From 4ad0a8e110060ea1c719949b36f7d3a342d61a2b Mon Sep 17 00:00:00 2001 From: zhuojie Date: Fri, 4 Sep 2020 17:05:33 -0700 Subject: [PATCH] Fix mysql8 compatibility --- docker-compose.kafka.yml | 24 +- go.mod | 9 +- go.sum | 37 +- integration_tests/docker-compose.yml | 17 +- integration_tests/test.sh | 1 + pkg/entity/flag.go | 10 +- pkg/entity/segment.go | 4 +- .../go-sql-driver/mysql/.travis.yml | 28 +- vendor/github.com/go-sql-driver/mysql/AUTHORS | 16 + .../go-sql-driver/mysql/CHANGELOG.md | 39 + .../go-sql-driver/mysql/CONTRIBUTING.md | 23 - .../github.com/go-sql-driver/mysql/README.md | 39 +- vendor/github.com/go-sql-driver/mysql/auth.go | 44 +- .../github.com/go-sql-driver/mysql/buffer.go | 95 +- .../go-sql-driver/mysql/collations.go | 376 +- .../go-sql-driver/mysql/conncheck.go | 54 + .../{appengine.go => conncheck_dummy.go} | 12 +- .../go-sql-driver/mysql/connection.go | 220 +- .../go-sql-driver/mysql/connection_go18.go | 208 - .../go-sql-driver/mysql/connector.go | 146 + .../github.com/go-sql-driver/mysql/driver.go | 154 +- vendor/github.com/go-sql-driver/mysql/dsn.go | 239 +- vendor/github.com/go-sql-driver/mysql/go.mod | 3 + .../go-sql-driver/mysql/nulltime.go | 50 + .../go-sql-driver/mysql/nulltime_go113.go | 31 + .../go-sql-driver/mysql/nulltime_legacy.go | 34 + .../github.com/go-sql-driver/mysql/packets.go | 145 +- vendor/github.com/go-sql-driver/mysql/rows.go | 7 + .../go-sql-driver/mysql/statement.go | 11 +- .../github.com/go-sql-driver/mysql/utils.go | 311 +- .../go-sql-driver/mysql/utils_go17.go | 40 - .../go-sql-driver/mysql/utils_go18.go | 50 - .../github.com/jinzhu/gorm/.codeclimate.yml | 11 - vendor/github.com/jinzhu/gorm/.gitignore | 1 + vendor/github.com/jinzhu/gorm/README.md | 39 +- vendor/github.com/jinzhu/gorm/association.go | 8 +- vendor/github.com/jinzhu/gorm/callback.go | 42 +- .../github.com/jinzhu/gorm/callback_create.go | 59 +- .../github.com/jinzhu/gorm/callback_delete.go | 4 +- .../github.com/jinzhu/gorm/callback_query.go | 7 +- .../jinzhu/gorm/callback_query_preload.go | 16 +- .../jinzhu/gorm/callback_row_query.go | 13 +- .../github.com/jinzhu/gorm/callback_update.go | 6 +- vendor/github.com/jinzhu/gorm/dialect.go | 11 +- .../github.com/jinzhu/gorm/dialect_common.go | 28 +- .../github.com/jinzhu/gorm/dialect_mysql.go | 73 +- .../jinzhu/gorm/dialect_postgres.go | 4 + .../jinzhu/gorm/dialects/postgres/postgres.go | 1 + vendor/github.com/jinzhu/gorm/errors.go | 14 +- vendor/github.com/jinzhu/gorm/field.go | 10 +- vendor/github.com/jinzhu/gorm/go.mod | 14 + vendor/github.com/jinzhu/gorm/go.sum | 33 + vendor/github.com/jinzhu/gorm/interface.go | 6 +- vendor/github.com/jinzhu/gorm/logger.go | 26 +- vendor/github.com/jinzhu/gorm/main.go | 130 +- vendor/github.com/jinzhu/gorm/model_struct.go | 138 +- vendor/github.com/jinzhu/gorm/scope.go | 100 +- vendor/github.com/jinzhu/gorm/search.go | 54 +- vendor/github.com/jinzhu/gorm/utils.go | 6 +- vendor/github.com/jinzhu/gorm/wercker.yml | 45 +- vendor/github.com/jinzhu/inflection/go.mod | 1 + vendor/github.com/lib/pq/.travis.sh | 2 +- vendor/github.com/lib/pq/.travis.yml | 4 +- vendor/github.com/lib/pq/README.md | 2 +- vendor/github.com/lib/pq/conn.go | 257 +- vendor/github.com/lib/pq/conn_go18.go | 28 +- vendor/github.com/lib/pq/connector.go | 91 +- vendor/github.com/lib/pq/doc.go | 2 +- vendor/github.com/lib/pq/encode.go | 9 +- vendor/github.com/lib/pq/scram/scram.go | 264 + vendor/github.com/lib/pq/ssl.go | 8 +- vendor/github.com/lib/pq/ssl_go1.7.go | 14 - vendor/github.com/lib/pq/ssl_renegotiation.go | 8 - .../github.com/mattn/go-sqlite3/.travis.yml | 36 +- vendor/github.com/mattn/go-sqlite3/README.md | 114 +- vendor/github.com/mattn/go-sqlite3/backup.go | 8 +- .../github.com/mattn/go-sqlite3/callback.go | 34 +- vendor/github.com/mattn/go-sqlite3/convert.go | 299 + vendor/github.com/mattn/go-sqlite3/error.go | 21 +- vendor/github.com/mattn/go-sqlite3/go.mod | 8 + vendor/github.com/mattn/go-sqlite3/go.sum | 13 + .../mattn/go-sqlite3/sqlite3-binding.c | 109034 +++++++++------ .../mattn/go-sqlite3/sqlite3-binding.h | 1392 +- vendor/github.com/mattn/go-sqlite3/sqlite3.go | 431 +- .../mattn/go-sqlite3/sqlite3_context.go | 2 +- .../mattn/go-sqlite3/sqlite3_func_crypt.go | 12 +- .../mattn/go-sqlite3/sqlite3_go18.go | 6 +- .../mattn/go-sqlite3/sqlite3_libsqlite3.go | 3 +- .../go-sqlite3/sqlite3_load_extension.go | 40 +- .../go-sqlite3/sqlite3_load_extension_omit.go | 2 +- .../sqlite3_opt_allow_uri_authority.go | 2 +- .../mattn/go-sqlite3/sqlite3_opt_app_armor.go | 4 +- .../go-sqlite3/sqlite3_opt_foreign_keys.go | 2 +- .../mattn/go-sqlite3/sqlite3_opt_fts5.go | 2 +- .../mattn/go-sqlite3/sqlite3_opt_icu.go | 2 +- .../go-sqlite3/sqlite3_opt_introspect.go | 2 +- .../mattn/go-sqlite3/sqlite3_opt_json1.go | 2 +- .../mattn/go-sqlite3/sqlite3_opt_preupdate.go | 20 + .../go-sqlite3/sqlite3_opt_preupdate_hook.go | 112 + .../go-sqlite3/sqlite3_opt_preupdate_omit.go | 21 + .../go-sqlite3/sqlite3_opt_secure_delete.go | 2 +- .../sqlite3_opt_secure_delete_fast.go | 2 +- .../mattn/go-sqlite3/sqlite3_opt_stat4.go | 2 +- .../go-sqlite3/sqlite3_opt_unlock_notify.c | 85 + .../go-sqlite3/sqlite3_opt_unlock_notify.go | 93 + .../go-sqlite3/sqlite3_opt_vacuum_full.go | 2 +- .../go-sqlite3/sqlite3_opt_vacuum_incr.go | 2 +- .../mattn/go-sqlite3/sqlite3_opt_vtable.go | 2 +- .../mattn/go-sqlite3/sqlite3_other.go | 5 +- .../mattn/go-sqlite3/sqlite3_solaris.go | 2 +- .../mattn/go-sqlite3/sqlite3_trace.go | 5 +- .../mattn/go-sqlite3/sqlite3_type.go | 5 + .../mattn/go-sqlite3/sqlite3_windows.go | 2 +- .../github.com/mattn/go-sqlite3/sqlite3ext.h | 44 + .../mattn/go-sqlite3/static_mock.go | 30 +- .../x/crypto/poly1305/bits_compat.go | 39 + .../x/crypto/poly1305/bits_go1.13.go | 21 + .../golang.org/x/crypto/poly1305/poly1305.go | 10 +- .../golang.org/x/crypto/poly1305/sum_amd64.go | 56 +- .../golang.org/x/crypto/poly1305/sum_amd64.s | 40 - .../golang.org/x/crypto/poly1305/sum_arm.go | 7 +- .../x/crypto/poly1305/sum_generic.go | 391 +- .../golang.org/x/crypto/poly1305/sum_noasm.go | 5 +- .../x/crypto/poly1305/sum_ppc64le.go | 56 +- .../x/crypto/poly1305/sum_ppc64le.s | 66 - .../golang.org/x/crypto/poly1305/sum_s390x.go | 5 +- .../appengine/cloudsql/cloudsql.go | 62 - .../appengine/cloudsql/cloudsql_classic.go | 17 - .../appengine/cloudsql/cloudsql_vm.go | 16 - vendor/modules.txt | 24 +- 130 files changed, 68639 insertions(+), 48149 deletions(-) delete mode 100644 vendor/github.com/go-sql-driver/mysql/CONTRIBUTING.md create mode 100644 vendor/github.com/go-sql-driver/mysql/conncheck.go rename vendor/github.com/go-sql-driver/mysql/{appengine.go => conncheck_dummy.go} (59%) delete mode 100644 vendor/github.com/go-sql-driver/mysql/connection_go18.go create mode 100644 vendor/github.com/go-sql-driver/mysql/connector.go create mode 100644 vendor/github.com/go-sql-driver/mysql/go.mod create mode 100644 vendor/github.com/go-sql-driver/mysql/nulltime.go create mode 100644 vendor/github.com/go-sql-driver/mysql/nulltime_go113.go create mode 100644 vendor/github.com/go-sql-driver/mysql/nulltime_legacy.go delete mode 100644 vendor/github.com/go-sql-driver/mysql/utils_go17.go delete mode 100644 vendor/github.com/go-sql-driver/mysql/utils_go18.go delete mode 100644 vendor/github.com/jinzhu/gorm/.codeclimate.yml create mode 100644 vendor/github.com/jinzhu/gorm/go.mod create mode 100644 vendor/github.com/jinzhu/gorm/go.sum create mode 100644 vendor/github.com/jinzhu/inflection/go.mod create mode 100644 vendor/github.com/lib/pq/scram/scram.go delete mode 100644 vendor/github.com/lib/pq/ssl_go1.7.go delete mode 100644 vendor/github.com/lib/pq/ssl_renegotiation.go create mode 100644 vendor/github.com/mattn/go-sqlite3/convert.go create mode 100644 vendor/github.com/mattn/go-sqlite3/go.mod create mode 100644 vendor/github.com/mattn/go-sqlite3/go.sum create mode 100644 vendor/github.com/mattn/go-sqlite3/sqlite3_opt_preupdate.go create mode 100644 vendor/github.com/mattn/go-sqlite3/sqlite3_opt_preupdate_hook.go create mode 100644 vendor/github.com/mattn/go-sqlite3/sqlite3_opt_preupdate_omit.go create mode 100644 vendor/github.com/mattn/go-sqlite3/sqlite3_opt_unlock_notify.c create mode 100644 vendor/github.com/mattn/go-sqlite3/sqlite3_opt_unlock_notify.go create mode 100644 vendor/golang.org/x/crypto/poly1305/bits_compat.go create mode 100644 vendor/golang.org/x/crypto/poly1305/bits_go1.13.go delete mode 100644 vendor/google.golang.org/appengine/cloudsql/cloudsql.go delete mode 100644 vendor/google.golang.org/appengine/cloudsql/cloudsql_classic.go delete mode 100644 vendor/google.golang.org/appengine/cloudsql/cloudsql_vm.go diff --git a/docker-compose.kafka.yml b/docker-compose.kafka.yml index e73b39ff..d038aff0 100644 --- a/docker-compose.kafka.yml +++ b/docker-compose.kafka.yml @@ -1,14 +1,32 @@ mysql: image: mysql:5.6 + container_name: flagr-mysql + environment: + - MYSQL_ALLOW_EMPTY_PASSWORD=yes + - MYSQL_DATABASE=flagr + - MYSQL_USER=root ports: - "18100:3306" + +mysql8: + image: mysql:8 container_name: flagr-mysql environment: - MYSQL_ALLOW_EMPTY_PASSWORD=yes - MYSQL_DATABASE=flagr - MYSQL_USER=root - volumes: - - ./_data/mysql:/var/lib/mysql + ports: + - "18101:3306" + +postgres: + image: postgres:9 + container_name: flagr-postgres + environment: + POSTGRES_PASSWORD: "test" + POSTGRES_USER: "test" + POSTGRES_DB: "flagr" + ports: + - "18102:5432" zookeeper: image: confluentinc/cp-zookeeper @@ -35,8 +53,6 @@ kafka: - "9092:9092" extra_hosts: - "moby:127.0.0.1" - volumes: - - ./_data/kafka:/var/lib/kafka kafka-manager: image: sheepkiller/kafka-manager diff --git a/go.mod b/go.mod index ef38fe0d..17b44faf 100644 --- a/go.mod +++ b/go.mod @@ -17,9 +17,7 @@ require ( github.com/codegangsta/negroni v1.0.0 // indirect github.com/davecgh/go-spew v1.1.1 github.com/dchest/uniuri v0.0.0-20160212164326-8902c56451e9 - github.com/denisenkom/go-mssqldb v0.0.0-20190418034912-35416408c946 // indirect github.com/dgrijalva/jwt-go v3.2.0+incompatible - github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5 // indirect github.com/evalphobia/logrus_sentry v0.4.6 github.com/getsentry/raven-go v0.0.0-20180903072508-084a9de9eb03 github.com/go-ini/ini v1.38.2 // indirect @@ -30,16 +28,11 @@ require ( github.com/go-openapi/strfmt v0.19.5 github.com/go-openapi/swag v0.19.9 github.com/go-openapi/validate v0.19.10 - github.com/go-sql-driver/mysql v1.4.0 // indirect github.com/gohttp/pprof v0.0.0-20141119085724-c9d246cbb3ba github.com/gorilla/mux v1.7.1 // indirect github.com/jessevdk/go-flags v1.4.0 - github.com/jinzhu/gorm v0.0.0-20180909231100-123d4f50ef8a - github.com/jinzhu/inflection v0.0.0-20180308033659-04140366298a // indirect - github.com/jinzhu/now v1.0.0 // indirect + github.com/jinzhu/gorm v1.9.16 github.com/jpillora/backoff v0.0.0-20170918002102-8eab2debe79d // indirect - github.com/lib/pq v1.0.0 // indirect - github.com/mattn/go-sqlite3 v1.9.0 // indirect github.com/meatballhat/negroni-logrus v0.0.0-20170801195057-31067281800f github.com/newrelic/go-agent v2.1.0+incompatible github.com/onsi/ginkgo v1.8.0 // indirect diff --git a/go.sum b/go.sum index 756b8966..ab19c481 100644 --- a/go.sum +++ b/go.sum @@ -6,6 +6,7 @@ github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/DataDog/datadog-go v0.0.0-20180330214955-e67964b4021a h1:zpQSzEApXM0qkXcpdjeJ4OpnBWhD/X8zT/iT1wYLiVU= github.com/DataDog/datadog-go v0.0.0-20180330214955-e67964b4021a/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= +github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= github.com/PuerkitoBio/purell v1.1.0 h1:rmGxhojJlM0tuKtfdvliR84CFHljx9ag64t2xmVkjK4= github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= @@ -20,6 +21,7 @@ github.com/a8m/kinesis-producer v0.0.0-20180723062609-03228a9f79b3 h1:B3CxuHrRSJ github.com/a8m/kinesis-producer v0.0.0-20180723062609-03228a9f79b3/go.mod h1:CxoFe0Y49udKMnQPkC5S5VmZZy6a+Bef9otuoH96Pv0= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= @@ -51,8 +53,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dchest/uniuri v0.0.0-20160212164326-8902c56451e9 h1:74lLNRzvsdIlkTgfDSMuaPjBr4cf6k7pwQQANm/yLKU= github.com/dchest/uniuri v0.0.0-20160212164326-8902c56451e9/go.mod h1:GgB8SF9nRG+GqaDtLcwJZsQFhcogVCJ79j4EdT0c2V4= -github.com/denisenkom/go-mssqldb v0.0.0-20190418034912-35416408c946 h1:xn+jBHAqNYs6CnHhJwfWZQspPHMEL+LzUk0vpqWj6eo= -github.com/denisenkom/go-mssqldb v0.0.0-20190418034912-35416408c946/go.mod h1:zAg7JM8CkOJ43xKXIj7eRO9kmWm/TW578qo+oDO6tuM= +github.com/denisenkom/go-mssqldb v0.0.0-20191124224453-732737034ffd h1:83Wprp6ROGeiHFAP8WJdI2RoxALQYgdllERc3N5N2DM= +github.com/denisenkom/go-mssqldb v0.0.0-20191124224453-732737034ffd/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/docker/go-units v0.3.3 h1:Xk8S3Xj5sLGlG5g67hJmYMmUgXv5N4PhkjJHHqrwnTk= @@ -146,8 +148,8 @@ github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2K github.com/go-openapi/validate v0.19.3/go.mod h1:90Vh6jjkTn+OT1Eefm0ZixWNFjhtOH7vS9k0lo6zwJo= github.com/go-openapi/validate v0.19.10 h1:tG3SZ5DC5KF4cyt7nqLVcQXGj5A7mpaYkAcNPlDK+Yk= github.com/go-openapi/validate v0.19.10/go.mod h1:RKEZTUWDkxKQxN2jDT7ZnZi2bhZlbNMAuKvKB+IaGx8= -github.com/go-sql-driver/mysql v1.4.0 h1:7LxgVwFb2hIQtMm87NdgAVfXjnt4OePseqT1tKx+opk= -github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs= +github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0= @@ -179,6 +181,8 @@ github.com/gogo/protobuf v1.2.0 h1:xU6/SpYbvkNYiptHJYEDRseDLvYE7wSqhYYNy0QSUzI= github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gohttp/pprof v0.0.0-20141119085724-c9d246cbb3ba h1:OckY4Dk1WhEEEz4zYYMsXG5f6necMtGAyAs19vcpRXk= github.com/gohttp/pprof v0.0.0-20141119085724-c9d246cbb3ba/go.mod h1:V97TX7IXWIioKfmy0IKnnBzsC1jRXP2VicslN9O8IIQ= +github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe h1:lXe2qZdvpiX5WZkZR4hgp4KJVfY3nMkvmwbVkpv1rVY= +github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -217,12 +221,12 @@ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpO github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jessevdk/go-flags v1.4.0 h1:4IU2WS7AumrZ/40jfhf4QVDMsQwqA7VEHozFRrGARJA= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= -github.com/jinzhu/gorm v0.0.0-20180909231100-123d4f50ef8a h1:Z+fo5W6ecb0uvnWoEtzYoQKB8e9NFHT/19aB9ihFsLM= -github.com/jinzhu/gorm v0.0.0-20180909231100-123d4f50ef8a/go.mod h1:Vla75njaFJ8clLU1W44h34PjIkijhjHIYnZxMqCdxqo= -github.com/jinzhu/inflection v0.0.0-20180308033659-04140366298a h1:eeaG9XMUvRBYXJi4pg1ZKM7nxc5AfXfojeLLW7O5J3k= -github.com/jinzhu/inflection v0.0.0-20180308033659-04140366298a/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= -github.com/jinzhu/now v1.0.0 h1:6WV8LvwPpDhKjo5U9O6b4+xdG/jTXNPwlDme/MTo8Ns= -github.com/jinzhu/now v1.0.0/go.mod h1:oHTiXerJ20+SfYcrdlBO7rzZRJWGwSTQ0iUY2jI6Gfc= +github.com/jinzhu/gorm v1.9.16 h1:+IyIjPEABKRpsu/F8OvDPy9fyQlgsg2luMV2ZIH5i5o= +github.com/jinzhu/gorm v1.9.16/go.mod h1:G3LB3wezTOWM2ITLzPxEXgSkOXAntiLHS7UdBefADcs= +github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= +github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= +github.com/jinzhu/now v1.0.1 h1:HjfetcXq097iXP0uoPCdnM4Efp5/9MsM0/M+XOTeR3M= +github.com/jinzhu/now v1.0.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8 h1:12VvqtR6Aowv3l/EQUlocDHW2Cp4G9WJVH7uyH8QFJE= github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= @@ -251,8 +255,8 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/lib/pq v1.0.0 h1:X5PMW56eZitiTeO7tKzZxFCSpbFZJtkMMooicw2us9A= -github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.1.1 h1:sJZmqHoEaY7f+NPP8pgLB/WxulyR3fewgCM2qaSlBb4= +github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329 h1:2gxZ0XQIU/5z3Z3bUBu+FXuk2pFbkN6tcwi/pjyaDic= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -262,8 +266,8 @@ github.com/mailru/easyjson v0.7.1 h1:mdxE1MF9o53iCb2Ghj1VfWvh7ZOwHpnVG/xwXrV90U8 github.com/mailru/easyjson v0.7.1/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= -github.com/mattn/go-sqlite3 v1.9.0 h1:pDRiWfl+++eC2FEFRy6jXmQlvp4Yh3z1MJKg4UeYM/4= -github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/mattn/go-sqlite3 v1.14.0 h1:mLyGNKR8+Vv9CAU7PphKa2hkEqxxhn8i32J6FPj1/QA= +github.com/mattn/go-sqlite3 v1.14.0/go.mod h1:JIl7NbARA7phWnGvh0LKTyg7S9BA+6gx71ShQilpsus= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/meatballhat/negroni-logrus v0.0.0-20170801195057-31067281800f h1:V6GHkMOIsnpGDasS1iYiNxEYTY8TmyjQXEF8PqYkKQ8= @@ -380,10 +384,13 @@ golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56 h1:ZpKuNIejY8P0ExLOVyKhb0WsgG8UdvHXe6TWjY7eL6k= golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191205180655-e7c4368fe9dd h1:GGJVjV8waZKRHrgwvtH66z9ZGVurTD1MT0n1Bb+q4aM= +golang.org/x/crypto v0.0.0-20191205180655-e7c4368fe9dd/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -399,7 +406,9 @@ golang.org/x/net v0.0.0-20190320064053-1272bf9dcd53/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200602114024-627f9648deb9 h1:pNX+40auqi2JqRfOP1akLGtYcn15TUbkhwuCO3foqqM= golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= diff --git a/integration_tests/docker-compose.yml b/integration_tests/docker-compose.yml index 0f51505b..a74862b8 100644 --- a/integration_tests/docker-compose.yml +++ b/integration_tests/docker-compose.yml @@ -14,6 +14,14 @@ services: MYSQL_DATABASE: "flagr" MYSQL_USER: "root" + mysql8: + image: mysql:8 + container_name: flagr-mysql8 + environment: + MYSQL_ALLOW_EMPTY_PASSWORD: "yes" + MYSQL_DATABASE: "flagr" + MYSQL_USER: "root" + postgres: image: postgres:9 container_name: flagr-postgres @@ -31,13 +39,18 @@ services: flagr_with_mysql: image: flagr_integration_tests - ports: - - "18002:18000" environment: FLAGR_DB_DBDRIVER: "mysql" FLAGR_DB_DBCONNECTIONSTR: "root:@tcp(mysql:3306)/flagr?parseTime=true" command: sh -c "sleep 15 && ./flagr" + flagr_with_mysql8: + image: flagr_integration_tests + environment: + FLAGR_DB_DBDRIVER: "mysql" + FLAGR_DB_DBCONNECTIONSTR: "root:@tcp(mysql8:3306)/flagr?parseTime=true" + command: sh -c "sleep 15 && ./flagr" + flagr_with_postgres: image: flagr_integration_tests environment: diff --git a/integration_tests/test.sh b/integration_tests/test.sh index 76024dd5..da351898 100644 --- a/integration_tests/test.sh +++ b/integration_tests/test.sh @@ -392,6 +392,7 @@ start_test() start(){ start_test flagr_with_sqlite start_test flagr_with_mysql + start_test flagr_with_mysql8 start_test flagr_with_postgres } diff --git a/pkg/entity/flag.go b/pkg/entity/flag.go index 8aad4fbd..1d80a185 100644 --- a/pkg/entity/flag.go +++ b/pkg/entity/flag.go @@ -36,7 +36,7 @@ type FlagEvaluation struct { // Preloads just the tags func PreloadFlagTags(db *gorm.DB) *gorm.DB { return db.Preload("Tags", func(db *gorm.DB) *gorm.DB { - return db.Order("id ASC") + return db.Order("id") }) } @@ -45,14 +45,14 @@ func PreloadSegmentsVariantsTags(db *gorm.DB) *gorm.DB { return db. Preload("Segments", func(db *gorm.DB) *gorm.DB { return PreloadConstraintsDistribution(db). - Order("rank ASC"). - Order("id ASC") + Order("rank"). + Order("id") }). Preload("Variants", func(db *gorm.DB) *gorm.DB { - return db.Order("id ASC") + return db.Order("id") }). Preload("Tags", func(db *gorm.DB) *gorm.DB { - return db.Order("id ASC") + return db.Order("id") }) } diff --git a/pkg/entity/segment.go b/pkg/entity/segment.go index b5938a14..5c3eb860 100644 --- a/pkg/entity/segment.go +++ b/pkg/entity/segment.go @@ -27,10 +27,10 @@ type Segment struct { func PreloadConstraintsDistribution(db *gorm.DB) *gorm.DB { return db. Preload("Distributions", func(db *gorm.DB) *gorm.DB { - return db.Order("variant_id ASC") + return db.Order("variant_id") }). Preload("Constraints", func(db *gorm.DB) *gorm.DB { - return db.Order("created_at ASC") + return db.Order("created_at") }) } diff --git a/vendor/github.com/go-sql-driver/mysql/.travis.yml b/vendor/github.com/go-sql-driver/mysql/.travis.yml index cc1268c3..56fcf25f 100644 --- a/vendor/github.com/go-sql-driver/mysql/.travis.yml +++ b/vendor/github.com/go-sql-driver/mysql/.travis.yml @@ -1,10 +1,10 @@ sudo: false language: go go: - - 1.7.x - - 1.8.x - - 1.9.x - 1.10.x + - 1.11.x + - 1.12.x + - 1.13.x - master before_install: @@ -99,6 +99,28 @@ matrix: - export MYSQL_TEST_ADDR=127.0.0.1:3307 - export MYSQL_TEST_CONCURRENT=1 + - os: osx + osx_image: xcode10.1 + addons: + homebrew: + packages: + - mysql + update: true + go: 1.12.x + before_install: + - go get golang.org/x/tools/cmd/cover + - go get github.com/mattn/goveralls + before_script: + - echo -e "[server]\ninnodb_log_file_size=256MB\ninnodb_buffer_pool_size=512MB\nmax_allowed_packet=16MB\nlocal_infile=1" >> /usr/local/etc/my.cnf + - mysql.server start + - mysql -uroot -e 'CREATE USER gotest IDENTIFIED BY "secret"' + - mysql -uroot -e 'GRANT ALL ON *.* TO gotest' + - mysql -uroot -e 'create database gotest;' + - export MYSQL_TEST_USER=gotest + - export MYSQL_TEST_PASS=secret + - export MYSQL_TEST_ADDR=127.0.0.1:3306 + - export MYSQL_TEST_CONCURRENT=1 + script: - go test -v -covermode=count -coverprofile=coverage.out - go vet ./... diff --git a/vendor/github.com/go-sql-driver/mysql/AUTHORS b/vendor/github.com/go-sql-driver/mysql/AUTHORS index 73ff68fb..ad598980 100644 --- a/vendor/github.com/go-sql-driver/mysql/AUTHORS +++ b/vendor/github.com/go-sql-driver/mysql/AUTHORS @@ -27,6 +27,7 @@ Daniël van Eeden Dave Protasowski DisposaBoy Egor Smolyakov +Erwan Martin Evan Shaw Frederick Mayle Gustavo Kristic @@ -34,12 +35,16 @@ Hajime Nakagami Hanno Braun Henri Yandell Hirotaka Yamamoto +Huyiguang ICHINOSE Shogo +Ilia Cimpoes INADA Naoki Jacek Szwec James Harr Jeff Hodges Jeffrey Charles +Jerome Meyer +Jiajia Zhong Jian Zhen Joshua Prunier Julien Lefevre @@ -58,6 +63,7 @@ Lucas Liu Luke Scott Maciej Zimnoch Michael Woolnough +Nathanial Murphy Nicola Peduzzi Olivier Mengué oscarzhao @@ -69,9 +75,15 @@ Richard Wilkes Robert Russell Runrioter Wung Shuode Li +Simon J Mudd Soroush Pour Stan Putrya Stanley Gunawan +Steven Hartland +Thomas Wodarek +Tim Ruffles +Tom Jenkinson +Vladimir Kovpak Xiangyu Hu Xiaobing Jiang Xiuming Chen @@ -81,9 +93,13 @@ Zhenye Xie Barracuda Networks, Inc. Counting Ltd. +DigitalOcean Inc. +Facebook Inc. +GitHub Inc. Google Inc. InfoSum Ltd. Keybase Inc. +Multiplay Ltd. Percona LLC Pivotal Inc. Stripe Inc. diff --git a/vendor/github.com/go-sql-driver/mysql/CHANGELOG.md b/vendor/github.com/go-sql-driver/mysql/CHANGELOG.md index 2d87d74c..9cb97b38 100644 --- a/vendor/github.com/go-sql-driver/mysql/CHANGELOG.md +++ b/vendor/github.com/go-sql-driver/mysql/CHANGELOG.md @@ -1,3 +1,42 @@ +## Version 1.5 (2020-01-07) + +Changes: + + - Dropped support Go 1.9 and lower (#823, #829, #886, #1016, #1017) + - Improve buffer handling (#890) + - Document potentially insecure TLS configs (#901) + - Use a double-buffering scheme to prevent data races (#943) + - Pass uint64 values without converting them to string (#838, #955) + - Update collations and make utf8mb4 default (#877, #1054) + - Make NullTime compatible with sql.NullTime in Go 1.13+ (#995) + - Removed CloudSQL support (#993, #1007) + - Add Go Module support (#1003) + +New Features: + + - Implement support of optional TLS (#900) + - Check connection liveness (#934, #964, #997, #1048, #1051, #1052) + - Implement Connector Interface (#941, #958, #1020, #1035) + +Bugfixes: + + - Mark connections as bad on error during ping (#875) + - Mark connections as bad on error during dial (#867) + - Fix connection leak caused by rapid context cancellation (#1024) + - Mark connections as bad on error during Conn.Prepare (#1030) + + +## Version 1.4.1 (2018-11-14) + +Bugfixes: + + - Fix TIME format for binary columns (#818) + - Fix handling of empty auth plugin names (#835) + - Fix caching_sha2_password with empty password (#826) + - Fix canceled context broke mysqlConn (#862) + - Fix OldAuthSwitchRequest support (#870) + - Fix Auth Response packet for cleartext password (#887) + ## Version 1.4 (2018-06-03) Changes: diff --git a/vendor/github.com/go-sql-driver/mysql/CONTRIBUTING.md b/vendor/github.com/go-sql-driver/mysql/CONTRIBUTING.md deleted file mode 100644 index 8fe16bcb..00000000 --- a/vendor/github.com/go-sql-driver/mysql/CONTRIBUTING.md +++ /dev/null @@ -1,23 +0,0 @@ -# Contributing Guidelines - -## Reporting Issues - -Before creating a new Issue, please check first if a similar Issue [already exists](https://github.com/go-sql-driver/mysql/issues?state=open) or was [recently closed](https://github.com/go-sql-driver/mysql/issues?direction=desc&page=1&sort=updated&state=closed). - -## Contributing Code - -By contributing to this project, you share your code under the Mozilla Public License 2, as specified in the LICENSE file. -Don't forget to add yourself to the AUTHORS file. - -### Code Review - -Everyone is invited to review and comment on pull requests. -If it looks fine to you, comment with "LGTM" (Looks good to me). - -If changes are required, notice the reviewers with "PTAL" (Please take another look) after committing the fixes. - -Before merging the Pull Request, at least one [team member](https://github.com/go-sql-driver?tab=members) must have commented with "LGTM". - -## Development Ideas - -If you are looking for ideas for code contributions, please check our [Development Ideas](https://github.com/go-sql-driver/mysql/wiki/Development-Ideas) Wiki page. diff --git a/vendor/github.com/go-sql-driver/mysql/README.md b/vendor/github.com/go-sql-driver/mysql/README.md index 2e9b07ee..d2627a41 100644 --- a/vendor/github.com/go-sql-driver/mysql/README.md +++ b/vendor/github.com/go-sql-driver/mysql/README.md @@ -40,7 +40,7 @@ A MySQL-Driver for Go's [database/sql](https://golang.org/pkg/database/sql/) pac * Optional placeholder interpolation ## Requirements - * Go 1.7 or higher. We aim to support the 3 latest versions of Go. + * Go 1.10 or higher. We aim to support the 3 latest versions of Go. * MySQL (4.1+), MariaDB, Percona Server, Google CloudSQL or Sphinx (2.2.3+) --------------------------------------- @@ -166,18 +166,34 @@ Sets the charset used for client-server interaction (`"SET NAMES "`). If Usage of the `charset` parameter is discouraged because it issues additional queries to the server. Unless you need the fallback behavior, please use `collation` instead. +##### `checkConnLiveness` + +``` +Type: bool +Valid Values: true, false +Default: true +``` + +On supported platforms connections retrieved from the connection pool are checked for liveness before using them. If the check fails, the respective connection is marked as bad and the query retried with another connection. +`checkConnLiveness=false` disables this liveness check of connections. + ##### `collation` ``` Type: string Valid Values: -Default: utf8_general_ci +Default: utf8mb4_general_ci ``` Sets the collation used for client-server interaction on connection. In contrast to `charset`, `collation` does not issue additional queries. If the specified collation is unavailable on the target server, the connection will fail. A list of valid charsets for a server is retrievable with `SHOW COLLATION`. +The default collation (`utf8mb4_general_ci`) is supported from MySQL 5.5. You should use an older collation (e.g. `utf8_general_ci`) for older MySQL. + +Collations for charset "ucs2", "utf16", "utf16le", and "utf32" can not be used ([ref](https://dev.mysql.com/doc/refman/5.7/en/charset-connection.html#charset-connection-impermissible-client-charset)). + + ##### `clientFoundRows` ``` @@ -328,11 +344,11 @@ Timeout for establishing connections, aka dial timeout. The value must be a deci ``` Type: bool / string -Valid Values: true, false, skip-verify, +Valid Values: true, false, skip-verify, preferred, Default: false ``` -`tls=true` enables TLS / SSL encrypted connection to the server. Use `skip-verify` if you want to use a self-signed or invalid certificate (server side). Use a custom value registered with [`mysql.RegisterTLSConfig`](https://godoc.org/github.com/go-sql-driver/mysql#RegisterTLSConfig). +`tls=true` enables TLS / SSL encrypted connection to the server. Use `skip-verify` if you want to use a self-signed or invalid certificate (server side) or use `preferred` to use TLS only when advertised by the server. This is similar to `skip-verify`, but additionally allows a fallback to a connection which is not encrypted. Neither `skip-verify` nor `preferred` add any reliable security. You can use a custom TLS config after registering it with [`mysql.RegisterTLSConfig`](https://godoc.org/github.com/go-sql-driver/mysql#RegisterTLSConfig). ##### `writeTimeout` @@ -391,14 +407,9 @@ TCP on a remote host, e.g. Amazon RDS: id:password@tcp(your-amazonaws-uri.com:3306)/dbname ``` -Google Cloud SQL on App Engine (First Generation MySQL Server): -``` -user@cloudsql(project-id:instance-name)/dbname -``` - -Google Cloud SQL on App Engine (Second Generation MySQL Server): +Google Cloud SQL on App Engine: ``` -user@cloudsql(project-id:regionname:instance-name)/dbname +user:password@unix(/cloudsql/project-id:region-name:instance-name)/dbname ``` TCP using default port (3306) on localhost: @@ -444,7 +455,7 @@ See the [godoc of Go-MySQL-Driver](https://godoc.org/github.com/go-sql-driver/my ### `time.Time` support The default internal output type of MySQL `DATE` and `DATETIME` values is `[]byte` which allows you to scan the value into a `[]byte`, `string` or `sql.RawBytes` variable in your program. -However, many want to scan MySQL `DATE` and `DATETIME` values into `time.Time` variables, which is the logical opposite in Go to `DATE` and `DATETIME` in MySQL. You can do that by changing the internal output type from `[]byte` to `time.Time` with the DSN parameter `parseTime=true`. You can set the default [`time.Time` location](https://golang.org/pkg/time/#Location) with the `loc` DSN parameter. +However, many want to scan MySQL `DATE` and `DATETIME` values into `time.Time` variables, which is the logical equivalent in Go to `DATE` and `DATETIME` in MySQL. You can do that by changing the internal output type from `[]byte` to `time.Time` with the DSN parameter `parseTime=true`. You can set the default [`time.Time` location](https://golang.org/pkg/time/#Location) with the `loc` DSN parameter. **Caution:** As of Go 1.1, this makes `time.Time` the only variable type you can scan `DATE` and `DATETIME` values into. This breaks for example [`sql.RawBytes` support](https://github.com/go-sql-driver/mysql/wiki/Examples#rawbytes). @@ -452,13 +463,13 @@ Alternatively you can use the [`NullTime`](https://godoc.org/github.com/go-sql-d ### Unicode support -Since version 1.1 Go-MySQL-Driver automatically uses the collation `utf8_general_ci` by default. +Since version 1.5 Go-MySQL-Driver automatically uses the collation ` utf8mb4_general_ci` by default. Other collations / charsets can be set using the [`collation`](#collation) DSN parameter. Version 1.0 of the driver recommended adding `&charset=utf8` (alias for `SET NAMES utf8`) to the DSN to enable proper UTF-8 support. This is not necessary anymore. The [`collation`](#collation) parameter should be preferred to set another collation / charset than the default. -See http://dev.mysql.com/doc/refman/5.7/en/charset-unicode.html for more details on MySQL's Unicode support. +See http://dev.mysql.com/doc/refman/8.0/en/charset-unicode.html for more details on MySQL's Unicode support. ## Testing / Development To run the driver tests you may need to adjust the configuration. See the [Testing Wiki-Page](https://github.com/go-sql-driver/mysql/wiki/Testing "Testing") for details. diff --git a/vendor/github.com/go-sql-driver/mysql/auth.go b/vendor/github.com/go-sql-driver/mysql/auth.go index 0b59f52e..fec7040d 100644 --- a/vendor/github.com/go-sql-driver/mysql/auth.go +++ b/vendor/github.com/go-sql-driver/mysql/auth.go @@ -234,64 +234,64 @@ func (mc *mysqlConn) sendEncryptedPassword(seed []byte, pub *rsa.PublicKey) erro if err != nil { return err } - return mc.writeAuthSwitchPacket(enc, false) + return mc.writeAuthSwitchPacket(enc) } -func (mc *mysqlConn) auth(authData []byte, plugin string) ([]byte, bool, error) { +func (mc *mysqlConn) auth(authData []byte, plugin string) ([]byte, error) { switch plugin { case "caching_sha2_password": authResp := scrambleSHA256Password(authData, mc.cfg.Passwd) - return authResp, (authResp == nil), nil + return authResp, nil case "mysql_old_password": if !mc.cfg.AllowOldPasswords { - return nil, false, ErrOldPassword + return nil, ErrOldPassword } // Note: there are edge cases where this should work but doesn't; // this is currently "wontfix": // https://github.com/go-sql-driver/mysql/issues/184 - authResp := scrambleOldPassword(authData[:8], mc.cfg.Passwd) - return authResp, true, nil + authResp := append(scrambleOldPassword(authData[:8], mc.cfg.Passwd), 0) + return authResp, nil case "mysql_clear_password": if !mc.cfg.AllowCleartextPasswords { - return nil, false, ErrCleartextPassword + return nil, ErrCleartextPassword } // http://dev.mysql.com/doc/refman/5.7/en/cleartext-authentication-plugin.html // http://dev.mysql.com/doc/refman/5.7/en/pam-authentication-plugin.html - return []byte(mc.cfg.Passwd), true, nil + return append([]byte(mc.cfg.Passwd), 0), nil case "mysql_native_password": if !mc.cfg.AllowNativePasswords { - return nil, false, ErrNativePassword + return nil, ErrNativePassword } // https://dev.mysql.com/doc/internals/en/secure-password-authentication.html // Native password authentication only need and will need 20-byte challenge. authResp := scramblePassword(authData[:20], mc.cfg.Passwd) - return authResp, false, nil + return authResp, nil case "sha256_password": if len(mc.cfg.Passwd) == 0 { - return nil, true, nil + return []byte{0}, nil } if mc.cfg.tls != nil || mc.cfg.Net == "unix" { // write cleartext auth packet - return []byte(mc.cfg.Passwd), true, nil + return append([]byte(mc.cfg.Passwd), 0), nil } pubKey := mc.cfg.pubKey if pubKey == nil { // request public key from server - return []byte{1}, false, nil + return []byte{1}, nil } // encrypted password enc, err := encryptPassword(mc.cfg.Passwd, authData, pubKey) - return enc, false, err + return enc, err default: errLog.Print("unknown auth plugin:", plugin) - return nil, false, ErrUnknownPlugin + return nil, ErrUnknownPlugin } } @@ -315,11 +315,11 @@ func (mc *mysqlConn) handleAuthResult(oldAuthData []byte, plugin string) error { plugin = newPlugin - authResp, addNUL, err := mc.auth(authData, plugin) + authResp, err := mc.auth(authData, plugin) if err != nil { return err } - if err = mc.writeAuthSwitchPacket(authResp, addNUL); err != nil { + if err = mc.writeAuthSwitchPacket(authResp); err != nil { return err } @@ -352,7 +352,7 @@ func (mc *mysqlConn) handleAuthResult(oldAuthData []byte, plugin string) error { case cachingSha2PasswordPerformFullAuthentication: if mc.cfg.tls != nil || mc.cfg.Net == "unix" { // write cleartext auth packet - err = mc.writeAuthSwitchPacket([]byte(mc.cfg.Passwd), true) + err = mc.writeAuthSwitchPacket(append([]byte(mc.cfg.Passwd), 0)) if err != nil { return err } @@ -360,13 +360,15 @@ func (mc *mysqlConn) handleAuthResult(oldAuthData []byte, plugin string) error { pubKey := mc.cfg.pubKey if pubKey == nil { // request public key from server - data := mc.buf.takeSmallBuffer(4 + 1) + data, err := mc.buf.takeSmallBuffer(4 + 1) + if err != nil { + return err + } data[4] = cachingSha2PasswordRequestPublicKey mc.writePacket(data) // parse public key - data, err := mc.readPacket() - if err != nil { + if data, err = mc.readPacket(); err != nil { return err } diff --git a/vendor/github.com/go-sql-driver/mysql/buffer.go b/vendor/github.com/go-sql-driver/mysql/buffer.go index eb4748bf..0774c5c8 100644 --- a/vendor/github.com/go-sql-driver/mysql/buffer.go +++ b/vendor/github.com/go-sql-driver/mysql/buffer.go @@ -15,47 +15,69 @@ import ( ) const defaultBufSize = 4096 +const maxCachedBufSize = 256 * 1024 // A buffer which is used for both reading and writing. // This is possible since communication on each connection is synchronous. // In other words, we can't write and read simultaneously on the same connection. // The buffer is similar to bufio.Reader / Writer but zero-copy-ish // Also highly optimized for this particular use case. +// This buffer is backed by two byte slices in a double-buffering scheme type buffer struct { - buf []byte + buf []byte // buf is a byte buffer who's length and capacity are equal. nc net.Conn idx int length int timeout time.Duration + dbuf [2][]byte // dbuf is an array with the two byte slices that back this buffer + flipcnt uint // flipccnt is the current buffer counter for double-buffering } +// newBuffer allocates and returns a new buffer. func newBuffer(nc net.Conn) buffer { - var b [defaultBufSize]byte + fg := make([]byte, defaultBufSize) return buffer{ - buf: b[:], - nc: nc, + buf: fg, + nc: nc, + dbuf: [2][]byte{fg, nil}, } } +// flip replaces the active buffer with the background buffer +// this is a delayed flip that simply increases the buffer counter; +// the actual flip will be performed the next time we call `buffer.fill` +func (b *buffer) flip() { + b.flipcnt += 1 +} + // fill reads into the buffer until at least _need_ bytes are in it func (b *buffer) fill(need int) error { n := b.length + // fill data into its double-buffering target: if we've called + // flip on this buffer, we'll be copying to the background buffer, + // and then filling it with network data; otherwise we'll just move + // the contents of the current buffer to the front before filling it + dest := b.dbuf[b.flipcnt&1] + + // grow buffer if necessary to fit the whole packet. + if need > len(dest) { + // Round up to the next multiple of the default size + dest = make([]byte, ((need/defaultBufSize)+1)*defaultBufSize) - // move existing data to the beginning - if n > 0 && b.idx > 0 { - copy(b.buf[0:n], b.buf[b.idx:]) + // if the allocated buffer is not too large, move it to backing storage + // to prevent extra allocations on applications that perform large reads + if len(dest) <= maxCachedBufSize { + b.dbuf[b.flipcnt&1] = dest + } } - // grow buffer if necessary - // TODO: let the buffer shrink again at some point - // Maybe keep the org buf slice and swap back? - if need > len(b.buf) { - // Round up to the next multiple of the default size - newBuf := make([]byte, ((need/defaultBufSize)+1)*defaultBufSize) - copy(newBuf, b.buf) - b.buf = newBuf + // if we're filling the fg buffer, move the existing data to the start of it. + // if we're filling the bg buffer, copy over the data + if n > 0 { + copy(dest[:n], b.buf[b.idx:]) } + b.buf = dest b.idx = 0 for { @@ -105,43 +127,56 @@ func (b *buffer) readNext(need int) ([]byte, error) { return b.buf[offset:b.idx], nil } -// returns a buffer with the requested size. +// takeBuffer returns a buffer with the requested size. // If possible, a slice from the existing buffer is returned. // Otherwise a bigger buffer is made. // Only one buffer (total) can be used at a time. -func (b *buffer) takeBuffer(length int) []byte { +func (b *buffer) takeBuffer(length int) ([]byte, error) { if b.length > 0 { - return nil + return nil, ErrBusyBuffer } // test (cheap) general case first - if length <= defaultBufSize || length <= cap(b.buf) { - return b.buf[:length] + if length <= cap(b.buf) { + return b.buf[:length], nil } if length < maxPacketSize { b.buf = make([]byte, length) - return b.buf + return b.buf, nil } - return make([]byte, length) + + // buffer is larger than we want to store. + return make([]byte, length), nil } -// shortcut which can be used if the requested buffer is guaranteed to be -// smaller than defaultBufSize +// takeSmallBuffer is shortcut which can be used if length is +// known to be smaller than defaultBufSize. // Only one buffer (total) can be used at a time. -func (b *buffer) takeSmallBuffer(length int) []byte { +func (b *buffer) takeSmallBuffer(length int) ([]byte, error) { if b.length > 0 { - return nil + return nil, ErrBusyBuffer } - return b.buf[:length] + return b.buf[:length], nil } // takeCompleteBuffer returns the complete existing buffer. // This can be used if the necessary buffer size is unknown. +// cap and len of the returned buffer will be equal. // Only one buffer (total) can be used at a time. -func (b *buffer) takeCompleteBuffer() []byte { +func (b *buffer) takeCompleteBuffer() ([]byte, error) { + if b.length > 0 { + return nil, ErrBusyBuffer + } + return b.buf, nil +} + +// store stores buf, an updated buffer, if its suitable to do so. +func (b *buffer) store(buf []byte) error { if b.length > 0 { - return nil + return ErrBusyBuffer + } else if cap(buf) <= maxPacketSize && cap(buf) > cap(b.buf) { + b.buf = buf[:cap(buf)] } - return b.buf + return nil } diff --git a/vendor/github.com/go-sql-driver/mysql/collations.go b/vendor/github.com/go-sql-driver/mysql/collations.go index 136c9e4d..8d2b5567 100644 --- a/vendor/github.com/go-sql-driver/mysql/collations.go +++ b/vendor/github.com/go-sql-driver/mysql/collations.go @@ -8,183 +8,190 @@ package mysql -const defaultCollation = "utf8_general_ci" +const defaultCollation = "utf8mb4_general_ci" const binaryCollation = "binary" // A list of available collations mapped to the internal ID. // To update this map use the following MySQL query: -// SELECT COLLATION_NAME, ID FROM information_schema.COLLATIONS +// SELECT COLLATION_NAME, ID FROM information_schema.COLLATIONS WHERE ID<256 ORDER BY ID +// +// Handshake packet have only 1 byte for collation_id. So we can't use collations with ID > 255. +// +// ucs2, utf16, and utf32 can't be used for connection charset. +// https://dev.mysql.com/doc/refman/5.7/en/charset-connection.html#charset-connection-impermissible-client-charset +// They are commented out to reduce this map. var collations = map[string]byte{ - "big5_chinese_ci": 1, - "latin2_czech_cs": 2, - "dec8_swedish_ci": 3, - "cp850_general_ci": 4, - "latin1_german1_ci": 5, - "hp8_english_ci": 6, - "koi8r_general_ci": 7, - "latin1_swedish_ci": 8, - "latin2_general_ci": 9, - "swe7_swedish_ci": 10, - "ascii_general_ci": 11, - "ujis_japanese_ci": 12, - "sjis_japanese_ci": 13, - "cp1251_bulgarian_ci": 14, - "latin1_danish_ci": 15, - "hebrew_general_ci": 16, - "tis620_thai_ci": 18, - "euckr_korean_ci": 19, - "latin7_estonian_cs": 20, - "latin2_hungarian_ci": 21, - "koi8u_general_ci": 22, - "cp1251_ukrainian_ci": 23, - "gb2312_chinese_ci": 24, - "greek_general_ci": 25, - "cp1250_general_ci": 26, - "latin2_croatian_ci": 27, - "gbk_chinese_ci": 28, - "cp1257_lithuanian_ci": 29, - "latin5_turkish_ci": 30, - "latin1_german2_ci": 31, - "armscii8_general_ci": 32, - "utf8_general_ci": 33, - "cp1250_czech_cs": 34, - "ucs2_general_ci": 35, - "cp866_general_ci": 36, - "keybcs2_general_ci": 37, - "macce_general_ci": 38, - "macroman_general_ci": 39, - "cp852_general_ci": 40, - "latin7_general_ci": 41, - "latin7_general_cs": 42, - "macce_bin": 43, - "cp1250_croatian_ci": 44, - "utf8mb4_general_ci": 45, - "utf8mb4_bin": 46, - "latin1_bin": 47, - "latin1_general_ci": 48, - "latin1_general_cs": 49, - "cp1251_bin": 50, - "cp1251_general_ci": 51, - "cp1251_general_cs": 52, - "macroman_bin": 53, - "utf16_general_ci": 54, - "utf16_bin": 55, - "utf16le_general_ci": 56, - "cp1256_general_ci": 57, - "cp1257_bin": 58, - "cp1257_general_ci": 59, - "utf32_general_ci": 60, - "utf32_bin": 61, - "utf16le_bin": 62, - "binary": 63, - "armscii8_bin": 64, - "ascii_bin": 65, - "cp1250_bin": 66, - "cp1256_bin": 67, - "cp866_bin": 68, - "dec8_bin": 69, - "greek_bin": 70, - "hebrew_bin": 71, - "hp8_bin": 72, - "keybcs2_bin": 73, - "koi8r_bin": 74, - "koi8u_bin": 75, - "latin2_bin": 77, - "latin5_bin": 78, - "latin7_bin": 79, - "cp850_bin": 80, - "cp852_bin": 81, - "swe7_bin": 82, - "utf8_bin": 83, - "big5_bin": 84, - "euckr_bin": 85, - "gb2312_bin": 86, - "gbk_bin": 87, - "sjis_bin": 88, - "tis620_bin": 89, - "ucs2_bin": 90, - "ujis_bin": 91, - "geostd8_general_ci": 92, - "geostd8_bin": 93, - "latin1_spanish_ci": 94, - "cp932_japanese_ci": 95, - "cp932_bin": 96, - "eucjpms_japanese_ci": 97, - "eucjpms_bin": 98, - "cp1250_polish_ci": 99, - "utf16_unicode_ci": 101, - "utf16_icelandic_ci": 102, - "utf16_latvian_ci": 103, - "utf16_romanian_ci": 104, - "utf16_slovenian_ci": 105, - "utf16_polish_ci": 106, - "utf16_estonian_ci": 107, - "utf16_spanish_ci": 108, - "utf16_swedish_ci": 109, - "utf16_turkish_ci": 110, - "utf16_czech_ci": 111, - "utf16_danish_ci": 112, - "utf16_lithuanian_ci": 113, - "utf16_slovak_ci": 114, - "utf16_spanish2_ci": 115, - "utf16_roman_ci": 116, - "utf16_persian_ci": 117, - "utf16_esperanto_ci": 118, - "utf16_hungarian_ci": 119, - "utf16_sinhala_ci": 120, - "utf16_german2_ci": 121, - "utf16_croatian_ci": 122, - "utf16_unicode_520_ci": 123, - "utf16_vietnamese_ci": 124, - "ucs2_unicode_ci": 128, - "ucs2_icelandic_ci": 129, - "ucs2_latvian_ci": 130, - "ucs2_romanian_ci": 131, - "ucs2_slovenian_ci": 132, - "ucs2_polish_ci": 133, - "ucs2_estonian_ci": 134, - "ucs2_spanish_ci": 135, - "ucs2_swedish_ci": 136, - "ucs2_turkish_ci": 137, - "ucs2_czech_ci": 138, - "ucs2_danish_ci": 139, - "ucs2_lithuanian_ci": 140, - "ucs2_slovak_ci": 141, - "ucs2_spanish2_ci": 142, - "ucs2_roman_ci": 143, - "ucs2_persian_ci": 144, - "ucs2_esperanto_ci": 145, - "ucs2_hungarian_ci": 146, - "ucs2_sinhala_ci": 147, - "ucs2_german2_ci": 148, - "ucs2_croatian_ci": 149, - "ucs2_unicode_520_ci": 150, - "ucs2_vietnamese_ci": 151, - "ucs2_general_mysql500_ci": 159, - "utf32_unicode_ci": 160, - "utf32_icelandic_ci": 161, - "utf32_latvian_ci": 162, - "utf32_romanian_ci": 163, - "utf32_slovenian_ci": 164, - "utf32_polish_ci": 165, - "utf32_estonian_ci": 166, - "utf32_spanish_ci": 167, - "utf32_swedish_ci": 168, - "utf32_turkish_ci": 169, - "utf32_czech_ci": 170, - "utf32_danish_ci": 171, - "utf32_lithuanian_ci": 172, - "utf32_slovak_ci": 173, - "utf32_spanish2_ci": 174, - "utf32_roman_ci": 175, - "utf32_persian_ci": 176, - "utf32_esperanto_ci": 177, - "utf32_hungarian_ci": 178, - "utf32_sinhala_ci": 179, - "utf32_german2_ci": 180, - "utf32_croatian_ci": 181, - "utf32_unicode_520_ci": 182, - "utf32_vietnamese_ci": 183, + "big5_chinese_ci": 1, + "latin2_czech_cs": 2, + "dec8_swedish_ci": 3, + "cp850_general_ci": 4, + "latin1_german1_ci": 5, + "hp8_english_ci": 6, + "koi8r_general_ci": 7, + "latin1_swedish_ci": 8, + "latin2_general_ci": 9, + "swe7_swedish_ci": 10, + "ascii_general_ci": 11, + "ujis_japanese_ci": 12, + "sjis_japanese_ci": 13, + "cp1251_bulgarian_ci": 14, + "latin1_danish_ci": 15, + "hebrew_general_ci": 16, + "tis620_thai_ci": 18, + "euckr_korean_ci": 19, + "latin7_estonian_cs": 20, + "latin2_hungarian_ci": 21, + "koi8u_general_ci": 22, + "cp1251_ukrainian_ci": 23, + "gb2312_chinese_ci": 24, + "greek_general_ci": 25, + "cp1250_general_ci": 26, + "latin2_croatian_ci": 27, + "gbk_chinese_ci": 28, + "cp1257_lithuanian_ci": 29, + "latin5_turkish_ci": 30, + "latin1_german2_ci": 31, + "armscii8_general_ci": 32, + "utf8_general_ci": 33, + "cp1250_czech_cs": 34, + //"ucs2_general_ci": 35, + "cp866_general_ci": 36, + "keybcs2_general_ci": 37, + "macce_general_ci": 38, + "macroman_general_ci": 39, + "cp852_general_ci": 40, + "latin7_general_ci": 41, + "latin7_general_cs": 42, + "macce_bin": 43, + "cp1250_croatian_ci": 44, + "utf8mb4_general_ci": 45, + "utf8mb4_bin": 46, + "latin1_bin": 47, + "latin1_general_ci": 48, + "latin1_general_cs": 49, + "cp1251_bin": 50, + "cp1251_general_ci": 51, + "cp1251_general_cs": 52, + "macroman_bin": 53, + //"utf16_general_ci": 54, + //"utf16_bin": 55, + //"utf16le_general_ci": 56, + "cp1256_general_ci": 57, + "cp1257_bin": 58, + "cp1257_general_ci": 59, + //"utf32_general_ci": 60, + //"utf32_bin": 61, + //"utf16le_bin": 62, + "binary": 63, + "armscii8_bin": 64, + "ascii_bin": 65, + "cp1250_bin": 66, + "cp1256_bin": 67, + "cp866_bin": 68, + "dec8_bin": 69, + "greek_bin": 70, + "hebrew_bin": 71, + "hp8_bin": 72, + "keybcs2_bin": 73, + "koi8r_bin": 74, + "koi8u_bin": 75, + "utf8_tolower_ci": 76, + "latin2_bin": 77, + "latin5_bin": 78, + "latin7_bin": 79, + "cp850_bin": 80, + "cp852_bin": 81, + "swe7_bin": 82, + "utf8_bin": 83, + "big5_bin": 84, + "euckr_bin": 85, + "gb2312_bin": 86, + "gbk_bin": 87, + "sjis_bin": 88, + "tis620_bin": 89, + //"ucs2_bin": 90, + "ujis_bin": 91, + "geostd8_general_ci": 92, + "geostd8_bin": 93, + "latin1_spanish_ci": 94, + "cp932_japanese_ci": 95, + "cp932_bin": 96, + "eucjpms_japanese_ci": 97, + "eucjpms_bin": 98, + "cp1250_polish_ci": 99, + //"utf16_unicode_ci": 101, + //"utf16_icelandic_ci": 102, + //"utf16_latvian_ci": 103, + //"utf16_romanian_ci": 104, + //"utf16_slovenian_ci": 105, + //"utf16_polish_ci": 106, + //"utf16_estonian_ci": 107, + //"utf16_spanish_ci": 108, + //"utf16_swedish_ci": 109, + //"utf16_turkish_ci": 110, + //"utf16_czech_ci": 111, + //"utf16_danish_ci": 112, + //"utf16_lithuanian_ci": 113, + //"utf16_slovak_ci": 114, + //"utf16_spanish2_ci": 115, + //"utf16_roman_ci": 116, + //"utf16_persian_ci": 117, + //"utf16_esperanto_ci": 118, + //"utf16_hungarian_ci": 119, + //"utf16_sinhala_ci": 120, + //"utf16_german2_ci": 121, + //"utf16_croatian_ci": 122, + //"utf16_unicode_520_ci": 123, + //"utf16_vietnamese_ci": 124, + //"ucs2_unicode_ci": 128, + //"ucs2_icelandic_ci": 129, + //"ucs2_latvian_ci": 130, + //"ucs2_romanian_ci": 131, + //"ucs2_slovenian_ci": 132, + //"ucs2_polish_ci": 133, + //"ucs2_estonian_ci": 134, + //"ucs2_spanish_ci": 135, + //"ucs2_swedish_ci": 136, + //"ucs2_turkish_ci": 137, + //"ucs2_czech_ci": 138, + //"ucs2_danish_ci": 139, + //"ucs2_lithuanian_ci": 140, + //"ucs2_slovak_ci": 141, + //"ucs2_spanish2_ci": 142, + //"ucs2_roman_ci": 143, + //"ucs2_persian_ci": 144, + //"ucs2_esperanto_ci": 145, + //"ucs2_hungarian_ci": 146, + //"ucs2_sinhala_ci": 147, + //"ucs2_german2_ci": 148, + //"ucs2_croatian_ci": 149, + //"ucs2_unicode_520_ci": 150, + //"ucs2_vietnamese_ci": 151, + //"ucs2_general_mysql500_ci": 159, + //"utf32_unicode_ci": 160, + //"utf32_icelandic_ci": 161, + //"utf32_latvian_ci": 162, + //"utf32_romanian_ci": 163, + //"utf32_slovenian_ci": 164, + //"utf32_polish_ci": 165, + //"utf32_estonian_ci": 166, + //"utf32_spanish_ci": 167, + //"utf32_swedish_ci": 168, + //"utf32_turkish_ci": 169, + //"utf32_czech_ci": 170, + //"utf32_danish_ci": 171, + //"utf32_lithuanian_ci": 172, + //"utf32_slovak_ci": 173, + //"utf32_spanish2_ci": 174, + //"utf32_roman_ci": 175, + //"utf32_persian_ci": 176, + //"utf32_esperanto_ci": 177, + //"utf32_hungarian_ci": 178, + //"utf32_sinhala_ci": 179, + //"utf32_german2_ci": 180, + //"utf32_croatian_ci": 181, + //"utf32_unicode_520_ci": 182, + //"utf32_vietnamese_ci": 183, "utf8_unicode_ci": 192, "utf8_icelandic_ci": 193, "utf8_latvian_ci": 194, @@ -234,18 +241,25 @@ var collations = map[string]byte{ "utf8mb4_croatian_ci": 245, "utf8mb4_unicode_520_ci": 246, "utf8mb4_vietnamese_ci": 247, + "gb18030_chinese_ci": 248, + "gb18030_bin": 249, + "gb18030_unicode_520_ci": 250, + "utf8mb4_0900_ai_ci": 255, } // A blacklist of collations which is unsafe to interpolate parameters. // These multibyte encodings may contains 0x5c (`\`) in their trailing bytes. var unsafeCollations = map[string]bool{ - "big5_chinese_ci": true, - "sjis_japanese_ci": true, - "gbk_chinese_ci": true, - "big5_bin": true, - "gb2312_bin": true, - "gbk_bin": true, - "sjis_bin": true, - "cp932_japanese_ci": true, - "cp932_bin": true, + "big5_chinese_ci": true, + "sjis_japanese_ci": true, + "gbk_chinese_ci": true, + "big5_bin": true, + "gb2312_bin": true, + "gbk_bin": true, + "sjis_bin": true, + "cp932_japanese_ci": true, + "cp932_bin": true, + "gb18030_chinese_ci": true, + "gb18030_bin": true, + "gb18030_unicode_520_ci": true, } diff --git a/vendor/github.com/go-sql-driver/mysql/conncheck.go b/vendor/github.com/go-sql-driver/mysql/conncheck.go new file mode 100644 index 00000000..024eb285 --- /dev/null +++ b/vendor/github.com/go-sql-driver/mysql/conncheck.go @@ -0,0 +1,54 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2019 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +// +build linux darwin dragonfly freebsd netbsd openbsd solaris illumos + +package mysql + +import ( + "errors" + "io" + "net" + "syscall" +) + +var errUnexpectedRead = errors.New("unexpected read from socket") + +func connCheck(conn net.Conn) error { + var sysErr error + + sysConn, ok := conn.(syscall.Conn) + if !ok { + return nil + } + rawConn, err := sysConn.SyscallConn() + if err != nil { + return err + } + + err = rawConn.Read(func(fd uintptr) bool { + var buf [1]byte + n, err := syscall.Read(int(fd), buf[:]) + switch { + case n == 0 && err == nil: + sysErr = io.EOF + case n > 0: + sysErr = errUnexpectedRead + case err == syscall.EAGAIN || err == syscall.EWOULDBLOCK: + sysErr = nil + default: + sysErr = err + } + return true + }) + if err != nil { + return err + } + + return sysErr +} diff --git a/vendor/github.com/go-sql-driver/mysql/appengine.go b/vendor/github.com/go-sql-driver/mysql/conncheck_dummy.go similarity index 59% rename from vendor/github.com/go-sql-driver/mysql/appengine.go rename to vendor/github.com/go-sql-driver/mysql/conncheck_dummy.go index be41f2ee..ea7fb607 100644 --- a/vendor/github.com/go-sql-driver/mysql/appengine.go +++ b/vendor/github.com/go-sql-driver/mysql/conncheck_dummy.go @@ -1,19 +1,17 @@ // Go MySQL Driver - A MySQL-Driver for Go's database/sql package // -// Copyright 2013 The Go-MySQL-Driver Authors. All rights reserved. +// Copyright 2019 The Go-MySQL-Driver Authors. All rights reserved. // // This Source Code Form is subject to the terms of the Mozilla Public // License, v. 2.0. If a copy of the MPL was not distributed with this file, // You can obtain one at http://mozilla.org/MPL/2.0/. -// +build appengine +// +build !linux,!darwin,!dragonfly,!freebsd,!netbsd,!openbsd,!solaris,!illumos package mysql -import ( - "google.golang.org/appengine/cloudsql" -) +import "net" -func init() { - RegisterDial("cloudsql", cloudsql.Dial) +func connCheck(conn net.Conn) error { + return nil } diff --git a/vendor/github.com/go-sql-driver/mysql/connection.go b/vendor/github.com/go-sql-driver/mysql/connection.go index e5706141..e4bb59e6 100644 --- a/vendor/github.com/go-sql-driver/mysql/connection.go +++ b/vendor/github.com/go-sql-driver/mysql/connection.go @@ -9,6 +9,8 @@ package mysql import ( + "context" + "database/sql" "database/sql/driver" "io" "net" @@ -17,19 +19,10 @@ import ( "time" ) -// a copy of context.Context for Go 1.7 and earlier -type mysqlContext interface { - Done() <-chan struct{} - Err() error - - // defined in context.Context, but not used in this driver: - // Deadline() (deadline time.Time, ok bool) - // Value(key interface{}) interface{} -} - type mysqlConn struct { buf buffer netConn net.Conn + rawConn net.Conn // underlying connection when netConn is TLS connection. affectedRows uint64 insertId uint64 cfg *Config @@ -40,10 +33,11 @@ type mysqlConn struct { status statusFlag sequence uint8 parseTime bool + reset bool // set when the Go SQL package calls ResetSession // for context support (Go 1.8+) watching bool - watcher chan<- mysqlContext + watcher chan<- context.Context closech chan struct{} finished chan<- struct{} canceled atomicError // set non-nil if conn is canceled @@ -160,7 +154,9 @@ func (mc *mysqlConn) Prepare(query string) (driver.Stmt, error) { // Send command err := mc.writeCommandPacketStr(comStmtPrepare, query) if err != nil { - return nil, mc.markBadConn(err) + // STMT_PREPARE is safe to retry. So we can return ErrBadConn here. + errLog.Print(err) + return nil, driver.ErrBadConn } stmt := &mysqlStmt{ @@ -190,10 +186,10 @@ func (mc *mysqlConn) interpolateParams(query string, args []driver.Value) (strin return "", driver.ErrSkip } - buf := mc.buf.takeCompleteBuffer() - if buf == nil { + buf, err := mc.buf.takeCompleteBuffer() + if err != nil { // can not take the buffer. Something must be wrong with the connection - errLog.Print(ErrBusyBuffer) + errLog.Print(err) return "", ErrInvalidConn } buf = buf[:0] @@ -219,6 +215,9 @@ func (mc *mysqlConn) interpolateParams(query string, args []driver.Value) (strin switch v := arg.(type) { case int64: buf = strconv.AppendInt(buf, v, 10) + case uint64: + // Handle uint64 explicitly because our custom ConvertValue emits unsigned values + buf = strconv.AppendUint(buf, v, 10) case float64: buf = strconv.AppendFloat(buf, v, 'g', -1, 64) case bool: @@ -459,3 +458,194 @@ func (mc *mysqlConn) finish() { case <-mc.closech: } } + +// Ping implements driver.Pinger interface +func (mc *mysqlConn) Ping(ctx context.Context) (err error) { + if mc.closed.IsSet() { + errLog.Print(ErrInvalidConn) + return driver.ErrBadConn + } + + if err = mc.watchCancel(ctx); err != nil { + return + } + defer mc.finish() + + if err = mc.writeCommandPacket(comPing); err != nil { + return mc.markBadConn(err) + } + + return mc.readResultOK() +} + +// BeginTx implements driver.ConnBeginTx interface +func (mc *mysqlConn) BeginTx(ctx context.Context, opts driver.TxOptions) (driver.Tx, error) { + if err := mc.watchCancel(ctx); err != nil { + return nil, err + } + defer mc.finish() + + if sql.IsolationLevel(opts.Isolation) != sql.LevelDefault { + level, err := mapIsolationLevel(opts.Isolation) + if err != nil { + return nil, err + } + err = mc.exec("SET TRANSACTION ISOLATION LEVEL " + level) + if err != nil { + return nil, err + } + } + + return mc.begin(opts.ReadOnly) +} + +func (mc *mysqlConn) QueryContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Rows, error) { + dargs, err := namedValueToValue(args) + if err != nil { + return nil, err + } + + if err := mc.watchCancel(ctx); err != nil { + return nil, err + } + + rows, err := mc.query(query, dargs) + if err != nil { + mc.finish() + return nil, err + } + rows.finish = mc.finish + return rows, err +} + +func (mc *mysqlConn) ExecContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Result, error) { + dargs, err := namedValueToValue(args) + if err != nil { + return nil, err + } + + if err := mc.watchCancel(ctx); err != nil { + return nil, err + } + defer mc.finish() + + return mc.Exec(query, dargs) +} + +func (mc *mysqlConn) PrepareContext(ctx context.Context, query string) (driver.Stmt, error) { + if err := mc.watchCancel(ctx); err != nil { + return nil, err + } + + stmt, err := mc.Prepare(query) + mc.finish() + if err != nil { + return nil, err + } + + select { + default: + case <-ctx.Done(): + stmt.Close() + return nil, ctx.Err() + } + return stmt, nil +} + +func (stmt *mysqlStmt) QueryContext(ctx context.Context, args []driver.NamedValue) (driver.Rows, error) { + dargs, err := namedValueToValue(args) + if err != nil { + return nil, err + } + + if err := stmt.mc.watchCancel(ctx); err != nil { + return nil, err + } + + rows, err := stmt.query(dargs) + if err != nil { + stmt.mc.finish() + return nil, err + } + rows.finish = stmt.mc.finish + return rows, err +} + +func (stmt *mysqlStmt) ExecContext(ctx context.Context, args []driver.NamedValue) (driver.Result, error) { + dargs, err := namedValueToValue(args) + if err != nil { + return nil, err + } + + if err := stmt.mc.watchCancel(ctx); err != nil { + return nil, err + } + defer stmt.mc.finish() + + return stmt.Exec(dargs) +} + +func (mc *mysqlConn) watchCancel(ctx context.Context) error { + if mc.watching { + // Reach here if canceled, + // so the connection is already invalid + mc.cleanup() + return nil + } + // When ctx is already cancelled, don't watch it. + if err := ctx.Err(); err != nil { + return err + } + // When ctx is not cancellable, don't watch it. + if ctx.Done() == nil { + return nil + } + // When watcher is not alive, can't watch it. + if mc.watcher == nil { + return nil + } + + mc.watching = true + mc.watcher <- ctx + return nil +} + +func (mc *mysqlConn) startWatcher() { + watcher := make(chan context.Context, 1) + mc.watcher = watcher + finished := make(chan struct{}) + mc.finished = finished + go func() { + for { + var ctx context.Context + select { + case ctx = <-watcher: + case <-mc.closech: + return + } + + select { + case <-ctx.Done(): + mc.cancel(ctx.Err()) + case <-finished: + case <-mc.closech: + return + } + } + }() +} + +func (mc *mysqlConn) CheckNamedValue(nv *driver.NamedValue) (err error) { + nv.Value, err = converter{}.ConvertValue(nv.Value) + return +} + +// ResetSession implements driver.SessionResetter. +// (From Go 1.10) +func (mc *mysqlConn) ResetSession(ctx context.Context) error { + if mc.closed.IsSet() { + return driver.ErrBadConn + } + mc.reset = true + return nil +} diff --git a/vendor/github.com/go-sql-driver/mysql/connection_go18.go b/vendor/github.com/go-sql-driver/mysql/connection_go18.go deleted file mode 100644 index 62796bfc..00000000 --- a/vendor/github.com/go-sql-driver/mysql/connection_go18.go +++ /dev/null @@ -1,208 +0,0 @@ -// Go MySQL Driver - A MySQL-Driver for Go's database/sql package -// -// Copyright 2012 The Go-MySQL-Driver Authors. All rights reserved. -// -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this file, -// You can obtain one at http://mozilla.org/MPL/2.0/. - -// +build go1.8 - -package mysql - -import ( - "context" - "database/sql" - "database/sql/driver" -) - -// Ping implements driver.Pinger interface -func (mc *mysqlConn) Ping(ctx context.Context) (err error) { - if mc.closed.IsSet() { - errLog.Print(ErrInvalidConn) - return driver.ErrBadConn - } - - if err = mc.watchCancel(ctx); err != nil { - return - } - defer mc.finish() - - if err = mc.writeCommandPacket(comPing); err != nil { - return - } - - return mc.readResultOK() -} - -// BeginTx implements driver.ConnBeginTx interface -func (mc *mysqlConn) BeginTx(ctx context.Context, opts driver.TxOptions) (driver.Tx, error) { - if err := mc.watchCancel(ctx); err != nil { - return nil, err - } - defer mc.finish() - - if sql.IsolationLevel(opts.Isolation) != sql.LevelDefault { - level, err := mapIsolationLevel(opts.Isolation) - if err != nil { - return nil, err - } - err = mc.exec("SET TRANSACTION ISOLATION LEVEL " + level) - if err != nil { - return nil, err - } - } - - return mc.begin(opts.ReadOnly) -} - -func (mc *mysqlConn) QueryContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Rows, error) { - dargs, err := namedValueToValue(args) - if err != nil { - return nil, err - } - - if err := mc.watchCancel(ctx); err != nil { - return nil, err - } - - rows, err := mc.query(query, dargs) - if err != nil { - mc.finish() - return nil, err - } - rows.finish = mc.finish - return rows, err -} - -func (mc *mysqlConn) ExecContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Result, error) { - dargs, err := namedValueToValue(args) - if err != nil { - return nil, err - } - - if err := mc.watchCancel(ctx); err != nil { - return nil, err - } - defer mc.finish() - - return mc.Exec(query, dargs) -} - -func (mc *mysqlConn) PrepareContext(ctx context.Context, query string) (driver.Stmt, error) { - if err := mc.watchCancel(ctx); err != nil { - return nil, err - } - - stmt, err := mc.Prepare(query) - mc.finish() - if err != nil { - return nil, err - } - - select { - default: - case <-ctx.Done(): - stmt.Close() - return nil, ctx.Err() - } - return stmt, nil -} - -func (stmt *mysqlStmt) QueryContext(ctx context.Context, args []driver.NamedValue) (driver.Rows, error) { - dargs, err := namedValueToValue(args) - if err != nil { - return nil, err - } - - if err := stmt.mc.watchCancel(ctx); err != nil { - return nil, err - } - - rows, err := stmt.query(dargs) - if err != nil { - stmt.mc.finish() - return nil, err - } - rows.finish = stmt.mc.finish - return rows, err -} - -func (stmt *mysqlStmt) ExecContext(ctx context.Context, args []driver.NamedValue) (driver.Result, error) { - dargs, err := namedValueToValue(args) - if err != nil { - return nil, err - } - - if err := stmt.mc.watchCancel(ctx); err != nil { - return nil, err - } - defer stmt.mc.finish() - - return stmt.Exec(dargs) -} - -func (mc *mysqlConn) watchCancel(ctx context.Context) error { - if mc.watching { - // Reach here if canceled, - // so the connection is already invalid - mc.cleanup() - return nil - } - if ctx.Done() == nil { - return nil - } - - mc.watching = true - select { - default: - case <-ctx.Done(): - return ctx.Err() - } - if mc.watcher == nil { - return nil - } - - mc.watcher <- ctx - - return nil -} - -func (mc *mysqlConn) startWatcher() { - watcher := make(chan mysqlContext, 1) - mc.watcher = watcher - finished := make(chan struct{}) - mc.finished = finished - go func() { - for { - var ctx mysqlContext - select { - case ctx = <-watcher: - case <-mc.closech: - return - } - - select { - case <-ctx.Done(): - mc.cancel(ctx.Err()) - case <-finished: - case <-mc.closech: - return - } - } - }() -} - -func (mc *mysqlConn) CheckNamedValue(nv *driver.NamedValue) (err error) { - nv.Value, err = converter{}.ConvertValue(nv.Value) - return -} - -// ResetSession implements driver.SessionResetter. -// (From Go 1.10) -func (mc *mysqlConn) ResetSession(ctx context.Context) error { - if mc.closed.IsSet() { - return driver.ErrBadConn - } - return nil -} diff --git a/vendor/github.com/go-sql-driver/mysql/connector.go b/vendor/github.com/go-sql-driver/mysql/connector.go new file mode 100644 index 00000000..d567b4e4 --- /dev/null +++ b/vendor/github.com/go-sql-driver/mysql/connector.go @@ -0,0 +1,146 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2018 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package mysql + +import ( + "context" + "database/sql/driver" + "net" +) + +type connector struct { + cfg *Config // immutable private copy. +} + +// Connect implements driver.Connector interface. +// Connect returns a connection to the database. +func (c *connector) Connect(ctx context.Context) (driver.Conn, error) { + var err error + + // New mysqlConn + mc := &mysqlConn{ + maxAllowedPacket: maxPacketSize, + maxWriteSize: maxPacketSize - 1, + closech: make(chan struct{}), + cfg: c.cfg, + } + mc.parseTime = mc.cfg.ParseTime + + // Connect to Server + dialsLock.RLock() + dial, ok := dials[mc.cfg.Net] + dialsLock.RUnlock() + if ok { + dctx := ctx + if mc.cfg.Timeout > 0 { + var cancel context.CancelFunc + dctx, cancel = context.WithTimeout(ctx, c.cfg.Timeout) + defer cancel() + } + mc.netConn, err = dial(dctx, mc.cfg.Addr) + } else { + nd := net.Dialer{Timeout: mc.cfg.Timeout} + mc.netConn, err = nd.DialContext(ctx, mc.cfg.Net, mc.cfg.Addr) + } + + if err != nil { + return nil, err + } + + // Enable TCP Keepalives on TCP connections + if tc, ok := mc.netConn.(*net.TCPConn); ok { + if err := tc.SetKeepAlive(true); err != nil { + // Don't send COM_QUIT before handshake. + mc.netConn.Close() + mc.netConn = nil + return nil, err + } + } + + // Call startWatcher for context support (From Go 1.8) + mc.startWatcher() + if err := mc.watchCancel(ctx); err != nil { + mc.cleanup() + return nil, err + } + defer mc.finish() + + mc.buf = newBuffer(mc.netConn) + + // Set I/O timeouts + mc.buf.timeout = mc.cfg.ReadTimeout + mc.writeTimeout = mc.cfg.WriteTimeout + + // Reading Handshake Initialization Packet + authData, plugin, err := mc.readHandshakePacket() + if err != nil { + mc.cleanup() + return nil, err + } + + if plugin == "" { + plugin = defaultAuthPlugin + } + + // Send Client Authentication Packet + authResp, err := mc.auth(authData, plugin) + if err != nil { + // try the default auth plugin, if using the requested plugin failed + errLog.Print("could not use requested auth plugin '"+plugin+"': ", err.Error()) + plugin = defaultAuthPlugin + authResp, err = mc.auth(authData, plugin) + if err != nil { + mc.cleanup() + return nil, err + } + } + if err = mc.writeHandshakeResponsePacket(authResp, plugin); err != nil { + mc.cleanup() + return nil, err + } + + // Handle response to auth packet, switch methods if possible + if err = mc.handleAuthResult(authData, plugin); err != nil { + // Authentication failed and MySQL has already closed the connection + // (https://dev.mysql.com/doc/internals/en/authentication-fails.html). + // Do not send COM_QUIT, just cleanup and return the error. + mc.cleanup() + return nil, err + } + + if mc.cfg.MaxAllowedPacket > 0 { + mc.maxAllowedPacket = mc.cfg.MaxAllowedPacket + } else { + // Get max allowed packet size + maxap, err := mc.getSystemVar("max_allowed_packet") + if err != nil { + mc.Close() + return nil, err + } + mc.maxAllowedPacket = stringToInt(maxap) - 1 + } + if mc.maxAllowedPacket < maxPacketSize { + mc.maxWriteSize = mc.maxAllowedPacket + } + + // Handle DSN Params + err = mc.handleParams() + if err != nil { + mc.Close() + return nil, err + } + + return mc, nil +} + +// Driver implements driver.Connector interface. +// Driver returns &MySQLDriver{}. +func (c *connector) Driver() driver.Driver { + return &MySQLDriver{} +} diff --git a/vendor/github.com/go-sql-driver/mysql/driver.go b/vendor/github.com/go-sql-driver/mysql/driver.go index 1a75a16e..c1bdf119 100644 --- a/vendor/github.com/go-sql-driver/mysql/driver.go +++ b/vendor/github.com/go-sql-driver/mysql/driver.go @@ -17,153 +17,91 @@ package mysql import ( + "context" "database/sql" "database/sql/driver" "net" "sync" ) -// watcher interface is used for context support (From Go 1.8) -type watcher interface { - startWatcher() -} - // MySQLDriver is exported to make the driver directly accessible. // In general the driver is used via the database/sql package. type MySQLDriver struct{} // DialFunc is a function which can be used to establish the network connection. // Custom dial functions must be registered with RegisterDial +// +// Deprecated: users should register a DialContextFunc instead type DialFunc func(addr string) (net.Conn, error) +// DialContextFunc is a function which can be used to establish the network connection. +// Custom dial functions must be registered with RegisterDialContext +type DialContextFunc func(ctx context.Context, addr string) (net.Conn, error) + var ( dialsLock sync.RWMutex - dials map[string]DialFunc + dials map[string]DialContextFunc ) -// RegisterDial registers a custom dial function. It can then be used by the +// RegisterDialContext registers a custom dial function. It can then be used by the // network address mynet(addr), where mynet is the registered new network. -// addr is passed as a parameter to the dial function. -func RegisterDial(net string, dial DialFunc) { +// The current context for the connection and its address is passed to the dial function. +func RegisterDialContext(net string, dial DialContextFunc) { dialsLock.Lock() defer dialsLock.Unlock() if dials == nil { - dials = make(map[string]DialFunc) + dials = make(map[string]DialContextFunc) } dials[net] = dial } +// RegisterDial registers a custom dial function. It can then be used by the +// network address mynet(addr), where mynet is the registered new network. +// addr is passed as a parameter to the dial function. +// +// Deprecated: users should call RegisterDialContext instead +func RegisterDial(network string, dial DialFunc) { + RegisterDialContext(network, func(_ context.Context, addr string) (net.Conn, error) { + return dial(addr) + }) +} + // Open new Connection. // See https://github.com/go-sql-driver/mysql#dsn-data-source-name for how -// the DSN string is formated +// the DSN string is formatted func (d MySQLDriver) Open(dsn string) (driver.Conn, error) { - var err error - - // New mysqlConn - mc := &mysqlConn{ - maxAllowedPacket: maxPacketSize, - maxWriteSize: maxPacketSize - 1, - closech: make(chan struct{}), - } - mc.cfg, err = ParseDSN(dsn) + cfg, err := ParseDSN(dsn) if err != nil { return nil, err } - mc.parseTime = mc.cfg.ParseTime - - // Connect to Server - dialsLock.RLock() - dial, ok := dials[mc.cfg.Net] - dialsLock.RUnlock() - if ok { - mc.netConn, err = dial(mc.cfg.Addr) - } else { - nd := net.Dialer{Timeout: mc.cfg.Timeout} - mc.netConn, err = nd.Dial(mc.cfg.Net, mc.cfg.Addr) - } - if err != nil { - return nil, err - } - - // Enable TCP Keepalives on TCP connections - if tc, ok := mc.netConn.(*net.TCPConn); ok { - if err := tc.SetKeepAlive(true); err != nil { - // Don't send COM_QUIT before handshake. - mc.netConn.Close() - mc.netConn = nil - return nil, err - } - } - - // Call startWatcher for context support (From Go 1.8) - if s, ok := interface{}(mc).(watcher); ok { - s.startWatcher() - } - - mc.buf = newBuffer(mc.netConn) - - // Set I/O timeouts - mc.buf.timeout = mc.cfg.ReadTimeout - mc.writeTimeout = mc.cfg.WriteTimeout - - // Reading Handshake Initialization Packet - authData, plugin, err := mc.readHandshakePacket() - if err != nil { - mc.cleanup() - return nil, err + c := &connector{ + cfg: cfg, } + return c.Connect(context.Background()) +} - // Send Client Authentication Packet - authResp, addNUL, err := mc.auth(authData, plugin) - if err != nil { - // try the default auth plugin, if using the requested plugin failed - errLog.Print("could not use requested auth plugin '"+plugin+"': ", err.Error()) - plugin = defaultAuthPlugin - authResp, addNUL, err = mc.auth(authData, plugin) - if err != nil { - mc.cleanup() - return nil, err - } - } - if err = mc.writeHandshakeResponsePacket(authResp, addNUL, plugin); err != nil { - mc.cleanup() - return nil, err - } +func init() { + sql.Register("mysql", &MySQLDriver{}) +} - // Handle response to auth packet, switch methods if possible - if err = mc.handleAuthResult(authData, plugin); err != nil { - // Authentication failed and MySQL has already closed the connection - // (https://dev.mysql.com/doc/internals/en/authentication-fails.html). - // Do not send COM_QUIT, just cleanup and return the error. - mc.cleanup() +// NewConnector returns new driver.Connector. +func NewConnector(cfg *Config) (driver.Connector, error) { + cfg = cfg.Clone() + // normalize the contents of cfg so calls to NewConnector have the same + // behavior as MySQLDriver.OpenConnector + if err := cfg.normalize(); err != nil { return nil, err } + return &connector{cfg: cfg}, nil +} - if mc.cfg.MaxAllowedPacket > 0 { - mc.maxAllowedPacket = mc.cfg.MaxAllowedPacket - } else { - // Get max allowed packet size - maxap, err := mc.getSystemVar("max_allowed_packet") - if err != nil { - mc.Close() - return nil, err - } - mc.maxAllowedPacket = stringToInt(maxap) - 1 - } - if mc.maxAllowedPacket < maxPacketSize { - mc.maxWriteSize = mc.maxAllowedPacket - } - - // Handle DSN Params - err = mc.handleParams() +// OpenConnector implements driver.DriverContext. +func (d MySQLDriver) OpenConnector(dsn string) (driver.Connector, error) { + cfg, err := ParseDSN(dsn) if err != nil { - mc.Close() return nil, err } - - return mc, nil -} - -func init() { - sql.Register("mysql", &MySQLDriver{}) + return &connector{ + cfg: cfg, + }, nil } diff --git a/vendor/github.com/go-sql-driver/mysql/dsn.go b/vendor/github.com/go-sql-driver/mysql/dsn.go index be014bab..75c8c248 100644 --- a/vendor/github.com/go-sql-driver/mysql/dsn.go +++ b/vendor/github.com/go-sql-driver/mysql/dsn.go @@ -14,6 +14,7 @@ import ( "crypto/tls" "errors" "fmt" + "math/big" "net" "net/url" "sort" @@ -54,6 +55,7 @@ type Config struct { AllowCleartextPasswords bool // Allows the cleartext client side plugin AllowNativePasswords bool // Allows the native password authentication method AllowOldPasswords bool // Allows the old insecure password method + CheckConnLiveness bool // Check connections for liveness before using them ClientFoundRows bool // Return number of matching rows instead of rows changed ColumnsWithAlias bool // Prepend table alias to column names InterpolateParams bool // Interpolate placeholders into query string @@ -69,9 +71,30 @@ func NewConfig() *Config { Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, + CheckConnLiveness: true, } } +func (cfg *Config) Clone() *Config { + cp := *cfg + if cp.tls != nil { + cp.tls = cfg.tls.Clone() + } + if len(cp.Params) > 0 { + cp.Params = make(map[string]string, len(cfg.Params)) + for k, v := range cfg.Params { + cp.Params[k] = v + } + } + if cfg.pubKey != nil { + cp.pubKey = &rsa.PublicKey{ + N: new(big.Int).Set(cfg.pubKey.N), + E: cfg.pubKey.E, + } + } + return &cp +} + func (cfg *Config) normalize() error { if cfg.InterpolateParams && unsafeCollations[cfg.Collation] { return errInvalidDSNUnsafeCollation @@ -92,23 +115,54 @@ func (cfg *Config) normalize() error { default: return errors.New("default addr for network '" + cfg.Net + "' unknown") } - } else if cfg.Net == "tcp" { cfg.Addr = ensureHavePort(cfg.Addr) } - if cfg.tls != nil { - if cfg.tls.ServerName == "" && !cfg.tls.InsecureSkipVerify { - host, _, err := net.SplitHostPort(cfg.Addr) - if err == nil { - cfg.tls.ServerName = host - } + switch cfg.TLSConfig { + case "false", "": + // don't set anything + case "true": + cfg.tls = &tls.Config{} + case "skip-verify", "preferred": + cfg.tls = &tls.Config{InsecureSkipVerify: true} + default: + cfg.tls = getTLSConfigClone(cfg.TLSConfig) + if cfg.tls == nil { + return errors.New("invalid value / unknown config name: " + cfg.TLSConfig) + } + } + + if cfg.tls != nil && cfg.tls.ServerName == "" && !cfg.tls.InsecureSkipVerify { + host, _, err := net.SplitHostPort(cfg.Addr) + if err == nil { + cfg.tls.ServerName = host + } + } + + if cfg.ServerPubKey != "" { + cfg.pubKey = getServerPubKey(cfg.ServerPubKey) + if cfg.pubKey == nil { + return errors.New("invalid value / unknown server pub key name: " + cfg.ServerPubKey) } } return nil } +func writeDSNParam(buf *bytes.Buffer, hasParam *bool, name, value string) { + buf.Grow(1 + len(name) + 1 + len(value)) + if !*hasParam { + *hasParam = true + buf.WriteByte('?') + } else { + buf.WriteByte('&') + } + buf.WriteString(name) + buf.WriteByte('=') + buf.WriteString(value) +} + // FormatDSN formats the given Config into a DSN string which can be passed to // the driver. func (cfg *Config) FormatDSN() string { @@ -147,165 +201,75 @@ func (cfg *Config) FormatDSN() string { } if cfg.AllowCleartextPasswords { - if hasParam { - buf.WriteString("&allowCleartextPasswords=true") - } else { - hasParam = true - buf.WriteString("?allowCleartextPasswords=true") - } + writeDSNParam(&buf, &hasParam, "allowCleartextPasswords", "true") } if !cfg.AllowNativePasswords { - if hasParam { - buf.WriteString("&allowNativePasswords=false") - } else { - hasParam = true - buf.WriteString("?allowNativePasswords=false") - } + writeDSNParam(&buf, &hasParam, "allowNativePasswords", "false") } if cfg.AllowOldPasswords { - if hasParam { - buf.WriteString("&allowOldPasswords=true") - } else { - hasParam = true - buf.WriteString("?allowOldPasswords=true") - } + writeDSNParam(&buf, &hasParam, "allowOldPasswords", "true") + } + + if !cfg.CheckConnLiveness { + writeDSNParam(&buf, &hasParam, "checkConnLiveness", "false") } if cfg.ClientFoundRows { - if hasParam { - buf.WriteString("&clientFoundRows=true") - } else { - hasParam = true - buf.WriteString("?clientFoundRows=true") - } + writeDSNParam(&buf, &hasParam, "clientFoundRows", "true") } if col := cfg.Collation; col != defaultCollation && len(col) > 0 { - if hasParam { - buf.WriteString("&collation=") - } else { - hasParam = true - buf.WriteString("?collation=") - } - buf.WriteString(col) + writeDSNParam(&buf, &hasParam, "collation", col) } if cfg.ColumnsWithAlias { - if hasParam { - buf.WriteString("&columnsWithAlias=true") - } else { - hasParam = true - buf.WriteString("?columnsWithAlias=true") - } + writeDSNParam(&buf, &hasParam, "columnsWithAlias", "true") } if cfg.InterpolateParams { - if hasParam { - buf.WriteString("&interpolateParams=true") - } else { - hasParam = true - buf.WriteString("?interpolateParams=true") - } + writeDSNParam(&buf, &hasParam, "interpolateParams", "true") } if cfg.Loc != time.UTC && cfg.Loc != nil { - if hasParam { - buf.WriteString("&loc=") - } else { - hasParam = true - buf.WriteString("?loc=") - } - buf.WriteString(url.QueryEscape(cfg.Loc.String())) + writeDSNParam(&buf, &hasParam, "loc", url.QueryEscape(cfg.Loc.String())) } if cfg.MultiStatements { - if hasParam { - buf.WriteString("&multiStatements=true") - } else { - hasParam = true - buf.WriteString("?multiStatements=true") - } + writeDSNParam(&buf, &hasParam, "multiStatements", "true") } if cfg.ParseTime { - if hasParam { - buf.WriteString("&parseTime=true") - } else { - hasParam = true - buf.WriteString("?parseTime=true") - } + writeDSNParam(&buf, &hasParam, "parseTime", "true") } if cfg.ReadTimeout > 0 { - if hasParam { - buf.WriteString("&readTimeout=") - } else { - hasParam = true - buf.WriteString("?readTimeout=") - } - buf.WriteString(cfg.ReadTimeout.String()) + writeDSNParam(&buf, &hasParam, "readTimeout", cfg.ReadTimeout.String()) } if cfg.RejectReadOnly { - if hasParam { - buf.WriteString("&rejectReadOnly=true") - } else { - hasParam = true - buf.WriteString("?rejectReadOnly=true") - } + writeDSNParam(&buf, &hasParam, "rejectReadOnly", "true") } if len(cfg.ServerPubKey) > 0 { - if hasParam { - buf.WriteString("&serverPubKey=") - } else { - hasParam = true - buf.WriteString("?serverPubKey=") - } - buf.WriteString(url.QueryEscape(cfg.ServerPubKey)) + writeDSNParam(&buf, &hasParam, "serverPubKey", url.QueryEscape(cfg.ServerPubKey)) } if cfg.Timeout > 0 { - if hasParam { - buf.WriteString("&timeout=") - } else { - hasParam = true - buf.WriteString("?timeout=") - } - buf.WriteString(cfg.Timeout.String()) + writeDSNParam(&buf, &hasParam, "timeout", cfg.Timeout.String()) } if len(cfg.TLSConfig) > 0 { - if hasParam { - buf.WriteString("&tls=") - } else { - hasParam = true - buf.WriteString("?tls=") - } - buf.WriteString(url.QueryEscape(cfg.TLSConfig)) + writeDSNParam(&buf, &hasParam, "tls", url.QueryEscape(cfg.TLSConfig)) } if cfg.WriteTimeout > 0 { - if hasParam { - buf.WriteString("&writeTimeout=") - } else { - hasParam = true - buf.WriteString("?writeTimeout=") - } - buf.WriteString(cfg.WriteTimeout.String()) + writeDSNParam(&buf, &hasParam, "writeTimeout", cfg.WriteTimeout.String()) } if cfg.MaxAllowedPacket != defaultMaxAllowedPacket { - if hasParam { - buf.WriteString("&maxAllowedPacket=") - } else { - hasParam = true - buf.WriteString("?maxAllowedPacket=") - } - buf.WriteString(strconv.Itoa(cfg.MaxAllowedPacket)) - + writeDSNParam(&buf, &hasParam, "maxAllowedPacket", strconv.Itoa(cfg.MaxAllowedPacket)) } // other params @@ -316,16 +280,7 @@ func (cfg *Config) FormatDSN() string { } sort.Strings(params) for _, param := range params { - if hasParam { - buf.WriteByte('&') - } else { - hasParam = true - buf.WriteByte('?') - } - - buf.WriteString(param) - buf.WriteByte('=') - buf.WriteString(url.QueryEscape(cfg.Params[param])) + writeDSNParam(&buf, &hasParam, param, url.QueryEscape(cfg.Params[param])) } } @@ -452,6 +407,14 @@ func parseDSNParams(cfg *Config, params string) (err error) { return errors.New("invalid bool value: " + value) } + // Check connections for Liveness before using them + case "checkConnLiveness": + var isBool bool + cfg.CheckConnLiveness, isBool = readBool(value) + if !isBool { + return errors.New("invalid bool value: " + value) + } + // Switch "rowsAffected" mode case "clientFoundRows": var isBool bool @@ -531,13 +494,7 @@ func parseDSNParams(cfg *Config, params string) (err error) { if err != nil { return fmt.Errorf("invalid value for server pub key name: %v", err) } - - if pubKey := getServerPubKey(name); pubKey != nil { - cfg.ServerPubKey = name - cfg.pubKey = pubKey - } else { - return errors.New("invalid value / unknown server pub key name: " + name) - } + cfg.ServerPubKey = name // Strict mode case "strict": @@ -556,25 +513,17 @@ func parseDSNParams(cfg *Config, params string) (err error) { if isBool { if boolValue { cfg.TLSConfig = "true" - cfg.tls = &tls.Config{} } else { cfg.TLSConfig = "false" } - } else if vl := strings.ToLower(value); vl == "skip-verify" { + } else if vl := strings.ToLower(value); vl == "skip-verify" || vl == "preferred" { cfg.TLSConfig = vl - cfg.tls = &tls.Config{InsecureSkipVerify: true} } else { name, err := url.QueryUnescape(value) if err != nil { return fmt.Errorf("invalid value for TLS config name: %v", err) } - - if tlsConfig := getTLSConfigClone(name); tlsConfig != nil { - cfg.TLSConfig = name - cfg.tls = tlsConfig - } else { - return errors.New("invalid value / unknown config name: " + name) - } + cfg.TLSConfig = name } // I/O write Timeout diff --git a/vendor/github.com/go-sql-driver/mysql/go.mod b/vendor/github.com/go-sql-driver/mysql/go.mod new file mode 100644 index 00000000..fffbf6a9 --- /dev/null +++ b/vendor/github.com/go-sql-driver/mysql/go.mod @@ -0,0 +1,3 @@ +module github.com/go-sql-driver/mysql + +go 1.10 diff --git a/vendor/github.com/go-sql-driver/mysql/nulltime.go b/vendor/github.com/go-sql-driver/mysql/nulltime.go new file mode 100644 index 00000000..afa8a89e --- /dev/null +++ b/vendor/github.com/go-sql-driver/mysql/nulltime.go @@ -0,0 +1,50 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2013 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package mysql + +import ( + "database/sql/driver" + "fmt" + "time" +) + +// Scan implements the Scanner interface. +// The value type must be time.Time or string / []byte (formatted time-string), +// otherwise Scan fails. +func (nt *NullTime) Scan(value interface{}) (err error) { + if value == nil { + nt.Time, nt.Valid = time.Time{}, false + return + } + + switch v := value.(type) { + case time.Time: + nt.Time, nt.Valid = v, true + return + case []byte: + nt.Time, err = parseDateTime(string(v), time.UTC) + nt.Valid = (err == nil) + return + case string: + nt.Time, err = parseDateTime(v, time.UTC) + nt.Valid = (err == nil) + return + } + + nt.Valid = false + return fmt.Errorf("Can't convert %T to time.Time", value) +} + +// Value implements the driver Valuer interface. +func (nt NullTime) Value() (driver.Value, error) { + if !nt.Valid { + return nil, nil + } + return nt.Time, nil +} diff --git a/vendor/github.com/go-sql-driver/mysql/nulltime_go113.go b/vendor/github.com/go-sql-driver/mysql/nulltime_go113.go new file mode 100644 index 00000000..c392594d --- /dev/null +++ b/vendor/github.com/go-sql-driver/mysql/nulltime_go113.go @@ -0,0 +1,31 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2013 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +// +build go1.13 + +package mysql + +import ( + "database/sql" +) + +// NullTime represents a time.Time that may be NULL. +// NullTime implements the Scanner interface so +// it can be used as a scan destination: +// +// var nt NullTime +// err := db.QueryRow("SELECT time FROM foo WHERE id=?", id).Scan(&nt) +// ... +// if nt.Valid { +// // use nt.Time +// } else { +// // NULL value +// } +// +// This NullTime implementation is not driver-specific +type NullTime sql.NullTime diff --git a/vendor/github.com/go-sql-driver/mysql/nulltime_legacy.go b/vendor/github.com/go-sql-driver/mysql/nulltime_legacy.go new file mode 100644 index 00000000..86d159d4 --- /dev/null +++ b/vendor/github.com/go-sql-driver/mysql/nulltime_legacy.go @@ -0,0 +1,34 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2013 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +// +build !go1.13 + +package mysql + +import ( + "time" +) + +// NullTime represents a time.Time that may be NULL. +// NullTime implements the Scanner interface so +// it can be used as a scan destination: +// +// var nt NullTime +// err := db.QueryRow("SELECT time FROM foo WHERE id=?", id).Scan(&nt) +// ... +// if nt.Valid { +// // use nt.Time +// } else { +// // NULL value +// } +// +// This NullTime implementation is not driver-specific +type NullTime struct { + Time time.Time + Valid bool // Valid is true if Time is not NULL +} diff --git a/vendor/github.com/go-sql-driver/mysql/packets.go b/vendor/github.com/go-sql-driver/mysql/packets.go index d873a97b..82ad7a20 100644 --- a/vendor/github.com/go-sql-driver/mysql/packets.go +++ b/vendor/github.com/go-sql-driver/mysql/packets.go @@ -51,7 +51,7 @@ func (mc *mysqlConn) readPacket() ([]byte, error) { mc.sequence++ // packets with length 0 terminate a previous packet which is a - // multiple of (2^24)−1 bytes long + // multiple of (2^24)-1 bytes long if pktLen == 0 { // there was no previous packet if prevData == nil { @@ -96,6 +96,35 @@ func (mc *mysqlConn) writePacket(data []byte) error { return ErrPktTooLarge } + // Perform a stale connection check. We only perform this check for + // the first query on a connection that has been checked out of the + // connection pool: a fresh connection from the pool is more likely + // to be stale, and it has not performed any previous writes that + // could cause data corruption, so it's safe to return ErrBadConn + // if the check fails. + if mc.reset { + mc.reset = false + conn := mc.netConn + if mc.rawConn != nil { + conn = mc.rawConn + } + var err error + // If this connection has a ReadTimeout which we've been setting on + // reads, reset it to its default value before we attempt a non-blocking + // read, otherwise the scheduler will just time us out before we can read + if mc.cfg.ReadTimeout != 0 { + err = conn.SetReadDeadline(time.Time{}) + } + if err == nil && mc.cfg.CheckConnLiveness { + err = connCheck(conn) + } + if err != nil { + errLog.Print("closing bad idle connection: ", err) + mc.Close() + return driver.ErrBadConn + } + } + for { var size int if pktLen >= maxPacketSize { @@ -154,15 +183,15 @@ func (mc *mysqlConn) writePacket(data []byte) error { // Handshake Initialization Packet // http://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::Handshake -func (mc *mysqlConn) readHandshakePacket() ([]byte, string, error) { - data, err := mc.readPacket() +func (mc *mysqlConn) readHandshakePacket() (data []byte, plugin string, err error) { + data, err = mc.readPacket() if err != nil { // for init we can rewrite this to ErrBadConn for sql.Driver to retry, since // in connection initialization we don't risk retrying non-idempotent actions. if err == ErrInvalidConn { return nil, "", driver.ErrBadConn } - return nil, "", err + return } if data[0] == iERR { @@ -194,11 +223,14 @@ func (mc *mysqlConn) readHandshakePacket() ([]byte, string, error) { return nil, "", ErrOldProtocol } if mc.flags&clientSSL == 0 && mc.cfg.tls != nil { - return nil, "", ErrNoTLS + if mc.cfg.TLSConfig == "preferred" { + mc.cfg.tls = nil + } else { + return nil, "", ErrNoTLS + } } pos += 2 - plugin := "" if len(data) > pos { // character set [1 byte] // status flags [2 bytes] @@ -236,8 +268,6 @@ func (mc *mysqlConn) readHandshakePacket() ([]byte, string, error) { return b[:], plugin, nil } - plugin = defaultAuthPlugin - // make a memory safe copy of the cipher slice var b [8]byte copy(b[:], authData) @@ -246,7 +276,7 @@ func (mc *mysqlConn) readHandshakePacket() ([]byte, string, error) { // Client Authentication Packet // http://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::HandshakeResponse -func (mc *mysqlConn) writeHandshakeResponsePacket(authResp []byte, addNUL bool, plugin string) error { +func (mc *mysqlConn) writeHandshakeResponsePacket(authResp []byte, plugin string) error { // Adjust client flags based on server support clientFlags := clientProtocol41 | clientSecureConn | @@ -272,7 +302,8 @@ func (mc *mysqlConn) writeHandshakeResponsePacket(authResp []byte, addNUL bool, // encode length of the auth plugin data var authRespLEIBuf [9]byte - authRespLEI := appendLengthEncodedInteger(authRespLEIBuf[:0], uint64(len(authResp))) + authRespLen := len(authResp) + authRespLEI := appendLengthEncodedInteger(authRespLEIBuf[:0], uint64(authRespLen)) if len(authRespLEI) > 1 { // if the length can not be written in 1 byte, it must be written as a // length encoded integer @@ -280,9 +311,6 @@ func (mc *mysqlConn) writeHandshakeResponsePacket(authResp []byte, addNUL bool, } pktLen := 4 + 4 + 1 + 23 + len(mc.cfg.User) + 1 + len(authRespLEI) + len(authResp) + 21 + 1 - if addNUL { - pktLen++ - } // To specify a db name if n := len(mc.cfg.DBName); n > 0 { @@ -291,10 +319,10 @@ func (mc *mysqlConn) writeHandshakeResponsePacket(authResp []byte, addNUL bool, } // Calculate packet length and get buffer with that size - data := mc.buf.takeSmallBuffer(pktLen + 4) - if data == nil { + data, err := mc.buf.takeSmallBuffer(pktLen + 4) + if err != nil { // cannot take the buffer. Something must be wrong with the connection - errLog.Print(ErrBusyBuffer) + errLog.Print(err) return errBadConnNoWrite } @@ -333,6 +361,7 @@ func (mc *mysqlConn) writeHandshakeResponsePacket(authResp []byte, addNUL bool, if err := tlsConn.Handshake(); err != nil { return err } + mc.rawConn = mc.netConn mc.netConn = tlsConn mc.buf.nc = tlsConn } @@ -353,10 +382,6 @@ func (mc *mysqlConn) writeHandshakeResponsePacket(authResp []byte, addNUL bool, // Auth Data [length encoded integer] pos += copy(data[pos:], authRespLEI) pos += copy(data[pos:], authResp) - if addNUL { - data[pos] = 0x00 - pos++ - } // Databasename [null terminated string] if len(mc.cfg.DBName) > 0 { @@ -367,30 +392,24 @@ func (mc *mysqlConn) writeHandshakeResponsePacket(authResp []byte, addNUL bool, pos += copy(data[pos:], plugin) data[pos] = 0x00 + pos++ // Send Auth packet - return mc.writePacket(data) + return mc.writePacket(data[:pos]) } // http://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::AuthSwitchResponse -func (mc *mysqlConn) writeAuthSwitchPacket(authData []byte, addNUL bool) error { +func (mc *mysqlConn) writeAuthSwitchPacket(authData []byte) error { pktLen := 4 + len(authData) - if addNUL { - pktLen++ - } - data := mc.buf.takeSmallBuffer(pktLen) - if data == nil { + data, err := mc.buf.takeSmallBuffer(pktLen) + if err != nil { // cannot take the buffer. Something must be wrong with the connection - errLog.Print(ErrBusyBuffer) + errLog.Print(err) return errBadConnNoWrite } // Add the auth data [EOF] copy(data[4:], authData) - if addNUL { - data[pktLen-1] = 0x00 - } - return mc.writePacket(data) } @@ -402,10 +421,10 @@ func (mc *mysqlConn) writeCommandPacket(command byte) error { // Reset Packet Sequence mc.sequence = 0 - data := mc.buf.takeSmallBuffer(4 + 1) - if data == nil { + data, err := mc.buf.takeSmallBuffer(4 + 1) + if err != nil { // cannot take the buffer. Something must be wrong with the connection - errLog.Print(ErrBusyBuffer) + errLog.Print(err) return errBadConnNoWrite } @@ -421,10 +440,10 @@ func (mc *mysqlConn) writeCommandPacketStr(command byte, arg string) error { mc.sequence = 0 pktLen := 1 + len(arg) - data := mc.buf.takeBuffer(pktLen + 4) - if data == nil { + data, err := mc.buf.takeBuffer(pktLen + 4) + if err != nil { // cannot take the buffer. Something must be wrong with the connection - errLog.Print(ErrBusyBuffer) + errLog.Print(err) return errBadConnNoWrite } @@ -442,10 +461,10 @@ func (mc *mysqlConn) writeCommandPacketUint32(command byte, arg uint32) error { // Reset Packet Sequence mc.sequence = 0 - data := mc.buf.takeSmallBuffer(4 + 1 + 4) - if data == nil { + data, err := mc.buf.takeSmallBuffer(4 + 1 + 4) + if err != nil { // cannot take the buffer. Something must be wrong with the connection - errLog.Print(ErrBusyBuffer) + errLog.Print(err) return errBadConnNoWrite } @@ -482,7 +501,7 @@ func (mc *mysqlConn) readAuthResult() ([]byte, string, error) { return data[1:], "", err case iEOF: - if len(data) < 1 { + if len(data) == 1 { // https://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::OldAuthSwitchRequest return nil, "mysql_old_password", nil } @@ -898,7 +917,7 @@ func (stmt *mysqlStmt) writeExecutePacket(args []driver.Value) error { const minPktLen = 4 + 1 + 4 + 1 + 4 mc := stmt.mc - // Determine threshould dynamically to avoid packet size shortage. + // Determine threshold dynamically to avoid packet size shortage. longDataSize := mc.maxAllowedPacket / (stmt.paramCount + 1) if longDataSize < 64 { longDataSize = 64 @@ -908,15 +927,17 @@ func (stmt *mysqlStmt) writeExecutePacket(args []driver.Value) error { mc.sequence = 0 var data []byte + var err error if len(args) == 0 { - data = mc.buf.takeBuffer(minPktLen) + data, err = mc.buf.takeBuffer(minPktLen) } else { - data = mc.buf.takeCompleteBuffer() + data, err = mc.buf.takeCompleteBuffer() + // In this case the len(data) == cap(data) which is used to optimise the flow below. } - if data == nil { + if err != nil { // cannot take the buffer. Something must be wrong with the connection - errLog.Print(ErrBusyBuffer) + errLog.Print(err) return errBadConnNoWrite } @@ -942,7 +963,7 @@ func (stmt *mysqlStmt) writeExecutePacket(args []driver.Value) error { pos := minPktLen var nullMask []byte - if maskLen, typesLen := (len(args)+7)/8, 1+2*len(args); pos+maskLen+typesLen >= len(data) { + if maskLen, typesLen := (len(args)+7)/8, 1+2*len(args); pos+maskLen+typesLen >= cap(data) { // buffer has to be extended but we don't know by how much so // we depend on append after all data with known sizes fit. // We stop at that because we deal with a lot of columns here @@ -951,10 +972,11 @@ func (stmt *mysqlStmt) writeExecutePacket(args []driver.Value) error { copy(tmp[:pos], data[:pos]) data = tmp nullMask = data[pos : pos+maskLen] + // No need to clean nullMask as make ensures that. pos += maskLen } else { nullMask = data[pos : pos+maskLen] - for i := 0; i < maskLen; i++ { + for i := range nullMask { nullMask[i] = 0 } pos += maskLen @@ -999,6 +1021,22 @@ func (stmt *mysqlStmt) writeExecutePacket(args []driver.Value) error { ) } + case uint64: + paramTypes[i+i] = byte(fieldTypeLongLong) + paramTypes[i+i+1] = 0x80 // type is unsigned + + if cap(paramValues)-len(paramValues)-8 >= 0 { + paramValues = paramValues[:len(paramValues)+8] + binary.LittleEndian.PutUint64( + paramValues[len(paramValues)-8:], + uint64(v), + ) + } else { + paramValues = append(paramValues, + uint64ToBytes(uint64(v))..., + ) + } + case float64: paramTypes[i+i] = byte(fieldTypeDouble) paramTypes[i+i+1] = 0x00 @@ -1091,7 +1129,10 @@ func (stmt *mysqlStmt) writeExecutePacket(args []driver.Value) error { // In that case we must build the data packet with the new values buffer if valuesCap != cap(paramValues) { data = append(data[:pos], paramValues...) - mc.buf.buf = data + if err = mc.buf.store(data); err != nil { + errLog.Print(err) + return errBadConnNoWrite + } } pos += len(paramValues) @@ -1261,7 +1302,7 @@ func (rows *binaryRows) readRow(dest []driver.Value) error { rows.rs.columns[i].decimals, ) } - dest[i], err = formatBinaryDateTime(data[pos:pos+int(num)], dstlen, true) + dest[i], err = formatBinaryTime(data[pos:pos+int(num)], dstlen) case rows.mc.parseTime: dest[i], err = parseBinaryDateTime(num, data[pos:], rows.mc.cfg.Loc) default: @@ -1281,7 +1322,7 @@ func (rows *binaryRows) readRow(dest []driver.Value) error { ) } } - dest[i], err = formatBinaryDateTime(data[pos:pos+int(num)], dstlen, false) + dest[i], err = formatBinaryDateTime(data[pos:pos+int(num)], dstlen) } if err == nil { diff --git a/vendor/github.com/go-sql-driver/mysql/rows.go b/vendor/github.com/go-sql-driver/mysql/rows.go index d3b1e282..888bdb5f 100644 --- a/vendor/github.com/go-sql-driver/mysql/rows.go +++ b/vendor/github.com/go-sql-driver/mysql/rows.go @@ -111,6 +111,13 @@ func (rows *mysqlRows) Close() (err error) { return err } + // flip the buffer for this connection if we need to drain it. + // note that for a successful query (i.e. one where rows.next() + // has been called until it returns false), `rows.mc` will be nil + // by the time the user calls `(*Rows).Close`, so we won't reach this + // see: https://github.com/golang/go/commit/651ddbdb5056ded455f47f9c494c67b389622a47 + mc.buf.flip() + // Remove unread packets from stream if !rows.rs.done { err = mc.readUntilEOF() diff --git a/vendor/github.com/go-sql-driver/mysql/statement.go b/vendor/github.com/go-sql-driver/mysql/statement.go index ce7fe4cd..f7e37093 100644 --- a/vendor/github.com/go-sql-driver/mysql/statement.go +++ b/vendor/github.com/go-sql-driver/mysql/statement.go @@ -13,7 +13,6 @@ import ( "fmt" "io" "reflect" - "strconv" ) type mysqlStmt struct { @@ -164,14 +163,8 @@ func (c converter) ConvertValue(v interface{}) (driver.Value, error) { } case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: return rv.Int(), nil - case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32: - return int64(rv.Uint()), nil - case reflect.Uint64: - u64 := rv.Uint() - if u64 >= 1<<63 { - return strconv.FormatUint(u64, 10), nil - } - return int64(u64), nil + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + return rv.Uint(), nil case reflect.Float32, reflect.Float64: return rv.Float(), nil case reflect.Bool: diff --git a/vendor/github.com/go-sql-driver/mysql/utils.go b/vendor/github.com/go-sql-driver/mysql/utils.go index 84d595b6..9552e80b 100644 --- a/vendor/github.com/go-sql-driver/mysql/utils.go +++ b/vendor/github.com/go-sql-driver/mysql/utils.go @@ -10,10 +10,13 @@ package mysql import ( "crypto/tls" + "database/sql" "database/sql/driver" "encoding/binary" + "errors" "fmt" "io" + "strconv" "strings" "sync" "sync/atomic" @@ -53,7 +56,7 @@ var ( // db, err := sql.Open("mysql", "user@tcp(localhost:3306)/test?tls=custom") // func RegisterTLSConfig(key string, config *tls.Config) error { - if _, isBool := readBool(key); isBool || strings.ToLower(key) == "skip-verify" { + if _, isBool := readBool(key); isBool || strings.ToLower(key) == "skip-verify" || strings.ToLower(key) == "preferred" { return fmt.Errorf("key '%s' is reserved", key) } @@ -79,7 +82,7 @@ func DeregisterTLSConfig(key string) { func getTLSConfigClone(key string) (config *tls.Config) { tlsConfigLock.RLock() if v, ok := tlsConfigRegistry[key]; ok { - config = cloneTLSConfig(v) + config = v.Clone() } tlsConfigLock.RUnlock() return @@ -103,60 +106,6 @@ func readBool(input string) (value bool, valid bool) { * Time related utils * ******************************************************************************/ -// NullTime represents a time.Time that may be NULL. -// NullTime implements the Scanner interface so -// it can be used as a scan destination: -// -// var nt NullTime -// err := db.QueryRow("SELECT time FROM foo WHERE id=?", id).Scan(&nt) -// ... -// if nt.Valid { -// // use nt.Time -// } else { -// // NULL value -// } -// -// This NullTime implementation is not driver-specific -type NullTime struct { - Time time.Time - Valid bool // Valid is true if Time is not NULL -} - -// Scan implements the Scanner interface. -// The value type must be time.Time or string / []byte (formatted time-string), -// otherwise Scan fails. -func (nt *NullTime) Scan(value interface{}) (err error) { - if value == nil { - nt.Time, nt.Valid = time.Time{}, false - return - } - - switch v := value.(type) { - case time.Time: - nt.Time, nt.Valid = v, true - return - case []byte: - nt.Time, err = parseDateTime(string(v), time.UTC) - nt.Valid = (err == nil) - return - case string: - nt.Time, err = parseDateTime(v, time.UTC) - nt.Valid = (err == nil) - return - } - - nt.Valid = false - return fmt.Errorf("Can't convert %T to time.Time", value) -} - -// Value implements the driver Valuer interface. -func (nt NullTime) Value() (driver.Value, error) { - if !nt.Valid { - return nil, nil - } - return nt.Time, nil -} - func parseDateTime(str string, loc *time.Location) (t time.Time, err error) { base := "0000-00-00 00:00:00.0000000" switch len(str) { @@ -227,141 +176,156 @@ var zeroDateTime = []byte("0000-00-00 00:00:00.000000") const digits01 = "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789" const digits10 = "0000000000111111111122222222223333333333444444444455555555556666666666777777777788888888889999999999" -func formatBinaryDateTime(src []byte, length uint8, justTime bool) (driver.Value, error) { - // length expects the deterministic length of the zero value, - // negative time and 100+ hours are automatically added if needed - if len(src) == 0 { - if justTime { - return zeroDateTime[11 : 11+length], nil - } - return zeroDateTime[:length], nil - } - var dst []byte // return value - var pt, p1, p2, p3 byte // current digit pair - var zOffs byte // offset of value in zeroDateTime - if justTime { - switch length { - case - 8, // time (can be up to 10 when negative and 100+ hours) - 10, 11, 12, 13, 14, 15: // time with fractional seconds - default: - return nil, fmt.Errorf("illegal TIME length %d", length) - } - switch len(src) { - case 8, 12: - default: - return nil, fmt.Errorf("invalid TIME packet length %d", len(src)) - } - // +2 to enable negative time and 100+ hours - dst = make([]byte, 0, length+2) - if src[0] == 1 { - dst = append(dst, '-') - } - if src[1] != 0 { - hour := uint16(src[1])*24 + uint16(src[5]) - pt = byte(hour / 100) - p1 = byte(hour - 100*uint16(pt)) - dst = append(dst, digits01[pt]) - } else { - p1 = src[5] - } - zOffs = 11 - src = src[6:] - } else { - switch length { - case 10, 19, 21, 22, 23, 24, 25, 26: - default: - t := "DATE" - if length > 10 { - t += "TIME" - } - return nil, fmt.Errorf("illegal %s length %d", t, length) - } - switch len(src) { - case 4, 7, 11: - default: - t := "DATE" - if length > 10 { - t += "TIME" - } - return nil, fmt.Errorf("illegal %s packet length %d", t, len(src)) - } - dst = make([]byte, 0, length) - // start with the date - year := binary.LittleEndian.Uint16(src[:2]) - pt = byte(year / 100) - p1 = byte(year - 100*uint16(pt)) - p2, p3 = src[2], src[3] - dst = append(dst, - digits10[pt], digits01[pt], - digits10[p1], digits01[p1], '-', - digits10[p2], digits01[p2], '-', - digits10[p3], digits01[p3], - ) - if length == 10 { - return dst, nil - } - if len(src) == 4 { - return append(dst, zeroDateTime[10:length]...), nil - } - dst = append(dst, ' ') - p1 = src[4] // hour - src = src[5:] +func appendMicrosecs(dst, src []byte, decimals int) []byte { + if decimals <= 0 { + return dst } - // p1 is 2-digit hour, src is after hour - p2, p3 = src[0], src[1] - dst = append(dst, - digits10[p1], digits01[p1], ':', - digits10[p2], digits01[p2], ':', - digits10[p3], digits01[p3], - ) - if length <= byte(len(dst)) { - return dst, nil - } - src = src[2:] if len(src) == 0 { - return append(dst, zeroDateTime[19:zOffs+length]...), nil + return append(dst, ".000000"[:decimals+1]...) } + microsecs := binary.LittleEndian.Uint32(src[:4]) - p1 = byte(microsecs / 10000) + p1 := byte(microsecs / 10000) microsecs -= 10000 * uint32(p1) - p2 = byte(microsecs / 100) + p2 := byte(microsecs / 100) microsecs -= 100 * uint32(p2) - p3 = byte(microsecs) - switch decimals := zOffs + length - 20; decimals { + p3 := byte(microsecs) + + switch decimals { default: return append(dst, '.', digits10[p1], digits01[p1], digits10[p2], digits01[p2], digits10[p3], digits01[p3], - ), nil + ) case 1: return append(dst, '.', digits10[p1], - ), nil + ) case 2: return append(dst, '.', digits10[p1], digits01[p1], - ), nil + ) case 3: return append(dst, '.', digits10[p1], digits01[p1], digits10[p2], - ), nil + ) case 4: return append(dst, '.', digits10[p1], digits01[p1], digits10[p2], digits01[p2], - ), nil + ) case 5: return append(dst, '.', digits10[p1], digits01[p1], digits10[p2], digits01[p2], digits10[p3], - ), nil + ) } } +func formatBinaryDateTime(src []byte, length uint8) (driver.Value, error) { + // length expects the deterministic length of the zero value, + // negative time and 100+ hours are automatically added if needed + if len(src) == 0 { + return zeroDateTime[:length], nil + } + var dst []byte // return value + var p1, p2, p3 byte // current digit pair + + switch length { + case 10, 19, 21, 22, 23, 24, 25, 26: + default: + t := "DATE" + if length > 10 { + t += "TIME" + } + return nil, fmt.Errorf("illegal %s length %d", t, length) + } + switch len(src) { + case 4, 7, 11: + default: + t := "DATE" + if length > 10 { + t += "TIME" + } + return nil, fmt.Errorf("illegal %s packet length %d", t, len(src)) + } + dst = make([]byte, 0, length) + // start with the date + year := binary.LittleEndian.Uint16(src[:2]) + pt := year / 100 + p1 = byte(year - 100*uint16(pt)) + p2, p3 = src[2], src[3] + dst = append(dst, + digits10[pt], digits01[pt], + digits10[p1], digits01[p1], '-', + digits10[p2], digits01[p2], '-', + digits10[p3], digits01[p3], + ) + if length == 10 { + return dst, nil + } + if len(src) == 4 { + return append(dst, zeroDateTime[10:length]...), nil + } + dst = append(dst, ' ') + p1 = src[4] // hour + src = src[5:] + + // p1 is 2-digit hour, src is after hour + p2, p3 = src[0], src[1] + dst = append(dst, + digits10[p1], digits01[p1], ':', + digits10[p2], digits01[p2], ':', + digits10[p3], digits01[p3], + ) + return appendMicrosecs(dst, src[2:], int(length)-20), nil +} + +func formatBinaryTime(src []byte, length uint8) (driver.Value, error) { + // length expects the deterministic length of the zero value, + // negative time and 100+ hours are automatically added if needed + if len(src) == 0 { + return zeroDateTime[11 : 11+length], nil + } + var dst []byte // return value + + switch length { + case + 8, // time (can be up to 10 when negative and 100+ hours) + 10, 11, 12, 13, 14, 15: // time with fractional seconds + default: + return nil, fmt.Errorf("illegal TIME length %d", length) + } + switch len(src) { + case 8, 12: + default: + return nil, fmt.Errorf("invalid TIME packet length %d", len(src)) + } + // +2 to enable negative time and 100+ hours + dst = make([]byte, 0, length+2) + if src[0] == 1 { + dst = append(dst, '-') + } + days := binary.LittleEndian.Uint32(src[1:5]) + hours := int64(days)*24 + int64(src[5]) + + if hours >= 100 { + dst = strconv.AppendInt(dst, hours, 10) + } else { + dst = append(dst, digits10[hours], digits01[hours]) + } + + min, sec := src[6], src[7] + dst = append(dst, ':', + digits10[min], digits01[min], ':', + digits10[sec], digits01[sec], + ) + return appendMicrosecs(dst, src[8:], int(length)-9), nil +} + /****************************************************************************** * Convert from and to bytes * ******************************************************************************/ @@ -666,7 +630,7 @@ type atomicBool struct { value uint32 } -// IsSet returns wether the current boolean value is true +// IsSet returns whether the current boolean value is true func (ab *atomicBool) IsSet() bool { return atomic.LoadUint32(&ab.value) > 0 } @@ -680,7 +644,7 @@ func (ab *atomicBool) Set(value bool) { } } -// TrySet sets the value of the bool and returns wether the value changed +// TrySet sets the value of the bool and returns whether the value changed func (ab *atomicBool) TrySet(value bool) bool { if value { return atomic.SwapUint32(&ab.value, 1) == 0 @@ -708,3 +672,30 @@ func (ae *atomicError) Value() error { } return nil } + +func namedValueToValue(named []driver.NamedValue) ([]driver.Value, error) { + dargs := make([]driver.Value, len(named)) + for n, param := range named { + if len(param.Name) > 0 { + // TODO: support the use of Named Parameters #561 + return nil, errors.New("mysql: driver does not support the use of Named Parameters") + } + dargs[n] = param.Value + } + return dargs, nil +} + +func mapIsolationLevel(level driver.IsolationLevel) (string, error) { + switch sql.IsolationLevel(level) { + case sql.LevelRepeatableRead: + return "REPEATABLE READ", nil + case sql.LevelReadCommitted: + return "READ COMMITTED", nil + case sql.LevelReadUncommitted: + return "READ UNCOMMITTED", nil + case sql.LevelSerializable: + return "SERIALIZABLE", nil + default: + return "", fmt.Errorf("mysql: unsupported isolation level: %v", level) + } +} diff --git a/vendor/github.com/go-sql-driver/mysql/utils_go17.go b/vendor/github.com/go-sql-driver/mysql/utils_go17.go deleted file mode 100644 index f5956345..00000000 --- a/vendor/github.com/go-sql-driver/mysql/utils_go17.go +++ /dev/null @@ -1,40 +0,0 @@ -// Go MySQL Driver - A MySQL-Driver for Go's database/sql package -// -// Copyright 2017 The Go-MySQL-Driver Authors. All rights reserved. -// -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this file, -// You can obtain one at http://mozilla.org/MPL/2.0/. - -// +build go1.7 -// +build !go1.8 - -package mysql - -import "crypto/tls" - -func cloneTLSConfig(c *tls.Config) *tls.Config { - return &tls.Config{ - Rand: c.Rand, - Time: c.Time, - Certificates: c.Certificates, - NameToCertificate: c.NameToCertificate, - GetCertificate: c.GetCertificate, - RootCAs: c.RootCAs, - NextProtos: c.NextProtos, - ServerName: c.ServerName, - ClientAuth: c.ClientAuth, - ClientCAs: c.ClientCAs, - InsecureSkipVerify: c.InsecureSkipVerify, - CipherSuites: c.CipherSuites, - PreferServerCipherSuites: c.PreferServerCipherSuites, - SessionTicketsDisabled: c.SessionTicketsDisabled, - SessionTicketKey: c.SessionTicketKey, - ClientSessionCache: c.ClientSessionCache, - MinVersion: c.MinVersion, - MaxVersion: c.MaxVersion, - CurvePreferences: c.CurvePreferences, - DynamicRecordSizingDisabled: c.DynamicRecordSizingDisabled, - Renegotiation: c.Renegotiation, - } -} diff --git a/vendor/github.com/go-sql-driver/mysql/utils_go18.go b/vendor/github.com/go-sql-driver/mysql/utils_go18.go deleted file mode 100644 index c35c2a6a..00000000 --- a/vendor/github.com/go-sql-driver/mysql/utils_go18.go +++ /dev/null @@ -1,50 +0,0 @@ -// Go MySQL Driver - A MySQL-Driver for Go's database/sql package -// -// Copyright 2017 The Go-MySQL-Driver Authors. All rights reserved. -// -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this file, -// You can obtain one at http://mozilla.org/MPL/2.0/. - -// +build go1.8 - -package mysql - -import ( - "crypto/tls" - "database/sql" - "database/sql/driver" - "errors" - "fmt" -) - -func cloneTLSConfig(c *tls.Config) *tls.Config { - return c.Clone() -} - -func namedValueToValue(named []driver.NamedValue) ([]driver.Value, error) { - dargs := make([]driver.Value, len(named)) - for n, param := range named { - if len(param.Name) > 0 { - // TODO: support the use of Named Parameters #561 - return nil, errors.New("mysql: driver does not support the use of Named Parameters") - } - dargs[n] = param.Value - } - return dargs, nil -} - -func mapIsolationLevel(level driver.IsolationLevel) (string, error) { - switch sql.IsolationLevel(level) { - case sql.LevelRepeatableRead: - return "REPEATABLE READ", nil - case sql.LevelReadCommitted: - return "READ COMMITTED", nil - case sql.LevelReadUncommitted: - return "READ UNCOMMITTED", nil - case sql.LevelSerializable: - return "SERIALIZABLE", nil - default: - return "", fmt.Errorf("mysql: unsupported isolation level: %v", level) - } -} diff --git a/vendor/github.com/jinzhu/gorm/.codeclimate.yml b/vendor/github.com/jinzhu/gorm/.codeclimate.yml deleted file mode 100644 index 51aba50c..00000000 --- a/vendor/github.com/jinzhu/gorm/.codeclimate.yml +++ /dev/null @@ -1,11 +0,0 @@ ---- -engines: - gofmt: - enabled: true - govet: - enabled: true - golint: - enabled: true -ratings: - paths: - - "**.go" diff --git a/vendor/github.com/jinzhu/gorm/.gitignore b/vendor/github.com/jinzhu/gorm/.gitignore index 01dc5ce0..117f92f5 100644 --- a/vendor/github.com/jinzhu/gorm/.gitignore +++ b/vendor/github.com/jinzhu/gorm/.gitignore @@ -1,2 +1,3 @@ documents +coverage.txt _book diff --git a/vendor/github.com/jinzhu/gorm/README.md b/vendor/github.com/jinzhu/gorm/README.md index 0c5c7ea6..85588a79 100644 --- a/vendor/github.com/jinzhu/gorm/README.md +++ b/vendor/github.com/jinzhu/gorm/README.md @@ -1,40 +1,5 @@ # GORM -The fantastic ORM library for Golang, aims to be developer friendly. +GORM V2 moved to https://github.com/go-gorm/gorm -[![go report card](https://goreportcard.com/badge/github.com/jinzhu/gorm "go report card")](https://goreportcard.com/report/github.com/jinzhu/gorm) -[![wercker status](https://app.wercker.com/status/8596cace912c9947dd9c8542ecc8cb8b/s/master "wercker status")](https://app.wercker.com/project/byKey/8596cace912c9947dd9c8542ecc8cb8b) -[![Join the chat at https://gitter.im/jinzhu/gorm](https://img.shields.io/gitter/room/jinzhu/gorm.svg)](https://gitter.im/jinzhu/gorm?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) -[![Open Collective Backer](https://opencollective.com/gorm/tiers/backer/badge.svg?label=backer&color=brightgreen "Open Collective Backer")](https://opencollective.com/gorm) -[![Open Collective Sponsor](https://opencollective.com/gorm/tiers/sponsor/badge.svg?label=sponsor&color=brightgreen "Open Collective Sponsor")](https://opencollective.com/gorm) -[![MIT license](http://img.shields.io/badge/license-MIT-brightgreen.svg)](http://opensource.org/licenses/MIT) -[![GoDoc](https://godoc.org/github.com/jinzhu/gorm?status.svg)](https://godoc.org/github.com/jinzhu/gorm) - -## Overview - -* Full-Featured ORM (almost) -* Associations (Has One, Has Many, Belongs To, Many To Many, Polymorphism) -* Hooks (Before/After Create/Save/Update/Delete/Find) -* Preloading (eager loading) -* Transactions -* Composite Primary Key -* SQL Builder -* Auto Migrations -* Logger -* Extendable, write Plugins based on GORM callbacks -* Every feature comes with tests -* Developer Friendly - -## Getting Started - -* GORM Guides [http://gorm.io](http://gorm.io) - -## Contributing - -[You can help to deliver a better GORM, check out things you can do](http://gorm.io/contribute.html) - -## License - -© Jinzhu, 2013~time.Now - -Released under the [MIT License](https://github.com/jinzhu/gorm/blob/master/License) +GORM V1 Doc https://v1.gorm.io/ diff --git a/vendor/github.com/jinzhu/gorm/association.go b/vendor/github.com/jinzhu/gorm/association.go index 8c6d9864..a73344fe 100644 --- a/vendor/github.com/jinzhu/gorm/association.go +++ b/vendor/github.com/jinzhu/gorm/association.go @@ -267,15 +267,16 @@ func (association *Association) Count() int { query = scope.DB() ) - if relationship.Kind == "many_to_many" { + switch relationship.Kind { + case "many_to_many": query = relationship.JoinTableHandler.JoinWith(relationship.JoinTableHandler, query, scope.Value) - } else if relationship.Kind == "has_many" || relationship.Kind == "has_one" { + case "has_many", "has_one": primaryKeys := scope.getColumnAsArray(relationship.AssociationForeignFieldNames, scope.Value) query = query.Where( fmt.Sprintf("%v IN (%v)", toQueryCondition(scope, relationship.ForeignDBNames), toQueryMarks(primaryKeys)), toQueryValues(primaryKeys)..., ) - } else if relationship.Kind == "belongs_to" { + case "belongs_to": primaryKeys := scope.getColumnAsArray(relationship.ForeignFieldNames, scope.Value) query = query.Where( fmt.Sprintf("%v IN (%v)", toQueryCondition(scope, relationship.AssociationForeignDBNames), toQueryMarks(primaryKeys)), @@ -367,6 +368,7 @@ func (association *Association) saveAssociations(values ...interface{}) *Associa return association } +// setErr set error when the error is not nil. And return Association. func (association *Association) setErr(err error) *Association { if err != nil { association.Error = err diff --git a/vendor/github.com/jinzhu/gorm/callback.go b/vendor/github.com/jinzhu/gorm/callback.go index a4382147..1f0e3c79 100644 --- a/vendor/github.com/jinzhu/gorm/callback.go +++ b/vendor/github.com/jinzhu/gorm/callback.go @@ -1,9 +1,9 @@ package gorm -import "log" +import "fmt" // DefaultCallback default callbacks defined by gorm -var DefaultCallback = &Callback{} +var DefaultCallback = &Callback{logger: nopLogger{}} // Callback is a struct that contains all CRUD callbacks // Field `creates` contains callbacks will be call when creating object @@ -13,6 +13,7 @@ var DefaultCallback = &Callback{} // Field `rowQueries` contains callbacks will be call when querying object with Row, Rows... // Field `processors` contains all callback processors, will be used to generate above callbacks in order type Callback struct { + logger logger creates []*func(scope *Scope) updates []*func(scope *Scope) deletes []*func(scope *Scope) @@ -23,6 +24,7 @@ type Callback struct { // CallbackProcessor contains callback informations type CallbackProcessor struct { + logger logger name string // current callback's name before string // register current callback before a callback after string // register current callback after a callback @@ -33,8 +35,9 @@ type CallbackProcessor struct { parent *Callback } -func (c *Callback) clone() *Callback { +func (c *Callback) clone(logger logger) *Callback { return &Callback{ + logger: logger, creates: c.creates, updates: c.updates, deletes: c.deletes, @@ -53,28 +56,28 @@ func (c *Callback) clone() *Callback { // scope.Err(errors.New("error")) // }) func (c *Callback) Create() *CallbackProcessor { - return &CallbackProcessor{kind: "create", parent: c} + return &CallbackProcessor{logger: c.logger, kind: "create", parent: c} } // Update could be used to register callbacks for updating object, refer `Create` for usage func (c *Callback) Update() *CallbackProcessor { - return &CallbackProcessor{kind: "update", parent: c} + return &CallbackProcessor{logger: c.logger, kind: "update", parent: c} } // Delete could be used to register callbacks for deleting object, refer `Create` for usage func (c *Callback) Delete() *CallbackProcessor { - return &CallbackProcessor{kind: "delete", parent: c} + return &CallbackProcessor{logger: c.logger, kind: "delete", parent: c} } // Query could be used to register callbacks for querying objects with query methods like `Find`, `First`, `Related`, `Association`... // Refer `Create` for usage func (c *Callback) Query() *CallbackProcessor { - return &CallbackProcessor{kind: "query", parent: c} + return &CallbackProcessor{logger: c.logger, kind: "query", parent: c} } // RowQuery could be used to register callbacks for querying objects with `Row`, `Rows`, refer `Create` for usage func (c *Callback) RowQuery() *CallbackProcessor { - return &CallbackProcessor{kind: "row_query", parent: c} + return &CallbackProcessor{logger: c.logger, kind: "row_query", parent: c} } // After insert a new callback after callback `callbackName`, refer `Callbacks.Create` @@ -93,11 +96,12 @@ func (cp *CallbackProcessor) Before(callbackName string) *CallbackProcessor { func (cp *CallbackProcessor) Register(callbackName string, callback func(scope *Scope)) { if cp.kind == "row_query" { if cp.before == "" && cp.after == "" && callbackName != "gorm:row_query" { - log.Printf("Registing RowQuery callback %v without specify order with Before(), After(), applying Before('gorm:row_query') by default for compatibility...\n", callbackName) + cp.logger.Print("info", fmt.Sprintf("Registering RowQuery callback %v without specify order with Before(), After(), applying Before('gorm:row_query') by default for compatibility...", callbackName)) cp.before = "gorm:row_query" } } + cp.logger.Print("info", fmt.Sprintf("[info] registering callback `%v` from %v", callbackName, fileWithLineNum())) cp.name = callbackName cp.processor = &callback cp.parent.processors = append(cp.parent.processors, cp) @@ -107,7 +111,7 @@ func (cp *CallbackProcessor) Register(callbackName string, callback func(scope * // Remove a registered callback // db.Callback().Create().Remove("gorm:update_time_stamp_when_create") func (cp *CallbackProcessor) Remove(callbackName string) { - log.Printf("[info] removing callback `%v` from %v\n", callbackName, fileWithLineNum()) + cp.logger.Print("info", fmt.Sprintf("[info] removing callback `%v` from %v", callbackName, fileWithLineNum())) cp.name = callbackName cp.remove = true cp.parent.processors = append(cp.parent.processors, cp) @@ -116,11 +120,11 @@ func (cp *CallbackProcessor) Remove(callbackName string) { // Replace a registered callback with new callback // db.Callback().Create().Replace("gorm:update_time_stamp_when_create", func(*Scope) { -// scope.SetColumn("Created", now) -// scope.SetColumn("Updated", now) +// scope.SetColumn("CreatedAt", now) +// scope.SetColumn("UpdatedAt", now) // }) func (cp *CallbackProcessor) Replace(callbackName string, callback func(scope *Scope)) { - log.Printf("[info] replacing callback `%v` from %v\n", callbackName, fileWithLineNum()) + cp.logger.Print("info", fmt.Sprintf("[info] replacing callback `%v` from %v", callbackName, fileWithLineNum())) cp.name = callbackName cp.processor = &callback cp.replace = true @@ -132,11 +136,15 @@ func (cp *CallbackProcessor) Replace(callbackName string, callback func(scope *S // db.Callback().Create().Get("gorm:create") func (cp *CallbackProcessor) Get(callbackName string) (callback func(scope *Scope)) { for _, p := range cp.parent.processors { - if p.name == callbackName && p.kind == cp.kind && !cp.remove { - return *p.processor + if p.name == callbackName && p.kind == cp.kind { + if p.remove { + callback = nil + } else { + callback = *p.processor + } } } - return nil + return } // getRIndex get right index from string slice @@ -159,7 +167,7 @@ func sortProcessors(cps []*CallbackProcessor) []*func(scope *Scope) { for _, cp := range cps { // show warning message the callback name already exists if index := getRIndex(allNames, cp.name); index > -1 && !cp.replace && !cp.remove { - log.Printf("[warning] duplicated callback `%v` from %v\n", cp.name, fileWithLineNum()) + cp.logger.Print("warning", fmt.Sprintf("[warning] duplicated callback `%v` from %v", cp.name, fileWithLineNum())) } allNames = append(allNames, cp.name) } diff --git a/vendor/github.com/jinzhu/gorm/callback_create.go b/vendor/github.com/jinzhu/gorm/callback_create.go index 2ab05d3b..c4d25f37 100644 --- a/vendor/github.com/jinzhu/gorm/callback_create.go +++ b/vendor/github.com/jinzhu/gorm/callback_create.go @@ -31,7 +31,7 @@ func beforeCreateCallback(scope *Scope) { // updateTimeStampForCreateCallback will set `CreatedAt`, `UpdatedAt` when creating func updateTimeStampForCreateCallback(scope *Scope) { if !scope.HasError() { - now := NowFunc() + now := scope.db.nowFunc() if createdAtField, ok := scope.FieldByName("CreatedAt"); ok { if createdAtField.IsBlank { @@ -83,21 +83,33 @@ func createCallback(scope *Scope) { quotedTableName = scope.QuotedTableName() primaryField = scope.PrimaryField() extraOption string + insertModifier string ) if str, ok := scope.Get("gorm:insert_option"); ok { extraOption = fmt.Sprint(str) } + if str, ok := scope.Get("gorm:insert_modifier"); ok { + insertModifier = strings.ToUpper(fmt.Sprint(str)) + if insertModifier == "INTO" { + insertModifier = "" + } + } if primaryField != nil { returningColumn = scope.Quote(primaryField.DBName) } - lastInsertIDReturningSuffix := scope.Dialect().LastInsertIDReturningSuffix(quotedTableName, returningColumn) + lastInsertIDOutputInterstitial := scope.Dialect().LastInsertIDOutputInterstitial(quotedTableName, returningColumn, columns) + var lastInsertIDReturningSuffix string + if lastInsertIDOutputInterstitial == "" { + lastInsertIDReturningSuffix = scope.Dialect().LastInsertIDReturningSuffix(quotedTableName, returningColumn) + } if len(columns) == 0 { scope.Raw(fmt.Sprintf( - "INSERT INTO %v %v%v%v", + "INSERT%v INTO %v %v%v%v", + addExtraSpaceIfExist(insertModifier), quotedTableName, scope.Dialect().DefaultValueStr(), addExtraSpaceIfExist(extraOption), @@ -105,17 +117,19 @@ func createCallback(scope *Scope) { )) } else { scope.Raw(fmt.Sprintf( - "INSERT INTO %v (%v) VALUES (%v)%v%v", + "INSERT%v INTO %v (%v)%v VALUES (%v)%v%v", + addExtraSpaceIfExist(insertModifier), scope.QuotedTableName(), strings.Join(columns, ","), + addExtraSpaceIfExist(lastInsertIDOutputInterstitial), strings.Join(placeholders, ","), addExtraSpaceIfExist(extraOption), addExtraSpaceIfExist(lastInsertIDReturningSuffix), )) } - // execute create sql - if lastInsertIDReturningSuffix == "" || primaryField == nil { + // execute create sql: no primaryField + if primaryField == nil { if result, err := scope.SQLDB().Exec(scope.SQL, scope.SQLVars...); scope.Err(err) == nil { // set rows affected count scope.db.RowsAffected, _ = result.RowsAffected() @@ -127,16 +141,35 @@ func createCallback(scope *Scope) { } } } - } else { - if primaryField.Field.CanAddr() { - if err := scope.SQLDB().QueryRow(scope.SQL, scope.SQLVars...).Scan(primaryField.Field.Addr().Interface()); scope.Err(err) == nil { - primaryField.IsBlank = false - scope.db.RowsAffected = 1 + return + } + + // execute create sql: lastInsertID implemention for majority of dialects + if lastInsertIDReturningSuffix == "" && lastInsertIDOutputInterstitial == "" { + if result, err := scope.SQLDB().Exec(scope.SQL, scope.SQLVars...); scope.Err(err) == nil { + // set rows affected count + scope.db.RowsAffected, _ = result.RowsAffected() + + // set primary value to primary field + if primaryField != nil && primaryField.IsBlank { + if primaryValue, err := result.LastInsertId(); scope.Err(err) == nil { + scope.Err(primaryField.Set(primaryValue)) + } } - } else { - scope.Err(ErrUnaddressable) } + return + } + + // execute create sql: dialects with additional lastInsertID requirements (currently postgres & mssql) + if primaryField.Field.CanAddr() { + if err := scope.SQLDB().QueryRow(scope.SQL, scope.SQLVars...).Scan(primaryField.Field.Addr().Interface()); scope.Err(err) == nil { + primaryField.IsBlank = false + scope.db.RowsAffected = 1 + } + } else { + scope.Err(ErrUnaddressable) } + return } } diff --git a/vendor/github.com/jinzhu/gorm/callback_delete.go b/vendor/github.com/jinzhu/gorm/callback_delete.go index 73d90880..48b97acb 100644 --- a/vendor/github.com/jinzhu/gorm/callback_delete.go +++ b/vendor/github.com/jinzhu/gorm/callback_delete.go @@ -17,7 +17,7 @@ func init() { // beforeDeleteCallback will invoke `BeforeDelete` method before deleting func beforeDeleteCallback(scope *Scope) { if scope.DB().HasBlockGlobalUpdate() && !scope.hasConditions() { - scope.Err(errors.New("Missing WHERE clause while deleting")) + scope.Err(errors.New("missing WHERE clause while deleting")) return } if !scope.HasError() { @@ -40,7 +40,7 @@ func deleteCallback(scope *Scope) { "UPDATE %v SET %v=%v%v%v", scope.QuotedTableName(), scope.Quote(deletedAtField.DBName), - scope.AddToVars(NowFunc()), + scope.AddToVars(scope.db.nowFunc()), addExtraSpaceIfExist(scope.CombinedConditionSql()), addExtraSpaceIfExist(extraOption), )).Exec() diff --git a/vendor/github.com/jinzhu/gorm/callback_query.go b/vendor/github.com/jinzhu/gorm/callback_query.go index 593e5d30..544afd63 100644 --- a/vendor/github.com/jinzhu/gorm/callback_query.go +++ b/vendor/github.com/jinzhu/gorm/callback_query.go @@ -18,7 +18,7 @@ func queryCallback(scope *Scope) { if _, skip := scope.InstanceGet("gorm:skip_query_callback"); skip { return } - + //we are only preloading relations, dont touch base model if _, skip := scope.InstanceGet("gorm:only_preload"); skip { return @@ -60,6 +60,11 @@ func queryCallback(scope *Scope) { if !scope.HasError() { scope.db.RowsAffected = 0 + + if str, ok := scope.Get("gorm:query_hint"); ok { + scope.SQL = fmt.Sprint(str) + scope.SQL + } + if str, ok := scope.Get("gorm:query_option"); ok { scope.SQL += addExtraSpaceIfExist(fmt.Sprint(str)) } diff --git a/vendor/github.com/jinzhu/gorm/callback_query_preload.go b/vendor/github.com/jinzhu/gorm/callback_query_preload.go index d7c8a133..a936180a 100644 --- a/vendor/github.com/jinzhu/gorm/callback_query_preload.go +++ b/vendor/github.com/jinzhu/gorm/callback_query_preload.go @@ -391,14 +391,20 @@ func (scope *Scope) handleManyToManyPreload(field *Field, conditions []interface key := toString(getValueFromFields(indirectScopeValue, foreignFieldNames)) fieldsSourceMap[key] = append(fieldsSourceMap[key], indirectScopeValue.FieldByName(field.Name)) } - for source, link := range linkHash { - for i, field := range fieldsSourceMap[source] { + + for source, fields := range fieldsSourceMap { + for _, f := range fields { //If not 0 this means Value is a pointer and we already added preloaded models to it - if fieldsSourceMap[source][i].Len() != 0 { + if f.Len() != 0 { continue } - field.Set(reflect.Append(fieldsSourceMap[source][i], link...)) - } + v := reflect.MakeSlice(f.Type(), 0, 0) + if len(linkHash[source]) > 0 { + v = reflect.Append(f, linkHash[source]...) + } + + f.Set(v) + } } } diff --git a/vendor/github.com/jinzhu/gorm/callback_row_query.go b/vendor/github.com/jinzhu/gorm/callback_row_query.go index c2ff4a08..323b1605 100644 --- a/vendor/github.com/jinzhu/gorm/callback_row_query.go +++ b/vendor/github.com/jinzhu/gorm/callback_row_query.go @@ -1,6 +1,9 @@ package gorm -import "database/sql" +import ( + "database/sql" + "fmt" +) // Define callbacks for row query func init() { @@ -21,6 +24,14 @@ func rowQueryCallback(scope *Scope) { if result, ok := scope.InstanceGet("row_query_result"); ok { scope.prepareQuerySQL() + if str, ok := scope.Get("gorm:query_hint"); ok { + scope.SQL = fmt.Sprint(str) + scope.SQL + } + + if str, ok := scope.Get("gorm:query_option"); ok { + scope.SQL += addExtraSpaceIfExist(fmt.Sprint(str)) + } + if rowResult, ok := result.(*RowQueryResult); ok { rowResult.Row = scope.SQLDB().QueryRow(scope.SQL, scope.SQLVars...) } else if rowsResult, ok := result.(*RowsQueryResult); ok { diff --git a/vendor/github.com/jinzhu/gorm/callback_update.go b/vendor/github.com/jinzhu/gorm/callback_update.go index f6ba0ffd..699e534b 100644 --- a/vendor/github.com/jinzhu/gorm/callback_update.go +++ b/vendor/github.com/jinzhu/gorm/callback_update.go @@ -34,7 +34,7 @@ func assignUpdatingAttributesCallback(scope *Scope) { // beforeUpdateCallback will invoke `BeforeSave`, `BeforeUpdate` method before updating func beforeUpdateCallback(scope *Scope) { if scope.DB().HasBlockGlobalUpdate() && !scope.hasConditions() { - scope.Err(errors.New("Missing WHERE clause while updating")) + scope.Err(errors.New("missing WHERE clause while updating")) return } if _, ok := scope.Get("gorm:update_column"); !ok { @@ -50,7 +50,7 @@ func beforeUpdateCallback(scope *Scope) { // updateTimeStampForUpdateCallback will set `UpdatedAt` when updating func updateTimeStampForUpdateCallback(scope *Scope) { if _, ok := scope.Get("gorm:update_column"); !ok { - scope.SetColumn("UpdatedAt", NowFunc()) + scope.SetColumn("UpdatedAt", scope.db.nowFunc()) } } @@ -75,7 +75,7 @@ func updateCallback(scope *Scope) { } else { for _, field := range scope.Fields() { if scope.changeableField(field) { - if !field.IsPrimaryKey && field.IsNormal { + if !field.IsPrimaryKey && field.IsNormal && (field.Name != "CreatedAt" || !field.IsBlank) { if !field.IsForeignKey || !field.IsBlank || !field.HasDefaultValue { sqls = append(sqls, fmt.Sprintf("%v = %v", scope.Quote(field.DBName), scope.AddToVars(field.Field.Interface()))) } diff --git a/vendor/github.com/jinzhu/gorm/dialect.go b/vendor/github.com/jinzhu/gorm/dialect.go index 27b308af..749587f4 100644 --- a/vendor/github.com/jinzhu/gorm/dialect.go +++ b/vendor/github.com/jinzhu/gorm/dialect.go @@ -37,9 +37,11 @@ type Dialect interface { ModifyColumn(tableName string, columnName string, typ string) error // LimitAndOffsetSQL return generated SQL with Limit and Offset, as mssql has special case - LimitAndOffsetSQL(limit, offset interface{}) string + LimitAndOffsetSQL(limit, offset interface{}) (string, error) // SelectFromDummyTable return select values, for most dbs, `SELECT values` just works, mysql needs `SELECT value FROM DUAL` SelectFromDummyTable() string + // LastInsertIDOutputInterstitial most dbs support LastInsertId, but mssql needs to use `OUTPUT` + LastInsertIDOutputInterstitial(tableName, columnName string, columns []string) string // LastInsertIdReturningSuffix most dbs support LastInsertId, but postgres needs to use `RETURNING` LastInsertIDReturningSuffix(tableName, columnName string) string // DefaultValueStr @@ -48,6 +50,9 @@ type Dialect interface { // BuildKeyName returns a valid key name (foreign key, index key) for the given table, field and reference BuildKeyName(kind, tableName string, fields ...string) string + // NormalizeIndexAndColumn returns valid index name and column name depending on each dialect + NormalizeIndexAndColumn(indexName, columnName string) (string, string) + // CurrentDatabase return current database name CurrentDatabase() string } @@ -126,6 +131,10 @@ var ParseFieldStructForDialect = func(field *StructField, dialect Dialect) (fiel additionalType = additionalType + " DEFAULT " + value } + if value, ok := field.TagSettingsGet("COMMENT"); ok { + additionalType = additionalType + " COMMENT " + value + } + return fieldValue, dataType, size, strings.TrimSpace(additionalType) } diff --git a/vendor/github.com/jinzhu/gorm/dialect_common.go b/vendor/github.com/jinzhu/gorm/dialect_common.go index a479be79..d549510c 100644 --- a/vendor/github.com/jinzhu/gorm/dialect_common.go +++ b/vendor/github.com/jinzhu/gorm/dialect_common.go @@ -9,6 +9,8 @@ import ( "time" ) +var keyNameRegex = regexp.MustCompile("[^a-zA-Z0-9]+") + // DefaultForeignKeyNamer contains the default foreign key name generator method type DefaultForeignKeyNamer struct { } @@ -137,14 +139,19 @@ func (s commonDialect) CurrentDatabase() (name string) { return } -func (commonDialect) LimitAndOffsetSQL(limit, offset interface{}) (sql string) { +// LimitAndOffsetSQL return generated SQL with Limit and Offset +func (s commonDialect) LimitAndOffsetSQL(limit, offset interface{}) (sql string, err error) { if limit != nil { - if parsedLimit, err := strconv.ParseInt(fmt.Sprint(limit), 0, 0); err == nil && parsedLimit >= 0 { + if parsedLimit, err := s.parseInt(limit); err != nil { + return "", err + } else if parsedLimit >= 0 { sql += fmt.Sprintf(" LIMIT %d", parsedLimit) } } if offset != nil { - if parsedOffset, err := strconv.ParseInt(fmt.Sprint(offset), 0, 0); err == nil && parsedOffset >= 0 { + if parsedOffset, err := s.parseInt(offset); err != nil { + return "", err + } else if parsedOffset >= 0 { sql += fmt.Sprintf(" OFFSET %d", parsedOffset) } } @@ -155,6 +162,10 @@ func (commonDialect) SelectFromDummyTable() string { return "" } +func (commonDialect) LastInsertIDOutputInterstitial(tableName, columnName string, columns []string) string { + return "" +} + func (commonDialect) LastInsertIDReturningSuffix(tableName, columnName string) string { return "" } @@ -166,10 +177,19 @@ func (commonDialect) DefaultValueStr() string { // BuildKeyName returns a valid key name (foreign key, index key) for the given table, field and reference func (DefaultForeignKeyNamer) BuildKeyName(kind, tableName string, fields ...string) string { keyName := fmt.Sprintf("%s_%s_%s", kind, tableName, strings.Join(fields, "_")) - keyName = regexp.MustCompile("[^a-zA-Z0-9]+").ReplaceAllString(keyName, "_") + keyName = keyNameRegex.ReplaceAllString(keyName, "_") return keyName } +// NormalizeIndexAndColumn returns argument's index name and column name without doing anything +func (commonDialect) NormalizeIndexAndColumn(indexName, columnName string) (string, string) { + return indexName, columnName +} + +func (commonDialect) parseInt(value interface{}) (int64, error) { + return strconv.ParseInt(fmt.Sprint(value), 0, 0) +} + // IsByteArrayOrSlice returns true of the reflected value is an array or slice func IsByteArrayOrSlice(value reflect.Value) bool { return (value.Kind() == reflect.Array || value.Kind() == reflect.Slice) && value.Type().Elem() == reflect.TypeOf(uint8(0)) diff --git a/vendor/github.com/jinzhu/gorm/dialect_mysql.go b/vendor/github.com/jinzhu/gorm/dialect_mysql.go index 5d63e5cd..b4467ffa 100644 --- a/vendor/github.com/jinzhu/gorm/dialect_mysql.go +++ b/vendor/github.com/jinzhu/gorm/dialect_mysql.go @@ -2,15 +2,17 @@ package gorm import ( "crypto/sha1" + "database/sql" "fmt" "reflect" "regexp" - "strconv" "strings" "time" "unicode/utf8" ) +var mysqlIndexRegex = regexp.MustCompile(`^(.+)\((\d+)\)$`) + type mysql struct { commonDialect } @@ -100,10 +102,10 @@ func (s *mysql) DataTypeOf(field *StructField) string { precision = fmt.Sprintf("(%s)", p) } - if _, ok := field.TagSettingsGet("NOT NULL"); ok { - sqlType = fmt.Sprintf("timestamp%v", precision) + if _, ok := field.TagSettings["NOT NULL"]; ok || field.IsPrimaryKey { + sqlType = fmt.Sprintf("DATETIME%v", precision) } else { - sqlType = fmt.Sprintf("timestamp%v NULL", precision) + sqlType = fmt.Sprintf("DATETIME%v NULL", precision) } } default: @@ -118,7 +120,7 @@ func (s *mysql) DataTypeOf(field *StructField) string { } if sqlType == "" { - panic(fmt.Sprintf("invalid sql type %s (%s) for mysql", dataValue.Type().Name(), dataValue.Kind().String())) + panic(fmt.Sprintf("invalid sql type %s (%s) in field %s for mysql", dataValue.Type().Name(), dataValue.Kind().String(), field.Name)) } if strings.TrimSpace(additionalType) == "" { @@ -137,13 +139,21 @@ func (s mysql) ModifyColumn(tableName string, columnName string, typ string) err return err } -func (s mysql) LimitAndOffsetSQL(limit, offset interface{}) (sql string) { +func (s mysql) LimitAndOffsetSQL(limit, offset interface{}) (sql string, err error) { if limit != nil { - if parsedLimit, err := strconv.ParseInt(fmt.Sprint(limit), 0, 0); err == nil && parsedLimit >= 0 { + parsedLimit, err := s.parseInt(limit) + if err != nil { + return "", err + } + if parsedLimit >= 0 { sql += fmt.Sprintf(" LIMIT %d", parsedLimit) if offset != nil { - if parsedOffset, err := strconv.ParseInt(fmt.Sprint(offset), 0, 0); err == nil && parsedOffset >= 0 { + parsedOffset, err := s.parseInt(offset) + if err != nil { + return "", err + } + if parsedOffset >= 0 { sql += fmt.Sprintf(" OFFSET %d", parsedOffset) } } @@ -159,6 +169,40 @@ func (s mysql) HasForeignKey(tableName string, foreignKeyName string) bool { return count > 0 } +func (s mysql) HasTable(tableName string) bool { + currentDatabase, tableName := currentDatabaseAndTable(&s, tableName) + var name string + // allow mysql database name with '-' character + if err := s.db.QueryRow(fmt.Sprintf("SHOW TABLES FROM `%s` WHERE `Tables_in_%s` = ?", currentDatabase, currentDatabase), tableName).Scan(&name); err != nil { + if err == sql.ErrNoRows { + return false + } + panic(err) + } else { + return true + } +} + +func (s mysql) HasIndex(tableName string, indexName string) bool { + currentDatabase, tableName := currentDatabaseAndTable(&s, tableName) + if rows, err := s.db.Query(fmt.Sprintf("SHOW INDEXES FROM `%s` FROM `%s` WHERE Key_name = ?", tableName, currentDatabase), indexName); err != nil { + panic(err) + } else { + defer rows.Close() + return rows.Next() + } +} + +func (s mysql) HasColumn(tableName string, columnName string) bool { + currentDatabase, tableName := currentDatabaseAndTable(&s, tableName) + if rows, err := s.db.Query(fmt.Sprintf("SHOW COLUMNS FROM `%s` FROM `%s` WHERE Field = ?", tableName, currentDatabase), columnName); err != nil { + panic(err) + } else { + defer rows.Close() + return rows.Next() + } +} + func (s mysql) CurrentDatabase() (name string) { s.db.QueryRow("SELECT DATABASE()").Scan(&name) return @@ -178,7 +222,7 @@ func (s mysql) BuildKeyName(kind, tableName string, fields ...string) string { bs := h.Sum(nil) // sha1 is 40 characters, keep first 24 characters of destination - destRunes := []rune(regexp.MustCompile("[^a-zA-Z0-9]+").ReplaceAllString(fields[0], "_")) + destRunes := []rune(keyNameRegex.ReplaceAllString(fields[0], "_")) if len(destRunes) > 24 { destRunes = destRunes[:24] } @@ -186,6 +230,17 @@ func (s mysql) BuildKeyName(kind, tableName string, fields ...string) string { return fmt.Sprintf("%s%x", string(destRunes), bs) } +// NormalizeIndexAndColumn returns index name and column name for specify an index prefix length if needed +func (mysql) NormalizeIndexAndColumn(indexName, columnName string) (string, string) { + submatch := mysqlIndexRegex.FindStringSubmatch(indexName) + if len(submatch) != 3 { + return indexName, columnName + } + indexName = submatch[1] + columnName = fmt.Sprintf("%s(%s)", columnName, submatch[2]) + return indexName, columnName +} + func (mysql) DefaultValueStr() string { return "VALUES()" } diff --git a/vendor/github.com/jinzhu/gorm/dialect_postgres.go b/vendor/github.com/jinzhu/gorm/dialect_postgres.go index 53d31388..d2df3131 100644 --- a/vendor/github.com/jinzhu/gorm/dialect_postgres.go +++ b/vendor/github.com/jinzhu/gorm/dialect_postgres.go @@ -120,6 +120,10 @@ func (s postgres) CurrentDatabase() (name string) { return } +func (s postgres) LastInsertIDOutputInterstitial(tableName, key string, columns []string) string { + return "" +} + func (s postgres) LastInsertIDReturningSuffix(tableName, key string) string { return fmt.Sprintf("RETURNING %v.%v", tableName, key) } diff --git a/vendor/github.com/jinzhu/gorm/dialects/postgres/postgres.go b/vendor/github.com/jinzhu/gorm/dialects/postgres/postgres.go index 424e8bdc..e6c088b1 100644 --- a/vendor/github.com/jinzhu/gorm/dialects/postgres/postgres.go +++ b/vendor/github.com/jinzhu/gorm/dialects/postgres/postgres.go @@ -7,6 +7,7 @@ import ( "encoding/json" "errors" "fmt" + _ "github.com/lib/pq" "github.com/lib/pq/hstore" ) diff --git a/vendor/github.com/jinzhu/gorm/errors.go b/vendor/github.com/jinzhu/gorm/errors.go index 27c9a92d..d5ef8d57 100644 --- a/vendor/github.com/jinzhu/gorm/errors.go +++ b/vendor/github.com/jinzhu/gorm/errors.go @@ -6,11 +6,11 @@ import ( ) var ( - // ErrRecordNotFound record not found error, happens when only haven't find any matched data when looking up with a struct, finding a slice won't return this error + // ErrRecordNotFound returns a "record not found error". Occurs only when attempting to query the database with a struct; querying with a slice won't return this error ErrRecordNotFound = errors.New("record not found") - // ErrInvalidSQL invalid SQL error, happens when you passed invalid SQL + // ErrInvalidSQL occurs when you attempt a query with invalid SQL ErrInvalidSQL = errors.New("invalid SQL") - // ErrInvalidTransaction invalid transaction when you are trying to `Commit` or `Rollback` + // ErrInvalidTransaction occurs when you are trying to `Commit` or `Rollback` ErrInvalidTransaction = errors.New("no valid transaction") // ErrCantStartTransaction can't start transaction when you are trying to start one with `Begin` ErrCantStartTransaction = errors.New("can't start transaction") @@ -21,7 +21,7 @@ var ( // Errors contains all happened errors type Errors []error -// IsRecordNotFoundError returns current error has record not found error or not +// IsRecordNotFoundError returns true if error contains a RecordNotFound error func IsRecordNotFoundError(err error) bool { if errs, ok := err.(Errors); ok { for _, err := range errs { @@ -33,12 +33,12 @@ func IsRecordNotFoundError(err error) bool { return err == ErrRecordNotFound } -// GetErrors gets all happened errors +// GetErrors gets all errors that have occurred and returns a slice of errors (Error type) func (errs Errors) GetErrors() []error { return errs } -// Add adds an error +// Add adds an error to a given slice of errors func (errs Errors) Add(newErrors ...error) Errors { for _, err := range newErrors { if err == nil { @@ -62,7 +62,7 @@ func (errs Errors) Add(newErrors ...error) Errors { return errs } -// Error format happened errors +// Error takes a slice of all errors that have occurred and returns it as a formatted string func (errs Errors) Error() string { var errors = []string{} for _, e := range errs { diff --git a/vendor/github.com/jinzhu/gorm/field.go b/vendor/github.com/jinzhu/gorm/field.go index 11c410b0..acd06e20 100644 --- a/vendor/github.com/jinzhu/gorm/field.go +++ b/vendor/github.com/jinzhu/gorm/field.go @@ -2,6 +2,7 @@ package gorm import ( "database/sql" + "database/sql/driver" "errors" "fmt" "reflect" @@ -44,7 +45,14 @@ func (field *Field) Set(value interface{}) (err error) { if reflectValue.Type().ConvertibleTo(fieldValue.Type()) { fieldValue.Set(reflectValue.Convert(fieldValue.Type())) } else if scanner, ok := fieldValue.Addr().Interface().(sql.Scanner); ok { - err = scanner.Scan(reflectValue.Interface()) + v := reflectValue.Interface() + if valuer, ok := v.(driver.Valuer); ok { + if v, err = valuer.Value(); err == nil { + err = scanner.Scan(v) + } + } else { + err = scanner.Scan(v) + } } else { err = fmt.Errorf("could not convert argument of field %s from %s to %s", field.Name, reflectValue.Type(), fieldValue.Type()) } diff --git a/vendor/github.com/jinzhu/gorm/go.mod b/vendor/github.com/jinzhu/gorm/go.mod new file mode 100644 index 00000000..64a42a4e --- /dev/null +++ b/vendor/github.com/jinzhu/gorm/go.mod @@ -0,0 +1,14 @@ +module github.com/jinzhu/gorm + +go 1.12 + +require ( + github.com/denisenkom/go-mssqldb v0.0.0-20191124224453-732737034ffd + github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5 + github.com/go-sql-driver/mysql v1.5.0 + github.com/jinzhu/inflection v1.0.0 + github.com/jinzhu/now v1.0.1 + github.com/lib/pq v1.1.1 + github.com/mattn/go-sqlite3 v1.14.0 + golang.org/x/crypto v0.0.0-20191205180655-e7c4368fe9dd // indirect +) diff --git a/vendor/github.com/jinzhu/gorm/go.sum b/vendor/github.com/jinzhu/gorm/go.sum new file mode 100644 index 00000000..d30630b9 --- /dev/null +++ b/vendor/github.com/jinzhu/gorm/go.sum @@ -0,0 +1,33 @@ +github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= +github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= +github.com/denisenkom/go-mssqldb v0.0.0-20191124224453-732737034ffd h1:83Wprp6ROGeiHFAP8WJdI2RoxALQYgdllERc3N5N2DM= +github.com/denisenkom/go-mssqldb v0.0.0-20191124224453-732737034ffd/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= +github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5 h1:Yzb9+7DPaBjB8zlTR87/ElzFsnQfuHnVUVqpZZIcV5Y= +github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0= +github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs= +github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe h1:lXe2qZdvpiX5WZkZR4hgp4KJVfY3nMkvmwbVkpv1rVY= +github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= +github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= +github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= +github.com/jinzhu/now v1.0.1 h1:HjfetcXq097iXP0uoPCdnM4Efp5/9MsM0/M+XOTeR3M= +github.com/jinzhu/now v1.0.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/lib/pq v1.1.1 h1:sJZmqHoEaY7f+NPP8pgLB/WxulyR3fewgCM2qaSlBb4= +github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/mattn/go-sqlite3 v1.14.0 h1:mLyGNKR8+Vv9CAU7PphKa2hkEqxxhn8i32J6FPj1/QA= +github.com/mattn/go-sqlite3 v1.14.0/go.mod h1:JIl7NbARA7phWnGvh0LKTyg7S9BA+6gx71ShQilpsus= +github.com/mattn/go-sqlite3 v2.0.1+incompatible h1:xQ15muvnzGBHpIpdrNi1DA5x0+TcBZzsIDwmw9uTHzw= +github.com/mattn/go-sqlite3 v2.0.1+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c h1:Vj5n4GlwjmQteupaxJ9+0FNOmBrHfq7vN4btdGoDZgI= +golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191205180655-e7c4368fe9dd h1:GGJVjV8waZKRHrgwvtH66z9ZGVurTD1MT0n1Bb+q4aM= +golang.org/x/crypto v0.0.0-20191205180655-e7c4368fe9dd/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/vendor/github.com/jinzhu/gorm/interface.go b/vendor/github.com/jinzhu/gorm/interface.go index 55128f7f..fe649231 100644 --- a/vendor/github.com/jinzhu/gorm/interface.go +++ b/vendor/github.com/jinzhu/gorm/interface.go @@ -1,6 +1,9 @@ package gorm -import "database/sql" +import ( + "context" + "database/sql" +) // SQLCommon is the minimal database connection functionality gorm requires. Implemented by *sql.DB. type SQLCommon interface { @@ -12,6 +15,7 @@ type SQLCommon interface { type sqlDb interface { Begin() (*sql.Tx, error) + BeginTx(ctx context.Context, opts *sql.TxOptions) (*sql.Tx, error) } type sqlTx interface { diff --git a/vendor/github.com/jinzhu/gorm/logger.go b/vendor/github.com/jinzhu/gorm/logger.go index 4324a2e4..88e167dd 100644 --- a/vendor/github.com/jinzhu/gorm/logger.go +++ b/vendor/github.com/jinzhu/gorm/logger.go @@ -39,6 +39,15 @@ var LogFormatter = func(values ...interface{}) (messages []interface{}) { messages = []interface{}{source, currentTime} + if len(values) == 2 { + //remove the line break + currentTime = currentTime[1:] + //remove the brackets + source = fmt.Sprintf("\033[35m%v\033[0m", values[1]) + + messages = []interface{}{currentTime, source} + } + if level == "sql" { // duration messages = append(messages, fmt.Sprintf(" \033[36;1m[%.2fms]\033[0m ", float64(values[2].(time.Duration).Nanoseconds()/1e4)/100.0)) @@ -49,7 +58,11 @@ var LogFormatter = func(values ...interface{}) (messages []interface{}) { if indirectValue.IsValid() { value = indirectValue.Interface() if t, ok := value.(time.Time); ok { - formattedValues = append(formattedValues, fmt.Sprintf("'%v'", t.Format("2006-01-02 15:04:05"))) + if t.IsZero() { + formattedValues = append(formattedValues, fmt.Sprintf("'%v'", "0000-00-00 00:00:00")) + } else { + formattedValues = append(formattedValues, fmt.Sprintf("'%v'", t.Format("2006-01-02 15:04:05"))) + } } else if b, ok := value.([]byte); ok { if str := string(b); isPrintable(str) { formattedValues = append(formattedValues, fmt.Sprintf("'%v'", str)) @@ -63,7 +76,12 @@ var LogFormatter = func(values ...interface{}) (messages []interface{}) { formattedValues = append(formattedValues, "NULL") } } else { - formattedValues = append(formattedValues, fmt.Sprintf("'%v'", value)) + switch value.(type) { + case int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64, float32, float64, bool: + formattedValues = append(formattedValues, fmt.Sprintf("%v", value)) + default: + formattedValues = append(formattedValues, fmt.Sprintf("'%v'", value)) + } } } else { formattedValues = append(formattedValues, "NULL") @@ -117,3 +135,7 @@ type Logger struct { func (logger Logger) Print(values ...interface{}) { logger.Println(LogFormatter(values...)...) } + +type nopLogger struct{} + +func (nopLogger) Print(values ...interface{}) {} diff --git a/vendor/github.com/jinzhu/gorm/main.go b/vendor/github.com/jinzhu/gorm/main.go index 17c75ed3..466e80c3 100644 --- a/vendor/github.com/jinzhu/gorm/main.go +++ b/vendor/github.com/jinzhu/gorm/main.go @@ -1,6 +1,7 @@ package gorm import ( + "context" "database/sql" "errors" "fmt" @@ -12,6 +13,7 @@ import ( // DB contains information for current db connection type DB struct { + sync.RWMutex Value interface{} Error error RowsAffected int64 @@ -19,7 +21,7 @@ type DB struct { // single db db SQLCommon blockGlobalUpdate bool - logMode int + logMode logModeValue logger logger search *search values sync.Map @@ -29,8 +31,19 @@ type DB struct { callbacks *Callback dialect Dialect singularTable bool + + // function to be used to override the creating of a new timestamp + nowFuncOverride func() time.Time } +type logModeValue int + +const ( + defaultLogMode logModeValue = iota + noLogMode + detailedLogMode +) + // Open initialize a new db connection, need to import driver first, e.g: // // import _ "github.com/go-sql-driver/mysql" @@ -111,7 +124,10 @@ func (s *DB) Close() error { // DB get `*sql.DB` from current connection // If the underlying database connection is not a *sql.DB, returns nil func (s *DB) DB() *sql.DB { - db, _ := s.db.(*sql.DB) + db, ok := s.db.(*sql.DB) + if !ok { + panic("can't support full GORM on currently status, maybe this is a TX instance.") + } return db } @@ -129,7 +145,7 @@ func (s *DB) Dialect() Dialect { // db.Callback().Create().Register("update_created_at", updateCreated) // Refer https://jinzhu.github.io/gorm/development.html#callbacks func (s *DB) Callback() *Callback { - s.parent.callbacks = s.parent.callbacks.clone() + s.parent.callbacks = s.parent.callbacks.clone(s.logger) return s.parent.callbacks } @@ -141,13 +157,29 @@ func (s *DB) SetLogger(log logger) { // LogMode set log mode, `true` for detailed logs, `false` for no log, default, will only print error logs func (s *DB) LogMode(enable bool) *DB { if enable { - s.logMode = 2 + s.logMode = detailedLogMode } else { - s.logMode = 1 + s.logMode = noLogMode } return s } +// SetNowFuncOverride set the function to be used when creating a new timestamp +func (s *DB) SetNowFuncOverride(nowFuncOverride func() time.Time) *DB { + s.nowFuncOverride = nowFuncOverride + return s +} + +// Get a new timestamp, using the provided nowFuncOverride on the DB instance if set, +// otherwise defaults to the global NowFunc() +func (s *DB) nowFunc() time.Time { + if s.nowFuncOverride != nil { + return s.nowFuncOverride() + } + + return NowFunc() +} + // BlockGlobalUpdate if true, generates an error on update/delete without where clause. // This is to prevent eventual error with empty objects updates/deletions func (s *DB) BlockGlobalUpdate(enable bool) *DB { @@ -162,7 +194,8 @@ func (s *DB) HasBlockGlobalUpdate() bool { // SingularTable use singular table by default func (s *DB) SingularTable(enable bool) { - modelStructsMap = sync.Map{} + s.parent.Lock() + defer s.parent.Unlock() s.parent.singularTable = enable } @@ -170,11 +203,17 @@ func (s *DB) SingularTable(enable bool) { func (s *DB) NewScope(value interface{}) *Scope { dbClone := s.clone() dbClone.Value = value - return &Scope{db: dbClone, Search: dbClone.search.clone(), Value: value} + scope := &Scope{db: dbClone, Value: value} + if s.search != nil { + scope.Search = s.search.clone() + } else { + scope.Search = &search{} + } + return scope } -// QueryExpr returns the query as expr object -func (s *DB) QueryExpr() *expr { +// QueryExpr returns the query as SqlExpr object +func (s *DB) QueryExpr() *SqlExpr { scope := s.NewScope(s.Value) scope.InstanceSet("skip_bindvar", true) scope.prepareQuerySQL() @@ -183,7 +222,7 @@ func (s *DB) QueryExpr() *expr { } // SubQuery returns the query as sub query -func (s *DB) SubQuery() *expr { +func (s *DB) SubQuery() *SqlExpr { scope := s.NewScope(s.Value) scope.InstanceSet("skip_bindvar", true) scope.prepareQuerySQL() @@ -290,6 +329,7 @@ func (s *DB) Assign(attrs ...interface{}) *DB { func (s *DB) First(out interface{}, where ...interface{}) *DB { newScope := s.NewScope(out) newScope.Search.Limit(1) + return newScope.Set("gorm:order_by_primary_key", "ASC"). inlineCondition(where...).callCallbacks(s.parent.callbacks.queries).db } @@ -397,6 +437,7 @@ func (s *DB) FirstOrCreate(out interface{}, where ...interface{}) *DB { } // Update update attributes with callbacks, refer: https://jinzhu.github.io/gorm/crud.html#update +// WARNING when update with struct, GORM will not update fields that with zero value func (s *DB) Update(attrs ...interface{}) *DB { return s.Updates(toSearchableMap(attrs...), true) } @@ -429,7 +470,7 @@ func (s *DB) Save(value interface{}) *DB { if !scope.PrimaryKeyZero() { newDB := scope.callCallbacks(s.parent.callbacks.updates).db if newDB.Error == nil && newDB.RowsAffected == 0 { - return s.New().FirstOrCreate(value) + return s.New().Table(scope.TableName()).FirstOrCreate(value) } return newDB } @@ -443,6 +484,7 @@ func (s *DB) Create(value interface{}) *DB { } // Delete delete value match given conditions, if the value has primary key, then will including the primary key as condition +// WARNING If model has DeletedAt field, GORM will only set field DeletedAt's value to current time func (s *DB) Delete(value interface{}, where ...interface{}) *DB { return s.NewScope(value).inlineCondition(where...).callCallbacks(s.parent.callbacks.deletes).db } @@ -486,11 +528,43 @@ func (s *DB) Debug() *DB { return s.clone().LogMode(true) } -// Begin begin a transaction +// Transaction start a transaction as a block, +// return error will rollback, otherwise to commit. +func (s *DB) Transaction(fc func(tx *DB) error) (err error) { + + if _, ok := s.db.(*sql.Tx); ok { + return fc(s) + } + + panicked := true + tx := s.Begin() + defer func() { + // Make sure to rollback when panic, Block error or Commit error + if panicked || err != nil { + tx.Rollback() + } + }() + + err = fc(tx) + + if err == nil { + err = tx.Commit().Error + } + + panicked = false + return +} + +// Begin begins a transaction func (s *DB) Begin() *DB { + return s.BeginTx(context.Background(), &sql.TxOptions{}) +} + +// BeginTx begins a transaction with options +func (s *DB) BeginTx(ctx context.Context, opts *sql.TxOptions) *DB { c := s.clone() if db, ok := c.db.(sqlDb); ok && db != nil { - tx, err := db.Begin() + tx, err := db.BeginTx(ctx, opts) c.db = interface{}(tx).(SQLCommon) c.dialect.SetDB(c.db) @@ -516,7 +590,26 @@ func (s *DB) Commit() *DB { func (s *DB) Rollback() *DB { var emptySQLTx *sql.Tx if db, ok := s.db.(sqlTx); ok && db != nil && db != emptySQLTx { - s.AddError(db.Rollback()) + if err := db.Rollback(); err != nil && err != sql.ErrTxDone { + s.AddError(err) + } + } else { + s.AddError(ErrInvalidTransaction) + } + return s +} + +// RollbackUnlessCommitted rollback a transaction if it has not yet been +// committed. +func (s *DB) RollbackUnlessCommitted() *DB { + var emptySQLTx *sql.Tx + if db, ok := s.db.(sqlTx); ok && db != nil && db != emptySQLTx { + err := db.Rollback() + // Ignore the error indicating that the transaction has already + // been committed. + if err != sql.ErrTxDone { + s.AddError(err) + } } else { s.AddError(ErrInvalidTransaction) } @@ -716,8 +809,8 @@ func (s *DB) SetJoinTableHandler(source interface{}, column string, handler Join func (s *DB) AddError(err error) error { if err != nil { if err != ErrRecordNotFound { - if s.logMode == 0 { - go s.print(fileWithLineNum(), err) + if s.logMode == defaultLogMode { + go s.print("error", fileWithLineNum(), err) } else { s.log(err) } @@ -758,6 +851,7 @@ func (s *DB) clone() *DB { Error: s.Error, blockGlobalUpdate: s.blockGlobalUpdate, dialect: newDialect(s.dialect.GetName(), s.db), + nowFuncOverride: s.nowFuncOverride, } s.values.Range(func(k, v interface{}) bool { @@ -780,13 +874,13 @@ func (s *DB) print(v ...interface{}) { } func (s *DB) log(v ...interface{}) { - if s != nil && s.logMode == 2 { + if s != nil && s.logMode == detailedLogMode { s.print(append([]interface{}{"log", fileWithLineNum()}, v...)...) } } func (s *DB) slog(sql string, t time.Time, vars ...interface{}) { - if s.logMode == 2 { + if s.logMode == detailedLogMode { s.print("sql", fileWithLineNum(), NowFunc().Sub(t), sql, vars, s.RowsAffected) } } diff --git a/vendor/github.com/jinzhu/gorm/model_struct.go b/vendor/github.com/jinzhu/gorm/model_struct.go index 12860e67..57dbec38 100644 --- a/vendor/github.com/jinzhu/gorm/model_struct.go +++ b/vendor/github.com/jinzhu/gorm/model_struct.go @@ -17,27 +17,38 @@ var DefaultTableNameHandler = func(db *DB, defaultTableName string) string { return defaultTableName } +// lock for mutating global cached model metadata +var structsLock sync.Mutex + +// global cache of model metadata var modelStructsMap sync.Map // ModelStruct model definition type ModelStruct struct { - PrimaryFields []*StructField - StructFields []*StructField - ModelType reflect.Type + PrimaryFields []*StructField + StructFields []*StructField + ModelType reflect.Type + defaultTableName string + l sync.Mutex } // TableName returns model's table name func (s *ModelStruct) TableName(db *DB) string { + s.l.Lock() + defer s.l.Unlock() + if s.defaultTableName == "" && db != nil && s.ModelType != nil { // Set default table name if tabler, ok := reflect.New(s.ModelType).Interface().(tabler); ok { s.defaultTableName = tabler.TableName() } else { tableName := ToTableName(s.ModelType.Name()) - if db == nil || !db.parent.singularTable { + db.parent.RLock() + if db == nil || (db.parent != nil && !db.parent.singularTable) { tableName = inflection.Plural(tableName) } + db.parent.RUnlock() s.defaultTableName = tableName } } @@ -65,52 +76,52 @@ type StructField struct { } // TagSettingsSet Sets a tag in the tag settings map -func (s *StructField) TagSettingsSet(key, val string) { - s.tagSettingsLock.Lock() - defer s.tagSettingsLock.Unlock() - s.TagSettings[key] = val +func (sf *StructField) TagSettingsSet(key, val string) { + sf.tagSettingsLock.Lock() + defer sf.tagSettingsLock.Unlock() + sf.TagSettings[key] = val } // TagSettingsGet returns a tag from the tag settings -func (s *StructField) TagSettingsGet(key string) (string, bool) { - s.tagSettingsLock.RLock() - defer s.tagSettingsLock.RUnlock() - val, ok := s.TagSettings[key] +func (sf *StructField) TagSettingsGet(key string) (string, bool) { + sf.tagSettingsLock.RLock() + defer sf.tagSettingsLock.RUnlock() + val, ok := sf.TagSettings[key] return val, ok } // TagSettingsDelete deletes a tag -func (s *StructField) TagSettingsDelete(key string) { - s.tagSettingsLock.Lock() - defer s.tagSettingsLock.Unlock() - delete(s.TagSettings, key) +func (sf *StructField) TagSettingsDelete(key string) { + sf.tagSettingsLock.Lock() + defer sf.tagSettingsLock.Unlock() + delete(sf.TagSettings, key) } -func (structField *StructField) clone() *StructField { +func (sf *StructField) clone() *StructField { clone := &StructField{ - DBName: structField.DBName, - Name: structField.Name, - Names: structField.Names, - IsPrimaryKey: structField.IsPrimaryKey, - IsNormal: structField.IsNormal, - IsIgnored: structField.IsIgnored, - IsScanner: structField.IsScanner, - HasDefaultValue: structField.HasDefaultValue, - Tag: structField.Tag, + DBName: sf.DBName, + Name: sf.Name, + Names: sf.Names, + IsPrimaryKey: sf.IsPrimaryKey, + IsNormal: sf.IsNormal, + IsIgnored: sf.IsIgnored, + IsScanner: sf.IsScanner, + HasDefaultValue: sf.HasDefaultValue, + Tag: sf.Tag, TagSettings: map[string]string{}, - Struct: structField.Struct, - IsForeignKey: structField.IsForeignKey, + Struct: sf.Struct, + IsForeignKey: sf.IsForeignKey, } - if structField.Relationship != nil { - relationship := *structField.Relationship + if sf.Relationship != nil { + relationship := *sf.Relationship clone.Relationship = &relationship } // copy the struct field tagSettings, they should be read-locked while they are copied - structField.tagSettingsLock.Lock() - defer structField.tagSettingsLock.Unlock() - for key, value := range structField.TagSettings { + sf.tagSettingsLock.Lock() + defer sf.tagSettingsLock.Unlock() + for key, value := range sf.TagSettings { clone.TagSettings[key] = value } @@ -141,6 +152,10 @@ func getForeignField(column string, fields []*StructField) *StructField { // GetModelStruct get value's model struct, relationships based on struct and tag definition func (scope *Scope) GetModelStruct() *ModelStruct { + return scope.getModelStruct(scope, make([]*StructField, 0)) +} + +func (scope *Scope) getModelStruct(rootScope *Scope, allFields []*StructField) *ModelStruct { var modelStruct ModelStruct // Scope value can't be nil if scope.Value == nil { @@ -158,7 +173,18 @@ func (scope *Scope) GetModelStruct() *ModelStruct { } // Get Cached model struct - if value, ok := modelStructsMap.Load(reflectType); ok && value != nil { + isSingularTable := false + if scope.db != nil && scope.db.parent != nil { + scope.db.parent.RLock() + isSingularTable = scope.db.parent.singularTable + scope.db.parent.RUnlock() + } + + hashKey := struct { + singularTable bool + reflectType reflect.Type + }{isSingularTable, reflectType} + if value, ok := modelStructsMap.Load(hashKey); ok && value != nil { return value.(*ModelStruct) } @@ -184,7 +210,7 @@ func (scope *Scope) GetModelStruct() *ModelStruct { modelStruct.PrimaryFields = append(modelStruct.PrimaryFields, field) } - if _, ok := field.TagSettingsGet("DEFAULT"); ok { + if _, ok := field.TagSettingsGet("DEFAULT"); ok && !field.IsPrimaryKey { field.HasDefaultValue = true } @@ -215,7 +241,7 @@ func (scope *Scope) GetModelStruct() *ModelStruct { field.IsNormal = true } else if _, ok := field.TagSettingsGet("EMBEDDED"); ok || fieldStruct.Anonymous { // is embedded struct - for _, subField := range scope.New(fieldValue).GetModelStruct().StructFields { + for _, subField := range scope.New(fieldValue).getModelStruct(rootScope, allFields).StructFields { subField = subField.clone() subField.Names = append([]string{fieldStruct.Name}, subField.Names...) if prefix, ok := field.TagSettingsGet("EMBEDDED_PREFIX"); ok { @@ -239,6 +265,7 @@ func (scope *Scope) GetModelStruct() *ModelStruct { } modelStruct.StructFields = append(modelStruct.StructFields, subField) + allFields = append(allFields, subField) } continue } else { @@ -335,7 +362,7 @@ func (scope *Scope) GetModelStruct() *ModelStruct { } joinTableHandler := JoinTableHandler{} - joinTableHandler.Setup(relationship, ToTableName(many2many), reflectType, elemType) + joinTableHandler.Setup(relationship, many2many, reflectType, elemType) relationship.JoinTableHandler = &joinTableHandler field.Relationship = relationship } else { @@ -372,7 +399,7 @@ func (scope *Scope) GetModelStruct() *ModelStruct { } else { // generate foreign keys from defined association foreign keys for _, scopeFieldName := range associationForeignKeys { - if foreignField := getForeignField(scopeFieldName, modelStruct.StructFields); foreignField != nil { + if foreignField := getForeignField(scopeFieldName, allFields); foreignField != nil { foreignKeys = append(foreignKeys, associationType+foreignField.Name) associationForeignKeys = append(associationForeignKeys, foreignField.Name) } @@ -384,13 +411,13 @@ func (scope *Scope) GetModelStruct() *ModelStruct { for _, foreignKey := range foreignKeys { if strings.HasPrefix(foreignKey, associationType) { associationForeignKey := strings.TrimPrefix(foreignKey, associationType) - if foreignField := getForeignField(associationForeignKey, modelStruct.StructFields); foreignField != nil { + if foreignField := getForeignField(associationForeignKey, allFields); foreignField != nil { associationForeignKeys = append(associationForeignKeys, associationForeignKey) } } } if len(associationForeignKeys) == 0 && len(foreignKeys) == 1 { - associationForeignKeys = []string{scope.PrimaryKey()} + associationForeignKeys = []string{rootScope.PrimaryKey()} } } else if len(foreignKeys) != len(associationForeignKeys) { scope.Err(errors.New("invalid foreign keys, should have same length")) @@ -400,9 +427,13 @@ func (scope *Scope) GetModelStruct() *ModelStruct { for idx, foreignKey := range foreignKeys { if foreignField := getForeignField(foreignKey, toFields); foreignField != nil { - if associationField := getForeignField(associationForeignKeys[idx], modelStruct.StructFields); associationField != nil { - // source foreign keys + if associationField := getForeignField(associationForeignKeys[idx], allFields); associationField != nil { + // mark field as foreignkey, use global lock to avoid race + structsLock.Lock() foreignField.IsForeignKey = true + structsLock.Unlock() + + // association foreign keys relationship.AssociationForeignFieldNames = append(relationship.AssociationForeignFieldNames, associationField.Name) relationship.AssociationForeignDBNames = append(relationship.AssociationForeignDBNames, associationField.DBName) @@ -476,7 +507,7 @@ func (scope *Scope) GetModelStruct() *ModelStruct { } else { // generate foreign keys form association foreign keys for _, associationForeignKey := range tagAssociationForeignKeys { - if foreignField := getForeignField(associationForeignKey, modelStruct.StructFields); foreignField != nil { + if foreignField := getForeignField(associationForeignKey, allFields); foreignField != nil { foreignKeys = append(foreignKeys, associationType+foreignField.Name) associationForeignKeys = append(associationForeignKeys, foreignField.Name) } @@ -488,13 +519,13 @@ func (scope *Scope) GetModelStruct() *ModelStruct { for _, foreignKey := range foreignKeys { if strings.HasPrefix(foreignKey, associationType) { associationForeignKey := strings.TrimPrefix(foreignKey, associationType) - if foreignField := getForeignField(associationForeignKey, modelStruct.StructFields); foreignField != nil { + if foreignField := getForeignField(associationForeignKey, allFields); foreignField != nil { associationForeignKeys = append(associationForeignKeys, associationForeignKey) } } } if len(associationForeignKeys) == 0 && len(foreignKeys) == 1 { - associationForeignKeys = []string{scope.PrimaryKey()} + associationForeignKeys = []string{rootScope.PrimaryKey()} } } else if len(foreignKeys) != len(associationForeignKeys) { scope.Err(errors.New("invalid foreign keys, should have same length")) @@ -504,9 +535,13 @@ func (scope *Scope) GetModelStruct() *ModelStruct { for idx, foreignKey := range foreignKeys { if foreignField := getForeignField(foreignKey, toFields); foreignField != nil { - if scopeField := getForeignField(associationForeignKeys[idx], modelStruct.StructFields); scopeField != nil { + if scopeField := getForeignField(associationForeignKeys[idx], allFields); scopeField != nil { + // mark field as foreignkey, use global lock to avoid race + structsLock.Lock() foreignField.IsForeignKey = true - // source foreign keys + structsLock.Unlock() + + // association foreign keys relationship.AssociationForeignFieldNames = append(relationship.AssociationForeignFieldNames, scopeField.Name) relationship.AssociationForeignDBNames = append(relationship.AssociationForeignDBNames, scopeField.DBName) @@ -564,7 +599,10 @@ func (scope *Scope) GetModelStruct() *ModelStruct { for idx, foreignKey := range foreignKeys { if foreignField := getForeignField(foreignKey, modelStruct.StructFields); foreignField != nil { if associationField := getForeignField(associationForeignKeys[idx], toFields); associationField != nil { + // mark field as foreignkey, use global lock to avoid race + structsLock.Lock() foreignField.IsForeignKey = true + structsLock.Unlock() // association foreign keys relationship.AssociationForeignFieldNames = append(relationship.AssociationForeignFieldNames, associationField.Name) @@ -597,6 +635,7 @@ func (scope *Scope) GetModelStruct() *ModelStruct { } modelStruct.StructFields = append(modelStruct.StructFields, field) + allFields = append(allFields, field) } } @@ -607,7 +646,7 @@ func (scope *Scope) GetModelStruct() *ModelStruct { } } - modelStructsMap.Store(reflectType, &modelStruct) + modelStructsMap.Store(hashKey, &modelStruct) return &modelStruct } @@ -620,6 +659,9 @@ func (scope *Scope) GetStructFields() (fields []*StructField) { func parseTagSetting(tags reflect.StructTag) map[string]string { setting := map[string]string{} for _, str := range []string{tags.Get("sql"), tags.Get("gorm")} { + if str == "" { + continue + } tags := strings.Split(str, ";") for _, value := range tags { v := strings.Split(value, ":") diff --git a/vendor/github.com/jinzhu/gorm/scope.go b/vendor/github.com/jinzhu/gorm/scope.go index fa521ca2..56c3d6e5 100644 --- a/vendor/github.com/jinzhu/gorm/scope.go +++ b/vendor/github.com/jinzhu/gorm/scope.go @@ -68,7 +68,7 @@ func (scope *Scope) Dialect() Dialect { // Quote used to quote string to escape them for database func (scope *Scope) Quote(str string) string { - if strings.Index(str, ".") != -1 { + if strings.Contains(str, ".") { newStrs := []string{} for _, str := range strings.Split(str, ".") { newStrs = append(newStrs, scope.Dialect().Quote(str)) @@ -225,7 +225,7 @@ func (scope *Scope) SetColumn(column interface{}, value interface{}) error { updateAttrs[field.DBName] = value return field.Set(value) } - if (field.DBName == dbName) || (field.Name == name && mostMatchedField == nil) { + if !field.IsIgnored && ((field.DBName == dbName) || (field.Name == name && mostMatchedField == nil)) { mostMatchedField = field } } @@ -257,7 +257,7 @@ func (scope *Scope) CallMethod(methodName string) { func (scope *Scope) AddToVars(value interface{}) string { _, skipBindVar := scope.InstanceGet("skip_bindvar") - if expr, ok := value.(*expr); ok { + if expr, ok := value.(*SqlExpr); ok { exp := expr.expr for _, arg := range expr.args { if skipBindVar { @@ -330,7 +330,7 @@ func (scope *Scope) TableName() string { // QuotedTableName return quoted table name func (scope *Scope) QuotedTableName() (name string) { if scope.Search != nil && len(scope.Search.tableName) > 0 { - if strings.Index(scope.Search.tableName, " ") != -1 { + if strings.Contains(scope.Search.tableName, " ") { return scope.Search.tableName } return scope.Quote(scope.Search.tableName) @@ -402,7 +402,7 @@ func (scope *Scope) InstanceGet(name string) (interface{}, bool) { // Begin start a transaction func (scope *Scope) Begin() *Scope { if db, ok := scope.SQLDB().(sqlDb); ok { - if tx, err := db.Begin(); err == nil { + if tx, err := db.Begin(); scope.Err(err) == nil { scope.db.db = interface{}(tx).(SQLCommon) scope.InstanceSet("gorm:started_transaction", true) } @@ -590,7 +590,7 @@ func (scope *Scope) buildCondition(clause map[string]interface{}, include bool) } scopeQuotedTableName := newScope.QuotedTableName() for _, field := range newScope.Fields() { - if !field.IsIgnored && !field.IsBlank { + if !field.IsIgnored && !field.IsBlank && field.Relationship == nil { sqls = append(sqls, fmt.Sprintf("(%v.%v %s %v)", scopeQuotedTableName, scope.Quote(field.DBName), equalSQL, scope.AddToVars(field.Field.Interface()))) } } @@ -785,7 +785,7 @@ func (scope *Scope) orderSQL() string { for _, order := range scope.Search.orders { if str, ok := order.(string); ok { orders = append(orders, scope.quoteIfPossible(str)) - } else if expr, ok := order.(*expr); ok { + } else if expr, ok := order.(*SqlExpr); ok { exp := expr.expr for _, arg := range expr.args { exp = strings.Replace(exp, "?", scope.AddToVars(arg), 1) @@ -797,7 +797,9 @@ func (scope *Scope) orderSQL() string { } func (scope *Scope) limitAndOffsetSQL() string { - return scope.Dialect().LimitAndOffsetSQL(scope.Search.limit, scope.Search.offset) + sql, err := scope.Dialect().LimitAndOffsetSQL(scope.Search.limit, scope.Search.offset) + scope.Err(err) + return sql } func (scope *Scope) groupSQL() string { @@ -855,6 +857,14 @@ func (scope *Scope) inlineCondition(values ...interface{}) *Scope { } func (scope *Scope) callCallbacks(funcs []*func(s *Scope)) *Scope { + defer func() { + if err := recover(); err != nil { + if db, ok := scope.db.db.(sqlTx); ok { + db.Rollback() + } + panic(err) + } + }() for _, f := range funcs { (*f)(scope) if scope.skipLeft { @@ -864,7 +874,7 @@ func (scope *Scope) callCallbacks(funcs []*func(s *Scope)) *Scope { return scope } -func convertInterfaceToMap(values interface{}, withIgnoredField bool) map[string]interface{} { +func convertInterfaceToMap(values interface{}, withIgnoredField bool, db *DB) map[string]interface{} { var attrs = map[string]interface{}{} switch value := values.(type) { @@ -872,7 +882,7 @@ func convertInterfaceToMap(values interface{}, withIgnoredField bool) map[string return value case []interface{}: for _, v := range value { - for key, value := range convertInterfaceToMap(v, withIgnoredField) { + for key, value := range convertInterfaceToMap(v, withIgnoredField, db) { attrs[key] = value } } @@ -885,7 +895,7 @@ func convertInterfaceToMap(values interface{}, withIgnoredField bool) map[string attrs[ToColumnName(key.Interface().(string))] = reflectValue.MapIndex(key).Interface() } default: - for _, field := range (&Scope{Value: values}).Fields() { + for _, field := range (&Scope{Value: values, db: db}).Fields() { if !field.IsBlank && (withIgnoredField || !field.IsIgnored) { attrs[field.DBName] = field.Field.Interface() } @@ -897,27 +907,31 @@ func convertInterfaceToMap(values interface{}, withIgnoredField bool) map[string func (scope *Scope) updatedAttrsWithValues(value interface{}) (results map[string]interface{}, hasUpdate bool) { if scope.IndirectValue().Kind() != reflect.Struct { - return convertInterfaceToMap(value, false), true + return convertInterfaceToMap(value, false, scope.db), true } results = map[string]interface{}{} - for key, value := range convertInterfaceToMap(value, true) { - if field, ok := scope.FieldByName(key); ok && scope.changeableField(field) { - if _, ok := value.(*expr); ok { - hasUpdate = true - results[field.DBName] = value - } else { - err := field.Set(value) - if field.IsNormal && !field.IsIgnored { + for key, value := range convertInterfaceToMap(value, true, scope.db) { + if field, ok := scope.FieldByName(key); ok { + if scope.changeableField(field) { + if _, ok := value.(*SqlExpr); ok { hasUpdate = true - if err == ErrUnaddressable { - results[field.DBName] = value - } else { - results[field.DBName] = field.Field.Interface() + results[field.DBName] = value + } else { + err := field.Set(value) + if field.IsNormal && !field.IsIgnored { + hasUpdate = true + if err == ErrUnaddressable { + results[field.DBName] = value + } else { + results[field.DBName] = field.Field.Interface() + } } } } + } else { + results[key] = value } } return @@ -976,6 +990,10 @@ func (scope *Scope) pluck(column string, value interface{}) *Scope { return scope } + if dest.Len() > 0 { + dest.Set(reflect.Zero(dest.Type())) + } + if query, ok := scope.Search.selects["query"]; !ok || !scope.isQueryForColumn(query, column) { scope.Search.Select(column) } @@ -999,8 +1017,15 @@ func (scope *Scope) pluck(column string, value interface{}) *Scope { func (scope *Scope) count(value interface{}) *Scope { if query, ok := scope.Search.selects["query"]; !ok || !countingQueryRegexp.MatchString(fmt.Sprint(query)) { if len(scope.Search.group) != 0 { - scope.Search.Select("count(*) FROM ( SELECT count(*) as name ") - scope.Search.group += " ) AS count_table" + if len(scope.Search.havingConditions) != 0 { + scope.prepareQuerySQL() + scope.Search = &search{} + scope.Search.Select("count(*)") + scope.Search.Table(fmt.Sprintf("( %s ) AS count_table", scope.SQL)) + } else { + scope.Search.Select("count(*) FROM ( SELECT count(*) as name ") + scope.Search.group += " ) AS count_table" + } } else { scope.Search.Select("count(*)") } @@ -1175,7 +1200,7 @@ func (scope *Scope) createTable() *Scope { } func (scope *Scope) dropTable() *Scope { - scope.Raw(fmt.Sprintf("DROP TABLE %v%s", scope.QuotedTableName(), scope.getTableOptions())).Exec() + scope.Raw(fmt.Sprintf("DROP TABLE %v", scope.QuotedTableName())).Exec() return scope } @@ -1269,7 +1294,8 @@ func (scope *Scope) autoIndex() *Scope { if name == "INDEX" || name == "" { name = scope.Dialect().BuildKeyName("idx", scope.TableName(), field.DBName) } - indexes[name] = append(indexes[name], field.DBName) + name, column := scope.Dialect().NormalizeIndexAndColumn(name, field.DBName) + indexes[name] = append(indexes[name], column) } } @@ -1280,7 +1306,8 @@ func (scope *Scope) autoIndex() *Scope { if name == "UNIQUE_INDEX" || name == "" { name = scope.Dialect().BuildKeyName("uix", scope.TableName(), field.DBName) } - uniqueIndexes[name] = append(uniqueIndexes[name], field.DBName) + name, column := scope.Dialect().NormalizeIndexAndColumn(name, field.DBName) + uniqueIndexes[name] = append(uniqueIndexes[name], column) } } } @@ -1301,6 +1328,7 @@ func (scope *Scope) autoIndex() *Scope { } func (scope *Scope) getColumnAsArray(columns []string, values ...interface{}) (results [][]interface{}) { + resultMap := make(map[string][]interface{}) for _, value := range values { indirectValue := indirect(reflect.ValueOf(value)) @@ -1319,7 +1347,10 @@ func (scope *Scope) getColumnAsArray(columns []string, values ...interface{}) (r } if hasValue { - results = append(results, result) + h := fmt.Sprint(result...) + if _, exist := resultMap[h]; !exist { + resultMap[h] = result + } } } case reflect.Struct: @@ -1334,11 +1365,16 @@ func (scope *Scope) getColumnAsArray(columns []string, values ...interface{}) (r } if hasValue { - results = append(results, result) + h := fmt.Sprint(result...) + if _, exist := resultMap[h]; !exist { + resultMap[h] = result + } } } } - + for _, v := range resultMap { + results = append(results, v) + } return } diff --git a/vendor/github.com/jinzhu/gorm/search.go b/vendor/github.com/jinzhu/gorm/search.go index 90138595..52ae2efc 100644 --- a/vendor/github.com/jinzhu/gorm/search.go +++ b/vendor/github.com/jinzhu/gorm/search.go @@ -32,7 +32,57 @@ type searchPreload struct { } func (s *search) clone() *search { - clone := *s + clone := search{ + db: s.db, + whereConditions: make([]map[string]interface{}, len(s.whereConditions)), + orConditions: make([]map[string]interface{}, len(s.orConditions)), + notConditions: make([]map[string]interface{}, len(s.notConditions)), + havingConditions: make([]map[string]interface{}, len(s.havingConditions)), + joinConditions: make([]map[string]interface{}, len(s.joinConditions)), + initAttrs: make([]interface{}, len(s.initAttrs)), + assignAttrs: make([]interface{}, len(s.assignAttrs)), + selects: s.selects, + omits: make([]string, len(s.omits)), + orders: make([]interface{}, len(s.orders)), + preload: make([]searchPreload, len(s.preload)), + offset: s.offset, + limit: s.limit, + group: s.group, + tableName: s.tableName, + raw: s.raw, + Unscoped: s.Unscoped, + ignoreOrderQuery: s.ignoreOrderQuery, + } + for i, value := range s.whereConditions { + clone.whereConditions[i] = value + } + for i, value := range s.orConditions { + clone.orConditions[i] = value + } + for i, value := range s.notConditions { + clone.notConditions[i] = value + } + for i, value := range s.havingConditions { + clone.havingConditions[i] = value + } + for i, value := range s.joinConditions { + clone.joinConditions[i] = value + } + for i, value := range s.initAttrs { + clone.initAttrs[i] = value + } + for i, value := range s.assignAttrs { + clone.assignAttrs[i] = value + } + for i, value := range s.omits { + clone.omits[i] = value + } + for i, value := range s.orders { + clone.orders[i] = value + } + for i, value := range s.preload { + clone.preload[i] = value + } return &clone } @@ -98,7 +148,7 @@ func (s *search) Group(query string) *search { } func (s *search) Having(query interface{}, values ...interface{}) *search { - if val, ok := query.(*expr); ok { + if val, ok := query.(*SqlExpr); ok { s.havingConditions = append(s.havingConditions, map[string]interface{}{"query": val.expr, "args": val.args}) } else { s.havingConditions = append(s.havingConditions, map[string]interface{}{"query": query, "args": values}) diff --git a/vendor/github.com/jinzhu/gorm/utils.go b/vendor/github.com/jinzhu/gorm/utils.go index e58e57a5..d2ae9465 100644 --- a/vendor/github.com/jinzhu/gorm/utils.go +++ b/vendor/github.com/jinzhu/gorm/utils.go @@ -58,15 +58,15 @@ func newSafeMap() *safeMap { } // SQL expression -type expr struct { +type SqlExpr struct { expr string args []interface{} } // Expr generate raw SQL expression, for example: // DB.Model(&product).Update("price", gorm.Expr("price * ? + ?", 2, 100)) -func Expr(expression string, args ...interface{}) *expr { - return &expr{expr: expression, args: args} +func Expr(expression string, args ...interface{}) *SqlExpr { + return &SqlExpr{expr: expression, args: args} } func indirect(reflectValue reflect.Value) reflect.Value { diff --git a/vendor/github.com/jinzhu/gorm/wercker.yml b/vendor/github.com/jinzhu/gorm/wercker.yml index 0c3e73ef..1de947b8 100644 --- a/vendor/github.com/jinzhu/gorm/wercker.yml +++ b/vendor/github.com/jinzhu/gorm/wercker.yml @@ -9,22 +9,22 @@ services: MYSQL_USER: gorm MYSQL_PASSWORD: gorm MYSQL_RANDOM_ROOT_PASSWORD: "yes" - - name: mysql57 - id: mysql:5.7 + - name: mysql + id: mysql:latest env: MYSQL_DATABASE: gorm MYSQL_USER: gorm MYSQL_PASSWORD: gorm MYSQL_RANDOM_ROOT_PASSWORD: "yes" - - name: mysql56 - id: mysql:5.6 + - name: mysql57 + id: mysql:5.7 env: MYSQL_DATABASE: gorm MYSQL_USER: gorm MYSQL_PASSWORD: gorm MYSQL_RANDOM_ROOT_PASSWORD: "yes" - - name: mysql55 - id: mysql:5.5 + - name: mysql56 + id: mysql:5.6 env: MYSQL_DATABASE: gorm MYSQL_USER: gorm @@ -83,7 +83,7 @@ build: code: | cd $WERCKER_SOURCE_DIR go version - go get -t ./... + go get -t -v ./... # Build the project - script: @@ -95,54 +95,55 @@ build: - script: name: test sqlite code: | - go test ./... + go test -race -v ./... - script: name: test mariadb code: | - GORM_DIALECT=mysql GORM_DSN="gorm:gorm@tcp(mariadb:3306)/gorm?charset=utf8&parseTime=True" go test ./... + GORM_DIALECT=mysql GORM_DSN="gorm:gorm@tcp(mariadb:3306)/gorm?charset=utf8&parseTime=True" go test -race ./... - script: - name: test mysql5.7 + name: test mysql code: | - GORM_DIALECT=mysql GORM_DSN="gorm:gorm@tcp(mysql57:3306)/gorm?charset=utf8&parseTime=True" go test ./... + GORM_DIALECT=mysql GORM_DSN="gorm:gorm@tcp(mysql:3306)/gorm?charset=utf8&parseTime=True" go test -race ./... - script: - name: test mysql5.6 + name: test mysql5.7 code: | - GORM_DIALECT=mysql GORM_DSN="gorm:gorm@tcp(mysql56:3306)/gorm?charset=utf8&parseTime=True" go test ./... + GORM_DIALECT=mysql GORM_DSN="gorm:gorm@tcp(mysql57:3306)/gorm?charset=utf8&parseTime=True" go test -race ./... - script: - name: test mysql5.5 + name: test mysql5.6 code: | - GORM_DIALECT=mysql GORM_DSN="gorm:gorm@tcp(mysql55:3306)/gorm?charset=utf8&parseTime=True" go test ./... + GORM_DIALECT=mysql GORM_DSN="gorm:gorm@tcp(mysql56:3306)/gorm?charset=utf8&parseTime=True" go test -race ./... - script: name: test postgres code: | - GORM_DIALECT=postgres GORM_DSN="host=postgres user=gorm password=gorm DB.name=gorm port=5432 sslmode=disable" go test ./... + GORM_DIALECT=postgres GORM_DSN="host=postgres user=gorm password=gorm DB.name=gorm port=5432 sslmode=disable" go test -race ./... - script: name: test postgres96 code: | - GORM_DIALECT=postgres GORM_DSN="host=postgres96 user=gorm password=gorm DB.name=gorm port=5432 sslmode=disable" go test ./... + GORM_DIALECT=postgres GORM_DSN="host=postgres96 user=gorm password=gorm DB.name=gorm port=5432 sslmode=disable" go test -race ./... - script: name: test postgres95 code: | - GORM_DIALECT=postgres GORM_DSN="host=postgres95 user=gorm password=gorm DB.name=gorm port=5432 sslmode=disable" go test ./... + GORM_DIALECT=postgres GORM_DSN="host=postgres95 user=gorm password=gorm DB.name=gorm port=5432 sslmode=disable" go test -race ./... - script: name: test postgres94 code: | - GORM_DIALECT=postgres GORM_DSN="host=postgres94 user=gorm password=gorm DB.name=gorm port=5432 sslmode=disable" go test ./... + GORM_DIALECT=postgres GORM_DSN="host=postgres94 user=gorm password=gorm DB.name=gorm port=5432 sslmode=disable" go test -race ./... - script: name: test postgres93 code: | - GORM_DIALECT=postgres GORM_DSN="host=postgres93 user=gorm password=gorm DB.name=gorm port=5432 sslmode=disable" go test ./... + GORM_DIALECT=postgres GORM_DSN="host=postgres93 user=gorm password=gorm DB.name=gorm port=5432 sslmode=disable" go test -race ./... - script: - name: test mssql + name: codecov code: | - GORM_DIALECT=mssql GORM_DSN="sqlserver://gorm:LoremIpsum86@mssql:1433?database=gorm" go test ./... + go test -race -coverprofile=coverage.txt -covermode=atomic ./... + bash <(curl -s https://codecov.io/bash) diff --git a/vendor/github.com/jinzhu/inflection/go.mod b/vendor/github.com/jinzhu/inflection/go.mod new file mode 100644 index 00000000..2fb76904 --- /dev/null +++ b/vendor/github.com/jinzhu/inflection/go.mod @@ -0,0 +1 @@ +module github.com/jinzhu/inflection diff --git a/vendor/github.com/lib/pq/.travis.sh b/vendor/github.com/lib/pq/.travis.sh index a297dc45..21a52644 100644 --- a/vendor/github.com/lib/pq/.travis.sh +++ b/vendor/github.com/lib/pq/.travis.sh @@ -80,7 +80,7 @@ megacheck_install() { } golint_install() { - go get github.com/golang/lint/golint + go get golang.org/x/lint/golint } $1 diff --git a/vendor/github.com/lib/pq/.travis.yml b/vendor/github.com/lib/pq/.travis.yml index 18556e08..f0305809 100644 --- a/vendor/github.com/lib/pq/.travis.yml +++ b/vendor/github.com/lib/pq/.travis.yml @@ -1,9 +1,9 @@ language: go go: - - 1.8.x - 1.9.x - 1.10.x + - 1.11.x - master sudo: true @@ -44,7 +44,7 @@ script: - > goimports -d -e $(find -name '*.go') | awk '{ print } END { exit NR == 0 ? 0 : 1 }' - go vet ./... - - megacheck -go 1.8 ./... + - megacheck -go 1.9 ./... - golint ./... - PQTEST_BINARY_PARAMETERS=no go test -race -v ./... - PQTEST_BINARY_PARAMETERS=yes go test -race -v ./... diff --git a/vendor/github.com/lib/pq/README.md b/vendor/github.com/lib/pq/README.md index d71f3c2c..385fe735 100644 --- a/vendor/github.com/lib/pq/README.md +++ b/vendor/github.com/lib/pq/README.md @@ -10,7 +10,7 @@ ## Docs For detailed documentation and basic usage examples, please see the package -documentation at . +documentation at . ## Tests diff --git a/vendor/github.com/lib/pq/conn.go b/vendor/github.com/lib/pq/conn.go index 43c8df29..62551a14 100644 --- a/vendor/github.com/lib/pq/conn.go +++ b/vendor/github.com/lib/pq/conn.go @@ -2,7 +2,9 @@ package pq import ( "bufio" + "context" "crypto/md5" + "crypto/sha256" "database/sql" "database/sql/driver" "encoding/binary" @@ -20,6 +22,7 @@ import ( "unicode" "github.com/lib/pq/oid" + "github.com/lib/pq/scram" ) // Common error types @@ -89,13 +92,24 @@ type Dialer interface { DialTimeout(network, address string, timeout time.Duration) (net.Conn, error) } -type defaultDialer struct{} +type DialerContext interface { + DialContext(ctx context.Context, network, address string) (net.Conn, error) +} + +type defaultDialer struct { + d net.Dialer +} -func (d defaultDialer) Dial(ntw, addr string) (net.Conn, error) { - return net.Dial(ntw, addr) +func (d defaultDialer) Dial(network, address string) (net.Conn, error) { + return d.d.Dial(network, address) } -func (d defaultDialer) DialTimeout(ntw, addr string, timeout time.Duration) (net.Conn, error) { - return net.DialTimeout(ntw, addr, timeout) +func (d defaultDialer) DialTimeout(network, address string, timeout time.Duration) (net.Conn, error) { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + return d.DialContext(ctx, network, address) +} +func (d defaultDialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) { + return d.d.DialContext(ctx, network, address) } type conn struct { @@ -244,90 +258,35 @@ func (cn *conn) writeBuf(b byte) *writeBuf { } } -// Open opens a new connection to the database. name is a connection string. +// Open opens a new connection to the database. dsn is a connection string. // Most users should only use it through database/sql package from the standard // library. -func Open(name string) (_ driver.Conn, err error) { - return DialOpen(defaultDialer{}, name) +func Open(dsn string) (_ driver.Conn, err error) { + return DialOpen(defaultDialer{}, dsn) } // DialOpen opens a new connection to the database using a dialer. -func DialOpen(d Dialer, name string) (_ driver.Conn, err error) { +func DialOpen(d Dialer, dsn string) (_ driver.Conn, err error) { + c, err := NewConnector(dsn) + if err != nil { + return nil, err + } + c.dialer = d + return c.open(context.Background()) +} + +func (c *Connector) open(ctx context.Context) (cn *conn, err error) { // Handle any panics during connection initialization. Note that we // specifically do *not* want to use errRecover(), as that would turn any // connection errors into ErrBadConns, hiding the real error message from // the user. defer errRecoverNoErrBadConn(&err) - o := make(values) + o := c.opts - // A number of defaults are applied here, in this order: - // - // * Very low precedence defaults applied in every situation - // * Environment variables - // * Explicitly passed connection information - o["host"] = "localhost" - o["port"] = "5432" - // N.B.: Extra float digits should be set to 3, but that breaks - // Postgres 8.4 and older, where the max is 2. - o["extra_float_digits"] = "2" - for k, v := range parseEnviron(os.Environ()) { - o[k] = v - } - - if strings.HasPrefix(name, "postgres://") || strings.HasPrefix(name, "postgresql://") { - name, err = ParseURL(name) - if err != nil { - return nil, err - } - } - - if err := parseOpts(name, o); err != nil { - return nil, err - } - - // Use the "fallback" application name if necessary - if fallback, ok := o["fallback_application_name"]; ok { - if _, ok := o["application_name"]; !ok { - o["application_name"] = fallback - } - } - - // We can't work with any client_encoding other than UTF-8 currently. - // However, we have historically allowed the user to set it to UTF-8 - // explicitly, and there's no reason to break such programs, so allow that. - // Note that the "options" setting could also set client_encoding, but - // parsing its value is not worth it. Instead, we always explicitly send - // client_encoding as a separate run-time parameter, which should override - // anything set in options. - if enc, ok := o["client_encoding"]; ok && !isUTF8(enc) { - return nil, errors.New("client_encoding must be absent or 'UTF8'") - } - o["client_encoding"] = "UTF8" - // DateStyle needs a similar treatment. - if datestyle, ok := o["datestyle"]; ok { - if datestyle != "ISO, MDY" { - panic(fmt.Sprintf("setting datestyle must be absent or %v; got %v", - "ISO, MDY", datestyle)) - } - } else { - o["datestyle"] = "ISO, MDY" - } - - // If a user is not provided by any other means, the last - // resort is to use the current operating system provided user - // name. - if _, ok := o["user"]; !ok { - u, err := userCurrent() - if err != nil { - return nil, err - } - o["user"] = u - } - - cn := &conn{ + cn = &conn{ opts: o, - dialer: d, + dialer: c.dialer, } err = cn.handleDriverSettings(o) if err != nil { @@ -335,7 +294,7 @@ func DialOpen(d Dialer, name string) (_ driver.Conn, err error) { } cn.handlePgpass(o) - cn.c, err = dial(d, o) + cn.c, err = dial(ctx, c.dialer, o) if err != nil { return nil, err } @@ -364,10 +323,10 @@ func DialOpen(d Dialer, name string) (_ driver.Conn, err error) { return cn, err } -func dial(d Dialer, o values) (net.Conn, error) { - ntw, addr := network(o) +func dial(ctx context.Context, d Dialer, o values) (net.Conn, error) { + network, address := network(o) // SSL is not necessary or supported over UNIX domain sockets - if ntw == "unix" { + if network == "unix" { o["sslmode"] = "disable" } @@ -378,19 +337,30 @@ func dial(d Dialer, o values) (net.Conn, error) { return nil, fmt.Errorf("invalid value for parameter connect_timeout: %s", err) } duration := time.Duration(seconds) * time.Second + // connect_timeout should apply to the entire connection establishment // procedure, so we both use a timeout for the TCP connection // establishment and set a deadline for doing the initial handshake. // The deadline is then reset after startup() is done. deadline := time.Now().Add(duration) - conn, err := d.DialTimeout(ntw, addr, duration) + var conn net.Conn + if dctx, ok := d.(DialerContext); ok { + ctx, cancel := context.WithTimeout(ctx, duration) + defer cancel() + conn, err = dctx.DialContext(ctx, network, address) + } else { + conn, err = d.DialTimeout(network, address, duration) + } if err != nil { return nil, err } err = conn.SetDeadline(deadline) return conn, err } - return d.Dial(ntw, addr) + if dctx, ok := d.(DialerContext); ok { + return dctx.DialContext(ctx, network, address) + } + return d.Dial(network, address) } func network(o values) (string, string) { @@ -704,7 +674,7 @@ func (cn *conn) simpleQuery(q string) (res *rows, err error) { // res might be non-nil here if we received a previous // CommandComplete, but that's fine; just overwrite it res = &rows{cn: cn} - res.colNames, res.colFmts, res.colTyps = parsePortalRowDescribe(r) + res.rowsHeader = parsePortalRowDescribe(r) // To work around a bug in QueryRow in Go 1.2 and earlier, wait // until the first DataRow has been received. @@ -861,17 +831,15 @@ func (cn *conn) query(query string, args []driver.Value) (_ *rows, err error) { cn.readParseResponse() cn.readBindResponse() rows := &rows{cn: cn} - rows.colNames, rows.colFmts, rows.colTyps = cn.readPortalDescribeResponse() + rows.rowsHeader = cn.readPortalDescribeResponse() cn.postExecuteWorkaround() return rows, nil } st := cn.prepareTo(query, "") st.exec(args) return &rows{ - cn: cn, - colNames: st.colNames, - colTyps: st.colTyps, - colFmts: st.colFmts, + cn: cn, + rowsHeader: st.rowsHeader, }, nil } @@ -992,7 +960,6 @@ func (cn *conn) recv() (t byte, r *readBuf) { if err != nil { panic(err) } - switch t { case 'E': panic(parseError(r)) @@ -1163,6 +1130,55 @@ func (cn *conn) auth(r *readBuf, o values) { if r.int32() != 0 { errorf("unexpected authentication response: %q", t) } + case 10: + sc := scram.NewClient(sha256.New, o["user"], o["password"]) + sc.Step(nil) + if sc.Err() != nil { + errorf("SCRAM-SHA-256 error: %s", sc.Err().Error()) + } + scOut := sc.Out() + + w := cn.writeBuf('p') + w.string("SCRAM-SHA-256") + w.int32(len(scOut)) + w.bytes(scOut) + cn.send(w) + + t, r := cn.recv() + if t != 'R' { + errorf("unexpected password response: %q", t) + } + + if r.int32() != 11 { + errorf("unexpected authentication response: %q", t) + } + + nextStep := r.next(len(*r)) + sc.Step(nextStep) + if sc.Err() != nil { + errorf("SCRAM-SHA-256 error: %s", sc.Err().Error()) + } + + scOut = sc.Out() + w = cn.writeBuf('p') + w.bytes(scOut) + cn.send(w) + + t, r = cn.recv() + if t != 'R' { + errorf("unexpected password response: %q", t) + } + + if r.int32() != 12 { + errorf("unexpected authentication response: %q", t) + } + + nextStep = r.next(len(*r)) + sc.Step(nextStep) + if sc.Err() != nil { + errorf("SCRAM-SHA-256 error: %s", sc.Err().Error()) + } + default: errorf("unknown authentication response: %d", code) } @@ -1180,12 +1196,10 @@ var colFmtDataAllBinary = []byte{0, 1, 0, 1} var colFmtDataAllText = []byte{0, 0} type stmt struct { - cn *conn - name string - colNames []string - colFmts []format + cn *conn + name string + rowsHeader colFmtData []byte - colTyps []fieldDesc paramTyps []oid.Oid closed bool } @@ -1231,10 +1245,8 @@ func (st *stmt) Query(v []driver.Value) (r driver.Rows, err error) { st.exec(v) return &rows{ - cn: st.cn, - colNames: st.colNames, - colTyps: st.colTyps, - colFmts: st.colFmts, + cn: st.cn, + rowsHeader: st.rowsHeader, }, nil } @@ -1344,16 +1356,22 @@ func (cn *conn) parseComplete(commandTag string) (driver.Result, string) { return driver.RowsAffected(n), commandTag } -type rows struct { - cn *conn - finish func() +type rowsHeader struct { colNames []string colTyps []fieldDesc colFmts []format - done bool - rb readBuf - result driver.Result - tag string +} + +type rows struct { + cn *conn + finish func() + rowsHeader + done bool + rb readBuf + result driver.Result + tag string + + next *rowsHeader } func (rs *rows) Close() error { @@ -1440,7 +1458,8 @@ func (rs *rows) Next(dest []driver.Value) (err error) { } return case 'T': - rs.colNames, rs.colFmts, rs.colTyps = parsePortalRowDescribe(&rs.rb) + next := parsePortalRowDescribe(&rs.rb) + rs.next = &next return io.EOF default: errorf("unexpected message after execute: %q", t) @@ -1449,10 +1468,16 @@ func (rs *rows) Next(dest []driver.Value) (err error) { } func (rs *rows) HasNextResultSet() bool { - return !rs.done + hasNext := rs.next != nil && !rs.done + return hasNext } func (rs *rows) NextResultSet() error { + if rs.next == nil { + return io.EOF + } + rs.rowsHeader = *rs.next + rs.next = nil return nil } @@ -1630,13 +1655,13 @@ func (cn *conn) readStatementDescribeResponse() (paramTyps []oid.Oid, colNames [ } } -func (cn *conn) readPortalDescribeResponse() (colNames []string, colFmts []format, colTyps []fieldDesc) { +func (cn *conn) readPortalDescribeResponse() rowsHeader { t, r := cn.recv1() switch t { case 'T': return parsePortalRowDescribe(r) case 'n': - return nil, nil, nil + return rowsHeader{} case 'E': err := parseError(r) cn.readReadyForQuery() @@ -1742,11 +1767,11 @@ func parseStatementRowDescribe(r *readBuf) (colNames []string, colTyps []fieldDe return } -func parsePortalRowDescribe(r *readBuf) (colNames []string, colFmts []format, colTyps []fieldDesc) { +func parsePortalRowDescribe(r *readBuf) rowsHeader { n := r.int16() - colNames = make([]string, n) - colFmts = make([]format, n) - colTyps = make([]fieldDesc, n) + colNames := make([]string, n) + colFmts := make([]format, n) + colTyps := make([]fieldDesc, n) for i := range colNames { colNames[i] = r.string() r.next(6) @@ -1755,7 +1780,11 @@ func parsePortalRowDescribe(r *readBuf) (colNames []string, colFmts []format, co colTyps[i].Mod = r.int32() colFmts[i] = format(r.int16()) } - return + return rowsHeader{ + colNames: colNames, + colFmts: colFmts, + colTyps: colTyps, + } } // parseEnviron tries to mimic some of libpq's environment handling diff --git a/vendor/github.com/lib/pq/conn_go18.go b/vendor/github.com/lib/pq/conn_go18.go index a5254f2b..0fdd06a6 100644 --- a/vendor/github.com/lib/pq/conn_go18.go +++ b/vendor/github.com/lib/pq/conn_go18.go @@ -1,5 +1,3 @@ -// +build go1.8 - package pq import ( @@ -9,6 +7,7 @@ import ( "fmt" "io" "io/ioutil" + "time" ) // Implement the "QueryerContext" interface @@ -76,13 +75,32 @@ func (cn *conn) BeginTx(ctx context.Context, opts driver.TxOptions) (driver.Tx, return tx, nil } +func (cn *conn) Ping(ctx context.Context) error { + if finish := cn.watchCancel(ctx); finish != nil { + defer finish() + } + rows, err := cn.simpleQuery("SELECT 'lib/pq ping test';") + if err != nil { + return driver.ErrBadConn // https://golang.org/pkg/database/sql/driver/#Pinger + } + rows.Close() + return nil +} + func (cn *conn) watchCancel(ctx context.Context) func() { if done := ctx.Done(); done != nil { finished := make(chan struct{}) go func() { select { case <-done: - _ = cn.cancel() + // At this point the function level context is canceled, + // so it must not be used for the additional network + // request to cancel the query. + // Create a new context to pass into the dial. + ctxCancel, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + + _ = cn.cancel(ctxCancel) finished <- struct{}{} case <-finished: } @@ -97,8 +115,8 @@ func (cn *conn) watchCancel(ctx context.Context) func() { return nil } -func (cn *conn) cancel() error { - c, err := dial(cn.dialer, cn.opts) +func (cn *conn) cancel(ctx context.Context) error { + c, err := dial(ctx, cn.dialer, cn.opts) if err != nil { return err } diff --git a/vendor/github.com/lib/pq/connector.go b/vendor/github.com/lib/pq/connector.go index 9e66eb5d..2f8ced67 100644 --- a/vendor/github.com/lib/pq/connector.go +++ b/vendor/github.com/lib/pq/connector.go @@ -1,10 +1,12 @@ -// +build go1.10 - package pq import ( "context" "database/sql/driver" + "errors" + "fmt" + "os" + "strings" ) // Connector represents a fixed configuration for the pq driver with a given @@ -14,30 +16,95 @@ import ( // // See https://golang.org/pkg/database/sql/driver/#Connector. // See https://golang.org/pkg/database/sql/#OpenDB. -type connector struct { - name string +type Connector struct { + opts values + dialer Dialer } // Connect returns a connection to the database using the fixed configuration // of this Connector. Context is not used. -func (c *connector) Connect(_ context.Context) (driver.Conn, error) { - return (&Driver{}).Open(c.name) +func (c *Connector) Connect(ctx context.Context) (driver.Conn, error) { + return c.open(ctx) } // Driver returnst the underlying driver of this Connector. -func (c *connector) Driver() driver.Driver { +func (c *Connector) Driver() driver.Driver { return &Driver{} } -var _ driver.Connector = &connector{} - // NewConnector returns a connector for the pq driver in a fixed configuration -// with the given name. The returned connector can be used to create any number +// with the given dsn. The returned connector can be used to create any number // of equivalent Conn's. The returned connector is intended to be used with // database/sql.OpenDB. // // See https://golang.org/pkg/database/sql/driver/#Connector. // See https://golang.org/pkg/database/sql/#OpenDB. -func NewConnector(name string) (driver.Connector, error) { - return &connector{name: name}, nil +func NewConnector(dsn string) (*Connector, error) { + var err error + o := make(values) + + // A number of defaults are applied here, in this order: + // + // * Very low precedence defaults applied in every situation + // * Environment variables + // * Explicitly passed connection information + o["host"] = "localhost" + o["port"] = "5432" + // N.B.: Extra float digits should be set to 3, but that breaks + // Postgres 8.4 and older, where the max is 2. + o["extra_float_digits"] = "2" + for k, v := range parseEnviron(os.Environ()) { + o[k] = v + } + + if strings.HasPrefix(dsn, "postgres://") || strings.HasPrefix(dsn, "postgresql://") { + dsn, err = ParseURL(dsn) + if err != nil { + return nil, err + } + } + + if err := parseOpts(dsn, o); err != nil { + return nil, err + } + + // Use the "fallback" application name if necessary + if fallback, ok := o["fallback_application_name"]; ok { + if _, ok := o["application_name"]; !ok { + o["application_name"] = fallback + } + } + + // We can't work with any client_encoding other than UTF-8 currently. + // However, we have historically allowed the user to set it to UTF-8 + // explicitly, and there's no reason to break such programs, so allow that. + // Note that the "options" setting could also set client_encoding, but + // parsing its value is not worth it. Instead, we always explicitly send + // client_encoding as a separate run-time parameter, which should override + // anything set in options. + if enc, ok := o["client_encoding"]; ok && !isUTF8(enc) { + return nil, errors.New("client_encoding must be absent or 'UTF8'") + } + o["client_encoding"] = "UTF8" + // DateStyle needs a similar treatment. + if datestyle, ok := o["datestyle"]; ok { + if datestyle != "ISO, MDY" { + return nil, fmt.Errorf("setting datestyle must be absent or %v; got %v", "ISO, MDY", datestyle) + } + } else { + o["datestyle"] = "ISO, MDY" + } + + // If a user is not provided by any other means, the last + // resort is to use the current operating system provided user + // name. + if _, ok := o["user"]; !ok { + u, err := userCurrent() + if err != nil { + return nil, err + } + o["user"] = u + } + + return &Connector{opts: o, dialer: defaultDialer{}}, nil } diff --git a/vendor/github.com/lib/pq/doc.go b/vendor/github.com/lib/pq/doc.go index a1b02971..2a60054e 100644 --- a/vendor/github.com/lib/pq/doc.go +++ b/vendor/github.com/lib/pq/doc.go @@ -239,7 +239,7 @@ for more information). Note that the channel name will be truncated to 63 bytes by the PostgreSQL server. You can find a complete, working example of Listener usage at -http://godoc.org/github.com/lib/pq/example/listen. +https://godoc.org/github.com/lib/pq/example/listen. */ package pq diff --git a/vendor/github.com/lib/pq/encode.go b/vendor/github.com/lib/pq/encode.go index 3b0d365f..a6902fae 100644 --- a/vendor/github.com/lib/pq/encode.go +++ b/vendor/github.com/lib/pq/encode.go @@ -117,11 +117,10 @@ func textDecode(parameterStatus *parameterStatus, s []byte, typ oid.Oid) interfa } return i case oid.T_float4, oid.T_float8: - bits := 64 - if typ == oid.T_float4 { - bits = 32 - } - f, err := strconv.ParseFloat(string(s), bits) + // We always use 64 bit parsing, regardless of whether the input text is for + // a float4 or float8, because clients expect float64s for all float datatypes + // and returning a 32-bit parsed float64 produces lossy results. + f, err := strconv.ParseFloat(string(s), 64) if err != nil { errorf("%s", err) } diff --git a/vendor/github.com/lib/pq/scram/scram.go b/vendor/github.com/lib/pq/scram/scram.go new file mode 100644 index 00000000..5d0358f8 --- /dev/null +++ b/vendor/github.com/lib/pq/scram/scram.go @@ -0,0 +1,264 @@ +// Copyright (c) 2014 - Gustavo Niemeyer +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this +// list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// Pacakage scram implements a SCRAM-{SHA-1,etc} client per RFC5802. +// +// http://tools.ietf.org/html/rfc5802 +// +package scram + +import ( + "bytes" + "crypto/hmac" + "crypto/rand" + "encoding/base64" + "fmt" + "hash" + "strconv" + "strings" +) + +// Client implements a SCRAM-* client (SCRAM-SHA-1, SCRAM-SHA-256, etc). +// +// A Client may be used within a SASL conversation with logic resembling: +// +// var in []byte +// var client = scram.NewClient(sha1.New, user, pass) +// for client.Step(in) { +// out := client.Out() +// // send out to server +// in := serverOut +// } +// if client.Err() != nil { +// // auth failed +// } +// +type Client struct { + newHash func() hash.Hash + + user string + pass string + step int + out bytes.Buffer + err error + + clientNonce []byte + serverNonce []byte + saltedPass []byte + authMsg bytes.Buffer +} + +// NewClient returns a new SCRAM-* client with the provided hash algorithm. +// +// For SCRAM-SHA-256, for example, use: +// +// client := scram.NewClient(sha256.New, user, pass) +// +func NewClient(newHash func() hash.Hash, user, pass string) *Client { + c := &Client{ + newHash: newHash, + user: user, + pass: pass, + } + c.out.Grow(256) + c.authMsg.Grow(256) + return c +} + +// Out returns the data to be sent to the server in the current step. +func (c *Client) Out() []byte { + if c.out.Len() == 0 { + return nil + } + return c.out.Bytes() +} + +// Err returns the error that ocurred, or nil if there were no errors. +func (c *Client) Err() error { + return c.err +} + +// SetNonce sets the client nonce to the provided value. +// If not set, the nonce is generated automatically out of crypto/rand on the first step. +func (c *Client) SetNonce(nonce []byte) { + c.clientNonce = nonce +} + +var escaper = strings.NewReplacer("=", "=3D", ",", "=2C") + +// Step processes the incoming data from the server and makes the +// next round of data for the server available via Client.Out. +// Step returns false if there are no errors and more data is +// still expected. +func (c *Client) Step(in []byte) bool { + c.out.Reset() + if c.step > 2 || c.err != nil { + return false + } + c.step++ + switch c.step { + case 1: + c.err = c.step1(in) + case 2: + c.err = c.step2(in) + case 3: + c.err = c.step3(in) + } + return c.step > 2 || c.err != nil +} + +func (c *Client) step1(in []byte) error { + if len(c.clientNonce) == 0 { + const nonceLen = 16 + buf := make([]byte, nonceLen+b64.EncodedLen(nonceLen)) + if _, err := rand.Read(buf[:nonceLen]); err != nil { + return fmt.Errorf("cannot read random SCRAM-SHA-256 nonce from operating system: %v", err) + } + c.clientNonce = buf[nonceLen:] + b64.Encode(c.clientNonce, buf[:nonceLen]) + } + c.authMsg.WriteString("n=") + escaper.WriteString(&c.authMsg, c.user) + c.authMsg.WriteString(",r=") + c.authMsg.Write(c.clientNonce) + + c.out.WriteString("n,,") + c.out.Write(c.authMsg.Bytes()) + return nil +} + +var b64 = base64.StdEncoding + +func (c *Client) step2(in []byte) error { + c.authMsg.WriteByte(',') + c.authMsg.Write(in) + + fields := bytes.Split(in, []byte(",")) + if len(fields) != 3 { + return fmt.Errorf("expected 3 fields in first SCRAM-SHA-256 server message, got %d: %q", len(fields), in) + } + if !bytes.HasPrefix(fields[0], []byte("r=")) || len(fields[0]) < 2 { + return fmt.Errorf("server sent an invalid SCRAM-SHA-256 nonce: %q", fields[0]) + } + if !bytes.HasPrefix(fields[1], []byte("s=")) || len(fields[1]) < 6 { + return fmt.Errorf("server sent an invalid SCRAM-SHA-256 salt: %q", fields[1]) + } + if !bytes.HasPrefix(fields[2], []byte("i=")) || len(fields[2]) < 6 { + return fmt.Errorf("server sent an invalid SCRAM-SHA-256 iteration count: %q", fields[2]) + } + + c.serverNonce = fields[0][2:] + if !bytes.HasPrefix(c.serverNonce, c.clientNonce) { + return fmt.Errorf("server SCRAM-SHA-256 nonce is not prefixed by client nonce: got %q, want %q+\"...\"", c.serverNonce, c.clientNonce) + } + + salt := make([]byte, b64.DecodedLen(len(fields[1][2:]))) + n, err := b64.Decode(salt, fields[1][2:]) + if err != nil { + return fmt.Errorf("cannot decode SCRAM-SHA-256 salt sent by server: %q", fields[1]) + } + salt = salt[:n] + iterCount, err := strconv.Atoi(string(fields[2][2:])) + if err != nil { + return fmt.Errorf("server sent an invalid SCRAM-SHA-256 iteration count: %q", fields[2]) + } + c.saltPassword(salt, iterCount) + + c.authMsg.WriteString(",c=biws,r=") + c.authMsg.Write(c.serverNonce) + + c.out.WriteString("c=biws,r=") + c.out.Write(c.serverNonce) + c.out.WriteString(",p=") + c.out.Write(c.clientProof()) + return nil +} + +func (c *Client) step3(in []byte) error { + var isv, ise bool + var fields = bytes.Split(in, []byte(",")) + if len(fields) == 1 { + isv = bytes.HasPrefix(fields[0], []byte("v=")) + ise = bytes.HasPrefix(fields[0], []byte("e=")) + } + if ise { + return fmt.Errorf("SCRAM-SHA-256 authentication error: %s", fields[0][2:]) + } else if !isv { + return fmt.Errorf("unsupported SCRAM-SHA-256 final message from server: %q", in) + } + if !bytes.Equal(c.serverSignature(), fields[0][2:]) { + return fmt.Errorf("cannot authenticate SCRAM-SHA-256 server signature: %q", fields[0][2:]) + } + return nil +} + +func (c *Client) saltPassword(salt []byte, iterCount int) { + mac := hmac.New(c.newHash, []byte(c.pass)) + mac.Write(salt) + mac.Write([]byte{0, 0, 0, 1}) + ui := mac.Sum(nil) + hi := make([]byte, len(ui)) + copy(hi, ui) + for i := 1; i < iterCount; i++ { + mac.Reset() + mac.Write(ui) + mac.Sum(ui[:0]) + for j, b := range ui { + hi[j] ^= b + } + } + c.saltedPass = hi +} + +func (c *Client) clientProof() []byte { + mac := hmac.New(c.newHash, c.saltedPass) + mac.Write([]byte("Client Key")) + clientKey := mac.Sum(nil) + hash := c.newHash() + hash.Write(clientKey) + storedKey := hash.Sum(nil) + mac = hmac.New(c.newHash, storedKey) + mac.Write(c.authMsg.Bytes()) + clientProof := mac.Sum(nil) + for i, b := range clientKey { + clientProof[i] ^= b + } + clientProof64 := make([]byte, b64.EncodedLen(len(clientProof))) + b64.Encode(clientProof64, clientProof) + return clientProof64 +} + +func (c *Client) serverSignature() []byte { + mac := hmac.New(c.newHash, c.saltedPass) + mac.Write([]byte("Server Key")) + serverKey := mac.Sum(nil) + + mac = hmac.New(c.newHash, serverKey) + mac.Write(c.authMsg.Bytes()) + serverSignature := mac.Sum(nil) + + encoded := make([]byte, b64.EncodedLen(len(serverSignature))) + b64.Encode(encoded, serverSignature) + return encoded +} diff --git a/vendor/github.com/lib/pq/ssl.go b/vendor/github.com/lib/pq/ssl.go index e1a326a0..d9020845 100644 --- a/vendor/github.com/lib/pq/ssl.go +++ b/vendor/github.com/lib/pq/ssl.go @@ -58,7 +58,13 @@ func ssl(o values) (func(net.Conn) (net.Conn, error), error) { if err != nil { return nil, err } - sslRenegotiation(&tlsConf) + + // Accept renegotiation requests initiated by the backend. + // + // Renegotiation was deprecated then removed from PostgreSQL 9.5, but + // the default configuration of older versions has it enabled. Redshift + // also initiates renegotiations and cannot be reconfigured. + tlsConf.Renegotiation = tls.RenegotiateFreelyAsClient return func(conn net.Conn) (net.Conn, error) { client := tls.Client(conn, &tlsConf) diff --git a/vendor/github.com/lib/pq/ssl_go1.7.go b/vendor/github.com/lib/pq/ssl_go1.7.go deleted file mode 100644 index d7ba43b3..00000000 --- a/vendor/github.com/lib/pq/ssl_go1.7.go +++ /dev/null @@ -1,14 +0,0 @@ -// +build go1.7 - -package pq - -import "crypto/tls" - -// Accept renegotiation requests initiated by the backend. -// -// Renegotiation was deprecated then removed from PostgreSQL 9.5, but -// the default configuration of older versions has it enabled. Redshift -// also initiates renegotiations and cannot be reconfigured. -func sslRenegotiation(conf *tls.Config) { - conf.Renegotiation = tls.RenegotiateFreelyAsClient -} diff --git a/vendor/github.com/lib/pq/ssl_renegotiation.go b/vendor/github.com/lib/pq/ssl_renegotiation.go deleted file mode 100644 index 85ed5e43..00000000 --- a/vendor/github.com/lib/pq/ssl_renegotiation.go +++ /dev/null @@ -1,8 +0,0 @@ -// +build !go1.7 - -package pq - -import "crypto/tls" - -// Renegotiation is not supported by crypto/tls until Go 1.7. -func sslRenegotiation(*tls.Config) {} diff --git a/vendor/github.com/mattn/go-sqlite3/.travis.yml b/vendor/github.com/mattn/go-sqlite3/.travis.yml index a21c8170..afc7c09b 100644 --- a/vendor/github.com/mattn/go-sqlite3/.travis.yml +++ b/vendor/github.com/mattn/go-sqlite3/.travis.yml @@ -8,34 +8,28 @@ addons: apt: update: true -env: - matrix: - - GOTAGS= - - GOTAGS=libsqlite3 - - GOTAGS="sqlite_allow_uri_authority sqlite_app_armor sqlite_foreign_keys sqlite_fts5 sqlite_icu sqlite_introspect sqlite_json sqlite_secure_delete sqlite_see sqlite_stat4 sqlite_trace sqlite_userauth sqlite_vacuum_incr sqlite_vtable" - - GOTAGS=sqlite_vacuum_full - go: - 1.9.x - 1.10.x + - 1.11.x + - 1.12.x + - 1.13.x + - master before_install: - | - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then + if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew update - brew upgrade icu4c - fi - - | - go get github.com/smartystreets/goconvey - if [[ "${GOOS}" != "windows" ]]; then - go get github.com/mattn/goveralls - go get golang.org/x/tools/cmd/cover fi + - go get github.com/mattn/goveralls + - go get golang.org/x/tools/cmd/cover script: - - GOOS=$(go env GOOS) GOARCH=$(go env GOARCH) go build -v -tags "${GOTAGS}" . - - | - if [[ "${GOOS}" != "windows" ]]; then - $HOME/gopath/bin/goveralls -repotoken 3qJVUE0iQwqnCbmNcDsjYu1nh4J4KIFXx - go test -race -v . -tags "${GOTAGS}" - fi + - $HOME/gopath/bin/goveralls -repotoken 3qJVUE0iQwqnCbmNcDsjYu1nh4J4KIFXx -parallel + - go test -race -v . -tags "" + - go test -race -v . -tags "libsqlite3" + - go test -race -v . -tags "sqlite_allow_uri_authority sqlite_app_armor sqlite_foreign_keys sqlite_fts5 sqlite_icu sqlite_introspect sqlite_json sqlite_preupdate_hook sqlite_secure_delete sqlite_see sqlite_stat4 sqlite_trace sqlite_userauth sqlite_vacuum_incr sqlite_vtable sqlite_unlock_notify" + - go test -race -v . -tags "sqlite_vacuum_full" + +notifications: + webhooks: https://coveralls.io/webhook?repo_token=3qJVUE0iQwqnCbmNcDsjYu1nh4J4KIFXx diff --git a/vendor/github.com/mattn/go-sqlite3/README.md b/vendor/github.com/mattn/go-sqlite3/README.md index 37d017a1..8e6e2223 100644 --- a/vendor/github.com/mattn/go-sqlite3/README.md +++ b/vendor/github.com/mattn/go-sqlite3/README.md @@ -3,30 +3,37 @@ go-sqlite3 [![GoDoc Reference](https://godoc.org/github.com/mattn/go-sqlite3?status.svg)](http://godoc.org/github.com/mattn/go-sqlite3) [![Build Status](https://travis-ci.org/mattn/go-sqlite3.svg?branch=master)](https://travis-ci.org/mattn/go-sqlite3) +[![Financial Contributors on Open Collective](https://opencollective.com/mattn-go-sqlite3/all/badge.svg?label=financial+contributors)](https://opencollective.com/mattn-go-sqlite3) [![Coverage Status](https://coveralls.io/repos/mattn/go-sqlite3/badge.svg?branch=master)](https://coveralls.io/r/mattn/go-sqlite3?branch=master) [![Go Report Card](https://goreportcard.com/badge/github.com/mattn/go-sqlite3)](https://goreportcard.com/report/github.com/mattn/go-sqlite3) +**NOTE:** The increase to v2 was an accident. There were no major changes or features. + # Description sqlite3 driver conforming to the built-in database/sql interface -Supported Golang version: -- 1.9.x -- 1.10.x +Supported Golang version: See .travis.yml [This package follows the official Golang Release Policy.](https://golang.org/doc/devel/release.html#policy) ### Overview +- [go-sqlite3](#go-sqlite3) +- [Description](#description) + - [Overview](#overview) - [Installation](#installation) - [API Reference](#api-reference) - [Connection String](#connection-string) + - [DSN Examples](#dsn-examples) - [Features](#features) + - [Usage](#usage) + - [Feature / Extension List](#feature--extension-list) - [Compilation](#compilation) - [Android](#android) - - [ARM](#arm) - - [Cross Compile](#cross-compile) - - [Google Cloud Platform](#google-cloud-platform) +- [ARM](#arm) +- [Cross Compile](#cross-compile) +- [Google Cloud Platform](#google-cloud-platform) - [Linux](#linux) - [Alpine](#alpine) - [Fedora](#fedora) @@ -36,11 +43,22 @@ Supported Golang version: - [Errors](#errors) - [User Authentication](#user-authentication) - [Compile](#compile) - - [Usage](#usage) + - [Usage](#usage-1) + - [Create protected database](#create-protected-database) + - [Password Encoding](#password-encoding) + - [Available Encoders](#available-encoders) + - [Restrictions](#restrictions) + - [Support](#support) + - [User Management](#user-management) + - [SQL](#sql) + - [Examples](#examples) + - [*SQLiteConn](#sqliteconn) + - [Attached database](#attached-database) - [Extensions](#extensions) - [Spatialite](#spatialite) - [FAQ](#faq) - [License](#license) +- [Author](#author) # Installation @@ -67,6 +85,7 @@ This is also known as a DSN string. (Data Source Name). Options are append after the filename of the SQLite database. The database filename and options are seperated by an `?` (Question Mark). +Options should be URL-encoded (see [url.QueryEscape](https://golang.org/pkg/net/url/#QueryEscape)). This also applies when using an in-memory database instead of a file. @@ -150,6 +169,7 @@ go build --tags "icu json1 fts5 secure_delete" | International Components for Unicode | sqlite_icu | This option causes the International Components for Unicode or "ICU" extension to SQLite to be added to the build | | Introspect PRAGMAS | sqlite_introspect | This option adds some extra PRAGMA statements.
  • PRAGMA function_list
  • PRAGMA module_list
  • PRAGMA pragma_list
| | JSON SQL Functions | sqlite_json | When this option is defined in the amalgamation, the JSON SQL functions are added to the build automatically | +| Pre Update Hook | sqlite_preupdate_hook | Registers a callback function that is invoked prior to each INSERT, UPDATE, and DELETE operation on a database table. | | Secure Delete | sqlite_secure_delete | This compile-time option changes the default setting of the secure_delete pragma.

When this option is not used, secure_delete defaults to off. When this option is present, secure_delete defaults to on.

The secure_delete setting causes deleted content to be overwritten with zeros. There is a small performance penalty since additional I/O must occur.

On the other hand, secure_delete can prevent fragments of sensitive information from lingering in unused parts of the database file after it has been deleted. See the documentation on the secure_delete pragma for additional information | | Secure Delete (FAST) | sqlite_secure_delete_fast | For more information see [PRAGMA secure_delete](https://www.sqlite.org/pragma.html#pragma_secure_delete) | | Tracing / Debug | sqlite_trace | Activate trace functions | @@ -192,13 +212,19 @@ This library can be cross-compiled. In some cases you are required to the `CC` environment variable with the cross compiler. -Additional information: -- [#491](https://github.com/mattn/go-sqlite3/issues/491) -- [#560](https://github.com/mattn/go-sqlite3/issues/560) +## Cross Compiling from MAC OSX +The simplest way to cross compile from OSX is to use [xgo](https://github.com/karalabe/xgo). + +Steps: +- Install [xgo](https://github.com/karalabe/xgo) (`go get github.com/karalabe/xgo`). +- Ensure that your project is within your `GOPATH`. +- Run `xgo local/path/to/project`. + +Please refer to the project's [README](https://github.com/karalabe/xgo/blob/master/README.md) for further information. # Google Cloud Platform -Building on GCP is not possible because `Google Cloud Platform does not allow `gcc` to be executed. +Building on GCP is not possible because Google Cloud Platform does not allow `gcc` to be executed. Please work only with compiled final binaries. @@ -248,7 +274,7 @@ Required dependency brew install sqlite3 ``` -For OSX there is an additional package install which is required if you whish to build the `icu` extension. +For OSX there is an additional package install which is required if you wish to build the `icu` extension. This additional package can be installed with `homebrew`. @@ -281,7 +307,7 @@ To compile this package on Windows OS you must have the `gcc` compiler installed 3) Open a terminal for the TDM-GCC toolchain, can be found in the Windows Start menu. 4) Navigate to your project folder and run the `go build ...` command for this package. -For example the TDM-GCC Toolchain can be found [here](ttps://sourceforge.net/projects/tdm-gcc/). +For example the TDM-GCC Toolchain can be found [here](https://sourceforge.net/projects/tdm-gcc/). ## Errors @@ -290,7 +316,7 @@ For example the TDM-GCC Toolchain can be found [here](ttps://sourceforge.net/pro When receiving a compile time error referencing recompile with `-FPIC` then you are probably using a hardend system. - You can copile the library on a hardend system with the following command. + You can compile the library on a hardend system with the following command. ```bash go build -ldflags '-extldflags=-fno-PIC' @@ -433,6 +459,16 @@ If you want your own extension to be listed here or you want to add a reference Spatialite is available as an extension to SQLite, and can be used in combination with this repository. For an example see [shaxbee/go-spatialite](https://github.com/shaxbee/go-spatialite). +## extension-functions.c from SQLite3 Contrib + +extension-functions.c is available as an extension to SQLite, and provides the following functions: + +- Math: acos, asin, atan, atn2, atan2, acosh, asinh, atanh, difference, degrees, radians, cos, sin, tan, cot, cosh, sinh, tanh, coth, exp, log, log10, power, sign, sqrt, square, ceil, floor, pi. +- String: replicate, charindex, leftstr, rightstr, ltrim, rtrim, trim, replace, reverse, proper, padl, padr, padc, strfilter. +- Aggregate: stdev, variance, mode, median, lower_quartile, upper_quartile + +For an example see [dinedal/go-sqlite3-extension-functions](https://github.com/dinedal/go-sqlite3-extension-functions). + # FAQ - Getting insert error while query is opened. @@ -457,15 +493,19 @@ For an example see [shaxbee/go-spatialite](https://github.com/shaxbee/go-spatial Why is it racy if I use a `sql.Open("sqlite3", ":memory:")` database? - Each connection to :memory: opens a brand new in-memory sql database, so if + Each connection to `":memory:"` opens a brand new in-memory sql database, so if the stdlib's sql engine happens to open another connection and you've only - specified ":memory:", that connection will see a brand new database. A - workaround is to use "file::memory:?mode=memory&cache=shared". Every - connection to this string will point to the same in-memory database. + specified `":memory:"`, that connection will see a brand new database. A + workaround is to use `"file::memory:?cache=shared"` (or `"file:foobar?mode=memory&cache=shared"`). Every + connection to this string will point to the same in-memory database. + + Note that if the last database connection in the pool closes, the in-memory database is deleted. Make sure the [max idle connection limit](https://golang.org/pkg/database/sql/#DB.SetMaxIdleConns) is > 0, and the [connection lifetime](https://golang.org/pkg/database/sql/#DB.SetConnMaxLifetime) is infinite. For more information see * [#204](https://github.com/mattn/go-sqlite3/issues/204) * [#511](https://github.com/mattn/go-sqlite3/issues/511) + * https://www.sqlite.org/sharedcache.html#shared_cache_and_in_memory_databases + * https://www.sqlite.org/inmemorydb.html#sharedmemdb - Reading from database with large amount of goroutines fails on OSX. @@ -473,18 +513,18 @@ For an example see [shaxbee/go-spatialite](https://github.com/shaxbee/go-spatial For more information see [#289](https://github.com/mattn/go-sqlite3/issues/289) -- Trying to execure a `.` (dot) command throws an error. +- Trying to execute a `.` (dot) command throws an error. Error: `Error: near ".": syntax error` Dot command are part of SQLite3 CLI not of this library. You need to implement the feature or call the sqlite3 cli. - More infomation see [#305](https://github.com/mattn/go-sqlite3/issues/305) + More information see [#305](https://github.com/mattn/go-sqlite3/issues/305) - Error: `database is locked` - When you get an database is locked. Please use the following options. + When you get a database is locked. Please use the following options. Add to DSN: `cache=shared` @@ -496,11 +536,41 @@ For an example see [shaxbee/go-spatialite](https://github.com/shaxbee/go-spatial Second please set the database connections of the SQL package to 1. ```go - db.SetMaxOpenConn(1) + db.SetMaxOpenConns(1) ``` More information see [#209](https://github.com/mattn/go-sqlite3/issues/209) +## Contributors + +### Code Contributors + +This project exists thanks to all the people who contribute. [[Contribute](CONTRIBUTING.md)]. + + +### Financial Contributors + +Become a financial contributor and help us sustain our community. [[Contribute](https://opencollective.com/mattn-go-sqlite3/contribute)] + +#### Individuals + + + +#### Organizations + +Support this project with your organization. Your logo will show up here with a link to your website. [[Contribute](https://opencollective.com/mattn-go-sqlite3/contribute)] + + + + + + + + + + + + # License MIT: http://mattn.mit-license.org/2018 diff --git a/vendor/github.com/mattn/go-sqlite3/backup.go b/vendor/github.com/mattn/go-sqlite3/backup.go index 5ab3a54d..e222cc88 100644 --- a/vendor/github.com/mattn/go-sqlite3/backup.go +++ b/vendor/github.com/mattn/go-sqlite3/backup.go @@ -1,4 +1,4 @@ -// Copyright (C) 2014 Yasuhiro Matsumoto . +// Copyright (C) 2019 Yasuhiro Matsumoto . // // Use of this source code is governed by an MIT-style // license that can be found in the LICENSE file. @@ -25,18 +25,18 @@ type SQLiteBackup struct { } // Backup make backup from src to dest. -func (c *SQLiteConn) Backup(dest string, conn *SQLiteConn, src string) (*SQLiteBackup, error) { +func (destConn *SQLiteConn) Backup(dest string, srcConn *SQLiteConn, src string) (*SQLiteBackup, error) { destptr := C.CString(dest) defer C.free(unsafe.Pointer(destptr)) srcptr := C.CString(src) defer C.free(unsafe.Pointer(srcptr)) - if b := C.sqlite3_backup_init(c.db, destptr, conn.db, srcptr); b != nil { + if b := C.sqlite3_backup_init(destConn.db, destptr, srcConn.db, srcptr); b != nil { bb := &SQLiteBackup{b: b} runtime.SetFinalizer(bb, (*SQLiteBackup).Finish) return bb, nil } - return nil, c.lastError() + return nil, destConn.lastError() } // Step to backs up for one step. Calls the underlying `sqlite3_backup_step` diff --git a/vendor/github.com/mattn/go-sqlite3/callback.go b/vendor/github.com/mattn/go-sqlite3/callback.go index 5a735c03..64505751 100644 --- a/vendor/github.com/mattn/go-sqlite3/callback.go +++ b/vendor/github.com/mattn/go-sqlite3/callback.go @@ -1,4 +1,4 @@ -// Copyright (C) 2014 Yasuhiro Matsumoto . +// Copyright (C) 2019 Yasuhiro Matsumoto . // // Use of this source code is governed by an MIT-style // license that can be found in the LICENSE file. @@ -77,8 +77,28 @@ func updateHookTrampoline(handle uintptr, op int, db *C.char, table *C.char, row callback(op, C.GoString(db), C.GoString(table), rowid) } -// Use handles to avoid passing Go pointers to C. +//export authorizerTrampoline +func authorizerTrampoline(handle uintptr, op int, arg1 *C.char, arg2 *C.char, arg3 *C.char) int { + callback := lookupHandle(handle).(func(int, string, string, string) int) + return callback(op, C.GoString(arg1), C.GoString(arg2), C.GoString(arg3)) +} + +//export preUpdateHookTrampoline +func preUpdateHookTrampoline(handle uintptr, dbHandle uintptr, op int, db *C.char, table *C.char, oldrowid int64, newrowid int64) { + hval := lookupHandleVal(handle) + data := SQLitePreUpdateData{ + Conn: hval.db, + Op: op, + DatabaseName: C.GoString(db), + TableName: C.GoString(table), + OldRowID: oldrowid, + NewRowID: newrowid, + } + callback := hval.val.(func(SQLitePreUpdateData)) + callback(data) +} +// Use handles to avoid passing Go pointers to C. type handleVal struct { db *SQLiteConn val interface{} @@ -97,7 +117,7 @@ func newHandle(db *SQLiteConn, v interface{}) uintptr { return i } -func lookupHandle(handle uintptr) interface{} { +func lookupHandleVal(handle uintptr) handleVal { handleLock.Lock() defer handleLock.Unlock() r, ok := handleVals[handle] @@ -108,7 +128,11 @@ func lookupHandle(handle uintptr) interface{} { panic("invalid handle") } } - return r.val + return r +} + +func lookupHandle(handle uintptr) interface{} { + return lookupHandleVal(handle).val } func deleteHandles(db *SQLiteConn) { @@ -362,7 +386,7 @@ func callbackRet(typ reflect.Type) (callbackRetConverter, error) { func callbackError(ctx *C.sqlite3_context, err error) { cstr := C.CString(err.Error()) defer C.free(unsafe.Pointer(cstr)) - C.sqlite3_result_error(ctx, cstr, -1) + C.sqlite3_result_error(ctx, cstr, C.int(-1)) } // Test support code. Tests are not allowed to import "C", so we can't diff --git a/vendor/github.com/mattn/go-sqlite3/convert.go b/vendor/github.com/mattn/go-sqlite3/convert.go new file mode 100644 index 00000000..03850737 --- /dev/null +++ b/vendor/github.com/mattn/go-sqlite3/convert.go @@ -0,0 +1,299 @@ +// Extracted from Go database/sql source code + +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Type conversions for Scan. + +package sqlite3 + +import ( + "database/sql" + "database/sql/driver" + "errors" + "fmt" + "reflect" + "strconv" + "time" +) + +var errNilPtr = errors.New("destination pointer is nil") // embedded in descriptive error + +// convertAssign copies to dest the value in src, converting it if possible. +// An error is returned if the copy would result in loss of information. +// dest should be a pointer type. +func convertAssign(dest, src interface{}) error { + // Common cases, without reflect. + switch s := src.(type) { + case string: + switch d := dest.(type) { + case *string: + if d == nil { + return errNilPtr + } + *d = s + return nil + case *[]byte: + if d == nil { + return errNilPtr + } + *d = []byte(s) + return nil + case *sql.RawBytes: + if d == nil { + return errNilPtr + } + *d = append((*d)[:0], s...) + return nil + } + case []byte: + switch d := dest.(type) { + case *string: + if d == nil { + return errNilPtr + } + *d = string(s) + return nil + case *interface{}: + if d == nil { + return errNilPtr + } + *d = cloneBytes(s) + return nil + case *[]byte: + if d == nil { + return errNilPtr + } + *d = cloneBytes(s) + return nil + case *sql.RawBytes: + if d == nil { + return errNilPtr + } + *d = s + return nil + } + case time.Time: + switch d := dest.(type) { + case *time.Time: + *d = s + return nil + case *string: + *d = s.Format(time.RFC3339Nano) + return nil + case *[]byte: + if d == nil { + return errNilPtr + } + *d = []byte(s.Format(time.RFC3339Nano)) + return nil + case *sql.RawBytes: + if d == nil { + return errNilPtr + } + *d = s.AppendFormat((*d)[:0], time.RFC3339Nano) + return nil + } + case nil: + switch d := dest.(type) { + case *interface{}: + if d == nil { + return errNilPtr + } + *d = nil + return nil + case *[]byte: + if d == nil { + return errNilPtr + } + *d = nil + return nil + case *sql.RawBytes: + if d == nil { + return errNilPtr + } + *d = nil + return nil + } + } + + var sv reflect.Value + + switch d := dest.(type) { + case *string: + sv = reflect.ValueOf(src) + switch sv.Kind() { + case reflect.Bool, + reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, + reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, + reflect.Float32, reflect.Float64: + *d = asString(src) + return nil + } + case *[]byte: + sv = reflect.ValueOf(src) + if b, ok := asBytes(nil, sv); ok { + *d = b + return nil + } + case *sql.RawBytes: + sv = reflect.ValueOf(src) + if b, ok := asBytes([]byte(*d)[:0], sv); ok { + *d = sql.RawBytes(b) + return nil + } + case *bool: + bv, err := driver.Bool.ConvertValue(src) + if err == nil { + *d = bv.(bool) + } + return err + case *interface{}: + *d = src + return nil + } + + if scanner, ok := dest.(sql.Scanner); ok { + return scanner.Scan(src) + } + + dpv := reflect.ValueOf(dest) + if dpv.Kind() != reflect.Ptr { + return errors.New("destination not a pointer") + } + if dpv.IsNil() { + return errNilPtr + } + + if !sv.IsValid() { + sv = reflect.ValueOf(src) + } + + dv := reflect.Indirect(dpv) + if sv.IsValid() && sv.Type().AssignableTo(dv.Type()) { + switch b := src.(type) { + case []byte: + dv.Set(reflect.ValueOf(cloneBytes(b))) + default: + dv.Set(sv) + } + return nil + } + + if dv.Kind() == sv.Kind() && sv.Type().ConvertibleTo(dv.Type()) { + dv.Set(sv.Convert(dv.Type())) + return nil + } + + // The following conversions use a string value as an intermediate representation + // to convert between various numeric types. + // + // This also allows scanning into user defined types such as "type Int int64". + // For symmetry, also check for string destination types. + switch dv.Kind() { + case reflect.Ptr: + if src == nil { + dv.Set(reflect.Zero(dv.Type())) + return nil + } + dv.Set(reflect.New(dv.Type().Elem())) + return convertAssign(dv.Interface(), src) + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + s := asString(src) + i64, err := strconv.ParseInt(s, 10, dv.Type().Bits()) + if err != nil { + err = strconvErr(err) + return fmt.Errorf("converting driver.Value type %T (%q) to a %s: %v", src, s, dv.Kind(), err) + } + dv.SetInt(i64) + return nil + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + s := asString(src) + u64, err := strconv.ParseUint(s, 10, dv.Type().Bits()) + if err != nil { + err = strconvErr(err) + return fmt.Errorf("converting driver.Value type %T (%q) to a %s: %v", src, s, dv.Kind(), err) + } + dv.SetUint(u64) + return nil + case reflect.Float32, reflect.Float64: + s := asString(src) + f64, err := strconv.ParseFloat(s, dv.Type().Bits()) + if err != nil { + err = strconvErr(err) + return fmt.Errorf("converting driver.Value type %T (%q) to a %s: %v", src, s, dv.Kind(), err) + } + dv.SetFloat(f64) + return nil + case reflect.String: + switch v := src.(type) { + case string: + dv.SetString(v) + return nil + case []byte: + dv.SetString(string(v)) + return nil + } + } + + return fmt.Errorf("unsupported Scan, storing driver.Value type %T into type %T", src, dest) +} + +func strconvErr(err error) error { + if ne, ok := err.(*strconv.NumError); ok { + return ne.Err + } + return err +} + +func cloneBytes(b []byte) []byte { + if b == nil { + return nil + } + c := make([]byte, len(b)) + copy(c, b) + return c +} + +func asString(src interface{}) string { + switch v := src.(type) { + case string: + return v + case []byte: + return string(v) + } + rv := reflect.ValueOf(src) + switch rv.Kind() { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return strconv.FormatInt(rv.Int(), 10) + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + return strconv.FormatUint(rv.Uint(), 10) + case reflect.Float64: + return strconv.FormatFloat(rv.Float(), 'g', -1, 64) + case reflect.Float32: + return strconv.FormatFloat(rv.Float(), 'g', -1, 32) + case reflect.Bool: + return strconv.FormatBool(rv.Bool()) + } + return fmt.Sprintf("%v", src) +} + +func asBytes(buf []byte, rv reflect.Value) (b []byte, ok bool) { + switch rv.Kind() { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return strconv.AppendInt(buf, rv.Int(), 10), true + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + return strconv.AppendUint(buf, rv.Uint(), 10), true + case reflect.Float32: + return strconv.AppendFloat(buf, rv.Float(), 'g', -1, 32), true + case reflect.Float64: + return strconv.AppendFloat(buf, rv.Float(), 'g', -1, 64), true + case reflect.Bool: + return strconv.AppendBool(buf, rv.Bool()), true + case reflect.String: + s := rv.String() + return append(buf, s...), true + } + return +} diff --git a/vendor/github.com/mattn/go-sqlite3/error.go b/vendor/github.com/mattn/go-sqlite3/error.go index 49ab8903..696281c7 100644 --- a/vendor/github.com/mattn/go-sqlite3/error.go +++ b/vendor/github.com/mattn/go-sqlite3/error.go @@ -1,11 +1,19 @@ -// Copyright (C) 2014 Yasuhiro Matsumoto . +// Copyright (C) 2019 Yasuhiro Matsumoto . // // Use of this source code is governed by an MIT-style // license that can be found in the LICENSE file. package sqlite3 +/* +#ifndef USE_LIBSQLITE3 +#include +#else +#include +#endif +*/ import "C" +import "syscall" // ErrNo inherit errno. type ErrNo int @@ -20,6 +28,7 @@ type ErrNoExtended int type Error struct { Code ErrNo /* The error code returned by SQLite */ ExtendedCode ErrNoExtended /* The extended error code returned by SQLite */ + SystemErrno syscall.Errno /* The system errno returned by the OS through SQLite, if applicable */ err string /* The error string returned by sqlite3_errmsg(), this usually contains more specific details. */ } @@ -72,10 +81,16 @@ func (err ErrNoExtended) Error() string { } func (err Error) Error() string { + var str string if err.err != "" { - return err.err + str = err.err + } else { + str = C.GoString(C.sqlite3_errstr(C.int(err.Code))) } - return errorString(err) + if err.SystemErrno != 0 { + str += ": " + err.SystemErrno.Error() + } + return str } // result codes from http://www.sqlite.org/c3ref/c_abort_rollback.html diff --git a/vendor/github.com/mattn/go-sqlite3/go.mod b/vendor/github.com/mattn/go-sqlite3/go.mod new file mode 100644 index 00000000..d3a1571c --- /dev/null +++ b/vendor/github.com/mattn/go-sqlite3/go.mod @@ -0,0 +1,8 @@ +module github.com/mattn/go-sqlite3 + +go 1.10 + +require ( + github.com/PuerkitoBio/goquery v1.5.1 + golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e // indirect +) diff --git a/vendor/github.com/mattn/go-sqlite3/go.sum b/vendor/github.com/mattn/go-sqlite3/go.sum new file mode 100644 index 00000000..d2963a7f --- /dev/null +++ b/vendor/github.com/mattn/go-sqlite3/go.sum @@ -0,0 +1,13 @@ +github.com/PuerkitoBio/goquery v1.5.1 h1:PSPBGne8NIUWw+/7vFBV+kG2J/5MOjbzc7154OaKCSE= +github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= +github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5zzsLTo= +github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e h1:3G+cUijn7XD+S4eJFddp53Pv7+slrESplyjG25HgL+k= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/vendor/github.com/mattn/go-sqlite3/sqlite3-binding.c b/vendor/github.com/mattn/go-sqlite3/sqlite3-binding.c index f077152a..aeb87fbf 100644 --- a/vendor/github.com/mattn/go-sqlite3/sqlite3-binding.c +++ b/vendor/github.com/mattn/go-sqlite3/sqlite3-binding.c @@ -1,7 +1,7 @@ #ifndef USE_LIBSQLITE3 /****************************************************************************** ** This file is an amalgamation of many separate C source files from SQLite -** version 3.24.0. By combining all the individual C code files into this +** version 3.32.2. By combining all the individual C code files into this ** single large file, the entire code can be compiled as a single translation ** unit. This allows many compilers to do optimizations that would not be ** possible if the files were compiled separately. Performance improvements @@ -40,7 +40,7 @@ ** SQLite was built with. */ -#ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS +#ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS /* IMP: R-16824-07538 */ /* ** Include the configuration header output by 'configure' if we're using the @@ -56,6 +56,12 @@ #define CTIMEOPT_VAL_(opt) #opt #define CTIMEOPT_VAL(opt) CTIMEOPT_VAL_(opt) +/* Like CTIMEOPT_VAL, but especially for SQLITE_DEFAULT_LOOKASIDE. This +** option requires a separate macro because legal values contain a single +** comma. e.g. (-DSQLITE_DEFAULT_LOOKASIDE="100,100") */ +#define CTIMEOPT_VAL2_(opt1,opt2) #opt1 "," #opt2 +#define CTIMEOPT_VAL2(opt) CTIMEOPT_VAL2_(opt) + /* ** An array of names of all compile-time options. This array should ** be sorted A-Z. @@ -139,7 +145,7 @@ static const char * const sqlite3azCompileOpt[] = { "DEFAULT_LOCKING_MODE=" CTIMEOPT_VAL(SQLITE_DEFAULT_LOCKING_MODE), #endif #ifdef SQLITE_DEFAULT_LOOKASIDE - "DEFAULT_LOOKASIDE=" CTIMEOPT_VAL(SQLITE_DEFAULT_LOOKASIDE), + "DEFAULT_LOOKASIDE=" CTIMEOPT_VAL2(SQLITE_DEFAULT_LOOKASIDE), #endif #if SQLITE_DEFAULT_MEMSTATUS "DEFAULT_MEMSTATUS", @@ -213,6 +219,9 @@ static const char * const sqlite3azCompileOpt[] = { #if SQLITE_ENABLE_BATCH_ATOMIC_WRITE "ENABLE_BATCH_ATOMIC_WRITE", #endif +#if SQLITE_ENABLE_BYTECODE_VTAB + "ENABLE_BYTECODE_VTAB", +#endif #if SQLITE_ENABLE_CEROD "ENABLE_CEROD=" CTIMEOPT_VAL(SQLITE_ENABLE_CEROD), #endif @@ -255,6 +264,9 @@ static const char * const sqlite3azCompileOpt[] = { #if SQLITE_ENABLE_FTS5 "ENABLE_FTS5", #endif +#if SQLITE_ENABLE_GEOPOLY + "ENABLE_GEOPOLY", +#endif #if SQLITE_ENABLE_HIDDEN_COLUMNS "ENABLE_HIDDEN_COLUMNS", #endif @@ -285,6 +297,9 @@ static const char * const sqlite3azCompileOpt[] = { #if SQLITE_ENABLE_MULTIPLEX "ENABLE_MULTIPLEX", #endif +#if SQLITE_ENABLE_NORMALIZE + "ENABLE_NORMALIZE", +#endif #if SQLITE_ENABLE_NULL_TRIM "ENABLE_NULL_TRIM", #endif @@ -320,8 +335,6 @@ static const char * const sqlite3azCompileOpt[] = { #endif #if defined(SQLITE_ENABLE_STAT4) "ENABLE_STAT4", -#elif defined(SQLITE_ENABLE_STAT3) - "ENABLE_STAT3", #endif #if SQLITE_ENABLE_STMTVTAB "ENABLE_STMTVTAB", @@ -371,9 +384,6 @@ static const char * const sqlite3azCompileOpt[] = { #if SQLITE_FTS5_NO_WITHOUT_ROWID "FTS5_NO_WITHOUT_ROWID", #endif -#if SQLITE_HAS_CODEC - "HAS_CODEC", -#endif #if HAVE_ISNAN || SQLITE_HAVE_ISNAN "HAVE_ISNAN", #endif @@ -530,9 +540,6 @@ static const char * const sqlite3azCompileOpt[] = { #if SQLITE_OMIT_BLOB_LITERAL "OMIT_BLOB_LITERAL", #endif -#if SQLITE_OMIT_BTREECOUNT - "OMIT_BTREECOUNT", -#endif #if SQLITE_OMIT_CAST "OMIT_CAST", #endif @@ -877,6 +884,11 @@ SQLITE_PRIVATE const char **sqlite3CompileOptions(int *pnOpt){ #pragma warning(disable : 4706) #endif /* defined(_MSC_VER) */ +#if defined(_MSC_VER) && !defined(_WIN64) +#undef SQLITE_4_BYTE_ALIGNED_MALLOC +#define SQLITE_4_BYTE_ALIGNED_MALLOC +#endif /* defined(_MSC_VER) && !defined(_WIN64) */ + #endif /* SQLITE_MSVC_H */ /************** End of msvc.h ************************************************/ @@ -1151,9 +1163,9 @@ extern "C" { ** [sqlite3_libversion_number()], [sqlite3_sourceid()], ** [sqlite_version()] and [sqlite_source_id()]. */ -#define SQLITE_VERSION "3.24.0" -#define SQLITE_VERSION_NUMBER 3024000 -#define SQLITE_SOURCE_ID "2018-06-04 19:24:41 c7ee0833225bfd8c5ec2f9bf62b97c4e04d03bd9566366d5221ac8fb199a87ca" +#define SQLITE_VERSION "3.32.2" +#define SQLITE_VERSION_NUMBER 3032002 +#define SQLITE_SOURCE_ID "2020-06-04 12:58:43 ec02243ea6ce33b090870ae55ab8aa2534b54d216d45c4aa2fdbb00e86861e8c" /* ** CAPI3REF: Run-Time Library Version Numbers @@ -1217,6 +1229,9 @@ SQLITE_API int sqlite3_libversion_number(void); #ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS SQLITE_API int sqlite3_compileoption_used(const char *zOptName); SQLITE_API const char *sqlite3_compileoption_get(int N); +#else +# define sqlite3_compileoption_used(X) 0 +# define sqlite3_compileoption_get(X) ((void*)0) #endif /* @@ -1324,26 +1339,22 @@ typedef sqlite_uint64 sqlite3_uint64; ** the [sqlite3] object is successfully destroyed and all associated ** resources are deallocated. ** -** ^If the database connection is associated with unfinalized prepared -** statements or unfinished sqlite3_backup objects then sqlite3_close() -** will leave the database connection open and return [SQLITE_BUSY]. -** ^If sqlite3_close_v2() is called with unfinalized prepared statements -** and/or unfinished sqlite3_backups, then the database connection becomes -** an unusable "zombie" which will automatically be deallocated when the -** last prepared statement is finalized or the last sqlite3_backup is -** finished. The sqlite3_close_v2() interface is intended for use with -** host languages that are garbage collected, and where the order in which -** destructors are called is arbitrary. -** -** Applications should [sqlite3_finalize | finalize] all [prepared statements], -** [sqlite3_blob_close | close] all [BLOB handles], and +** Ideally, applications should [sqlite3_finalize | finalize] all +** [prepared statements], [sqlite3_blob_close | close] all [BLOB handles], and ** [sqlite3_backup_finish | finish] all [sqlite3_backup] objects associated -** with the [sqlite3] object prior to attempting to close the object. ^If -** sqlite3_close_v2() is called on a [database connection] that still has -** outstanding [prepared statements], [BLOB handles], and/or -** [sqlite3_backup] objects then it returns [SQLITE_OK] and the deallocation -** of resources is deferred until all [prepared statements], [BLOB handles], -** and [sqlite3_backup] objects are also destroyed. +** with the [sqlite3] object prior to attempting to close the object. +** ^If the database connection is associated with unfinalized prepared +** statements, BLOB handlers, and/or unfinished sqlite3_backup objects then +** sqlite3_close() will leave the database connection open and return +** [SQLITE_BUSY]. ^If sqlite3_close_v2() is called with unfinalized prepared +** statements, unclosed BLOB handlers, and/or unfinished sqlite3_backups, +** it returns [SQLITE_OK] regardless, but instead of deallocating the database +** connection immediately, it marks the database connection as an unusable +** "zombie" and makes arrangements to automatically deallocate the database +** connection after all prepared statements are finalized, all BLOB handles +** are closed, and all backups have finished. The sqlite3_close_v2() interface +** is intended for use with host languages that are garbage collected, and +** where the order in which destructors are called is arbitrary. ** ** ^If an [sqlite3] object is destroyed while a transaction is open, ** the transaction is automatically rolled back. @@ -1500,6 +1511,7 @@ SQLITE_API int sqlite3_exec( */ #define SQLITE_ERROR_MISSING_COLLSEQ (SQLITE_ERROR | (1<<8)) #define SQLITE_ERROR_RETRY (SQLITE_ERROR | (2<<8)) +#define SQLITE_ERROR_SNAPSHOT (SQLITE_ERROR | (3<<8)) #define SQLITE_IOERR_READ (SQLITE_IOERR | (1<<8)) #define SQLITE_IOERR_SHORT_READ (SQLITE_IOERR | (2<<8)) #define SQLITE_IOERR_WRITE (SQLITE_IOERR | (3<<8)) @@ -1531,16 +1543,21 @@ SQLITE_API int sqlite3_exec( #define SQLITE_IOERR_BEGIN_ATOMIC (SQLITE_IOERR | (29<<8)) #define SQLITE_IOERR_COMMIT_ATOMIC (SQLITE_IOERR | (30<<8)) #define SQLITE_IOERR_ROLLBACK_ATOMIC (SQLITE_IOERR | (31<<8)) +#define SQLITE_IOERR_DATA (SQLITE_IOERR | (32<<8)) #define SQLITE_LOCKED_SHAREDCACHE (SQLITE_LOCKED | (1<<8)) #define SQLITE_LOCKED_VTAB (SQLITE_LOCKED | (2<<8)) #define SQLITE_BUSY_RECOVERY (SQLITE_BUSY | (1<<8)) #define SQLITE_BUSY_SNAPSHOT (SQLITE_BUSY | (2<<8)) +#define SQLITE_BUSY_TIMEOUT (SQLITE_BUSY | (3<<8)) #define SQLITE_CANTOPEN_NOTEMPDIR (SQLITE_CANTOPEN | (1<<8)) #define SQLITE_CANTOPEN_ISDIR (SQLITE_CANTOPEN | (2<<8)) #define SQLITE_CANTOPEN_FULLPATH (SQLITE_CANTOPEN | (3<<8)) #define SQLITE_CANTOPEN_CONVPATH (SQLITE_CANTOPEN | (4<<8)) +#define SQLITE_CANTOPEN_DIRTYWAL (SQLITE_CANTOPEN | (5<<8)) /* Not Used */ +#define SQLITE_CANTOPEN_SYMLINK (SQLITE_CANTOPEN | (6<<8)) #define SQLITE_CORRUPT_VTAB (SQLITE_CORRUPT | (1<<8)) #define SQLITE_CORRUPT_SEQUENCE (SQLITE_CORRUPT | (2<<8)) +#define SQLITE_CORRUPT_INDEX (SQLITE_CORRUPT | (3<<8)) #define SQLITE_READONLY_RECOVERY (SQLITE_READONLY | (1<<8)) #define SQLITE_READONLY_CANTLOCK (SQLITE_READONLY | (2<<8)) #define SQLITE_READONLY_ROLLBACK (SQLITE_READONLY | (3<<8)) @@ -1558,11 +1575,13 @@ SQLITE_API int sqlite3_exec( #define SQLITE_CONSTRAINT_UNIQUE (SQLITE_CONSTRAINT | (8<<8)) #define SQLITE_CONSTRAINT_VTAB (SQLITE_CONSTRAINT | (9<<8)) #define SQLITE_CONSTRAINT_ROWID (SQLITE_CONSTRAINT |(10<<8)) +#define SQLITE_CONSTRAINT_PINNED (SQLITE_CONSTRAINT |(11<<8)) #define SQLITE_NOTICE_RECOVER_WAL (SQLITE_NOTICE | (1<<8)) #define SQLITE_NOTICE_RECOVER_ROLLBACK (SQLITE_NOTICE | (2<<8)) #define SQLITE_WARNING_AUTOINDEX (SQLITE_WARNING | (1<<8)) #define SQLITE_AUTH_USER (SQLITE_AUTH | (1<<8)) #define SQLITE_OK_LOAD_PERMANENTLY (SQLITE_OK | (1<<8)) +#define SQLITE_OK_SYMLINK (SQLITE_OK | (2<<8)) /* ** CAPI3REF: Flags For File Open Operations @@ -1591,6 +1610,7 @@ SQLITE_API int sqlite3_exec( #define SQLITE_OPEN_SHAREDCACHE 0x00020000 /* Ok for sqlite3_open_v2() */ #define SQLITE_OPEN_PRIVATECACHE 0x00040000 /* Ok for sqlite3_open_v2() */ #define SQLITE_OPEN_WAL 0x00080000 /* VFS only */ +#define SQLITE_OPEN_NOFOLLOW 0x01000000 /* Ok for sqlite3_open_v2() */ /* Reserved: 0x00F00000 */ @@ -1849,6 +1869,15 @@ struct sqlite3_io_methods { ** file space based on this hint in order to help writes to the database ** file run faster. ** +**
  • [[SQLITE_FCNTL_SIZE_LIMIT]] +** The [SQLITE_FCNTL_SIZE_LIMIT] opcode is used by in-memory VFS that +** implements [sqlite3_deserialize()] to set an upper bound on the size +** of the in-memory database. The argument is a pointer to a [sqlite3_int64]. +** If the integer pointed to is negative, then it is filled in with the +** current limit. Otherwise the limit is set to the larger of the value +** of the integer pointed to and the current database size. The integer +** pointed to is set to the new limit. +** **
  • [[SQLITE_FCNTL_CHUNK_SIZE]] ** The [SQLITE_FCNTL_CHUNK_SIZE] opcode is used to request that the VFS ** extends and truncates the database file in chunks of a size specified @@ -1914,7 +1943,8 @@ struct sqlite3_io_methods { **
  • [[SQLITE_FCNTL_PERSIST_WAL]] ** ^The [SQLITE_FCNTL_PERSIST_WAL] opcode is used to set or query the ** persistent [WAL | Write Ahead Log] setting. By default, the auxiliary -** write ahead log and shared memory files used for transaction control +** write ahead log ([WAL file]) and shared memory +** files used for transaction control ** are automatically deleted when the latest connection to the database ** closes. Setting persistent WAL mode causes those files to persist after ** close. Persisting the files is useful when other processes that do not @@ -1992,16 +2022,16 @@ struct sqlite3_io_methods { ** ^The [SQLITE_FCNTL_BUSYHANDLER] ** file-control may be invoked by SQLite on the database file handle ** shortly after it is opened in order to provide a custom VFS with access -** to the connections busy-handler callback. The argument is of type (void **) +** to the connection's busy-handler callback. The argument is of type (void**) ** - an array of two (void *) values. The first (void *) actually points -** to a function of type (int (*)(void *)). In order to invoke the connections +** to a function of type (int (*)(void *)). In order to invoke the connection's ** busy-handler, this function should be invoked with the second (void *) in ** the array as the only argument. If it returns non-zero, then the operation ** should be retried. If it returns zero, the custom VFS should abandon the ** current operation. ** **
  • [[SQLITE_FCNTL_TEMPFILENAME]] -** ^Application can invoke the [SQLITE_FCNTL_TEMPFILENAME] file-control +** ^Applications can invoke the [SQLITE_FCNTL_TEMPFILENAME] file-control ** to have SQLite generate a ** temporary filename using the same algorithm that is followed to generate ** temporary filenames for TEMP tables and other internal uses. The @@ -2096,10 +2126,43 @@ struct sqlite3_io_methods { ** a prior successful call to [SQLITE_FCNTL_BEGIN_ATOMIC_WRITE]. ** **
  • [[SQLITE_FCNTL_LOCK_TIMEOUT]] -** The [SQLITE_FCNTL_LOCK_TIMEOUT] opcode causes attempts to obtain -** a file lock using the xLock or xShmLock methods of the VFS to wait -** for up to M milliseconds before failing, where M is the single -** unsigned integer parameter. +** The [SQLITE_FCNTL_LOCK_TIMEOUT] opcode is used to configure a VFS +** to block for up to M milliseconds before failing when attempting to +** obtain a file lock using the xLock or xShmLock methods of the VFS. +** The parameter is a pointer to a 32-bit signed integer that contains +** the value that M is to be set to. Before returning, the 32-bit signed +** integer is overwritten with the previous value of M. +** +**
  • [[SQLITE_FCNTL_DATA_VERSION]] +** The [SQLITE_FCNTL_DATA_VERSION] opcode is used to detect changes to +** a database file. The argument is a pointer to a 32-bit unsigned integer. +** The "data version" for the pager is written into the pointer. The +** "data version" changes whenever any change occurs to the corresponding +** database file, either through SQL statements on the same database +** connection or through transactions committed by separate database +** connections possibly in other processes. The [sqlite3_total_changes()] +** interface can be used to find if any database on the connection has changed, +** but that interface responds to changes on TEMP as well as MAIN and does +** not provide a mechanism to detect changes to MAIN only. Also, the +** [sqlite3_total_changes()] interface responds to internal changes only and +** omits changes made by other database connections. The +** [PRAGMA data_version] command provides a mechanism to detect changes to +** a single attached database that occur due to other database connections, +** but omits changes implemented by the database connection on which it is +** called. This file control is the only mechanism to detect changes that +** happen either internally or externally and that are associated with +** a particular attached database. +** +**
  • [[SQLITE_FCNTL_CKPT_START]] +** The [SQLITE_FCNTL_CKPT_START] opcode is invoked from within a checkpoint +** in wal mode before the client starts to copy pages from the wal +** file to the database file. +** +**
  • [[SQLITE_FCNTL_CKPT_DONE]] +** The [SQLITE_FCNTL_CKPT_DONE] opcode is invoked from within a checkpoint +** in wal mode after the client has finished copying pages from the wal +** file to the database file, but before the *-shm file is updated to +** record the fact that the pages have been checkpointed. ** */ #define SQLITE_FCNTL_LOCKSTATE 1 @@ -2135,6 +2198,11 @@ struct sqlite3_io_methods { #define SQLITE_FCNTL_COMMIT_ATOMIC_WRITE 32 #define SQLITE_FCNTL_ROLLBACK_ATOMIC_WRITE 33 #define SQLITE_FCNTL_LOCK_TIMEOUT 34 +#define SQLITE_FCNTL_DATA_VERSION 35 +#define SQLITE_FCNTL_SIZE_LIMIT 36 +#define SQLITE_FCNTL_CKPT_DONE 37 +#define SQLITE_FCNTL_RESERVE_BYTES 38 +#define SQLITE_FCNTL_CKPT_START 39 /* deprecated names */ #define SQLITE_GET_LOCKPROXYFILE SQLITE_FCNTL_GET_LOCKPROXYFILE @@ -2180,10 +2248,10 @@ typedef struct sqlite3_api_routines sqlite3_api_routines; ** to 3 with SQLite [version 3.7.6] on [dateof:3.7.6]. Additional fields ** may be appended to the sqlite3_vfs object and the iVersion value ** may increase again in future versions of SQLite. -** Note that the structure -** of the sqlite3_vfs object changes in the transition from +** Note that due to an oversight, the structure +** of the sqlite3_vfs object changed in the transition from ** SQLite [version 3.5.9] to [version 3.6.0] on [dateof:3.6.0] -** and yet the iVersion field was not modified. +** and yet the iVersion field was not increased. ** ** The szOsFile field is the size of the subclassed [sqlite3_file] ** structure used by this VFS. mxPathname is the maximum length of @@ -2274,7 +2342,7 @@ typedef struct sqlite3_api_routines sqlite3_api_routines; ** for exclusive access. ** ** ^At least szOsFile bytes of memory are allocated by SQLite -** to hold the [sqlite3_file] structure passed as the third +** to hold the [sqlite3_file] structure passed as the third ** argument to xOpen. The xOpen method does not have to ** allocate the structure; it should just fill it in. Note that ** the xOpen method must set the sqlite3_file.pMethods to either @@ -2287,8 +2355,14 @@ typedef struct sqlite3_api_routines sqlite3_api_routines; ** ^The flags argument to xAccess() may be [SQLITE_ACCESS_EXISTS] ** to test for the existence of a file, or [SQLITE_ACCESS_READWRITE] to ** test whether a file is readable and writable, or [SQLITE_ACCESS_READ] -** to test whether a file is at least readable. The file can be a -** directory. +** to test whether a file is at least readable. The SQLITE_ACCESS_READ +** flag is never actually used and is not implemented in the built-in +** VFSes of SQLite. The file is named by the second argument and can be a +** directory. The xAccess method returns [SQLITE_OK] on success or some +** non-zero error code if there is an I/O error or if the name of +** the file given in the second argument is illegal. If SQLITE_OK +** is returned, then non-zero or zero is written into *pResOut to indicate +** whether or not the file is accessible. ** ** ^SQLite will always allocate at least mxPathname+1 bytes for the ** output buffer xFullPathname. The exact size of the output buffer @@ -2605,7 +2679,7 @@ SQLITE_API int sqlite3_db_config(sqlite3*, int op, ...); ** that causes the corresponding memory allocation to fail. ** ** The xInit method initializes the memory allocator. For example, -** it might allocate any require mutexes or initialize internal data +** it might allocate any required mutexes or initialize internal data ** structures. The xShutdown method is invoked (indirectly) by ** [sqlite3_shutdown()] and should deallocate any resources acquired ** by xInit. The pAppData pointer is used as the only parameter to @@ -2727,6 +2801,7 @@ struct sqlite3_mem_methods { ** memory allocation statistics. ^(When memory allocation statistics are ** disabled, the following SQLite interfaces become non-operational: **
      +**
    • [sqlite3_hard_heap_limit64()] **
    • [sqlite3_memory_used()] **
    • [sqlite3_memory_highwater()] **
    • [sqlite3_soft_heap_limit64()] @@ -2745,7 +2820,7 @@ struct sqlite3_mem_methods { **
      ^The SQLITE_CONFIG_PAGECACHE option specifies a memory pool ** that SQLite can use for the database page cache with the default page ** cache implementation. -** This configuration option is a no-op if an application-define page +** This configuration option is a no-op if an application-defined page ** cache implementation is loaded using the [SQLITE_CONFIG_PCACHE2]. ** ^There are three arguments to SQLITE_CONFIG_PAGECACHE: A pointer to ** 8-byte aligned memory (pMem), the size of each page cache line (sz), @@ -2976,6 +3051,17 @@ struct sqlite3_mem_methods { ** negative value for this option restores the default behaviour. ** This option is only available if SQLite is compiled with the ** [SQLITE_ENABLE_SORTER_REFERENCES] compile-time option. +** +** [[SQLITE_CONFIG_MEMDB_MAXSIZE]] +**
      SQLITE_CONFIG_MEMDB_MAXSIZE +**
      The SQLITE_CONFIG_MEMDB_MAXSIZE option accepts a single parameter +** [sqlite3_int64] parameter which is the default maximum size for an in-memory +** database created using [sqlite3_deserialize()]. This default maximum +** size can be adjusted up or down for individual databases using the +** [SQLITE_FCNTL_SIZE_LIMIT] [sqlite3_file_control|file-control]. If this +** configuration setting is never used, then the default maximum is determined +** by the [SQLITE_MEMDB_DEFAULT_MAXSIZE] compile-time option. If that +** compile-time option is not set, then the default maximum is 1073741824. ** */ #define SQLITE_CONFIG_SINGLETHREAD 1 /* nil */ @@ -3006,6 +3092,7 @@ struct sqlite3_mem_methods { #define SQLITE_CONFIG_STMTJRNL_SPILL 26 /* int nByte */ #define SQLITE_CONFIG_SMALL_MALLOC 27 /* boolean */ #define SQLITE_CONFIG_SORTERREF_SIZE 28 /* int nByte */ +#define SQLITE_CONFIG_MEMDB_MAXSIZE 29 /* sqlite3_int64 */ /* ** CAPI3REF: Database Connection Configuration Options @@ -3021,6 +3108,7 @@ struct sqlite3_mem_methods { ** is invoked. ** **
      +** [[SQLITE_DBCONFIG_LOOKASIDE]] **
      SQLITE_DBCONFIG_LOOKASIDE
      **
      ^This option takes three additional arguments that determine the ** [lookaside memory allocator] configuration for the [database connection]. @@ -3043,6 +3131,7 @@ struct sqlite3_mem_methods { ** memory is in use leaves the configuration unchanged and returns ** [SQLITE_BUSY].)^
      ** +** [[SQLITE_DBCONFIG_ENABLE_FKEY]] **
      SQLITE_DBCONFIG_ENABLE_FKEY
      **
      ^This option is used to enable or disable the enforcement of ** [foreign key constraints]. There should be two additional arguments. @@ -3053,6 +3142,7 @@ struct sqlite3_mem_methods { ** following this call. The second parameter may be a NULL pointer, in ** which case the FK enforcement setting is not reported back.
      ** +** [[SQLITE_DBCONFIG_ENABLE_TRIGGER]] **
      SQLITE_DBCONFIG_ENABLE_TRIGGER
      **
      ^This option is used to enable or disable [CREATE TRIGGER | triggers]. ** There should be two additional arguments. @@ -3063,9 +3153,21 @@ struct sqlite3_mem_methods { ** following this call. The second parameter may be a NULL pointer, in ** which case the trigger setting is not reported back.
      ** +** [[SQLITE_DBCONFIG_ENABLE_VIEW]] +**
      SQLITE_DBCONFIG_ENABLE_VIEW
      +**
      ^This option is used to enable or disable [CREATE VIEW | views]. +** There should be two additional arguments. +** The first argument is an integer which is 0 to disable views, +** positive to enable views or negative to leave the setting unchanged. +** The second parameter is a pointer to an integer into which +** is written 0 or 1 to indicate whether views are disabled or enabled +** following this call. The second parameter may be a NULL pointer, in +** which case the view setting is not reported back.
      +** +** [[SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER]] **
      SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER
      -**
      ^This option is used to enable or disable the two-argument -** version of the [fts3_tokenizer()] function which is part of the +**
      ^This option is used to enable or disable the +** [fts3_tokenizer()] function which is part of the ** [FTS3] full-text search engine extension. ** There should be two additional arguments. ** The first argument is an integer which is 0 to disable fts3_tokenizer() or @@ -3076,6 +3178,7 @@ struct sqlite3_mem_methods { ** following this call. The second parameter may be a NULL pointer, in ** which case the new setting is not reported back.
      ** +** [[SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION]] **
      SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION
      **
      ^This option is used to enable or disable the [sqlite3_load_extension()] ** interface independently of the [load_extension()] SQL function. @@ -3093,7 +3196,7 @@ struct sqlite3_mem_methods { ** be a NULL pointer, in which case the new setting is not reported back. **
      ** -**
      SQLITE_DBCONFIG_MAINDBNAME
      +** [[SQLITE_DBCONFIG_MAINDBNAME]]
      SQLITE_DBCONFIG_MAINDBNAME
      **
      ^This option is used to change the name of the "main" database ** schema. ^The sole argument is a pointer to a constant UTF8 string ** which will become the new schema name in place of "main". ^SQLite @@ -3102,6 +3205,7 @@ struct sqlite3_mem_methods { ** until after the database connection closes. **
      ** +** [[SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE]] **
      SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE
      **
      Usually, when a database in wal mode is closed or detached from a ** database handle, SQLite checks if this will mean that there are now no @@ -3115,7 +3219,7 @@ struct sqlite3_mem_methods { ** have been disabled - 0 if they are not disabled, 1 if they are. **
      ** -**
      SQLITE_DBCONFIG_ENABLE_QPSG
      +** [[SQLITE_DBCONFIG_ENABLE_QPSG]]
      SQLITE_DBCONFIG_ENABLE_QPSG
      **
      ^(The SQLITE_DBCONFIG_ENABLE_QPSG option activates or deactivates ** the [query planner stability guarantee] (QPSG). When the QPSG is active, ** a single SQL query statement will always use the same algorithm regardless @@ -3131,7 +3235,7 @@ struct sqlite3_mem_methods { ** following this call. **
      ** -**
      SQLITE_DBCONFIG_TRIGGER_EQP
      +** [[SQLITE_DBCONFIG_TRIGGER_EQP]]
      SQLITE_DBCONFIG_TRIGGER_EQP
      **
      By default, the output of EXPLAIN QUERY PLAN commands does not ** include output for any operations performed by trigger programs. This ** option is used to set or clear (the default) a flag that governs this @@ -3143,12 +3247,18 @@ struct sqlite3_mem_methods { ** it is not disabled, 1 if it is. **
      ** -**
      SQLITE_DBCONFIG_RESET_DATABASE
      +** [[SQLITE_DBCONFIG_RESET_DATABASE]]
      SQLITE_DBCONFIG_RESET_DATABASE
      **
      Set the SQLITE_DBCONFIG_RESET_DATABASE flag and then run ** [VACUUM] in order to reset a database back to an empty database ** with no schema and no content. The following process works even for ** a badly corrupted database file: **
        +**
      1. If the database connection is newly opened, make sure it has read the +** database schema by preparing then discarding some query against the +** database, or calling sqlite3_table_column_metadata(), ignoring any +** errors. This step is only necessary if the application desires to keep +** the database in WAL mode after the reset if it was in WAL mode before +** the reset. **
      2. sqlite3_db_config(db, SQLITE_DBCONFIG_RESET_DATABASE, 1, 0); **
      3. [sqlite3_exec](db, "[VACUUM]", 0, 0, 0); **
      4. sqlite3_db_config(db, SQLITE_DBCONFIG_RESET_DATABASE, 0, 0); @@ -3156,6 +3266,101 @@ struct sqlite3_mem_methods { ** Because resetting a database is destructive and irreversible, the ** process requires the use of this obscure API and multiple steps to help ** ensure that it does not happen by accident. +** +** [[SQLITE_DBCONFIG_DEFENSIVE]]
        SQLITE_DBCONFIG_DEFENSIVE
        +**
        The SQLITE_DBCONFIG_DEFENSIVE option activates or deactivates the +** "defensive" flag for a database connection. When the defensive +** flag is enabled, language features that allow ordinary SQL to +** deliberately corrupt the database file are disabled. The disabled +** features include but are not limited to the following: +**
          +**
        • The [PRAGMA writable_schema=ON] statement. +**
        • The [PRAGMA journal_mode=OFF] statement. +**
        • Writes to the [sqlite_dbpage] virtual table. +**
        • Direct writes to [shadow tables]. +**
        +**
        +** +** [[SQLITE_DBCONFIG_WRITABLE_SCHEMA]]
        SQLITE_DBCONFIG_WRITABLE_SCHEMA
        +**
        The SQLITE_DBCONFIG_WRITABLE_SCHEMA option activates or deactivates the +** "writable_schema" flag. This has the same effect and is logically equivalent +** to setting [PRAGMA writable_schema=ON] or [PRAGMA writable_schema=OFF]. +** The first argument to this setting is an integer which is 0 to disable +** the writable_schema, positive to enable writable_schema, or negative to +** leave the setting unchanged. The second parameter is a pointer to an +** integer into which is written 0 or 1 to indicate whether the writable_schema +** is enabled or disabled following this call. +**
        +** +** [[SQLITE_DBCONFIG_LEGACY_ALTER_TABLE]] +**
        SQLITE_DBCONFIG_LEGACY_ALTER_TABLE
        +**
        The SQLITE_DBCONFIG_LEGACY_ALTER_TABLE option activates or deactivates +** the legacy behavior of the [ALTER TABLE RENAME] command such it +** behaves as it did prior to [version 3.24.0] (2018-06-04). See the +** "Compatibility Notice" on the [ALTER TABLE RENAME documentation] for +** additional information. This feature can also be turned on and off +** using the [PRAGMA legacy_alter_table] statement. +**
        +** +** [[SQLITE_DBCONFIG_DQS_DML]] +**
        SQLITE_DBCONFIG_DQS_DML +**
        The SQLITE_DBCONFIG_DQS_DML option activates or deactivates +** the legacy [double-quoted string literal] misfeature for DML statements +** only, that is DELETE, INSERT, SELECT, and UPDATE statements. The +** default value of this setting is determined by the [-DSQLITE_DQS] +** compile-time option. +**
        +** +** [[SQLITE_DBCONFIG_DQS_DDL]] +**
        SQLITE_DBCONFIG_DQS_DDL +**
        The SQLITE_DBCONFIG_DQS option activates or deactivates +** the legacy [double-quoted string literal] misfeature for DDL statements, +** such as CREATE TABLE and CREATE INDEX. The +** default value of this setting is determined by the [-DSQLITE_DQS] +** compile-time option. +**
        +** +** [[SQLITE_DBCONFIG_TRUSTED_SCHEMA]] +**
        SQLITE_DBCONFIG_TRUSTED_SCHEMA +**
        The SQLITE_DBCONFIG_TRUSTED_SCHEMA option tells SQLite to +** assume that database schemas (the contents of the [sqlite_master] tables) +** are untainted by malicious content. +** When the SQLITE_DBCONFIG_TRUSTED_SCHEMA option is disabled, SQLite +** takes additional defensive steps to protect the application from harm +** including: +**
          +**
        • Prohibit the use of SQL functions inside triggers, views, +** CHECK constraints, DEFAULT clauses, expression indexes, +** partial indexes, or generated columns +** unless those functions are tagged with [SQLITE_INNOCUOUS]. +**
        • Prohibit the use of virtual tables inside of triggers or views +** unless those virtual tables are tagged with [SQLITE_VTAB_INNOCUOUS]. +**
        +** This setting defaults to "on" for legacy compatibility, however +** all applications are advised to turn it off if possible. This setting +** can also be controlled using the [PRAGMA trusted_schema] statement. +**
        +** +** [[SQLITE_DBCONFIG_LEGACY_FILE_FORMAT]] +**
        SQLITE_DBCONFIG_LEGACY_FILE_FORMAT +**
        The SQLITE_DBCONFIG_LEGACY_FILE_FORMAT option activates or deactivates +** the legacy file format flag. When activated, this flag causes all newly +** created database file to have a schema format version number (the 4-byte +** integer found at offset 44 into the database header) of 1. This in turn +** means that the resulting database file will be readable and writable by +** any SQLite version back to 3.0.0 ([dateof:3.0.0]). Without this setting, +** newly created databases are generally not understandable by SQLite versions +** prior to 3.3.0 ([dateof:3.3.0]). As these words are written, there +** is now scarcely any need to generated database files that are compatible +** all the way back to version 3.0.0, and so this setting is of little +** practical use, but is provided so that SQLite can continue to claim the +** ability to generate new database files that are compatible with version +** 3.0.0. +**

        Note that when the SQLITE_DBCONFIG_LEGACY_FILE_FORMAT setting is on, +** the [VACUUM] command will fail with an obscure error when attempting to +** process a table with generated columns and a descending index. This is +** not considered a bug since SQLite versions 3.3.0 and earlier do not support +** either generated columns or decending indexes. **

        **
      */ @@ -3169,7 +3374,15 @@ struct sqlite3_mem_methods { #define SQLITE_DBCONFIG_ENABLE_QPSG 1007 /* int int* */ #define SQLITE_DBCONFIG_TRIGGER_EQP 1008 /* int int* */ #define SQLITE_DBCONFIG_RESET_DATABASE 1009 /* int int* */ -#define SQLITE_DBCONFIG_MAX 1009 /* Largest DBCONFIG */ +#define SQLITE_DBCONFIG_DEFENSIVE 1010 /* int int* */ +#define SQLITE_DBCONFIG_WRITABLE_SCHEMA 1011 /* int int* */ +#define SQLITE_DBCONFIG_LEGACY_ALTER_TABLE 1012 /* int int* */ +#define SQLITE_DBCONFIG_DQS_DML 1013 /* int int* */ +#define SQLITE_DBCONFIG_DQS_DDL 1014 /* int int* */ +#define SQLITE_DBCONFIG_ENABLE_VIEW 1015 /* int int* */ +#define SQLITE_DBCONFIG_LEGACY_FILE_FORMAT 1016 /* int int* */ +#define SQLITE_DBCONFIG_TRUSTED_SCHEMA 1017 /* int int* */ +#define SQLITE_DBCONFIG_MAX 1017 /* Largest DBCONFIG */ /* ** CAPI3REF: Enable Or Disable Extended Result Codes @@ -3297,12 +3510,17 @@ SQLITE_API void sqlite3_set_last_insert_rowid(sqlite3*,sqlite3_int64); ** program, the value returned reflects the number of rows modified by the ** previous INSERT, UPDATE or DELETE statement within the same trigger. ** -** See also the [sqlite3_total_changes()] interface, the -** [count_changes pragma], and the [changes() SQL function]. -** ** If a separate thread makes changes on the same database connection ** while [sqlite3_changes()] is running then the value returned ** is unpredictable and not meaningful. +** +** See also: +**
        +**
      • the [sqlite3_total_changes()] interface +**
      • the [count_changes pragma] +**
      • the [changes() SQL function] +**
      • the [data_version pragma] +**
      */ SQLITE_API int sqlite3_changes(sqlite3*); @@ -3320,13 +3538,26 @@ SQLITE_API int sqlite3_changes(sqlite3*); ** count, but those made as part of REPLACE constraint resolution are ** not. ^Changes to a view that are intercepted by INSTEAD OF triggers ** are not counted. -** -** See also the [sqlite3_changes()] interface, the -** [count_changes pragma], and the [total_changes() SQL function]. ** +** The [sqlite3_total_changes(D)] interface only reports the number +** of rows that changed due to SQL statement run against database +** connection D. Any changes by other database connections are ignored. +** To detect changes against a database file from other database +** connections use the [PRAGMA data_version] command or the +** [SQLITE_FCNTL_DATA_VERSION] [file control]. +** ** If a separate thread makes changes on the same database connection ** while [sqlite3_total_changes()] is running then the value ** returned is unpredictable and not meaningful. +** +** See also: +**
        +**
      • the [sqlite3_changes()] interface +**
      • the [count_changes pragma] +**
      • the [changes() SQL function] +**
      • the [data_version pragma] +**
      • the [SQLITE_FCNTL_DATA_VERSION] [file control] +**
      */ SQLITE_API int sqlite3_total_changes(sqlite3*); @@ -3357,7 +3588,7 @@ SQLITE_API int sqlite3_total_changes(sqlite3*); ** ^The sqlite3_interrupt(D) call is in effect until all currently running ** SQL statements on [database connection] D complete. ^Any new SQL statements ** that are started after the sqlite3_interrupt() call and before the -** running statements reaches zero are interrupted as if they had been +** running statement count reaches zero are interrupted as if they had been ** running prior to the sqlite3_interrupt() call. ^New SQL statements ** that are started after the running statement count reaches zero are ** not effected by the sqlite3_interrupt(). @@ -3525,9 +3756,9 @@ SQLITE_API int sqlite3_busy_timeout(sqlite3*, int ms); ** Cindy | 21 ** ** -** There are two column (M==2) and three rows (N==3). Thus the +** There are two columns (M==2) and three rows (N==3). Thus the ** result table has 8 entries. Suppose the result table is stored -** in an array names azResult. Then azResult holds this content: +** in an array named azResult. Then azResult holds this content: ** ** 
       **        azResult[0] = "Name";
@@ -3620,7 +3851,7 @@ SQLITE_API char *sqlite3_vsnprintf(int,char*,const char*, va_list);
 **
 ** The SQLite core uses these three routines for all of its own
 ** internal memory allocation needs. "Core" in the previous sentence
-** does not include operating-system specific VFS implementation.  The
+** does not include operating-system specific [VFS] implementation.  The
 ** Windows VFS uses native malloc() and free() for some operations.
 **
 ** ^The sqlite3_malloc() routine returns a pointer to a block
@@ -3681,19 +3912,6 @@ SQLITE_API char *sqlite3_vsnprintf(int,char*,const char*, va_list);
 ** 4 byte boundary if the [SQLITE_4_BYTE_ALIGNED_MALLOC] compile-time
 ** option is used.
 **
-** In SQLite version 3.5.0 and 3.5.1, it was possible to define
-** the SQLITE_OMIT_MEMORY_ALLOCATION which would cause the built-in
-** implementation of these routines to be omitted.  That capability
-** is no longer provided.  Only built-in memory allocators can be used.
-**
-** Prior to SQLite version 3.7.10, the Windows OS interface layer called
-** the system malloc() and free() directly when converting
-** filenames between the UTF-8 encoding used by SQLite
-** and whatever filename encoding is used by the particular Windows
-** installation.  Memory allocation errors were detected, but
-** they were reported back as [SQLITE_CANTOPEN] or
-** [SQLITE_IOERR] rather than [SQLITE_NOMEM].
-**
 ** The pointer arguments to [sqlite3_free()] and [sqlite3_realloc()]
 ** must be either NULL or else pointers obtained from a prior
 ** invocation of [sqlite3_malloc()] or [sqlite3_realloc()] that have
@@ -3742,7 +3960,7 @@ SQLITE_API sqlite3_int64 sqlite3_memory_highwater(int resetFlag);
 ** SQLite contains a high-quality pseudo-random number generator (PRNG) used to
 ** select random [ROWID | ROWIDs] when inserting new records into a table that
 ** already uses the largest possible [ROWID].  The PRNG is also used for
-** the build-in random() and randomblob() SQL functions.  This interface allows
+** the built-in random() and randomblob() SQL functions.  This interface allows
 ** applications to access the same PRNG for other purposes.
 **
 ** ^A call to this routine stores N bytes of randomness into buffer P.
@@ -3952,9 +4170,9 @@ SQLITE_API int sqlite3_set_authorizer(
 ** time is in units of nanoseconds, however the current implementation
 ** is only capable of millisecond resolution so the six least significant
 ** digits in the time are meaningless.  Future versions of SQLite
-** might provide greater resolution on the profiler callback.  The
-** sqlite3_profile() function is considered experimental and is
-** subject to change in future versions of SQLite.
+** might provide greater resolution on the profiler callback.  Invoking
+** either [sqlite3_trace()] or [sqlite3_trace_v2()] will cancel the
+** profile callback.
 */
 SQLITE_API SQLITE_DEPRECATED void *sqlite3_trace(sqlite3*,
    void(*xTrace)(void*,const char*), void*);
@@ -4116,10 +4334,8 @@ SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*);
 ** The sqlite3_open_v2() interface works like sqlite3_open()
 ** except that it accepts two additional parameters for additional control
 ** over the new database connection.  ^(The flags parameter to
-** sqlite3_open_v2() can take one of
-** the following three values, optionally combined with the 
-** [SQLITE_OPEN_NOMUTEX], [SQLITE_OPEN_FULLMUTEX], [SQLITE_OPEN_SHAREDCACHE],
-** [SQLITE_OPEN_PRIVATECACHE], and/or [SQLITE_OPEN_URI] flags:)^
+** sqlite3_open_v2() must include, at a minimum, one of the following
+** three flag combinations:)^
 **
 ** 
      
 ** ^(
      [SQLITE_OPEN_READONLY]
      
@@ -4137,23 +4353,51 @@ SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*);
 ** sqlite3_open() and sqlite3_open16().
      )^ ** ** +** In addition to the required flags, the following optional flags are +** also supported: +** +**
      +** ^(
      [SQLITE_OPEN_URI]
      +**
      The filename can be interpreted as a URI if this flag is set.
      )^ +** +** ^(
      [SQLITE_OPEN_MEMORY]
      +**
      The database will be opened as an in-memory database. The database +** is named by the "filename" argument for the purposes of cache-sharing, +** if shared cache mode is enabled, but the "filename" is otherwise ignored. +**
      )^ +** +** ^(
      [SQLITE_OPEN_NOMUTEX]
      +**
      The new database connection will use the "multi-thread" +** [threading mode].)^ This means that separate threads are allowed +** to use SQLite at the same time, as long as each thread is using +** a different [database connection]. +** +** ^(
      [SQLITE_OPEN_FULLMUTEX]
      +**
      The new database connection will use the "serialized" +** [threading mode].)^ This means the multiple threads can safely +** attempt to use the same database connection at the same time. +** (Mutexes will block any actual concurrency, but in this mode +** there is no harm in trying.) +** +** ^(
      [SQLITE_OPEN_SHAREDCACHE]
      +**
      The database is opened [shared cache] enabled, overriding +** the default shared cache setting provided by +** [sqlite3_enable_shared_cache()].)^ +** +** ^(
      [SQLITE_OPEN_PRIVATECACHE]
      +**
      The database is opened [shared cache] disabled, overriding +** the default shared cache setting provided by +** [sqlite3_enable_shared_cache()].)^ +** +** [[OPEN_NOFOLLOW]] ^(
      [SQLITE_OPEN_NOFOLLOW]
      +**
      The database filename is not allowed to be a symbolic link
      +**
      )^ +** ** If the 3rd parameter to sqlite3_open_v2() is not one of the -** combinations shown above optionally combined with other +** required combinations shown above optionally combined with other ** [SQLITE_OPEN_READONLY | SQLITE_OPEN_* bits] ** then the behavior is undefined. ** -** ^If the [SQLITE_OPEN_NOMUTEX] flag is set, then the database connection -** opens in the multi-thread [threading mode] as long as the single-thread -** mode has not been set at compile-time or start-time. ^If the -** [SQLITE_OPEN_FULLMUTEX] flag is set then the database connection opens -** in the serialized [threading mode] unless single-thread was -** previously selected at compile-time or start-time. -** ^The [SQLITE_OPEN_SHAREDCACHE] flag causes the database connection to be -** eligible to use [shared cache mode], regardless of whether or not shared -** cache is enabled using [sqlite3_enable_shared_cache()]. ^The -** [SQLITE_OPEN_PRIVATECACHE] flag causes the database connection to not -** participate in [shared cache mode] even if it is enabled. -** ** ^The fourth parameter to sqlite3_open_v2() is the name of the ** [sqlite3_vfs] object that defines the operating system interface that ** the new database connection should use. ^If the fourth parameter is @@ -4333,17 +4577,27 @@ SQLITE_API int sqlite3_open_v2( /* ** CAPI3REF: Obtain Values For URI Parameters ** -** These are utility routines, useful to VFS implementations, that check -** to see if a database file was a URI that contained a specific query +** These are utility routines, useful to [VFS|custom VFS implementations], +** that check if a database file was a URI that contained a specific query ** parameter, and if so obtains the value of that query parameter. ** -** If F is the database filename pointer passed into the xOpen() method of -** a VFS implementation when the flags parameter to xOpen() has one or -** more of the [SQLITE_OPEN_URI] or [SQLITE_OPEN_MAIN_DB] bits set and -** P is the name of the query parameter, then +** The first parameter to these interfaces (hereafter referred to +** as F) must be one of: +**
        +**
      • A database filename pointer created by the SQLite core and +** passed into the xOpen() method of a VFS implemention, or +**
      • A filename obtained from [sqlite3_db_filename()], or +**
      • A new filename constructed using [sqlite3_create_filename()]. +**
      +** If the F parameter is not one of the above, then the behavior is +** undefined and probably undesirable. Older versions of SQLite were +** more tolerant of invalid F parameters than newer versions. +** +** If F is a suitable filename (as described in the previous paragraph) +** and if P is the name of the query parameter, then ** sqlite3_uri_parameter(F,P) returns the value of the P ** parameter if it exists or a NULL pointer if P does not appear as a -** query parameter on F. If P is a query parameter of F +** query parameter on F. If P is a query parameter of F and it ** has no explicit value, then sqlite3_uri_parameter(F,P) returns ** a pointer to an empty string. ** @@ -4355,24 +4609,145 @@ SQLITE_API int sqlite3_open_v2( ** sqlite3_uri_boolean(F,P,B) routines returns false (0) if the value of ** query parameter P is one of "no", "false", or "off" in any case or ** if the value begins with a numeric zero. If P is not a query -** parameter on F or if the value of P is does not match any of the +** parameter on F or if the value of P does not match any of the ** above, then sqlite3_uri_boolean(F,P,B) returns (B!=0). ** ** The sqlite3_uri_int64(F,P,D) routine converts the value of P into a ** 64-bit signed integer and returns that integer, or D if P does not ** exist. If the value of P is something other than an integer, then ** zero is returned. +** +** The sqlite3_uri_key(F,N) returns a pointer to the name (not +** the value) of the N-th query parameter for filename F, or a NULL +** pointer if N is less than zero or greater than the number of query +** parameters minus 1. The N value is zero-based so N should be 0 to obtain +** the name of the first query parameter, 1 for the second parameter, and +** so forth. ** ** If F is a NULL pointer, then sqlite3_uri_parameter(F,P) returns NULL and ** sqlite3_uri_boolean(F,P,B) returns B. If F is not a NULL pointer and -** is not a database file pathname pointer that SQLite passed into the xOpen -** VFS method, then the behavior of this routine is undefined and probably -** undesirable. +** is not a database file pathname pointer that the SQLite core passed +** into the xOpen VFS method, then the behavior of this routine is undefined +** and probably undesirable. +** +** Beginning with SQLite [version 3.31.0] ([dateof:3.31.0]) the input F +** parameter can also be the name of a rollback journal file or WAL file +** in addition to the main database file. Prior to version 3.31.0, these +** routines would only work if F was the name of the main database file. +** When the F parameter is the name of the rollback journal or WAL file, +** it has access to all the same query parameters as were found on the +** main database file. +** +** See the [URI filename] documentation for additional information. */ SQLITE_API const char *sqlite3_uri_parameter(const char *zFilename, const char *zParam); SQLITE_API int sqlite3_uri_boolean(const char *zFile, const char *zParam, int bDefault); SQLITE_API sqlite3_int64 sqlite3_uri_int64(const char*, const char*, sqlite3_int64); +SQLITE_API const char *sqlite3_uri_key(const char *zFilename, int N); + +/* +** CAPI3REF: Translate filenames +** +** These routines are available to [VFS|custom VFS implementations] for +** translating filenames between the main database file, the journal file, +** and the WAL file. +** +** If F is the name of an sqlite database file, journal file, or WAL file +** passed by the SQLite core into the VFS, then sqlite3_filename_database(F) +** returns the name of the corresponding database file. +** +** If F is the name of an sqlite database file, journal file, or WAL file +** passed by the SQLite core into the VFS, or if F is a database filename +** obtained from [sqlite3_db_filename()], then sqlite3_filename_journal(F) +** returns the name of the corresponding rollback journal file. +** +** If F is the name of an sqlite database file, journal file, or WAL file +** that was passed by the SQLite core into the VFS, or if F is a database +** filename obtained from [sqlite3_db_filename()], then +** sqlite3_filename_wal(F) returns the name of the corresponding +** WAL file. +** +** In all of the above, if F is not the name of a database, journal or WAL +** filename passed into the VFS from the SQLite core and F is not the +** return value from [sqlite3_db_filename()], then the result is +** undefined and is likely a memory access violation. +*/ +SQLITE_API const char *sqlite3_filename_database(const char*); +SQLITE_API const char *sqlite3_filename_journal(const char*); +SQLITE_API const char *sqlite3_filename_wal(const char*); +/* +** CAPI3REF: Database File Corresponding To A Journal +** +** ^If X is the name of a rollback or WAL-mode journal file that is +** passed into the xOpen method of [sqlite3_vfs], then +** sqlite3_database_file_object(X) returns a pointer to the [sqlite3_file] +** object that represents the main database file. +** +** This routine is intended for use in custom [VFS] implementations +** only. It is not a general-purpose interface. +** The argument sqlite3_file_object(X) must be a filename pointer that +** has been passed into [sqlite3_vfs].xOpen method where the +** flags parameter to xOpen contains one of the bits +** [SQLITE_OPEN_MAIN_JOURNAL] or [SQLITE_OPEN_WAL]. Any other use +** of this routine results in undefined and probably undesirable +** behavior. +*/ +SQLITE_API sqlite3_file *sqlite3_database_file_object(const char*); + +/* +** CAPI3REF: Create and Destroy VFS Filenames +** +** These interfces are provided for use by [VFS shim] implementations and +** are not useful outside of that context. +** +** The sqlite3_create_filename(D,J,W,N,P) allocates memory to hold a version of +** database filename D with corresponding journal file J and WAL file W and +** with N URI parameters key/values pairs in the array P. The result from +** sqlite3_create_filename(D,J,W,N,P) is a pointer to a database filename that +** is safe to pass to routines like: +**
        +**
      • [sqlite3_uri_parameter()], +**
      • [sqlite3_uri_boolean()], +**
      • [sqlite3_uri_int64()], +**
      • [sqlite3_uri_key()], +**
      • [sqlite3_filename_database()], +**
      • [sqlite3_filename_journal()], or +**
      • [sqlite3_filename_wal()]. +**
      +** If a memory allocation error occurs, sqlite3_create_filename() might +** return a NULL pointer. The memory obtained from sqlite3_create_filename(X) +** must be released by a corresponding call to sqlite3_free_filename(Y). +** +** The P parameter in sqlite3_create_filename(D,J,W,N,P) should be an array +** of 2*N pointers to strings. Each pair of pointers in this array corresponds +** to a key and value for a query parameter. The P parameter may be a NULL +** pointer if N is zero. None of the 2*N pointers in the P array may be +** NULL pointers and key pointers should not be empty strings. +** None of the D, J, or W parameters to sqlite3_create_filename(D,J,W,N,P) may +** be NULL pointers, though they can be empty strings. +** +** The sqlite3_free_filename(Y) routine releases a memory allocation +** previously obtained from sqlite3_create_filename(). Invoking +** sqlite3_free_filename(Y) where Y is a NULL pointer is a harmless no-op. +** +** If the Y parameter to sqlite3_free_filename(Y) is anything other +** than a NULL pointer or a pointer previously acquired from +** sqlite3_create_filename(), then bad things such as heap +** corruption or segfaults may occur. The value Y should be +** used again after sqlite3_free_filename(Y) has been called. This means +** that if the [sqlite3_vfs.xOpen()] method of a VFS has been called using Y, +** then the corresponding [sqlite3_module.xClose() method should also be +** invoked prior to calling sqlite3_free_filename(Y). +*/ +SQLITE_API char *sqlite3_create_filename( + const char *zDatabase, + const char *zJournal, + const char *zWal, + int nParam, + const char **azParam +); +SQLITE_API void sqlite3_free_filename(char*); /* ** CAPI3REF: Error Codes And Messages @@ -4382,13 +4757,24 @@ SQLITE_API sqlite3_int64 sqlite3_uri_int64(const char*, const char*, sqlite3_int ** [database connection] D failed, then the sqlite3_errcode(D) interface ** returns the numeric [result code] or [extended result code] for that ** API call. -** If the most recent API call was successful, -** then the return value from sqlite3_errcode() is undefined. ** ^The sqlite3_extended_errcode() ** interface is the same except that it always returns the ** [extended result code] even when extended result codes are ** disabled. ** +** The values returned by sqlite3_errcode() and/or +** sqlite3_extended_errcode() might change with each API call. +** Except, there are some interfaces that are guaranteed to never +** change the value of the error code. The error-code preserving +** interfaces are: +** +**
        +**
      • sqlite3_errcode() +**
      • sqlite3_extended_errcode() +**
      • sqlite3_errmsg() +**
      • sqlite3_errmsg16() +**
      +** ** ^The sqlite3_errmsg() and sqlite3_errmsg16() return English-language ** text that describes the error, as either UTF-8 or UTF-16 respectively. ** ^(Memory to hold the error message string is managed internally. @@ -4578,9 +4964,24 @@ SQLITE_API int sqlite3_limit(sqlite3*, int id, int newVal); ** on this hint by avoiding the use of [lookaside memory] so as not to ** deplete the limited store of lookaside memory. Future versions of ** SQLite may act on this hint differently. +** +** [[SQLITE_PREPARE_NORMALIZE]]
      SQLITE_PREPARE_NORMALIZE
      +**
      The SQLITE_PREPARE_NORMALIZE flag is a no-op. This flag used +** to be required for any prepared statement that wanted to use the +** [sqlite3_normalized_sql()] interface. However, the +** [sqlite3_normalized_sql()] interface is now available to all +** prepared statements, regardless of whether or not they use this +** flag. +** +** [[SQLITE_PREPARE_NO_VTAB]]
      SQLITE_PREPARE_NO_VTAB
      +**
      The SQLITE_PREPARE_NO_VTAB flag causes the SQL compiler +** to return an error (error code SQLITE_ERROR) if the statement uses +** any virtual tables. ** */ #define SQLITE_PREPARE_PERSISTENT 0x01 +#define SQLITE_PREPARE_NORMALIZE 0x02 +#define SQLITE_PREPARE_NO_VTAB 0x04 /* ** CAPI3REF: Compiling An SQL Statement @@ -4664,15 +5065,15 @@ SQLITE_API int sqlite3_limit(sqlite3*, int id, int newVal); **
      • ** **
    • -** ^If the specific value bound to [parameter | host parameter] in the +** ^If the specific value bound to a [parameter | host parameter] in the ** WHERE clause might influence the choice of query plan for a statement, ** then the statement will be automatically recompiled, as if there had been -** a schema change, on the first [sqlite3_step()] call following any change +** a schema change, on the first [sqlite3_step()] call following any change ** to the [sqlite3_bind_text | bindings] of that [parameter]. -** ^The specific value of WHERE-clause [parameter] might influence the +** ^The specific value of a WHERE-clause [parameter] might influence the ** choice of query plan if the parameter is the left-hand side of a [LIKE] ** or [GLOB] operator or if the parameter is compared to an indexed column -** and the [SQLITE_ENABLE_STAT3] compile-time option is enabled. +** and the [SQLITE_ENABLE_STAT4] compile-time option is enabled. **
      • ** ** @@ -4738,6 +5139,11 @@ SQLITE_API int sqlite3_prepare16_v3( ** ^The sqlite3_expanded_sql(P) interface returns a pointer to a UTF-8 ** string containing the SQL text of prepared statement P with ** [bound parameters] expanded. +** ^The sqlite3_normalized_sql(P) interface returns a pointer to a UTF-8 +** string containing the normalized SQL text of prepared statement P. The +** semantics used to normalize a SQL statement are unspecified and subject +** to change. At a minimum, literal values will be replaced with suitable +** placeholders. ** ** ^(For example, if a prepared statement is created using the SQL ** text "SELECT $abc,:xyz" and if parameter $abc is bound to integer 2345 @@ -4753,14 +5159,16 @@ SQLITE_API int sqlite3_prepare16_v3( ** bound parameter expansions. ^The [SQLITE_OMIT_TRACE] compile-time ** option causes sqlite3_expanded_sql() to always return NULL. ** -** ^The string returned by sqlite3_sql(P) is managed by SQLite and is -** automatically freed when the prepared statement is finalized. +** ^The strings returned by sqlite3_sql(P) and sqlite3_normalized_sql(P) +** are managed by SQLite and are automatically freed when the prepared +** statement is finalized. ** ^The string returned by sqlite3_expanded_sql(P), on the other hand, ** is obtained from [sqlite3_malloc()] and must be free by the application ** by passing it to [sqlite3_free()]. */ SQLITE_API const char *sqlite3_sql(sqlite3_stmt *pStmt); SQLITE_API char *sqlite3_expanded_sql(sqlite3_stmt *pStmt); +SQLITE_API const char *sqlite3_normalized_sql(sqlite3_stmt *pStmt); /* ** CAPI3REF: Determine If An SQL Statement Writes The Database @@ -4798,6 +5206,18 @@ SQLITE_API char *sqlite3_expanded_sql(sqlite3_stmt *pStmt); */ SQLITE_API int sqlite3_stmt_readonly(sqlite3_stmt *pStmt); +/* +** CAPI3REF: Query The EXPLAIN Setting For A Prepared Statement +** METHOD: sqlite3_stmt +** +** ^The sqlite3_stmt_isexplain(S) interface returns 1 if the +** prepared statement S is an EXPLAIN statement, or 2 if the +** statement S is an EXPLAIN QUERY PLAN. +** ^The sqlite3_stmt_isexplain(S) interface returns 0 if S is +** an ordinary statement or a NULL pointer. +*/ +SQLITE_API int sqlite3_stmt_isexplain(sqlite3_stmt *pStmt); + /* ** CAPI3REF: Determine If A Prepared Statement Has Been Reset ** METHOD: sqlite3_stmt @@ -4910,12 +5330,30 @@ typedef struct sqlite3_context sqlite3_context; ** [sqlite3_bind_parameter_index()] API if desired. ^The index ** for "?NNN" parameters is the value of NNN. ** ^The NNN value must be between 1 and the [sqlite3_limit()] -** parameter [SQLITE_LIMIT_VARIABLE_NUMBER] (default value: 999). +** parameter [SQLITE_LIMIT_VARIABLE_NUMBER] (default value: 32766). ** ** ^The third argument is the value to bind to the parameter. ** ^If the third parameter to sqlite3_bind_text() or sqlite3_bind_text16() ** or sqlite3_bind_blob() is a NULL pointer then the fourth parameter ** is ignored and the end result is the same as sqlite3_bind_null(). +** ^If the third parameter to sqlite3_bind_text() is not NULL, then +** it should be a pointer to well-formed UTF8 text. +** ^If the third parameter to sqlite3_bind_text16() is not NULL, then +** it should be a pointer to well-formed UTF16 text. +** ^If the third parameter to sqlite3_bind_text64() is not NULL, then +** it should be a pointer to a well-formed unicode string that is +** either UTF8 if the sixth parameter is SQLITE_UTF8, or UTF16 +** otherwise. +** +** [[byte-order determination rules]] ^The byte-order of +** UTF16 input text is determined by the byte-order mark (BOM, U+FEFF) +** found in first character, which is removed, or in the absence of a BOM +** the byte order is the native byte order of the host +** machine for sqlite3_bind_text16() or the byte order specified in +** the 6th parameter for sqlite3_bind_text64().)^ +** ^If UTF16 input text contains invalid unicode +** characters, then SQLite might change those invalid characters +** into the unicode replacement character: U+FFFD. ** ** ^(In those routines that have a fourth argument, its value is the ** number of bytes in the parameter. To be clear: the value is the @@ -4929,7 +5367,7 @@ typedef struct sqlite3_context sqlite3_context; ** or sqlite3_bind_text16() or sqlite3_bind_text64() then ** that parameter must be the byte offset ** where the NUL terminator would occur assuming the string were NUL -** terminated. If any NUL characters occur at byte offsets less than +** terminated. If any NUL characters occurs at byte offsets less than ** the value of the fourth parameter then the resulting string value will ** contain embedded NULs. The result of expressions involving strings ** with embedded NULs is undefined. @@ -4937,7 +5375,9 @@ typedef struct sqlite3_context sqlite3_context; ** ^The fifth argument to the BLOB and string binding interfaces ** is a destructor used to dispose of the BLOB or ** string after SQLite has finished with it. ^The destructor is called -** to dispose of the BLOB or string even if the call to bind API fails. +** to dispose of the BLOB or string even if the call to the bind API fails, +** except the destructor is not called if the third parameter is a NULL +** pointer or the fourth parameter is negative. ** ^If the fifth argument is ** the special value [SQLITE_STATIC], then SQLite assumes that the ** information is in static, unmanaged space and does not need to be freed. @@ -5157,7 +5597,7 @@ SQLITE_API const void *sqlite3_column_name16(sqlite3_stmt*, int N); ** ** ^If the Nth column returned by the statement is an expression or ** subquery and is not a column value, then all of these functions return -** NULL. ^These routine might also return NULL if a memory allocation error +** NULL. ^These routines might also return NULL if a memory allocation error ** occurs. ^Otherwise, they return the name of the attached database, table, ** or column that query result column was extracted from. ** @@ -5167,10 +5607,6 @@ SQLITE_API const void *sqlite3_column_name16(sqlite3_stmt*, int N); ** ^These APIs are only available if the library was compiled with the ** [SQLITE_ENABLE_COLUMN_METADATA] C-preprocessor symbol. ** -** If two or more threads call one or more of these routines against the same -** prepared statement and column at the same time then the results are -** undefined. -** ** If two or more threads call one or more ** [sqlite3_column_database_name | column metadata interfaces] ** for the same [prepared statement] and result column @@ -5307,7 +5743,7 @@ SQLITE_API int sqlite3_step(sqlite3_stmt*); ** ^The sqlite3_data_count(P) interface returns the number of columns in the ** current row of the result set of [prepared statement] P. ** ^If prepared statement P does not have results ready to return -** (via calls to the [sqlite3_column_int | sqlite3_column_*()] of +** (via calls to the [sqlite3_column_int | sqlite3_column()] family of ** interfaces) then sqlite3_data_count(P) returns 0. ** ^The sqlite3_data_count(P) routine also returns 0 if P is a NULL pointer. ** ^The sqlite3_data_count(P) routine returns 0 if the previous call to @@ -5542,11 +5978,25 @@ SQLITE_API int sqlite3_data_count(sqlite3_stmt *pStmt); ** from [sqlite3_column_blob()], [sqlite3_column_text()], etc. into ** [sqlite3_free()]. ** -** ^(If a memory allocation error occurs during the evaluation of any -** of these routines, a default value is returned. The default value -** is either the integer 0, the floating point number 0.0, or a NULL -** pointer. Subsequent calls to [sqlite3_errcode()] will return -** [SQLITE_NOMEM].)^ +** As long as the input parameters are correct, these routines will only +** fail if an out-of-memory error occurs during a format conversion. +** Only the following subset of interfaces are subject to out-of-memory +** errors: +** +**
        +**
      • sqlite3_column_blob() +**
      • sqlite3_column_text() +**
      • sqlite3_column_text16() +**
      • sqlite3_column_bytes() +**
      • sqlite3_column_bytes16() +**
      +** +** If an out-of-memory error occurs, then the return value from these +** routines is the same as if the column had contained an SQL NULL value. +** Valid SQL NULL returns can be distinguished from out-of-memory errors +** by invoking the [sqlite3_errcode()] immediately after the suspect +** return value is obtained and before any +** other SQLite interface is called on the same [database connection]. */ SQLITE_API const void *sqlite3_column_blob(sqlite3_stmt*, int iCol); SQLITE_API double sqlite3_column_double(sqlite3_stmt*, int iCol); @@ -5617,17 +6067,17 @@ SQLITE_API int sqlite3_reset(sqlite3_stmt *pStmt); /* ** CAPI3REF: Create Or Redefine SQL Functions ** KEYWORDS: {function creation routines} -** KEYWORDS: {application-defined SQL function} -** KEYWORDS: {application-defined SQL functions} ** METHOD: sqlite3 ** ** ^These functions (collectively known as "function creation routines") ** are used to add SQL functions or aggregates or to redefine the behavior -** of existing SQL functions or aggregates. The only differences between -** these routines are the text encoding expected for -** the second parameter (the name of the function being created) -** and the presence or absence of a destructor callback for -** the application data pointer. +** of existing SQL functions or aggregates. The only differences between +** the three "sqlite3_create_function*" routines are the text encoding +** expected for the second parameter (the name of the function being +** created) and the presence or absence of a destructor callback for +** the application data pointer. Function sqlite3_create_window_function() +** is similar, but allows the user to supply the extra callback functions +** needed by [aggregate window functions]. ** ** ^The first parameter is the [database connection] to which the SQL ** function is to be added. ^If an application uses more than one database @@ -5670,10 +6120,28 @@ SQLITE_API int sqlite3_reset(sqlite3_stmt *pStmt); ** perform additional optimizations on deterministic functions, so use ** of the [SQLITE_DETERMINISTIC] flag is recommended where possible. ** +** ^The fourth parameter may also optionally include the [SQLITE_DIRECTONLY] +** flag, which if present prevents the function from being invoked from +** within VIEWs, TRIGGERs, CHECK constraints, generated column expressions, +** index expressions, or the WHERE clause of partial indexes. +** +** +** For best security, the [SQLITE_DIRECTONLY] flag is recommended for +** all application-defined SQL functions that do not need to be +** used inside of triggers, view, CHECK constraints, or other elements of +** the database schema. This flags is especially recommended for SQL +** functions that have side effects or reveal internal application state. +** Without this flag, an attacker might be able to modify the schema of +** a database file to include invocations of the function with parameters +** chosen by the attacker, which the application will then execute when +** the database file is opened and read. +** +** ** ^(The fifth parameter is an arbitrary pointer. The implementation of the ** function can gain access to this pointer using [sqlite3_user_data()].)^ ** -** ^The sixth, seventh and eighth parameters, xFunc, xStep and xFinal, are +** ^The sixth, seventh and eighth parameters passed to the three +** "sqlite3_create_function*" functions, xFunc, xStep and xFinal, are ** pointers to C-language functions that implement the SQL function or ** aggregate. ^A scalar SQL function requires an implementation of the xFunc ** callback only; NULL pointers must be passed as the xStep and xFinal @@ -5682,15 +6150,24 @@ SQLITE_API int sqlite3_reset(sqlite3_stmt *pStmt); ** SQL function or aggregate, pass NULL pointers for all three function ** callbacks. ** -** ^(If the ninth parameter to sqlite3_create_function_v2() is not NULL, -** then it is destructor for the application data pointer. -** The destructor is invoked when the function is deleted, either by being -** overloaded or when the database connection closes.)^ -** ^The destructor is also invoked if the call to -** sqlite3_create_function_v2() fails. -** ^When the destructor callback of the tenth parameter is invoked, it -** is passed a single argument which is a copy of the application data -** pointer which was the fifth parameter to sqlite3_create_function_v2(). +** ^The sixth, seventh, eighth and ninth parameters (xStep, xFinal, xValue +** and xInverse) passed to sqlite3_create_window_function are pointers to +** C-language callbacks that implement the new function. xStep and xFinal +** must both be non-NULL. xValue and xInverse may either both be NULL, in +** which case a regular aggregate function is created, or must both be +** non-NULL, in which case the new function may be used as either an aggregate +** or aggregate window function. More details regarding the implementation +** of aggregate window functions are +** [user-defined window functions|available here]. +** +** ^(If the final parameter to sqlite3_create_function_v2() or +** sqlite3_create_window_function() is not NULL, then it is destructor for +** the application data pointer. The destructor is invoked when the function +** is deleted, either by being overloaded or when the database connection +** closes.)^ ^The destructor is also invoked if the call to +** sqlite3_create_function_v2() fails. ^When the destructor callback is +** invoked, it is passed a single argument which is a copy of the application +** data pointer which was the fifth parameter to sqlite3_create_function_v2(). ** ** ^It is permitted to register multiple implementations of the same ** functions with the same name but with either differing numbers of @@ -5743,6 +6220,18 @@ SQLITE_API int sqlite3_create_function_v2( void (*xFinal)(sqlite3_context*), void(*xDestroy)(void*) ); +SQLITE_API int sqlite3_create_window_function( + sqlite3 *db, + const char *zFunctionName, + int nArg, + int eTextRep, + void *pApp, + void (*xStep)(sqlite3_context*,int,sqlite3_value**), + void (*xFinal)(sqlite3_context*), + void (*xValue)(sqlite3_context*), + void (*xInverse)(sqlite3_context*,int,sqlite3_value**), + void(*xDestroy)(void*) +); /* ** CAPI3REF: Text Encodings @@ -5764,8 +6253,68 @@ SQLITE_API int sqlite3_create_function_v2( ** [SQLITE_UTF8 | preferred text encoding] as the fourth argument ** to [sqlite3_create_function()], [sqlite3_create_function16()], or ** [sqlite3_create_function_v2()]. +** +**
      +** [[SQLITE_DETERMINISTIC]]
      SQLITE_DETERMINISTIC
      +** The SQLITE_DETERMINISTIC flag means that the new function always gives +** the same output when the input parameters are the same. +** The [abs|abs() function] is deterministic, for example, but +** [randomblob|randomblob()] is not. Functions must +** be deterministic in order to be used in certain contexts such as +** with the WHERE clause of [partial indexes] or in [generated columns]. +** SQLite might also optimize deterministic functions by factoring them +** out of inner loops. +**
      +** +** [[SQLITE_DIRECTONLY]]
      SQLITE_DIRECTONLY
      +** The SQLITE_DIRECTONLY flag means that the function may only be invoked +** from top-level SQL, and cannot be used in VIEWs or TRIGGERs nor in +** schema structures such as [CHECK constraints], [DEFAULT clauses], +** [expression indexes], [partial indexes], or [generated columns]. +** The SQLITE_DIRECTONLY flags is a security feature which is recommended +** for all [application-defined SQL functions], and especially for functions +** that have side-effects or that could potentially leak sensitive +** information. +**
      +** +** [[SQLITE_INNOCUOUS]]
      SQLITE_INNOCUOUS
      +** The SQLITE_INNOCUOUS flag means that the function is unlikely +** to cause problems even if misused. An innocuous function should have +** no side effects and should not depend on any values other than its +** input parameters. The [abs|abs() function] is an example of an +** innocuous function. +** The [load_extension() SQL function] is not innocuous because of its +** side effects. +**

      SQLITE_INNOCUOUS is similar to SQLITE_DETERMINISTIC, but is not +** exactly the same. The [random|random() function] is an example of a +** function that is innocuous but not deterministic. +**

      Some heightened security settings +** ([SQLITE_DBCONFIG_TRUSTED_SCHEMA] and [PRAGMA trusted_schema=OFF]) +** disable the use of SQL functions inside views and triggers and in +** schema structures such as [CHECK constraints], [DEFAULT clauses], +** [expression indexes], [partial indexes], and [generated columns] unless +** the function is tagged with SQLITE_INNOCUOUS. Most built-in functions +** are innocuous. Developers are advised to avoid using the +** SQLITE_INNOCUOUS flag for application-defined functions unless the +** function has been carefully audited and found to be free of potentially +** security-adverse side-effects and information-leaks. +**

      +** +** [[SQLITE_SUBTYPE]]
      SQLITE_SUBTYPE
      +** The SQLITE_SUBTYPE flag indicates to SQLite that a function may call +** [sqlite3_value_subtype()] to inspect the sub-types of its arguments. +** Specifying this flag makes no difference for scalar or aggregate user +** functions. However, if it is not specified for a user-defined window +** function, then any sub-types belonging to arguments passed to the window +** function may be discarded before the window function is called (i.e. +** sqlite3_value_subtype() will always return 0). +**
      +**
      */ -#define SQLITE_DETERMINISTIC 0x800 +#define SQLITE_DETERMINISTIC 0x000000800 +#define SQLITE_DIRECTONLY 0x000080000 +#define SQLITE_SUBTYPE 0x000100000 +#define SQLITE_INNOCUOUS 0x000200000 /* ** CAPI3REF: Deprecated Functions @@ -5816,14 +6365,16 @@ SQLITE_API SQLITE_DEPRECATED int sqlite3_memory_alarm(void(*)(void*,sqlite3_int6 ** sqlite3_value_nochange   ** →  True if the column is unchanged in an UPDATE ** against a virtual table. +** sqlite3_value_frombind   +** →  True if value originated from a [bound parameter] ** ** ** Details: ** ** These routines extract type, size, and content information from ** [protected sqlite3_value] objects. Protected sqlite3_value objects -** are used to pass parameter information into implementation of -** [application-defined SQL functions] and [virtual tables]. +** are used to pass parameter information into the functions that +** implement [application-defined SQL functions] and [virtual tables]. ** ** These routines work only with [protected sqlite3_value] objects. ** Any attempt to use these routines on an [unprotected sqlite3_value] @@ -5877,6 +6428,11 @@ SQLITE_API SQLITE_DEPRECATED int sqlite3_memory_alarm(void(*)(void*,sqlite3_int6 ** than within an [xUpdate] method call for an UPDATE statement, then ** the return value is arbitrary and meaningless. ** +** ^The sqlite3_value_frombind(X) interface returns non-zero if the +** value X originated from one of the [sqlite3_bind_int|sqlite3_bind()] +** interfaces. ^If X comes from an SQL literal value, or a table column, +** or an expression, then sqlite3_value_frombind(X) returns zero. +** ** Please pay particular attention to the fact that the pointer returned ** from [sqlite3_value_blob()], [sqlite3_value_text()], or ** [sqlite3_value_text16()] can be invalidated by a subsequent call to @@ -5885,6 +6441,28 @@ SQLITE_API SQLITE_DEPRECATED int sqlite3_memory_alarm(void(*)(void*,sqlite3_int6 ** ** These routines must be called from the same thread as ** the SQL function that supplied the [sqlite3_value*] parameters. +** +** As long as the input parameter is correct, these routines can only +** fail if an out-of-memory error occurs during a format conversion. +** Only the following subset of interfaces are subject to out-of-memory +** errors: +** +**
        +**
      • sqlite3_value_blob() +**
      • sqlite3_value_text() +**
      • sqlite3_value_text16() +**
      • sqlite3_value_text16le() +**
      • sqlite3_value_text16be() +**
      • sqlite3_value_bytes() +**
      • sqlite3_value_bytes16() +**
      +** +** If an out-of-memory error occurs, then the return value from these +** routines is the same as if the column had contained an SQL NULL value. +** Valid SQL NULL returns can be distinguished from out-of-memory errors +** by invoking the [sqlite3_errcode()] immediately after the suspect +** return value is obtained and before any +** other SQLite interface is called on the same [database connection]. */ SQLITE_API const void *sqlite3_value_blob(sqlite3_value*); SQLITE_API double sqlite3_value_double(sqlite3_value*); @@ -5900,6 +6478,7 @@ SQLITE_API int sqlite3_value_bytes16(sqlite3_value*); SQLITE_API int sqlite3_value_type(sqlite3_value*); SQLITE_API int sqlite3_value_numeric_type(sqlite3_value*); SQLITE_API int sqlite3_value_nochange(sqlite3_value*); +SQLITE_API int sqlite3_value_frombind(sqlite3_value*); /* ** CAPI3REF: Finding The Subtype Of SQL Values @@ -5938,8 +6517,8 @@ SQLITE_API void sqlite3_value_free(sqlite3_value*); ** routine to allocate memory for storing their state. ** ** ^The first time the sqlite3_aggregate_context(C,N) routine is called -** for a particular aggregate function, SQLite -** allocates N of memory, zeroes out that memory, and returns a pointer +** for a particular aggregate function, SQLite allocates +** N bytes of memory, zeroes out that memory, and returns a pointer ** to the new memory. ^On second and subsequent calls to ** sqlite3_aggregate_context() for the same aggregate function instance, ** the same buffer is returned. Sqlite3_aggregate_context() is normally @@ -5956,7 +6535,7 @@ SQLITE_API void sqlite3_value_free(sqlite3_value*); ** ** ^(The amount of space allocated by sqlite3_aggregate_context(C,N) is ** determined by the N parameter on first successful call. Changing the -** value of N in subsequent call to sqlite3_aggregate_context() within +** value of N in any subsequent call to sqlite3_aggregate_context() within ** the same aggregate function instance will not resize the memory ** allocation.)^ Within the xFinal callback, it is customary to set ** N=0 in calls to sqlite3_aggregate_context(C,N) so that no @@ -6113,8 +6692,9 @@ typedef void (*sqlite3_destructor_type)(void*); ** 2nd parameter of sqlite3_result_error() or sqlite3_result_error16() ** as the text of an error message. ^SQLite interprets the error ** message string from sqlite3_result_error() as UTF-8. ^SQLite -** interprets the string from sqlite3_result_error16() as UTF-16 in native -** byte order. ^If the third parameter to sqlite3_result_error() +** interprets the string from sqlite3_result_error16() as UTF-16 using +** the same [byte-order determination rules] as [sqlite3_bind_text16()]. +** ^If the third parameter to sqlite3_result_error() ** or sqlite3_result_error16() is negative then SQLite takes as the error ** message all text up through the first zero character. ** ^If the third parameter to sqlite3_result_error() or @@ -6182,6 +6762,25 @@ typedef void (*sqlite3_destructor_type)(void*); ** then SQLite makes a copy of the result into space obtained ** from [sqlite3_malloc()] before it returns. ** +** ^For the sqlite3_result_text16(), sqlite3_result_text16le(), and +** sqlite3_result_text16be() routines, and for sqlite3_result_text64() +** when the encoding is not UTF8, if the input UTF16 begins with a +** byte-order mark (BOM, U+FEFF) then the BOM is removed from the +** string and the rest of the string is interpreted according to the +** byte-order specified by the BOM. ^The byte-order specified by +** the BOM at the beginning of the text overrides the byte-order +** specified by the interface procedure. ^So, for example, if +** sqlite3_result_text16le() is invoked with text that begins +** with bytes 0xfe, 0xff (a big-endian byte-order mark) then the +** first two bytes of input are skipped and the remaining input +** is interpreted as UTF16BE text. +** +** ^For UTF16 input text to the sqlite3_result_text16(), +** sqlite3_result_text16be(), sqlite3_result_text16le(), and +** sqlite3_result_text64() routines, if the text contains invalid +** UTF16 characters, the invalid characters might be converted +** into the unicode replacement character, U+FFFD. +** ** ^The sqlite3_result_value() interface sets the result of ** the application-defined function to be a copy of the ** [unprotected sqlite3_value] object specified by the 2nd parameter. ^The @@ -6267,7 +6866,7 @@ SQLITE_API void sqlite3_result_subtype(sqlite3_context*,unsigned int); **
    • [SQLITE_UTF16_ALIGNED]. **
    )^ ** ^The eTextRep argument determines the encoding of strings passed -** to the collating function callback, xCallback. +** to the collating function callback, xCompare. ** ^The [SQLITE_UTF16] and [SQLITE_UTF16_ALIGNED] values for eTextRep ** force strings to be UTF16 with native byte order. ** ^The [SQLITE_UTF16_ALIGNED] value for eTextRep forces strings to begin @@ -6276,18 +6875,19 @@ SQLITE_API void sqlite3_result_subtype(sqlite3_context*,unsigned int); ** ^The fourth argument, pArg, is an application data pointer that is passed ** through as the first argument to the collating function callback. ** -** ^The fifth argument, xCallback, is a pointer to the collating function. +** ^The fifth argument, xCompare, is a pointer to the collating function. ** ^Multiple collating functions can be registered using the same name but ** with different eTextRep parameters and SQLite will use whichever ** function requires the least amount of data transformation. -** ^If the xCallback argument is NULL then the collating function is +** ^If the xCompare argument is NULL then the collating function is ** deleted. ^When all collating functions having the same name are deleted, ** that collation is no longer usable. ** ** ^The collating function callback is invoked with a copy of the pArg ** application data pointer and with two strings in the encoding specified -** by the eTextRep argument. The collating function must return an -** integer that is negative, zero, or positive +** by the eTextRep argument. The two integer parameters to the collating +** function callback are the length of the two strings, in bytes. The collating +** function must return an integer that is negative, zero, or positive ** if the first string is less than, equal to, or greater than the second, ** respectively. A collating function must always return the same answer ** given the same inputs. If two or more collating functions are registered @@ -6304,7 +6904,7 @@ SQLITE_API void sqlite3_result_subtype(sqlite3_context*,unsigned int); ** ** ** If a collating function fails any of the above constraints and that -** collating function is registered and used, then the behavior of SQLite +** collating function is registered and used, then the behavior of SQLite ** is undefined. ** ** ^The sqlite3_create_collation_v2() works like sqlite3_create_collation() @@ -6386,51 +6986,6 @@ SQLITE_API int sqlite3_collation_needed16( void(*)(void*,sqlite3*,int eTextRep,const void*) ); -#ifdef SQLITE_HAS_CODEC -/* -** Specify the key for an encrypted database. This routine should be -** called right after sqlite3_open(). -** -** The code to implement this API is not available in the public release -** of SQLite. -*/ -SQLITE_API int sqlite3_key( - sqlite3 *db, /* Database to be rekeyed */ - const void *pKey, int nKey /* The key */ -); -SQLITE_API int sqlite3_key_v2( - sqlite3 *db, /* Database to be rekeyed */ - const char *zDbName, /* Name of the database */ - const void *pKey, int nKey /* The key */ -); - -/* -** Change the key on an open database. If the current database is not -** encrypted, this routine will encrypt it. If pNew==0 or nNew==0, the -** database is decrypted. -** -** The code to implement this API is not available in the public release -** of SQLite. -*/ -SQLITE_API int sqlite3_rekey( - sqlite3 *db, /* Database to be rekeyed */ - const void *pKey, int nKey /* The new key */ -); -SQLITE_API int sqlite3_rekey_v2( - sqlite3 *db, /* Database to be rekeyed */ - const char *zDbName, /* Name of the database */ - const void *pKey, int nKey /* The new key */ -); - -/* -** Specify the activation key for a SEE database. Unless -** activated, none of the SEE routines will work. -*/ -SQLITE_API void sqlite3_activate_see( - const char *zPassPhrase /* Activation phrase */ -); -#endif - #ifdef SQLITE_ENABLE_CEROD /* ** Specify the activation key for a CEROD database. Unless @@ -6631,16 +7186,31 @@ SQLITE_API sqlite3 *sqlite3_db_handle(sqlite3_stmt*); ** CAPI3REF: Return The Filename For A Database Connection ** METHOD: sqlite3 ** -** ^The sqlite3_db_filename(D,N) interface returns a pointer to a filename -** associated with database N of connection D. ^The main database file -** has the name "main". If there is no attached database N on the database +** ^The sqlite3_db_filename(D,N) interface returns a pointer to the filename +** associated with database N of connection D. +** ^If there is no attached database N on the database ** connection D, or if database N is a temporary or in-memory database, then -** a NULL pointer is returned. +** this function will return either a NULL pointer or an empty string. +** +** ^The string value returned by this routine is owned and managed by +** the database connection. ^The value will be valid until the database N +** is [DETACH]-ed or until the database connection closes. ** ** ^The filename returned by this function is the output of the ** xFullPathname method of the [VFS]. ^In other words, the filename ** will be an absolute pathname, even if the filename used ** to open the database originally was a URI or relative pathname. +** +** If the filename pointer returned by this routine is not NULL, then it +** can be used as the filename input parameter to these routines: +**
      +**
    • [sqlite3_uri_parameter()] +**
    • [sqlite3_uri_boolean()] +**
    • [sqlite3_uri_int64()] +**
    • [sqlite3_filename_database()] +**
    • [sqlite3_filename_journal()] +**
    • [sqlite3_filename_wal()] +**
    */ SQLITE_API const char *sqlite3_db_filename(sqlite3 *db, const char *zDbName); @@ -6790,15 +7360,19 @@ SQLITE_API void *sqlite3_update_hook( ** ** ^(The cache sharing mode set by this interface effects all subsequent ** calls to [sqlite3_open()], [sqlite3_open_v2()], and [sqlite3_open16()]. -** Existing database connections continue use the sharing mode +** Existing database connections continue to use the sharing mode ** that was in effect at the time they were opened.)^ ** ** ^(This routine returns [SQLITE_OK] if shared cache was enabled or disabled ** successfully. An [error code] is returned otherwise.)^ ** -** ^Shared cache is disabled by default. But this might change in -** future releases of SQLite. Applications that care about shared -** cache setting should set it explicitly. +** ^Shared cache is disabled by default. It is recommended that it stay +** that way. In other words, do not use this routine. This interface +** continues to be provided for historical compatibility, but its use is +** discouraged. Any use of shared cache is discouraged. If shared cache +** must be used, it is recommended that shared cache only be enabled for +** individual database connections using the [sqlite3_open_v2()] interface +** with the [SQLITE_OPEN_SHAREDCACHE] flag. ** ** Note: This method is disabled on MacOS X 10.7 and iOS version 5.0 ** and will always return SQLITE_MISUSE. On those systems, @@ -6845,6 +7419,9 @@ SQLITE_API int sqlite3_db_release_memory(sqlite3*); /* ** CAPI3REF: Impose A Limit On Heap Size ** +** These interfaces impose limits on the amount of heap memory that will be +** by all database connections within a single process. +** ** ^The sqlite3_soft_heap_limit64() interface sets and/or queries the ** soft limit on the amount of heap memory that may be allocated by SQLite. ** ^SQLite strives to keep heap memory utilization below the soft heap @@ -6855,20 +7432,41 @@ SQLITE_API int sqlite3_db_release_memory(sqlite3*); ** an [SQLITE_NOMEM] error. In other words, the soft heap limit ** is advisory only. ** -** ^The return value from sqlite3_soft_heap_limit64() is the size of -** the soft heap limit prior to the call, or negative in the case of an -** error. ^If the argument N is negative -** then no change is made to the soft heap limit. Hence, the current -** size of the soft heap limit can be determined by invoking -** sqlite3_soft_heap_limit64() with a negative argument. -** -** ^If the argument N is zero then the soft heap limit is disabled. +** ^The sqlite3_hard_heap_limit64(N) interface sets a hard upper bound of +** N bytes on the amount of memory that will be allocated. ^The +** sqlite3_hard_heap_limit64(N) interface is similar to +** sqlite3_soft_heap_limit64(N) except that memory allocations will fail +** when the hard heap limit is reached. ** -** ^(The soft heap limit is not enforced in the current implementation +** ^The return value from both sqlite3_soft_heap_limit64() and +** sqlite3_hard_heap_limit64() is the size of +** the heap limit prior to the call, or negative in the case of an +** error. ^If the argument N is negative +** then no change is made to the heap limit. Hence, the current +** size of heap limits can be determined by invoking +** sqlite3_soft_heap_limit64(-1) or sqlite3_hard_heap_limit(-1). +** +** ^Setting the heap limits to zero disables the heap limiter mechanism. +** +** ^The soft heap limit may not be greater than the hard heap limit. +** ^If the hard heap limit is enabled and if sqlite3_soft_heap_limit(N) +** is invoked with a value of N that is greater than the hard heap limit, +** the the soft heap limit is set to the value of the hard heap limit. +** ^The soft heap limit is automatically enabled whenever the hard heap +** limit is enabled. ^When sqlite3_hard_heap_limit64(N) is invoked and +** the soft heap limit is outside the range of 1..N, then the soft heap +** limit is set to N. ^Invoking sqlite3_soft_heap_limit64(0) when the +** hard heap limit is enabled makes the soft heap limit equal to the +** hard heap limit. +** +** The memory allocation limits can also be adjusted using +** [PRAGMA soft_heap_limit] and [PRAGMA hard_heap_limit]. +** +** ^(The heap limits are not enforced in the current implementation ** if one or more of following conditions are true: ** **
      -**
    • The soft heap limit is set to zero. +**
    • The limit value is set to zero. **
    • Memory accounting is disabled using a combination of the ** [sqlite3_config]([SQLITE_CONFIG_MEMSTATUS],...) start-time option and ** the [SQLITE_DEFAULT_MEMSTATUS] compile-time option. @@ -6879,21 +7477,11 @@ SQLITE_API int sqlite3_db_release_memory(sqlite3*); ** from the heap. **
    )^ ** -** Beginning with SQLite [version 3.7.3] ([dateof:3.7.3]), -** the soft heap limit is enforced -** regardless of whether or not the [SQLITE_ENABLE_MEMORY_MANAGEMENT] -** compile-time option is invoked. With [SQLITE_ENABLE_MEMORY_MANAGEMENT], -** the soft heap limit is enforced on every memory allocation. Without -** [SQLITE_ENABLE_MEMORY_MANAGEMENT], the soft heap limit is only enforced -** when memory is allocated by the page cache. Testing suggests that because -** the page cache is the predominate memory user in SQLite, most -** applications will achieve adequate soft heap limit enforcement without -** the use of [SQLITE_ENABLE_MEMORY_MANAGEMENT]. -** -** The circumstances under which SQLite will enforce the soft heap limit may +** The circumstances under which SQLite will enforce the heap limits may ** changes in future releases of SQLite. */ SQLITE_API sqlite3_int64 sqlite3_soft_heap_limit64(sqlite3_int64 N); +SQLITE_API sqlite3_int64 sqlite3_hard_heap_limit64(sqlite3_int64 N); /* ** CAPI3REF: Deprecated Soft Heap Limit Interface @@ -6917,7 +7505,7 @@ SQLITE_API SQLITE_DEPRECATED void sqlite3_soft_heap_limit(int N); ** interface returns SQLITE_OK and fills in the non-NULL pointers in ** the final five arguments with appropriate values if the specified ** column exists. ^The sqlite3_table_column_metadata() interface returns -** SQLITE_ERROR and if the specified column does not exist. +** SQLITE_ERROR if the specified column does not exist. ** ^If the column-name parameter to sqlite3_table_column_metadata() is a ** NULL pointer, then this routine simply checks for the existence of the ** table and returns SQLITE_OK if the table exists and SQLITE_ERROR if it @@ -7059,7 +7647,7 @@ SQLITE_API int sqlite3_load_extension( ** to enable or disable only the C-API.)^ ** ** Security warning: It is recommended that extension loading -** be disabled using the [SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION] method +** be enabled using the [SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION] method ** rather than this interface, so the [load_extension()] SQL function ** remains disabled. This will prevent SQL injections from giving attackers ** access to extension loading capabilities. @@ -7146,7 +7734,7 @@ typedef struct sqlite3_module sqlite3_module; ** KEYWORDS: sqlite3_module {virtual table module} ** ** This structure, sometimes called a "virtual table module", -** defines the implementation of a [virtual tables]. +** defines the implementation of a [virtual table]. ** This structure consists mostly of methods for the module. ** ** ^A virtual table module is created by filling in a persistent @@ -7190,6 +7778,9 @@ struct sqlite3_module { int (*xSavepoint)(sqlite3_vtab *pVTab, int); int (*xRelease)(sqlite3_vtab *pVTab, int); int (*xRollbackTo)(sqlite3_vtab *pVTab, int); + /* The methods above are in versions 1 and 2 of the sqlite_module object. + ** Those below are for version 3 and greater. */ + int (*xShadowName)(const char*); }; /* @@ -7240,7 +7831,13 @@ struct sqlite3_module { ** the right-hand side of the corresponding aConstraint[] is evaluated ** and becomes the argvIndex-th entry in argv. ^(If aConstraintUsage[].omit ** is true, then the constraint is assumed to be fully handled by the -** virtual table and is not checked again by SQLite.)^ +** virtual table and might not be checked again by the byte code.)^ ^(The +** aConstraintUsage[].omit flag is an optimization hint. When the omit flag +** is left in its default setting of false, the constraint will always be +** checked separately in byte code. If the omit flag is change to true, then +** the constraint may or may not be checked in byte code. In other words, +** when the omit flag is true there is no guarantee that the constraint will +** not be checked again using byte code.)^ ** ** ^The idxNum and idxPtr values are recorded and passed into the ** [xFilter] method. @@ -7280,7 +7877,7 @@ struct sqlite3_module { ** If a virtual table extension is ** used with an SQLite version earlier than 3.8.2, the results of attempting ** to read or write the estimatedRows field are undefined (but are likely -** to included crashing the application). The estimatedRows field should +** to include crashing the application). The estimatedRows field should ** therefore only be used if [sqlite3_libversion_number()] returns a ** value greater than or equal to 3008002. Similarly, the idxFlags field ** was added for [version 3.9.0] ([dateof:3.9.0]). @@ -7332,7 +7929,7 @@ struct sqlite3_index_info { /* ** CAPI3REF: Virtual Table Constraint Operator Codes ** -** These macros defined the allowed values for the +** These macros define the allowed values for the ** [sqlite3_index_info].aConstraint[].op field. Each value represents ** an operator that is part of a constraint term in the wHERE clause of ** a query that uses a [virtual table]. @@ -7351,6 +7948,7 @@ struct sqlite3_index_info { #define SQLITE_INDEX_CONSTRAINT_ISNOTNULL 70 #define SQLITE_INDEX_CONSTRAINT_ISNULL 71 #define SQLITE_INDEX_CONSTRAINT_IS 72 +#define SQLITE_INDEX_CONSTRAINT_FUNCTION 150 /* ** CAPI3REF: Register A Virtual Table Implementation @@ -7377,6 +7975,12 @@ struct sqlite3_index_info { ** ^The sqlite3_create_module() ** interface is equivalent to sqlite3_create_module_v2() with a NULL ** destructor. +** +** ^If the third parameter (the pointer to the sqlite3_module object) is +** NULL then no new module is create and any existing modules with the +** same name are dropped. +** +** See also: [sqlite3_drop_modules()] */ SQLITE_API int sqlite3_create_module( sqlite3 *db, /* SQLite connection to register module with */ @@ -7392,6 +7996,23 @@ SQLITE_API int sqlite3_create_module_v2( void(*xDestroy)(void*) /* Module destructor function */ ); +/* +** CAPI3REF: Remove Unnecessary Virtual Table Implementations +** METHOD: sqlite3 +** +** ^The sqlite3_drop_modules(D,L) interface removes all virtual +** table modules from database connection D except those named on list L. +** The L parameter must be either NULL or a pointer to an array of pointers +** to strings where the array is terminated by a single NULL pointer. +** ^If the L parameter is NULL, then all virtual table modules are removed. +** +** See also: [sqlite3_create_module()] +*/ +SQLITE_API int sqlite3_drop_modules( + sqlite3 *db, /* Remove modules from this connection */ + const char **azKeep /* Except, do not remove the ones named here */ +); + /* ** CAPI3REF: Virtual Table Instance Object ** KEYWORDS: sqlite3_vtab @@ -7918,7 +8539,7 @@ SQLITE_API void sqlite3_mutex_leave(sqlite3_mutex*); ** The only difference is that the public sqlite3_XXX functions enumerated ** above silently ignore any invocations that pass a NULL pointer instead ** of a valid mutex handle. The implementations of the methods defined -** by this structure are not required to handle this case, the results +** by this structure are not required to handle this case. The results ** of passing a NULL pointer instead of a valid mutex handle are undefined ** (i.e. it is acceptable to provide an implementation that segfaults if ** it is passed a NULL pointer). @@ -8027,6 +8648,7 @@ SQLITE_API sqlite3_mutex *sqlite3_db_mutex(sqlite3*); /* ** CAPI3REF: Low-Level Control Of Database Files ** METHOD: sqlite3 +** KEYWORDS: {file control} ** ** ^The [sqlite3_file_control()] interface makes a direct call to the ** xFileControl method for the [sqlite3_io_methods] object associated @@ -8041,11 +8663,18 @@ SQLITE_API sqlite3_mutex *sqlite3_db_mutex(sqlite3*); ** the xFileControl method. ^The return value of the xFileControl ** method becomes the return value of this routine. ** +** A few opcodes for [sqlite3_file_control()] are handled directly +** by the SQLite core and never invoke the +** sqlite3_io_methods.xFileControl method. ** ^The [SQLITE_FCNTL_FILE_POINTER] value for the op parameter causes ** a pointer to the underlying [sqlite3_file] object to be written into -** the space pointed to by the 4th parameter. ^The [SQLITE_FCNTL_FILE_POINTER] -** case is a short-circuit path which does not actually invoke the -** underlying sqlite3_io_methods.xFileControl method. +** the space pointed to by the 4th parameter. The +** [SQLITE_FCNTL_JOURNAL_POINTER] works similarly except that it returns +** the [sqlite3_file] object associated with the journal file instead of +** the main database. The [SQLITE_FCNTL_VFS_POINTER] opcode returns +** a pointer to the underlying [sqlite3_vfs] object for the file. +** The [SQLITE_FCNTL_DATA_VERSION] returns the data version counter +** from the pager. ** ** ^If the second parameter (zDbName) does not match the name of any ** open database file, then SQLITE_ERROR is returned. ^This error @@ -8092,17 +8721,18 @@ SQLITE_API int sqlite3_test_control(int op, ...); #define SQLITE_TESTCTRL_FIRST 5 #define SQLITE_TESTCTRL_PRNG_SAVE 5 #define SQLITE_TESTCTRL_PRNG_RESTORE 6 -#define SQLITE_TESTCTRL_PRNG_RESET 7 +#define SQLITE_TESTCTRL_PRNG_RESET 7 /* NOT USED */ #define SQLITE_TESTCTRL_BITVEC_TEST 8 #define SQLITE_TESTCTRL_FAULT_INSTALL 9 #define SQLITE_TESTCTRL_BENIGN_MALLOC_HOOKS 10 #define SQLITE_TESTCTRL_PENDING_BYTE 11 #define SQLITE_TESTCTRL_ASSERT 12 #define SQLITE_TESTCTRL_ALWAYS 13 -#define SQLITE_TESTCTRL_RESERVE 14 +#define SQLITE_TESTCTRL_RESERVE 14 /* NOT USED */ #define SQLITE_TESTCTRL_OPTIMIZATIONS 15 #define SQLITE_TESTCTRL_ISKEYWORD 16 /* NOT USED */ #define SQLITE_TESTCTRL_SCRATCHMALLOC 17 /* NOT USED */ +#define SQLITE_TESTCTRL_INTERNAL_FUNCTIONS 17 #define SQLITE_TESTCTRL_LOCALTIME_FAULT 18 #define SQLITE_TESTCTRL_EXPLAIN_STMT 19 /* NOT USED */ #define SQLITE_TESTCTRL_ONCE_RESET_THRESHOLD 19 @@ -8113,7 +8743,10 @@ SQLITE_API int sqlite3_test_control(int op, ...); #define SQLITE_TESTCTRL_SORTER_MMAP 24 #define SQLITE_TESTCTRL_IMPOSTER 25 #define SQLITE_TESTCTRL_PARSER_COVERAGE 26 -#define SQLITE_TESTCTRL_LAST 26 /* Largest TESTCTRL */ +#define SQLITE_TESTCTRL_RESULT_INTREAL 27 +#define SQLITE_TESTCTRL_PRNG_SEED 28 +#define SQLITE_TESTCTRL_EXTRA_SCHEMA_CHECKS 29 +#define SQLITE_TESTCTRL_LAST 29 /* Largest TESTCTRL */ /* ** CAPI3REF: SQL Keyword Checking @@ -8379,7 +9012,7 @@ SQLITE_API int sqlite3_status64( ** ** [[SQLITE_STATUS_PAGECACHE_SIZE]] ^(
    SQLITE_STATUS_PAGECACHE_SIZE
    **
    This parameter records the largest memory allocation request -** handed to [pagecache memory allocator]. Only the value returned in the +** handed to the [pagecache memory allocator]. Only the value returned in the ** *pHighwater parameter to [sqlite3_status()] is of interest. ** The value written into the *pCurrent parameter is undefined.
    )^ ** @@ -8455,7 +9088,7 @@ SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int r ** checked out.)^ ** ** [[SQLITE_DBSTATUS_LOOKASIDE_HIT]] ^(
    SQLITE_DBSTATUS_LOOKASIDE_HIT
    -**
    This parameter returns the number malloc attempts that were +**
    This parameter returns the number of malloc attempts that were ** satisfied using lookaside memory. Only the high-water value is meaningful; ** the current value is always zero.)^ ** @@ -8537,7 +9170,7 @@ SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int r ** cache overflowing. Transactions are more efficient if they are written ** to disk all at once. When pages spill mid-transaction, that introduces ** additional overhead. This parameter can be used help identify -** inefficiencies that can be resolve by increasing the cache size. +** inefficiencies that can be resolved by increasing the cache size. **
    ** ** [[SQLITE_DBSTATUS_DEFERRED_FKS]] ^(
    SQLITE_DBSTATUS_DEFERRED_FKS
    @@ -8626,7 +9259,7 @@ SQLITE_API int sqlite3_stmt_status(sqlite3_stmt*, int op,int resetFlg); ** ** [[SQLITE_STMTSTATUS_REPREPARE]]
    SQLITE_STMTSTATUS_REPREPARE
    **
    ^This is the number of times that the prepare statement has been -** automatically regenerated due to schema changes or change to +** automatically regenerated due to schema changes or changes to ** [bound parameters] that might affect the query plan. ** ** [[SQLITE_STMTSTATUS_RUN]]
    SQLITE_STMTSTATUS_RUN
    @@ -8797,7 +9430,7 @@ struct sqlite3_pcache_page { ** ** ^(SQLite will normally invoke xFetch() with a createFlag of 0 or 1. SQLite ** will only use a createFlag of 2 after a prior call with a createFlag of 1 -** failed.)^ In between the to xFetch() calls, SQLite may +** failed.)^ In between the xFetch() calls, SQLite may ** attempt to unpin one or more cache pages by spilling the content of ** pinned pages to disk and synching the operating system disk cache. ** @@ -9115,7 +9748,7 @@ SQLITE_API int sqlite3_backup_pagecount(sqlite3_backup *p); ** the first argument to register for a callback that will be invoked ** when the blocking connections current transaction is concluded. ^The ** callback is invoked from within the [sqlite3_step] or [sqlite3_close] -** call that concludes the blocking connections transaction. +** call that concludes the blocking connection's transaction. ** ** ^(If sqlite3_unlock_notify() is called in a multi-threaded application, ** there is a chance that the blocking connection will have already @@ -9153,7 +9786,7 @@ SQLITE_API int sqlite3_backup_pagecount(sqlite3_backup *p); ** an unlock-notify callback is a pointer to an array of void* pointers, ** and the second is the number of entries in the array. ** -** When a blocking connections transaction is concluded, there may be +** When a blocking connection's transaction is concluded, there may be ** more than one blocked connection that has registered for an unlock-notify ** callback. ^If two or more such blocked connections have specified the ** same callback function, then instead of invoking the callback function @@ -9501,21 +10134,28 @@ SQLITE_API int sqlite3_wal_checkpoint_v2( ** If this interface is invoked outside the context of an xConnect or ** xCreate virtual table method then the behavior is undefined. ** -** At present, there is only one option that may be configured using -** this function. (See [SQLITE_VTAB_CONSTRAINT_SUPPORT].) Further options -** may be added in the future. +** In the call sqlite3_vtab_config(D,C,...) the D parameter is the +** [database connection] in which the virtual table is being created and +** which is passed in as the first argument to the [xConnect] or [xCreate] +** method that is invoking sqlite3_vtab_config(). The C parameter is one +** of the [virtual table configuration options]. The presence and meaning +** of parameters after C depend on which [virtual table configuration option] +** is used. */ SQLITE_API int sqlite3_vtab_config(sqlite3*, int op, ...); /* ** CAPI3REF: Virtual Table Configuration Options +** KEYWORDS: {virtual table configuration options} +** KEYWORDS: {virtual table configuration option} ** ** These macros define the various options to the ** [sqlite3_vtab_config()] interface that [virtual table] implementations ** can use to customize and optimize their behavior. ** **
    -**
    SQLITE_VTAB_CONSTRAINT_SUPPORT +** [[SQLITE_VTAB_CONSTRAINT_SUPPORT]] +**
    SQLITE_VTAB_CONSTRAINT_SUPPORT
    **
    Calls of the form ** [sqlite3_vtab_config](db,SQLITE_VTAB_CONSTRAINT_SUPPORT,X) are supported, ** where X is an integer. If X is zero, then the [virtual table] whose @@ -9544,9 +10184,31 @@ SQLITE_API int sqlite3_vtab_config(sqlite3*, int op, ...); ** return SQLITE_OK. Or, if this is not possible, it may return ** SQLITE_CONSTRAINT, in which case SQLite falls back to OR ABORT ** constraint handling. +**
    +** +** [[SQLITE_VTAB_DIRECTONLY]]
    SQLITE_VTAB_DIRECTONLY
    +**
    Calls of the form +** [sqlite3_vtab_config](db,SQLITE_VTAB_DIRECTONLY) from within the +** the [xConnect] or [xCreate] methods of a [virtual table] implmentation +** prohibits that virtual table from being used from within triggers and +** views. +**
    +** +** [[SQLITE_VTAB_INNOCUOUS]]
    SQLITE_VTAB_INNOCUOUS
    +**
    Calls of the form +** [sqlite3_vtab_config](db,SQLITE_VTAB_INNOCUOUS) from within the +** the [xConnect] or [xCreate] methods of a [virtual table] implmentation +** identify that virtual table as being safe to use from within triggers +** and views. Conceptually, the SQLITE_VTAB_INNOCUOUS tag means that the +** virtual table can do no serious harm even if it is controlled by a +** malicious hacker. Developers should avoid setting the SQLITE_VTAB_INNOCUOUS +** flag unless absolutely necessary. +**
    **
    */ #define SQLITE_VTAB_CONSTRAINT_SUPPORT 1 +#define SQLITE_VTAB_INNOCUOUS 2 +#define SQLITE_VTAB_DIRECTONLY 3 /* ** CAPI3REF: Determine The Virtual Table Conflict Policy @@ -9626,15 +10288,15 @@ SQLITE_API SQLITE_EXPERIMENTAL const char *sqlite3_vtab_collation(sqlite3_index_ ** **
    ** [[SQLITE_SCANSTAT_NLOOP]]
    SQLITE_SCANSTAT_NLOOP
    -**
    ^The [sqlite3_int64] variable pointed to by the T parameter will be +**
    ^The [sqlite3_int64] variable pointed to by the V parameter will be ** set to the total number of times that the X-th loop has run.
    ** ** [[SQLITE_SCANSTAT_NVISIT]]
    SQLITE_SCANSTAT_NVISIT
    -**
    ^The [sqlite3_int64] variable pointed to by the T parameter will be set +**
    ^The [sqlite3_int64] variable pointed to by the V parameter will be set ** to the total number of rows examined by all iterations of the X-th loop.
    ** ** [[SQLITE_SCANSTAT_EST]]
    SQLITE_SCANSTAT_EST
    -**
    ^The "double" variable pointed to by the T parameter will be set to the +**
    ^The "double" variable pointed to by the V parameter will be set to the ** query planner's estimate for the average number of rows output from each ** iteration of the X-th loop. If the query planner's estimates was accurate, ** then this value will approximate the quotient NVISIT/NLOOP and the @@ -9642,17 +10304,17 @@ SQLITE_API SQLITE_EXPERIMENTAL const char *sqlite3_vtab_collation(sqlite3_index_ ** be the NLOOP value for the current loop. ** ** [[SQLITE_SCANSTAT_NAME]]
    SQLITE_SCANSTAT_NAME
    -**
    ^The "const char *" variable pointed to by the T parameter will be set +**
    ^The "const char *" variable pointed to by the V parameter will be set ** to a zero-terminated UTF-8 string containing the name of the index or table ** used for the X-th loop. ** ** [[SQLITE_SCANSTAT_EXPLAIN]]
    SQLITE_SCANSTAT_EXPLAIN
    -**
    ^The "const char *" variable pointed to by the T parameter will be set +**
    ^The "const char *" variable pointed to by the V parameter will be set ** to a zero-terminated UTF-8 string containing the [EXPLAIN QUERY PLAN] ** description for the X-th loop. ** ** [[SQLITE_SCANSTAT_SELECTID]]
    SQLITE_SCANSTAT_SELECT
    -**
    ^The "int" variable pointed to by the T parameter will be set to the +**
    ^The "int" variable pointed to by the V parameter will be set to the ** "select-id" for the X-th loop. The select-id identifies which query or ** subquery the loop is part of. The main query has a select-id of zero. ** The select-id is the same value as is output in the first column @@ -9864,7 +10526,6 @@ SQLITE_API int sqlite3_system_errno(sqlite3*); /* ** CAPI3REF: Database Snapshot ** KEYWORDS: {snapshot} {sqlite3_snapshot} -** EXPERIMENTAL ** ** An instance of the snapshot object records the state of a [WAL mode] ** database for some specific point in history. @@ -9881,11 +10542,6 @@ SQLITE_API int sqlite3_system_errno(sqlite3*); ** version of the database file so that it is possible to later open a new read ** transaction that sees that historical version of the database rather than ** the most recent version. -** -** The constructor for this object is [sqlite3_snapshot_get()]. The -** [sqlite3_snapshot_open()] method causes a fresh read transaction to refer -** to an historical snapshot (if possible). The destructor for -** sqlite3_snapshot objects is [sqlite3_snapshot_free()]. */ typedef struct sqlite3_snapshot { unsigned char hidden[48]; @@ -9893,7 +10549,7 @@ typedef struct sqlite3_snapshot { /* ** CAPI3REF: Record A Database Snapshot -** EXPERIMENTAL +** CONSTRUCTOR: sqlite3_snapshot ** ** ^The [sqlite3_snapshot_get(D,S,P)] interface attempts to make a ** new [sqlite3_snapshot] object that records the current state of @@ -9909,7 +10565,7 @@ typedef struct sqlite3_snapshot { ** in this case. ** **
      -**
    • The database handle must be in [autocommit mode]. +**
    • The database handle must not be in [autocommit mode]. ** **
    • Schema S of [database connection] D must be a [WAL mode] database. ** @@ -9932,7 +10588,7 @@ typedef struct sqlite3_snapshot { ** to avoid a memory leak. ** ** The [sqlite3_snapshot_get()] interface is only available when the -** SQLITE_ENABLE_SNAPSHOT compile-time option is used. +** [SQLITE_ENABLE_SNAPSHOT] compile-time option is used. */ SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_get( sqlite3 *db, @@ -9942,24 +10598,35 @@ SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_get( /* ** CAPI3REF: Start a read transaction on an historical snapshot -** EXPERIMENTAL +** METHOD: sqlite3_snapshot +** +** ^The [sqlite3_snapshot_open(D,S,P)] interface either starts a new read +** transaction or upgrades an existing one for schema S of +** [database connection] D such that the read transaction refers to +** historical [snapshot] P, rather than the most recent change to the +** database. ^The [sqlite3_snapshot_open()] interface returns SQLITE_OK +** on success or an appropriate [error code] if it fails. +** +** ^In order to succeed, the database connection must not be in +** [autocommit mode] when [sqlite3_snapshot_open(D,S,P)] is called. If there +** is already a read transaction open on schema S, then the database handle +** must have no active statements (SELECT statements that have been passed +** to sqlite3_step() but not sqlite3_reset() or sqlite3_finalize()). +** SQLITE_ERROR is returned if either of these conditions is violated, or +** if schema S does not exist, or if the snapshot object is invalid. +** +** ^A call to sqlite3_snapshot_open() will fail to open if the specified +** snapshot has been overwritten by a [checkpoint]. In this case +** SQLITE_ERROR_SNAPSHOT is returned. +** +** If there is already a read transaction open when this function is +** invoked, then the same read transaction remains open (on the same +** database snapshot) if SQLITE_ERROR, SQLITE_BUSY or SQLITE_ERROR_SNAPSHOT +** is returned. If another error code - for example SQLITE_PROTOCOL or an +** SQLITE_IOERR error code - is returned, then the final state of the +** read transaction is undefined. If SQLITE_OK is returned, then the +** read transaction is now open on database snapshot P. ** -** ^The [sqlite3_snapshot_open(D,S,P)] interface starts a -** read transaction for schema S of -** [database connection] D such that the read transaction -** refers to historical [snapshot] P, rather than the most -** recent change to the database. -** ^The [sqlite3_snapshot_open()] interface returns SQLITE_OK on success -** or an appropriate [error code] if it fails. -** -** ^In order to succeed, a call to [sqlite3_snapshot_open(D,S,P)] must be -** the first operation following the [BEGIN] that takes the schema S -** out of [autocommit mode]. -** ^In other words, schema S must not currently be in -** a transaction for [sqlite3_snapshot_open(D,S,P)] to work, but the -** database connection D must be out of [autocommit mode]. -** ^A [snapshot] will fail to open if it has been overwritten by a -** [checkpoint]. ** ^(A call to [sqlite3_snapshot_open(D,S,P)] will fail if the ** database connection D does not know that the database file for ** schema S is in [WAL mode]. A database connection might not know @@ -9970,7 +10637,7 @@ SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_get( ** database connection in order to make it ready to use snapshots.) ** ** The [sqlite3_snapshot_open()] interface is only available when the -** SQLITE_ENABLE_SNAPSHOT compile-time option is used. +** [SQLITE_ENABLE_SNAPSHOT] compile-time option is used. */ SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_open( sqlite3 *db, @@ -9980,20 +10647,20 @@ SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_open( /* ** CAPI3REF: Destroy a snapshot -** EXPERIMENTAL +** DESTRUCTOR: sqlite3_snapshot ** ** ^The [sqlite3_snapshot_free(P)] interface destroys [sqlite3_snapshot] P. ** The application must eventually free every [sqlite3_snapshot] object ** using this routine to avoid a memory leak. ** ** The [sqlite3_snapshot_free()] interface is only available when the -** SQLITE_ENABLE_SNAPSHOT compile-time option is used. +** [SQLITE_ENABLE_SNAPSHOT] compile-time option is used. */ SQLITE_API SQLITE_EXPERIMENTAL void sqlite3_snapshot_free(sqlite3_snapshot*); /* ** CAPI3REF: Compare the ages of two snapshot handles. -** EXPERIMENTAL +** METHOD: sqlite3_snapshot ** ** The sqlite3_snapshot_cmp(P1, P2) interface is used to compare the ages ** of two valid snapshot handles. @@ -10012,6 +10679,9 @@ SQLITE_API SQLITE_EXPERIMENTAL void sqlite3_snapshot_free(sqlite3_snapshot*); ** Otherwise, this API returns a negative value if P1 refers to an older ** snapshot than P2, zero if the two handles refer to the same database ** snapshot, and a positive value if P1 is a newer snapshot than P2. +** +** This interface is only available if SQLite is compiled with the +** [SQLITE_ENABLE_SNAPSHOT] option. */ SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_cmp( sqlite3_snapshot *p1, @@ -10020,23 +10690,26 @@ SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_cmp( /* ** CAPI3REF: Recover snapshots from a wal file -** EXPERIMENTAL +** METHOD: sqlite3_snapshot ** -** If all connections disconnect from a database file but do not perform -** a checkpoint, the existing wal file is opened along with the database -** file the next time the database is opened. At this point it is only -** possible to successfully call sqlite3_snapshot_open() to open the most -** recent snapshot of the database (the one at the head of the wal file), -** even though the wal file may contain other valid snapshots for which -** clients have sqlite3_snapshot handles. +** If a [WAL file] remains on disk after all database connections close +** (either through the use of the [SQLITE_FCNTL_PERSIST_WAL] [file control] +** or because the last process to have the database opened exited without +** calling [sqlite3_close()]) and a new connection is subsequently opened +** on that database and [WAL file], the [sqlite3_snapshot_open()] interface +** will only be able to open the last transaction added to the WAL file +** even though the WAL file contains other valid transactions. ** -** This function attempts to scan the wal file associated with database zDb +** This function attempts to scan the WAL file associated with database zDb ** of database handle db and make all valid snapshots available to ** sqlite3_snapshot_open(). It is an error if there is already a read -** transaction open on the database, or if the database is not a wal mode +** transaction open on the database, or if the database is not a WAL mode ** database. ** ** SQLITE_OK is returned if successful, or an SQLite error code otherwise. +** +** This interface is only available if SQLite is compiled with the +** [SQLITE_ENABLE_SNAPSHOT] option. */ SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_recover(sqlite3 *db, const char *zDb); @@ -10147,7 +10820,7 @@ SQLITE_API int sqlite3_deserialize( ** in the P argument is held in memory obtained from [sqlite3_malloc64()] ** and that SQLite should take ownership of this memory and automatically ** free it when it has finished using it. Without this flag, the caller -** is resposible for freeing any dynamically allocated memory. +** is responsible for freeing any dynamically allocated memory. ** ** The SQLITE_DESERIALIZE_RESIZEABLE flag means that SQLite is allowed to ** grow the size of the database using calls to [sqlite3_realloc64()]. This @@ -10273,7 +10946,7 @@ struct sqlite3_rtree_query_info { sqlite3_int64 iRowid; /* Rowid for current entry */ sqlite3_rtree_dbl rParentScore; /* Score of parent node */ int eParentWithin; /* Visibility of parent node */ - int eWithin; /* OUT: Visiblity */ + int eWithin; /* OUT: Visibility */ sqlite3_rtree_dbl rScore; /* OUT: Write the score here */ /* The following fields are only available in 3.8.11 and later */ sqlite3_value **apSqlParam; /* Original SQL values of parameters */ @@ -10496,7 +11169,7 @@ SQLITE_API int sqlite3session_attach( ** The second argument (xFilter) is the "filter callback". For changes to rows ** in tables that are not attached to the Session object, the filter is called ** to determine whether changes to the table's rows should be tracked or not. -** If xFilter returns 0, changes is not tracked. Note that once a table is +** If xFilter returns 0, changes are not tracked. Note that once a table is ** attached, xFilter will not be called again. */ SQLITE_API void sqlite3session_table_filter( @@ -10670,7 +11343,7 @@ SQLITE_API int sqlite3session_changeset( ** It an error if database zFrom does not exist or does not contain the ** required compatible table. ** -** If the operation successful, SQLITE_OK is returned. Otherwise, an SQLite +** If the operation is successful, SQLITE_OK is returned. Otherwise, an SQLite ** error code. In this case, if argument pzErrMsg is not NULL, *pzErrMsg ** may be set to point to a buffer containing an English language error ** message. It is the responsibility of the caller to free this buffer using @@ -10769,19 +11442,45 @@ SQLITE_API int sqlite3session_isempty(sqlite3_session *pSession); ** consecutively. There is no chance that the iterator will visit a change ** the applies to table X, then one for table Y, and then later on visit ** another change for table X. +** +** The behavior of sqlite3changeset_start_v2() and its streaming equivalent +** may be modified by passing a combination of +** [SQLITE_CHANGESETSTART_INVERT | supported flags] as the 4th parameter. +** +** Note that the sqlite3changeset_start_v2() API is still experimental +** and therefore subject to change. */ SQLITE_API int sqlite3changeset_start( sqlite3_changeset_iter **pp, /* OUT: New changeset iterator handle */ int nChangeset, /* Size of changeset blob in bytes */ void *pChangeset /* Pointer to blob containing changeset */ ); +SQLITE_API int sqlite3changeset_start_v2( + sqlite3_changeset_iter **pp, /* OUT: New changeset iterator handle */ + int nChangeset, /* Size of changeset blob in bytes */ + void *pChangeset, /* Pointer to blob containing changeset */ + int flags /* SESSION_CHANGESETSTART_* flags */ +); + +/* +** CAPI3REF: Flags for sqlite3changeset_start_v2 +** +** The following flags may passed via the 4th parameter to +** [sqlite3changeset_start_v2] and [sqlite3changeset_start_v2_strm]: +** +**
      SQLITE_CHANGESETAPPLY_INVERT
      +** Invert the changeset while iterating through it. This is equivalent to +** inverting a changeset using sqlite3changeset_invert() before applying it. +** It is an error to specify this flag with a patchset. +*/ +#define SQLITE_CHANGESETSTART_INVERT 0x0002 /* ** CAPI3REF: Advance A Changeset Iterator ** METHOD: sqlite3_changeset_iter ** -** This function may only be used with iterators created by function +** This function may only be used with iterators created by the function ** [sqlite3changeset_start()]. If it is called on an iterator passed to ** a conflict-handler callback by [sqlite3changeset_apply()], SQLITE_MISUSE ** is returned and the call has no effect. @@ -10818,7 +11517,7 @@ SQLITE_API int sqlite3changeset_next(sqlite3_changeset_iter *pIter); ** sqlite3changeset_next() is called on the iterator or until the ** conflict-handler function returns. If pnCol is not NULL, then *pnCol is ** set to the number of columns in the table affected by the change. If -** pbIncorrect is not NULL, then *pbIndirect is set to true (1) if the change +** pbIndirect is not NULL, then *pbIndirect is set to true (1) if the change ** is an indirect change, or false (0) otherwise. See the documentation for ** [sqlite3session_indirect()] for a description of direct and indirect ** changes. Finally, if pOp is not NULL, then *pOp is set to one of @@ -11197,8 +11896,8 @@ SQLITE_API int sqlite3changegroup_new(sqlite3_changegroup **pp); ** case, this function fails with SQLITE_SCHEMA. If the input changeset ** appears to be corrupt and the corruption is detected, SQLITE_CORRUPT is ** returned. Or, if an out-of-memory condition occurs during processing, this -** function returns SQLITE_NOMEM. In all cases, if an error occurs the -** final contents of the changegroup is undefined. +** function returns SQLITE_NOMEM. In all cases, if an error occurs the state +** of the final contents of the changegroup is undefined. ** ** If no error occurs, SQLITE_OK is returned. */ @@ -11373,7 +12072,7 @@ SQLITE_API void sqlite3changegroup_delete(sqlite3_changegroup*); ** ** It is safe to execute SQL statements, including those that write to the ** table that the callback related to, from within the xConflict callback. -** This can be used to further customize the applications conflict +** This can be used to further customize the application's conflict ** resolution strategy. ** ** All changes made by these functions are enclosed in a savepoint transaction. @@ -11429,7 +12128,7 @@ SQLITE_API int sqlite3changeset_apply_v2( ), void *pCtx, /* First argument passed to xConflict */ void **ppRebase, int *pnRebase, /* OUT: Rebase data */ - int flags /* Combination of SESSION_APPLY_* flags */ + int flags /* SESSION_CHANGESETAPPLY_* flags */ ); /* @@ -11447,8 +12146,14 @@ SQLITE_API int sqlite3changeset_apply_v2( ** causes the sessions module to omit this savepoint. In this case, if the ** caller has an open transaction or savepoint when apply_v2() is called, ** it may revert the partially applied changeset by rolling it back. +** +**
      SQLITE_CHANGESETAPPLY_INVERT
      +** Invert the changeset before applying it. This is equivalent to inverting +** a changeset using sqlite3changeset_invert() before applying it. It is +** an error to specify this flag with a patchset. */ #define SQLITE_CHANGESETAPPLY_NOSAVEPOINT 0x0001 +#define SQLITE_CHANGESETAPPLY_INVERT 0x0002 /* ** CAPI3REF: Constants Passed To The Conflict Handler @@ -11677,9 +12382,9 @@ SQLITE_API int sqlite3rebaser_configure( ** ** Argument pIn must point to a buffer containing a changeset nIn bytes ** in size. This function allocates and populates a buffer with a copy -** of the changeset rebased rebased according to the configuration of the +** of the changeset rebased according to the configuration of the ** rebaser object passed as the first argument. If successful, (*ppOut) -** is set to point to the new buffer containing the rebased changset and +** is set to point to the new buffer containing the rebased changeset and ** (*pnOut) to its size in bytes and SQLITE_OK returned. It is the ** responsibility of the caller to eventually free the new buffer using ** sqlite3_free(). Otherwise, if an error occurs, (*ppOut) and (*pnOut) @@ -11842,6 +12547,12 @@ SQLITE_API int sqlite3changeset_start_strm( int (*xInput)(void *pIn, void *pData, int *pnData), void *pIn ); +SQLITE_API int sqlite3changeset_start_v2_strm( + sqlite3_changeset_iter **pp, + int (*xInput)(void *pIn, void *pData, int *pnData), + void *pIn, + int flags +); SQLITE_API int sqlite3session_changeset_strm( sqlite3_session *pSession, int (*xOutput)(void *pOut, const void *pData, int nData), @@ -11868,6 +12579,45 @@ SQLITE_API int sqlite3rebaser_rebase_strm( void *pOut ); +/* +** CAPI3REF: Configure global parameters +** +** The sqlite3session_config() interface is used to make global configuration +** changes to the sessions module in order to tune it to the specific needs +** of the application. +** +** The sqlite3session_config() interface is not threadsafe. If it is invoked +** while any other thread is inside any other sessions method then the +** results are undefined. Furthermore, if it is invoked after any sessions +** related objects have been created, the results are also undefined. +** +** The first argument to the sqlite3session_config() function must be one +** of the SQLITE_SESSION_CONFIG_XXX constants defined below. The +** interpretation of the (void*) value passed as the second parameter and +** the effect of calling this function depends on the value of the first +** parameter. +** +**
      +**
      SQLITE_SESSION_CONFIG_STRMSIZE
      +** By default, the sessions module streaming interfaces attempt to input +** and output data in approximately 1 KiB chunks. This operand may be used +** to set and query the value of this configuration setting. The pointer +** passed as the second argument must point to a value of type (int). +** If this value is greater than 0, it is used as the new streaming data +** chunk size for both input and output. Before returning, the (int) value +** pointed to by pArg is set to the final value of the streaming interface +** chunk size. +**
      +** +** This function returns SQLITE_OK if successful, or an SQLite error code +** otherwise. +*/ +SQLITE_API int sqlite3session_config(int op, void *pArg); + +/* +** CAPI3REF: Values for sqlite3session_config(). +*/ +#define SQLITE_SESSION_CONFIG_STRMSIZE 1 /* ** Make sure we can call this stuff from C++. @@ -12001,12 +12751,8 @@ struct Fts5PhraseIter { ** ** Usually, output parameter *piPhrase is set to the phrase number, *piCol ** to the column in which it occurs and *piOff the token offset of the -** first token of the phrase. The exception is if the table was created -** with the offsets=0 option specified. In this case *piOff is always -** set to -1. -** -** Returns SQLITE_OK if successful, or an error code (i.e. SQLITE_NOMEM) -** if an error occurs. +** first token of the phrase. Returns SQLITE_OK if successful, or an error +** code (i.e. SQLITE_NOMEM) if an error occurs. ** ** This API can be quite slow if used with an FTS5 table created with the ** "detail=none" or "detail=column" option. @@ -12044,10 +12790,10 @@ struct Fts5PhraseIter { ** ** xSetAuxdata(pFts5, pAux, xDelete) ** -** Save the pointer passed as the second argument as the extension functions +** Save the pointer passed as the second argument as the extension function's ** "auxiliary data". The pointer may then be retrieved by the current or any ** future invocation of the same fts5 extension function made as part of -** of the same MATCH query using the xGetAuxdata() API. +** the same MATCH query using the xGetAuxdata() API. ** ** Each extension function is allocated a single auxiliary data slot for ** each FTS query (MATCH expression). If the extension function is invoked @@ -12062,7 +12808,7 @@ struct Fts5PhraseIter { ** The xDelete callback, if one is specified, is also invoked on the ** auxiliary data pointer after the FTS5 query has finished. ** -** If an error (e.g. an OOM condition) occurs within this function, an +** If an error (e.g. an OOM condition) occurs within this function, ** the auxiliary data is set to NULL and an error code returned. If the ** xDelete parameter was not NULL, it is invoked on the auxiliary data ** pointer before returning. @@ -12286,8 +13032,8 @@ struct Fts5ExtensionApi { ** ** There are several ways to approach this in FTS5: ** -**
      1. By mapping all synonyms to a single token. In this case, the -** In the above example, this means that the tokenizer returns the +**
        1. By mapping all synonyms to a single token. In this case, using +** the above example, this means that the tokenizer returns the ** same token for inputs "first" and "1st". Say that token is in ** fact "first", so that when the user inserts the document "I won ** 1st place" entries are added to the index for tokens "i", "won", @@ -12295,11 +13041,11 @@ struct Fts5ExtensionApi { ** the tokenizer substitutes "first" for "1st" and the query works ** as expected. ** -**
        2. By adding multiple synonyms for a single term to the FTS index. -** In this case, when tokenizing query text, the tokenizer may -** provide multiple synonyms for a single term within the document. -** FTS5 then queries the index for each synonym individually. For -** example, faced with the query: +**
        3. By querying the index for all synonyms of each query term +** separately. In this case, when tokenizing query text, the +** tokenizer may provide multiple synonyms for a single term +** within the document. FTS5 then queries the index for each +** synonym individually. For example, faced with the query: ** ** ** ... MATCH 'first place' @@ -12323,9 +13069,9 @@ struct Fts5ExtensionApi { ** "place". ** ** This way, even if the tokenizer does not provide synonyms -** when tokenizing query text (it should not - to do would be +** when tokenizing query text (it should not - to do so would be ** inefficient), it doesn't matter if the user queries for -** 'first + place' or '1st + place', as there are entires in the +** 'first + place' or '1st + place', as there are entries in the ** FTS index corresponding to both forms of the first token. **
        ** @@ -12353,7 +13099,7 @@ struct Fts5ExtensionApi { ** extra data to the FTS index or require FTS5 to query for multiple terms, ** so it is efficient in terms of disk space and query speed. However, it ** does not support prefix queries very well. If, as suggested above, the -** token "first" is subsituted for "1st" by the tokenizer, then the query: +** token "first" is substituted for "1st" by the tokenizer, then the query: ** ** ** ... MATCH '1s*' @@ -12608,9 +13354,12 @@ struct fts5_api { /* ** The maximum value of a ?nnn wildcard that the parser will accept. +** If the value exceeds 32767 then extra space is required for the Expr +** structure. But otherwise, we believe that the number can be as large +** as a signed 32-bit integer can hold. */ #ifndef SQLITE_MAX_VARIABLE_NUMBER -# define SQLITE_MAX_VARIABLE_NUMBER 999 +# define SQLITE_MAX_VARIABLE_NUMBER 32766 #endif /* Maximum page size. The upper bound on this value is 65536. This a limit @@ -12699,6 +13448,21 @@ struct fts5_api { #pragma warn -spa /* Suspicious pointer arithmetic */ #endif +/* +** WAL mode depends on atomic aligned 32-bit loads and stores in a few +** places. The following macros try to make this explicit. +*/ +#ifndef __has_feature +# define __has_feature(x) 0 /* compatibility with non-clang compilers */ +#endif +#if GCC_VERSION>=4007000 || __has_feature(c_atomic) +# define AtomicLoad(PTR) __atomic_load_n((PTR),__ATOMIC_RELAXED) +# define AtomicStore(PTR,VAL) __atomic_store_n((PTR),(VAL),__ATOMIC_RELAXED) +#else +# define AtomicLoad(PTR) (*(PTR)) +# define AtomicStore(PTR,VAL) (*(PTR) = (VAL)) +#endif + /* ** Include standard header files as necessary */ @@ -12725,15 +13489,15 @@ struct fts5_api { ** So we have to define the macros in different ways depending on the ** compiler. */ -#if defined(__PTRDIFF_TYPE__) /* This case should work for GCC */ +#if defined(HAVE_STDINT_H) /* Use this case if we have ANSI headers */ +# define SQLITE_INT_TO_PTR(X) ((void*)(intptr_t)(X)) +# define SQLITE_PTR_TO_INT(X) ((int)(intptr_t)(X)) +#elif defined(__PTRDIFF_TYPE__) /* This case should work for GCC */ # define SQLITE_INT_TO_PTR(X) ((void*)(__PTRDIFF_TYPE__)(X)) # define SQLITE_PTR_TO_INT(X) ((int)(__PTRDIFF_TYPE__)(X)) #elif !defined(__GNUC__) /* Works for compilers other than LLVM */ # define SQLITE_INT_TO_PTR(X) ((void*)&((char*)0)[X]) # define SQLITE_PTR_TO_INT(X) ((int)(((char*)X)-(char*)0)) -#elif defined(HAVE_STDINT_H) /* Use this case if we have ANSI headers */ -# define SQLITE_INT_TO_PTR(X) ((void*)(intptr_t)(X)) -# define SQLITE_PTR_TO_INT(X) ((int)(intptr_t)(X)) #else /* Generates a warning - but it always works */ # define SQLITE_INT_TO_PTR(X) ((void*)(X)) # define SQLITE_PTR_TO_INT(X) ((int)(X)) @@ -12959,6 +13723,26 @@ SQLITE_PRIVATE void sqlite3Coverage(int); # define NEVER(X) (X) #endif +/* +** The harmless(X) macro indicates that expression X is usually false +** but can be true without causing any problems, but we don't know of +** any way to cause X to be true. +** +** In debugging and testing builds, this macro will abort if X is ever +** true. In this way, developers are alerted to a possible test case +** that causes X to be true. If a harmless macro ever fails, that is +** an opportunity to change the macro into a testcase() and add a new +** test case to the test suite. +** +** For normal production builds, harmless(X) is a no-op, since it does +** not matter whether expression X is true or false. +*/ +#ifdef SQLITE_DEBUG +# define harmless(X) assert(!(X)); +#else +# define harmless(X) +#endif + /* ** Some conditionals are optimizations only. In other words, if the ** conditionals are replaced with a constant 1 (true) or 0 (false) then @@ -13088,7 +13872,7 @@ struct Hash { unsigned int count; /* Number of entries in this table */ HashElem *first; /* The first element of the array */ struct _ht { /* the hash table */ - int count; /* Number of entries with this hash */ + unsigned int count; /* Number of entries with this hash */ HashElem *chain; /* Pointer to first entry with this hash */ } *ht; }; @@ -13217,101 +14001,111 @@ SQLITE_PRIVATE void sqlite3HashClear(Hash*); #define TK_REPLACE 73 #define TK_RESTRICT 74 #define TK_ROW 75 -#define TK_TRIGGER 76 -#define TK_VACUUM 77 -#define TK_VIEW 78 -#define TK_VIRTUAL 79 -#define TK_WITH 80 -#define TK_REINDEX 81 -#define TK_RENAME 82 -#define TK_CTIME_KW 83 -#define TK_ANY 84 -#define TK_BITAND 85 -#define TK_BITOR 86 -#define TK_LSHIFT 87 -#define TK_RSHIFT 88 -#define TK_PLUS 89 -#define TK_MINUS 90 -#define TK_STAR 91 -#define TK_SLASH 92 -#define TK_REM 93 -#define TK_CONCAT 94 -#define TK_COLLATE 95 -#define TK_BITNOT 96 -#define TK_ON 97 -#define TK_INDEXED 98 -#define TK_STRING 99 -#define TK_JOIN_KW 100 -#define TK_CONSTRAINT 101 -#define TK_DEFAULT 102 -#define TK_NULL 103 -#define TK_PRIMARY 104 -#define TK_UNIQUE 105 -#define TK_CHECK 106 -#define TK_REFERENCES 107 -#define TK_AUTOINCR 108 -#define TK_INSERT 109 -#define TK_DELETE 110 -#define TK_UPDATE 111 -#define TK_SET 112 -#define TK_DEFERRABLE 113 -#define TK_FOREIGN 114 -#define TK_DROP 115 -#define TK_UNION 116 -#define TK_ALL 117 -#define TK_EXCEPT 118 -#define TK_INTERSECT 119 -#define TK_SELECT 120 -#define TK_VALUES 121 -#define TK_DISTINCT 122 -#define TK_DOT 123 -#define TK_FROM 124 -#define TK_JOIN 125 -#define TK_USING 126 -#define TK_ORDER 127 -#define TK_GROUP 128 -#define TK_HAVING 129 -#define TK_LIMIT 130 -#define TK_WHERE 131 -#define TK_INTO 132 -#define TK_NOTHING 133 -#define TK_FLOAT 134 -#define TK_BLOB 135 -#define TK_INTEGER 136 -#define TK_VARIABLE 137 -#define TK_CASE 138 -#define TK_WHEN 139 -#define TK_THEN 140 -#define TK_ELSE 141 -#define TK_INDEX 142 -#define TK_ALTER 143 -#define TK_ADD 144 -#define TK_TRUEFALSE 145 -#define TK_ISNOT 146 -#define TK_FUNCTION 147 -#define TK_COLUMN 148 -#define TK_AGG_FUNCTION 149 -#define TK_AGG_COLUMN 150 -#define TK_UMINUS 151 -#define TK_UPLUS 152 -#define TK_TRUTH 153 -#define TK_REGISTER 154 -#define TK_VECTOR 155 -#define TK_SELECT_COLUMN 156 -#define TK_IF_NULL_ROW 157 -#define TK_ASTERISK 158 -#define TK_SPAN 159 -#define TK_END_OF_FILE 160 -#define TK_UNCLOSED_STRING 161 -#define TK_SPACE 162 -#define TK_ILLEGAL 163 - -/* The token codes above must all fit in 8 bits */ -#define TKFLG_MASK 0xff - -/* Flags that can be added to a token code when it is not -** being stored in a u8: */ -#define TKFLG_DONTFOLD 0x100 /* Omit constant folding optimizations */ +#define TK_ROWS 76 +#define TK_TRIGGER 77 +#define TK_VACUUM 78 +#define TK_VIEW 79 +#define TK_VIRTUAL 80 +#define TK_WITH 81 +#define TK_NULLS 82 +#define TK_FIRST 83 +#define TK_LAST 84 +#define TK_CURRENT 85 +#define TK_FOLLOWING 86 +#define TK_PARTITION 87 +#define TK_PRECEDING 88 +#define TK_RANGE 89 +#define TK_UNBOUNDED 90 +#define TK_EXCLUDE 91 +#define TK_GROUPS 92 +#define TK_OTHERS 93 +#define TK_TIES 94 +#define TK_GENERATED 95 +#define TK_ALWAYS 96 +#define TK_REINDEX 97 +#define TK_RENAME 98 +#define TK_CTIME_KW 99 +#define TK_ANY 100 +#define TK_BITAND 101 +#define TK_BITOR 102 +#define TK_LSHIFT 103 +#define TK_RSHIFT 104 +#define TK_PLUS 105 +#define TK_MINUS 106 +#define TK_STAR 107 +#define TK_SLASH 108 +#define TK_REM 109 +#define TK_CONCAT 110 +#define TK_COLLATE 111 +#define TK_BITNOT 112 +#define TK_ON 113 +#define TK_INDEXED 114 +#define TK_STRING 115 +#define TK_JOIN_KW 116 +#define TK_CONSTRAINT 117 +#define TK_DEFAULT 118 +#define TK_NULL 119 +#define TK_PRIMARY 120 +#define TK_UNIQUE 121 +#define TK_CHECK 122 +#define TK_REFERENCES 123 +#define TK_AUTOINCR 124 +#define TK_INSERT 125 +#define TK_DELETE 126 +#define TK_UPDATE 127 +#define TK_SET 128 +#define TK_DEFERRABLE 129 +#define TK_FOREIGN 130 +#define TK_DROP 131 +#define TK_UNION 132 +#define TK_ALL 133 +#define TK_EXCEPT 134 +#define TK_INTERSECT 135 +#define TK_SELECT 136 +#define TK_VALUES 137 +#define TK_DISTINCT 138 +#define TK_DOT 139 +#define TK_FROM 140 +#define TK_JOIN 141 +#define TK_USING 142 +#define TK_ORDER 143 +#define TK_GROUP 144 +#define TK_HAVING 145 +#define TK_LIMIT 146 +#define TK_WHERE 147 +#define TK_INTO 148 +#define TK_NOTHING 149 +#define TK_FLOAT 150 +#define TK_BLOB 151 +#define TK_INTEGER 152 +#define TK_VARIABLE 153 +#define TK_CASE 154 +#define TK_WHEN 155 +#define TK_THEN 156 +#define TK_ELSE 157 +#define TK_INDEX 158 +#define TK_ALTER 159 +#define TK_ADD 160 +#define TK_WINDOW 161 +#define TK_OVER 162 +#define TK_FILTER 163 +#define TK_COLUMN 164 +#define TK_AGG_FUNCTION 165 +#define TK_AGG_COLUMN 166 +#define TK_TRUEFALSE 167 +#define TK_ISNOT 168 +#define TK_FUNCTION 169 +#define TK_UMINUS 170 +#define TK_UPLUS 171 +#define TK_TRUTH 172 +#define TK_REGISTER 173 +#define TK_VECTOR 174 +#define TK_SELECT_COLUMN 175 +#define TK_IF_NULL_ROW 176 +#define TK_ASTERISK 177 +#define TK_SPAN 178 +#define TK_SPACE 179 +#define TK_ILLEGAL 180 /************** End of parse.h ***********************************************/ /************** Continuing where we left off in sqliteInt.h ******************/ @@ -13578,7 +14372,8 @@ typedef INT16_TYPE LogEst; # if defined(__SIZEOF_POINTER__) # define SQLITE_PTRSIZE __SIZEOF_POINTER__ # elif defined(i386) || defined(__i386__) || defined(_M_IX86) || \ - defined(_M_ARM) || defined(__arm__) || defined(__x86) + defined(_M_ARM) || defined(__arm__) || defined(__x86) || \ + (defined(__TOS_AIX__) && !defined(__64BIT__)) # define SQLITE_PTRSIZE 4 # else # define SQLITE_PTRSIZE 8 @@ -13616,12 +14411,13 @@ typedef INT16_TYPE LogEst; ** at run-time. */ #ifndef SQLITE_BYTEORDER -# if defined(i386) || defined(__i386__) || defined(_M_IX86) || \ - defined(__x86_64) || defined(__x86_64__) || defined(_M_X64) || \ - defined(_M_AMD64) || defined(_M_ARM) || defined(__x86) || \ - defined(__arm__) +# if defined(i386) || defined(__i386__) || defined(_M_IX86) || \ + defined(__x86_64) || defined(__x86_64__) || defined(_M_X64) || \ + defined(_M_AMD64) || defined(_M_ARM) || defined(__x86) || \ + defined(__ARMEL__) || defined(__AARCH64EL__) || defined(_M_ARM64) # define SQLITE_BYTEORDER 1234 -# elif defined(sparc) || defined(__ppc__) +# elif defined(sparc) || defined(__ppc__) || \ + defined(__ARMEB__) || defined(__AARCH64EB__) # define SQLITE_BYTEORDER 4321 # else # define SQLITE_BYTEORDER 0 @@ -13720,20 +14516,6 @@ typedef INT16_TYPE LogEst; # define SQLITE_DEFAULT_MMAP_SIZE SQLITE_MAX_MMAP_SIZE #endif -/* -** Only one of SQLITE_ENABLE_STAT3 or SQLITE_ENABLE_STAT4 can be defined. -** Priority is given to SQLITE_ENABLE_STAT4. If either are defined, also -** define SQLITE_ENABLE_STAT3_OR_STAT4 -*/ -#ifdef SQLITE_ENABLE_STAT4 -# undef SQLITE_ENABLE_STAT3 -# define SQLITE_ENABLE_STAT3_OR_STAT4 1 -#elif SQLITE_ENABLE_STAT3 -# define SQLITE_ENABLE_STAT3_OR_STAT4 1 -#elif SQLITE_ENABLE_STAT3_OR_STAT4 -# undef SQLITE_ENABLE_STAT3_OR_STAT4 -#endif - /* ** SELECTTRACE_ENABLED will be either 1 or 0 depending on whether or not ** the Select query generator tracing logic is turned on. @@ -13758,7 +14540,6 @@ struct BusyHandler { int (*xBusyHandler)(void *,int); /* The busy callback */ void *pBusyArg; /* First arg to busy callback */ int nBusy; /* Incremented with each busy call */ - u8 bExtraFileArg; /* Include sqlite3_file as callback arg */ }; /* @@ -13874,6 +14655,7 @@ typedef struct NameContext NameContext; typedef struct Parse Parse; typedef struct PreUpdate PreUpdate; typedef struct PrintfArguments PrintfArguments; +typedef struct RenameToken RenameToken; typedef struct RowSet RowSet; typedef struct Savepoint Savepoint; typedef struct Select Select; @@ -13894,8 +14676,36 @@ typedef struct VTable VTable; typedef struct VtabCtx VtabCtx; typedef struct Walker Walker; typedef struct WhereInfo WhereInfo; +typedef struct Window Window; typedef struct With With; + +/* +** The bitmask datatype defined below is used for various optimizations. +** +** Changing this from a 64-bit to a 32-bit type limits the number of +** tables in a join to 32 instead of 64. But it also reduces the size +** of the library by 738 bytes on ix86. +*/ +#ifdef SQLITE_BITMASK_TYPE + typedef SQLITE_BITMASK_TYPE Bitmask; +#else + typedef u64 Bitmask; +#endif + +/* +** The number of bits in a Bitmask. "BMS" means "BitMask Size". +*/ +#define BMS ((int)(sizeof(Bitmask)*8)) + +/* +** A bit in a Bitmask +*/ +#define MASKBIT(n) (((Bitmask)1)<<(n)) +#define MASKBIT64(n) (((u64)1)<<(n)) +#define MASKBIT32(n) (((unsigned int)1)<<(n)) +#define ALLBITS ((Bitmask)-1) + /* A VList object records a mapping between parameters/variables/wildcards ** in the SQL statement (such as $abc, @pqr, or :xyz) and the integer ** variable number associated with that parameter. See the format description @@ -13987,11 +14797,11 @@ SQLITE_PRIVATE int sqlite3BtreeGetPageSize(Btree*); SQLITE_PRIVATE int sqlite3BtreeMaxPageCount(Btree*,int); SQLITE_PRIVATE u32 sqlite3BtreeLastPage(Btree*); SQLITE_PRIVATE int sqlite3BtreeSecureDelete(Btree*,int); -SQLITE_PRIVATE int sqlite3BtreeGetOptimalReserve(Btree*); +SQLITE_PRIVATE int sqlite3BtreeGetRequestedReserve(Btree*); SQLITE_PRIVATE int sqlite3BtreeGetReserveNoMutex(Btree *p); SQLITE_PRIVATE int sqlite3BtreeSetAutoVacuum(Btree *, int); SQLITE_PRIVATE int sqlite3BtreeGetAutoVacuum(Btree *); -SQLITE_PRIVATE int sqlite3BtreeBeginTrans(Btree*,int); +SQLITE_PRIVATE int sqlite3BtreeBeginTrans(Btree*,int,int*); SQLITE_PRIVATE int sqlite3BtreeCommitPhaseOne(Btree*, const char *zMaster); SQLITE_PRIVATE int sqlite3BtreeCommitPhaseTwo(Btree*, int); SQLITE_PRIVATE int sqlite3BtreeCommit(Btree*); @@ -14219,14 +15029,17 @@ SQLITE_PRIVATE int sqlite3BtreeNext(BtCursor*, int flags); SQLITE_PRIVATE int sqlite3BtreeEof(BtCursor*); SQLITE_PRIVATE int sqlite3BtreePrevious(BtCursor*, int flags); SQLITE_PRIVATE i64 sqlite3BtreeIntegerKey(BtCursor*); +SQLITE_PRIVATE void sqlite3BtreeCursorPin(BtCursor*); +SQLITE_PRIVATE void sqlite3BtreeCursorUnpin(BtCursor*); #ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC SQLITE_PRIVATE i64 sqlite3BtreeOffset(BtCursor*); #endif SQLITE_PRIVATE int sqlite3BtreePayload(BtCursor*, u32 offset, u32 amt, void*); SQLITE_PRIVATE const void *sqlite3BtreePayloadFetch(BtCursor*, u32 *pAmt); SQLITE_PRIVATE u32 sqlite3BtreePayloadSize(BtCursor*); +SQLITE_PRIVATE sqlite3_int64 sqlite3BtreeMaxRecordSize(BtCursor*); -SQLITE_PRIVATE char *sqlite3BtreeIntegrityCheck(Btree*, int *aRoot, int nRoot, int, int*); +SQLITE_PRIVATE char *sqlite3BtreeIntegrityCheck(sqlite3*,Btree*,int*aRoot,int nRoot,int,int*); SQLITE_PRIVATE struct Pager *sqlite3BtreePager(Btree*); SQLITE_PRIVATE i64 sqlite3BtreeRowCountEst(BtCursor*); @@ -14246,9 +15059,7 @@ SQLITE_PRIVATE int sqlite3BtreeCursorIsValid(BtCursor*); #endif SQLITE_PRIVATE int sqlite3BtreeCursorIsValidNN(BtCursor*); -#ifndef SQLITE_OMIT_BTREECOUNT -SQLITE_PRIVATE int sqlite3BtreeCount(BtCursor *, i64 *); -#endif +SQLITE_PRIVATE int sqlite3BtreeCount(sqlite3*, BtCursor*, i64*); #ifdef SQLITE_TEST SQLITE_PRIVATE int sqlite3BtreeCursorInfo(BtCursor*, int*, int); @@ -14381,7 +15192,8 @@ struct VdbeOp { u64 cycles; /* Total time spent executing this instruction */ #endif #ifdef SQLITE_VDBE_COVERAGE - int iSrcLine; /* Source-code line that generated this opcode */ + u32 iSrcLine; /* Source-code line that generated this opcode + ** with flags in the upper 8 bits */ #endif }; typedef struct VdbeOp VdbeOp; @@ -14463,12 +15275,11 @@ typedef struct VdbeOpList VdbeOpList; #endif /* -** The following macro converts a relative address in the p2 field -** of a VdbeOp structure into a negative number so that -** sqlite3VdbeAddOpList() knows that the address is relative. Calling -** the macro again restores the address. +** The following macro converts a label returned by sqlite3VdbeMakeLabel() +** into an index into the Parse.aLabel[] array that contains the resolved +** address of that label. */ -#define ADDR(X) (-1-(X)) +#define ADDR(X) (~(X)) /* ** The makefile scans the vdbe.c source file and creates the "opcodes.h" @@ -14482,30 +15293,30 @@ typedef struct VdbeOpList VdbeOpList; #define OP_AutoCommit 1 #define OP_Transaction 2 #define OP_SorterNext 3 /* jump */ -#define OP_PrevIfOpen 4 /* jump */ -#define OP_NextIfOpen 5 /* jump */ -#define OP_Prev 6 /* jump */ -#define OP_Next 7 /* jump */ -#define OP_Checkpoint 8 -#define OP_JournalMode 9 -#define OP_Vacuum 10 -#define OP_VFilter 11 /* jump, synopsis: iplan=r[P3] zplan='P4' */ -#define OP_VUpdate 12 /* synopsis: data=r[P3@P2] */ -#define OP_Goto 13 /* jump */ -#define OP_Gosub 14 /* jump */ -#define OP_InitCoroutine 15 /* jump */ -#define OP_Yield 16 /* jump */ -#define OP_MustBeInt 17 /* jump */ -#define OP_Jump 18 /* jump */ +#define OP_Prev 4 /* jump */ +#define OP_Next 5 /* jump */ +#define OP_Checkpoint 6 +#define OP_JournalMode 7 +#define OP_Vacuum 8 +#define OP_VFilter 9 /* jump, synopsis: iplan=r[P3] zplan='P4' */ +#define OP_VUpdate 10 /* synopsis: data=r[P3@P2] */ +#define OP_Goto 11 /* jump */ +#define OP_Gosub 12 /* jump */ +#define OP_InitCoroutine 13 /* jump */ +#define OP_Yield 14 /* jump */ +#define OP_MustBeInt 15 /* jump */ +#define OP_Jump 16 /* jump */ +#define OP_Once 17 /* jump */ +#define OP_If 18 /* jump */ #define OP_Not 19 /* same as TK_NOT, synopsis: r[P2]= !r[P1] */ -#define OP_Once 20 /* jump */ -#define OP_If 21 /* jump */ -#define OP_IfNot 22 /* jump */ -#define OP_IfNullRow 23 /* jump, synopsis: if P1.nullRow then r[P3]=NULL, goto P2 */ -#define OP_SeekLT 24 /* jump, synopsis: key=r[P3@P4] */ -#define OP_SeekLE 25 /* jump, synopsis: key=r[P3@P4] */ -#define OP_SeekGE 26 /* jump, synopsis: key=r[P3@P4] */ -#define OP_SeekGT 27 /* jump, synopsis: key=r[P3@P4] */ +#define OP_IfNot 20 /* jump */ +#define OP_IfNullRow 21 /* jump, synopsis: if P1.nullRow then r[P3]=NULL, goto P2 */ +#define OP_SeekLT 22 /* jump, synopsis: key=r[P3@P4] */ +#define OP_SeekLE 23 /* jump, synopsis: key=r[P3@P4] */ +#define OP_SeekGE 24 /* jump, synopsis: key=r[P3@P4] */ +#define OP_SeekGT 25 /* jump, synopsis: key=r[P3@P4] */ +#define OP_IfNotOpen 26 /* jump, synopsis: if( !csr[P1] ) goto P2 */ +#define OP_IfNoHope 27 /* jump, synopsis: key=r[P3@P4] */ #define OP_NoConflict 28 /* jump, synopsis: key=r[P3@P4] */ #define OP_NotFound 29 /* jump, synopsis: key=r[P3@P4] */ #define OP_Found 30 /* jump, synopsis: key=r[P3@P4] */ @@ -14541,115 +15352,119 @@ typedef struct VdbeOpList VdbeOpList; #define OP_IncrVacuum 60 /* jump */ #define OP_VNext 61 /* jump */ #define OP_Init 62 /* jump, synopsis: Start at P2 */ -#define OP_Return 63 -#define OP_EndCoroutine 64 -#define OP_HaltIfNull 65 /* synopsis: if r[P3]=null halt */ -#define OP_Halt 66 -#define OP_Integer 67 /* synopsis: r[P2]=P1 */ -#define OP_Int64 68 /* synopsis: r[P2]=P4 */ -#define OP_String 69 /* synopsis: r[P2]='P4' (len=P1) */ -#define OP_Null 70 /* synopsis: r[P2..P3]=NULL */ -#define OP_SoftNull 71 /* synopsis: r[P1]=NULL */ -#define OP_Blob 72 /* synopsis: r[P2]=P4 (len=P1) */ -#define OP_Variable 73 /* synopsis: r[P2]=parameter(P1,P4) */ -#define OP_Move 74 /* synopsis: r[P2@P3]=r[P1@P3] */ -#define OP_Copy 75 /* synopsis: r[P2@P3+1]=r[P1@P3+1] */ -#define OP_SCopy 76 /* synopsis: r[P2]=r[P1] */ -#define OP_IntCopy 77 /* synopsis: r[P2]=r[P1] */ -#define OP_ResultRow 78 /* synopsis: output=r[P1@P2] */ -#define OP_CollSeq 79 -#define OP_AddImm 80 /* synopsis: r[P1]=r[P1]+P2 */ -#define OP_RealAffinity 81 -#define OP_Cast 82 /* synopsis: affinity(r[P1]) */ -#define OP_Permutation 83 -#define OP_Compare 84 /* synopsis: r[P1@P3] <-> r[P2@P3] */ -#define OP_BitAnd 85 /* same as TK_BITAND, synopsis: r[P3]=r[P1]&r[P2] */ -#define OP_BitOr 86 /* same as TK_BITOR, synopsis: r[P3]=r[P1]|r[P2] */ -#define OP_ShiftLeft 87 /* same as TK_LSHIFT, synopsis: r[P3]=r[P2]<>r[P1] */ -#define OP_Add 89 /* same as TK_PLUS, synopsis: r[P3]=r[P1]+r[P2] */ -#define OP_Subtract 90 /* same as TK_MINUS, synopsis: r[P3]=r[P2]-r[P1] */ -#define OP_Multiply 91 /* same as TK_STAR, synopsis: r[P3]=r[P1]*r[P2] */ -#define OP_Divide 92 /* same as TK_SLASH, synopsis: r[P3]=r[P2]/r[P1] */ -#define OP_Remainder 93 /* same as TK_REM, synopsis: r[P3]=r[P2]%r[P1] */ -#define OP_Concat 94 /* same as TK_CONCAT, synopsis: r[P3]=r[P2]+r[P1] */ -#define OP_IsTrue 95 /* synopsis: r[P2] = coalesce(r[P1]==TRUE,P3) ^ P4 */ -#define OP_BitNot 96 /* same as TK_BITNOT, synopsis: r[P1]= ~r[P1] */ -#define OP_Offset 97 /* synopsis: r[P3] = sqlite_offset(P1) */ -#define OP_Column 98 /* synopsis: r[P3]=PX */ -#define OP_String8 99 /* same as TK_STRING, synopsis: r[P2]='P4' */ -#define OP_Affinity 100 /* synopsis: affinity(r[P1@P2]) */ -#define OP_MakeRecord 101 /* synopsis: r[P3]=mkrec(r[P1@P2]) */ -#define OP_Count 102 /* synopsis: r[P2]=count() */ -#define OP_ReadCookie 103 -#define OP_SetCookie 104 -#define OP_ReopenIdx 105 /* synopsis: root=P2 iDb=P3 */ -#define OP_OpenRead 106 /* synopsis: root=P2 iDb=P3 */ -#define OP_OpenWrite 107 /* synopsis: root=P2 iDb=P3 */ -#define OP_OpenDup 108 -#define OP_OpenAutoindex 109 /* synopsis: nColumn=P2 */ -#define OP_OpenEphemeral 110 /* synopsis: nColumn=P2 */ +#define OP_PureFunc 63 /* synopsis: r[P3]=func(r[P2@NP]) */ +#define OP_Function 64 /* synopsis: r[P3]=func(r[P2@NP]) */ +#define OP_Return 65 +#define OP_EndCoroutine 66 +#define OP_HaltIfNull 67 /* synopsis: if r[P3]=null halt */ +#define OP_Halt 68 +#define OP_Integer 69 /* synopsis: r[P2]=P1 */ +#define OP_Int64 70 /* synopsis: r[P2]=P4 */ +#define OP_String 71 /* synopsis: r[P2]='P4' (len=P1) */ +#define OP_Null 72 /* synopsis: r[P2..P3]=NULL */ +#define OP_SoftNull 73 /* synopsis: r[P1]=NULL */ +#define OP_Blob 74 /* synopsis: r[P2]=P4 (len=P1) */ +#define OP_Variable 75 /* synopsis: r[P2]=parameter(P1,P4) */ +#define OP_Move 76 /* synopsis: r[P2@P3]=r[P1@P3] */ +#define OP_Copy 77 /* synopsis: r[P2@P3+1]=r[P1@P3+1] */ +#define OP_SCopy 78 /* synopsis: r[P2]=r[P1] */ +#define OP_IntCopy 79 /* synopsis: r[P2]=r[P1] */ +#define OP_ResultRow 80 /* synopsis: output=r[P1@P2] */ +#define OP_CollSeq 81 +#define OP_AddImm 82 /* synopsis: r[P1]=r[P1]+P2 */ +#define OP_RealAffinity 83 +#define OP_Cast 84 /* synopsis: affinity(r[P1]) */ +#define OP_Permutation 85 +#define OP_Compare 86 /* synopsis: r[P1@P3] <-> r[P2@P3] */ +#define OP_IsTrue 87 /* synopsis: r[P2] = coalesce(r[P1]==TRUE,P3) ^ P4 */ +#define OP_Offset 88 /* synopsis: r[P3] = sqlite_offset(P1) */ +#define OP_Column 89 /* synopsis: r[P3]=PX */ +#define OP_Affinity 90 /* synopsis: affinity(r[P1@P2]) */ +#define OP_MakeRecord 91 /* synopsis: r[P3]=mkrec(r[P1@P2]) */ +#define OP_Count 92 /* synopsis: r[P2]=count() */ +#define OP_ReadCookie 93 +#define OP_SetCookie 94 +#define OP_ReopenIdx 95 /* synopsis: root=P2 iDb=P3 */ +#define OP_OpenRead 96 /* synopsis: root=P2 iDb=P3 */ +#define OP_OpenWrite 97 /* synopsis: root=P2 iDb=P3 */ +#define OP_OpenDup 98 +#define OP_OpenAutoindex 99 /* synopsis: nColumn=P2 */ +#define OP_OpenEphemeral 100 /* synopsis: nColumn=P2 */ +#define OP_BitAnd 101 /* same as TK_BITAND, synopsis: r[P3]=r[P1]&r[P2] */ +#define OP_BitOr 102 /* same as TK_BITOR, synopsis: r[P3]=r[P1]|r[P2] */ +#define OP_ShiftLeft 103 /* same as TK_LSHIFT, synopsis: r[P3]=r[P2]<>r[P1] */ +#define OP_Add 105 /* same as TK_PLUS, synopsis: r[P3]=r[P1]+r[P2] */ +#define OP_Subtract 106 /* same as TK_MINUS, synopsis: r[P3]=r[P2]-r[P1] */ +#define OP_Multiply 107 /* same as TK_STAR, synopsis: r[P3]=r[P1]*r[P2] */ +#define OP_Divide 108 /* same as TK_SLASH, synopsis: r[P3]=r[P2]/r[P1] */ +#define OP_Remainder 109 /* same as TK_REM, synopsis: r[P3]=r[P2]%r[P1] */ +#define OP_Concat 110 /* same as TK_CONCAT, synopsis: r[P3]=r[P2]+r[P1] */ #define OP_SorterOpen 111 -#define OP_SequenceTest 112 /* synopsis: if( cursor[P1].ctr++ ) pc = P2 */ -#define OP_OpenPseudo 113 /* synopsis: P3 columns in r[P2] */ -#define OP_Close 114 -#define OP_ColumnsUsed 115 -#define OP_Sequence 116 /* synopsis: r[P2]=cursor[P1].ctr++ */ -#define OP_NewRowid 117 /* synopsis: r[P2]=rowid */ -#define OP_Insert 118 /* synopsis: intkey=r[P3] data=r[P2] */ -#define OP_InsertInt 119 /* synopsis: intkey=P3 data=r[P2] */ -#define OP_Delete 120 -#define OP_ResetCount 121 -#define OP_SorterCompare 122 /* synopsis: if key(P1)!=trim(r[P3],P4) goto P2 */ -#define OP_SorterData 123 /* synopsis: r[P2]=data */ -#define OP_RowData 124 /* synopsis: r[P2]=data */ -#define OP_Rowid 125 /* synopsis: r[P2]=rowid */ -#define OP_NullRow 126 -#define OP_SeekEnd 127 -#define OP_SorterInsert 128 /* synopsis: key=r[P2] */ -#define OP_IdxInsert 129 /* synopsis: key=r[P2] */ -#define OP_IdxDelete 130 /* synopsis: key=r[P2@P3] */ -#define OP_DeferredSeek 131 /* synopsis: Move P3 to P1.rowid if needed */ -#define OP_IdxRowid 132 /* synopsis: r[P2]=rowid */ -#define OP_Destroy 133 -#define OP_Real 134 /* same as TK_FLOAT, synopsis: r[P2]=P4 */ -#define OP_Clear 135 -#define OP_ResetSorter 136 -#define OP_CreateBtree 137 /* synopsis: r[P2]=root iDb=P1 flags=P3 */ -#define OP_SqlExec 138 -#define OP_ParseSchema 139 -#define OP_LoadAnalysis 140 -#define OP_DropTable 141 -#define OP_DropIndex 142 -#define OP_DropTrigger 143 -#define OP_IntegrityCk 144 -#define OP_RowSetAdd 145 /* synopsis: rowset(P1)=r[P2] */ -#define OP_Param 146 -#define OP_FkCounter 147 /* synopsis: fkctr[P1]+=P2 */ -#define OP_MemMax 148 /* synopsis: r[P1]=max(r[P1],r[P2]) */ -#define OP_OffsetLimit 149 /* synopsis: if r[P1]>0 then r[P2]=r[P1]+max(0,r[P3]) else r[P2]=(-1) */ -#define OP_AggStep0 150 /* synopsis: accum=r[P3] step(r[P2@P5]) */ -#define OP_AggStep 151 /* synopsis: accum=r[P3] step(r[P2@P5]) */ -#define OP_AggFinal 152 /* synopsis: accum=r[P1] N=P2 */ -#define OP_Expire 153 -#define OP_TableLock 154 /* synopsis: iDb=P1 root=P2 write=P3 */ -#define OP_VBegin 155 -#define OP_VCreate 156 -#define OP_VDestroy 157 -#define OP_VOpen 158 -#define OP_VColumn 159 /* synopsis: r[P3]=vcolumn(P2) */ -#define OP_VRename 160 -#define OP_Pagecount 161 -#define OP_MaxPgcnt 162 -#define OP_PureFunc0 163 -#define OP_Function0 164 /* synopsis: r[P3]=func(r[P2@P5]) */ -#define OP_PureFunc 165 -#define OP_Function 166 /* synopsis: r[P3]=func(r[P2@P5]) */ -#define OP_Trace 167 -#define OP_CursorHint 168 -#define OP_Noop 169 -#define OP_Explain 170 -#define OP_Abortable 171 +#define OP_BitNot 112 /* same as TK_BITNOT, synopsis: r[P2]= ~r[P1] */ +#define OP_SequenceTest 113 /* synopsis: if( cursor[P1].ctr++ ) pc = P2 */ +#define OP_OpenPseudo 114 /* synopsis: P3 columns in r[P2] */ +#define OP_String8 115 /* same as TK_STRING, synopsis: r[P2]='P4' */ +#define OP_Close 116 +#define OP_ColumnsUsed 117 +#define OP_SeekHit 118 /* synopsis: seekHit=P2 */ +#define OP_Sequence 119 /* synopsis: r[P2]=cursor[P1].ctr++ */ +#define OP_NewRowid 120 /* synopsis: r[P2]=rowid */ +#define OP_Insert 121 /* synopsis: intkey=r[P3] data=r[P2] */ +#define OP_Delete 122 +#define OP_ResetCount 123 +#define OP_SorterCompare 124 /* synopsis: if key(P1)!=trim(r[P3],P4) goto P2 */ +#define OP_SorterData 125 /* synopsis: r[P2]=data */ +#define OP_RowData 126 /* synopsis: r[P2]=data */ +#define OP_Rowid 127 /* synopsis: r[P2]=rowid */ +#define OP_NullRow 128 +#define OP_SeekEnd 129 +#define OP_IdxInsert 130 /* synopsis: key=r[P2] */ +#define OP_SorterInsert 131 /* synopsis: key=r[P2] */ +#define OP_IdxDelete 132 /* synopsis: key=r[P2@P3] */ +#define OP_DeferredSeek 133 /* synopsis: Move P3 to P1.rowid if needed */ +#define OP_IdxRowid 134 /* synopsis: r[P2]=rowid */ +#define OP_FinishSeek 135 +#define OP_Destroy 136 +#define OP_Clear 137 +#define OP_ResetSorter 138 +#define OP_CreateBtree 139 /* synopsis: r[P2]=root iDb=P1 flags=P3 */ +#define OP_SqlExec 140 +#define OP_ParseSchema 141 +#define OP_LoadAnalysis 142 +#define OP_DropTable 143 +#define OP_DropIndex 144 +#define OP_DropTrigger 145 +#define OP_IntegrityCk 146 +#define OP_RowSetAdd 147 /* synopsis: rowset(P1)=r[P2] */ +#define OP_Param 148 +#define OP_FkCounter 149 /* synopsis: fkctr[P1]+=P2 */ +#define OP_Real 150 /* same as TK_FLOAT, synopsis: r[P2]=P4 */ +#define OP_MemMax 151 /* synopsis: r[P1]=max(r[P1],r[P2]) */ +#define OP_OffsetLimit 152 /* synopsis: if r[P1]>0 then r[P2]=r[P1]+max(0,r[P3]) else r[P2]=(-1) */ +#define OP_AggInverse 153 /* synopsis: accum=r[P3] inverse(r[P2@P5]) */ +#define OP_AggStep 154 /* synopsis: accum=r[P3] step(r[P2@P5]) */ +#define OP_AggStep1 155 /* synopsis: accum=r[P3] step(r[P2@P5]) */ +#define OP_AggValue 156 /* synopsis: r[P3]=value N=P2 */ +#define OP_AggFinal 157 /* synopsis: accum=r[P1] N=P2 */ +#define OP_Expire 158 +#define OP_CursorLock 159 +#define OP_CursorUnlock 160 +#define OP_TableLock 161 /* synopsis: iDb=P1 root=P2 write=P3 */ +#define OP_VBegin 162 +#define OP_VCreate 163 +#define OP_VDestroy 164 +#define OP_VOpen 165 +#define OP_VColumn 166 /* synopsis: r[P3]=vcolumn(P2) */ +#define OP_VRename 167 +#define OP_Pagecount 168 +#define OP_MaxPgcnt 169 +#define OP_Trace 170 +#define OP_CursorHint 171 +#define OP_ReleaseReg 172 /* synopsis: release r[P1@P2] mask P3 */ +#define OP_Noop 173 +#define OP_Explain 174 +#define OP_Abortable 175 /* Properties such as "out2" or "jump" that are specified in ** comments following the "case" for each opcode in the vdbe.c @@ -14662,28 +15477,29 @@ typedef struct VdbeOpList VdbeOpList; #define OPFLG_OUT2 0x10 /* out2: P2 is an output */ #define OPFLG_OUT3 0x20 /* out3: P3 is an output */ #define OPFLG_INITIALIZER {\ -/* 0 */ 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01,\ -/* 8 */ 0x00, 0x10, 0x00, 0x01, 0x00, 0x01, 0x01, 0x01,\ -/* 16 */ 0x03, 0x03, 0x01, 0x12, 0x01, 0x03, 0x03, 0x01,\ -/* 24 */ 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09,\ +/* 0 */ 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x00, 0x10,\ +/* 8 */ 0x00, 0x01, 0x00, 0x01, 0x01, 0x01, 0x03, 0x03,\ +/* 16 */ 0x01, 0x01, 0x03, 0x12, 0x03, 0x01, 0x09, 0x09,\ +/* 24 */ 0x09, 0x09, 0x01, 0x09, 0x09, 0x09, 0x09, 0x09,\ /* 32 */ 0x09, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,\ /* 40 */ 0x01, 0x01, 0x23, 0x26, 0x26, 0x0b, 0x01, 0x01,\ /* 48 */ 0x03, 0x03, 0x03, 0x03, 0x0b, 0x0b, 0x0b, 0x0b,\ -/* 56 */ 0x0b, 0x0b, 0x01, 0x03, 0x01, 0x01, 0x01, 0x02,\ -/* 64 */ 0x02, 0x08, 0x00, 0x10, 0x10, 0x10, 0x10, 0x00,\ -/* 72 */ 0x10, 0x10, 0x00, 0x00, 0x10, 0x10, 0x00, 0x00,\ -/* 80 */ 0x02, 0x02, 0x02, 0x00, 0x00, 0x26, 0x26, 0x26,\ -/* 88 */ 0x26, 0x26, 0x26, 0x26, 0x26, 0x26, 0x26, 0x12,\ -/* 96 */ 0x12, 0x20, 0x00, 0x10, 0x00, 0x00, 0x10, 0x10,\ -/* 104 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,\ -/* 112 */ 0x00, 0x00, 0x00, 0x00, 0x10, 0x10, 0x00, 0x00,\ -/* 120 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00,\ -/* 128 */ 0x04, 0x04, 0x00, 0x00, 0x10, 0x10, 0x10, 0x00,\ -/* 136 */ 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,\ -/* 144 */ 0x00, 0x06, 0x10, 0x00, 0x04, 0x1a, 0x00, 0x00,\ -/* 152 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,\ -/* 160 */ 0x00, 0x10, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00,\ -/* 168 */ 0x00, 0x00, 0x00, 0x00,} +/* 56 */ 0x0b, 0x0b, 0x01, 0x03, 0x01, 0x01, 0x01, 0x00,\ +/* 64 */ 0x00, 0x02, 0x02, 0x08, 0x00, 0x10, 0x10, 0x10,\ +/* 72 */ 0x10, 0x00, 0x10, 0x10, 0x00, 0x00, 0x10, 0x10,\ +/* 80 */ 0x00, 0x00, 0x02, 0x02, 0x02, 0x00, 0x00, 0x12,\ +/* 88 */ 0x20, 0x00, 0x00, 0x00, 0x10, 0x10, 0x00, 0x00,\ +/* 96 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x26, 0x26, 0x26,\ +/* 104 */ 0x26, 0x26, 0x26, 0x26, 0x26, 0x26, 0x26, 0x00,\ +/* 112 */ 0x12, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x10,\ +/* 120 */ 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10,\ +/* 128 */ 0x00, 0x00, 0x04, 0x04, 0x00, 0x00, 0x10, 0x00,\ +/* 136 */ 0x10, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00,\ +/* 144 */ 0x00, 0x00, 0x00, 0x06, 0x10, 0x00, 0x10, 0x04,\ +/* 152 */ 0x1a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,\ +/* 160 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,\ +/* 168 */ 0x10, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,\ +} /* The sqlite3P2Values() routine is able to run faster if it knows ** the value of the largest JUMP opcode. The smaller the maximum @@ -14707,6 +15523,7 @@ typedef struct VdbeOpList VdbeOpList; ** for a description of what each of these routines does. */ SQLITE_PRIVATE Vdbe *sqlite3VdbeCreate(Parse*); +SQLITE_PRIVATE Parse *sqlite3VdbeParser(Vdbe*); SQLITE_PRIVATE int sqlite3VdbeAddOp0(Vdbe*,int); SQLITE_PRIVATE int sqlite3VdbeAddOp1(Vdbe*,int,int); SQLITE_PRIVATE int sqlite3VdbeAddOp2(Vdbe*,int,int,int); @@ -14717,6 +15534,7 @@ SQLITE_PRIVATE int sqlite3VdbeAddOp3(Vdbe*,int,int,int,int); SQLITE_PRIVATE int sqlite3VdbeAddOp4(Vdbe*,int,int,int,int,const char *zP4,int); SQLITE_PRIVATE int sqlite3VdbeAddOp4Dup8(Vdbe*,int,int,int,int,const u8*,int); SQLITE_PRIVATE int sqlite3VdbeAddOp4Int(Vdbe*,int,int,int,int,int); +SQLITE_PRIVATE int sqlite3VdbeAddFunctionCall(Parse*,int,int,int,int,const FuncDef*,int); SQLITE_PRIVATE void sqlite3VdbeEndCoroutine(Vdbe*,int); #if defined(SQLITE_DEBUG) && !defined(SQLITE_TEST_REALLOC_STRESS) SQLITE_PRIVATE void sqlite3VdbeVerifyNoMallocRequired(Vdbe *p, int N); @@ -14742,22 +15560,34 @@ SQLITE_PRIVATE int sqlite3VdbeExplainParent(Parse*); # define ExplainQueryPlan(P) # define ExplainQueryPlanPop(P) # define ExplainQueryPlanParent(P) 0 +# define sqlite3ExplainBreakpoint(A,B) /*no-op*/ +#endif +#if defined(SQLITE_DEBUG) && !defined(SQLITE_OMIT_EXPLAIN) +SQLITE_PRIVATE void sqlite3ExplainBreakpoint(const char*,const char*); +#else +# define sqlite3ExplainBreakpoint(A,B) /*no-op*/ #endif SQLITE_PRIVATE void sqlite3VdbeAddParseSchemaOp(Vdbe*,int,char*); -SQLITE_PRIVATE void sqlite3VdbeChangeOpcode(Vdbe*, u32 addr, u8); -SQLITE_PRIVATE void sqlite3VdbeChangeP1(Vdbe*, u32 addr, int P1); -SQLITE_PRIVATE void sqlite3VdbeChangeP2(Vdbe*, u32 addr, int P2); -SQLITE_PRIVATE void sqlite3VdbeChangeP3(Vdbe*, u32 addr, int P3); +SQLITE_PRIVATE void sqlite3VdbeChangeOpcode(Vdbe*, int addr, u8); +SQLITE_PRIVATE void sqlite3VdbeChangeP1(Vdbe*, int addr, int P1); +SQLITE_PRIVATE void sqlite3VdbeChangeP2(Vdbe*, int addr, int P2); +SQLITE_PRIVATE void sqlite3VdbeChangeP3(Vdbe*, int addr, int P3); SQLITE_PRIVATE void sqlite3VdbeChangeP5(Vdbe*, u16 P5); SQLITE_PRIVATE void sqlite3VdbeJumpHere(Vdbe*, int addr); +SQLITE_PRIVATE void sqlite3VdbeJumpHereOrPopInst(Vdbe*, int addr); SQLITE_PRIVATE int sqlite3VdbeChangeToNoop(Vdbe*, int addr); SQLITE_PRIVATE int sqlite3VdbeDeletePriorOpcode(Vdbe*, u8 op); +#ifdef SQLITE_DEBUG +SQLITE_PRIVATE void sqlite3VdbeReleaseRegisters(Parse*,int addr, int n, u32 mask, int); +#else +# define sqlite3VdbeReleaseRegisters(P,A,N,M,F) +#endif SQLITE_PRIVATE void sqlite3VdbeChangeP4(Vdbe*, int addr, const char *zP4, int N); SQLITE_PRIVATE void sqlite3VdbeAppendP4(Vdbe*, void *pP4, int p4type); SQLITE_PRIVATE void sqlite3VdbeSetP4KeyInfo(Parse*, Index*); SQLITE_PRIVATE void sqlite3VdbeUsesBtree(Vdbe*, int); SQLITE_PRIVATE VdbeOp *sqlite3VdbeGetOp(Vdbe*, int); -SQLITE_PRIVATE int sqlite3VdbeMakeLabel(Vdbe*); +SQLITE_PRIVATE int sqlite3VdbeMakeLabel(Parse*); SQLITE_PRIVATE void sqlite3VdbeRunOnlyOnce(Vdbe*); SQLITE_PRIVATE void sqlite3VdbeReusable(Vdbe*); SQLITE_PRIVATE void sqlite3VdbeDelete(Vdbe*); @@ -14765,9 +15595,6 @@ SQLITE_PRIVATE void sqlite3VdbeClearObject(sqlite3*,Vdbe*); SQLITE_PRIVATE void sqlite3VdbeMakeReady(Vdbe*,Parse*); SQLITE_PRIVATE int sqlite3VdbeFinalize(Vdbe*); SQLITE_PRIVATE void sqlite3VdbeResolveLabel(Vdbe*, int); -#ifdef SQLITE_COVERAGE_TEST -SQLITE_PRIVATE int sqlite3VdbeLabelHasBeenResolved(Vdbe*,int); -#endif SQLITE_PRIVATE int sqlite3VdbeCurrentAddr(Vdbe*); #ifdef SQLITE_DEBUG SQLITE_PRIVATE int sqlite3VdbeAssertMayAbort(Vdbe *, int); @@ -14781,6 +15608,10 @@ SQLITE_PRIVATE void sqlite3VdbeCountChanges(Vdbe*); SQLITE_PRIVATE sqlite3 *sqlite3VdbeDb(Vdbe*); SQLITE_PRIVATE u8 sqlite3VdbePrepareFlags(Vdbe*); SQLITE_PRIVATE void sqlite3VdbeSetSql(Vdbe*, const char *z, int n, u8); +#ifdef SQLITE_ENABLE_NORMALIZE +SQLITE_PRIVATE void sqlite3VdbeAddDblquoteStr(sqlite3*,Vdbe*,const char*); +SQLITE_PRIVATE int sqlite3VdbeUsesDoubleQuotedString(Vdbe*,const char*); +#endif SQLITE_PRIVATE void sqlite3VdbeSwap(Vdbe*,Vdbe*); SQLITE_PRIVATE VdbeOp *sqlite3VdbeTakeOpArray(Vdbe*, int*, int*); SQLITE_PRIVATE sqlite3_value *sqlite3VdbeGetBoundValue(Vdbe*, int, u8); @@ -14789,6 +15620,7 @@ SQLITE_PRIVATE void sqlite3VdbeSetVarmask(Vdbe*, int); SQLITE_PRIVATE char *sqlite3VdbeExpandSql(Vdbe*, const char*); #endif SQLITE_PRIVATE int sqlite3MemCompare(const Mem*, const Mem*, const CollSeq*); +SQLITE_PRIVATE int sqlite3BlobCompare(const Mem*, const Mem*); SQLITE_PRIVATE void sqlite3VdbeRecordUnpack(KeyInfo*,int,const void*,UnpackedRecord*); SQLITE_PRIVATE int sqlite3VdbeRecordCompare(int,const void*,UnpackedRecord*); @@ -14798,11 +15630,13 @@ SQLITE_PRIVATE UnpackedRecord *sqlite3VdbeAllocUnpackedRecord(KeyInfo*); typedef int (*RecordCompare)(int,const void*,UnpackedRecord*); SQLITE_PRIVATE RecordCompare sqlite3VdbeFindCompare(UnpackedRecord*); -#ifndef SQLITE_OMIT_TRIGGER SQLITE_PRIVATE void sqlite3VdbeLinkSubProgram(Vdbe *, SubProgram *); -#endif +SQLITE_PRIVATE int sqlite3VdbeHasSubProgram(Vdbe*); SQLITE_PRIVATE int sqlite3NotPureFunc(sqlite3_context*); +#ifdef SQLITE_ENABLE_BYTECODE_VTAB +SQLITE_PRIVATE int sqlite3VdbeBytecodeVtabInit(sqlite3*); +#endif /* Use SQLITE_ENABLE_COMMENTS to enable generation of extra comments on ** each VDBE opcode. @@ -14844,23 +15678,52 @@ SQLITE_PRIVATE void sqlite3VdbeNoopComment(Vdbe*, const char*, ...); ** ** VdbeCoverageNeverTaken(v) // Previous branch is never taken ** +** VdbeCoverageNeverNull(v) // Previous three-way branch is only +** // taken on the first two ways. The +** // NULL option is not possible +** +** VdbeCoverageEqNe(v) // Previous OP_Jump is only interested +** // in distingishing equal and not-equal. +** ** Every VDBE branch operation must be tagged with one of the macros above. ** If not, then when "make test" is run with -DSQLITE_VDBE_COVERAGE and ** -DSQLITE_DEBUG then an ALWAYS() will fail in the vdbeTakeBranch() ** routine in vdbe.c, alerting the developer to the missed tag. +** +** During testing, the test application will invoke +** sqlite3_test_control(SQLITE_TESTCTRL_VDBE_COVERAGE,...) to set a callback +** routine that is invoked as each bytecode branch is taken. The callback +** contains the sqlite3.c source line number ov the VdbeCoverage macro and +** flags to indicate whether or not the branch was taken. The test application +** is responsible for keeping track of this and reporting byte-code branches +** that are never taken. +** +** See the VdbeBranchTaken() macro and vdbeTakeBranch() function in the +** vdbe.c source file for additional information. */ #ifdef SQLITE_VDBE_COVERAGE SQLITE_PRIVATE void sqlite3VdbeSetLineNumber(Vdbe*,int); # define VdbeCoverage(v) sqlite3VdbeSetLineNumber(v,__LINE__) # define VdbeCoverageIf(v,x) if(x)sqlite3VdbeSetLineNumber(v,__LINE__) -# define VdbeCoverageAlwaysTaken(v) sqlite3VdbeSetLineNumber(v,2); -# define VdbeCoverageNeverTaken(v) sqlite3VdbeSetLineNumber(v,1); +# define VdbeCoverageAlwaysTaken(v) \ + sqlite3VdbeSetLineNumber(v,__LINE__|0x5000000); +# define VdbeCoverageNeverTaken(v) \ + sqlite3VdbeSetLineNumber(v,__LINE__|0x6000000); +# define VdbeCoverageNeverNull(v) \ + sqlite3VdbeSetLineNumber(v,__LINE__|0x4000000); +# define VdbeCoverageNeverNullIf(v,x) \ + if(x)sqlite3VdbeSetLineNumber(v,__LINE__|0x4000000); +# define VdbeCoverageEqNe(v) \ + sqlite3VdbeSetLineNumber(v,__LINE__|0x8000000); # define VDBE_OFFSET_LINENO(x) (__LINE__+x) #else # define VdbeCoverage(v) # define VdbeCoverageIf(v,x) # define VdbeCoverageAlwaysTaken(v) # define VdbeCoverageNeverTaken(v) +# define VdbeCoverageNeverNull(v) +# define VdbeCoverageNeverNullIf(v,x) +# define VdbeCoverageEqNe(v) # define VDBE_OFFSET_LINENO(x) 0 #endif @@ -14870,6 +15733,10 @@ SQLITE_PRIVATE void sqlite3VdbeScanStatus(Vdbe*, int, int, int, LogEst, const ch # define sqlite3VdbeScanStatus(a,b,c,d,e) #endif +#if defined(SQLITE_DEBUG) || defined(VDBE_PROFILE) +SQLITE_PRIVATE void sqlite3VdbePrintOp(FILE*, int, VdbeOp*); +#endif + #endif /* SQLITE_VDBE_H */ /************** End of vdbe.h ************************************************/ @@ -15006,9 +15873,6 @@ SQLITE_PRIVATE int sqlite3PagerReadFileheader(Pager*, int, unsigned char*); /* Functions used to configure a Pager object. */ SQLITE_PRIVATE void sqlite3PagerSetBusyHandler(Pager*, int(*)(void *), void *); SQLITE_PRIVATE int sqlite3PagerSetPagesize(Pager*, u32*, int); -#ifdef SQLITE_HAS_CODEC -SQLITE_PRIVATE void sqlite3PagerAlignReserve(Pager*,Pager*); -#endif SQLITE_PRIVATE int sqlite3PagerMaxPageCount(Pager*, int); SQLITE_PRIVATE void sqlite3PagerSetCachesize(Pager*, int); SQLITE_PRIVATE int sqlite3PagerSetSpillsize(Pager*, int); @@ -15057,16 +15921,25 @@ SQLITE_PRIVATE int sqlite3PagerWalSupported(Pager *pPager); SQLITE_PRIVATE int sqlite3PagerWalCallback(Pager *pPager); SQLITE_PRIVATE int sqlite3PagerOpenWal(Pager *pPager, int *pisOpen); SQLITE_PRIVATE int sqlite3PagerCloseWal(Pager *pPager, sqlite3*); -# ifdef SQLITE_DIRECT_OVERFLOW_READ -SQLITE_PRIVATE int sqlite3PagerUseWal(Pager *pPager, Pgno); -# endif # ifdef SQLITE_ENABLE_SNAPSHOT -SQLITE_PRIVATE int sqlite3PagerSnapshotGet(Pager *pPager, sqlite3_snapshot **ppSnapshot); -SQLITE_PRIVATE int sqlite3PagerSnapshotOpen(Pager *pPager, sqlite3_snapshot *pSnapshot); +SQLITE_PRIVATE int sqlite3PagerSnapshotGet(Pager*, sqlite3_snapshot **ppSnapshot); +SQLITE_PRIVATE int sqlite3PagerSnapshotOpen(Pager*, sqlite3_snapshot *pSnapshot); SQLITE_PRIVATE int sqlite3PagerSnapshotRecover(Pager *pPager); +SQLITE_PRIVATE int sqlite3PagerSnapshotCheck(Pager *pPager, sqlite3_snapshot *pSnapshot); +SQLITE_PRIVATE void sqlite3PagerSnapshotUnlock(Pager *pPager); # endif +#endif + +#if !defined(SQLITE_OMIT_WAL) && defined(SQLITE_ENABLE_SETLK_TIMEOUT) +SQLITE_PRIVATE int sqlite3PagerWalWriteLock(Pager*, int); +SQLITE_PRIVATE void sqlite3PagerWalDb(Pager*, sqlite3*); #else -# define sqlite3PagerUseWal(x,y) 0 +# define sqlite3PagerWalWriteLock(y,z) SQLITE_OK +# define sqlite3PagerWalDb(x,y) +#endif + +#ifdef SQLITE_DIRECT_OVERFLOW_READ +SQLITE_PRIVATE int sqlite3PagerDirectReadOk(Pager *pPager, Pgno pgno); #endif #ifdef SQLITE_ENABLE_ZIPVFS @@ -15080,7 +15953,7 @@ SQLITE_PRIVATE u32 sqlite3PagerDataVersion(Pager*); SQLITE_PRIVATE int sqlite3PagerRefcount(Pager*); #endif SQLITE_PRIVATE int sqlite3PagerMemUsed(Pager*); -SQLITE_PRIVATE const char *sqlite3PagerFilename(Pager*, int); +SQLITE_PRIVATE const char *sqlite3PagerFilename(const Pager*, int); SQLITE_PRIVATE sqlite3_vfs *sqlite3PagerVfs(Pager*); SQLITE_PRIVATE sqlite3_file *sqlite3PagerFile(Pager*); SQLITE_PRIVATE sqlite3_file *sqlite3PagerJrnlFile(Pager*); @@ -15090,21 +15963,12 @@ SQLITE_PRIVATE int sqlite3PagerIsMemdb(Pager*); SQLITE_PRIVATE void sqlite3PagerCacheStat(Pager *, int, int, int *); SQLITE_PRIVATE void sqlite3PagerClearCache(Pager*); SQLITE_PRIVATE int sqlite3SectorSize(sqlite3_file *); -#ifdef SQLITE_ENABLE_SETLK_TIMEOUT -SQLITE_PRIVATE void sqlite3PagerResetLockTimeout(Pager *pPager); -#else -# define sqlite3PagerResetLockTimeout(X) -#endif /* Functions used to truncate the database file. */ SQLITE_PRIVATE void sqlite3PagerTruncateImage(Pager*,Pgno); SQLITE_PRIVATE void sqlite3PagerRekey(DbPage*, Pgno, u16); -#if defined(SQLITE_HAS_CODEC) && !defined(SQLITE_OMIT_WAL) -SQLITE_PRIVATE void *sqlite3PagerCodec(DbPage *); -#endif - /* Functions to support testing and debugging. */ #if !defined(NDEBUG) || defined(SQLITE_TEST) SQLITE_PRIVATE Pgno sqlite3PagerPagenumber(DbPage*); @@ -15311,6 +16175,10 @@ SQLITE_PRIVATE int sqlite3HeaderSizePcache1(void); /* Number of dirty pages as a percentage of the configured cache size */ SQLITE_PRIVATE int sqlite3PCachePercentDirty(PCache*); +#ifdef SQLITE_DIRECT_OVERFLOW_READ +SQLITE_PRIVATE int sqlite3PCacheIsDirty(PCache *pCache); +#endif + #endif /* _PCACHE_H_ */ /************** End of pcache.h **********************************************/ @@ -15664,6 +16532,7 @@ SQLITE_PRIVATE void sqlite3OsCloseFree(sqlite3_file *); #define MUTEX_LOGIC(X) #else #define MUTEX_LOGIC(X) X +SQLITE_API int sqlite3_mutex_held(sqlite3_mutex*); #endif /* defined(SQLITE_MUTEX_OMIT) */ /************** End of mutex.h ***********************************************/ @@ -15767,7 +16636,6 @@ struct Schema { */ #define DB_SchemaLoaded 0x0001 /* The schema has been loaded */ #define DB_UnresetViews 0x0002 /* Some views have defined column names */ -#define DB_Empty 0x0004 /* The file is empty (length 0 bytes) */ #define DB_ResetWanted 0x0008 /* Reset the schema when nSchemaLock==0 */ /* @@ -15795,15 +16663,47 @@ struct Schema { ** is shared by multiple database connections. Therefore, while parsing ** schema information, the Lookaside.bEnabled flag is cleared so that ** lookaside allocations are not used to construct the schema objects. +** +** New lookaside allocations are only allowed if bDisable==0. When +** bDisable is greater than zero, sz is set to zero which effectively +** disables lookaside without adding a new test for the bDisable flag +** in a performance-critical path. sz should be set by to szTrue whenever +** bDisable changes back to zero. +** +** Lookaside buffers are initially held on the pInit list. As they are +** used and freed, they are added back to the pFree list. New allocations +** come off of pFree first, then pInit as a fallback. This dual-list +** allows use to compute a high-water mark - the maximum number of allocations +** outstanding at any point in the past - by subtracting the number of +** allocations on the pInit list from the total number of allocations. +** +** Enhancement on 2019-12-12: Two-size-lookaside +** The default lookaside configuration is 100 slots of 1200 bytes each. +** The larger slot sizes are important for performance, but they waste +** a lot of space, as most lookaside allocations are less than 128 bytes. +** The two-size-lookaside enhancement breaks up the lookaside allocation +** into two pools: One of 128-byte slots and the other of the default size +** (1200-byte) slots. Allocations are filled from the small-pool first, +** failing over to the full-size pool if that does not work. Thus more +** lookaside slots are available while also using less memory. +** This enhancement can be omitted by compiling with +** SQLITE_OMIT_TWOSIZE_LOOKASIDE. */ struct Lookaside { u32 bDisable; /* Only operate the lookaside when zero */ u16 sz; /* Size of each buffer in bytes */ + u16 szTrue; /* True value of sz, even if disabled */ u8 bMalloced; /* True if pStart obtained from sqlite3_malloc() */ u32 nSlot; /* Number of lookaside slots allocated */ u32 anStat[3]; /* 0: hits. 1: size misses. 2: full misses */ LookasideSlot *pInit; /* List of buffers not previously used */ LookasideSlot *pFree; /* List of available buffers */ +#ifndef SQLITE_OMIT_TWOSIZE_LOOKASIDE + LookasideSlot *pSmallInit; /* List of small buffers not prediously used */ + LookasideSlot *pSmallFree; /* List of available small buffers */ + void *pMiddle; /* First byte past end of full-size buffers and + ** the first byte of LOOKASIDE_SMALL buffers */ +#endif /* SQLITE_OMIT_TWOSIZE_LOOKASIDE */ void *pStart; /* First byte of available memory space */ void *pEnd; /* First byte past end of available space */ }; @@ -15811,17 +16711,30 @@ struct LookasideSlot { LookasideSlot *pNext; /* Next buffer in the list of free buffers */ }; +#define DisableLookaside db->lookaside.bDisable++;db->lookaside.sz=0 +#define EnableLookaside db->lookaside.bDisable--;\ + db->lookaside.sz=db->lookaside.bDisable?0:db->lookaside.szTrue + +/* Size of the smaller allocations in two-size lookside */ +#ifdef SQLITE_OMIT_TWOSIZE_LOOKASIDE +# define LOOKASIDE_SMALL 0 +#else +# define LOOKASIDE_SMALL 128 +#endif + /* ** A hash table for built-in function definitions. (Application-defined ** functions use a regular table table from hash.h.) ** ** Hash each FuncDef structure into one of the FuncDefHash.a[] slots. -** Collisions are on the FuncDef.u.pHash chain. +** Collisions are on the FuncDef.u.pHash chain. Use the SQLITE_FUNC_HASH() +** macro to compute a hash on the function name. */ #define SQLITE_FUNC_HASH_SZ 23 struct FuncDefHash { FuncDef *a[SQLITE_FUNC_HASH_SZ]; /* Hash table for functions */ }; +#define SQLITE_FUNC_HASH(C,L) (((C)+(L))%SQLITE_FUNC_HASH_SZ) #ifdef SQLITE_USER_AUTHENTICATION /* @@ -15865,10 +16778,13 @@ SQLITE_PRIVATE void sqlite3CryptFunc(sqlite3_context*,int,sqlite3_value**); /* This is an extra SQLITE_TRACE macro that indicates "legacy" tracing ** in the style of sqlite3_trace() */ -#define SQLITE_TRACE_LEGACY 0x80 +#define SQLITE_TRACE_LEGACY 0x40 /* Use the legacy xTrace */ +#define SQLITE_TRACE_XPROFILE 0x80 /* Use the legacy xProfile */ #else -#define SQLITE_TRACE_LEGACY 0 +#define SQLITE_TRACE_LEGACY 0 +#define SQLITE_TRACE_XPROFILE 0 #endif /* SQLITE_OMIT_DEPRECATED */ +#define SQLITE_TRACE_NONLEGACY_MASK 0x0f /* Normal flags */ /* @@ -15877,12 +16793,12 @@ SQLITE_PRIVATE void sqlite3CryptFunc(sqlite3_context*,int,sqlite3_value**); struct sqlite3 { sqlite3_vfs *pVfs; /* OS Interface */ struct Vdbe *pVdbe; /* List of active virtual machines */ - CollSeq *pDfltColl; /* The default collating sequence (BINARY) */ + CollSeq *pDfltColl; /* BINARY collseq for the database encoding */ sqlite3_mutex *mutex; /* Connection mutex */ Db *aDb; /* All backends */ int nDb; /* Number of backends currently in use */ u32 mDbFlags; /* flags recording internal state */ - u32 flags; /* flags settable by pragmas. See below */ + u64 flags; /* flags settable by pragmas. See below */ i64 lastRowid; /* ROWID of most recent insert (see above) */ i64 szMmap; /* Default mmap_size setting */ u32 nSchemaLock; /* Do not reset the schema when non-zero */ @@ -15917,6 +16833,7 @@ struct sqlite3 { unsigned orphanTrigger : 1; /* Last statement is orphaned TEMP trigger */ unsigned imposterTable : 1; /* Building an imposter table */ unsigned reopenMemdb : 1; /* ATTACH is really a reopen using MemDB */ + char **azInit; /* "type", "name", and "tbl_name" columns */ } init; int nVdbeActive; /* Number of VDBEs currently running */ int nVdbeRead; /* Number of active VDBEs that read or write */ @@ -15927,14 +16844,17 @@ struct sqlite3 { void **aExtension; /* Array of shared library handles */ int (*xTrace)(u32,void*,void*,void*); /* Trace function */ void *pTraceArg; /* Argument to the trace function */ +#ifndef SQLITE_OMIT_DEPRECATED void (*xProfile)(void*,const char*,u64); /* Profiling function */ void *pProfileArg; /* Argument to profile function */ +#endif void *pCommitArg; /* Argument to xCommitCallback() */ int (*xCommitCallback)(void*); /* Invoked at every commit. */ void *pRollbackArg; /* Argument to xRollbackCallback() */ void (*xRollbackCallback)(void*); /* Invoked at every commit. */ void *pUpdateArg; void (*xUpdateCallback)(void*,int, const char*,const char*,sqlite_int64); + Parse *pParse; /* Current parse */ #ifdef SQLITE_ENABLE_PREUPDATE_HOOK void *pPreUpdateArg; /* First argument to xPreUpdateCallback */ void (*xPreUpdateCallback)( /* Registered using sqlite3_preupdate_hook() */ @@ -15976,6 +16896,7 @@ struct sqlite3 { BusyHandler busyHandler; /* Busy callback */ Db aDbStatic[2]; /* Static space for the 2 default backends */ Savepoint *pSavepoint; /* List of active savepoints */ + int nAnalysisLimit; /* Number of index rows to ANALYZE */ int busyTimeout; /* Busy handler timeout, in msec */ int nSavepoint; /* Number of non-transaction savepoints */ int nStatement; /* Number of nested statement-transactions */ @@ -16010,6 +16931,13 @@ struct sqlite3 { #define SCHEMA_ENC(db) ((db)->aDb[0].pSchema->enc) #define ENC(db) ((db)->enc) +/* +** A u64 constant where the lower 32 bits are all zeros. Only the +** upper 32 bits are included in the argument. Necessary because some +** C-compilers still do not accept LL integer literals. +*/ +#define HI(X) ((u64)(X)<<32) + /* ** Possible values for the sqlite3.flags. ** @@ -16025,9 +16953,8 @@ struct sqlite3 { #define SQLITE_CkptFullFSync 0x00000010 /* Use full fsync for checkpoint */ #define SQLITE_CacheSpill 0x00000020 /* OK to spill pager cache */ #define SQLITE_ShortColNames 0x00000040 /* Show short columns names */ -#define SQLITE_CountRows 0x00000080 /* Count rows changed by INSERT, */ - /* DELETE, or UPDATE and return */ - /* the count using a callback. */ +#define SQLITE_TrustedSchema 0x00000080 /* Allow unsafe functions and + ** vtabs in the schema definition */ #define SQLITE_NullCallback 0x00000100 /* Invoke the callback once if the */ /* result set is empty */ #define SQLITE_IgnoreChecks 0x00000200 /* Do not enforce check constraints */ @@ -16047,14 +16974,24 @@ struct sqlite3 { #define SQLITE_EnableQPSG 0x00800000 /* Query Planner Stability Guarantee*/ #define SQLITE_TriggerEQP 0x01000000 /* Show trigger EXPLAIN QUERY PLAN */ #define SQLITE_ResetDatabase 0x02000000 /* Reset the database */ +#define SQLITE_LegacyAlter 0x04000000 /* Legacy ALTER TABLE behaviour */ +#define SQLITE_NoSchemaError 0x08000000 /* Do not report schema parse errors*/ +#define SQLITE_Defensive 0x10000000 /* Input SQL is likely hostile */ +#define SQLITE_DqsDDL 0x20000000 /* dbl-quoted strings allowed in DDL*/ +#define SQLITE_DqsDML 0x40000000 /* dbl-quoted strings allowed in DML*/ +#define SQLITE_EnableView 0x80000000 /* Enable the use of views */ +#define SQLITE_CountRows HI(0x00001) /* Count rows changed by INSERT, */ + /* DELETE, or UPDATE and return */ + /* the count using a callback. */ /* Flags used only if debugging */ #ifdef SQLITE_DEBUG -#define SQLITE_SqlTrace 0x08000000 /* Debug print SQL as it executes */ -#define SQLITE_VdbeListing 0x10000000 /* Debug listings of VDBE programs */ -#define SQLITE_VdbeTrace 0x20000000 /* True to trace VDBE execution */ -#define SQLITE_VdbeAddopTrace 0x40000000 /* Trace sqlite3VdbeAddOp() calls */ -#define SQLITE_VdbeEQP 0x80000000 /* Debug EXPLAIN QUERY PLAN */ +#define SQLITE_SqlTrace HI(0x0100000) /* Debug print SQL as it executes */ +#define SQLITE_VdbeListing HI(0x0200000) /* Debug listings of VDBE progs */ +#define SQLITE_VdbeTrace HI(0x0400000) /* True to trace VDBE execution */ +#define SQLITE_VdbeAddopTrace HI(0x0800000) /* Trace sqlite3VdbeAddOp() calls */ +#define SQLITE_VdbeEQP HI(0x1000000) /* Debug EXPLAIN QUERY PLAN */ +#define SQLITE_ParserTrace HI(0x2000000) /* PRAGMA parser_trace=ON */ #endif /* @@ -16063,7 +17000,10 @@ struct sqlite3 { #define DBFLAG_SchemaChange 0x0001 /* Uncommitted Hash table changes */ #define DBFLAG_PreferBuiltin 0x0002 /* Preference to built-in funcs */ #define DBFLAG_Vacuum 0x0004 /* Currently in a VACUUM */ -#define DBFLAG_SchemaKnownOk 0x0008 /* Schema is known to be valid */ +#define DBFLAG_VacuumInto 0x0008 /* Currently running VACUUM INTO */ +#define DBFLAG_SchemaKnownOk 0x0010 /* Schema is known to be valid */ +#define DBFLAG_InternalFunc 0x0020 /* Allow use of internal functions */ +#define DBFLAG_EncodingFixed 0x0040 /* No longer possible to change enc. */ /* ** Bits of the sqlite3.dbOptFlags field that are used by the @@ -16071,7 +17011,7 @@ struct sqlite3 { ** selectively disable various optimizations. */ #define SQLITE_QueryFlattener 0x0001 /* Query flattening */ -#define SQLITE_ColumnCache 0x0002 /* Column cache */ +#define SQLITE_WindowFunc 0x0002 /* Use xInverse for window functions */ #define SQLITE_GroupByOrder 0x0004 /* GROUPBY cover of ORDERBY */ #define SQLITE_FactorOutConst 0x0008 /* Constant factoring */ #define SQLITE_DistinctOpt 0x0010 /* DISTINCT using indexes */ @@ -16081,10 +17021,12 @@ struct sqlite3 { #define SQLITE_OmitNoopJoin 0x0100 /* Omit unused tables in joins */ #define SQLITE_CountOfView 0x0200 /* The count-of-view optimization */ #define SQLITE_CursorHints 0x0400 /* Add OP_CursorHint opcodes */ -#define SQLITE_Stat34 0x0800 /* Use STAT3 or STAT4 data */ - /* TH3 expects the Stat34 ^^^^^^ value to be 0x0800. Don't change it */ +#define SQLITE_Stat4 0x0800 /* Use STAT4 data */ + /* TH3 expects the Stat4 ^^^^^^ value to be 0x0800. Don't change it */ #define SQLITE_PushDown 0x1000 /* The push-down optimization */ #define SQLITE_SimplifyJoin 0x2000 /* Convert LEFT JOIN to JOIN */ +#define SQLITE_SkipScan 0x4000 /* Skip-scans */ +#define SQLITE_PropagateConst 0x8000 /* The constant propagation opt */ #define SQLITE_AllOpts 0xffff /* All optimizations */ /* @@ -16123,11 +17065,13 @@ struct sqlite3 { */ struct FuncDef { i8 nArg; /* Number of arguments. -1 means unlimited */ - u16 funcFlags; /* Some combination of SQLITE_FUNC_* */ + u32 funcFlags; /* Some combination of SQLITE_FUNC_* */ void *pUserData; /* User data parameter */ FuncDef *pNext; /* Next function with same name */ void (*xSFunc)(sqlite3_context*,int,sqlite3_value**); /* func or agg-step */ void (*xFinalize)(sqlite3_context*); /* Agg finalizer */ + void (*xValue)(sqlite3_context*); /* Current agg value */ + void (*xInverse)(sqlite3_context*,int,sqlite3_value**); /* inverse agg-step */ const char *zName; /* SQL name of the function. */ union { FuncDef *pHash; /* Next with a different name but the same hash */ @@ -16166,6 +17110,8 @@ struct FuncDestructor { ** SQLITE_FUNC_LENGTH == OPFLAG_LENGTHARG ** SQLITE_FUNC_TYPEOF == OPFLAG_TYPEOFARG ** SQLITE_FUNC_CONSTANT == SQLITE_DETERMINISTIC from the API +** SQLITE_FUNC_DIRECT == SQLITE_DIRECTONLY from the API +** SQLITE_FUNC_UNSAFE == SQLITE_INNOCUOUS ** SQLITE_FUNC_ENCMASK depends on SQLITE_UTF* macros in the API */ #define SQLITE_FUNC_ENCMASK 0x0003 /* SQLITE_UTF8, SQLITE_UTF16BE or UTF16LE */ @@ -16176,14 +17122,29 @@ struct FuncDestructor { #define SQLITE_FUNC_LENGTH 0x0040 /* Built-in length() function */ #define SQLITE_FUNC_TYPEOF 0x0080 /* Built-in typeof() function */ #define SQLITE_FUNC_COUNT 0x0100 /* Built-in count(*) aggregate */ -#define SQLITE_FUNC_COALESCE 0x0200 /* Built-in coalesce() or ifnull() */ +/* 0x0200 -- available for reuse */ #define SQLITE_FUNC_UNLIKELY 0x0400 /* Built-in unlikely() function */ #define SQLITE_FUNC_CONSTANT 0x0800 /* Constant inputs give a constant output */ #define SQLITE_FUNC_MINMAX 0x1000 /* True for min() and max() aggregates */ #define SQLITE_FUNC_SLOCHNG 0x2000 /* "Slow Change". Value constant during a ** single query - might change over time */ -#define SQLITE_FUNC_AFFINITY 0x4000 /* Built-in affinity() function */ +#define SQLITE_FUNC_TEST 0x4000 /* Built-in testing functions */ #define SQLITE_FUNC_OFFSET 0x8000 /* Built-in sqlite_offset() function */ +#define SQLITE_FUNC_WINDOW 0x00010000 /* Built-in window-only function */ +#define SQLITE_FUNC_INTERNAL 0x00040000 /* For use by NestedParse() only */ +#define SQLITE_FUNC_DIRECT 0x00080000 /* Not for use in TRIGGERs or VIEWs */ +#define SQLITE_FUNC_SUBTYPE 0x00100000 /* Result likely to have sub-type */ +#define SQLITE_FUNC_UNSAFE 0x00200000 /* Function has side effects */ +#define SQLITE_FUNC_INLINE 0x00400000 /* Functions implemented in-line */ + +/* Identifier numbers for each in-line function */ +#define INLINEFUNC_coalesce 0 +#define INLINEFUNC_implies_nonnull_row 1 +#define INLINEFUNC_expr_implies_expr 2 +#define INLINEFUNC_expr_compare 3 +#define INLINEFUNC_affinity 4 +#define INLINEFUNC_iif 5 +#define INLINEFUNC_unlikely 99 /* Default case */ /* ** The following three macros, FUNCTION(), LIKEFUNC() and AGGREGATE() are @@ -16199,6 +17160,22 @@ struct FuncDestructor { ** VFUNCTION(zName, nArg, iArg, bNC, xFunc) ** Like FUNCTION except it omits the SQLITE_FUNC_CONSTANT flag. ** +** SFUNCTION(zName, nArg, iArg, bNC, xFunc) +** Like FUNCTION except it omits the SQLITE_FUNC_CONSTANT flag and +** adds the SQLITE_DIRECTONLY flag. +** +** INLINE_FUNC(zName, nArg, iFuncId, mFlags) +** zName is the name of a function that is implemented by in-line +** byte code rather than by the usual callbacks. The iFuncId +** parameter determines the function id. The mFlags parameter is +** optional SQLITE_FUNC_ flags for this function. +** +** TEST_FUNC(zName, nArg, iFuncId, mFlags) +** zName is the name of a test-only function implemented by in-line +** byte code rather than by the usual callbacks. The iFuncId +** parameter determines the function id. The mFlags parameter is +** optional SQLITE_FUNC_ flags for this function. +** ** DFUNCTION(zName, nArg, iArg, bNC, xFunc) ** Like FUNCTION except it omits the SQLITE_FUNC_CONSTANT flag and ** adds the SQLITE_FUNC_SLOCHNG flag. Used for date & time functions @@ -16218,6 +17195,12 @@ struct FuncDestructor { ** are interpreted in the same way as the first 4 parameters to ** FUNCTION(). ** +** WFUNCTION(zName, nArg, iArg, xStep, xFinal, xValue, xInverse) +** Used to create an aggregate function definition implemented by +** the C functions xStep and xFinal. The first four parameters +** are interpreted in the same way as the first 4 parameters to +** FUNCTION(). +** ** LIKEFUNC(zName, nArg, pArg, flags) ** Used to create a scalar function definition of a function zName ** that accepts nArg arguments and is implemented by a call to C @@ -16228,31 +17211,42 @@ struct FuncDestructor { */ #define FUNCTION(zName, nArg, iArg, bNC, xFunc) \ {nArg, SQLITE_FUNC_CONSTANT|SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL), \ - SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, #zName, {0} } + SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, 0, #zName, {0} } #define VFUNCTION(zName, nArg, iArg, bNC, xFunc) \ {nArg, SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL), \ - SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, #zName, {0} } + SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, 0, #zName, {0} } +#define SFUNCTION(zName, nArg, iArg, bNC, xFunc) \ + {nArg, SQLITE_UTF8|SQLITE_DIRECTONLY|SQLITE_FUNC_UNSAFE, \ + SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, 0, #zName, {0} } +#define INLINE_FUNC(zName, nArg, iArg, mFlags) \ + {nArg, SQLITE_UTF8|SQLITE_FUNC_INLINE|SQLITE_FUNC_CONSTANT|(mFlags), \ + SQLITE_INT_TO_PTR(iArg), 0, noopFunc, 0, 0, 0, #zName, {0} } +#define TEST_FUNC(zName, nArg, iArg, mFlags) \ + {nArg, SQLITE_UTF8|SQLITE_FUNC_INTERNAL|SQLITE_FUNC_TEST| \ + SQLITE_FUNC_INLINE|SQLITE_FUNC_CONSTANT|(mFlags), \ + SQLITE_INT_TO_PTR(iArg), 0, noopFunc, 0, 0, 0, #zName, {0} } #define DFUNCTION(zName, nArg, iArg, bNC, xFunc) \ {nArg, SQLITE_FUNC_SLOCHNG|SQLITE_UTF8, \ - 0, 0, xFunc, 0, #zName, {0} } + 0, 0, xFunc, 0, 0, 0, #zName, {0} } #define PURE_DATE(zName, nArg, iArg, bNC, xFunc) \ {nArg, SQLITE_FUNC_SLOCHNG|SQLITE_UTF8|SQLITE_FUNC_CONSTANT, \ - (void*)&sqlite3Config, 0, xFunc, 0, #zName, {0} } + (void*)&sqlite3Config, 0, xFunc, 0, 0, 0, #zName, {0} } #define FUNCTION2(zName, nArg, iArg, bNC, xFunc, extraFlags) \ {nArg,SQLITE_FUNC_CONSTANT|SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL)|extraFlags,\ - SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, #zName, {0} } + SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, 0, #zName, {0} } #define STR_FUNCTION(zName, nArg, pArg, bNC, xFunc) \ {nArg, SQLITE_FUNC_SLOCHNG|SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL), \ - pArg, 0, xFunc, 0, #zName, } + pArg, 0, xFunc, 0, 0, 0, #zName, } #define LIKEFUNC(zName, nArg, arg, flags) \ {nArg, SQLITE_FUNC_CONSTANT|SQLITE_UTF8|flags, \ - (void *)arg, 0, likeFunc, 0, #zName, {0} } -#define AGGREGATE(zName, nArg, arg, nc, xStep, xFinal) \ - {nArg, SQLITE_UTF8|(nc*SQLITE_FUNC_NEEDCOLL), \ - SQLITE_INT_TO_PTR(arg), 0, xStep,xFinal,#zName, {0}} -#define AGGREGATE2(zName, nArg, arg, nc, xStep, xFinal, extraFlags) \ - {nArg, SQLITE_UTF8|(nc*SQLITE_FUNC_NEEDCOLL)|extraFlags, \ - SQLITE_INT_TO_PTR(arg), 0, xStep,xFinal,#zName, {0}} + (void *)arg, 0, likeFunc, 0, 0, 0, #zName, {0} } +#define WAGGREGATE(zName, nArg, arg, nc, xStep, xFinal, xValue, xInverse, f) \ + {nArg, SQLITE_UTF8|(nc*SQLITE_FUNC_NEEDCOLL)|f, \ + SQLITE_INT_TO_PTR(arg), 0, xStep,xFinal,xValue,xInverse,#zName, {0}} +#define INTERNAL_FUNCTION(zName, nArg, xFunc) \ + {nArg, SQLITE_FUNC_INTERNAL|SQLITE_UTF8|SQLITE_FUNC_CONSTANT, \ + 0, 0, xFunc, 0, 0, 0, #zName, {0} } + /* ** All current savepoints are stored in a linked list starting at @@ -16284,32 +17278,53 @@ struct Savepoint { struct Module { const sqlite3_module *pModule; /* Callback pointers */ const char *zName; /* Name passed to create_module() */ + int nRefModule; /* Number of pointers to this object */ void *pAux; /* pAux passed to create_module() */ void (*xDestroy)(void *); /* Module destructor function */ Table *pEpoTab; /* Eponymous table for this module */ }; /* -** information about each column of an SQL table is held in an instance -** of this structure. +** Information about each column of an SQL table is held in an instance +** of the Column structure, in the Table.aCol[] array. +** +** Definitions: +** +** "table column index" This is the index of the column in the +** Table.aCol[] array, and also the index of +** the column in the original CREATE TABLE stmt. +** +** "storage column index" This is the index of the column in the +** record BLOB generated by the OP_MakeRecord +** opcode. The storage column index is less than +** or equal to the table column index. It is +** equal if and only if there are no VIRTUAL +** columns to the left. */ struct Column { char *zName; /* Name of this column, \000, then the type */ - Expr *pDflt; /* Default value of this column */ + Expr *pDflt; /* Default value or GENERATED ALWAYS AS value */ char *zColl; /* Collating sequence. If NULL, use the default */ u8 notNull; /* An OE_ code for handling a NOT NULL constraint */ char affinity; /* One of the SQLITE_AFF_... values */ u8 szEst; /* Estimated size of value in this column. sizeof(INT)==1 */ - u8 colFlags; /* Boolean properties. See COLFLAG_ defines below */ + u8 hName; /* Column name hash for faster lookup */ + u16 colFlags; /* Boolean properties. See COLFLAG_ defines below */ }; /* Allowed values for Column.colFlags: */ -#define COLFLAG_PRIMKEY 0x0001 /* Column is part of the primary key */ -#define COLFLAG_HIDDEN 0x0002 /* A hidden column in a virtual table */ -#define COLFLAG_HASTYPE 0x0004 /* Type name follows column name */ -#define COLFLAG_UNIQUE 0x0008 /* Column def contains "UNIQUE" or "PK" */ +#define COLFLAG_PRIMKEY 0x0001 /* Column is part of the primary key */ +#define COLFLAG_HIDDEN 0x0002 /* A hidden column in a virtual table */ +#define COLFLAG_HASTYPE 0x0004 /* Type name follows column name */ +#define COLFLAG_UNIQUE 0x0008 /* Column def contains "UNIQUE" or "PK" */ #define COLFLAG_SORTERREF 0x0010 /* Use sorter-refs with this column */ +#define COLFLAG_VIRTUAL 0x0020 /* GENERATED ALWAYS AS ... VIRTUAL */ +#define COLFLAG_STORED 0x0040 /* GENERATED ALWAYS AS ... STORED */ +#define COLFLAG_NOTAVAIL 0x0080 /* STORED column not yet calculated */ +#define COLFLAG_BUSY 0x0100 /* Blocks recursion on GENERATED columns */ +#define COLFLAG_GENERATED 0x0060 /* Combo: _STORED, _VIRTUAL */ +#define COLFLAG_NOINSERT 0x0062 /* Combo: _HIDDEN, _STORED, _VIRTUAL */ /* ** A "Collating Sequence" is defined by an instance of the following @@ -16349,11 +17364,12 @@ struct CollSeq { ** Note also that the numeric types are grouped together so that testing ** for a numeric type is a single comparison. And the BLOB type is first. */ -#define SQLITE_AFF_BLOB 'A' -#define SQLITE_AFF_TEXT 'B' -#define SQLITE_AFF_NUMERIC 'C' -#define SQLITE_AFF_INTEGER 'D' -#define SQLITE_AFF_REAL 'E' +#define SQLITE_AFF_NONE 0x40 /* '@' */ +#define SQLITE_AFF_BLOB 0x41 /* 'A' */ +#define SQLITE_AFF_TEXT 0x42 /* 'B' */ +#define SQLITE_AFF_NUMERIC 0x43 /* 'C' */ +#define SQLITE_AFF_INTEGER 0x44 /* 'D' */ +#define SQLITE_AFF_REAL 0x45 /* 'E' */ #define sqlite3IsNumericAffinity(X) ((X)>=SQLITE_AFF_NUMERIC) @@ -16426,10 +17442,17 @@ struct VTable { sqlite3_vtab *pVtab; /* Pointer to vtab instance */ int nRef; /* Number of pointers to this structure */ u8 bConstraint; /* True if constraints are supported */ + u8 eVtabRisk; /* Riskiness of allowing hacker access */ int iSavepoint; /* Depth of the SAVEPOINT stack */ VTable *pNext; /* Next in linked list (see above) */ }; +/* Allowed values for VTable.eVtabRisk +*/ +#define SQLITE_VTABRISK_Low 0 +#define SQLITE_VTABRISK_Normal 1 +#define SQLITE_VTABRISK_High 2 + /* ** The schema for each SQL table and view is represented in memory ** by an instance of the following structure. @@ -16448,6 +17471,7 @@ struct Table { u32 tabFlags; /* Mask of TF_* values */ i16 iPKey; /* If not negative, use aCol[iPKey] as the rowid */ i16 nCol; /* Number of columns in this table */ + i16 nNVCol; /* Number of columns that are not VIRTUAL */ LogEst nRowLogEst; /* Estimated rows in table - from sqlite_stat1 table */ LogEst szTabRow; /* Estimated size of each table row in bytes */ #ifdef SQLITE_ENABLE_COSTMULT @@ -16474,19 +17498,28 @@ struct Table { ** followed by non-hidden columns. Example: "CREATE VIRTUAL TABLE x USING ** vtab1(a HIDDEN, b);". Since "b" is a non-hidden column but "a" is hidden, ** the TF_OOOHidden attribute would apply in this case. Such tables require -** special handling during INSERT processing. +** special handling during INSERT processing. The "OOO" means "Out Of Order". +** +** Constraints: +** +** TF_HasVirtual == COLFLAG_Virtual +** TF_HasStored == COLFLAG_Stored */ #define TF_Readonly 0x0001 /* Read-only system table */ #define TF_Ephemeral 0x0002 /* An ephemeral table */ #define TF_HasPrimaryKey 0x0004 /* Table has a primary key */ #define TF_Autoincrement 0x0008 /* Integer primary key is autoincrement */ #define TF_HasStat1 0x0010 /* nRowLogEst set from sqlite_stat1 */ -#define TF_WithoutRowid 0x0020 /* No rowid. PRIMARY KEY is the key */ -#define TF_NoVisibleRowid 0x0040 /* No user-visible "rowid" column */ -#define TF_OOOHidden 0x0080 /* Out-of-Order hidden columns */ +#define TF_HasVirtual 0x0020 /* Has one or more VIRTUAL columns */ +#define TF_HasStored 0x0040 /* Has one or more STORED columns */ +#define TF_HasGenerated 0x0060 /* Combo: HasVirtual + HasStored */ +#define TF_WithoutRowid 0x0080 /* No rowid. PRIMARY KEY is the key */ #define TF_StatsUsed 0x0100 /* Query planner decisions affected by ** Index.aiRowLogEst[] values */ -#define TF_HasNotNull 0x0200 /* Contains NOT NULL constraints */ +#define TF_NoVisibleRowid 0x0200 /* No user-visible "rowid" column */ +#define TF_OOOHidden 0x0400 /* Out-of-Order hidden columns */ +#define TF_HasNotNull 0x0800 /* Contains NOT NULL constraints */ +#define TF_Shadow 0x1000 /* True for a shadow table */ /* ** Test to see whether or not a table is a virtual table. This is @@ -16495,8 +17528,11 @@ struct Table { */ #ifndef SQLITE_OMIT_VIRTUALTABLE # define IsVirtual(X) ((X)->nModuleArg) +# define ExprIsVtab(X) \ + ((X)->op==TK_COLUMN && (X)->y.pTab!=0 && (X)->y.pTab->nModuleArg) #else # define IsVirtual(X) 0 +# define ExprIsVtab(X) 0 #endif /* @@ -16620,10 +17656,16 @@ struct KeyInfo { u16 nKeyField; /* Number of key columns in the index */ u16 nAllField; /* Total columns, including key plus others */ sqlite3 *db; /* The database connection */ - u8 *aSortOrder; /* Sort order for each column. */ + u8 *aSortFlags; /* Sort order for each column. */ CollSeq *aColl[1]; /* Collating sequence for each term of the key */ }; +/* +** Allowed bit values for entries in the KeyInfo.aSortFlags[] array. +*/ +#define KEYINFO_ORDER_DESC 0x01 /* DESC sort order */ +#define KEYINFO_ORDER_BIGNULL 0x02 /* NULL is larger than any other value */ + /* ** This object holds a record which has been parsed out into individual ** fields, for the purposes of doing a comparison. @@ -16722,7 +17764,7 @@ struct Index { u16 nKeyCol; /* Number of columns forming the key */ u16 nColumn; /* Number of columns stored in the index */ u8 onError; /* OE_Abort, OE_Ignore, OE_Replace, or OE_None */ - unsigned idxType:2; /* 1==UNIQUE, 2==PRIMARY KEY, 0==CREATE INDEX */ + unsigned idxType:2; /* 0:Normal 1:UNIQUE, 2:PRIMARY KEY, 3:IPK */ unsigned bUnordered:1; /* Use this index for == or IN queries only */ unsigned uniqNotNull:1; /* True if UNIQUE and NOT NULL for all columns */ unsigned isResized:1; /* True if resizeIndexObject() has been called */ @@ -16730,7 +17772,9 @@ struct Index { unsigned noSkipScan:1; /* Do not try to use skip-scan if true */ unsigned hasStat1:1; /* aiRowLogEst values come from sqlite_stat1 */ unsigned bNoQuery:1; /* Do not use this index to optimize queries */ -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 + unsigned bAscKeyBug:1; /* True if the bba7b69f9849b5bf bug applies */ + unsigned bHasVCol:1; /* Index references one or more VIRTUAL columns */ +#ifdef SQLITE_ENABLE_STAT4 int nSample; /* Number of elements in aSample[] */ int nSampleCol; /* Size of IndexSample.anEq[] and so on */ tRowcnt *aAvgEq; /* Average nEq values for keys not in aSample */ @@ -16738,6 +17782,7 @@ struct Index { tRowcnt *aiRowEst; /* Non-logarithmic stat1 data for this index */ tRowcnt nRowEst0; /* Non-logarithmic number of rows in the index */ #endif + Bitmask colNotIdxed; /* 0 for unindexed columns in pTab */ }; /* @@ -16746,6 +17791,7 @@ struct Index { #define SQLITE_IDXTYPE_APPDEF 0 /* Created using CREATE INDEX */ #define SQLITE_IDXTYPE_UNIQUE 1 /* Implements a UNIQUE constraint */ #define SQLITE_IDXTYPE_PRIMARYKEY 2 /* Is the PRIMARY KEY for the table */ +#define SQLITE_IDXTYPE_IPK 3 /* INTEGER PRIMARY KEY index */ /* Return true if index X is a PRIMARY KEY index */ #define IsPrimaryKeyIndex(X) ((X)->idxType==SQLITE_IDXTYPE_PRIMARYKEY) @@ -16760,7 +17806,7 @@ struct Index { #define XN_EXPR (-2) /* Indexed column is an expression */ /* -** Each sample stored in the sqlite_stat3 table is represented in memory +** Each sample stored in the sqlite_stat4 table is represented in memory ** using a structure of this type. See documentation at the top of the ** analyze.c source file for additional information. */ @@ -16772,13 +17818,21 @@ struct IndexSample { tRowcnt *anDLt; /* Est. number of distinct keys less than this sample */ }; +/* +** Possible values to use within the flags argument to sqlite3GetToken(). +*/ +#define SQLITE_TOKEN_QUOTED 0x1 /* Token is a quoted identifier. */ +#define SQLITE_TOKEN_KEYWORD 0x2 /* Token is a keyword. */ + /* ** Each token coming out of the lexer is an instance of ** this structure. Tokens are also used as part of an expression. ** -** Note if Token.z==0 then Token.dyn and Token.n are undefined and -** may contain random values. Do not make any assumptions about Token.dyn -** and Token.n when Token.z==0. +** The memory that "z" points to is owned by other objects. Take care +** that the owner of the "z" string does not deallocate the string before +** the Token goes out of scope! Very often, the "z" points to some place +** in the middle of the Parse.zSql text. But it might also point to a +** static string. */ struct Token { const char *z; /* Text of the token. Not NULL-terminated! */ @@ -16790,7 +17844,7 @@ struct Token { ** code for a SELECT that contains aggregate functions. ** ** If Expr.op==TK_AGG_COLUMN or TK_AGG_FUNCTION then Expr.pAggInfo is a -** pointer to this structure. The Expr.iColumn field is the index in +** pointer to this structure. The Expr.iAgg field is the index in ** AggInfo.aCol[] or AggInfo.aFunc[] of information needed to generate ** code for that node. ** @@ -16836,10 +17890,10 @@ struct AggInfo { ** it uses less memory in the Expr object, which is a big memory user ** in systems with lots of prepared statements. And few applications ** need more than about 10 or 20 variables. But some extreme users want -** to have prepared statements with over 32767 variables, and for them +** to have prepared statements with over 32766 variables, and for them ** the option is available (at compile-time). */ -#if SQLITE_MAX_VARIABLE_NUMBER<=32767 +#if SQLITE_MAX_VARIABLE_NUMBER<32767 typedef i16 ynVar; #else typedef int ynVar; @@ -16910,7 +17964,14 @@ typedef int ynVar; */ struct Expr { u8 op; /* Operation performed by this node */ - char affinity; /* The affinity of the column or 0 if not a column */ + char affExpr; /* affinity, or RAISE type */ + u8 op2; /* TK_REGISTER/TK_TRUTH: original value of Expr.op + ** TK_COLUMN: the value of p5 for OP_Column + ** TK_AGG_FUNCTION: nesting depth + ** TK_FUNCTION: NC_SelfRef flag if needs OP_PureFunc */ +#ifdef SQLITE_DEBUG + u8 vvaFlags; /* Verification flags. */ +#endif u32 flags; /* Various flags. EP_* See below */ union { char *zToken; /* Token value. Zero terminated and dequoted */ @@ -16941,47 +18002,65 @@ struct Expr { ** TK_REGISTER: register number ** TK_TRIGGER: 1 -> new, 0 -> old ** EP_Unlikely: 134217728 times likelihood + ** TK_IN: ephemerial table holding RHS + ** TK_SELECT_COLUMN: Number of columns on the LHS ** TK_SELECT: 1st register of result vector */ ynVar iColumn; /* TK_COLUMN: column index. -1 for rowid. ** TK_VARIABLE: variable number (always >= 1). ** TK_SELECT_COLUMN: column of the result vector */ i16 iAgg; /* Which entry in pAggInfo->aCol[] or ->aFunc[] */ i16 iRightJoinTable; /* If EP_FromJoin, the right table of the join */ - u8 op2; /* TK_REGISTER: original value of Expr.op - ** TK_COLUMN: the value of p5 for OP_Column - ** TK_AGG_FUNCTION: nesting depth */ AggInfo *pAggInfo; /* Used by TK_AGG_COLUMN and TK_AGG_FUNCTION */ - Table *pTab; /* Table for TK_COLUMN expressions. Can be NULL - ** for a column of an index on an expression */ + union { + Table *pTab; /* TK_COLUMN: Table containing column. Can be NULL + ** for a column of an index on an expression */ + Window *pWin; /* EP_WinFunc: Window/Filter defn for a function */ + struct { /* TK_IN, TK_SELECT, and TK_EXISTS */ + int iAddr; /* Subroutine entry address */ + int regReturn; /* Register used to hold return address */ + } sub; + } y; }; /* ** The following are the meanings of bits in the Expr.flags field. -*/ -#define EP_FromJoin 0x000001 /* Originates in ON/USING clause of outer join */ -#define EP_Agg 0x000002 /* Contains one or more aggregate functions */ -#define EP_HasFunc 0x000004 /* Contains one or more functions of any kind */ - /* 0x000008 // available for use */ -#define EP_Distinct 0x000010 /* Aggregate function with DISTINCT keyword */ -#define EP_VarSelect 0x000020 /* pSelect is correlated, not constant */ -#define EP_DblQuoted 0x000040 /* token.z was originally in "..." */ -#define EP_InfixFunc 0x000080 /* True for an infix function: LIKE, GLOB, etc */ -#define EP_Collate 0x000100 /* Tree contains a TK_COLLATE operator */ -#define EP_Generic 0x000200 /* Ignore COLLATE or affinity on this tree */ -#define EP_IntValue 0x000400 /* Integer value contained in u.iValue */ -#define EP_xIsSelect 0x000800 /* x.pSelect is valid (otherwise x.pList is) */ -#define EP_Skip 0x001000 /* COLLATE, AS, or UNLIKELY */ -#define EP_Reduced 0x002000 /* Expr struct EXPR_REDUCEDSIZE bytes only */ -#define EP_TokenOnly 0x004000 /* Expr struct EXPR_TOKENONLYSIZE bytes only */ -#define EP_Static 0x008000 /* Held in memory not obtained from malloc() */ -#define EP_MemToken 0x010000 /* Need to sqlite3DbFree() Expr.zToken */ -#define EP_NoReduce 0x020000 /* Cannot EXPRDUP_REDUCE this Expr */ -#define EP_Unlikely 0x040000 /* unlikely() or likelihood() function */ -#define EP_ConstFunc 0x080000 /* A SQLITE_FUNC_CONSTANT or _SLOCHNG function */ -#define EP_CanBeNull 0x100000 /* Can be null despite NOT NULL constraint */ -#define EP_Subquery 0x200000 /* Tree contains a TK_SELECT operator */ -#define EP_Alias 0x400000 /* Is an alias for a result set column */ -#define EP_Leaf 0x800000 /* Expr.pLeft, .pRight, .u.pSelect all NULL */ +** Value restrictions: +** +** EP_Agg == NC_HasAgg == SF_HasAgg +** EP_Win == NC_HasWin +*/ +#define EP_FromJoin 0x000001 /* Originates in ON/USING clause of outer join */ +#define EP_Distinct 0x000002 /* Aggregate function with DISTINCT keyword */ +#define EP_HasFunc 0x000004 /* Contains one or more functions of any kind */ +#define EP_FixedCol 0x000008 /* TK_Column with a known fixed value */ +#define EP_Agg 0x000010 /* Contains one or more aggregate functions */ +#define EP_VarSelect 0x000020 /* pSelect is correlated, not constant */ +#define EP_DblQuoted 0x000040 /* token.z was originally in "..." */ +#define EP_InfixFunc 0x000080 /* True for an infix function: LIKE, GLOB, etc */ +#define EP_Collate 0x000100 /* Tree contains a TK_COLLATE operator */ +#define EP_Commuted 0x000200 /* Comparison operator has been commuted */ +#define EP_IntValue 0x000400 /* Integer value contained in u.iValue */ +#define EP_xIsSelect 0x000800 /* x.pSelect is valid (otherwise x.pList is) */ +#define EP_Skip 0x001000 /* Operator does not contribute to affinity */ +#define EP_Reduced 0x002000 /* Expr struct EXPR_REDUCEDSIZE bytes only */ +#define EP_TokenOnly 0x004000 /* Expr struct EXPR_TOKENONLYSIZE bytes only */ +#define EP_Win 0x008000 /* Contains window functions */ +#define EP_MemToken 0x010000 /* Need to sqlite3DbFree() Expr.zToken */ + /* 0x020000 // available for reuse */ +#define EP_Unlikely 0x040000 /* unlikely() or likelihood() function */ +#define EP_ConstFunc 0x080000 /* A SQLITE_FUNC_CONSTANT or _SLOCHNG function */ +#define EP_CanBeNull 0x100000 /* Can be null despite NOT NULL constraint */ +#define EP_Subquery 0x200000 /* Tree contains a TK_SELECT operator */ +#define EP_Alias 0x400000 /* Is an alias for a result set column */ +#define EP_Leaf 0x800000 /* Expr.pLeft, .pRight, .u.pSelect all NULL */ +#define EP_WinFunc 0x1000000 /* TK_FUNCTION with Expr.y.pWin set */ +#define EP_Subrtn 0x2000000 /* Uses Expr.y.sub. TK_IN, _SELECT, or _EXISTS */ +#define EP_Quoted 0x4000000 /* TK_ID was originally quoted */ +#define EP_Static 0x8000000 /* Held in memory not obtained from malloc() */ +#define EP_IsTrue 0x10000000 /* Always has boolean value of TRUE */ +#define EP_IsFalse 0x20000000 /* Always has boolean value of FALSE */ +#define EP_FromDDL 0x40000000 /* Originates from sqlite_master */ + /* 0x80000000 // Available */ /* ** The EP_Propagate mask is a set of properties that automatically propagate @@ -16997,15 +18076,27 @@ struct Expr { #define ExprHasAllProperty(E,P) (((E)->flags&(P))==(P)) #define ExprSetProperty(E,P) (E)->flags|=(P) #define ExprClearProperty(E,P) (E)->flags&=~(P) +#define ExprAlwaysTrue(E) (((E)->flags&(EP_FromJoin|EP_IsTrue))==EP_IsTrue) +#define ExprAlwaysFalse(E) (((E)->flags&(EP_FromJoin|EP_IsFalse))==EP_IsFalse) + + +/* Flags for use with Expr.vvaFlags +*/ +#define EP_NoReduce 0x01 /* Cannot EXPRDUP_REDUCE this Expr */ +#define EP_Immutable 0x02 /* Do not change this Expr node */ /* The ExprSetVVAProperty() macro is used for Verification, Validation, ** and Accreditation only. It works like ExprSetProperty() during VVA ** processes but is a no-op for delivery. */ #ifdef SQLITE_DEBUG -# define ExprSetVVAProperty(E,P) (E)->flags|=(P) +# define ExprSetVVAProperty(E,P) (E)->vvaFlags|=(P) +# define ExprHasVVAProperty(E,P) (((E)->vvaFlags&(P))!=0) +# define ExprClearVVAProperties(E) (E)->vvaFlags = 0 #else # define ExprSetVVAProperty(E,P) +# define ExprHasVVAProperty(E,P) 0 +# define ExprClearVVAProperties(E) #endif /* @@ -17023,6 +18114,18 @@ struct Expr { */ #define EXPRDUP_REDUCE 0x0001 /* Used reduced-size Expr nodes */ +/* +** True if the expression passed as an argument was a function with +** an OVER() clause (a window function). +*/ +#ifdef SQLITE_OMIT_WINDOWFUNC +# define IsWindowFunc(p) 0 +#else +# define IsWindowFunc(p) ( \ + ExprHasProperty((p), EP_WinFunc) && p->y.pWin->eFrmType!=TK_FILTER \ + ) +#endif + /* ** A list of expressions. Each expression may optionally have a ** name. An expr/name combination can be used in several ways, such @@ -17031,25 +18134,31 @@ struct Expr { ** also be used as the argument to a function, in which case the a.zName ** field is not used. ** -** By default the Expr.zSpan field holds a human-readable description of -** the expression that is used in the generation of error messages and -** column labels. In this case, Expr.zSpan is typically the text of a -** column expression as it exists in a SELECT statement. However, if -** the bSpanIsTab flag is set, then zSpan is overloaded to mean the name -** of the result column in the form: DATABASE.TABLE.COLUMN. This later -** form is used for name resolution with nested FROM clauses. +** In order to try to keep memory usage down, the Expr.a.zEName field +** is used for multiple purposes: +** +** eEName Usage +** ---------- ------------------------- +** ENAME_NAME (1) the AS of result set column +** (2) COLUMN= of an UPDATE +** +** ENAME_TAB DB.TABLE.NAME used to resolve names +** of subqueries +** +** ENAME_SPAN Text of the original result set +** expression. */ struct ExprList { int nExpr; /* Number of expressions on the list */ struct ExprList_item { /* For each expression in the list */ Expr *pExpr; /* The parse tree for this expression */ - char *zName; /* Token associated with this expression */ - char *zSpan; /* Original text of the expression */ - u8 sortOrder; /* 1 for DESC or 0 for ASC */ + char *zEName; /* Token associated with this expression */ + u8 sortFlags; /* Mask of KEYINFO_ORDER_* flags */ + unsigned eEName :2; /* Meaning of zEName */ unsigned done :1; /* A flag to indicate when processing is finished */ - unsigned bSpanIsTab :1; /* zSpan holds DB.TABLE.COLUMN */ unsigned reusable :1; /* Constant expression is reusable */ unsigned bSorterRef :1; /* Defer evaluation until after sorting */ + unsigned bNulls: 1; /* True if explicit "NULLS FIRST/LAST" */ union { struct { u16 iOrderByCol; /* For ORDER BY, column number in result set */ @@ -17060,6 +18169,13 @@ struct ExprList { } a[1]; /* One slot for each expression in the list */ }; +/* +** Allowed values for Expr.a.eEName +*/ +#define ENAME_NAME 0 /* The AS clause of a result set */ +#define ENAME_SPAN 1 /* Complete text of the result set expression */ +#define ENAME_TAB 2 /* "DB.TABLE.NAME" for the result set */ + /* ** An instance of this structure can hold a simple list of identifiers, ** such as the list "a,b,c" in the following statements: @@ -17083,31 +18199,6 @@ struct IdList { int nId; /* Number of identifiers on the list */ }; -/* -** The bitmask datatype defined below is used for various optimizations. -** -** Changing this from a 64-bit to a 32-bit type limits the number of -** tables in a join to 32 instead of 64. But it also reduces the size -** of the library by 738 bytes on ix86. -*/ -#ifdef SQLITE_BITMASK_TYPE - typedef SQLITE_BITMASK_TYPE Bitmask; -#else - typedef u64 Bitmask; -#endif - -/* -** The number of bits in a Bitmask. "BMS" means "BitMask Size". -*/ -#define BMS ((int)(sizeof(Bitmask)*8)) - -/* -** A bit in a Bitmask -*/ -#define MASKBIT(n) (((Bitmask)1)<<(n)) -#define MASKBIT32(n) (((unsigned int)1)<<(n)) -#define ALLBITS ((Bitmask)-1) - /* ** The following structure describes the FROM clause of a SELECT statement. ** Each table or subquery in the FROM clause is a separate element of @@ -17148,6 +18239,7 @@ struct SrcList { unsigned isCorrelated :1; /* True if sub-query is correlated */ unsigned viaCoroutine :1; /* Implemented as a co-routine */ unsigned isRecursive :1; /* True for recursive reference in WITH */ + unsigned fromDDL :1; /* Comes from sqlite_master */ } fg; int iCursor; /* The VDBE cursor number used to access this table */ Expr *pOn; /* The ON clause of a join */ @@ -17238,29 +18330,37 @@ struct NameContext { NameContext *pNext; /* Next outer name context. NULL for outermost */ int nRef; /* Number of names resolved by this context */ int nErr; /* Number of errors encountered while resolving names */ - u16 ncFlags; /* Zero or more NC_* flags defined below */ + int ncFlags; /* Zero or more NC_* flags defined below */ + Select *pWinSelect; /* SELECT statement for any window functions */ }; /* ** Allowed values for the NameContext, ncFlags field. ** ** Value constraints (all checked via assert()): -** NC_HasAgg == SF_HasAgg +** NC_HasAgg == SF_HasAgg == EP_Agg ** NC_MinMaxAgg == SF_MinMaxAgg == SQLITE_FUNC_MINMAX -** -*/ -#define NC_AllowAgg 0x0001 /* Aggregate functions are allowed here */ -#define NC_PartIdx 0x0002 /* True if resolving a partial index WHERE */ -#define NC_IsCheck 0x0004 /* True if resolving names in a CHECK constraint */ -#define NC_InAggFunc 0x0008 /* True if analyzing arguments to an agg func */ -#define NC_HasAgg 0x0010 /* One or more aggregate functions seen */ -#define NC_IdxExpr 0x0020 /* True if resolving columns of CREATE INDEX */ -#define NC_VarSelect 0x0040 /* A correlated subquery has been seen */ -#define NC_UEList 0x0080 /* True if uNC.pEList is used */ -#define NC_UAggInfo 0x0100 /* True if uNC.pAggInfo is used */ -#define NC_UUpsert 0x0200 /* True if uNC.pUpsert is used */ -#define NC_MinMaxAgg 0x1000 /* min/max aggregates seen. See note above */ -#define NC_Complex 0x2000 /* True if a function or subquery seen */ +** NC_HasWin == EP_Win +** +*/ +#define NC_AllowAgg 0x00001 /* Aggregate functions are allowed here */ +#define NC_PartIdx 0x00002 /* True if resolving a partial index WHERE */ +#define NC_IsCheck 0x00004 /* True if resolving a CHECK constraint */ +#define NC_GenCol 0x00008 /* True for a GENERATED ALWAYS AS clause */ +#define NC_HasAgg 0x00010 /* One or more aggregate functions seen */ +#define NC_IdxExpr 0x00020 /* True if resolving columns of CREATE INDEX */ +#define NC_SelfRef 0x0002e /* Combo: PartIdx, isCheck, GenCol, and IdxExpr */ +#define NC_VarSelect 0x00040 /* A correlated subquery has been seen */ +#define NC_UEList 0x00080 /* True if uNC.pEList is used */ +#define NC_UAggInfo 0x00100 /* True if uNC.pAggInfo is used */ +#define NC_UUpsert 0x00200 /* True if uNC.pUpsert is used */ +#define NC_MinMaxAgg 0x01000 /* min/max aggregates seen. See note above */ +#define NC_Complex 0x02000 /* True if a function or subquery seen */ +#define NC_AllowWin 0x04000 /* Window functions are allowed here */ +#define NC_HasWin 0x08000 /* One or more window functions seen */ +#define NC_IsDDL 0x10000 /* Resolving names in a CREATE statement */ +#define NC_InAggFunc 0x20000 /* True if analyzing arguments to an agg func */ +#define NC_FromDDL 0x40000 /* SQL text comes from sqlite_master */ /* ** An instance of the following object describes a single ON CONFLICT @@ -17310,15 +18410,13 @@ struct Upsert { ** sequences for the ORDER BY clause. */ struct Select { - ExprList *pEList; /* The fields of the result */ u8 op; /* One of: TK_UNION TK_ALL TK_INTERSECT TK_EXCEPT */ LogEst nSelectRow; /* Estimated number of result rows */ u32 selFlags; /* Various SF_* values */ int iLimit, iOffset; /* Memory registers holding LIMIT & OFFSET counters */ -#if SELECTTRACE_ENABLED - char zSelName[12]; /* Symbolic name of this SELECT use for debugging */ -#endif + u32 selId; /* Unique identifier number for this SELECT */ int addrOpenEphm[2]; /* OP_OpenEphem opcodes related to this select */ + ExprList *pEList; /* The fields of the result */ SrcList *pSrc; /* The FROM clause */ Expr *pWhere; /* The WHERE clause */ ExprList *pGroupBy; /* The GROUP BY clause */ @@ -17328,6 +18426,10 @@ struct Select { Select *pNext; /* Next select to the left in a compound */ Expr *pLimit; /* LIMIT expression. NULL means not used. */ With *pWith; /* WITH clause attached to this select. Or NULL. */ +#ifndef SQLITE_OMIT_WINDOWFUNC + Window *pWin; /* List of window functions */ + Window *pWinDefn; /* List of named window definitions */ +#endif }; /* @@ -17339,25 +18441,28 @@ struct Select { ** SF_MinMaxAgg == NC_MinMaxAgg == SQLITE_FUNC_MINMAX ** SF_FixedLimit == WHERE_USE_LIMIT */ -#define SF_Distinct 0x00001 /* Output should be DISTINCT */ -#define SF_All 0x00002 /* Includes the ALL keyword */ -#define SF_Resolved 0x00004 /* Identifiers have been resolved */ -#define SF_Aggregate 0x00008 /* Contains agg functions or a GROUP BY */ -#define SF_HasAgg 0x00010 /* Contains aggregate functions */ -#define SF_UsesEphemeral 0x00020 /* Uses the OpenEphemeral opcode */ -#define SF_Expanded 0x00040 /* sqlite3SelectExpand() called on this */ -#define SF_HasTypeInfo 0x00080 /* FROM subqueries have Table metadata */ -#define SF_Compound 0x00100 /* Part of a compound query */ -#define SF_Values 0x00200 /* Synthesized from VALUES clause */ -#define SF_MultiValue 0x00400 /* Single VALUES term with multiple rows */ -#define SF_NestedFrom 0x00800 /* Part of a parenthesized FROM clause */ -#define SF_MinMaxAgg 0x01000 /* Aggregate containing min() or max() */ -#define SF_Recursive 0x02000 /* The recursive part of a recursive CTE */ -#define SF_FixedLimit 0x04000 /* nSelectRow set by a constant LIMIT */ -#define SF_MaybeConvert 0x08000 /* Need convertCompoundSelectToSubquery() */ -#define SF_Converted 0x10000 /* By convertCompoundSelectToSubquery() */ -#define SF_IncludeHidden 0x20000 /* Include hidden columns in output */ -#define SF_ComplexResult 0x40000 /* Result contains subquery or function */ +#define SF_Distinct 0x0000001 /* Output should be DISTINCT */ +#define SF_All 0x0000002 /* Includes the ALL keyword */ +#define SF_Resolved 0x0000004 /* Identifiers have been resolved */ +#define SF_Aggregate 0x0000008 /* Contains agg functions or a GROUP BY */ +#define SF_HasAgg 0x0000010 /* Contains aggregate functions */ +#define SF_UsesEphemeral 0x0000020 /* Uses the OpenEphemeral opcode */ +#define SF_Expanded 0x0000040 /* sqlite3SelectExpand() called on this */ +#define SF_HasTypeInfo 0x0000080 /* FROM subqueries have Table metadata */ +#define SF_Compound 0x0000100 /* Part of a compound query */ +#define SF_Values 0x0000200 /* Synthesized from VALUES clause */ +#define SF_MultiValue 0x0000400 /* Single VALUES term with multiple rows */ +#define SF_NestedFrom 0x0000800 /* Part of a parenthesized FROM clause */ +#define SF_MinMaxAgg 0x0001000 /* Aggregate containing min() or max() */ +#define SF_Recursive 0x0002000 /* The recursive part of a recursive CTE */ +#define SF_FixedLimit 0x0004000 /* nSelectRow set by a constant LIMIT */ +#define SF_MaybeConvert 0x0008000 /* Need convertCompoundSelectToSubquery() */ +#define SF_Converted 0x0010000 /* By convertCompoundSelectToSubquery() */ +#define SF_IncludeHidden 0x0020000 /* Include hidden columns in output */ +#define SF_ComplexResult 0x0040000 /* Result contains subquery or function */ +#define SF_WhereBegin 0x0080000 /* Really a WhereBegin() call. Debug Only */ +#define SF_WinRewrite 0x0100000 /* Window function rewrite accomplished */ +#define SF_View 0x0200000 /* SELECT statement is a view */ /* ** The results of a SELECT can be distributed in several ways, as defined @@ -17471,13 +18576,6 @@ struct AutoincInfo { int regCtr; /* Memory register holding the rowid counter */ }; -/* -** Size of the column cache -*/ -#ifndef SQLITE_N_COLCACHE -# define SQLITE_N_COLCACHE 10 -#endif - /* ** At least one instance of the following structure is created for each ** trigger that may be fired while parsing an INSERT, UPDATE or DELETE @@ -17553,19 +18651,17 @@ struct Parse { u8 hasCompound; /* Need to invoke convertCompoundSelectToSubquery() */ u8 okConstFactor; /* OK to factor out constants */ u8 disableLookaside; /* Number of times lookaside has been disabled */ - u8 nColCache; /* Number of entries in aColCache[] */ + u8 disableVtab; /* Disable all virtual tables for this parse */ int nRangeReg; /* Size of the temporary register block */ int iRangeReg; /* First register in temporary register block */ int nErr; /* Number of errors seen */ int nTab; /* Number of previously allocated VDBE cursors */ int nMem; /* Number of memory cells used so far */ - int nOpAlloc; /* Number of slots allocated for Vdbe.aOp[] */ int szOpAlloc; /* Bytes of memory space allocated for Vdbe.aOp[] */ int iSelfTab; /* Table associated with an index on expr, or negative ** of the base register during check-constraint eval */ - int iCacheLevel; /* ColCache valid when aColCache[].iLevel<=iCacheLevel */ - int iCacheCnt; /* Counter used to generate aColCache[].lru values */ - int nLabel; /* Number of labels used */ + int nLabel; /* The *negative* of the number of labels used */ + int nLabelAlloc; /* Number of slots in aLabel */ int *aLabel; /* Space to hold the labels */ ExprList *pConstExpr;/* Constant expressions */ Token constraintName;/* Name of the constraint currently being parsed */ @@ -17574,9 +18670,7 @@ struct Parse { int regRowid; /* Register holding rowid of CREATE TABLE entry */ int regRoot; /* Register holding root page number for new objects */ int nMaxArg; /* Max args passed to user function by sub-program */ -#if SELECTTRACE_ENABLED - int nSelect; /* Number of SELECT statements seen */ -#endif + int nSelect; /* Number of SELECT stmts. Counter for Select.selId */ #ifndef SQLITE_OMIT_SHARED_CACHE int nTableLock; /* Number of locks in aTableLock */ TableLock *aTableLock; /* Required table locks for shared-cache mode */ @@ -17584,6 +18678,7 @@ struct Parse { AutoincInfo *pAinc; /* Information about AUTOINCREMENT counters */ Parse *pToplevel; /* Parse structure for main program (or NULL) */ Table *pTriggerTab; /* Table triggers are being coded for */ + Parse *pParentParse; /* Parent parser if this parser is nested */ int addrCrTab; /* Address of OP_CreateBtree opcode on CREATE TABLE */ u32 nQueryLoop; /* Est number of iterations of a query (10*log2(N)) */ u32 oldmask; /* Mask of old.* columns referenced */ @@ -17596,17 +18691,9 @@ struct Parse { ** Fields above must be initialized to zero. The fields that follow, ** down to the beginning of the recursive section, do not need to be ** initialized as they will be set before being used. The boundary is - ** determined by offsetof(Parse,aColCache). + ** determined by offsetof(Parse,aTempReg). **************************************************************************/ - struct yColCache { - int iTable; /* Table cursor number */ - i16 iColumn; /* Table column number */ - u8 tempReg; /* iReg is a temp register that needs to be freed */ - int iLevel; /* Nesting level */ - int iReg; /* Reg with value of this column. 0 means none. */ - int lru; /* Least recently used entry has the smallest value */ - } aColCache[SQLITE_N_COLCACHE]; /* One for each column cache entry */ int aTempReg[8]; /* Holding area for temporary registers */ Token sNameToken; /* Token with unqualified schema object name */ @@ -17621,8 +18708,10 @@ struct Parse { ynVar nVar; /* Number of '?' variables seen in the SQL so far */ u8 iPkSortOrder; /* ASC or DESC for INTEGER PRIMARY KEY */ u8 explain; /* True if the EXPLAIN flag is found on the query */ +#if !(defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_OMIT_ALTERTABLE)) + u8 eParseMode; /* PARSE_MODE_XXX constant */ +#endif #ifndef SQLITE_OMIT_VIRTUALTABLE - u8 declareVtab; /* True if inside sqlite3_declare_vtab() */ int nVtabLock; /* Number of virtual tables to lock */ #endif int nHeight; /* Expression tree height of current sub-select */ @@ -17633,6 +18722,9 @@ struct Parse { Vdbe *pReprepare; /* VM being reprepared (sqlite3Reprepare()) */ const char *zTail; /* All SQL text past the last semicolon parsed */ Table *pNewTable; /* A table being constructed by CREATE TABLE */ + Index *pNewIndex; /* An index being constructed by CREATE INDEX. + ** Also used to hold redundant UNIQUE constraints + ** during a RENAME COLUMN */ Trigger *pNewTrigger; /* Trigger under construct by a CREATE TRIGGER */ const char *zAuthContext; /* The 6th parameter to db->xAuth callbacks */ #ifndef SQLITE_OMIT_VIRTUALTABLE @@ -17643,12 +18735,20 @@ struct Parse { TriggerPrg *pTriggerPrg; /* Linked list of coded triggers */ With *pWith; /* Current WITH clause, or NULL */ With *pWithToFree; /* Free this WITH object at the end of the parse */ +#ifndef SQLITE_OMIT_ALTERTABLE + RenameToken *pRename; /* Tokens subject to renaming by ALTER TABLE */ +#endif }; +#define PARSE_MODE_NORMAL 0 +#define PARSE_MODE_DECLARE_VTAB 1 +#define PARSE_MODE_RENAME 2 +#define PARSE_MODE_UNMAP 3 + /* ** Sizes and pointers of various parts of the Parse object. */ -#define PARSE_HDR_SZ offsetof(Parse,aColCache) /* Recursive part w/o aColCache*/ +#define PARSE_HDR_SZ offsetof(Parse,aTempReg) /* Recursive part w/o aColCache*/ #define PARSE_RECURSE_SZ offsetof(Parse,sLastToken) /* Recursive part */ #define PARSE_TAIL_SZ (sizeof(Parse)-PARSE_RECURSE_SZ) /* Non-recursive part */ #define PARSE_TAIL(X) (((char*)(X))+PARSE_RECURSE_SZ) /* Pointer to tail */ @@ -17659,7 +18759,19 @@ struct Parse { #ifdef SQLITE_OMIT_VIRTUALTABLE #define IN_DECLARE_VTAB 0 #else - #define IN_DECLARE_VTAB (pParse->declareVtab) + #define IN_DECLARE_VTAB (pParse->eParseMode==PARSE_MODE_DECLARE_VTAB) +#endif + +#if defined(SQLITE_OMIT_ALTERTABLE) + #define IN_RENAME_OBJECT 0 +#else + #define IN_RENAME_OBJECT (pParse->eParseMode>=PARSE_MODE_RENAME) +#endif + +#if defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_OMIT_ALTERTABLE) + #define IN_SPECIAL_PARSE 0 +#else + #define IN_SPECIAL_PARSE (pParse->eParseMode!=PARSE_MODE_NORMAL) #endif /* @@ -17685,6 +18797,7 @@ struct AuthContext { */ #define OPFLAG_NCHANGE 0x01 /* OP_Insert: Set to update db->nChange */ /* Also used in P2 (not P5) of OP_Delete */ +#define OPFLAG_NOCHNG 0x01 /* OP_VColumn nochange for UPDATE */ #define OPFLAG_EPHEM 0x01 /* OP_Column: Ephemeral output is ok */ #define OPFLAG_LASTROWID 0x20 /* Set to update db->lastRowid */ #define OPFLAG_ISUPDATE 0x04 /* This OP_Insert is an sql UPDATE */ @@ -17803,7 +18916,7 @@ typedef struct DbFixer DbFixer; struct DbFixer { Parse *pParse; /* The parsing context. Error messages written here */ Schema *pSchema; /* Fix items to this schema */ - int bVarOnly; /* Check for variable references only */ + u8 bTemp; /* True for TEMP schema entries */ const char *zDb; /* Make sure all objects are contained in this database */ const char *zType; /* Type of the container - used for error messages */ const Token *pName; /* Name of the container - used for error messages */ @@ -17838,8 +18951,15 @@ typedef struct { char **pzErrMsg; /* Error message stored here */ int iDb; /* 0 for main database. 1 for TEMP, 2.. for ATTACHed */ int rc; /* Result code stored here */ + u32 mInitFlags; /* Flags controlling error messages */ + u32 nInitRow; /* Number of rows processed */ } InitData; +/* +** Allowed values for mInitFlags +*/ +#define INITFLAG_AlterTable 0x0001 /* This is a reparse after ALTER TABLE */ + /* ** Structure containing global configuration data for the SQLite library. ** @@ -17847,11 +18967,12 @@ typedef struct { */ struct Sqlite3Config { int bMemstat; /* True to enable memory status */ - int bCoreMutex; /* True to enable core mutexing */ - int bFullMutex; /* True to enable full mutexing */ - int bOpenUri; /* True to interpret filenames as URIs */ - int bUseCis; /* Use covering indices for full-scans */ - int bSmallMalloc; /* Avoid large memory allocations if true */ + u8 bCoreMutex; /* True to enable core mutexing */ + u8 bFullMutex; /* True to enable full mutexing */ + u8 bOpenUri; /* True to interpret filenames as URIs */ + u8 bUseCis; /* Use covering indices for full-scans */ + u8 bSmallMalloc; /* Avoid large memory allocations if true */ + u8 bExtraSchemaChecks; /* Verify type,name,tbl_name in schema */ int mxStrlen; /* Maximum string length */ int neverCorrupt; /* Database is always well-formed */ int szLookaside; /* Default lookaside buffer size */ @@ -17890,15 +19011,19 @@ struct Sqlite3Config { /* The following callback (if not NULL) is invoked on every VDBE branch ** operation. Set the callback using SQLITE_TESTCTRL_VDBE_COVERAGE. */ - void (*xVdbeBranch)(void*,int iSrcLine,u8 eThis,u8 eMx); /* Callback */ + void (*xVdbeBranch)(void*,unsigned iSrcLine,u8 eThis,u8 eMx); /* Callback */ void *pVdbeBranchArg; /* 1st argument */ #endif +#ifdef SQLITE_ENABLE_DESERIALIZE + sqlite3_int64 mxMemdbSize; /* Default max memdb size */ +#endif #ifndef SQLITE_UNTESTABLE int (*xTestCallback)(int); /* Invoked by sqlite3FaultSim() */ #endif int bLocaltimeFault; /* True to fail localtime() calls */ int iOnceResetThreshold; /* When to reset OP_Once counters */ u32 szSorterRef; /* Min size in bytes to use sorter-refs */ + unsigned int iPrngSeed; /* Alternative fixed seed for the PRNG */ }; /* @@ -17928,7 +19053,7 @@ struct Walker { int (*xSelectCallback)(Walker*,Select*); /* Callback for SELECTs */ void (*xSelectCallback2)(Walker*,Select*);/* Second callback for SELECTs */ int walkerDepth; /* Number of subqueries */ - u8 eCode; /* A small processing code */ + u16 eCode; /* A small processing code */ union { /* Extra data for callback */ NameContext *pNC; /* Naming context */ int n; /* A counter */ @@ -17941,6 +19066,11 @@ struct Walker { struct IdxExprTrans *pIdxTrans; /* Convert idxed expr to column */ ExprList *pGroupBy; /* GROUP BY clause */ Select *pSelect; /* HAVING to WHERE clause ctx */ + struct WindowRewrite *pRewrite; /* Window rewrite context */ + struct WhereConst *pConst; /* WHERE clause constants */ + struct RenameCtx *pRename; /* RENAME COLUMN context */ + struct Table *pTab; /* Table of generated column */ + struct SrcList_item *pSrcItem; /* A single FROM clause item */ } u; }; @@ -17953,6 +19083,9 @@ SQLITE_PRIVATE int sqlite3WalkSelectFrom(Walker*, Select*); SQLITE_PRIVATE int sqlite3ExprWalkNoop(Walker*, Expr*); SQLITE_PRIVATE int sqlite3SelectWalkNoop(Walker*, Select*); SQLITE_PRIVATE int sqlite3SelectWalkFail(Walker*, Select*); +SQLITE_PRIVATE int sqlite3WalkerDepthIncrease(Walker*,Select*); +SQLITE_PRIVATE void sqlite3WalkerDepthDecrease(Walker*,Select*); + #ifdef SQLITE_DEBUG SQLITE_PRIVATE void sqlite3SelectWalkAssert2(Walker*, Select*); #endif @@ -17991,6 +19124,85 @@ struct TreeView { }; #endif /* SQLITE_DEBUG */ +/* +** This object is used in various ways, most (but not all) related to window +** functions. +** +** (1) A single instance of this structure is attached to the +** the Expr.y.pWin field for each window function in an expression tree. +** This object holds the information contained in the OVER clause, +** plus additional fields used during code generation. +** +** (2) All window functions in a single SELECT form a linked-list +** attached to Select.pWin. The Window.pFunc and Window.pExpr +** fields point back to the expression that is the window function. +** +** (3) The terms of the WINDOW clause of a SELECT are instances of this +** object on a linked list attached to Select.pWinDefn. +** +** (4) For an aggregate function with a FILTER clause, an instance +** of this object is stored in Expr.y.pWin with eFrmType set to +** TK_FILTER. In this case the only field used is Window.pFilter. +** +** The uses (1) and (2) are really the same Window object that just happens +** to be accessible in two different ways. Use case (3) are separate objects. +*/ +struct Window { + char *zName; /* Name of window (may be NULL) */ + char *zBase; /* Name of base window for chaining (may be NULL) */ + ExprList *pPartition; /* PARTITION BY clause */ + ExprList *pOrderBy; /* ORDER BY clause */ + u8 eFrmType; /* TK_RANGE, TK_GROUPS, TK_ROWS, or 0 */ + u8 eStart; /* UNBOUNDED, CURRENT, PRECEDING or FOLLOWING */ + u8 eEnd; /* UNBOUNDED, CURRENT, PRECEDING or FOLLOWING */ + u8 bImplicitFrame; /* True if frame was implicitly specified */ + u8 eExclude; /* TK_NO, TK_CURRENT, TK_TIES, TK_GROUP, or 0 */ + Expr *pStart; /* Expression for " PRECEDING" */ + Expr *pEnd; /* Expression for " FOLLOWING" */ + Window **ppThis; /* Pointer to this object in Select.pWin list */ + Window *pNextWin; /* Next window function belonging to this SELECT */ + Expr *pFilter; /* The FILTER expression */ + FuncDef *pFunc; /* The function */ + int iEphCsr; /* Partition buffer or Peer buffer */ + int regAccum; /* Accumulator */ + int regResult; /* Interim result */ + int csrApp; /* Function cursor (used by min/max) */ + int regApp; /* Function register (also used by min/max) */ + int regPart; /* Array of registers for PARTITION BY values */ + Expr *pOwner; /* Expression object this window is attached to */ + int nBufferCol; /* Number of columns in buffer table */ + int iArgCol; /* Offset of first argument for this function */ + int regOne; /* Register containing constant value 1 */ + int regStartRowid; + int regEndRowid; + u8 bExprArgs; /* Defer evaluation of window function arguments + ** due to the SQLITE_SUBTYPE flag */ +}; + +#ifndef SQLITE_OMIT_WINDOWFUNC +SQLITE_PRIVATE void sqlite3WindowDelete(sqlite3*, Window*); +SQLITE_PRIVATE void sqlite3WindowUnlinkFromSelect(Window*); +SQLITE_PRIVATE void sqlite3WindowListDelete(sqlite3 *db, Window *p); +SQLITE_PRIVATE Window *sqlite3WindowAlloc(Parse*, int, int, Expr*, int , Expr*, u8); +SQLITE_PRIVATE void sqlite3WindowAttach(Parse*, Expr*, Window*); +SQLITE_PRIVATE void sqlite3WindowLink(Select *pSel, Window *pWin); +SQLITE_PRIVATE int sqlite3WindowCompare(Parse*, Window*, Window*, int); +SQLITE_PRIVATE void sqlite3WindowCodeInit(Parse*, Select*); +SQLITE_PRIVATE void sqlite3WindowCodeStep(Parse*, Select*, WhereInfo*, int, int); +SQLITE_PRIVATE int sqlite3WindowRewrite(Parse*, Select*); +SQLITE_PRIVATE int sqlite3ExpandSubquery(Parse*, struct SrcList_item*); +SQLITE_PRIVATE void sqlite3WindowUpdate(Parse*, Window*, Window*, FuncDef*); +SQLITE_PRIVATE Window *sqlite3WindowDup(sqlite3 *db, Expr *pOwner, Window *p); +SQLITE_PRIVATE Window *sqlite3WindowListDup(sqlite3 *db, Window *p); +SQLITE_PRIVATE void sqlite3WindowFunctions(void); +SQLITE_PRIVATE void sqlite3WindowChain(Parse*, Window*, Window*); +SQLITE_PRIVATE Window *sqlite3WindowAssemble(Parse*, Window*, ExprList*, ExprList*, Token*); +#else +# define sqlite3WindowDelete(a,b) +# define sqlite3WindowFunctions() +# define sqlite3WindowAttach(a,b,c) +#endif + /* ** Assuming zIn points to the first byte of a UTF-8 character, ** advance zIn to point to the first byte of the next UTF-8 character. @@ -18018,13 +19230,16 @@ SQLITE_PRIVATE int sqlite3CantopenError(int); #ifdef SQLITE_DEBUG SQLITE_PRIVATE int sqlite3NomemError(int); SQLITE_PRIVATE int sqlite3IoerrnomemError(int); -SQLITE_PRIVATE int sqlite3CorruptPgnoError(int,Pgno); # define SQLITE_NOMEM_BKPT sqlite3NomemError(__LINE__) # define SQLITE_IOERR_NOMEM_BKPT sqlite3IoerrnomemError(__LINE__) -# define SQLITE_CORRUPT_PGNO(P) sqlite3CorruptPgnoError(__LINE__,(P)) #else # define SQLITE_NOMEM_BKPT SQLITE_NOMEM # define SQLITE_IOERR_NOMEM_BKPT SQLITE_IOERR_NOMEM +#endif +#if defined(SQLITE_DEBUG) || defined(SQLITE_ENABLE_CORRUPT_PGNO) +SQLITE_PRIVATE int sqlite3CorruptPgnoError(int,Pgno); +# define SQLITE_CORRUPT_PGNO(P) sqlite3CorruptPgnoError(__LINE__,(P)) +#else # define SQLITE_CORRUPT_PGNO(P) sqlite3CorruptError(__LINE__) #endif @@ -18078,15 +19293,14 @@ SQLITE_PRIVATE int sqlite3CorruptPgnoError(int,Pgno); # define sqlite3Tolower(x) tolower((unsigned char)(x)) # define sqlite3Isquote(x) ((x)=='"'||(x)=='\''||(x)=='['||(x)=='`') #endif -#ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS SQLITE_PRIVATE int sqlite3IsIdChar(u8); -#endif /* ** Internal function prototypes */ SQLITE_PRIVATE int sqlite3StrICmp(const char*,const char*); SQLITE_PRIVATE int sqlite3Strlen30(const char*); +#define sqlite3Strlen30NN(C) (strlen(C)&0x3fffffff) SQLITE_PRIVATE char *sqlite3ColumnType(Column*,char*); #define sqlite3StrNICmp sqlite3_strnicmp @@ -18175,8 +19389,12 @@ SQLITE_PRIVATE void sqlite3MutexWarnOnContention(sqlite3_mutex*); #endif #ifndef SQLITE_OMIT_FLOATING_POINT +# define EXP754 (((u64)0x7ff)<<52) +# define MAN754 ((((u64)1)<<52)-1) +# define IsNaN(X) (((X)&EXP754)==EXP754 && ((X)&MAN754)!=0) SQLITE_PRIVATE int sqlite3IsNaN(double); #else +# define IsNaN(X) 0 # define sqlite3IsNaN(X) 0 #endif @@ -18203,14 +19421,21 @@ SQLITE_PRIVATE void *sqlite3TestTextToPtr(const char*); SQLITE_PRIVATE void sqlite3TreeViewExpr(TreeView*, const Expr*, u8); SQLITE_PRIVATE void sqlite3TreeViewBareExprList(TreeView*, const ExprList*, const char*); SQLITE_PRIVATE void sqlite3TreeViewExprList(TreeView*, const ExprList*, u8, const char*); +SQLITE_PRIVATE void sqlite3TreeViewSrcList(TreeView*, const SrcList*); SQLITE_PRIVATE void sqlite3TreeViewSelect(TreeView*, const Select*, u8); SQLITE_PRIVATE void sqlite3TreeViewWith(TreeView*, const With*, u8); +#ifndef SQLITE_OMIT_WINDOWFUNC +SQLITE_PRIVATE void sqlite3TreeViewWindow(TreeView*, const Window*, u8); +SQLITE_PRIVATE void sqlite3TreeViewWinFunc(TreeView*, const Window*, u8); +#endif #endif SQLITE_PRIVATE void sqlite3SetString(char **, sqlite3*, const char*); SQLITE_PRIVATE void sqlite3ErrorMsg(Parse*, const char*, ...); +SQLITE_PRIVATE int sqlite3ErrorToParser(sqlite3*,int); SQLITE_PRIVATE void sqlite3Dequote(char*); +SQLITE_PRIVATE void sqlite3DequoteExpr(Expr*); SQLITE_PRIVATE void sqlite3TokenInit(Token*,char*); SQLITE_PRIVATE int sqlite3KeywordCode(const unsigned char*, int); SQLITE_PRIVATE int sqlite3RunParser(Parse*, const char*, char **); @@ -18228,19 +19453,24 @@ SQLITE_PRIVATE Expr *sqlite3Expr(sqlite3*,int,const char*); SQLITE_PRIVATE void sqlite3ExprAttachSubtrees(sqlite3*,Expr*,Expr*,Expr*); SQLITE_PRIVATE Expr *sqlite3PExpr(Parse*, int, Expr*, Expr*); SQLITE_PRIVATE void sqlite3PExprAddSelect(Parse*, Expr*, Select*); -SQLITE_PRIVATE Expr *sqlite3ExprAnd(sqlite3*,Expr*, Expr*); -SQLITE_PRIVATE Expr *sqlite3ExprFunction(Parse*,ExprList*, Token*); +SQLITE_PRIVATE Expr *sqlite3ExprAnd(Parse*,Expr*, Expr*); +SQLITE_PRIVATE Expr *sqlite3ExprSimplifiedAndOr(Expr*); +SQLITE_PRIVATE Expr *sqlite3ExprFunction(Parse*,ExprList*, Token*, int); +SQLITE_PRIVATE void sqlite3ExprFunctionUsable(Parse*,Expr*,FuncDef*); SQLITE_PRIVATE void sqlite3ExprAssignVarNumber(Parse*, Expr*, u32); SQLITE_PRIVATE void sqlite3ExprDelete(sqlite3*, Expr*); +SQLITE_PRIVATE void sqlite3ExprUnmapAndDelete(Parse*, Expr*); SQLITE_PRIVATE ExprList *sqlite3ExprListAppend(Parse*,ExprList*,Expr*); SQLITE_PRIVATE ExprList *sqlite3ExprListAppendVector(Parse*,ExprList*,IdList*,Expr*); -SQLITE_PRIVATE void sqlite3ExprListSetSortOrder(ExprList*,int); +SQLITE_PRIVATE void sqlite3ExprListSetSortOrder(ExprList*,int,int); SQLITE_PRIVATE void sqlite3ExprListSetName(Parse*,ExprList*,Token*,int); SQLITE_PRIVATE void sqlite3ExprListSetSpan(Parse*,ExprList*,const char*,const char*); SQLITE_PRIVATE void sqlite3ExprListDelete(sqlite3*, ExprList*); SQLITE_PRIVATE u32 sqlite3ExprListFlags(const ExprList*); +SQLITE_PRIVATE int sqlite3IndexHasDuplicateRootPage(Index*); SQLITE_PRIVATE int sqlite3Init(sqlite3*, char**); SQLITE_PRIVATE int sqlite3InitCallback(void*, int, char**, char**); +SQLITE_PRIVATE int sqlite3InitOne(sqlite3*, int, char**, u32); SQLITE_PRIVATE void sqlite3Pragma(Parse*,Token*,Token*,Token*,int); #ifndef SQLITE_OMIT_VIRTUALTABLE SQLITE_PRIVATE Module *sqlite3PragmaVtabRegister(sqlite3*,const char *zName); @@ -18251,11 +19481,18 @@ SQLITE_PRIVATE void sqlite3CollapseDatabaseArray(sqlite3*); SQLITE_PRIVATE void sqlite3CommitInternalChanges(sqlite3*); SQLITE_PRIVATE void sqlite3DeleteColumnNames(sqlite3*,Table*); SQLITE_PRIVATE int sqlite3ColumnsFromExprList(Parse*,ExprList*,i16*,Column**); -SQLITE_PRIVATE void sqlite3SelectAddColumnTypeAndCollation(Parse*,Table*,Select*); -SQLITE_PRIVATE Table *sqlite3ResultSetOfSelect(Parse*,Select*); +SQLITE_PRIVATE void sqlite3SelectAddColumnTypeAndCollation(Parse*,Table*,Select*,char); +SQLITE_PRIVATE Table *sqlite3ResultSetOfSelect(Parse*,Select*,char); SQLITE_PRIVATE void sqlite3OpenMasterTable(Parse *, int); SQLITE_PRIVATE Index *sqlite3PrimaryKeyIndex(Table*); -SQLITE_PRIVATE i16 sqlite3ColumnOfIndex(Index*, i16); +SQLITE_PRIVATE i16 sqlite3TableColumnToIndex(Index*, i16); +#ifdef SQLITE_OMIT_GENERATED_COLUMNS +# define sqlite3TableColumnToStorage(T,X) (X) /* No-op pass-through */ +# define sqlite3StorageColumnToTable(T,X) (X) /* No-op pass-through */ +#else +SQLITE_PRIVATE i16 sqlite3TableColumnToStorage(Table*, i16); +SQLITE_PRIVATE i16 sqlite3StorageColumnToTable(Table*, i16); +#endif SQLITE_PRIVATE void sqlite3StartTable(Parse*,Token*,Token*,int,int,int,int); #if SQLITE_ENABLE_HIDDEN_COLUMNS SQLITE_PRIVATE void sqlite3ColumnPropertiesFromName(Table*, Column*); @@ -18268,9 +19505,11 @@ SQLITE_PRIVATE void sqlite3AddPrimaryKey(Parse*, ExprList*, int, int, int); SQLITE_PRIVATE void sqlite3AddCheckConstraint(Parse*, Expr*); SQLITE_PRIVATE void sqlite3AddDefaultValue(Parse*,Expr*,const char*,const char*); SQLITE_PRIVATE void sqlite3AddCollateType(Parse*, Token*); +SQLITE_PRIVATE void sqlite3AddGenerated(Parse*,Expr*,Token*); SQLITE_PRIVATE void sqlite3EndTable(Parse*,Token*,Token*,u8,Select*); SQLITE_PRIVATE int sqlite3ParseUri(const char*,const char*,unsigned int*, sqlite3_vfs**,char**,char **); +#define sqlite3CodecQueryParameters(A,B,C) 0 SQLITE_PRIVATE Btree *sqlite3DbNameToBtree(sqlite3*,const char*); #ifdef SQLITE_UNTESTABLE @@ -18290,8 +19529,9 @@ SQLITE_PRIVATE u32 sqlite3BitvecSize(Bitvec*); SQLITE_PRIVATE int sqlite3BitvecBuiltinTest(int,int*); #endif -SQLITE_PRIVATE RowSet *sqlite3RowSetInit(sqlite3*, void*, unsigned int); -SQLITE_PRIVATE void sqlite3RowSetClear(RowSet*); +SQLITE_PRIVATE RowSet *sqlite3RowSetInit(sqlite3*); +SQLITE_PRIVATE void sqlite3RowSetDelete(void*); +SQLITE_PRIVATE void sqlite3RowSetClear(void*); SQLITE_PRIVATE void sqlite3RowSetInsert(RowSet*, i64); SQLITE_PRIVATE int sqlite3RowSetTest(RowSet*, int iBatch, i64); SQLITE_PRIVATE int sqlite3RowSetNext(RowSet*, i64*); @@ -18310,6 +19550,7 @@ SQLITE_PRIVATE int sqlite3DbMaskAllZero(yDbMask); SQLITE_PRIVATE void sqlite3DropTable(Parse*, SrcList*, int, int); SQLITE_PRIVATE void sqlite3CodeDropTable(Parse*, Table*, int, int); SQLITE_PRIVATE void sqlite3DeleteTable(sqlite3*, Table*); +SQLITE_PRIVATE void sqlite3FreeIndex(sqlite3*, Index*); #ifndef SQLITE_OMIT_AUTOINCREMENT SQLITE_PRIVATE void sqlite3AutoincrementBegin(Parse *pParse); SQLITE_PRIVATE void sqlite3AutoincrementEnd(Parse *pParse); @@ -18318,11 +19559,14 @@ SQLITE_PRIVATE void sqlite3AutoincrementEnd(Parse *pParse); # define sqlite3AutoincrementEnd(X) #endif SQLITE_PRIVATE void sqlite3Insert(Parse*, SrcList*, Select*, IdList*, int, Upsert*); +#ifndef SQLITE_OMIT_GENERATED_COLUMNS +SQLITE_PRIVATE void sqlite3ComputeGeneratedColumns(Parse*, int, Table*); +#endif SQLITE_PRIVATE void *sqlite3ArrayAllocate(sqlite3*,void*,int,int*,int*); -SQLITE_PRIVATE IdList *sqlite3IdListAppend(sqlite3*, IdList*, Token*); +SQLITE_PRIVATE IdList *sqlite3IdListAppend(Parse*, IdList*, Token*); SQLITE_PRIVATE int sqlite3IdListIndex(IdList*,const char*); -SQLITE_PRIVATE SrcList *sqlite3SrcListEnlarge(sqlite3*, SrcList*, int, int); -SQLITE_PRIVATE SrcList *sqlite3SrcListAppend(sqlite3*, SrcList*, Token*, Token*); +SQLITE_PRIVATE SrcList *sqlite3SrcListEnlarge(Parse*, SrcList*, int, int); +SQLITE_PRIVATE SrcList *sqlite3SrcListAppend(Parse*, SrcList*, Token*, Token*); SQLITE_PRIVATE SrcList *sqlite3SrcListAppendFromTerm(Parse*, SrcList*, Token*, Token*, Token*, Select*, Expr*, IdList*); SQLITE_PRIVATE void sqlite3SrcListIndexedBy(Parse *, SrcList *, Token *); @@ -18340,6 +19584,7 @@ SQLITE_PRIVATE int sqlite3Select(Parse*, Select*, SelectDest*); SQLITE_PRIVATE Select *sqlite3SelectNew(Parse*,ExprList*,SrcList*,Expr*,ExprList*, Expr*,ExprList*,u32,Expr*); SQLITE_PRIVATE void sqlite3SelectDelete(sqlite3*, Select*); +SQLITE_PRIVATE void sqlite3SelectReset(Parse*, Select*); SQLITE_PRIVATE Table *sqlite3SrcListLookup(Parse*, SrcList*); SQLITE_PRIVATE int sqlite3IsReadOnly(Parse*, Table*, int); SQLITE_PRIVATE void sqlite3OpenTable(Parse*, int iCur, int iDb, Table*, int); @@ -18354,7 +19599,7 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo*); SQLITE_PRIVATE LogEst sqlite3WhereOutputRowCount(WhereInfo*); SQLITE_PRIVATE int sqlite3WhereIsDistinct(WhereInfo*); SQLITE_PRIVATE int sqlite3WhereIsOrdered(WhereInfo*); -SQLITE_PRIVATE int sqlite3WhereOrderedInnerLoop(WhereInfo*); +SQLITE_PRIVATE int sqlite3WhereOrderByLimitOptLabel(WhereInfo*); SQLITE_PRIVATE int sqlite3WhereIsSorted(WhereInfo*); SQLITE_PRIVATE int sqlite3WhereContinueLabel(WhereInfo*); SQLITE_PRIVATE int sqlite3WhereBreakLabel(WhereInfo*); @@ -18362,24 +19607,20 @@ SQLITE_PRIVATE int sqlite3WhereOkOnePass(WhereInfo*, int*); #define ONEPASS_OFF 0 /* Use of ONEPASS not allowed */ #define ONEPASS_SINGLE 1 /* ONEPASS valid for a single row update */ #define ONEPASS_MULTI 2 /* ONEPASS is valid for multiple rows */ +SQLITE_PRIVATE int sqlite3WhereUsesDeferredSeek(WhereInfo*); SQLITE_PRIVATE void sqlite3ExprCodeLoadIndexColumn(Parse*, Index*, int, int, int); SQLITE_PRIVATE int sqlite3ExprCodeGetColumn(Parse*, Table*, int, int, int, u8); -SQLITE_PRIVATE void sqlite3ExprCodeGetColumnToReg(Parse*, Table*, int, int, int); SQLITE_PRIVATE void sqlite3ExprCodeGetColumnOfTable(Vdbe*, Table*, int, int, int); SQLITE_PRIVATE void sqlite3ExprCodeMove(Parse*, int, int, int); -SQLITE_PRIVATE void sqlite3ExprCacheStore(Parse*, int, int, int); -SQLITE_PRIVATE void sqlite3ExprCachePush(Parse*); -SQLITE_PRIVATE void sqlite3ExprCachePop(Parse*); -SQLITE_PRIVATE void sqlite3ExprCacheRemove(Parse*, int, int); -SQLITE_PRIVATE void sqlite3ExprCacheClear(Parse*); -SQLITE_PRIVATE void sqlite3ExprCacheAffinityChange(Parse*, int, int); SQLITE_PRIVATE void sqlite3ExprCode(Parse*, Expr*, int); +#ifndef SQLITE_OMIT_GENERATED_COLUMNS +SQLITE_PRIVATE void sqlite3ExprCodeGeneratedColumn(Parse*, Column*, int); +#endif SQLITE_PRIVATE void sqlite3ExprCodeCopy(Parse*, Expr*, int); SQLITE_PRIVATE void sqlite3ExprCodeFactorable(Parse*, Expr*, int); -SQLITE_PRIVATE int sqlite3ExprCodeAtInit(Parse*, Expr*, int); +SQLITE_PRIVATE int sqlite3ExprCodeRunJustOnce(Parse*, Expr*, int); SQLITE_PRIVATE int sqlite3ExprCodeTemp(Parse*, Expr*, int*); SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse*, Expr*, int); -SQLITE_PRIVATE void sqlite3ExprCodeAndCache(Parse*, Expr*, int); SQLITE_PRIVATE int sqlite3ExprCodeExprList(Parse*, ExprList*, int, int, u8); #define SQLITE_ECEL_DUP 0x01 /* Deep, not shallow copies */ #define SQLITE_ECEL_FACTOR 0x02 /* Factor out constant terms */ @@ -18396,8 +19637,8 @@ SQLITE_PRIVATE Table *sqlite3LocateTableItem(Parse*,u32 flags,struct SrcList_ite SQLITE_PRIVATE Index *sqlite3FindIndex(sqlite3*,const char*, const char*); SQLITE_PRIVATE void sqlite3UnlinkAndDeleteTable(sqlite3*,int,const char*); SQLITE_PRIVATE void sqlite3UnlinkAndDeleteIndex(sqlite3*,int,const char*); -SQLITE_PRIVATE void sqlite3Vacuum(Parse*,Token*); -SQLITE_PRIVATE int sqlite3RunVacuum(char**, sqlite3*, int); +SQLITE_PRIVATE void sqlite3Vacuum(Parse*,Token*,Expr*); +SQLITE_PRIVATE int sqlite3RunVacuum(char**, sqlite3*, int, sqlite3_value*); SQLITE_PRIVATE char *sqlite3NameFromToken(sqlite3*, Token*); SQLITE_PRIVATE int sqlite3ExprCompare(Parse*,Expr*, Expr*, int); SQLITE_PRIVATE int sqlite3ExprCompareSkip(Expr*, Expr*, int); @@ -18421,6 +19662,7 @@ SQLITE_PRIVATE void sqlite3EndTransaction(Parse*,int); SQLITE_PRIVATE void sqlite3Savepoint(Parse*, int, Token*); SQLITE_PRIVATE void sqlite3CloseSavepoints(sqlite3 *); SQLITE_PRIVATE void sqlite3LeaveMutexAndCloseZombie(sqlite3*); +SQLITE_PRIVATE u32 sqlite3IsTrueOrFalse(const char*); SQLITE_PRIVATE int sqlite3ExprIdToTrueFalse(Expr*); SQLITE_PRIVATE int sqlite3ExprTruthValue(const Expr*); SQLITE_PRIVATE int sqlite3ExprIsConstant(Expr*); @@ -18440,6 +19682,7 @@ SQLITE_PRIVATE void sqlite3GenerateRowDelete( SQLITE_PRIVATE void sqlite3GenerateRowIndexDelete(Parse*, Table*, int, int, int*, int); SQLITE_PRIVATE int sqlite3GenerateIndexKey(Parse*, Index*, int, int, int, int*,Index*,int); SQLITE_PRIVATE void sqlite3ResolvePartIdxLabel(Parse*,int); +SQLITE_PRIVATE int sqlite3ExprReferencesUpdatedColumn(Expr*,int*,int); SQLITE_PRIVATE void sqlite3GenerateConstraintChecks(Parse*,Table*,int*,int,int,int,int, u8,u8,int,int*,int*,Upsert*); #ifdef SQLITE_ENABLE_NULL_TRIM @@ -18460,11 +19703,7 @@ SQLITE_PRIVATE ExprList *sqlite3ExprListDup(sqlite3*,ExprList*,int); SQLITE_PRIVATE SrcList *sqlite3SrcListDup(sqlite3*,SrcList*,int); SQLITE_PRIVATE IdList *sqlite3IdListDup(sqlite3*,IdList*); SQLITE_PRIVATE Select *sqlite3SelectDup(sqlite3*,Select*,int); -#if SELECTTRACE_ENABLED -SQLITE_PRIVATE void sqlite3SelectSetName(Select*,const char*); -#else -# define sqlite3SelectSetName(A,B) -#endif +SQLITE_PRIVATE FuncDef *sqlite3FunctionSearch(int,const char*); SQLITE_PRIVATE void sqlite3InsertBuiltinFuncs(FuncDef*,int); SQLITE_PRIVATE FuncDef *sqlite3FindFunction(sqlite3*,const char*,int,u8,u8); SQLITE_PRIVATE void sqlite3RegisterBuiltinFunctions(void); @@ -18493,12 +19732,12 @@ SQLITE_PRIVATE void sqlite3CodeRowTriggerDirect(Parse *, Trigger *, Table *, i SQLITE_PRIVATE void sqlite3DeleteTriggerStep(sqlite3*, TriggerStep*); SQLITE_PRIVATE TriggerStep *sqlite3TriggerSelectStep(sqlite3*,Select*, const char*,const char*); -SQLITE_PRIVATE TriggerStep *sqlite3TriggerInsertStep(sqlite3*,Token*, IdList*, +SQLITE_PRIVATE TriggerStep *sqlite3TriggerInsertStep(Parse*,Token*, IdList*, Select*,u8,Upsert*, const char*,const char*); -SQLITE_PRIVATE TriggerStep *sqlite3TriggerUpdateStep(sqlite3*,Token*,ExprList*, Expr*, u8, +SQLITE_PRIVATE TriggerStep *sqlite3TriggerUpdateStep(Parse*,Token*,ExprList*, Expr*, u8, const char*,const char*); -SQLITE_PRIVATE TriggerStep *sqlite3TriggerDeleteStep(sqlite3*,Token*, Expr*, +SQLITE_PRIVATE TriggerStep *sqlite3TriggerDeleteStep(Parse*,Token*, Expr*, const char*,const char*); SQLITE_PRIVATE void sqlite3DeleteTrigger(sqlite3*, Trigger*); SQLITE_PRIVATE void sqlite3UnlinkAndDeleteTrigger(sqlite3*,int,const char*); @@ -18519,6 +19758,7 @@ SQLITE_PRIVATE u32 sqlite3TriggerColmask(Parse*,Trigger*,ExprList*,int,int,Tab #endif SQLITE_PRIVATE int sqlite3JoinType(Parse*, Token*, Token*, Token*); +SQLITE_PRIVATE void sqlite3SetJoinExpr(Expr*,int); SQLITE_PRIVATE void sqlite3CreateForeignKey(Parse*, ExprList*, Token*, ExprList*, int); SQLITE_PRIVATE void sqlite3DeferForeignKey(Parse*, int); #ifndef SQLITE_OMIT_AUTHORIZATION @@ -18533,6 +19773,7 @@ SQLITE_PRIVATE int sqlite3AuthReadCol(Parse*, const char *, const char *, int) # define sqlite3AuthContextPush(a,b,c) # define sqlite3AuthContextPop(a) ((void)(a)) #endif +SQLITE_PRIVATE int sqlite3DbIsNamed(sqlite3 *db, int iDb, const char *zName); SQLITE_PRIVATE void sqlite3Attach(Parse*, Expr*, Expr*, Expr*); SQLITE_PRIVATE void sqlite3Detach(Parse*, Expr*); SQLITE_PRIVATE void sqlite3FixInit(DbFixer*, Parse*, int, const char*, const Token*); @@ -18541,6 +19782,7 @@ SQLITE_PRIVATE int sqlite3FixSelect(DbFixer*, Select*); SQLITE_PRIVATE int sqlite3FixExpr(DbFixer*, Expr*); SQLITE_PRIVATE int sqlite3FixExprList(DbFixer*, ExprList*); SQLITE_PRIVATE int sqlite3FixTriggerStep(DbFixer*, TriggerStep*); +SQLITE_PRIVATE int sqlite3RealSameAsInt(double,sqlite3_int64); SQLITE_PRIVATE int sqlite3AtoF(const char *z, double*, int, u8); SQLITE_PRIVATE int sqlite3GetInt32(const char *, int*); SQLITE_PRIVATE int sqlite3Atoi(const char*); @@ -18555,7 +19797,7 @@ SQLITE_PRIVATE LogEst sqlite3LogEstAdd(LogEst,LogEst); SQLITE_PRIVATE LogEst sqlite3LogEstFromDouble(double); #endif #if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || \ - defined(SQLITE_ENABLE_STAT3_OR_STAT4) || \ + defined(SQLITE_ENABLE_STAT4) || \ defined(SQLITE_EXPLAIN_ESTIMATED_ROWS) SQLITE_PRIVATE u64 sqlite3LogEstToInt(LogEst); #endif @@ -18580,6 +19822,8 @@ SQLITE_PRIVATE int sqlite3VarintLen(u64 v); */ #define getVarint32(A,B) \ (u8)((*(A)<(u8)0x80)?((B)=(u32)*(A)),1:sqlite3GetVarint32((A),(u32 *)&(B))) +#define getVarint32NR(A,B) \ + B=(u32)*(A);if(B>=0x80)sqlite3GetVarint32((A),(u32*)&(B)) #define putVarint32(A,B) \ (u8)(((u32)(B)<(u32)0x80)?(*(A)=(unsigned char)(B)),1:\ sqlite3PutVarint((A),(B))) @@ -18589,10 +19833,10 @@ SQLITE_PRIVATE int sqlite3VarintLen(u64 v); SQLITE_PRIVATE const char *sqlite3IndexAffinityStr(sqlite3*, Index*); SQLITE_PRIVATE void sqlite3TableAffinity(Vdbe*, Table*, int); -SQLITE_PRIVATE char sqlite3CompareAffinity(Expr *pExpr, char aff2); -SQLITE_PRIVATE int sqlite3IndexAffinityOk(Expr *pExpr, char idx_affinity); +SQLITE_PRIVATE char sqlite3CompareAffinity(const Expr *pExpr, char aff2); +SQLITE_PRIVATE int sqlite3IndexAffinityOk(const Expr *pExpr, char idx_affinity); SQLITE_PRIVATE char sqlite3TableColumnAffinity(Table*,int); -SQLITE_PRIVATE char sqlite3ExprAffinity(Expr *pExpr); +SQLITE_PRIVATE char sqlite3ExprAffinity(const Expr *pExpr); SQLITE_PRIVATE int sqlite3Atoi64(const char*, i64*, int, u8); SQLITE_PRIVATE int sqlite3DecOrHexToI64(const char*, i64*); SQLITE_PRIVATE void sqlite3ErrorWithMsg(sqlite3*, int, const char*,...); @@ -18613,15 +19857,19 @@ SQLITE_PRIVATE int sqlite3MemdbInit(void); SQLITE_PRIVATE const char *sqlite3ErrStr(int); SQLITE_PRIVATE int sqlite3ReadSchema(Parse *pParse); SQLITE_PRIVATE CollSeq *sqlite3FindCollSeq(sqlite3*,u8 enc, const char*,int); +SQLITE_PRIVATE int sqlite3IsBinary(const CollSeq*); SQLITE_PRIVATE CollSeq *sqlite3LocateCollSeq(Parse *pParse, const char*zName); -SQLITE_PRIVATE CollSeq *sqlite3ExprCollSeq(Parse *pParse, Expr *pExpr); -SQLITE_PRIVATE CollSeq *sqlite3ExprNNCollSeq(Parse *pParse, Expr *pExpr); -SQLITE_PRIVATE int sqlite3ExprCollSeqMatch(Parse*,Expr*,Expr*); +SQLITE_PRIVATE void sqlite3SetTextEncoding(sqlite3 *db, u8); +SQLITE_PRIVATE CollSeq *sqlite3ExprCollSeq(Parse *pParse, const Expr *pExpr); +SQLITE_PRIVATE CollSeq *sqlite3ExprNNCollSeq(Parse *pParse, const Expr *pExpr); +SQLITE_PRIVATE int sqlite3ExprCollSeqMatch(Parse*,const Expr*,const Expr*); SQLITE_PRIVATE Expr *sqlite3ExprAddCollateToken(Parse *pParse, Expr*, const Token*, int); SQLITE_PRIVATE Expr *sqlite3ExprAddCollateString(Parse*,Expr*,const char*); SQLITE_PRIVATE Expr *sqlite3ExprSkipCollate(Expr*); +SQLITE_PRIVATE Expr *sqlite3ExprSkipCollateAndLikely(Expr*); SQLITE_PRIVATE int sqlite3CheckCollSeq(Parse *, CollSeq *); -SQLITE_PRIVATE int sqlite3CheckObjectName(Parse *, const char *); +SQLITE_PRIVATE int sqlite3WritableSchema(sqlite3*); +SQLITE_PRIVATE int sqlite3CheckObjectName(Parse*, const char*,const char*,const char*); SQLITE_PRIVATE void sqlite3VdbeSetChanges(sqlite3 *, int); SQLITE_PRIVATE int sqlite3AddInt64(i64*,i64); SQLITE_PRIVATE int sqlite3SubInt64(i64*,i64); @@ -18640,6 +19888,9 @@ SQLITE_PRIVATE void sqlite3ValueSetStr(sqlite3_value*, int, const void *,u8, void(*)(void*)); SQLITE_PRIVATE void sqlite3ValueSetNull(sqlite3_value*); SQLITE_PRIVATE void sqlite3ValueFree(sqlite3_value*); +#ifndef SQLITE_UNTESTABLE +SQLITE_PRIVATE void sqlite3ResultIntReal(sqlite3_context*); +#endif SQLITE_PRIVATE sqlite3_value *sqlite3ValueNew(sqlite3 *); #ifndef SQLITE_OMIT_UTF16 SQLITE_PRIVATE char *sqlite3Utf16to8(sqlite3 *, const void*, int, u8); @@ -18651,7 +19902,6 @@ SQLITE_PRIVATE const unsigned char sqlite3OpcodeProperty[]; SQLITE_PRIVATE const char sqlite3StrBINARY[]; SQLITE_PRIVATE const unsigned char sqlite3UpperToLower[]; SQLITE_PRIVATE const unsigned char sqlite3CtypeMap[]; -SQLITE_PRIVATE const Token sqlite3IntTokens[]; SQLITE_PRIVATE SQLITE_WSD struct Sqlite3Config sqlite3Config; SQLITE_PRIVATE FuncDefHash sqlite3BuiltinFunctions; #ifndef SQLITE_OMIT_WSD @@ -18665,25 +19915,38 @@ SQLITE_PRIVATE void sqlite3RootPageMoved(sqlite3*, int, int, int); SQLITE_PRIVATE void sqlite3Reindex(Parse*, Token*, Token*); SQLITE_PRIVATE void sqlite3AlterFunctions(void); SQLITE_PRIVATE void sqlite3AlterRenameTable(Parse*, SrcList*, Token*); +SQLITE_PRIVATE void sqlite3AlterRenameColumn(Parse*, SrcList*, Token*, Token*); SQLITE_PRIVATE int sqlite3GetToken(const unsigned char *, int *); SQLITE_PRIVATE void sqlite3NestedParse(Parse*, const char*, ...); -SQLITE_PRIVATE void sqlite3ExpirePreparedStatements(sqlite3*); -SQLITE_PRIVATE int sqlite3CodeSubselect(Parse*, Expr *, int, int); +SQLITE_PRIVATE void sqlite3ExpirePreparedStatements(sqlite3*, int); +SQLITE_PRIVATE void sqlite3CodeRhsOfIN(Parse*, Expr*, int); +SQLITE_PRIVATE int sqlite3CodeSubselect(Parse*, Expr*); SQLITE_PRIVATE void sqlite3SelectPrep(Parse*, Select*, NameContext*); SQLITE_PRIVATE void sqlite3SelectWrongNumTermsError(Parse *pParse, Select *p); -SQLITE_PRIVATE int sqlite3MatchSpanName(const char*, const char*, const char*, const char*); +SQLITE_PRIVATE int sqlite3MatchEName( + const struct ExprList_item*, + const char*, + const char*, + const char* +); +SQLITE_PRIVATE Bitmask sqlite3ExprColUsed(Expr*); +SQLITE_PRIVATE u8 sqlite3StrIHash(const char*); SQLITE_PRIVATE int sqlite3ResolveExprNames(NameContext*, Expr*); SQLITE_PRIVATE int sqlite3ResolveExprListNames(NameContext*, ExprList*); SQLITE_PRIVATE void sqlite3ResolveSelectNames(Parse*, Select*, NameContext*); -SQLITE_PRIVATE void sqlite3ResolveSelfReference(Parse*,Table*,int,Expr*,ExprList*); +SQLITE_PRIVATE int sqlite3ResolveSelfReference(Parse*,Table*,int,Expr*,ExprList*); SQLITE_PRIVATE int sqlite3ResolveOrderGroupBy(Parse*, Select*, ExprList*, const char*); SQLITE_PRIVATE void sqlite3ColumnDefault(Vdbe *, Table *, int, int); SQLITE_PRIVATE void sqlite3AlterFinishAddColumn(Parse *, Token *); SQLITE_PRIVATE void sqlite3AlterBeginAddColumn(Parse *, SrcList *); +SQLITE_PRIVATE void *sqlite3RenameTokenMap(Parse*, void*, Token*); +SQLITE_PRIVATE void sqlite3RenameTokenRemap(Parse*, void *pTo, void *pFrom); +SQLITE_PRIVATE void sqlite3RenameExprUnmap(Parse*, Expr*); +SQLITE_PRIVATE void sqlite3RenameExprlistUnmap(Parse*, ExprList*); SQLITE_PRIVATE CollSeq *sqlite3GetCollSeq(Parse*, u8, CollSeq *, const char*); SQLITE_PRIVATE char sqlite3AffinityType(const char*, Column*); SQLITE_PRIVATE void sqlite3Analyze(Parse*, Token*, Token*); -SQLITE_PRIVATE int sqlite3InvokeBusyHandler(BusyHandler*, sqlite3_file*); +SQLITE_PRIVATE int sqlite3InvokeBusyHandler(BusyHandler*); SQLITE_PRIVATE int sqlite3FindDb(sqlite3*, Token*); SQLITE_PRIVATE int sqlite3FindDbName(sqlite3 *, const char *); SQLITE_PRIVATE int sqlite3AnalysisLoad(sqlite3*,int iDB); @@ -18698,12 +19961,18 @@ SQLITE_PRIVATE KeyInfo *sqlite3KeyInfoAlloc(sqlite3*,int,int); SQLITE_PRIVATE void sqlite3KeyInfoUnref(KeyInfo*); SQLITE_PRIVATE KeyInfo *sqlite3KeyInfoRef(KeyInfo*); SQLITE_PRIVATE KeyInfo *sqlite3KeyInfoOfIndex(Parse*, Index*); +SQLITE_PRIVATE KeyInfo *sqlite3KeyInfoFromExprList(Parse*, ExprList*, int, int); +SQLITE_PRIVATE int sqlite3HasExplicitNulls(Parse*, ExprList*); + #ifdef SQLITE_DEBUG SQLITE_PRIVATE int sqlite3KeyInfoIsWriteable(KeyInfo*); #endif SQLITE_PRIVATE int sqlite3CreateFunc(sqlite3 *, const char *, int, int, void *, void (*)(sqlite3_context*,int,sqlite3_value **), - void (*)(sqlite3_context*,int,sqlite3_value **), void (*)(sqlite3_context*), + void (*)(sqlite3_context*,int,sqlite3_value **), + void (*)(sqlite3_context*), + void (*)(sqlite3_context*), + void (*)(sqlite3_context*,int,sqlite3_value **), FuncDestructor *pDestructor ); SQLITE_PRIVATE void sqlite3NoopDestructor(void*); @@ -18726,8 +19995,7 @@ SQLITE_PRIVATE int sqlite3ExprCheckIN(Parse*, Expr*); # define sqlite3ExprCheckIN(x,y) SQLITE_OK #endif -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 -SQLITE_PRIVATE void sqlite3AnalyzeFunctions(void); +#ifdef SQLITE_ENABLE_STAT4 SQLITE_PRIVATE int sqlite3Stat4ProbeSetValue( Parse*,Index*,UnpackedRecord**,Expr*,int,int,int*); SQLITE_PRIVATE int sqlite3Stat4ValueFromExpr(Parse*, Expr*, u8, sqlite3_value**); @@ -18744,6 +20012,7 @@ SQLITE_PRIVATE void *sqlite3ParserAlloc(void*(*)(u64), Parse*); SQLITE_PRIVATE void sqlite3ParserFree(void*, void(*)(void*)); #endif SQLITE_PRIVATE void sqlite3Parser(void*, int, Token); +SQLITE_PRIVATE int sqlite3ParserFallback(int); #ifdef YYTRACKMAXSTACKDEPTH SQLITE_PRIVATE int sqlite3ParserStackPeak(void*); #endif @@ -18773,6 +20042,7 @@ SQLITE_PRIVATE int sqlite3Utf8To8(unsigned char*); # define sqlite3VtabInSync(db) 0 # define sqlite3VtabLock(X) # define sqlite3VtabUnlock(X) +# define sqlite3VtabModuleUnref(D,X) # define sqlite3VtabUnlockList(X) # define sqlite3VtabSavepoint(X, Y, Z) SQLITE_OK # define sqlite3GetVTable(X,Y) ((VTable*)0) @@ -18784,6 +20054,7 @@ SQLITE_PRIVATE int sqlite3VtabRollback(sqlite3 *db); SQLITE_PRIVATE int sqlite3VtabCommit(sqlite3 *db); SQLITE_PRIVATE void sqlite3VtabLock(VTable *); SQLITE_PRIVATE void sqlite3VtabUnlock(VTable *); +SQLITE_PRIVATE void sqlite3VtabModuleUnref(sqlite3*,Module*); SQLITE_PRIVATE void sqlite3VtabUnlockList(sqlite3*); SQLITE_PRIVATE int sqlite3VtabSavepoint(sqlite3 *, int, int); SQLITE_PRIVATE void sqlite3VtabImportErrmsg(Vdbe*, sqlite3_vtab*); @@ -18797,6 +20068,14 @@ SQLITE_PRIVATE Module *sqlite3VtabCreateModule( ); # define sqlite3VtabInSync(db) ((db)->nVTrans>0 && (db)->aVTrans==0) #endif +SQLITE_PRIVATE int sqlite3ReadOnlyShadowTables(sqlite3 *db); +#ifndef SQLITE_OMIT_VIRTUALTABLE +SQLITE_PRIVATE int sqlite3ShadowTableName(sqlite3 *db, const char *zName); +SQLITE_PRIVATE int sqlite3IsShadowTableOf(sqlite3*,Table*,const char*); +#else +# define sqlite3ShadowTableName(A,B) 0 +# define sqlite3IsShadowTableOf(A,B,C) 0 +#endif SQLITE_PRIVATE int sqlite3VtabEponymousTableInit(Parse*,Module*); SQLITE_PRIVATE void sqlite3VtabEponymousTableClear(sqlite3*,Module*); SQLITE_PRIVATE void sqlite3VtabMakeWritable(Parse*,Table*); @@ -18813,9 +20092,13 @@ SQLITE_PRIVATE sqlite3_int64 sqlite3StmtCurrentTime(sqlite3_context*); SQLITE_PRIVATE int sqlite3VdbeParameterIndex(Vdbe*, const char*, int); SQLITE_PRIVATE int sqlite3TransferBindings(sqlite3_stmt *, sqlite3_stmt *); SQLITE_PRIVATE void sqlite3ParserReset(Parse*); +#ifdef SQLITE_ENABLE_NORMALIZE +SQLITE_PRIVATE char *sqlite3Normalize(Vdbe*, const char*); +#endif SQLITE_PRIVATE int sqlite3Reprepare(Vdbe*); SQLITE_PRIVATE void sqlite3ExprListCheckLength(Parse*, ExprList*, const char*); -SQLITE_PRIVATE CollSeq *sqlite3BinaryCompareCollSeq(Parse *, Expr *, Expr *); +SQLITE_PRIVATE CollSeq *sqlite3ExprCompareCollSeq(Parse*,const Expr*); +SQLITE_PRIVATE CollSeq *sqlite3BinaryCompareCollSeq(Parse *, const Expr*, const Expr*); SQLITE_PRIVATE int sqlite3TempInMemory(const sqlite3*); SQLITE_PRIVATE const char *sqlite3JournalModename(int); #ifndef SQLITE_OMIT_WAL @@ -18907,7 +20190,7 @@ SQLITE_PRIVATE void sqlite3EndBenignMalloc(void); #define IN_INDEX_NOOP_OK 0x0001 /* OK to return IN_INDEX_NOOP */ #define IN_INDEX_MEMBERSHIP 0x0002 /* IN operator used for membership test */ #define IN_INDEX_LOOP 0x0004 /* IN operator used as a loop */ -SQLITE_PRIVATE int sqlite3FindInIndex(Parse *, Expr *, u32, int*, int*); +SQLITE_PRIVATE int sqlite3FindInIndex(Parse *, Expr *, u32, int*, int*, int*); SQLITE_PRIVATE int sqlite3JournalOpen(sqlite3_vfs *, const char *, sqlite3_file *, int, int); SQLITE_PRIVATE int sqlite3JournalSize(sqlite3_vfs *); @@ -19121,7 +20404,6 @@ SQLITE_PRIVATE const unsigned char sqlite3UpperToLower[] = { ** non-ASCII UTF character. Hence the test for whether or not a character is ** part of an identifier is 0x46. */ -#ifdef SQLITE_ASCII SQLITE_PRIVATE const unsigned char sqlite3CtypeMap[256] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 00..07 ........ */ 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x00, 0x00, /* 08..0f ........ */ @@ -19159,7 +20441,6 @@ SQLITE_PRIVATE const unsigned char sqlite3CtypeMap[256] = { 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* f0..f7 ........ */ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40 /* f8..ff ........ */ }; -#endif /* EVIDENCE-OF: R-02982-34736 In order to maintain full backwards ** compatibility for legacy applications, the URI filename capability is @@ -19171,24 +20452,24 @@ SQLITE_PRIVATE const unsigned char sqlite3CtypeMap[256] = { ** EVIDENCE-OF: R-43642-56306 By default, URI handling is globally ** disabled. The default value may be changed by compiling with the ** SQLITE_USE_URI symbol defined. -** -** URI filenames are enabled by default if SQLITE_HAS_CODEC is -** enabled. */ #ifndef SQLITE_USE_URI -# ifdef SQLITE_HAS_CODEC -# define SQLITE_USE_URI 1 -# else -# define SQLITE_USE_URI 0 -# endif +# define SQLITE_USE_URI 0 #endif /* EVIDENCE-OF: R-38720-18127 The default setting is determined by the ** SQLITE_ALLOW_COVERING_INDEX_SCAN compile-time option, or is "on" if ** that compile-time option is omitted. */ -#ifndef SQLITE_ALLOW_COVERING_INDEX_SCAN +#if !defined(SQLITE_ALLOW_COVERING_INDEX_SCAN) # define SQLITE_ALLOW_COVERING_INDEX_SCAN 1 +#else +# if !SQLITE_ALLOW_COVERING_INDEX_SCAN +# error "Compile-time disabling of covering index scan using the\ + -DSQLITE_ALLOW_COVERING_INDEX_SCAN=0 option is deprecated.\ + Contact SQLite developers if this is a problem for you, and\ + delete this #error macro to continue with your build." +# endif #endif /* The minimum PMA size is set to this value multiplied by the database @@ -19217,12 +20498,28 @@ SQLITE_PRIVATE const unsigned char sqlite3CtypeMap[256] = { ** changed as start-time using sqlite3_config(SQLITE_CONFIG_LOOKASIDE) ** or at run-time for an individual database connection using ** sqlite3_db_config(db, SQLITE_DBCONFIG_LOOKASIDE); +** +** With the two-size-lookaside enhancement, less lookaside is required. +** The default configuration of 1200,40 actually provides 30 1200-byte slots +** and 93 128-byte slots, which is more lookaside than is available +** using the older 1200,100 configuration without two-size-lookaside. */ #ifndef SQLITE_DEFAULT_LOOKASIDE -# define SQLITE_DEFAULT_LOOKASIDE 1200,100 +# ifdef SQLITE_OMIT_TWOSIZE_LOOKASIDE +# define SQLITE_DEFAULT_LOOKASIDE 1200,100 /* 120KB of memory */ +# else +# define SQLITE_DEFAULT_LOOKASIDE 1200,40 /* 48KB of memory */ +# endif #endif +/* The default maximum size of an in-memory database created using +** sqlite3_deserialize() +*/ +#ifndef SQLITE_MEMDB_DEFAULT_MAXSIZE +# define SQLITE_MEMDB_DEFAULT_MAXSIZE 1073741824 +#endif + /* ** The following singleton contains the global configuration for ** the SQLite library. @@ -19234,6 +20531,7 @@ SQLITE_PRIVATE SQLITE_WSD struct Sqlite3Config sqlite3Config = { SQLITE_USE_URI, /* bOpenUri */ SQLITE_ALLOW_COVERING_INDEX_SCAN, /* bUseCis */ 0, /* bSmallMalloc */ + 1, /* bExtraSchemaChecks */ 0x7ffffffe, /* mxStrlen */ 0, /* neverCorrupt */ SQLITE_DEFAULT_LOOKASIDE, /* szLookaside, nLookaside */ @@ -19270,12 +20568,16 @@ SQLITE_PRIVATE SQLITE_WSD struct Sqlite3Config sqlite3Config = { 0, /* xVdbeBranch */ 0, /* pVbeBranchArg */ #endif +#ifdef SQLITE_ENABLE_DESERIALIZE + SQLITE_MEMDB_DEFAULT_MAXSIZE, /* mxMemdbSize */ +#endif #ifndef SQLITE_UNTESTABLE 0, /* xTestCallback */ #endif 0, /* bLocaltimeFault */ 0x7ffffffe, /* iOnceResetThreshold */ - SQLITE_DEFAULT_SORTERREF_SIZE /* szSorterRef */ + SQLITE_DEFAULT_SORTERREF_SIZE, /* szSorterRef */ + 0, /* iPrngSeed */ }; /* @@ -19285,14 +20587,6 @@ SQLITE_PRIVATE SQLITE_WSD struct Sqlite3Config sqlite3Config = { */ SQLITE_PRIVATE FuncDefHash sqlite3BuiltinFunctions; -/* -** Constant tokens for values 0 and 1. -*/ -SQLITE_PRIVATE const Token sqlite3IntTokens[] = { - { "0", 1 }, - { "1", 1 } -}; - #ifdef VDBE_PROFILE /* ** The following performance counter can be used in place of @@ -19390,7 +20684,8 @@ SQLITE_PRIVATE const char sqlite3StrBINARY[] = "BINARY"; ** "explain" P4 display logic is enabled. */ #if !defined(SQLITE_OMIT_EXPLAIN) || !defined(NDEBUG) \ - || defined(VDBE_PROFILE) || defined(SQLITE_DEBUG) + || defined(VDBE_PROFILE) || defined(SQLITE_DEBUG) \ + || defined(SQLITE_ENABLE_BYTECODE_VTAB) # define VDBE_DISPLAY_P4 1 #else # define VDBE_DISPLAY_P4 0 @@ -19444,6 +20739,7 @@ struct VdbeCursor { Bool isEphemeral:1; /* True for an ephemeral table */ Bool useRandomRowid:1; /* Generate new record numbers semi-randomly */ Bool isOrdered:1; /* True if the table is not BTREE_UNORDERED */ + Bool seekHit:1; /* See the OP_SeekHit and OP_IfNoHope opcodes */ Btree *pBtx; /* Separate file holding temporary table */ i64 seqCount; /* Sequence counter */ int *aAltMap; /* Mapping from table to index column numbers */ @@ -19527,6 +20823,9 @@ struct VdbeFrame { void *token; /* Copy of SubProgram.token */ i64 lastRowid; /* Last insert rowid (sqlite3.lastRowid) */ AuxData *pAuxData; /* Linked list of auxdata allocations */ +#if SQLITE_DEBUG + u32 iFrameMagic; /* magic number for sanity checking */ +#endif int nCursor; /* Number of entries in apCsr */ int pc; /* Program Counter in parent (calling) frame */ int nOp; /* Size of aOp array */ @@ -19537,6 +20836,13 @@ struct VdbeFrame { int nDbChange; /* Value of db->nChange */ }; +/* Magic number for sanity checking on VdbeFrame objects */ +#define SQLITE_FRAME_MAGIC 0x879fb71e + +/* +** Return a pointer to the array of registers allocated for use +** by a VdbeFrame. +*/ #define VdbeFrameMem(p) ((Mem *)&((u8 *)p)[ROUND8(sizeof(VdbeFrame))]) /* @@ -19551,8 +20857,6 @@ struct sqlite3_value { int nZero; /* Extra zero bytes when MEM_Zero and MEM_Blob set */ const char *zPType; /* Pointer type when MEM_Term|MEM_Subtype|MEM_Null */ FuncDef *pDef; /* Used only when flags==MEM_Agg */ - RowSet *pRowSet; /* Used only when flags==MEM_RowSet */ - VdbeFrame *pFrame; /* Used when flags==MEM_Frame */ } u; u16 flags; /* Some combination of MEM_Null, MEM_Str, MEM_Dyn, etc. */ u8 enc; /* SQLITE_UTF8, SQLITE_UTF16BE, SQLITE_UTF16LE */ @@ -19567,7 +20871,7 @@ struct sqlite3_value { void (*xDel)(void*);/* Destructor for Mem.z - only valid if MEM_Dyn */ #ifdef SQLITE_DEBUG Mem *pScopyFrom; /* This Mem is a shallow copy of pScopyFrom */ - void *pFiller; /* So that sizeof(Mem) is a multiple of 8 */ + u16 mScopyFlags; /* flags value immediately after the shallow copy */ #endif }; @@ -19595,12 +20899,12 @@ struct sqlite3_value { #define MEM_Int 0x0004 /* Value is an integer */ #define MEM_Real 0x0008 /* Value is a real number */ #define MEM_Blob 0x0010 /* Value is a BLOB */ -#define MEM_AffMask 0x001f /* Mask of affinity bits */ -#define MEM_RowSet 0x0020 /* Value is a RowSet object */ -#define MEM_Frame 0x0040 /* Value is a VdbeFrame object */ +#define MEM_IntReal 0x0020 /* MEM_Int that stringifies like MEM_Real */ +#define MEM_AffMask 0x003f /* Mask of affinity bits */ +#define MEM_FromBind 0x0040 /* Value originates from sqlite3_bind() */ #define MEM_Undefined 0x0080 /* Value is undefined */ #define MEM_Cleared 0x0100 /* NULL set by OP_Null, not from data */ -#define MEM_TypeMask 0xc1ff /* Mask of type bits */ +#define MEM_TypeMask 0xc1bf /* Mask of type bits */ /* Whenever Mem contains a valid string or blob representation, one of @@ -19624,7 +20928,7 @@ struct sqlite3_value { ** that needs to be deallocated to avoid a leak. */ #define VdbeMemDynamic(X) \ - (((X)->flags&(MEM_Agg|MEM_Dyn|MEM_RowSet|MEM_Frame))!=0) + (((X)->flags&(MEM_Agg|MEM_Dyn))!=0) /* ** Clear any existing type flags from a Mem and replace them with f @@ -19632,6 +20936,13 @@ struct sqlite3_value { #define MemSetTypeFlag(p, f) \ ((p)->flags = ((p)->flags&~(MEM_TypeMask|MEM_Zero))|f) +/* +** True if Mem X is a NULL-nochng type. +*/ +#define MemNullNochng(X) \ + (((X)->flags&MEM_TypeMask)==(MEM_Null|MEM_Zero) \ + && (X)->n==0 && (X)->u.nZero==0) + /* ** Return true if a memory cell is not marked as invalid. This macro ** is for use inside assert() statements only. @@ -19685,6 +20996,9 @@ struct sqlite3_context { */ typedef unsigned bft; /* Bit Field Type */ +/* The ScanStatus object holds a single value for the +** sqlite3_stmt_scanstatus() interface. +*/ typedef struct ScanStatus ScanStatus; struct ScanStatus { int addrExplain; /* OP_Explain for loop */ @@ -19695,6 +21009,19 @@ struct ScanStatus { char *zName; /* Name of table or index */ }; +/* The DblquoteStr object holds the text of a double-quoted +** string for a prepared statement. A linked list of these objects +** is constructed during statement parsing and is held on Vdbe.pDblStr. +** When computing a normalized SQL statement for an SQL statement, that +** list is consulted for each double-quoted identifier to see if the +** identifier should really be a string literal. +*/ +typedef struct DblquoteStr DblquoteStr; +struct DblquoteStr { + DblquoteStr *pNextStr; /* Next string literal in the list */ + char z[8]; /* Dequoted value for the string */ +}; + /* ** An instance of the virtual machine. This structure contains the complete ** state of the virtual machine. @@ -19714,28 +21041,29 @@ struct Vdbe { int pc; /* The program counter */ int rc; /* Value to return */ int nChange; /* Number of db changes made since last reset */ - int iStatement; /* Statement number (or 0 if has not opened stmt) */ + int iStatement; /* Statement number (or 0 if has no opened stmt) */ i64 iCurrentTime; /* Value of julianday('now') for this statement */ i64 nFkConstraint; /* Number of imm. FK constraints this VM */ i64 nStmtDefCons; /* Number of def. constraints when stmt started */ i64 nStmtDefImmCons; /* Number of def. imm constraints when stmt started */ + Mem *aMem; /* The memory locations */ + Mem **apArg; /* Arguments to currently executing user function */ + VdbeCursor **apCsr; /* One element of this array for each open cursor */ + Mem *aVar; /* Values for the OP_Variable opcode. */ /* When allocating a new Vdbe object, all of the fields below should be ** initialized to zero or NULL */ Op *aOp; /* Space to hold the virtual machine's program */ - Mem *aMem; /* The memory locations */ - Mem **apArg; /* Arguments to currently executing user function */ + int nOp; /* Number of instructions in the program */ + int nOpAlloc; /* Slots allocated for aOp[] */ Mem *aColName; /* Column names to return */ Mem *pResultSet; /* Pointer to an array of results */ char *zErrMsg; /* Error message written here */ - VdbeCursor **apCsr; /* One element of this array for each open cursor */ - Mem *aVar; /* Values for the OP_Variable opcode. */ VList *pVList; /* Name of variables */ #ifndef SQLITE_OMIT_TRACE i64 startTime; /* Time when query started - used for profiling */ #endif - int nOp; /* Number of instructions in the program */ #ifdef SQLITE_DEBUG int rcApp; /* errcode set by sqlite3_result_error_code() */ u32 nWrite; /* Number of write operations that have occurred */ @@ -19744,8 +21072,8 @@ struct Vdbe { u8 errorAction; /* Recovery action to do in case of an error */ u8 minWriteFileFormat; /* Minimum file format for writable database files */ u8 prepFlags; /* SQLITE_PREPARE_* flags */ - bft expired:1; /* True if the VM needs to be recompiled */ - bft doingRerun:1; /* True if rerunning after an auto-reprepare */ + u8 doingRerun; /* True if rerunning after an auto-reprepare */ + bft expired:2; /* 1: recompile VM immediately 2: when convenient */ bft explain:2; /* True if EXPLAIN present on SQL command */ bft changeCntOn:1; /* True to update the change-counter */ bft runOnlyOnce:1; /* Automatically expire on reset */ @@ -19756,6 +21084,10 @@ struct Vdbe { yDbMask lockMask; /* Subset of btreeMask that requires a lock */ u32 aCounter[7]; /* Counters used by sqlite3_stmt_status() */ char *zSql; /* Text of the SQL statement that generated this */ +#ifdef SQLITE_ENABLE_NORMALIZE + char *zNormSql; /* Normalization of the associated SQL statement */ + DblquoteStr *pDblStr; /* List of double-quoted string literals */ +#endif void *pFree; /* Free this when deleting the vdbe */ VdbeFrame *pFrame; /* Parent frame */ VdbeFrame *pDelFrame; /* List of frame objects to free on VM reset */ @@ -19805,14 +21137,11 @@ struct PreUpdate { SQLITE_PRIVATE void sqlite3VdbeError(Vdbe*, const char *, ...); SQLITE_PRIVATE void sqlite3VdbeFreeCursor(Vdbe *, VdbeCursor*); void sqliteVdbePopStack(Vdbe*,int); +SQLITE_PRIVATE int SQLITE_NOINLINE sqlite3VdbeFinishMoveto(VdbeCursor*); SQLITE_PRIVATE int sqlite3VdbeCursorMoveto(VdbeCursor**, int*); SQLITE_PRIVATE int sqlite3VdbeCursorRestore(VdbeCursor*); -#if defined(SQLITE_DEBUG) || defined(VDBE_PROFILE) -SQLITE_PRIVATE void sqlite3VdbePrintOp(FILE*, int, Op*); -#endif SQLITE_PRIVATE u32 sqlite3VdbeSerialTypeLen(u32); SQLITE_PRIVATE u8 sqlite3VdbeOneByteSerialTypeLen(u8); -SQLITE_PRIVATE u32 sqlite3VdbeSerialType(Mem*, int, u32*); SQLITE_PRIVATE u32 sqlite3VdbeSerialPut(unsigned char*, Mem*, u32); SQLITE_PRIVATE u32 sqlite3VdbeSerialGet(const unsigned char*, u32, Mem*); SQLITE_PRIVATE void sqlite3VdbeDeleteAuxData(sqlite3*, AuxData**, int, int); @@ -19821,7 +21150,16 @@ int sqlite2BtreeKeyCompare(BtCursor *, const void *, int, int, int *); SQLITE_PRIVATE int sqlite3VdbeIdxKeyCompare(sqlite3*,VdbeCursor*,UnpackedRecord*,int*); SQLITE_PRIVATE int sqlite3VdbeIdxRowid(sqlite3*, BtCursor*, i64*); SQLITE_PRIVATE int sqlite3VdbeExec(Vdbe*); +#if !defined(SQLITE_OMIT_EXPLAIN) || defined(SQLITE_ENABLE_BYTECODE_VTAB) +SQLITE_PRIVATE int sqlite3VdbeNextOpcode(Vdbe*,Mem*,int,int*,int*,Op**); +SQLITE_PRIVATE char *sqlite3VdbeDisplayP4(sqlite3*,Op*); +#endif +#if defined(SQLITE_ENABLE_EXPLAIN_COMMENTS) +SQLITE_PRIVATE char *sqlite3VdbeDisplayComment(sqlite3*,const Op*,const char*); +#endif +#if !defined(SQLITE_OMIT_EXPLAIN) SQLITE_PRIVATE int sqlite3VdbeList(Vdbe*); +#endif SQLITE_PRIVATE int sqlite3VdbeHalt(Vdbe*); SQLITE_PRIVATE int sqlite3VdbeChangeEncoding(Mem *, int); SQLITE_PRIVATE int sqlite3VdbeMemTooBig(Mem*); @@ -19840,7 +21178,10 @@ SQLITE_PRIVATE void sqlite3VdbeMemSetPointer(Mem*, void*, const char*, void(*)(v SQLITE_PRIVATE void sqlite3VdbeMemInit(Mem*,sqlite3*,u16); SQLITE_PRIVATE void sqlite3VdbeMemSetNull(Mem*); SQLITE_PRIVATE void sqlite3VdbeMemSetZeroBlob(Mem*,int); -SQLITE_PRIVATE void sqlite3VdbeMemSetRowSet(Mem*); +#ifdef SQLITE_DEBUG +SQLITE_PRIVATE int sqlite3VdbeMemIsRowSet(const Mem*); +#endif +SQLITE_PRIVATE int sqlite3VdbeMemSetRowSet(Mem*); SQLITE_PRIVATE int sqlite3VdbeMemMakeWriteable(Mem*); SQLITE_PRIVATE int sqlite3VdbeMemStringify(Mem*, u8, u8); SQLITE_PRIVATE i64 sqlite3VdbeIntValue(Mem*); @@ -19850,15 +21191,25 @@ SQLITE_PRIVATE int sqlite3VdbeBooleanValue(Mem*, int ifNull); SQLITE_PRIVATE void sqlite3VdbeIntegerAffinity(Mem*); SQLITE_PRIVATE int sqlite3VdbeMemRealify(Mem*); SQLITE_PRIVATE int sqlite3VdbeMemNumerify(Mem*); -SQLITE_PRIVATE void sqlite3VdbeMemCast(Mem*,u8,u8); +SQLITE_PRIVATE int sqlite3VdbeMemCast(Mem*,u8,u8); SQLITE_PRIVATE int sqlite3VdbeMemFromBtree(BtCursor*,u32,u32,Mem*); +SQLITE_PRIVATE int sqlite3VdbeMemFromBtreeZeroOffset(BtCursor*,u32,Mem*); SQLITE_PRIVATE void sqlite3VdbeMemRelease(Mem *p); SQLITE_PRIVATE int sqlite3VdbeMemFinalize(Mem*, FuncDef*); +#ifndef SQLITE_OMIT_WINDOWFUNC +SQLITE_PRIVATE int sqlite3VdbeMemAggValue(Mem*, Mem*, FuncDef*); +#endif +#if !defined(SQLITE_OMIT_EXPLAIN) || defined(SQLITE_ENABLE_BYTECODE_VTAB) SQLITE_PRIVATE const char *sqlite3OpcodeName(int); +#endif SQLITE_PRIVATE int sqlite3VdbeMemGrow(Mem *pMem, int n, int preserve); SQLITE_PRIVATE int sqlite3VdbeMemClearAndResize(Mem *pMem, int n); SQLITE_PRIVATE int sqlite3VdbeCloseStatement(Vdbe *, int); -SQLITE_PRIVATE void sqlite3VdbeFrameDelete(VdbeFrame*); +#ifdef SQLITE_DEBUG +SQLITE_PRIVATE int sqlite3VdbeFrameIsValid(VdbeFrame*); +#endif +SQLITE_PRIVATE void sqlite3VdbeFrameMemDel(void*); /* Destructor on Mem */ +SQLITE_PRIVATE void sqlite3VdbeFrameDelete(VdbeFrame*); /* Actually deletes the Frame */ SQLITE_PRIVATE int sqlite3VdbeFrameRestore(VdbeFrame *); #ifdef SQLITE_ENABLE_PREUPDATE_HOOK SQLITE_PRIVATE void sqlite3VdbePreUpdateHook(Vdbe*,VdbeCursor*,int,const char*,Table*,i64,int); @@ -19907,7 +21258,7 @@ SQLITE_PRIVATE int sqlite3VdbeCheckFk(Vdbe *, int); #ifdef SQLITE_DEBUG SQLITE_PRIVATE void sqlite3VdbePrintSql(Vdbe*); -SQLITE_PRIVATE void sqlite3VdbeMemPrettyPrint(Mem *pMem, char *zBuf); +SQLITE_PRIVATE void sqlite3VdbeMemPrettyPrint(Mem *pMem, StrAccum *pStr); #endif #ifndef SQLITE_OMIT_UTF16 SQLITE_PRIVATE int sqlite3VdbeMemTranslate(Mem*, u8); @@ -20099,6 +21450,10 @@ static u32 countLookasideSlots(LookasideSlot *p){ SQLITE_PRIVATE int sqlite3LookasideUsed(sqlite3 *db, int *pHighwater){ u32 nInit = countLookasideSlots(db->lookaside.pInit); u32 nFree = countLookasideSlots(db->lookaside.pFree); +#ifndef SQLITE_OMIT_TWOSIZE_LOOKASIDE + nInit += countLookasideSlots(db->lookaside.pSmallInit); + nFree += countLookasideSlots(db->lookaside.pSmallFree); +#endif /* SQLITE_OMIT_TWOSIZE_LOOKASIDE */ if( pHighwater ) *pHighwater = db->lookaside.nSlot - nInit; return db->lookaside.nSlot - (nInit+nFree); } @@ -20131,6 +21486,15 @@ SQLITE_API int sqlite3_db_status( db->lookaside.pInit = db->lookaside.pFree; db->lookaside.pFree = 0; } +#ifndef SQLITE_OMIT_TWOSIZE_LOOKASIDE + p = db->lookaside.pSmallFree; + if( p ){ + while( p->pNext ) p = p->pNext; + p->pNext = db->lookaside.pSmallInit; + db->lookaside.pSmallInit = db->lookaside.pSmallFree; + db->lookaside.pSmallFree = 0; + } +#endif } break; } @@ -20682,7 +22046,7 @@ static int parseDateOrTime( return 0; }else if( sqlite3StrICmp(zDate,"now")==0 && sqlite3NotPureFunc(context) ){ return setDateTimeToCurrent(context, p); - }else if( sqlite3AtoF(zDate, &r, sqlite3Strlen30(zDate), SQLITE_UTF8) ){ + }else if( sqlite3AtoF(zDate, &r, sqlite3Strlen30(zDate), SQLITE_UTF8)>0 ){ setRawDateNumber(p, r); return 0; } @@ -20915,12 +22279,12 @@ static const struct { double rLimit; /* Maximum NNN value for this transform */ double rXform; /* Constant used for this transform */ } aXformType[] = { - { 0, 6, "second", 464269060800.0, 86400000.0/(24.0*60.0*60.0) }, - { 0, 6, "minute", 7737817680.0, 86400000.0/(24.0*60.0) }, - { 0, 4, "hour", 128963628.0, 86400000.0/24.0 }, - { 0, 3, "day", 5373485.0, 86400000.0 }, - { 1, 5, "month", 176546.0, 30.0*86400000.0 }, - { 2, 4, "year", 14713.0, 365.0*86400000.0 }, + { 0, 6, "second", 464269060800.0, 1000.0 }, + { 0, 6, "minute", 7737817680.0, 60000.0 }, + { 0, 4, "hour", 128963628.0, 3600000.0 }, + { 0, 3, "day", 5373485.0, 86400000.0 }, + { 1, 5, "month", 176546.0, 2592000000.0 }, + { 2, 4, "year", 14713.0, 31536000000.0 }, }; /* @@ -20982,7 +22346,7 @@ static int parseModifier( r = p->s*1000.0 + 210866760000000.0; if( r>=0.0 && r<464269060800000.0 ){ clearYMD_HMS_TZ(p); - p->iJD = (sqlite3_int64)r; + p->iJD = (sqlite3_int64)(r + 0.5); p->validJD = 1; p->rawS = 0; rc = 0; @@ -21016,7 +22380,7 @@ static int parseModifier( ** date is already on the appropriate weekday, this is a no-op. */ if( sqlite3_strnicmp(z, "weekday ", 8)==0 - && sqlite3AtoF(&z[8], &r, sqlite3Strlen30(&z[8]), SQLITE_UTF8) + && sqlite3AtoF(&z[8], &r, sqlite3Strlen30(&z[8]), SQLITE_UTF8)>0 && (n=(int)r)==r && n>=0 && r<7 ){ sqlite3_int64 Z; computeYMD_HMS(p); @@ -21075,7 +22439,7 @@ static int parseModifier( double rRounder; int i; for(n=1; z[n] && z[n]!=':' && !sqlite3Isspace(z[n]); n++){} - if( !sqlite3AtoF(z, &r, n, SQLITE_UTF8) ){ + if( sqlite3AtoF(z, &r, n, SQLITE_UTF8)<=0 ){ rc = 1; break; } @@ -21761,7 +23125,7 @@ SQLITE_PRIVATE int sqlite3OsOpen( ** down into the VFS layer. Some SQLITE_OPEN_ flags (for example, ** SQLITE_OPEN_FULLMUTEX or SQLITE_OPEN_SHAREDCACHE) are blocked before ** reaching the VFS. */ - rc = pVfs->xOpen(pVfs, zPath, pFile, flags & 0x87f7f, pFlagsOut); + rc = pVfs->xOpen(pVfs, zPath, pFile, flags & 0x1087f7f, pFlagsOut); assert( rc==SQLITE_OK || pFile->pMethods==0 ); return rc; } @@ -21804,7 +23168,15 @@ SQLITE_PRIVATE void sqlite3OsDlClose(sqlite3_vfs *pVfs, void *pHandle){ } #endif /* SQLITE_OMIT_LOAD_EXTENSION */ SQLITE_PRIVATE int sqlite3OsRandomness(sqlite3_vfs *pVfs, int nByte, char *zBufOut){ - return pVfs->xRandomness(pVfs, nByte, zBufOut); + if( sqlite3Config.iPrngSeed ){ + memset(zBufOut, 0, nByte); + if( ALWAYS(nByte>(signed)sizeof(unsigned)) ) nByte = sizeof(unsigned int); + memcpy(zBufOut, &sqlite3Config.iPrngSeed, nByte); + return SQLITE_OK; + }else{ + return pVfs->xRandomness(pVfs, nByte, zBufOut); + } + } SQLITE_PRIVATE int sqlite3OsSleep(sqlite3_vfs *pVfs, int nMicro){ return pVfs->xSleep(pVfs, nMicro); @@ -21956,9 +23328,12 @@ SQLITE_API int sqlite3_vfs_register(sqlite3_vfs *pVfs, int makeDflt){ ** Unregister a VFS so that it is no longer accessible. */ SQLITE_API int sqlite3_vfs_unregister(sqlite3_vfs *pVfs){ -#if SQLITE_THREADSAFE - sqlite3_mutex *mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); + MUTEX_LOGIC(sqlite3_mutex *mutex;) +#ifndef SQLITE_OMIT_AUTOINIT + int rc = sqlite3_initialize(); + if( rc ) return rc; #endif + MUTEX_LOGIC( mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); ) sqlite3_mutex_enter(mutex); vfsUnlink(pVfs); sqlite3_mutex_leave(mutex); @@ -24469,6 +25844,7 @@ SQLITE_PRIVATE int sqlite3MutexInit(void){ GLOBAL(int, mutexIsInit) = 1; #endif + sqlite3MemoryBarrier(); return rc; } @@ -25268,7 +26644,7 @@ SQLITE_PRIVATE sqlite3_mutex_methods const *sqlite3DefaultMutex(void){ ****************************************************************************** ** ** This file contains inline asm code for retrieving "high-performance" -** counters for x86 class CPUs. +** counters for x86 and x86_64 class CPUs. */ #ifndef SQLITE_HWTIME_H #define SQLITE_HWTIME_H @@ -25279,8 +26655,9 @@ SQLITE_PRIVATE sqlite3_mutex_methods const *sqlite3DefaultMutex(void){ ** processor and returns that value. This can be used for high-res ** profiling. */ -#if (defined(__GNUC__) || defined(_MSC_VER)) && \ - (defined(i386) || defined(__i386__) || defined(_M_IX86)) +#if !defined(__STRICT_ANSI__) && \ + (defined(__GNUC__) || defined(_MSC_VER)) && \ + (defined(i386) || defined(__i386__) || defined(_M_IX86)) #if defined(__GNUC__) @@ -25301,7 +26678,7 @@ SQLITE_PRIVATE sqlite3_mutex_methods const *sqlite3DefaultMutex(void){ #endif -#elif (defined(__GNUC__) && defined(__x86_64__)) +#elif !defined(__STRICT_ANSI__) && (defined(__GNUC__) && defined(__x86_64__)) __inline__ sqlite_uint64 sqlite3Hwtime(void){ unsigned long val; @@ -25309,7 +26686,7 @@ SQLITE_PRIVATE sqlite3_mutex_methods const *sqlite3DefaultMutex(void){ return val; } -#elif (defined(__GNUC__) && defined(__ppc__)) +#elif !defined(__STRICT_ANSI__) && (defined(__GNUC__) && defined(__ppc__)) __inline__ sqlite_uint64 sqlite3Hwtime(void){ unsigned long long retval; @@ -25326,14 +26703,13 @@ SQLITE_PRIVATE sqlite3_mutex_methods const *sqlite3DefaultMutex(void){ #else - #error Need implementation of sqlite3Hwtime() for your platform. - /* - ** To compile without implementing sqlite3Hwtime() for your platform, - ** you can remove the above #error and use the following - ** stub function. You will lose timing support for many - ** of the debugging and testing utilities, but it should at - ** least compile and run. + ** asm() is needed for hardware timing support. Without asm(), + ** disable the sqlite3Hwtime() routine. + ** + ** sqlite3Hwtime() is only used for some obscure debugging + ** and analysis configurations, not in any deliverable, so this + ** should not be a great loss. */ SQLITE_PRIVATE sqlite_uint64 sqlite3Hwtime(void){ return ((sqlite_uint64)0); } @@ -25917,19 +27293,27 @@ SQLITE_API int sqlite3_release_memory(int n){ #endif } +/* +** Default value of the hard heap limit. 0 means "no limit". +*/ +#ifndef SQLITE_MAX_MEMORY +# define SQLITE_MAX_MEMORY 0 +#endif + /* ** State information local to the memory allocation subsystem. */ static SQLITE_WSD struct Mem0Global { sqlite3_mutex *mutex; /* Mutex to serialize access */ sqlite3_int64 alarmThreshold; /* The soft heap limit */ + sqlite3_int64 hardLimit; /* The hard upper bound on memory */ /* ** True if heap is nearly "full" where "full" is defined by the ** sqlite3_soft_heap_limit() setting. */ int nearlyFull; -} mem0 = { 0, 0, 0 }; +} mem0 = { 0, SQLITE_MAX_MEMORY, SQLITE_MAX_MEMORY, 0 }; #define mem0 GLOBAL(struct Mem0Global, mem0) @@ -25959,8 +27343,15 @@ SQLITE_API int sqlite3_memory_alarm( #endif /* -** Set the soft heap-size limit for the library. Passing a zero or -** negative value indicates no limit. +** Set the soft heap-size limit for the library. An argument of +** zero disables the limit. A negative argument is a no-op used to +** obtain the return value. +** +** The return value is the value of the heap limit just before this +** interface was called. +** +** If the hard heap limit is enabled, then the soft heap limit cannot +** be disabled nor raised above the hard heap limit. */ SQLITE_API sqlite3_int64 sqlite3_soft_heap_limit64(sqlite3_int64 n){ sqlite3_int64 priorLimit; @@ -25976,9 +27367,12 @@ SQLITE_API sqlite3_int64 sqlite3_soft_heap_limit64(sqlite3_int64 n){ sqlite3_mutex_leave(mem0.mutex); return priorLimit; } + if( mem0.hardLimit>0 && (n>mem0.hardLimit || n==0) ){ + n = mem0.hardLimit; + } mem0.alarmThreshold = n; nUsed = sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED); - mem0.nearlyFull = (n>0 && n<=nUsed); + AtomicStore(&mem0.nearlyFull, n>0 && n<=nUsed); sqlite3_mutex_leave(mem0.mutex); excess = sqlite3_memory_used() - n; if( excess>0 ) sqlite3_release_memory((int)(excess & 0x7fffffff)); @@ -25989,6 +27383,37 @@ SQLITE_API void sqlite3_soft_heap_limit(int n){ sqlite3_soft_heap_limit64(n); } +/* +** Set the hard heap-size limit for the library. An argument of zero +** disables the hard heap limit. A negative argument is a no-op used +** to obtain the return value without affecting the hard heap limit. +** +** The return value is the value of the hard heap limit just prior to +** calling this interface. +** +** Setting the hard heap limit will also activate the soft heap limit +** and constrain the soft heap limit to be no more than the hard heap +** limit. +*/ +SQLITE_API sqlite3_int64 sqlite3_hard_heap_limit64(sqlite3_int64 n){ + sqlite3_int64 priorLimit; +#ifndef SQLITE_OMIT_AUTOINIT + int rc = sqlite3_initialize(); + if( rc ) return -1; +#endif + sqlite3_mutex_enter(mem0.mutex); + priorLimit = mem0.hardLimit; + if( n>=0 ){ + mem0.hardLimit = n; + if( nSQLITE_MAX_MEMORY ){ - *pp = 0; - return; - } -#endif - sqlite3StatusHighwater(SQLITE_STATUS_MALLOC_SIZE, n); if( mem0.alarmThreshold>0 ){ sqlite3_int64 nUsed = sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED); if( nUsed >= mem0.alarmThreshold - nFull ){ - mem0.nearlyFull = 1; + AtomicStore(&mem0.nearlyFull, 1); sqlite3MallocAlarm(nFull); + if( mem0.hardLimit ){ + nUsed = sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED); + if( nUsed >= mem0.hardLimit - nFull ){ + *pp = 0; + return; + } + } }else{ - mem0.nearlyFull = 0; + AtomicStore(&mem0.nearlyFull, 0); } } p = sqlite3GlobalConfig.m.xMalloc(nFull); @@ -26168,10 +27593,17 @@ SQLITE_PRIVATE int sqlite3MallocSize(void *p){ assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) ); return sqlite3GlobalConfig.m.xSize(p); } +static int lookasideMallocSize(sqlite3 *db, void *p){ +#ifndef SQLITE_OMIT_TWOSIZE_LOOKASIDE + return plookaside.pMiddle ? db->lookaside.szTrue : LOOKASIDE_SMALL; +#else + return db->lookaside.szTrue; +#endif +} SQLITE_PRIVATE int sqlite3DbMallocSize(sqlite3 *db, void *p){ assert( p!=0 ); - if( db==0 || !isLookaside(db,p) ){ #ifdef SQLITE_DEBUG + if( db==0 || !isLookaside(db,p) ){ if( db==0 ){ assert( sqlite3MemdebugNoType(p, (u8)~MEMTYPE_HEAP) ); assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) ); @@ -26179,12 +27611,23 @@ SQLITE_PRIVATE int sqlite3DbMallocSize(sqlite3 *db, void *p){ assert( sqlite3MemdebugHasType(p, (MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); assert( sqlite3MemdebugNoType(p, (u8)~(MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); } + } #endif - return sqlite3GlobalConfig.m.xSize(p); - }else{ - assert( sqlite3_mutex_held(db->mutex) ); - return db->lookaside.sz; + if( db ){ + if( ((uptr)p)<(uptr)(db->lookaside.pEnd) ){ +#ifndef SQLITE_OMIT_TWOSIZE_LOOKASIDE + if( ((uptr)p)>=(uptr)(db->lookaside.pMiddle) ){ + assert( sqlite3_mutex_held(db->mutex) ); + return LOOKASIDE_SMALL; + } +#endif + if( ((uptr)p)>=(uptr)(db->lookaside.pStart) ){ + assert( sqlite3_mutex_held(db->mutex) ); + return db->lookaside.szTrue; + } + } } + return sqlite3GlobalConfig.m.xSize(p); } SQLITE_API sqlite3_uint64 sqlite3_msize(void *p){ assert( sqlite3MemdebugNoType(p, (u8)~MEMTYPE_HEAP) ); @@ -26231,15 +27674,27 @@ SQLITE_PRIVATE void sqlite3DbFreeNN(sqlite3 *db, void *p){ measureAllocationSize(db, p); return; } - if( isLookaside(db, p) ){ - LookasideSlot *pBuf = (LookasideSlot*)p; + if( ((uptr)p)<(uptr)(db->lookaside.pEnd) ){ +#ifndef SQLITE_OMIT_TWOSIZE_LOOKASIDE + if( ((uptr)p)>=(uptr)(db->lookaside.pMiddle) ){ + LookasideSlot *pBuf = (LookasideSlot*)p; #ifdef SQLITE_DEBUG - /* Trash all content in the buffer being freed */ - memset(p, 0xaa, db->lookaside.sz); + memset(p, 0xaa, LOOKASIDE_SMALL); /* Trash freed content */ #endif - pBuf->pNext = db->lookaside.pFree; - db->lookaside.pFree = pBuf; - return; + pBuf->pNext = db->lookaside.pSmallFree; + db->lookaside.pSmallFree = pBuf; + return; + } +#endif /* SQLITE_OMIT_TWOSIZE_LOOKASIDE */ + if( ((uptr)p)>=(uptr)(db->lookaside.pStart) ){ + LookasideSlot *pBuf = (LookasideSlot*)p; +#ifdef SQLITE_DEBUG + memset(p, 0xaa, db->lookaside.szTrue); /* Trash freed content */ +#endif + pBuf->pNext = db->lookaside.pFree; + db->lookaside.pFree = pBuf; + return; + } } } assert( sqlite3MemdebugHasType(p, (MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); @@ -26288,10 +27743,12 @@ SQLITE_PRIVATE void *sqlite3Realloc(void *pOld, u64 nBytes){ sqlite3MallocAlarm(nDiff); } pNew = sqlite3GlobalConfig.m.xRealloc(pOld, nNew); +#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT if( pNew==0 && mem0.alarmThreshold>0 ){ sqlite3MallocAlarm((int)nBytes); pNew = sqlite3GlobalConfig.m.xRealloc(pOld, nNew); } +#endif if( pNew ){ nNew = sqlite3MallocSize(pNew); sqlite3StatusUp(SQLITE_STATUS_MEMORY_USED, nNew-nOld); @@ -26395,23 +27852,37 @@ SQLITE_PRIVATE void *sqlite3DbMallocRawNN(sqlite3 *db, u64 n){ assert( db!=0 ); assert( sqlite3_mutex_held(db->mutex) ); assert( db->pnBytesFreed==0 ); - if( db->lookaside.bDisable==0 ){ - assert( db->mallocFailed==0 ); - if( n>db->lookaside.sz ){ - db->lookaside.anStat[1]++; - }else if( (pBuf = db->lookaside.pFree)!=0 ){ - db->lookaside.pFree = pBuf->pNext; + if( n>db->lookaside.sz ){ + if( !db->lookaside.bDisable ){ + db->lookaside.anStat[1]++; + }else if( db->mallocFailed ){ + return 0; + } + return dbMallocRawFinish(db, n); + } +#ifndef SQLITE_OMIT_TWOSIZE_LOOKASIDE + if( n<=LOOKASIDE_SMALL ){ + if( (pBuf = db->lookaside.pSmallFree)!=0 ){ + db->lookaside.pSmallFree = pBuf->pNext; db->lookaside.anStat[0]++; return (void*)pBuf; - }else if( (pBuf = db->lookaside.pInit)!=0 ){ - db->lookaside.pInit = pBuf->pNext; + }else if( (pBuf = db->lookaside.pSmallInit)!=0 ){ + db->lookaside.pSmallInit = pBuf->pNext; db->lookaside.anStat[0]++; return (void*)pBuf; - }else{ - db->lookaside.anStat[2]++; } - }else if( db->mallocFailed ){ - return 0; + } +#endif + if( (pBuf = db->lookaside.pFree)!=0 ){ + db->lookaside.pFree = pBuf->pNext; + db->lookaside.anStat[0]++; + return (void*)pBuf; + }else if( (pBuf = db->lookaside.pInit)!=0 ){ + db->lookaside.pInit = pBuf->pNext; + db->lookaside.anStat[0]++; + return (void*)pBuf; + }else{ + db->lookaside.anStat[2]++; } #else assert( db!=0 ); @@ -26435,7 +27906,16 @@ SQLITE_PRIVATE void *sqlite3DbRealloc(sqlite3 *db, void *p, u64 n){ assert( db!=0 ); if( p==0 ) return sqlite3DbMallocRawNN(db, n); assert( sqlite3_mutex_held(db->mutex) ); - if( isLookaside(db,p) && n<=db->lookaside.sz ) return p; + if( ((uptr)p)<(uptr)db->lookaside.pEnd ){ +#ifndef SQLITE_OMIT_TWOSIZE_LOOKASIDE + if( ((uptr)p)>=(uptr)db->lookaside.pMiddle ){ + if( n<=LOOKASIDE_SMALL ) return p; + }else +#endif + if( ((uptr)p)>=(uptr)db->lookaside.pStart ){ + if( n<=db->lookaside.szTrue ) return p; + } + } return dbReallocFinish(db, p, n); } static SQLITE_NOINLINE void *dbReallocFinish(sqlite3 *db, void *p, u64 n){ @@ -26446,14 +27926,14 @@ static SQLITE_NOINLINE void *dbReallocFinish(sqlite3 *db, void *p, u64 n){ if( isLookaside(db, p) ){ pNew = sqlite3DbMallocRawNN(db, n); if( pNew ){ - memcpy(pNew, p, db->lookaside.sz); + memcpy(pNew, p, lookasideMallocSize(db, p)); sqlite3DbFree(db, p); } }else{ assert( sqlite3MemdebugHasType(p, (MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); assert( sqlite3MemdebugNoType(p, (u8)~(MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); sqlite3MemdebugSetType(p, MEMTYPE_HEAP); - pNew = sqlite3_realloc64(p, n); + pNew = sqlite3Realloc(p, n); if( !pNew ){ sqlite3OomFault(db); } @@ -26543,9 +28023,12 @@ SQLITE_PRIVATE void sqlite3OomFault(sqlite3 *db){ if( db->mallocFailed==0 && db->bBenignMalloc==0 ){ db->mallocFailed = 1; if( db->nVdbeExec>0 ){ - db->u1.isInterrupted = 1; + AtomicStore(&db->u1.isInterrupted, 1); + } + DisableLookaside; + if( db->pParse ){ + db->pParse->rc = SQLITE_NOMEM_BKPT; } - db->lookaside.bDisable++; } } @@ -26559,9 +28042,9 @@ SQLITE_PRIVATE void sqlite3OomFault(sqlite3 *db){ SQLITE_PRIVATE void sqlite3OomClear(sqlite3 *db){ if( db->mallocFailed && db->nVdbeExec==0 ){ db->mallocFailed = 0; - db->u1.isInterrupted = 0; + AtomicStore(&db->u1.isInterrupted, 0); assert( db->lookaside.bDisable>0 ); - db->lookaside.bDisable--; + EnableLookaside; } } @@ -26702,6 +28185,12 @@ static const et_info fmtinfo[] = { { 'r', 10, 1, etORDINAL, 0, 0 }, }; +/* Floating point constants used for rounding */ +static const double arRound[] = { + 5.0e-01, 5.0e-02, 5.0e-03, 5.0e-04, 5.0e-05, + 5.0e-06, 5.0e-07, 5.0e-08, 5.0e-09, 5.0e-10, +}; + /* ** If SQLITE_OMIT_FLOATING_POINT is defined, then none of the floating point ** conversions will work. @@ -26739,7 +28228,8 @@ static char et_getdigit(LONGDOUBLE_TYPE *val, int *cnt){ static void setStrAccumError(StrAccum *p, u8 eError){ assert( eError==SQLITE_NOMEM || eError==SQLITE_TOOBIG ); p->accError = eError; - p->nAlloc = 0; + if( p->mxAlloc ) sqlite3_str_reset(p); + if( eError==SQLITE_TOOBIG ) sqlite3ErrorToParser(p->db, eError); } /* @@ -26758,6 +28248,28 @@ static char *getTextArg(PrintfArguments *p){ return (char*)sqlite3_value_text(p->apArg[p->nUsed++]); } +/* +** Allocate memory for a temporary buffer needed for printf rendering. +** +** If the requested size of the temp buffer is larger than the size +** of the output buffer in pAccum, then cause an SQLITE_TOOBIG error. +** Do the size check before the memory allocation to prevent rogue +** SQL from requesting large allocations using the precision or width +** field of the printf() function. +*/ +static char *printfTempBuf(sqlite3_str *pAccum, sqlite3_int64 n){ + char *z; + if( pAccum->accError ) return 0; + if( n>pAccum->nAlloc && n>pAccum->mxAlloc ){ + setStrAccumError(pAccum, SQLITE_TOOBIG); + return 0; + } + z = sqlite3DbMallocRaw(pAccum->db, n); + if( z==0 ){ + setStrAccumError(pAccum, SQLITE_NOMEM); + } + return z; +} /* ** On machines with a small stack size, you can redefine the @@ -26768,6 +28280,13 @@ static char *getTextArg(PrintfArguments *p){ #endif #define etBUFSIZE SQLITE_PRINT_BUF_SIZE /* Size of the output buffer */ +/* +** Hard limit on the precision of floating-point conversions. +*/ +#ifndef SQLITE_PRINTF_PRECISION_LIMIT +# define SQLITE_FP_PRECISION_LIMIT 100000000 +#endif + /* ** Render a string given by "fmt" into the StrAccum object. */ @@ -26840,6 +28359,9 @@ SQLITE_API void sqlite3_str_vappendf( flag_leftjustify = flag_prefix = cThousand = flag_alternateform = flag_altform2 = flag_zeropad = 0; done = 0; + width = 0; + flag_long = 0; + precision = -1; do{ switch( c ){ case '-': flag_leftjustify = 1; break; @@ -26850,80 +28372,93 @@ SQLITE_API void sqlite3_str_vappendf( case '0': flag_zeropad = 1; break; case ',': cThousand = ','; break; default: done = 1; break; - } - }while( !done && (c=(*++fmt))!=0 ); - /* Get the field width */ - if( c=='*' ){ - if( bArgList ){ - width = (int)getIntArg(pArgList); - }else{ - width = va_arg(ap,int); - } - if( width<0 ){ - flag_leftjustify = 1; - width = width >= -2147483647 ? -width : 0; - } - c = *++fmt; - }else{ - unsigned wx = 0; - while( c>='0' && c<='9' ){ - wx = wx*10 + c - '0'; - c = *++fmt; - } - testcase( wx>0x7fffffff ); - width = wx & 0x7fffffff; - } - assert( width>=0 ); + case 'l': { + flag_long = 1; + c = *++fmt; + if( c=='l' ){ + c = *++fmt; + flag_long = 2; + } + done = 1; + break; + } + case '1': case '2': case '3': case '4': case '5': + case '6': case '7': case '8': case '9': { + unsigned wx = c - '0'; + while( (c = *++fmt)>='0' && c<='9' ){ + wx = wx*10 + c - '0'; + } + testcase( wx>0x7fffffff ); + width = wx & 0x7fffffff; #ifdef SQLITE_PRINTF_PRECISION_LIMIT - if( width>SQLITE_PRINTF_PRECISION_LIMIT ){ - width = SQLITE_PRINTF_PRECISION_LIMIT; - } + if( width>SQLITE_PRINTF_PRECISION_LIMIT ){ + width = SQLITE_PRINTF_PRECISION_LIMIT; + } #endif - - /* Get the precision */ - if( c=='.' ){ - c = *++fmt; - if( c=='*' ){ - if( bArgList ){ - precision = (int)getIntArg(pArgList); - }else{ - precision = va_arg(ap,int); + if( c!='.' && c!='l' ){ + done = 1; + }else{ + fmt--; + } + break; } - c = *++fmt; - if( precision<0 ){ - precision = precision >= -2147483647 ? -precision : -1; + case '*': { + if( bArgList ){ + width = (int)getIntArg(pArgList); + }else{ + width = va_arg(ap,int); + } + if( width<0 ){ + flag_leftjustify = 1; + width = width >= -2147483647 ? -width : 0; + } +#ifdef SQLITE_PRINTF_PRECISION_LIMIT + if( width>SQLITE_PRINTF_PRECISION_LIMIT ){ + width = SQLITE_PRINTF_PRECISION_LIMIT; + } +#endif + if( (c = fmt[1])!='.' && c!='l' ){ + c = *++fmt; + done = 1; + } + break; } - }else{ - unsigned px = 0; - while( c>='0' && c<='9' ){ - px = px*10 + c - '0'; + case '.': { c = *++fmt; - } - testcase( px>0x7fffffff ); - precision = px & 0x7fffffff; - } - }else{ - precision = -1; - } - assert( precision>=(-1) ); + if( c=='*' ){ + if( bArgList ){ + precision = (int)getIntArg(pArgList); + }else{ + precision = va_arg(ap,int); + } + if( precision<0 ){ + precision = precision >= -2147483647 ? -precision : -1; + } + c = *++fmt; + }else{ + unsigned px = 0; + while( c>='0' && c<='9' ){ + px = px*10 + c - '0'; + c = *++fmt; + } + testcase( px>0x7fffffff ); + precision = px & 0x7fffffff; + } #ifdef SQLITE_PRINTF_PRECISION_LIMIT - if( precision>SQLITE_PRINTF_PRECISION_LIMIT ){ - precision = SQLITE_PRINTF_PRECISION_LIMIT; - } + if( precision>SQLITE_PRINTF_PRECISION_LIMIT ){ + precision = SQLITE_PRINTF_PRECISION_LIMIT; + } #endif - - - /* Get the conversion type modifier */ - if( c=='l' ){ - flag_long = 1; - c = *++fmt; - if( c=='l' ){ - flag_long = 2; - c = *++fmt; + if( c=='l' ){ + --fmt; + }else{ + done = 1; + } + break; + } } - }else{ - flag_long = 0; - } + }while( !done && (c=(*++fmt))!=0 ); + /* Fetch the info entry for the field */ infop = &fmtinfo[0]; xtype = etINVALID; @@ -26952,6 +28487,8 @@ SQLITE_API void sqlite3_str_vappendf( ** xtype The class of the conversion. ** infop Pointer to the appropriate info struct. */ + assert( width>=0 ); + assert( precision>=(-1) ); switch( xtype ){ case etPOINTER: flag_long = sizeof(char*)==sizeof(i64) ? 2 : @@ -27008,12 +28545,11 @@ SQLITE_API void sqlite3_str_vappendf( nOut = etBUFSIZE; zOut = buf; }else{ - u64 n = (u64)precision + 10 + precision/3; - zOut = zExtra = sqlite3Malloc( n ); - if( zOut==0 ){ - setStrAccumError(pAccum, SQLITE_NOMEM); - return; - } + u64 n; + n = (u64)precision + 10; + if( cThousand ) n += precision/3; + zOut = zExtra = printfTempBuf(pAccum, n); + if( zOut==0 ) return; nOut = (int)n; } bufpt = &zOut[nOut-1]; @@ -27074,6 +28610,11 @@ SQLITE_API void sqlite3_str_vappendf( length = 0; #else if( precision<0 ) precision = 6; /* Set default precision */ +#ifdef SQLITE_FP_PRECISION_LIMIT + if( precision>SQLITE_FP_PRECISION_LIMIT ){ + precision = SQLITE_FP_PRECISION_LIMIT; + } +#endif if( realvalue<0.0 ){ realvalue = -realvalue; prefix = '-'; @@ -27082,8 +28623,18 @@ SQLITE_API void sqlite3_str_vappendf( } if( xtype==etGENERIC && precision>0 ) precision--; testcase( precision>0xfff ); - for(idx=precision&0xfff, rounder=0.5; idx>0; idx--, rounder*=0.1){} - if( xtype==etFLOAT ) realvalue += rounder; + idx = precision & 0xfff; + rounder = arRound[idx%10]; + while( idx>=10 ){ rounder *= 1.0e-10; idx -= 10; } + if( xtype==etFLOAT ){ + double rx = (double)realvalue; + sqlite3_uint64 u; + int ex; + memcpy(&u, &rx, sizeof(u)); + ex = -1023 + (int)((u>>52)&0x7ff); + if( precision+(ex/3) < 15 ) rounder += realvalue*3e-16; + realvalue += rounder; + } /* Normalize realvalue to within 10.0 > realvalue >= 1.0 */ exp = 0; if( sqlite3IsNaN((double)realvalue) ){ @@ -27132,12 +28683,12 @@ SQLITE_API void sqlite3_str_vappendf( }else{ e2 = exp; } - if( MAX(e2,0)+(i64)precision+(i64)width > etBUFSIZE - 15 ){ - bufpt = zExtra - = sqlite3Malloc( MAX(e2,0)+(i64)precision+(i64)width+15 ); - if( bufpt==0 ){ - setStrAccumError(pAccum, SQLITE_NOMEM); - return; + { + i64 szBufNeeded; /* Size of a temporary buffer needed */ + szBufNeeded = MAX(e2,0)+(i64)precision+(i64)width+15; + if( szBufNeeded > etBUFSIZE ){ + bufpt = zExtra = printfTempBuf(pAccum, szBufNeeded); + if( bufpt==0 ) return; } } zOut = bufpt; @@ -27289,7 +28840,12 @@ SQLITE_API void sqlite3_str_vappendf( if( bufpt==0 ){ bufpt = ""; }else if( xtype==etDYNSTRING ){ - if( pAccum->nChar==0 && pAccum->mxAlloc && width==0 && precision<0 ){ + if( pAccum->nChar==0 + && pAccum->mxAlloc + && width==0 + && precision<0 + && pAccum->accError==0 + ){ /* Special optimization for sqlite3_mprintf("%z..."): ** Extend an existing memory allocation rather than creating ** a new one. */ @@ -27341,7 +28897,7 @@ SQLITE_API void sqlite3_str_vappendf( } isnull = escarg==0; if( isnull ) escarg = (xtype==etSQLESCAPE2 ? "NULL" : "(NULL)"); - /* For %q, %Q, and %w, the precision is the number of byte (or + /* For %q, %Q, and %w, the precision is the number of bytes (or ** characters if the ! flags is present) to use from the input. ** Because of the extra quoting characters inserted, the number ** of output characters may be larger than the precision. @@ -27356,11 +28912,8 @@ SQLITE_API void sqlite3_str_vappendf( needQuote = !isnull && xtype==etSQLESCAPE2; n += i + 3; if( n>etBUFSIZE ){ - bufpt = zExtra = sqlite3Malloc( n ); - if( bufpt==0 ){ - setStrAccumError(pAccum, SQLITE_NOMEM); - return; - } + bufpt = zExtra = printfTempBuf(pAccum, n); + if( bufpt==0 ) return; }else{ bufpt = buf; } @@ -27450,9 +29003,8 @@ static int sqlite3StrAccumEnlarge(StrAccum *p, int N){ return 0; } if( p->mxAlloc==0 ){ - N = p->nAlloc - p->nChar - 1; setStrAccumError(p, SQLITE_TOOBIG); - return N; + return p->nAlloc - p->nChar - 1; }else{ char *zOld = isMalloced(p) ? p->zText : 0; i64 szNew = p->nChar; @@ -27472,7 +29024,7 @@ static int sqlite3StrAccumEnlarge(StrAccum *p, int N){ if( p->db ){ zNew = sqlite3DbRealloc(p->db, zOld, p->nAlloc); }else{ - zNew = sqlite3_realloc64(zOld, p->nAlloc); + zNew = sqlite3Realloc(zOld, p->nAlloc); } if( zNew ){ assert( p->zText!=0 || p->nChar==0 ); @@ -27524,7 +29076,7 @@ SQLITE_API void sqlite3_str_append(sqlite3_str *p, const char *z, int N){ assert( z!=0 || N==0 ); assert( p->zText!=0 || p->nChar==0 || p->accError ); assert( N>=0 ); - assert( p->accError==0 || p->nAlloc==0 ); + assert( p->accError==0 || p->nAlloc==0 || p->mxAlloc==0 ); if( p->nChar+N >= p->nAlloc ){ enlargeAndAppend(p,z,N); }else if( N ){ @@ -27814,7 +29366,7 @@ SQLITE_API void sqlite3_log(int iErrCode, const char *zFormat, ...){ SQLITE_PRIVATE void sqlite3DebugPrintf(const char *zFormat, ...){ va_list ap; StrAccum acc; - char zBuf[500]; + char zBuf[SQLITE_PRINT_BUF_SIZE*10]; sqlite3StrAccumInit(&acc, 0, zBuf, sizeof(zBuf), 0); va_start(ap,zFormat); sqlite3_str_vappendf(&acc, zFormat, ap); @@ -27914,7 +29466,7 @@ static void sqlite3TreeViewLine(TreeView *p, const char *zFormat, ...){ va_start(ap, zFormat); sqlite3_str_vappendf(&acc, zFormat, ap); va_end(ap); - assert( acc.nChar>0 ); + assert( acc.nChar>0 || acc.accError ); sqlite3_str_append(&acc, "\n", 1); } sqlite3StrAccumFinish(&acc); @@ -27954,7 +29506,7 @@ SQLITE_PRIVATE void sqlite3TreeViewWith(TreeView *pView, const With *pWith, u8 m char cSep = '('; int j; for(j=0; jpCols->nExpr; j++){ - sqlite3_str_appendf(&x, "%c%s", cSep, pCte->pCols->a[j].zName); + sqlite3_str_appendf(&x, "%c%s", cSep, pCte->pCols->a[j].zEName); cSep = ','; } sqlite3_str_appendf(&x, ")"); @@ -27969,6 +29521,46 @@ SQLITE_PRIVATE void sqlite3TreeViewWith(TreeView *pView, const With *pWith, u8 m } } +/* +** Generate a human-readable description of a SrcList object. +*/ +SQLITE_PRIVATE void sqlite3TreeViewSrcList(TreeView *pView, const SrcList *pSrc){ + int i; + for(i=0; inSrc; i++){ + const struct SrcList_item *pItem = &pSrc->a[i]; + StrAccum x; + char zLine[100]; + sqlite3StrAccumInit(&x, 0, zLine, sizeof(zLine), 0); + sqlite3_str_appendf(&x, "{%d:*}", pItem->iCursor); + if( pItem->zDatabase ){ + sqlite3_str_appendf(&x, " %s.%s", pItem->zDatabase, pItem->zName); + }else if( pItem->zName ){ + sqlite3_str_appendf(&x, " %s", pItem->zName); + } + if( pItem->pTab ){ + sqlite3_str_appendf(&x, " tab=%Q nCol=%d ptr=%p used=%llx", + pItem->pTab->zName, pItem->pTab->nCol, pItem->pTab, pItem->colUsed); + } + if( pItem->zAlias ){ + sqlite3_str_appendf(&x, " (AS %s)", pItem->zAlias); + } + if( pItem->fg.jointype & JT_LEFT ){ + sqlite3_str_appendf(&x, " LEFT-JOIN"); + } + if( pItem->fg.fromDDL ){ + sqlite3_str_appendf(&x, " DDL"); + } + sqlite3StrAccumFinish(&x); + sqlite3TreeViewItem(pView, zLine, inSrc-1); + if( pItem->pSelect ){ + sqlite3TreeViewSelect(pView, pItem->pSelect, 0); + } + if( pItem->fg.isTabFunc ){ + sqlite3TreeViewExprList(pView, pItem->u1.pFuncArg, 0, "func-args:"); + } + sqlite3TreeViewPop(pView); + } +} /* ** Generate a human-readable description of a Select object. @@ -27987,21 +29579,17 @@ SQLITE_PRIVATE void sqlite3TreeViewSelect(TreeView *pView, const Select *p, u8 m sqlite3TreeViewPush(pView, 1); } do{ -#if SELECTTRACE_ENABLED - sqlite3TreeViewLine(pView, - "SELECT%s%s (%s/%p) selFlags=0x%x nSelectRow=%d", - ((p->selFlags & SF_Distinct) ? " DISTINCT" : ""), - ((p->selFlags & SF_Aggregate) ? " agg_flag" : ""), - p->zSelName, p, p->selFlags, - (int)p->nSelectRow - ); -#else - sqlite3TreeViewLine(pView, "SELECT%s%s (0x%p) selFlags=0x%x nSelectRow=%d", - ((p->selFlags & SF_Distinct) ? " DISTINCT" : ""), - ((p->selFlags & SF_Aggregate) ? " agg_flag" : ""), p, p->selFlags, - (int)p->nSelectRow - ); -#endif + if( p->selFlags & SF_WhereBegin ){ + sqlite3TreeViewLine(pView, "sqlite3WhereBegin()"); + }else{ + sqlite3TreeViewLine(pView, + "SELECT%s%s (%u/%p) selFlags=0x%x nSelectRow=%d", + ((p->selFlags & SF_Distinct) ? " DISTINCT" : ""), + ((p->selFlags & SF_Aggregate) ? " agg_flag" : ""), + p->selId, p, p->selFlags, + (int)p->nSelectRow + ); + } if( cnt++ ) sqlite3TreeViewPop(pView); if( p->pPrior ){ n = 1000; @@ -28013,42 +29601,30 @@ SQLITE_PRIVATE void sqlite3TreeViewSelect(TreeView *pView, const Select *p, u8 m if( p->pHaving ) n++; if( p->pOrderBy ) n++; if( p->pLimit ) n++; +#ifndef SQLITE_OMIT_WINDOWFUNC + if( p->pWin ) n++; + if( p->pWinDefn ) n++; +#endif + } + if( p->pEList ){ + sqlite3TreeViewExprList(pView, p->pEList, n>0, "result-set"); } - sqlite3TreeViewExprList(pView, p->pEList, (n--)>0, "result-set"); + n--; +#ifndef SQLITE_OMIT_WINDOWFUNC + if( p->pWin ){ + Window *pX; + pView = sqlite3TreeViewPush(pView, (n--)>0); + sqlite3TreeViewLine(pView, "window-functions"); + for(pX=p->pWin; pX; pX=pX->pNextWin){ + sqlite3TreeViewWinFunc(pView, pX, pX->pNextWin!=0); + } + sqlite3TreeViewPop(pView); + } +#endif if( p->pSrc && p->pSrc->nSrc ){ - int i; pView = sqlite3TreeViewPush(pView, (n--)>0); sqlite3TreeViewLine(pView, "FROM"); - for(i=0; ipSrc->nSrc; i++){ - struct SrcList_item *pItem = &p->pSrc->a[i]; - StrAccum x; - char zLine[100]; - sqlite3StrAccumInit(&x, 0, zLine, sizeof(zLine), 0); - sqlite3_str_appendf(&x, "{%d,*}", pItem->iCursor); - if( pItem->zDatabase ){ - sqlite3_str_appendf(&x, " %s.%s", pItem->zDatabase, pItem->zName); - }else if( pItem->zName ){ - sqlite3_str_appendf(&x, " %s", pItem->zName); - } - if( pItem->pTab ){ - sqlite3_str_appendf(&x, " tabname=%Q", pItem->pTab->zName); - } - if( pItem->zAlias ){ - sqlite3_str_appendf(&x, " (AS %s)", pItem->zAlias); - } - if( pItem->fg.jointype & JT_LEFT ){ - sqlite3_str_appendf(&x, " LEFT-JOIN"); - } - sqlite3StrAccumFinish(&x); - sqlite3TreeViewItem(pView, zLine, ipSrc->nSrc-1); - if( pItem->pSelect ){ - sqlite3TreeViewSelect(pView, pItem->pSelect, 0); - } - if( pItem->fg.isTabFunc ){ - sqlite3TreeViewExprList(pView, pItem->u1.pFuncArg, 0, "func-args:"); - } - sqlite3TreeViewPop(pView); - } + sqlite3TreeViewSrcList(pView, p->pSrc); sqlite3TreeViewPop(pView); } if( p->pWhere ){ @@ -28064,6 +29640,16 @@ SQLITE_PRIVATE void sqlite3TreeViewSelect(TreeView *pView, const Select *p, u8 m sqlite3TreeViewExpr(pView, p->pHaving, 0); sqlite3TreeViewPop(pView); } +#ifndef SQLITE_OMIT_WINDOWFUNC + if( p->pWinDefn ){ + Window *pX; + sqlite3TreeViewItem(pView, "WINDOW", (n--)>0); + for(pX=p->pWinDefn; pX; pX=pX->pNextWin){ + sqlite3TreeViewWindow(pView, pX, pX->pNextWin!=0); + } + sqlite3TreeViewPop(pView); + } +#endif if( p->pOrderBy ){ sqlite3TreeViewExprList(pView, p->pOrderBy, (n--)>0, "ORDERBY"); } @@ -28091,26 +29677,149 @@ SQLITE_PRIVATE void sqlite3TreeViewSelect(TreeView *pView, const Select *p, u8 m sqlite3TreeViewPop(pView); } +#ifndef SQLITE_OMIT_WINDOWFUNC +/* +** Generate a description of starting or stopping bounds +*/ +SQLITE_PRIVATE void sqlite3TreeViewBound( + TreeView *pView, /* View context */ + u8 eBound, /* UNBOUNDED, CURRENT, PRECEDING, FOLLOWING */ + Expr *pExpr, /* Value for PRECEDING or FOLLOWING */ + u8 moreToFollow /* True if more to follow */ +){ + switch( eBound ){ + case TK_UNBOUNDED: { + sqlite3TreeViewItem(pView, "UNBOUNDED", moreToFollow); + sqlite3TreeViewPop(pView); + break; + } + case TK_CURRENT: { + sqlite3TreeViewItem(pView, "CURRENT", moreToFollow); + sqlite3TreeViewPop(pView); + break; + } + case TK_PRECEDING: { + sqlite3TreeViewItem(pView, "PRECEDING", moreToFollow); + sqlite3TreeViewExpr(pView, pExpr, 0); + sqlite3TreeViewPop(pView); + break; + } + case TK_FOLLOWING: { + sqlite3TreeViewItem(pView, "FOLLOWING", moreToFollow); + sqlite3TreeViewExpr(pView, pExpr, 0); + sqlite3TreeViewPop(pView); + break; + } + } +} +#endif /* SQLITE_OMIT_WINDOWFUNC */ + +#ifndef SQLITE_OMIT_WINDOWFUNC +/* +** Generate a human-readable explanation for a Window object +*/ +SQLITE_PRIVATE void sqlite3TreeViewWindow(TreeView *pView, const Window *pWin, u8 more){ + int nElement = 0; + if( pWin->pFilter ){ + sqlite3TreeViewItem(pView, "FILTER", 1); + sqlite3TreeViewExpr(pView, pWin->pFilter, 0); + sqlite3TreeViewPop(pView); + } + pView = sqlite3TreeViewPush(pView, more); + if( pWin->zName ){ + sqlite3TreeViewLine(pView, "OVER %s (%p)", pWin->zName, pWin); + }else{ + sqlite3TreeViewLine(pView, "OVER (%p)", pWin); + } + if( pWin->zBase ) nElement++; + if( pWin->pOrderBy ) nElement++; + if( pWin->eFrmType ) nElement++; + if( pWin->eExclude ) nElement++; + if( pWin->zBase ){ + sqlite3TreeViewPush(pView, (--nElement)>0); + sqlite3TreeViewLine(pView, "window: %s", pWin->zBase); + sqlite3TreeViewPop(pView); + } + if( pWin->pPartition ){ + sqlite3TreeViewExprList(pView, pWin->pPartition, nElement>0,"PARTITION-BY"); + } + if( pWin->pOrderBy ){ + sqlite3TreeViewExprList(pView, pWin->pOrderBy, (--nElement)>0, "ORDER-BY"); + } + if( pWin->eFrmType ){ + char zBuf[30]; + const char *zFrmType = "ROWS"; + if( pWin->eFrmType==TK_RANGE ) zFrmType = "RANGE"; + if( pWin->eFrmType==TK_GROUPS ) zFrmType = "GROUPS"; + sqlite3_snprintf(sizeof(zBuf),zBuf,"%s%s",zFrmType, + pWin->bImplicitFrame ? " (implied)" : ""); + sqlite3TreeViewItem(pView, zBuf, (--nElement)>0); + sqlite3TreeViewBound(pView, pWin->eStart, pWin->pStart, 1); + sqlite3TreeViewBound(pView, pWin->eEnd, pWin->pEnd, 0); + sqlite3TreeViewPop(pView); + } + if( pWin->eExclude ){ + char zBuf[30]; + const char *zExclude; + switch( pWin->eExclude ){ + case TK_NO: zExclude = "NO OTHERS"; break; + case TK_CURRENT: zExclude = "CURRENT ROW"; break; + case TK_GROUP: zExclude = "GROUP"; break; + case TK_TIES: zExclude = "TIES"; break; + default: + sqlite3_snprintf(sizeof(zBuf),zBuf,"invalid(%d)", pWin->eExclude); + zExclude = zBuf; + break; + } + sqlite3TreeViewPush(pView, 0); + sqlite3TreeViewLine(pView, "EXCLUDE %s", zExclude); + sqlite3TreeViewPop(pView); + } + sqlite3TreeViewPop(pView); +} +#endif /* SQLITE_OMIT_WINDOWFUNC */ + +#ifndef SQLITE_OMIT_WINDOWFUNC +/* +** Generate a human-readable explanation for a Window Function object +*/ +SQLITE_PRIVATE void sqlite3TreeViewWinFunc(TreeView *pView, const Window *pWin, u8 more){ + pView = sqlite3TreeViewPush(pView, more); + sqlite3TreeViewLine(pView, "WINFUNC %s(%d)", + pWin->pFunc->zName, pWin->pFunc->nArg); + sqlite3TreeViewWindow(pView, pWin, 0); + sqlite3TreeViewPop(pView); +} +#endif /* SQLITE_OMIT_WINDOWFUNC */ + /* ** Generate a human-readable explanation of an expression tree. */ SQLITE_PRIVATE void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 moreToFollow){ const char *zBinOp = 0; /* Binary operator */ const char *zUniOp = 0; /* Unary operator */ - char zFlgs[60]; + char zFlgs[200]; pView = sqlite3TreeViewPush(pView, moreToFollow); if( pExpr==0 ){ sqlite3TreeViewLine(pView, "nil"); sqlite3TreeViewPop(pView); return; } - if( pExpr->flags ){ + if( pExpr->flags || pExpr->affExpr || pExpr->vvaFlags ){ + StrAccum x; + sqlite3StrAccumInit(&x, 0, zFlgs, sizeof(zFlgs), 0); + sqlite3_str_appendf(&x, " fg.af=%x.%c", + pExpr->flags, pExpr->affExpr ? pExpr->affExpr : 'n'); if( ExprHasProperty(pExpr, EP_FromJoin) ){ - sqlite3_snprintf(sizeof(zFlgs),zFlgs," flags=0x%x iRJT=%d", - pExpr->flags, pExpr->iRightJoinTable); - }else{ - sqlite3_snprintf(sizeof(zFlgs),zFlgs," flags=0x%x",pExpr->flags); + sqlite3_str_appendf(&x, " iRJT=%d", pExpr->iRightJoinTable); + } + if( ExprHasProperty(pExpr, EP_FromDDL) ){ + sqlite3_str_appendf(&x, " DDL"); + } + if( ExprHasVVAProperty(pExpr, EP_Immutable) ){ + sqlite3_str_appendf(&x, " IMMUTABLE"); } + sqlite3StrAccumFinish(&x); }else{ zFlgs[0] = 0; } @@ -28123,10 +29832,21 @@ SQLITE_PRIVATE void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 m case TK_COLUMN: { if( pExpr->iTable<0 ){ /* This only happens when coding check constraints */ - sqlite3TreeViewLine(pView, "COLUMN(%d)%s", pExpr->iColumn, zFlgs); + char zOp2[16]; + if( pExpr->op2 ){ + sqlite3_snprintf(sizeof(zOp2),zOp2," op2=0x%02x",pExpr->op2); + }else{ + zOp2[0] = 0; + } + sqlite3TreeViewLine(pView, "COLUMN(%d)%s%s", + pExpr->iColumn, zFlgs, zOp2); }else{ - sqlite3TreeViewLine(pView, "{%d:%d}%s", - pExpr->iTable, pExpr->iColumn, zFlgs); + sqlite3TreeViewLine(pView, "{%d:%d} pTab=%p%s", + pExpr->iTable, pExpr->iColumn, + pExpr->y.pTab, zFlgs); + } + if( ExprHasProperty(pExpr, EP_FixedCol) ){ + sqlite3TreeViewExpr(pView, pExpr->pLeft, 0); } break; } @@ -28205,6 +29925,7 @@ SQLITE_PRIVATE void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 m case TK_RSHIFT: zBinOp = "RSHIFT"; break; case TK_CONCAT: zBinOp = "CONCAT"; break; case TK_DOT: zBinOp = "DOT"; break; + case TK_LIMIT: zBinOp = "LIMIT"; break; case TK_UMINUS: zUniOp = "UMINUS"; break; case TK_UPLUS: zUniOp = "UPLUS"; break; @@ -28220,7 +29941,7 @@ SQLITE_PRIVATE void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 m }; assert( pExpr->op2==TK_IS || pExpr->op2==TK_ISNOT ); assert( pExpr->pRight ); - assert( pExpr->pRight->op==TK_TRUEFALSE ); + assert( sqlite3ExprSkipCollate(pExpr->pRight)->op==TK_TRUEFALSE ); x = (pExpr->op2==TK_ISNOT)*2 + sqlite3ExprTruthValue(pExpr->pRight); zUniOp = azOp[x]; break; @@ -28233,7 +29954,14 @@ SQLITE_PRIVATE void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 m } case TK_COLLATE: { - sqlite3TreeViewLine(pView, "COLLATE %Q", pExpr->u.zToken); + /* COLLATE operators without the EP_Collate flag are intended to + ** emulate collation associated with a table column. These show + ** up in the treeview output as "SOFT-COLLATE". Explicit COLLATE + ** operators that appear in the original SQL always have the + ** EP_Collate bit set and appear in treeview output as just "COLLATE" */ + sqlite3TreeViewLine(pView, "%sCOLLATE %Q%s", + !ExprHasProperty(pExpr, EP_Collate) ? "SOFT-" : "", + pExpr->u.zToken, zFlgs); sqlite3TreeViewExpr(pView, pExpr->pLeft, 0); break; } @@ -28241,20 +29969,44 @@ SQLITE_PRIVATE void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 m case TK_AGG_FUNCTION: case TK_FUNCTION: { ExprList *pFarg; /* List of function arguments */ + Window *pWin; if( ExprHasProperty(pExpr, EP_TokenOnly) ){ pFarg = 0; + pWin = 0; }else{ pFarg = pExpr->x.pList; +#ifndef SQLITE_OMIT_WINDOWFUNC + pWin = ExprHasProperty(pExpr, EP_WinFunc) ? pExpr->y.pWin : 0; +#else + pWin = 0; +#endif } if( pExpr->op==TK_AGG_FUNCTION ){ - sqlite3TreeViewLine(pView, "AGG_FUNCTION%d %Q", - pExpr->op2, pExpr->u.zToken); + sqlite3TreeViewLine(pView, "AGG_FUNCTION%d %Q%s iAgg=%d agg=%p", + pExpr->op2, pExpr->u.zToken, zFlgs, + pExpr->iAgg, pExpr->pAggInfo); + }else if( pExpr->op2!=0 ){ + const char *zOp2; + char zBuf[8]; + sqlite3_snprintf(sizeof(zBuf),zBuf,"0x%02x",pExpr->op2); + zOp2 = zBuf; + if( pExpr->op2==NC_IsCheck ) zOp2 = "NC_IsCheck"; + if( pExpr->op2==NC_IdxExpr ) zOp2 = "NC_IdxExpr"; + if( pExpr->op2==NC_PartIdx ) zOp2 = "NC_PartIdx"; + if( pExpr->op2==NC_GenCol ) zOp2 = "NC_GenCol"; + sqlite3TreeViewLine(pView, "FUNCTION %Q%s op2=%s", + pExpr->u.zToken, zFlgs, zOp2); }else{ - sqlite3TreeViewLine(pView, "FUNCTION %Q", pExpr->u.zToken); + sqlite3TreeViewLine(pView, "FUNCTION %Q%s", pExpr->u.zToken, zFlgs); } if( pFarg ){ - sqlite3TreeViewExprList(pView, pFarg, 0, 0); + sqlite3TreeViewExprList(pView, pFarg, pWin!=0, 0); } +#ifndef SQLITE_OMIT_WINDOWFUNC + if( pWin ){ + sqlite3TreeViewWindow(pView, pWin, 0); + } +#endif break; } #ifndef SQLITE_OMIT_SUBQUERY @@ -28322,7 +30074,7 @@ SQLITE_PRIVATE void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 m #ifndef SQLITE_OMIT_TRIGGER case TK_RAISE: { const char *zType = "unk"; - switch( pExpr->affinity ){ + switch( pExpr->affExpr ){ case OE_Rollback: zType = "rollback"; break; case OE_Abort: zType = "abort"; break; case OE_Fail: zType = "fail"; break; @@ -28339,7 +30091,9 @@ SQLITE_PRIVATE void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 m break; } case TK_VECTOR: { - sqlite3TreeViewBareExprList(pView, pExpr->x.pList, "VECTOR"); + char *z = sqlite3_mprintf("VECTOR%s",zFlgs); + sqlite3TreeViewBareExprList(pView, pExpr->x.pList, z); + sqlite3_free(z); break; } case TK_SELECT_COLUMN: { @@ -28363,7 +30117,7 @@ SQLITE_PRIVATE void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 m sqlite3TreeViewExpr(pView, pExpr->pRight, 0); }else if( zUniOp ){ sqlite3TreeViewLine(pView, "%s%s", zUniOp, zFlgs); - sqlite3TreeViewExpr(pView, pExpr->pLeft, 0); + sqlite3TreeViewExpr(pView, pExpr->pLeft, 0); } sqlite3TreeViewPop(pView); } @@ -28385,8 +30139,9 @@ SQLITE_PRIVATE void sqlite3TreeViewBareExprList( sqlite3TreeViewLine(pView, "%s", zLabel); for(i=0; inExpr; i++){ int j = pList->a[i].u.x.iOrderByCol; - char *zName = pList->a[i].zName; + char *zName = pList->a[i].zEName; int moreToFollow = inExpr - 1; + if( pList->a[i].eEName!=ENAME_NAME ) zName = 0; if( j || zName ){ sqlite3TreeViewPush(pView, moreToFollow); moreToFollow = 0; @@ -28943,26 +30698,6 @@ static const unsigned char sqlite3Utf8Trans1[] = { } \ } -#define READ_UTF16LE(zIn, TERM, c){ \ - c = (*zIn++); \ - c += ((*zIn++)<<8); \ - if( c>=0xD800 && c<0xE000 && TERM ){ \ - int c2 = (*zIn++); \ - c2 += ((*zIn++)<<8); \ - c = (c2&0x03FF) + ((c&0x003F)<<10) + (((c&0x03C0)+0x0040)<<10); \ - } \ -} - -#define READ_UTF16BE(zIn, TERM, c){ \ - c = ((*zIn++)<<8); \ - c += (*zIn++); \ - if( c>=0xD800 && c<0xE000 && TERM ){ \ - int c2 = ((*zIn++)<<8); \ - c2 += (*zIn++); \ - c = (c2&0x03FF) + ((c&0x003F)<<10) + (((c&0x03C0)+0x0040)<<10); \ - } \ -} - /* ** Translate a single UTF-8 character. Return the unicode value. ** @@ -29038,11 +30773,11 @@ SQLITE_PRIVATE u32 sqlite3Utf8Read( ** encoding, or if *pMem does not contain a string value. */ SQLITE_PRIVATE SQLITE_NOINLINE int sqlite3VdbeMemTranslate(Mem *pMem, u8 desiredEnc){ - int len; /* Maximum length of output string in bytes */ - unsigned char *zOut; /* Output buffer */ - unsigned char *zIn; /* Input iterator */ - unsigned char *zTerm; /* End of input */ - unsigned char *z; /* Output iterator */ + sqlite3_int64 len; /* Maximum length of output string in bytes */ + unsigned char *zOut; /* Output buffer */ + unsigned char *zIn; /* Input iterator */ + unsigned char *zTerm; /* End of input */ + unsigned char *z; /* Output iterator */ unsigned int c; assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); @@ -29053,9 +30788,11 @@ SQLITE_PRIVATE SQLITE_NOINLINE int sqlite3VdbeMemTranslate(Mem *pMem, u8 desired #if defined(TRANSLATE_TRACE) && defined(SQLITE_DEBUG) { - char zBuf[100]; - sqlite3VdbeMemPrettyPrint(pMem, zBuf); - fprintf(stderr, "INPUT: %s\n", zBuf); + StrAccum acc; + char zBuf[1000]; + sqlite3StrAccumInit(&acc, 0, zBuf, sizeof(zBuf), 0); + sqlite3VdbeMemPrettyPrint(pMem, &acc); + fprintf(stderr, "INPUT: %s\n", sqlite3StrAccumFinish(&acc)); } #endif @@ -29091,14 +30828,14 @@ SQLITE_PRIVATE SQLITE_NOINLINE int sqlite3VdbeMemTranslate(Mem *pMem, u8 desired ** nul-terminator. */ pMem->n &= ~1; - len = pMem->n * 2 + 1; + len = 2 * (sqlite3_int64)pMem->n + 1; }else{ /* When converting from UTF-8 to UTF-16 the maximum growth is caused ** when a 1-byte UTF-8 character is translated into a 2-byte UTF-16 ** character. Two bytes are required in the output buffer for the ** nul-terminator. */ - len = pMem->n * 2 + 2; + len = 2 * (sqlite3_int64)pMem->n + 2; } /* Set zIn to point at the start of the input buffer and zTerm to point 1 @@ -29137,13 +30874,59 @@ SQLITE_PRIVATE SQLITE_NOINLINE int sqlite3VdbeMemTranslate(Mem *pMem, u8 desired if( pMem->enc==SQLITE_UTF16LE ){ /* UTF-16 Little-endian -> UTF-8 */ while( zIn=0xd800 && c<0xe000 ){ +#ifdef SQLITE_REPLACE_INVALID_UTF + if( c>=0xdc00 || zIn>=zTerm ){ + c = 0xfffd; + }else{ + int c2 = *(zIn++); + c2 += (*(zIn++))<<8; + if( c2<0xdc00 || c2>=0xe000 ){ + zIn -= 2; + c = 0xfffd; + }else{ + c = ((c&0x3ff)<<10) + (c2&0x3ff) + 0x10000; + } + } +#else + if( zIn UTF-8 */ while( zIn=0xd800 && c<0xe000 ){ +#ifdef SQLITE_REPLACE_INVALID_UTF + if( c>=0xdc00 || zIn>=zTerm ){ + c = 0xfffd; + }else{ + int c2 = (*(zIn++))<<8; + c2 += *(zIn++); + if( c2<0xdc00 || c2>=0xe000 ){ + zIn -= 2; + c = 0xfffd; + }else{ + c = ((c&0x3ff)<<10) + (c2&0x3ff) + 0x10000; + } + } +#else + if( zIn=0xd8 && c<0xdc && z[0]>=0xdc && z[0]<0xe0 ) z += 2; + n++; } - return (int)(z-(unsigned char const *)zIn); + return (int)(z-(unsigned char const *)zIn) + - (SQLITE_UTF16NATIVE==SQLITE_UTF16LE); } #if defined(SQLITE_TEST) @@ -29341,30 +31123,6 @@ SQLITE_PRIVATE void sqlite3UtfSelfTest(void){ assert( c==t ); assert( (z-zBuf)==n ); } - for(i=0; i<0x00110000; i++){ - if( i>=0xD800 && i<0xE000 ) continue; - z = zBuf; - WRITE_UTF16LE(z, i); - n = (int)(z-zBuf); - assert( n>0 && n<=4 ); - z[0] = 0; - z = zBuf; - READ_UTF16LE(z, 1, c); - assert( c==i ); - assert( (z-zBuf)==n ); - } - for(i=0; i<0x00110000; i++){ - if( i>=0xD800 && i<0xE000 ) continue; - z = zBuf; - WRITE_UTF16BE(z, i); - n = (int)(z-zBuf); - assert( n>0 && n<=4 ); - z[0] = 0; - z = zBuf; - READ_UTF16BE(z, 1, c); - assert( c==i ); - assert( (z-zBuf)==n ); - } } #endif /* SQLITE_TEST */ #endif /* SQLITE_OMIT_UTF16 */ @@ -29390,8 +31148,8 @@ SQLITE_PRIVATE void sqlite3UtfSelfTest(void){ */ /* #include "sqliteInt.h" */ /* #include */ -#if HAVE_ISNAN || SQLITE_HAVE_ISNAN -# include +#ifndef SQLITE_OMIT_FLOATING_POINT +#include #endif /* @@ -29405,15 +31163,23 @@ SQLITE_PRIVATE void sqlite3Coverage(int x){ #endif /* -** Give a callback to the test harness that can be used to simulate faults -** in places where it is difficult or expensive to do so purely by means -** of inputs. +** Calls to sqlite3FaultSim() are used to simulate a failure during testing, +** or to bypass normal error detection during testing in order to let +** execute proceed futher downstream. +** +** In deployment, sqlite3FaultSim() *always* return SQLITE_OK (0). The +** sqlite3FaultSim() function only returns non-zero during testing. ** -** The intent of the integer argument is to let the fault simulator know -** which of multiple sqlite3FaultSim() calls has been hit. +** During testing, if the test harness has set a fault-sim callback using +** a call to sqlite3_test_control(SQLITE_TESTCTRL_FAULT_INSTALL), then +** each call to sqlite3FaultSim() is relayed to that application-supplied +** callback and the integer return value form the application-supplied +** callback is returned by sqlite3FaultSim(). ** -** Return whatever integer value the test callback returns, or return -** SQLITE_OK if no test callback is installed. +** The integer argument to sqlite3FaultSim() is a code to identify which +** sqlite3FaultSim() instance is being invoked. Each call to sqlite3FaultSim() +** should have a unique code. To prevent legacy testing applications from +** breaking, the codes should not be changed or reused. */ #ifndef SQLITE_UNTESTABLE SQLITE_PRIVATE int sqlite3FaultSim(int iTest){ @@ -29425,47 +31191,11 @@ SQLITE_PRIVATE int sqlite3FaultSim(int iTest){ #ifndef SQLITE_OMIT_FLOATING_POINT /* ** Return true if the floating point value is Not a Number (NaN). -** -** Use the math library isnan() function if compiled with SQLITE_HAVE_ISNAN. -** Otherwise, we have our own implementation that works on most systems. */ SQLITE_PRIVATE int sqlite3IsNaN(double x){ - int rc; /* The value return */ -#if !SQLITE_HAVE_ISNAN && !HAVE_ISNAN - /* - ** Systems that support the isnan() library function should probably - ** make use of it by compiling with -DSQLITE_HAVE_ISNAN. But we have - ** found that many systems do not have a working isnan() function so - ** this implementation is provided as an alternative. - ** - ** This NaN test sometimes fails if compiled on GCC with -ffast-math. - ** On the other hand, the use of -ffast-math comes with the following - ** warning: - ** - ** This option [-ffast-math] should never be turned on by any - ** -O option since it can result in incorrect output for programs - ** which depend on an exact implementation of IEEE or ISO - ** rules/specifications for math functions. - ** - ** Under MSVC, this NaN test may fail if compiled with a floating- - ** point precision mode other than /fp:precise. From the MSDN - ** documentation: - ** - ** The compiler [with /fp:precise] will properly handle comparisons - ** involving NaN. For example, x != x evaluates to true if x is NaN - ** ... - */ -#ifdef __FAST_MATH__ -# error SQLite will not work correctly with the -ffast-math option of GCC. -#endif - volatile double y = x; - volatile double z = y; - rc = (y!=z); -#else /* if HAVE_ISNAN */ - rc = isnan(x); -#endif /* HAVE_ISNAN */ - testcase( rc ); - return rc; + u64 y; + memcpy(&y,&x,sizeof(y)); + return IsNaN(y); } #endif /* SQLITE_OMIT_FLOATING_POINT */ @@ -29595,9 +31325,23 @@ SQLITE_PRIVATE void sqlite3ErrorMsg(Parse *pParse, const char *zFormat, ...){ sqlite3DbFree(db, pParse->zErrMsg); pParse->zErrMsg = zMsg; pParse->rc = SQLITE_ERROR; + pParse->pWith = 0; } } +/* +** If database connection db is currently parsing SQL, then transfer +** error code errCode to that parser if the parser has not already +** encountered some other kind of error. +*/ +SQLITE_PRIVATE int sqlite3ErrorToParser(sqlite3 *db, int errCode){ + Parse *pParse; + if( db==0 || (pParse = db->pParse)==0 ) return errCode; + pParse->rc = errCode; + pParse->nErr++; + return errCode; +} + /* ** Convert an SQL-style quoted string into a normal string by removing ** the quote characters. The conversion is done in-place. If the @@ -29611,7 +31355,7 @@ SQLITE_PRIVATE void sqlite3ErrorMsg(Parse *pParse, const char *zFormat, ...){ ** dequoted string, exclusive of the zero terminator, if dequoting does ** occur. ** -** 2002-Feb-14: This routine is extended to remove MS-Access style +** 2002-02-14: This routine is extended to remove MS-Access style ** brackets from around identifiers. For example: "[a-b-c]" becomes ** "a-b-c". */ @@ -29637,6 +31381,11 @@ SQLITE_PRIVATE void sqlite3Dequote(char *z){ } z[j] = 0; } +SQLITE_PRIVATE void sqlite3DequoteExpr(Expr *p){ + assert( sqlite3Isquote(p->u.zToken[0]) ); + p->flags |= p->u.zToken[0]=='"' ? EP_Quoted|EP_DblQuoted : EP_Quoted; + sqlite3Dequote(p->u.zToken); +} /* ** Generate a Token object from a string @@ -29669,12 +31418,18 @@ SQLITE_API int sqlite3_stricmp(const char *zLeft, const char *zRight){ } SQLITE_PRIVATE int sqlite3StrICmp(const char *zLeft, const char *zRight){ unsigned char *a, *b; - int c; + int c, x; a = (unsigned char *)zLeft; b = (unsigned char *)zRight; for(;;){ - c = (int)UpperToLower[*a] - (int)UpperToLower[*b]; - if( c || *a==0 ) break; + c = *a; + x = *b; + if( c==x ){ + if( c==0 ) break; + }else{ + c = (int)UpperToLower[c] - (int)UpperToLower[x]; + if( c ) break; + } a++; b++; } @@ -29693,6 +31448,19 @@ SQLITE_API int sqlite3_strnicmp(const char *zLeft, const char *zRight, int N){ return N<0 ? 0 : UpperToLower[*a] - UpperToLower[*b]; } +/* +** Compute an 8-bit hash on a string that is insensitive to case differences +*/ +SQLITE_PRIVATE u8 sqlite3StrIHash(const char *z){ + u8 h = 0; + if( z==0 ) return 0; + while( z[0] ){ + h += UpperToLower[(unsigned char)z[0]]; + z++; + } + return h; +} + /* ** Compute 10 to the E-th power. Examples: E==1 results in 10. ** E==2 results in 100. E==50 results in 1.0e50. @@ -29702,15 +31470,15 @@ SQLITE_API int sqlite3_strnicmp(const char *zLeft, const char *zRight, int N){ static LONGDOUBLE_TYPE sqlite3Pow10(int E){ #if defined(_MSC_VER) static const LONGDOUBLE_TYPE x[] = { - 1.0e+001, - 1.0e+002, - 1.0e+004, - 1.0e+008, - 1.0e+016, - 1.0e+032, - 1.0e+064, - 1.0e+128, - 1.0e+256 + 1.0e+001L, + 1.0e+002L, + 1.0e+004L, + 1.0e+008L, + 1.0e+016L, + 1.0e+032L, + 1.0e+064L, + 1.0e+128L, + 1.0e+256L }; LONGDOUBLE_TYPE r = 1.0; int i; @@ -29740,8 +31508,15 @@ static LONGDOUBLE_TYPE sqlite3Pow10(int E){ ** uses the encoding enc. The string is not necessarily zero-terminated. ** ** Return TRUE if the result is a valid real number (or integer) and FALSE -** if the string is empty or contains extraneous text. Valid numbers -** are in one of these formats: +** if the string is empty or contains extraneous text. More specifically +** return +** 1 => The input string is a pure integer +** 2 or more => The input has a decimal point or eNNN clause +** 0 or less => The input string is not a valid number +** -1 => Not a valid number, but has a valid prefix which +** includes a decimal point and/or an eNNN clause +** +** Valid numbers are in one of these formats: ** ** [+-]digits[E[+-]digits] ** [+-]digits.[digits][E[+-]digits] @@ -29754,10 +31529,13 @@ static LONGDOUBLE_TYPE sqlite3Pow10(int E){ ** returns FALSE but it still converts the prefix and writes the result ** into *pResult. */ +#if defined(_MSC_VER) +#pragma warning(disable : 4756) +#endif SQLITE_PRIVATE int sqlite3AtoF(const char *z, double *pResult, int length, u8 enc){ #ifndef SQLITE_OMIT_FLOATING_POINT int incr; - const char *zEnd = z + length; + const char *zEnd; /* sign * significand * (10 ^ (esign * exponent)) */ int sign = 1; /* sign of significand */ i64 s = 0; /* significand */ @@ -29766,20 +31544,25 @@ SQLITE_PRIVATE int sqlite3AtoF(const char *z, double *pResult, int length, u8 en int e = 0; /* exponent */ int eValid = 1; /* True exponent is either not used or is well-formed */ double result; - int nDigits = 0; - int nonNum = 0; /* True if input contains UTF16 with high byte non-zero */ + int nDigit = 0; /* Number of digits processed */ + int eType = 1; /* 1: pure integer, 2+: fractional -1 or less: bad UTF16 */ assert( enc==SQLITE_UTF8 || enc==SQLITE_UTF16LE || enc==SQLITE_UTF16BE ); *pResult = 0.0; /* Default return value, in case of an error */ + if( length==0 ) return 0; if( enc==SQLITE_UTF8 ){ incr = 1; + zEnd = z + length; }else{ int i; incr = 2; + length &= ~1; assert( SQLITE_UTF16LE==2 && SQLITE_UTF16BE==3 ); + testcase( enc==SQLITE_UTF16LE ); + testcase( enc==SQLITE_UTF16BE ); for(i=3-enc; i=((LARGEST_INT64-9)/10) ){ + /* skip non-significant significand digits + ** (increase exponent by d to shift decimal left) */ + while( z=zEnd ) goto do_atof_calc; /* if decimal point is present */ if( *z=='.' ){ z+=incr; + eType++; /* copy digits from after decimal to significand ** (decrease exponent by d to shift decimal right) */ while( z=zEnd ) goto do_atof_calc; @@ -29826,6 +31612,7 @@ SQLITE_PRIVATE int sqlite3AtoF(const char *z, double *pResult, int length, u8 en if( *z=='e' || *z=='E' ){ z+=incr; eValid = 0; + eType++; /* This branch is needed to avoid a (harmless) buffer overread. The ** special comment alerts the mutation tester that the correct answer @@ -29924,11 +31711,20 @@ SQLITE_PRIVATE int sqlite3AtoF(const char *z, double *pResult, int length, u8 en *pResult = result; /* return true if number and no extra non-whitespace chracters after */ - return z==zEnd && nDigits>0 && eValid && nonNum==0; + if( z==zEnd && nDigit>0 && eValid && eType>0 ){ + return eType; + }else if( eType>=2 && (eType==3 || eValid) && nDigit>0 ){ + return -1; + }else{ + return 0; + } #else return !sqlite3Atoi64(z, pResult, length, enc); #endif /* SQLITE_OMIT_FLOATING_POINT */ } +#if defined(_MSC_VER) +#pragma warning(default : 4756) +#endif /* ** Compare the 19-character string zNum against the text representation @@ -29967,6 +31763,7 @@ static int compare2pow63(const char *zNum, int incr){ ** ** Returns: ** +** -1 Not even a prefix of the input text looks like an integer ** 0 Successful transformation. Fits in a 64-bit signed integer. ** 1 Excess non-space text after the integer value ** 2 Integer too large for a 64-bit signed integer or is malformed @@ -30026,9 +31823,9 @@ SQLITE_PRIVATE int sqlite3Atoi64(const char *zNum, i64 *pNum, int length, u8 enc *pNum = (i64)u; } rc = 0; - if( (i==0 && zStart==zNum) /* No digits */ - || nonNum /* UTF16 with high-order bytes non-zero */ - ){ + if( i==0 && zStart==zNum ){ /* No digits */ + rc = -1; + }else if( nonNum ){ /* UTF16 with high-order bytes non-zero */ rc = 1; }else if( &zNum[i]=0 ){ + *v = *p; return 1; } - - p++; - b = *p; - /* b: p1 (unmasked) */ - if (!(b&0x80)) - { - a &= 0x7f; - a = a<<7; - a |= b; - *v = a; + if( ((signed char*)p)[1]>=0 ){ + *v = ((u32)(p[0]&0x7f)<<7) | p[1]; return 2; } @@ -30283,8 +32069,9 @@ SQLITE_PRIVATE u8 sqlite3GetVarint(const unsigned char *p, u64 *v){ assert( SLOT_2_0 == ((0x7f<<14) | (0x7f)) ); assert( SLOT_4_2_0 == ((0xfU<<28) | (0x7f<<14) | (0x7f)) ); - p++; - a = a<<14; + a = ((u32)p[0])<<14; + b = p[1]; + p += 2; a |= *p; /* a: p0<<14 | p2 (unmasked) */ if (!(a&0x80)) @@ -30613,7 +32400,7 @@ SQLITE_PRIVATE u8 sqlite3HexToInt(int h){ return (u8)(h & 0xf); } -#if !defined(SQLITE_OMIT_BLOB_LITERAL) || defined(SQLITE_HAS_CODEC) +#if !defined(SQLITE_OMIT_BLOB_LITERAL) /* ** Convert a BLOB literal of the form "x'hhhhhh'" into its binary ** value. Return a pointer to its binary value. Space to hold the @@ -30634,7 +32421,7 @@ SQLITE_PRIVATE void *sqlite3HexToBlob(sqlite3 *db, const char *z, int n){ } return zBlob; } -#endif /* !SQLITE_OMIT_BLOB_LITERAL || SQLITE_HAS_CODEC */ +#endif /* !SQLITE_OMIT_BLOB_LITERAL */ /* ** Log an error that is an API call on a connection pointer that should @@ -30867,7 +32654,7 @@ SQLITE_PRIVATE LogEst sqlite3LogEstFromDouble(double x){ #endif /* SQLITE_OMIT_VIRTUALTABLE */ #if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || \ - defined(SQLITE_ENABLE_STAT3_OR_STAT4) || \ + defined(SQLITE_ENABLE_STAT4) || \ defined(SQLITE_EXPLAIN_ESTIMATED_ROWS) /* ** Convert a LogEst into an integer. @@ -30885,7 +32672,7 @@ SQLITE_PRIVATE u64 sqlite3LogEstToInt(LogEst x){ defined(SQLITE_EXPLAIN_ESTIMATED_ROWS) if( x>60 ) return (u64)LARGEST_INT64; #else - /* If only SQLITE_ENABLE_STAT3_OR_STAT4 is on, then the largest input + /* If only SQLITE_ENABLE_STAT4 is on, then the largest input ** possible to this routine is 310, resulting in a maximum x of 31 */ assert( x<=60 ); #endif @@ -30944,7 +32731,7 @@ SQLITE_PRIVATE VList *sqlite3VListAdd( assert( pIn==0 || pIn[0]>=3 ); /* Verify ok to add new elements */ if( pIn==0 || pIn[1]+nInt > pIn[0] ){ /* Enlarge the allocation */ - int nAlloc = (pIn ? pIn[0]*2 : 10) + nInt; + sqlite3_int64 nAlloc = (pIn ? 2*(sqlite3_int64)pIn[0] : 10) + nInt; VList *pOut = sqlite3DbRealloc(db, pIn, nAlloc*sizeof(int)); if( pOut==0 ) return pIn; if( pIn==0 ) pOut[1] = 2; @@ -31150,7 +32937,7 @@ static HashElem *findElementWithHash( unsigned int *pHash /* Write the hash value here */ ){ HashElem *elem; /* Used to loop thru the element list */ - int count; /* Number of elements left to test */ + unsigned int count; /* Number of elements left to test */ unsigned int h; /* The computed hash */ static HashElem nullElement = { 0, 0, 0, 0 }; @@ -31198,8 +32985,8 @@ static void removeElementGivenHash( if( pEntry->chain==elem ){ pEntry->chain = elem->next; } + assert( pEntry->count>0 ); pEntry->count--; - assert( pEntry->count>=0 ); } sqlite3_free( elem ); pH->count--; @@ -31286,30 +33073,30 @@ SQLITE_PRIVATE const char *sqlite3OpcodeName(int i){ /* 1 */ "AutoCommit" OpHelp(""), /* 2 */ "Transaction" OpHelp(""), /* 3 */ "SorterNext" OpHelp(""), - /* 4 */ "PrevIfOpen" OpHelp(""), - /* 5 */ "NextIfOpen" OpHelp(""), - /* 6 */ "Prev" OpHelp(""), - /* 7 */ "Next" OpHelp(""), - /* 8 */ "Checkpoint" OpHelp(""), - /* 9 */ "JournalMode" OpHelp(""), - /* 10 */ "Vacuum" OpHelp(""), - /* 11 */ "VFilter" OpHelp("iplan=r[P3] zplan='P4'"), - /* 12 */ "VUpdate" OpHelp("data=r[P3@P2]"), - /* 13 */ "Goto" OpHelp(""), - /* 14 */ "Gosub" OpHelp(""), - /* 15 */ "InitCoroutine" OpHelp(""), - /* 16 */ "Yield" OpHelp(""), - /* 17 */ "MustBeInt" OpHelp(""), - /* 18 */ "Jump" OpHelp(""), + /* 4 */ "Prev" OpHelp(""), + /* 5 */ "Next" OpHelp(""), + /* 6 */ "Checkpoint" OpHelp(""), + /* 7 */ "JournalMode" OpHelp(""), + /* 8 */ "Vacuum" OpHelp(""), + /* 9 */ "VFilter" OpHelp("iplan=r[P3] zplan='P4'"), + /* 10 */ "VUpdate" OpHelp("data=r[P3@P2]"), + /* 11 */ "Goto" OpHelp(""), + /* 12 */ "Gosub" OpHelp(""), + /* 13 */ "InitCoroutine" OpHelp(""), + /* 14 */ "Yield" OpHelp(""), + /* 15 */ "MustBeInt" OpHelp(""), + /* 16 */ "Jump" OpHelp(""), + /* 17 */ "Once" OpHelp(""), + /* 18 */ "If" OpHelp(""), /* 19 */ "Not" OpHelp("r[P2]= !r[P1]"), - /* 20 */ "Once" OpHelp(""), - /* 21 */ "If" OpHelp(""), - /* 22 */ "IfNot" OpHelp(""), - /* 23 */ "IfNullRow" OpHelp("if P1.nullRow then r[P3]=NULL, goto P2"), - /* 24 */ "SeekLT" OpHelp("key=r[P3@P4]"), - /* 25 */ "SeekLE" OpHelp("key=r[P3@P4]"), - /* 26 */ "SeekGE" OpHelp("key=r[P3@P4]"), - /* 27 */ "SeekGT" OpHelp("key=r[P3@P4]"), + /* 20 */ "IfNot" OpHelp(""), + /* 21 */ "IfNullRow" OpHelp("if P1.nullRow then r[P3]=NULL, goto P2"), + /* 22 */ "SeekLT" OpHelp("key=r[P3@P4]"), + /* 23 */ "SeekLE" OpHelp("key=r[P3@P4]"), + /* 24 */ "SeekGE" OpHelp("key=r[P3@P4]"), + /* 25 */ "SeekGT" OpHelp("key=r[P3@P4]"), + /* 26 */ "IfNotOpen" OpHelp("if( !csr[P1] ) goto P2"), + /* 27 */ "IfNoHope" OpHelp("key=r[P3@P4]"), /* 28 */ "NoConflict" OpHelp("key=r[P3@P4]"), /* 29 */ "NotFound" OpHelp("key=r[P3@P4]"), /* 30 */ "Found" OpHelp("key=r[P3@P4]"), @@ -31345,115 +33132,119 @@ SQLITE_PRIVATE const char *sqlite3OpcodeName(int i){ /* 60 */ "IncrVacuum" OpHelp(""), /* 61 */ "VNext" OpHelp(""), /* 62 */ "Init" OpHelp("Start at P2"), - /* 63 */ "Return" OpHelp(""), - /* 64 */ "EndCoroutine" OpHelp(""), - /* 65 */ "HaltIfNull" OpHelp("if r[P3]=null halt"), - /* 66 */ "Halt" OpHelp(""), - /* 67 */ "Integer" OpHelp("r[P2]=P1"), - /* 68 */ "Int64" OpHelp("r[P2]=P4"), - /* 69 */ "String" OpHelp("r[P2]='P4' (len=P1)"), - /* 70 */ "Null" OpHelp("r[P2..P3]=NULL"), - /* 71 */ "SoftNull" OpHelp("r[P1]=NULL"), - /* 72 */ "Blob" OpHelp("r[P2]=P4 (len=P1)"), - /* 73 */ "Variable" OpHelp("r[P2]=parameter(P1,P4)"), - /* 74 */ "Move" OpHelp("r[P2@P3]=r[P1@P3]"), - /* 75 */ "Copy" OpHelp("r[P2@P3+1]=r[P1@P3+1]"), - /* 76 */ "SCopy" OpHelp("r[P2]=r[P1]"), - /* 77 */ "IntCopy" OpHelp("r[P2]=r[P1]"), - /* 78 */ "ResultRow" OpHelp("output=r[P1@P2]"), - /* 79 */ "CollSeq" OpHelp(""), - /* 80 */ "AddImm" OpHelp("r[P1]=r[P1]+P2"), - /* 81 */ "RealAffinity" OpHelp(""), - /* 82 */ "Cast" OpHelp("affinity(r[P1])"), - /* 83 */ "Permutation" OpHelp(""), - /* 84 */ "Compare" OpHelp("r[P1@P3] <-> r[P2@P3]"), - /* 85 */ "BitAnd" OpHelp("r[P3]=r[P1]&r[P2]"), - /* 86 */ "BitOr" OpHelp("r[P3]=r[P1]|r[P2]"), - /* 87 */ "ShiftLeft" OpHelp("r[P3]=r[P2]<>r[P1]"), - /* 89 */ "Add" OpHelp("r[P3]=r[P1]+r[P2]"), - /* 90 */ "Subtract" OpHelp("r[P3]=r[P2]-r[P1]"), - /* 91 */ "Multiply" OpHelp("r[P3]=r[P1]*r[P2]"), - /* 92 */ "Divide" OpHelp("r[P3]=r[P2]/r[P1]"), - /* 93 */ "Remainder" OpHelp("r[P3]=r[P2]%r[P1]"), - /* 94 */ "Concat" OpHelp("r[P3]=r[P2]+r[P1]"), - /* 95 */ "IsTrue" OpHelp("r[P2] = coalesce(r[P1]==TRUE,P3) ^ P4"), - /* 96 */ "BitNot" OpHelp("r[P1]= ~r[P1]"), - /* 97 */ "Offset" OpHelp("r[P3] = sqlite_offset(P1)"), - /* 98 */ "Column" OpHelp("r[P3]=PX"), - /* 99 */ "String8" OpHelp("r[P2]='P4'"), - /* 100 */ "Affinity" OpHelp("affinity(r[P1@P2])"), - /* 101 */ "MakeRecord" OpHelp("r[P3]=mkrec(r[P1@P2])"), - /* 102 */ "Count" OpHelp("r[P2]=count()"), - /* 103 */ "ReadCookie" OpHelp(""), - /* 104 */ "SetCookie" OpHelp(""), - /* 105 */ "ReopenIdx" OpHelp("root=P2 iDb=P3"), - /* 106 */ "OpenRead" OpHelp("root=P2 iDb=P3"), - /* 107 */ "OpenWrite" OpHelp("root=P2 iDb=P3"), - /* 108 */ "OpenDup" OpHelp(""), - /* 109 */ "OpenAutoindex" OpHelp("nColumn=P2"), - /* 110 */ "OpenEphemeral" OpHelp("nColumn=P2"), + /* 63 */ "PureFunc" OpHelp("r[P3]=func(r[P2@NP])"), + /* 64 */ "Function" OpHelp("r[P3]=func(r[P2@NP])"), + /* 65 */ "Return" OpHelp(""), + /* 66 */ "EndCoroutine" OpHelp(""), + /* 67 */ "HaltIfNull" OpHelp("if r[P3]=null halt"), + /* 68 */ "Halt" OpHelp(""), + /* 69 */ "Integer" OpHelp("r[P2]=P1"), + /* 70 */ "Int64" OpHelp("r[P2]=P4"), + /* 71 */ "String" OpHelp("r[P2]='P4' (len=P1)"), + /* 72 */ "Null" OpHelp("r[P2..P3]=NULL"), + /* 73 */ "SoftNull" OpHelp("r[P1]=NULL"), + /* 74 */ "Blob" OpHelp("r[P2]=P4 (len=P1)"), + /* 75 */ "Variable" OpHelp("r[P2]=parameter(P1,P4)"), + /* 76 */ "Move" OpHelp("r[P2@P3]=r[P1@P3]"), + /* 77 */ "Copy" OpHelp("r[P2@P3+1]=r[P1@P3+1]"), + /* 78 */ "SCopy" OpHelp("r[P2]=r[P1]"), + /* 79 */ "IntCopy" OpHelp("r[P2]=r[P1]"), + /* 80 */ "ResultRow" OpHelp("output=r[P1@P2]"), + /* 81 */ "CollSeq" OpHelp(""), + /* 82 */ "AddImm" OpHelp("r[P1]=r[P1]+P2"), + /* 83 */ "RealAffinity" OpHelp(""), + /* 84 */ "Cast" OpHelp("affinity(r[P1])"), + /* 85 */ "Permutation" OpHelp(""), + /* 86 */ "Compare" OpHelp("r[P1@P3] <-> r[P2@P3]"), + /* 87 */ "IsTrue" OpHelp("r[P2] = coalesce(r[P1]==TRUE,P3) ^ P4"), + /* 88 */ "Offset" OpHelp("r[P3] = sqlite_offset(P1)"), + /* 89 */ "Column" OpHelp("r[P3]=PX"), + /* 90 */ "Affinity" OpHelp("affinity(r[P1@P2])"), + /* 91 */ "MakeRecord" OpHelp("r[P3]=mkrec(r[P1@P2])"), + /* 92 */ "Count" OpHelp("r[P2]=count()"), + /* 93 */ "ReadCookie" OpHelp(""), + /* 94 */ "SetCookie" OpHelp(""), + /* 95 */ "ReopenIdx" OpHelp("root=P2 iDb=P3"), + /* 96 */ "OpenRead" OpHelp("root=P2 iDb=P3"), + /* 97 */ "OpenWrite" OpHelp("root=P2 iDb=P3"), + /* 98 */ "OpenDup" OpHelp(""), + /* 99 */ "OpenAutoindex" OpHelp("nColumn=P2"), + /* 100 */ "OpenEphemeral" OpHelp("nColumn=P2"), + /* 101 */ "BitAnd" OpHelp("r[P3]=r[P1]&r[P2]"), + /* 102 */ "BitOr" OpHelp("r[P3]=r[P1]|r[P2]"), + /* 103 */ "ShiftLeft" OpHelp("r[P3]=r[P2]<>r[P1]"), + /* 105 */ "Add" OpHelp("r[P3]=r[P1]+r[P2]"), + /* 106 */ "Subtract" OpHelp("r[P3]=r[P2]-r[P1]"), + /* 107 */ "Multiply" OpHelp("r[P3]=r[P1]*r[P2]"), + /* 108 */ "Divide" OpHelp("r[P3]=r[P2]/r[P1]"), + /* 109 */ "Remainder" OpHelp("r[P3]=r[P2]%r[P1]"), + /* 110 */ "Concat" OpHelp("r[P3]=r[P2]+r[P1]"), /* 111 */ "SorterOpen" OpHelp(""), - /* 112 */ "SequenceTest" OpHelp("if( cursor[P1].ctr++ ) pc = P2"), - /* 113 */ "OpenPseudo" OpHelp("P3 columns in r[P2]"), - /* 114 */ "Close" OpHelp(""), - /* 115 */ "ColumnsUsed" OpHelp(""), - /* 116 */ "Sequence" OpHelp("r[P2]=cursor[P1].ctr++"), - /* 117 */ "NewRowid" OpHelp("r[P2]=rowid"), - /* 118 */ "Insert" OpHelp("intkey=r[P3] data=r[P2]"), - /* 119 */ "InsertInt" OpHelp("intkey=P3 data=r[P2]"), - /* 120 */ "Delete" OpHelp(""), - /* 121 */ "ResetCount" OpHelp(""), - /* 122 */ "SorterCompare" OpHelp("if key(P1)!=trim(r[P3],P4) goto P2"), - /* 123 */ "SorterData" OpHelp("r[P2]=data"), - /* 124 */ "RowData" OpHelp("r[P2]=data"), - /* 125 */ "Rowid" OpHelp("r[P2]=rowid"), - /* 126 */ "NullRow" OpHelp(""), - /* 127 */ "SeekEnd" OpHelp(""), - /* 128 */ "SorterInsert" OpHelp("key=r[P2]"), - /* 129 */ "IdxInsert" OpHelp("key=r[P2]"), - /* 130 */ "IdxDelete" OpHelp("key=r[P2@P3]"), - /* 131 */ "DeferredSeek" OpHelp("Move P3 to P1.rowid if needed"), - /* 132 */ "IdxRowid" OpHelp("r[P2]=rowid"), - /* 133 */ "Destroy" OpHelp(""), - /* 134 */ "Real" OpHelp("r[P2]=P4"), - /* 135 */ "Clear" OpHelp(""), - /* 136 */ "ResetSorter" OpHelp(""), - /* 137 */ "CreateBtree" OpHelp("r[P2]=root iDb=P1 flags=P3"), - /* 138 */ "SqlExec" OpHelp(""), - /* 139 */ "ParseSchema" OpHelp(""), - /* 140 */ "LoadAnalysis" OpHelp(""), - /* 141 */ "DropTable" OpHelp(""), - /* 142 */ "DropIndex" OpHelp(""), - /* 143 */ "DropTrigger" OpHelp(""), - /* 144 */ "IntegrityCk" OpHelp(""), - /* 145 */ "RowSetAdd" OpHelp("rowset(P1)=r[P2]"), - /* 146 */ "Param" OpHelp(""), - /* 147 */ "FkCounter" OpHelp("fkctr[P1]+=P2"), - /* 148 */ "MemMax" OpHelp("r[P1]=max(r[P1],r[P2])"), - /* 149 */ "OffsetLimit" OpHelp("if r[P1]>0 then r[P2]=r[P1]+max(0,r[P3]) else r[P2]=(-1)"), - /* 150 */ "AggStep0" OpHelp("accum=r[P3] step(r[P2@P5])"), - /* 151 */ "AggStep" OpHelp("accum=r[P3] step(r[P2@P5])"), - /* 152 */ "AggFinal" OpHelp("accum=r[P1] N=P2"), - /* 153 */ "Expire" OpHelp(""), - /* 154 */ "TableLock" OpHelp("iDb=P1 root=P2 write=P3"), - /* 155 */ "VBegin" OpHelp(""), - /* 156 */ "VCreate" OpHelp(""), - /* 157 */ "VDestroy" OpHelp(""), - /* 158 */ "VOpen" OpHelp(""), - /* 159 */ "VColumn" OpHelp("r[P3]=vcolumn(P2)"), - /* 160 */ "VRename" OpHelp(""), - /* 161 */ "Pagecount" OpHelp(""), - /* 162 */ "MaxPgcnt" OpHelp(""), - /* 163 */ "PureFunc0" OpHelp(""), - /* 164 */ "Function0" OpHelp("r[P3]=func(r[P2@P5])"), - /* 165 */ "PureFunc" OpHelp(""), - /* 166 */ "Function" OpHelp("r[P3]=func(r[P2@P5])"), - /* 167 */ "Trace" OpHelp(""), - /* 168 */ "CursorHint" OpHelp(""), - /* 169 */ "Noop" OpHelp(""), - /* 170 */ "Explain" OpHelp(""), - /* 171 */ "Abortable" OpHelp(""), + /* 112 */ "BitNot" OpHelp("r[P2]= ~r[P1]"), + /* 113 */ "SequenceTest" OpHelp("if( cursor[P1].ctr++ ) pc = P2"), + /* 114 */ "OpenPseudo" OpHelp("P3 columns in r[P2]"), + /* 115 */ "String8" OpHelp("r[P2]='P4'"), + /* 116 */ "Close" OpHelp(""), + /* 117 */ "ColumnsUsed" OpHelp(""), + /* 118 */ "SeekHit" OpHelp("seekHit=P2"), + /* 119 */ "Sequence" OpHelp("r[P2]=cursor[P1].ctr++"), + /* 120 */ "NewRowid" OpHelp("r[P2]=rowid"), + /* 121 */ "Insert" OpHelp("intkey=r[P3] data=r[P2]"), + /* 122 */ "Delete" OpHelp(""), + /* 123 */ "ResetCount" OpHelp(""), + /* 124 */ "SorterCompare" OpHelp("if key(P1)!=trim(r[P3],P4) goto P2"), + /* 125 */ "SorterData" OpHelp("r[P2]=data"), + /* 126 */ "RowData" OpHelp("r[P2]=data"), + /* 127 */ "Rowid" OpHelp("r[P2]=rowid"), + /* 128 */ "NullRow" OpHelp(""), + /* 129 */ "SeekEnd" OpHelp(""), + /* 130 */ "IdxInsert" OpHelp("key=r[P2]"), + /* 131 */ "SorterInsert" OpHelp("key=r[P2]"), + /* 132 */ "IdxDelete" OpHelp("key=r[P2@P3]"), + /* 133 */ "DeferredSeek" OpHelp("Move P3 to P1.rowid if needed"), + /* 134 */ "IdxRowid" OpHelp("r[P2]=rowid"), + /* 135 */ "FinishSeek" OpHelp(""), + /* 136 */ "Destroy" OpHelp(""), + /* 137 */ "Clear" OpHelp(""), + /* 138 */ "ResetSorter" OpHelp(""), + /* 139 */ "CreateBtree" OpHelp("r[P2]=root iDb=P1 flags=P3"), + /* 140 */ "SqlExec" OpHelp(""), + /* 141 */ "ParseSchema" OpHelp(""), + /* 142 */ "LoadAnalysis" OpHelp(""), + /* 143 */ "DropTable" OpHelp(""), + /* 144 */ "DropIndex" OpHelp(""), + /* 145 */ "DropTrigger" OpHelp(""), + /* 146 */ "IntegrityCk" OpHelp(""), + /* 147 */ "RowSetAdd" OpHelp("rowset(P1)=r[P2]"), + /* 148 */ "Param" OpHelp(""), + /* 149 */ "FkCounter" OpHelp("fkctr[P1]+=P2"), + /* 150 */ "Real" OpHelp("r[P2]=P4"), + /* 151 */ "MemMax" OpHelp("r[P1]=max(r[P1],r[P2])"), + /* 152 */ "OffsetLimit" OpHelp("if r[P1]>0 then r[P2]=r[P1]+max(0,r[P3]) else r[P2]=(-1)"), + /* 153 */ "AggInverse" OpHelp("accum=r[P3] inverse(r[P2@P5])"), + /* 154 */ "AggStep" OpHelp("accum=r[P3] step(r[P2@P5])"), + /* 155 */ "AggStep1" OpHelp("accum=r[P3] step(r[P2@P5])"), + /* 156 */ "AggValue" OpHelp("r[P3]=value N=P2"), + /* 157 */ "AggFinal" OpHelp("accum=r[P1] N=P2"), + /* 158 */ "Expire" OpHelp(""), + /* 159 */ "CursorLock" OpHelp(""), + /* 160 */ "CursorUnlock" OpHelp(""), + /* 161 */ "TableLock" OpHelp("iDb=P1 root=P2 write=P3"), + /* 162 */ "VBegin" OpHelp(""), + /* 163 */ "VCreate" OpHelp(""), + /* 164 */ "VDestroy" OpHelp(""), + /* 165 */ "VOpen" OpHelp(""), + /* 166 */ "VColumn" OpHelp("r[P3]=vcolumn(P2)"), + /* 167 */ "VRename" OpHelp(""), + /* 168 */ "Pagecount" OpHelp(""), + /* 169 */ "MaxPgcnt" OpHelp(""), + /* 170 */ "Trace" OpHelp(""), + /* 171 */ "CursorHint" OpHelp(""), + /* 172 */ "ReleaseReg" OpHelp("release r[P1@P2] mask P3"), + /* 173 */ "Noop" OpHelp(""), + /* 174 */ "Explain" OpHelp(""), + /* 175 */ "Abortable" OpHelp(""), }; return azName[i]; } @@ -31568,13 +33359,29 @@ SQLITE_PRIVATE const char *sqlite3OpcodeName(int i){ # include #endif /* SQLITE_ENABLE_LOCKING_STYLE */ -#if defined(__APPLE__) && ((__MAC_OS_X_VERSION_MIN_REQUIRED > 1050) || \ - (__IPHONE_OS_VERSION_MIN_REQUIRED > 2000)) -# if (!defined(TARGET_OS_EMBEDDED) || (TARGET_OS_EMBEDDED==0)) \ - && (!defined(TARGET_IPHONE_SIMULATOR) || (TARGET_IPHONE_SIMULATOR==0)) -# define HAVE_GETHOSTUUID 1 -# else -# warning "gethostuuid() is disabled." +/* +** Try to determine if gethostuuid() is available based on standard +** macros. This might sometimes compute the wrong value for some +** obscure platforms. For those cases, simply compile with one of +** the following: +** +** -DHAVE_GETHOSTUUID=0 +** -DHAVE_GETHOSTUUID=1 +** +** None if this matters except when building on Apple products with +** -DSQLITE_ENABLE_LOCKING_STYLE. +*/ +#ifndef HAVE_GETHOSTUUID +# define HAVE_GETHOSTUUID 0 +# if defined(__APPLE__) && ((__MAC_OS_X_VERSION_MIN_REQUIRED > 1050) || \ + (__IPHONE_OS_VERSION_MIN_REQUIRED > 2000)) +# if (!defined(TARGET_OS_EMBEDDED) || (TARGET_OS_EMBEDDED==0)) \ + && (!defined(TARGET_IPHONE_SIMULATOR) || (TARGET_IPHONE_SIMULATOR==0)) +# undef HAVE_GETHOSTUUID +# define HAVE_GETHOSTUUID 1 +# else +# warning "gethostuuid() is disabled." +# endif # endif #endif @@ -31599,12 +33406,10 @@ SQLITE_PRIVATE const char *sqlite3OpcodeName(int i){ #define SQLITE_FSFLAGS_IS_MSDOS 0x1 /* -** If we are to be thread-safe, include the pthreads header and define -** the SQLITE_UNIX_THREADS macro. +** If we are to be thread-safe, include the pthreads header. */ #if SQLITE_THREADSAFE /* # include */ -# define SQLITE_UNIX_THREADS 1 #endif /* @@ -31803,7 +33608,7 @@ static pid_t randomnessPid = 0; ****************************************************************************** ** ** This file contains inline asm code for retrieving "high-performance" -** counters for x86 class CPUs. +** counters for x86 and x86_64 class CPUs. */ #ifndef SQLITE_HWTIME_H #define SQLITE_HWTIME_H @@ -31814,8 +33619,9 @@ static pid_t randomnessPid = 0; ** processor and returns that value. This can be used for high-res ** profiling. */ -#if (defined(__GNUC__) || defined(_MSC_VER)) && \ - (defined(i386) || defined(__i386__) || defined(_M_IX86)) +#if !defined(__STRICT_ANSI__) && \ + (defined(__GNUC__) || defined(_MSC_VER)) && \ + (defined(i386) || defined(__i386__) || defined(_M_IX86)) #if defined(__GNUC__) @@ -31836,7 +33642,7 @@ static pid_t randomnessPid = 0; #endif -#elif (defined(__GNUC__) && defined(__x86_64__)) +#elif !defined(__STRICT_ANSI__) && (defined(__GNUC__) && defined(__x86_64__)) __inline__ sqlite_uint64 sqlite3Hwtime(void){ unsigned long val; @@ -31844,7 +33650,7 @@ static pid_t randomnessPid = 0; return val; } -#elif (defined(__GNUC__) && defined(__ppc__)) +#elif !defined(__STRICT_ANSI__) && (defined(__GNUC__) && defined(__ppc__)) __inline__ sqlite_uint64 sqlite3Hwtime(void){ unsigned long long retval; @@ -31861,14 +33667,13 @@ static pid_t randomnessPid = 0; #else - #error Need implementation of sqlite3Hwtime() for your platform. - /* - ** To compile without implementing sqlite3Hwtime() for your platform, - ** you can remove the above #error and use the following - ** stub function. You will lose timing support for many - ** of the debugging and testing utilities, but it should at - ** least compile and run. + ** asm() is needed for hardware timing support. Without asm(), + ** disable the sqlite3Hwtime() routine. + ** + ** sqlite3Hwtime() is only used for some obscure debugging + ** and analysis configurations, not in any deliverable, so this + ** should not be a great loss. */ SQLITE_PRIVATE sqlite_uint64 sqlite3Hwtime(void){ return ((sqlite_uint64)0); } @@ -32182,11 +33987,16 @@ static struct unix_syscall { #define osLstat ((int(*)(const char*,struct stat*))aSyscall[27].pCurrent) #if defined(__linux__) && defined(SQLITE_ENABLE_BATCH_ATOMIC_WRITE) +# ifdef __ANDROID__ + { "ioctl", (sqlite3_syscall_ptr)(int(*)(int, int, ...))ioctl, 0 }, +#define osIoctl ((int(*)(int,int,...))aSyscall[28].pCurrent) +# else { "ioctl", (sqlite3_syscall_ptr)ioctl, 0 }, +#define osIoctl ((int(*)(int,unsigned long,...))aSyscall[28].pCurrent) +# endif #else { "ioctl", (sqlite3_syscall_ptr)0, 0 }, #endif -#define osIoctl ((int(*)(int,int,...))aSyscall[28].pCurrent) }; /* End of the overrideable system calls */ @@ -32331,7 +34141,7 @@ static int robust_open(const char *z, int f, mode_t m){ sqlite3_log(SQLITE_WARNING, "attempt to open \"%s\" as file descriptor %d", z, fd); fd = -1; - if( osOpen("/dev/null", f, m)<0 ) break; + if( osOpen("/dev/null", O_RDONLY, m)<0 ) break; } if( fd>=0 ){ if( m!=0 ){ @@ -32363,12 +34173,25 @@ static int robust_open(const char *z, int f, mode_t m){ ** unixEnterMutex() ** assert( unixMutexHeld() ); ** unixEnterLeave() +** +** To prevent deadlock, the global unixBigLock must must be acquired +** before the unixInodeInfo.pLockMutex mutex, if both are held. It is +** OK to get the pLockMutex without holding unixBigLock first, but if +** that happens, the unixBigLock mutex must not be acquired until after +** pLockMutex is released. +** +** OK: enter(unixBigLock), enter(pLockInfo) +** OK: enter(unixBigLock) +** OK: enter(pLockInfo) +** ERROR: enter(pLockInfo), enter(unixBigLock) */ static sqlite3_mutex *unixBigLock = 0; static void unixEnterMutex(void){ + assert( sqlite3_mutex_notheld(unixBigLock) ); /* Not a recursive mutex */ sqlite3_mutex_enter(unixBigLock); } static void unixLeaveMutex(void){ + assert( sqlite3_mutex_held(unixBigLock) ); sqlite3_mutex_leave(unixBigLock); } #ifdef SQLITE_DEBUG @@ -32763,22 +34586,39 @@ struct unixFileId { /* ** An instance of the following structure is allocated for each open -** inode. Or, on LinuxThreads, there is one of these structures for -** each inode opened by each thread. +** inode. ** ** A single inode can have multiple file descriptors, so each unixFile ** structure contains a pointer to an instance of this object and this ** object keeps a count of the number of unixFile pointing to it. +** +** Mutex rules: +** +** (1) Only the pLockMutex mutex must be held in order to read or write +** any of the locking fields: +** nShared, nLock, eFileLock, bProcessLock, pUnused +** +** (2) When nRef>0, then the following fields are unchanging and can +** be read (but not written) without holding any mutex: +** fileId, pLockMutex +** +** (3) With the exceptions above, all the fields may only be read +** or written while holding the global unixBigLock mutex. +** +** Deadlock prevention: The global unixBigLock mutex may not +** be acquired while holding the pLockMutex mutex. If both unixBigLock +** and pLockMutex are needed, then unixBigLock must be acquired first. */ struct unixInodeInfo { struct unixFileId fileId; /* The lookup key */ - int nShared; /* Number of SHARED locks held */ - unsigned char eFileLock; /* One of SHARED_LOCK, RESERVED_LOCK etc. */ - unsigned char bProcessLock; /* An exclusive process lock is held */ + sqlite3_mutex *pLockMutex; /* Hold this mutex for... */ + int nShared; /* Number of SHARED locks held */ + int nLock; /* Number of outstanding file locks */ + unsigned char eFileLock; /* One of SHARED_LOCK, RESERVED_LOCK etc. */ + unsigned char bProcessLock; /* An exclusive process lock is held */ + UnixUnusedFd *pUnused; /* Unused file descriptors to close */ int nRef; /* Number of pointers to this structure */ unixShmNode *pShmNode; /* Shared memory associated with this inode */ - int nLock; /* Number of outstanding file locks */ - UnixUnusedFd *pUnused; /* Unused file descriptors to close */ unixInodeInfo *pNext; /* List of all unixInodeInfo objects */ unixInodeInfo *pPrev; /* .... doubly linked */ #if SQLITE_ENABLE_LOCKING_STYLE @@ -32792,9 +34632,26 @@ struct unixInodeInfo { /* ** A lists of all unixInodeInfo objects. +** +** Must hold unixBigLock in order to read or write this variable. */ static unixInodeInfo *inodeList = 0; /* All unixInodeInfo objects */ -static unsigned int nUnusedFd = 0; /* Total unused file descriptors */ + +#ifdef SQLITE_DEBUG +/* +** True if the inode mutex (on the unixFile.pFileMutex field) is held, or not. +** This routine is used only within assert() to help verify correct mutex +** usage. +*/ +int unixFileMutexHeld(unixFile *pFile){ + assert( pFile->pInode ); + return sqlite3_mutex_held(pFile->pInode->pLockMutex); +} +int unixFileMutexNotheld(unixFile *pFile){ + assert( pFile->pInode ); + return sqlite3_mutex_notheld(pFile->pInode->pLockMutex); +} +#endif /* ** @@ -32900,11 +34757,11 @@ static void closePendingFds(unixFile *pFile){ unixInodeInfo *pInode = pFile->pInode; UnixUnusedFd *p; UnixUnusedFd *pNext; + assert( unixFileMutexHeld(pFile) ); for(p=pInode->pUnused; p; p=pNext){ pNext = p->pNext; robust_close(pFile, p->fd, __LINE__); sqlite3_free(p); - nUnusedFd--; } pInode->pUnused = 0; } @@ -32912,17 +34769,20 @@ static void closePendingFds(unixFile *pFile){ /* ** Release a unixInodeInfo structure previously allocated by findInodeInfo(). ** -** The mutex entered using the unixEnterMutex() function must be held -** when this function is called. +** The global mutex must be held when this routine is called, but the mutex +** on the inode being deleted must NOT be held. */ static void releaseInodeInfo(unixFile *pFile){ unixInodeInfo *pInode = pFile->pInode; assert( unixMutexHeld() ); + assert( unixFileMutexNotheld(pFile) ); if( ALWAYS(pInode) ){ pInode->nRef--; if( pInode->nRef==0 ){ assert( pInode->pShmNode==0 ); + sqlite3_mutex_enter(pInode->pLockMutex); closePendingFds(pFile); + sqlite3_mutex_leave(pInode->pLockMutex); if( pInode->pPrev ){ assert( pInode->pPrev->pNext==pInode ); pInode->pPrev->pNext = pInode->pNext; @@ -32934,10 +34794,10 @@ static void releaseInodeInfo(unixFile *pFile){ assert( pInode->pNext->pPrev==pInode ); pInode->pNext->pPrev = pInode->pPrev; } + sqlite3_mutex_free(pInode->pLockMutex); sqlite3_free(pInode); } } - assert( inodeList!=0 || nUnusedFd==0 ); } /* @@ -32945,8 +34805,7 @@ static void releaseInodeInfo(unixFile *pFile){ ** describes that file descriptor. Create a new one if necessary. The ** return value might be uninitialized if an error occurs. ** -** The mutex entered using the unixEnterMutex() function must be held -** when this function is called. +** The global mutex must held when calling this routine. ** ** Return an appropriate error code. */ @@ -33007,7 +34866,7 @@ static int findInodeInfo( #else fileId.ino = (u64)statbuf.st_ino; #endif - assert( inodeList!=0 || nUnusedFd==0 ); + assert( unixMutexHeld() ); pInode = inodeList; while( pInode && memcmp(&fileId, &pInode->fileId, sizeof(fileId)) ){ pInode = pInode->pNext; @@ -33019,7 +34878,15 @@ static int findInodeInfo( } memset(pInode, 0, sizeof(*pInode)); memcpy(&pInode->fileId, &fileId, sizeof(fileId)); + if( sqlite3GlobalConfig.bCoreMutex ){ + pInode->pLockMutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); + if( pInode->pLockMutex==0 ){ + sqlite3_free(pInode); + return SQLITE_NOMEM_BKPT; + } + } pInode->nRef = 1; + assert( unixMutexHeld() ); pInode->pNext = inodeList; pInode->pPrev = 0; if( inodeList ) inodeList->pPrev = pInode; @@ -33097,7 +34964,7 @@ static int unixCheckReservedLock(sqlite3_file *id, int *pResOut){ assert( pFile ); assert( pFile->eFileLock<=SHARED_LOCK ); - unixEnterMutex(); /* Because pFile->pInode is shared across threads */ + sqlite3_mutex_enter(pFile->pInode->pLockMutex); /* Check if a thread in this process holds such a lock */ if( pFile->pInode->eFileLock>SHARED_LOCK ){ @@ -33122,7 +34989,7 @@ static int unixCheckReservedLock(sqlite3_file *id, int *pResOut){ } #endif - unixLeaveMutex(); + sqlite3_mutex_leave(pFile->pInode->pLockMutex); OSTRACE(("TEST WR-LOCK %d %d %d (unix)\n", pFile->h, rc, reserved)); *pResOut = reserved; @@ -33150,8 +35017,9 @@ static int osSetPosixAdvisoryLock( struct flock *pLock, /* The description of the lock */ unixFile *pFile /* Structure holding timeout value */ ){ + int tm = pFile->iBusyTimeout; int rc = osFcntl(h,F_SETLK,pLock); - while( rc<0 && pFile->iBusyTimeout>0 ){ + while( rc<0 && tm>0 ){ /* On systems that support some kind of blocking file lock with a timeout, ** make appropriate changes here to invoke that blocking file lock. On ** generic posix, however, there is no such API. So we simply try the @@ -33159,7 +35027,7 @@ static int osSetPosixAdvisoryLock( ** the lock is obtained. */ usleep(1000); rc = osFcntl(h,F_SETLK,pLock); - pFile->iBusyTimeout--; + tm--; } return rc; } @@ -33188,8 +35056,8 @@ static int osSetPosixAdvisoryLock( static int unixFileLock(unixFile *pFile, struct flock *pLock){ int rc; unixInodeInfo *pInode = pFile->pInode; - assert( unixMutexHeld() ); assert( pInode!=0 ); + assert( sqlite3_mutex_held(pInode->pLockMutex) ); if( (pFile->ctrlFlags & (UNIXFILE_EXCL|UNIXFILE_RDONLY))==UNIXFILE_EXCL ){ if( pInode->bProcessLock==0 ){ struct flock lock; @@ -33308,8 +35176,8 @@ static int unixLock(sqlite3_file *id, int eFileLock){ /* This mutex is needed because pFile->pInode is shared across threads */ - unixEnterMutex(); pInode = pFile->pInode; + sqlite3_mutex_enter(pInode->pLockMutex); /* If some thread using this PID has a lock via a different unixFile* ** handle that precludes the requested lock, return BUSY. @@ -33452,7 +35320,7 @@ static int unixLock(sqlite3_file *id, int eFileLock){ } end_lock: - unixLeaveMutex(); + sqlite3_mutex_leave(pInode->pLockMutex); OSTRACE(("LOCK %d %s %s (unix)\n", pFile->h, azFileLock(eFileLock), rc==SQLITE_OK ? "ok" : "failed")); return rc; @@ -33465,11 +35333,11 @@ static int unixLock(sqlite3_file *id, int eFileLock){ static void setPendingFd(unixFile *pFile){ unixInodeInfo *pInode = pFile->pInode; UnixUnusedFd *p = pFile->pPreallocatedUnused; + assert( unixFileMutexHeld(pFile) ); p->pNext = pInode->pUnused; pInode->pUnused = p; pFile->h = -1; pFile->pPreallocatedUnused = 0; - nUnusedFd++; } /* @@ -33500,8 +35368,8 @@ static int posixUnlock(sqlite3_file *id, int eFileLock, int handleNFSUnlock){ if( pFile->eFileLock<=eFileLock ){ return SQLITE_OK; } - unixEnterMutex(); pInode = pFile->pInode; + sqlite3_mutex_enter(pInode->pLockMutex); assert( pInode->nShared!=0 ); if( pFile->eFileLock>SHARED_LOCK ){ assert( pInode->eFileLock==pFile->eFileLock ); @@ -33627,14 +35495,14 @@ static int posixUnlock(sqlite3_file *id, int eFileLock, int handleNFSUnlock){ */ pInode->nLock--; assert( pInode->nLock>=0 ); - if( pInode->nLock==0 ){ - closePendingFds(pFile); - } + if( pInode->nLock==0 ) closePendingFds(pFile); } end_unlock: - unixLeaveMutex(); - if( rc==SQLITE_OK ) pFile->eFileLock = eFileLock; + sqlite3_mutex_leave(pInode->pLockMutex); + if( rc==SQLITE_OK ){ + pFile->eFileLock = eFileLock; + } return rc; } @@ -33705,15 +35573,20 @@ static int closeUnixFile(sqlite3_file *id){ static int unixClose(sqlite3_file *id){ int rc = SQLITE_OK; unixFile *pFile = (unixFile *)id; + unixInodeInfo *pInode = pFile->pInode; + + assert( pInode!=0 ); verifyDbFile(pFile); unixUnlock(id, NO_LOCK); + assert( unixFileMutexNotheld(pFile) ); unixEnterMutex(); /* unixFile.pInode is always valid here. Otherwise, a different close ** routine (e.g. nolockClose()) would be called instead. */ assert( pFile->pInode->nLock>0 || pFile->pInode->bProcessLock==0 ); - if( ALWAYS(pFile->pInode) && pFile->pInode->nLock ){ + sqlite3_mutex_enter(pInode->pLockMutex); + if( pInode->nLock ){ /* If there are outstanding locks, do not actually close the file just ** yet because that would clear those locks. Instead, add the file ** descriptor to pInode->pUnused list. It will be automatically closed @@ -33721,6 +35594,7 @@ static int unixClose(sqlite3_file *id){ */ setPendingFd(pFile); } + sqlite3_mutex_leave(pInode->pLockMutex); releaseInodeInfo(pFile); rc = closeUnixFile(id); unixLeaveMutex(); @@ -34318,6 +36192,7 @@ static int semXClose(sqlite3_file *id) { unixFile *pFile = (unixFile*)id; semXUnlock(id, NO_LOCK); assert( pFile ); + assert( unixFileMutexNotheld(pFile) ); unixEnterMutex(); releaseInodeInfo(pFile); unixLeaveMutex(); @@ -34432,8 +36307,7 @@ static int afpCheckReservedLock(sqlite3_file *id, int *pResOut){ *pResOut = 1; return SQLITE_OK; } - unixEnterMutex(); /* Because pFile->pInode is shared across threads */ - + sqlite3_mutex_enter(pFile->pInode->pLockMutex); /* Check if a thread in this process holds such a lock */ if( pFile->pInode->eFileLock>SHARED_LOCK ){ reserved = 1; @@ -34457,7 +36331,7 @@ static int afpCheckReservedLock(sqlite3_file *id, int *pResOut){ } } - unixLeaveMutex(); + sqlite3_mutex_leave(pFile->pInode->pLockMutex); OSTRACE(("TEST WR-LOCK %d %d %d (afp)\n", pFile->h, rc, reserved)); *pResOut = reserved; @@ -34520,8 +36394,8 @@ static int afpLock(sqlite3_file *id, int eFileLock){ /* This mutex is needed because pFile->pInode is shared across threads */ - unixEnterMutex(); pInode = pFile->pInode; + sqlite3_mutex_enter(pInode->pLockMutex); /* If some thread using this PID has a lock via a different unixFile* ** handle that precludes the requested lock, return BUSY. @@ -34657,7 +36531,7 @@ static int afpLock(sqlite3_file *id, int eFileLock){ } afp_end_lock: - unixLeaveMutex(); + sqlite3_mutex_leave(pInode->pLockMutex); OSTRACE(("LOCK %d %s %s (afp)\n", pFile->h, azFileLock(eFileLock), rc==SQLITE_OK ? "ok" : "failed")); return rc; @@ -34689,8 +36563,8 @@ static int afpUnlock(sqlite3_file *id, int eFileLock) { if( pFile->eFileLock<=eFileLock ){ return SQLITE_OK; } - unixEnterMutex(); pInode = pFile->pInode; + sqlite3_mutex_enter(pInode->pLockMutex); assert( pInode->nShared!=0 ); if( pFile->eFileLock>SHARED_LOCK ){ assert( pInode->eFileLock==pFile->eFileLock ); @@ -34759,14 +36633,14 @@ static int afpUnlock(sqlite3_file *id, int eFileLock) { if( rc==SQLITE_OK ){ pInode->nLock--; assert( pInode->nLock>=0 ); - if( pInode->nLock==0 ){ - closePendingFds(pFile); - } + if( pInode->nLock==0 ) closePendingFds(pFile); } } - unixLeaveMutex(); - if( rc==SQLITE_OK ) pFile->eFileLock = eFileLock; + sqlite3_mutex_leave(pInode->pLockMutex); + if( rc==SQLITE_OK ){ + pFile->eFileLock = eFileLock; + } return rc; } @@ -34778,14 +36652,20 @@ static int afpClose(sqlite3_file *id) { unixFile *pFile = (unixFile*)id; assert( id!=0 ); afpUnlock(id, NO_LOCK); + assert( unixFileMutexNotheld(pFile) ); unixEnterMutex(); - if( pFile->pInode && pFile->pInode->nLock ){ - /* If there are outstanding locks, do not actually close the file just - ** yet because that would clear those locks. Instead, add the file - ** descriptor to pInode->aPending. It will be automatically closed when - ** the last lock is cleared. - */ - setPendingFd(pFile); + if( pFile->pInode ){ + unixInodeInfo *pInode = pFile->pInode; + sqlite3_mutex_enter(pInode->pLockMutex); + if( pInode->nLock ){ + /* If there are outstanding locks, do not actually close the file just + ** yet because that would clear those locks. Instead, add the file + ** descriptor to pInode->aPending. It will be automatically closed when + ** the last lock is cleared. + */ + setPendingFd(pFile); + } + sqlite3_mutex_leave(pInode->pLockMutex); } releaseInodeInfo(pFile); sqlite3_free(pFile->lockingContext); @@ -35568,7 +37448,9 @@ static int unixFileControl(sqlite3_file *id, int op, void *pArg){ } #ifdef SQLITE_ENABLE_SETLK_TIMEOUT case SQLITE_FCNTL_LOCK_TIMEOUT: { + int iOld = pFile->iBusyTimeout; pFile->iBusyTimeout = *(int*)pArg; + *(int*)pArg = iOld; return SQLITE_OK; } #endif @@ -35805,18 +37687,18 @@ static int unixGetpagesize(void){ ** ** The following fields are read-only after the object is created: ** -** fid +** hShm ** zFilename ** -** Either unixShmNode.mutex must be held or unixShmNode.nRef==0 and +** Either unixShmNode.pShmMutex must be held or unixShmNode.nRef==0 and ** unixMutexHeld() is true when reading or writing any other field ** in this structure. */ struct unixShmNode { unixInodeInfo *pInode; /* unixInodeInfo that owns this SHM node */ - sqlite3_mutex *mutex; /* Mutex to access this object */ + sqlite3_mutex *pShmMutex; /* Mutex to access this object */ char *zFilename; /* Name of the mmapped file */ - int h; /* Open file descriptor */ + int hShm; /* Open file descriptor */ int szRegion; /* Size of shared-memory regions */ u16 nRegion; /* Size of array apRegion */ u8 isReadonly; /* True if read-only */ @@ -35838,16 +37720,16 @@ struct unixShmNode { ** The following fields are initialized when this object is created and ** are read-only thereafter: ** -** unixShm.pFile +** unixShm.pShmNode ** unixShm.id ** -** All other fields are read/write. The unixShm.pFile->mutex must be held -** while accessing any read/write fields. +** All other fields are read/write. The unixShm.pShmNode->pShmMutex must +** be held while accessing any read/write fields. */ struct unixShm { unixShmNode *pShmNode; /* The underlying unixShmNode object */ unixShm *pNext; /* Next unixShm with the same unixShmNode */ - u8 hasMutex; /* True if holding the unixShmNode mutex */ + u8 hasMutex; /* True if holding the unixShmNode->pShmMutex */ u8 id; /* Id of this connection within its unixShmNode */ u16 sharedMask; /* Mask of shared locks held */ u16 exclMask; /* Mask of exclusive locks held */ @@ -35877,7 +37759,8 @@ static int unixShmSystemLock( /* Access to the unixShmNode object is serialized by the caller */ pShmNode = pFile->pInode->pShmNode; - assert( pShmNode->nRef==0 || sqlite3_mutex_held(pShmNode->mutex) ); + assert( pShmNode->nRef==0 || sqlite3_mutex_held(pShmNode->pShmMutex) ); + assert( pShmNode->nRef>0 || unixMutexHeld() ); /* Shared locks never span more than one byte */ assert( n==1 || lockType!=F_RDLCK ); @@ -35885,14 +37768,21 @@ static int unixShmSystemLock( /* Locks are within range */ assert( n>=1 && n<=SQLITE_SHM_NLOCK ); - if( pShmNode->h>=0 ){ + if( pShmNode->hShm>=0 ){ + int res; /* Initialize the locking parameters */ f.l_type = lockType; f.l_whence = SEEK_SET; f.l_start = ofst; f.l_len = n; - rc = osSetPosixAdvisoryLock(pShmNode->h, &f, pFile); - rc = (rc!=(-1)) ? SQLITE_OK : SQLITE_BUSY; + res = osSetPosixAdvisoryLock(pShmNode->hShm, &f, pFile); + if( res==-1 ){ +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + rc = (pFile->iBusyTimeout ? SQLITE_BUSY_TIMEOUT : SQLITE_BUSY); +#else + rc = SQLITE_BUSY; +#endif + } } /* Update the global lock state and do debug tracing */ @@ -35963,18 +37853,18 @@ static void unixShmPurge(unixFile *pFd){ int nShmPerMap = unixShmRegionPerMap(); int i; assert( p->pInode==pFd->pInode ); - sqlite3_mutex_free(p->mutex); + sqlite3_mutex_free(p->pShmMutex); for(i=0; inRegion; i+=nShmPerMap){ - if( p->h>=0 ){ + if( p->hShm>=0 ){ osMunmap(p->apRegion[i], p->szRegion); }else{ sqlite3_free(p->apRegion[i]); } } sqlite3_free(p->apRegion); - if( p->h>=0 ){ - robust_close(pFd, p->h, __LINE__); - p->h = -1; + if( p->hShm>=0 ){ + robust_close(pFd, p->hShm, __LINE__); + p->hShm = -1; } p->pInode->pShmNode = 0; sqlite3_free(p); @@ -36016,7 +37906,7 @@ static int unixLockSharedMemory(unixFile *pDbFd, unixShmNode *pShmNode){ lock.l_start = UNIX_SHM_DMS; lock.l_len = 1; lock.l_type = F_WRLCK; - if( osFcntl(pShmNode->h, F_GETLK, &lock)!=0 ) { + if( osFcntl(pShmNode->hShm, F_GETLK, &lock)!=0 ) { rc = SQLITE_IOERR_LOCK; }else if( lock.l_type==F_UNLCK ){ if( pShmNode->isReadonly ){ @@ -36024,7 +37914,12 @@ static int unixLockSharedMemory(unixFile *pDbFd, unixShmNode *pShmNode){ rc = SQLITE_READONLY_CANTINIT; }else{ rc = unixShmSystemLock(pDbFd, F_WRLCK, UNIX_SHM_DMS, 1); - if( rc==SQLITE_OK && robust_ftruncate(pShmNode->h, 0) ){ + /* The first connection to attach must truncate the -shm file. We + ** truncate to 3 bytes (an arbitrary small number, less than the + ** -shm header size) rather than 0 as a system debugging aid, to + ** help detect if a -shm file truncation is legitimate or is the work + ** or a rogue process. */ + if( rc==SQLITE_OK && robust_ftruncate(pShmNode->hShm, 3) ){ rc = unixLogError(SQLITE_IOERR_SHMOPEN,"ftruncate",pShmNode->zFilename); } } @@ -36091,6 +37986,7 @@ static int unixOpenSharedMemory(unixFile *pDbFd){ /* Check to see if a unixShmNode object already exists. Reuse an existing ** one if present. Create a new one if necessary. */ + assert( unixFileMutexNotheld(pDbFd) ); unixEnterMutex(); pInode = pDbFd->pInode; pShmNode = pInode->pShmNode; @@ -36129,12 +38025,12 @@ static int unixOpenSharedMemory(unixFile *pDbFd){ sqlite3_snprintf(nShmFilename, zShm, "%s-shm", zBasePath); sqlite3FileSuffix3(pDbFd->zPath, zShm); #endif - pShmNode->h = -1; + pShmNode->hShm = -1; pDbFd->pInode->pShmNode = pShmNode; pShmNode->pInode = pDbFd->pInode; if( sqlite3GlobalConfig.bCoreMutex ){ - pShmNode->mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); - if( pShmNode->mutex==0 ){ + pShmNode->pShmMutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); + if( pShmNode->pShmMutex==0 ){ rc = SQLITE_NOMEM_BKPT; goto shm_open_err; } @@ -36142,11 +38038,13 @@ static int unixOpenSharedMemory(unixFile *pDbFd){ if( pInode->bProcessLock==0 ){ if( 0==sqlite3_uri_boolean(pDbFd->zPath, "readonly_shm", 0) ){ - pShmNode->h = robust_open(zShm, O_RDWR|O_CREAT, (sStat.st_mode&0777)); + pShmNode->hShm = robust_open(zShm, O_RDWR|O_CREAT|O_NOFOLLOW, + (sStat.st_mode&0777)); } - if( pShmNode->h<0 ){ - pShmNode->h = robust_open(zShm, O_RDONLY, (sStat.st_mode&0777)); - if( pShmNode->h<0 ){ + if( pShmNode->hShm<0 ){ + pShmNode->hShm = robust_open(zShm, O_RDONLY|O_NOFOLLOW, + (sStat.st_mode&0777)); + if( pShmNode->hShm<0 ){ rc = unixLogError(SQLITE_CANTOPEN_BKPT, "open", zShm); goto shm_open_err; } @@ -36157,7 +38055,7 @@ static int unixOpenSharedMemory(unixFile *pDbFd){ ** is owned by the same user that owns the original database. Otherwise, ** the original owner will not be able to connect. */ - robustFchown(pShmNode->h, sStat.st_uid, sStat.st_gid); + robustFchown(pShmNode->hShm, sStat.st_uid, sStat.st_gid); rc = unixLockSharedMemory(pDbFd, pShmNode); if( rc!=SQLITE_OK && rc!=SQLITE_READONLY_CANTINIT ) goto shm_open_err; @@ -36177,13 +38075,13 @@ static int unixOpenSharedMemory(unixFile *pDbFd){ ** the cover of the unixEnterMutex() mutex and the pointer from the ** new (struct unixShm) object to the pShmNode has been set. All that is ** left to do is to link the new object into the linked list starting - ** at pShmNode->pFirst. This must be done while holding the pShmNode->mutex - ** mutex. + ** at pShmNode->pFirst. This must be done while holding the + ** pShmNode->pShmMutex. */ - sqlite3_mutex_enter(pShmNode->mutex); + sqlite3_mutex_enter(pShmNode->pShmMutex); p->pNext = pShmNode->pFirst; pShmNode->pFirst = p; - sqlite3_mutex_leave(pShmNode->mutex); + sqlite3_mutex_leave(pShmNode->pShmMutex); return rc; /* Jump here on any error */ @@ -36235,7 +38133,7 @@ static int unixShmMap( p = pDbFd->pShm; pShmNode = p->pShmNode; - sqlite3_mutex_enter(pShmNode->mutex); + sqlite3_mutex_enter(pShmNode->pShmMutex); if( pShmNode->isUnlocked ){ rc = unixLockSharedMemory(pDbFd, pShmNode); if( rc!=SQLITE_OK ) goto shmpage_out; @@ -36243,8 +38141,8 @@ static int unixShmMap( } assert( szRegion==pShmNode->szRegion || pShmNode->nRegion==0 ); assert( pShmNode->pInode==pDbFd->pInode ); - assert( pShmNode->h>=0 || pDbFd->pInode->bProcessLock==1 ); - assert( pShmNode->h<0 || pDbFd->pInode->bProcessLock==0 ); + assert( pShmNode->hShm>=0 || pDbFd->pInode->bProcessLock==1 ); + assert( pShmNode->hShm<0 || pDbFd->pInode->bProcessLock==0 ); /* Minimum number of regions required to be mapped. */ nReqRegion = ((iRegion+nShmPerMap) / nShmPerMap) * nShmPerMap; @@ -36256,12 +38154,12 @@ static int unixShmMap( pShmNode->szRegion = szRegion; - if( pShmNode->h>=0 ){ + if( pShmNode->hShm>=0 ){ /* The requested region is not mapped into this processes address space. ** Check to see if it has been allocated (i.e. if the wal-index file is ** large enough to contain the requested region). */ - if( osFstat(pShmNode->h, &sStat) ){ + if( osFstat(pShmNode->hShm, &sStat) ){ rc = SQLITE_IOERR_SHMSIZE; goto shmpage_out; } @@ -36289,7 +38187,7 @@ static int unixShmMap( assert( (nByte % pgsz)==0 ); for(iPg=(sStat.st_size/pgsz); iPg<(nByte/pgsz); iPg++){ int x = 0; - if( seekAndWriteFd(pShmNode->h, iPg*pgsz + pgsz-1, "", 1, &x)!=1 ){ + if( seekAndWriteFd(pShmNode->hShm, iPg*pgsz + pgsz-1,"",1,&x)!=1 ){ const char *zFile = pShmNode->zFilename; rc = unixLogError(SQLITE_IOERR_SHMSIZE, "write", zFile); goto shmpage_out; @@ -36312,22 +38210,22 @@ static int unixShmMap( int nMap = szRegion*nShmPerMap; int i; void *pMem; - if( pShmNode->h>=0 ){ + if( pShmNode->hShm>=0 ){ pMem = osMmap(0, nMap, pShmNode->isReadonly ? PROT_READ : PROT_READ|PROT_WRITE, - MAP_SHARED, pShmNode->h, szRegion*(i64)pShmNode->nRegion + MAP_SHARED, pShmNode->hShm, szRegion*(i64)pShmNode->nRegion ); if( pMem==MAP_FAILED ){ rc = unixLogError(SQLITE_IOERR_SHMMAP, "mmap", pShmNode->zFilename); goto shmpage_out; } }else{ - pMem = sqlite3_malloc64(szRegion); + pMem = sqlite3_malloc64(nMap); if( pMem==0 ){ rc = SQLITE_NOMEM_BKPT; goto shmpage_out; } - memset(pMem, 0, szRegion); + memset(pMem, 0, nMap); } for(i=0; iisReadonly && rc==SQLITE_OK ) rc = SQLITE_READONLY; - sqlite3_mutex_leave(pShmNode->mutex); + sqlite3_mutex_leave(pShmNode->pShmMutex); return rc; } @@ -36378,12 +38276,31 @@ static int unixShmLock( || flags==(SQLITE_SHM_UNLOCK | SQLITE_SHM_SHARED) || flags==(SQLITE_SHM_UNLOCK | SQLITE_SHM_EXCLUSIVE) ); assert( n==1 || (flags & SQLITE_SHM_EXCLUSIVE)!=0 ); - assert( pShmNode->h>=0 || pDbFd->pInode->bProcessLock==1 ); - assert( pShmNode->h<0 || pDbFd->pInode->bProcessLock==0 ); + assert( pShmNode->hShm>=0 || pDbFd->pInode->bProcessLock==1 ); + assert( pShmNode->hShm<0 || pDbFd->pInode->bProcessLock==0 ); + + /* Check that, if this to be a blocking lock, no locks that occur later + ** in the following list than the lock being obtained are already held: + ** + ** 1. Checkpointer lock (ofst==1). + ** 2. Write lock (ofst==0). + ** 3. Read locks (ofst>=3 && ofstiBusyTimeout==0 || ( + (ofst!=2) /* not RECOVER */ + && (ofst!=1 || (p->exclMask|p->sharedMask)==0) + && (ofst!=0 || (p->exclMask|p->sharedMask)<3) + && (ofst<3 || (p->exclMask|p->sharedMask)<(1<1 || mask==(1<mutex); + sqlite3_mutex_enter(pShmNode->pShmMutex); if( flags & SQLITE_SHM_UNLOCK ){ u16 allMask = 0; /* Mask of locks held by siblings */ @@ -36456,7 +38373,7 @@ static int unixShmLock( } } } - sqlite3_mutex_leave(pShmNode->mutex); + sqlite3_mutex_leave(pShmNode->pShmMutex); OSTRACE(("SHM-LOCK shmid-%d, pid-%d got %03x,%03x\n", p->id, osGetpid(0), p->sharedMask, p->exclMask)); return rc; @@ -36473,6 +38390,9 @@ static void unixShmBarrier( ){ UNUSED_PARAMETER(fd); sqlite3MemoryBarrier(); /* compiler-defined memory barrier */ + assert( fd->pMethods->xLock==nolockLock + || unixFileMutexNotheld((unixFile*)fd) + ); unixEnterMutex(); /* Also mutex, for redundancy */ unixLeaveMutex(); } @@ -36503,22 +38423,23 @@ static int unixShmUnmap( /* Remove connection p from the set of connections associated ** with pShmNode */ - sqlite3_mutex_enter(pShmNode->mutex); + sqlite3_mutex_enter(pShmNode->pShmMutex); for(pp=&pShmNode->pFirst; (*pp)!=p; pp = &(*pp)->pNext){} *pp = p->pNext; /* Free the connection p */ sqlite3_free(p); pDbFd->pShm = 0; - sqlite3_mutex_leave(pShmNode->mutex); + sqlite3_mutex_leave(pShmNode->pShmMutex); /* If pShmNode->nRef has reached 0, then close the underlying ** shared-memory file, too */ + assert( unixFileMutexNotheld(pDbFd) ); unixEnterMutex(); assert( pShmNode->nRef>0 ); pShmNode->nRef--; if( pShmNode->nRef==0 ){ - if( deleteFlag && pShmNode->h>=0 ){ + if( deleteFlag && pShmNode->hShm>=0 ){ osUnlink(pShmNode->zFilename); } unixShmPurge(pDbFd); @@ -36840,7 +38761,7 @@ IOMETHODS( IOMETHODS( nolockIoFinder, /* Finder function name */ nolockIoMethods, /* sqlite3_io_methods object name */ - 3, /* shared memory is disabled */ + 3, /* shared memory and mmap are enabled */ nolockClose, /* xClose method */ nolockLock, /* xLock method */ nolockUnlock, /* xUnlock method */ @@ -37336,7 +39257,7 @@ static UnixUnusedFd *findReusableFd(const char *zPath, int flags){ ** ** Even if a subsequent open() call does succeed, the consequences of ** not searching for a reusable file descriptor are not dire. */ - if( nUnusedFd>0 && 0==osStat(zPath, &sStat) ){ + if( inodeList!=0 && 0==osStat(zPath, &sStat) ){ unixInodeInfo *pInode; pInode = inodeList; @@ -37346,12 +39267,15 @@ static UnixUnusedFd *findReusableFd(const char *zPath, int flags){ } if( pInode ){ UnixUnusedFd **pp; + assert( sqlite3_mutex_notheld(pInode->pLockMutex) ); + sqlite3_mutex_enter(pInode->pLockMutex); + flags &= (SQLITE_OPEN_READONLY|SQLITE_OPEN_READWRITE); for(pp=&pInode->pUnused; *pp && (*pp)->flags!=flags; pp=&((*pp)->pNext)); pUnused = *pp; if( pUnused ){ - nUnusedFd--; *pp = pUnused->pNext; } + sqlite3_mutex_leave(pInode->pLockMutex); } } unixLeaveMutex(); @@ -37399,7 +39323,7 @@ static int getFileMode( ** If the SQLITE_ENABLE_8_3_NAMES option is enabled, then the ** original filename is unavailable. But 8_3_NAMES is only used for ** FAT filesystems and permissions do not matter there, so just use -** the default permissions. +** the default permissions. In 8_3_NAMES mode, leave *pMode set to zero. */ static int findCreateFileMode( const char *zPath, /* Path of file (possibly) being created */ @@ -37488,7 +39412,7 @@ static int unixOpen( unixFile *p = (unixFile *)pFile; int fd = -1; /* File descriptor returned by open() */ int openFlags = 0; /* Flags to pass to open() */ - int eType = flags&0xFFFFFF00; /* Type of file to open */ + int eType = flags&0x0FFF00; /* Type of file to open */ int noLock; /* True to omit locking primitives */ int rc = SQLITE_OK; /* Function Return Code */ int ctrlFlags = 0; /* UNIXFILE_* flags */ @@ -37598,7 +39522,7 @@ static int unixOpen( if( isReadWrite ) openFlags |= O_RDWR; if( isCreate ) openFlags |= O_CREAT; if( isExclusive ) openFlags |= (O_EXCL|O_NOFOLLOW); - openFlags |= (O_LARGEFILE|O_BINARY); + openFlags |= (O_LARGEFILE|O_BINARY|O_NOFOLLOW); if( fd<0 ){ mode_t openMode; /* Permissions to create file with */ @@ -37634,11 +39558,19 @@ static int unixOpen( goto open_finished; } - /* If this process is running as root and if creating a new rollback - ** journal or WAL file, set the ownership of the journal or WAL to be - ** the same as the original database. + /* The owner of the rollback journal or WAL file should always be the + ** same as the owner of the database file. Try to ensure that this is + ** the case. The chown() system call will be a no-op if the current + ** process lacks root privileges, be we should at least try. Without + ** this step, if a root process opens a database file, it can leave + ** behinds a journal/WAL that is owned by root and hence make the + ** database inaccessible to unprivileged processes. + ** + ** If openMode==0, then that means uid and gid are not set correctly + ** (probably because SQLite is configured to use 8+3 filename mode) and + ** in that case we do not want to attempt the chown(). */ - if( flags & (SQLITE_OPEN_WAL|SQLITE_OPEN_MAIN_JOURNAL) ){ + if( openMode && (flags & (SQLITE_OPEN_WAL|SQLITE_OPEN_MAIN_JOURNAL))!=0 ){ robustFchown(fd, uid, gid); } } @@ -37649,7 +39581,8 @@ static int unixOpen( if( p->pPreallocatedUnused ){ p->pPreallocatedUnused->fd = fd; - p->pPreallocatedUnused->flags = flags; + p->pPreallocatedUnused->flags = + flags & (SQLITE_OPEN_READONLY|SQLITE_OPEN_READWRITE); } if( isDelete ){ @@ -37807,7 +39740,8 @@ static int unixAccess( if( flags==SQLITE_ACCESS_EXISTS ){ struct stat buf; - *pResOut = (0==osStat(zPath, &buf) && buf.st_size>0); + *pResOut = 0==osStat(zPath, &buf) && + (!S_ISREG(buf.st_mode) || buf.st_size>0); }else{ *pResOut = osAccess(zPath, W_OK|R_OK)==0; } @@ -37861,7 +39795,7 @@ static int unixFullPathname( #else int rc = SQLITE_OK; int nByte; - int nLink = 1; /* Number of symbolic links followed so far */ + int nLink = 0; /* Number of symbolic links followed so far */ const char *zIn = zPath; /* Input path for each iteration of loop */ char *zDel = 0; @@ -37890,10 +39824,11 @@ static int unixFullPathname( } if( bLink ){ + nLink++; if( zDel==0 ){ zDel = sqlite3_malloc(nOut); if( zDel==0 ) rc = SQLITE_NOMEM_BKPT; - }else if( ++nLink>SQLITE_MAX_SYMLINKS ){ + }else if( nLink>=SQLITE_MAX_SYMLINKS ){ rc = SQLITE_CANTOPEN_BKPT; } @@ -37929,6 +39864,7 @@ static int unixFullPathname( }while( rc==SQLITE_OK ); sqlite3_free(zDel); + if( rc==SQLITE_OK && nLink ) rc = SQLITE_OK_SYMLINK; return rc; #endif /* HAVE_READLINK && HAVE_LSTAT */ } @@ -38414,7 +40350,7 @@ static int proxyCreateUnixFile( int fd = -1; unixFile *pNew; int rc = SQLITE_OK; - int openFlags = O_RDWR | O_CREAT; + int openFlags = O_RDWR | O_CREAT | O_NOFOLLOW; sqlite3_vfs dummyVfs; int terrno = 0; UnixUnusedFd *pUnused = NULL; @@ -38444,7 +40380,7 @@ static int proxyCreateUnixFile( } } if( fd<0 ){ - openFlags = O_RDONLY; + openFlags = O_RDONLY | O_NOFOLLOW; fd = robust_open(path, openFlags, 0); terrno = errno; } @@ -38495,7 +40431,7 @@ SQLITE_API int sqlite3_hostid_num = 0; #define PROXY_HOSTIDLEN 16 /* conch file host id length */ -#ifdef HAVE_GETHOSTUUID +#if HAVE_GETHOSTUUID /* Not always defined in the headers as it ought to be */ extern int gethostuuid(uuid_t id, const struct timespec *wait); #endif @@ -38506,7 +40442,7 @@ extern int gethostuuid(uuid_t id, const struct timespec *wait); static int proxyGetHostID(unsigned char *pHostID, int *pError){ assert(PROXY_HOSTIDLEN == sizeof(uuid_t)); memset(pHostID, 0, PROXY_HOSTIDLEN); -#ifdef HAVE_GETHOSTUUID +#if HAVE_GETHOSTUUID { struct timespec timeout = {1, 0}; /* 1 sec timeout */ if( gethostuuid(pHostID, &timeout) ){ @@ -38570,7 +40506,7 @@ static int proxyBreakConchLock(unixFile *pFile, uuid_t myHostID){ goto end_breaklock; } /* write it out to the temporary break file */ - fd = robust_open(tPath, (O_RDWR|O_CREAT|O_EXCL), 0); + fd = robust_open(tPath, (O_RDWR|O_CREAT|O_EXCL|O_NOFOLLOW), 0); if( fd<0 ){ sqlite3_snprintf(sizeof(errmsg), errmsg, "create failed (%d)", errno); goto end_breaklock; @@ -39180,7 +41116,7 @@ static int proxyFileControl(sqlite3_file *id, int op, void *pArg){ assert( 0 ); /* The call assures that only valid opcodes are sent */ } } - /*NOTREACHED*/ + /*NOTREACHED*/ assert(0); return SQLITE_ERROR; } @@ -39528,7 +41464,7 @@ SQLITE_API int sqlite3_os_end(void){ ****************************************************************************** ** ** This file contains inline asm code for retrieving "high-performance" -** counters for x86 class CPUs. +** counters for x86 and x86_64 class CPUs. */ #ifndef SQLITE_HWTIME_H #define SQLITE_HWTIME_H @@ -39539,8 +41475,9 @@ SQLITE_API int sqlite3_os_end(void){ ** processor and returns that value. This can be used for high-res ** profiling. */ -#if (defined(__GNUC__) || defined(_MSC_VER)) && \ - (defined(i386) || defined(__i386__) || defined(_M_IX86)) +#if !defined(__STRICT_ANSI__) && \ + (defined(__GNUC__) || defined(_MSC_VER)) && \ + (defined(i386) || defined(__i386__) || defined(_M_IX86)) #if defined(__GNUC__) @@ -39561,7 +41498,7 @@ SQLITE_API int sqlite3_os_end(void){ #endif -#elif (defined(__GNUC__) && defined(__x86_64__)) +#elif !defined(__STRICT_ANSI__) && (defined(__GNUC__) && defined(__x86_64__)) __inline__ sqlite_uint64 sqlite3Hwtime(void){ unsigned long val; @@ -39569,7 +41506,7 @@ SQLITE_API int sqlite3_os_end(void){ return val; } -#elif (defined(__GNUC__) && defined(__ppc__)) +#elif !defined(__STRICT_ANSI__) && (defined(__GNUC__) && defined(__ppc__)) __inline__ sqlite_uint64 sqlite3Hwtime(void){ unsigned long long retval; @@ -39586,14 +41523,13 @@ SQLITE_API int sqlite3_os_end(void){ #else - #error Need implementation of sqlite3Hwtime() for your platform. - /* - ** To compile without implementing sqlite3Hwtime() for your platform, - ** you can remove the above #error and use the following - ** stub function. You will lose timing support for many - ** of the debugging and testing utilities, but it should at - ** least compile and run. + ** asm() is needed for hardware timing support. Without asm(), + ** disable the sqlite3Hwtime() routine. + ** + ** sqlite3Hwtime() is only used for some obscure debugging + ** and analysis configurations, not in any deliverable, so this + ** should not be a great loss. */ SQLITE_PRIVATE sqlite_uint64 sqlite3Hwtime(void){ return ((sqlite_uint64)0); } @@ -39934,8 +41870,7 @@ struct winFile { int nFetchOut; /* Number of outstanding xFetch references */ HANDLE hMap; /* Handle for accessing memory mapping */ void *pMapRegion; /* Area memory mapped */ - sqlite3_int64 mmapSize; /* Usable size of mapped region */ - sqlite3_int64 mmapSizeActual; /* Actual size of mapped region */ + sqlite3_int64 mmapSize; /* Size of mapped region */ sqlite3_int64 mmapSizeMax; /* Configured FCNTL_MMAP_SIZE value */ #endif }; @@ -42554,6 +44489,29 @@ static int winTruncate(sqlite3_file *id, sqlite3_int64 nByte){ winFile *pFile = (winFile*)id; /* File handle object */ int rc = SQLITE_OK; /* Return code for this function */ DWORD lastErrno; +#if SQLITE_MAX_MMAP_SIZE>0 + sqlite3_int64 oldMmapSize; + if( pFile->nFetchOut>0 ){ + /* File truncation is a no-op if there are outstanding memory mapped + ** pages. This is because truncating the file means temporarily unmapping + ** the file, and that might delete memory out from under existing cursors. + ** + ** This can result in incremental vacuum not truncating the file, + ** if there is an active read cursor when the incremental vacuum occurs. + ** No real harm comes of this - the database file is not corrupted, + ** though some folks might complain that the file is bigger than it + ** needs to be. + ** + ** The only feasible work-around is to defer the truncation until after + ** all references to memory-mapped content are closed. That is doable, + ** but involves adding a few branches in the common write code path which + ** could slow down normal operations slightly. Hence, we have decided for + ** now to simply make trancations a no-op if there are pending reads. We + ** can maybe revisit this decision in the future. + */ + return SQLITE_OK; + } +#endif assert( pFile ); SimulateIOError(return SQLITE_IOERR_TRUNCATE); @@ -42569,6 +44527,15 @@ static int winTruncate(sqlite3_file *id, sqlite3_int64 nByte){ nByte = ((nByte + pFile->szChunk - 1)/pFile->szChunk) * pFile->szChunk; } +#if SQLITE_MAX_MMAP_SIZE>0 + if( pFile->pMapRegion ){ + oldMmapSize = pFile->mmapSize; + }else{ + oldMmapSize = 0; + } + winUnmapfile(pFile); +#endif + /* SetEndOfFile() returns non-zero when successful, or zero when it fails. */ if( winSeekFile(pFile, nByte) ){ rc = winLogError(SQLITE_IOERR_TRUNCATE, pFile->lastErrno, @@ -42581,12 +44548,12 @@ static int winTruncate(sqlite3_file *id, sqlite3_int64 nByte){ } #if SQLITE_MAX_MMAP_SIZE>0 - /* If the file was truncated to a size smaller than the currently - ** mapped region, reduce the effective mapping size as well. SQLite will - ** use read() and write() to access data beyond this point from now on. - */ - if( pFile->pMapRegion && nBytemmapSize ){ - pFile->mmapSize = nByte; + if( rc==SQLITE_OK && oldMmapSize>0 ){ + if( oldMmapSize>nByte ){ + winMapfile(pFile, -1); + }else{ + winMapfile(pFile, oldMmapSize); + } } #endif @@ -43121,6 +45088,7 @@ static void winModeBit(winFile *pFile, unsigned char mask, int *pArg){ /* Forward references to VFS helper methods used for temporary files */ static int winGetTempname(sqlite3_vfs *, char **); static int winIsDir(const void *); +static BOOL winIsLongPathPrefix(const char *); static BOOL winIsDriveLetterAndColon(const char *); /* @@ -43834,6 +45802,7 @@ static int winShmMap( rc = winOpenSharedMemory(pDbFd); if( rc!=SQLITE_OK ) return rc; pShm = pDbFd->pShm; + assert( pShm!=0 ); } pShmNode = pShm->pShmNode; @@ -43972,9 +45941,9 @@ static int winShmMap( static int winUnmapfile(winFile *pFile){ assert( pFile!=0 ); OSTRACE(("UNMAP-FILE pid=%lu, pFile=%p, hMap=%p, pMapRegion=%p, " - "mmapSize=%lld, mmapSizeActual=%lld, mmapSizeMax=%lld\n", + "mmapSize=%lld, mmapSizeMax=%lld\n", osGetCurrentProcessId(), pFile, pFile->hMap, pFile->pMapRegion, - pFile->mmapSize, pFile->mmapSizeActual, pFile->mmapSizeMax)); + pFile->mmapSize, pFile->mmapSizeMax)); if( pFile->pMapRegion ){ if( !osUnmapViewOfFile(pFile->pMapRegion) ){ pFile->lastErrno = osGetLastError(); @@ -43986,7 +45955,6 @@ static int winUnmapfile(winFile *pFile){ } pFile->pMapRegion = 0; pFile->mmapSize = 0; - pFile->mmapSizeActual = 0; } if( pFile->hMap!=NULL ){ if( !osCloseHandle(pFile->hMap) ){ @@ -44097,7 +46065,6 @@ static int winMapfile(winFile *pFd, sqlite3_int64 nByte){ } pFd->pMapRegion = pNew; pFd->mmapSize = nMap; - pFd->mmapSizeActual = nMap; } OSTRACE(("MAP-FILE pid=%lu, pFile=%p, rc=SQLITE_OK\n", @@ -44138,6 +46105,7 @@ static int winFetch(sqlite3_file *fd, i64 iOff, int nAmt, void **pp){ } } if( pFd->mmapSize >= iOff+nAmt ){ + assert( pFd->pMapRegion!=0 ); *pp = &((u8 *)pFd->pMapRegion)[iOff]; pFd->nFetchOut++; } @@ -44890,7 +46858,9 @@ static int winOpen( if( isReadonly ){ pFile->ctrlFlags |= WINFILE_RDONLY; } - if( sqlite3_uri_boolean(zName, "psow", SQLITE_POWERSAFE_OVERWRITE) ){ + if( (flags & SQLITE_OPEN_MAIN_DB) + && sqlite3_uri_boolean(zName, "psow", SQLITE_POWERSAFE_OVERWRITE) + ){ pFile->ctrlFlags |= WINFILE_PSOW; } pFile->lastErrno = NO_ERROR; @@ -44899,7 +46869,6 @@ static int winOpen( pFile->hMap = NULL; pFile->pMapRegion = 0; pFile->mmapSize = 0; - pFile->mmapSizeActual = 0; pFile->mmapSizeMax = sqlite3GlobalConfig.szMmap; #endif @@ -45101,6 +47070,17 @@ static int winAccess( return SQLITE_OK; } +/* +** Returns non-zero if the specified path name starts with the "long path" +** prefix. +*/ +static BOOL winIsLongPathPrefix( + const char *zPathname +){ + return ( zPathname[0]=='\\' && zPathname[1]=='\\' + && zPathname[2]=='?' && zPathname[3]=='\\' ); +} + /* ** Returns non-zero if the specified path name starts with a drive letter ** followed by a colon character. @@ -45165,10 +47145,11 @@ static int winFullPathname( char *zOut; #endif - /* If this path name begins with "/X:", where "X" is any alphabetic - ** character, discard the initial "/" from the pathname. + /* If this path name begins with "/X:" or "\\?\", where "X" is any + ** alphabetic character, discard the initial "/" from the pathname. */ - if( zRelative[0]=='/' && winIsDriveLetterAndColon(zRelative+1) ){ + if( zRelative[0]=='/' && (winIsDriveLetterAndColon(zRelative+1) + || winIsLongPathPrefix(zRelative+1)) ){ zRelative++; } @@ -45774,8 +47755,8 @@ SQLITE_API int sqlite3_os_end(void){ ** This file also implements interface sqlite3_serialize() and ** sqlite3_deserialize(). */ -#ifdef SQLITE_ENABLE_DESERIALIZE /* #include "sqliteInt.h" */ +#ifdef SQLITE_ENABLE_DESERIALIZE /* ** Forward declaration of objects used by this utility @@ -45792,7 +47773,8 @@ typedef struct MemFile MemFile; struct MemFile { sqlite3_file base; /* IO methods */ sqlite3_int64 sz; /* Size of the file */ - sqlite3_int64 szMax; /* Space allocated to aData */ + sqlite3_int64 szAlloc; /* Space allocated to aData */ + sqlite3_int64 szMax; /* Maximum allowed size of the file */ unsigned char *aData; /* content of the file */ int nMmap; /* Number of memory mapped pages */ unsigned mFlags; /* Flags */ @@ -45918,10 +47900,15 @@ static int memdbEnlarge(MemFile *p, sqlite3_int64 newSz){ if( (p->mFlags & SQLITE_DESERIALIZE_RESIZEABLE)==0 || p->nMmap>0 ){ return SQLITE_FULL; } - pNew = sqlite3_realloc64(p->aData, newSz); + if( newSz>p->szMax ){ + return SQLITE_FULL; + } + newSz *= 2; + if( newSz>p->szMax ) newSz = p->szMax; + pNew = sqlite3Realloc(p->aData, newSz); if( pNew==0 ) return SQLITE_NOMEM; p->aData = pNew; - p->szMax = newSz; + p->szAlloc = newSz; return SQLITE_OK; } @@ -45935,10 +47922,11 @@ static int memdbWrite( sqlite_int64 iOfst ){ MemFile *p = (MemFile *)pFile; + if( NEVER(p->mFlags & SQLITE_DESERIALIZE_READONLY) ) return SQLITE_READONLY; if( iOfst+iAmt>p->sz ){ int rc; - if( iOfst+iAmt>p->szMax - && (rc = memdbEnlarge(p, (iOfst+iAmt)*2))!=SQLITE_OK + if( iOfst+iAmt>p->szAlloc + && (rc = memdbEnlarge(p, iOfst+iAmt))!=SQLITE_OK ){ return rc; } @@ -45984,6 +47972,11 @@ static int memdbFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){ */ static int memdbLock(sqlite3_file *pFile, int eLock){ MemFile *p = (MemFile *)pFile; + if( eLock>SQLITE_LOCK_SHARED + && (p->mFlags & SQLITE_DESERIALIZE_READONLY)!=0 + ){ + return SQLITE_READONLY; + } p->eLock = eLock; return SQLITE_OK; } @@ -46008,6 +48001,19 @@ static int memdbFileControl(sqlite3_file *pFile, int op, void *pArg){ *(char**)pArg = sqlite3_mprintf("memdb(%p,%lld)", p->aData, p->sz); rc = SQLITE_OK; } + if( op==SQLITE_FCNTL_SIZE_LIMIT ){ + sqlite3_int64 iLimit = *(sqlite3_int64*)pArg; + if( iLimitsz ){ + if( iLimit<0 ){ + iLimit = p->szMax; + }else{ + iLimit = p->sz; + } + } + p->szMax = iLimit; + *(sqlite3_int64*)pArg = iLimit; + rc = SQLITE_OK; + } return rc; } @@ -46038,8 +48044,12 @@ static int memdbFetch( void **pp ){ MemFile *p = (MemFile *)pFile; - p->nMmap++; - *pp = (void*)(p->aData + iOfst); + if( iOfst+iAmt>p->sz ){ + *pp = 0; + }else{ + p->nMmap++; + *pp = (void*)(p->aData + iOfst); + } return SQLITE_OK; } @@ -46069,6 +48079,7 @@ static int memdbOpen( assert( pOutFlags!=0 ); /* True because flags==SQLITE_OPEN_MAIN_DB */ *pOutFlags = flags | SQLITE_OPEN_MEMORY; p->base.pMethods = &memdb_io_methods; + p->szMax = sqlite3GlobalConfig.mxMemdbSize; return SQLITE_OK; } @@ -46318,7 +48329,11 @@ SQLITE_API int sqlite3_deserialize( }else{ p->aData = pData; p->sz = szDb; + p->szAlloc = szBuf; p->szMax = szBuf; + if( p->szMaxszMax = sqlite3GlobalConfig.mxMemdbSize; + } p->mFlags = mFlags; rc = SQLITE_OK; } @@ -46337,10 +48352,11 @@ SQLITE_PRIVATE int sqlite3MemdbInit(void){ sqlite3_vfs *pLower = sqlite3_vfs_find(0); int sz = pLower->szOsFile; memdb_vfs.pAppData = pLower; - /* In all known configurations of SQLite, the size of a default - ** sqlite3_file is greater than the size of a memdb sqlite3_file. - ** Should that ever change, remove the following NEVER() */ - if( NEVER(szszCache; }else{ - /* IMPLEMENTATION-OF: R-61436-13639 If the argument N is negative, then - ** the number of cache pages is adjusted to use approximately abs(N*1024) - ** bytes of memory. */ + /* IMPLEMANTATION-OF: R-59858-46238 If the argument N is negative, then the + ** number of cache pages is adjusted to be a number of pages that would + ** use approximately abs(N*1024) bytes of memory based on the current + ** page size. */ return (int)((-1024*(i64)p->szCache)/(p->szPage+p->szExtra)); } } @@ -47025,6 +49042,7 @@ SQLITE_PRIVATE int sqlite3PcacheInitialize(void){ ** built-in default page cache is used instead of the application defined ** page cache. */ sqlite3PCacheSetDefault(); + assert( sqlite3GlobalConfig.pcache2.xInit!=0 ); } return sqlite3GlobalConfig.pcache2.xInit(sqlite3GlobalConfig.pcache2.pArg); } @@ -47620,6 +49638,15 @@ SQLITE_PRIVATE int sqlite3PCachePercentDirty(PCache *pCache){ return nCache ? (int)(((i64)nDirty * 100) / nCache) : 0; } +#ifdef SQLITE_DIRECT_OVERFLOW_READ +/* +** Return true if there are one or more dirty pages in the cache. Else false. +*/ +SQLITE_PRIVATE int sqlite3PCacheIsDirty(PCache *pCache){ + return (pCache->pDirty!=0); +} +#endif + #if defined(SQLITE_CHECK_PAGES) || defined(SQLITE_DEBUG) /* ** For all dirty pages currently in the cache, invoke the specified @@ -47730,20 +49757,32 @@ typedef struct PGroup PGroup; ** structure. Unless SQLITE_PCACHE_SEPARATE_HEADER is defined, a buffer of ** PgHdr1.pCache->szPage bytes is allocated directly before this structure ** in memory. +** +** Note: Variables isBulkLocal and isAnchor were once type "u8". That works, +** but causes a 2-byte gap in the structure for most architectures (since +** pointers must be either 4 or 8-byte aligned). As this structure is located +** in memory directly after the associated page data, if the database is +** corrupt, code at the b-tree layer may overread the page buffer and +** read part of this structure before the corruption is detected. This +** can cause a valgrind error if the unitialized gap is accessed. Using u16 +** ensures there is no such gap, and therefore no bytes of unitialized memory +** in the structure. */ struct PgHdr1 { sqlite3_pcache_page page; /* Base class. Must be first. pBuf & pExtra */ unsigned int iKey; /* Key value (page number) */ - u8 isBulkLocal; /* This page from bulk local storage */ - u8 isAnchor; /* This is the PGroup.lru element */ + u16 isBulkLocal; /* This page from bulk local storage */ + u16 isAnchor; /* This is the PGroup.lru element */ PgHdr1 *pNext; /* Next in hash table chain */ PCache1 *pCache; /* Cache that currently owns this page */ PgHdr1 *pLruNext; /* Next in LRU list of unpinned pages */ PgHdr1 *pLruPrev; /* Previous in LRU list of unpinned pages */ + /* NB: pLruPrev is only valid if pLruNext!=0 */ }; /* -** A page is pinned if it is no on the LRU list +** A page is pinned if it is not on the LRU list. To be "pinned" means +** that the page is in active use and must not be deallocated. */ #define PAGE_IS_PINNED(p) ((p)->pLruNext==0) #define PAGE_IS_UNPINNED(p) ((p)->pLruNext!=0) @@ -47804,6 +49843,7 @@ struct PCache1 { unsigned int nMax; /* Configured "cache_size" value */ unsigned int n90pct; /* nMax*9/10 */ unsigned int iMaxKey; /* Largest key seen since xTruncate() */ + unsigned int nPurgeableDummy; /* pnPurgeable points here when not used*/ /* Hash table of all pages. The following variables may only be accessed ** when the accessor is holding the PGroup mutex. @@ -47938,6 +49978,7 @@ static int pcache1InitBulk(PCache1 *pCache){ pX->isBulkLocal = 1; pX->isAnchor = 0; pX->pNext = pCache->pFree; + pX->pLruPrev = 0; /* Initializing this saves a valgrind error */ pCache->pFree = pX; zBulk += pCache->szAlloc; }while( --nBulk ); @@ -48048,6 +50089,7 @@ static PgHdr1 *pcache1AllocPage(PCache1 *pCache, int benignMalloc){ assert( sqlite3_mutex_held(pCache->pGroup->mutex) ); if( pCache->pFree || (pCache->nPage==0 && pcache1InitBulk(pCache)) ){ + assert( pCache->pFree!=0 ); p = pCache->pFree; pCache->pFree = p->pNext; p->pNext = 0; @@ -48071,13 +50113,15 @@ static PgHdr1 *pcache1AllocPage(PCache1 *pCache, int benignMalloc){ } #else pPg = pcache1Alloc(pCache->szAlloc); - p = (PgHdr1 *)&((u8 *)pPg)[pCache->szPage]; #endif if( benignMalloc ){ sqlite3EndBenignMalloc(); } #ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT pcache1EnterMutex(pCache->pGroup); #endif if( pPg==0 ) return 0; +#ifndef SQLITE_PCACHE_SEPARATE_HEADER + p = (PgHdr1 *)&((u8 *)pPg)[pCache->szPage]; +#endif p->page.pBuf = pPg; p->page.pExtra = &p[1]; p->isBulkLocal = 0; @@ -48113,6 +50157,7 @@ static void pcache1FreePage(PgHdr1 *p){ ** exists, this function falls back to sqlite3Malloc(). */ SQLITE_PRIVATE void *sqlite3PageMalloc(int sz){ + assert( sz<=65536+8 ); /* These allocations are never very large */ return pcache1Alloc(sz); } @@ -48207,7 +50252,8 @@ static PgHdr1 *pcache1PinPage(PgHdr1 *pPage){ pPage->pLruPrev->pLruNext = pPage->pLruNext; pPage->pLruNext->pLruPrev = pPage->pLruPrev; pPage->pLruNext = 0; - pPage->pLruPrev = 0; + /* pPage->pLruPrev = 0; + ** No need to clear pLruPrev as it is never accessed if pLruNext is 0 */ assert( pPage->isAnchor==0 ); assert( pPage->pCache->pGroup->lru.isAnchor==1 ); pPage->pCache->nRecyclable--; @@ -48400,6 +50446,7 @@ static sqlite3_pcache *pcache1Create(int szPage, int szExtra, int bPurgeable){ }else{ pGroup = &pcache1.grp; } + pcache1EnterMutex(pGroup); if( pGroup->lru.isAnchor==0 ){ pGroup->lru.isAnchor = 1; pGroup->lru.pLruPrev = pGroup->lru.pLruNext = &pGroup->lru; @@ -48409,7 +50456,6 @@ static sqlite3_pcache *pcache1Create(int szPage, int szExtra, int bPurgeable){ pCache->szExtra = szExtra; pCache->szAlloc = szPage + szExtra + ROUND8(sizeof(PgHdr1)); pCache->bPurgeable = (bPurgeable ? 1 : 0); - pcache1EnterMutex(pGroup); pcache1ResizeHash(pCache); if( bPurgeable ){ pCache->nMin = 10; @@ -48417,8 +50463,7 @@ static sqlite3_pcache *pcache1Create(int szPage, int szExtra, int bPurgeable){ pGroup->mxPinned = pGroup->nMaxPage + 10 - pGroup->nMinPage; pCache->pnPurgeable = &pGroup->nPurgeable; }else{ - static unsigned int dummyCurrentPage; - pCache->pnPurgeable = &dummyCurrentPage; + pCache->pnPurgeable = &pCache->nPurgeableDummy; } pcache1LeaveMutex(pGroup); if( pCache->nHash==0 ){ @@ -48545,8 +50590,9 @@ static SQLITE_NOINLINE PgHdr1 *pcache1FetchStage2( pPage->iKey = iKey; pPage->pNext = pCache->apHash[h]; pPage->pCache = pCache; - pPage->pLruPrev = 0; pPage->pLruNext = 0; + /* pPage->pLruPrev = 0; + ** No need to clear pLruPrev since it is not accessed when pLruNext==0 */ *(void **)pPage->page.pExtra = 0; pCache->apHash[h] = pPage; if( iKey>pCache->iMaxKey ){ @@ -48706,7 +50752,7 @@ static void pcache1Unpin( /* It is an error to call this function if the page is already ** part of the PGroup LRU list. */ - assert( pPage->pLruPrev==0 && pPage->pLruNext==0 ); + assert( pPage->pLruNext==0 ); assert( PAGE_IS_PINNED(pPage) ); if( reuseUnlikely || pGroup->nPurgeable>pGroup->nMaxPage ){ @@ -49023,30 +51069,23 @@ struct RowSet { #define ROWSET_NEXT 0x02 /* True if sqlite3RowSetNext() has been called */ /* -** Turn bulk memory into a RowSet object. N bytes of memory -** are available at pSpace. The db pointer is used as a memory context -** for any subsequent allocations that need to occur. -** Return a pointer to the new RowSet object. -** -** It must be the case that N is sufficient to make a Rowset. If not -** an assertion fault occurs. -** -** If N is larger than the minimum, use the surplus as an initial -** allocation of entries available to be filled. +** Allocate a RowSet object. Return NULL if a memory allocation +** error occurs. */ -SQLITE_PRIVATE RowSet *sqlite3RowSetInit(sqlite3 *db, void *pSpace, unsigned int N){ - RowSet *p; - assert( N >= ROUND8(sizeof(*p)) ); - p = pSpace; - p->pChunk = 0; - p->db = db; - p->pEntry = 0; - p->pLast = 0; - p->pForest = 0; - p->pFresh = (struct RowSetEntry*)(ROUND8(sizeof(*p)) + (char*)p); - p->nFresh = (u16)((N - ROUND8(sizeof(*p)))/sizeof(struct RowSetEntry)); - p->rsFlags = ROWSET_SORTED; - p->iBatch = 0; +SQLITE_PRIVATE RowSet *sqlite3RowSetInit(sqlite3 *db){ + RowSet *p = sqlite3DbMallocRawNN(db, sizeof(*p)); + if( p ){ + int N = sqlite3DbMallocSize(db, p); + p->pChunk = 0; + p->db = db; + p->pEntry = 0; + p->pLast = 0; + p->pForest = 0; + p->pFresh = (struct RowSetEntry*)(ROUND8(sizeof(*p)) + (char*)p); + p->nFresh = (u16)((N - ROUND8(sizeof(*p)))/sizeof(struct RowSetEntry)); + p->rsFlags = ROWSET_SORTED; + p->iBatch = 0; + } return p; } @@ -49055,7 +51094,8 @@ SQLITE_PRIVATE RowSet *sqlite3RowSetInit(sqlite3 *db, void *pSpace, unsigned int ** the RowSet has allocated over its lifetime. This routine is ** the destructor for the RowSet. */ -SQLITE_PRIVATE void sqlite3RowSetClear(RowSet *p){ +SQLITE_PRIVATE void sqlite3RowSetClear(void *pArg){ + RowSet *p = (RowSet*)pArg; struct RowSetChunk *pChunk, *pNextChunk; for(pChunk=p->pChunk; pChunk; pChunk = pNextChunk){ pNextChunk = pChunk->pNextChunk; @@ -49069,10 +51109,20 @@ SQLITE_PRIVATE void sqlite3RowSetClear(RowSet *p){ p->rsFlags = ROWSET_SORTED; } +/* +** Deallocate all chunks from a RowSet. This frees all memory that +** the RowSet has allocated over its lifetime. This routine is +** the destructor for the RowSet. +*/ +SQLITE_PRIVATE void sqlite3RowSetDelete(void *pArg){ + sqlite3RowSetClear(pArg); + sqlite3DbFree(((RowSet*)pArg)->db, pArg); +} + /* ** Allocate a new RowSetEntry object that is associated with the ** given RowSet. Return a pointer to the new and completely uninitialized -** objected. +** object. ** ** In an OOM situation, the RowSet.db->mallocFailed flag is set and this ** routine returns NULL. @@ -49348,7 +51398,7 @@ SQLITE_PRIVATE int sqlite3RowSetTest(RowSet *pRowSet, int iBatch, sqlite3_int64 if( p ){ struct RowSetEntry **ppPrevTree = &pRowSet->pForest; if( (pRowSet->rsFlags & ROWSET_SORTED)==0 ){ /*OPTIMIZATION-IF-FALSE*/ - /* Only sort the current set of entiries if they need it */ + /* Only sort the current set of entries if they need it */ p = rowSetEntrySort(p); } for(pTree = pRowSet->pForest; pTree; pTree=pTree->pRight){ @@ -49556,6 +51606,8 @@ SQLITE_PRIVATE int sqlite3WalHeapMemory(Wal *pWal); SQLITE_PRIVATE int sqlite3WalSnapshotGet(Wal *pWal, sqlite3_snapshot **ppSnapshot); SQLITE_PRIVATE void sqlite3WalSnapshotOpen(Wal *pWal, sqlite3_snapshot *pSnapshot); SQLITE_PRIVATE int sqlite3WalSnapshotRecover(Wal *pWal); +SQLITE_PRIVATE int sqlite3WalSnapshotCheck(Wal *pWal, sqlite3_snapshot *pSnapshot); +SQLITE_PRIVATE void sqlite3WalSnapshotUnlock(Wal *pWal); #endif #ifdef SQLITE_ENABLE_ZIPVFS @@ -49568,6 +51620,11 @@ SQLITE_PRIVATE int sqlite3WalFramesize(Wal *pWal); /* Return the sqlite3_file object for the WAL file */ SQLITE_PRIVATE sqlite3_file *sqlite3WalFile(Wal *pWal); +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT +SQLITE_PRIVATE int sqlite3WalWriteLock(Wal *pWal, int bLock); +SQLITE_PRIVATE void sqlite3WalDb(Wal *pWal, sqlite3 *db); +#endif + #endif /* ifndef SQLITE_OMIT_WAL */ #endif /* SQLITE_WAL_H */ @@ -49958,20 +52015,6 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ */ #define UNKNOWN_LOCK (EXCLUSIVE_LOCK+1) -/* -** A macro used for invoking the codec if there is one -*/ -#ifdef SQLITE_HAS_CODEC -# define CODEC1(P,D,N,X,E) \ - if( P->xCodec && P->xCodec(P->pCodec,D,N,X)==0 ){ E; } -# define CODEC2(P,D,N,X,E,O) \ - if( P->xCodec==0 ){ O=(char*)D; }else \ - if( (O=(char*)(P->xCodec(P->pCodec,D,N,X)))==0 ){ E; } -#else -# define CODEC1(P,D,N,X,E) /* NO-OP */ -# define CODEC2(P,D,N,X,E,O) O=(char*)D -#endif - /* ** The maximum allowed sector size. 64KiB. If the xSectorsize() method ** returns a value larger than this, then MAX_SECTOR_SIZE is used instead. @@ -50257,12 +52300,6 @@ struct Pager { #endif void (*xReiniter)(DbPage*); /* Call this routine when reloading pages */ int (*xGet)(Pager*,Pgno,DbPage**,int); /* Routine to fetch a patch */ -#ifdef SQLITE_HAS_CODEC - void *(*xCodec)(void*,void*,Pgno,int); /* Routine for en/decoding data */ - void (*xCodecSizeChng)(void*,int,int); /* Notify of page size changes */ - void (*xCodecFree)(void*); /* Destructor for the codec */ - void *pCodec; /* First argument to xCodec... methods */ -#endif char *pTmpSpace; /* Pager.pageSize bytes of space for tmp use */ PCache *pPCache; /* Pointer to page cache object */ #ifndef SQLITE_OMIT_WAL @@ -50377,19 +52414,30 @@ static const unsigned char aJournalMagic[] = { */ #define isOpen(pFd) ((pFd)->pMethods!=0) +#ifdef SQLITE_DIRECT_OVERFLOW_READ /* -** Return true if this pager uses a write-ahead log to read page pgno. -** Return false if the pager reads pgno directly from the database. +** Return true if page pgno can be read directly from the database file +** by the b-tree layer. This is the case if: +** +** * the database file is open, +** * there are no dirty pages in the cache, and +** * the desired page is not currently in the wal file. */ -#if !defined(SQLITE_OMIT_WAL) && defined(SQLITE_DIRECT_OVERFLOW_READ) -SQLITE_PRIVATE int sqlite3PagerUseWal(Pager *pPager, Pgno pgno){ - u32 iRead = 0; - int rc; - if( pPager->pWal==0 ) return 0; - rc = sqlite3WalFindFrame(pPager->pWal, pgno, &iRead); - return rc || iRead; +SQLITE_PRIVATE int sqlite3PagerDirectReadOk(Pager *pPager, Pgno pgno){ + if( pPager->fd->pMethods==0 ) return 0; + if( sqlite3PCacheIsDirty(pPager->pPCache) ) return 0; +#ifndef SQLITE_OMIT_WAL + if( pPager->pWal ){ + u32 iRead = 0; + int rc; + rc = sqlite3WalFindFrame(pPager->pWal, pgno, &iRead); + return (rc==SQLITE_OK && iRead==0); + } +#endif + return 1; } #endif + #ifndef SQLITE_OMIT_WAL # define pagerUseWal(x) ((x)->pWal!=0) #else @@ -50549,8 +52597,12 @@ static int assert_pager_state(Pager *p){ ** to "print *pPager" in gdb: ** ** (gdb) printf "%s", print_pager_state(pPager) +** +** This routine has external linkage in order to suppress compiler warnings +** about an unused function. It is enclosed within SQLITE_DEBUG and so does +** not appear in normal builds. */ -static char *print_pager_state(Pager *p){ +char *print_pager_state(Pager *p){ static char zRet[1024]; sqlite3_snprintf(1024, zRet, @@ -50607,11 +52659,7 @@ static void setGetterMethod(Pager *pPager){ if( pPager->errCode ){ pPager->xGet = getPageError; #if SQLITE_MAX_MMAP_SIZE>0 - }else if( USEFETCH(pPager) -#ifdef SQLITE_HAS_CODEC - && pPager->xCodec==0 -#endif - ){ + }else if( USEFETCH(pPager) ){ pPager->xGet = getPageMMap; #endif /* SQLITE_MAX_MMAP_SIZE>0 */ }else{ @@ -50706,6 +52754,7 @@ static int pagerUnlockDb(Pager *pPager, int eLock){ } IOTRACE(("UNLOCK %p %d\n", pPager, eLock)) } + pPager->changeCountDone = pPager->tempFile; /* ticket fb3b3024ea238d5c */ return rc; } @@ -50893,6 +52942,7 @@ static int readMasterJournal(sqlite3_file *pJrnl, char *zMaster, u32 nMaster){ len = 0; } zMaster[len] = '\0'; + zMaster[len+1] = '\0'; return SQLITE_OK; } @@ -51316,7 +53366,6 @@ static void pager_reset(Pager *pPager){ ** Return the pPager->iDataVersion value */ SQLITE_PRIVATE u32 sqlite3PagerDataVersion(Pager *pPager){ - assert( pPager->eState>PAGER_OPEN ); return pPager->iDataVersion; } @@ -51427,7 +53476,6 @@ static void pager_unlock(Pager *pPager){ ** code is cleared and the cache reset in the block below. */ assert( pPager->errCode || pPager->eState!=PAGER_ERROR ); - pPager->changeCountDone = 0; pPager->eState = PAGER_OPEN; } @@ -51691,7 +53739,6 @@ static int pager_end_transaction(Pager *pPager, int hasMaster, int bCommit){ && (!pagerUseWal(pPager) || sqlite3WalExclusiveMode(pPager->pWal, 0)) ){ rc2 = pagerUnlockDb(pPager, SHARED_LOCK); - pPager->changeCountDone = 0; } pPager->eState = PAGER_READER; pPager->setMaster = 0; @@ -51760,35 +53807,6 @@ static u32 pager_cksum(Pager *pPager, const u8 *aData){ return cksum; } -/* -** Report the current page size and number of reserved bytes back -** to the codec. -*/ -#ifdef SQLITE_HAS_CODEC -static void pagerReportSize(Pager *pPager){ - if( pPager->xCodecSizeChng ){ - pPager->xCodecSizeChng(pPager->pCodec, pPager->pageSize, - (int)pPager->nReserve); - } -} -#else -# define pagerReportSize(X) /* No-op if we do not support a codec */ -#endif - -#ifdef SQLITE_HAS_CODEC -/* -** Make sure the number of reserved bits is the same in the destination -** pager as it is in the source. This comes up when a VACUUM changes the -** number of reserved bits to the "optimal" amount. -*/ -SQLITE_PRIVATE void sqlite3PagerAlignReserve(Pager *pDest, Pager *pSrc){ - if( pDest->nReserve!=pSrc->nReserve ){ - pDest->nReserve = pSrc->nReserve; - pagerReportSize(pDest); - } -} -#endif - /* ** Read a single page from either the journal file (if isMainJrnl==1) or ** from the sub-journal (if isMainJrnl==0) and playback that page. @@ -51840,11 +53858,6 @@ static int pager_playback_one_page( char *aData; /* Temporary storage for the page */ sqlite3_file *jfd; /* The file descriptor for the journal file */ int isSynced; /* True if journal page is synced */ -#ifdef SQLITE_HAS_CODEC - /* The jrnlEnc flag is true if Journal pages should be passed through - ** the codec. It is false for pure in-memory journals. */ - const int jrnlEnc = (isMainJrnl || pPager->subjInMemory==0); -#endif assert( (isMainJrnl&~1)==0 ); /* isMainJrnl is 0 or 1 */ assert( (isSavepnt&~1)==0 ); /* isSavepnt is 0 or 1 */ @@ -51907,7 +53920,6 @@ static int pager_playback_one_page( */ if( pgno==1 && pPager->nReserve!=((u8*)aData)[20] ){ pPager->nReserve = ((u8*)aData)[20]; - pagerReportSize(pPager); } /* If the pager is in CACHEMOD state, then there must be a copy of this @@ -51975,26 +53987,12 @@ static int pager_playback_one_page( ** is if the data was just read from an in-memory sub-journal. In that ** case it must be encrypted here before it is copied into the database ** file. */ -#ifdef SQLITE_HAS_CODEC - if( !jrnlEnc ){ - CODEC2(pPager, aData, pgno, 7, rc=SQLITE_NOMEM_BKPT, aData); - rc = sqlite3OsWrite(pPager->fd, (u8 *)aData, pPager->pageSize, ofst); - CODEC1(pPager, aData, pgno, 3, rc=SQLITE_NOMEM_BKPT); - }else -#endif rc = sqlite3OsWrite(pPager->fd, (u8 *)aData, pPager->pageSize, ofst); if( pgno>pPager->dbFileSize ){ pPager->dbFileSize = pgno; } if( pPager->pBackup ){ -#ifdef SQLITE_HAS_CODEC - if( jrnlEnc ){ - CODEC1(pPager, aData, pgno, 3, rc=SQLITE_NOMEM_BKPT); - sqlite3BackupUpdate(pPager->pBackup, pgno, (u8*)aData); - CODEC2(pPager, aData, pgno, 7, rc=SQLITE_NOMEM_BKPT,aData); - }else -#endif sqlite3BackupUpdate(pPager->pBackup, pgno, (u8*)aData); } }else if( !isMainJrnl && pPg==0 ){ @@ -52045,11 +54043,6 @@ static int pager_playback_one_page( if( pgno==1 ){ memcpy(&pPager->dbFileVers, &((u8*)pData)[24],sizeof(pPager->dbFileVers)); } - - /* Decode the page just read from disk */ -#if SQLITE_HAS_CODEC - if( jrnlEnc ){ CODEC1(pPager, pData, pPg->pgno, 3, rc=SQLITE_NOMEM_BKPT); } -#endif sqlite3PcacheRelease(pPg); } return rc; @@ -52130,15 +54123,16 @@ static int pager_delmaster(Pager *pPager, const char *zMaster){ rc = sqlite3OsFileSize(pMaster, &nMasterJournal); if( rc!=SQLITE_OK ) goto delmaster_out; nMasterPtr = pVfs->mxPathname+1; - zMasterJournal = sqlite3Malloc(nMasterJournal + nMasterPtr + 1); + zMasterJournal = sqlite3Malloc(nMasterJournal + nMasterPtr + 2); if( !zMasterJournal ){ rc = SQLITE_NOMEM_BKPT; goto delmaster_out; } - zMasterPtr = &zMasterJournal[nMasterJournal+1]; + zMasterPtr = &zMasterJournal[nMasterJournal+2]; rc = sqlite3OsRead(pMaster, zMasterJournal, (int)nMasterJournal, 0); if( rc!=SQLITE_OK ) goto delmaster_out; zMasterJournal[nMasterJournal] = 0; + zMasterJournal[nMasterJournal+1] = 0; zJournal = zMasterJournal; while( (zJournal-zMasterJournal)dbFileVers, dbFileVers, sizeof(pPager->dbFileVers)); } } - CODEC1(pPager, pPg->pData, pPg->pgno, 3, rc = SQLITE_NOMEM_BKPT); - PAGER_INCR(sqlite3_pager_readdb_count); PAGER_INCR(pPager->nRead); IOTRACE(("PGIN %p %d\n", pPager, pPg->pgno)); @@ -53324,8 +55319,14 @@ SQLITE_PRIVATE int sqlite3PagerSetPagesize(Pager *pPager, u32 *pPageSize, int nR rc = sqlite3OsFileSize(pPager->fd, &nByte); } if( rc==SQLITE_OK ){ - pNew = (char *)sqlite3PageMalloc(pageSize); - if( !pNew ) rc = SQLITE_NOMEM_BKPT; + /* 8 bytes of zeroed overrun space is sufficient so that the b-tree + * cell header parser will never run off the end of the allocation */ + pNew = (char *)sqlite3PageMalloc(pageSize+8); + if( !pNew ){ + rc = SQLITE_NOMEM_BKPT; + }else{ + memset(pNew+pageSize, 0, 8); + } } if( rc==SQLITE_OK ){ @@ -53347,7 +55348,6 @@ SQLITE_PRIVATE int sqlite3PagerSetPagesize(Pager *pPager, u32 *pPageSize, int nR if( nReserve<0 ) nReserve = pPager->nReserve; assert( nReserve>=0 && nReserve<1000 ); pPager->nReserve = (i16)nReserve; - pagerReportSize(pPager); pagerFixMaplimit(pPager); } return rc; @@ -53377,7 +55377,10 @@ SQLITE_PRIVATE int sqlite3PagerMaxPageCount(Pager *pPager, int mxPage){ pPager->mxPgno = mxPage; } assert( pPager->eState!=PAGER_OPEN ); /* Called only by OP_MaxPgcnt */ - assert( pPager->mxPgno>=pPager->dbSize ); /* OP_MaxPgcnt enforces this */ + /* assert( pPager->mxPgno>=pPager->dbSize ); */ + /* OP_MaxPgcnt ensures that the parameter passed to this function is not + ** less than the total number of valid pages in the database. But this + ** may be less than Pager.dbSize, and so the assert() above is not valid */ return pPager->mxPgno; } @@ -53740,11 +55743,6 @@ SQLITE_PRIVATE int sqlite3PagerClose(Pager *pPager, sqlite3 *db){ sqlite3OsClose(pPager->fd); sqlite3PageFree(pTmp); sqlite3PcacheClose(pPager->pPCache); - -#ifdef SQLITE_HAS_CODEC - if( pPager->xCodecFree ) pPager->xCodecFree(pPager->pCodec); -#endif - assert( !pPager->aSavepoint && !pPager->pInJournal ); assert( !isOpen(pPager->jfd) && !isOpen(pPager->sjfd) ); @@ -53995,8 +55993,7 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ assert( (pList->flags&PGHDR_NEED_SYNC)==0 ); if( pList->pgno==1 ) pager_write_changecounter(pList); - /* Encode the database */ - CODEC2(pPager, pList->pData, pgno, 6, return SQLITE_NOMEM_BKPT, pData); + pData = pList->pData; /* Write out the page data. */ rc = sqlite3OsWrite(pPager->fd, pData, pPager->pageSize, offset); @@ -54085,12 +56082,6 @@ static int subjournalPage(PgHdr *pPg){ void *pData = pPg->pData; i64 offset = (i64)pPager->nSubRec*(4+pPager->pageSize); char *pData2; - -#if SQLITE_HAS_CODEC - if( !pPager->subjInMemory ){ - CODEC2(pPager, pData, pPg->pgno, 7, return SQLITE_NOMEM_BKPT, pData2); - }else -#endif pData2 = pData; PAGERTRACE(("STMT-JOURNAL %d page %d\n", PAGERID(pPager), pPg->pgno)); rc = write32bits(pPager->sjfd, offset, pPg->pgno); @@ -54286,7 +56277,8 @@ SQLITE_PRIVATE int sqlite3PagerOpen( int pcacheSize = sqlite3PcacheSize(); /* Bytes to allocate for PCache */ u32 szPageDflt = SQLITE_DEFAULT_PAGE_SIZE; /* Default page size */ const char *zUri = 0; /* URI args to copy */ - int nUri = 0; /* Number of bytes of URI args at *zUri */ + int nUriByte = 1; /* Number of bytes of URI args at *zUri */ + int nUri = 0; /* Number of URI parameters */ /* Figure out how much space is required for each journal file-handle ** (there are two of them, the main journal and the sub-journal). */ @@ -54320,14 +56312,24 @@ SQLITE_PRIVATE int sqlite3PagerOpen( } zPathname[0] = 0; /* Make sure initialized even if FullPathname() fails */ rc = sqlite3OsFullPathname(pVfs, zFilename, nPathname, zPathname); + if( rc!=SQLITE_OK ){ + if( rc==SQLITE_OK_SYMLINK ){ + if( vfsFlags & SQLITE_OPEN_NOFOLLOW ){ + rc = SQLITE_CANTOPEN_SYMLINK; + }else{ + rc = SQLITE_OK; + } + } + } nPathname = sqlite3Strlen30(zPathname); z = zUri = &zFilename[sqlite3Strlen30(zFilename)+1]; while( *z ){ - z += sqlite3Strlen30(z)+1; - z += sqlite3Strlen30(z)+1; + z += strlen(z)+1; + z += strlen(z)+1; + nUri++; } - nUri = (int)(&z[1] - zUri); - assert( nUri>=0 ); + nUriByte = (int)(&z[1] - zUri); + assert( nUriByte>=1 ); if( rc==SQLITE_OK && nPathname+8>pVfs->mxPathname ){ /* This branch is taken when the journal path required by ** the database being opened will be more than pVfs->mxPathname @@ -54352,50 +56354,111 @@ SQLITE_PRIVATE int sqlite3PagerOpen( ** Database file handle (pVfs->szOsFile bytes) ** Sub-journal file handle (journalFileSize bytes) ** Main journal file handle (journalFileSize bytes) + ** Ptr back to the Pager (sizeof(Pager*) bytes) + ** \0\0\0\0 database prefix (4 bytes) ** Database file name (nPathname+1 bytes) - ** Journal file name (nPathname+8+1 bytes) + ** URI query parameters (nUriByte bytes) + ** Journal filename (nPathname+8+1 bytes) + ** WAL filename (nPathname+4+1 bytes) + ** \0\0\0 terminator (3 bytes) + ** + ** Some 3rd-party software, over which we have no control, depends on + ** the specific order of the filenames and the \0 separators between them + ** so that it can (for example) find the database filename given the WAL + ** filename without using the sqlite3_filename_database() API. This is a + ** misuse of SQLite and a bug in the 3rd-party software, but the 3rd-party + ** software is in widespread use, so we try to avoid changing the filename + ** order and formatting if possible. In particular, the details of the + ** filename format expected by 3rd-party software should be as follows: + ** + ** - Main Database Path + ** - \0 + ** - Multiple URI components consisting of: + ** - Key + ** - \0 + ** - Value + ** - \0 + ** - \0 + ** - Journal Path + ** - \0 + ** - WAL Path (zWALName) + ** - \0 + ** + ** The sqlite3_create_filename() interface and the databaseFilename() utility + ** that is used by sqlite3_filename_database() and kin also depend on the + ** specific formatting and order of the various filenames, so if the format + ** changes here, be sure to change it there as well. */ pPtr = (u8 *)sqlite3MallocZero( - ROUND8(sizeof(*pPager)) + /* Pager structure */ - ROUND8(pcacheSize) + /* PCache object */ - ROUND8(pVfs->szOsFile) + /* The main db file */ - journalFileSize * 2 + /* The two journal files */ - nPathname + 1 + nUri + /* zFilename */ - nPathname + 8 + 2 /* zJournal */ + ROUND8(sizeof(*pPager)) + /* Pager structure */ + ROUND8(pcacheSize) + /* PCache object */ + ROUND8(pVfs->szOsFile) + /* The main db file */ + journalFileSize * 2 + /* The two journal files */ + sizeof(pPager) + /* Space to hold a pointer */ + 4 + /* Database prefix */ + nPathname + 1 + /* database filename */ + nUriByte + /* query parameters */ + nPathname + 8 + 1 + /* Journal filename */ #ifndef SQLITE_OMIT_WAL - + nPathname + 4 + 2 /* zWal */ + nPathname + 4 + 1 + /* WAL filename */ #endif + 3 /* Terminator */ ); assert( EIGHT_BYTE_ALIGNMENT(SQLITE_INT_TO_PTR(journalFileSize)) ); if( !pPtr ){ sqlite3DbFree(0, zPathname); return SQLITE_NOMEM_BKPT; } - pPager = (Pager*)(pPtr); - pPager->pPCache = (PCache*)(pPtr += ROUND8(sizeof(*pPager))); - pPager->fd = (sqlite3_file*)(pPtr += ROUND8(pcacheSize)); - pPager->sjfd = (sqlite3_file*)(pPtr += ROUND8(pVfs->szOsFile)); - pPager->jfd = (sqlite3_file*)(pPtr += journalFileSize); - pPager->zFilename = (char*)(pPtr += journalFileSize); + pPager = (Pager*)pPtr; pPtr += ROUND8(sizeof(*pPager)); + pPager->pPCache = (PCache*)pPtr; pPtr += ROUND8(pcacheSize); + pPager->fd = (sqlite3_file*)pPtr; pPtr += ROUND8(pVfs->szOsFile); + pPager->sjfd = (sqlite3_file*)pPtr; pPtr += journalFileSize; + pPager->jfd = (sqlite3_file*)pPtr; pPtr += journalFileSize; assert( EIGHT_BYTE_ALIGNMENT(pPager->jfd) ); + memcpy(pPtr, &pPager, sizeof(pPager)); pPtr += sizeof(pPager); + + /* Fill in the Pager.zFilename and pPager.zQueryParam fields */ + pPtr += 4; /* Skip zero prefix */ + pPager->zFilename = (char*)pPtr; + if( nPathname>0 ){ + memcpy(pPtr, zPathname, nPathname); pPtr += nPathname + 1; + if( zUri ){ + memcpy(pPtr, zUri, nUriByte); pPtr += nUriByte; + }else{ + pPtr++; + } + } + + + /* Fill in Pager.zJournal */ + if( nPathname>0 ){ + pPager->zJournal = (char*)pPtr; + memcpy(pPtr, zPathname, nPathname); pPtr += nPathname; + memcpy(pPtr, "-journal",8); pPtr += 8 + 1; +#ifdef SQLITE_ENABLE_8_3_NAMES + sqlite3FileSuffix3(zFilename,pPager->zJournal); + pPtr = (u8*)(pPager->zJournal + sqlite3Strlen30(pPager->zJournal)+1); +#endif + }else{ + pPager->zJournal = 0; + } - /* Fill in the Pager.zFilename and Pager.zJournal buffers, if required. */ - if( zPathname ){ - assert( nPathname>0 ); - pPager->zJournal = (char*)(pPtr += nPathname + 1 + nUri); - memcpy(pPager->zFilename, zPathname, nPathname); - if( nUri ) memcpy(&pPager->zFilename[nPathname+1], zUri, nUri); - memcpy(pPager->zJournal, zPathname, nPathname); - memcpy(&pPager->zJournal[nPathname], "-journal\000", 8+2); - sqlite3FileSuffix3(pPager->zFilename, pPager->zJournal); #ifndef SQLITE_OMIT_WAL - pPager->zWal = &pPager->zJournal[nPathname+8+1]; - memcpy(pPager->zWal, zPathname, nPathname); - memcpy(&pPager->zWal[nPathname], "-wal\000", 4+1); - sqlite3FileSuffix3(pPager->zFilename, pPager->zWal); + /* Fill in Pager.zWal */ + if( nPathname>0 ){ + pPager->zWal = (char*)pPtr; + memcpy(pPtr, zPathname, nPathname); pPtr += nPathname; + memcpy(pPtr, "-wal", 4); pPtr += 4 + 1; +#ifdef SQLITE_ENABLE_8_3_NAMES + sqlite3FileSuffix3(zFilename, pPager->zWal); + pPtr = (u8*)(pPager->zWal + sqlite3Strlen30(pPager->zWal)+1); #endif - sqlite3DbFree(0, zPathname); + }else{ + pPager->zWal = 0; } +#endif + + if( nPathname ) sqlite3DbFree(0, zPathname); pPager->pVfs = pVfs; pPager->vfsFlags = vfsFlags; @@ -54444,9 +56507,9 @@ SQLITE_PRIVATE int sqlite3PagerOpen( } #endif } - pPager->noLock = sqlite3_uri_boolean(zFilename, "nolock", 0); + pPager->noLock = sqlite3_uri_boolean(pPager->zFilename, "nolock", 0); if( (iDc & SQLITE_IOCAP_IMMUTABLE)!=0 - || sqlite3_uri_boolean(zFilename, "immutable", 0) ){ + || sqlite3_uri_boolean(pPager->zFilename, "immutable", 0) ){ vfsFlags |= SQLITE_OPEN_READONLY; goto act_like_temp_file; } @@ -54553,6 +56616,19 @@ SQLITE_PRIVATE int sqlite3PagerOpen( return SQLITE_OK; } +/* +** Return the sqlite3_file for the main database given the name +** of the corresonding WAL or Journal name as passed into +** xOpen. +*/ +SQLITE_API sqlite3_file *sqlite3_database_file_object(const char *zName){ + Pager *pPager; + while( zName[-1]!=0 || zName[-2]!=0 || zName[-3]!=0 || zName[-4]!=0 ){ + zName--; + } + pPager = *(Pager**)(zName - 4 - sizeof(Pager*)); + return pPager->fd; +} /* @@ -55108,9 +57184,6 @@ static int getPageMMap( ); assert( USEFETCH(pPager) ); -#ifdef SQLITE_HAS_CODEC - assert( pPager->xCodec==0 ); -#endif /* Optimization note: Adding the "pgno<=1" term before "pgno==0" here ** allows the compiler optimizer to reuse the results of the "pgno>1" @@ -55241,7 +57314,6 @@ SQLITE_PRIVATE void sqlite3PagerUnrefPageOne(DbPage *pPg){ assert( pPg->pgno==1 ); assert( (pPg->flags & PGHDR_MMAP)==0 ); /* Page1 is never memory mapped */ pPager = pPg->pPager; - sqlite3PagerResetLockTimeout(pPager); sqlite3PcacheRelease(pPg); pagerUnlockIfUnused(pPager); } @@ -55439,7 +57511,7 @@ static SQLITE_NOINLINE int pagerAddPageToRollbackJournal(PgHdr *pPg){ assert( pPg->pgno!=PAGER_MJ_PGNO(pPager) ); assert( pPager->journalHdr<=pPager->journalOff ); - CODEC2(pPager, pPg->pData, pPg->pgno, 7, return SQLITE_NOMEM_BKPT, pData2); + pData2 = pPg->pData; cksum = pager_cksum(pPager, (u8*)pData2); /* Even if an IO or diskfull error occurs while journalling the @@ -55804,7 +57876,7 @@ static int pager_incr_changecounter(Pager *pPager, int isDirectMode){ if( DIRECT_MODE ){ const void *zBuf; assert( pPager->dbFileSize>0 ); - CODEC2(pPager, pPgHdr->pData, 1, 6, rc=SQLITE_NOMEM_BKPT, zBuf); + zBuf = pPgHdr->pData; if( rc==SQLITE_OK ){ rc = sqlite3OsWrite(pPager->fd, zBuf, pPager->pageSize, 0); pPager->aStat[PAGER_STAT_WRITE]++; @@ -55934,9 +58006,10 @@ SQLITE_PRIVATE int sqlite3PagerCommitPhaseOne( ** backup in progress needs to be restarted. */ sqlite3BackupRestart(pPager->pBackup); }else{ + PgHdr *pList; if( pagerUseWal(pPager) ){ - PgHdr *pList = sqlite3PcacheDirtyList(pPager->pPCache); PgHdr *pPageOne = 0; + pList = sqlite3PcacheDirtyList(pPager->pPCache); if( pList==0 ){ /* Must have at least one page for the WAL commit flag. ** Ticket [2d1a5c67dfc2363e44f29d9bbd57f] 2011-05-18 */ @@ -55957,14 +58030,14 @@ SQLITE_PRIVATE int sqlite3PagerCommitPhaseOne( ** should be used. No rollback journal is created if batch-atomic-write ** is enabled. */ - sqlite3_file *fd = pPager->fd; #ifdef SQLITE_ENABLE_BATCH_ATOMIC_WRITE - const int bBatch = zMaster==0 /* An SQLITE_IOCAP_BATCH_ATOMIC commit */ + sqlite3_file *fd = pPager->fd; + int bBatch = zMaster==0 /* An SQLITE_IOCAP_BATCH_ATOMIC commit */ && (sqlite3OsDeviceCharacteristics(fd) & SQLITE_IOCAP_BATCH_ATOMIC) && !pPager->noSync && sqlite3JournalIsInMemory(pPager->jfd); #else -# define bBatch 0 +# define bBatch 0 #endif #ifdef SQLITE_ENABLE_ATOMIC_WRITE @@ -56016,15 +58089,16 @@ SQLITE_PRIVATE int sqlite3PagerCommitPhaseOne( } } } -#else +#else /* SQLITE_ENABLE_ATOMIC_WRITE */ #ifdef SQLITE_ENABLE_BATCH_ATOMIC_WRITE if( zMaster ){ rc = sqlite3JournalCreate(pPager->jfd); if( rc!=SQLITE_OK ) goto commit_phase_one_exit; + assert( bBatch==0 ); } #endif rc = pager_incr_changecounter(pPager, 0); -#endif +#endif /* !SQLITE_ENABLE_ATOMIC_WRITE */ if( rc!=SQLITE_OK ) goto commit_phase_one_exit; /* Write the master journal name into the journal file. If a master @@ -56048,24 +58122,36 @@ SQLITE_PRIVATE int sqlite3PagerCommitPhaseOne( rc = syncJournal(pPager, 0); if( rc!=SQLITE_OK ) goto commit_phase_one_exit; + pList = sqlite3PcacheDirtyList(pPager->pPCache); +#ifdef SQLITE_ENABLE_BATCH_ATOMIC_WRITE if( bBatch ){ - /* The pager is now in DBMOD state. But regardless of what happens - ** next, attempting to play the journal back into the database would - ** be unsafe. Close it now to make sure that does not happen. */ - sqlite3OsClose(pPager->jfd); rc = sqlite3OsFileControl(fd, SQLITE_FCNTL_BEGIN_ATOMIC_WRITE, 0); - if( rc!=SQLITE_OK ) goto commit_phase_one_exit; - } - rc = pager_write_pagelist(pPager,sqlite3PcacheDirtyList(pPager->pPCache)); - if( bBatch ){ if( rc==SQLITE_OK ){ - rc = sqlite3OsFileControl(fd, SQLITE_FCNTL_COMMIT_ATOMIC_WRITE, 0); + rc = pager_write_pagelist(pPager, pList); + if( rc==SQLITE_OK ){ + rc = sqlite3OsFileControl(fd, SQLITE_FCNTL_COMMIT_ATOMIC_WRITE, 0); + } + if( rc!=SQLITE_OK ){ + sqlite3OsFileControlHint(fd, SQLITE_FCNTL_ROLLBACK_ATOMIC_WRITE, 0); + } } - if( rc!=SQLITE_OK ){ - sqlite3OsFileControlHint(fd, SQLITE_FCNTL_ROLLBACK_ATOMIC_WRITE, 0); + + if( (rc&0xFF)==SQLITE_IOERR && rc!=SQLITE_IOERR_NOMEM ){ + rc = sqlite3JournalCreate(pPager->jfd); + if( rc!=SQLITE_OK ){ + sqlite3OsClose(pPager->jfd); + goto commit_phase_one_exit; + } + bBatch = 0; + }else{ + sqlite3OsClose(pPager->jfd); } } +#endif /* SQLITE_ENABLE_BATCH_ATOMIC_WRITE */ + if( bBatch==0 ){ + rc = pager_write_pagelist(pPager, pList); + } if( rc!=SQLITE_OK ){ assert( rc!=SQLITE_IOERR_BLOCKED ); goto commit_phase_one_exit; @@ -56123,6 +58209,7 @@ SQLITE_PRIVATE int sqlite3PagerCommitPhaseTwo(Pager *pPager){ ** But if (due to a coding error elsewhere in the system) it does get ** called, just return the same error code without doing anything. */ if( NEVER(pPager->errCode) ) return pPager->errCode; + pPager->iDataVersion++; assert( pPager->eState==PAGER_WRITER_LOCKED || pPager->eState==PAGER_WRITER_FINISHED @@ -56151,7 +58238,6 @@ SQLITE_PRIVATE int sqlite3PagerCommitPhaseTwo(Pager *pPager){ } PAGERTRACE(("COMMIT %d\n", PAGERID(pPager))); - pPager->iDataVersion++; rc = pager_end_transaction(pPager, pPager->setMaster, 1); return pager_error(pPager, rc); } @@ -56495,9 +58581,13 @@ SQLITE_PRIVATE int sqlite3PagerSavepoint(Pager *pPager, int op, int iSavepoint){ ** behavior. But when the Btree needs to know the filename for matching to ** shared cache, it uses nullIfMemDb==0 so that in-memory databases can ** participate in shared-cache. +** +** The return value to this routine is always safe to use with +** sqlite3_uri_parameter() and sqlite3_filename_database() and friends. */ -SQLITE_PRIVATE const char *sqlite3PagerFilename(Pager *pPager, int nullIfMemDb){ - return (nullIfMemDb && pPager->memDb) ? "" : pPager->zFilename; +SQLITE_PRIVATE const char *sqlite3PagerFilename(const Pager *pPager, int nullIfMemDb){ + static const char zFake[8] = { 0, 0, 0, 0, 0, 0, 0, 0 }; + return (nullIfMemDb && pPager->memDb) ? &zFake[4] : pPager->zFilename; } /* @@ -56516,16 +58606,6 @@ SQLITE_PRIVATE sqlite3_file *sqlite3PagerFile(Pager *pPager){ return pPager->fd; } -#ifdef SQLITE_ENABLE_SETLK_TIMEOUT -/* -** Reset the lock timeout for pager. -*/ -SQLITE_PRIVATE void sqlite3PagerResetLockTimeout(Pager *pPager){ - int x = 0; - sqlite3OsFileControl(pPager->fd, SQLITE_FCNTL_LOCK_TIMEOUT, &x); -} -#endif - /* ** Return the file handle for the journal file (if it exists). ** This will be either the rollback journal or the WAL file. @@ -56545,50 +58625,6 @@ SQLITE_PRIVATE const char *sqlite3PagerJournalname(Pager *pPager){ return pPager->zJournal; } -#ifdef SQLITE_HAS_CODEC -/* -** Set or retrieve the codec for this pager -*/ -SQLITE_PRIVATE void sqlite3PagerSetCodec( - Pager *pPager, - void *(*xCodec)(void*,void*,Pgno,int), - void (*xCodecSizeChng)(void*,int,int), - void (*xCodecFree)(void*), - void *pCodec -){ - if( pPager->xCodecFree ) pPager->xCodecFree(pPager->pCodec); - pPager->xCodec = pPager->memDb ? 0 : xCodec; - pPager->xCodecSizeChng = xCodecSizeChng; - pPager->xCodecFree = xCodecFree; - pPager->pCodec = pCodec; - setGetterMethod(pPager); - pagerReportSize(pPager); -} -SQLITE_PRIVATE void *sqlite3PagerGetCodec(Pager *pPager){ - return pPager->pCodec; -} - -/* -** This function is called by the wal module when writing page content -** into the log file. -** -** This function returns a pointer to a buffer containing the encrypted -** page content. If a malloc fails, this function may return NULL. -*/ -SQLITE_PRIVATE void *sqlite3PagerCodec(PgHdr *pPg){ - void *aData = 0; - CODEC2(pPg->pPager, pPg->pData, pPg->pgno, 6, return 0, aData); - return aData; -} - -/* -** Return the current pager state -*/ -SQLITE_PRIVATE int sqlite3PagerState(Pager *pPager){ - return pPager->eState; -} -#endif /* SQLITE_HAS_CODEC */ - #ifndef SQLITE_OMIT_AUTOVACUUM /* ** Move the page pPg to location pgno in the file. @@ -56685,8 +58721,12 @@ SQLITE_PRIVATE int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, i */ pPg->flags &= ~PGHDR_NEED_SYNC; pPgOld = sqlite3PagerLookup(pPager, pgno); - assert( !pPgOld || pPgOld->nRef==1 ); + assert( !pPgOld || pPgOld->nRef==1 || CORRUPT_DB ); if( pPgOld ){ + if( pPgOld->nRef>1 ){ + sqlite3PagerUnrefNotNull(pPgOld); + return SQLITE_CORRUPT_BKPT; + } pPg->flags |= (pPgOld->flags&PGHDR_NEED_SYNC); if( pPager->tempFile ){ /* Do not discard pages from an in-memory database since we might @@ -56817,13 +58857,6 @@ SQLITE_PRIVATE int sqlite3PagerLockingMode(Pager *pPager, int eMode){ SQLITE_PRIVATE int sqlite3PagerSetJournalMode(Pager *pPager, int eMode){ u8 eOld = pPager->journalMode; /* Prior journalmode */ -#ifdef SQLITE_DEBUG - /* The print_pager_state() routine is intended to be used by the debugger - ** only. We invoke it once here to suppress a compiler warning. */ - print_pager_state(pPager); -#endif - - /* The eMode parameter is always valid */ assert( eMode==PAGER_JOURNALMODE_DELETE || eMode==PAGER_JOURNALMODE_TRUNCATE @@ -56986,7 +59019,6 @@ SQLITE_PRIVATE int sqlite3PagerCheckpoint( pPager->walSyncFlags, pPager->pageSize, (u8 *)pPager->pTmpSpace, pnLog, pnCkpt ); - sqlite3PagerResetLockTimeout(pPager); } return rc; } @@ -57151,6 +59183,32 @@ SQLITE_PRIVATE int sqlite3PagerCloseWal(Pager *pPager, sqlite3 *db){ return rc; } +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT +/* +** If pager pPager is a wal-mode database not in exclusive locking mode, +** invoke the sqlite3WalWriteLock() function on the associated Wal object +** with the same db and bLock parameters as were passed to this function. +** Return an SQLite error code if an error occurs, or SQLITE_OK otherwise. +*/ +SQLITE_PRIVATE int sqlite3PagerWalWriteLock(Pager *pPager, int bLock){ + int rc = SQLITE_OK; + if( pagerUseWal(pPager) && pPager->exclusiveMode==0 ){ + rc = sqlite3WalWriteLock(pPager->pWal, bLock); + } + return rc; +} + +/* +** Set the database handle used by the wal layer to determine if +** blocking locks are required. +*/ +SQLITE_PRIVATE void sqlite3PagerWalDb(Pager *pPager, sqlite3 *db){ + if( pagerUseWal(pPager) ){ + sqlite3WalDb(pPager->pWal, db); + } +} +#endif + #ifdef SQLITE_ENABLE_SNAPSHOT /* ** If this is a WAL database, obtain a snapshot handle for the snapshot @@ -57169,7 +59227,10 @@ SQLITE_PRIVATE int sqlite3PagerSnapshotGet(Pager *pPager, sqlite3_snapshot **ppS ** read transaction is opened, attempt to read from the snapshot it ** identifies. If this is not a WAL database, return an error. */ -SQLITE_PRIVATE int sqlite3PagerSnapshotOpen(Pager *pPager, sqlite3_snapshot *pSnapshot){ +SQLITE_PRIVATE int sqlite3PagerSnapshotOpen( + Pager *pPager, + sqlite3_snapshot *pSnapshot +){ int rc = SQLITE_OK; if( pPager->pWal ){ sqlite3WalSnapshotOpen(pPager->pWal, pSnapshot); @@ -57192,6 +59253,38 @@ SQLITE_PRIVATE int sqlite3PagerSnapshotRecover(Pager *pPager){ } return rc; } + +/* +** The caller currently has a read transaction open on the database. +** If this is not a WAL database, SQLITE_ERROR is returned. Otherwise, +** this function takes a SHARED lock on the CHECKPOINTER slot and then +** checks if the snapshot passed as the second argument is still +** available. If so, SQLITE_OK is returned. +** +** If the snapshot is not available, SQLITE_ERROR is returned. Or, if +** the CHECKPOINTER lock cannot be obtained, SQLITE_BUSY. If any error +** occurs (any value other than SQLITE_OK is returned), the CHECKPOINTER +** lock is released before returning. +*/ +SQLITE_PRIVATE int sqlite3PagerSnapshotCheck(Pager *pPager, sqlite3_snapshot *pSnapshot){ + int rc; + if( pPager->pWal ){ + rc = sqlite3WalSnapshotCheck(pPager->pWal, pSnapshot); + }else{ + rc = SQLITE_ERROR; + } + return rc; +} + +/* +** Release a lock obtained by an earlier successful call to +** sqlite3PagerSnapshotCheck(). +*/ +SQLITE_PRIVATE void sqlite3PagerSnapshotUnlock(Pager *pPager){ + assert( pPager->pWal ); + sqlite3WalSnapshotUnlock(pPager->pWal); +} + #endif /* SQLITE_ENABLE_SNAPSHOT */ #endif /* !SQLITE_OMIT_WAL */ @@ -57682,6 +59775,9 @@ struct Wal { #ifdef SQLITE_ENABLE_SNAPSHOT WalIndexHdr *pSnapshot; /* Start transaction here if not NULL */ #endif +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + sqlite3 *db; +#endif }; /* @@ -57778,9 +59874,9 @@ static SQLITE_NOINLINE int walIndexPageRealloc( /* Enlarge the pWal->apWiData[] array if required */ if( pWal->nWiData<=iPage ){ - int nByte = sizeof(u32*)*(iPage+1); + sqlite3_int64 nByte = sizeof(u32*)*(iPage+1); volatile u32 **apNew; - apNew = (volatile u32 **)sqlite3_realloc64((void *)pWal->apWiData, nByte); + apNew = (volatile u32 **)sqlite3Realloc((void *)pWal->apWiData, nByte); if( !apNew ){ *ppPage = 0; return SQLITE_NOMEM_BKPT; @@ -57882,6 +59978,7 @@ static void walChecksumBytes( assert( nByte>=8 ); assert( (nByte&0x00000007)==0 ); + assert( nByte<=65536 ); if( nativeCksum ){ do { @@ -57900,18 +59997,35 @@ static void walChecksumBytes( aOut[1] = s2; } +/* +** If there is the possibility of concurrent access to the SHM file +** from multiple threads and/or processes, then do a memory barrier. +*/ static void walShmBarrier(Wal *pWal){ if( pWal->exclusiveMode!=WAL_HEAPMEMORY_MODE ){ sqlite3OsShmBarrier(pWal->pDbFd); } } +/* +** Add the SQLITE_NO_TSAN as part of the return-type of a function +** definition as a hint that the function contains constructs that +** might give false-positive TSAN warnings. +** +** See tag-20200519-1. +*/ +#if defined(__clang__) && !defined(SQLITE_NO_TSAN) +# define SQLITE_NO_TSAN __attribute__((no_sanitize_thread)) +#else +# define SQLITE_NO_TSAN +#endif + /* ** Write the header information in pWal->hdr into the wal-index. ** ** The checksum on pWal->hdr is updated before it is written. */ -static void walIndexWriteHdr(Wal *pWal){ +static SQLITE_NO_TSAN void walIndexWriteHdr(Wal *pWal){ volatile WalIndexHdr *aHdr = walIndexHdr(pWal); const int nCksum = offsetof(WalIndexHdr, aCksum); @@ -57919,6 +60033,7 @@ static void walIndexWriteHdr(Wal *pWal){ pWal->hdr.isInit = 1; pWal->hdr.iVersion = WALINDEX_MAX_VERSION; walChecksumBytes(1, (u8*)&pWal->hdr, nCksum, 0, pWal->hdr.aCksum); + /* Possible TSAN false-positive. See tag-20200519-1 */ memcpy((void*)&aHdr[1], (const void*)&pWal->hdr, sizeof(WalIndexHdr)); walShmBarrier(pWal); memcpy((void*)&aHdr[0], (const void*)&pWal->hdr, sizeof(WalIndexHdr)); @@ -58054,7 +60169,7 @@ static int walLockShared(Wal *pWal, int lockIdx){ SQLITE_SHM_LOCK | SQLITE_SHM_SHARED); WALTRACE(("WAL%p: acquire SHARED-%s %s\n", pWal, walLockName(lockIdx), rc ? "failed" : "ok")); - VVA_ONLY( pWal->lockError = (u8)(rc!=SQLITE_OK && rc!=SQLITE_BUSY); ) + VVA_ONLY( pWal->lockError = (u8)(rc!=SQLITE_OK && (rc&0xFF)!=SQLITE_BUSY); ) return rc; } static void walUnlockShared(Wal *pWal, int lockIdx){ @@ -58070,7 +60185,7 @@ static int walLockExclusive(Wal *pWal, int lockIdx, int n){ SQLITE_SHM_LOCK | SQLITE_SHM_EXCLUSIVE); WALTRACE(("WAL%p: acquire EXCLUSIVE-%s cnt=%d %s\n", pWal, walLockName(lockIdx), n, rc ? "failed" : "ok")); - VVA_ONLY( pWal->lockError = (u8)(rc!=SQLITE_OK && rc!=SQLITE_BUSY); ) + VVA_ONLY( pWal->lockError = (u8)(rc!=SQLITE_OK && (rc&0xFF)!=SQLITE_BUSY); ) return rc; } static void walUnlockExclusive(Wal *pWal, int lockIdx, int n){ @@ -58095,48 +60210,51 @@ static int walNextHash(int iPriorHash){ return (iPriorHash+1)&(HASHTABLE_NSLOT-1); } +/* +** An instance of the WalHashLoc object is used to describe the location +** of a page hash table in the wal-index. This becomes the return value +** from walHashGet(). +*/ +typedef struct WalHashLoc WalHashLoc; +struct WalHashLoc { + volatile ht_slot *aHash; /* Start of the wal-index hash table */ + volatile u32 *aPgno; /* aPgno[1] is the page of first frame indexed */ + u32 iZero; /* One less than the frame number of first indexed*/ +}; + /* ** Return pointers to the hash table and page number array stored on ** page iHash of the wal-index. The wal-index is broken into 32KB pages ** numbered starting from 0. ** -** Set output variable *paHash to point to the start of the hash table -** in the wal-index file. Set *piZero to one less than the frame +** Set output variable pLoc->aHash to point to the start of the hash table +** in the wal-index file. Set pLoc->iZero to one less than the frame ** number of the first frame indexed by this hash table. If a ** slot in the hash table is set to N, it refers to frame number -** (*piZero+N) in the log. +** (pLoc->iZero+N) in the log. ** -** Finally, set *paPgno so that *paPgno[1] is the page number of the -** first frame indexed by the hash table, frame (*piZero+1). +** Finally, set pLoc->aPgno so that pLoc->aPgno[1] is the page number of the +** first frame indexed by the hash table, frame (pLoc->iZero+1). */ static int walHashGet( Wal *pWal, /* WAL handle */ int iHash, /* Find the iHash'th table */ - volatile ht_slot **paHash, /* OUT: Pointer to hash index */ - volatile u32 **paPgno, /* OUT: Pointer to page number array */ - u32 *piZero /* OUT: Frame associated with *paPgno[0] */ + WalHashLoc *pLoc /* OUT: Hash table location */ ){ int rc; /* Return code */ - volatile u32 *aPgno; - rc = walIndexPage(pWal, iHash, &aPgno); + rc = walIndexPage(pWal, iHash, &pLoc->aPgno); assert( rc==SQLITE_OK || iHash>0 ); if( rc==SQLITE_OK ){ - u32 iZero; - volatile ht_slot *aHash; - - aHash = (volatile ht_slot *)&aPgno[HASHTABLE_NPAGE]; + pLoc->aHash = (volatile ht_slot *)&pLoc->aPgno[HASHTABLE_NPAGE]; if( iHash==0 ){ - aPgno = &aPgno[WALINDEX_HDR_SIZE/sizeof(u32)]; - iZero = 0; + pLoc->aPgno = &pLoc->aPgno[WALINDEX_HDR_SIZE/sizeof(u32)]; + pLoc->iZero = 0; }else{ - iZero = HASHTABLE_NPAGE_ONE + (iHash-1)*HASHTABLE_NPAGE; + pLoc->iZero = HASHTABLE_NPAGE_ONE + (iHash-1)*HASHTABLE_NPAGE; } - - *paPgno = &aPgno[-1]; - *paHash = aHash; - *piZero = iZero; + pLoc->aPgno = &pLoc->aPgno[-1]; } return rc; } @@ -58182,12 +60300,11 @@ static u32 walFramePgno(Wal *pWal, u32 iFrame){ ** actually needed. */ static void walCleanupHash(Wal *pWal){ - volatile ht_slot *aHash = 0; /* Pointer to hash table to clear */ - volatile u32 *aPgno = 0; /* Page number array for hash table */ - u32 iZero = 0; /* frame == (aHash[x]+iZero) */ + WalHashLoc sLoc; /* Hash table location */ int iLimit = 0; /* Zero values greater than this */ int nByte; /* Number of bytes to zero in aPgno[] */ int i; /* Used to iterate through aHash[] */ + int rc; /* Return code form walHashGet() */ assert( pWal->writeLock ); testcase( pWal->hdr.mxFrame==HASHTABLE_NPAGE_ONE-1 ); @@ -58198,28 +60315,29 @@ static void walCleanupHash(Wal *pWal){ /* Obtain pointers to the hash-table and page-number array containing ** the entry that corresponds to frame pWal->hdr.mxFrame. It is guaranteed - ** that the page said hash-table and array reside on is already mapped. + ** that the page said hash-table and array reside on is already mapped.(1) */ assert( pWal->nWiData>walFramePage(pWal->hdr.mxFrame) ); assert( pWal->apWiData[walFramePage(pWal->hdr.mxFrame)] ); - walHashGet(pWal, walFramePage(pWal->hdr.mxFrame), &aHash, &aPgno, &iZero); + rc = walHashGet(pWal, walFramePage(pWal->hdr.mxFrame), &sLoc); + if( NEVER(rc) ) return; /* Defense-in-depth, in case (1) above is wrong */ /* Zero all hash-table entries that correspond to frame numbers greater ** than pWal->hdr.mxFrame. */ - iLimit = pWal->hdr.mxFrame - iZero; + iLimit = pWal->hdr.mxFrame - sLoc.iZero; assert( iLimit>0 ); for(i=0; iiLimit ){ - aHash[i] = 0; + if( sLoc.aHash[i]>iLimit ){ + sLoc.aHash[i] = 0; } } /* Zero the entries in the aPgno array that correspond to frames with ** frame numbers greater than pWal->hdr.mxFrame. */ - nByte = (int)((char *)aHash - (char *)&aPgno[iLimit+1]); - memset((void *)&aPgno[iLimit+1], 0, nByte); + nByte = (int)((char *)sLoc.aHash - (char *)&sLoc.aPgno[iLimit+1]); + memset((void *)&sLoc.aPgno[iLimit+1], 0, nByte); #ifdef SQLITE_ENABLE_EXPENSIVE_ASSERT /* Verify that the every entry in the mapping region is still reachable @@ -58229,10 +60347,10 @@ static void walCleanupHash(Wal *pWal){ int j; /* Loop counter */ int iKey; /* Hash key */ for(j=1; j<=iLimit; j++){ - for(iKey=walHash(aPgno[j]); aHash[iKey]; iKey=walNextHash(iKey)){ - if( aHash[iKey]==j ) break; + for(iKey=walHash(sLoc.aPgno[j]);sLoc.aHash[iKey];iKey=walNextHash(iKey)){ + if( sLoc.aHash[iKey]==j ) break; } - assert( aHash[iKey]==j ); + assert( sLoc.aHash[iKey]==j ); } } #endif /* SQLITE_ENABLE_EXPENSIVE_ASSERT */ @@ -58245,11 +60363,9 @@ static void walCleanupHash(Wal *pWal){ */ static int walIndexAppend(Wal *pWal, u32 iFrame, u32 iPage){ int rc; /* Return code */ - u32 iZero = 0; /* One less than frame number of aPgno[1] */ - volatile u32 *aPgno = 0; /* Page number array */ - volatile ht_slot *aHash = 0; /* Hash table */ + WalHashLoc sLoc; /* Wal-index hash table location */ - rc = walHashGet(pWal, walFramePage(iFrame), &aHash, &aPgno, &iZero); + rc = walHashGet(pWal, walFramePage(iFrame), &sLoc); /* Assuming the wal-index file was successfully mapped, populate the ** page number array and hash table entry. @@ -58259,15 +60375,16 @@ static int walIndexAppend(Wal *pWal, u32 iFrame, u32 iPage){ int idx; /* Value to write to hash-table slot */ int nCollide; /* Number of hash collisions */ - idx = iFrame - iZero; + idx = iFrame - sLoc.iZero; assert( idx <= HASHTABLE_NSLOT/2 + 1 ); /* If this is the first entry to be added to this hash-table, zero the ** entire hash table and aPgno[] array before proceeding. */ if( idx==1 ){ - int nByte = (int)((u8 *)&aHash[HASHTABLE_NSLOT] - (u8 *)&aPgno[1]); - memset((void*)&aPgno[1], 0, nByte); + int nByte = (int)((u8 *)&sLoc.aHash[HASHTABLE_NSLOT] + - (u8 *)&sLoc.aPgno[1]); + memset((void*)&sLoc.aPgno[1], 0, nByte); } /* If the entry in aPgno[] is already set, then the previous writer @@ -58276,18 +60393,18 @@ static int walIndexAppend(Wal *pWal, u32 iFrame, u32 iPage){ ** Remove the remnants of that writers uncommitted transaction from ** the hash-table before writing any new entries. */ - if( aPgno[idx] ){ + if( sLoc.aPgno[idx] ){ walCleanupHash(pWal); - assert( !aPgno[idx] ); + assert( !sLoc.aPgno[idx] ); } /* Write the aPgno[] array entry and the hash-table slot. */ nCollide = idx; - for(iKey=walHash(iPage); aHash[iKey]; iKey=walNextHash(iKey)){ + for(iKey=walHash(iPage); sLoc.aHash[iKey]; iKey=walNextHash(iKey)){ if( (nCollide--)==0 ) return SQLITE_CORRUPT_BKPT; } - aPgno[idx] = iPage; - aHash[iKey] = (ht_slot)idx; + sLoc.aPgno[idx] = iPage; + sLoc.aHash[iKey] = (ht_slot)idx; #ifdef SQLITE_ENABLE_EXPENSIVE_ASSERT /* Verify that the number of entries in the hash table exactly equals @@ -58296,7 +60413,7 @@ static int walIndexAppend(Wal *pWal, u32 iFrame, u32 iPage){ { int i; /* Loop counter */ int nEntry = 0; /* Number of entries in the hash table */ - for(i=0; iaSegment[p->nSegment])[iZero]; - iZero++; + aIndex = &((ht_slot *)&p->aSegment[p->nSegment])[sLoc.iZero]; + sLoc.iZero++; for(j=0; jaSegment[i].iZero = iZero; + walMergesort((u32 *)sLoc.aPgno, aTmp, aIndex, &nEntry); + p->aSegment[i].iZero = sLoc.iZero; p->aSegment[i].nEntry = nEntry; p->aSegment[i].aIndex = aIndex; - p->aSegment[i].aPgno = (u32 *)aPgno; + p->aSegment[i].aPgno = (u32 *)sLoc.aPgno; } } sqlite3_free(aTmp); @@ -58888,6 +61005,89 @@ static int walIteratorInit(Wal *pWal, u32 nBackfill, WalIterator **pp){ return rc; } +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT +/* +** Attempt to enable blocking locks. Blocking locks are enabled only if (a) +** they are supported by the VFS, and (b) the database handle is configured +** with a busy-timeout. Return 1 if blocking locks are successfully enabled, +** or 0 otherwise. +*/ +static int walEnableBlocking(Wal *pWal){ + int res = 0; + if( pWal->db ){ + int tmout = pWal->db->busyTimeout; + if( tmout ){ + int rc; + rc = sqlite3OsFileControl( + pWal->pDbFd, SQLITE_FCNTL_LOCK_TIMEOUT, (void*)&tmout + ); + res = (rc==SQLITE_OK); + } + } + return res; +} + +/* +** Disable blocking locks. +*/ +static void walDisableBlocking(Wal *pWal){ + int tmout = 0; + sqlite3OsFileControl(pWal->pDbFd, SQLITE_FCNTL_LOCK_TIMEOUT, (void*)&tmout); +} + +/* +** If parameter bLock is true, attempt to enable blocking locks, take +** the WRITER lock, and then disable blocking locks. If blocking locks +** cannot be enabled, no attempt to obtain the WRITER lock is made. Return +** an SQLite error code if an error occurs, or SQLITE_OK otherwise. It is not +** an error if blocking locks can not be enabled. +** +** If the bLock parameter is false and the WRITER lock is held, release it. +*/ +SQLITE_PRIVATE int sqlite3WalWriteLock(Wal *pWal, int bLock){ + int rc = SQLITE_OK; + assert( pWal->readLock<0 || bLock==0 ); + if( bLock ){ + assert( pWal->db ); + if( walEnableBlocking(pWal) ){ + rc = walLockExclusive(pWal, WAL_WRITE_LOCK, 1); + if( rc==SQLITE_OK ){ + pWal->writeLock = 1; + } + walDisableBlocking(pWal); + } + }else if( pWal->writeLock ){ + walUnlockExclusive(pWal, WAL_WRITE_LOCK, 1); + pWal->writeLock = 0; + } + return rc; +} + +/* +** Set the database handle used to determine if blocking locks are required. +*/ +SQLITE_PRIVATE void sqlite3WalDb(Wal *pWal, sqlite3 *db){ + pWal->db = db; +} + +/* +** Take an exclusive WRITE lock. Blocking if so configured. +*/ +static int walLockWriter(Wal *pWal){ + int rc; + walEnableBlocking(pWal); + rc = walLockExclusive(pWal, WAL_WRITE_LOCK, 1); + walDisableBlocking(pWal); + return rc; +} +#else +# define walEnableBlocking(x) 0 +# define walDisableBlocking(x) +# define walLockWriter(pWal) walLockExclusive((pWal), WAL_WRITE_LOCK, 1) +# define sqlite3WalDb(pWal, db) +#endif /* ifdef SQLITE_ENABLE_SETLK_TIMEOUT */ + + /* ** Attempt to obtain the exclusive WAL lock defined by parameters lockIdx and ** n. If the attempt fails and parameter xBusy is not NULL, then it is a @@ -58905,6 +61105,12 @@ static int walBusyLock( do { rc = walLockExclusive(pWal, lockIdx, n); }while( xBusy && rc==SQLITE_BUSY && xBusy(pBusyArg) ); +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + if( rc==SQLITE_BUSY_TIMEOUT ){ + walDisableBlocking(pWal); + rc = SQLITE_BUSY; + } +#endif return rc; } @@ -58942,7 +61148,7 @@ static void walRestartHdr(Wal *pWal, u32 salt1){ sqlite3Put4byte((u8*)&aSalt[0], 1 + sqlite3Get4byte((u8*)&aSalt[0])); memcpy(&pWal->hdr.aSalt[1], &salt1, 4); walIndexWriteHdr(pWal); - pInfo->nBackfill = 0; + AtomicStore(&pInfo->nBackfill, 0); pInfo->nBackfillAttempted = 0; pInfo->aReadMark[1] = 0; for(i=2; iaReadMark[i] = READMARK_NOT_USED; @@ -59017,20 +61223,13 @@ static int walCheckpoint( mxSafeFrame = pWal->hdr.mxFrame; mxPage = pWal->hdr.nPage; for(i=1; iaReadMark[i]; + u32 y = AtomicLoad(pInfo->aReadMark+i); if( mxSafeFrame>y ){ assert( y<=pWal->hdr.mxFrame ); rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(i), 1); if( rc==SQLITE_OK ){ - pInfo->aReadMark[i] = (i==1 ? mxSafeFrame : READMARK_NOT_USED); + u32 iMark = (i==1 ? mxSafeFrame : READMARK_NOT_USED); + AtomicStore(pInfo->aReadMark+i, iMark); walUnlockExclusive(pWal, WAL_READ_LOCK(i), 1); }else if( rc==SQLITE_BUSY ){ mxSafeFrame = y; @@ -59048,9 +61247,8 @@ static int walCheckpoint( } if( pIter - && (rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(0),1))==SQLITE_OK + && (rc = walBusyLock(pWal,xBusy,pBusyArg,WAL_READ_LOCK(0),1))==SQLITE_OK ){ - i64 nSize; /* Current size of database file */ u32 nBackfill = pInfo->nBackfill; pInfo->nBackfillAttempted = mxSafeFrame; @@ -59063,6 +61261,8 @@ static int walCheckpoint( */ if( rc==SQLITE_OK ){ i64 nReq = ((i64)mxPage * szPage); + i64 nSize; /* Current size of database file */ + sqlite3OsFileControl(pWal->pDbFd, SQLITE_FCNTL_CKPT_START, 0); rc = sqlite3OsFileSize(pWal->pDbFd, &nSize); if( rc==SQLITE_OK && nSizepDbFd, SQLITE_FCNTL_SIZE_HINT, &nReq); @@ -59074,7 +61274,7 @@ static int walCheckpoint( while( rc==SQLITE_OK && 0==walIteratorNext(pIter, &iDbpage, &iFrame) ){ i64 iOffset; assert( walFramePgno(pWal, iFrame)==iDbpage ); - if( db->u1.isInterrupted ){ + if( AtomicLoad(&db->u1.isInterrupted) ){ rc = db->mallocFailed ? SQLITE_NOMEM_BKPT : SQLITE_INTERRUPT; break; } @@ -59090,6 +61290,7 @@ static int walCheckpoint( rc = sqlite3OsWrite(pWal->pDbFd, zBuf, szPage, iOffset); if( rc!=SQLITE_OK ) break; } + sqlite3OsFileControl(pWal->pDbFd, SQLITE_FCNTL_CKPT_DONE, 0); /* If work was actually accomplished... */ if( rc==SQLITE_OK ){ @@ -59102,7 +61303,7 @@ static int walCheckpoint( } } if( rc==SQLITE_OK ){ - pInfo->nBackfill = mxSafeFrame; + AtomicStore(&pInfo->nBackfill, mxSafeFrame); } } @@ -59261,7 +61462,7 @@ SQLITE_PRIVATE int sqlite3WalClose( ** If the checksum cannot be verified return non-zero. If the header ** is read successfully and the checksum verified, return zero. */ -static int walIndexTryHdr(Wal *pWal, int *pChanged){ +static SQLITE_NO_TSAN int walIndexTryHdr(Wal *pWal, int *pChanged){ u32 aCksum[2]; /* Checksum on the header content */ WalIndexHdr h1, h2; /* Two copies of the header content */ WalIndexHdr volatile *aHdr; /* Header in shared memory */ @@ -59274,13 +61475,19 @@ static int walIndexTryHdr(Wal *pWal, int *pChanged){ ** meaning it is possible that an inconsistent snapshot is read ** from the file. If this happens, return non-zero. ** + ** tag-20200519-1: ** There are two copies of the header at the beginning of the wal-index. ** When reading, read [0] first then [1]. Writes are in the reverse order. ** Memory barriers are used to prevent the compiler or the hardware from - ** reordering the reads and writes. + ** reordering the reads and writes. TSAN and similar tools can sometimes + ** give false-positive warnings about these accesses because the tools do not + ** account for the double-read and the memory barrier. The use of mutexes + ** here would be problematic as the memory being accessed is potentially + ** shared among multiple processes and not all mutex implementions work + ** reliably in that environment. */ aHdr = walIndexHdr(pWal); - memcpy(&h1, (void *)&aHdr[0], sizeof(h1)); + memcpy(&h1, (void *)&aHdr[0], sizeof(h1)); /* Possible TSAN false-positive */ walShmBarrier(pWal); memcpy(&h2, (void *)&aHdr[1], sizeof(h2)); @@ -59370,28 +61577,32 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){ /* If the first attempt failed, it might have been due to a race ** with a writer. So get a WRITE lock and try again. */ - assert( badHdr==0 || pWal->writeLock==0 ); if( badHdr ){ if( pWal->bShmUnreliable==0 && (pWal->readOnly & WAL_SHM_RDONLY) ){ if( SQLITE_OK==(rc = walLockShared(pWal, WAL_WRITE_LOCK)) ){ walUnlockShared(pWal, WAL_WRITE_LOCK); rc = SQLITE_READONLY_RECOVERY; } - }else if( SQLITE_OK==(rc = walLockExclusive(pWal, WAL_WRITE_LOCK, 1)) ){ - pWal->writeLock = 1; - if( SQLITE_OK==(rc = walIndexPage(pWal, 0, &page0)) ){ - badHdr = walIndexTryHdr(pWal, pChanged); - if( badHdr ){ - /* If the wal-index header is still malformed even while holding - ** a WRITE lock, it can only mean that the header is corrupted and - ** needs to be reconstructed. So run recovery to do exactly that. - */ - rc = walIndexRecover(pWal); - *pChanged = 1; + }else{ + int bWriteLock = pWal->writeLock; + if( bWriteLock || SQLITE_OK==(rc = walLockWriter(pWal)) ){ + pWal->writeLock = 1; + if( SQLITE_OK==(rc = walIndexPage(pWal, 0, &page0)) ){ + badHdr = walIndexTryHdr(pWal, pChanged); + if( badHdr ){ + /* If the wal-index header is still malformed even while holding + ** a WRITE lock, it can only mean that the header is corrupted and + ** needs to be reconstructed. So run recovery to do exactly that. + */ + rc = walIndexRecover(pWal); + *pChanged = 1; + } + } + if( bWriteLock==0 ){ + pWal->writeLock = 0; + walUnlockExclusive(pWal, WAL_WRITE_LOCK, 1); } } - pWal->writeLock = 0; - walUnlockExclusive(pWal, WAL_WRITE_LOCK, 1); } } @@ -59721,7 +61932,7 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ assert( pWal->nWiData>0 ); assert( pWal->apWiData[0]!=0 ); pInfo = walCkptInfo(pWal); - if( !useWal && pInfo->nBackfill==pWal->hdr.mxFrame + if( !useWal && AtomicLoad(&pInfo->nBackfill)==pWal->hdr.mxFrame #ifdef SQLITE_ENABLE_SNAPSHOT && (pWal->pSnapshot==0 || pWal->hdr.mxFrame==0) #endif @@ -59770,7 +61981,7 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ } #endif for(i=1; iaReadMark[i]; + u32 thisMark = AtomicLoad(pInfo->aReadMark+i); if( mxReadMark<=thisMark && thisMark<=mxFrame ){ assert( thisMark!=READMARK_NOT_USED ); mxReadMark = thisMark; @@ -59783,7 +61994,8 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ for(i=1; iaReadMark[i] = mxFrame; + AtomicStore(pInfo->aReadMark+i,mxFrame); + mxReadMark = mxFrame; mxI = i; walUnlockExclusive(pWal, WAL_READ_LOCK(i), 1); break; @@ -59835,9 +62047,9 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ ** we can guarantee that the checkpointer that set nBackfill could not ** see any pages past pWal->hdr.mxFrame, this problem does not come up. */ - pWal->minFrame = pInfo->nBackfill+1; + pWal->minFrame = AtomicLoad(&pInfo->nBackfill)+1; walShmBarrier(pWal); - if( pInfo->aReadMark[mxI]!=mxReadMark + if( AtomicLoad(pInfo->aReadMark+mxI)!=mxReadMark || memcmp((void *)walIndexHdr(pWal), &pWal->hdr, sizeof(WalIndexHdr)) ){ walUnlockShared(pWal, WAL_READ_LOCK(mxI)); @@ -59887,17 +62099,15 @@ SQLITE_PRIVATE int sqlite3WalSnapshotRecover(Wal *pWal){ rc = SQLITE_NOMEM; }else{ u32 i = pInfo->nBackfillAttempted; - for(i=pInfo->nBackfillAttempted; i>pInfo->nBackfill; i--){ - volatile ht_slot *dummy; - volatile u32 *aPgno; /* Array of page numbers */ - u32 iZero; /* Frame corresponding to aPgno[0] */ + for(i=pInfo->nBackfillAttempted; i>AtomicLoad(&pInfo->nBackfill); i--){ + WalHashLoc sLoc; /* Hash table location */ u32 pgno; /* Page number in db file */ i64 iDbOff; /* Offset of db file entry */ i64 iWalOff; /* Offset of wal file entry */ - rc = walHashGet(pWal, walFramePage(i), &dummy, &aPgno, &iZero); + rc = walHashGet(pWal, walFramePage(i), &sLoc); if( rc!=SQLITE_OK ) break; - pgno = aPgno[i-iZero]; + pgno = sLoc.aPgno[i-sLoc.iZero]; iDbOff = (i64)(pgno-1) * szPage; if( iDbOff+szPage<=szDb ){ @@ -59938,18 +62148,41 @@ SQLITE_PRIVATE int sqlite3WalSnapshotRecover(Wal *pWal){ ** ** If the database contents have changes since the previous read ** transaction, then *pChanged is set to 1 before returning. The -** Pager layer will use this to know that is cache is stale and +** Pager layer will use this to know that its cache is stale and ** needs to be flushed. */ SQLITE_PRIVATE int sqlite3WalBeginReadTransaction(Wal *pWal, int *pChanged){ int rc; /* Return code */ int cnt = 0; /* Number of TryBeginRead attempts */ - #ifdef SQLITE_ENABLE_SNAPSHOT int bChanged = 0; WalIndexHdr *pSnapshot = pWal->pSnapshot; - if( pSnapshot && memcmp(pSnapshot, &pWal->hdr, sizeof(WalIndexHdr))!=0 ){ - bChanged = 1; +#endif + + assert( pWal->ckptLock==0 ); + +#ifdef SQLITE_ENABLE_SNAPSHOT + if( pSnapshot ){ + if( memcmp(pSnapshot, &pWal->hdr, sizeof(WalIndexHdr))!=0 ){ + bChanged = 1; + } + + /* It is possible that there is a checkpointer thread running + ** concurrent with this code. If this is the case, it may be that the + ** checkpointer has already determined that it will checkpoint + ** snapshot X, where X is later in the wal file than pSnapshot, but + ** has not yet set the pInfo->nBackfillAttempted variable to indicate + ** its intent. To avoid the race condition this leads to, ensure that + ** there is no checkpointer process by taking a shared CKPT lock + ** before checking pInfo->nBackfillAttempted. */ + (void)walEnableBlocking(pWal); + rc = walLockShared(pWal, WAL_CKPT_LOCK); + walDisableBlocking(pWal); + + if( rc!=SQLITE_OK ){ + return rc; + } + pWal->ckptLock = 1; } #endif @@ -59982,47 +62215,42 @@ SQLITE_PRIVATE int sqlite3WalBeginReadTransaction(Wal *pWal, int *pChanged){ assert( pWal->readLock>0 || pWal->hdr.mxFrame==0 ); assert( pInfo->aReadMark[pWal->readLock]<=pSnapshot->mxFrame ); - /* It is possible that there is a checkpointer thread running - ** concurrent with this code. If this is the case, it may be that the - ** checkpointer has already determined that it will checkpoint - ** snapshot X, where X is later in the wal file than pSnapshot, but - ** has not yet set the pInfo->nBackfillAttempted variable to indicate - ** its intent. To avoid the race condition this leads to, ensure that - ** there is no checkpointer process by taking a shared CKPT lock - ** before checking pInfo->nBackfillAttempted. - ** - ** TODO: Does the aReadMark[] lock prevent a checkpointer from doing - ** this already? - */ - rc = walLockShared(pWal, WAL_CKPT_LOCK); - - if( rc==SQLITE_OK ){ - /* Check that the wal file has not been wrapped. Assuming that it has - ** not, also check that no checkpointer has attempted to checkpoint any - ** frames beyond pSnapshot->mxFrame. If either of these conditions are - ** true, return SQLITE_BUSY_SNAPSHOT. Otherwise, overwrite pWal->hdr - ** with *pSnapshot and set *pChanged as appropriate for opening the - ** snapshot. */ - if( !memcmp(pSnapshot->aSalt, pWal->hdr.aSalt, sizeof(pWal->hdr.aSalt)) - && pSnapshot->mxFrame>=pInfo->nBackfillAttempted - ){ - assert( pWal->readLock>0 ); - memcpy(&pWal->hdr, pSnapshot, sizeof(WalIndexHdr)); - *pChanged = bChanged; - }else{ - rc = SQLITE_BUSY_SNAPSHOT; - } - - /* Release the shared CKPT lock obtained above. */ - walUnlockShared(pWal, WAL_CKPT_LOCK); + /* Check that the wal file has not been wrapped. Assuming that it has + ** not, also check that no checkpointer has attempted to checkpoint any + ** frames beyond pSnapshot->mxFrame. If either of these conditions are + ** true, return SQLITE_ERROR_SNAPSHOT. Otherwise, overwrite pWal->hdr + ** with *pSnapshot and set *pChanged as appropriate for opening the + ** snapshot. */ + if( !memcmp(pSnapshot->aSalt, pWal->hdr.aSalt, sizeof(pWal->hdr.aSalt)) + && pSnapshot->mxFrame>=pInfo->nBackfillAttempted + ){ + assert( pWal->readLock>0 ); + memcpy(&pWal->hdr, pSnapshot, sizeof(WalIndexHdr)); + *pChanged = bChanged; + }else{ + rc = SQLITE_ERROR_SNAPSHOT; } + /* A client using a non-current snapshot may not ignore any frames + ** from the start of the wal file. This is because, for a system + ** where (minFrame < iSnapshot < maxFrame), a checkpointer may + ** have omitted to checkpoint a frame earlier than minFrame in + ** the file because there exists a frame after iSnapshot that + ** is the same database page. */ + pWal->minFrame = 1; if( rc!=SQLITE_OK ){ sqlite3WalEndReadTransaction(pWal); } } } + + /* Release the shared CKPT lock obtained above. */ + if( pWal->ckptLock ){ + assert( pSnapshot ); + walUnlockShared(pWal, WAL_CKPT_LOCK); + pWal->ckptLock = 0; + } #endif return rc; } @@ -60098,27 +62326,28 @@ SQLITE_PRIVATE int sqlite3WalFindFrame( */ iMinHash = walFramePage(pWal->minFrame); for(iHash=walFramePage(iLast); iHash>=iMinHash; iHash--){ - volatile ht_slot *aHash; /* Pointer to hash table */ - volatile u32 *aPgno; /* Pointer to array of page numbers */ - u32 iZero; /* Frame number corresponding to aPgno[0] */ + WalHashLoc sLoc; /* Hash table location */ int iKey; /* Hash slot index */ int nCollide; /* Number of hash collisions remaining */ int rc; /* Error code */ + u32 iH; - rc = walHashGet(pWal, iHash, &aHash, &aPgno, &iZero); + rc = walHashGet(pWal, iHash, &sLoc); if( rc!=SQLITE_OK ){ return rc; } nCollide = HASHTABLE_NSLOT; - for(iKey=walHash(pgno); aHash[iKey]; iKey=walNextHash(iKey)){ - u32 iFrame = aHash[iKey] + iZero; - if( iFrame<=iLast && iFrame>=pWal->minFrame && aPgno[aHash[iKey]]==pgno ){ + iKey = walHash(pgno); + while( (iH = AtomicLoad(&sLoc.aHash[iKey]))!=0 ){ + u32 iFrame = iH + sLoc.iZero; + if( iFrame<=iLast && iFrame>=pWal->minFrame && sLoc.aPgno[iH]==pgno ){ assert( iFrame>iRead || CORRUPT_DB ); iRead = iFrame; } if( (nCollide--)==0 ){ return SQLITE_CORRUPT_BKPT; } + iKey = walNextHash(iKey); } if( iRead ) break; } @@ -60194,6 +62423,16 @@ SQLITE_PRIVATE Pgno sqlite3WalDbsize(Wal *pWal){ SQLITE_PRIVATE int sqlite3WalBeginWriteTransaction(Wal *pWal){ int rc; +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + /* If the write-lock is already held, then it was obtained before the + ** read-transaction was even opened, making this call a no-op. + ** Return early. */ + if( pWal->writeLock ){ + assert( !memcmp(&pWal->hdr,(void *)walIndexHdr(pWal),sizeof(WalIndexHdr)) ); + return SQLITE_OK; + } +#endif + /* Cannot start a write transaction without first holding a read ** transaction. */ assert( pWal->readLock>=0 ); @@ -60439,11 +62678,7 @@ static int walWriteOneFrame( int rc; /* Result code from subfunctions */ void *pData; /* Data actually written */ u8 aFrame[WAL_FRAME_HDRSIZE]; /* Buffer to assemble frame-header in */ -#if defined(SQLITE_HAS_CODEC) - if( (pData = sqlite3PagerCodec(pPage))==0 ) return SQLITE_NOMEM_BKPT; -#else pData = pPage->pData; -#endif walEncodeFrame(p->pWal, pPage->pgno, nTruncate, pData, aFrame); rc = walWriteToLog(p, aFrame, sizeof(aFrame), iOffset); if( rc ) return rc; @@ -60626,11 +62861,7 @@ SQLITE_PRIVATE int sqlite3WalFrames( if( pWal->iReCksum==0 || iWriteiReCksum ){ pWal->iReCksum = iWrite; } -#if defined(SQLITE_HAS_CODEC) - if( (pData = sqlite3PagerCodec(p))==0 ) return SQLITE_NOMEM; -#else pData = p->pData; -#endif rc = sqlite3OsWrite(pWal->pWalFd, pData, szPage, iOff); if( rc ) return rc; p->flags &= ~PGHDR_WAL_APPEND; @@ -60680,6 +62911,7 @@ SQLITE_PRIVATE int sqlite3WalFrames( if( rc ) return rc; iOffset += szFrame; nExtra++; + assert( pLast!=0 ); } } if( bSync ){ @@ -60712,6 +62944,7 @@ SQLITE_PRIVATE int sqlite3WalFrames( iFrame++; rc = walIndexAppend(pWal, iFrame, p->pgno); } + assert( pLast!=0 || nExtra==0 ); while( rc==SQLITE_OK && nExtra>0 ){ iFrame++; nExtra--; @@ -60776,45 +63009,52 @@ SQLITE_PRIVATE int sqlite3WalCheckpoint( if( pWal->readOnly ) return SQLITE_READONLY; WALTRACE(("WAL%p: checkpoint begins\n", pWal)); + /* Enable blocking locks, if possible. If blocking locks are successfully + ** enabled, set xBusy2=0 so that the busy-handler is never invoked. */ + sqlite3WalDb(pWal, db); + (void)walEnableBlocking(pWal); + /* IMPLEMENTATION-OF: R-62028-47212 All calls obtain an exclusive - ** "checkpoint" lock on the database file. */ + ** "checkpoint" lock on the database file. + ** EVIDENCE-OF: R-10421-19736 If any other process is running a + ** checkpoint operation at the same time, the lock cannot be obtained and + ** SQLITE_BUSY is returned. + ** EVIDENCE-OF: R-53820-33897 Even if there is a busy-handler configured, + ** it will not be invoked in this case. + */ rc = walLockExclusive(pWal, WAL_CKPT_LOCK, 1); - if( rc ){ - /* EVIDENCE-OF: R-10421-19736 If any other process is running a - ** checkpoint operation at the same time, the lock cannot be obtained and - ** SQLITE_BUSY is returned. - ** EVIDENCE-OF: R-53820-33897 Even if there is a busy-handler configured, - ** it will not be invoked in this case. - */ - testcase( rc==SQLITE_BUSY ); - testcase( xBusy!=0 ); - return rc; - } - pWal->ckptLock = 1; + testcase( rc==SQLITE_BUSY ); + testcase( rc!=SQLITE_OK && xBusy2!=0 ); + if( rc==SQLITE_OK ){ + pWal->ckptLock = 1; - /* IMPLEMENTATION-OF: R-59782-36818 The SQLITE_CHECKPOINT_FULL, RESTART and - ** TRUNCATE modes also obtain the exclusive "writer" lock on the database - ** file. - ** - ** EVIDENCE-OF: R-60642-04082 If the writer lock cannot be obtained - ** immediately, and a busy-handler is configured, it is invoked and the - ** writer lock retried until either the busy-handler returns 0 or the - ** lock is successfully obtained. - */ - if( eMode!=SQLITE_CHECKPOINT_PASSIVE ){ - rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_WRITE_LOCK, 1); - if( rc==SQLITE_OK ){ - pWal->writeLock = 1; - }else if( rc==SQLITE_BUSY ){ - eMode2 = SQLITE_CHECKPOINT_PASSIVE; - xBusy2 = 0; - rc = SQLITE_OK; + /* IMPLEMENTATION-OF: R-59782-36818 The SQLITE_CHECKPOINT_FULL, RESTART and + ** TRUNCATE modes also obtain the exclusive "writer" lock on the database + ** file. + ** + ** EVIDENCE-OF: R-60642-04082 If the writer lock cannot be obtained + ** immediately, and a busy-handler is configured, it is invoked and the + ** writer lock retried until either the busy-handler returns 0 or the + ** lock is successfully obtained. + */ + if( eMode!=SQLITE_CHECKPOINT_PASSIVE ){ + rc = walBusyLock(pWal, xBusy2, pBusyArg, WAL_WRITE_LOCK, 1); + if( rc==SQLITE_OK ){ + pWal->writeLock = 1; + }else if( rc==SQLITE_BUSY ){ + eMode2 = SQLITE_CHECKPOINT_PASSIVE; + xBusy2 = 0; + rc = SQLITE_OK; + } } } + /* Read the wal-index header. */ if( rc==SQLITE_OK ){ + walDisableBlocking(pWal); rc = walIndexReadHdr(pWal, &isChanged); + (void)walEnableBlocking(pWal); if( isChanged && pWal->pDbFd->pMethods->iVersion>=3 ){ sqlite3OsUnfetch(pWal->pDbFd, 0, 0); } @@ -60846,11 +63086,19 @@ SQLITE_PRIVATE int sqlite3WalCheckpoint( memset(&pWal->hdr, 0, sizeof(WalIndexHdr)); } + walDisableBlocking(pWal); + sqlite3WalDb(pWal, 0); + /* Release the locks. */ sqlite3WalEndWriteTransaction(pWal); - walUnlockExclusive(pWal, WAL_CKPT_LOCK, 1); - pWal->ckptLock = 0; + if( pWal->ckptLock ){ + walUnlockExclusive(pWal, WAL_CKPT_LOCK, 1); + pWal->ckptLock = 0; + } WALTRACE(("WAL%p: checkpoint %s\n", pWal, rc ? "failed" : "ok")); +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + if( rc==SQLITE_BUSY_TIMEOUT ) rc = SQLITE_BUSY; +#endif return (rc==SQLITE_OK && eMode!=eMode2 ? SQLITE_BUSY : rc); } @@ -60967,7 +63215,10 @@ SQLITE_PRIVATE int sqlite3WalSnapshotGet(Wal *pWal, sqlite3_snapshot **ppSnapsho /* Try to open on pSnapshot when the next read-transaction starts */ -SQLITE_PRIVATE void sqlite3WalSnapshotOpen(Wal *pWal, sqlite3_snapshot *pSnapshot){ +SQLITE_PRIVATE void sqlite3WalSnapshotOpen( + Wal *pWal, + sqlite3_snapshot *pSnapshot +){ pWal->pSnapshot = (WalIndexHdr*)pSnapshot; } @@ -60987,6 +63238,43 @@ SQLITE_API int sqlite3_snapshot_cmp(sqlite3_snapshot *p1, sqlite3_snapshot *p2){ if( pHdr1->mxFrame>pHdr2->mxFrame ) return +1; return 0; } + +/* +** The caller currently has a read transaction open on the database. +** This function takes a SHARED lock on the CHECKPOINTER slot and then +** checks if the snapshot passed as the second argument is still +** available. If so, SQLITE_OK is returned. +** +** If the snapshot is not available, SQLITE_ERROR is returned. Or, if +** the CHECKPOINTER lock cannot be obtained, SQLITE_BUSY. If any error +** occurs (any value other than SQLITE_OK is returned), the CHECKPOINTER +** lock is released before returning. +*/ +SQLITE_PRIVATE int sqlite3WalSnapshotCheck(Wal *pWal, sqlite3_snapshot *pSnapshot){ + int rc; + rc = walLockShared(pWal, WAL_CKPT_LOCK); + if( rc==SQLITE_OK ){ + WalIndexHdr *pNew = (WalIndexHdr*)pSnapshot; + if( memcmp(pNew->aSalt, pWal->hdr.aSalt, sizeof(pWal->hdr.aSalt)) + || pNew->mxFramenBackfillAttempted + ){ + rc = SQLITE_ERROR_SNAPSHOT; + walUnlockShared(pWal, WAL_CKPT_LOCK); + } + } + return rc; +} + +/* +** Release a lock obtained by an earlier successful call to +** sqlite3WalSnapshotCheck(). +*/ +SQLITE_PRIVATE void sqlite3WalSnapshotUnlock(Wal *pWal){ + assert( pWal ); + walUnlockShared(pWal, WAL_CKPT_LOCK); +} + + #endif /* SQLITE_ENABLE_SNAPSHOT */ #ifdef SQLITE_ENABLE_ZIPVFS @@ -61318,7 +63606,7 @@ struct MemPage { u16 maxLocal; /* Copy of BtShared.maxLocal or BtShared.maxLeaf */ u16 minLocal; /* Copy of BtShared.minLocal or BtShared.minLeaf */ u16 cellOffset; /* Index in aData of first cell pointer */ - u16 nFree; /* Number of free bytes on the page */ + int nFree; /* Number of free bytes on the page. -1 for unknown */ u16 nCell; /* Number of cells on this page, local and ovfl */ u16 maskPage; /* Mask for page offset */ u16 aiOvfl[4]; /* Insert the i-th overflow cell before the aiOvfl-th @@ -61449,9 +63737,7 @@ struct BtShared { #endif u8 inTransaction; /* Transaction state */ u8 max1bytePayload; /* Maximum first byte of cell for a 1-byte payload */ -#ifdef SQLITE_HAS_CODEC - u8 optimalReserve; /* Desired amount of reserved space per page */ -#endif + u8 nReserveWanted; /* Desired number of extra bytes per page */ u16 btsFlags; /* Boolean parameters. See BTS_* macros below */ u16 maxLocal; /* Maximum local payload in non-LEAFDATA tables */ u16 minLocal; /* Minimum local payload in non-LEAFDATA tables */ @@ -61526,9 +63812,16 @@ struct CellInfo { ** found at self->pBt->mutex. ** ** skipNext meaning: -** eState==SKIPNEXT && skipNext>0: Next sqlite3BtreeNext() is no-op. -** eState==SKIPNEXT && skipNext<0: Next sqlite3BtreePrevious() is no-op. -** eState==FAULT: Cursor fault with skipNext as error code. +** The meaning of skipNext depends on the value of eState: +** +** eState Meaning of skipNext +** VALID skipNext is meaningless and is ignored +** INVALID skipNext is meaningless and is ignored +** SKIPNEXT sqlite3BtreeNext() is a no-op if skipNext>0 and +** sqlite3BtreePrevious() is no-op if skipNext<0. +** REQUIRESEEK restoreCursorPosition() restores the cursor to +** eState=SKIPNEXT if skipNext!=0 +** FAULT skipNext holds the cursor fault error code. */ struct BtCursor { u8 eState; /* One of the CURSOR_XXX constants (see below) */ @@ -61567,6 +63860,7 @@ struct BtCursor { #define BTCF_AtLast 0x08 /* Cursor is pointing ot the last entry */ #define BTCF_Incrblob 0x10 /* True if an incremental I/O handle */ #define BTCF_Multiple 0x20 /* Maybe another cursor on the same btree */ +#define BTCF_Pinned 0x40 /* Cursor is busy and cannot be moved */ /* ** Potential values for BtCursor.eState. @@ -61710,6 +64004,7 @@ struct IntegrityCk { int v1, v2; /* Values for up to two %d fields in zPfx */ StrAccum errMsg; /* Accumulate the error message text here */ u32 *heap; /* Min-heap used for analyzing cell coverage */ + sqlite3 *db; /* Database connection running the check */ }; /* @@ -62643,7 +64938,7 @@ static int btreeSetHasContent(BtShared *pBt, Pgno pgno){ */ static int btreeGetHasContent(BtShared *pBt, Pgno pgno){ Bitvec *p = pBt->pHasContent; - return (p && (pgno>sqlite3BitvecSize(p) || sqlite3BitvecTest(p, pgno))); + return p && (pgno>sqlite3BitvecSize(p) || sqlite3BitvecTestNotNull(p, pgno)); } /* @@ -62692,13 +64987,19 @@ static int saveCursorKey(BtCursor *pCur){ /* Only the rowid is required for a table btree */ pCur->nKey = sqlite3BtreeIntegerKey(pCur); }else{ - /* For an index btree, save the complete key content */ + /* For an index btree, save the complete key content. It is possible + ** that the current key is corrupt. In that case, it is possible that + ** the sqlite3VdbeRecordUnpack() function may overread the buffer by + ** up to the size of 1 varint plus 1 8-byte value when the cursor + ** position is restored. Hence the 17 bytes of padding allocated + ** below. */ void *pKey; pCur->nKey = sqlite3BtreePayloadSize(pCur); - pKey = sqlite3Malloc( pCur->nKey ); + pKey = sqlite3Malloc( pCur->nKey + 9 + 8 ); if( pKey ){ rc = sqlite3BtreePayload(pCur, 0, (int)pCur->nKey, pKey); if( rc==SQLITE_OK ){ + memset(((u8*)pKey)+pCur->nKey, 0, 9+8); pCur->pKey = pKey; }else{ sqlite3_free(pKey); @@ -62725,6 +65026,9 @@ static int saveCursorPosition(BtCursor *pCur){ assert( 0==pCur->pKey ); assert( cursorHoldsMutex(pCur) ); + if( pCur->curFlags & BTCF_Pinned ){ + return SQLITE_CONSTRAINT_PINNED; + } if( pCur->eState==CURSOR_SKIPNEXT ){ pCur->eState = CURSOR_VALID; }else{ @@ -62830,11 +65134,12 @@ static int btreeMoveto( UnpackedRecord *pIdxKey; /* Unpacked index key */ if( pKey ){ + KeyInfo *pKeyInfo = pCur->pKeyInfo; assert( nKey==(i64)(int)nKey ); - pIdxKey = sqlite3VdbeAllocUnpackedRecord(pCur->pKeyInfo); + pIdxKey = sqlite3VdbeAllocUnpackedRecord(pKeyInfo); if( pIdxKey==0 ) return SQLITE_NOMEM_BKPT; - sqlite3VdbeRecordUnpack(pCur->pKeyInfo, (int)nKey, pKey, pIdxKey); - if( pIdxKey->nField==0 ){ + sqlite3VdbeRecordUnpack(pKeyInfo, (int)nKey, pKey, pIdxKey); + if( pIdxKey->nField==0 || pIdxKey->nField>pKeyInfo->nAllField ){ rc = SQLITE_CORRUPT_BKPT; goto moveto_done; } @@ -62858,19 +65163,23 @@ static int btreeMoveto( */ static int btreeRestoreCursorPosition(BtCursor *pCur){ int rc; - int skipNext; + int skipNext = 0; assert( cursorOwnsBtShared(pCur) ); assert( pCur->eState>=CURSOR_REQUIRESEEK ); if( pCur->eState==CURSOR_FAULT ){ return pCur->skipNext; } pCur->eState = CURSOR_INVALID; - rc = btreeMoveto(pCur, pCur->pKey, pCur->nKey, 0, &skipNext); + if( sqlite3FaultSim(410) ){ + rc = SQLITE_IOERR; + }else{ + rc = btreeMoveto(pCur, pCur->pKey, pCur->nKey, 0, &skipNext); + } if( rc==SQLITE_OK ){ sqlite3_free(pCur->pKey); pCur->pKey = 0; assert( pCur->eState==CURSOR_VALID || pCur->eState==CURSOR_INVALID ); - pCur->skipNext |= skipNext; + if( skipNext ) pCur->skipNext = skipNext; if( pCur->skipNext && pCur->eState==CURSOR_VALID ){ pCur->eState = CURSOR_SKIPNEXT; } @@ -62940,7 +65249,6 @@ SQLITE_PRIVATE int sqlite3BtreeCursorRestore(BtCursor *pCur, int *pDifferentRow) if( pCur->eState!=CURSOR_VALID ){ *pDifferentRow = 1; }else{ - assert( pCur->skipNext==0 ); *pDifferentRow = 0; } return SQLITE_OK; @@ -63024,6 +65332,13 @@ static void ptrmapPut(BtShared *pBt, Pgno key, u8 eType, Pgno parent, int *pRC){ *pRC = rc; return; } + if( ((char*)sqlite3PagerGetExtra(pDbPage))[0]!=0 ){ + /* The first byte of the extra data is the MemPage.isInit byte. + ** If that byte is set, it means this page is also being used + ** as a btree page. */ + *pRC = SQLITE_CORRUPT_BKPT; + goto ptrmap_exit; + } offset = PTRMAP_PTROFFSET(iPtrmap, key); if( offset<0 ){ *pRC = SQLITE_CORRUPT_BKPT; @@ -63086,7 +65401,7 @@ static int ptrmapGet(BtShared *pBt, Pgno key, u8 *pEType, Pgno *pPgno){ #else /* if defined SQLITE_OMIT_AUTOVACUUM */ #define ptrmapPut(w,x,y,z,rc) #define ptrmapGet(w,x,y,z) SQLITE_OK - #define ptrmapPutOvflPtr(x, y, rc) + #define ptrmapPutOvflPtr(x, y, z, rc) #endif /* @@ -63379,17 +65694,24 @@ static u16 cellSize(MemPage *pPage, int iCell){ #ifndef SQLITE_OMIT_AUTOVACUUM /* -** If the cell pCell, part of page pPage contains a pointer -** to an overflow page, insert an entry into the pointer-map -** for the overflow page. +** The cell pCell is currently part of page pSrc but will ultimately be part +** of pPage. (pSrc and pPager are often the same.) If pCell contains a +** pointer to an overflow page, insert an entry into the pointer-map for +** the overflow page that will be valid after pCell has been moved to pPage. */ -static void ptrmapPutOvflPtr(MemPage *pPage, u8 *pCell, int *pRC){ +static void ptrmapPutOvflPtr(MemPage *pPage, MemPage *pSrc, u8 *pCell,int *pRC){ CellInfo info; if( *pRC ) return; assert( pCell!=0 ); pPage->xParseCell(pPage, pCell, &info); if( info.nLocalaDataEnd, pCell, pCell+info.nLocal) ){ + testcase( pSrc!=pPage ); + *pRC = SQLITE_CORRUPT_BKPT; + return; + } + ovfl = get4byte(&pCell[info.nSize-4]); ptrmapPut(pPage->pBt, ovfl, PTRMAP_OVERFLOW1, pPage->pgno, pRC); } } @@ -63433,7 +65755,7 @@ static int defragmentPage(MemPage *pPage, int nMaxFrag){ hdr = pPage->hdrOffset; cellOffset = pPage->cellOffset; nCell = pPage->nCell; - assert( nCell==get2byte(&data[hdr+3]) ); + assert( nCell==get2byte(&data[hdr+3]) || CORRUPT_DB ); iCellFirst = cellOffset + 2*nCell; usableSize = pPage->pBt->usableSize; @@ -63444,19 +65766,10 @@ static int defragmentPage(MemPage *pPage, int nMaxFrag){ ** reconstruct the entire page. */ if( (int)data[hdr+7]<=nMaxFrag ){ int iFree = get2byte(&data[hdr+1]); + if( iFree>usableSize-4 ) return SQLITE_CORRUPT_PAGE(pPage); if( iFree ){ int iFree2 = get2byte(&data[iFree]); - - /* pageFindSlot() has already verified that free blocks are sorted - ** in order of offset within the page, and that no block extends - ** past the end of the page. Provided the two free slots do not - ** overlap, this guarantees that the memmove() calls below will not - ** overwrite the usableSize byte buffer, even if the database page - ** is corrupt. */ - assert( iFree2==0 || iFree2>iFree ); - assert( iFree+get2byte(&data[iFree+2]) <= usableSize ); - assert( iFree2==0 || iFree2+get2byte(&data[iFree2+2]) <= usableSize ); - + if( iFree2>usableSize-4 ) return SQLITE_CORRUPT_PAGE(pPage); if( 0==iFree2 || (data[iFree2]==0 && data[iFree2+1]==0) ){ u8 *pEnd = &data[cellOffset + nCell*2]; u8 *pAddr; @@ -63467,12 +65780,15 @@ static int defragmentPage(MemPage *pPage, int nMaxFrag){ return SQLITE_CORRUPT_PAGE(pPage); } if( iFree2 ){ - assert( iFree+sz<=iFree2 ); /* Verified by pageFindSlot() */ + if( iFree+sz>iFree2 ) return SQLITE_CORRUPT_PAGE(pPage); sz2 = get2byte(&data[iFree2+2]); - assert( iFree+sz+sz2+iFree2-(iFree+sz) <= usableSize ); + if( iFree2+sz2 > usableSize ) return SQLITE_CORRUPT_PAGE(pPage); memmove(&data[iFree+sz+sz2], &data[iFree+sz], iFree2-(iFree+sz)); sz += sz2; + }else if( NEVER(iFree+sz>usableSize) ){ + return SQLITE_CORRUPT_PAGE(pPage); } + cbrk = top+sz; assert( cbrk+(iFree-top) <= usableSize ); memmove(&data[cbrk], &data[top], iFree-top); @@ -63523,6 +65839,7 @@ static int defragmentPage(MemPage *pPage, int nMaxFrag){ data[hdr+7] = 0; defragment_out: + assert( pPage->nFree>=0 ); if( data[hdr+7]+cbrk-iCellFirst!=pPage->nFree ){ return SQLITE_CORRUPT_PAGE(pPage); } @@ -63550,16 +65867,16 @@ static int defragmentPage(MemPage *pPage, int nMaxFrag){ ** causes the fragmentation count to exceed 60. */ static u8 *pageFindSlot(MemPage *pPg, int nByte, int *pRc){ - const int hdr = pPg->hdrOffset; - u8 * const aData = pPg->aData; - int iAddr = hdr + 1; - int pc = get2byte(&aData[iAddr]); - int x; - int usableSize = pPg->pBt->usableSize; - int size; /* Size of the free slot */ + const int hdr = pPg->hdrOffset; /* Offset to page header */ + u8 * const aData = pPg->aData; /* Page data */ + int iAddr = hdr + 1; /* Address of ptr to pc */ + int pc = get2byte(&aData[iAddr]); /* Address of a free slot */ + int x; /* Excess size of the slot */ + int maxPC = pPg->pBt->usableSize - nByte; /* Max address for a usable slot */ + int size; /* Size of the free slot */ assert( pc>0 ); - while( pc<=usableSize-4 ){ + while( pc<=maxPC ){ /* EVIDENCE-OF: R-22710-53328 The third and fourth bytes of each ** freeblock form a big-endian integer which is the size of the freeblock ** in bytes, including the 4-byte header. */ @@ -63567,10 +65884,7 @@ static u8 *pageFindSlot(MemPage *pPg, int nByte, int *pRc){ if( (x = size - nByte)>=0 ){ testcase( x==4 ); testcase( x==3 ); - if( size+pc > usableSize ){ - *pRc = SQLITE_CORRUPT_PAGE(pPg); - return 0; - }else if( x<4 ){ + if( x<4 ){ /* EVIDENCE-OF: R-11498-58022 In a well-formed b-tree page, the total ** number of bytes in fragments may not exceed 60. */ if( aData[hdr+7]>57 ) return 0; @@ -63579,21 +65893,31 @@ static u8 *pageFindSlot(MemPage *pPg, int nByte, int *pRc){ ** fragmented bytes within the page. */ memcpy(&aData[iAddr], &aData[pc], 2); aData[hdr+7] += (u8)x; + }else if( x+pc > maxPC ){ + /* This slot extends off the end of the usable part of the page */ + *pRc = SQLITE_CORRUPT_PAGE(pPg); + return 0; }else{ /* The slot remains on the free-list. Reduce its size to account - ** for the portion used by the new allocation. */ + ** for the portion used by the new allocation. */ put2byte(&aData[pc+2], x); } return &aData[pc + x]; } iAddr = pc; pc = get2byte(&aData[pc]); - if( pcmaxPC+nByte-4 ){ + /* The free slot chain extends off the end of the page */ *pRc = SQLITE_CORRUPT_PAGE(pPg); } - return 0; } @@ -63634,7 +65958,7 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ ** However, that integer is too large to be stored in a 2-byte unsigned ** integer, so a value of 0 is used in its place. */ top = get2byte(&data[hdr+5]); - assert( top<=(int)pPage->pBt->usableSize ); /* Prevent by getAndInitPage() */ + assert( top<=(int)pPage->pBt->usableSize ); /* by btreeComputeFreeSpace() */ if( gap>top ){ if( top==0 && pPage->pBt->usableSize==65536 ){ top = 65536; @@ -63643,9 +65967,9 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ } } - /* If there is enough space between gap and top for one more cell pointer - ** array entry offset, and if the freelist is not empty, then search the - ** freelist looking for a free slot big enough to satisfy the request. + /* If there is enough space between gap and top for one more cell pointer, + ** and if the freelist is not empty, then search the + ** freelist looking for a slot big enough to satisfy the request. */ testcase( gap+2==top ); testcase( gap+1==top ); @@ -63653,9 +65977,14 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ if( (data[hdr+2] || data[hdr+1]) && gap+2<=top ){ u8 *pSpace = pageFindSlot(pPage, nByte, &rc); if( pSpace ){ - assert( pSpace>=data && (pSpace - data)<65536 ); - *pIdx = (int)(pSpace - data); - return SQLITE_OK; + int g2; + assert( pSpace+nByte<=data+pPage->pBt->usableSize ); + *pIdx = g2 = (int)(pSpace-data); + if( NEVER(g2<=gap) ){ + return SQLITE_CORRUPT_PAGE(pPage); + }else{ + return SQLITE_OK; + } }else if( rc ){ return rc; } @@ -63667,6 +65996,7 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ testcase( gap+2+nByte==top ); if( gap+2+nByte>top ){ assert( pPage->nCell>0 || CORRUPT_DB ); + assert( pPage->nFree>=0 ); rc = defragmentPage(pPage, MIN(4, pPage->nFree - (2+nByte))); if( rc ) return rc; top = get2byteNotZero(&data[hdr+5]); @@ -63675,7 +66005,7 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ /* Allocate memory from the gap in between the cell pointer array - ** and the cell content area. The btreeInitPage() call has already + ** and the cell content area. The btreeComputeFreeSpace() call has already ** validated the freelist. Given that the freelist is valid, there ** is no way that the allocation can extend off the end of the page. ** The assert() below verifies the previous sentence. @@ -63694,7 +66024,7 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ ** ** Adjacent freeblocks are coalesced. ** -** Note that even though the freeblock list was checked by btreeInitPage(), +** Even though the freeblock list was checked by btreeComputeFreeSpace(), ** that routine will not detect overlap between cells or freeblocks. Nor ** does it detect cells or freeblocks that encrouch into the reserved bytes ** at the end of the page. So do additional corruption checks inside this @@ -63728,12 +66058,12 @@ static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){ }else{ while( (iFreeBlk = get2byte(&data[iPtr]))pPage->pBt->usableSize-4 ){ + if( iFreeBlk>pPage->pBt->usableSize-4 ){ /* TH3: corrupt081.100 */ return SQLITE_CORRUPT_PAGE(pPage); } assert( iFreeBlk>iPtr || iFreeBlk==0 ); @@ -63748,7 +66078,7 @@ static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){ nFrag = iFreeBlk - iEnd; if( iEnd>iFreeBlk ) return SQLITE_CORRUPT_PAGE(pPage); iEnd = iFreeBlk + get2byte(&data[iFreeBlk+2]); - if( iEnd > pPage->pBt->usableSize ){ + if( NEVER(iEnd > pPage->pBt->usableSize) ){ return SQLITE_CORRUPT_PAGE(pPage); } iSize = iEnd - iStart; @@ -63776,7 +66106,8 @@ static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){ /* The new freeblock is at the beginning of the cell content area, ** so just extend the cell content area rather than create another ** freelist entry */ - if( iStartnFree field. */ -static int btreeInitPage(MemPage *pPage){ +static int btreeComputeFreeSpace(MemPage *pPage){ int pc; /* Address of a freeblock within pPage->aData[] */ u8 hdr; /* Offset to beginning of page header */ u8 *data; /* Equal to pPage->aData */ - BtShared *pBt; /* The main btree structure */ int usableSize; /* Amount of usable space on each page */ - u16 cellOffset; /* Offset from start of page to first cell pointer */ int nFree; /* Number of unused bytes on the page */ int top; /* First byte of the cell content area */ int iCellFirst; /* First allowable cell or freeblock offset */ @@ -63882,71 +66206,18 @@ static int btreeInitPage(MemPage *pPage){ assert( pPage->pgno==sqlite3PagerPagenumber(pPage->pDbPage) ); assert( pPage == sqlite3PagerGetExtra(pPage->pDbPage) ); assert( pPage->aData == sqlite3PagerGetData(pPage->pDbPage) ); - assert( pPage->isInit==0 ); + assert( pPage->isInit==1 ); + assert( pPage->nFree<0 ); - pBt = pPage->pBt; + usableSize = pPage->pBt->usableSize; hdr = pPage->hdrOffset; data = pPage->aData; - /* EVIDENCE-OF: R-28594-02890 The one-byte flag at offset 0 indicating - ** the b-tree page type. */ - if( decodeFlags(pPage, data[hdr]) ){ - return SQLITE_CORRUPT_PAGE(pPage); - } - assert( pBt->pageSize>=512 && pBt->pageSize<=65536 ); - pPage->maskPage = (u16)(pBt->pageSize - 1); - pPage->nOverflow = 0; - usableSize = pBt->usableSize; - pPage->cellOffset = cellOffset = hdr + 8 + pPage->childPtrSize; - pPage->aDataEnd = &data[usableSize]; - pPage->aCellIdx = &data[cellOffset]; - pPage->aDataOfst = &data[pPage->childPtrSize]; /* EVIDENCE-OF: R-58015-48175 The two-byte integer at offset 5 designates ** the start of the cell content area. A zero value for this integer is ** interpreted as 65536. */ top = get2byteNotZero(&data[hdr+5]); - /* EVIDENCE-OF: R-37002-32774 The two-byte integer at offset 3 gives the - ** number of cells on the page. */ - pPage->nCell = get2byte(&data[hdr+3]); - if( pPage->nCell>MX_CELL(pBt) ){ - /* To many cells for a single page. The page must be corrupt */ - return SQLITE_CORRUPT_PAGE(pPage); - } - testcase( pPage->nCell==MX_CELL(pBt) ); - /* EVIDENCE-OF: R-24089-57979 If a page contains no cells (which is only - ** possible for a root page of a table that contains no rows) then the - ** offset to the cell content area will equal the page size minus the - ** bytes of reserved space. */ - assert( pPage->nCell>0 || top==usableSize || CORRUPT_DB ); - - /* A malformed database page might cause us to read past the end - ** of page when parsing a cell. - ** - ** The following block of code checks early to see if a cell extends - ** past the end of a page boundary and causes SQLITE_CORRUPT to be - ** returned if it does. - */ - iCellFirst = cellOffset + 2*pPage->nCell; + iCellFirst = hdr + 8 + pPage->childPtrSize + 2*pPage->nCell; iCellLast = usableSize - 4; - if( pBt->db->flags & SQLITE_CellSizeCk ){ - int i; /* Index into the cell pointer array */ - int sz; /* Size of a cell */ - - if( !pPage->leaf ) iCellLast--; - for(i=0; inCell; i++){ - pc = get2byteAligned(&data[cellOffset+i*2]); - testcase( pc==iCellFirst ); - testcase( pc==iCellLast ); - if( pciCellLast ){ - return SQLITE_CORRUPT_PAGE(pPage); - } - sz = pPage->xCellSize(pPage, &data[pc]); - testcase( pc+sz==usableSize ); - if( pc+sz>usableSize ){ - return SQLITE_CORRUPT_PAGE(pPage); - } - } - if( !pPage->leaf ) iCellLast++; - } /* Compute the total free space on the page ** EVIDENCE-OF: R-23588-34450 The two-byte integer at offset 1 gives the @@ -63956,7 +66227,7 @@ static int btreeInitPage(MemPage *pPage){ nFree = data[hdr+7] + top; /* Init nFree to non-freeblock free space */ if( pc>0 ){ u32 next, size; - if( pcusableSize ){ + if( nFree>usableSize || nFreenFree = (u16)(nFree - iCellFirst); + return SQLITE_OK; +} + +/* +** Do additional sanity check after btreeInitPage() if +** PRAGMA cell_size_check=ON +*/ +static SQLITE_NOINLINE int btreeCellSizeCheck(MemPage *pPage){ + int iCellFirst; /* First allowable cell or freeblock offset */ + int iCellLast; /* Last possible cell or freeblock offset */ + int i; /* Index into the cell pointer array */ + int sz; /* Size of a cell */ + int pc; /* Address of a freeblock within pPage->aData[] */ + u8 *data; /* Equal to pPage->aData */ + int usableSize; /* Maximum usable space on the page */ + int cellOffset; /* Start of cell content area */ + + iCellFirst = pPage->cellOffset + 2*pPage->nCell; + usableSize = pPage->pBt->usableSize; + iCellLast = usableSize - 4; + data = pPage->aData; + cellOffset = pPage->cellOffset; + if( !pPage->leaf ) iCellLast--; + for(i=0; inCell; i++){ + pc = get2byteAligned(&data[cellOffset+i*2]); + testcase( pc==iCellFirst ); + testcase( pc==iCellLast ); + if( pciCellLast ){ + return SQLITE_CORRUPT_PAGE(pPage); + } + sz = pPage->xCellSize(pPage, &data[pc]); + testcase( pc+sz==usableSize ); + if( pc+sz>usableSize ){ + return SQLITE_CORRUPT_PAGE(pPage); + } + } + return SQLITE_OK; +} + +/* +** Initialize the auxiliary information for a disk block. +** +** Return SQLITE_OK on success. If we see that the page does +** not contain a well-formed database page, then return +** SQLITE_CORRUPT. Note that a return of SQLITE_OK does not +** guarantee that the page is well-formed. It only shows that +** we failed to detect any corruption. +*/ +static int btreeInitPage(MemPage *pPage){ + u8 *data; /* Equal to pPage->aData */ + BtShared *pBt; /* The main btree structure */ + + assert( pPage->pBt!=0 ); + assert( pPage->pBt->db!=0 ); + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + assert( pPage->pgno==sqlite3PagerPagenumber(pPage->pDbPage) ); + assert( pPage == sqlite3PagerGetExtra(pPage->pDbPage) ); + assert( pPage->aData == sqlite3PagerGetData(pPage->pDbPage) ); + assert( pPage->isInit==0 ); + + pBt = pPage->pBt; + data = pPage->aData + pPage->hdrOffset; + /* EVIDENCE-OF: R-28594-02890 The one-byte flag at offset 0 indicating + ** the b-tree page type. */ + if( decodeFlags(pPage, data[0]) ){ + return SQLITE_CORRUPT_PAGE(pPage); + } + assert( pBt->pageSize>=512 && pBt->pageSize<=65536 ); + pPage->maskPage = (u16)(pBt->pageSize - 1); + pPage->nOverflow = 0; + pPage->cellOffset = pPage->hdrOffset + 8 + pPage->childPtrSize; + pPage->aCellIdx = data + pPage->childPtrSize + 8; + pPage->aDataEnd = pPage->aData + pBt->usableSize; + pPage->aDataOfst = pPage->aData + pPage->childPtrSize; + /* EVIDENCE-OF: R-37002-32774 The two-byte integer at offset 3 gives the + ** number of cells on the page. */ + pPage->nCell = get2byte(&data[3]); + if( pPage->nCell>MX_CELL(pBt) ){ + /* To many cells for a single page. The page must be corrupt */ + return SQLITE_CORRUPT_PAGE(pPage); + } + testcase( pPage->nCell==MX_CELL(pBt) ); + /* EVIDENCE-OF: R-24089-57979 If a page contains no cells (which is only + ** possible for a root page of a table that contains no rows) then the + ** offset to the cell content area will equal the page size minus the + ** bytes of reserved space. */ + assert( pPage->nCell>0 + || get2byteNotZero(&data[5])==(int)pBt->usableSize + || CORRUPT_DB ); + pPage->nFree = -1; /* Indicate that this value is yet uncomputed */ pPage->isInit = 1; + if( pBt->db->flags & SQLITE_CellSizeCk ){ + return btreeCellSizeCheck(pPage); + } return SQLITE_OK; } @@ -64100,12 +66464,12 @@ static MemPage *btreePageLookup(BtShared *pBt, Pgno pgno){ ** error, return ((unsigned int)-1). */ static Pgno btreePagecount(BtShared *pBt){ + assert( (pBt->nPage & 0x80000000)==0 || CORRUPT_DB ); return pBt->nPage; } SQLITE_PRIVATE u32 sqlite3BtreeLastPage(Btree *p){ assert( sqlite3BtreeHoldsMutex(p) ); - assert( ((p->pBt->nPage)&0x80000000)==0 ); - return btreePagecount(p->pBt); + return btreePagecount(p->pBt) & 0x7fffffff; } /* @@ -64137,19 +66501,18 @@ static int getAndInitPage( if( pgno>btreePagecount(pBt) ){ rc = SQLITE_CORRUPT_BKPT; - goto getAndInitPage_error; + goto getAndInitPage_error1; } rc = sqlite3PagerGet(pBt->pPager, pgno, (DbPage**)&pDbPage, bReadOnly); if( rc ){ - goto getAndInitPage_error; + goto getAndInitPage_error1; } *ppPage = (MemPage*)sqlite3PagerGetExtra(pDbPage); if( (*ppPage)->isInit==0 ){ btreePageFromDbPage(pDbPage, pgno, pBt); rc = btreeInitPage(*ppPage); if( rc!=SQLITE_OK ){ - releasePage(*ppPage); - goto getAndInitPage_error; + goto getAndInitPage_error2; } } assert( (*ppPage)->pgno==pgno ); @@ -64159,12 +66522,13 @@ static int getAndInitPage( ** compatible with the root page. */ if( pCur && ((*ppPage)->nCell<1 || (*ppPage)->intKey!=pCur->curIntKey) ){ rc = SQLITE_CORRUPT_PGNO(pgno); - releasePage(*ppPage); - goto getAndInitPage_error; + goto getAndInitPage_error2; } return SQLITE_OK; -getAndInitPage_error: +getAndInitPage_error2: + releasePage(*ppPage); +getAndInitPage_error1: if( pCur ){ pCur->iPage--; pCur->pPage = pCur->apPage[pCur->iPage]; @@ -64267,8 +66631,7 @@ static int btreeInvokeBusyHandler(void *pArg){ BtShared *pBt = (BtShared*)pArg; assert( pBt->db ); assert( sqlite3_mutex_held(pBt->db->mutex) ); - return sqlite3InvokeBusyHandler(&pBt->db->busyHandler, - sqlite3PagerFile(pBt->pPager)); + return sqlite3InvokeBusyHandler(&pBt->db->busyHandler); } /* @@ -64372,9 +66735,13 @@ SQLITE_PRIVATE int sqlite3BtreeOpen( rc = sqlite3OsFullPathname(pVfs, zFilename, nFullPathname, zFullPathname); if( rc ){ - sqlite3_free(zFullPathname); - sqlite3_free(p); - return rc; + if( rc==SQLITE_OK_SYMLINK ){ + rc = SQLITE_OK; + }else{ + sqlite3_free(zFullPathname); + sqlite3_free(p); + return rc; + } } } #if SQLITE_THREADSAFE @@ -64815,19 +67182,17 @@ SQLITE_PRIVATE int sqlite3BtreeSetPagerFlags( */ SQLITE_PRIVATE int sqlite3BtreeSetPageSize(Btree *p, int pageSize, int nReserve, int iFix){ int rc = SQLITE_OK; + int x; BtShared *pBt = p->pBt; - assert( nReserve>=-1 && nReserve<=255 ); + assert( nReserve>=0 && nReserve<=255 ); sqlite3BtreeEnter(p); -#if SQLITE_HAS_CODEC - if( nReserve>pBt->optimalReserve ) pBt->optimalReserve = (u8)nReserve; -#endif + pBt->nReserveWanted = nReserve; + x = pBt->pageSize - pBt->usableSize; + if( nReservebtsFlags & BTS_PAGESIZE_FIXED ){ sqlite3BtreeLeave(p); return SQLITE_READONLY; } - if( nReserve<0 ){ - nReserve = pBt->pageSize - pBt->usableSize; - } assert( nReserve>=0 && nReserve<=255 ); if( pageSize>=512 && pageSize<=SQLITE_MAX_PAGE_SIZE && ((pageSize-1)&pageSize)==0 ){ @@ -64873,19 +67238,17 @@ SQLITE_PRIVATE int sqlite3BtreeGetReserveNoMutex(Btree *p){ ** are intentually left unused. This is the "reserved" space that is ** sometimes used by extensions. ** -** If SQLITE_HAS_MUTEX is defined then the number returned is the -** greater of the current reserved space and the maximum requested -** reserve space. +** The value returned is the larger of the current reserve size and +** the latest reserve size requested by SQLITE_FILECTRL_RESERVE_BYTES. +** The amount of reserve can only grow - never shrink. */ -SQLITE_PRIVATE int sqlite3BtreeGetOptimalReserve(Btree *p){ - int n; +SQLITE_PRIVATE int sqlite3BtreeGetRequestedReserve(Btree *p){ + int n1, n2; sqlite3BtreeEnter(p); - n = sqlite3BtreeGetReserveNoMutex(p); -#ifdef SQLITE_HAS_CODEC - if( npBt->optimalReserve ) n = p->pBt->optimalReserve; -#endif + n1 = (int)p->pBt->nReserveWanted; + n2 = sqlite3BtreeGetReserveNoMutex(p); sqlite3BtreeLeave(p); - return n; + return n1>n2 ? n1 : n2; } @@ -65024,9 +67387,9 @@ static int newDatabase(BtShared*); static int lockBtree(BtShared *pBt){ int rc; /* Result code from subfunctions */ MemPage *pPage1; /* Page 1 of the database file */ - int nPage; /* Number of pages in the database */ - int nPageFile = 0; /* Number of pages in the database file */ - int nPageHeader; /* Number of pages in the database according to hdr */ + u32 nPage; /* Number of pages in the database */ + u32 nPageFile = 0; /* Number of pages in the database file */ + u32 nPageHeader; /* Number of pages in the database according to hdr */ assert( sqlite3_mutex_held(pBt->mutex) ); assert( pBt->pPage1==0 ); @@ -65039,7 +67402,7 @@ static int lockBtree(BtShared *pBt){ ** a valid database file. */ nPage = nPageHeader = get4byte(28+(u8*)pPage1->aData); - sqlite3PagerPagecount(pBt->pPager, &nPageFile); + sqlite3PagerPagecount(pBt->pPager, (int*)&nPageFile); if( nPage==0 || memcmp(24+(u8*)pPage1->aData, 92+(u8*)pPage1->aData,4)!=0 ){ nPage = nPageFile; } @@ -65120,6 +67483,7 @@ static int lockBtree(BtShared *pBt){ ){ goto page1_init_failed; } + pBt->btsFlags |= BTS_PAGESIZE_FIXED; assert( (pageSize & 7)==0 ); /* EVIDENCE-OF: R-59310-51205 The "reserved space" size in the 1-byte ** integer at offset 20 is the number of bytes of space at the end of @@ -65144,7 +67508,7 @@ static int lockBtree(BtShared *pBt){ pageSize-usableSize); return rc; } - if( (pBt->db->flags & SQLITE_WriteSchema)==0 && nPage>nPageFile ){ + if( sqlite3WritableSchema(pBt->db)==0 && nPage>nPageFile ){ rc = SQLITE_CORRUPT_BKPT; goto page1_init_failed; } @@ -65332,8 +67696,9 @@ SQLITE_PRIVATE int sqlite3BtreeNewDb(Btree *p){ ** when A already has a read lock, we encourage A to give up and let B ** proceed. */ -SQLITE_PRIVATE int sqlite3BtreeBeginTrans(Btree *p, int wrflag){ +SQLITE_PRIVATE int sqlite3BtreeBeginTrans(Btree *p, int wrflag, int *pSchemaVersion){ BtShared *pBt = p->pBt; + Pager *pPager = pBt->pPager; int rc = SQLITE_OK; sqlite3BtreeEnter(p); @@ -65348,6 +67713,12 @@ SQLITE_PRIVATE int sqlite3BtreeBeginTrans(Btree *p, int wrflag){ } assert( pBt->inTransaction==TRANS_WRITE || IfNotOmitAV(pBt->bDoTruncate)==0 ); + if( (p->db->flags & SQLITE_ResetDatabase) + && sqlite3PagerIsreadonly(pPager)==0 + ){ + pBt->btsFlags &= ~BTS_READ_ONLY; + } + /* Write transactions are not possible on a read-only database */ if( (pBt->btsFlags & BTS_READ_ONLY)!=0 && wrflag ){ rc = SQLITE_READONLY; @@ -65391,6 +67762,18 @@ SQLITE_PRIVATE int sqlite3BtreeBeginTrans(Btree *p, int wrflag){ pBt->btsFlags &= ~BTS_INITIALLY_EMPTY; if( pBt->nPage==0 ) pBt->btsFlags |= BTS_INITIALLY_EMPTY; do { + sqlite3PagerWalDb(pPager, p->db); + +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + /* If transitioning from no transaction directly to a write transaction, + ** block for the WRITER lock first if possible. */ + if( pBt->pPage1==0 && wrflag ){ + assert( pBt->inTransaction==TRANS_NONE ); + rc = sqlite3PagerWalWriteLock(pPager, 1); + if( rc!=SQLITE_BUSY && rc!=SQLITE_OK ) break; + } +#endif + /* Call lockBtree() until either pBt->pPage1 is populated or ** lockBtree() returns something other than SQLITE_OK. lockBtree() ** may return SQLITE_OK but leave pBt->pPage1 set to 0 if after @@ -65404,19 +67787,28 @@ SQLITE_PRIVATE int sqlite3BtreeBeginTrans(Btree *p, int wrflag){ if( (pBt->btsFlags & BTS_READ_ONLY)!=0 ){ rc = SQLITE_READONLY; }else{ - rc = sqlite3PagerBegin(pBt->pPager,wrflag>1,sqlite3TempInMemory(p->db)); + rc = sqlite3PagerBegin(pPager, wrflag>1, sqlite3TempInMemory(p->db)); if( rc==SQLITE_OK ){ rc = newDatabase(pBt); + }else if( rc==SQLITE_BUSY_SNAPSHOT && pBt->inTransaction==TRANS_NONE ){ + /* if there was no transaction opened when this function was + ** called and SQLITE_BUSY_SNAPSHOT is returned, change the error + ** code to SQLITE_BUSY. */ + rc = SQLITE_BUSY; } } } if( rc!=SQLITE_OK ){ + (void)sqlite3PagerWalWriteLock(pPager, 0); unlockBtreeIfUnused(pBt); } }while( (rc&0xFF)==SQLITE_BUSY && pBt->inTransaction==TRANS_NONE && btreeInvokeBusyHandler(pBt) ); - sqlite3PagerResetLockTimeout(pBt->pPager); + sqlite3PagerWalDb(pPager, 0); +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + if( rc==SQLITE_BUSY_TIMEOUT ) rc = SQLITE_BUSY; +#endif if( rc==SQLITE_OK ){ if( p->inTrans==TRANS_NONE ){ @@ -65458,14 +67850,18 @@ SQLITE_PRIVATE int sqlite3BtreeBeginTrans(Btree *p, int wrflag){ } } - trans_begun: - if( rc==SQLITE_OK && wrflag ){ - /* This call makes sure that the pager has the correct number of - ** open savepoints. If the second parameter is greater than 0 and - ** the sub-journal is not already open, then it will be opened here. - */ - rc = sqlite3PagerOpenSavepoint(pBt->pPager, p->db->nSavepoint); + if( rc==SQLITE_OK ){ + if( pSchemaVersion ){ + *pSchemaVersion = get4byte(&pBt->pPage1->aData[40]); + } + if( wrflag ){ + /* This call makes sure that the pager has the correct number of + ** open savepoints. If the second parameter is greater than 0 and + ** the sub-journal is not already open, then it will be opened here. + */ + rc = sqlite3PagerOpenSavepoint(pPager, p->db->nSavepoint); + } } btreeIntegrity(p); @@ -65495,7 +67891,7 @@ static int setChildPtrmaps(MemPage *pPage){ for(i=0; ileaf ){ Pgno childPgno = get4byte(pCell); @@ -65603,6 +67999,7 @@ static int relocatePage( eType==PTRMAP_BTREE || eType==PTRMAP_ROOTPAGE ); assert( sqlite3_mutex_held(pBt->mutex) ); assert( pDbPage->pBt==pBt ); + if( iDbPage<3 ) return SQLITE_CORRUPT_BKPT; /* Move page iDbPage from its current location to page number iFreePage */ TRACE(("AUTOVACUUM: Moving %d to free page %d (ptr page %d type %d)\n", @@ -66108,6 +68505,18 @@ SQLITE_PRIVATE int sqlite3BtreeTripAllCursors(Btree *pBtree, int errCode, int wr return rc; } +/* +** Set the pBt->nPage field correctly, according to the current +** state of the database. Assume pBt->pPage1 is valid. +*/ +static void btreeSetNPage(BtShared *pBt, MemPage *pPage1){ + int nPage = get4byte(&pPage1->aData[28]); + testcase( nPage==0 ); + if( nPage==0 ) sqlite3PagerPagecount(pBt->pPager, &nPage); + testcase( pBt->nPage!=nPage ); + pBt->nPage = nPage; +} + /* ** Rollback the transaction in progress. ** @@ -66153,11 +68562,7 @@ SQLITE_PRIVATE int sqlite3BtreeRollback(Btree *p, int tripCode, int writeOnly){ ** call btreeGetPage() on page 1 again to make ** sure pPage1->aData is set correctly. */ if( btreeGetPage(pBt, 1, &pPage1, 0)==SQLITE_OK ){ - int nPage = get4byte(28+(u8*)pPage1->aData); - testcase( nPage==0 ); - if( nPage==0 ) sqlite3PagerPagecount(pBt->pPager, &nPage); - testcase( pBt->nPage!=nPage ); - pBt->nPage = nPage; + btreeSetNPage(pBt, pPage1); releasePageOne(pPage1); } assert( countValidCursors(pBt, 1)==0 ); @@ -66237,12 +68642,11 @@ SQLITE_PRIVATE int sqlite3BtreeSavepoint(Btree *p, int op, int iSavepoint){ pBt->nPage = 0; } rc = newDatabase(pBt); - pBt->nPage = get4byte(28 + pBt->pPage1->aData); + btreeSetNPage(pBt, pBt->pPage1); - /* The database size was written into the offset 28 of the header - ** when the transaction started, so we know that the value at offset - ** 28 is nonzero. */ - assert( pBt->nPage>0 ); + /* pBt->nPage might be zero if the database was corrupt when + ** the transaction was started. Otherwise, it must be at least 1. */ + assert( CORRUPT_DB || pBt->nPage>0 ); } sqlite3BtreeLeave(p); } @@ -66310,8 +68714,9 @@ static int btreeCursor( /* The following assert statements verify that if this is a sharable ** b-tree database, the connection is holding the required table locks, ** and that no other connection has any open cursor that conflicts with - ** this lock. */ - assert( hasSharedCacheTableLock(p, iTable, pKeyInfo!=0, (wrFlag?2:1)) ); + ** this lock. The iTable<1 term disables the check for corrupt schemas. */ + assert( hasSharedCacheTableLock(p, iTable, pKeyInfo!=0, (wrFlag?2:1)) + || iTable<1 ); assert( wrFlag==0 || !hasReadConflicts(p, iTable) ); /* Assert that the caller has opened the required transaction. */ @@ -66324,9 +68729,13 @@ static int btreeCursor( allocateTempSpace(pBt); if( pBt->pTmpSpace==0 ) return SQLITE_NOMEM_BKPT; } - if( iTable==1 && btreePagecount(pBt)==0 ){ - assert( wrFlag==0 ); - iTable = 0; + if( iTable<=1 ){ + if( iTable<1 ){ + return SQLITE_CORRUPT_BKPT; + }else if( btreePagecount(pBt)==0 ){ + assert( wrFlag==0 ); + iTable = 0; + } } /* Now that no other errors can occur, finish filling in the BtCursor @@ -66351,6 +68760,19 @@ static int btreeCursor( pCur->eState = CURSOR_INVALID; return SQLITE_OK; } +static int btreeCursorWithLock( + Btree *p, /* The btree */ + int iTable, /* Root page of table to open */ + int wrFlag, /* 1 to write. 0 read-only */ + struct KeyInfo *pKeyInfo, /* First arg to comparison function */ + BtCursor *pCur /* Space for new cursor */ +){ + int rc; + sqlite3BtreeEnter(p); + rc = btreeCursor(p, iTable, wrFlag, pKeyInfo, pCur); + sqlite3BtreeLeave(p); + return rc; +} SQLITE_PRIVATE int sqlite3BtreeCursor( Btree *p, /* The btree */ int iTable, /* Root page of table to open */ @@ -66358,15 +68780,11 @@ SQLITE_PRIVATE int sqlite3BtreeCursor( struct KeyInfo *pKeyInfo, /* First arg to xCompare() */ BtCursor *pCur /* Write new cursor here */ ){ - int rc; - if( iTable<1 ){ - rc = SQLITE_CORRUPT_BKPT; + if( p->sharable ){ + return btreeCursorWithLock(p, iTable, wrFlag, pKeyInfo, pCur); }else{ - sqlite3BtreeEnter(p); - rc = btreeCursor(p, iTable, wrFlag, pKeyInfo, pCur); - sqlite3BtreeLeave(p); + return btreeCursor(p, iTable, wrFlag, pKeyInfo, pCur); } - return rc; } /* @@ -66420,6 +68838,7 @@ SQLITE_PRIVATE int sqlite3BtreeCloseCursor(BtCursor *pCur){ sqlite3_free(pCur->aOverflow); sqlite3_free(pCur->pKey); sqlite3BtreeLeave(pBtree); + pCur->pBtree = 0; } return SQLITE_OK; } @@ -66488,6 +68907,18 @@ SQLITE_PRIVATE i64 sqlite3BtreeIntegerKey(BtCursor *pCur){ return pCur->info.nKey; } +/* +** Pin or unpin a cursor. +*/ +SQLITE_PRIVATE void sqlite3BtreeCursorPin(BtCursor *pCur){ + assert( (pCur->curFlags & BTCF_Pinned)==0 ); + pCur->curFlags |= BTCF_Pinned; +} +SQLITE_PRIVATE void sqlite3BtreeCursorUnpin(BtCursor *pCur){ + assert( (pCur->curFlags & BTCF_Pinned)!=0 ); + pCur->curFlags &= ~BTCF_Pinned; +} + #ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC /* ** Return the offset into the database file for the start of the @@ -66518,6 +68949,25 @@ SQLITE_PRIVATE u32 sqlite3BtreePayloadSize(BtCursor *pCur){ return pCur->info.nPayload; } +/* +** Return an upper bound on the size of any record for the table +** that the cursor is pointing into. +** +** This is an optimization. Everything will still work if this +** routine always returns 2147483647 (which is the largest record +** that SQLite can handle) or more. But returning a smaller value might +** prevent large memory allocations when trying to interpret a +** corrupt datrabase. +** +** The current implementation merely returns the size of the underlying +** database file. +*/ +SQLITE_PRIVATE sqlite3_int64 sqlite3BtreeMaxRecordSize(BtCursor *pCur){ + assert( cursorHoldsMutex(pCur) ); + assert( pCur->eState==CURSOR_VALID ); + return pCur->pBt->pageSize * (sqlite3_int64)pCur->pBt->nPage; +} + /* ** Given the page number of an overflow page in the database (parameter ** ovfl), this function finds the page number of the next page in the @@ -66774,9 +69224,6 @@ static int accessPayload( /* Need to read this page properly. It contains some of the ** range of data that is being read (eOp==0) or written (eOp!=0). */ -#ifdef SQLITE_DIRECT_OVERFLOW_READ - sqlite3_file *fd; /* File from which to do direct overflow read */ -#endif int a = amt; if( a + offset > ovflSize ){ a = ovflSize - offset; @@ -66787,7 +69234,7 @@ static int accessPayload( ** ** 1) this is a read operation, and ** 2) data is required from the start of this overflow page, and - ** 3) there is no open write-transaction, and + ** 3) there are no dirty pages in the page-cache ** 4) the database is file-backed, and ** 5) the page is not in the WAL file ** 6) at least 4 bytes have already been read into the output buffer @@ -66798,16 +69245,16 @@ static int accessPayload( */ if( eOp==0 /* (1) */ && offset==0 /* (2) */ - && pBt->inTransaction==TRANS_READ /* (3) */ - && (fd = sqlite3PagerFile(pBt->pPager))->pMethods /* (4) */ - && 0==sqlite3PagerUseWal(pBt->pPager, nextPage) /* (5) */ + && sqlite3PagerDirectReadOk(pBt->pPager, nextPage) /* (3,4,5) */ && &pBuf[-4]>=pBufStart /* (6) */ ){ + sqlite3_file *fd = sqlite3PagerFile(pBt->pPager); u8 aSave[4]; u8 *aWrite = &pBuf[-4]; assert( aWrite>=pBufStart ); /* due to (6) */ memcpy(aSave, aWrite, 4); rc = sqlite3OsRead(fd, aWrite, a+4, (i64)pBt->pageSize*(nextPage-1)); + if( rc && nextPage>pBt->nPage ) rc = SQLITE_CORRUPT_BKPT; nextPage = get4byte(aWrite); memcpy(aWrite, aSave, 4); }else @@ -67234,6 +69681,7 @@ SQLITE_PRIVATE int sqlite3BtreeLast(BtCursor *pCur, int *pRes){ assert( pCur->ix==pCur->pPage->nCell-1 ); assert( pCur->pPage->leaf ); #endif + *pRes = 0; return SQLITE_OK; } @@ -67319,7 +69767,7 @@ SQLITE_PRIVATE int sqlite3BtreeMovetoUnpacked( ** try to get there using sqlite3BtreeNext() rather than a full ** binary search. This is an optimization only. The correct answer ** is still obtained without this case, only a little more slowely */ - if( pCur->info.nKey+1==intKey && !pCur->skipNext ){ + if( pCur->info.nKey+1==intKey ){ *pRes = 0; rc = sqlite3BtreeNext(pCur, 0); if( rc==SQLITE_OK ){ @@ -67455,29 +69903,31 @@ SQLITE_PRIVATE int sqlite3BtreeMovetoUnpacked( ** case this happens. */ void *pCellKey; u8 * const pCellBody = pCell - pPage->childPtrSize; + const int nOverrun = 18; /* Size of the overrun padding */ pPage->xParseCell(pPage, pCellBody, &pCur->info); nCell = (int)pCur->info.nKey; testcase( nCell<0 ); /* True if key size is 2^32 or more */ testcase( nCell==0 ); /* Invalid key size: 0x80 0x80 0x00 */ testcase( nCell==1 ); /* Invalid key size: 0x80 0x80 0x01 */ testcase( nCell==2 ); /* Minimum legal index key size */ - if( nCell<2 ){ + if( nCell<2 || nCell/pCur->pBt->usableSize>pCur->pBt->nPage ){ rc = SQLITE_CORRUPT_PAGE(pPage); goto moveto_finish; } - pCellKey = sqlite3Malloc( nCell+18 ); + pCellKey = sqlite3Malloc( nCell+nOverrun ); if( pCellKey==0 ){ rc = SQLITE_NOMEM_BKPT; goto moveto_finish; } pCur->ix = (u16)idx; rc = accessPayload(pCur, 0, nCell, (unsigned char*)pCellKey, 0); + memset(((u8*)pCellKey)+nCell,0,nOverrun); /* Fix uninit warnings */ pCur->curFlags &= ~BTCF_ValidOvfl; if( rc ){ sqlite3_free(pCellKey); goto moveto_finish; } - c = xRecordCompare(nCell, pCellKey, pIdxKey); + c = sqlite3VdbeRecordCompare(nCell, pCellKey, pIdxKey); sqlite3_free(pCellKey); } assert( @@ -67593,7 +70043,6 @@ static SQLITE_NOINLINE int btreeNext(BtCursor *pCur){ MemPage *pPage; assert( cursorOwnsBtShared(pCur) ); - assert( pCur->skipNext==0 || pCur->eState!=CURSOR_VALID ); if( pCur->eState!=CURSOR_VALID ){ assert( (pCur->curFlags & BTCF_ValidOvfl)==0 ); rc = restoreCursorPosition(pCur); @@ -67603,27 +70052,34 @@ static SQLITE_NOINLINE int btreeNext(BtCursor *pCur){ if( CURSOR_INVALID==pCur->eState ){ return SQLITE_DONE; } - if( pCur->skipNext ){ - assert( pCur->eState==CURSOR_VALID || pCur->eState==CURSOR_SKIPNEXT ); + if( pCur->eState==CURSOR_SKIPNEXT ){ pCur->eState = CURSOR_VALID; - if( pCur->skipNext>0 ){ - pCur->skipNext = 0; - return SQLITE_OK; - } - pCur->skipNext = 0; + if( pCur->skipNext>0 ) return SQLITE_OK; } } pPage = pCur->pPage; idx = ++pCur->ix; - assert( pPage->isInit ); + if( !pPage->isInit ){ + /* The only known way for this to happen is for there to be a + ** recursive SQL function that does a DELETE operation as part of a + ** SELECT which deletes content out from under an active cursor + ** in a corrupt database file where the table being DELETE-ed from + ** has pages in common with the table being queried. See TH3 + ** module cov1/btree78.test testcase 220 (2018-06-08) for an + ** example. */ + return SQLITE_CORRUPT_BKPT; + } /* If the database file is corrupt, it is possible for the value of idx ** to be invalid here. This can only occur if a second cursor modifies ** the page while cursor pCur is holding a reference to it. Which can ** only happen if the database is corrupt in such a way as to link the - ** page into more than one b-tree structure. */ - testcase( idx>pPage->nCell ); + ** page into more than one b-tree structure. + ** + ** Update 2019-12-23: appears to long longer be possible after the + ** addition of anotherValidCursor() condition on balance_deeper(). */ + harmless( idx>pPage->nCell ); if( idx>=pPage->nCell ){ if( !pPage->leaf ){ @@ -67656,7 +70112,6 @@ SQLITE_PRIVATE int sqlite3BtreeNext(BtCursor *pCur, int flags){ UNUSED_PARAMETER( flags ); /* Used in COMDB2 but not native SQLite */ assert( cursorOwnsBtShared(pCur) ); assert( flags==0 || flags==1 ); - assert( pCur->skipNext==0 || pCur->eState!=CURSOR_VALID ); pCur->info.nSize = 0; pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl); if( pCur->eState!=CURSOR_VALID ) return btreeNext(pCur); @@ -67697,7 +70152,6 @@ static SQLITE_NOINLINE int btreePrevious(BtCursor *pCur){ MemPage *pPage; assert( cursorOwnsBtShared(pCur) ); - assert( pCur->skipNext==0 || pCur->eState!=CURSOR_VALID ); assert( (pCur->curFlags & (BTCF_AtLast|BTCF_ValidOvfl|BTCF_ValidNKey))==0 ); assert( pCur->info.nSize==0 ); if( pCur->eState!=CURSOR_VALID ){ @@ -67708,14 +70162,9 @@ static SQLITE_NOINLINE int btreePrevious(BtCursor *pCur){ if( CURSOR_INVALID==pCur->eState ){ return SQLITE_DONE; } - if( pCur->skipNext ){ - assert( pCur->eState==CURSOR_VALID || pCur->eState==CURSOR_SKIPNEXT ); + if( CURSOR_SKIPNEXT==pCur->eState ){ pCur->eState = CURSOR_VALID; - if( pCur->skipNext<0 ){ - pCur->skipNext = 0; - return SQLITE_OK; - } - pCur->skipNext = 0; + if( pCur->skipNext<0 ) return SQLITE_OK; } } @@ -67750,7 +70199,6 @@ static SQLITE_NOINLINE int btreePrevious(BtCursor *pCur){ SQLITE_PRIVATE int sqlite3BtreePrevious(BtCursor *pCur, int flags){ assert( cursorOwnsBtShared(pCur) ); assert( flags==0 || flags==1 ); - assert( pCur->skipNext==0 || pCur->eState!=CURSOR_VALID ); UNUSED_PARAMETER( flags ); /* Used in COMDB2 but not native SQLite */ pCur->curFlags &= ~(BTCF_AtLast|BTCF_ValidOvfl|BTCF_ValidNKey); pCur->info.nSize = 0; @@ -68086,7 +70534,7 @@ static int allocateBtreePage( TRACE(("ALLOCATE: %d from end of file\n", *pPgno)); } - assert( *pPgno!=PENDING_BYTE_PAGE(pBt) ); + assert( CORRUPT_DB || *pPgno!=PENDING_BYTE_PAGE(pBt) ); end_allocate_page: releasePage(pTrunk); @@ -68114,13 +70562,15 @@ static int freePage2(BtShared *pBt, MemPage *pMemPage, Pgno iPage){ MemPage *pPage1 = pBt->pPage1; /* Local reference to page 1 */ MemPage *pPage; /* Page being freed. May be NULL. */ int rc; /* Return Code */ - int nFree; /* Initial number of pages on free-list */ + u32 nFree; /* Initial number of pages on free-list */ assert( sqlite3_mutex_held(pBt->mutex) ); assert( CORRUPT_DB || iPage>1 ); assert( !pMemPage || pMemPage->pgno==iPage ); - if( iPage<2 ) return SQLITE_CORRUPT_BKPT; + if( iPage<2 || iPage>pBt->nPage ){ + return SQLITE_CORRUPT_BKPT; + } if( pMemPage ){ pPage = pMemPage; sqlite3PagerRef(pPage->pDbPage); @@ -68531,6 +70981,7 @@ static void dropCell(MemPage *pPage, int idx, int sz, int *pRC){ assert( CORRUPT_DB || sz==cellSize(pPage, idx) ); assert( sqlite3PagerIswriteable(pPage->pDbPage) ); assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + assert( pPage->nFree>=0 ); data = pPage->aData; ptr = &pPage->aCellIdx[2*idx]; pc = get2byte(ptr); @@ -68595,12 +71046,8 @@ static void insertCell( assert( pPage->nOverflow<=ArraySize(pPage->apOvfl) ); assert( ArraySize(pPage->apOvfl)==ArraySize(pPage->aiOvfl) ); assert( sqlite3_mutex_held(pPage->pBt->mutex) ); - /* The cell should normally be sized correctly. However, when moving a - ** malformed cell from a leaf page to an interior page, if the cell size - ** wanted to be less than 4 but got rounded up to 4 on the leaf, then size - ** might be less than 8 (leaf-size + pointer) on the interior node. Hence - ** the term after the || in the following assert(). */ - assert( sz==pPage->xCellSize(pPage, pCell) || (sz==8 && iChild>0) ); + assert( sz==pPage->xCellSize(pPage, pCell) || CORRUPT_DB ); + assert( pPage->nFree>=0 ); if( pPage->nOverflow || sz+2>pPage->nFree ){ if( pTemp ){ memcpy(pTemp, pCell, sz); @@ -68641,9 +71088,16 @@ static void insertCell( assert( idx >= pPage->cellOffset+2*pPage->nCell+2 || CORRUPT_DB ); assert( idx+sz <= (int)pPage->pBt->usableSize ); pPage->nFree -= (u16)(2 + sz); - memcpy(&data[idx], pCell, sz); if( iChild ){ + /* In a corrupt database where an entry in the cell index section of + ** a btree page has a value of 3 or less, the pCell value might point + ** as many as 4 bytes in front of the start of the aData buffer for + ** the source page. Make sure this does not cause problems by not + ** reading the first 4 bytes */ + memcpy(&data[idx+4], pCell+4, sz-4); put4byte(&data[idx], iChild); + }else{ + memcpy(&data[idx], pCell, sz); } pIns = pPage->aCellIdx + i*2; memmove(pIns+2, pIns, 2*(pPage->nCell - i)); @@ -68651,21 +71105,100 @@ static void insertCell( pPage->nCell++; /* increment the cell count */ if( (++data[pPage->hdrOffset+4])==0 ) data[pPage->hdrOffset+3]++; - assert( get2byte(&data[pPage->hdrOffset+3])==pPage->nCell ); + assert( get2byte(&data[pPage->hdrOffset+3])==pPage->nCell || CORRUPT_DB ); #ifndef SQLITE_OMIT_AUTOVACUUM if( pPage->pBt->autoVacuum ){ /* The cell may contain a pointer to an overflow page. If so, write ** the entry for the overflow page into the pointer map. */ - ptrmapPutOvflPtr(pPage, pCell, pRC); + ptrmapPutOvflPtr(pPage, pPage, pCell, pRC); } #endif } } +/* +** The following parameters determine how many adjacent pages get involved +** in a balancing operation. NN is the number of neighbors on either side +** of the page that participate in the balancing operation. NB is the +** total number of pages that participate, including the target page and +** NN neighbors on either side. +** +** The minimum value of NN is 1 (of course). Increasing NN above 1 +** (to 2 or 3) gives a modest improvement in SELECT and DELETE performance +** in exchange for a larger degradation in INSERT and UPDATE performance. +** The value of NN appears to give the best results overall. +** +** (Later:) The description above makes it seem as if these values are +** tunable - as if you could change them and recompile and it would all work. +** But that is unlikely. NB has been 3 since the inception of SQLite and +** we have never tested any other value. +*/ +#define NN 1 /* Number of neighbors on either side of pPage */ +#define NB 3 /* (NN*2+1): Total pages involved in the balance */ + /* ** A CellArray object contains a cache of pointers and sizes for a ** consecutive sequence of cells that might be held on multiple pages. +** +** The cells in this array are the divider cell or cells from the pParent +** page plus up to three child pages. There are a total of nCell cells. +** +** pRef is a pointer to one of the pages that contributes cells. This is +** used to access information such as MemPage.intKey and MemPage.pBt->pageSize +** which should be common to all pages that contribute cells to this array. +** +** apCell[] and szCell[] hold, respectively, pointers to the start of each +** cell and the size of each cell. Some of the apCell[] pointers might refer +** to overflow cells. In other words, some apCel[] pointers might not point +** to content area of the pages. +** +** A szCell[] of zero means the size of that cell has not yet been computed. +** +** The cells come from as many as four different pages: +** +** ----------- +** | Parent | +** ----------- +** / | \ +** / | \ +** --------- --------- --------- +** |Child-1| |Child-2| |Child-3| +** --------- --------- --------- +** +** The order of cells is in the array is for an index btree is: +** +** 1. All cells from Child-1 in order +** 2. The first divider cell from Parent +** 3. All cells from Child-2 in order +** 4. The second divider cell from Parent +** 5. All cells from Child-3 in order +** +** For a table-btree (with rowids) the items 2 and 4 are empty because +** content exists only in leaves and there are no divider cells. +** +** For an index btree, the apEnd[] array holds pointer to the end of page +** for Child-1, the Parent, Child-2, the Parent (again), and Child-3, +** respectively. The ixNx[] array holds the number of cells contained in +** each of these 5 stages, and all stages to the left. Hence: +** +** ixNx[0] = Number of cells in Child-1. +** ixNx[1] = Number of cells in Child-1 plus 1 for first divider. +** ixNx[2] = Number of cells in Child-1 and Child-2 + 1 for 1st divider. +** ixNx[3] = Number of cells in Child-1 and Child-2 + both divider cells +** ixNx[4] = Total number of cells. +** +** For a table-btree, the concept is similar, except only apEnd[0]..apEnd[2] +** are used and they point to the leaf pages only, and the ixNx value are: +** +** ixNx[0] = Number of cells in Child-1. +** ixNx[1] = Number of cells in Child-1 and Child-2. +** ixNx[2] = Total number of cells. +** +** Sometimes when deleting, a child page can have zero cells. In those +** cases, ixNx[] entries with higher indexes, and the corresponding apEnd[] +** entries, shift down. The end result is that each ixNx[] entry should +** be larger than the previous */ typedef struct CellArray CellArray; struct CellArray { @@ -68673,6 +71206,8 @@ struct CellArray { MemPage *pRef; /* Reference page */ u8 **apCell; /* All cells begin balanced */ u16 *szCell; /* Local size of all cells in apCell[] */ + u8 *apEnd[NB*2]; /* MemPage.aDataEnd values */ + int ixNx[NB*2]; /* Index of at which we move to the next apEnd[] */ }; /* @@ -68723,36 +71258,59 @@ static u16 cachedCellSize(CellArray *p, int N){ ** responsibility of the caller to set it correctly. */ static int rebuildPage( - MemPage *pPg, /* Edit this page */ + CellArray *pCArray, /* Content to be added to page pPg */ + int iFirst, /* First cell in pCArray to use */ int nCell, /* Final number of cells on page */ - u8 **apCell, /* Array of cells */ - u16 *szCell /* Array of cell sizes */ + MemPage *pPg /* The page to be reconstructed */ ){ const int hdr = pPg->hdrOffset; /* Offset of header on pPg */ u8 * const aData = pPg->aData; /* Pointer to data for pPg */ const int usableSize = pPg->pBt->usableSize; u8 * const pEnd = &aData[usableSize]; - int i; + int i = iFirst; /* Which cell to copy from pCArray*/ + u32 j; /* Start of cell content area */ + int iEnd = i+nCell; /* Loop terminator */ u8 *pCellptr = pPg->aCellIdx; u8 *pTmp = sqlite3PagerTempSpace(pPg->pBt->pPager); u8 *pData; + int k; /* Current slot in pCArray->apEnd[] */ + u8 *pSrcEnd; /* Current pCArray->apEnd[k] value */ + + assert( i(u32)usableSize) ){ j = 0; } + memcpy(&pTmp[j], &aData[j], usableSize - j); - i = get2byte(&aData[hdr+5]); - memcpy(&pTmp[i], &aData[i], usableSize - i); + for(k=0; pCArray->ixNx[k]<=i && ALWAYS(kapEnd[k]; pData = pEnd; - for(i=0; iapCell[i]; + u16 sz = pCArray->szCell[i]; + assert( sz>0 ); if( SQLITE_WITHIN(pCell,aData,pEnd) ){ + if( ((uptr)(pCell+sz))>(uptr)pEnd ) return SQLITE_CORRUPT_BKPT; pCell = &pTmp[pCell - aData]; + }else if( (uptr)(pCell+sz)>(uptr)pSrcEnd + && (uptr)(pCell)<(uptr)pSrcEnd + ){ + return SQLITE_CORRUPT_BKPT; } - pData -= szCell[i]; + + pData -= sz; put2byte(pCellptr, (pData - aData)); pCellptr += 2; if( pData < pCellptr ) return SQLITE_CORRUPT_BKPT; - memcpy(pData, pCell, szCell[i]); - assert( szCell[i]==pPg->xCellSize(pPg, pCell) || CORRUPT_DB ); - testcase( szCell[i]!=pPg->xCellSize(pPg,pCell) ); + memcpy(pData, pCell, sz); + assert( sz==pPg->xCellSize(pPg, pCell) || CORRUPT_DB ); + testcase( sz!=pPg->xCellSize(pPg,pCell) ) + i++; + if( i>=iEnd ) break; + if( pCArray->ixNx[k]<=i ){ + k++; + pSrcEnd = pCArray->apEnd[k]; + } } /* The pPg->nFree field is now set incorrectly. The caller will fix it. */ @@ -68767,12 +71325,11 @@ static int rebuildPage( } /* -** Array apCell[] contains nCell pointers to b-tree cells. Array szCell -** contains the size in bytes of each such cell. This function attempts to -** add the cells stored in the array to page pPg. If it cannot (because -** the page needs to be defragmented before the cells will fit), non-zero -** is returned. Otherwise, if the cells are added successfully, zero is -** returned. +** The pCArray objects contains pointers to b-tree cells and the cell sizes. +** This function attempts to add the cells stored in the array to page pPg. +** If it cannot (because the page needs to be defragmented before the cells +** will fit), non-zero is returned. Otherwise, if the cells are added +** successfully, zero is returned. ** ** Argument pCellptr points to the first entry in the cell-pointer array ** (part of page pPg) to populate. After cell apCell[0] is written to the @@ -68794,21 +71351,27 @@ static int rebuildPage( static int pageInsertArray( MemPage *pPg, /* Page to add cells to */ u8 *pBegin, /* End of cell-pointer array */ - u8 **ppData, /* IN/OUT: Page content -area pointer */ + u8 **ppData, /* IN/OUT: Page content-area pointer */ u8 *pCellptr, /* Pointer to cell-pointer area */ int iFirst, /* Index of first cell to add */ int nCell, /* Number of cells to add to pPg */ CellArray *pCArray /* Array of cells */ ){ - int i; - u8 *aData = pPg->aData; - u8 *pData = *ppData; - int iEnd = iFirst + nCell; + int i = iFirst; /* Loop counter - cell index to insert */ + u8 *aData = pPg->aData; /* Complete page */ + u8 *pData = *ppData; /* Content area. A subset of aData[] */ + int iEnd = iFirst + nCell; /* End of loop. One past last cell to ins */ + int k; /* Current slot in pCArray->apEnd[] */ + u8 *pEnd; /* Maximum extent of cell data */ assert( CORRUPT_DB || pPg->hdrOffset==0 ); /* Never called on page 1 */ - for(i=iFirst; iixNx[k]<=i && ALWAYS(kapEnd[k]; + while( 1 /*Exit by break*/ ){ int sz, rc; u8 *pSlot; - sz = cachedCellSize(pCArray, i); + assert( pCArray->szCell[i]!=0 ); + sz = pCArray->szCell[i]; if( (aData[1]==0 && aData[2]==0) || (pSlot = pageFindSlot(pPg,sz,&rc))==0 ){ if( (pData - pBegin)apCell[i] || pSlot>=(pCArray->apCell[i]+sz) || CORRUPT_DB ); + if( (uptr)(pCArray->apCell[i]+sz)>(uptr)pEnd + && (uptr)(pCArray->apCell[i])<(uptr)pEnd + ){ + assert( CORRUPT_DB ); + (void)SQLITE_CORRUPT_BKPT; + return 1; + } memmove(pSlot, pCArray->apCell[i], sz); put2byte(pCellptr, (pSlot - aData)); pCellptr += 2; + i++; + if( i>=iEnd ) break; + if( pCArray->ixNx[k]<=i ){ + k++; + pEnd = pCArray->apEnd[k]; + } } *ppData = pData; return 0; } /* -** Array apCell[] contains nCell pointers to b-tree cells. Array szCell -** contains the size in bytes of each such cell. This function adds the -** space associated with each cell in the array that is currently stored -** within the body of pPg to the pPg free-list. The cell-pointers and other -** fields of the page are not updated. +** The pCArray object contains pointers to b-tree cells and their sizes. +** +** This function adds the space associated with each cell in the array +** that is currently stored within the body of pPg to the pPg free-list. +** The cell-pointers and other fields of the page are not updated. ** ** This function returns the total number of cells added to the free-list. */ @@ -68883,9 +71459,9 @@ static int pageFreeArray( } /* -** apCell[] and szCell[] contains pointers to and sizes of all cells in the -** pages being balanced. The current page, pPg, has pPg->nCell cells starting -** with apCell[iOld]. After balancing, this page should hold nNew cells +** pCArray contains pointers to and sizes of all cells in the page being +** balanced. The current page, pPg, has pPg->nCell cells starting with +** pCArray->apCell[iOld]. After balancing, this page should hold nNew cells ** starting at apCell[iNew]. ** ** This routine makes the necessary adjustments to pPg so that it contains @@ -68917,13 +71493,17 @@ static int editPage( #endif /* Remove cells from the start and end of the page */ + assert( nCell>=0 ); if( iOldnCell) ) return SQLITE_CORRUPT_BKPT; memmove(pPg->aCellIdx, &pPg->aCellIdx[nShift*2], nCell*2); nCell -= nShift; } if( iNewEnd < iOldEnd ){ - nCell -= pageFreeArray(pPg, iNewEnd, iOldEnd - iNewEnd, pCArray); + int nTail = pageFreeArray(pPg, iNewEnd, iOldEnd - iNewEnd, pCArray); + assert( nCell>=nTail ); + nCell -= nTail; } pData = &aData[get2byteNotZero(&aData[hdr+5])]; @@ -68933,6 +71513,7 @@ static int editPage( if( iNew=0 ); pCellptr = pPg->aCellIdx; memmove(&pCellptr[nAdd*2], pCellptr, nCell*2); if( pageInsertArray( @@ -68947,8 +71528,11 @@ static int editPage( int iCell = (iOld + pPg->aiOvfl[i]) - iNew; if( iCell>=0 && iCellaCellIdx[iCell * 2]; - memmove(&pCellptr[2], pCellptr, (nCell - iCell) * 2); + if( nCell>iCell ){ + memmove(&pCellptr[2], pCellptr, (nCell - iCell) * 2); + } nCell++; + cachedCellSize(pCArray, iCell+iNew); if( pageInsertArray( pPg, pBegin, &pData, pCellptr, iCell+iNew, 1, pCArray @@ -68957,6 +71541,7 @@ static int editPage( } /* Append cells to the end of the page */ + assert( nCell>=0 ); pCellptr = &pPg->aCellIdx[nCell*2]; if( pageInsertArray( pPg, pBegin, &pData, pCellptr, @@ -68985,24 +71570,9 @@ static int editPage( editpage_fail: /* Unable to edit this page. Rebuild it from scratch instead. */ populateCellCache(pCArray, iNew, nNew); - return rebuildPage(pPg, nNew, &pCArray->apCell[iNew], &pCArray->szCell[iNew]); + return rebuildPage(pCArray, iNew, nNew, pPg); } -/* -** The following parameters determine how many adjacent pages get involved -** in a balancing operation. NN is the number of neighbors on either side -** of the page that participate in the balancing operation. NB is the -** total number of pages that participate, including the target page and -** NN neighbors on either side. -** -** The minimum value of NN is 1 (of course). Increasing NN above 1 -** (to 2 or 3) gives a modest improvement in SELECT and DELETE performance -** in exchange for a larger degradation in INSERT and UPDATE performance. -** The value of NN appears to give the best results overall. -*/ -#define NN 1 /* Number of neighbors on either side of pPage */ -#define NB (NN*2+1) /* Total pages involved in the balance */ - #ifndef SQLITE_OMIT_QUICKBALANCE /* @@ -69037,9 +71607,10 @@ static int balance_quick(MemPage *pParent, MemPage *pPage, u8 *pSpace){ assert( sqlite3_mutex_held(pPage->pBt->mutex) ); assert( sqlite3PagerIswriteable(pParent->pDbPage) ); assert( pPage->nOverflow==1 ); - - /* This error condition is now caught prior to reaching this function */ - if( NEVER(pPage->nCell==0) ) return SQLITE_CORRUPT_BKPT; + + if( pPage->nCell==0 ) return SQLITE_CORRUPT_BKPT; /* dbfuzz001.test */ + assert( pPage->nFree>=0 ); + assert( pParent->nFree>=0 ); /* Allocate a new page. This page will become the right-sibling of ** pPage. Make the parent page writable, so that the new divider cell @@ -69053,12 +71624,22 @@ static int balance_quick(MemPage *pParent, MemPage *pPage, u8 *pSpace){ u8 *pCell = pPage->apOvfl[0]; u16 szCell = pPage->xCellSize(pPage, pCell); u8 *pStop; + CellArray b; assert( sqlite3PagerIswriteable(pNew->pDbPage) ); - assert( pPage->aData[0]==(PTF_INTKEY|PTF_LEAFDATA|PTF_LEAF) ); + assert( CORRUPT_DB || pPage->aData[0]==(PTF_INTKEY|PTF_LEAFDATA|PTF_LEAF) ); zeroPage(pNew, PTF_INTKEY|PTF_LEAFDATA|PTF_LEAF); - rc = rebuildPage(pNew, 1, &pCell, &szCell); - if( NEVER(rc) ) return rc; + b.nCell = 1; + b.pRef = pPage; + b.apCell = &pCell; + b.szCell = &szCell; + b.apEnd[0] = pPage->aDataEnd; + b.ixNx[0] = 2; + rc = rebuildPage(&b, 0, 1, pNew); + if( NEVER(rc) ){ + releasePage(pNew); + return rc; + } pNew->nFree = pBt->usableSize - pNew->cellOffset - 2 - szCell; /* If this is an auto-vacuum database, update the pointer map @@ -69073,7 +71654,7 @@ static int balance_quick(MemPage *pParent, MemPage *pPage, u8 *pSpace){ if( ISAUTOVACUUM ){ ptrmapPut(pBt, pgnoNew, PTRMAP_BTREE, pParent->pgno, &rc); if( szCell>pNew->minLocal ){ - ptrmapPutOvflPtr(pNew, pCell, &rc); + ptrmapPutOvflPtr(pNew, pNew, pCell, &rc); } } @@ -69199,6 +71780,7 @@ static void copyNodeContent(MemPage *pFrom, MemPage *pTo, int *pRC){ */ pTo->isInit = 0; rc = btreeInitPage(pTo); + if( rc==SQLITE_OK ) rc = btreeComputeFreeSpace(pTo); if( rc!=SQLITE_OK ){ *pRC = rc; return; @@ -69296,10 +71878,6 @@ static int balance_nonroot( assert( sqlite3_mutex_held(pBt->mutex) ); assert( sqlite3PagerIswriteable(pParent->pDbPage) ); -#if 0 - TRACE(("BALANCE: begin page %d child of %d\n", pPage->pgno, pParent->pgno)); -#endif - /* At this point pParent may have at most one overflow cell. And if ** this overflow cell is present, it must be the cell with ** index iParentIdx. This scenario comes about when this function @@ -69311,6 +71889,7 @@ static int balance_nonroot( if( !aOvflSpace ){ return SQLITE_NOMEM_BKPT; } + assert( pParent->nFree>=0 ); /* Find the sibling pages to balance. Also locate the cells in pParent ** that divide the siblings. An attempt is made to find NN siblings on @@ -69350,7 +71929,13 @@ static int balance_nonroot( memset(apOld, 0, (i+1)*sizeof(MemPage*)); goto balance_cleanup; } - nMaxCells += 1+apOld[i]->nCell+apOld[i]->nOverflow; + if( apOld[i]->nFree<0 ){ + rc = btreeComputeFreeSpace(apOld[i]); + if( rc ){ + memset(apOld, 0, (i)*sizeof(MemPage*)); + goto balance_cleanup; + } + } if( (i--)==0 ) break; if( pParent->nOverflow && i+nxDiv==pParent->aiOvfl[0] ){ @@ -69394,6 +71979,7 @@ static int balance_nonroot( /* Make nMaxCells a multiple of 4 in order to preserve 8-byte ** alignment */ + nMaxCells = nOld*(MX_CELL(pBt) + ArraySize(pParent->apOvfl)); nMaxCells = (nMaxCells + 3)&~3; /* @@ -69404,7 +71990,7 @@ static int balance_nonroot( + nMaxCells*sizeof(u16) /* b.szCell */ + pBt->pageSize; /* aSpace1 */ - assert( szScratch<=6*(int)pBt->pageSize ); + assert( szScratch<=7*(int)pBt->pageSize ); b.apCell = sqlite3StackAllocRaw(0, szScratch ); if( b.apCell==0 ){ rc = SQLITE_NOMEM_BKPT; @@ -69440,6 +72026,7 @@ static int balance_nonroot( u16 maskPage = pOld->maskPage; u8 *piCell = aData + pOld->cellOffset; u8 *piEnd; + VVA_ONLY( int nCellAtStart = b.nCell; ) /* Verify that all sibling pages are of the same "type" (table-leaf, ** table-interior, index-leaf, or index-interior). @@ -69468,6 +72055,10 @@ static int balance_nonroot( */ memset(&b.szCell[b.nCell], 0, sizeof(b.szCell[0])*(limit+pOld->nOverflow)); if( pOld->nOverflow>0 ){ + if( NEVER(limitaiOvfl[0]) ){ + rc = SQLITE_CORRUPT_BKPT; + goto balance_cleanup; + } limit = pOld->aiOvfl[0]; for(j=0; jnCell+pOld->nOverflow) ); cntOld[i] = b.nCell; if( iusableSize - 12 + leafCorrection; - for(i=0; iaDataEnd; + b.ixNx[k] = cntOld[i]; + if( k && b.ixNx[k]==b.ixNx[k-1] ){ + k--; /* Omit b.ixNx[] entry for child pages with no cells */ + } + if( !leafData ){ + k++; + b.apEnd[k] = pParent->aDataEnd; + b.ixNx[k] = cntOld[i]+1; + } + assert( p->nFree>=0 ); szNew[i] = usableSpace - p->nFree; for(j=0; jnOverflow; j++){ szNew[i] += 2 + p->xCellSize(p, p->apOvfl[j]); @@ -69738,6 +72341,8 @@ static int balance_nonroot( )); assert( sqlite3PagerIswriteable(pParent->pDbPage) ); + assert( nNew>=1 && nNew<=ArraySize(apNew) ); + assert( apNew[nNew-1]!=0 ); put4byte(pRight, apNew[nNew-1]->pgno); /* If the sibling pages are not leaves, ensure that the right-child pointer @@ -69765,19 +72370,20 @@ static int balance_nonroot( ** populated, not here. */ if( ISAUTOVACUUM ){ - MemPage *pNew = apNew[0]; - u8 *aOld = pNew->aData; + MemPage *pOld; + MemPage *pNew = pOld = apNew[0]; int cntOldNext = pNew->nCell + pNew->nOverflow; - int usableSize = pBt->usableSize; int iNew = 0; int iOld = 0; for(i=0; i=0 && iOldnCell + pOld->nOverflow + !leafData; - aOld = pOld->aData; } if( i==cntNew[iNew] ){ pNew = apNew[++iNew]; @@ -69792,13 +72398,13 @@ static int balance_nonroot( ** overflow cell), we can skip updating the pointer map entries. */ if( iOld>=nNew || pNew->pgno!=aPgno[iOld] - || !SQLITE_WITHIN(pCell,aOld,&aOld[usableSize]) + || !SQLITE_WITHIN(pCell,pOld->aData,pOld->aDataEnd) ){ if( !leafCorrection ){ ptrmapPut(pBt, get4byte(pCell), PTRMAP_BTREE, pNew->pgno, &rc); } if( cachedCellSize(&b,i)>pNew->minLocal ){ - ptrmapPutOvflPtr(pNew, pCell, &rc); + ptrmapPutOvflPtr(pNew, pOld, pCell, &rc); } if( rc ) goto balance_cleanup; } @@ -69943,7 +72549,8 @@ static int balance_nonroot( rc = defragmentPage(apNew[0], -1); testcase( rc!=SQLITE_OK ); assert( apNew[0]->nFree == - (get2byte(&apNew[0]->aData[5])-apNew[0]->cellOffset-apNew[0]->nCell*2) + (get2byteNotZero(&apNew[0]->aData[5]) - apNew[0]->cellOffset + - apNew[0]->nCell*2) || rc!=SQLITE_OK ); copyNodeContent(apNew[0], pParent, &rc); @@ -70042,7 +72649,7 @@ static int balance_deeper(MemPage *pRoot, MemPage **ppChild){ } assert( sqlite3PagerIswriteable(pChild->pDbPage) ); assert( sqlite3PagerIswriteable(pRoot->pDbPage) ); - assert( pChild->nCell==pRoot->nCell ); + assert( pChild->nCell==pRoot->nCell || CORRUPT_DB ); TRACE(("BALANCE: copy root %d into %d\n", pRoot->pgno, pChild->pgno)); @@ -70061,6 +72668,30 @@ static int balance_deeper(MemPage *pRoot, MemPage **ppChild){ return SQLITE_OK; } +/* +** Return SQLITE_CORRUPT if any cursor other than pCur is currently valid +** on the same B-tree as pCur. +** +** This can if a database is corrupt with two or more SQL tables +** pointing to the same b-tree. If an insert occurs on one SQL table +** and causes a BEFORE TRIGGER to do a secondary insert on the other SQL +** table linked to the same b-tree. If the secondary insert causes a +** rebalance, that can change content out from under the cursor on the +** first SQL table, violating invariants on the first insert. +*/ +static int anotherValidCursor(BtCursor *pCur){ + BtCursor *pOther; + for(pOther=pCur->pBt->pCursor; pOther; pOther=pOther->pNext){ + if( pOther!=pCur + && pOther->eState==CURSOR_VALID + && pOther->pPage==pCur->pPage + ){ + return SQLITE_CORRUPT_BKPT; + } + } + return SQLITE_OK; +} + /* ** The page that pCur currently points to has just been modified in ** some way. This function figures out if this modification means the @@ -70081,11 +72712,14 @@ static int balance(BtCursor *pCur){ VVA_ONLY( int balance_deeper_called = 0 ); do { - int iPage = pCur->iPage; + int iPage; MemPage *pPage = pCur->pPage; - if( iPage==0 ){ - if( pPage->nOverflow ){ + if( NEVER(pPage->nFree<0) && btreeComputeFreeSpace(pPage) ) break; + if( pPage->nOverflow==0 && pPage->nFree<=nMin ){ + break; + }else if( (iPage = pCur->iPage)==0 ){ + if( pPage->nOverflow && (rc = anotherValidCursor(pCur))==SQLITE_OK ){ /* The root page of the b-tree is overfull. In this case call the ** balance_deeper() function to create a new child for the root-page ** and copy the current contents of the root-page to it. The @@ -70105,13 +72739,14 @@ static int balance(BtCursor *pCur){ }else{ break; } - }else if( pPage->nOverflow==0 && pPage->nFree<=nMin ){ - break; }else{ MemPage * const pParent = pCur->apPage[iPage-1]; int const iIdx = pCur->aiIdx[iPage-1]; rc = sqlite3PagerWrite(pParent->pDbPage); + if( rc==SQLITE_OK && pParent->nFree<0 ){ + rc = btreeComputeFreeSpace(pParent); + } if( rc==SQLITE_OK ){ #ifndef SQLITE_OMIT_QUICKBALANCE if( pPage->intKeyLeaf @@ -70222,7 +72857,11 @@ static int btreeOverwriteContent( if( memcmp(pDest, ((u8*)pX->pData) + iOffset, iAmt)!=0 ){ int rc = sqlite3PagerWrite(pPage->pDbPage); if( rc ) return rc; - memcpy(pDest, ((u8*)pX->pData) + iOffset, iAmt); + /* In a corrupt database, it is possible for the source and destination + ** buffers to overlap. This is harmless since the database is already + ** corrupt but it does cause valgrind and ASAN warnings. So use + ** memmove(). */ + memmove(pDest, ((u8*)pX->pData) + iOffset, iAmt); } } return SQLITE_OK; @@ -70241,7 +72880,9 @@ static int btreeOverwriteCell(BtCursor *pCur, const BtreePayload *pX){ Pgno ovflPgno; /* Next overflow page to write */ u32 ovflPageSize; /* Size to write on overflow page */ - if( pCur->info.pPayload + pCur->info.nLocal > pPage->aDataEnd ){ + if( pCur->info.pPayload + pCur->info.nLocal > pPage->aDataEnd + || pCur->info.pPayload < pPage->aData + pPage->cellOffset + ){ return SQLITE_CORRUPT_BKPT; } /* Overwrite the local portion first */ @@ -70374,7 +73015,6 @@ SQLITE_PRIVATE int sqlite3BtreeInsert( if( flags & BTREE_SAVEPOSITION ){ assert( pCur->curFlags & BTCF_ValidNKey ); assert( pX->nKey==pCur->info.nKey ); - assert( pCur->info.nSize!=0 ); assert( loc==0 ); } #endif @@ -70449,11 +73089,17 @@ SQLITE_PRIVATE int sqlite3BtreeInsert( } } - assert( pCur->eState==CURSOR_VALID || (pCur->eState==CURSOR_INVALID && loc) ); + assert( pCur->eState==CURSOR_VALID + || (pCur->eState==CURSOR_INVALID && loc) + || CORRUPT_DB ); pPage = pCur->pPage; assert( pPage->intKey || pX->nKey>=0 ); assert( pPage->leaf || !pPage->intKey ); + if( pPage->nFree<0 ){ + rc = btreeComputeFreeSpace(pPage); + if( rc ) return rc; + } TRACE(("INSERT: table=%d nkey=%lld ndata=%d page=%d %s\n", pCur->pgnoRoot, pX->nKey, pX->nData, pPage->pgno, @@ -70478,6 +73124,8 @@ SQLITE_PRIVATE int sqlite3BtreeInsert( memcpy(newCell, oldCell, 4); } rc = clearCell(pPage, oldCell, &info); + testcase( pCur->curFlags & BTCF_ValidOvfl ); + invalidateOverflowCache(pCur); if( info.nSize==szNew && info.nLocal==info.nPayload && (!ISAUTOVACUUM || szNewminLocal) ){ @@ -70491,7 +73139,12 @@ SQLITE_PRIVATE int sqlite3BtreeInsert( ** new entry uses overflow pages, as the insertCell() call below is ** necessary to add the PTRMAP_OVERFLOW1 pointer-map entry. */ assert( rc==SQLITE_OK ); /* clearCell never fails when nLocal==nPayload */ - if( oldCell+szNew > pPage->aDataEnd ) return SQLITE_CORRUPT_BKPT; + if( oldCell < pPage->aData+pPage->hdrOffset+10 ){ + return SQLITE_CORRUPT_BKPT; + } + if( oldCell+szNew > pPage->aDataEnd ){ + return SQLITE_CORRUPT_BKPT; + } memcpy(oldCell, newCell, szNew); return SQLITE_OK; } @@ -70596,14 +73249,18 @@ SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor *pCur, u8 flags){ assert( pCur->curFlags & BTCF_WriteFlag ); assert( hasSharedCacheTableLock(p, pCur->pgnoRoot, pCur->pKeyInfo!=0, 2) ); assert( !hasReadConflicts(p, pCur->pgnoRoot) ); - assert( pCur->ixpPage->nCell ); - assert( pCur->eState==CURSOR_VALID ); assert( (flags & ~(BTREE_SAVEPOSITION | BTREE_AUXDELETE))==0 ); + if( pCur->eState==CURSOR_REQUIRESEEK ){ + rc = btreeRestoreCursorPosition(pCur); + if( rc ) return rc; + } + assert( pCur->eState==CURSOR_VALID ); iCellDepth = pCur->iPage; iCellIdx = pCur->ix; pPage = pCur->pPage; pCell = findCell(pPage, iCellIdx); + if( pPage->nFree<0 && btreeComputeFreeSpace(pPage) ) return SQLITE_CORRUPT; /* If the bPreserve flag is set to true, then the cursor position must ** be preserved following this delete operation. If the current delete @@ -70617,6 +73274,7 @@ SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor *pCur, u8 flags){ if( bPreserve ){ if( !pPage->leaf || (pPage->nFree+cellSizePtr(pPage,pCell)+2)>(int)(pBt->usableSize*2/3) + || pPage->nCell==1 /* See dbfuzz001.test for a test case */ ){ /* A b-tree rebalance will be required after deleting this entry. ** Save the cursor key. */ @@ -70673,6 +73331,10 @@ SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor *pCur, u8 flags){ Pgno n; unsigned char *pTmp; + if( pLeaf->nFree<0 ){ + rc = btreeComputeFreeSpace(pLeaf); + if( rc ) return rc; + } if( iCellDepthiPage-1 ){ n = pCur->apPage[iCellDepth+1]->pgno; }else{ @@ -71031,6 +73693,9 @@ static int btreeDropTable(Btree *p, Pgno iTable, int *piMoved){ assert( sqlite3BtreeHoldsMutex(p) ); assert( p->inTrans==TRANS_WRITE ); assert( iTable>=2 ); + if( iTable>btreePagecount(pBt) ){ + return SQLITE_CORRUPT_BKPT; + } rc = btreeGetPage(pBt, (Pgno)iTable, &pPage, 0); if( rc ) return rc; @@ -71188,7 +73853,6 @@ SQLITE_PRIVATE int sqlite3BtreeUpdateMeta(Btree *p, int idx, u32 iMeta){ return rc; } -#ifndef SQLITE_OMIT_BTREECOUNT /* ** The first argument, pCur, is a cursor opened on some b-tree. Count the ** number of entries in the b-tree and write the result to *pnEntry. @@ -71197,7 +73861,7 @@ SQLITE_PRIVATE int sqlite3BtreeUpdateMeta(Btree *p, int idx, u32 iMeta){ ** Otherwise, if an error is encountered (i.e. an IO error or database ** corruption) an SQLite error code is returned. */ -SQLITE_PRIVATE int sqlite3BtreeCount(BtCursor *pCur, i64 *pnEntry){ +SQLITE_PRIVATE int sqlite3BtreeCount(sqlite3 *db, BtCursor *pCur, i64 *pnEntry){ i64 nEntry = 0; /* Value to return in *pnEntry */ int rc; /* Return code */ @@ -71210,7 +73874,7 @@ SQLITE_PRIVATE int sqlite3BtreeCount(BtCursor *pCur, i64 *pnEntry){ /* Unless an error occurs, the following loop runs one iteration for each ** page in the B-Tree structure (not including overflow pages). */ - while( rc==SQLITE_OK ){ + while( rc==SQLITE_OK && !AtomicLoad(&db->u1.isInterrupted) ){ int iIdx; /* Index of child node in parent */ MemPage *pPage; /* Current page of the b-tree */ @@ -71261,7 +73925,6 @@ SQLITE_PRIVATE int sqlite3BtreeCount(BtCursor *pCur, i64 *pnEntry){ /* An error has occurred. Return an error code. */ return rc; } -#endif /* ** Return the pager associated with a BTree. This routine is used for @@ -71328,8 +73991,7 @@ static void setPageReferenced(IntegrityCk *pCheck, Pgno iPg){ ** Also check that the page number is in bounds. */ static int checkRef(IntegrityCk *pCheck, Pgno iPage){ - if( iPage==0 ) return 1; - if( iPage>pCheck->nPage ){ + if( iPage>pCheck->nPage || iPage==0 ){ checkAppendMsg(pCheck, "invalid page number %d", iPage); return 1; } @@ -71337,6 +73999,7 @@ static int checkRef(IntegrityCk *pCheck, Pgno iPage){ checkAppendMsg(pCheck, "2nd reference to page %d", iPage); return 1; } + if( AtomicLoad(&pCheck->db->u1.isInterrupted) ) return 1; setPageReferenced(pCheck, iPage); return 0; } @@ -71380,39 +74043,34 @@ static void checkList( IntegrityCk *pCheck, /* Integrity checking context */ int isFreeList, /* True for a freelist. False for overflow page list */ int iPage, /* Page number for first page in the list */ - int N /* Expected number of pages in the list */ + u32 N /* Expected number of pages in the list */ ){ int i; - int expected = N; - int iFirst = iPage; - while( N-- > 0 && pCheck->mxErr ){ + u32 expected = N; + int nErrAtStart = pCheck->nErr; + while( iPage!=0 && pCheck->mxErr ){ DbPage *pOvflPage; unsigned char *pOvflData; - if( iPage<1 ){ - checkAppendMsg(pCheck, - "%d of %d pages missing from overflow list starting at %d", - N+1, expected, iFirst); - break; - } if( checkRef(pCheck, iPage) ) break; + N--; if( sqlite3PagerGet(pCheck->pPager, (Pgno)iPage, &pOvflPage, 0) ){ checkAppendMsg(pCheck, "failed to get page %d", iPage); break; } pOvflData = (unsigned char *)sqlite3PagerGetData(pOvflPage); if( isFreeList ){ - int n = get4byte(&pOvflData[4]); + u32 n = (u32)get4byte(&pOvflData[4]); #ifndef SQLITE_OMIT_AUTOVACUUM if( pCheck->pBt->autoVacuum ){ checkPtrmap(pCheck, iPage, PTRMAP_FREEPAGE, 0); } #endif - if( n>(int)pCheck->pBt->usableSize/4-2 ){ + if( n>pCheck->pBt->usableSize/4-2 ){ checkAppendMsg(pCheck, "freelist leaf count too big on page %d", iPage); N--; }else{ - for(i=0; ipBt->autoVacuum ){ @@ -71438,10 +74096,12 @@ static void checkList( #endif iPage = get4byte(pOvflData); sqlite3PagerUnref(pOvflPage); - - if( isFreeList && N<(iPage!=0) ){ - checkAppendMsg(pCheck, "free-page count in header is too small"); - } + } + if( N && nErrAtStart==pCheck->nErr ){ + checkAppendMsg(pCheck, + "%s is %d but should be %d", + isFreeList ? "size" : "overflow list length", + expected-N, expected); } } #endif /* SQLITE_OMIT_INTEGRITY_CHECK */ @@ -71568,6 +74228,11 @@ static int checkTreePage( "btreeInitPage() returns error code %d", rc); goto end_of_check; } + if( (rc = btreeComputeFreeSpace(pPage))!=0 ){ + assert( rc==SQLITE_CORRUPT ); + checkAppendMsg(pCheck, "free space corruption", rc); + goto end_of_check; + } data = pPage->aData; hdr = pPage->hdrOffset; @@ -71640,7 +74305,7 @@ static int checkTreePage( /* Check the content overflow list */ if( info.nPayload>info.nLocal ){ - int nPage; /* Number of pages on the overflow chain */ + u32 nPage; /* Number of pages on the overflow chain */ Pgno pgnoOvfl; /* First page of the overflow chain */ assert( pc + info.nSize - 4 <= usableSize ); nPage = (info.nPayload - info.nLocal + usableSize - 5)/(usableSize - 4); @@ -71700,9 +74365,9 @@ static int checkTreePage( i = get2byte(&data[hdr+1]); while( i>0 ){ int size, j; - assert( (u32)i<=usableSize-4 ); /* Enforced by btreeInitPage() */ + assert( (u32)i<=usableSize-4 ); /* Enforced by btreeComputeFreeSpace() */ size = get2byte(&data[i+2]); - assert( (u32)(i+size)<=usableSize ); /* Enforced by btreeInitPage() */ + assert( (u32)(i+size)<=usableSize ); /* due to btreeComputeFreeSpace() */ btreeHeapInsert(heap, (((u32)i)<<16)|(i+size-1)); /* EVIDENCE-OF: R-58208-19414 The first 2 bytes of a freeblock are a ** big-endian integer which is the offset in the b-tree page of the next @@ -71711,8 +74376,8 @@ static int checkTreePage( j = get2byte(&data[i]); /* EVIDENCE-OF: R-06866-39125 Freeblocks are always connected in order of ** increasing offset. */ - assert( j==0 || j>i+size ); /* Enforced by btreeInitPage() */ - assert( (u32)j<=usableSize-4 ); /* Enforced by btreeInitPage() */ + assert( j==0 || j>i+size ); /* Enforced by btreeComputeFreeSpace() */ + assert( (u32)j<=usableSize-4 ); /* Enforced by btreeComputeFreeSpace() */ i = j; } /* Analyze the min-heap looking for overlap between cells and/or @@ -71778,6 +74443,7 @@ static int checkTreePage( ** returned. If a memory allocation error occurs, NULL is returned. */ SQLITE_PRIVATE char *sqlite3BtreeIntegrityCheck( + sqlite3 *db, /* Database connection that is running the check */ Btree *p, /* The btree to be checked */ int *aRoot, /* An array of root pages numbers for individual trees */ int nRoot, /* Number of entries in aRoot[] */ @@ -71787,7 +74453,7 @@ SQLITE_PRIVATE char *sqlite3BtreeIntegrityCheck( Pgno i; IntegrityCk sCheck; BtShared *pBt = p->pBt; - int savedDbFlags = pBt->db->flags; + u64 savedDbFlags = pBt->db->flags; char zErr[100]; VVA_ONLY( int nRef ); @@ -71795,6 +74461,7 @@ SQLITE_PRIVATE char *sqlite3BtreeIntegrityCheck( assert( p->inTrans>TRANS_NONE && pBt->inTransaction>TRANS_NONE ); VVA_ONLY( nRef = sqlite3PagerRefcount(pBt->pPager) ); assert( nRef>=0 ); + sCheck.db = db; sCheck.pBt = pBt; sCheck.pPager = pBt->pPager; sCheck.nPage = btreePagecount(sCheck.pBt); @@ -71835,8 +74502,26 @@ SQLITE_PRIVATE char *sqlite3BtreeIntegrityCheck( /* Check all the tables. */ +#ifndef SQLITE_OMIT_AUTOVACUUM + if( pBt->autoVacuum ){ + int mx = 0; + int mxInHdr; + for(i=0; (int)ipPage1->aData[52]); + if( mx!=mxInHdr ){ + checkAppendMsg(&sCheck, + "max rootpage (%d) disagrees with header (%d)", + mx, mxInHdr + ); + } + }else if( get4byte(&pBt->pPage1->aData[64])!=0 ){ + checkAppendMsg(&sCheck, + "incremental_vacuum enabled with a max rootpage of zero" + ); + } +#endif testcase( pBt->db->flags & SQLITE_CellSizeCk ); - pBt->db->flags &= ~SQLITE_CellSizeCk; + pBt->db->flags &= ~(u64)SQLITE_CellSizeCk; for(i=0; (int)ibtsFlags &= ~BTS_NO_WAL; if( iVersion==1 ) pBt->btsFlags |= BTS_NO_WAL; - rc = sqlite3BtreeBeginTrans(pBtree, 0); + rc = sqlite3BtreeBeginTrans(pBtree, 0, 0); if( rc==SQLITE_OK ){ u8 *aData = pBt->pPage1->aData; if( aData[18]!=(u8)iVersion || aData[19]!=(u8)iVersion ){ - rc = sqlite3BtreeBeginTrans(pBtree, 2); + rc = sqlite3BtreeBeginTrans(pBtree, 2, 0); if( rc==SQLITE_OK ){ rc = sqlite3PagerWrite(pBt->pPage1->pDbPage); if( rc==SQLITE_OK ){ @@ -72290,7 +74975,7 @@ static Btree *findBtree(sqlite3 *pErrorDb, sqlite3 *pDb, const char *zDb){ */ static int setDestPgsz(sqlite3_backup *p){ int rc; - rc = sqlite3BtreeSetPageSize(p->pDest,sqlite3BtreeGetPageSize(p->pSrc),-1,0); + rc = sqlite3BtreeSetPageSize(p->pDest,sqlite3BtreeGetPageSize(p->pSrc),0,0); return rc; } @@ -72413,13 +75098,6 @@ static int backupOnePage( int nDestPgsz = sqlite3BtreeGetPageSize(p->pDest); const int nCopy = MIN(nSrcPgsz, nDestPgsz); const i64 iEnd = (i64)iSrcPg*(i64)nSrcPgsz; -#ifdef SQLITE_HAS_CODEC - /* Use BtreeGetReserveNoMutex() for the source b-tree, as although it is - ** guaranteed that the shared-mutex is held by this thread, handle - ** p->pSrc may not actually be the owner. */ - int nSrcReserve = sqlite3BtreeGetReserveNoMutex(p->pSrc); - int nDestReserve = sqlite3BtreeGetOptimalReserve(p->pDest); -#endif int rc = SQLITE_OK; i64 iOff; @@ -72436,26 +75114,6 @@ static int backupOnePage( rc = SQLITE_READONLY; } -#ifdef SQLITE_HAS_CODEC - /* Backup is not possible if the page size of the destination is changing - ** and a codec is in use. - */ - if( nSrcPgsz!=nDestPgsz && sqlite3PagerGetCodec(pDestPager)!=0 ){ - rc = SQLITE_READONLY; - } - - /* Backup is not possible if the number of bytes of reserve space differ - ** between source and destination. If there is a difference, try to - ** fix the destination to agree with the source. If that is not possible, - ** then the backup cannot proceed. - */ - if( nSrcReserve!=nDestReserve ){ - u32 newPgsz = nSrcPgsz; - rc = sqlite3PagerSetPagesize(pDestPager, &newPgsz, nSrcReserve); - if( rc==SQLITE_OK && newPgsz!=nSrcPgsz ) rc = SQLITE_READONLY; - } -#endif - /* This loop runs once for each destination page spanned by the source ** page. For each iteration, variable iOff is set to the byte offset ** of the destination page. @@ -72560,7 +75218,7 @@ SQLITE_API int sqlite3_backup_step(sqlite3_backup *p, int nPage){ ** before this function exits. */ if( rc==SQLITE_OK && 0==sqlite3BtreeIsInReadTrans(p->pSrc) ){ - rc = sqlite3BtreeBeginTrans(p->pSrc, 0); + rc = sqlite3BtreeBeginTrans(p->pSrc, 0, 0); bCloseTrans = 1; } @@ -72576,10 +75234,10 @@ SQLITE_API int sqlite3_backup_step(sqlite3_backup *p, int nPage){ /* Lock the destination database, if it is not locked already. */ if( SQLITE_OK==rc && p->bDestLocked==0 - && SQLITE_OK==(rc = sqlite3BtreeBeginTrans(p->pDest, 2)) + && SQLITE_OK==(rc = sqlite3BtreeBeginTrans(p->pDest, 2, + (int*)&p->iDestSchema)) ){ p->bDestLocked = 1; - sqlite3BtreeGetMeta(p->pDest, BTREE_SCHEMA_VERSION, &p->iDestSchema); } /* Do not allow backup if the destination database is in WAL mode @@ -72797,8 +75455,10 @@ SQLITE_API int sqlite3_backup_finish(sqlite3_backup *p){ } if( p->isAttached ){ pp = sqlite3PagerBackupPtr(sqlite3BtreePager(p->pSrc)); + assert( pp!=0 ); while( *pp!=p ){ pp = &(*pp)->pNext; + assert( pp!=0 ); } *pp = p->pNext; } @@ -72949,10 +75609,6 @@ SQLITE_PRIVATE int sqlite3BtreeCopyFile(Btree *pTo, Btree *pFrom){ b.pDest = pTo; b.iNext = 1; -#ifdef SQLITE_HAS_CODEC - sqlite3PagerAlignReserve(sqlite3BtreePager(pTo), sqlite3BtreePager(pFrom)); -#endif - /* 0x7FFFFFFF is the hard limit for the number of pages in a database ** file. By passing this as the number of pages to copy to ** sqlite3_backup_step(), we can guarantee that the copy finishes @@ -72999,6 +75655,11 @@ SQLITE_PRIVATE int sqlite3BtreeCopyFile(Btree *pTo, Btree *pFrom){ /* #include "sqliteInt.h" */ /* #include "vdbeInt.h" */ +/* True if X is a power of two. 0 is considered a power of two here. +** In other words, return true if X has at most one bit set. +*/ +#define ISPOWEROF2(X) (((X)&((X)-1))==0) + #ifdef SQLITE_DEBUG /* ** Check invariants on a Mem object. @@ -73018,13 +75679,12 @@ SQLITE_PRIVATE int sqlite3VdbeCheckMemInvariants(Mem *p){ ** That saves a few cycles in inner loops. */ assert( (p->flags & MEM_Dyn)==0 || p->szMalloc==0 ); - /* Cannot be both MEM_Int and MEM_Real at the same time */ - assert( (p->flags & (MEM_Int|MEM_Real))!=(MEM_Int|MEM_Real) ); + /* Cannot have more than one of MEM_Int, MEM_Real, or MEM_IntReal */ + assert( ISPOWEROF2(p->flags & (MEM_Int|MEM_Real|MEM_IntReal)) ); if( p->flags & MEM_Null ){ /* Cannot be both MEM_Null and some other type */ - assert( (p->flags & (MEM_Int|MEM_Real|MEM_Str|MEM_Blob - |MEM_RowSet|MEM_Frame|MEM_Agg))==0 ); + assert( (p->flags & (MEM_Int|MEM_Real|MEM_Str|MEM_Blob|MEM_Agg))==0 ); /* If MEM_Null is set, then either the value is a pure NULL (the usual ** case) or it is a pointer set using sqlite3_bind_pointer() or @@ -73039,7 +75699,7 @@ SQLITE_PRIVATE int sqlite3VdbeCheckMemInvariants(Mem *p){ ((p->flags&MEM_Static)!=0 ? 1 : 0) <= 1 ); /* No other bits set */ - assert( (p->flags & ~(MEM_Null|MEM_Term|MEM_Subtype + assert( (p->flags & ~(MEM_Null|MEM_Term|MEM_Subtype|MEM_FromBind |MEM_Dyn|MEM_Ephem|MEM_Static))==0 ); }else{ /* A pure NULL might have other flags, such as MEM_Static, MEM_Dyn, @@ -73074,9 +75734,31 @@ SQLITE_PRIVATE int sqlite3VdbeCheckMemInvariants(Mem *p){ } #endif +/* +** Render a Mem object which is one of MEM_Int, MEM_Real, or MEM_IntReal +** into a buffer. +*/ +static void vdbeMemRenderNum(int sz, char *zBuf, Mem *p){ + StrAccum acc; + assert( p->flags & (MEM_Int|MEM_Real|MEM_IntReal) ); + sqlite3StrAccumInit(&acc, 0, zBuf, sz, 0); + if( p->flags & MEM_Int ){ + sqlite3_str_appendf(&acc, "%lld", p->u.i); + }else if( p->flags & MEM_IntReal ){ + sqlite3_str_appendf(&acc, "%!.15g", (double)p->u.i); + }else{ + sqlite3_str_appendf(&acc, "%!.15g", p->u.r); + } + assert( acc.zText==zBuf && acc.mxAlloc<=0 ); + zBuf[acc.nChar] = 0; /* Fast version of sqlite3StrAccumFinish(&acc) */ +} + #ifdef SQLITE_DEBUG /* -** Check that string value of pMem agrees with its integer or real value. +** Validity checks on pMem. pMem holds a string. +** +** (1) Check that string value of pMem agrees with its integer or real value. +** (2) Check that the string is correctly zero terminated ** ** A single int or real value always converts to the same strings. But ** many different strings can be converted into the same int or real. @@ -73094,17 +75776,24 @@ SQLITE_PRIVATE int sqlite3VdbeCheckMemInvariants(Mem *p){ ** ** This routine is for use inside of assert() statements only. */ -SQLITE_PRIVATE int sqlite3VdbeMemConsistentDualRep(Mem *p){ +SQLITE_PRIVATE int sqlite3VdbeMemValidStrRep(Mem *p){ char zBuf[100]; char *z; int i, j, incr; if( (p->flags & MEM_Str)==0 ) return 1; - if( (p->flags & (MEM_Int|MEM_Real))==0 ) return 1; - if( p->flags & MEM_Int ){ - sqlite3_snprintf(sizeof(zBuf),zBuf,"%lld",p->u.i); - }else{ - sqlite3_snprintf(sizeof(zBuf),zBuf,"%!.15g",p->u.r); - } + if( p->flags & MEM_Term ){ + /* Insure that the string is properly zero-terminated. Pay particular + ** attention to the case where p->n is odd */ + if( p->szMalloc>0 && p->z==p->zMalloc ){ + assert( p->enc==SQLITE_UTF8 || p->szMalloc >= ((p->n+1)&~1)+2 ); + assert( p->enc!=SQLITE_UTF8 || p->szMalloc >= p->n+1 ); + } + assert( p->z[p->n]==0 ); + assert( p->enc==SQLITE_UTF8 || p->z[(p->n+1)&~1]==0 ); + assert( p->enc==SQLITE_UTF8 || p->z[((p->n+1)&~1)+1]==0 ); + } + if( (p->flags & (MEM_Int|MEM_Real|MEM_IntReal))==0 ) return 1; + vdbeMemRenderNum(sizeof(zBuf), zBuf, p); z = p->z; i = j = 0; incr = 1; @@ -73137,7 +75826,7 @@ SQLITE_PRIVATE int sqlite3VdbeChangeEncoding(Mem *pMem, int desiredEnc){ #ifndef SQLITE_OMIT_UTF16 int rc; #endif - assert( (pMem->flags&MEM_RowSet)==0 ); + assert( !sqlite3VdbeMemIsRowSet(pMem) ); assert( desiredEnc==SQLITE_UTF8 || desiredEnc==SQLITE_UTF16LE || desiredEnc==SQLITE_UTF16BE ); if( !(pMem->flags&MEM_Str) || pMem->enc==desiredEnc ){ @@ -73160,8 +75849,7 @@ SQLITE_PRIVATE int sqlite3VdbeChangeEncoding(Mem *pMem, int desiredEnc){ } /* -** Make sure pMem->z points to a writable allocation of at least -** min(n,32) bytes. +** Make sure pMem->z points to a writable allocation of at least n bytes. ** ** If the bPreserve argument is true, then copy of the content of ** pMem->z into the new allocation. pMem must be either a string or @@ -73170,7 +75858,7 @@ SQLITE_PRIVATE int sqlite3VdbeChangeEncoding(Mem *pMem, int desiredEnc){ */ SQLITE_PRIVATE SQLITE_NOINLINE int sqlite3VdbeMemGrow(Mem *pMem, int n, int bPreserve){ assert( sqlite3VdbeCheckMemInvariants(pMem) ); - assert( (pMem->flags&MEM_RowSet)==0 ); + assert( !sqlite3VdbeMemIsRowSet(pMem) ); testcase( pMem->db==0 ); /* If the bPreserve flag is set to true, then the memory cell must already @@ -73180,9 +75868,14 @@ SQLITE_PRIVATE SQLITE_NOINLINE int sqlite3VdbeMemGrow(Mem *pMem, int n, int bPre assert( pMem->szMalloc==0 || pMem->szMalloc==sqlite3DbMallocSize(pMem->db, pMem->zMalloc) ); - if( n<32 ) n = 32; if( pMem->szMalloc>0 && bPreserve && pMem->z==pMem->zMalloc ){ - pMem->z = pMem->zMalloc = sqlite3DbReallocOrFree(pMem->db, pMem->z, n); + if( pMem->db ){ + pMem->z = pMem->zMalloc = sqlite3DbReallocOrFree(pMem->db, pMem->z, n); + }else{ + pMem->zMalloc = sqlite3Realloc(pMem->z, n); + if( pMem->zMalloc==0 ) sqlite3_free(pMem->z); + pMem->z = pMem->zMalloc; + } bPreserve = 0; }else{ if( pMem->szMalloc>0 ) sqlite3DbFreeNN(pMem->db, pMem->zMalloc); @@ -73218,34 +75911,40 @@ SQLITE_PRIVATE SQLITE_NOINLINE int sqlite3VdbeMemGrow(Mem *pMem, int n, int bPre ** ** Any prior string or blob content in the pMem object may be discarded. ** The pMem->xDel destructor is called, if it exists. Though MEM_Str -** and MEM_Blob values may be discarded, MEM_Int, MEM_Real, and MEM_Null -** values are preserved. +** and MEM_Blob values may be discarded, MEM_Int, MEM_Real, MEM_IntReal, +** and MEM_Null values are preserved. ** ** Return SQLITE_OK on success or an error code (probably SQLITE_NOMEM) ** if unable to complete the resizing. */ SQLITE_PRIVATE int sqlite3VdbeMemClearAndResize(Mem *pMem, int szNew){ - assert( szNew>0 ); + assert( CORRUPT_DB || szNew>0 ); assert( (pMem->flags & MEM_Dyn)==0 || pMem->szMalloc==0 ); if( pMem->szMallocflags & MEM_Dyn)==0 ); pMem->z = pMem->zMalloc; - pMem->flags &= (MEM_Null|MEM_Int|MEM_Real); + pMem->flags &= (MEM_Null|MEM_Int|MEM_Real|MEM_IntReal); return SQLITE_OK; } /* ** It is already known that pMem contains an unterminated string. ** Add the zero terminator. +** +** Three bytes of zero are added. In this way, there is guaranteed +** to be a double-zero byte at an even byte boundary in order to +** terminate a UTF16 string, even if the initial size of the buffer +** is an odd number of bytes. */ static SQLITE_NOINLINE int vdbeMemAddTerminator(Mem *pMem){ - if( sqlite3VdbeMemGrow(pMem, pMem->n+2, 1) ){ + if( sqlite3VdbeMemGrow(pMem, pMem->n+3, 1) ){ return SQLITE_NOMEM_BKPT; } pMem->z[pMem->n] = 0; pMem->z[pMem->n+1] = 0; + pMem->z[pMem->n+2] = 0; pMem->flags |= MEM_Term; return SQLITE_OK; } @@ -73258,7 +75957,7 @@ static SQLITE_NOINLINE int vdbeMemAddTerminator(Mem *pMem){ */ SQLITE_PRIVATE int sqlite3VdbeMemMakeWriteable(Mem *pMem){ assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); - assert( (pMem->flags&MEM_RowSet)==0 ); + assert( !sqlite3VdbeMemIsRowSet(pMem) ); if( (pMem->flags & (MEM_Str|MEM_Blob))!=0 ){ if( ExpandBlob(pMem) ) return SQLITE_NOMEM; if( pMem->szMalloc==0 || pMem->z!=pMem->zMalloc ){ @@ -73282,13 +75981,15 @@ SQLITE_PRIVATE int sqlite3VdbeMemMakeWriteable(Mem *pMem){ SQLITE_PRIVATE int sqlite3VdbeMemExpandBlob(Mem *pMem){ int nByte; assert( pMem->flags & MEM_Zero ); - assert( pMem->flags&MEM_Blob ); - assert( (pMem->flags&MEM_RowSet)==0 ); + assert( (pMem->flags&MEM_Blob)!=0 || MemNullNochng(pMem) ); + testcase( sqlite3_value_nochange(pMem) ); + assert( !sqlite3VdbeMemIsRowSet(pMem) ); assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); /* Set nByte to the number of bytes required to store the expanded blob. */ nByte = pMem->n + pMem->u.nZero; if( nByte<=0 ){ + if( (pMem->flags & MEM_Blob)==0 ) return SQLITE_OK; nByte = 1; } if( sqlite3VdbeMemGrow(pMem, nByte, 1) ){ @@ -73317,12 +76018,12 @@ SQLITE_PRIVATE int sqlite3VdbeMemNulTerminate(Mem *pMem){ } /* -** Add MEM_Str to the set of representations for the given Mem. Numbers -** are converted using sqlite3_snprintf(). Converting a BLOB to a string -** is a no-op. +** Add MEM_Str to the set of representations for the given Mem. This +** routine is only called if pMem is a number of some kind, not a NULL +** or a BLOB. ** -** Existing representations MEM_Int and MEM_Real are invalidated if -** bForce is true but are retained if bForce is false. +** Existing representations MEM_Int, MEM_Real, or MEM_IntReal are invalidated +** if bForce is true but are retained if bForce is false. ** ** A MEM_Null value will never be passed to this function. This function is ** used for converting values to text for returning to the user (i.e. via @@ -73331,14 +76032,13 @@ SQLITE_PRIVATE int sqlite3VdbeMemNulTerminate(Mem *pMem){ ** user and the latter is an internal programming error. */ SQLITE_PRIVATE int sqlite3VdbeMemStringify(Mem *pMem, u8 enc, u8 bForce){ - int fg = pMem->flags; const int nByte = 32; assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); - assert( !(fg&MEM_Zero) ); - assert( !(fg&(MEM_Str|MEM_Blob)) ); - assert( fg&(MEM_Int|MEM_Real) ); - assert( (pMem->flags&MEM_RowSet)==0 ); + assert( !(pMem->flags&MEM_Zero) ); + assert( !(pMem->flags&(MEM_Str|MEM_Blob)) ); + assert( pMem->flags&(MEM_Int|MEM_Real|MEM_IntReal) ); + assert( !sqlite3VdbeMemIsRowSet(pMem) ); assert( EIGHT_BYTE_ALIGNMENT(pMem) ); @@ -73347,22 +76047,12 @@ SQLITE_PRIVATE int sqlite3VdbeMemStringify(Mem *pMem, u8 enc, u8 bForce){ return SQLITE_NOMEM_BKPT; } - /* For a Real or Integer, use sqlite3_snprintf() to produce the UTF-8 - ** string representation of the value. Then, if the required encoding - ** is UTF-16le or UTF-16be do a translation. - ** - ** FIX ME: It would be better if sqlite3_snprintf() could do UTF-16. - */ - if( fg & MEM_Int ){ - sqlite3_snprintf(nByte, pMem->z, "%lld", pMem->u.i); - }else{ - assert( fg & MEM_Real ); - sqlite3_snprintf(nByte, pMem->z, "%!.15g", pMem->u.r); - } - pMem->n = sqlite3Strlen30(pMem->z); + vdbeMemRenderNum(nByte, pMem->z, pMem); + assert( pMem->z!=0 ); + pMem->n = sqlite3Strlen30NN(pMem->z); pMem->enc = SQLITE_UTF8; pMem->flags |= MEM_Str|MEM_Term; - if( bForce ) pMem->flags &= ~(MEM_Int|MEM_Real); + if( bForce ) pMem->flags &= ~(MEM_Int|MEM_Real|MEM_IntReal); sqlite3VdbeChangeEncoding(pMem, enc); return SQLITE_OK; } @@ -73396,6 +76086,31 @@ SQLITE_PRIVATE int sqlite3VdbeMemFinalize(Mem *pMem, FuncDef *pFunc){ return ctx.isError; } +/* +** Memory cell pAccum contains the context of an aggregate function. +** This routine calls the xValue method for that function and stores +** the results in memory cell pMem. +** +** SQLITE_ERROR is returned if xValue() reports an error. SQLITE_OK +** otherwise. +*/ +#ifndef SQLITE_OMIT_WINDOWFUNC +SQLITE_PRIVATE int sqlite3VdbeMemAggValue(Mem *pAccum, Mem *pOut, FuncDef *pFunc){ + sqlite3_context ctx; + assert( pFunc!=0 ); + assert( pFunc->xValue!=0 ); + assert( (pAccum->flags & MEM_Null)!=0 || pFunc==pAccum->u.pDef ); + assert( pAccum->db==0 || sqlite3_mutex_held(pAccum->db->mutex) ); + memset(&ctx, 0, sizeof(ctx)); + sqlite3VdbeMemSetNull(pOut); + ctx.pOut = pOut; + ctx.pMem = pAccum; + ctx.pFunc = pFunc; + pFunc->xValue(&ctx); + return ctx.isError; +} +#endif /* SQLITE_OMIT_WINDOWFUNC */ + /* ** If the memory cell contains a value that must be freed by ** invoking the external callback in Mem.xDel, then this routine @@ -73414,15 +76129,8 @@ static SQLITE_NOINLINE void vdbeMemClearExternAndSetNull(Mem *p){ testcase( p->flags & MEM_Dyn ); } if( p->flags&MEM_Dyn ){ - assert( (p->flags&MEM_RowSet)==0 ); assert( p->xDel!=SQLITE_DYNAMIC && p->xDel!=0 ); p->xDel((void *)p->z); - }else if( p->flags&MEM_RowSet ){ - sqlite3RowSetClear(p->u.pRowSet); - }else if( p->flags&MEM_Frame ){ - VdbeFrame *pFrame = p->u.pFrame; - pFrame->pParent = pFrame->v->pDelFrame; - pFrame->v->pDelFrame = pFrame; } p->flags = MEM_Null; } @@ -73514,12 +76222,12 @@ SQLITE_PRIVATE i64 sqlite3VdbeIntValue(Mem *pMem){ assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); assert( EIGHT_BYTE_ALIGNMENT(pMem) ); flags = pMem->flags; - if( flags & MEM_Int ){ + if( flags & (MEM_Int|MEM_IntReal) ){ + testcase( flags & MEM_IntReal ); return pMem->u.i; }else if( flags & MEM_Real ){ return doubleToInt64(pMem->u.r); - }else if( flags & (MEM_Str|MEM_Blob) ){ - assert( pMem->z || pMem->n==0 ); + }else if( (flags & (MEM_Str|MEM_Blob))!=0 && pMem->z!=0 ){ return memIntValue(pMem); }else{ return 0; @@ -73543,7 +76251,8 @@ SQLITE_PRIVATE double sqlite3VdbeRealValue(Mem *pMem){ assert( EIGHT_BYTE_ALIGNMENT(pMem) ); if( pMem->flags & MEM_Real ){ return pMem->u.r; - }else if( pMem->flags & MEM_Int ){ + }else if( pMem->flags & (MEM_Int|MEM_IntReal) ){ + testcase( pMem->flags & MEM_IntReal ); return (double)pMem->u.i; }else if( pMem->flags & (MEM_Str|MEM_Blob) ){ return memRealValue(pMem); @@ -73558,7 +76267,8 @@ SQLITE_PRIVATE double sqlite3VdbeRealValue(Mem *pMem){ ** Return the value ifNull if pMem is NULL. */ SQLITE_PRIVATE int sqlite3VdbeBooleanValue(Mem *pMem, int ifNull){ - if( pMem->flags & MEM_Int ) return pMem->u.i!=0; + testcase( pMem->flags & MEM_IntReal ); + if( pMem->flags & (MEM_Int|MEM_IntReal) ) return pMem->u.i!=0; if( pMem->flags & MEM_Null ) return ifNull; return sqlite3VdbeRealValue(pMem)!=0.0; } @@ -73570,7 +76280,7 @@ SQLITE_PRIVATE int sqlite3VdbeBooleanValue(Mem *pMem, int ifNull){ SQLITE_PRIVATE void sqlite3VdbeIntegerAffinity(Mem *pMem){ i64 ix; assert( pMem->flags & MEM_Real ); - assert( (pMem->flags & MEM_RowSet)==0 ); + assert( !sqlite3VdbeMemIsRowSet(pMem) ); assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); assert( EIGHT_BYTE_ALIGNMENT(pMem) ); @@ -73597,7 +76307,7 @@ SQLITE_PRIVATE void sqlite3VdbeIntegerAffinity(Mem *pMem){ */ SQLITE_PRIVATE int sqlite3VdbeMemIntegerify(Mem *pMem){ assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); - assert( (pMem->flags & MEM_RowSet)==0 ); + assert( !sqlite3VdbeMemIsRowSet(pMem) ); assert( EIGHT_BYTE_ALIGNMENT(pMem) ); pMem->u.i = sqlite3VdbeIntValue(pMem); @@ -73621,17 +76331,21 @@ SQLITE_PRIVATE int sqlite3VdbeMemRealify(Mem *pMem){ /* Compare a floating point value to an integer. Return true if the two ** values are the same within the precision of the floating point value. ** +** This function assumes that i was obtained by assignment from r1. +** ** For some versions of GCC on 32-bit machines, if you do the more obvious ** comparison of "r1==(double)i" you sometimes get an answer of false even ** though the r1 and (double)i values are bit-for-bit the same. */ -static int sqlite3RealSameAsInt(double r1, sqlite3_int64 i){ +SQLITE_PRIVATE int sqlite3RealSameAsInt(double r1, sqlite3_int64 i){ double r2 = (double)i; - return memcmp(&r1, &r2, sizeof(r1))==0; + return r1==0.0 + || (memcmp(&r1, &r2, sizeof(r1))==0 + && i >= -2251799813685248LL && i < 2251799813685248LL); } /* -** Convert pMem so that it has types MEM_Real or MEM_Int or both. +** Convert pMem so that it has type MEM_Real or MEM_Int. ** Invalidate any prior representations. ** ** Every effort is made to force the conversion, even if the input @@ -73639,25 +76353,26 @@ static int sqlite3RealSameAsInt(double r1, sqlite3_int64 i){ ** as much of the string as we can and ignore the rest. */ SQLITE_PRIVATE int sqlite3VdbeMemNumerify(Mem *pMem){ - if( (pMem->flags & (MEM_Int|MEM_Real|MEM_Null))==0 ){ + testcase( pMem->flags & MEM_Int ); + testcase( pMem->flags & MEM_Real ); + testcase( pMem->flags & MEM_IntReal ); + testcase( pMem->flags & MEM_Null ); + if( (pMem->flags & (MEM_Int|MEM_Real|MEM_IntReal|MEM_Null))==0 ){ int rc; + sqlite3_int64 ix; assert( (pMem->flags & (MEM_Blob|MEM_Str))!=0 ); assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); - rc = sqlite3Atoi64(pMem->z, &pMem->u.i, pMem->n, pMem->enc); - if( rc==0 ){ + rc = sqlite3AtoF(pMem->z, &pMem->u.r, pMem->n, pMem->enc); + if( ((rc==0 || rc==1) && sqlite3Atoi64(pMem->z, &ix, pMem->n, pMem->enc)<=1) + || sqlite3RealSameAsInt(pMem->u.r, (ix = (i64)pMem->u.r)) + ){ + pMem->u.i = ix; MemSetTypeFlag(pMem, MEM_Int); }else{ - i64 i = pMem->u.i; - sqlite3AtoF(pMem->z, &pMem->u.r, pMem->n, pMem->enc); - if( rc==1 && sqlite3RealSameAsInt(pMem->u.r, i) ){ - pMem->u.i = i; - MemSetTypeFlag(pMem, MEM_Int); - }else{ - MemSetTypeFlag(pMem, MEM_Real); - } + MemSetTypeFlag(pMem, MEM_Real); } } - assert( (pMem->flags & (MEM_Int|MEM_Real|MEM_Null))!=0 ); + assert( (pMem->flags & (MEM_Int|MEM_Real|MEM_IntReal|MEM_Null))!=0 ); pMem->flags &= ~(MEM_Str|MEM_Blob|MEM_Zero); return SQLITE_OK; } @@ -73669,8 +76384,8 @@ SQLITE_PRIVATE int sqlite3VdbeMemNumerify(Mem *pMem){ ** affinity even if that results in loss of data. This routine is ** used (for example) to implement the SQL "cast()" operator. */ -SQLITE_PRIVATE void sqlite3VdbeMemCast(Mem *pMem, u8 aff, u8 encoding){ - if( pMem->flags & MEM_Null ) return; +SQLITE_PRIVATE int sqlite3VdbeMemCast(Mem *pMem, u8 aff, u8 encoding){ + if( pMem->flags & MEM_Null ) return SQLITE_OK; switch( aff ){ case SQLITE_AFF_BLOB: { /* Really a cast to BLOB */ if( (pMem->flags & MEM_Blob)==0 ){ @@ -73700,10 +76415,11 @@ SQLITE_PRIVATE void sqlite3VdbeMemCast(Mem *pMem, u8 aff, u8 encoding){ pMem->flags |= (pMem->flags&MEM_Blob)>>3; sqlite3ValueApplyAffinity(pMem, SQLITE_AFF_TEXT, encoding); assert( pMem->flags & MEM_Str || pMem->db->mallocFailed ); - pMem->flags &= ~(MEM_Int|MEM_Real|MEM_Blob|MEM_Zero); - break; + pMem->flags &= ~(MEM_Int|MEM_Real|MEM_IntReal|MEM_Blob|MEM_Zero); + return sqlite3VdbeChangeEncoding(pMem, encoding); } } + return SQLITE_OK; } /* @@ -73815,26 +76531,36 @@ SQLITE_PRIVATE void sqlite3VdbeMemSetDouble(Mem *pMem, double val){ } #endif +#ifdef SQLITE_DEBUG +/* +** Return true if the Mem holds a RowSet object. This routine is intended +** for use inside of assert() statements. +*/ +SQLITE_PRIVATE int sqlite3VdbeMemIsRowSet(const Mem *pMem){ + return (pMem->flags&(MEM_Blob|MEM_Dyn))==(MEM_Blob|MEM_Dyn) + && pMem->xDel==sqlite3RowSetDelete; +} +#endif + /* ** Delete any previous value and set the value of pMem to be an ** empty boolean index. +** +** Return SQLITE_OK on success and SQLITE_NOMEM if a memory allocation +** error occurs. */ -SQLITE_PRIVATE void sqlite3VdbeMemSetRowSet(Mem *pMem){ +SQLITE_PRIVATE int sqlite3VdbeMemSetRowSet(Mem *pMem){ sqlite3 *db = pMem->db; + RowSet *p; assert( db!=0 ); - assert( (pMem->flags & MEM_RowSet)==0 ); + assert( !sqlite3VdbeMemIsRowSet(pMem) ); sqlite3VdbeMemRelease(pMem); - pMem->zMalloc = sqlite3DbMallocRawNN(db, 64); - if( db->mallocFailed ){ - pMem->flags = MEM_Null; - pMem->szMalloc = 0; - }else{ - assert( pMem->zMalloc ); - pMem->szMalloc = sqlite3DbMallocSize(db, pMem->zMalloc); - pMem->u.pRowSet = sqlite3RowSetInit(db, pMem->zMalloc, pMem->szMalloc); - assert( pMem->u.pRowSet!=0 ); - pMem->flags = MEM_RowSet; - } + p = sqlite3RowSetInit(db); + if( p==0 ) return SQLITE_NOMEM; + pMem->z = (char*)p; + pMem->flags = MEM_Blob|MEM_Dyn; + pMem->xDel = sqlite3RowSetDelete; + return SQLITE_OK; } /* @@ -73859,15 +76585,31 @@ SQLITE_PRIVATE int sqlite3VdbeMemTooBig(Mem *p){ ** its link to a shallow copy and by marking any current shallow ** copies of this cell as invalid. ** -** This is used for testing and debugging only - to make sure shallow -** copies are not misused. +** This is used for testing and debugging only - to help ensure that shallow +** copies (created by OP_SCopy) are not misused. */ SQLITE_PRIVATE void sqlite3VdbeMemAboutToChange(Vdbe *pVdbe, Mem *pMem){ int i; Mem *pX; - for(i=0, pX=pVdbe->aMem; inMem; i++, pX++){ + for(i=1, pX=pVdbe->aMem+1; inMem; i++, pX++){ if( pX->pScopyFrom==pMem ){ - pX->flags |= MEM_Undefined; + u16 mFlags; + if( pVdbe->db->flags & SQLITE_VdbeTrace ){ + sqlite3DebugPrintf("Invalidate R[%d] due to change in R[%d]\n", + (int)(pX - pVdbe->aMem), (int)(pMem - pVdbe->aMem)); + } + /* If pX is marked as a shallow copy of pMem, then try to verify that + ** no significant changes have been made to pX since the OP_SCopy. + ** A significant change would indicated a missed call to this + ** function for pX. Minor changes, such as adding or removing a + ** dual type, are allowed, as long as the underlying value is the + ** same. */ + mFlags = pMem->flags & pX->flags & pX->mScopyFlags; + assert( (mFlags&(MEM_Int|MEM_IntReal))==0 || pMem->u.i==pX->u.i ); + + /* pMem is the register that is changing. But also mark pX as + ** undefined so that we can quickly detect the shallow-copy error */ + pX->flags = MEM_Undefined; pX->pScopyFrom = 0; } } @@ -73875,7 +76617,6 @@ SQLITE_PRIVATE void sqlite3VdbeMemAboutToChange(Vdbe *pVdbe, Mem *pMem){ } #endif /* SQLITE_DEBUG */ - /* ** Make an shallow copy of pFrom into pTo. Prior contents of ** pTo are freed. The pFrom->z field is not duplicated. If @@ -73888,7 +76629,7 @@ static SQLITE_NOINLINE void vdbeClrCopy(Mem *pTo, const Mem *pFrom, int eType){ sqlite3VdbeMemShallowCopy(pTo, pFrom, eType); } SQLITE_PRIVATE void sqlite3VdbeMemShallowCopy(Mem *pTo, const Mem *pFrom, int srcType){ - assert( (pFrom->flags & MEM_RowSet)==0 ); + assert( !sqlite3VdbeMemIsRowSet(pFrom) ); assert( pTo->db==pFrom->db ); if( VdbeMemDynamic(pTo) ){ vdbeClrCopy(pTo,pFrom,srcType); return; } memcpy(pTo, pFrom, MEMCELLSIZE); @@ -73906,7 +76647,7 @@ SQLITE_PRIVATE void sqlite3VdbeMemShallowCopy(Mem *pTo, const Mem *pFrom, int sr SQLITE_PRIVATE int sqlite3VdbeMemCopy(Mem *pTo, const Mem *pFrom){ int rc = SQLITE_OK; - assert( (pFrom->flags & MEM_RowSet)==0 ); + assert( !sqlite3VdbeMemIsRowSet(pFrom) ); if( VdbeMemDynamic(pTo) ) vdbeMemClearExternAndSetNull(pTo); memcpy(pTo, pFrom, MEMCELLSIZE); pTo->flags &= ~MEM_Dyn; @@ -73964,7 +76705,7 @@ SQLITE_PRIVATE int sqlite3VdbeMemSetStr( u16 flags = 0; /* New value for pMem->flags */ assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); - assert( (pMem->flags & MEM_RowSet)==0 ); + assert( !sqlite3VdbeMemIsRowSet(pMem) ); /* If z is a NULL pointer, set pMem to contain an SQL NULL. */ if( !z ){ @@ -73982,7 +76723,6 @@ SQLITE_PRIVATE int sqlite3VdbeMemSetStr( assert( enc!=0 ); if( enc==SQLITE_UTF8 ){ nByte = 0x7fffffff & (int)strlen(z); - if( nByte>iLimit ) nByte = iLimit+1; }else{ for(nByte=0; nByte<=iLimit && (z[nByte] | z[nByte+1]); nByte+=2){} } @@ -73994,37 +76734,47 @@ SQLITE_PRIVATE int sqlite3VdbeMemSetStr( ** management (one of MEM_Dyn or MEM_Static). */ if( xDel==SQLITE_TRANSIENT ){ - int nAlloc = nByte; + u32 nAlloc = nByte; if( flags&MEM_Term ){ nAlloc += (enc==SQLITE_UTF8?1:2); } if( nByte>iLimit ){ - return SQLITE_TOOBIG; + return sqlite3ErrorToParser(pMem->db, SQLITE_TOOBIG); } testcase( nAlloc==0 ); testcase( nAlloc==31 ); testcase( nAlloc==32 ); - if( sqlite3VdbeMemClearAndResize(pMem, MAX(nAlloc,32)) ){ + if( sqlite3VdbeMemClearAndResize(pMem, (int)MAX(nAlloc,32)) ){ return SQLITE_NOMEM_BKPT; } memcpy(pMem->z, z, nAlloc); - }else if( xDel==SQLITE_DYNAMIC ){ - sqlite3VdbeMemRelease(pMem); - pMem->zMalloc = pMem->z = (char *)z; - pMem->szMalloc = sqlite3DbMallocSize(pMem->db, pMem->zMalloc); }else{ sqlite3VdbeMemRelease(pMem); pMem->z = (char *)z; - pMem->xDel = xDel; - flags |= ((xDel==SQLITE_STATIC)?MEM_Static:MEM_Dyn); + if( xDel==SQLITE_DYNAMIC ){ + pMem->zMalloc = pMem->z; + pMem->szMalloc = sqlite3DbMallocSize(pMem->db, pMem->zMalloc); + }else{ + pMem->xDel = xDel; + flags |= ((xDel==SQLITE_STATIC)?MEM_Static:MEM_Dyn); + } } pMem->n = nByte; pMem->flags = flags; - pMem->enc = (enc==0 ? SQLITE_UTF8 : enc); + if( enc ){ + pMem->enc = enc; +#ifdef SQLITE_ENABLE_SESSION + }else if( pMem->db==0 ){ + pMem->enc = SQLITE_UTF8; +#endif + }else{ + assert( pMem->db!=0 ); + pMem->enc = ENC(pMem->db); + } #ifndef SQLITE_OMIT_UTF16 - if( pMem->enc!=SQLITE_UTF8 && sqlite3VdbeMemHandleBom(pMem) ){ + if( enc>SQLITE_UTF8 && sqlite3VdbeMemHandleBom(pMem) ){ return SQLITE_NOMEM_BKPT; } #endif @@ -74051,7 +76801,7 @@ SQLITE_PRIVATE int sqlite3VdbeMemSetStr( ** If this routine fails for any reason (malloc returns NULL or unable ** to read from the disk) then the pMem is left in an inconsistent state. */ -static SQLITE_NOINLINE int vdbeMemFromBtreeResize( +SQLITE_PRIVATE int sqlite3VdbeMemFromBtree( BtCursor *pCur, /* Cursor pointing at record to retrieve. */ u32 offset, /* Offset from the start of data to return bytes from. */ u32 amt, /* Number of bytes to return. */ @@ -74059,6 +76809,9 @@ static SQLITE_NOINLINE int vdbeMemFromBtreeResize( ){ int rc; pMem->flags = MEM_Null; + if( sqlite3BtreeMaxRecordSize(pCur)z); if( rc==SQLITE_OK ){ @@ -74071,13 +76824,11 @@ static SQLITE_NOINLINE int vdbeMemFromBtreeResize( } return rc; } -SQLITE_PRIVATE int sqlite3VdbeMemFromBtree( +SQLITE_PRIVATE int sqlite3VdbeMemFromBtreeZeroOffset( BtCursor *pCur, /* Cursor pointing at record to retrieve. */ - u32 offset, /* Offset from the start of data to return bytes from. */ u32 amt, /* Number of bytes to return. */ Mem *pMem /* OUT: Return data in this Mem structure. */ ){ - char *zData; /* Data from the btree layer */ u32 available = 0; /* Number of bytes available on the local btree page */ int rc = SQLITE_OK; /* Return code */ @@ -74086,16 +76837,15 @@ SQLITE_PRIVATE int sqlite3VdbeMemFromBtree( /* Note: the calls to BtreeKeyFetch() and DataFetch() below assert() ** that both the BtShared and database handle mutexes are held. */ - assert( (pMem->flags & MEM_RowSet)==0 ); - zData = (char *)sqlite3BtreePayloadFetch(pCur, &available); - assert( zData!=0 ); + assert( !sqlite3VdbeMemIsRowSet(pMem) ); + pMem->z = (char *)sqlite3BtreePayloadFetch(pCur, &available); + assert( pMem->z!=0 ); - if( offset+amt<=available ){ - pMem->z = &zData[offset]; + if( amt<=available ){ pMem->flags = MEM_Blob|MEM_Ephem; pMem->n = (int)amt; }else{ - rc = vdbeMemFromBtreeResize(pCur, offset, amt, pMem); + rc = sqlite3VdbeMemFromBtree(pCur, 0, amt, pMem); } return rc; @@ -74110,7 +76860,7 @@ static SQLITE_NOINLINE const void *valueToText(sqlite3_value* pVal, u8 enc){ assert( pVal!=0 ); assert( pVal->db==0 || sqlite3_mutex_held(pVal->db->mutex) ); assert( (enc&3)==(enc&~SQLITE_UTF16_ALIGNED) ); - assert( (pVal->flags & MEM_RowSet)==0 ); + assert( !sqlite3VdbeMemIsRowSet(pVal) ); assert( (pVal->flags & (MEM_Null))==0 ); if( pVal->flags & (MEM_Blob|MEM_Str) ){ if( ExpandBlob(pVal) ) return 0; @@ -74132,7 +76882,7 @@ static SQLITE_NOINLINE const void *valueToText(sqlite3_value* pVal, u8 enc){ assert(pVal->enc==(enc & ~SQLITE_UTF16_ALIGNED) || pVal->db==0 || pVal->db->mallocFailed ); if( pVal->enc==(enc & ~SQLITE_UTF16_ALIGNED) ){ - assert( sqlite3VdbeMemConsistentDualRep(pVal) ); + assert( sqlite3VdbeMemValidStrRep(pVal) ); return pVal->z; }else{ return 0; @@ -74153,9 +76903,9 @@ SQLITE_PRIVATE const void *sqlite3ValueText(sqlite3_value* pVal, u8 enc){ if( !pVal ) return 0; assert( pVal->db==0 || sqlite3_mutex_held(pVal->db->mutex) ); assert( (enc&3)==(enc&~SQLITE_UTF16_ALIGNED) ); - assert( (pVal->flags & MEM_RowSet)==0 ); + assert( !sqlite3VdbeMemIsRowSet(pVal) ); if( (pVal->flags&(MEM_Str|MEM_Term))==(MEM_Str|MEM_Term) && pVal->enc==enc ){ - assert( sqlite3VdbeMemConsistentDualRep(pVal) ); + assert( sqlite3VdbeMemValidStrRep(pVal) ); return pVal->z; } if( pVal->flags&MEM_Null ){ @@ -74199,7 +76949,7 @@ struct ValueNewStat4Ctx { ** an sqlite3_value within the UnpackedRecord.a[] array. */ static sqlite3_value *valueNew(sqlite3 *db, struct ValueNewStat4Ctx *p){ -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +#ifdef SQLITE_ENABLE_STAT4 if( p ){ UnpackedRecord *pRec = p->ppRec[0]; @@ -74235,7 +76985,7 @@ static sqlite3_value *valueNew(sqlite3 *db, struct ValueNewStat4Ctx *p){ } #else UNUSED_PARAMETER(p); -#endif /* defined(SQLITE_ENABLE_STAT3_OR_STAT4) */ +#endif /* defined(SQLITE_ENABLE_STAT4) */ return sqlite3ValueNew(db); } @@ -74259,7 +77009,7 @@ static sqlite3_value *valueNew(sqlite3 *db, struct ValueNewStat4Ctx *p){ ** and sets (*ppVal) to NULL. Or, if an error occurs, (*ppVal) is set to ** NULL and an SQLite error code returned. */ -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +#ifdef SQLITE_ENABLE_STAT4 static int valueFromFunction( sqlite3 *db, /* The database connection */ Expr *p, /* The expression to evaluate */ @@ -74342,7 +77092,7 @@ static int valueFromFunction( } #else # define valueFromFunction(a,b,c,d,e,f) SQLITE_OK -#endif /* defined(SQLITE_ENABLE_STAT3_OR_STAT4) */ +#endif /* defined(SQLITE_ENABLE_STAT4) */ /* ** Extract a value from the supplied expression in the manner described @@ -74371,7 +77121,7 @@ static int valueFromExpr( assert( pExpr!=0 ); while( (op = pExpr->op)==TK_UPLUS || op==TK_SPAN ) pExpr = pExpr->pLeft; -#if defined(SQLITE_ENABLE_STAT3_OR_STAT4) +#if defined(SQLITE_ENABLE_STAT4) if( op==TK_REGISTER ) op = pExpr->op2; #else if( NEVER(op==TK_REGISTER) ) op = pExpr->op2; @@ -74420,7 +77170,12 @@ static int valueFromExpr( }else{ sqlite3ValueApplyAffinity(pVal, affinity, SQLITE_UTF8); } - if( pVal->flags & (MEM_Int|MEM_Real) ) pVal->flags &= ~MEM_Str; + assert( (pVal->flags & MEM_IntReal)==0 ); + if( pVal->flags & (MEM_Int|MEM_IntReal|MEM_Real) ){ + testcase( pVal->flags & MEM_Int ); + testcase( pVal->flags & MEM_Real ); + pVal->flags &= ~MEM_Str; + } if( enc!=SQLITE_UTF8 ){ rc = sqlite3VdbeChangeEncoding(pVal, enc); } @@ -74433,7 +77188,11 @@ static int valueFromExpr( if( pVal->flags & MEM_Real ){ pVal->u.r = -pVal->u.r; }else if( pVal->u.i==SMALLEST_INT64 ){ +#ifndef SQLITE_OMIT_FLOATING_POINT pVal->u.r = -(double)SMALLEST_INT64; +#else + pVal->u.r = LARGEST_INT64; +#endif MemSetTypeFlag(pVal, MEM_Real); }else{ pVal->u.i = -pVal->u.i; @@ -74443,7 +77202,7 @@ static int valueFromExpr( }else if( op==TK_NULL ){ pVal = valueNew(db, pCtx); if( pVal==0 ) goto no_mem; - sqlite3VdbeMemNumerify(pVal); + sqlite3VdbeMemSetNull(pVal); } #ifndef SQLITE_OMIT_BLOB_LITERAL else if( op==TK_BLOB ){ @@ -74459,28 +77218,30 @@ static int valueFromExpr( 0, SQLITE_DYNAMIC); } #endif -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +#ifdef SQLITE_ENABLE_STAT4 else if( op==TK_FUNCTION && pCtx!=0 ){ rc = valueFromFunction(db, pExpr, enc, affinity, &pVal, pCtx); } #endif else if( op==TK_TRUEFALSE ){ - pVal = valueNew(db, pCtx); - pVal->flags = MEM_Int; - pVal->u.i = pExpr->u.zToken[4]==0; + pVal = valueNew(db, pCtx); + if( pVal ){ + pVal->flags = MEM_Int; + pVal->u.i = pExpr->u.zToken[4]==0; + } } *ppVal = pVal; return rc; no_mem: -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +#ifdef SQLITE_ENABLE_STAT4 if( pCtx==0 || pCtx->pParse->nErr==0 ) #endif sqlite3OomFault(db); sqlite3DbFree(db, zVal); assert( *ppVal==0 ); -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +#ifdef SQLITE_ENABLE_STAT4 if( pCtx==0 ) sqlite3ValueFree(pVal); #else assert( pCtx==0 ); sqlite3ValueFree(pVal); @@ -74508,56 +77269,7 @@ SQLITE_PRIVATE int sqlite3ValueFromExpr( return pExpr ? valueFromExpr(db, pExpr, enc, affinity, ppVal, 0) : 0; } -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 -/* -** The implementation of the sqlite_record() function. This function accepts -** a single argument of any type. The return value is a formatted database -** record (a blob) containing the argument value. -** -** This is used to convert the value stored in the 'sample' column of the -** sqlite_stat3 table to the record format SQLite uses internally. -*/ -static void recordFunc( - sqlite3_context *context, - int argc, - sqlite3_value **argv -){ - const int file_format = 1; - u32 iSerial; /* Serial type */ - int nSerial; /* Bytes of space for iSerial as varint */ - u32 nVal; /* Bytes of space required for argv[0] */ - int nRet; - sqlite3 *db; - u8 *aRet; - - UNUSED_PARAMETER( argc ); - iSerial = sqlite3VdbeSerialType(argv[0], file_format, &nVal); - nSerial = sqlite3VarintLen(iSerial); - db = sqlite3_context_db_handle(context); - - nRet = 1 + nSerial + nVal; - aRet = sqlite3DbMallocRawNN(db, nRet); - if( aRet==0 ){ - sqlite3_result_error_nomem(context); - }else{ - aRet[0] = nSerial+1; - putVarint32(&aRet[1], iSerial); - sqlite3VdbeSerialPut(&aRet[1+nSerial], argv[0], iSerial); - sqlite3_result_blob(context, aRet, nRet, SQLITE_TRANSIENT); - sqlite3DbFreeNN(db, aRet); - } -} - -/* -** Register built-in functions used to help read ANALYZE data. -*/ -SQLITE_PRIVATE void sqlite3AnalyzeFunctions(void){ - static FuncDef aAnalyzeTableFuncs[] = { - FUNCTION(sqlite_record, 1, 0, 0, recordFunc), - }; - sqlite3InsertBuiltinFuncs(aAnalyzeTableFuncs, ArraySize(aAnalyzeTableFuncs)); -} - +#ifdef SQLITE_ENABLE_STAT4 /* ** Attempt to extract a value from pExpr and use it to construct *ppVal. ** @@ -74720,11 +77432,11 @@ SQLITE_PRIVATE int sqlite3Stat4Column( int iCol, /* Column to extract */ sqlite3_value **ppVal /* OUT: Extracted value */ ){ - u32 t; /* a column type code */ + u32 t = 0; /* a column type code */ int nHdr; /* Size of the header in the record */ int iHdr; /* Next unread header byte */ int iField; /* Next unread data byte */ - int szField; /* Size of the current data field */ + int szField = 0; /* Size of the current data field */ int i; /* Column index */ u8 *a = (u8*)pRec; /* Typecast byte array */ Mem *pMem = *ppVal; /* Write result into this Mem object */ @@ -74839,6 +77551,10 @@ SQLITE_PRIVATE int sqlite3ValueBytes(sqlite3_value *pVal, u8 enc){ /* #include "sqliteInt.h" */ /* #include "vdbeInt.h" */ +/* Forward references */ +static void freeEphemeralFunction(sqlite3 *db, FuncDef *pDef); +static void vdbeFreeOpArray(sqlite3 *, Op *, int); + /* ** Create a new virtual database engine. */ @@ -74860,12 +77576,19 @@ SQLITE_PRIVATE Vdbe *sqlite3VdbeCreate(Parse *pParse){ pParse->pVdbe = p; assert( pParse->aLabel==0 ); assert( pParse->nLabel==0 ); - assert( pParse->nOpAlloc==0 ); + assert( p->nOpAlloc==0 ); assert( pParse->szOpAlloc==0 ); sqlite3VdbeAddOp2(p, OP_Init, 0, 1); return p; } +/* +** Return the Parse object that owns a Vdbe object. +*/ +SQLITE_PRIVATE Parse *sqlite3VdbeParser(Vdbe *p){ + return p->pParse; +} + /* ** Change the error string stored in Vdbe.zErrMsg */ @@ -74890,6 +77613,43 @@ SQLITE_PRIVATE void sqlite3VdbeSetSql(Vdbe *p, const char *z, int n, u8 prepFlag p->zSql = sqlite3DbStrNDup(p->db, z, n); } +#ifdef SQLITE_ENABLE_NORMALIZE +/* +** Add a new element to the Vdbe->pDblStr list. +*/ +SQLITE_PRIVATE void sqlite3VdbeAddDblquoteStr(sqlite3 *db, Vdbe *p, const char *z){ + if( p ){ + int n = sqlite3Strlen30(z); + DblquoteStr *pStr = sqlite3DbMallocRawNN(db, + sizeof(*pStr)+n+1-sizeof(pStr->z)); + if( pStr ){ + pStr->pNextStr = p->pDblStr; + p->pDblStr = pStr; + memcpy(pStr->z, z, n+1); + } + } +} +#endif + +#ifdef SQLITE_ENABLE_NORMALIZE +/* +** zId of length nId is a double-quoted identifier. Check to see if +** that identifier is really used as a string literal. +*/ +SQLITE_PRIVATE int sqlite3VdbeUsesDoubleQuotedString( + Vdbe *pVdbe, /* The prepared statement */ + const char *zId /* The double-quoted identifier, already dequoted */ +){ + DblquoteStr *pStr; + assert( zId!=0 ); + if( pVdbe->pDblStr==0 ) return 0; + for(pStr=pVdbe->pDblStr; pStr; pStr=pStr->pNextStr){ + if( strcmp(zId, pStr->z)==0 ) return 1; + } + return 0; +} +#endif + /* ** Swap all content between two VDBE structures. */ @@ -74909,6 +77669,11 @@ SQLITE_PRIVATE void sqlite3VdbeSwap(Vdbe *pA, Vdbe *pB){ zTmp = pA->zSql; pA->zSql = pB->zSql; pB->zSql = zTmp; +#ifdef SQLITE_ENABLE_NORMALIZE + zTmp = pA->zNormSql; + pA->zNormSql = pB->zNormSql; + pB->zNormSql = zTmp; +#endif pB->expmask = pA->expmask; pB->prepFlags = pA->prepFlags; memcpy(pB->aCounter, pA->aCounter, sizeof(pB->aCounter)); @@ -74921,7 +77686,7 @@ SQLITE_PRIVATE void sqlite3VdbeSwap(Vdbe *pA, Vdbe *pB){ ** to 1024/sizeof(Op). ** ** If an out-of-memory error occurs while resizing the array, return -** SQLITE_NOMEM. In this case Vdbe.aOp and Parse.nOpAlloc remain +** SQLITE_NOMEM. In this case Vdbe.aOp and Vdbe.nOpAlloc remain ** unchanged (this is so that any opcodes already allocated can be ** correctly deallocated along with the rest of the Vdbe). */ @@ -74937,9 +77702,11 @@ static int growOpArray(Vdbe *v, int nOp){ ** operation (without SQLITE_TEST_REALLOC_STRESS) is to double the current ** size of the op array or add 1KB of space, whichever is smaller. */ #ifdef SQLITE_TEST_REALLOC_STRESS - int nNew = (p->nOpAlloc>=512 ? p->nOpAlloc*2 : p->nOpAlloc+nOp); + sqlite3_int64 nNew = (v->nOpAlloc>=512 ? 2*(sqlite3_int64)v->nOpAlloc + : (sqlite3_int64)v->nOpAlloc+nOp); #else - int nNew = (p->nOpAlloc ? p->nOpAlloc*2 : (int)(1024/sizeof(Op))); + sqlite3_int64 nNew = (v->nOpAlloc ? 2*(sqlite3_int64)v->nOpAlloc + : (sqlite3_int64)(1024/sizeof(Op))); UNUSED_PARAMETER(nOp); #endif @@ -74950,11 +77717,11 @@ static int growOpArray(Vdbe *v, int nOp){ } assert( nOp<=(1024/sizeof(Op)) ); - assert( nNew>=(p->nOpAlloc+nOp) ); + assert( nNew>=(v->nOpAlloc+nOp) ); pNew = sqlite3DbRealloc(p->db, v->aOp, nNew*sizeof(Op)); if( pNew ){ p->szOpAlloc = sqlite3DbMallocSize(p->db, pNew); - p->nOpAlloc = p->szOpAlloc/sizeof(Op); + v->nOpAlloc = p->szOpAlloc/sizeof(Op); v->aOp = pNew; } return (pNew ? SQLITE_OK : SQLITE_NOMEM_BKPT); @@ -74963,9 +77730,16 @@ static int growOpArray(Vdbe *v, int nOp){ #ifdef SQLITE_DEBUG /* This routine is just a convenient place to set a breakpoint that will ** fire after each opcode is inserted and displayed using -** "PRAGMA vdbe_addoptrace=on". +** "PRAGMA vdbe_addoptrace=on". Parameters "pc" (program counter) and +** pOp are available to make the breakpoint conditional. +** +** Other useful labels for breakpoints include: +** test_trace_breakpoint(pc,pOp) +** sqlite3CorruptError(lineno) +** sqlite3MisuseError(lineno) +** sqlite3CantopenError(lineno) */ -static void test_addop_breakpoint(void){ +static void test_addop_breakpoint(int pc, Op *pOp){ static int n = 0; n++; } @@ -74988,9 +77762,9 @@ static void test_addop_breakpoint(void){ ** operand. */ static SQLITE_NOINLINE int growOp3(Vdbe *p, int op, int p1, int p2, int p3){ - assert( p->pParse->nOpAlloc<=p->nOp ); + assert( p->nOpAlloc<=p->nOp ); if( growOpArray(p, 1) ) return 1; - assert( p->pParse->nOpAlloc>p->nOp ); + assert( p->nOpAlloc>p->nOp ); return sqlite3VdbeAddOp3(p, op, p1, p2, p3); } SQLITE_PRIVATE int sqlite3VdbeAddOp3(Vdbe *p, int op, int p1, int p2, int p3){ @@ -75000,7 +77774,7 @@ SQLITE_PRIVATE int sqlite3VdbeAddOp3(Vdbe *p, int op, int p1, int p2, int p3){ i = p->nOp; assert( p->magic==VDBE_MAGIC_INIT ); assert( op>=0 && op<0xff ); - if( p->pParse->nOpAlloc<=i ){ + if( p->nOpAlloc<=i ){ return growOp3(p, op, p1, p2, p3); } p->nOp++; @@ -75017,16 +77791,8 @@ SQLITE_PRIVATE int sqlite3VdbeAddOp3(Vdbe *p, int op, int p1, int p2, int p3){ #endif #ifdef SQLITE_DEBUG if( p->db->flags & SQLITE_VdbeAddopTrace ){ - int jj, kk; - Parse *pParse = p->pParse; - for(jj=kk=0; jjnColCache; jj++){ - struct yColCache *x = pParse->aColCache + jj; - printf(" r[%d]={%d:%d}", x->iReg, x->iTable, x->iColumn); - kk++; - } - if( kk ) printf("\n"); sqlite3VdbePrintOp(0, i, &p->aOp[i]); - test_addop_breakpoint(); + test_addop_breakpoint(i, &p->aOp[i]); } #endif #ifdef VDBE_PROFILE @@ -75109,6 +77875,49 @@ SQLITE_PRIVATE int sqlite3VdbeAddOp4( return addr; } +/* +** Add an OP_Function or OP_PureFunc opcode. +** +** The eCallCtx argument is information (typically taken from Expr.op2) +** that describes the calling context of the function. 0 means a general +** function call. NC_IsCheck means called by a check constraint, +** NC_IdxExpr means called as part of an index expression. NC_PartIdx +** means in the WHERE clause of a partial index. NC_GenCol means called +** while computing a generated column value. 0 is the usual case. +*/ +SQLITE_PRIVATE int sqlite3VdbeAddFunctionCall( + Parse *pParse, /* Parsing context */ + int p1, /* Constant argument mask */ + int p2, /* First argument register */ + int p3, /* Register into which results are written */ + int nArg, /* Number of argument */ + const FuncDef *pFunc, /* The function to be invoked */ + int eCallCtx /* Calling context */ +){ + Vdbe *v = pParse->pVdbe; + int nByte; + int addr; + sqlite3_context *pCtx; + assert( v ); + nByte = sizeof(*pCtx) + (nArg-1)*sizeof(sqlite3_value*); + pCtx = sqlite3DbMallocRawNN(pParse->db, nByte); + if( pCtx==0 ){ + assert( pParse->db->mallocFailed ); + freeEphemeralFunction(pParse->db, (FuncDef*)pFunc); + return 0; + } + pCtx->pOut = 0; + pCtx->pFunc = (FuncDef*)pFunc; + pCtx->pVdbe = 0; + pCtx->isError = 0; + pCtx->argc = nArg; + pCtx->iOp = sqlite3VdbeCurrentAddr(v); + addr = sqlite3VdbeAddOp4(v, eCallCtx ? OP_PureFunc : OP_Function, + p1, p2, p3, (char*)pCtx, P4_FUNCCTX); + sqlite3VdbeChangeP5(v, eCallCtx & NC_SelfRef); + return addr; +} + /* ** Add an opcode that includes the p4 value with a P4_INT64 or ** P4_REAL type. @@ -75139,6 +77948,17 @@ SQLITE_PRIVATE int sqlite3VdbeExplainParent(Parse *pParse){ return pOp->p2; } +/* +** Set a debugger breakpoint on the following routine in order to +** monitor the EXPLAIN QUERY PLAN code generation. +*/ +#if defined(SQLITE_DEBUG) +SQLITE_PRIVATE void sqlite3ExplainBreakpoint(const char *z1, const char *z2){ + (void)z1; + (void)z2; +} +#endif + /* ** Add a new OP_Explain opcode. ** @@ -75146,9 +77966,14 @@ SQLITE_PRIVATE int sqlite3VdbeExplainParent(Parse *pParse){ ** subsequent Explains until sqlite3VdbeExplainPop() is called. */ SQLITE_PRIVATE void sqlite3VdbeExplain(Parse *pParse, u8 bPush, const char *zFmt, ...){ - if( pParse->explain==2 ){ +#ifndef SQLITE_DEBUG + /* Always include the OP_Explain opcodes if SQLITE_DEBUG is defined. + ** But omit them (for performance) during production builds */ + if( pParse->explain==2 ) +#endif + { char *zMsg; - Vdbe *v = pParse->pVdbe; + Vdbe *v; va_list ap; int iThis; va_start(ap, zFmt); @@ -75158,7 +77983,10 @@ SQLITE_PRIVATE void sqlite3VdbeExplain(Parse *pParse, u8 bPush, const char *zFmt iThis = v->nOp; sqlite3VdbeAddOp4(v, OP_Explain, iThis, pParse->addrExplain, 0, zMsg, P4_DYNAMIC); - if( bPush) pParse->addrExplain = iThis; + sqlite3ExplainBreakpoint(bPush?"PUSH":"", sqlite3VdbeGetOp(v,-1)->p4.z); + if( bPush){ + pParse->addrExplain = iThis; + } } } @@ -75166,6 +77994,7 @@ SQLITE_PRIVATE void sqlite3VdbeExplain(Parse *pParse, u8 bPush, const char *zFmt ** Pop the EXPLAIN QUERY PLAN stack one level. */ SQLITE_PRIVATE void sqlite3VdbeExplainPop(Parse *pParse){ + sqlite3ExplainBreakpoint("POP", 0); pParse->addrExplain = sqlite3VdbeExplainParent(pParse); } #endif /* SQLITE_OMIT_EXPLAIN */ @@ -75230,21 +78059,22 @@ SQLITE_PRIVATE void sqlite3VdbeEndCoroutine(Vdbe *v, int regYield){ ** The VDBE knows that a P2 value is a label because labels are ** always negative and P2 values are suppose to be non-negative. ** Hence, a negative P2 value is a label that has yet to be resolved. +** (Later:) This is only true for opcodes that have the OPFLG_JUMP +** property. ** -** Zero is returned if a malloc() fails. +** Variable usage notes: +** +** Parse.aLabel[x] Stores the address that the x-th label resolves +** into. For testing (SQLITE_DEBUG), unresolved +** labels stores -1, but that is not required. +** Parse.nLabelAlloc Number of slots allocated to Parse.aLabel[] +** Parse.nLabel The *negative* of the number of labels that have +** been issued. The negative is stored because +** that gives a performance improvement over storing +** the equivalent positive value. */ -SQLITE_PRIVATE int sqlite3VdbeMakeLabel(Vdbe *v){ - Parse *p = v->pParse; - int i = p->nLabel++; - assert( v->magic==VDBE_MAGIC_INIT ); - if( (i & (i-1))==0 ){ - p->aLabel = sqlite3DbReallocOrFree(p->db, p->aLabel, - (i*2+1)*sizeof(p->aLabel[0])); - } - if( p->aLabel ){ - p->aLabel[i] = -1; - } - return ADDR(i); +SQLITE_PRIVATE int sqlite3VdbeMakeLabel(Parse *pParse){ + return --pParse->nLabel; } /* @@ -75252,36 +78082,40 @@ SQLITE_PRIVATE int sqlite3VdbeMakeLabel(Vdbe *v){ ** be inserted. The parameter "x" must have been obtained from ** a prior call to sqlite3VdbeMakeLabel(). */ +static SQLITE_NOINLINE void resizeResolveLabel(Parse *p, Vdbe *v, int j){ + int nNewSize = 10 - p->nLabel; + p->aLabel = sqlite3DbReallocOrFree(p->db, p->aLabel, + nNewSize*sizeof(p->aLabel[0])); + if( p->aLabel==0 ){ + p->nLabelAlloc = 0; + }else{ +#ifdef SQLITE_DEBUG + int i; + for(i=p->nLabelAlloc; iaLabel[i] = -1; +#endif + p->nLabelAlloc = nNewSize; + p->aLabel[j] = v->nOp; + } +} SQLITE_PRIVATE void sqlite3VdbeResolveLabel(Vdbe *v, int x){ Parse *p = v->pParse; int j = ADDR(x); assert( v->magic==VDBE_MAGIC_INIT ); - assert( jnLabel ); + assert( j<-p->nLabel ); assert( j>=0 ); - if( p->aLabel ){ #ifdef SQLITE_DEBUG - if( p->db->flags & SQLITE_VdbeAddopTrace ){ - printf("RESOLVE LABEL %d to %d\n", x, v->nOp); - } + if( p->db->flags & SQLITE_VdbeAddopTrace ){ + printf("RESOLVE LABEL %d to %d\n", x, v->nOp); + } #endif + if( p->nLabelAlloc + p->nLabel < 0 ){ + resizeResolveLabel(p,v,j); + }else{ assert( p->aLabel[j]==(-1) ); /* Labels may only be resolved once */ p->aLabel[j] = v->nOp; } } -#ifdef SQLITE_COVERAGE_TEST -/* -** Return TRUE if and only if the label x has already been resolved. -** Return FALSE (zero) if label x is still unresolved. -** -** This routine is only used inside of testcase() macros, and so it -** only exists when measuring test coverage. -*/ -SQLITE_PRIVATE int sqlite3VdbeLabelHasBeenResolved(Vdbe *v, int x){ - return v->pParse->aLabel && v->pParse->aLabel[ADDR(x)]>=0; -} -#endif /* SQLITE_COVERAGE_TEST */ - /* ** Mark the VDBE as one that can only be run one time. */ @@ -75376,6 +78210,7 @@ static Op *opIterNext(VdbeOpIter *p){ ** * OP_HaltIfNull with P1=SQLITE_CONSTRAINT and P2=OE_Abort. ** * OP_Destroy ** * OP_VUpdate +** * OP_VCreate ** * OP_VRename ** * OP_FkCounter with P2==0 (immediate foreign key constraint) ** * OP_CreateBtree/BTREE_INTKEY and OP_InitCoroutine @@ -75392,6 +78227,7 @@ SQLITE_PRIVATE int sqlite3VdbeAssertMayAbort(Vdbe *v, int mayAbort){ int hasAbort = 0; int hasFkCounter = 0; int hasCreateTable = 0; + int hasCreateIndex = 0; int hasInitCoroutine = 0; Op *pOp; VdbeOpIter sIter; @@ -75401,13 +78237,24 @@ SQLITE_PRIVATE int sqlite3VdbeAssertMayAbort(Vdbe *v, int mayAbort){ while( (pOp = opIterNext(&sIter))!=0 ){ int opcode = pOp->opcode; if( opcode==OP_Destroy || opcode==OP_VUpdate || opcode==OP_VRename + || opcode==OP_VDestroy + || opcode==OP_VCreate + || (opcode==OP_ParseSchema && pOp->p4.z==0) || ((opcode==OP_Halt || opcode==OP_HaltIfNull) - && ((pOp->p1&0xff)==SQLITE_CONSTRAINT && pOp->p2==OE_Abort)) + && ((pOp->p1)!=SQLITE_OK && pOp->p2==OE_Abort)) ){ hasAbort = 1; break; } if( opcode==OP_CreateBtree && pOp->p3==BTREE_INTKEY ) hasCreateTable = 1; + if( mayAbort ){ + /* hasCreateIndex may also be set for some DELETE statements that use + ** OP_Clear. So this routine may end up returning true in the case + ** where a "DELETE FROM tbl" has a statement-journal but does not + ** require one. This is not so bad - it is an inefficiency, not a bug. */ + if( opcode==OP_CreateBtree && pOp->p3==BTREE_BLOBKEY ) hasCreateIndex = 1; + if( opcode==OP_Clear ) hasCreateIndex = 1; + } if( opcode==OP_InitCoroutine ) hasInitCoroutine = 1; #ifndef SQLITE_OMIT_FOREIGN_KEY if( opcode==OP_FkCounter && pOp->p1==0 && pOp->p2==1 ){ @@ -75423,7 +78270,8 @@ SQLITE_PRIVATE int sqlite3VdbeAssertMayAbort(Vdbe *v, int mayAbort){ ** true for this case to prevent the assert() in the callers frame ** from failing. */ return ( v->db->mallocFailed || hasAbort==mayAbort || hasFkCounter - || (hasCreateTable && hasInitCoroutine) ); + || (hasCreateTable && hasInitCoroutine) || hasCreateIndex + ); } #endif /* SQLITE_DEBUG - the sqlite3AssertMayAbort() function */ @@ -75513,7 +78361,6 @@ static void resolveP2Values(Vdbe *p, int *pMaxFuncArgs){ break; } case OP_Next: - case OP_NextIfOpen: case OP_SorterNext: { pOp->p4.xAdvance = sqlite3BtreeNext; pOp->p4type = P4_ADVANCE; @@ -75523,8 +78370,7 @@ static void resolveP2Values(Vdbe *p, int *pMaxFuncArgs){ assert( pOp->p2>=0 ); break; } - case OP_Prev: - case OP_PrevIfOpen: { + case OP_Prev: { pOp->p4.xAdvance = sqlite3BtreePrevious; pOp->p4type = P4_ADVANCE; /* The code generator never codes any of these opcodes as a jump @@ -75553,7 +78399,7 @@ static void resolveP2Values(Vdbe *p, int *pMaxFuncArgs){ ** non-jump opcodes less than SQLITE_MX_JUMP_CODE are guaranteed to ** have non-negative values for P2. */ assert( (sqlite3OpcodeProperty[pOp->opcode] & OPFLG_JUMP)!=0 ); - assert( ADDR(pOp->p2)nLabel ); + assert( ADDR(pOp->p2)<-pParse->nLabel ); pOp->p2 = aLabel[ADDR(pOp->p2)]; } break; @@ -75592,7 +78438,7 @@ SQLITE_PRIVATE int sqlite3VdbeCurrentAddr(Vdbe *p){ */ #if defined(SQLITE_DEBUG) && !defined(SQLITE_TEST_REALLOC_STRESS) SQLITE_PRIVATE void sqlite3VdbeVerifyNoMallocRequired(Vdbe *p, int N){ - assert( p->nOp + N <= p->pParse->nOpAlloc ); + assert( p->nOp + N <= p->nOpAlloc ); } #endif @@ -75664,7 +78510,7 @@ SQLITE_PRIVATE VdbeOp *sqlite3VdbeAddOpList( VdbeOp *pOut, *pFirst; assert( nOp>0 ); assert( p->magic==VDBE_MAGIC_INIT ); - if( p->nOp + nOp > p->pParse->nOpAlloc && growOpArray(p, nOp) ){ + if( p->nOp + nOp > p->nOpAlloc && growOpArray(p, nOp) ){ return 0; } pFirst = pOut = &p->aOp[p->nOp]; @@ -75710,7 +78556,7 @@ SQLITE_PRIVATE void sqlite3VdbeScanStatus( LogEst nEst, /* Estimated number of output rows */ const char *zName /* Name of table or index being scanned */ ){ - int nByte = (p->nScan+1) * sizeof(ScanStatus); + sqlite3_int64 nByte = (p->nScan+1) * sizeof(ScanStatus); ScanStatus *aNew; aNew = (ScanStatus*)sqlite3DbRealloc(p->db, p->aScan, nByte); if( aNew ){ @@ -75730,16 +78576,16 @@ SQLITE_PRIVATE void sqlite3VdbeScanStatus( ** Change the value of the opcode, or P1, P2, P3, or P5 operands ** for a specific instruction. */ -SQLITE_PRIVATE void sqlite3VdbeChangeOpcode(Vdbe *p, u32 addr, u8 iNewOpcode){ +SQLITE_PRIVATE void sqlite3VdbeChangeOpcode(Vdbe *p, int addr, u8 iNewOpcode){ sqlite3VdbeGetOp(p,addr)->opcode = iNewOpcode; } -SQLITE_PRIVATE void sqlite3VdbeChangeP1(Vdbe *p, u32 addr, int val){ +SQLITE_PRIVATE void sqlite3VdbeChangeP1(Vdbe *p, int addr, int val){ sqlite3VdbeGetOp(p,addr)->p1 = val; } -SQLITE_PRIVATE void sqlite3VdbeChangeP2(Vdbe *p, u32 addr, int val){ +SQLITE_PRIVATE void sqlite3VdbeChangeP2(Vdbe *p, int addr, int val){ sqlite3VdbeGetOp(p,addr)->p2 = val; } -SQLITE_PRIVATE void sqlite3VdbeChangeP3(Vdbe *p, u32 addr, int val){ +SQLITE_PRIVATE void sqlite3VdbeChangeP3(Vdbe *p, int addr, int val){ sqlite3VdbeGetOp(p,addr)->p3 = val; } SQLITE_PRIVATE void sqlite3VdbeChangeP5(Vdbe *p, u16 p5){ @@ -75755,6 +78601,34 @@ SQLITE_PRIVATE void sqlite3VdbeJumpHere(Vdbe *p, int addr){ sqlite3VdbeChangeP2(p, addr, p->nOp); } +/* +** Change the P2 operand of the jump instruction at addr so that +** the jump lands on the next opcode. Or if the jump instruction was +** the previous opcode (and is thus a no-op) then simply back up +** the next instruction counter by one slot so that the jump is +** overwritten by the next inserted opcode. +** +** This routine is an optimization of sqlite3VdbeJumpHere() that +** strives to omit useless byte-code like this: +** +** 7 Once 0 8 0 +** 8 ... +*/ +SQLITE_PRIVATE void sqlite3VdbeJumpHereOrPopInst(Vdbe *p, int addr){ + if( addr==p->nOp-1 ){ + assert( p->aOp[addr].opcode==OP_Once + || p->aOp[addr].opcode==OP_If + || p->aOp[addr].opcode==OP_FkIfZero ); + assert( p->aOp[addr].p4type==0 ); +#ifdef SQLITE_VDBE_COVERAGE + sqlite3VdbeGetOp(p,-1)->iSrcLine = 0; /* Erase VdbeCoverage() macros */ +#endif + p->nOp--; + }else{ + sqlite3VdbeChangeP2(p, addr, p->nOp); + } +} + /* ** If the input FuncDef structure is ephemeral, then free it. If @@ -75766,8 +78640,6 @@ static void freeEphemeralFunction(sqlite3 *db, FuncDef *pDef){ } } -static void vdbeFreeOpArray(sqlite3 *, Op *, int); - /* ** Delete a P4 value if necessary. */ @@ -75777,7 +78649,7 @@ static SQLITE_NOINLINE void freeP4Mem(sqlite3 *db, Mem *p){ } static SQLITE_NOINLINE void freeP4FuncCtx(sqlite3 *db, sqlite3_context *p){ freeEphemeralFunction(db, p->pFunc); - sqlite3DbFreeNN(db, p); + sqlite3DbFreeNN(db, p); } static void freeP4(sqlite3 *db, int p4type, void *p4){ assert( db ); @@ -75851,6 +78723,13 @@ SQLITE_PRIVATE void sqlite3VdbeLinkSubProgram(Vdbe *pVdbe, SubProgram *p){ pVdbe->pProgram = p; } +/* +** Return true if the given Vdbe has any SubPrograms. +*/ +SQLITE_PRIVATE int sqlite3VdbeHasSubProgram(Vdbe *pVdbe){ + return pVdbe->pProgram!=0; +} + /* ** Change the opcode at addr into OP_Noop */ @@ -75878,6 +78757,41 @@ SQLITE_PRIVATE int sqlite3VdbeDeletePriorOpcode(Vdbe *p, u8 op){ } } +#ifdef SQLITE_DEBUG +/* +** Generate an OP_ReleaseReg opcode to indicate that a range of +** registers, except any identified by mask, are no longer in use. +*/ +SQLITE_PRIVATE void sqlite3VdbeReleaseRegisters( + Parse *pParse, /* Parsing context */ + int iFirst, /* Index of first register to be released */ + int N, /* Number of registers to release */ + u32 mask, /* Mask of registers to NOT release */ + int bUndefine /* If true, mark registers as undefined */ +){ + if( N==0 ) return; + assert( pParse->pVdbe ); + assert( iFirst>=1 ); + assert( iFirst+N-1<=pParse->nMem ); + if( N<=31 && mask!=0 ){ + while( N>0 && (mask&1)!=0 ){ + mask >>= 1; + iFirst++; + N--; + } + while( N>0 && N<=32 && (mask & MASKBIT32(N-1))!=0 ){ + mask &= ~MASKBIT32(N-1); + N--; + } + } + if( N>0 ){ + sqlite3VdbeAddOp3(pParse->pVdbe, OP_ReleaseReg, iFirst, N, *(int*)&mask); + if( bUndefine ) sqlite3VdbeChangeP5(pParse->pVdbe, 1); + } +} +#endif /* SQLITE_DEBUG */ + + /* ** Change the value of the P4 operand for a specific instruction. ** This routine is useful when a large program is loaded from a @@ -75995,7 +78909,8 @@ SQLITE_PRIVATE void sqlite3VdbeSetP4KeyInfo(Parse *pParse, Index *pIdx){ */ static void vdbeVComment(Vdbe *p, const char *zFormat, va_list ap){ assert( p->nOp>0 || p->aOp==0 ); - assert( p->aOp==0 || p->aOp[p->nOp-1].zComment==0 || p->db->mallocFailed ); + assert( p->aOp==0 || p->aOp[p->nOp-1].zComment==0 || p->db->mallocFailed + || p->pParse->nErr>0 ); if( p->nOp ){ assert( p->aOp ); sqlite3DbFree(p->db, p->aOp[p->nOp-1].zComment); @@ -76085,17 +79000,19 @@ static int translateP(char c, const Op *pOp){ ** "PX@PY+1" -> "r[X..X+Y]" or "r[x]" if y is 0 ** "PY..PY" -> "r[X..Y]" or "r[x]" if y<=x */ -static int displayComment( +SQLITE_PRIVATE char *sqlite3VdbeDisplayComment( + sqlite3 *db, /* Optional - Oom error reporting only */ const Op *pOp, /* The opcode to be commented */ - const char *zP4, /* Previously obtained value for P4 */ - char *zTemp, /* Write result here */ - int nTemp /* Space available in zTemp[] */ + const char *zP4 /* Previously obtained value for P4 */ ){ const char *zOpName; const char *zSynopsis; int nOpName; - int ii, jj; + int ii; char zAlt[50]; + StrAccum x; + + sqlite3StrAccumInit(&x, 0, 0, 0, SQLITE_MAX_LENGTH); zOpName = sqlite3OpcodeName(pOp->opcode); nOpName = sqlite3Strlen30(zOpName); if( zOpName[nOpName+1] ){ @@ -76110,53 +79027,64 @@ static int displayComment( } zSynopsis = zAlt; } - for(ii=jj=0; jjzComment); + sqlite3_str_appendall(&x, pOp->zComment); seenCom = 1; }else{ int v1 = translateP(c, pOp); int v2; - sqlite3_snprintf(nTemp-jj, zTemp+jj, "%d", v1); if( strncmp(zSynopsis+ii+1, "@P", 2)==0 ){ ii += 3; - jj += sqlite3Strlen30(zTemp+jj); v2 = translateP(zSynopsis[ii], pOp); if( strncmp(zSynopsis+ii+1,"+1",2)==0 ){ ii += 2; v2++; } - if( v2>1 ){ - sqlite3_snprintf(nTemp-jj, zTemp+jj, "..%d", v1+v2-1); + if( v2<2 ){ + sqlite3_str_appendf(&x, "%d", v1); + }else{ + sqlite3_str_appendf(&x, "%d..%d", v1, v1+v2-1); + } + }else if( strncmp(zSynopsis+ii+1, "@NP", 3)==0 ){ + sqlite3_context *pCtx = pOp->p4.pCtx; + if( pOp->p4type!=P4_FUNCCTX || pCtx->argc==1 ){ + sqlite3_str_appendf(&x, "%d", v1); + }else if( pCtx->argc>1 ){ + sqlite3_str_appendf(&x, "%d..%d", v1, v1+pCtx->argc-1); + }else{ + assert( x.nChar>2 ); + x.nChar -= 2; + ii++; + } + ii += 3; + }else{ + sqlite3_str_appendf(&x, "%d", v1); + if( strncmp(zSynopsis+ii+1, "..P3", 4)==0 && pOp->p3==0 ){ + ii += 4; } - }else if( strncmp(zSynopsis+ii+1, "..P3", 4)==0 && pOp->p3==0 ){ - ii += 4; } } - jj += sqlite3Strlen30(zTemp+jj); }else{ - zTemp[jj++] = c; + sqlite3_str_appendchar(&x, 1, c); } } - if( !seenCom && jjzComment ){ - sqlite3_snprintf(nTemp-jj, zTemp+jj, "; %s", pOp->zComment); - jj += sqlite3Strlen30(zTemp+jj); + if( !seenCom && pOp->zComment ){ + sqlite3_str_appendf(&x, "; %s", pOp->zComment); } - if( jjzComment ){ - sqlite3_snprintf(nTemp, zTemp, "%s", pOp->zComment); - jj = sqlite3Strlen30(zTemp); - }else{ - zTemp[0] = 0; - jj = 0; + sqlite3_str_appendall(&x, pOp->zComment); + } + if( (x.accError & SQLITE_NOMEM)!=0 && db!=0 ){ + sqlite3OomFault(db); } - return jj; + return sqlite3StrAccumFinish(&x); } -#endif /* SQLITE_DEBUG */ +#endif /* SQLITE_ENABLE_EXPLAIN_COMMENTS */ #if VDBE_DISPLAY_P4 && defined(SQLITE_ENABLE_CURSOR_HINTS) /* @@ -76237,23 +79165,25 @@ static void displayP4Expr(StrAccum *p, Expr *pExpr){ ** Compute a string that describes the P4 parameter for an opcode. ** Use zTemp for any required temporary buffer space. */ -static char *displayP4(Op *pOp, char *zTemp, int nTemp){ - char *zP4 = zTemp; +SQLITE_PRIVATE char *sqlite3VdbeDisplayP4(sqlite3 *db, Op *pOp){ + char *zP4 = 0; StrAccum x; - assert( nTemp>=20 ); - sqlite3StrAccumInit(&x, 0, zTemp, nTemp, 0); + + sqlite3StrAccumInit(&x, 0, 0, 0, SQLITE_MAX_LENGTH); switch( pOp->p4type ){ case P4_KEYINFO: { int j; KeyInfo *pKeyInfo = pOp->p4.pKeyInfo; - assert( pKeyInfo->aSortOrder!=0 ); + assert( pKeyInfo->aSortFlags!=0 ); sqlite3_str_appendf(&x, "k(%d", pKeyInfo->nKeyField); for(j=0; jnKeyField; j++){ CollSeq *pColl = pKeyInfo->aColl[j]; const char *zColl = pColl ? pColl->zName : ""; if( strcmp(zColl, "BINARY")==0 ) zColl = "B"; - sqlite3_str_appendf(&x, ",%s%s", - pKeyInfo->aSortOrder[j] ? "-" : "", zColl); + sqlite3_str_appendf(&x, ",%s%s%s", + (pKeyInfo->aSortFlags[j] & KEYINFO_ORDER_DESC) ? "-" : "", + (pKeyInfo->aSortFlags[j] & KEYINFO_ORDER_BIGNULL)? "N." : "", + zColl); } sqlite3_str_append(&x, ")", 1); break; @@ -76265,8 +79195,11 @@ static char *displayP4(Op *pOp, char *zTemp, int nTemp){ } #endif case P4_COLLSEQ: { + static const char *const encnames[] = {"?", "8", "16LE", "16BE"}; CollSeq *pColl = pOp->p4.pColl; - sqlite3_str_appendf(&x, "(%.20s)", pColl->zName); + assert( pColl->enc>=0 && pColl->enc<4 ); + sqlite3_str_appendf(&x, "%.18s-%s", pColl->zName, + encnames[pColl->enc]); break; } case P4_FUNCDEF: { @@ -76274,13 +79207,11 @@ static char *displayP4(Op *pOp, char *zTemp, int nTemp){ sqlite3_str_appendf(&x, "%s(%d)", pDef->zName, pDef->nArg); break; } -#if defined(SQLITE_DEBUG) || defined(VDBE_PROFILE) case P4_FUNCCTX: { FuncDef *pDef = pOp->p4.pCtx->pFunc; sqlite3_str_appendf(&x, "%s(%d)", pDef->zName, pDef->nArg); break; } -#endif case P4_INT64: { sqlite3_str_appendf(&x, "%lld", *pOp->p4.pI64); break; @@ -76297,7 +79228,7 @@ static char *displayP4(Op *pOp, char *zTemp, int nTemp){ Mem *pMem = pOp->p4.pMem; if( pMem->flags & MEM_Str ){ zP4 = pMem->z; - }else if( pMem->flags & MEM_Int ){ + }else if( pMem->flags & (MEM_Int|MEM_IntReal) ){ sqlite3_str_appendf(&x, "%lld", pMem->u.i); }else if( pMem->flags & MEM_Real ){ sqlite3_str_appendf(&x, "%.16g", pMem->u.r); @@ -76322,36 +79253,32 @@ static char *displayP4(Op *pOp, char *zTemp, int nTemp){ int n = ai[0]; /* The first element of an INTARRAY is always the ** count of the number of elements to follow */ for(i=1; i<=n; i++){ - sqlite3_str_appendf(&x, ",%d", ai[i]); + sqlite3_str_appendf(&x, "%c%d", (i==1 ? '[' : ','), ai[i]); } - zTemp[0] = '['; sqlite3_str_append(&x, "]", 1); break; } case P4_SUBPROGRAM: { - sqlite3_str_appendf(&x, "program"); + zP4 = "program"; break; } case P4_DYNBLOB: case P4_ADVANCE: { - zTemp[0] = 0; break; } case P4_TABLE: { - sqlite3_str_appendf(&x, "%s", pOp->p4.pTab->zName); + zP4 = pOp->p4.pTab->zName; break; } default: { zP4 = pOp->p4.z; - if( zP4==0 ){ - zP4 = zTemp; - zTemp[0] = 0; - } } } - sqlite3StrAccumFinish(&x); - assert( zP4!=0 ); - return zP4; + if( zP4 ) sqlite3_str_appendall(&x, zP4); + if( (x.accError & SQLITE_NOMEM)!=0 ){ + sqlite3OomFault(db); + } + return sqlite3StrAccumFinish(&x); } #endif /* VDBE_DISPLAY_P4 */ @@ -76439,26 +79366,32 @@ SQLITE_PRIVATE void sqlite3VdbeLeave(Vdbe *p){ /* ** Print a single opcode. This routine is used for debugging only. */ -SQLITE_PRIVATE void sqlite3VdbePrintOp(FILE *pOut, int pc, Op *pOp){ +SQLITE_PRIVATE void sqlite3VdbePrintOp(FILE *pOut, int pc, VdbeOp *pOp){ char *zP4; - char zPtr[50]; - char zCom[100]; + char *zCom; + sqlite3 dummyDb; static const char *zFormat1 = "%4d %-13s %4d %4d %4d %-13s %.2X %s\n"; if( pOut==0 ) pOut = stdout; - zP4 = displayP4(pOp, zPtr, sizeof(zPtr)); + sqlite3BeginBenignMalloc(); + dummyDb.mallocFailed = 1; + zP4 = sqlite3VdbeDisplayP4(&dummyDb, pOp); #ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS - displayComment(pOp, zP4, zCom, sizeof(zCom)); + zCom = sqlite3VdbeDisplayComment(0, pOp, zP4); #else - zCom[0] = 0; + zCom = 0; #endif /* NB: The sqlite3OpcodeName() function is implemented by code created ** by the mkopcodeh.awk and mkopcodec.awk scripts which extract the ** information from the vdbe.c source text */ fprintf(pOut, zFormat1, pc, - sqlite3OpcodeName(pOp->opcode), pOp->p1, pOp->p2, pOp->p3, zP4, pOp->p5, - zCom + sqlite3OpcodeName(pOp->opcode), pOp->p1, pOp->p2, pOp->p3, + zP4 ? zP4 : "", pOp->p5, + zCom ? zCom : "" ); fflush(pOut); + sqlite3_free(zP4); + sqlite3_free(zCom); + sqlite3EndBenignMalloc(); } #endif @@ -76508,9 +79441,8 @@ static void releaseMemArray(Mem *p, int N){ */ testcase( p->flags & MEM_Agg ); testcase( p->flags & MEM_Dyn ); - testcase( p->flags & MEM_Frame ); - testcase( p->flags & MEM_RowSet ); - if( p->flags&(MEM_Agg|MEM_Dyn|MEM_Frame|MEM_RowSet) ){ + testcase( p->xDel==sqlite3VdbeFrameMemDel ); + if( p->flags&(MEM_Agg|MEM_Dyn) ){ sqlite3VdbeMemRelease(p); }else if( p->szMalloc ){ sqlite3DbFreeNN(db, p->zMalloc); @@ -76522,6 +79454,150 @@ static void releaseMemArray(Mem *p, int N){ } } +#ifdef SQLITE_DEBUG +/* +** Verify that pFrame is a valid VdbeFrame pointer. Return true if it is +** and false if something is wrong. +** +** This routine is intended for use inside of assert() statements only. +*/ +SQLITE_PRIVATE int sqlite3VdbeFrameIsValid(VdbeFrame *pFrame){ + if( pFrame->iFrameMagic!=SQLITE_FRAME_MAGIC ) return 0; + return 1; +} +#endif + + +/* +** This is a destructor on a Mem object (which is really an sqlite3_value) +** that deletes the Frame object that is attached to it as a blob. +** +** This routine does not delete the Frame right away. It merely adds the +** frame to a list of frames to be deleted when the Vdbe halts. +*/ +SQLITE_PRIVATE void sqlite3VdbeFrameMemDel(void *pArg){ + VdbeFrame *pFrame = (VdbeFrame*)pArg; + assert( sqlite3VdbeFrameIsValid(pFrame) ); + pFrame->pParent = pFrame->v->pDelFrame; + pFrame->v->pDelFrame = pFrame; +} + +#if defined(SQLITE_ENABLE_BYTECODE_VTAB) || !defined(SQLITE_OMIT_EXPLAIN) +/* +** Locate the next opcode to be displayed in EXPLAIN or EXPLAIN +** QUERY PLAN output. +** +** Return SQLITE_ROW on success. Return SQLITE_DONE if there are no +** more opcodes to be displayed. +*/ +SQLITE_PRIVATE int sqlite3VdbeNextOpcode( + Vdbe *p, /* The statement being explained */ + Mem *pSub, /* Storage for keeping track of subprogram nesting */ + int eMode, /* 0: normal. 1: EQP. 2: TablesUsed */ + int *piPc, /* IN/OUT: Current rowid. Overwritten with next rowid */ + int *piAddr, /* OUT: Write index into (*paOp)[] here */ + Op **paOp /* OUT: Write the opcode array here */ +){ + int nRow; /* Stop when row count reaches this */ + int nSub = 0; /* Number of sub-vdbes seen so far */ + SubProgram **apSub = 0; /* Array of sub-vdbes */ + int i; /* Next instruction address */ + int rc = SQLITE_OK; /* Result code */ + Op *aOp = 0; /* Opcode array */ + int iPc; /* Rowid. Copy of value in *piPc */ + + /* When the number of output rows reaches nRow, that means the + ** listing has finished and sqlite3_step() should return SQLITE_DONE. + ** nRow is the sum of the number of rows in the main program, plus + ** the sum of the number of rows in all trigger subprograms encountered + ** so far. The nRow value will increase as new trigger subprograms are + ** encountered, but p->pc will eventually catch up to nRow. + */ + nRow = p->nOp; + if( pSub!=0 ){ + if( pSub->flags&MEM_Blob ){ + /* pSub is initiallly NULL. It is initialized to a BLOB by + ** the P4_SUBPROGRAM processing logic below */ + nSub = pSub->n/sizeof(Vdbe*); + apSub = (SubProgram **)pSub->z; + } + for(i=0; inOp; + } + } + iPc = *piPc; + while(1){ /* Loop exits via break */ + i = iPc++; + if( i>=nRow ){ + p->rc = SQLITE_OK; + rc = SQLITE_DONE; + break; + } + if( inOp ){ + /* The rowid is small enough that we are still in the + ** main program. */ + aOp = p->aOp; + }else{ + /* We are currently listing subprograms. Figure out which one and + ** pick up the appropriate opcode. */ + int j; + i -= p->nOp; + assert( apSub!=0 ); + assert( nSub>0 ); + for(j=0; i>=apSub[j]->nOp; j++){ + i -= apSub[j]->nOp; + assert( inOp || j+1aOp; + } + + /* When an OP_Program opcode is encounter (the only opcode that has + ** a P4_SUBPROGRAM argument), expand the size of the array of subprograms + ** kept in p->aMem[9].z to hold the new program - assuming this subprogram + ** has not already been seen. + */ + if( pSub!=0 && aOp[i].p4type==P4_SUBPROGRAM ){ + int nByte = (nSub+1)*sizeof(SubProgram*); + int j; + for(j=0; jrc = sqlite3VdbeMemGrow(pSub, nByte, nSub!=0); + if( p->rc!=SQLITE_OK ){ + rc = SQLITE_ERROR; + break; + } + apSub = (SubProgram **)pSub->z; + apSub[nSub++] = aOp[i].p4.pProgram; + MemSetTypeFlag(pSub, MEM_Blob); + pSub->n = nSub*sizeof(SubProgram*); + nRow += aOp[i].p4.pProgram->nOp; + } + } + if( eMode==0 ) break; +#ifdef SQLITE_ENABLE_BYTECODE_VTAB + if( eMode==2 ){ + Op *pOp = aOp + i; + if( pOp->opcode==OP_OpenRead ) break; + if( pOp->opcode==OP_OpenWrite && (pOp->p5 & OPFLAG_P2ISREG)==0 ) break; + if( pOp->opcode==OP_ReopenIdx ) break; + }else +#endif + { + assert( eMode==1 ); + if( aOp[i].opcode==OP_Explain ) break; + if( aOp[i].opcode==OP_Init && iPc>1 ) break; + } + } + *piPc = iPc; + *piAddr = i; + *paOp = aOp; + return rc; +} +#endif /* SQLITE_ENABLE_BYTECODE_VTAB || !SQLITE_OMIT_EXPLAIN */ + + /* ** Delete a VdbeFrame object and its contents. VdbeFrame objects are ** allocated by the OP_Program opcode in sqlite3VdbeExec(). @@ -76530,6 +79606,7 @@ SQLITE_PRIVATE void sqlite3VdbeFrameDelete(VdbeFrame *p){ int i; Mem *aMem = VdbeFrameMem(p); VdbeCursor **apCsr = (VdbeCursor **)&aMem[p->nChildMem]; + assert( sqlite3VdbeFrameIsValid(p) ); for(i=0; inChildCsr; i++){ sqlite3VdbeFreeCursor(p->v, apCsr[i]); } @@ -76560,16 +79637,14 @@ SQLITE_PRIVATE void sqlite3VdbeFrameDelete(VdbeFrame *p){ SQLITE_PRIVATE int sqlite3VdbeList( Vdbe *p /* The VDBE */ ){ - int nRow; /* Stop when row count reaches this */ - int nSub = 0; /* Number of sub-vdbes seen so far */ - SubProgram **apSub = 0; /* Array of sub-vdbes */ Mem *pSub = 0; /* Memory cell hold array of subprogs */ sqlite3 *db = p->db; /* The database connection */ int i; /* Loop counter */ int rc = SQLITE_OK; /* Return code */ Mem *pMem = &p->aMem[1]; /* First Mem of result set */ int bListSubprogs = (p->explain==1 || (db->flags & SQLITE_TriggerEQP)!=0); - Op *pOp = 0; + Op *aOp; /* Array of opcodes */ + Op *pOp; /* Current opcode */ assert( p->explain ); assert( p->magic==VDBE_MAGIC_RUN ); @@ -76589,14 +79664,6 @@ SQLITE_PRIVATE int sqlite3VdbeList( return SQLITE_ERROR; } - /* When the number of output rows reaches nRow, that means the - ** listing has finished and sqlite3_step() should return SQLITE_DONE. - ** nRow is the sum of the number of rows in the main program, plus - ** the sum of the number of rows in all trigger subprograms encountered - ** so far. The nRow value will increase as new trigger subprograms are - ** encountered, but p->pc will eventually catch up to nRow. - */ - nRow = p->nOp; if( bListSubprogs ){ /* The first 8 memory cells are used for the result set. So we will ** commandeer the 9th cell to use as storage for an array of pointers @@ -76604,144 +79671,55 @@ SQLITE_PRIVATE int sqlite3VdbeList( ** cells. */ assert( p->nMem>9 ); pSub = &p->aMem[9]; - if( pSub->flags&MEM_Blob ){ - /* On the first call to sqlite3_step(), pSub will hold a NULL. It is - ** initialized to a BLOB by the P4_SUBPROGRAM processing logic below */ - nSub = pSub->n/sizeof(Vdbe*); - apSub = (SubProgram **)pSub->z; - } - for(i=0; inOp; - } + }else{ + pSub = 0; } - while(1){ /* Loop exits via break */ - i = p->pc++; - if( i>=nRow ){ - p->rc = SQLITE_OK; - rc = SQLITE_DONE; - break; - } - if( inOp ){ - /* The output line number is small enough that we are still in the - ** main program. */ - pOp = &p->aOp[i]; - }else{ - /* We are currently listing subprograms. Figure out which one and - ** pick up the appropriate opcode. */ - int j; - i -= p->nOp; - for(j=0; i>=apSub[j]->nOp; j++){ - i -= apSub[j]->nOp; - } - pOp = &apSub[j]->aOp[i]; - } - - /* When an OP_Program opcode is encounter (the only opcode that has - ** a P4_SUBPROGRAM argument), expand the size of the array of subprograms - ** kept in p->aMem[9].z to hold the new program - assuming this subprogram - ** has not already been seen. - */ - if( bListSubprogs && pOp->p4type==P4_SUBPROGRAM ){ - int nByte = (nSub+1)*sizeof(SubProgram*); - int j; - for(j=0; jp4.pProgram ) break; - } - if( j==nSub ){ - p->rc = sqlite3VdbeMemGrow(pSub, nByte, nSub!=0); - if( p->rc!=SQLITE_OK ){ - rc = SQLITE_ERROR; - break; - } - apSub = (SubProgram **)pSub->z; - apSub[nSub++] = pOp->p4.pProgram; - pSub->flags |= MEM_Blob; - pSub->n = nSub*sizeof(SubProgram*); - nRow += pOp->p4.pProgram->nOp; - } - } - if( p->explain<2 ) break; - if( pOp->opcode==OP_Explain ) break; - if( pOp->opcode==OP_Init && p->pc>1 ) break; - } + /* Figure out which opcode is next to display */ + rc = sqlite3VdbeNextOpcode(p, pSub, p->explain==2, &p->pc, &i, &aOp); if( rc==SQLITE_OK ){ - if( db->u1.isInterrupted ){ + pOp = aOp + i; + if( AtomicLoad(&db->u1.isInterrupted) ){ p->rc = SQLITE_INTERRUPT; rc = SQLITE_ERROR; sqlite3VdbeError(p, sqlite3ErrStr(p->rc)); }else{ - char *zP4; - if( p->explain==1 ){ - pMem->flags = MEM_Int; - pMem->u.i = i; /* Program counter */ - pMem++; - - pMem->flags = MEM_Static|MEM_Str|MEM_Term; - pMem->z = (char*)sqlite3OpcodeName(pOp->opcode); /* Opcode */ - assert( pMem->z!=0 ); - pMem->n = sqlite3Strlen30(pMem->z); - pMem->enc = SQLITE_UTF8; - pMem++; - } - - pMem->flags = MEM_Int; - pMem->u.i = pOp->p1; /* P1 */ - pMem++; - - pMem->flags = MEM_Int; - pMem->u.i = pOp->p2; /* P2 */ - pMem++; - - pMem->flags = MEM_Int; - pMem->u.i = pOp->p3; /* P3 */ - pMem++; - - if( sqlite3VdbeMemClearAndResize(pMem, 100) ){ /* P4 */ - assert( p->db->mallocFailed ); - return SQLITE_ERROR; - } - pMem->flags = MEM_Str|MEM_Term; - zP4 = displayP4(pOp, pMem->z, pMem->szMalloc); - if( zP4!=pMem->z ){ - pMem->n = 0; - sqlite3VdbeMemSetStr(pMem, zP4, -1, SQLITE_UTF8, 0); + char *zP4 = sqlite3VdbeDisplayP4(db, pOp); + if( p->explain==2 ){ + sqlite3VdbeMemSetInt64(pMem, pOp->p1); + sqlite3VdbeMemSetInt64(pMem+1, pOp->p2); + sqlite3VdbeMemSetInt64(pMem+2, pOp->p3); + sqlite3VdbeMemSetStr(pMem+3, zP4, -1, SQLITE_UTF8, sqlite3_free); + p->nResColumn = 4; }else{ - assert( pMem->z!=0 ); - pMem->n = sqlite3Strlen30(pMem->z); - pMem->enc = SQLITE_UTF8; - } - pMem++; - - if( p->explain==1 ){ - if( sqlite3VdbeMemClearAndResize(pMem, 4) ){ - assert( p->db->mallocFailed ); - return SQLITE_ERROR; - } - pMem->flags = MEM_Str|MEM_Term; - pMem->n = 2; - sqlite3_snprintf(3, pMem->z, "%.2x", pOp->p5); /* P5 */ - pMem->enc = SQLITE_UTF8; - pMem++; - + sqlite3VdbeMemSetInt64(pMem+0, i); + sqlite3VdbeMemSetStr(pMem+1, (char*)sqlite3OpcodeName(pOp->opcode), + -1, SQLITE_UTF8, SQLITE_STATIC); + sqlite3VdbeMemSetInt64(pMem+2, pOp->p1); + sqlite3VdbeMemSetInt64(pMem+3, pOp->p2); + sqlite3VdbeMemSetInt64(pMem+4, pOp->p3); + /* pMem+5 for p4 is done last */ + sqlite3VdbeMemSetInt64(pMem+6, pOp->p5); #ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS - if( sqlite3VdbeMemClearAndResize(pMem, 500) ){ - assert( p->db->mallocFailed ); - return SQLITE_ERROR; + { + char *zCom = sqlite3VdbeDisplayComment(db, pOp, zP4); + sqlite3VdbeMemSetStr(pMem+7, zCom, -1, SQLITE_UTF8, sqlite3_free); } - pMem->flags = MEM_Str|MEM_Term; - pMem->n = displayComment(pOp, zP4, pMem->z, 500); - pMem->enc = SQLITE_UTF8; #else - pMem->flags = MEM_Null; /* Comment */ + sqlite3VdbeMemSetNull(pMem+7); #endif + sqlite3VdbeMemSetStr(pMem+5, zP4, -1, SQLITE_UTF8, sqlite3_free); + p->nResColumn = 8; + } + p->pResultSet = pMem; + if( db->mallocFailed ){ + p->rc = SQLITE_NOMEM; + rc = SQLITE_ERROR; + }else{ + p->rc = SQLITE_OK; + rc = SQLITE_ROW; } - - p->nResColumn = 8 - 4*(p->explain-1); - p->pResultSet = &p->aMem[1]; - p->rc = SQLITE_OK; - rc = SQLITE_ROW; } } return rc; @@ -76802,9 +79780,9 @@ SQLITE_PRIVATE void sqlite3VdbeIOTraceSql(Vdbe *p){ ** of a ReusableSpace object by the allocSpace() routine below. */ struct ReusableSpace { - u8 *pSpace; /* Available memory */ - int nFree; /* Bytes of available memory */ - int nNeeded; /* Total bytes that could not be allocated */ + u8 *pSpace; /* Available memory */ + sqlite3_int64 nFree; /* Bytes of available memory */ + sqlite3_int64 nNeeded; /* Total bytes that could not be allocated */ }; /* Try to allocate nByte bytes of 8-byte aligned bulk memory for pBuf @@ -76824,7 +79802,7 @@ struct ReusableSpace { static void *allocSpace( struct ReusableSpace *p, /* Bulk memory available for allocation */ void *pBuf, /* Pointer to a prior allocation */ - int nByte /* Bytes of memory needed */ + sqlite3_int64 nByte /* Bytes of memory needed */ ){ assert( EIGHT_BYTE_ALIGNMENT(p->pSpace) ); if( pBuf==0 ){ @@ -76942,8 +79920,27 @@ SQLITE_PRIVATE void sqlite3VdbeMakeReady( resolveP2Values(p, &nArg); p->usesStmtJournal = (u8)(pParse->isMultiWrite && pParse->mayAbort); - if( pParse->explain && nMem<10 ){ - nMem = 10; + if( pParse->explain ){ + static const char * const azColName[] = { + "addr", "opcode", "p1", "p2", "p3", "p4", "p5", "comment", + "id", "parent", "notused", "detail" + }; + int iFirst, mx, i; + if( nMem<10 ) nMem = 10; + p->explain = pParse->explain; + if( pParse->explain==2 ){ + sqlite3VdbeSetNumCols(p, 4); + iFirst = 8; + mx = 12; + }else{ + sqlite3VdbeSetNumCols(p, 8); + iFirst = 0; + mx = 8; + } + for(i=iFirst; iexpired = 0; @@ -76957,23 +79954,30 @@ SQLITE_PRIVATE void sqlite3VdbeMakeReady( ** the leftover memory at the end of the opcode array. This can significantly ** reduce the amount of memory held by a prepared statement. */ - do { - x.nNeeded = 0; - p->aMem = allocSpace(&x, p->aMem, nMem*sizeof(Mem)); - p->aVar = allocSpace(&x, p->aVar, nVar*sizeof(Mem)); - p->apArg = allocSpace(&x, p->apArg, nArg*sizeof(Mem*)); - p->apCsr = allocSpace(&x, p->apCsr, nCursor*sizeof(VdbeCursor*)); + x.nNeeded = 0; + p->aMem = allocSpace(&x, 0, nMem*sizeof(Mem)); + p->aVar = allocSpace(&x, 0, nVar*sizeof(Mem)); + p->apArg = allocSpace(&x, 0, nArg*sizeof(Mem*)); + p->apCsr = allocSpace(&x, 0, nCursor*sizeof(VdbeCursor*)); #ifdef SQLITE_ENABLE_STMT_SCANSTATUS - p->anExec = allocSpace(&x, p->anExec, p->nOp*sizeof(i64)); + p->anExec = allocSpace(&x, 0, p->nOp*sizeof(i64)); #endif - if( x.nNeeded==0 ) break; + if( x.nNeeded ){ x.pSpace = p->pFree = sqlite3DbMallocRawNN(db, x.nNeeded); x.nFree = x.nNeeded; - }while( !db->mallocFailed ); + if( !db->mallocFailed ){ + p->aMem = allocSpace(&x, p->aMem, nMem*sizeof(Mem)); + p->aVar = allocSpace(&x, p->aVar, nVar*sizeof(Mem)); + p->apArg = allocSpace(&x, p->apArg, nArg*sizeof(Mem*)); + p->apCsr = allocSpace(&x, p->apCsr, nCursor*sizeof(VdbeCursor*)); +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + p->anExec = allocSpace(&x, p->anExec, p->nOp*sizeof(i64)); +#endif + } + } p->pVList = pParse->pVList; pParse->pVList = 0; - p->explain = pParse->explain; if( db->mallocFailed ){ p->nVar = 0; p->nCursor = 0; @@ -77285,8 +80289,9 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){ /* Select a master journal file name */ nMainFile = sqlite3Strlen30(zMainFile); - zMaster = sqlite3MPrintf(db, "%s-mjXXXXXX9XXz", zMainFile); + zMaster = sqlite3MPrintf(db, "%.4c%s%.16c", 0,zMainFile,0); if( zMaster==0 ) return SQLITE_NOMEM_BKPT; + zMaster += 4; do { u32 iRandom; if( retryCount ){ @@ -77316,7 +80321,7 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){ ); } if( rc!=SQLITE_OK ){ - sqlite3DbFree(db, zMaster); + sqlite3DbFree(db, zMaster-4); return rc; } @@ -77339,7 +80344,7 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){ if( rc!=SQLITE_OK ){ sqlite3OsCloseFree(pMaster); sqlite3OsDelete(pVfs, zMaster, 0); - sqlite3DbFree(db, zMaster); + sqlite3DbFree(db, zMaster-4); return rc; } } @@ -77353,7 +80358,7 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){ ){ sqlite3OsCloseFree(pMaster); sqlite3OsDelete(pVfs, zMaster, 0); - sqlite3DbFree(db, zMaster); + sqlite3DbFree(db, zMaster-4); return rc; } @@ -77376,7 +80381,7 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){ sqlite3OsCloseFree(pMaster); assert( rc!=SQLITE_BUSY ); if( rc!=SQLITE_OK ){ - sqlite3DbFree(db, zMaster); + sqlite3DbFree(db, zMaster-4); return rc; } @@ -77385,7 +80390,7 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){ ** transaction files are deleted. */ rc = sqlite3OsDelete(pVfs, zMaster, 1); - sqlite3DbFree(db, zMaster); + sqlite3DbFree(db, zMaster-4); zMaster = 0; if( rc ){ return rc; @@ -77622,7 +80627,7 @@ SQLITE_PRIVATE int sqlite3VdbeHalt(Vdbe *p){ } /* Check for immediate foreign key violations. */ - if( p->rc==SQLITE_OK ){ + if( p->rc==SQLITE_OK || (p->errorAction==OE_Fail && !isSpecialError) ){ sqlite3VdbeCheckFk(p, 0); } @@ -77661,7 +80666,7 @@ SQLITE_PRIVATE int sqlite3VdbeHalt(Vdbe *p){ }else{ db->nDeferredCons = 0; db->nDeferredImmCons = 0; - db->flags &= ~SQLITE_DeferFKs; + db->flags &= ~(u64)SQLITE_DeferFKs; sqlite3CommitInternalChanges(db); } }else{ @@ -77826,7 +80831,7 @@ SQLITE_PRIVATE int sqlite3VdbeReset(Vdbe *p){ */ sqlite3VdbeHalt(p); - /* If the VDBE has be run even partially, then transfer the error code + /* If the VDBE has been run even partially, then transfer the error code ** and error message from the VDBE into the main database structure. But ** if the VDBE has just been set to run but has not actually executed any ** instructions yet, leave the main database error information unchanged. @@ -77974,6 +80979,16 @@ SQLITE_PRIVATE void sqlite3VdbeClearObject(sqlite3 *db, Vdbe *p){ vdbeFreeOpArray(db, p->aOp, p->nOp); sqlite3DbFree(db, p->aColName); sqlite3DbFree(db, p->zSql); +#ifdef SQLITE_ENABLE_NORMALIZE + sqlite3DbFree(db, p->zNormSql); + { + DblquoteStr *pThis, *pNext; + for(pThis=p->pDblStr; pThis; pThis=pNext){ + pNext = pThis->pNextStr; + sqlite3DbFree(db, pThis); + } + } +#endif #ifdef SQLITE_ENABLE_STMT_SCANSTATUS { int i; @@ -78014,7 +81029,7 @@ SQLITE_PRIVATE void sqlite3VdbeDelete(Vdbe *p){ ** carried out. Seek the cursor now. If an error occurs, return ** the appropriate error code. */ -static int SQLITE_NOINLINE handleDeferredMoveto(VdbeCursor *p){ +SQLITE_PRIVATE int SQLITE_NOINLINE sqlite3VdbeFinishMoveto(VdbeCursor *p){ int res, rc; #ifdef SQLITE_TEST extern int sqlite3_search_count; @@ -78081,12 +81096,12 @@ SQLITE_PRIVATE int sqlite3VdbeCursorMoveto(VdbeCursor **pp, int *piCol){ assert( p->eCurType==CURTYPE_BTREE || p->eCurType==CURTYPE_PSEUDO ); if( p->deferredMoveto ){ int iMap; - if( p->aAltMap && (iMap = p->aAltMap[1+*piCol])>0 ){ + if( p->aAltMap && (iMap = p->aAltMap[1+*piCol])>0 && !p->nullRow ){ *pp = p->pAltCursor; *piCol = iMap - 1; return SQLITE_OK; } - return handleDeferredMoveto(p); + return sqlite3VdbeFinishMoveto(p); } if( sqlite3BtreeCursorHasMoved(p->uc.pCursor) ){ return handleMovedCursor(p); @@ -78136,8 +81151,17 @@ SQLITE_PRIVATE int sqlite3VdbeCursorMoveto(VdbeCursor **pp, int *piCol){ ** of SQLite will not understand those serial types. */ +#if 0 /* Inlined into the OP_MakeRecord opcode */ /* ** Return the serial-type for the value stored in pMem. +** +** This routine might convert a large MEM_IntReal value into MEM_Real. +** +** 2019-07-11: The primary user of this subroutine was the OP_MakeRecord +** opcode in the byte-code engine. But by moving this routine in-line, we +** can omit some redundant tests and make that opcode a lot faster. So +** this routine is now only used by the STAT3 logic and STAT3 support has +** ended. The code is kept here for historical reference only. */ SQLITE_PRIVATE u32 sqlite3VdbeSerialType(Mem *pMem, int file_format, u32 *pLen){ int flags = pMem->flags; @@ -78148,11 +81172,13 @@ SQLITE_PRIVATE u32 sqlite3VdbeSerialType(Mem *pMem, int file_format, u32 *pLen){ *pLen = 0; return 0; } - if( flags&MEM_Int ){ + if( flags&(MEM_Int|MEM_IntReal) ){ /* Figure out whether to use 1, 2, 4, 6 or 8 bytes. */ # define MAX_6BYTE ((((i64)0x00008000)<<32)-1) i64 i = pMem->u.i; u64 u; + testcase( flags & MEM_Int ); + testcase( flags & MEM_IntReal ); if( i<0 ){ u = ~i; }else{ @@ -78172,6 +81198,15 @@ SQLITE_PRIVATE u32 sqlite3VdbeSerialType(Mem *pMem, int file_format, u32 *pLen){ if( u<=2147483647 ){ *pLen = 4; return 4; } if( u<=MAX_6BYTE ){ *pLen = 6; return 5; } *pLen = 8; + if( flags&MEM_IntReal ){ + /* If the value is IntReal and is going to take up 8 bytes to store + ** as an integer, then we might as well make it an 8-byte floating + ** point value */ + pMem->u.r = (double)pMem->u.i; + pMem->flags &= ~MEM_IntReal; + pMem->flags |= MEM_Real; + return 7; + } return 6; } if( flags&MEM_Real ){ @@ -78187,6 +81222,7 @@ SQLITE_PRIVATE u32 sqlite3VdbeSerialType(Mem *pMem, int file_format, u32 *pLen){ *pLen = n; return ((n*2) + 12 + ((flags&MEM_Str)!=0)); } +#endif /* inlined into OP_MakeRecord */ /* ** The sizes for serial types less than 128 @@ -78345,7 +81381,7 @@ SQLITE_PRIVATE u32 sqlite3VdbeSerialPut(u8 *buf, Mem *pMem, u32 serial_type){ ** routine so that in most cases the overhead of moving the stack pointer ** is avoided. */ -static u32 SQLITE_NOINLINE serialGet( +static u32 serialGet( const unsigned char *buf, /* Buffer to deserialize from */ u32 serial_type, /* Serial type to deserialize */ Mem *pMem /* Memory cell to write value into */ @@ -78377,7 +81413,7 @@ static u32 SQLITE_NOINLINE serialGet( assert( sizeof(x)==8 && sizeof(pMem->u.r)==8 ); swapMixedEndianFloat(x); memcpy(&pMem->u.r, &x, sizeof(x)); - pMem->flags = sqlite3IsNaN(pMem->u.r) ? MEM_Null : MEM_Real; + pMem->flags = IsNaN(x) ? MEM_Null : MEM_Real; } return 8; } @@ -78495,7 +81531,7 @@ SQLITE_PRIVATE UnpackedRecord *sqlite3VdbeAllocUnpackedRecord( p = (UnpackedRecord *)sqlite3DbMallocRaw(pKeyInfo->db, nByte); if( !p ) return 0; p->aMem = (Mem*)&((char*)p)[ROUND8(sizeof(UnpackedRecord))]; - assert( pKeyInfo->aSortOrder!=0 ); + assert( pKeyInfo->aSortFlags!=0 ); p->pKeyInfo = pKeyInfo; p->nField = pKeyInfo->nKeyField + 1; return p; @@ -78513,7 +81549,7 @@ SQLITE_PRIVATE void sqlite3VdbeRecordUnpack( UnpackedRecord *p /* Populate this structure before returning. */ ){ const unsigned char *aKey = (const unsigned char *)pKey; - int d; + u32 d; u32 idx; /* Offset in aKey[] to read from */ u16 u; /* Unsigned loop counter */ u32 szHdr; @@ -78524,7 +81560,7 @@ SQLITE_PRIVATE void sqlite3VdbeRecordUnpack( idx = getVarint32(aKey, szHdr); d = szHdr; u = 0; - while( idx=p->nField ) break; } + if( d>(u32)nKey && u ){ + assert( CORRUPT_DB ); + /* In a corrupt record entry, the last pMem might have been set up using + ** uninitialized memory. Overwrite its value with NULL, to prevent + ** warnings from MSAN. */ + sqlite3VdbeMemSetNull(pMem-1); + } assert( u<=pKeyInfo->nKeyField + 1 ); p->nField = u; } @@ -78587,7 +81630,7 @@ static int vdbeRecordCompareDebug( if( szHdr1>98307 ) return SQLITE_CORRUPT; d1 = szHdr1; assert( pKeyInfo->nAllField>=pPKey2->nField || CORRUPT_DB ); - assert( pKeyInfo->aSortOrder!=0 ); + assert( pKeyInfo->aSortFlags!=0 ); assert( pKeyInfo->nKeyField>0 ); assert( idx1<=szHdr1 || CORRUPT_DB ); do{ @@ -78602,8 +81645,8 @@ static int vdbeRecordCompareDebug( ** Use that approximation to avoid the more expensive call to ** sqlite3VdbeSerialTypeLen() in the common case. */ - if( d1+serial_type1+2>(u32)nKey1 - && d1+sqlite3VdbeSerialTypeLen(serial_type1)>(u32)nKey1 + if( d1+(u64)serial_type1+2>(u64)nKey1 + && d1+(u64)sqlite3VdbeSerialTypeLen(serial_type1)>(u64)nKey1 ){ break; } @@ -78614,10 +81657,16 @@ static int vdbeRecordCompareDebug( /* Do the comparison */ - rc = sqlite3MemCompare(&mem1, &pPKey2->aMem[i], pKeyInfo->aColl[i]); + rc = sqlite3MemCompare(&mem1, &pPKey2->aMem[i], + pKeyInfo->nAllField>i ? pKeyInfo->aColl[i] : 0); if( rc!=0 ){ assert( mem1.szMalloc==0 ); /* See comment below */ - if( pKeyInfo->aSortOrder[i] ){ + if( (pKeyInfo->aSortFlags[i] & KEYINFO_ORDER_BIGNULL) + && ((mem1.flags & MEM_Null) || (pPKey2->aMem[i].flags & MEM_Null)) + ){ + rc = -rc; + } + if( pKeyInfo->aSortFlags[i] & KEYINFO_ORDER_DESC ){ rc = -rc; /* Invert the result for DESC sort order. */ } goto debugCompareEnd; @@ -78738,7 +81787,7 @@ static int isAllZero(const char *z, int n){ ** is less than, equal to, or greater than the second, respectively. ** If one blob is a prefix of the other, then the shorter is the lessor. */ -static SQLITE_NOINLINE int sqlite3BlobCompare(const Mem *pB1, const Mem *pB2){ +SQLITE_PRIVATE SQLITE_NOINLINE int sqlite3BlobCompare(const Mem *pB1, const Mem *pB2){ int c; int n1 = pB1->n; int n2 = pB2->n; @@ -78808,7 +81857,7 @@ SQLITE_PRIVATE int sqlite3MemCompare(const Mem *pMem1, const Mem *pMem2, const C f1 = pMem1->flags; f2 = pMem2->flags; combined_flags = f1|f2; - assert( (combined_flags & MEM_RowSet)==0 ); + assert( !sqlite3VdbeMemIsRowSet(pMem1) && !sqlite3VdbeMemIsRowSet(pMem2) ); /* If one value is NULL, it is less than the other. If both values ** are NULL, return 0. @@ -78819,8 +81868,13 @@ SQLITE_PRIVATE int sqlite3MemCompare(const Mem *pMem1, const Mem *pMem2, const C /* At least one of the two values is a number */ - if( combined_flags&(MEM_Int|MEM_Real) ){ - if( (f1 & f2 & MEM_Int)!=0 ){ + if( combined_flags&(MEM_Int|MEM_Real|MEM_IntReal) ){ + testcase( combined_flags & MEM_Int ); + testcase( combined_flags & MEM_Real ); + testcase( combined_flags & MEM_IntReal ); + if( (f1 & f2 & (MEM_Int|MEM_IntReal))!=0 ){ + testcase( f1 & f2 & MEM_Int ); + testcase( f1 & f2 & MEM_IntReal ); if( pMem1->u.i < pMem2->u.i ) return -1; if( pMem1->u.i > pMem2->u.i ) return +1; return 0; @@ -78830,15 +81884,23 @@ SQLITE_PRIVATE int sqlite3MemCompare(const Mem *pMem1, const Mem *pMem2, const C if( pMem1->u.r > pMem2->u.r ) return +1; return 0; } - if( (f1&MEM_Int)!=0 ){ + if( (f1&(MEM_Int|MEM_IntReal))!=0 ){ + testcase( f1 & MEM_Int ); + testcase( f1 & MEM_IntReal ); if( (f2&MEM_Real)!=0 ){ return sqlite3IntFloatCompare(pMem1->u.i, pMem2->u.r); + }else if( (f2&(MEM_Int|MEM_IntReal))!=0 ){ + if( pMem1->u.i < pMem2->u.i ) return -1; + if( pMem1->u.i > pMem2->u.i ) return +1; + return 0; }else{ return -1; } } if( (f1&MEM_Real)!=0 ){ - if( (f2&MEM_Int)!=0 ){ + if( (f2&(MEM_Int|MEM_IntReal))!=0 ){ + testcase( f2 & MEM_Int ); + testcase( f2 & MEM_IntReal ); return -sqlite3IntFloatCompare(pMem2->u.i, pMem1->u.r); }else{ return -1; @@ -78953,7 +82015,7 @@ SQLITE_PRIVATE int sqlite3VdbeRecordCompareWithSkip( u32 idx1; /* Offset of first type in header */ int rc = 0; /* Return value */ Mem *pRhs = pPKey2->aMem; /* Next field of pPKey2 to compare */ - KeyInfo *pKeyInfo = pPKey2->pKeyInfo; + KeyInfo *pKeyInfo; const unsigned char *aKey1 = (const unsigned char *)pKey1; Mem mem1; @@ -78970,24 +82032,26 @@ SQLITE_PRIVATE int sqlite3VdbeRecordCompareWithSkip( }else{ idx1 = getVarint32(aKey1, szHdr1); d1 = szHdr1; - if( d1>(unsigned)nKey1 ){ - pPKey2->errCode = (u8)SQLITE_CORRUPT_BKPT; - return 0; /* Corruption */ - } i = 0; } + if( d1>(unsigned)nKey1 ){ + pPKey2->errCode = (u8)SQLITE_CORRUPT_BKPT; + return 0; /* Corruption */ + } VVA_ONLY( mem1.szMalloc = 0; ) /* Only needed by assert() statements */ assert( pPKey2->pKeyInfo->nAllField>=pPKey2->nField || CORRUPT_DB ); - assert( pPKey2->pKeyInfo->aSortOrder!=0 ); + assert( pPKey2->pKeyInfo->aSortFlags!=0 ); assert( pPKey2->pKeyInfo->nKeyField>0 ); assert( idx1<=szHdr1 || CORRUPT_DB ); do{ u32 serial_type; /* RHS is an integer */ - if( pRhs->flags & MEM_Int ){ + if( pRhs->flags & (MEM_Int|MEM_IntReal) ){ + testcase( pRhs->flags & MEM_Int ); + testcase( pRhs->flags & MEM_IntReal ); serial_type = aKey1[idx1]; testcase( serial_type==12 ); if( serial_type>=10 ){ @@ -79035,7 +82099,7 @@ SQLITE_PRIVATE int sqlite3VdbeRecordCompareWithSkip( /* RHS is a string */ else if( pRhs->flags & MEM_Str ){ - getVarint32(&aKey1[idx1], serial_type); + getVarint32NR(&aKey1[idx1], serial_type); testcase( serial_type==12 ); if( serial_type<12 ){ rc = -1; @@ -79045,7 +82109,9 @@ SQLITE_PRIVATE int sqlite3VdbeRecordCompareWithSkip( mem1.n = (serial_type - 12) / 2; testcase( (d1+mem1.n)==(unsigned)nKey1 ); testcase( (d1+mem1.n+1)==(unsigned)nKey1 ); - if( (d1+mem1.n) > (unsigned)nKey1 ){ + if( (d1+mem1.n) > (unsigned)nKey1 + || (pKeyInfo = pPKey2->pKeyInfo)->nAllField<=i + ){ pPKey2->errCode = (u8)SQLITE_CORRUPT_BKPT; return 0; /* Corruption */ }else if( pKeyInfo->aColl[i] ){ @@ -79067,7 +82133,7 @@ SQLITE_PRIVATE int sqlite3VdbeRecordCompareWithSkip( /* RHS is a blob */ else if( pRhs->flags & MEM_Blob ){ assert( (pRhs->flags & MEM_Zero)==0 || pRhs->n==0 ); - getVarint32(&aKey1[idx1], serial_type); + getVarint32NR(&aKey1[idx1], serial_type); testcase( serial_type==12 ); if( serial_type<12 || (serial_type & 0x01) ){ rc = -1; @@ -79099,8 +82165,14 @@ SQLITE_PRIVATE int sqlite3VdbeRecordCompareWithSkip( } if( rc!=0 ){ - if( pKeyInfo->aSortOrder[i] ){ - rc = -rc; + int sortFlags = pPKey2->pKeyInfo->aSortFlags[i]; + if( sortFlags ){ + if( (sortFlags & KEYINFO_ORDER_BIGNULL)==0 + || ((sortFlags & KEYINFO_ORDER_DESC) + !=(serial_type==0 || (pRhs->flags&MEM_Null))) + ){ + rc = -rc; + } } assert( vdbeRecordCompareDebug(nKey1, pKey1, pPKey2, rc) ); assert( mem1.szMalloc==0 ); /* See comment below */ @@ -79108,10 +82180,11 @@ SQLITE_PRIVATE int sqlite3VdbeRecordCompareWithSkip( } i++; + if( i==pPKey2->nField ) break; pRhs++; d1 += sqlite3VdbeSerialTypeLen(serial_type); idx1 += sqlite3VarintLen(serial_type); - }while( idx1<(unsigned)szHdr1 && inField && d1<=(unsigned)nKey1 ); + }while( idx1<(unsigned)szHdr1 && d1<=(unsigned)nKey1 ); /* No memory allocation is ever used on mem1. Prove this using ** the following assert(). If the assert() fails, it indicates a @@ -79123,7 +82196,7 @@ SQLITE_PRIVATE int sqlite3VdbeRecordCompareWithSkip( ** value. */ assert( CORRUPT_DB || vdbeRecordCompareDebug(nKey1, pKey1, pPKey2, pPKey2->default_rc) - || pKeyInfo->db->mallocFailed + || pPKey2->pKeyInfo->db->mallocFailed ); pPKey2->eqSeen = 1; return pPKey2->default_rc; @@ -79249,7 +82322,10 @@ static int vdbeRecordCompareString( assert( pPKey2->aMem[0].flags & MEM_Str ); vdbeAssertFieldCountWithinLimits(nKey1, pKey1, pPKey2->pKeyInfo); - getVarint32(&aKey1[1], serial_type); + serial_type = (u8)(aKey1[1]); + if( serial_type >= 0x80 ){ + sqlite3GetVarint32(&aKey1[1], (u32*)&serial_type); + } if( serial_type<12 ){ res = pPKey2->r1; /* (pKey1/nKey1) is a number or a null */ }else if( !(serial_type & 0x01) ){ @@ -79267,7 +82343,11 @@ static int vdbeRecordCompareString( nCmp = MIN( pPKey2->aMem[0].n, nStr ); res = memcmp(&aKey1[szHdr], pPKey2->aMem[0].z, nCmp); - if( res==0 ){ + if( res>0 ){ + res = pPKey2->r2; + }else if( res<0 ){ + res = pPKey2->r1; + }else{ res = nStr - pPKey2->aMem[0].n; if( res==0 ){ if( pPKey2->nField>1 ){ @@ -79281,10 +82361,6 @@ static int vdbeRecordCompareString( }else{ res = pPKey2->r1; } - }else if( res>0 ){ - res = pPKey2->r2; - }else{ - res = pPKey2->r1; } } @@ -79316,7 +82392,10 @@ SQLITE_PRIVATE RecordCompare sqlite3VdbeFindCompare(UnpackedRecord *p){ ** header size is (12*5 + 1 + 1) bytes. */ if( p->pKeyInfo->nAllField<=13 ){ int flags = p->aMem[0].flags; - if( p->pKeyInfo->aSortOrder[0] ){ + if( p->pKeyInfo->aSortFlags[0] ){ + if( p->pKeyInfo->aSortFlags[0] & KEYINFO_ORDER_BIGNULL ){ + return sqlite3VdbeRecordCompare; + } p->r1 = 1; p->r2 = -1; }else{ @@ -79329,7 +82408,9 @@ SQLITE_PRIVATE RecordCompare sqlite3VdbeFindCompare(UnpackedRecord *p){ testcase( flags & MEM_Real ); testcase( flags & MEM_Null ); testcase( flags & MEM_Blob ); - if( (flags & (MEM_Real|MEM_Null|MEM_Blob))==0 && p->pKeyInfo->aColl[0]==0 ){ + if( (flags & (MEM_Real|MEM_IntReal|MEM_Null|MEM_Blob))==0 + && p->pKeyInfo->aColl[0]==0 + ){ assert( flags & MEM_Str ); return vdbeRecordCompareString; } @@ -79365,22 +82446,24 @@ SQLITE_PRIVATE int sqlite3VdbeIdxRowid(sqlite3 *db, BtCursor *pCur, i64 *rowid){ /* Read in the complete content of the index entry */ sqlite3VdbeMemInit(&m, db, 0); - rc = sqlite3VdbeMemFromBtree(pCur, 0, (u32)nCellKey, &m); + rc = sqlite3VdbeMemFromBtreeZeroOffset(pCur, (u32)nCellKey, &m); if( rc ){ return rc; } /* The index entry must begin with a header size */ - (void)getVarint32((u8*)m.z, szHdr); + getVarint32NR((u8*)m.z, szHdr); testcase( szHdr==3 ); testcase( szHdr==m.n ); - if( unlikely(szHdr<3 || (int)szHdr>m.n) ){ + testcase( szHdr>0x7fffffff ); + assert( m.n>=0 ); + if( unlikely(szHdr<3 || szHdr>(unsigned)m.n) ){ goto idx_rowid_corruption; } /* The last field of the index should be an integer - the ROWID. ** Verify that the last entry really is an integer. */ - (void)getVarint32((u8*)&m.z[szHdr-1], typeRowid); + getVarint32NR((u8*)&m.z[szHdr-1], typeRowid); testcase( typeRowid==1 ); testcase( typeRowid==2 ); testcase( typeRowid==3 ); @@ -79445,11 +82528,11 @@ SQLITE_PRIVATE int sqlite3VdbeIdxKeyCompare( return SQLITE_CORRUPT_BKPT; } sqlite3VdbeMemInit(&m, db, 0); - rc = sqlite3VdbeMemFromBtree(pCur, 0, (u32)nCellKey, &m); + rc = sqlite3VdbeMemFromBtreeZeroOffset(pCur, (u32)nCellKey, &m); if( rc ){ return rc; } - *res = sqlite3VdbeRecordCompare(m.n, m.z, pUnpacked); + *res = sqlite3VdbeRecordCompareWithSkip(m.n, m.z, pUnpacked, 0); sqlite3VdbeMemRelease(&m); return SQLITE_OK; } @@ -79481,11 +82564,19 @@ SQLITE_PRIVATE void sqlite3VdbeCountChanges(Vdbe *v){ ** programs obsolete. Removing user-defined functions or collating ** sequences, or changing an authorization function are the types of ** things that make prepared statements obsolete. +** +** If iCode is 1, then expiration is advisory. The statement should +** be reprepared before being restarted, but if it is already running +** it is allowed to run to completion. +** +** Internally, this function just sets the Vdbe.expired flag on all +** prepared statements. The flag is set to 1 for an immediate expiration +** and set to 2 for an advisory expiration. */ -SQLITE_PRIVATE void sqlite3ExpirePreparedStatements(sqlite3 *db){ +SQLITE_PRIVATE void sqlite3ExpirePreparedStatements(sqlite3 *db, int iCode){ Vdbe *p; for(p = db->pVdbe; p; p=p->pNext){ - p->expired = 1; + p->expired = iCode+1; } } @@ -79553,13 +82644,25 @@ SQLITE_PRIVATE void sqlite3VdbeSetVarmask(Vdbe *v, int iVar){ ** features such as 'now'. */ SQLITE_PRIVATE int sqlite3NotPureFunc(sqlite3_context *pCtx){ -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 + const VdbeOp *pOp; +#ifdef SQLITE_ENABLE_STAT4 if( pCtx->pVdbe==0 ) return 1; #endif - if( pCtx->pVdbe->aOp[pCtx->iOp].opcode==OP_PureFunc ){ - sqlite3_result_error(pCtx, - "non-deterministic function in index expression or CHECK constraint", - -1); + pOp = pCtx->pVdbe->aOp + pCtx->iOp; + if( pOp->opcode==OP_PureFunc ){ + const char *zContext; + char *zMsg; + if( pOp->p5 & NC_IsCheck ){ + zContext = "a CHECK constraint"; + }else if( pOp->p5 & NC_GenCol ){ + zContext = "a generated column"; + }else{ + zContext = "an index"; + } + zMsg = sqlite3_mprintf("non-deterministic use of %s() in %s", + pCtx->pFunc->zName, zContext); + sqlite3_result_error(pCtx, zMsg, -1); + sqlite3_free(zMsg); return 0; } return 1; @@ -79650,7 +82753,7 @@ SQLITE_PRIVATE void sqlite3VdbePreUpdateHook( preupdate.keyinfo.db = db; preupdate.keyinfo.enc = ENC(db); preupdate.keyinfo.nKeyField = pTab->nCol; - preupdate.keyinfo.aSortOrder = (u8*)&fakeSortOrder; + preupdate.keyinfo.aSortFlags = (u8*)&fakeSortOrder; preupdate.iKey1 = iKey1; preupdate.iKey2 = iKey2; preupdate.pTab = pTab; @@ -79737,14 +82840,16 @@ static SQLITE_NOINLINE void invokeProfileCallback(sqlite3 *db, Vdbe *p){ sqlite3_int64 iNow; sqlite3_int64 iElapse; assert( p->startTime>0 ); - assert( db->xProfile!=0 || (db->mTrace & SQLITE_TRACE_PROFILE)!=0 ); + assert( (db->mTrace & (SQLITE_TRACE_PROFILE|SQLITE_TRACE_XPROFILE))!=0 ); assert( db->init.busy==0 ); assert( p->zSql!=0 ); sqlite3OsCurrentTimeInt64(db->pVfs, &iNow); iElapse = (iNow - p->startTime)*1000000; +#ifndef SQLITE_OMIT_DEPRECATED if( db->xProfile ){ db->xProfile(db->pProfileArg, p->zSql, iElapse); } +#endif if( db->mTrace & SQLITE_TRACE_PROFILE ){ db->xTrace(SQLITE_TRACE_PROFILE, db->pTraceArg, p, (void*)&iElapse); } @@ -79907,39 +83012,86 @@ SQLITE_API const void *sqlite3_value_text16le(sqlite3_value *pVal){ */ SQLITE_API int sqlite3_value_type(sqlite3_value* pVal){ static const u8 aType[] = { - SQLITE_BLOB, /* 0x00 */ - SQLITE_NULL, /* 0x01 */ - SQLITE_TEXT, /* 0x02 */ - SQLITE_NULL, /* 0x03 */ - SQLITE_INTEGER, /* 0x04 */ - SQLITE_NULL, /* 0x05 */ - SQLITE_INTEGER, /* 0x06 */ - SQLITE_NULL, /* 0x07 */ - SQLITE_FLOAT, /* 0x08 */ - SQLITE_NULL, /* 0x09 */ - SQLITE_FLOAT, /* 0x0a */ - SQLITE_NULL, /* 0x0b */ - SQLITE_INTEGER, /* 0x0c */ - SQLITE_NULL, /* 0x0d */ - SQLITE_INTEGER, /* 0x0e */ - SQLITE_NULL, /* 0x0f */ - SQLITE_BLOB, /* 0x10 */ - SQLITE_NULL, /* 0x11 */ - SQLITE_TEXT, /* 0x12 */ - SQLITE_NULL, /* 0x13 */ - SQLITE_INTEGER, /* 0x14 */ - SQLITE_NULL, /* 0x15 */ - SQLITE_INTEGER, /* 0x16 */ - SQLITE_NULL, /* 0x17 */ - SQLITE_FLOAT, /* 0x18 */ - SQLITE_NULL, /* 0x19 */ - SQLITE_FLOAT, /* 0x1a */ - SQLITE_NULL, /* 0x1b */ - SQLITE_INTEGER, /* 0x1c */ - SQLITE_NULL, /* 0x1d */ - SQLITE_INTEGER, /* 0x1e */ - SQLITE_NULL, /* 0x1f */ + SQLITE_BLOB, /* 0x00 (not possible) */ + SQLITE_NULL, /* 0x01 NULL */ + SQLITE_TEXT, /* 0x02 TEXT */ + SQLITE_NULL, /* 0x03 (not possible) */ + SQLITE_INTEGER, /* 0x04 INTEGER */ + SQLITE_NULL, /* 0x05 (not possible) */ + SQLITE_INTEGER, /* 0x06 INTEGER + TEXT */ + SQLITE_NULL, /* 0x07 (not possible) */ + SQLITE_FLOAT, /* 0x08 FLOAT */ + SQLITE_NULL, /* 0x09 (not possible) */ + SQLITE_FLOAT, /* 0x0a FLOAT + TEXT */ + SQLITE_NULL, /* 0x0b (not possible) */ + SQLITE_INTEGER, /* 0x0c (not possible) */ + SQLITE_NULL, /* 0x0d (not possible) */ + SQLITE_INTEGER, /* 0x0e (not possible) */ + SQLITE_NULL, /* 0x0f (not possible) */ + SQLITE_BLOB, /* 0x10 BLOB */ + SQLITE_NULL, /* 0x11 (not possible) */ + SQLITE_TEXT, /* 0x12 (not possible) */ + SQLITE_NULL, /* 0x13 (not possible) */ + SQLITE_INTEGER, /* 0x14 INTEGER + BLOB */ + SQLITE_NULL, /* 0x15 (not possible) */ + SQLITE_INTEGER, /* 0x16 (not possible) */ + SQLITE_NULL, /* 0x17 (not possible) */ + SQLITE_FLOAT, /* 0x18 FLOAT + BLOB */ + SQLITE_NULL, /* 0x19 (not possible) */ + SQLITE_FLOAT, /* 0x1a (not possible) */ + SQLITE_NULL, /* 0x1b (not possible) */ + SQLITE_INTEGER, /* 0x1c (not possible) */ + SQLITE_NULL, /* 0x1d (not possible) */ + SQLITE_INTEGER, /* 0x1e (not possible) */ + SQLITE_NULL, /* 0x1f (not possible) */ + SQLITE_FLOAT, /* 0x20 INTREAL */ + SQLITE_NULL, /* 0x21 (not possible) */ + SQLITE_TEXT, /* 0x22 INTREAL + TEXT */ + SQLITE_NULL, /* 0x23 (not possible) */ + SQLITE_FLOAT, /* 0x24 (not possible) */ + SQLITE_NULL, /* 0x25 (not possible) */ + SQLITE_FLOAT, /* 0x26 (not possible) */ + SQLITE_NULL, /* 0x27 (not possible) */ + SQLITE_FLOAT, /* 0x28 (not possible) */ + SQLITE_NULL, /* 0x29 (not possible) */ + SQLITE_FLOAT, /* 0x2a (not possible) */ + SQLITE_NULL, /* 0x2b (not possible) */ + SQLITE_FLOAT, /* 0x2c (not possible) */ + SQLITE_NULL, /* 0x2d (not possible) */ + SQLITE_FLOAT, /* 0x2e (not possible) */ + SQLITE_NULL, /* 0x2f (not possible) */ + SQLITE_BLOB, /* 0x30 (not possible) */ + SQLITE_NULL, /* 0x31 (not possible) */ + SQLITE_TEXT, /* 0x32 (not possible) */ + SQLITE_NULL, /* 0x33 (not possible) */ + SQLITE_FLOAT, /* 0x34 (not possible) */ + SQLITE_NULL, /* 0x35 (not possible) */ + SQLITE_FLOAT, /* 0x36 (not possible) */ + SQLITE_NULL, /* 0x37 (not possible) */ + SQLITE_FLOAT, /* 0x38 (not possible) */ + SQLITE_NULL, /* 0x39 (not possible) */ + SQLITE_FLOAT, /* 0x3a (not possible) */ + SQLITE_NULL, /* 0x3b (not possible) */ + SQLITE_FLOAT, /* 0x3c (not possible) */ + SQLITE_NULL, /* 0x3d (not possible) */ + SQLITE_FLOAT, /* 0x3e (not possible) */ + SQLITE_NULL, /* 0x3f (not possible) */ }; +#ifdef SQLITE_DEBUG + { + int eType = SQLITE_BLOB; + if( pVal->flags & MEM_Null ){ + eType = SQLITE_NULL; + }else if( pVal->flags & (MEM_Real|MEM_IntReal) ){ + eType = SQLITE_FLOAT; + }else if( pVal->flags & MEM_Int ){ + eType = SQLITE_INTEGER; + }else if( pVal->flags & MEM_Str ){ + eType = SQLITE_TEXT; + } + assert( eType == aType[pVal->flags&MEM_AffMask] ); + } +#endif return aType[pVal->flags&MEM_AffMask]; } @@ -79948,6 +83100,11 @@ SQLITE_API int sqlite3_value_nochange(sqlite3_value *pVal){ return (pVal->flags&(MEM_Null|MEM_Zero))==(MEM_Null|MEM_Zero); } +/* Return true if a parameter value originated from an sqlite3_bind() */ +SQLITE_API int sqlite3_value_frombind(sqlite3_value *pVal){ + return (pVal->flags&MEM_FromBind)!=0; +} + /* Make a copy of an sqlite3_value object */ SQLITE_API sqlite3_value *sqlite3_value_dup(const sqlite3_value *pOrig){ @@ -80184,6 +83341,21 @@ SQLITE_API void sqlite3_result_error_nomem(sqlite3_context *pCtx){ sqlite3OomFault(pCtx->pOut->db); } +#ifndef SQLITE_UNTESTABLE +/* Force the INT64 value currently stored as the result to be +** a MEM_IntReal value. See the SQLITE_TESTCTRL_RESULT_INTREAL +** test-control. +*/ +SQLITE_PRIVATE void sqlite3ResultIntReal(sqlite3_context *pCtx){ + assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); + if( pCtx->pOut->flags & MEM_Int ){ + pCtx->pOut->flags &= ~MEM_Int; + pCtx->pOut->flags |= MEM_IntReal; + } +} +#endif + + /* ** This function is called after a transaction has been committed. It ** invokes callbacks registered with sqlite3_wal_hook() as required. @@ -80258,7 +83430,7 @@ static int sqlite3Step(Vdbe *p){ return SQLITE_NOMEM_BKPT; } - if( p->pc<=0 && p->expired ){ + if( p->pc<0 && p->expired ){ p->rc = SQLITE_SCHEMA; rc = SQLITE_ERROR; goto end_of_step; @@ -80269,7 +83441,7 @@ static int sqlite3Step(Vdbe *p){ ** from interrupting a statement that has not yet started. */ if( db->nVdbeActive==0 ){ - db->u1.isInterrupted = 0; + AtomicStore(&db->u1.isInterrupted, 0); } assert( db->nVdbeWrite>0 || db->autoCommit==0 @@ -80277,7 +83449,7 @@ static int sqlite3Step(Vdbe *p){ ); #ifndef SQLITE_OMIT_TRACE - if( (db->xProfile || (db->mTrace & SQLITE_TRACE_PROFILE)!=0) + if( (db->mTrace & (SQLITE_TRACE_PROFILE|SQLITE_TRACE_XPROFILE))!=0 && !db->init.busy && p->zSql ){ sqlite3OsCurrentTimeInt64(db->pVfs, &p->startTime); }else{ @@ -80304,16 +83476,18 @@ static int sqlite3Step(Vdbe *p){ db->nVdbeExec--; } + if( rc!=SQLITE_ROW ){ #ifndef SQLITE_OMIT_TRACE - /* If the statement completed successfully, invoke the profile callback */ - if( rc!=SQLITE_ROW ) checkProfileCallback(db, p); + /* If the statement completed successfully, invoke the profile callback */ + checkProfileCallback(db, p); #endif - if( rc==SQLITE_DONE && db->autoCommit ){ - assert( p->rc==SQLITE_OK ); - p->rc = doWalCallbacks(db); - if( p->rc!=SQLITE_OK ){ - rc = SQLITE_ERROR; + if( rc==SQLITE_DONE && db->autoCommit ){ + assert( p->rc==SQLITE_OK ); + p->rc = doWalCallbacks(db); + if( p->rc!=SQLITE_OK ){ + rc = SQLITE_ERROR; + } } } @@ -80333,9 +83507,9 @@ static int sqlite3Step(Vdbe *p){ || (rc&0xff)==SQLITE_BUSY || rc==SQLITE_MISUSE ); assert( (p->rc!=SQLITE_ROW && p->rc!=SQLITE_DONE) || p->rc==p->rcApp ); - if( (p->prepFlags & SQLITE_PREPARE_SAVESQL)!=0 - && rc!=SQLITE_ROW - && rc!=SQLITE_DONE + if( rc!=SQLITE_ROW + && rc!=SQLITE_DONE + && (p->prepFlags & SQLITE_PREPARE_SAVESQL)!=0 ){ /* If this statement was prepared using saved SQL and an ** error has occurred, then return the error code in p->rc to the @@ -80448,7 +83622,7 @@ SQLITE_API int sqlite3_vtab_nochange(sqlite3_context *p){ */ SQLITE_PRIVATE sqlite3_int64 sqlite3StmtCurrentTime(sqlite3_context *p){ int rc; -#ifndef SQLITE_ENABLE_STAT3_OR_STAT4 +#ifndef SQLITE_ENABLE_STAT4 sqlite3_int64 *piTime = &p->pVdbe->iCurrentTime; assert( p->pVdbe!=0 ); #else @@ -80513,7 +83687,7 @@ SQLITE_API void *sqlite3_get_auxdata(sqlite3_context *pCtx, int iArg){ AuxData *pAuxData; assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); -#if SQLITE_ENABLE_STAT3_OR_STAT4 +#if SQLITE_ENABLE_STAT4 if( pCtx->pVdbe==0 ) return 0; #else assert( pCtx->pVdbe!=0 ); @@ -80547,7 +83721,7 @@ SQLITE_API void sqlite3_set_auxdata( Vdbe *pVdbe = pCtx->pVdbe; assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +#ifdef SQLITE_ENABLE_STAT4 if( pVdbe==0 ) goto failed; #else assert( pVdbe!=0 ); @@ -80645,7 +83819,7 @@ static const Mem *columnNullValue(void){ /* .xDel = */ (void(*)(void*))0, #ifdef SQLITE_DEBUG /* .pScopyFrom = */ (Mem*)0, - /* .pFiller = */ (void*)0, + /* .mScopyFlags= */ 0, #endif }; return &nullMem; @@ -80791,10 +83965,10 @@ SQLITE_API int sqlite3_column_type(sqlite3_stmt *pStmt, int i){ ** or a constant) then useTypes 2, 3, and 4 return NULL. */ static const void *columnName( - sqlite3_stmt *pStmt, - int N, - const void *(*xFunc)(Mem*), - int useType + sqlite3_stmt *pStmt, /* The statement */ + int N, /* Which column to get the name for */ + int useUtf16, /* True to return the name as UTF16 */ + int useType /* What type of name */ ){ const void *ret; Vdbe *p; @@ -80815,8 +83989,15 @@ static const void *columnName( N += useType*n; sqlite3_mutex_enter(db->mutex); assert( db->mallocFailed==0 ); - ret = xFunc(&p->aColName[N]); - /* A malloc may have failed inside of the xFunc() call. If this +#ifndef SQLITE_OMIT_UTF16 + if( useUtf16 ){ + ret = sqlite3_value_text16((sqlite3_value*)&p->aColName[N]); + }else +#endif + { + ret = sqlite3_value_text((sqlite3_value*)&p->aColName[N]); + } + /* A malloc may have failed inside of the _text() call. If this ** is the case, clear the mallocFailed flag and return NULL. */ if( db->mallocFailed ){ @@ -80833,13 +84014,11 @@ static const void *columnName( ** statement pStmt. */ SQLITE_API const char *sqlite3_column_name(sqlite3_stmt *pStmt, int N){ - return columnName( - pStmt, N, (const void*(*)(Mem*))sqlite3_value_text, COLNAME_NAME); + return columnName(pStmt, N, 0, COLNAME_NAME); } #ifndef SQLITE_OMIT_UTF16 SQLITE_API const void *sqlite3_column_name16(sqlite3_stmt *pStmt, int N){ - return columnName( - pStmt, N, (const void*(*)(Mem*))sqlite3_value_text16, COLNAME_NAME); + return columnName(pStmt, N, 1, COLNAME_NAME); } #endif @@ -80858,13 +84037,11 @@ SQLITE_API const void *sqlite3_column_name16(sqlite3_stmt *pStmt, int N){ ** of the result set of SQL statement pStmt. */ SQLITE_API const char *sqlite3_column_decltype(sqlite3_stmt *pStmt, int N){ - return columnName( - pStmt, N, (const void*(*)(Mem*))sqlite3_value_text, COLNAME_DECLTYPE); + return columnName(pStmt, N, 0, COLNAME_DECLTYPE); } #ifndef SQLITE_OMIT_UTF16 SQLITE_API const void *sqlite3_column_decltype16(sqlite3_stmt *pStmt, int N){ - return columnName( - pStmt, N, (const void*(*)(Mem*))sqlite3_value_text16, COLNAME_DECLTYPE); + return columnName(pStmt, N, 1, COLNAME_DECLTYPE); } #endif /* SQLITE_OMIT_UTF16 */ #endif /* SQLITE_OMIT_DECLTYPE */ @@ -80876,13 +84053,11 @@ SQLITE_API const void *sqlite3_column_decltype16(sqlite3_stmt *pStmt, int N){ ** anything else which is not an unambiguous reference to a database column. */ SQLITE_API const char *sqlite3_column_database_name(sqlite3_stmt *pStmt, int N){ - return columnName( - pStmt, N, (const void*(*)(Mem*))sqlite3_value_text, COLNAME_DATABASE); + return columnName(pStmt, N, 0, COLNAME_DATABASE); } #ifndef SQLITE_OMIT_UTF16 SQLITE_API const void *sqlite3_column_database_name16(sqlite3_stmt *pStmt, int N){ - return columnName( - pStmt, N, (const void*(*)(Mem*))sqlite3_value_text16, COLNAME_DATABASE); + return columnName(pStmt, N, 1, COLNAME_DATABASE); } #endif /* SQLITE_OMIT_UTF16 */ @@ -80892,13 +84067,11 @@ SQLITE_API const void *sqlite3_column_database_name16(sqlite3_stmt *pStmt, int N ** anything else which is not an unambiguous reference to a database column. */ SQLITE_API const char *sqlite3_column_table_name(sqlite3_stmt *pStmt, int N){ - return columnName( - pStmt, N, (const void*(*)(Mem*))sqlite3_value_text, COLNAME_TABLE); + return columnName(pStmt, N, 0, COLNAME_TABLE); } #ifndef SQLITE_OMIT_UTF16 SQLITE_API const void *sqlite3_column_table_name16(sqlite3_stmt *pStmt, int N){ - return columnName( - pStmt, N, (const void*(*)(Mem*))sqlite3_value_text16, COLNAME_TABLE); + return columnName(pStmt, N, 1, COLNAME_TABLE); } #endif /* SQLITE_OMIT_UTF16 */ @@ -80908,13 +84081,11 @@ SQLITE_API const void *sqlite3_column_table_name16(sqlite3_stmt *pStmt, int N){ ** anything else which is not an unambiguous reference to a database column. */ SQLITE_API const char *sqlite3_column_origin_name(sqlite3_stmt *pStmt, int N){ - return columnName( - pStmt, N, (const void*(*)(Mem*))sqlite3_value_text, COLNAME_COLUMN); + return columnName(pStmt, N, 0, COLNAME_COLUMN); } #ifndef SQLITE_OMIT_UTF16 SQLITE_API const void *sqlite3_column_origin_name16(sqlite3_stmt *pStmt, int N){ - return columnName( - pStmt, N, (const void*(*)(Mem*))sqlite3_value_text16, COLNAME_COLUMN); + return columnName(pStmt, N, 1, COLNAME_COLUMN); } #endif /* SQLITE_OMIT_UTF16 */ #endif /* SQLITE_ENABLE_COLUMN_METADATA */ @@ -80957,12 +84128,12 @@ static int vdbeUnbind(Vdbe *p, int i){ pVar = &p->aVar[i]; sqlite3VdbeMemRelease(pVar); pVar->flags = MEM_Null; - sqlite3Error(p->db, SQLITE_OK); + p->db->errCode = SQLITE_OK; /* If the bit corresponding to this variable in Vdbe.expmask is set, then ** binding a new value to this variable invalidates the current query plan. ** - ** IMPLEMENTATION-OF: R-48440-37595 If the specific value bound to host + ** IMPLEMENTATION-OF: R-57496-20354 If the specific value bound to a host ** parameter in the WHERE clause might influence the choice of query plan ** for a statement, then the statement will be automatically recompiled, ** as if there had been a schema change, on the first sqlite3_step() call @@ -81282,6 +84453,14 @@ SQLITE_API int sqlite3_stmt_readonly(sqlite3_stmt *pStmt){ return pStmt ? ((Vdbe*)pStmt)->readOnly : 1; } +/* +** Return 1 if the statement is an EXPLAIN and return 2 if the +** statement is an EXPLAIN QUERY PLAN +*/ +SQLITE_API int sqlite3_stmt_isexplain(sqlite3_stmt *pStmt){ + return pStmt ? ((Vdbe*)pStmt)->explain : 0; +} + /* ** Return true if the prepared statement is in need of being reset. */ @@ -81377,6 +84556,22 @@ SQLITE_API char *sqlite3_expanded_sql(sqlite3_stmt *pStmt){ #endif } +#ifdef SQLITE_ENABLE_NORMALIZE +/* +** Return the normalized SQL associated with a prepared statement. +*/ +SQLITE_API const char *sqlite3_normalized_sql(sqlite3_stmt *pStmt){ + Vdbe *p = (Vdbe *)pStmt; + if( p==0 ) return 0; + if( p->zNormSql==0 && ALWAYS(p->zSql!=0) ){ + sqlite3_mutex_enter(p->db->mutex); + p->zNormSql = sqlite3Normalize(p, p->zSql); + sqlite3_mutex_leave(p->db->mutex); + } + return p->zNormSql; +} +#endif /* SQLITE_ENABLE_NORMALIZE */ + #ifdef SQLITE_ENABLE_PREUPDATE_HOOK /* ** Allocate and populate an UnpackedRecord structure based on the serialized @@ -81414,7 +84609,7 @@ SQLITE_API int sqlite3_preupdate_old(sqlite3 *db, int iIdx, sqlite3_value **ppVa goto preupdate_old_out; } if( p->pPk ){ - iIdx = sqlite3ColumnOfIndex(p->pPk, iIdx); + iIdx = sqlite3TableColumnToIndex(p->pPk, iIdx); } if( iIdx>=p->pCsr->nField || iIdx<0 ){ rc = SQLITE_RANGE; @@ -81447,7 +84642,9 @@ SQLITE_API int sqlite3_preupdate_old(sqlite3 *db, int iIdx, sqlite3_value **ppVa }else if( iIdx>=p->pUnpacked->nField ){ *ppValue = (sqlite3_value *)columnNullValue(); }else if( p->pTab->aCol[iIdx].affinity==SQLITE_AFF_REAL ){ - if( pMem->flags & MEM_Int ){ + if( pMem->flags & (MEM_Int|MEM_IntReal) ){ + testcase( pMem->flags & MEM_Int ); + testcase( pMem->flags & MEM_IntReal ); sqlite3VdbeMemRealify(pMem); } } @@ -81502,7 +84699,7 @@ SQLITE_API int sqlite3_preupdate_new(sqlite3 *db, int iIdx, sqlite3_value **ppVa goto preupdate_new_out; } if( p->pPk && p->op!=SQLITE_UPDATE ){ - iIdx = sqlite3ColumnOfIndex(p->pPk, iIdx); + iIdx = sqlite3TableColumnToIndex(p->pPk, iIdx); } if( iIdx>=p->pCsr->nField || iIdx<0 ){ rc = SQLITE_RANGE; @@ -81766,7 +84963,7 @@ SQLITE_PRIVATE char *sqlite3VdbeExpandSql( pVar = &p->aVar[idx-1]; if( pVar->flags & MEM_Null ){ sqlite3_str_append(&out, "NULL", 4); - }else if( pVar->flags & MEM_Int ){ + }else if( pVar->flags & (MEM_Int|MEM_IntReal) ){ sqlite3_str_appendf(&out, "%lld", pVar->u.i); }else if( pVar->flags & MEM_Real ){ sqlite3_str_appendf(&out, "%!.15g", pVar->u.r); @@ -81950,48 +85147,104 @@ SQLITE_API int sqlite3_found_count = 0; # define UPDATE_MAX_BLOBSIZE(P) #endif +#ifdef SQLITE_DEBUG +/* This routine provides a convenient place to set a breakpoint during +** tracing with PRAGMA vdbe_trace=on. The breakpoint fires right after +** each opcode is printed. Variables "pc" (program counter) and pOp are +** available to add conditionals to the breakpoint. GDB example: +** +** break test_trace_breakpoint if pc=22 +** +** Other useful labels for breakpoints include: +** test_addop_breakpoint(pc,pOp) +** sqlite3CorruptError(lineno) +** sqlite3MisuseError(lineno) +** sqlite3CantopenError(lineno) +*/ +static void test_trace_breakpoint(int pc, Op *pOp, Vdbe *v){ + static int n = 0; + n++; +} +#endif + /* ** Invoke the VDBE coverage callback, if that callback is defined. This ** feature is used for test suite validation only and does not appear an ** production builds. ** -** M is an integer, 2 or 3, that indices how many different ways the -** branch can go. It is usually 2. "I" is the direction the branch -** goes. 0 means falls through. 1 means branch is taken. 2 means the -** second alternative branch is taken. +** M is the type of branch. I is the direction taken for this instance of +** the branch. +** +** M: 2 - two-way branch (I=0: fall-thru 1: jump ) +** 3 - two-way + NULL (I=0: fall-thru 1: jump 2: NULL ) +** 4 - OP_Jump (I=0: jump p1 1: jump p2 2: jump p3) +** +** In other words, if M is 2, then I is either 0 (for fall-through) or +** 1 (for when the branch is taken). If M is 3, the I is 0 for an +** ordinary fall-through, I is 1 if the branch was taken, and I is 2 +** if the result of comparison is NULL. For M=3, I=2 the jump may or +** may not be taken, depending on the SQLITE_JUMPIFNULL flags in p5. +** When M is 4, that means that an OP_Jump is being run. I is 0, 1, or 2 +** depending on if the operands are less than, equal, or greater than. ** ** iSrcLine is the source code line (from the __LINE__ macro) that -** generated the VDBE instruction. This instrumentation assumes that all -** source code is in a single file (the amalgamation). Special values 1 -** and 2 for the iSrcLine parameter mean that this particular branch is -** always taken or never taken, respectively. +** generated the VDBE instruction combined with flag bits. The source +** code line number is in the lower 24 bits of iSrcLine and the upper +** 8 bytes are flags. The lower three bits of the flags indicate +** values for I that should never occur. For example, if the branch is +** always taken, the flags should be 0x05 since the fall-through and +** alternate branch are never taken. If a branch is never taken then +** flags should be 0x06 since only the fall-through approach is allowed. +** +** Bit 0x08 of the flags indicates an OP_Jump opcode that is only +** interested in equal or not-equal. In other words, I==0 and I==2 +** should be treated as equivalent +** +** Since only a line number is retained, not the filename, this macro +** only works for amalgamation builds. But that is ok, since these macros +** should be no-ops except for special builds used to measure test coverage. */ #if !defined(SQLITE_VDBE_COVERAGE) # define VdbeBranchTaken(I,M) #else # define VdbeBranchTaken(I,M) vdbeTakeBranch(pOp->iSrcLine,I,M) - static void vdbeTakeBranch(int iSrcLine, u8 I, u8 M){ - if( iSrcLine<=2 && ALWAYS(iSrcLine>0) ){ - M = iSrcLine; - /* Assert the truth of VdbeCoverageAlwaysTaken() and - ** VdbeCoverageNeverTaken() */ - assert( (M & I)==I ); - }else{ - if( sqlite3GlobalConfig.xVdbeBranch==0 ) return; /*NO_TEST*/ - sqlite3GlobalConfig.xVdbeBranch(sqlite3GlobalConfig.pVdbeBranchArg, - iSrcLine,I,M); + static void vdbeTakeBranch(u32 iSrcLine, u8 I, u8 M){ + u8 mNever; + assert( I<=2 ); /* 0: fall through, 1: taken, 2: alternate taken */ + assert( M<=4 ); /* 2: two-way branch, 3: three-way branch, 4: OP_Jump */ + assert( I> 24; + assert( (I & mNever)==0 ); + if( sqlite3GlobalConfig.xVdbeBranch==0 ) return; /*NO_TEST*/ + /* Invoke the branch coverage callback with three arguments: + ** iSrcLine - the line number of the VdbeCoverage() macro, with + ** flags removed. + ** I - Mask of bits 0x07 indicating which cases are are + ** fulfilled by this instance of the jump. 0x01 means + ** fall-thru, 0x02 means taken, 0x04 means NULL. Any + ** impossible cases (ex: if the comparison is never NULL) + ** are filled in automatically so that the coverage + ** measurement logic does not flag those impossible cases + ** as missed coverage. + ** M - Type of jump. Same as M argument above + */ + I |= mNever; + if( M==2 ) I |= 0x04; + if( M==4 ){ + I |= 0x08; + if( (mNever&0x08)!=0 && (I&0x05)!=0) I |= 0x05; /*NO_TEST*/ } + sqlite3GlobalConfig.xVdbeBranch(sqlite3GlobalConfig.pVdbeBranchArg, + iSrcLine&0xffffff, I, M); } #endif -/* -** Convert the given register into a string if it isn't one -** already. Return non-zero if a malloc() fails. -*/ -#define Stringify(P, enc) \ - if(((P)->flags&(MEM_Str|MEM_Blob))==0 && sqlite3VdbeMemStringify(P,enc,0)) \ - { goto no_mem; } - /* ** An ephemeral string value (signified by the MEM_Ephem flag) contains ** a pointer to a dynamically allocated string where some other entity @@ -82049,6 +85302,11 @@ static VdbeCursor *allocateCursor( assert( iCur>=0 && iCurnCursor ); if( p->apCsr[iCur] ){ /*OPTIMIZATION-IF-FALSE*/ + /* Before calling sqlite3VdbeFreeCursor(), ensure the isEphemeral flag + ** is clear. Otherwise, if this is an ephemeral cursor created by + ** OP_OpenDup, the cursor will not be closed and will still be part + ** of a BtShared.pCursor list. */ + if( p->apCsr[iCur]->pBtx==0 ) p->apCsr[iCur]->isEphemeral = 0; sqlite3VdbeFreeCursor(p, p->apCsr[iCur]); p->apCsr[iCur] = 0; } @@ -82068,6 +85326,21 @@ static VdbeCursor *allocateCursor( return pCx; } +/* +** The string in pRec is known to look like an integer and to have a +** floating point value of rValue. Return true and set *piValue to the +** integer value if the string is in range to be an integer. Otherwise, +** return false. +*/ +static int alsoAnInt(Mem *pRec, double rValue, i64 *piValue){ + i64 iValue = (double)rValue; + if( sqlite3RealSameAsInt(rValue,iValue) ){ + *piValue = iValue; + return 1; + } + return 0==sqlite3Atoi64(pRec->z, piValue, pRec->n, pRec->enc); +} + /* ** Try to convert a value into a numeric representation if we can ** do so without loss of information. In other words, if the string @@ -82085,12 +85358,12 @@ static VdbeCursor *allocateCursor( */ static void applyNumericAffinity(Mem *pRec, int bTryForInt){ double rValue; - i64 iValue; u8 enc = pRec->enc; - assert( (pRec->flags & (MEM_Str|MEM_Int|MEM_Real))==MEM_Str ); - if( sqlite3AtoF(pRec->z, &rValue, pRec->n, enc)==0 ) return; - if( 0==sqlite3Atoi64(pRec->z, &iValue, pRec->n, enc) ){ - pRec->u.i = iValue; + int rc; + assert( (pRec->flags & (MEM_Str|MEM_Int|MEM_Real|MEM_IntReal))==MEM_Str ); + rc = sqlite3AtoF(pRec->z, &rValue, pRec->n, enc); + if( rc<=0 ) return; + if( rc==1 && alsoAnInt(pRec, rValue, &pRec->u.i) ){ pRec->flags |= MEM_Int; }else{ pRec->u.r = rValue; @@ -82120,6 +85393,7 @@ static void applyNumericAffinity(Mem *pRec, int bTryForInt){ ** Convert pRec to a text representation. ** ** SQLITE_AFF_BLOB: +** SQLITE_AFF_NONE: ** No-op. pRec is unchanged. */ static void applyAffinity( @@ -82144,11 +85418,14 @@ static void applyAffinity( ** there is already a string rep, but it is pointless to waste those ** CPU cycles. */ if( 0==(pRec->flags&MEM_Str) ){ /*OPTIMIZATION-IF-FALSE*/ - if( (pRec->flags&(MEM_Real|MEM_Int)) ){ + if( (pRec->flags&(MEM_Real|MEM_Int|MEM_IntReal)) ){ + testcase( pRec->flags & MEM_Int ); + testcase( pRec->flags & MEM_Real ); + testcase( pRec->flags & MEM_IntReal ); sqlite3VdbeMemStringify(pRec, enc, 1); } } - pRec->flags &= ~(MEM_Real|MEM_Int); + pRec->flags &= ~(MEM_Real|MEM_Int|MEM_IntReal); } } @@ -82187,12 +85464,21 @@ SQLITE_PRIVATE void sqlite3ValueApplyAffinity( ** accordingly. */ static u16 SQLITE_NOINLINE computeNumericType(Mem *pMem){ - assert( (pMem->flags & (MEM_Int|MEM_Real))==0 ); + int rc; + sqlite3_int64 ix; + assert( (pMem->flags & (MEM_Int|MEM_Real|MEM_IntReal))==0 ); assert( (pMem->flags & (MEM_Str|MEM_Blob))!=0 ); - if( sqlite3AtoF(pMem->z, &pMem->u.r, pMem->n, pMem->enc)==0 ){ - return 0; - } - if( sqlite3Atoi64(pMem->z, &pMem->u.i, pMem->n, pMem->enc)==0 ){ + ExpandBlob(pMem); + rc = sqlite3AtoF(pMem->z, &pMem->u.r, pMem->n, pMem->enc); + if( rc<=0 ){ + if( rc==0 && sqlite3Atoi64(pMem->z, &ix, pMem->n, pMem->enc)<=1 ){ + pMem->u.i = ix; + return MEM_Int; + }else{ + return MEM_Real; + } + }else if( rc==1 && sqlite3Atoi64(pMem->z, &ix, pMem->n, pMem->enc)==0 ){ + pMem->u.i = ix; return MEM_Int; } return MEM_Real; @@ -82206,10 +85492,15 @@ static u16 SQLITE_NOINLINE computeNumericType(Mem *pMem){ ** But it does set pMem->u.r and pMem->u.i appropriately. */ static u16 numericType(Mem *pMem){ - if( pMem->flags & (MEM_Int|MEM_Real) ){ - return pMem->flags & (MEM_Int|MEM_Real); + if( pMem->flags & (MEM_Int|MEM_Real|MEM_IntReal) ){ + testcase( pMem->flags & MEM_Int ); + testcase( pMem->flags & MEM_Real ); + testcase( pMem->flags & MEM_IntReal ); + return pMem->flags & (MEM_Int|MEM_Real|MEM_IntReal); } if( pMem->flags & (MEM_Str|MEM_Blob) ){ + testcase( pMem->flags & MEM_Str ); + testcase( pMem->flags & MEM_Blob ); return computeNumericType(pMem); } return 0; @@ -82220,12 +85511,9 @@ static u16 numericType(Mem *pMem){ ** Write a nice string representation of the contents of cell pMem ** into buffer zBuf, length nBuf. */ -SQLITE_PRIVATE void sqlite3VdbeMemPrettyPrint(Mem *pMem, char *zBuf){ - char *zCsr = zBuf; +SQLITE_PRIVATE void sqlite3VdbeMemPrettyPrint(Mem *pMem, StrAccum *pStr){ int f = pMem->flags; - static const char *const encnames[] = {"(X)", "(8)", "(16LE)", "(16BE)"}; - if( f&MEM_Blob ){ int i; char c; @@ -82241,55 +85529,40 @@ SQLITE_PRIVATE void sqlite3VdbeMemPrettyPrint(Mem *pMem, char *zBuf){ }else{ c = 's'; } - *(zCsr++) = c; - sqlite3_snprintf(100, zCsr, "%d[", pMem->n); - zCsr += sqlite3Strlen30(zCsr); - for(i=0; i<16 && in; i++){ - sqlite3_snprintf(100, zCsr, "%02X", ((int)pMem->z[i] & 0xFF)); - zCsr += sqlite3Strlen30(zCsr); + sqlite3_str_appendf(pStr, "%cx[", c); + for(i=0; i<25 && in; i++){ + sqlite3_str_appendf(pStr, "%02X", ((int)pMem->z[i] & 0xFF)); } - for(i=0; i<16 && in; i++){ + sqlite3_str_appendf(pStr, "|"); + for(i=0; i<25 && in; i++){ char z = pMem->z[i]; - if( z<32 || z>126 ) *zCsr++ = '.'; - else *zCsr++ = z; + sqlite3_str_appendchar(pStr, 1, (z<32||z>126)?'.':z); } - *(zCsr++) = ']'; + sqlite3_str_appendf(pStr,"]"); if( f & MEM_Zero ){ - sqlite3_snprintf(100, zCsr,"+%dz",pMem->u.nZero); - zCsr += sqlite3Strlen30(zCsr); + sqlite3_str_appendf(pStr, "+%dz",pMem->u.nZero); } - *zCsr = '\0'; }else if( f & MEM_Str ){ - int j, k; - zBuf[0] = ' '; + int j; + u8 c; if( f & MEM_Dyn ){ - zBuf[1] = 'z'; + c = 'z'; assert( (f & (MEM_Static|MEM_Ephem))==0 ); }else if( f & MEM_Static ){ - zBuf[1] = 't'; + c = 't'; assert( (f & (MEM_Dyn|MEM_Ephem))==0 ); }else if( f & MEM_Ephem ){ - zBuf[1] = 'e'; + c = 'e'; assert( (f & (MEM_Static|MEM_Dyn))==0 ); }else{ - zBuf[1] = 's'; + c = 's'; } - k = 2; - sqlite3_snprintf(100, &zBuf[k], "%d", pMem->n); - k += sqlite3Strlen30(&zBuf[k]); - zBuf[k++] = '['; - for(j=0; j<15 && jn; j++){ - u8 c = pMem->z[j]; - if( c>=0x20 && c<0x7f ){ - zBuf[k++] = c; - }else{ - zBuf[k++] = '.'; - } + sqlite3_str_appendf(pStr, " %c%d[", c, pMem->n); + for(j=0; j<25 && jn; j++){ + c = pMem->z[j]; + sqlite3_str_appendchar(pStr, 1, (c>=0x20&&c<=0x7f) ? c : '.'); } - zBuf[k++] = ']'; - sqlite3_snprintf(100,&zBuf[k], encnames[pMem->enc]); - k += sqlite3Strlen30(&zBuf[k]); - zBuf[k++] = 0; + sqlite3_str_appendf(pStr, "]%s", encnames[pMem->enc]); } } #endif @@ -82305,29 +85578,48 @@ static void memTracePrint(Mem *p){ printf(p->flags & MEM_Zero ? " NULL-nochng" : " NULL"); }else if( (p->flags & (MEM_Int|MEM_Str))==(MEM_Int|MEM_Str) ){ printf(" si:%lld", p->u.i); + }else if( (p->flags & (MEM_IntReal))!=0 ){ + printf(" ir:%lld", p->u.i); }else if( p->flags & MEM_Int ){ printf(" i:%lld", p->u.i); #ifndef SQLITE_OMIT_FLOATING_POINT }else if( p->flags & MEM_Real ){ - printf(" r:%g", p->u.r); + printf(" r:%.17g", p->u.r); #endif - }else if( p->flags & MEM_RowSet ){ + }else if( sqlite3VdbeMemIsRowSet(p) ){ printf(" (rowset)"); }else{ - char zBuf[200]; - sqlite3VdbeMemPrettyPrint(p, zBuf); - printf(" %s", zBuf); + StrAccum acc; + char zBuf[1000]; + sqlite3StrAccumInit(&acc, 0, zBuf, sizeof(zBuf), 0); + sqlite3VdbeMemPrettyPrint(p, &acc); + printf(" %s", sqlite3StrAccumFinish(&acc)); } if( p->flags & MEM_Subtype ) printf(" subtype=0x%02x", p->eSubtype); } static void registerTrace(int iReg, Mem *p){ - printf("REG[%d] = ", iReg); + printf("R[%d] = ", iReg); memTracePrint(p); + if( p->pScopyFrom ){ + printf(" <== R[%d]", (int)(p->pScopyFrom - &p[-iReg])); + } printf("\n"); sqlite3VdbeCheckMemInvariants(p); } #endif +#ifdef SQLITE_DEBUG +/* +** Show the values of all registers in the virtual machine. Used for +** interactive debugging. +*/ +SQLITE_PRIVATE void sqlite3VdbeRegisterDump(Vdbe *v){ + int i; + for(i=1; inMem; i++) registerTrace(i, v->aMem+i); +} +#endif /* SQLITE_DEBUG */ + + #ifdef SQLITE_DEBUG # define REGISTER_TRACE(R,M) if(db->flags&SQLITE_VdbeTrace)registerTrace(R,M) #else @@ -82356,7 +85648,7 @@ static void registerTrace(int iReg, Mem *p){ ****************************************************************************** ** ** This file contains inline asm code for retrieving "high-performance" -** counters for x86 class CPUs. +** counters for x86 and x86_64 class CPUs. */ #ifndef SQLITE_HWTIME_H #define SQLITE_HWTIME_H @@ -82367,8 +85659,9 @@ static void registerTrace(int iReg, Mem *p){ ** processor and returns that value. This can be used for high-res ** profiling. */ -#if (defined(__GNUC__) || defined(_MSC_VER)) && \ - (defined(i386) || defined(__i386__) || defined(_M_IX86)) +#if !defined(__STRICT_ANSI__) && \ + (defined(__GNUC__) || defined(_MSC_VER)) && \ + (defined(i386) || defined(__i386__) || defined(_M_IX86)) #if defined(__GNUC__) @@ -82389,7 +85682,7 @@ static void registerTrace(int iReg, Mem *p){ #endif -#elif (defined(__GNUC__) && defined(__x86_64__)) +#elif !defined(__STRICT_ANSI__) && (defined(__GNUC__) && defined(__x86_64__)) __inline__ sqlite_uint64 sqlite3Hwtime(void){ unsigned long val; @@ -82397,7 +85690,7 @@ static void registerTrace(int iReg, Mem *p){ return val; } -#elif (defined(__GNUC__) && defined(__ppc__)) +#elif !defined(__STRICT_ANSI__) && (defined(__GNUC__) && defined(__ppc__)) __inline__ sqlite_uint64 sqlite3Hwtime(void){ unsigned long long retval; @@ -82414,14 +85707,13 @@ static void registerTrace(int iReg, Mem *p){ #else - #error Need implementation of sqlite3Hwtime() for your platform. - /* - ** To compile without implementing sqlite3Hwtime() for your platform, - ** you can remove the above #error and use the following - ** stub function. You will lose timing support for many - ** of the debugging and testing utilities, but it should at - ** least compile and run. + ** asm() is needed for hardware timing support. Without asm(), + ** disable the sqlite3Hwtime() routine. + ** + ** sqlite3Hwtime() is only used for some obscure debugging + ** and analysis configurations, not in any deliverable, so this + ** should not be a great loss. */ SQLITE_PRIVATE sqlite_uint64 sqlite3Hwtime(void){ return ((sqlite_uint64)0); } @@ -82514,28 +85806,30 @@ SQLITE_PRIVATE int sqlite3VdbeExec( assert( p->magic==VDBE_MAGIC_RUN ); /* sqlite3_step() verifies this */ sqlite3VdbeEnter(p); +#ifndef SQLITE_OMIT_PROGRESS_CALLBACK + if( db->xProgress ){ + u32 iPrior = p->aCounter[SQLITE_STMTSTATUS_VM_STEP]; + assert( 0 < db->nProgressOps ); + nProgressLimit = db->nProgressOps - (iPrior % db->nProgressOps); + }else{ + nProgressLimit = 0xffffffff; + } +#endif if( p->rc==SQLITE_NOMEM ){ /* This happens if a malloc() inside a call to sqlite3_column_text() or ** sqlite3_column_text16() failed. */ goto no_mem; } assert( p->rc==SQLITE_OK || (p->rc&0xff)==SQLITE_BUSY ); + testcase( p->rc!=SQLITE_OK ); + p->rc = SQLITE_OK; assert( p->bIsReader || p->readOnly!=0 ); p->iCurrentTime = 0; assert( p->explain==0 ); p->pResultSet = 0; db->busyHandler.nBusy = 0; - if( db->u1.isInterrupted ) goto abort_due_to_interrupt; + if( AtomicLoad(&db->u1.isInterrupted) ) goto abort_due_to_interrupt; sqlite3VdbeIOTraceSql(p); -#ifndef SQLITE_OMIT_PROGRESS_CALLBACK - if( db->xProgress ){ - u32 iPrior = p->aCounter[SQLITE_STMTSTATUS_VM_STEP]; - assert( 0 < db->nProgressOps ); - nProgressLimit = db->nProgressOps - (iPrior % db->nProgressOps); - }else{ - nProgressLimit = 0xffffffff; - } -#endif #ifdef SQLITE_DEBUG sqlite3BeginBenignMalloc(); if( p->pc==0 @@ -82582,6 +85876,7 @@ SQLITE_PRIVATE int sqlite3VdbeExec( #ifdef SQLITE_DEBUG if( db->flags & SQLITE_VdbeTrace ){ sqlite3VdbePrintOp(stdout, (int)(pOp - aOp), pOp); + test_trace_breakpoint((int)(pOp - aOp),pOp,p); } #endif @@ -82689,6 +85984,20 @@ SQLITE_PRIVATE int sqlite3VdbeExec( ** to the current line should be indented for EXPLAIN output. */ case OP_Goto: { /* jump */ + +#ifdef SQLITE_DEBUG + /* In debuggging mode, when the p5 flags is set on an OP_Goto, that + ** means we should really jump back to the preceeding OP_ReleaseReg + ** instruction. */ + if( pOp->p5 ){ + assert( pOp->p2 < (int)(pOp - aOp) ); + assert( pOp->p2 > 1 ); + pOp = &aOp[pOp->p2 - 2]; + assert( pOp[1].opcode==OP_ReleaseReg ); + goto check_for_interrupt; + } +#endif + jump_to_p2_and_check_for_interrupt: pOp = &aOp[pOp->p2 - 1]; @@ -82703,7 +86012,7 @@ case OP_Goto: { /* jump */ ** checks on every opcode. This helps sqlite3_step() to run about 1.5% ** faster according to "valgrind --tool=cachegrind" */ check_for_interrupt: - if( db->u1.isInterrupted ) goto abort_due_to_interrupt; + if( AtomicLoad(&db->u1.isInterrupted) ) goto abort_due_to_interrupt; #ifndef SQLITE_OMIT_PROGRESS_CALLBACK /* Call the progress callback if it is configured and the required number ** of VDBE ops have been executed (either since this invocation of @@ -82711,10 +86020,11 @@ case OP_Goto: { /* jump */ ** If the progress callback returns non-zero, exit the virtual machine with ** a return code SQLITE_ABORT. */ - if( nVmStep>=nProgressLimit && db->xProgress!=0 ){ + while( nVmStep>=nProgressLimit && db->xProgress!=0 ){ assert( db->nProgressOps!=0 ); - nProgressLimit = nVmStep + db->nProgressOps - (nVmStep%db->nProgressOps); + nProgressLimit += db->nProgressOps; if( db->xProgress(db->pProgressArg) ){ + nProgressLimit = 0xffffffff; rc = SQLITE_INTERRUPT; goto abort_due_to_error; } @@ -82986,13 +86296,13 @@ case OP_Real: { /* same as TK_FLOAT, out2 */ case OP_String8: { /* same as TK_STRING, out2 */ assert( pOp->p4.z!=0 ); pOut = out2Prerelease(p, pOp); - pOp->opcode = OP_String; pOp->p1 = sqlite3Strlen30(pOp->p4.z); #ifndef SQLITE_OMIT_UTF16 if( encoding!=SQLITE_UTF8 ){ rc = sqlite3VdbeMemSetStr(pOut, pOp->p4.z, -1, SQLITE_UTF8, SQLITE_STATIC); assert( rc==SQLITE_OK || rc==SQLITE_TOOBIG ); + if( rc ) goto too_big; if( SQLITE_OK!=sqlite3VdbeChangeEncoding(pOut, encoding) ) goto no_mem; assert( pOut->szMalloc>0 && pOut->zMalloc==pOut->z ); assert( VdbeMemDynamic(pOut)==0 ); @@ -83005,11 +86315,11 @@ case OP_String8: { /* same as TK_STRING, out2 */ pOp->p4.z = pOut->z; pOp->p1 = pOut->n; } - testcase( rc==SQLITE_TOOBIG ); #endif if( pOp->p1>db->aLimit[SQLITE_LIMIT_LENGTH] ){ goto too_big; } + pOp->opcode = OP_String; assert( rc==SQLITE_OK ); /* Fall through to the next case, OP_String */ } @@ -83065,6 +86375,9 @@ case OP_Null: { /* out2 */ assert( pOp->p3<=(p->nMem+1 - p->nCursor) ); pOut->flags = nullFlag = pOp->p1 ? (MEM_Null|MEM_Cleared) : MEM_Null; pOut->n = 0; +#ifdef SQLITE_DEBUG + pOut->uTemp = 0; +#endif while( cnt>0 ){ pOut++; memAboutToChange(p, pOut); @@ -83124,7 +86437,10 @@ case OP_Variable: { /* out2 */ goto too_big; } pOut = &aMem[pOp->p2]; - sqlite3VdbeMemShallowCopy(pOut, pVar, MEM_Static); + if( VdbeMemDynamic(pOut) ) sqlite3VdbeMemSetNull(pOut); + memcpy(pOut, pVar, MEMCELLSIZE); + pOut->flags &= ~(MEM_Dyn|MEM_Ephem); + pOut->flags |= MEM_Static|MEM_FromBind; UPDATE_MAX_BLOBSIZE(pOut); break; } @@ -83158,8 +86474,13 @@ case OP_Move: { memAboutToChange(p, pOut); sqlite3VdbeMemMove(pOut, pIn1); #ifdef SQLITE_DEBUG - if( pOut->pScopyFrom>=&aMem[p1] && pOut->pScopyFrompScopyFrom += pOp->p2 - p1; + pIn1->pScopyFrom = 0; + { int i; + for(i=1; inMem; i++){ + if( aMem[i].pScopyFrom==pIn1 ){ + aMem[i].pScopyFrom = pOut; + } + } } #endif Deephemeralize(pOut); @@ -83186,6 +86507,7 @@ case OP_Copy: { pOut = &aMem[pOp->p2]; assert( pOut!=pIn1 ); while( 1 ){ + memAboutToChange(p, pOut); sqlite3VdbeMemShallowCopy(pOut, pIn1, MEM_Ephem); Deephemeralize(pOut); #ifdef SQLITE_DEBUG @@ -83218,7 +86540,8 @@ case OP_SCopy: { /* out2 */ assert( pOut!=pIn1 ); sqlite3VdbeMemShallowCopy(pOut, pIn1, MEM_Ephem); #ifdef SQLITE_DEBUG - if( pOut->pScopyFrom==0 ) pOut->pScopyFrom = pIn1; + pOut->pScopyFrom = pIn1; + pOut->mScopyFlags = pIn1->flags; #endif break; } @@ -83255,18 +86578,6 @@ case OP_ResultRow: { assert( pOp->p1>0 ); assert( pOp->p1+pOp->p2<=(p->nMem+1 - p->nCursor)+1 ); -#ifndef SQLITE_OMIT_PROGRESS_CALLBACK - /* Run the progress counter just before returning. - */ - if( db->xProgress!=0 - && nVmStep>=nProgressLimit - && db->xProgress(db->pProgressArg)!=0 - ){ - rc = SQLITE_INTERRUPT; - goto abort_due_to_error; - } -#endif - /* If this statement has violated immediate foreign key constraints, do ** not return the number of rows modified. And do not RELEASE the statement ** transaction. It needs to be rolled back. */ @@ -83310,6 +86621,14 @@ case OP_ResultRow: { || (pMem[i].flags & (MEM_Str|MEM_Blob))==0 ); sqlite3VdbeMemNulTerminate(&pMem[i]); REGISTER_TRACE(pOp->p1+i, &pMem[i]); +#ifdef SQLITE_DEBUG + /* The registers in the result will not be used again when the + ** prepared statement restarts. This is because sqlite3_column() + ** APIs might have caused type conversions of made other changes to + ** the register values. Therefore, we can go ahead and break any + ** OP_SCopy dependencies. */ + pMem[i].pScopyFrom = 0; +#endif } if( db->mallocFailed ) goto no_mem; @@ -83317,6 +86636,7 @@ case OP_ResultRow: { db->xTrace(SQLITE_TRACE_ROW, db->pTraceArg, p, 0); } + /* Return SQLITE_ROW */ p->pc = (int)(pOp - aOp) + 1; @@ -83338,33 +86658,56 @@ case OP_ResultRow: { ** to avoid a memcpy(). */ case OP_Concat: { /* same as TK_CONCAT, in1, in2, out3 */ - i64 nByte; + i64 nByte; /* Total size of the output string or blob */ + u16 flags1; /* Initial flags for P1 */ + u16 flags2; /* Initial flags for P2 */ pIn1 = &aMem[pOp->p1]; pIn2 = &aMem[pOp->p2]; pOut = &aMem[pOp->p3]; + testcase( pOut==pIn2 ); assert( pIn1!=pOut ); - if( (pIn1->flags | pIn2->flags) & MEM_Null ){ + flags1 = pIn1->flags; + testcase( flags1 & MEM_Null ); + testcase( pIn2->flags & MEM_Null ); + if( (flags1 | pIn2->flags) & MEM_Null ){ sqlite3VdbeMemSetNull(pOut); break; } - if( ExpandBlob(pIn1) || ExpandBlob(pIn2) ) goto no_mem; - Stringify(pIn1, encoding); - Stringify(pIn2, encoding); + if( (flags1 & (MEM_Str|MEM_Blob))==0 ){ + if( sqlite3VdbeMemStringify(pIn1,encoding,0) ) goto no_mem; + flags1 = pIn1->flags & ~MEM_Str; + }else if( (flags1 & MEM_Zero)!=0 ){ + if( sqlite3VdbeMemExpandBlob(pIn1) ) goto no_mem; + flags1 = pIn1->flags & ~MEM_Str; + } + flags2 = pIn2->flags; + if( (flags2 & (MEM_Str|MEM_Blob))==0 ){ + if( sqlite3VdbeMemStringify(pIn2,encoding,0) ) goto no_mem; + flags2 = pIn2->flags & ~MEM_Str; + }else if( (flags2 & MEM_Zero)!=0 ){ + if( sqlite3VdbeMemExpandBlob(pIn2) ) goto no_mem; + flags2 = pIn2->flags & ~MEM_Str; + } nByte = pIn1->n + pIn2->n; if( nByte>db->aLimit[SQLITE_LIMIT_LENGTH] ){ goto too_big; } - if( sqlite3VdbeMemGrow(pOut, (int)nByte+2, pOut==pIn2) ){ + if( sqlite3VdbeMemGrow(pOut, (int)nByte+3, pOut==pIn2) ){ goto no_mem; } MemSetTypeFlag(pOut, MEM_Str); if( pOut!=pIn2 ){ memcpy(pOut->z, pIn2->z, pIn2->n); + assert( (pIn2->flags & MEM_Dyn) == (flags2 & MEM_Dyn) ); + pIn2->flags = flags2; } memcpy(&pOut->z[pIn2->n], pIn1->z, pIn1->n); + assert( (pIn1->flags & MEM_Dyn) == (flags1 & MEM_Dyn) ); + pIn1->flags = flags1; pOut->z[nByte]=0; pOut->z[nByte+1] = 0; + pOut->z[nByte+2] = 0; pOut->flags |= MEM_Term; pOut->n = (int)nByte; pOut->enc = encoding; @@ -83415,7 +86758,6 @@ case OP_Subtract: /* same as TK_MINUS, in1, in2, out3 */ case OP_Multiply: /* same as TK_STAR, in1, in2, out3 */ case OP_Divide: /* same as TK_SLASH, in1, in2, out3 */ case OP_Remainder: { /* same as TK_REM, in1, in2, out3 */ - char bIntint; /* Started out as two integer operands */ u16 flags; /* Combined MEM_* flags from both inputs */ u16 type1; /* Numeric type of left operand */ u16 type2; /* Numeric type of right operand */ @@ -83433,7 +86775,6 @@ case OP_Remainder: { /* same as TK_REM, in1, in2, out3 */ if( (type1 & type2 & MEM_Int)!=0 ){ iA = pIn1->u.i; iB = pIn2->u.i; - bIntint = 1; switch( pOp->opcode ){ case OP_Add: if( sqlite3AddInt64(&iB,iA) ) goto fp_math; break; case OP_Subtract: if( sqlite3SubInt64(&iB,iA) ) goto fp_math; break; @@ -83456,7 +86797,6 @@ case OP_Remainder: { /* same as TK_REM, in1, in2, out3 */ }else if( (flags & MEM_Null)!=0 ){ goto arithmetic_result_is_null; }else{ - bIntint = 0; fp_math: rA = sqlite3VdbeRealValue(pIn1); rB = sqlite3VdbeRealValue(pIn2); @@ -83471,8 +86811,8 @@ case OP_Remainder: { /* same as TK_REM, in1, in2, out3 */ break; } default: { - iA = (i64)rA; - iB = (i64)rB; + iA = sqlite3VdbeIntValue(pIn1); + iB = sqlite3VdbeIntValue(pIn2); if( iA==0 ) goto arithmetic_result_is_null; if( iA==-1 ) iA = 1; rB = (double)(iB % iA); @@ -83488,9 +86828,6 @@ case OP_Remainder: { /* same as TK_REM, in1, in2, out3 */ } pOut->u.r = rB; MemSetTypeFlag(pOut, MEM_Real); - if( ((type1|type2)&MEM_Real)==0 && !bIntint ){ - sqlite3VdbeIntegerAffinity(pOut); - } #endif } break; @@ -83632,8 +86969,8 @@ case OP_MustBeInt: { /* jump, in1 */ pIn1 = &aMem[pOp->p1]; if( (pIn1->flags & MEM_Int)==0 ){ applyAffinity(pIn1, SQLITE_AFF_NUMERIC, encoding); - VdbeBranchTaken((pIn1->flags&MEM_Int)==0, 2); if( (pIn1->flags & MEM_Int)==0 ){ + VdbeBranchTaken(1, 2); if( pOp->p2==0 ){ rc = SQLITE_MISMATCH; goto abort_due_to_error; @@ -83642,6 +86979,7 @@ case OP_MustBeInt: { /* jump, in1 */ } } } + VdbeBranchTaken(0, 2); MemSetTypeFlag(pIn1, MEM_Int); break; } @@ -83658,8 +86996,11 @@ case OP_MustBeInt: { /* jump, in1 */ */ case OP_RealAffinity: { /* in1 */ pIn1 = &aMem[pOp->p1]; - if( pIn1->flags & MEM_Int ){ + if( pIn1->flags & (MEM_Int|MEM_IntReal) ){ + testcase( pIn1->flags & MEM_Int ); + testcase( pIn1->flags & MEM_IntReal ); sqlite3VdbeMemRealify(pIn1); + REGISTER_TRACE(pOp->p1, pIn1); } break; } @@ -83691,9 +87032,11 @@ case OP_Cast: { /* in1 */ pIn1 = &aMem[pOp->p1]; memAboutToChange(p, pIn1); rc = ExpandBlob(pIn1); - sqlite3VdbeMemCast(pIn1, pOp->p2, encoding); - UPDATE_MAX_BLOBSIZE(pIn1); if( rc ) goto abort_due_to_error; + rc = sqlite3VdbeMemCast(pIn1, pOp->p2, encoding); + if( rc ) goto abort_due_to_error; + UPDATE_MAX_BLOBSIZE(pIn1); + REGISTER_TRACE(pOp->p1, pIn1); break; } #endif /* SQLITE_OMIT_CAST */ @@ -83816,15 +87159,15 @@ case OP_Ge: { /* same as TK_GE, jump, in1, in3 */ ** OP_Eq or OP_Ne) then take the jump or not depending on whether ** or not both operands are null. */ - assert( pOp->opcode==OP_Eq || pOp->opcode==OP_Ne ); assert( (flags1 & MEM_Cleared)==0 ); - assert( (pOp->p5 & SQLITE_JUMPIFNULL)==0 ); + assert( (pOp->p5 & SQLITE_JUMPIFNULL)==0 || CORRUPT_DB ); + testcase( (pOp->p5 & SQLITE_JUMPIFNULL)!=0 ); if( (flags1&flags3&MEM_Null)!=0 && (flags3&MEM_Cleared)==0 ){ res = 0; /* Operands are equal */ }else{ - res = 1; /* Operands are not equal */ + res = ((flags3 & MEM_Null) ? -1 : +1); /* Operands are not equal */ } }else{ /* SQLITE_NULLEQ is clear and at least one operand is NULL, @@ -83850,12 +87193,12 @@ case OP_Ge: { /* same as TK_GE, jump, in1, in3 */ affinity = pOp->p5 & SQLITE_AFF_MASK; if( affinity>=SQLITE_AFF_NUMERIC ){ if( (flags1 | flags3)&MEM_Str ){ - if( (flags1 & (MEM_Int|MEM_Real|MEM_Str))==MEM_Str ){ + if( (flags1 & (MEM_Int|MEM_IntReal|MEM_Real|MEM_Str))==MEM_Str ){ applyNumericAffinity(pIn1,0); - testcase( flags3!=pIn3->flags ); /* Possible if pIn1==pIn3 */ + assert( flags3==pIn3->flags ); flags3 = pIn3->flags; } - if( (flags3 & (MEM_Int|MEM_Real|MEM_Str))==MEM_Str ){ + if( (flags3 & (MEM_Int|MEM_IntReal|MEM_Real|MEM_Str))==MEM_Str ){ applyNumericAffinity(pIn3,0); } } @@ -83868,17 +87211,19 @@ case OP_Ge: { /* same as TK_GE, jump, in1, in3 */ goto compare_op; } }else if( affinity==SQLITE_AFF_TEXT ){ - if( (flags1 & MEM_Str)==0 && (flags1 & (MEM_Int|MEM_Real))!=0 ){ + if( (flags1 & MEM_Str)==0 && (flags1&(MEM_Int|MEM_Real|MEM_IntReal))!=0 ){ testcase( pIn1->flags & MEM_Int ); testcase( pIn1->flags & MEM_Real ); + testcase( pIn1->flags & MEM_IntReal ); sqlite3VdbeMemStringify(pIn1, encoding, 1); testcase( (flags1&MEM_Dyn) != (pIn1->flags&MEM_Dyn) ); flags1 = (pIn1->flags & ~MEM_TypeMask) | (flags1 & MEM_TypeMask); - assert( pIn1!=pIn3 ); + if( NEVER(pIn1==pIn3) ) flags3 = flags1 | MEM_Str; } - if( (flags3 & MEM_Str)==0 && (flags3 & (MEM_Int|MEM_Real))!=0 ){ + if( (flags3 & MEM_Str)==0 && (flags3&(MEM_Int|MEM_Real|MEM_IntReal))!=0 ){ testcase( pIn3->flags & MEM_Int ); testcase( pIn3->flags & MEM_Real ); + testcase( pIn3->flags & MEM_IntReal ); sqlite3VdbeMemStringify(pIn3, encoding, 1); testcase( (flags3&MEM_Dyn) != (pIn3->flags&MEM_Dyn) ); flags3 = (pIn3->flags & ~MEM_TypeMask) | (flags3 & MEM_TypeMask); @@ -83908,10 +87253,10 @@ case OP_Ge: { /* same as TK_GE, jump, in1, in3 */ } /* Undo any changes made by applyAffinity() to the input registers. */ - assert( (pIn1->flags & MEM_Dyn) == (flags1 & MEM_Dyn) ); - pIn1->flags = flags1; assert( (pIn3->flags & MEM_Dyn) == (flags3 & MEM_Dyn) ); pIn3->flags = flags3; + assert( (pIn1->flags & MEM_Dyn) == (flags1 & MEM_Dyn) ); + pIn1->flags = flags1; if( pOp->p5 & SQLITE_STOREP2 ){ pOut = &aMem[pOp->p2]; @@ -83937,7 +87282,7 @@ case OP_Ge: { /* same as TK_GE, jump, in1, in3 */ pOut->u.i = res2; REGISTER_TRACE(pOp->p2, pOut); }else{ - VdbeBranchTaken(res!=0, (pOp->p5 & SQLITE_NULLEQ)?2:3); + VdbeBranchTaken(res2!=0, (pOp->p5 & SQLITE_NULLEQ)?2:3); if( res2 ){ goto jump_to_p2; } @@ -83947,16 +87292,31 @@ case OP_Ge: { /* same as TK_GE, jump, in1, in3 */ /* Opcode: ElseNotEq * P2 * * * ** -** This opcode must immediately follow an OP_Lt or OP_Gt comparison operator. -** If result of an OP_Eq comparison on the same two operands -** would have be NULL or false (0), then then jump to P2. -** If the result of an OP_Eq comparison on the two previous operands -** would have been true (1), then fall through. +** This opcode must follow an OP_Lt or OP_Gt comparison operator. There +** can be zero or more OP_ReleaseReg opcodes intervening, but no other +** opcodes are allowed to occur between this instruction and the previous +** OP_Lt or OP_Gt. Furthermore, the prior OP_Lt or OP_Gt must have the +** SQLITE_STOREP2 bit set in the P5 field. +** +** If result of an OP_Eq comparison on the same two operands as the +** prior OP_Lt or OP_Gt would have been NULL or false (0), then then +** jump to P2. If the result of an OP_Eq comparison on the two previous +** operands would have been true (1), then fall through. */ case OP_ElseNotEq: { /* same as TK_ESCAPE, jump */ - assert( pOp>aOp ); - assert( pOp[-1].opcode==OP_Lt || pOp[-1].opcode==OP_Gt ); - assert( pOp[-1].p5 & SQLITE_STOREP2 ); + +#ifdef SQLITE_DEBUG + /* Verify the preconditions of this opcode - that it follows an OP_Lt or + ** OP_Gt with the SQLITE_STOREP2 flag set, with zero or more intervening + ** OP_ReleaseReg opcodes */ + int iAddr; + for(iAddr = (int)(pOp - aOp) - 1; ALWAYS(iAddr>=0); iAddr--){ + if( aOp[iAddr].opcode==OP_ReleaseReg ) continue; + assert( aOp[iAddr].opcode==OP_Lt || aOp[iAddr].opcode==OP_Gt ); + assert( aOp[iAddr].p5 & SQLITE_STOREP2 ); + break; + } +#endif /* SQLITE_DEBUG */ VdbeBranchTaken(iCompare!=0, 2); if( iCompare!=0 ) goto jump_to_p2; break; @@ -84048,9 +87408,14 @@ case OP_Compare: { REGISTER_TRACE(p2+idx, &aMem[p2+idx]); assert( inKeyField ); pColl = pKeyInfo->aColl[i]; - bRev = pKeyInfo->aSortOrder[i]; + bRev = (pKeyInfo->aSortFlags[i] & KEYINFO_ORDER_DESC); iCompare = sqlite3MemCompare(&aMem[p1+idx], &aMem[p2+idx], pColl); if( iCompare ){ + if( (pKeyInfo->aSortFlags[i] & KEYINFO_ORDER_BIGNULL) + && ((aMem[p1+idx].flags & MEM_Null) || (aMem[p2+idx].flags & MEM_Null)) + ){ + iCompare = -iCompare; + } if( bRev ) iCompare = -iCompare; break; } @@ -84066,11 +87431,11 @@ case OP_Compare: { */ case OP_Jump: { /* jump */ if( iCompare<0 ){ - VdbeBranchTaken(0,3); pOp = &aOp[pOp->p1 - 1]; + VdbeBranchTaken(0,4); pOp = &aOp[pOp->p1 - 1]; }else if( iCompare==0 ){ - VdbeBranchTaken(1,3); pOp = &aOp[pOp->p2 - 1]; + VdbeBranchTaken(1,4); pOp = &aOp[pOp->p2 - 1]; }else{ - VdbeBranchTaken(2,3); pOp = &aOp[pOp->p3 - 1]; + VdbeBranchTaken(2,4); pOp = &aOp[pOp->p3 - 1]; } break; } @@ -84167,7 +87532,7 @@ case OP_Not: { /* same as TK_NOT, in1, out2 */ } /* Opcode: BitNot P1 P2 * * * -** Synopsis: r[P1]= ~r[P1] +** Synopsis: r[P2]= ~r[P1] ** ** Interpret the content of register P1 as an integer. Store the ** ones-complement of the P1 value into register P2. If P1 holds @@ -84341,11 +87706,6 @@ case OP_Offset: { /* out3 */ ** if the P4 argument is a P4_MEM use the value of the P4 argument as ** the result. ** -** If the OPFLAG_CLEARCACHE bit is set on P5 and P1 is a pseudo-table cursor, -** then the cache of the cursor is reset prior to extracting the column. -** The first OP_Column against a pseudo-table after the value of the content -** register has changed should have this bit set. -** ** If the OPFLAG_LENGTHARG and OPFLAG_TYPEOFARG bits are set on P5 then ** the result is guaranteed to only be used as the argument of a length() ** or typeof() function, respectively. The loading of large blobs can be @@ -84367,7 +87727,9 @@ case OP_Column: { u32 t; /* A type code from the record header */ Mem *pReg; /* PseudoTable input register */ + assert( pOp->p1>=0 && pOp->p1nCursor ); pC = p->apCsr[pOp->p1]; + assert( pC!=0 ); p2 = pOp->p2; /* If the cursor cache is stale (meaning it is not currently point at @@ -84379,7 +87741,6 @@ case OP_Column: { assert( pOp->p3>0 && pOp->p3<=(p->nMem+1 - p->nCursor) ); pDest = &aMem[pOp->p3]; memAboutToChange(p, pDest); - assert( pOp->p1>=0 && pOp->p1nCursor ); assert( pC!=0 ); assert( p2nField ); aOffset = pC->aOffset; @@ -84472,7 +87833,7 @@ case OP_Column: { /* Make sure zData points to enough of the record to cover the header. */ if( pC->aRow==0 ){ memset(&sMem, 0, sizeof(sMem)); - rc = sqlite3VdbeMemFromBtree(pC->uc.pCursor, 0, aOffset[0], &sMem); + rc = sqlite3VdbeMemFromBtreeZeroOffset(pC->uc.pCursor,aOffset[0],&sMem); if( rc!=SQLITE_OK ) goto abort_due_to_error; zData = (u8*)sMem.z; }else{ @@ -84487,15 +87848,15 @@ case OP_Column: { zEndHdr = zData + aOffset[0]; testcase( zHdr>=zEndHdr ); do{ - if( (t = zHdr[0])<0x80 ){ + if( (pC->aType[i] = t = zHdr[0])<0x80 ){ zHdr++; offset64 += sqlite3VdbeOneByteSerialTypeLen(t); }else{ zHdr += sqlite3GetVarint32(zHdr, &t); + pC->aType[i] = t; offset64 += sqlite3VdbeSerialTypeLen(t); } - pC->aType[i++] = t; - aOffset[i] = (u32)(offset64 & 0xffffffff); + aOffset[++i] = (u32)(offset64 & 0xffffffff); }while( i<=p2 && zHdruc.pCursor, aOffset[p2], len, pDest); if( rc!=SQLITE_OK ) goto abort_due_to_error; @@ -84634,12 +87996,33 @@ case OP_Affinity: { assert( pOp->p2>0 ); assert( zAffinity[pOp->p2]==0 ); pIn1 = &aMem[pOp->p1]; - do{ + while( 1 /*exit-by-break*/ ){ assert( pIn1 <= &p->aMem[(p->nMem+1 - p->nCursor)] ); - assert( memIsValid(pIn1) ); - applyAffinity(pIn1, *(zAffinity++), encoding); + assert( zAffinity[0]==SQLITE_AFF_NONE || memIsValid(pIn1) ); + applyAffinity(pIn1, zAffinity[0], encoding); + if( zAffinity[0]==SQLITE_AFF_REAL && (pIn1->flags & MEM_Int)!=0 ){ + /* When applying REAL affinity, if the result is still an MEM_Int + ** that will fit in 6 bytes, then change the type to MEM_IntReal + ** so that we keep the high-resolution integer value but know that + ** the type really wants to be REAL. */ + testcase( pIn1->u.i==140737488355328LL ); + testcase( pIn1->u.i==140737488355327LL ); + testcase( pIn1->u.i==-140737488355328LL ); + testcase( pIn1->u.i==-140737488355329LL ); + if( pIn1->u.i<=140737488355327LL && pIn1->u.i>=-140737488355328LL ){ + pIn1->flags |= MEM_IntReal; + pIn1->flags &= ~MEM_Int; + }else{ + pIn1->u.r = (double)pIn1->u.i; + pIn1->flags |= MEM_Real; + pIn1->flags &= ~MEM_Int; + } + } + REGISTER_TRACE((int)(pIn1-aMem), pIn1); + zAffinity++; + if( zAffinity[0]==0 ) break; pIn1++; - }while( zAffinity[0] ); + } break; } @@ -84660,7 +88043,6 @@ case OP_Affinity: { ** If P4 is NULL then all index fields have the affinity BLOB. */ case OP_MakeRecord: { - u8 *zNewRecord; /* A buffer to hold the data for the new record */ Mem *pRec; /* The new record */ u64 nData; /* Number of bytes of data space */ int nHdr; /* Number of bytes of header space */ @@ -84673,9 +88055,9 @@ case OP_MakeRecord: { int nField; /* Number of fields in the record */ char *zAffinity; /* The affinity string for the record */ int file_format; /* File format to use for encoding */ - int i; /* Space used in zNewRecord[] header */ - int j; /* Space used in zNewRecord[] content */ u32 len; /* Length of a field */ + u8 *zHdr; /* Where to write next byte of the header */ + u8 *zPayload; /* Where to write next byte of the payload */ /* Assuming the record contains N fields, the record format looks ** like this: @@ -84714,7 +88096,14 @@ case OP_MakeRecord: { if( zAffinity ){ pRec = pData0; do{ - applyAffinity(pRec++, *(zAffinity++), encoding); + applyAffinity(pRec, zAffinity[0], encoding); + if( zAffinity[0]==SQLITE_AFF_REAL && (pRec->flags & MEM_Int) ){ + pRec->flags |= MEM_IntReal; + pRec->flags &= ~(MEM_Int); + } + REGISTER_TRACE((int)(pRec-aMem), pRec); + zAffinity++; + pRec++; assert( zAffinity[0]==0 || pRec<=pLast ); }while( zAffinity[0] ); } @@ -84734,14 +88123,36 @@ case OP_MakeRecord: { #endif /* Loop through the elements that will make up the record to figure - ** out how much space is required for the new record. + ** out how much space is required for the new record. After this loop, + ** the Mem.uTemp field of each term should hold the serial-type that will + ** be used for that term in the generated record: + ** + ** Mem.uTemp value type + ** --------------- --------------- + ** 0 NULL + ** 1 1-byte signed integer + ** 2 2-byte signed integer + ** 3 3-byte signed integer + ** 4 4-byte signed integer + ** 5 6-byte signed integer + ** 6 8-byte signed integer + ** 7 IEEE float + ** 8 Integer constant 0 + ** 9 Integer constant 1 + ** 10,11 reserved for expansion + ** N>=12 and even BLOB + ** N>=13 and odd text + ** + ** The following additional values are computed: + ** nHdr Number of bytes needed for the record header + ** nData Number of bytes of data space needed for the record + ** nZero Zero bytes at the end of the record */ pRec = pLast; do{ assert( memIsValid(pRec) ); - serial_type = sqlite3VdbeSerialType(pRec, file_format, &len); - if( pRec->flags & MEM_Zero ){ - if( serial_type==0 ){ + if( pRec->flags & MEM_Null ){ + if( pRec->flags & MEM_Zero ){ /* Values with MEM_Null and MEM_Zero are created by xColumn virtual ** table methods that never invoke sqlite3_result_xxxxx() while ** computing an unchanging column value in an UPDATE statement. @@ -84749,19 +88160,83 @@ case OP_MakeRecord: { ** so that they can be passed through to xUpdate and have ** a true sqlite3_value_nochange(). */ assert( pOp->p5==OPFLAG_NOCHNG_MAGIC || CORRUPT_DB ); - serial_type = 10; - }else if( nData ){ - if( sqlite3VdbeMemExpandBlob(pRec) ) goto no_mem; + pRec->uTemp = 10; + }else{ + pRec->uTemp = 0; + } + nHdr++; + }else if( pRec->flags & (MEM_Int|MEM_IntReal) ){ + /* Figure out whether to use 1, 2, 4, 6 or 8 bytes. */ + i64 i = pRec->u.i; + u64 uu; + testcase( pRec->flags & MEM_Int ); + testcase( pRec->flags & MEM_IntReal ); + if( i<0 ){ + uu = ~i; + }else{ + uu = i; + } + nHdr++; + testcase( uu==127 ); testcase( uu==128 ); + testcase( uu==32767 ); testcase( uu==32768 ); + testcase( uu==8388607 ); testcase( uu==8388608 ); + testcase( uu==2147483647 ); testcase( uu==2147483648 ); + testcase( uu==140737488355327LL ); testcase( uu==140737488355328LL ); + if( uu<=127 ){ + if( (i&1)==i && file_format>=4 ){ + pRec->uTemp = 8+(u32)uu; + }else{ + nData++; + pRec->uTemp = 1; + } + }else if( uu<=32767 ){ + nData += 2; + pRec->uTemp = 2; + }else if( uu<=8388607 ){ + nData += 3; + pRec->uTemp = 3; + }else if( uu<=2147483647 ){ + nData += 4; + pRec->uTemp = 4; + }else if( uu<=140737488355327LL ){ + nData += 6; + pRec->uTemp = 5; }else{ - nZero += pRec->u.nZero; - len -= pRec->u.nZero; + nData += 8; + if( pRec->flags & MEM_IntReal ){ + /* If the value is IntReal and is going to take up 8 bytes to store + ** as an integer, then we might as well make it an 8-byte floating + ** point value */ + pRec->u.r = (double)pRec->u.i; + pRec->flags &= ~MEM_IntReal; + pRec->flags |= MEM_Real; + pRec->uTemp = 7; + }else{ + pRec->uTemp = 6; + } + } + }else if( pRec->flags & MEM_Real ){ + nHdr++; + nData += 8; + pRec->uTemp = 7; + }else{ + assert( db->mallocFailed || pRec->flags&(MEM_Str|MEM_Blob) ); + assert( pRec->n>=0 ); + len = (u32)pRec->n; + serial_type = (len*2) + 12 + ((pRec->flags & MEM_Str)!=0); + if( pRec->flags & MEM_Zero ){ + serial_type += pRec->u.nZero*2; + if( nData ){ + if( sqlite3VdbeMemExpandBlob(pRec) ) goto no_mem; + len += pRec->u.nZero; + }else{ + nZero += pRec->u.nZero; + } } + nData += len; + nHdr += sqlite3VarintLen(serial_type); + pRec->uTemp = serial_type; } - nData += len; - testcase( serial_type==127 ); - testcase( serial_type==128 ); - nHdr += serial_type<=127 ? 1 : sqlite3VarintLen(serial_type); - pRec->uTemp = serial_type; if( pRec==pData0 ) break; pRec--; }while(1); @@ -84782,56 +88257,67 @@ case OP_MakeRecord: { if( nVarintdb->aLimit[SQLITE_LIMIT_LENGTH] ){ - goto too_big; - } /* Make sure the output register has a buffer large enough to store ** the new record. The output register (pOp->p3) is not allowed to ** be one of the input registers (because the following call to ** sqlite3VdbeMemClearAndResize() could clobber the value before it is used). */ - if( sqlite3VdbeMemClearAndResize(pOut, (int)nByte) ){ - goto no_mem; + if( nByte+nZero<=pOut->szMalloc ){ + /* The output register is already large enough to hold the record. + ** No error checks or buffer enlargement is required */ + pOut->z = pOut->zMalloc; + }else{ + /* Need to make sure that the output is not too big and then enlarge + ** the output register to hold the full result */ + if( nByte+nZero>db->aLimit[SQLITE_LIMIT_LENGTH] ){ + goto too_big; + } + if( sqlite3VdbeMemClearAndResize(pOut, (int)nByte) ){ + goto no_mem; + } + } + pOut->n = (int)nByte; + pOut->flags = MEM_Blob; + if( nZero ){ + pOut->u.nZero = nZero; + pOut->flags |= MEM_Zero; } - zNewRecord = (u8 *)pOut->z; + UPDATE_MAX_BLOBSIZE(pOut); + zHdr = (u8 *)pOut->z; + zPayload = zHdr + nHdr; /* Write the record */ - i = putVarint32(zNewRecord, nHdr); - j = nHdr; + zHdr += putVarint32(zHdr, nHdr); assert( pData0<=pLast ); pRec = pData0; do{ serial_type = pRec->uTemp; /* EVIDENCE-OF: R-06529-47362 Following the size varint are one or more ** additional varints, one per column. */ - i += putVarint32(&zNewRecord[i], serial_type); /* serial type */ + zHdr += putVarint32(zHdr, serial_type); /* serial type */ /* EVIDENCE-OF: R-64536-51728 The values for each column in the record ** immediately follow the header. */ - j += sqlite3VdbeSerialPut(&zNewRecord[j], pRec, serial_type); /* content */ + zPayload += sqlite3VdbeSerialPut(zPayload, pRec, serial_type); /* content */ }while( (++pRec)<=pLast ); - assert( i==nHdr ); - assert( j==nByte ); + assert( nHdr==(int)(zHdr - (u8*)pOut->z) ); + assert( nByte==(int)(zPayload - (u8*)pOut->z) ); assert( pOp->p3>0 && pOp->p3<=(p->nMem+1 - p->nCursor) ); - pOut->n = (int)nByte; - pOut->flags = MEM_Blob; - if( nZero ){ - pOut->u.nZero = nZero; - pOut->flags |= MEM_Zero; - } REGISTER_TRACE(pOp->p3, pOut); - UPDATE_MAX_BLOBSIZE(pOut); break; } -/* Opcode: Count P1 P2 * * * +/* Opcode: Count P1 P2 p3 * * ** Synopsis: r[P2]=count() ** ** Store the number of entries (an integer value) in the table or index -** opened by cursor P1 in register P2 +** opened by cursor P1 in register P2. +** +** If P3==0, then an exact count is obtained, which involves visiting +** every btree page of the table. But if P3 is non-zero, an estimate +** is returned based on the current cursor position. */ -#ifndef SQLITE_OMIT_BTREECOUNT case OP_Count: { /* out2 */ i64 nEntry; BtCursor *pCrsr; @@ -84839,20 +88325,24 @@ case OP_Count: { /* out2 */ assert( p->apCsr[pOp->p1]->eCurType==CURTYPE_BTREE ); pCrsr = p->apCsr[pOp->p1]->uc.pCursor; assert( pCrsr ); - nEntry = 0; /* Not needed. Only used to silence a warning. */ - rc = sqlite3BtreeCount(pCrsr, &nEntry); - if( rc ) goto abort_due_to_error; + if( pOp->p3 ){ + nEntry = sqlite3BtreeRowCountEst(pCrsr); + }else{ + nEntry = 0; /* Not needed. Only used to silence a warning. */ + rc = sqlite3BtreeCount(db, pCrsr, &nEntry); + if( rc ) goto abort_due_to_error; + } pOut = out2Prerelease(p, pOp); pOut->u.i = nEntry; - break; + goto check_for_interrupt; } -#endif /* Opcode: Savepoint P1 * * P4 * ** ** Open, release or rollback the savepoint named by parameter P4, depending -** on the value of P1. To open a new savepoint, P1==0. To release (commit) an -** existing savepoint, P1==1, or to rollback an existing savepoint P1==2. +** on the value of P1. To open a new savepoint set P1==0 (SAVEPOINT_BEGIN). +** To release (commit) an existing savepoint set P1==1 (SAVEPOINT_RELEASE). +** To rollback an existing savepoint set P1==2 (SAVEPOINT_ROLLBACK). */ case OP_Savepoint: { int p1; /* Value of P1 operand */ @@ -84920,6 +88410,7 @@ case OP_Savepoint: { } } }else{ + assert( p1==SAVEPOINT_RELEASE || p1==SAVEPOINT_ROLLBACK ); iSavepoint = 0; /* Find the named savepoint. If there is no such savepoint, then an @@ -84959,8 +88450,12 @@ case OP_Savepoint: { p->rc = rc = SQLITE_BUSY; goto vdbe_return; } - db->isTransactionSavepoint = 0; rc = p->rc; + if( rc ){ + db->autoCommit = 0; + }else{ + db->isTransactionSavepoint = 0; + } }else{ int isSchemaChange; iSavepoint = db->nSavepoint - iSavepoint - 1; @@ -84973,6 +88468,7 @@ case OP_Savepoint: { if( rc!=SQLITE_OK ) goto abort_due_to_error; } }else{ + assert( p1==SAVEPOINT_RELEASE ); isSchemaChange = 0; } for(ii=0; iinDb; ii++){ @@ -84982,11 +88478,12 @@ case OP_Savepoint: { } } if( isSchemaChange ){ - sqlite3ExpirePreparedStatements(db); + sqlite3ExpirePreparedStatements(db, 0); sqlite3ResetAllSchemasOfConnection(db); db->mDbFlags |= DBFLAG_SchemaChange; } } + if( rc ) goto abort_due_to_error; /* Regardless of whether this is a RELEASE or ROLLBACK, destroy all ** savepoints nested inside of the savepoint being operated on. */ @@ -85009,6 +88506,7 @@ case OP_Savepoint: { db->nSavepoint--; } }else{ + assert( p1==SAVEPOINT_ROLLBACK ); db->nDeferredCons = pSavepoint->nDeferredCons; db->nDeferredImmCons = pSavepoint->nDeferredImmCons; } @@ -85068,7 +88566,6 @@ case OP_AutoCommit: { p->rc = rc = SQLITE_BUSY; goto vdbe_return; } - assert( db->nStatement==0 ); sqlite3CloseSavepoints(db); if( p->rc==SQLITE_OK ){ rc = SQLITE_DONE; @@ -85085,7 +88582,7 @@ case OP_AutoCommit: { rc = SQLITE_ERROR; goto abort_due_to_error; } - break; + /*NOTREACHED*/ assert(0); } /* Opcode: Transaction P1 P2 P3 P4 P5 @@ -85124,8 +88621,7 @@ case OP_AutoCommit: { */ case OP_Transaction: { Btree *pBt; - int iMeta; - int iGen; + int iMeta = 0; assert( p->bIsReader ); assert( p->readOnly==0 || pOp->p2==0 ); @@ -85138,7 +88634,7 @@ case OP_Transaction: { pBt = db->aDb[pOp->p1].pBt; if( pBt ){ - rc = sqlite3BtreeBeginTrans(pBt, pOp->p2); + rc = sqlite3BtreeBeginTrans(pBt, pOp->p2, &iMeta); testcase( rc==SQLITE_BUSY_SNAPSHOT ); testcase( rc==SQLITE_BUSY_RECOVERY ); if( rc!=SQLITE_OK ){ @@ -85150,7 +88646,8 @@ case OP_Transaction: { goto abort_due_to_error; } - if( pOp->p2 && p->usesStmtJournal + if( p->usesStmtJournal + && pOp->p2 && (db->autoCommit==0 || db->nVdbeRead>1) ){ assert( sqlite3BtreeIsInTrans(pBt) ); @@ -85171,19 +88668,17 @@ case OP_Transaction: { p->nStmtDefCons = db->nDeferredCons; p->nStmtDefImmCons = db->nDeferredImmCons; } - - /* Gather the schema version number for checking: + } + assert( pOp->p5==0 || pOp->p4type==P4_INT32 ); + if( pOp->p5 + && (iMeta!=pOp->p3 + || db->aDb[pOp->p1].pSchema->iGeneration!=pOp->p4.i) + ){ + /* ** IMPLEMENTATION-OF: R-03189-51135 As each SQL statement runs, the schema ** version is checked to ensure that the schema has not changed since the ** SQL statement was prepared. */ - sqlite3BtreeGetMeta(pBt, BTREE_SCHEMA_VERSION, (u32 *)&iMeta); - iGen = db->aDb[pOp->p1].pSchema->iGeneration; - }else{ - iGen = iMeta = 0; - } - assert( pOp->p5==0 || pOp->p4type==P4_INT32 ); - if( pOp->p5 && (iMeta!=pOp->p3 || iGen!=pOp->p4.i) ){ sqlite3DbFree(db, p->zErrMsg); p->zErrMsg = sqlite3DbStrDup(db, "database schema has changed"); /* If the schema-cookie from the database file matches the cookie @@ -85274,7 +88769,7 @@ case OP_SetCookie: { if( pOp->p1==1 ){ /* Invalidate all prepared statements whenever the TEMP database ** schema is changed. Ticket #1644 */ - sqlite3ExpirePreparedStatements(db); + sqlite3ExpirePreparedStatements(db, 0); p->expired = 0; } if( rc ) goto abort_due_to_error; @@ -85292,59 +88787,78 @@ case OP_SetCookie: { ** values need not be contiguous but all P1 values should be small integers. ** It is an error for P1 to be negative. ** -** If P5!=0 then use the content of register P2 as the root page, not -** the value of P2 itself. -** -** There will be a read lock on the database whenever there is an -** open cursor. If the database was unlocked prior to this instruction -** then a read lock is acquired as part of this instruction. A read -** lock allows other processes to read the database but prohibits -** any other process from modifying the database. The read lock is -** released when all cursors are closed. If this instruction attempts -** to get a read lock but fails, the script terminates with an -** SQLITE_BUSY error code. +** Allowed P5 bits: +**
          +**
        • 0x02 OPFLAG_SEEKEQ: This cursor will only be used for +** equality lookups (implemented as a pair of opcodes OP_SeekGE/OP_IdxGT +** of OP_SeekLE/OP_IdxLT) +**
        ** ** The P4 value may be either an integer (P4_INT32) or a pointer to ** a KeyInfo structure (P4_KEYINFO). If it is a pointer to a KeyInfo -** structure, then said structure defines the content and collating -** sequence of the index being opened. Otherwise, if P4 is an integer -** value, it is set to the number of columns in the table. +** object, then table being opened must be an [index b-tree] where the +** KeyInfo object defines the content and collating +** sequence of that index b-tree. Otherwise, if P4 is an integer +** value, then the table being opened must be a [table b-tree] with a +** number of columns no less than the value of P4. ** ** See also: OpenWrite, ReopenIdx */ /* Opcode: ReopenIdx P1 P2 P3 P4 P5 ** Synopsis: root=P2 iDb=P3 ** -** The ReopenIdx opcode works exactly like ReadOpen except that it first -** checks to see if the cursor on P1 is already open with a root page -** number of P2 and if it is this opcode becomes a no-op. In other words, +** The ReopenIdx opcode works like OP_OpenRead except that it first +** checks to see if the cursor on P1 is already open on the same +** b-tree and if it is this opcode becomes a no-op. In other words, ** if the cursor is already open, do not reopen it. ** -** The ReopenIdx opcode may only be used with P5==0 and with P4 being -** a P4_KEYINFO object. Furthermore, the P3 value must be the same as -** every other ReopenIdx or OpenRead for the same cursor number. +** The ReopenIdx opcode may only be used with P5==0 or P5==OPFLAG_SEEKEQ +** and with P4 being a P4_KEYINFO object. Furthermore, the P3 value must +** be the same as every other ReopenIdx or OpenRead for the same cursor +** number. +** +** Allowed P5 bits: +**
          +**
        • 0x02 OPFLAG_SEEKEQ: This cursor will only be used for +** equality lookups (implemented as a pair of opcodes OP_SeekGE/OP_IdxGT +** of OP_SeekLE/OP_IdxLT) +**
        ** -** See the OpenRead opcode documentation for additional information. +** See also: OP_OpenRead, OP_OpenWrite */ /* Opcode: OpenWrite P1 P2 P3 P4 P5 ** Synopsis: root=P2 iDb=P3 ** ** Open a read/write cursor named P1 on the table or index whose root -** page is P2. Or if P5!=0 use the content of register P2 to find the -** root page. +** page is P2 (or whose root page is held in register P2 if the +** OPFLAG_P2ISREG bit is set in P5 - see below). ** ** The P4 value may be either an integer (P4_INT32) or a pointer to ** a KeyInfo structure (P4_KEYINFO). If it is a pointer to a KeyInfo -** structure, then said structure defines the content and collating -** sequence of the index being opened. Otherwise, if P4 is an integer -** value, it is set to the number of columns in the table, or to the -** largest index of any column of the table that is actually used. +** object, then table being opened must be an [index b-tree] where the +** KeyInfo object defines the content and collating +** sequence of that index b-tree. Otherwise, if P4 is an integer +** value, then the table being opened must be a [table b-tree] with a +** number of columns no less than the value of P4. +** +** Allowed P5 bits: +**
          +**
        • 0x02 OPFLAG_SEEKEQ: This cursor will only be used for +** equality lookups (implemented as a pair of opcodes OP_SeekGE/OP_IdxGT +** of OP_SeekLE/OP_IdxLT) +**
        • 0x08 OPFLAG_FORDELETE: This cursor is used only to seek +** and subsequently delete entries in an index btree. This is a +** hint to the storage engine that the storage engine is allowed to +** ignore. The hint is not used by the official SQLite b*tree storage +** engine, but is used by COMDB2. +**
        • 0x10 OPFLAG_P2ISREG: Use the content of register P2 +** as the root page, not the value of P2 itself. +**
        ** -** This instruction works just like OpenRead except that it opens the cursor -** in read/write mode. For a given table, there can be one or more read-only -** cursors or a single read/write cursor but not both. +** This instruction works like OpenRead except that it opens the cursor +** in read/write mode. ** -** See also OpenRead. +** See also: OP_OpenRead, OP_ReopenIdx */ case OP_ReopenIdx: { int nField; @@ -85373,7 +88887,7 @@ case OP_OpenWrite: assert( pOp->opcode==OP_OpenRead || pOp->opcode==OP_ReopenIdx || p->readOnly==0 ); - if( p->expired ){ + if( p->expired==1 ){ rc = SQLITE_ABORT_ROLLBACK; goto abort_due_to_error; } @@ -85400,6 +88914,7 @@ case OP_OpenWrite: if( pOp->p5 & OPFLAG_P2ISREG ){ assert( p2>0 ); assert( p2<=(p->nMem+1 - p->nCursor) ); + assert( pOp->opcode==OP_OpenWrite ); pIn2 = &aMem[p2]; assert( memIsValid(pIn2) ); assert( (pIn2->flags & MEM_Int)!=0 ); @@ -85442,9 +88957,7 @@ case OP_OpenWrite: assert( OPFLAG_BULKCSR==BTREE_BULKLOAD ); assert( OPFLAG_SEEKEQ==BTREE_SEEK_EQ ); testcase( pOp->p5 & OPFLAG_BULKCSR ); -#ifdef SQLITE_ENABLE_CURSOR_HINTS testcase( pOp->p2 & OPFLAG_SEEKEQ ); -#endif sqlite3BtreeCursorHintFlags(pCur->uc.pCursor, (pOp->p5 & (OPFLAG_BULKCSR|OPFLAG_SEEKEQ))); if( rc ) goto abort_due_to_error; @@ -85464,6 +88977,7 @@ case OP_OpenDup: { VdbeCursor *pCx; /* The new cursor */ pOrig = p->apCsr[pOp->p2]; + assert( pOrig ); assert( pOrig->pBtx!=0 ); /* Only ephemeral cursors can be duplicated */ pCx = allocateCursor(p, pOp->p1, pOrig->nField, -1, CURTYPE_BTREE); @@ -85472,7 +88986,9 @@ case OP_OpenDup: { pCx->isEphemeral = 1; pCx->pKeyInfo = pOrig->pKeyInfo; pCx->isTable = pOrig->isTable; - rc = sqlite3BtreeCursor(pOrig->pBtx, MASTER_ROOT, BTREE_WRCSR, + pCx->pgnoRoot = pOrig->pgnoRoot; + pCx->isOrdered = pOrig->isOrdered; + rc = sqlite3BtreeCursor(pOrig->pBtx, pCx->pgnoRoot, BTREE_WRCSR, pCx->pKeyInfo, pCx->uc.pCursor); /* The sqlite3BtreeCursor() routine can only fail for the first cursor ** opened for a database. Since there is already an open cursor when this @@ -85490,6 +89006,9 @@ case OP_OpenDup: { ** the main database is read-only. The ephemeral ** table is deleted automatically when the cursor is closed. ** +** If the cursor P1 is already opened on an ephemeral table, the table +** is cleared (all content is erased). +** ** P2 is the number of columns in the ephemeral table. ** The cursor points to a BTree table if P4==0 and to a BTree index ** if P4 is not 0. If P4 is not NULL, it points to a KeyInfo structure @@ -85521,41 +89040,53 @@ case OP_OpenEphemeral: { SQLITE_OPEN_TRANSIENT_DB; assert( pOp->p1>=0 ); assert( pOp->p2>=0 ); - pCx = allocateCursor(p, pOp->p1, pOp->p2, -1, CURTYPE_BTREE); - if( pCx==0 ) goto no_mem; - pCx->nullRow = 1; - pCx->isEphemeral = 1; - rc = sqlite3BtreeOpen(db->pVfs, 0, db, &pCx->pBtx, - BTREE_OMIT_JOURNAL | BTREE_SINGLE | pOp->p5, vfsFlags); - if( rc==SQLITE_OK ){ - rc = sqlite3BtreeBeginTrans(pCx->pBtx, 1); - } - if( rc==SQLITE_OK ){ - /* If a transient index is required, create it by calling - ** sqlite3BtreeCreateTable() with the BTREE_BLOBKEY flag before - ** opening it. If a transient table is required, just use the - ** automatically created table with root-page 1 (an BLOB_INTKEY table). - */ - if( (pCx->pKeyInfo = pKeyInfo = pOp->p4.pKeyInfo)!=0 ){ - int pgno; - assert( pOp->p4type==P4_KEYINFO ); - rc = sqlite3BtreeCreateTable(pCx->pBtx, &pgno, BTREE_BLOBKEY | pOp->p5); - if( rc==SQLITE_OK ){ - assert( pgno==MASTER_ROOT+1 ); - assert( pKeyInfo->db==db ); - assert( pKeyInfo->enc==ENC(db) ); - rc = sqlite3BtreeCursor(pCx->pBtx, pgno, BTREE_WRCSR, - pKeyInfo, pCx->uc.pCursor); + pCx = p->apCsr[pOp->p1]; + if( pCx && pCx->pBtx ){ + /* If the ephermeral table is already open, erase all existing content + ** so that the table is empty again, rather than creating a new table. */ + assert( pCx->isEphemeral ); + pCx->seqCount = 0; + pCx->cacheStatus = CACHE_STALE; + rc = sqlite3BtreeClearTable(pCx->pBtx, pCx->pgnoRoot, 0); + }else{ + pCx = allocateCursor(p, pOp->p1, pOp->p2, -1, CURTYPE_BTREE); + if( pCx==0 ) goto no_mem; + pCx->isEphemeral = 1; + rc = sqlite3BtreeOpen(db->pVfs, 0, db, &pCx->pBtx, + BTREE_OMIT_JOURNAL | BTREE_SINGLE | pOp->p5, + vfsFlags); + if( rc==SQLITE_OK ){ + rc = sqlite3BtreeBeginTrans(pCx->pBtx, 1, 0); + } + if( rc==SQLITE_OK ){ + /* If a transient index is required, create it by calling + ** sqlite3BtreeCreateTable() with the BTREE_BLOBKEY flag before + ** opening it. If a transient table is required, just use the + ** automatically created table with root-page 1 (an BLOB_INTKEY table). + */ + if( (pCx->pKeyInfo = pKeyInfo = pOp->p4.pKeyInfo)!=0 ){ + assert( pOp->p4type==P4_KEYINFO ); + rc = sqlite3BtreeCreateTable(pCx->pBtx, (int*)&pCx->pgnoRoot, + BTREE_BLOBKEY | pOp->p5); + if( rc==SQLITE_OK ){ + assert( pCx->pgnoRoot==MASTER_ROOT+1 ); + assert( pKeyInfo->db==db ); + assert( pKeyInfo->enc==ENC(db) ); + rc = sqlite3BtreeCursor(pCx->pBtx, pCx->pgnoRoot, BTREE_WRCSR, + pKeyInfo, pCx->uc.pCursor); + } + pCx->isTable = 0; + }else{ + pCx->pgnoRoot = MASTER_ROOT; + rc = sqlite3BtreeCursor(pCx->pBtx, MASTER_ROOT, BTREE_WRCSR, + 0, pCx->uc.pCursor); + pCx->isTable = 1; } - pCx->isTable = 0; - }else{ - rc = sqlite3BtreeCursor(pCx->pBtx, MASTER_ROOT, BTREE_WRCSR, - 0, pCx->uc.pCursor); - pCx->isTable = 1; } + pCx->isOrdered = (pOp->p5!=BTREE_UNORDERED); } if( rc ) goto abort_due_to_error; - pCx->isOrdered = (pOp->p5!=BTREE_UNORDERED); + pCx->nullRow = 1; break; } @@ -85682,11 +89213,13 @@ case OP_ColumnsUsed: { ** greater than or equal to the key and P2 is not zero, then jump to P2. ** ** If the cursor P1 was opened using the OPFLAG_SEEKEQ flag, then this -** opcode will always land on a record that equally equals the key, or -** else jump immediately to P2. When the cursor is OPFLAG_SEEKEQ, this -** opcode must be followed by an IdxLE opcode with the same arguments. -** The IdxLE opcode will be skipped if this opcode succeeds, but the -** IdxLE opcode will be used on subsequent loop iterations. +** opcode will either land on a record that exactly matches the key, or +** else it will cause a jump to P2. When the cursor is OPFLAG_SEEKEQ, +** this opcode must be followed by an IdxLE opcode with the same arguments. +** The IdxGT opcode will be skipped if this opcode succeeds, but the +** IdxGT opcode will be used on subsequent loop iterations. The +** OPFLAG_SEEKEQ flags is a hint to the btree layer to say that this +** is an equality search. ** ** This opcode leaves the cursor configured to move in forward order, ** from the beginning toward the end. In other words, the cursor is @@ -85702,7 +89235,7 @@ case OP_ColumnsUsed: { ** to an SQL index, then P3 is the first in an array of P4 registers ** that are used as an unpacked index key. ** -** Reposition cursor P1 so that it points to the smallest entry that +** Reposition cursor P1 so that it points to the smallest entry that ** is greater than the key value. If there are no records greater than ** the key and P2 is not zero, then jump to P2. ** @@ -85747,18 +89280,20 @@ case OP_ColumnsUsed: { ** configured to use Prev, not Next. ** ** If the cursor P1 was opened using the OPFLAG_SEEKEQ flag, then this -** opcode will always land on a record that equally equals the key, or -** else jump immediately to P2. When the cursor is OPFLAG_SEEKEQ, this -** opcode must be followed by an IdxGE opcode with the same arguments. +** opcode will either land on a record that exactly matches the key, or +** else it will cause a jump to P2. When the cursor is OPFLAG_SEEKEQ, +** this opcode must be followed by an IdxLE opcode with the same arguments. ** The IdxGE opcode will be skipped if this opcode succeeds, but the -** IdxGE opcode will be used on subsequent loop iterations. +** IdxGE opcode will be used on subsequent loop iterations. The +** OPFLAG_SEEKEQ flags is a hint to the btree layer to say that this +** is an equality search. ** ** See also: Found, NotFound, SeekGt, SeekGe, SeekLt */ -case OP_SeekLT: /* jump, in3 */ -case OP_SeekLE: /* jump, in3 */ -case OP_SeekGE: /* jump, in3 */ -case OP_SeekGT: { /* jump, in3 */ +case OP_SeekLT: /* jump, in3, group */ +case OP_SeekLE: /* jump, in3, group */ +case OP_SeekGE: /* jump, in3, group */ +case OP_SeekGT: { /* jump, in3, group */ int res; /* Comparison result */ int oc; /* Opcode */ VdbeCursor *pC; /* The cursor to seek */ @@ -85784,8 +89319,11 @@ case OP_SeekGT: { /* jump, in3 */ pC->seekOp = pOp->opcode; #endif + pC->deferredMoveto = 0; + pC->cacheStatus = CACHE_STALE; if( pC->isTable ){ - /* The BTREE_SEEK_EQ flag is only set on index cursors */ + u16 flags3, newType; + /* The OPFLAG_SEEKEQ/BTREE_SEEK_EQ flag is only set on index cursors */ assert( sqlite3BtreeCursorHasHint(pC->uc.pCursor, BTREE_SEEK_EQ)==0 || CORRUPT_DB ); @@ -85793,20 +89331,27 @@ case OP_SeekGT: { /* jump, in3 */ ** blob, or NULL. But it needs to be an integer before we can do ** the seek, so convert it. */ pIn3 = &aMem[pOp->p3]; - if( (pIn3->flags & (MEM_Int|MEM_Real|MEM_Str))==MEM_Str ){ + flags3 = pIn3->flags; + if( (flags3 & (MEM_Int|MEM_Real|MEM_IntReal|MEM_Str))==MEM_Str ){ applyNumericAffinity(pIn3, 0); } - iKey = sqlite3VdbeIntValue(pIn3); + iKey = sqlite3VdbeIntValue(pIn3); /* Get the integer key value */ + newType = pIn3->flags; /* Record the type after applying numeric affinity */ + pIn3->flags = flags3; /* But convert the type back to its original */ /* If the P3 value could not be converted into an integer without ** loss of information, then special processing is required... */ - if( (pIn3->flags & MEM_Int)==0 ){ - if( (pIn3->flags & MEM_Real)==0 ){ - /* If the P3 value cannot be converted into any kind of a number, - ** then the seek is not possible, so jump to P2 */ - VdbeBranchTaken(1,2); goto jump_to_p2; - break; - } + if( (newType & (MEM_Int|MEM_IntReal))==0 ){ + if( (newType & MEM_Real)==0 ){ + if( (newType & MEM_Null) || oc>=OP_SeekGE ){ + VdbeBranchTaken(1,2); + goto jump_to_p2; + }else{ + rc = sqlite3BtreeLast(pC->uc.pCursor, &res); + if( rc!=SQLITE_OK ) goto abort_due_to_error; + goto seek_not_found; + } + }else /* If the approximation iKey is larger than the actual real search ** term, substitute >= for > and < for <=. e.g. if the search term @@ -85830,21 +89375,24 @@ case OP_SeekGT: { /* jump, in3 */ assert( (OP_SeekLT & 0x0001)==(OP_SeekGE & 0x0001) ); if( (oc & 0x0001)==(OP_SeekLT & 0x0001) ) oc++; } - } + } rc = sqlite3BtreeMovetoUnpacked(pC->uc.pCursor, 0, (u64)iKey, 0, &res); pC->movetoTarget = iKey; /* Used by OP_Delete */ if( rc!=SQLITE_OK ){ goto abort_due_to_error; } }else{ - /* For a cursor with the BTREE_SEEK_EQ hint, only the OP_SeekGE and - ** OP_SeekLE opcodes are allowed, and these must be immediately followed - ** by an OP_IdxGT or OP_IdxLT opcode, respectively, with the same key. + /* For a cursor with the OPFLAG_SEEKEQ/BTREE_SEEK_EQ hint, only the + ** OP_SeekGE and OP_SeekLE opcodes are allowed, and these must be + ** immediately followed by an OP_IdxGT or OP_IdxLT opcode, respectively, + ** with the same key. */ if( sqlite3BtreeCursorHasHint(pC->uc.pCursor, BTREE_SEEK_EQ) ){ eqOnly = 1; assert( pOp->opcode==OP_SeekGE || pOp->opcode==OP_SeekLE ); assert( pOp[1].opcode==OP_IdxLT || pOp[1].opcode==OP_IdxGT ); + assert( pOp->opcode==OP_SeekGE || pOp[1].opcode==OP_IdxLT ); + assert( pOp->opcode==OP_SeekLE || pOp[1].opcode==OP_IdxGT ); assert( pOp[1].p1==pOp[0].p1 ); assert( pOp[1].p2==pOp[0].p2 ); assert( pOp[1].p3==pOp[0].p3 ); @@ -85884,8 +89432,6 @@ case OP_SeekGT: { /* jump, in3 */ goto seek_not_found; } } - pC->deferredMoveto = 0; - pC->cacheStatus = CACHE_STALE; #ifdef SQLITE_TEST sqlite3_search_count++; #endif @@ -85936,6 +89482,39 @@ case OP_SeekGT: { /* jump, in3 */ break; } +/* Opcode: SeekHit P1 P2 * * * +** Synopsis: seekHit=P2 +** +** Set the seekHit flag on cursor P1 to the value in P2. +* The seekHit flag is used by the IfNoHope opcode. +** +** P1 must be a valid b-tree cursor. P2 must be a boolean value, +** either 0 or 1. +*/ +case OP_SeekHit: { + VdbeCursor *pC; + assert( pOp->p1>=0 && pOp->p1nCursor ); + pC = p->apCsr[pOp->p1]; + assert( pC!=0 ); + assert( pOp->p2==0 || pOp->p2==1 ); + pC->seekHit = pOp->p2 & 1; + break; +} + +/* Opcode: IfNotOpen P1 P2 * * * +** Synopsis: if( !csr[P1] ) goto P2 +** +** If cursor P1 is not open, jump to instruction P2. Otherwise, fall through. +*/ +case OP_IfNotOpen: { /* jump */ + assert( pOp->p1>=0 && pOp->p1nCursor ); + VdbeBranchTaken(p->apCsr[pOp->p1]==0, 2); + if( !p->apCsr[pOp->p1] ){ + goto jump_to_p2_and_check_for_interrupt; + } + break; +} + /* Opcode: Found P1 P2 P3 P4 * ** Synopsis: key=r[P3@P4] ** @@ -85970,7 +89549,34 @@ case OP_SeekGT: { /* jump, in3 */ ** advanced in either direction. In other words, the Next and Prev ** opcodes do not work after this operation. ** -** See also: Found, NotExists, NoConflict +** See also: Found, NotExists, NoConflict, IfNoHope +*/ +/* Opcode: IfNoHope P1 P2 P3 P4 * +** Synopsis: key=r[P3@P4] +** +** Register P3 is the first of P4 registers that form an unpacked +** record. +** +** Cursor P1 is on an index btree. If the seekHit flag is set on P1, then +** this opcode is a no-op. But if the seekHit flag of P1 is clear, then +** check to see if there is any entry in P1 that matches the +** prefix identified by P3 and P4. If no entry matches the prefix, +** jump to P2. Otherwise fall through. +** +** This opcode behaves like OP_NotFound if the seekHit +** flag is clear and it behaves like OP_Noop if the seekHit flag is set. +** +** This opcode is used in IN clause processing for a multi-column key. +** If an IN clause is attached to an element of the key other than the +** left-most element, and if there are no matches on the most recent +** seek over the whole key, then it might be that one of the key element +** to the left is prohibiting a match, and hence there is "no hope" of +** any match regardless of how many IN clause elements are checked. +** In such a case, we abandon the IN clause search early, using this +** opcode. The opcode name comes from the fact that the +** jump is taken if there is "no hope" of achieving a match. +** +** See also: NotFound, SeekHit */ /* Opcode: NoConflict P1 P2 P3 P4 * ** Synopsis: key=r[P3@P4] @@ -85995,6 +89601,14 @@ case OP_SeekGT: { /* jump, in3 */ ** ** See also: NotFound, Found, NotExists */ +case OP_IfNoHope: { /* jump, in3 */ + VdbeCursor *pC; + assert( pOp->p1>=0 && pOp->p1nCursor ); + pC = p->apCsr[pOp->p1]; + assert( pC!=0 ); + if( pC->seekHit ) break; + /* Fall through into OP_NotFound */ +} case OP_NoConflict: /* jump, in3 */ case OP_NotFound: /* jump, in3 */ case OP_Found: { /* jump, in3 */ @@ -86131,26 +89745,39 @@ case OP_SeekRowid: { /* jump, in3 */ u64 iKey; pIn3 = &aMem[pOp->p3]; - if( (pIn3->flags & MEM_Int)==0 ){ - applyAffinity(pIn3, SQLITE_AFF_NUMERIC, encoding); - if( (pIn3->flags & MEM_Int)==0 ) goto jump_to_p2; + testcase( pIn3->flags & MEM_Int ); + testcase( pIn3->flags & MEM_IntReal ); + testcase( pIn3->flags & MEM_Real ); + testcase( (pIn3->flags & (MEM_Str|MEM_Int))==MEM_Str ); + if( (pIn3->flags & (MEM_Int|MEM_IntReal))==0 ){ + /* If pIn3->u.i does not contain an integer, compute iKey as the + ** integer value of pIn3. Jump to P2 if pIn3 cannot be converted + ** into an integer without loss of information. Take care to avoid + ** changing the datatype of pIn3, however, as it is used by other + ** parts of the prepared statement. */ + Mem x = pIn3[0]; + applyAffinity(&x, SQLITE_AFF_NUMERIC, encoding); + if( (x.flags & MEM_Int)==0 ) goto jump_to_p2; + iKey = x.u.i; + goto notExistsWithKey; } /* Fall through into OP_NotExists */ case OP_NotExists: /* jump, in3 */ pIn3 = &aMem[pOp->p3]; - assert( pIn3->flags & MEM_Int ); + assert( (pIn3->flags & MEM_Int)!=0 || pOp->opcode==OP_SeekRowid ); assert( pOp->p1>=0 && pOp->p1nCursor ); + iKey = pIn3->u.i; +notExistsWithKey: pC = p->apCsr[pOp->p1]; assert( pC!=0 ); #ifdef SQLITE_DEBUG - pC->seekOp = 0; + if( pOp->opcode==OP_SeekRowid ) pC->seekOp = OP_SeekRowid; #endif assert( pC->isTable ); assert( pC->eCurType==CURTYPE_BTREE ); pCrsr = pC->uc.pCursor; assert( pCrsr!=0 ); res = 0; - iKey = pIn3->u.i; rc = sqlite3BtreeMovetoUnpacked(pCrsr, 0, iKey, 0, &res); assert( rc==SQLITE_OK || res==0 ); pC->movetoTarget = iKey; /* Used by OP_Delete */ @@ -86361,14 +89988,7 @@ case OP_NewRowid: { /* out2 */ ** This instruction only works on tables. The equivalent instruction ** for indices is OP_IdxInsert. */ -/* Opcode: InsertInt P1 P2 P3 P4 P5 -** Synopsis: intkey=P3 data=r[P2] -** -** This works exactly like OP_Insert except that the key is the -** integer value P3, not the value of the integer stored in register P3. -*/ -case OP_Insert: -case OP_InsertInt: { +case OP_Insert: { Mem *pData; /* MEM cell holding data for the record to be inserted */ Mem *pKey; /* MEM cell holding key for the record */ VdbeCursor *pC; /* Cursor to table into which insert is written */ @@ -86383,22 +90003,18 @@ case OP_InsertInt: { pC = p->apCsr[pOp->p1]; assert( pC!=0 ); assert( pC->eCurType==CURTYPE_BTREE ); + assert( pC->deferredMoveto==0 ); assert( pC->uc.pCursor!=0 ); assert( (pOp->p5 & OPFLAG_ISNOOP) || pC->isTable ); assert( pOp->p4type==P4_TABLE || pOp->p4type>=P4_STATIC ); REGISTER_TRACE(pOp->p2, pData); sqlite3VdbeIncrWriteCounter(p, pC); - if( pOp->opcode==OP_Insert ){ - pKey = &aMem[pOp->p3]; - assert( pKey->flags & MEM_Int ); - assert( memIsValid(pKey) ); - REGISTER_TRACE(pOp->p3, pKey); - x.nKey = pKey->u.i; - }else{ - assert( pOp->opcode==OP_InsertInt ); - x.nKey = pOp->p3; - } + pKey = &aMem[pOp->p3]; + assert( pKey->flags & MEM_Int ); + assert( memIsValid(pKey) ); + REGISTER_TRACE(pOp->p3, pKey); + x.nKey = pKey->u.i; if( pOp->p4type==P4_TABLE && HAS_UPDATE_HOOK(db) ){ assert( pC->iDb>=0 ); @@ -86505,12 +90121,16 @@ case OP_Delete: { sqlite3VdbeIncrWriteCounter(p, pC); #ifdef SQLITE_DEBUG - if( pOp->p4type==P4_TABLE && HasRowid(pOp->p4.pTab) && pOp->p5==0 ){ + if( pOp->p4type==P4_TABLE + && HasRowid(pOp->p4.pTab) + && pOp->p5==0 + && sqlite3BtreeCursorIsValidNN(pC->uc.pCursor) + ){ /* If p5 is zero, the seek operation that positioned the cursor prior to ** OP_Delete will have also set the pC->movetoTarget field to the rowid of ** the row that is being deleted */ i64 iKey = sqlite3BtreeIntegerKey(pC->uc.pCursor); - assert( pC->movetoTarget==iKey ); + assert( CORRUPT_DB || pC->movetoTarget==iKey ); } #endif @@ -86720,7 +90340,7 @@ case OP_RowData: { goto too_big; } testcase( n==0 ); - rc = sqlite3VdbeMemFromBtree(pCrsr, 0, n, pOut); + rc = sqlite3VdbeMemFromBtreeZeroOffset(pCrsr, n, pOut); if( rc ) goto abort_due_to_error; if( !pOp->p3 ) Deephemeralize(pOut); UPDATE_MAX_BLOBSIZE(pOut); @@ -86797,6 +90417,9 @@ case OP_NullRow: { assert( pC->uc.pCursor!=0 ); sqlite3BtreeClearCursor(pC->uc.pCursor); } +#ifdef SQLITE_DEBUG + if( pC->seekOp==0 ) pC->seekOp = OP_NullRow; +#endif break; } @@ -86933,6 +90556,7 @@ case OP_Rewind: { /* jump */ int res; assert( pOp->p1>=0 && pOp->p1nCursor ); + assert( pOp->p5==0 ); pC = p->apCsr[pOp->p1]; assert( pC!=0 ); assert( isSorter(pC)==(pOp->opcode==OP_SorterSort) ); @@ -86983,12 +90607,7 @@ case OP_Rewind: { /* jump */ ** If P5 is positive and the jump is taken, then event counter ** number P5-1 in the prepared statement is incremented. ** -** See also: Prev, NextIfOpen -*/ -/* Opcode: NextIfOpen P1 P2 P3 P4 P5 -** -** This opcode works just like Next except that if cursor P1 is not -** open it behaves a no-op. +** See also: Prev */ /* Opcode: Prev P1 P2 P3 P4 P5 ** @@ -87016,11 +90635,6 @@ case OP_Rewind: { /* jump */ ** If P5 is positive and the jump is taken, then event counter ** number P5-1 in the prepared statement is incremented. */ -/* Opcode: PrevIfOpen P1 P2 P3 P4 P5 -** -** This opcode works just like Prev except that if cursor P1 is not -** open it behaves a no-op. -*/ /* Opcode: SorterNext P1 P2 * * P5 ** ** This opcode works just like OP_Next except that P1 must be a @@ -87035,10 +90649,6 @@ case OP_SorterNext: { /* jump */ assert( isSorter(pC) ); rc = sqlite3VdbeSorterNext(db, pC); goto next_tail; -case OP_PrevIfOpen: /* jump */ -case OP_NextIfOpen: /* jump */ - if( p->apCsr[pOp->p1]==0 ) break; - /* Fall through */ case OP_Prev: /* jump */ case OP_Next: /* jump */ assert( pOp->p1>=0 && pOp->p1nCursor ); @@ -87049,17 +90659,18 @@ case OP_Next: /* jump */ assert( pC->eCurType==CURTYPE_BTREE ); assert( pOp->opcode!=OP_Next || pOp->p4.xAdvance==sqlite3BtreeNext ); assert( pOp->opcode!=OP_Prev || pOp->p4.xAdvance==sqlite3BtreePrevious ); - assert( pOp->opcode!=OP_NextIfOpen || pOp->p4.xAdvance==sqlite3BtreeNext ); - assert( pOp->opcode!=OP_PrevIfOpen || pOp->p4.xAdvance==sqlite3BtreePrevious); - /* The Next opcode is only used after SeekGT, SeekGE, and Rewind. + /* The Next opcode is only used after SeekGT, SeekGE, Rewind, and Found. ** The Prev opcode is only used after SeekLT, SeekLE, and Last. */ - assert( pOp->opcode!=OP_Next || pOp->opcode!=OP_NextIfOpen + assert( pOp->opcode!=OP_Next || pC->seekOp==OP_SeekGT || pC->seekOp==OP_SeekGE - || pC->seekOp==OP_Rewind || pC->seekOp==OP_Found); - assert( pOp->opcode!=OP_Prev || pOp->opcode!=OP_PrevIfOpen + || pC->seekOp==OP_Rewind || pC->seekOp==OP_Found + || pC->seekOp==OP_NullRow|| pC->seekOp==OP_SeekRowid + || pC->seekOp==OP_IfNoHope); + assert( pOp->opcode!=OP_Prev || pC->seekOp==OP_SeekLT || pC->seekOp==OP_SeekLE - || pC->seekOp==OP_Last ); + || pC->seekOp==OP_Last || pC->seekOp==OP_IfNoHope + || pC->seekOp==OP_NullRow); rc = pOp->p4.xAdvance(pC->uc.pCursor, pOp->p3); next_tail: @@ -87107,6 +90718,36 @@ case OP_Next: /* jump */ ** This instruction only works for indices. The equivalent instruction ** for tables is OP_Insert. */ +case OP_IdxInsert: { /* in2 */ + VdbeCursor *pC; + BtreePayload x; + + assert( pOp->p1>=0 && pOp->p1nCursor ); + pC = p->apCsr[pOp->p1]; + sqlite3VdbeIncrWriteCounter(p, pC); + assert( pC!=0 ); + assert( !isSorter(pC) ); + pIn2 = &aMem[pOp->p2]; + assert( pIn2->flags & MEM_Blob ); + if( pOp->p5 & OPFLAG_NCHANGE ) p->nChange++; + assert( pC->eCurType==CURTYPE_BTREE ); + assert( pC->isTable==0 ); + rc = ExpandBlob(pIn2); + if( rc ) goto abort_due_to_error; + x.nKey = pIn2->n; + x.pKey = pIn2->z; + x.aMem = aMem + pOp->p3; + x.nMem = (u16)pOp->p4.i; + rc = sqlite3BtreeInsert(pC->uc.pCursor, &x, + (pOp->p5 & (OPFLAG_APPEND|OPFLAG_SAVEPOSITION)), + ((pOp->p5 & OPFLAG_USESEEKRESULT) ? pC->seekResult : 0) + ); + assert( pC->deferredMoveto==0 ); + pC->cacheStatus = CACHE_STALE; + if( rc) goto abort_due_to_error; + break; +} + /* Opcode: SorterInsert P1 P2 * * * ** Synopsis: key=r[P2] ** @@ -87114,47 +90755,37 @@ case OP_Next: /* jump */ ** MakeRecord instructions. This opcode writes that key ** into the sorter P1. Data for the entry is nil. */ -case OP_SorterInsert: /* in2 */ -case OP_IdxInsert: { /* in2 */ +case OP_SorterInsert: { /* in2 */ VdbeCursor *pC; - BtreePayload x; assert( pOp->p1>=0 && pOp->p1nCursor ); pC = p->apCsr[pOp->p1]; sqlite3VdbeIncrWriteCounter(p, pC); assert( pC!=0 ); - assert( isSorter(pC)==(pOp->opcode==OP_SorterInsert) ); + assert( isSorter(pC) ); pIn2 = &aMem[pOp->p2]; assert( pIn2->flags & MEM_Blob ); - if( pOp->p5 & OPFLAG_NCHANGE ) p->nChange++; - assert( pC->eCurType==CURTYPE_BTREE || pOp->opcode==OP_SorterInsert ); assert( pC->isTable==0 ); rc = ExpandBlob(pIn2); if( rc ) goto abort_due_to_error; - if( pOp->opcode==OP_SorterInsert ){ - rc = sqlite3VdbeSorterWrite(pC, pIn2); - }else{ - x.nKey = pIn2->n; - x.pKey = pIn2->z; - x.aMem = aMem + pOp->p3; - x.nMem = (u16)pOp->p4.i; - rc = sqlite3BtreeInsert(pC->uc.pCursor, &x, - (pOp->p5 & (OPFLAG_APPEND|OPFLAG_SAVEPOSITION)), - ((pOp->p5 & OPFLAG_USESEEKRESULT) ? pC->seekResult : 0) - ); - assert( pC->deferredMoveto==0 ); - pC->cacheStatus = CACHE_STALE; - } + rc = sqlite3VdbeSorterWrite(pC, pIn2); if( rc) goto abort_due_to_error; break; } -/* Opcode: IdxDelete P1 P2 P3 * * +/* Opcode: IdxDelete P1 P2 P3 * P5 ** Synopsis: key=r[P2@P3] ** ** The content of P3 registers starting at register P2 form ** an unpacked index key. This opcode removes that entry from the ** index opened by cursor P1. +** +** If P5 is not zero, then raise an SQLITE_CORRUPT_INDEX error +** if no matching index entry is found. This happens when running +** an UPDATE or DELETE statement and the index entry to be updated +** or deleted is not found. For some uses of IdxDelete +** (example: the EXCEPT operator) it does not matter that no matching +** entry is found. For those cases, P5 is zero. */ case OP_IdxDelete: { VdbeCursor *pC; @@ -87171,7 +90802,6 @@ case OP_IdxDelete: { sqlite3VdbeIncrWriteCounter(p, pC); pCrsr = pC->uc.pCursor; assert( pCrsr!=0 ); - assert( pOp->p5==0 ); r.pKeyInfo = pC->pKeyInfo; r.nField = (u16)pOp->p3; r.default_rc = 0; @@ -87181,6 +90811,9 @@ case OP_IdxDelete: { if( res==0 ){ rc = sqlite3BtreeDelete(pCrsr, BTREE_AUXDELETE); if( rc ) goto abort_due_to_error; + }else if( pOp->p5 ){ + rc = SQLITE_CORRUPT_INDEX; + goto abort_due_to_error; } assert( pC->deferredMoveto==0 ); pC->cacheStatus = CACHE_STALE; @@ -87270,6 +90903,24 @@ case OP_IdxRowid: { /* out2 */ break; } +/* Opcode: FinishSeek P1 * * * * +** +** If cursor P1 was previously moved via OP_DeferredSeek, complete that +** seek operation now, without further delay. If the cursor seek has +** already occurred, this instruction is a no-op. +*/ +case OP_FinishSeek: { + VdbeCursor *pC; /* The P1 index cursor */ + + assert( pOp->p1>=0 && pOp->p1nCursor ); + pC = p->apCsr[pOp->p1]; + if( pC->deferredMoveto ){ + rc = sqlite3VdbeFinishMoveto(pC); + if( rc ) goto abort_due_to_error; + } + break; +} + /* Opcode: IdxGE P1 P2 P3 P4 P5 ** Synopsis: key=r[P3@P4] ** @@ -87342,7 +90993,13 @@ case OP_IdxGE: { /* jump */ } r.aMem = &aMem[pOp->p3]; #ifdef SQLITE_DEBUG - { int i; for(i=0; ip3+i, &aMem[pOp->p3+i]); + } + } #endif res = 0; /* Not needed. Only used to silence a warning. */ rc = sqlite3VdbeIdxKeyCompare(db, pC, &r, &res); @@ -87529,7 +91186,8 @@ case OP_SqlExec: { /* Opcode: ParseSchema P1 * * P4 * ** ** Read and parse all entries from the SQLITE_MASTER table of database P1 -** that match the WHERE clause P4. +** that match the WHERE clause P4. If P4 is a NULL pointer, then the +** entire schema for P1 is reparsed. ** ** This opcode invokes the parser to create a new virtual machine, ** then runs the new virtual machine. It is thus a re-entrant opcode. @@ -87553,13 +91211,24 @@ case OP_ParseSchema: { iDb = pOp->p1; assert( iDb>=0 && iDbnDb ); assert( DbHasProperty(db, iDb, DB_SchemaLoaded) ); - /* Used to be a conditional */ { + +#ifndef SQLITE_OMIT_ALTERTABLE + if( pOp->p4.z==0 ){ + sqlite3SchemaClear(db->aDb[iDb].pSchema); + db->mDbFlags &= ~DBFLAG_SchemaKnownOk; + rc = sqlite3InitOne(db, iDb, &p->zErrMsg, INITFLAG_AlterTable); + db->mDbFlags |= DBFLAG_SchemaChange; + p->expired = 0; + }else +#endif + { zMaster = MASTER_NAME; initData.db = db; - initData.iDb = pOp->p1; + initData.iDb = iDb; initData.pzErrMsg = &p->zErrMsg; + initData.mInitFlags = 0; zSql = sqlite3MPrintf(db, - "SELECT name, rootpage, sql FROM '%q'.%s WHERE %s ORDER BY rowid", + "SELECT*FROM\"%w\".%s WHERE %s ORDER BY rowid", db->aDb[iDb].zDbSName, zMaster, pOp->p4.z); if( zSql==0 ){ rc = SQLITE_NOMEM_BKPT; @@ -87567,9 +91236,16 @@ case OP_ParseSchema: { assert( db->init.busy==0 ); db->init.busy = 1; initData.rc = SQLITE_OK; + initData.nInitRow = 0; assert( !db->mallocFailed ); rc = sqlite3_exec(db, zSql, sqlite3InitCallback, &initData, 0); if( rc==SQLITE_OK ) rc = initData.rc; + if( rc==SQLITE_OK && initData.nInitRow==0 ){ + /* The OP_ParseSchema opcode with a non-NULL P4 argument should parse + ** at least one SQL statement. Any less than that indicates that + ** the sqlite_master table is corrupt. */ + rc = SQLITE_CORRUPT_BKPT; + } sqlite3DbFreeNN(db, zSql); db->init.busy = 0; } @@ -87681,7 +91357,7 @@ case OP_IntegrityCk: { pIn1 = &aMem[pOp->p1]; assert( pOp->p5nDb ); assert( DbMaskTest(p->btreeMask, pOp->p5) ); - z = sqlite3BtreeIntegrityCheck(db->aDb[pOp->p5].pBt, &aRoot[1], nRoot, + z = sqlite3BtreeIntegrityCheck(db, db->aDb[pOp->p5].pBt, &aRoot[1], nRoot, (int)pnErr->u.i+1, &nErr); sqlite3VdbeMemSetNull(pIn1); if( nErr==0 ){ @@ -87694,7 +91370,7 @@ case OP_IntegrityCk: { } UPDATE_MAX_BLOBSIZE(pIn1); sqlite3VdbeChangeEncoding(pIn1, encoding); - break; + goto check_for_interrupt; } #endif /* SQLITE_OMIT_INTEGRITY_CHECK */ @@ -87710,11 +91386,11 @@ case OP_RowSetAdd: { /* in1, in2 */ pIn1 = &aMem[pOp->p1]; pIn2 = &aMem[pOp->p2]; assert( (pIn2->flags & MEM_Int)!=0 ); - if( (pIn1->flags & MEM_RowSet)==0 ){ - sqlite3VdbeMemSetRowSet(pIn1); - if( (pIn1->flags & MEM_RowSet)==0 ) goto no_mem; + if( (pIn1->flags & MEM_Blob)==0 ){ + if( sqlite3VdbeMemSetRowSet(pIn1) ) goto no_mem; } - sqlite3RowSetInsert(pIn1->u.pRowSet, pIn2->u.i); + assert( sqlite3VdbeMemIsRowSet(pIn1) ); + sqlite3RowSetInsert((RowSet*)pIn1->z, pIn2->u.i); break; } @@ -87730,8 +91406,9 @@ case OP_RowSetRead: { /* jump, in1, out3 */ i64 val; pIn1 = &aMem[pOp->p1]; - if( (pIn1->flags & MEM_RowSet)==0 - || sqlite3RowSetNext(pIn1->u.pRowSet, &val)==0 + assert( (pIn1->flags & MEM_Blob)==0 || sqlite3VdbeMemIsRowSet(pIn1) ); + if( (pIn1->flags & MEM_Blob)==0 + || sqlite3RowSetNext((RowSet*)pIn1->z, &val)==0 ){ /* The boolean index is empty */ sqlite3VdbeMemSetNull(pIn1); @@ -87780,20 +91457,19 @@ case OP_RowSetTest: { /* jump, in1, in3 */ /* If there is anything other than a rowset object in memory cell P1, ** delete it now and initialize P1 with an empty rowset */ - if( (pIn1->flags & MEM_RowSet)==0 ){ - sqlite3VdbeMemSetRowSet(pIn1); - if( (pIn1->flags & MEM_RowSet)==0 ) goto no_mem; + if( (pIn1->flags & MEM_Blob)==0 ){ + if( sqlite3VdbeMemSetRowSet(pIn1) ) goto no_mem; } - + assert( sqlite3VdbeMemIsRowSet(pIn1) ); assert( pOp->p4type==P4_INT32 ); assert( iSet==-1 || iSet>=0 ); if( iSet ){ - exists = sqlite3RowSetTest(pIn1->u.pRowSet, iSet, pIn3->u.i); + exists = sqlite3RowSetTest((RowSet*)pIn1->z, iSet, pIn3->u.i); VdbeBranchTaken(exists!=0,2); if( exists ) goto jump_to_p2; } if( iSet>=0 ){ - sqlite3RowSetInsert(pIn1->u.pRowSet, pIn3->u.i); + sqlite3RowSetInsert((RowSet*)pIn1->z, pIn3->u.i); } break; } @@ -87857,7 +91533,7 @@ case OP_Program: { /* jump */ ** of the current program, and the memory required at runtime to execute ** the trigger program. If this trigger has been fired before, then pRt ** is already allocated. Otherwise, it must be initialized. */ - if( (pRt->flags&MEM_Frame)==0 ){ + if( (pRt->flags&MEM_Blob)==0 ){ /* SubProgram.nMem is set to the number of memory cells used by the ** program stored in SubProgram.aOp. As well as these, one memory ** cell is required for each cursor used by the program. Set local @@ -87875,8 +91551,10 @@ case OP_Program: { /* jump */ goto no_mem; } sqlite3VdbeMemRelease(pRt); - pRt->flags = MEM_Frame; - pRt->u.pFrame = pFrame; + pRt->flags = MEM_Blob|MEM_Dyn; + pRt->z = (char*)pFrame; + pRt->n = nByte; + pRt->xDel = sqlite3VdbeFrameMemDel; pFrame->v = p; pFrame->nChildMem = nMem; @@ -87892,6 +91570,9 @@ case OP_Program: { /* jump */ #ifdef SQLITE_ENABLE_STMT_SCANSTATUS pFrame->anExec = p->anExec; #endif +#ifdef SQLITE_DEBUG + pFrame->iFrameMagic = SQLITE_FRAME_MAGIC; +#endif pEnd = &VdbeFrameMem(pFrame)[pFrame->nChildMem]; for(pMem=VdbeFrameMem(pFrame); pMem!=pEnd; pMem++){ @@ -87899,7 +91580,8 @@ case OP_Program: { /* jump */ pMem->db = db; } }else{ - pFrame = pRt->u.pFrame; + pFrame = (VdbeFrame*)pRt->z; + assert( pRt->xDel==sqlite3VdbeFrameMemDel ); assert( pProgram->nMem+pProgram->nCsr==pFrame->nChildMem || (pProgram->nCsr==0 && pProgram->nMem+1==pFrame->nChildMem) ); assert( pProgram->nCsr==pFrame->nChildCsr ); @@ -87926,10 +91608,20 @@ case OP_Program: { /* jump */ p->nOp = pProgram->nOp; #ifdef SQLITE_ENABLE_STMT_SCANSTATUS p->anExec = 0; +#endif +#ifdef SQLITE_DEBUG + /* Verify that second and subsequent executions of the same trigger do not + ** try to reuse register values from the first use. */ + { + int i; + for(i=0; inMem; i++){ + aMem[i].pScopyFrom = 0; /* Prevent false-positive AboutToChange() errs */ + MemSetTypeFlag(&aMem[i], MEM_Undefined); /* Fault if this reg is reused */ + } + } #endif pOp = &aOp[-1]; - - break; + goto check_for_interrupt; } /* Opcode: Param P1 P2 * * * @@ -88128,24 +91820,35 @@ case OP_DecrJumpZero: { /* jump, in1 */ } -/* Opcode: AggStep0 * P2 P3 P4 P5 +/* Opcode: AggStep * P2 P3 P4 P5 ** Synopsis: accum=r[P3] step(r[P2@P5]) ** -** Execute the step function for an aggregate. The -** function has P5 arguments. P4 is a pointer to the FuncDef -** structure that specifies the function. Register P3 is the +** Execute the xStep function for an aggregate. +** The function has P5 arguments. P4 is a pointer to the +** FuncDef structure that specifies the function. Register P3 is the ** accumulator. ** ** The P5 arguments are taken from register P2 and its ** successors. */ -/* Opcode: AggStep * P2 P3 P4 P5 +/* Opcode: AggInverse * P2 P3 P4 P5 +** Synopsis: accum=r[P3] inverse(r[P2@P5]) +** +** Execute the xInverse function for an aggregate. +** The function has P5 arguments. P4 is a pointer to the +** FuncDef structure that specifies the function. Register P3 is the +** accumulator. +** +** The P5 arguments are taken from register P2 and its +** successors. +*/ +/* Opcode: AggStep1 P1 P2 P3 P4 P5 ** Synopsis: accum=r[P3] step(r[P2@P5]) ** -** Execute the step function for an aggregate. The -** function has P5 arguments. P4 is a pointer to an sqlite3_context -** object that is used to run the function. Register P3 is -** as the accumulator. +** Execute the xStep (if P1==0) or xInverse (if P1!=0) function for an +** aggregate. The function has P5 arguments. P4 is a pointer to the +** FuncDef structure that specifies the function. Register P3 is the +** accumulator. ** ** The P5 arguments are taken from register P2 and its ** successors. @@ -88156,7 +91859,8 @@ case OP_DecrJumpZero: { /* jump, in1 */ ** sqlite3_context only happens once, instead of on each call to the ** step function. */ -case OP_AggStep0: { +case OP_AggInverse: +case OP_AggStep: { int n; sqlite3_context *pCtx; @@ -88179,10 +91883,14 @@ case OP_AggStep0: { pCtx->argc = n; pOp->p4type = P4_FUNCCTX; pOp->p4.pCtx = pCtx; - pOp->opcode = OP_AggStep; + + /* OP_AggInverse must have P1==1 and OP_AggStep must have P1==0 */ + assert( pOp->p1==(pOp->opcode==OP_AggInverse) ); + + pOp->opcode = OP_AggStep1; /* Fall through into OP_AggStep */ } -case OP_AggStep: { +case OP_AggStep1: { int i; sqlite3_context *pCtx; Mem *pMem; @@ -88191,6 +91899,17 @@ case OP_AggStep: { pCtx = pOp->p4.pCtx; pMem = &aMem[pOp->p3]; +#ifdef SQLITE_DEBUG + if( pOp->p1 ){ + /* This is an OP_AggInverse call. Verify that xStep has always + ** been called at least once prior to any xInverse call. */ + assert( pMem->uTemp==0x1122e0e3 ); + }else{ + /* This is an OP_AggStep call. Mark it as such. */ + pMem->uTemp = 0x1122e0e3; + } +#endif + /* If this function is inside of a trigger, the register array in aMem[] ** might change from one evaluation to the next. The next block of code ** checks to see if the register array has changed, and if so it @@ -88211,7 +91930,13 @@ case OP_AggStep: { assert( pCtx->pOut->flags==MEM_Null ); assert( pCtx->isError==0 ); assert( pCtx->skipFlag==0 ); +#ifndef SQLITE_OMIT_WINDOWFUNC + if( pOp->p1 ){ + (pCtx->pFunc->xInverse)(pCtx,pCtx->argc,pCtx->argv); + }else +#endif (pCtx->pFunc->xSFunc)(pCtx,pCtx->argc,pCtx->argv); /* IMP: R-24505-23230 */ + if( pCtx->isError ){ if( pCtx->isError>0 ){ sqlite3VdbeError(p, "%s", sqlite3_value_text(pCtx->pOut)); @@ -88236,22 +91961,47 @@ case OP_AggStep: { /* Opcode: AggFinal P1 P2 * P4 * ** Synopsis: accum=r[P1] N=P2 ** -** Execute the finalizer function for an aggregate. P1 is -** the memory location that is the accumulator for the aggregate. +** P1 is the memory location that is the accumulator for an aggregate +** or window function. Execute the finalizer function +** for an aggregate and store the result in P1. ** ** P2 is the number of arguments that the step function takes and ** P4 is a pointer to the FuncDef for this function. The P2 ** argument is not used by this opcode. It is only there to disambiguate ** functions that can take varying numbers of arguments. The -** P4 argument is only needed for the degenerate case where +** P4 argument is only needed for the case where ** the step function was not previously called. */ +/* Opcode: AggValue * P2 P3 P4 * +** Synopsis: r[P3]=value N=P2 +** +** Invoke the xValue() function and store the result in register P3. +** +** P2 is the number of arguments that the step function takes and +** P4 is a pointer to the FuncDef for this function. The P2 +** argument is not used by this opcode. It is only there to disambiguate +** functions that can take varying numbers of arguments. The +** P4 argument is only needed for the case where +** the step function was not previously called. +*/ +case OP_AggValue: case OP_AggFinal: { Mem *pMem; assert( pOp->p1>0 && pOp->p1<=(p->nMem+1 - p->nCursor) ); + assert( pOp->p3==0 || pOp->opcode==OP_AggValue ); pMem = &aMem[pOp->p1]; assert( (pMem->flags & ~(MEM_Null|MEM_Agg))==0 ); - rc = sqlite3VdbeMemFinalize(pMem, pOp->p4.pFunc); +#ifndef SQLITE_OMIT_WINDOWFUNC + if( pOp->p3 ){ + memAboutToChange(p, &aMem[pOp->p3]); + rc = sqlite3VdbeMemAggValue(pMem, &aMem[pOp->p3], pOp->p4.pFunc); + pMem = &aMem[pOp->p3]; + }else +#endif + { + rc = sqlite3VdbeMemFinalize(pMem, pOp->p4.pFunc); + } + if( rc ){ sqlite3VdbeError(p, "%s", sqlite3_value_text(pMem)); goto abort_due_to_error; @@ -88408,14 +92158,19 @@ case OP_JournalMode: { /* out2 */ #endif /* SQLITE_OMIT_PRAGMA */ #if !defined(SQLITE_OMIT_VACUUM) && !defined(SQLITE_OMIT_ATTACH) -/* Opcode: Vacuum P1 * * * * +/* Opcode: Vacuum P1 P2 * * * ** ** Vacuum the entire database P1. P1 is 0 for "main", and 2 or more ** for an attached database. The "temp" database may not be vacuumed. +** +** If P2 is not zero, then it is a register holding a string which is +** the file into which the result of vacuum should be written. When +** P2 is zero, the vacuum overwrites the original database. */ case OP_Vacuum: { assert( p->readOnly==0 ); - rc = sqlite3RunVacuum(&p->zErrMsg, db, pOp->p1); + rc = sqlite3RunVacuum(&p->zErrMsg, db, pOp->p1, + pOp->p2 ? &aMem[pOp->p2] : 0); if( rc ) goto abort_due_to_error; break; } @@ -88446,7 +92201,7 @@ case OP_IncrVacuum: { /* jump */ } #endif -/* Opcode: Expire P1 * * * * +/* Opcode: Expire P1 P2 * * * ** ** Cause precompiled statements to expire. When an expired statement ** is executed using sqlite3_step() it will either automatically @@ -88455,16 +92210,53 @@ case OP_IncrVacuum: { /* jump */ ** ** If P1 is 0, then all SQL statements become expired. If P1 is non-zero, ** then only the currently executing statement is expired. +** +** If P2 is 0, then SQL statements are expired immediately. If P2 is 1, +** then running SQL statements are allowed to continue to run to completion. +** The P2==1 case occurs when a CREATE INDEX or similar schema change happens +** that might help the statement run faster but which does not affect the +** correctness of operation. */ case OP_Expire: { + assert( pOp->p2==0 || pOp->p2==1 ); if( !pOp->p1 ){ - sqlite3ExpirePreparedStatements(db); + sqlite3ExpirePreparedStatements(db, pOp->p2); }else{ - p->expired = 1; + p->expired = pOp->p2+1; } break; } +/* Opcode: CursorLock P1 * * * * +** +** Lock the btree to which cursor P1 is pointing so that the btree cannot be +** written by an other cursor. +*/ +case OP_CursorLock: { + VdbeCursor *pC; + assert( pOp->p1>=0 && pOp->p1nCursor ); + pC = p->apCsr[pOp->p1]; + assert( pC!=0 ); + assert( pC->eCurType==CURTYPE_BTREE ); + sqlite3BtreeCursorPin(pC->uc.pCursor); + break; +} + +/* Opcode: CursorUnlock P1 * * * * +** +** Unlock the btree to which cursor P1 is pointing so that it can be +** written by other cursors. +*/ +case OP_CursorUnlock: { + VdbeCursor *pC; + assert( pOp->p1>=0 && pOp->p1nCursor ); + pC = p->apCsr[pOp->p1]; + assert( pC!=0 ); + assert( pC->eCurType==CURTYPE_BTREE ); + sqlite3BtreeCursorUnpin(pC->uc.pCursor); + break; +} + #ifndef SQLITE_OMIT_SHARED_CACHE /* Opcode: TableLock P1 P2 P3 P4 * ** Synopsis: iDb=P1 root=P2 write=P3 @@ -88560,6 +92352,7 @@ case OP_VDestroy: { db->nVDestroy++; rc = sqlite3VtabCallDestroy(db, pOp->p1, pOp->p4.z); db->nVDestroy--; + assert( p->errorAction==OE_Abort && p->usesStmtJournal ); if( rc ) goto abort_due_to_error; break; } @@ -88682,10 +92475,11 @@ case OP_VFilter: { /* jump */ ** ** If the VColumn opcode is being used to fetch the value of ** an unchanging column during an UPDATE operation, then the P5 -** value is 1. Otherwise, P5 is 0. The P5 value is returned -** by sqlite3_vtab_nochange() routine and can be used -** by virtual table implementations to return special "no-change" -** marks which can be more efficient, depending on the virtual table. +** value is OPFLAG_NOCHNG. This will cause the sqlite3_vtab_nochange() +** function to return true inside the xColumn method of the virtual +** table implementation. The P5 column might also contain other +** bits (OPFLAG_LENGTHARG or OPFLAG_TYPEOFARG) but those bits are +** unused by OP_VColumn. */ case OP_VColumn: { sqlite3_vtab *pVtab; @@ -88707,7 +92501,8 @@ case OP_VColumn: { assert( pModule->xColumn ); memset(&sContext, 0, sizeof(sContext)); sContext.pOut = pDest; - if( pOp->p5 ){ + assert( pOp->p5==OPFLAG_NOCHNG || pOp->p5==0 ); + if( pOp->p5 & OPFLAG_NOCHNG ){ sqlite3VdbeMemSetNull(pDest); pDest->flags = MEM_Null|MEM_Zero; pDest->u.nZero = 0; @@ -88784,7 +92579,10 @@ case OP_VNext: { /* jump */ case OP_VRename: { sqlite3_vtab *pVtab; Mem *pName; - + int isLegacy; + + isLegacy = (db->flags & SQLITE_LegacyAlter); + db->flags |= SQLITE_LegacyAlter; pVtab = pOp->p4.pVtab->pVtab; pName = &aMem[pOp->p1]; assert( pVtab->pModule->xRename ); @@ -88798,6 +92596,7 @@ case OP_VRename: { rc = sqlite3VdbeChangeEncoding(pName, SQLITE_UTF8); if( rc ) goto abort_due_to_error; rc = pVtab->pModule->xRename(pVtab, pName->z); + if( isLegacy==0 ) db->flags &= ~(u64)SQLITE_LegacyAlter; sqlite3VtabImportErrmsg(p, pVtab); p->expired = 0; if( rc ) goto abort_due_to_error; @@ -88846,6 +92645,7 @@ case OP_VUpdate: { || pOp->p5==OE_Abort || pOp->p5==OE_Ignore || pOp->p5==OE_Replace ); assert( p->readOnly==0 ); + if( db->mallocFailed ) goto no_mem; sqlite3VdbeIncrWriteCounter(p, 0); pVtab = pOp->p4.pVtab->pVtab; if( pVtab==0 || NEVER(pVtab->pModule==0) ){ @@ -88926,13 +92726,15 @@ case OP_MaxPgcnt: { /* out2 */ } #endif -/* Opcode: Function0 P1 P2 P3 P4 P5 -** Synopsis: r[P3]=func(r[P2@P5]) +/* Opcode: Function P1 P2 P3 P4 * +** Synopsis: r[P3]=func(r[P2@NP]) ** -** Invoke a user function (P4 is a pointer to a FuncDef object that -** defines the function) with P5 arguments taken from register P2 and -** successors. The result of the function is stored in register P3. -** Register P3 must not be one of the function inputs. +** Invoke a user function (P4 is a pointer to an sqlite3_context object that +** contains a pointer to the function to be run) with arguments taken +** from register P2 and successors. The number of arguments is in +** the sqlite3_context object that P4 points to. +** The result of the function is stored +** in register P3. Register P3 must not be one of the function inputs. ** ** P1 is a 32-bit bitmask indicating whether or not each argument to the ** function was determined to be constant at compile time. If the first @@ -88941,14 +92743,16 @@ case OP_MaxPgcnt: { /* out2 */ ** sqlite3_set_auxdata() API may be safely retained until the next ** invocation of this opcode. ** -** See also: Function, AggStep, AggFinal +** See also: AggStep, AggFinal, PureFunc */ -/* Opcode: Function P1 P2 P3 P4 P5 -** Synopsis: r[P3]=func(r[P2@P5]) +/* Opcode: PureFunc P1 P2 P3 P4 * +** Synopsis: r[P3]=func(r[P2@NP]) ** ** Invoke a user function (P4 is a pointer to an sqlite3_context object that -** contains a pointer to the function to be run) with P5 arguments taken -** from register P2 and successors. The result of the function is stored +** contains a pointer to the function to be run) with arguments taken +** from register P2 and successors. The number of arguments is in +** the sqlite3_context object that P4 points to. +** The result of the function is stored ** in register P3. Register P3 must not be one of the function inputs. ** ** P1 is a 32-bit bitmask indicating whether or not each argument to the @@ -88958,42 +92762,18 @@ case OP_MaxPgcnt: { /* out2 */ ** sqlite3_set_auxdata() API may be safely retained until the next ** invocation of this opcode. ** -** SQL functions are initially coded as OP_Function0 with P4 pointing -** to a FuncDef object. But on first evaluation, the P4 operand is -** automatically converted into an sqlite3_context object and the operation -** changed to this OP_Function opcode. In this way, the initialization of -** the sqlite3_context object occurs only once, rather than once for each -** evaluation of the function. +** This opcode works exactly like OP_Function. The only difference is in +** its name. This opcode is used in places where the function must be +** purely non-deterministic. Some built-in date/time functions can be +** either determinitic of non-deterministic, depending on their arguments. +** When those function are used in a non-deterministic way, they will check +** to see if they were called using OP_PureFunc instead of OP_Function, and +** if they were, they throw an error. ** -** See also: Function0, AggStep, AggFinal +** See also: AggStep, AggFinal, Function */ -case OP_PureFunc0: -case OP_Function0: { - int n; - sqlite3_context *pCtx; - - assert( pOp->p4type==P4_FUNCDEF ); - n = pOp->p5; - assert( pOp->p3>0 && pOp->p3<=(p->nMem+1 - p->nCursor) ); - assert( n==0 || (pOp->p2>0 && pOp->p2+n<=(p->nMem+1 - p->nCursor)+1) ); - assert( pOp->p3p2 || pOp->p3>=pOp->p2+n ); - pCtx = sqlite3DbMallocRawNN(db, sizeof(*pCtx) + (n-1)*sizeof(sqlite3_value*)); - if( pCtx==0 ) goto no_mem; - pCtx->pOut = 0; - pCtx->pFunc = pOp->p4.pFunc; - pCtx->iOp = (int)(pOp - aOp); - pCtx->pVdbe = p; - pCtx->isError = 0; - pCtx->argc = n; - pOp->p4type = P4_FUNCCTX; - pOp->p4.pCtx = pCtx; - assert( OP_PureFunc == OP_PureFunc0+2 ); - assert( OP_Function == OP_Function0+2 ); - pOp->opcode += 2; - /* Fall through into OP_Function */ -} -case OP_PureFunc: -case OP_Function: { +case OP_PureFunc: /* group */ +case OP_Function: { /* group */ int i; sqlite3_context *pCtx; @@ -89006,9 +92786,11 @@ case OP_Function: { ** reinitializes the relavant parts of the sqlite3_context object */ pOut = &aMem[pOp->p3]; if( pCtx->pOut != pOut ){ + pCtx->pVdbe = p; pCtx->pOut = pOut; for(i=pCtx->argc-1; i>=0; i--) pCtx->argv[i] = &aMem[pOp->p2+i]; } + assert( pCtx->pVdbe==p ); memAboutToChange(p, pOut); #ifdef SQLITE_DEBUG @@ -89180,6 +92962,55 @@ case OP_Abortable: { } #endif +#ifdef SQLITE_DEBUG +/* Opcode: ReleaseReg P1 P2 P3 * P5 +** Synopsis: release r[P1@P2] mask P3 +** +** Release registers from service. Any content that was in the +** the registers is unreliable after this opcode completes. +** +** The registers released will be the P2 registers starting at P1, +** except if bit ii of P3 set, then do not release register P1+ii. +** In other words, P3 is a mask of registers to preserve. +** +** Releasing a register clears the Mem.pScopyFrom pointer. That means +** that if the content of the released register was set using OP_SCopy, +** a change to the value of the source register for the OP_SCopy will no longer +** generate an assertion fault in sqlite3VdbeMemAboutToChange(). +** +** If P5 is set, then all released registers have their type set +** to MEM_Undefined so that any subsequent attempt to read the released +** register (before it is reinitialized) will generate an assertion fault. +** +** P5 ought to be set on every call to this opcode. +** However, there are places in the code generator will release registers +** before their are used, under the (valid) assumption that the registers +** will not be reallocated for some other purpose before they are used and +** hence are safe to release. +** +** This opcode is only available in testing and debugging builds. It is +** not generated for release builds. The purpose of this opcode is to help +** validate the generated bytecode. This opcode does not actually contribute +** to computing an answer. +*/ +case OP_ReleaseReg: { + Mem *pMem; + int i; + u32 constMask; + assert( pOp->p1>0 ); + assert( pOp->p1+pOp->p2<=(p->nMem+1 - p->nCursor)+1 ); + pMem = &aMem[pOp->p1]; + constMask = pOp->p3; + for(i=0; ip2; i++, pMem++){ + if( i>=32 || (constMask & MASKBIT32(i))==0 ){ + pMem->pScopyFrom = 0; + if( i<32 && pOp->p5 ) MemSetTypeFlag(pMem, MEM_Undefined); + } + } + break; +} +#endif + /* Opcode: Noop * * * * * ** ** Do nothing. This instruction is often useful as a jump @@ -89231,6 +93062,12 @@ default: { /* This is really OP_Noop, OP_Explain */ if( opProperty & OPFLG_OUT3 ){ registerTrace(pOrigOp->p3, &aMem[pOrigOp->p3]); } + if( opProperty==0xff ){ + /* Never happens. This code exists to avoid a harmless linkage + ** warning aboud sqlite3VdbeRegisterDump() being defined but not + ** used. */ + sqlite3VdbeRegisterDump(p); + } } #endif /* SQLITE_DEBUG */ #endif /* NDEBUG */ @@ -89261,7 +93098,16 @@ default: { /* This is really OP_Noop, OP_Explain */ ** release the mutexes on btrees that were acquired at the ** top. */ vdbe_return: - testcase( nVmStep>0 ); +#ifndef SQLITE_OMIT_PROGRESS_CALLBACK + while( nVmStep>=nProgressLimit && db->xProgress!=0 ){ + nProgressLimit += db->nProgressOps; + if( db->xProgress(db->pProgressArg) ){ + nProgressLimit = 0xffffffff; + rc = SQLITE_INTERRUPT; + goto abort_due_to_error; + } + } +#endif p->aCounter[SQLITE_STMTSTATUS_VM_STEP] += (int)nVmStep; sqlite3VdbeLeave(p); assert( rc!=SQLITE_OK || nExtraDelete==0 @@ -89289,7 +93135,7 @@ default: { /* This is really OP_Noop, OP_Explain */ ** flag. */ abort_due_to_interrupt: - assert( db->u1.isInterrupted ); + assert( AtomicLoad(&db->u1.isInterrupted) ); rc = db->mallocFailed ? SQLITE_NOMEM_BKPT : SQLITE_INTERRUPT; p->rc = rc; sqlite3VdbeError(p, "%s", sqlite3ErrStr(rc)); @@ -89656,11 +93502,12 @@ SQLITE_API int sqlite3_blob_close(sqlite3_blob *pBlob){ sqlite3 *db; if( p ){ + sqlite3_stmt *pStmt = p->pStmt; db = p->db; sqlite3_mutex_enter(db->mutex); - rc = sqlite3_finalize(p->pStmt); sqlite3DbFree(db, p); sqlite3_mutex_leave(db->mutex); + rc = sqlite3_finalize(pStmt); }else{ rc = SQLITE_OK; } @@ -90348,7 +94195,7 @@ static int vdbePmaReadBlob( /* Extend the p->aAlloc[] allocation if required. */ if( p->nAllocnAlloc*2); + sqlite3_int64 nNew = MAX(128, 2*(sqlite3_int64)p->nAlloc); while( nByte>nNew ) nNew = nNew*2; aNew = sqlite3Realloc(p->aAlloc, nNew); if( !aNew ) return SQLITE_NOMEM_BKPT; @@ -90626,8 +94473,8 @@ static int vdbeSorterCompareText( int n2; int res; - getVarint32(&p1[1], n1); - getVarint32(&p2[1], n2); + getVarint32NR(&p1[1], n1); + getVarint32NR(&p2[1], n2); res = memcmp(v1, v2, (MIN(n1, n2) - 13)/2); if( res==0 ){ res = n1 - n2; @@ -90640,7 +94487,8 @@ static int vdbeSorterCompareText( ); } }else{ - if( pTask->pSorter->pKeyInfo->aSortOrder[0] ){ + assert( !(pTask->pSorter->pKeyInfo->aSortFlags[0]&KEYINFO_ORDER_BIGNULL) ); + if( pTask->pSorter->pKeyInfo->aSortFlags[0] ){ res = res * -1; } } @@ -90708,7 +94556,8 @@ static int vdbeSorterCompareInt( pTask, pbKey2Cached, pKey1, nKey1, pKey2, nKey2 ); } - }else if( pTask->pSorter->pKeyInfo->aSortOrder[0] ){ + }else if( pTask->pSorter->pKeyInfo->aSortFlags[0] ){ + assert( !(pTask->pSorter->pKeyInfo->aSortFlags[0]&KEYINFO_ORDER_BIGNULL) ); res = res * -1; } @@ -90823,6 +94672,7 @@ SQLITE_PRIVATE int sqlite3VdbeSorterInit( if( pKeyInfo->nAllField<13 && (pKeyInfo->aColl[0]==0 || pKeyInfo->aColl[0]==db->pDfltColl) + && (pKeyInfo->aSortFlags[0] & KEYINFO_ORDER_BIGNULL)==0 ){ pSorter->typeMask = SORTER_TYPE_INTEGER | SORTER_TYPE_TEXT; } @@ -91204,20 +95054,16 @@ static SorterCompare vdbeSorterGetCompare(VdbeSorter *p){ */ static int vdbeSorterSort(SortSubtask *pTask, SorterList *pList){ int i; - SorterRecord **aSlot; SorterRecord *p; int rc; + SorterRecord *aSlot[64]; rc = vdbeSortAllocUnpacked(pTask); if( rc!=SQLITE_OK ) return rc; p = pList->pList; pTask->xCompare = vdbeSorterGetCompare(pTask->pSorter); - - aSlot = (SorterRecord **)sqlite3MallocZero(64 * sizeof(SorterRecord *)); - if( !aSlot ){ - return SQLITE_NOMEM_BKPT; - } + memset(aSlot, 0, sizeof(aSlot)); while( p ){ SorterRecord *pNext; @@ -91242,13 +95088,12 @@ static int vdbeSorterSort(SortSubtask *pTask, SorterList *pList){ } p = 0; - for(i=0; i<64; i++){ + for(i=0; ipList = p; - sqlite3_free(aSlot); assert( pTask->pUnpacked->errCode==SQLITE_OK || pTask->pUnpacked->errCode==SQLITE_NOMEM ); @@ -91539,13 +95384,16 @@ static int vdbeSorterFlushPMA(VdbeSorter *pSorter){ rc = vdbeSorterListToPMA(&pSorter->aTask[nWorker], &pSorter->list); }else{ /* Launch a background thread for this operation */ - u8 *aMem = pTask->list.aMemory; - void *pCtx = (void*)pTask; + u8 *aMem; + void *pCtx; + assert( pTask!=0 ); assert( pTask->pThread==0 && pTask->bDone==0 ); assert( pTask->list.pList==0 ); assert( pTask->list.aMemory==0 || pSorter->list.aMemory!=0 ); + aMem = pTask->list.aMemory; + pCtx = (void*)pTask; pSorter->iPrev = (u8)(pTask - pSorter->aTask); pTask->list = pSorter->list; pSorter->list.pList = 0; @@ -91583,7 +95431,7 @@ SQLITE_PRIVATE int sqlite3VdbeSorterWrite( assert( pCsr->eCurType==CURTYPE_SORTER ); pSorter = pCsr->uc.pSorter; - getVarint32((const u8*)&pVal->z[1], t); + getVarint32NR((const u8*)&pVal->z[1], t); if( t>0 && t<10 && t!=7 ){ pSorter->typeMask &= SORTER_TYPE_INTEGER; }else if( t>10 && (t & 0x01) ){ @@ -91639,15 +95487,19 @@ SQLITE_PRIVATE int sqlite3VdbeSorterWrite( if( nMin>pSorter->nMemory ){ u8 *aNew; - int iListOff = (u8*)pSorter->list.pList - pSorter->list.aMemory; - int nNew = pSorter->nMemory * 2; + sqlite3_int64 nNew = 2 * (sqlite3_int64)pSorter->nMemory; + int iListOff = -1; + if( pSorter->list.pList ){ + iListOff = (u8*)pSorter->list.pList - pSorter->list.aMemory; + } while( nNew < nMin ) nNew = nNew*2; if( nNew > pSorter->mxPmaSize ) nNew = pSorter->mxPmaSize; if( nNew < nMin ) nNew = nMin; - aNew = sqlite3Realloc(pSorter->list.aMemory, nNew); if( !aNew ) return SQLITE_NOMEM_BKPT; - pSorter->list.pList = (SorterRecord*)&aNew[iListOff]; + if( iListOff>=0 ){ + pSorter->list.pList = (SorterRecord*)&aNew[iListOff]; + } pSorter->list.aMemory = aNew; pSorter->nMemory = nNew; } @@ -91918,7 +95770,11 @@ static int vdbeMergeEngineInit( ){ int rc = SQLITE_OK; /* Return code */ int i; /* For looping over PmaReader objects */ - int nTree = pMerger->nTree; + int nTree; /* Number of subtrees to merge */ + + /* Failure to allocate the merge would have been detected prior to + ** invoking this routine */ + assert( pMerger!=0 ); /* eMode is always INCRINIT_NORMAL in single-threaded mode */ assert( SQLITE_MAX_WORKER_THREADS>0 || eMode==INCRINIT_NORMAL ); @@ -91927,6 +95783,7 @@ static int vdbeMergeEngineInit( assert( pMerger->pTask==0 ); pMerger->pTask = pTask; + nTree = pMerger->nTree; for(i=0; i0 && eMode==INCRINIT_ROOT ){ /* PmaReaders should be normally initialized in order, as if they are @@ -92561,6 +96418,433 @@ SQLITE_PRIVATE int sqlite3VdbeSorterCompare( } /************** End of vdbesort.c ********************************************/ +/************** Begin file vdbevtab.c ****************************************/ +/* +** 2020-03-23 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file implements virtual-tables for examining the bytecode content +** of a prepared statement. +*/ +/* #include "sqliteInt.h" */ +#if defined(SQLITE_ENABLE_BYTECODE_VTAB) && !defined(SQLITE_OMIT_VIRTUALTABLE) +/* #include "vdbeInt.h" */ + +/* An instance of the bytecode() table-valued function. +*/ +typedef struct bytecodevtab bytecodevtab; +struct bytecodevtab { + sqlite3_vtab base; /* Base class - must be first */ + sqlite3 *db; /* Database connection */ + int bTablesUsed; /* 2 for tables_used(). 0 for bytecode(). */ +}; + +/* A cursor for scanning through the bytecode +*/ +typedef struct bytecodevtab_cursor bytecodevtab_cursor; +struct bytecodevtab_cursor { + sqlite3_vtab_cursor base; /* Base class - must be first */ + sqlite3_stmt *pStmt; /* The statement whose bytecode is displayed */ + int iRowid; /* The rowid of the output table */ + int iAddr; /* Address */ + int needFinalize; /* Cursors owns pStmt and must finalize it */ + int showSubprograms; /* Provide a listing of subprograms */ + Op *aOp; /* Operand array */ + char *zP4; /* Rendered P4 value */ + const char *zType; /* tables_used.type */ + const char *zSchema; /* tables_used.schema */ + const char *zName; /* tables_used.name */ + Mem sub; /* Subprograms */ +}; + +/* +** Create a new bytecode() table-valued function. +*/ +static int bytecodevtabConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + bytecodevtab *pNew; + int rc; + int isTabUsed = pAux!=0; + const char *azSchema[2] = { + /* bytecode() schema */ + "CREATE TABLE x(" + "addr INT," + "opcode TEXT," + "p1 INT," + "p2 INT," + "p3 INT," + "p4 TEXT," + "p5 INT," + "comment TEXT," + "subprog TEXT," + "stmt HIDDEN" + ");", + + /* Tables_used() schema */ + "CREATE TABLE x(" + "type TEXT," + "schema TEXT," + "name TEXT," + "wr INT," + "subprog TEXT," + "stmt HIDDEN" + ");" + }; + + rc = sqlite3_declare_vtab(db, azSchema[isTabUsed]); + if( rc==SQLITE_OK ){ + pNew = sqlite3_malloc( sizeof(*pNew) ); + *ppVtab = (sqlite3_vtab*)pNew; + if( pNew==0 ) return SQLITE_NOMEM; + memset(pNew, 0, sizeof(*pNew)); + pNew->db = db; + pNew->bTablesUsed = isTabUsed*2; + } + return rc; +} + +/* +** This method is the destructor for bytecodevtab objects. +*/ +static int bytecodevtabDisconnect(sqlite3_vtab *pVtab){ + bytecodevtab *p = (bytecodevtab*)pVtab; + sqlite3_free(p); + return SQLITE_OK; +} + +/* +** Constructor for a new bytecodevtab_cursor object. +*/ +static int bytecodevtabOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ + bytecodevtab *pVTab = (bytecodevtab*)p; + bytecodevtab_cursor *pCur; + pCur = sqlite3_malloc( sizeof(*pCur) ); + if( pCur==0 ) return SQLITE_NOMEM; + memset(pCur, 0, sizeof(*pCur)); + sqlite3VdbeMemInit(&pCur->sub, pVTab->db, 1); + *ppCursor = &pCur->base; + return SQLITE_OK; +} + +/* +** Clear all internal content from a bytecodevtab cursor. +*/ +static void bytecodevtabCursorClear(bytecodevtab_cursor *pCur){ + sqlite3_free(pCur->zP4); + pCur->zP4 = 0; + sqlite3VdbeMemRelease(&pCur->sub); + sqlite3VdbeMemSetNull(&pCur->sub); + if( pCur->needFinalize ){ + sqlite3_finalize(pCur->pStmt); + } + pCur->pStmt = 0; + pCur->needFinalize = 0; + pCur->zType = 0; + pCur->zSchema = 0; + pCur->zName = 0; +} + +/* +** Destructor for a bytecodevtab_cursor. +*/ +static int bytecodevtabClose(sqlite3_vtab_cursor *cur){ + bytecodevtab_cursor *pCur = (bytecodevtab_cursor*)cur; + bytecodevtabCursorClear(pCur); + sqlite3_free(pCur); + return SQLITE_OK; +} + + +/* +** Advance a bytecodevtab_cursor to its next row of output. +*/ +static int bytecodevtabNext(sqlite3_vtab_cursor *cur){ + bytecodevtab_cursor *pCur = (bytecodevtab_cursor*)cur; + bytecodevtab *pTab = (bytecodevtab*)cur->pVtab; + int rc; + if( pCur->zP4 ){ + sqlite3_free(pCur->zP4); + pCur->zP4 = 0; + } + if( pCur->zName ){ + pCur->zName = 0; + pCur->zType = 0; + pCur->zSchema = 0; + } + rc = sqlite3VdbeNextOpcode( + (Vdbe*)pCur->pStmt, + pCur->showSubprograms ? &pCur->sub : 0, + pTab->bTablesUsed, + &pCur->iRowid, + &pCur->iAddr, + &pCur->aOp); + if( rc!=SQLITE_OK ){ + sqlite3VdbeMemSetNull(&pCur->sub); + pCur->aOp = 0; + } + return SQLITE_OK; +} + +/* +** Return TRUE if the cursor has been moved off of the last +** row of output. +*/ +static int bytecodevtabEof(sqlite3_vtab_cursor *cur){ + bytecodevtab_cursor *pCur = (bytecodevtab_cursor*)cur; + return pCur->aOp==0; +} + +/* +** Return values of columns for the row at which the bytecodevtab_cursor +** is currently pointing. +*/ +static int bytecodevtabColumn( + sqlite3_vtab_cursor *cur, /* The cursor */ + sqlite3_context *ctx, /* First argument to sqlite3_result_...() */ + int i /* Which column to return */ +){ + bytecodevtab_cursor *pCur = (bytecodevtab_cursor*)cur; + bytecodevtab *pVTab = (bytecodevtab*)cur->pVtab; + Op *pOp = pCur->aOp + pCur->iAddr; + if( pVTab->bTablesUsed ){ + if( i==4 ){ + i = 8; + }else{ + if( i<=2 && pCur->zType==0 ){ + Schema *pSchema; + HashElem *k; + int iDb = pOp->p3; + int iRoot = pOp->p2; + sqlite3 *db = pVTab->db; + pSchema = db->aDb[iDb].pSchema; + pCur->zSchema = db->aDb[iDb].zDbSName; + for(k=sqliteHashFirst(&pSchema->tblHash); k; k=sqliteHashNext(k)){ + Table *pTab = (Table*)sqliteHashData(k); + if( !IsVirtual(pTab) && pTab->tnum==iRoot ){ + pCur->zName = pTab->zName; + pCur->zType = "table"; + break; + } + } + if( pCur->zName==0 ){ + for(k=sqliteHashFirst(&pSchema->idxHash); k; k=sqliteHashNext(k)){ + Index *pIdx = (Index*)sqliteHashData(k); + if( pIdx->tnum==iRoot ){ + pCur->zName = pIdx->zName; + pCur->zType = "index"; + } + } + } + } + i += 10; + } + } + switch( i ){ + case 0: /* addr */ + sqlite3_result_int(ctx, pCur->iAddr); + break; + case 1: /* opcode */ + sqlite3_result_text(ctx, (char*)sqlite3OpcodeName(pOp->opcode), + -1, SQLITE_STATIC); + break; + case 2: /* p1 */ + sqlite3_result_int(ctx, pOp->p1); + break; + case 3: /* p2 */ + sqlite3_result_int(ctx, pOp->p2); + break; + case 4: /* p3 */ + sqlite3_result_int(ctx, pOp->p3); + break; + case 5: /* p4 */ + case 7: /* comment */ + if( pCur->zP4==0 ){ + pCur->zP4 = sqlite3VdbeDisplayP4(pVTab->db, pOp); + } + if( i==5 ){ + sqlite3_result_text(ctx, pCur->zP4, -1, SQLITE_STATIC); + }else{ +#ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS + char *zCom = sqlite3VdbeDisplayComment(pVTab->db, pOp, pCur->zP4); + sqlite3_result_text(ctx, zCom, -1, sqlite3_free); +#endif + } + break; + case 6: /* p5 */ + sqlite3_result_int(ctx, pOp->p5); + break; + case 8: { /* subprog */ + Op *aOp = pCur->aOp; + assert( aOp[0].opcode==OP_Init ); + assert( aOp[0].p4.z==0 || strncmp(aOp[0].p4.z,"-" "- ",3)==0 ); + if( pCur->iRowid==pCur->iAddr+1 ){ + break; /* Result is NULL for the main program */ + }else if( aOp[0].p4.z!=0 ){ + sqlite3_result_text(ctx, aOp[0].p4.z+3, -1, SQLITE_STATIC); + }else{ + sqlite3_result_text(ctx, "(FK)", 4, SQLITE_STATIC); + } + break; + } + case 10: /* tables_used.type */ + sqlite3_result_text(ctx, pCur->zType, -1, SQLITE_STATIC); + break; + case 11: /* tables_used.schema */ + sqlite3_result_text(ctx, pCur->zSchema, -1, SQLITE_STATIC); + break; + case 12: /* tables_used.name */ + sqlite3_result_text(ctx, pCur->zName, -1, SQLITE_STATIC); + break; + case 13: /* tables_used.wr */ + sqlite3_result_int(ctx, pOp->opcode==OP_OpenWrite); + break; + } + return SQLITE_OK; +} + +/* +** Return the rowid for the current row. In this implementation, the +** rowid is the same as the output value. +*/ +static int bytecodevtabRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ + bytecodevtab_cursor *pCur = (bytecodevtab_cursor*)cur; + *pRowid = pCur->iRowid; + return SQLITE_OK; +} + +/* +** Initialize a cursor. +** +** idxNum==0 means show all subprograms +** idxNum==1 means show only the main bytecode and omit subprograms. +*/ +static int bytecodevtabFilter( + sqlite3_vtab_cursor *pVtabCursor, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + bytecodevtab_cursor *pCur = (bytecodevtab_cursor *)pVtabCursor; + bytecodevtab *pVTab = (bytecodevtab *)pVtabCursor->pVtab; + int rc = SQLITE_OK; + + bytecodevtabCursorClear(pCur); + pCur->iRowid = 0; + pCur->iAddr = 0; + pCur->showSubprograms = idxNum==0; + assert( argc==1 ); + if( sqlite3_value_type(argv[0])==SQLITE_TEXT ){ + const char *zSql = (const char*)sqlite3_value_text(argv[0]); + if( zSql==0 ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare_v2(pVTab->db, zSql, -1, &pCur->pStmt, 0); + pCur->needFinalize = 1; + } + }else{ + pCur->pStmt = (sqlite3_stmt*)sqlite3_value_pointer(argv[0],"stmt-pointer"); + } + if( pCur->pStmt==0 ){ + pVTab->base.zErrMsg = sqlite3_mprintf( + "argument to %s() is not a valid SQL statement", + pVTab->bTablesUsed ? "tables_used" : "bytecode" + ); + rc = SQLITE_ERROR; + }else{ + bytecodevtabNext(pVtabCursor); + } + return rc; +} + +/* +** We must have a single stmt=? constraint that will be passed through +** into the xFilter method. If there is no valid stmt=? constraint, +** then return an SQLITE_CONSTRAINT error. +*/ +static int bytecodevtabBestIndex( + sqlite3_vtab *tab, + sqlite3_index_info *pIdxInfo +){ + int i; + int rc = SQLITE_CONSTRAINT; + struct sqlite3_index_constraint *p; + bytecodevtab *pVTab = (bytecodevtab*)tab; + int iBaseCol = pVTab->bTablesUsed ? 4 : 8; + pIdxInfo->estimatedCost = (double)100; + pIdxInfo->estimatedRows = 100; + pIdxInfo->idxNum = 0; + for(i=0, p=pIdxInfo->aConstraint; inConstraint; i++, p++){ + if( p->usable==0 ) continue; + if( p->op==SQLITE_INDEX_CONSTRAINT_EQ && p->iColumn==iBaseCol+1 ){ + rc = SQLITE_OK; + pIdxInfo->aConstraintUsage[i].omit = 1; + pIdxInfo->aConstraintUsage[i].argvIndex = 1; + } + if( p->op==SQLITE_INDEX_CONSTRAINT_ISNULL && p->iColumn==iBaseCol ){ + pIdxInfo->aConstraintUsage[i].omit = 1; + pIdxInfo->idxNum = 1; + } + } + return rc; +} + +/* +** This following structure defines all the methods for the +** virtual table. +*/ +static sqlite3_module bytecodevtabModule = { + /* iVersion */ 0, + /* xCreate */ 0, + /* xConnect */ bytecodevtabConnect, + /* xBestIndex */ bytecodevtabBestIndex, + /* xDisconnect */ bytecodevtabDisconnect, + /* xDestroy */ 0, + /* xOpen */ bytecodevtabOpen, + /* xClose */ bytecodevtabClose, + /* xFilter */ bytecodevtabFilter, + /* xNext */ bytecodevtabNext, + /* xEof */ bytecodevtabEof, + /* xColumn */ bytecodevtabColumn, + /* xRowid */ bytecodevtabRowid, + /* xUpdate */ 0, + /* xBegin */ 0, + /* xSync */ 0, + /* xCommit */ 0, + /* xRollback */ 0, + /* xFindMethod */ 0, + /* xRename */ 0, + /* xSavepoint */ 0, + /* xRelease */ 0, + /* xRollbackTo */ 0, + /* xShadowName */ 0 +}; + + +SQLITE_PRIVATE int sqlite3VdbeBytecodeVtabInit(sqlite3 *db){ + int rc; + rc = sqlite3_create_module(db, "bytecode", &bytecodevtabModule, 0); + if( rc==SQLITE_OK ){ + rc = sqlite3_create_module(db, "tables_used", &bytecodevtabModule, &db); + } + return rc; +} +#elif defined(SQLITE_ENABLE_BYTECODE_VTAB) +SQLITE_PRIVATE int sqlite3VdbeBytecodeVtabInit(sqlite3 *db){ return SQLITE_OK; } +#endif /* SQLITE_ENABLE_BYTECODE_VTAB */ + +/************** End of vdbevtab.c ********************************************/ /************** Begin file memjournal.c **************************************/ /* ** 2008 October 7 @@ -92660,14 +96944,9 @@ static int memjrnlRead( int iChunkOffset; FileChunk *pChunk; -#if defined(SQLITE_ENABLE_ATOMIC_WRITE) \ - || defined(SQLITE_ENABLE_BATCH_ATOMIC_WRITE) if( (iAmt+iOfst)>p->endpoint.iOffset ){ return SQLITE_IOERR_SHORT_READ; } -#endif - - assert( (iAmt+iOfst)<=p->endpoint.iOffset ); assert( p->readpoint.iOffset==0 || p->readpoint.pChunk!=0 ); if( p->readpoint.iOffset!=iOfst || iOfst==0 ){ sqlite3_int64 iOff = 0; @@ -93018,6 +97297,35 @@ SQLITE_PRIVATE int sqlite3JournalSize(sqlite3_vfs *pVfs){ /* #include */ +#if !defined(SQLITE_OMIT_WINDOWFUNC) +/* +** Walk all expressions linked into the list of Window objects passed +** as the second argument. +*/ +static int walkWindowList(Walker *pWalker, Window *pList){ + Window *pWin; + for(pWin=pList; pWin; pWin=pWin->pNextWin){ + int rc; + rc = sqlite3WalkExprList(pWalker, pWin->pOrderBy); + if( rc ) return WRC_Abort; + rc = sqlite3WalkExprList(pWalker, pWin->pPartition); + if( rc ) return WRC_Abort; + rc = sqlite3WalkExpr(pWalker, pWin->pFilter); + if( rc ) return WRC_Abort; + + /* The next two are purely for calls to sqlite3RenameExprUnmap() + ** within sqlite3WindowOffsetExpr(). Because of constraints imposed + ** by sqlite3WindowOffsetExpr(), they can never fail. The results do + ** not matter anyhow. */ + rc = sqlite3WalkExpr(pWalker, pWin->pStart); + if( NEVER(rc) ) return WRC_Abort; + rc = sqlite3WalkExpr(pWalker, pWin->pEnd); + if( NEVER(rc) ) return WRC_Abort; + } + return WRC_Continue; +} +#endif + /* ** Walk an expression tree. Invoke the callback once for each node ** of the expression, while descending. (In other words, the callback @@ -93045,15 +97353,24 @@ static SQLITE_NOINLINE int walkExpr(Walker *pWalker, Expr *pExpr){ rc = pWalker->xExprCallback(pWalker, pExpr); if( rc ) return rc & WRC_Abort; if( !ExprHasProperty(pExpr,(EP_TokenOnly|EP_Leaf)) ){ + assert( pExpr->x.pList==0 || pExpr->pRight==0 ); if( pExpr->pLeft && walkExpr(pWalker, pExpr->pLeft) ) return WRC_Abort; - assert( pExpr->x.pList==0 || pExpr->pRight==0 ); if( pExpr->pRight ){ + assert( !ExprHasProperty(pExpr, EP_WinFunc) ); pExpr = pExpr->pRight; continue; }else if( ExprHasProperty(pExpr, EP_xIsSelect) ){ + assert( !ExprHasProperty(pExpr, EP_WinFunc) ); if( sqlite3WalkSelect(pWalker, pExpr->x.pSelect) ) return WRC_Abort; - }else if( pExpr->x.pList ){ - if( sqlite3WalkExprList(pWalker, pExpr->x.pList) ) return WRC_Abort; + }else{ + if( pExpr->x.pList ){ + if( sqlite3WalkExprList(pWalker, pExpr->x.pList) ) return WRC_Abort; + } +#ifndef SQLITE_OMIT_WINDOWFUNC + if( ExprHasProperty(pExpr, EP_WinFunc) ){ + if( walkWindowList(pWalker, pExpr->y.pWin) ) return WRC_Abort; + } +#endif } } break; @@ -93092,6 +97409,17 @@ SQLITE_PRIVATE int sqlite3WalkSelectExpr(Walker *pWalker, Select *p){ if( sqlite3WalkExpr(pWalker, p->pHaving) ) return WRC_Abort; if( sqlite3WalkExprList(pWalker, p->pOrderBy) ) return WRC_Abort; if( sqlite3WalkExpr(pWalker, p->pLimit) ) return WRC_Abort; +#if !defined(SQLITE_OMIT_WINDOWFUNC) && !defined(SQLITE_OMIT_ALTERTABLE) + { + Parse *pParse = pWalker->pParse; + if( pParse && IN_RENAME_OBJECT ){ + /* The following may return WRC_Abort if there are unresolvable + ** symbols (e.g. a table that does not exist) in a window definition. */ + int rc = walkWindowList(pWalker, p->pWinDefn); + return rc; + } + } +#endif return WRC_Continue; } @@ -93108,15 +97436,16 @@ SQLITE_PRIVATE int sqlite3WalkSelectFrom(Walker *pWalker, Select *p){ struct SrcList_item *pItem; pSrc = p->pSrc; - assert( pSrc!=0 ); - for(i=pSrc->nSrc, pItem=pSrc->a; i>0; i--, pItem++){ - if( pItem->pSelect && sqlite3WalkSelect(pWalker, pItem->pSelect) ){ - return WRC_Abort; - } - if( pItem->fg.isTabFunc - && sqlite3WalkExprList(pWalker, pItem->u1.pFuncArg) - ){ - return WRC_Abort; + if( pSrc ){ + for(i=pSrc->nSrc, pItem=pSrc->a; i>0; i--, pItem++){ + if( pItem->pSelect && sqlite3WalkSelect(pWalker, pItem->pSelect) ){ + return WRC_Abort; + } + if( pItem->fg.isTabFunc + && sqlite3WalkExprList(pWalker, pItem->u1.pFuncArg) + ){ + return WRC_Abort; + } } } return WRC_Continue; @@ -93159,6 +97488,43 @@ SQLITE_PRIVATE int sqlite3WalkSelect(Walker *pWalker, Select *p){ return WRC_Continue; } +/* Increase the walkerDepth when entering a subquery, and +** descrease when leaving the subquery. +*/ +SQLITE_PRIVATE int sqlite3WalkerDepthIncrease(Walker *pWalker, Select *pSelect){ + UNUSED_PARAMETER(pSelect); + pWalker->walkerDepth++; + return WRC_Continue; +} +SQLITE_PRIVATE void sqlite3WalkerDepthDecrease(Walker *pWalker, Select *pSelect){ + UNUSED_PARAMETER(pSelect); + pWalker->walkerDepth--; +} + + +/* +** No-op routine for the parse-tree walker. +** +** When this routine is the Walker.xExprCallback then expression trees +** are walked without any actions being taken at each node. Presumably, +** when this routine is used for Walker.xExprCallback then +** Walker.xSelectCallback is set to do something useful for every +** subquery in the parser tree. +*/ +SQLITE_PRIVATE int sqlite3ExprWalkNoop(Walker *NotUsed, Expr *NotUsed2){ + UNUSED_PARAMETER2(NotUsed, NotUsed2); + return WRC_Continue; +} + +/* +** No-op routine for the parse-tree walker for SELECT statements. +** subquery in the parser tree. +*/ +SQLITE_PRIVATE int sqlite3SelectWalkNoop(Walker *NotUsed, Select *NotUsed2){ + UNUSED_PARAMETER2(NotUsed, NotUsed2); + return WRC_Continue; +} + /************** End of walker.c **********************************************/ /************** Begin file resolve.c *****************************************/ /* @@ -93187,6 +97553,8 @@ SQLITE_PRIVATE int sqlite3WalkSelect(Walker *pWalker, Select *p){ ** ** incrAggFunctionDepth(pExpr,n) is the main routine. incrAggDepth(..) ** is a helper function - a callback for the tree walker. +** +** See also the sqlite3WindowExtraAggFuncDepth() routine in window.c */ static int incrAggDepth(Walker *pWalker, Expr *pExpr){ if( pExpr->op==TK_AGG_FUNCTION ) pExpr->op2 += pWalker->u.n; @@ -93243,7 +97611,6 @@ static void resolveAlias( if( pExpr->op==TK_COLLATE ){ pDup = sqlite3ExprAddCollateString(pParse, pDup, pExpr->u.zToken); } - ExprSetProperty(pDup, EP_Alias); /* Before calling sqlite3ExprDelete(), set the EP_Static flag. This ** prevents ExprDelete() from deleting the Expr structure itself, @@ -93260,6 +97627,13 @@ static void resolveAlias( pExpr->u.zToken = sqlite3DbStrDup(db, pExpr->u.zToken); pExpr->flags |= EP_MemToken; } + if( ExprHasProperty(pExpr, EP_WinFunc) ){ + if( pExpr->y.pWin!=0 ){ + pExpr->y.pWin->pOwner = pExpr; + }else{ + assert( db->mallocFailed ); + } + } sqlite3DbFree(db, pDup); } ExprSetProperty(pExpr, EP_Alias); @@ -93289,13 +97663,16 @@ static int nameInUsingClause(IdList *pUsing, const char *zCol){ ** and zCol. If any of zDb, zTab, and zCol are NULL then those fields will ** match anything. */ -SQLITE_PRIVATE int sqlite3MatchSpanName( - const char *zSpan, +SQLITE_PRIVATE int sqlite3MatchEName( + const struct ExprList_item *pItem, const char *zCol, const char *zTab, const char *zDb ){ int n; + const char *zSpan; + if( pItem->eEName!=ENAME_TAB ) return 0; + zSpan = pItem->zEName; for(n=0; ALWAYS(zSpan[n]) && zSpan[n]!='.'; n++){} if( zDb && (sqlite3StrNICmp(zSpan, zDb, n)!=0 || zDb[n]!=0) ){ return 0; @@ -93312,6 +97689,48 @@ SQLITE_PRIVATE int sqlite3MatchSpanName( return 1; } +/* +** Return TRUE if the double-quoted string mis-feature should be supported. +*/ +static int areDoubleQuotedStringsEnabled(sqlite3 *db, NameContext *pTopNC){ + if( db->init.busy ) return 1; /* Always support for legacy schemas */ + if( pTopNC->ncFlags & NC_IsDDL ){ + /* Currently parsing a DDL statement */ + if( sqlite3WritableSchema(db) && (db->flags & SQLITE_DqsDML)!=0 ){ + return 1; + } + return (db->flags & SQLITE_DqsDDL)!=0; + }else{ + /* Currently parsing a DML statement */ + return (db->flags & SQLITE_DqsDML)!=0; + } +} + +/* +** The argument is guaranteed to be a non-NULL Expr node of type TK_COLUMN. +** return the appropriate colUsed mask. +*/ +SQLITE_PRIVATE Bitmask sqlite3ExprColUsed(Expr *pExpr){ + int n; + Table *pExTab; + + n = pExpr->iColumn; + pExTab = pExpr->y.pTab; + assert( pExTab!=0 ); + if( (pExTab->tabFlags & TF_HasGenerated)!=0 + && (pExTab->aCol[n].colFlags & COLFLAG_GENERATED)!=0 + ){ + testcase( pExTab->nCol==BMS-1 ); + testcase( pExTab->nCol==BMS ); + return pExTab->nCol>=BMS ? ALLBITS : MASKBIT(pExTab->nCol)-1; + }else{ + testcase( n==BMS-1 ); + testcase( n==BMS ); + if( n>=BMS ) n = BMS-1; + return ((Bitmask)1)<iTable Set to the cursor number for the table obtained ** from pSrcList. -** pExpr->pTab Points to the Table structure of X.Y (even if +** pExpr->y.pTab Points to the Table structure of X.Y (even if ** X and/or Y are implied.) ** pExpr->iColumn Set to the column number within the table. ** pExpr->op Set to TK_COLUMN. @@ -93366,7 +97785,6 @@ static int lookupName( /* Initialize the node to no-match */ pExpr->iTable = -1; - pExpr->pTab = 0; ExprSetVVAProperty(pExpr, EP_NoReduce); /* Translate the schema name in zDb into a pointer to the corresponding @@ -93390,6 +97808,12 @@ static int lookupName( break; } } + if( i==db->nDb && sqlite3StrICmp("main", zDb)==0 ){ + /* This branch is taken when the main database has been renamed + ** using SQLITE_DBCONFIG_MAINDBNAME. */ + pSchema = db->aDb[0].pSchema; + zDb = db->aDb[0].zDbSName; + } } } @@ -93401,6 +97825,7 @@ static int lookupName( if( pSrcList ){ for(i=0, pItem=pSrcList->a; inSrc; i++, pItem++){ + u8 hCol; pTab = pItem->pTab; assert( pTab!=0 && pTab->zName!=0 ); assert( pTab->nCol>0 ); @@ -93408,7 +97833,7 @@ static int lookupName( int hit = 0; pEList = pItem->pSelect->pEList; for(j=0; jnExpr; j++){ - if( sqlite3MatchSpanName(pEList->a[j].zSpan, zCol, zTab, zDb) ){ + if( sqlite3MatchEName(&pEList->a[j], zCol, zTab, zDb) ){ cnt++; cntTab = 2; pMatch = pItem; @@ -93427,12 +97852,16 @@ static int lookupName( if( sqlite3StrICmp(zTabName, zTab)!=0 ){ continue; } + if( IN_RENAME_OBJECT && pItem->zAlias ){ + sqlite3RenameTokenRemap(pParse, 0, (void*)&pExpr->y.pTab); + } } if( 0==(cntTab++) ){ pMatch = pItem; } + hCol = sqlite3StrIHash(zCol); for(j=0, pCol=pTab->aCol; jnCol; j++, pCol++){ - if( sqlite3StrICmp(pCol->zName, zCol)==0 ){ + if( pCol->hName==hCol && sqlite3StrICmp(pCol->zName, zCol)==0 ){ /* If there has been exactly one prior match and this match ** is for the right-hand table of a NATURAL JOIN or is in a ** USING clause, then skip this match. @@ -93451,13 +97880,13 @@ static int lookupName( } if( pMatch ){ pExpr->iTable = pMatch->iCursor; - pExpr->pTab = pMatch->pTab; + pExpr->y.pTab = pMatch->pTab; /* RIGHT JOIN not (yet) supported */ assert( (pMatch->fg.jointype & JT_RIGHT)==0 ); if( (pMatch->fg.jointype & JT_LEFT)!=0 ){ ExprSetProperty(pExpr, EP_CanBeNull); } - pSchema = pExpr->pTab->pSchema; + pSchema = pExpr->y.pTab->pSchema; } } /* if( pSrcList ) */ @@ -93493,10 +97922,11 @@ static int lookupName( if( pTab ){ int iCol; + u8 hCol = sqlite3StrIHash(zCol); pSchema = pTab->pSchema; cntTab++; for(iCol=0, pCol=pTab->aCol; iColnCol; iCol++, pCol++){ - if( sqlite3StrICmp(pCol->zName, zCol)==0 ){ + if( pCol->hName==hCol && sqlite3StrICmp(pCol->zName, zCol)==0 ){ if( iCol==pTab->iPKey ){ iCol = -1; } @@ -93512,15 +97942,21 @@ static int lookupName( #ifndef SQLITE_OMIT_UPSERT if( pExpr->iTable==2 ){ testcase( iCol==(-1) ); - pExpr->iTable = pNC->uNC.pUpsert->regData + iCol; - eNewExprOp = TK_REGISTER; - ExprSetProperty(pExpr, EP_Alias); + if( IN_RENAME_OBJECT ){ + pExpr->iColumn = iCol; + pExpr->y.pTab = pTab; + eNewExprOp = TK_COLUMN; + }else{ + pExpr->iTable = pNC->uNC.pUpsert->regData + iCol; + eNewExprOp = TK_REGISTER; + ExprSetProperty(pExpr, EP_Alias); + } }else #endif /* SQLITE_OMIT_UPSERT */ { #ifndef SQLITE_OMIT_TRIGGER if( iCol<0 ){ - pExpr->affinity = SQLITE_AFF_INTEGER; + pExpr->affExpr = SQLITE_AFF_INTEGER; }else if( pExpr->iTable==0 ){ testcase( iCol==31 ); testcase( iCol==32 ); @@ -93530,7 +97966,7 @@ static int lookupName( testcase( iCol==32 ); pParse->newmask |= (iCol>=32 ? 0xffffffff : (((u32)1)<pTab = pTab; + pExpr->y.pTab = pTab; pExpr->iColumn = (i16)iCol; eNewExprOp = TK_TRIGGER; #endif /* SQLITE_OMIT_TRIGGER */ @@ -93546,13 +97982,13 @@ static int lookupName( if( cnt==0 && cntTab==1 && pMatch - && (pNC->ncFlags & NC_IdxExpr)==0 + && (pNC->ncFlags & (NC_IdxExpr|NC_GenCol))==0 && sqlite3IsRowid(zCol) && VisibleRowid(pMatch->pTab) ){ cnt = 1; pExpr->iColumn = -1; - pExpr->affinity = SQLITE_AFF_INTEGER; + pExpr->affExpr = SQLITE_AFF_INTEGER; } /* @@ -93580,8 +98016,10 @@ static int lookupName( pEList = pNC->uNC.pEList; assert( pEList!=0 ); for(j=0; jnExpr; j++){ - char *zAs = pEList->a[j].zName; - if( zAs!=0 && sqlite3StrICmp(zAs, zCol)==0 ){ + char *zAs = pEList->a[j].zEName; + if( pEList->a[j].eEName==ENAME_NAME + && sqlite3_stricmp(zAs, zCol)==0 + ){ Expr *pOrig; assert( pExpr->pLeft==0 && pExpr->pRight==0 ); assert( pExpr->x.pList==0 ); @@ -93591,6 +98029,12 @@ static int lookupName( sqlite3ErrorMsg(pParse, "misuse of aliased aggregate %s", zAs); return WRC_Abort; } + if( ExprHasProperty(pOrig, EP_Win) + && ((pNC->ncFlags&NC_AllowWin)==0 || pNC!=pTopNC ) + ){ + sqlite3ErrorMsg(pParse, "misuse of aliased window function %s",zAs); + return WRC_Abort; + } if( sqlite3ExprVectorSize(pOrig)!=1 ){ sqlite3ErrorMsg(pParse, "row value misused"); return WRC_Abort; @@ -93599,6 +98043,9 @@ static int lookupName( cnt = 1; pMatch = 0; assert( zTab==0 && zDb==0 ); + if( IN_RENAME_OBJECT ){ + sqlite3RenameTokenRemap(pParse, 0, (void*)pExpr); + } goto lookupname_end; } } @@ -93625,9 +98072,30 @@ static int lookupName( */ if( cnt==0 && zTab==0 ){ assert( pExpr->op==TK_ID ); - if( ExprHasProperty(pExpr,EP_DblQuoted) ){ + if( ExprHasProperty(pExpr,EP_DblQuoted) + && areDoubleQuotedStringsEnabled(db, pTopNC) + ){ + /* If a double-quoted identifier does not match any known column name, + ** then treat it as a string. + ** + ** This hack was added in the early days of SQLite in a misguided attempt + ** to be compatible with MySQL 3.x, which used double-quotes for strings. + ** I now sorely regret putting in this hack. The effect of this hack is + ** that misspelled identifier names are silently converted into strings + ** rather than causing an error, to the frustration of countless + ** programmers. To all those frustrated programmers, my apologies. + ** + ** Someday, I hope to get rid of this hack. Unfortunately there is + ** a huge amount of legacy SQL that uses it. So for now, we just + ** issue a warning. + */ + sqlite3_log(SQLITE_WARNING, + "double-quoted string literal: \"%w\"", zCol); +#ifdef SQLITE_ENABLE_NORMALIZE + sqlite3VdbeAddDblquoteStr(db, pParse->pVdbe, zCol); +#endif pExpr->op = TK_STRING; - pExpr->pTab = 0; + pExpr->y.pTab = 0; return WRC_Prune; } if( sqlite3ExprIdToTrueFalse(pExpr) ){ @@ -93655,18 +98123,20 @@ static int lookupName( /* If a column from a table in pSrcList is referenced, then record ** this fact in the pSrcList.a[].colUsed bitmask. Column 0 causes - ** bit 0 to be set. Column 1 sets bit 1. And so forth. If the - ** column number is greater than the number of bits in the bitmask - ** then set the high-order bit of the bitmask. + ** bit 0 to be set. Column 1 sets bit 1. And so forth. Bit 63 is + ** set if the 63rd or any subsequent column is used. + ** + ** The colUsed mask is an optimization used to help determine if an + ** index is a covering index. The correct answer is still obtained + ** if the mask contains extra set bits. However, it is important to + ** avoid setting bits beyond the maximum column number of the table. + ** (See ticket [b92e5e8ec2cdbaa1]). + ** + ** If a generated column is referenced, set bits for every column + ** of the table. */ if( pExpr->iColumn>=0 && pMatch!=0 ){ - int n = pExpr->iColumn; - testcase( n==BMS-1 ); - if( n>=BMS ){ - n = BMS-1; - } - assert( pMatch->iCursor==pExpr->iTable ); - pMatch->colUsed |= ((Bitmask)1)<colUsed |= sqlite3ExprColUsed(pExpr); } /* Clean up and return @@ -93705,15 +98175,23 @@ SQLITE_PRIVATE Expr *sqlite3CreateColumnExpr(sqlite3 *db, SrcList *pSrc, int iSr Expr *p = sqlite3ExprAlloc(db, TK_COLUMN, 0, 0); if( p ){ struct SrcList_item *pItem = &pSrc->a[iSrc]; - p->pTab = pItem->pTab; + Table *pTab = p->y.pTab = pItem->pTab; p->iTable = pItem->iCursor; - if( p->pTab->iPKey==iCol ){ + if( p->y.pTab->iPKey==iCol ){ p->iColumn = -1; }else{ p->iColumn = (ynVar)iCol; - testcase( iCol==BMS ); - testcase( iCol==BMS-1 ); - pItem->colUsed |= ((Bitmask)1)<<(iCol>=BMS ? BMS-1 : iCol); + if( (pTab->tabFlags & TF_HasGenerated)!=0 + && (pTab->aCol[iCol].colFlags & COLFLAG_GENERATED)!=0 + ){ + testcase( pTab->nCol==63 ); + testcase( pTab->nCol==64 ); + pItem->colUsed = pTab->nCol>=64 ? ALLBITS : MASKBIT(pTab->nCol)-1; + }else{ + testcase( iCol==BMS ); + testcase( iCol==BMS-1 ); + pItem->colUsed |= ((Bitmask)1)<<(iCol>=BMS ? BMS-1 : iCol); + } } } return p; @@ -93722,23 +98200,39 @@ SQLITE_PRIVATE Expr *sqlite3CreateColumnExpr(sqlite3 *db, SrcList *pSrc, int iSr /* ** Report an error that an expression is not valid for some set of ** pNC->ncFlags values determined by validMask. -*/ -static void notValid( - Parse *pParse, /* Leave error message here */ - NameContext *pNC, /* The name context */ - const char *zMsg, /* Type of error */ - int validMask /* Set of contexts for which prohibited */ -){ - assert( (validMask&~(NC_IsCheck|NC_PartIdx|NC_IdxExpr))==0 ); - if( (pNC->ncFlags & validMask)!=0 ){ - const char *zIn = "partial index WHERE clauses"; - if( pNC->ncFlags & NC_IdxExpr ) zIn = "index expressions"; +** +** static void notValid( +** Parse *pParse, // Leave error message here +** NameContext *pNC, // The name context +** const char *zMsg, // Type of error +** int validMask, // Set of contexts for which prohibited +** Expr *pExpr // Invalidate this expression on error +** ){...} +** +** As an optimization, since the conditional is almost always false +** (because errors are rare), the conditional is moved outside of the +** function call using a macro. +*/ +static void notValidImpl( + Parse *pParse, /* Leave error message here */ + NameContext *pNC, /* The name context */ + const char *zMsg, /* Type of error */ + Expr *pExpr /* Invalidate this expression on error */ +){ + const char *zIn = "partial index WHERE clauses"; + if( pNC->ncFlags & NC_IdxExpr ) zIn = "index expressions"; #ifndef SQLITE_OMIT_CHECK - else if( pNC->ncFlags & NC_IsCheck ) zIn = "CHECK constraints"; + else if( pNC->ncFlags & NC_IsCheck ) zIn = "CHECK constraints"; #endif - sqlite3ErrorMsg(pParse, "%s prohibited in %s", zMsg, zIn); - } +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + else if( pNC->ncFlags & NC_GenCol ) zIn = "generated columns"; +#endif + sqlite3ErrorMsg(pParse, "%s prohibited in %s", zMsg, zIn); + if( pExpr ) pExpr->op = TK_NULL; } +#define sqlite3ResolveNotValid(P,N,M,X,E) \ + assert( ((X)&~(NC_IsCheck|NC_PartIdx|NC_IdxExpr|NC_GenCol))==0 ); \ + if( ((N)->ncFlags & (X))!=0 ) notValidImpl(P,N,M,E); /* ** Expression p should encode a floating point value between 1.0 and 0.0. @@ -93797,10 +98291,10 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ pItem = pSrcList->a; assert( HasRowid(pItem->pTab) && pItem->pTab->pSelect==0 ); pExpr->op = TK_COLUMN; - pExpr->pTab = pItem->pTab; + pExpr->y.pTab = pItem->pTab; pExpr->iTable = pItem->iCursor; pExpr->iColumn = -1; - pExpr->affinity = SQLITE_AFF_INTEGER; + pExpr->affExpr = SQLITE_AFF_INTEGER; break; } #endif /* defined(SQLITE_ENABLE_UPDATE_DELETE_LIMIT) @@ -93826,17 +98320,25 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ zTable = 0; zColumn = pExpr->u.zToken; }else{ - notValid(pParse, pNC, "the \".\" operator", NC_IdxExpr); + Expr *pLeft = pExpr->pLeft; + testcase( pNC->ncFlags & NC_IdxExpr ); + testcase( pNC->ncFlags & NC_GenCol ); + sqlite3ResolveNotValid(pParse, pNC, "the \".\" operator", + NC_IdxExpr|NC_GenCol, 0); pRight = pExpr->pRight; if( pRight->op==TK_ID ){ zDb = 0; - zTable = pExpr->pLeft->u.zToken; - zColumn = pRight->u.zToken; }else{ assert( pRight->op==TK_DOT ); - zDb = pExpr->pLeft->u.zToken; - zTable = pRight->pLeft->u.zToken; - zColumn = pRight->pRight->u.zToken; + zDb = pLeft->u.zToken; + pLeft = pRight->pLeft; + pRight = pRight->pRight; + } + zTable = pLeft->u.zToken; + zColumn = pRight->u.zToken; + if( IN_RENAME_OBJECT ){ + sqlite3RenameTokenRemap(pParse, (void*)pExpr, (void*)pRight); + sqlite3RenameTokenRemap(pParse, (void*)&pExpr->y.pTab, (void*)pLeft); } } return lookupName(pParse, zDb, zTable, zColumn, pNC, pExpr); @@ -93854,7 +98356,10 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ const char *zId; /* The function name. */ FuncDef *pDef; /* Information about the function */ u8 enc = ENC(pParse->db); /* The database encoding */ - + int savedAllowFlags = (pNC->ncFlags & (NC_AllowAgg | NC_AllowWin)); +#ifndef SQLITE_OMIT_WINDOWFUNC + Window *pWin = (IsWindowFunc(pExpr) ? pExpr->y.pWin : 0); +#endif assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); zId = pExpr->u.zToken; nId = sqlite3Strlen30(zId); @@ -93869,7 +98374,7 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ }else{ is_agg = pDef->xFinalize!=0; if( pDef->funcFlags & SQLITE_FUNC_UNLIKELY ){ - ExprSetProperty(pExpr, EP_Unlikely|EP_Skip); + ExprSetProperty(pExpr, EP_Unlikely); if( n==2 ){ pExpr->iTable = exprProbability(pList->a[1].pExpr); if( pExpr->iTable<0 ){ @@ -93908,51 +98413,150 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ if( pDef->funcFlags & (SQLITE_FUNC_CONSTANT|SQLITE_FUNC_SLOCHNG) ){ /* For the purposes of the EP_ConstFunc flag, date and time ** functions and other functions that change slowly are considered - ** constant because they are constant for the duration of one query */ + ** constant because they are constant for the duration of one query. + ** This allows them to be factored out of inner loops. */ ExprSetProperty(pExpr,EP_ConstFunc); } if( (pDef->funcFlags & SQLITE_FUNC_CONSTANT)==0 ){ - /* Date/time functions that use 'now', and other functions like + /* Clearly non-deterministic functions like random(), but also + ** date/time functions that use 'now', and other functions like ** sqlite_version() that might change over time cannot be used - ** in an index. */ - notValid(pParse, pNC, "non-deterministic functions", - NC_IdxExpr|NC_PartIdx); + ** in an index or generated column. Curiously, they can be used + ** in a CHECK constraint. SQLServer, MySQL, and PostgreSQL all + ** all this. */ + sqlite3ResolveNotValid(pParse, pNC, "non-deterministic functions", + NC_IdxExpr|NC_PartIdx|NC_GenCol, 0); + }else{ + assert( (NC_SelfRef & 0xff)==NC_SelfRef ); /* Must fit in 8 bits */ + pExpr->op2 = pNC->ncFlags & NC_SelfRef; + if( pNC->ncFlags & NC_FromDDL ) ExprSetProperty(pExpr, EP_FromDDL); + } + if( (pDef->funcFlags & SQLITE_FUNC_INTERNAL)!=0 + && pParse->nested==0 + && (pParse->db->mDbFlags & DBFLAG_InternalFunc)==0 + ){ + /* Internal-use-only functions are disallowed unless the + ** SQL is being compiled using sqlite3NestedParse() or + ** the SQLITE_TESTCTRL_INTERNAL_FUNCTIONS test-control has be + ** used to activate internal functionsn for testing purposes */ + no_such_func = 1; + pDef = 0; + }else + if( (pDef->funcFlags & (SQLITE_FUNC_DIRECT|SQLITE_FUNC_UNSAFE))!=0 + && !IN_RENAME_OBJECT + ){ + sqlite3ExprFunctionUsable(pParse, pExpr, pDef); } } - if( is_agg && (pNC->ncFlags & NC_AllowAgg)==0 ){ - sqlite3ErrorMsg(pParse, "misuse of aggregate function %.*s()", nId,zId); - pNC->nErr++; - is_agg = 0; - }else if( no_such_func && pParse->db->init.busy==0 + + if( 0==IN_RENAME_OBJECT ){ +#ifndef SQLITE_OMIT_WINDOWFUNC + assert( is_agg==0 || (pDef->funcFlags & SQLITE_FUNC_MINMAX) + || (pDef->xValue==0 && pDef->xInverse==0) + || (pDef->xValue && pDef->xInverse && pDef->xSFunc && pDef->xFinalize) + ); + if( pDef && pDef->xValue==0 && pWin ){ + sqlite3ErrorMsg(pParse, + "%.*s() may not be used as a window function", nId, zId + ); + pNC->nErr++; + }else if( + (is_agg && (pNC->ncFlags & NC_AllowAgg)==0) + || (is_agg && (pDef->funcFlags&SQLITE_FUNC_WINDOW) && !pWin) + || (is_agg && pWin && (pNC->ncFlags & NC_AllowWin)==0) + ){ + const char *zType; + if( (pDef->funcFlags & SQLITE_FUNC_WINDOW) || pWin ){ + zType = "window"; + }else{ + zType = "aggregate"; + } + sqlite3ErrorMsg(pParse, "misuse of %s function %.*s()",zType,nId,zId); + pNC->nErr++; + is_agg = 0; + } +#else + if( (is_agg && (pNC->ncFlags & NC_AllowAgg)==0) ){ + sqlite3ErrorMsg(pParse,"misuse of aggregate function %.*s()",nId,zId); + pNC->nErr++; + is_agg = 0; + } +#endif + else if( no_such_func && pParse->db->init.busy==0 #ifdef SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION - && pParse->explain==0 + && pParse->explain==0 +#endif + ){ + sqlite3ErrorMsg(pParse, "no such function: %.*s", nId, zId); + pNC->nErr++; + }else if( wrong_num_args ){ + sqlite3ErrorMsg(pParse,"wrong number of arguments to function %.*s()", + nId, zId); + pNC->nErr++; + } +#ifndef SQLITE_OMIT_WINDOWFUNC + else if( is_agg==0 && ExprHasProperty(pExpr, EP_WinFunc) ){ + sqlite3ErrorMsg(pParse, + "FILTER may not be used with non-aggregate %.*s()", + nId, zId + ); + pNC->nErr++; + } +#endif + if( is_agg ){ + /* Window functions may not be arguments of aggregate functions. + ** Or arguments of other window functions. But aggregate functions + ** may be arguments for window functions. */ +#ifndef SQLITE_OMIT_WINDOWFUNC + pNC->ncFlags &= ~(NC_AllowWin | (!pWin ? NC_AllowAgg : 0)); +#else + pNC->ncFlags &= ~NC_AllowAgg; +#endif + } + } +#ifndef SQLITE_OMIT_WINDOWFUNC + else if( ExprHasProperty(pExpr, EP_WinFunc) ){ + is_agg = 1; + } #endif - ){ - sqlite3ErrorMsg(pParse, "no such function: %.*s", nId, zId); - pNC->nErr++; - }else if( wrong_num_args ){ - sqlite3ErrorMsg(pParse,"wrong number of arguments to function %.*s()", - nId, zId); - pNC->nErr++; - } - if( is_agg ) pNC->ncFlags &= ~NC_AllowAgg; sqlite3WalkExprList(pWalker, pList); if( is_agg ){ - NameContext *pNC2 = pNC; - pExpr->op = TK_AGG_FUNCTION; - pExpr->op2 = 0; - while( pNC2 && !sqlite3FunctionUsesThisSrc(pExpr, pNC2->pSrcList) ){ - pExpr->op2++; - pNC2 = pNC2->pNext; - } - assert( pDef!=0 ); - if( pNC2 ){ - assert( SQLITE_FUNC_MINMAX==NC_MinMaxAgg ); - testcase( (pDef->funcFlags & SQLITE_FUNC_MINMAX)!=0 ); - pNC2->ncFlags |= NC_HasAgg | (pDef->funcFlags & SQLITE_FUNC_MINMAX); +#ifndef SQLITE_OMIT_WINDOWFUNC + if( pWin ){ + Select *pSel = pNC->pWinSelect; + assert( pWin==pExpr->y.pWin ); + if( IN_RENAME_OBJECT==0 ){ + sqlite3WindowUpdate(pParse, pSel ? pSel->pWinDefn : 0, pWin, pDef); + } + sqlite3WalkExprList(pWalker, pWin->pPartition); + sqlite3WalkExprList(pWalker, pWin->pOrderBy); + sqlite3WalkExpr(pWalker, pWin->pFilter); + sqlite3WindowLink(pSel, pWin); + pNC->ncFlags |= NC_HasWin; + }else +#endif /* SQLITE_OMIT_WINDOWFUNC */ + { + NameContext *pNC2 = pNC; + pExpr->op = TK_AGG_FUNCTION; + pExpr->op2 = 0; +#ifndef SQLITE_OMIT_WINDOWFUNC + if( ExprHasProperty(pExpr, EP_WinFunc) ){ + sqlite3WalkExpr(pWalker, pExpr->y.pWin->pFilter); + } +#endif + while( pNC2 && !sqlite3FunctionUsesThisSrc(pExpr, pNC2->pSrcList) ){ + pExpr->op2++; + pNC2 = pNC2->pNext; + } + assert( pDef!=0 || IN_RENAME_OBJECT ); + if( pNC2 && pDef ){ + assert( SQLITE_FUNC_MINMAX==NC_MinMaxAgg ); + testcase( (pDef->funcFlags & SQLITE_FUNC_MINMAX)!=0 ); + pNC2->ncFlags |= NC_HasAgg | (pDef->funcFlags & SQLITE_FUNC_MINMAX); + } } - pNC->ncFlags |= NC_AllowAgg; + pNC->ncFlags |= savedAllowFlags; } /* FIX ME: Compute pExpr->affinity based on the expected return ** type of the function @@ -93967,7 +98571,12 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ testcase( pExpr->op==TK_IN ); if( ExprHasProperty(pExpr, EP_xIsSelect) ){ int nRef = pNC->nRef; - notValid(pParse, pNC, "subqueries", NC_IsCheck|NC_PartIdx|NC_IdxExpr); + testcase( pNC->ncFlags & NC_IsCheck ); + testcase( pNC->ncFlags & NC_PartIdx ); + testcase( pNC->ncFlags & NC_IdxExpr ); + testcase( pNC->ncFlags & NC_GenCol ); + sqlite3ResolveNotValid(pParse, pNC, "subqueries", + NC_IsCheck|NC_PartIdx|NC_IdxExpr|NC_GenCol, pExpr); sqlite3WalkSelect(pWalker, pExpr->x.pSelect); assert( pNC->nRef>=nRef ); if( nRef!=pNC->nRef ){ @@ -93978,16 +98587,21 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ break; } case TK_VARIABLE: { - notValid(pParse, pNC, "parameters", NC_IsCheck|NC_PartIdx|NC_IdxExpr); + testcase( pNC->ncFlags & NC_IsCheck ); + testcase( pNC->ncFlags & NC_PartIdx ); + testcase( pNC->ncFlags & NC_IdxExpr ); + testcase( pNC->ncFlags & NC_GenCol ); + sqlite3ResolveNotValid(pParse, pNC, "parameters", + NC_IsCheck|NC_PartIdx|NC_IdxExpr|NC_GenCol, pExpr); break; } case TK_IS: case TK_ISNOT: { - Expr *pRight; + Expr *pRight = sqlite3ExprSkipCollateAndLikely(pExpr->pRight); assert( !ExprHasProperty(pExpr, EP_Reduced) ); /* Handle special cases of "x IS TRUE", "x IS FALSE", "x IS NOT TRUE", ** and "x IS NOT FALSE". */ - if( (pRight = pExpr->pRight)->op==TK_ID ){ + if( pRight && pRight->op==TK_ID ){ int rc = resolveExprStep(pWalker, pRight); if( rc==WRC_Abort ) return WRC_Abort; if( pRight->op==TK_TRUEFALSE ){ @@ -94060,8 +98674,9 @@ static int resolveAsName( if( pE->op==TK_ID ){ char *zCol = pE->u.zToken; for(i=0; inExpr; i++){ - char *zAs = pEList->a[i].zName; - if( zAs!=0 && sqlite3StrICmp(zAs, zCol)==0 ){ + if( pEList->a[i].eEName==ENAME_NAME + && sqlite3_stricmp(pEList->a[i].zEName, zCol)==0 + ){ return i+1; } } @@ -94112,7 +98727,7 @@ static int resolveOrderByTermToExprList( nc.nErr = 0; db = pParse->db; savedSuppErr = db->suppressErr; - db->suppressErr = 1; + if( IN_RENAME_OBJECT==0 ) db->suppressErr = 1; rc = sqlite3ResolveExprNames(&nc, pE); db->suppressErr = savedSuppErr; if( rc ) return 0; @@ -94194,7 +98809,7 @@ static int resolveCompoundOrderBy( int iCol = -1; Expr *pE, *pDup; if( pItem->done ) continue; - pE = sqlite3ExprSkipCollate(pItem->pExpr); + pE = sqlite3ExprSkipCollateAndLikely(pItem->pExpr); if( sqlite3ExprIsInteger(pE, &iCol) ){ if( iCol<=0 || iCol>pEList->nExpr ){ resolveOutOfRangeError(pParse, "ORDER", i+1, pEList->nExpr); @@ -94203,32 +98818,53 @@ static int resolveCompoundOrderBy( }else{ iCol = resolveAsName(pParse, pEList, pE); if( iCol==0 ){ - pDup = sqlite3ExprDup(db, pE, 0); + /* Now test if expression pE matches one of the values returned + ** by pSelect. In the usual case this is done by duplicating the + ** expression, resolving any symbols in it, and then comparing + ** it against each expression returned by the SELECT statement. + ** Once the comparisons are finished, the duplicate expression + ** is deleted. + ** + ** Or, if this is running as part of an ALTER TABLE operation, + ** resolve the symbols in the actual expression, not a duplicate. + ** And, if one of the comparisons is successful, leave the expression + ** as is instead of transforming it to an integer as in the usual + ** case. This allows the code in alter.c to modify column + ** refererences within the ORDER BY expression as required. */ + if( IN_RENAME_OBJECT ){ + pDup = pE; + }else{ + pDup = sqlite3ExprDup(db, pE, 0); + } if( !db->mallocFailed ){ assert(pDup); iCol = resolveOrderByTermToExprList(pParse, pSelect, pDup); } - sqlite3ExprDelete(db, pDup); + if( !IN_RENAME_OBJECT ){ + sqlite3ExprDelete(db, pDup); + } } } if( iCol>0 ){ /* Convert the ORDER BY term into an integer column number iCol, ** taking care to preserve the COLLATE clause if it exists */ - Expr *pNew = sqlite3Expr(db, TK_INTEGER, 0); - if( pNew==0 ) return 1; - pNew->flags |= EP_IntValue; - pNew->u.iValue = iCol; - if( pItem->pExpr==pE ){ - pItem->pExpr = pNew; - }else{ - Expr *pParent = pItem->pExpr; - assert( pParent->op==TK_COLLATE ); - while( pParent->pLeft->op==TK_COLLATE ) pParent = pParent->pLeft; - assert( pParent->pLeft==pE ); - pParent->pLeft = pNew; + if( !IN_RENAME_OBJECT ){ + Expr *pNew = sqlite3Expr(db, TK_INTEGER, 0); + if( pNew==0 ) return 1; + pNew->flags |= EP_IntValue; + pNew->u.iValue = iCol; + if( pItem->pExpr==pE ){ + pItem->pExpr = pNew; + }else{ + Expr *pParent = pItem->pExpr; + assert( pParent->op==TK_COLLATE ); + while( pParent->pLeft->op==TK_COLLATE ) pParent = pParent->pLeft; + assert( pParent->pLeft==pE ); + pParent->pLeft = pNew; + } + sqlite3ExprDelete(db, pE); + pItem->u.x.iOrderByCol = (u16)iCol; } - sqlite3ExprDelete(db, pE); - pItem->u.x.iOrderByCol = (u16)iCol; pItem->done = 1; }else{ moreToDo = 1; @@ -94267,7 +98903,7 @@ SQLITE_PRIVATE int sqlite3ResolveOrderGroupBy( ExprList *pEList; struct ExprList_item *pItem; - if( pOrderBy==0 || pParse->db->mallocFailed ) return 0; + if( pOrderBy==0 || pParse->db->mallocFailed || IN_RENAME_OBJECT ) return 0; if( pOrderBy->nExpr>db->aLimit[SQLITE_LIMIT_COLUMN] ){ sqlite3ErrorMsg(pParse, "too many terms in %s BY clause", zType); return 1; @@ -94287,6 +98923,36 @@ SQLITE_PRIVATE int sqlite3ResolveOrderGroupBy( return 0; } +#ifndef SQLITE_OMIT_WINDOWFUNC +/* +** Walker callback for windowRemoveExprFromSelect(). +*/ +static int resolveRemoveWindowsCb(Walker *pWalker, Expr *pExpr){ + UNUSED_PARAMETER(pWalker); + if( ExprHasProperty(pExpr, EP_WinFunc) ){ + Window *pWin = pExpr->y.pWin; + sqlite3WindowUnlinkFromSelect(pWin); + } + return WRC_Continue; +} + +/* +** Remove any Window objects owned by the expression pExpr from the +** Select.pWin list of Select object pSelect. +*/ +static void windowRemoveExprFromSelect(Select *pSelect, Expr *pExpr){ + if( pSelect->pWin ){ + Walker sWalker; + memset(&sWalker, 0, sizeof(Walker)); + sWalker.xExprCallback = resolveRemoveWindowsCb; + sWalker.u.pSelect = pSelect; + sqlite3WalkExpr(&sWalker, pExpr); + } +} +#else +# define windowRemoveExprFromSelect(a, b) +#endif /* SQLITE_OMIT_WINDOWFUNC */ + /* ** pOrderBy is an ORDER BY or GROUP BY clause in SELECT statement pSelect. ** The Name context of the SELECT statement is pNC. zType is either @@ -94322,7 +98988,7 @@ static int resolveOrderGroupBy( pParse = pNC->pParse; for(i=0, pItem=pOrderBy->a; inExpr; i++, pItem++){ Expr *pE = pItem->pExpr; - Expr *pE2 = sqlite3ExprSkipCollate(pE); + Expr *pE2 = sqlite3ExprSkipCollateAndLikely(pE); if( zType[0]!='G' ){ iCol = resolveAsName(pParse, pSelect->pEList, pE2); if( iCol>0 ){ @@ -94353,6 +99019,10 @@ static int resolveOrderGroupBy( } for(j=0; jpEList->nExpr; j++){ if( sqlite3ExprCompare(0, pE, pSelect->pEList->a[j].pExpr, -1)==0 ){ + /* Since this expresion is being changed into a reference + ** to an identical expression in the result set, remove all Window + ** objects belonging to the expression from the Select.pWin list. */ + windowRemoveExprFromSelect(pSelect, pE); pItem->u.x.iOrderByCol = j+1; } } @@ -94409,6 +99079,7 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ */ memset(&sNC, 0, sizeof(sNC)); sNC.pParse = pParse; + sNC.pWinSelect = p; if( sqlite3ResolveExprNames(&sNC, p->pLimit) ){ return WRC_Abort; } @@ -94431,7 +99102,7 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ */ for(i=0; ipSrc->nSrc; i++){ struct SrcList_item *pItem = &p->pSrc->a[i]; - if( pItem->pSelect ){ + if( pItem->pSelect && (pItem->pSelect->selFlags & SF_Resolved)==0 ){ NameContext *pNC; /* Used to iterate name contexts */ int nRef = 0; /* Refcount for pOuterNC and outer contexts */ const char *zSavedContext = pParse->zAuthContext; @@ -94457,12 +99128,13 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ /* Set up the local name-context to pass to sqlite3ResolveExprNames() to ** resolve the result-set expression list. */ - sNC.ncFlags = NC_AllowAgg; + sNC.ncFlags = NC_AllowAgg|NC_AllowWin; sNC.pSrcList = p->pSrc; sNC.pNext = pOuterNC; /* Resolve names in the result set. */ if( sqlite3ResolveExprListNames(&sNC, p->pEList) ) return WRC_Abort; + sNC.ncFlags &= ~NC_AllowWin; /* If there are no aggregate functions in the result-set, and no GROUP BY ** expression, do not allow aggregates in any of the other expressions. @@ -94511,7 +99183,7 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ ** outer queries */ sNC.pNext = 0; - sNC.ncFlags |= NC_AllowAgg; + sNC.ncFlags |= NC_AllowAgg|NC_AllowWin; /* If this is a converted compound query, move the ORDER BY clause from ** the sub-query back to the parent query. At this point each term @@ -94542,6 +99214,7 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ if( db->mallocFailed ){ return WRC_Abort; } + sNC.ncFlags &= ~NC_AllowWin; /* Resolve the GROUP BY clause. At the same time, make sure ** the GROUP BY clause does not contain aggregate functions. @@ -94561,6 +99234,19 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ } } +#ifndef SQLITE_OMIT_WINDOWFUNC + if( IN_RENAME_OBJECT ){ + Window *pWin; + for(pWin=p->pWinDefn; pWin; pWin=pWin->pNextWin){ + if( sqlite3ResolveExprListNames(&sNC, pWin->pOrderBy) + || sqlite3ResolveExprListNames(&sNC, pWin->pPartition) + ){ + return WRC_Abort; + } + } + } +#endif + /* If this is part of a compound SELECT, check that it has the right ** number of expressions in the select list. */ if( p->pNext && p->pEList->nExpr!=p->pNext->pEList->nExpr ){ @@ -94636,12 +99322,12 @@ SQLITE_PRIVATE int sqlite3ResolveExprNames( NameContext *pNC, /* Namespace to resolve expressions in. */ Expr *pExpr /* The expression to be analyzed. */ ){ - u16 savedHasAgg; + int savedHasAgg; Walker w; if( pExpr==0 ) return SQLITE_OK; - savedHasAgg = pNC->ncFlags & (NC_HasAgg|NC_MinMaxAgg); - pNC->ncFlags &= ~(NC_HasAgg|NC_MinMaxAgg); + savedHasAgg = pNC->ncFlags & (NC_HasAgg|NC_MinMaxAgg|NC_HasWin); + pNC->ncFlags &= ~(NC_HasAgg|NC_MinMaxAgg|NC_HasWin); w.pParse = pNC->pParse; w.xExprCallback = resolveExprStep; w.xSelectCallback = resolveSelectStep; @@ -94657,9 +99343,11 @@ SQLITE_PRIVATE int sqlite3ResolveExprNames( #if SQLITE_MAX_EXPR_DEPTH>0 w.pParse->nHeight -= pExpr->nHeight; #endif - if( pNC->ncFlags & NC_HasAgg ){ - ExprSetProperty(pExpr, EP_Agg); - } + assert( EP_Agg==NC_HasAgg ); + assert( EP_Win==NC_HasWin ); + testcase( pNC->ncFlags & NC_HasAgg ); + testcase( pNC->ncFlags & NC_HasWin ); + ExprSetProperty(pExpr, pNC->ncFlags & (NC_HasAgg|NC_HasWin) ); pNC->ncFlags |= savedHasAgg; return pNC->nErr>0 || w.pParse->nErr>0; } @@ -94674,11 +99362,41 @@ SQLITE_PRIVATE int sqlite3ResolveExprListNames( ExprList *pList /* The expression list to be analyzed. */ ){ int i; - if( pList ){ - for(i=0; inExpr; i++){ - if( sqlite3ResolveExprNames(pNC, pList->a[i].pExpr) ) return WRC_Abort; + int savedHasAgg = 0; + Walker w; + if( pList==0 ) return WRC_Continue; + w.pParse = pNC->pParse; + w.xExprCallback = resolveExprStep; + w.xSelectCallback = resolveSelectStep; + w.xSelectCallback2 = 0; + w.u.pNC = pNC; + savedHasAgg = pNC->ncFlags & (NC_HasAgg|NC_MinMaxAgg|NC_HasWin); + pNC->ncFlags &= ~(NC_HasAgg|NC_MinMaxAgg|NC_HasWin); + for(i=0; inExpr; i++){ + Expr *pExpr = pList->a[i].pExpr; + if( pExpr==0 ) continue; +#if SQLITE_MAX_EXPR_DEPTH>0 + w.pParse->nHeight += pExpr->nHeight; + if( sqlite3ExprCheckHeight(w.pParse, w.pParse->nHeight) ){ + return WRC_Abort; + } +#endif + sqlite3WalkExpr(&w, pExpr); +#if SQLITE_MAX_EXPR_DEPTH>0 + w.pParse->nHeight -= pExpr->nHeight; +#endif + assert( EP_Agg==NC_HasAgg ); + assert( EP_Win==NC_HasWin ); + testcase( pNC->ncFlags & NC_HasAgg ); + testcase( pNC->ncFlags & NC_HasWin ); + if( pNC->ncFlags & (NC_HasAgg|NC_MinMaxAgg|NC_HasWin) ){ + ExprSetProperty(pExpr, pNC->ncFlags & (NC_HasAgg|NC_HasWin) ); + savedHasAgg |= pNC->ncFlags & (NC_HasAgg|NC_MinMaxAgg|NC_HasWin); + pNC->ncFlags &= ~(NC_HasAgg|NC_MinMaxAgg|NC_HasWin); } + if( pNC->nErr>0 || w.pParse->nErr>0 ) return WRC_Abort; } + pNC->ncFlags |= savedHasAgg; return WRC_Continue; } @@ -94711,38 +99429,56 @@ SQLITE_PRIVATE void sqlite3ResolveSelectNames( } /* -** Resolve names in expressions that can only reference a single table: +** Resolve names in expressions that can only reference a single table +** or which cannot reference any tables at all. Examples: ** -** * CHECK constraints -** * WHERE clauses on partial indices +** "type" flag +** ------------ +** (1) CHECK constraints NC_IsCheck +** (2) WHERE clauses on partial indices NC_PartIdx +** (3) Expressions in indexes on expressions NC_IdxExpr +** (4) Expression arguments to VACUUM INTO. 0 +** (5) GENERATED ALWAYS as expressions NC_GenCol ** -** The Expr.iTable value for Expr.op==TK_COLUMN nodes of the expression -** is set to -1 and the Expr.iColumn value is set to the column number. +** In all cases except (4), the Expr.iTable value for Expr.op==TK_COLUMN +** nodes of the expression is set to -1 and the Expr.iColumn value is +** set to the column number. In case (4), TK_COLUMN nodes cause an error. ** ** Any errors cause an error message to be set in pParse. */ -SQLITE_PRIVATE void sqlite3ResolveSelfReference( - Parse *pParse, /* Parsing context */ - Table *pTab, /* The table being referenced */ - int type, /* NC_IsCheck or NC_PartIdx or NC_IdxExpr */ - Expr *pExpr, /* Expression to resolve. May be NULL. */ - ExprList *pList /* Expression list to resolve. May be NULL. */ +SQLITE_PRIVATE int sqlite3ResolveSelfReference( + Parse *pParse, /* Parsing context */ + Table *pTab, /* The table being referenced, or NULL */ + int type, /* NC_IsCheck, NC_PartIdx, NC_IdxExpr, NC_GenCol, or 0 */ + Expr *pExpr, /* Expression to resolve. May be NULL. */ + ExprList *pList /* Expression list to resolve. May be NULL. */ ){ SrcList sSrc; /* Fake SrcList for pParse->pNewTable */ NameContext sNC; /* Name context for pParse->pNewTable */ + int rc; - assert( type==NC_IsCheck || type==NC_PartIdx || type==NC_IdxExpr ); + assert( type==0 || pTab!=0 ); + assert( type==NC_IsCheck || type==NC_PartIdx || type==NC_IdxExpr + || type==NC_GenCol || pTab==0 ); memset(&sNC, 0, sizeof(sNC)); memset(&sSrc, 0, sizeof(sSrc)); - sSrc.nSrc = 1; - sSrc.a[0].zName = pTab->zName; - sSrc.a[0].pTab = pTab; - sSrc.a[0].iCursor = -1; + if( pTab ){ + sSrc.nSrc = 1; + sSrc.a[0].zName = pTab->zName; + sSrc.a[0].pTab = pTab; + sSrc.a[0].iCursor = -1; + if( pTab->pSchema!=pParse->db->aDb[1].pSchema ){ + /* Cause EP_FromDDL to be set on TK_FUNCTION nodes of non-TEMP + ** schema elements */ + type |= NC_FromDDL; + } + } sNC.pParse = pParse; sNC.pSrcList = &sSrc; - sNC.ncFlags = type; - if( sqlite3ResolveExprNames(&sNC, pExpr) ) return; - if( pList ) sqlite3ResolveExprListNames(&sNC, pList); + sNC.ncFlags = type | NC_IsDDL; + if( (rc = sqlite3ResolveExprNames(&sNC, pExpr))!=SQLITE_OK ) return rc; + if( pList ) rc = sqlite3ResolveExprListNames(&sNC, pList); + return rc; } /************** End of resolve.c *********************************************/ @@ -94791,10 +99527,13 @@ SQLITE_PRIVATE char sqlite3TableColumnAffinity(Table *pTab, int iCol){ ** SELECT a AS b FROM t1 WHERE b; ** SELECT * FROM t1 WHERE (select a from t1); */ -SQLITE_PRIVATE char sqlite3ExprAffinity(Expr *pExpr){ +SQLITE_PRIVATE char sqlite3ExprAffinity(const Expr *pExpr){ int op; - pExpr = sqlite3ExprSkipCollate(pExpr); - if( pExpr->flags & EP_Generic ) return 0; + while( ExprHasProperty(pExpr, EP_Skip) ){ + assert( pExpr->op==TK_COLLATE || pExpr->op==TK_IF_NULL_ROW ); + pExpr = pExpr->pLeft; + assert( pExpr!=0 ); + } op = pExpr->op; if( op==TK_SELECT ){ assert( pExpr->flags&EP_xIsSelect ); @@ -94807,8 +99546,8 @@ SQLITE_PRIVATE char sqlite3ExprAffinity(Expr *pExpr){ return sqlite3AffinityType(pExpr->u.zToken, 0); } #endif - if( (op==TK_AGG_COLUMN || op==TK_COLUMN) && pExpr->pTab ){ - return sqlite3TableColumnAffinity(pExpr->pTab, pExpr->iColumn); + if( (op==TK_AGG_COLUMN || op==TK_COLUMN) && pExpr->y.pTab ){ + return sqlite3TableColumnAffinity(pExpr->y.pTab, pExpr->iColumn); } if( op==TK_SELECT_COLUMN ){ assert( pExpr->pLeft->flags&EP_xIsSelect ); @@ -94816,7 +99555,10 @@ SQLITE_PRIVATE char sqlite3ExprAffinity(Expr *pExpr){ pExpr->pLeft->x.pSelect->pEList->a[pExpr->iColumn].pExpr ); } - return pExpr->affinity; + if( op==TK_VECTOR ){ + return sqlite3ExprAffinity(pExpr->x.pList->a[0].pExpr); + } + return pExpr->affExpr; } /* @@ -94851,18 +99593,30 @@ SQLITE_PRIVATE Expr *sqlite3ExprAddCollateString(Parse *pParse, Expr *pExpr, con } /* -** Skip over any TK_COLLATE operators and any unlikely() -** or likelihood() function at the root of an expression. +** Skip over any TK_COLLATE operators. */ SQLITE_PRIVATE Expr *sqlite3ExprSkipCollate(Expr *pExpr){ while( pExpr && ExprHasProperty(pExpr, EP_Skip) ){ + assert( pExpr->op==TK_COLLATE || pExpr->op==TK_IF_NULL_ROW ); + pExpr = pExpr->pLeft; + } + return pExpr; +} + +/* +** Skip over any TK_COLLATE operators and/or any unlikely() +** or likelihood() or likely() functions at the root of an +** expression. +*/ +SQLITE_PRIVATE Expr *sqlite3ExprSkipCollateAndLikely(Expr *pExpr){ + while( pExpr && ExprHasProperty(pExpr, EP_Skip|EP_Unlikely) ){ if( ExprHasProperty(pExpr, EP_Unlikely) ){ assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); assert( pExpr->x.pList->nExpr>0 ); assert( pExpr->op==TK_FUNCTION ); pExpr = pExpr->x.pList->a[0].pExpr; }else{ - assert( pExpr->op==TK_COLLATE ); + assert( pExpr->op==TK_COLLATE || pExpr->op==TK_IF_NULL_ROW ); pExpr = pExpr->pLeft; } } @@ -94883,34 +99637,37 @@ SQLITE_PRIVATE Expr *sqlite3ExprSkipCollate(Expr *pExpr){ ** COLLATE operators take first precedence. Left operands take ** precedence over right operands. */ -SQLITE_PRIVATE CollSeq *sqlite3ExprCollSeq(Parse *pParse, Expr *pExpr){ +SQLITE_PRIVATE CollSeq *sqlite3ExprCollSeq(Parse *pParse, const Expr *pExpr){ sqlite3 *db = pParse->db; CollSeq *pColl = 0; - Expr *p = pExpr; + const Expr *p = pExpr; while( p ){ int op = p->op; - if( p->flags & EP_Generic ) break; - if( op==TK_CAST || op==TK_UPLUS ){ - p = p->pLeft; - continue; - } - if( op==TK_COLLATE || (op==TK_REGISTER && p->op2==TK_COLLATE) ){ - pColl = sqlite3GetCollSeq(pParse, ENC(db), 0, p->u.zToken); - break; - } - if( (op==TK_AGG_COLUMN || op==TK_COLUMN - || op==TK_REGISTER || op==TK_TRIGGER) - && p->pTab!=0 + if( op==TK_REGISTER ) op = p->op2; + if( (op==TK_AGG_COLUMN || op==TK_COLUMN || op==TK_TRIGGER) + && p->y.pTab!=0 ){ - /* op==TK_REGISTER && p->pTab!=0 happens when pExpr was originally + /* op==TK_REGISTER && p->y.pTab!=0 happens when pExpr was originally ** a TK_COLUMN but was previously evaluated and cached in a register */ int j = p->iColumn; if( j>=0 ){ - const char *zColl = p->pTab->aCol[j].zColl; + const char *zColl = p->y.pTab->aCol[j].zColl; pColl = sqlite3FindCollSeq(db, ENC(db), zColl, 0); } break; } + if( op==TK_CAST || op==TK_UPLUS ){ + p = p->pLeft; + continue; + } + if( op==TK_VECTOR ){ + p = p->x.pList->a[0].pExpr; + continue; + } + if( op==TK_COLLATE ){ + pColl = sqlite3GetCollSeq(pParse, ENC(db), 0, p->u.zToken); + break; + } if( p->flags & EP_Collate ){ if( p->pLeft && (p->pLeft->flags & EP_Collate)!=0 ){ p = p->pLeft; @@ -94918,12 +99675,12 @@ SQLITE_PRIVATE CollSeq *sqlite3ExprCollSeq(Parse *pParse, Expr *pExpr){ Expr *pNext = p->pRight; /* The Expr.x union is never used at the same time as Expr.pRight */ assert( p->x.pList==0 || p->pRight==0 ); - /* p->flags holds EP_Collate and p->pLeft->flags does not. And - ** p->x.pSelect cannot. So if p->x.pLeft exists, it must hold at - ** least one EP_Collate. Thus the following two ALWAYS. */ - if( p->x.pList!=0 && ALWAYS(!ExprHasProperty(p, EP_xIsSelect)) ){ + if( p->x.pList!=0 + && !db->mallocFailed + && ALWAYS(!ExprHasProperty(p, EP_xIsSelect)) + ){ int i; - for(i=0; ALWAYS(ix.pList->nExpr); i++){ + for(i=0; ix.pList->nExpr; i++){ if( ExprHasProperty(p->x.pList->a[i].pExpr, EP_Collate) ){ pNext = p->x.pList->a[i].pExpr; break; @@ -94952,7 +99709,7 @@ SQLITE_PRIVATE CollSeq *sqlite3ExprCollSeq(Parse *pParse, Expr *pExpr){ ** The sqlite3ExprCollSeq() routine works the same except that it ** returns NULL if there is no defined collation. */ -SQLITE_PRIVATE CollSeq *sqlite3ExprNNCollSeq(Parse *pParse, Expr *pExpr){ +SQLITE_PRIVATE CollSeq *sqlite3ExprNNCollSeq(Parse *pParse, const Expr *pExpr){ CollSeq *p = sqlite3ExprCollSeq(pParse, pExpr); if( p==0 ) p = pParse->db->pDfltColl; assert( p!=0 ); @@ -94962,7 +99719,7 @@ SQLITE_PRIVATE CollSeq *sqlite3ExprNNCollSeq(Parse *pParse, Expr *pExpr){ /* ** Return TRUE if the two expressions have equivalent collating sequences. */ -SQLITE_PRIVATE int sqlite3ExprCollSeqMatch(Parse *pParse, Expr *pE1, Expr *pE2){ +SQLITE_PRIVATE int sqlite3ExprCollSeqMatch(Parse *pParse, const Expr *pE1, const Expr *pE2){ CollSeq *pColl1 = sqlite3ExprNNCollSeq(pParse, pE1); CollSeq *pColl2 = sqlite3ExprNNCollSeq(pParse, pE2); return sqlite3StrICmp(pColl1->zName, pColl2->zName)==0; @@ -94973,9 +99730,9 @@ SQLITE_PRIVATE int sqlite3ExprCollSeqMatch(Parse *pParse, Expr *pE1, Expr *pE2){ ** type affinity of the other operand. This routine returns the ** type affinity that should be used for the comparison operator. */ -SQLITE_PRIVATE char sqlite3CompareAffinity(Expr *pExpr, char aff2){ +SQLITE_PRIVATE char sqlite3CompareAffinity(const Expr *pExpr, char aff2){ char aff1 = sqlite3ExprAffinity(pExpr); - if( aff1 && aff2 ){ + if( aff1>SQLITE_AFF_NONE && aff2>SQLITE_AFF_NONE ){ /* Both sides of the comparison are columns. If one has numeric ** affinity, use that. Otherwise use no affinity. */ @@ -94984,15 +99741,10 @@ SQLITE_PRIVATE char sqlite3CompareAffinity(Expr *pExpr, char aff2){ }else{ return SQLITE_AFF_BLOB; } - }else if( !aff1 && !aff2 ){ - /* Neither side of the comparison is a column. Compare the - ** results directly. - */ - return SQLITE_AFF_BLOB; }else{ /* One side is a column, the other is not. Use the columns affinity. */ - assert( aff1==0 || aff2==0 ); - return (aff1 + aff2); + assert( aff1<=SQLITE_AFF_NONE || aff2<=SQLITE_AFF_NONE ); + return (aff1<=SQLITE_AFF_NONE ? aff2 : aff1) | SQLITE_AFF_NONE; } } @@ -95000,7 +99752,7 @@ SQLITE_PRIVATE char sqlite3CompareAffinity(Expr *pExpr, char aff2){ ** pExpr is a comparison operator. Return the type affinity that should ** be applied to both operands prior to doing the comparison. */ -static char comparisonAffinity(Expr *pExpr){ +static char comparisonAffinity(const Expr *pExpr){ char aff; assert( pExpr->op==TK_EQ || pExpr->op==TK_IN || pExpr->op==TK_LT || pExpr->op==TK_GT || pExpr->op==TK_GE || pExpr->op==TK_LE || @@ -95023,23 +99775,26 @@ static char comparisonAffinity(Expr *pExpr){ ** if the index with affinity idx_affinity may be used to implement ** the comparison in pExpr. */ -SQLITE_PRIVATE int sqlite3IndexAffinityOk(Expr *pExpr, char idx_affinity){ +SQLITE_PRIVATE int sqlite3IndexAffinityOk(const Expr *pExpr, char idx_affinity){ char aff = comparisonAffinity(pExpr); - switch( aff ){ - case SQLITE_AFF_BLOB: - return 1; - case SQLITE_AFF_TEXT: - return idx_affinity==SQLITE_AFF_TEXT; - default: - return sqlite3IsNumericAffinity(idx_affinity); + if( affpRight, p->pLeft); + }else{ + return sqlite3BinaryCompareCollSeq(pParse, p->pLeft, p->pRight); + } +} + /* ** Generate code for a comparison operator. */ @@ -95087,13 +99858,19 @@ static int codeCompare( int opcode, /* The comparison opcode */ int in1, int in2, /* Register holding operands */ int dest, /* Jump here if true. */ - int jumpIfNull /* If true, jump if either operand is NULL */ + int jumpIfNull, /* If true, jump if either operand is NULL */ + int isCommuted /* The comparison has been commuted */ ){ int p5; int addr; CollSeq *p4; - p4 = sqlite3BinaryCompareCollSeq(pParse, pLeft, pRight); + if( pParse->nErr ) return 0; + if( isCommuted ){ + p4 = sqlite3BinaryCompareCollSeq(pParse, pRight, pLeft); + }else{ + p4 = sqlite3BinaryCompareCollSeq(pParse, pLeft, pRight); + } p5 = binaryCompareP5(pLeft, pRight, jumpIfNull); addr = sqlite3VdbeAddOp4(pParse->pVdbe, opcode, in2, dest, in1, (void*)p4, P4_COLLSEQ); @@ -95214,6 +99991,7 @@ SQLITE_PRIVATE Expr *sqlite3ExprForVectorField( }else{ if( pVector->op==TK_VECTOR ) pVector = pVector->x.pList->a[iField].pExpr; pRet = sqlite3ExprDup(pParse->db, pVector, 0); + sqlite3RenameTokenRemap(pParse, pRet, pVector); } return pRet; } @@ -95230,7 +100008,7 @@ static int exprCodeSubselect(Parse *pParse, Expr *pExpr){ int reg = 0; #ifndef SQLITE_OMIT_SUBQUERY if( pExpr->op==TK_SELECT ){ - reg = sqlite3CodeSubselect(pParse, pExpr, 0, 0); + reg = sqlite3CodeSubselect(pParse, pExpr); } #endif return reg; @@ -95302,8 +100080,11 @@ static void codeVectorCompare( int regLeft = 0; int regRight = 0; u8 opx = op; - int addrDone = sqlite3VdbeMakeLabel(v); + int addrDone = sqlite3VdbeMakeLabel(pParse); + int isCommuted = ExprHasProperty(pExpr,EP_Commuted); + assert( !ExprHasVVAProperty(pExpr,EP_Immutable) ); + if( pParse->nErr ) return; if( nLeft!=sqlite3ExprVectorSize(pRight) ){ sqlite3ErrorMsg(pParse, "row value misused"); return; @@ -95330,10 +100111,9 @@ static void codeVectorCompare( Expr *pL, *pR; int r1, r2; assert( i>=0 && i0 ) sqlite3ExprCachePush(pParse); r1 = exprVectorRegister(pParse, pLeft, i, regLeft, &pL, ®Free1); r2 = exprVectorRegister(pParse, pRight, i, regRight, &pR, ®Free2); - codeCompare(pParse, pL, pR, opx, r1, r2, dest, p5); + codeCompare(pParse, pL, pR, opx, r1, r2, dest, p5, isCommuted); testcase(op==OP_Lt); VdbeCoverageIf(v,op==OP_Lt); testcase(op==OP_Le); VdbeCoverageIf(v,op==OP_Le); testcase(op==OP_Gt); VdbeCoverageIf(v,op==OP_Gt); @@ -95342,7 +100122,6 @@ static void codeVectorCompare( testcase(op==OP_Ne); VdbeCoverageIf(v,op==OP_Ne); sqlite3ReleaseTempReg(pParse, regFree1); sqlite3ReleaseTempReg(pParse, regFree2); - if( i>0 ) sqlite3ExprCachePop(pParse); if( i==nLeft-1 ){ break; } @@ -95523,7 +100302,7 @@ SQLITE_PRIVATE Expr *sqlite3ExprAlloc( pNew->iAgg = -1; if( pToken ){ if( nExtra==0 ){ - pNew->flags |= EP_IntValue|EP_Leaf; + pNew->flags |= EP_IntValue|EP_Leaf|(iValue?EP_IsTrue:EP_IsFalse); pNew->u.iValue = iValue; }else{ pNew->u.zToken = (char*)&pNew[1]; @@ -95531,8 +100310,7 @@ SQLITE_PRIVATE Expr *sqlite3ExprAlloc( if( pToken->n ) memcpy(pNew->u.zToken, pToken->z, pToken->n); pNew->u.zToken[pToken->n] = 0; if( dequote && sqlite3Isquote(pNew->u.zToken[0]) ){ - if( pNew->u.zToken[0]=='"' ) pNew->flags |= EP_DblQuoted; - sqlite3Dequote(pNew->u.zToken); + sqlite3DequoteExpr(pNew); } } } @@ -95601,20 +100379,16 @@ SQLITE_PRIVATE Expr *sqlite3PExpr( Expr *pRight /* Right operand */ ){ Expr *p; - if( op==TK_AND && pParse->nErr==0 ){ - /* Take advantage of short-circuit false optimization for AND */ - p = sqlite3ExprAnd(pParse->db, pLeft, pRight); - }else{ - p = sqlite3DbMallocRawNN(pParse->db, sizeof(Expr)); - if( p ){ - memset(p, 0, sizeof(Expr)); - p->op = op & TKFLG_MASK; - p->iAgg = -1; - } + p = sqlite3DbMallocRawNN(pParse->db, sizeof(Expr)); + if( p ){ + memset(p, 0, sizeof(Expr)); + p->op = op & 0xff; + p->iAgg = -1; sqlite3ExprAttachSubtrees(pParse->db, p, pLeft, pRight); - } - if( p ) { sqlite3ExprCheckHeight(pParse, p->nHeight); + }else{ + sqlite3ExprDelete(pParse->db, pLeft); + sqlite3ExprDelete(pParse->db, pRight); } return p; } @@ -95635,33 +100409,6 @@ SQLITE_PRIVATE void sqlite3PExprAddSelect(Parse *pParse, Expr *pExpr, Select *pS } -/* -** If the expression is always either TRUE or FALSE (respectively), -** then return 1. If one cannot determine the truth value of the -** expression at compile-time return 0. -** -** This is an optimization. If is OK to return 0 here even if -** the expression really is always false or false (a false negative). -** But it is a bug to return 1 if the expression might have different -** boolean values in different circumstances (a false positive.) -** -** Note that if the expression is part of conditional for a -** LEFT JOIN, then we cannot determine at compile-time whether or not -** is it true or false, so always return 0. -*/ -static int exprAlwaysTrue(Expr *p){ - int v = 0; - if( ExprHasProperty(p, EP_FromJoin) ) return 0; - if( !sqlite3ExprIsInteger(p, &v) ) return 0; - return v!=0; -} -static int exprAlwaysFalse(Expr *p){ - int v = 0; - if( ExprHasProperty(p, EP_FromJoin) ) return 0; - if( !sqlite3ExprIsInteger(p, &v) ) return 0; - return v==0; -} - /* ** Join two expressions using an AND operator. If either expression is ** NULL, then just return the other expression. @@ -95670,19 +100417,20 @@ static int exprAlwaysFalse(Expr *p){ ** of returning an AND expression, just return a constant expression with ** a value of false. */ -SQLITE_PRIVATE Expr *sqlite3ExprAnd(sqlite3 *db, Expr *pLeft, Expr *pRight){ - if( pLeft==0 ){ +SQLITE_PRIVATE Expr *sqlite3ExprAnd(Parse *pParse, Expr *pLeft, Expr *pRight){ + sqlite3 *db = pParse->db; + if( pLeft==0 ){ return pRight; }else if( pRight==0 ){ return pLeft; - }else if( exprAlwaysFalse(pLeft) || exprAlwaysFalse(pRight) ){ + }else if( (ExprAlwaysFalse(pLeft) || ExprAlwaysFalse(pRight)) + && !IN_RENAME_OBJECT + ){ sqlite3ExprDelete(db, pLeft); sqlite3ExprDelete(db, pRight); - return sqlite3ExprAlloc(db, TK_INTEGER, &sqlite3IntTokens[0], 0); + return sqlite3Expr(db, TK_INTEGER, "0"); }else{ - Expr *pNew = sqlite3ExprAlloc(db, TK_AND, 0, 0); - sqlite3ExprAttachSubtrees(db, pNew, pLeft, pRight); - return pNew; + return sqlite3PExpr(pParse, TK_AND, pLeft, pRight); } } @@ -95690,7 +100438,12 @@ SQLITE_PRIVATE Expr *sqlite3ExprAnd(sqlite3 *db, Expr *pLeft, Expr *pRight){ ** Construct a new expression node for a function with multiple ** arguments. */ -SQLITE_PRIVATE Expr *sqlite3ExprFunction(Parse *pParse, ExprList *pList, Token *pToken){ +SQLITE_PRIVATE Expr *sqlite3ExprFunction( + Parse *pParse, /* Parsing context */ + ExprList *pList, /* Argument list */ + Token *pToken, /* Name of the function */ + int eDistinct /* SF_Distinct or SF_ALL or 0 */ +){ Expr *pNew; sqlite3 *db = pParse->db; assert( pToken ); @@ -95699,13 +100452,51 @@ SQLITE_PRIVATE Expr *sqlite3ExprFunction(Parse *pParse, ExprList *pList, Token * sqlite3ExprListDelete(db, pList); /* Avoid memory leak when malloc fails */ return 0; } + if( pList && pList->nExpr > pParse->db->aLimit[SQLITE_LIMIT_FUNCTION_ARG] ){ + sqlite3ErrorMsg(pParse, "too many arguments on function %T", pToken); + } pNew->x.pList = pList; ExprSetProperty(pNew, EP_HasFunc); assert( !ExprHasProperty(pNew, EP_xIsSelect) ); sqlite3ExprSetHeightAndFlags(pParse, pNew); + if( eDistinct==SF_Distinct ) ExprSetProperty(pNew, EP_Distinct); return pNew; } +/* +** Check to see if a function is usable according to current access +** rules: +** +** SQLITE_FUNC_DIRECT - Only usable from top-level SQL +** +** SQLITE_FUNC_UNSAFE - Usable if TRUSTED_SCHEMA or from +** top-level SQL +** +** If the function is not usable, create an error. +*/ +SQLITE_PRIVATE void sqlite3ExprFunctionUsable( + Parse *pParse, /* Parsing and code generating context */ + Expr *pExpr, /* The function invocation */ + FuncDef *pDef /* The function being invoked */ +){ + assert( !IN_RENAME_OBJECT ); + assert( (pDef->funcFlags & (SQLITE_FUNC_DIRECT|SQLITE_FUNC_UNSAFE))!=0 ); + if( ExprHasProperty(pExpr, EP_FromDDL) ){ + if( (pDef->funcFlags & SQLITE_FUNC_DIRECT)!=0 + || (pParse->db->flags & SQLITE_TrustedSchema)==0 + ){ + /* Functions prohibited in triggers and views if: + ** (1) tagged with SQLITE_DIRECTONLY + ** (2) not tagged with SQLITE_INNOCUOUS (which means it + ** is tagged with SQLITE_FUNC_UNSAFE) and + ** SQLITE_DBCONFIG_TRUSTED_SCHEMA is off (meaning + ** that the schema is possibly tainted). + */ + sqlite3ErrorMsg(pParse, "unsafe use of %s()", pDef->zName); + } + } +} + /* ** Assign a variable number to an expression that encodes a wildcard ** in the original SQL statement. @@ -95794,6 +100585,10 @@ static SQLITE_NOINLINE void sqlite3ExprDeleteNN(sqlite3 *db, Expr *p){ assert( p!=0 ); /* Sanity check: Assert that the IntValue is non-negative if it exists */ assert( !ExprHasProperty(p, EP_IntValue) || p->u.iValue>=0 ); + + assert( !ExprHasProperty(p, EP_WinFunc) || p->y.pWin!=0 || db->mallocFailed ); + assert( p->op!=TK_FUNCTION || ExprHasProperty(p, EP_TokenOnly|EP_Reduced) + || p->y.pWin==0 || ExprHasProperty(p, EP_WinFunc) ); #ifdef SQLITE_DEBUG if( ExprHasProperty(p, EP_Leaf) && !ExprHasProperty(p, EP_TokenOnly) ){ assert( p->pLeft==0 ); @@ -95806,11 +100601,18 @@ static SQLITE_NOINLINE void sqlite3ExprDeleteNN(sqlite3 *db, Expr *p){ assert( p->x.pList==0 || p->pRight==0 ); if( p->pLeft && p->op!=TK_SELECT_COLUMN ) sqlite3ExprDeleteNN(db, p->pLeft); if( p->pRight ){ + assert( !ExprHasProperty(p, EP_WinFunc) ); sqlite3ExprDeleteNN(db, p->pRight); }else if( ExprHasProperty(p, EP_xIsSelect) ){ + assert( !ExprHasProperty(p, EP_WinFunc) ); sqlite3SelectDelete(db, p->x.pSelect); }else{ sqlite3ExprListDelete(db, p->x.pList); +#ifndef SQLITE_OMIT_WINDOWFUNC + if( ExprHasProperty(p, EP_WinFunc) ){ + sqlite3WindowDelete(db, p->y.pWin); + } +#endif } } if( ExprHasProperty(p, EP_MemToken) ) sqlite3DbFree(db, p->u.zToken); @@ -95822,6 +100624,18 @@ SQLITE_PRIVATE void sqlite3ExprDelete(sqlite3 *db, Expr *p){ if( p ) sqlite3ExprDeleteNN(db, p); } +/* Invoke sqlite3RenameExprUnmap() and sqlite3ExprDelete() on the +** expression. +*/ +SQLITE_PRIVATE void sqlite3ExprUnmapAndDelete(Parse *pParse, Expr *p){ + if( p ){ + if( IN_RENAME_OBJECT ){ + sqlite3RenameExprUnmap(pParse, p); + } + sqlite3ExprDeleteNN(pParse->db, p); + } +} + /* ** Return the number of bytes allocated for the expression structure ** passed as the first argument. This is always one of EXPR_FULLSIZE, @@ -95860,7 +100674,7 @@ static int exprStructSize(Expr *p){ ** Note that with flags==EXPRDUP_REDUCE, this routines works on full-size ** (unreduced) Expr objects as they or originally constructed by the parser. ** During expression analysis, extra information is computed and moved into -** later parts of teh Expr object and that extra information might get chopped +** later parts of the Expr object and that extra information might get chopped ** off if the expression is reduced. Note also that it does not work to ** make an EXPRDUP_REDUCE copy of a reduced expression. It is only legal ** to reduce a pristine expression tree from the parser. The implementation @@ -95872,13 +100686,17 @@ static int dupedExprStructSize(Expr *p, int flags){ assert( flags==EXPRDUP_REDUCE || flags==0 ); /* Only one flag value allowed */ assert( EXPR_FULLSIZE<=0xfff ); assert( (0xfff & (EP_Reduced|EP_TokenOnly))==0 ); - if( 0==flags || p->op==TK_SELECT_COLUMN ){ + if( 0==flags || p->op==TK_SELECT_COLUMN +#ifndef SQLITE_OMIT_WINDOWFUNC + || ExprHasProperty(p, EP_WinFunc) +#endif + ){ nSize = EXPR_FULLSIZE; }else{ assert( !ExprHasProperty(p, EP_TokenOnly|EP_Reduced) ); assert( !ExprHasProperty(p, EP_FromJoin) ); assert( !ExprHasProperty(p, EP_MemToken) ); - assert( !ExprHasProperty(p, EP_NoReduce) ); + assert( !ExprHasVVAProperty(p, EP_NoReduce) ); if( p->pLeft || p->x.pList ){ nSize = EXPR_REDUCEDSIZE | EP_Reduced; }else{ @@ -95897,7 +100715,7 @@ static int dupedExprStructSize(Expr *p, int flags){ static int dupedExprNodeSize(Expr *p, int flags){ int nByte = dupedExprStructSize(p, flags) & 0xfff; if( !ExprHasProperty(p, EP_IntValue) && p->u.zToken ){ - nByte += sqlite3Strlen30(p->u.zToken)+1; + nByte += sqlite3Strlen30NN(p->u.zToken)+1; } return ROUND8(nByte); } @@ -95983,6 +100801,10 @@ static Expr *exprDup(sqlite3 *db, Expr *p, int dupFlags, u8 **pzBuffer){ pNew->flags &= ~(EP_Reduced|EP_TokenOnly|EP_Static|EP_MemToken); pNew->flags |= nStructSize & (EP_Reduced|EP_TokenOnly); pNew->flags |= staticFlag; + ExprClearVVAProperties(pNew); + if( dupFlags ){ + ExprSetVVAProperty(pNew, EP_Immutable); + } /* Copy the p->u.zToken string, if any. */ if( nToken ){ @@ -96000,7 +100822,7 @@ static Expr *exprDup(sqlite3 *db, Expr *p, int dupFlags, u8 **pzBuffer){ } /* Fill in pNew->pLeft and pNew->pRight. */ - if( ExprHasProperty(pNew, EP_Reduced|EP_TokenOnly) ){ + if( ExprHasProperty(pNew, EP_Reduced|EP_TokenOnly|EP_WinFunc) ){ zAlloc += dupedExprNodeSize(p, dupFlags); if( !ExprHasProperty(pNew, EP_TokenOnly|EP_Leaf) ){ pNew->pLeft = p->pLeft ? @@ -96008,6 +100830,12 @@ static Expr *exprDup(sqlite3 *db, Expr *p, int dupFlags, u8 **pzBuffer){ pNew->pRight = p->pRight ? exprDup(db, p->pRight, EXPRDUP_REDUCE, &zAlloc) : 0; } +#ifndef SQLITE_OMIT_WINDOWFUNC + if( ExprHasProperty(p, EP_WinFunc) ){ + pNew->y.pWin = sqlite3WindowDup(db, pNew, p->y.pWin); + assert( ExprHasProperty(pNew, EP_WinFunc) ); + } +#endif /* SQLITE_OMIT_WINDOWFUNC */ if( pzBuffer ){ *pzBuffer = zAlloc; } @@ -96036,7 +100864,7 @@ static Expr *exprDup(sqlite3 *db, Expr *p, int dupFlags, u8 **pzBuffer){ static With *withDup(sqlite3 *db, With *p){ With *pRet = 0; if( p ){ - int nByte = sizeof(*p) + sizeof(p->a[0]) * (p->nCte-1); + sqlite3_int64 nByte = sizeof(*p) + sizeof(p->a[0]) * (p->nCte-1); pRet = sqlite3DbMallocZero(db, nByte); if( pRet ){ int i; @@ -96054,6 +100882,39 @@ static With *withDup(sqlite3 *db, With *p){ # define withDup(x,y) 0 #endif +#ifndef SQLITE_OMIT_WINDOWFUNC +/* +** The gatherSelectWindows() procedure and its helper routine +** gatherSelectWindowsCallback() are used to scan all the expressions +** an a newly duplicated SELECT statement and gather all of the Window +** objects found there, assembling them onto the linked list at Select->pWin. +*/ +static int gatherSelectWindowsCallback(Walker *pWalker, Expr *pExpr){ + if( pExpr->op==TK_FUNCTION && ExprHasProperty(pExpr, EP_WinFunc) ){ + Select *pSelect = pWalker->u.pSelect; + Window *pWin = pExpr->y.pWin; + assert( pWin ); + assert( IsWindowFunc(pExpr) ); + assert( pWin->ppThis==0 ); + sqlite3WindowLink(pSelect, pWin); + } + return WRC_Continue; +} +static int gatherSelectWindowsSelectCallback(Walker *pWalker, Select *p){ + return p==pWalker->u.pSelect ? WRC_Continue : WRC_Prune; +} +static void gatherSelectWindows(Select *p){ + Walker w; + w.xExprCallback = gatherSelectWindowsCallback; + w.xSelectCallback = gatherSelectWindowsSelectCallback; + w.xSelectCallback2 = 0; + w.pParse = 0; + w.u.pSelect = p; + sqlite3WalkSelect(&w, p); +} +#endif + + /* ** The following group of routines make deep copies of expressions, ** expression lists, ID lists, and select statements. The copies can @@ -96107,11 +100968,11 @@ SQLITE_PRIVATE ExprList *sqlite3ExprListDup(sqlite3 *db, ExprList *p, int flags) pNewExpr->pLeft = pPriorSelectCol; } } - pItem->zName = sqlite3DbStrDup(db, pOldItem->zName); - pItem->zSpan = sqlite3DbStrDup(db, pOldItem->zSpan); - pItem->sortOrder = pOldItem->sortOrder; + pItem->zEName = sqlite3DbStrDup(db, pOldItem->zEName); + pItem->sortFlags = pOldItem->sortFlags; + pItem->eEName = pOldItem->eEName; pItem->done = 0; - pItem->bSpanIsTab = pOldItem->bSpanIsTab; + pItem->bNulls = pOldItem->bNulls; pItem->bSorterRef = pOldItem->bSorterRef; pItem->u = pOldItem->u; } @@ -96218,7 +101079,12 @@ SQLITE_PRIVATE Select *sqlite3SelectDup(sqlite3 *db, Select *pDup, int flags){ pNew->addrOpenEphm[1] = -1; pNew->nSelectRow = p->nSelectRow; pNew->pWith = withDup(db, p->pWith); - sqlite3SelectSetName(pNew, p->zSelName); +#ifndef SQLITE_OMIT_WINDOWFUNC + pNew->pWin = 0; + pNew->pWinDefn = sqlite3WindowListDup(db, p->pWinDefn); + if( p->pWin && db->mallocFailed==0 ) gatherSelectWindows(pNew); +#endif + pNew->selId = p->selId; *pp = pNew; pp = &pNew->pPrior; pNext = pNew; @@ -96266,16 +101132,16 @@ SQLITE_PRIVATE ExprList *sqlite3ExprListAppend( }else if( (pList->nExpr & (pList->nExpr-1))==0 ){ ExprList *pNew; pNew = sqlite3DbRealloc(db, pList, - sizeof(*pList)+(2*pList->nExpr - 1)*sizeof(pList->a[0])); + sizeof(*pList)+(2*(sqlite3_int64)pList->nExpr-1)*sizeof(pList->a[0])); if( pNew==0 ){ goto no_mem; } pList = pNew; } pItem = &pList->a[pList->nExpr++]; - assert( offsetof(struct ExprList_item,zName)==sizeof(pItem->pExpr) ); + assert( offsetof(struct ExprList_item,zEName)==sizeof(pItem->pExpr) ); assert( offsetof(struct ExprList_item,pExpr)==0 ); - memset(&pItem->zName,0,sizeof(*pItem)-offsetof(struct ExprList_item,zName)); + memset(&pItem->zEName,0,sizeof(*pItem)-offsetof(struct ExprList_item,zEName)); pItem->pExpr = pExpr; return pList; @@ -96325,10 +101191,14 @@ SQLITE_PRIVATE ExprList *sqlite3ExprListAppendVector( for(i=0; inId; i++){ Expr *pSubExpr = sqlite3ExprForVectorField(pParse, pExpr, i); + assert( pSubExpr!=0 || db->mallocFailed ); + assert( pSubExpr==0 || pSubExpr->iTable==0 ); + if( pSubExpr==0 ) continue; + pSubExpr->iTable = pColumns->nId; pList = sqlite3ExprListAppend(pParse, pList, pSubExpr); if( pList ){ assert( pList->nExpr==iFirst+i+1 ); - pList->a[pList->nExpr-1].zName = pColumns->a[i].zName; + pList->a[pList->nExpr-1].zEName = pColumns->a[i].zName; pColumns->a[i].zName = 0; } } @@ -96349,7 +101219,7 @@ SQLITE_PRIVATE ExprList *sqlite3ExprListAppendVector( } vector_append_error: - sqlite3ExprDelete(db, pExpr); + sqlite3ExprUnmapAndDelete(pParse, pExpr); sqlite3IdListDelete(db, pColumns); return pList; } @@ -96357,19 +101227,38 @@ SQLITE_PRIVATE ExprList *sqlite3ExprListAppendVector( /* ** Set the sort order for the last element on the given ExprList. */ -SQLITE_PRIVATE void sqlite3ExprListSetSortOrder(ExprList *p, int iSortOrder){ +SQLITE_PRIVATE void sqlite3ExprListSetSortOrder(ExprList *p, int iSortOrder, int eNulls){ + struct ExprList_item *pItem; if( p==0 ) return; - assert( SQLITE_SO_UNDEFINED<0 && SQLITE_SO_ASC>=0 && SQLITE_SO_DESC>0 ); assert( p->nExpr>0 ); - if( iSortOrder<0 ){ - assert( p->a[p->nExpr-1].sortOrder==SQLITE_SO_ASC ); - return; + + assert( SQLITE_SO_UNDEFINED<0 && SQLITE_SO_ASC==0 && SQLITE_SO_DESC>0 ); + assert( iSortOrder==SQLITE_SO_UNDEFINED + || iSortOrder==SQLITE_SO_ASC + || iSortOrder==SQLITE_SO_DESC + ); + assert( eNulls==SQLITE_SO_UNDEFINED + || eNulls==SQLITE_SO_ASC + || eNulls==SQLITE_SO_DESC + ); + + pItem = &p->a[p->nExpr-1]; + assert( pItem->bNulls==0 ); + if( iSortOrder==SQLITE_SO_UNDEFINED ){ + iSortOrder = SQLITE_SO_ASC; + } + pItem->sortFlags = (u8)iSortOrder; + + if( eNulls!=SQLITE_SO_UNDEFINED ){ + pItem->bNulls = 1; + if( iSortOrder!=eNulls ){ + pItem->sortFlags |= KEYINFO_ORDER_BIGNULL; + } } - p->a[p->nExpr-1].sortOrder = (u8)iSortOrder; } /* -** Set the ExprList.a[].zName element of the most recently added item +** Set the ExprList.a[].zEName element of the most recently added item ** on the expression list. ** ** pList might be NULL following an OOM error. But pName should never be @@ -96383,13 +101272,23 @@ SQLITE_PRIVATE void sqlite3ExprListSetName( int dequote /* True to cause the name to be dequoted */ ){ assert( pList!=0 || pParse->db->mallocFailed!=0 ); + assert( pParse->eParseMode!=PARSE_MODE_UNMAP || dequote==0 ); if( pList ){ struct ExprList_item *pItem; assert( pList->nExpr>0 ); pItem = &pList->a[pList->nExpr-1]; - assert( pItem->zName==0 ); - pItem->zName = sqlite3DbStrNDup(pParse->db, pName->z, pName->n); - if( dequote ) sqlite3Dequote(pItem->zName); + assert( pItem->zEName==0 ); + assert( pItem->eEName==ENAME_NAME ); + pItem->zEName = sqlite3DbStrNDup(pParse->db, pName->z, pName->n); + if( dequote ){ + /* If dequote==0, then pName->z does not point to part of a DDL + ** statement handled by the parser. And so no token need be added + ** to the token-map. */ + sqlite3Dequote(pItem->zEName); + if( IN_RENAME_OBJECT ){ + sqlite3RenameTokenMap(pParse, (void*)pItem->zEName, pName); + } + } } } @@ -96412,8 +101311,10 @@ SQLITE_PRIVATE void sqlite3ExprListSetSpan( if( pList ){ struct ExprList_item *pItem = &pList->a[pList->nExpr-1]; assert( pList->nExpr>0 ); - sqlite3DbFree(db, pItem->zSpan); - pItem->zSpan = sqlite3DbSpanDup(db, zStart, zEnd); + if( pItem->zEName==0 ){ + pItem->zEName = sqlite3DbSpanDup(db, zStart, zEnd); + pItem->eEName = ENAME_SPAN; + } } } @@ -96443,8 +101344,7 @@ static SQLITE_NOINLINE void exprListDeleteNN(sqlite3 *db, ExprList *pList){ assert( pList->nExpr>0 ); do{ sqlite3ExprDelete(db, pItem->pExpr); - sqlite3DbFree(db, pItem->zName); - sqlite3DbFree(db, pItem->zSpan); + sqlite3DbFree(db, pItem->zEName); pItem++; }while( --i>0 ); sqlite3DbFreeNN(db, pList); @@ -96482,17 +101382,34 @@ SQLITE_PRIVATE int sqlite3SelectWalkFail(Walker *pWalker, Select *NotUsed){ return WRC_Abort; } +/* +** Check the input string to see if it is "true" or "false" (in any case). +** +** If the string is.... Return +** "true" EP_IsTrue +** "false" EP_IsFalse +** anything else 0 +*/ +SQLITE_PRIVATE u32 sqlite3IsTrueOrFalse(const char *zIn){ + if( sqlite3StrICmp(zIn, "true")==0 ) return EP_IsTrue; + if( sqlite3StrICmp(zIn, "false")==0 ) return EP_IsFalse; + return 0; +} + + /* ** If the input expression is an ID with the name "true" or "false" ** then convert it into an TK_TRUEFALSE term. Return non-zero if ** the conversion happened, and zero if the expression is unaltered. */ SQLITE_PRIVATE int sqlite3ExprIdToTrueFalse(Expr *pExpr){ + u32 v; assert( pExpr->op==TK_ID || pExpr->op==TK_STRING ); - if( sqlite3StrICmp(pExpr->u.zToken, "true")==0 - || sqlite3StrICmp(pExpr->u.zToken, "false")==0 + if( !ExprHasProperty(pExpr, EP_Quoted) + && (v = sqlite3IsTrueOrFalse(pExpr->u.zToken))!=0 ){ pExpr->op = TK_TRUEFALSE; + ExprSetProperty(pExpr, v); return 1; } return 0; @@ -96503,12 +101420,40 @@ SQLITE_PRIVATE int sqlite3ExprIdToTrueFalse(Expr *pExpr){ ** and 0 if it is FALSE. */ SQLITE_PRIVATE int sqlite3ExprTruthValue(const Expr *pExpr){ + pExpr = sqlite3ExprSkipCollate((Expr*)pExpr); assert( pExpr->op==TK_TRUEFALSE ); assert( sqlite3StrICmp(pExpr->u.zToken,"true")==0 || sqlite3StrICmp(pExpr->u.zToken,"false")==0 ); return pExpr->u.zToken[4]==0; } +/* +** If pExpr is an AND or OR expression, try to simplify it by eliminating +** terms that are always true or false. Return the simplified expression. +** Or return the original expression if no simplification is possible. +** +** Examples: +** +** (x<10) AND true => (x<10) +** (x<10) AND false => false +** (x<10) AND (y=22 OR false) => (x<10) AND (y=22) +** (x<10) AND (y=22 OR true) => (x<10) +** (y=22) OR true => true +*/ +SQLITE_PRIVATE Expr *sqlite3ExprSimplifiedAndOr(Expr *pExpr){ + assert( pExpr!=0 ); + if( pExpr->op==TK_AND || pExpr->op==TK_OR ){ + Expr *pRight = sqlite3ExprSimplifiedAndOr(pExpr->pRight); + Expr *pLeft = sqlite3ExprSimplifiedAndOr(pExpr->pLeft); + if( ExprAlwaysTrue(pLeft) || ExprAlwaysFalse(pRight) ){ + pExpr = pExpr->op==TK_AND ? pRight : pLeft; + }else if( ExprAlwaysTrue(pRight) || ExprAlwaysFalse(pLeft) ){ + pExpr = pExpr->op==TK_AND ? pLeft : pRight; + } + } + return pExpr; +} + /* ** These routines are Walker callbacks used to check expressions to @@ -96526,10 +101471,11 @@ SQLITE_PRIVATE int sqlite3ExprTruthValue(const Expr *pExpr){ ** In all cases, the callbacks set Walker.eCode=0 and abort if the expression ** is found to not be a constant. ** -** The sqlite3ExprIsConstantOrFunction() is used for evaluating expressions -** in a CREATE TABLE statement. The Walker.eCode value is 5 when parsing -** an existing schema and 4 when processing a new statement. A bound -** parameter raises an error for new statements, but is silently converted +** The sqlite3ExprIsConstantOrFunction() is used for evaluating DEFAULT +** expressions in a CREATE TABLE statement. The Walker.eCode value is 5 +** when parsing an existing schema out of the sqlite_master table and 4 +** when processing a new CREATE TABLE statement. A bound parameter raises +** an error for new statements, but is silently converted ** to NULL for existing schemas. This allows sqlite_master tables that ** contain a bound parameter because they were generated by older versions ** of SQLite to be parsed by newer versions of SQLite without raising a @@ -96550,7 +101496,10 @@ static int exprNodeIsConstant(Walker *pWalker, Expr *pExpr){ ** and either pWalker->eCode==4 or 5 or the function has the ** SQLITE_FUNC_CONST flag. */ case TK_FUNCTION: - if( pWalker->eCode>=4 || ExprHasProperty(pExpr,EP_ConstFunc) ){ + if( (pWalker->eCode>=4 || ExprHasProperty(pExpr,EP_ConstFunc)) + && !ExprHasProperty(pExpr, EP_WinFunc) + ){ + if( pWalker->eCode==5 ) ExprSetProperty(pExpr, EP_FromDDL); return WRC_Continue; }else{ pWalker->eCode = 0; @@ -96570,6 +101519,9 @@ static int exprNodeIsConstant(Walker *pWalker, Expr *pExpr){ testcase( pExpr->op==TK_COLUMN ); testcase( pExpr->op==TK_AGG_FUNCTION ); testcase( pExpr->op==TK_AGG_COLUMN ); + if( ExprHasProperty(pExpr, EP_FixedCol) && pWalker->eCode!=2 ){ + return WRC_Continue; + } if( pWalker->eCode==3 && pExpr->iTable==pWalker->u.iCur ){ return WRC_Continue; } @@ -96625,10 +101577,17 @@ SQLITE_PRIVATE int sqlite3ExprIsConstant(Expr *p){ } /* -** Walk an expression tree. Return non-zero if the expression is constant -** that does no originate from the ON or USING clauses of a join. -** Return 0 if it involves variables or function calls or terms from -** an ON or USING clause. +** Walk an expression tree. Return non-zero if +** +** (1) the expression is constant, and +** (2) the expression does originate in the ON or USING clause +** of a LEFT JOIN, and +** (3) the expression does not contain any EP_FixedCol TK_COLUMN +** operands created by the constant propagation optimization. +** +** When this routine returns true, it indicates that the expression +** can be added to the pParse->pConstExpr list and evaluated once when +** the prepared statement starts up. See sqlite3ExprCodeRunJustOnce(). */ SQLITE_PRIVATE int sqlite3ExprIsConstantNotJoin(Expr *p){ return exprIsConst(p, 2, 0); @@ -96658,7 +101617,7 @@ static int exprNodeIsConstantOrGroupBy(Walker *pWalker, Expr *pExpr){ Expr *p = pGroupBy->a[i].pExpr; if( sqlite3ExprCompare(0, pExpr, p, -1)<2 ){ CollSeq *pColl = sqlite3ExprNNCollSeq(pWalker->pParse, p); - if( sqlite3_stricmp("BINARY", pColl->zName)==0 ){ + if( sqlite3IsBinary(pColl) ){ return WRC_Prune; } } @@ -96704,9 +101663,21 @@ SQLITE_PRIVATE int sqlite3ExprIsConstantOrGroupBy(Parse *pParse, Expr *p, ExprLi } /* -** Walk an expression tree. Return non-zero if the expression is constant -** or a function call with constant arguments. Return and 0 if there -** are any variables. +** Walk an expression tree for the DEFAULT field of a column definition +** in a CREATE TABLE statement. Return non-zero if the expression is +** acceptable for use as a DEFAULT. That is to say, return non-zero if +** the expression is constant or a function call with constant arguments. +** Return and 0 if there are any variables. +** +** isInit is true when parsing from sqlite_master. isInit is false when +** processing a new CREATE TABLE statement. When isInit is true, parameters +** (such as ? or $abc) in the expression are converted into NULL. When +** isInit is false, parameters raise an error. Parameters should not be +** allowed in a CREATE TABLE statement, but some legacy versions of SQLite +** allowed it, so we need to support it when reading sqlite_master for +** backwards compatibility. +** +** If isInit is true, set EP_FromDDL on every TK_FUNCTION node. ** ** For the purposes of this function, a double-quoted string (ex: "abc") ** is considered a variable but a single-quoted string (ex: 'abc') is @@ -96743,7 +101714,7 @@ SQLITE_PRIVATE int sqlite3ExprContainsSubquery(Expr *p){ */ SQLITE_PRIVATE int sqlite3ExprIsInteger(Expr *p, int *pValue){ int rc = 0; - if( p==0 ) return 0; /* Can only happen following on OOM */ + if( NEVER(p==0) ) return 0; /* Used to only happen following on OOM */ /* If an expression is an integer literal that fits in a signed 32-bit ** integer, then the EP_IntValue flag will have already been set */ @@ -96789,7 +101760,9 @@ SQLITE_PRIVATE int sqlite3ExprIsInteger(Expr *p, int *pValue){ */ SQLITE_PRIVATE int sqlite3ExprCanBeNull(const Expr *p){ u8 op; - while( p->op==TK_UPLUS || p->op==TK_UMINUS ){ p = p->pLeft; } + while( p->op==TK_UPLUS || p->op==TK_UMINUS ){ + p = p->pLeft; + } op = p->op; if( op==TK_REGISTER ) op = p->op2; switch( op ){ @@ -96800,8 +101773,10 @@ SQLITE_PRIVATE int sqlite3ExprCanBeNull(const Expr *p){ return 0; case TK_COLUMN: return ExprHasProperty(p, EP_CanBeNull) || - p->pTab==0 || /* Reference to column of index on expression */ - (p->iColumn>=0 && p->pTab->aCol[p->iColumn].notNull==0); + p->y.pTab==0 || /* Reference to column of index on expression */ + (p->iColumn>=0 + && ALWAYS(p->y.pTab->aCol!=0) /* Defense against OOM problems */ + && p->y.pTab->aCol[p->iColumn].notNull==0); default: return 1; } @@ -96819,27 +101794,30 @@ SQLITE_PRIVATE int sqlite3ExprCanBeNull(const Expr *p){ */ SQLITE_PRIVATE int sqlite3ExprNeedsNoAffinityChange(const Expr *p, char aff){ u8 op; + int unaryMinus = 0; if( aff==SQLITE_AFF_BLOB ) return 1; - while( p->op==TK_UPLUS || p->op==TK_UMINUS ){ p = p->pLeft; } + while( p->op==TK_UPLUS || p->op==TK_UMINUS ){ + if( p->op==TK_UMINUS ) unaryMinus = 1; + p = p->pLeft; + } op = p->op; if( op==TK_REGISTER ) op = p->op2; switch( op ){ case TK_INTEGER: { - return aff==SQLITE_AFF_INTEGER || aff==SQLITE_AFF_NUMERIC; + return aff>=SQLITE_AFF_NUMERIC; } case TK_FLOAT: { - return aff==SQLITE_AFF_REAL || aff==SQLITE_AFF_NUMERIC; + return aff>=SQLITE_AFF_NUMERIC; } case TK_STRING: { - return aff==SQLITE_AFF_TEXT; + return !unaryMinus && aff==SQLITE_AFF_TEXT; } case TK_BLOB: { - return 1; + return !unaryMinus; } case TK_COLUMN: { assert( p->iTable>=0 ); /* p cannot be part of a CHECK constraint */ - return p->iColumn<0 - && (aff==SQLITE_AFF_INTEGER || aff==SQLITE_AFF_NUMERIC); + return aff>=SQLITE_AFF_NUMERIC && p->iColumn<0; } default: { return 0; @@ -97022,10 +102000,11 @@ static int sqlite3InRhsIsConstant(Expr *pIn){ #ifndef SQLITE_OMIT_SUBQUERY SQLITE_PRIVATE int sqlite3FindInIndex( Parse *pParse, /* Parsing context */ - Expr *pX, /* The right-hand side (RHS) of the IN operator */ + Expr *pX, /* The IN expression */ u32 inFlags, /* IN_INDEX_LOOP, _MEMBERSHIP, and/or _NOOP_OK */ int *prRhsHasNull, /* Register holding NULL status. See notes */ - int *aiMap /* Mapping from Index fields to RHS fields */ + int *aiMap, /* Mapping from Index fields to RHS fields */ + int *piTab /* OUT: index to use */ ){ Select *p; /* SELECT to the right of IN operator */ int eType = 0; /* Type of RHS table. IN_INDEX_* */ @@ -97080,7 +102059,8 @@ SQLITE_PRIVATE int sqlite3FindInIndex( sqlite3OpenTable(pParse, iTab, iDb, pTab, OP_OpenRead); eType = IN_INDEX_ROWID; - + ExplainQueryPlan((pParse, 0, + "USING ROWID SEARCH ON TABLE %s FOR IN-OPERATOR",pTab->zName)); sqlite3VdbeJumpHere(v, iAddr); }else{ Index *pIdx; /* Iterator variable */ @@ -97119,6 +102099,7 @@ SQLITE_PRIVATE int sqlite3FindInIndex( Bitmask colUsed; /* Columns of the index used */ Bitmask mCol; /* Mask for the current column */ if( pIdx->nColumnpPartIdxWhere!=0 ) continue; /* Maximum nColumn is BMS-2, not BMS-1, so that we can compute ** BITMASK(nExpr) without overflowing */ testcase( pIdx->nColumn==BMS-2 ); @@ -97209,16 +102190,15 @@ SQLITE_PRIVATE int sqlite3FindInIndex( eType = IN_INDEX_EPH; if( inFlags & IN_INDEX_LOOP ){ pParse->nQueryLoop = 0; - if( pX->pLeft->iColumn<0 && !ExprHasProperty(pX, EP_xIsSelect) ){ - eType = IN_INDEX_ROWID; - } }else if( prRhsHasNull ){ *prRhsHasNull = rMayHaveNull = ++pParse->nMem; } - sqlite3CodeSubselect(pParse, pX, rMayHaveNull, eType==IN_INDEX_ROWID); + assert( pX->op==TK_IN ); + sqlite3CodeRhsOfIN(pParse, pX, iTab); + if( rMayHaveNull ){ + sqlite3SetHasNullFlag(v, iTab, rMayHaveNull); + } pParse->nQueryLoop = savedNQueryLoop; - }else{ - pX->iTable = iTab; } if( aiMap && eType!=IN_INDEX_INDEX_ASC && eType!=IN_INDEX_INDEX_DESC ){ @@ -97226,6 +102206,7 @@ SQLITE_PRIVATE int sqlite3FindInIndex( n = sqlite3ExprVectorSize(pX->pLeft); for(i=0; inErr==0 ){ + const char *zFmt = "sub-select returns %d columns - expected %d"; + sqlite3ErrorMsg(pParse, zFmt, nActual, nExpect); + } } #endif @@ -97299,264 +102282,320 @@ SQLITE_PRIVATE void sqlite3VectorErrorMsg(Parse *pParse, Expr *pExpr){ } } +#ifndef SQLITE_OMIT_SUBQUERY /* -** Generate code for scalar subqueries used as a subquery expression, EXISTS, -** or IN operators. Examples: +** Generate code that will construct an ephemeral table containing all terms +** in the RHS of an IN operator. The IN operator can be in either of two +** forms: ** -** (SELECT a FROM b) -- subquery -** EXISTS (SELECT a FROM b) -- EXISTS subquery ** x IN (4,5,11) -- IN operator with list on right-hand side ** x IN (SELECT a FROM b) -- IN operator with subquery on the right ** -** The pExpr parameter describes the expression that contains the IN -** operator or subquery. -** -** If parameter isRowid is non-zero, then expression pExpr is guaranteed -** to be of the form " IN (?, ?, ?)", where is a reference -** to some integer key column of a table B-Tree. In this case, use an -** intkey B-Tree to store the set of IN(...) values instead of the usual -** (slower) variable length keys B-Tree. -** -** If rMayHaveNull is non-zero, that means that the operation is an IN -** (not a SELECT or EXISTS) and that the RHS might contains NULLs. -** All this routine does is initialize the register given by rMayHaveNull -** to NULL. Calling routines will take care of changing this register -** value to non-NULL if the RHS is NULL-free. -** -** For a SELECT or EXISTS operator, return the register that holds the -** result. For a multi-column SELECT, the result is stored in a contiguous -** array of registers and the return value is the register of the left-most -** result column. Return 0 for IN operators or if an error occurs. -*/ -#ifndef SQLITE_OMIT_SUBQUERY -SQLITE_PRIVATE int sqlite3CodeSubselect( +** The pExpr parameter is the IN operator. The cursor number for the +** constructed ephermeral table is returned. The first time the ephemeral +** table is computed, the cursor number is also stored in pExpr->iTable, +** however the cursor number returned might not be the same, as it might +** have been duplicated using OP_OpenDup. +** +** If the LHS expression ("x" in the examples) is a column value, or +** the SELECT statement returns a column value, then the affinity of that +** column is used to build the index keys. If both 'x' and the +** SELECT... statement are columns, then numeric affinity is used +** if either column has NUMERIC or INTEGER affinity. If neither +** 'x' nor the SELECT... statement are columns, then numeric affinity +** is used. +*/ +SQLITE_PRIVATE void sqlite3CodeRhsOfIN( Parse *pParse, /* Parsing context */ - Expr *pExpr, /* The IN, SELECT, or EXISTS operator */ - int rHasNullFlag, /* Register that records whether NULLs exist in RHS */ - int isRowid /* If true, LHS of IN operator is a rowid */ + Expr *pExpr, /* The IN operator */ + int iTab /* Use this cursor number */ ){ - int jmpIfDynamic = -1; /* One-time test address */ - int rReg = 0; /* Register storing resulting */ - Vdbe *v = sqlite3GetVdbe(pParse); - if( NEVER(v==0) ) return 0; - sqlite3ExprCachePush(pParse); + int addrOnce = 0; /* Address of the OP_Once instruction at top */ + int addr; /* Address of OP_OpenEphemeral instruction */ + Expr *pLeft; /* the LHS of the IN operator */ + KeyInfo *pKeyInfo = 0; /* Key information */ + int nVal; /* Size of vector pLeft */ + Vdbe *v; /* The prepared statement under construction */ - /* The evaluation of the IN/EXISTS/SELECT must be repeated every time it + v = pParse->pVdbe; + assert( v!=0 ); + + /* The evaluation of the IN must be repeated every time it ** is encountered if any of the following is true: ** ** * The right-hand side is a correlated subquery ** * The right-hand side is an expression list containing variables ** * We are inside a trigger ** - ** If all of the above are false, then we can run this code just once - ** save the results, and reuse the same result on subsequent invocations. + ** If all of the above are false, then we can compute the RHS just once + ** and reuse it many names. */ - if( !ExprHasProperty(pExpr, EP_VarSelect) ){ - jmpIfDynamic = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); - } + if( !ExprHasProperty(pExpr, EP_VarSelect) && pParse->iSelfTab==0 ){ + /* Reuse of the RHS is allowed */ + /* If this routine has already been coded, but the previous code + ** might not have been invoked yet, so invoke it now as a subroutine. + */ + if( ExprHasProperty(pExpr, EP_Subrtn) ){ + addrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); + if( ExprHasProperty(pExpr, EP_xIsSelect) ){ + ExplainQueryPlan((pParse, 0, "REUSE LIST SUBQUERY %d", + pExpr->x.pSelect->selId)); + } + sqlite3VdbeAddOp2(v, OP_Gosub, pExpr->y.sub.regReturn, + pExpr->y.sub.iAddr); + sqlite3VdbeAddOp2(v, OP_OpenDup, iTab, pExpr->iTable); + sqlite3VdbeJumpHere(v, addrOnce); + return; + } - switch( pExpr->op ){ - case TK_IN: { - int addr; /* Address of OP_OpenEphemeral instruction */ - Expr *pLeft = pExpr->pLeft; /* the LHS of the IN operator */ - KeyInfo *pKeyInfo = 0; /* Key information */ - int nVal; /* Size of vector pLeft */ - - nVal = sqlite3ExprVectorSize(pLeft); - assert( !isRowid || nVal==1 ); + /* Begin coding the subroutine */ + ExprSetProperty(pExpr, EP_Subrtn); + assert( !ExprHasProperty(pExpr, EP_TokenOnly|EP_Reduced) ); + pExpr->y.sub.regReturn = ++pParse->nMem; + pExpr->y.sub.iAddr = + sqlite3VdbeAddOp2(v, OP_Integer, 0, pExpr->y.sub.regReturn) + 1; + VdbeComment((v, "return address")); - /* Whether this is an 'x IN(SELECT...)' or an 'x IN()' - ** expression it is handled the same way. An ephemeral table is - ** filled with index keys representing the results from the - ** SELECT or the . - ** - ** If the 'x' expression is a column value, or the SELECT... - ** statement returns a column value, then the affinity of that - ** column is used to build the index keys. If both 'x' and the - ** SELECT... statement are columns, then numeric affinity is used - ** if either column has NUMERIC or INTEGER affinity. If neither - ** 'x' nor the SELECT... statement are columns, then numeric affinity - ** is used. - */ - pExpr->iTable = pParse->nTab++; - addr = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, - pExpr->iTable, (isRowid?0:nVal)); - pKeyInfo = isRowid ? 0 : sqlite3KeyInfoAlloc(pParse->db, nVal, 1); + addrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); + } - if( ExprHasProperty(pExpr, EP_xIsSelect) ){ - /* Case 1: expr IN (SELECT ...) - ** - ** Generate code to write the results of the select into the temporary - ** table allocated and opened above. - */ - Select *pSelect = pExpr->x.pSelect; - ExprList *pEList = pSelect->pEList; - - ExplainQueryPlan((pParse, 1, "%sLIST SUBQUERY", - jmpIfDynamic>=0?"":"CORRELATED " - )); - assert( !isRowid ); - /* If the LHS and RHS of the IN operator do not match, that - ** error will have been caught long before we reach this point. */ - if( ALWAYS(pEList->nExpr==nVal) ){ - SelectDest dest; - int i; - sqlite3SelectDestInit(&dest, SRT_Set, pExpr->iTable); - dest.zAffSdst = exprINAffinity(pParse, pExpr); - pSelect->iLimit = 0; - testcase( pSelect->selFlags & SF_Distinct ); - testcase( pKeyInfo==0 ); /* Caused by OOM in sqlite3KeyInfoAlloc() */ - if( sqlite3Select(pParse, pSelect, &dest) ){ - sqlite3DbFree(pParse->db, dest.zAffSdst); - sqlite3KeyInfoUnref(pKeyInfo); - return 0; - } - sqlite3DbFree(pParse->db, dest.zAffSdst); - assert( pKeyInfo!=0 ); /* OOM will cause exit after sqlite3Select() */ - assert( pEList!=0 ); - assert( pEList->nExpr>0 ); - assert( sqlite3KeyInfoIsWriteable(pKeyInfo) ); - for(i=0; iaColl[i] = sqlite3BinaryCompareCollSeq( - pParse, p, pEList->a[i].pExpr - ); - } - } - }else if( ALWAYS(pExpr->x.pList!=0) ){ - /* Case 2: expr IN (exprlist) - ** - ** For each expression, build an index key from the evaluation and - ** store it in the temporary table. If is a column, then use - ** that columns affinity when building index keys. If is not - ** a column, use numeric affinity. - */ - char affinity; /* Affinity of the LHS of the IN */ - int i; - ExprList *pList = pExpr->x.pList; - struct ExprList_item *pItem; - int r1, r2, r3; - affinity = sqlite3ExprAffinity(pLeft); - if( !affinity ){ - affinity = SQLITE_AFF_BLOB; - } - if( pKeyInfo ){ - assert( sqlite3KeyInfoIsWriteable(pKeyInfo) ); - pKeyInfo->aColl[0] = sqlite3ExprCollSeq(pParse, pExpr->pLeft); - } + /* Check to see if this is a vector IN operator */ + pLeft = pExpr->pLeft; + nVal = sqlite3ExprVectorSize(pLeft); - /* Loop through each expression in . */ - r1 = sqlite3GetTempReg(pParse); - r2 = sqlite3GetTempReg(pParse); - if( isRowid ) sqlite3VdbeAddOp4(v, OP_Blob, 0, r2, 0, "", P4_STATIC); - for(i=pList->nExpr, pItem=pList->a; i>0; i--, pItem++){ - Expr *pE2 = pItem->pExpr; - int iValToIns; - - /* If the expression is not constant then we will need to - ** disable the test that was generated above that makes sure - ** this code only executes once. Because for a non-constant - ** expression we need to rerun this code each time. - */ - if( jmpIfDynamic>=0 && !sqlite3ExprIsConstant(pE2) ){ - sqlite3VdbeChangeToNoop(v, jmpIfDynamic); - jmpIfDynamic = -1; - } + /* Construct the ephemeral table that will contain the content of + ** RHS of the IN operator. + */ + pExpr->iTable = iTab; + addr = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pExpr->iTable, nVal); +#ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS + if( ExprHasProperty(pExpr, EP_xIsSelect) ){ + VdbeComment((v, "Result of SELECT %u", pExpr->x.pSelect->selId)); + }else{ + VdbeComment((v, "RHS of IN operator")); + } +#endif + pKeyInfo = sqlite3KeyInfoAlloc(pParse->db, nVal, 1); - /* Evaluate the expression and insert it into the temp table */ - if( isRowid && sqlite3ExprIsInteger(pE2, &iValToIns) ){ - sqlite3VdbeAddOp3(v, OP_InsertInt, pExpr->iTable, r2, iValToIns); - }else{ - r3 = sqlite3ExprCodeTarget(pParse, pE2, r1); - if( isRowid ){ - sqlite3VdbeAddOp2(v, OP_MustBeInt, r3, - sqlite3VdbeCurrentAddr(v)+2); - VdbeCoverage(v); - sqlite3VdbeAddOp3(v, OP_Insert, pExpr->iTable, r2, r3); - }else{ - sqlite3VdbeAddOp4(v, OP_MakeRecord, r3, 1, r2, &affinity, 1); - sqlite3ExprCacheAffinityChange(pParse, r3, 1); - sqlite3VdbeAddOp4Int(v, OP_IdxInsert, pExpr->iTable, r2, r3, 1); - } - } - } - sqlite3ReleaseTempReg(pParse, r1); - sqlite3ReleaseTempReg(pParse, r2); + if( ExprHasProperty(pExpr, EP_xIsSelect) ){ + /* Case 1: expr IN (SELECT ...) + ** + ** Generate code to write the results of the select into the temporary + ** table allocated and opened above. + */ + Select *pSelect = pExpr->x.pSelect; + ExprList *pEList = pSelect->pEList; + + ExplainQueryPlan((pParse, 1, "%sLIST SUBQUERY %d", + addrOnce?"":"CORRELATED ", pSelect->selId + )); + /* If the LHS and RHS of the IN operator do not match, that + ** error will have been caught long before we reach this point. */ + if( ALWAYS(pEList->nExpr==nVal) ){ + SelectDest dest; + int i; + sqlite3SelectDestInit(&dest, SRT_Set, iTab); + dest.zAffSdst = exprINAffinity(pParse, pExpr); + pSelect->iLimit = 0; + testcase( pSelect->selFlags & SF_Distinct ); + testcase( pKeyInfo==0 ); /* Caused by OOM in sqlite3KeyInfoAlloc() */ + if( sqlite3Select(pParse, pSelect, &dest) ){ + sqlite3DbFree(pParse->db, dest.zAffSdst); + sqlite3KeyInfoUnref(pKeyInfo); + return; } - if( pKeyInfo ){ - sqlite3VdbeChangeP4(v, addr, (void *)pKeyInfo, P4_KEYINFO); + sqlite3DbFree(pParse->db, dest.zAffSdst); + assert( pKeyInfo!=0 ); /* OOM will cause exit after sqlite3Select() */ + assert( pEList!=0 ); + assert( pEList->nExpr>0 ); + assert( sqlite3KeyInfoIsWriteable(pKeyInfo) ); + for(i=0; iaColl[i] = sqlite3BinaryCompareCollSeq( + pParse, p, pEList->a[i].pExpr + ); } - break; } - - case TK_EXISTS: - case TK_SELECT: - default: { - /* Case 3: (SELECT ... FROM ...) - ** or: EXISTS(SELECT ... FROM ...) - ** - ** For a SELECT, generate code to put the values for all columns of - ** the first row into an array of registers and return the index of - ** the first register. - ** - ** If this is an EXISTS, write an integer 0 (not exists) or 1 (exists) - ** into a register and return that register number. - ** - ** In both cases, the query is augmented with "LIMIT 1". Any - ** preexisting limit is discarded in place of the new LIMIT 1. + }else if( ALWAYS(pExpr->x.pList!=0) ){ + /* Case 2: expr IN (exprlist) + ** + ** For each expression, build an index key from the evaluation and + ** store it in the temporary table. If is a column, then use + ** that columns affinity when building index keys. If is not + ** a column, use numeric affinity. + */ + char affinity; /* Affinity of the LHS of the IN */ + int i; + ExprList *pList = pExpr->x.pList; + struct ExprList_item *pItem; + int r1, r2; + affinity = sqlite3ExprAffinity(pLeft); + if( affinity<=SQLITE_AFF_NONE ){ + affinity = SQLITE_AFF_BLOB; + }else if( affinity==SQLITE_AFF_REAL ){ + affinity = SQLITE_AFF_NUMERIC; + } + if( pKeyInfo ){ + assert( sqlite3KeyInfoIsWriteable(pKeyInfo) ); + pKeyInfo->aColl[0] = sqlite3ExprCollSeq(pParse, pExpr->pLeft); + } + + /* Loop through each expression in . */ + r1 = sqlite3GetTempReg(pParse); + r2 = sqlite3GetTempReg(pParse); + for(i=pList->nExpr, pItem=pList->a; i>0; i--, pItem++){ + Expr *pE2 = pItem->pExpr; + + /* If the expression is not constant then we will need to + ** disable the test that was generated above that makes sure + ** this code only executes once. Because for a non-constant + ** expression we need to rerun this code each time. */ - Select *pSel; /* SELECT statement to encode */ - SelectDest dest; /* How to deal with SELECT result */ - int nReg; /* Registers to allocate */ - Expr *pLimit; /* New limit expression */ - - testcase( pExpr->op==TK_EXISTS ); - testcase( pExpr->op==TK_SELECT ); - assert( pExpr->op==TK_EXISTS || pExpr->op==TK_SELECT ); - assert( ExprHasProperty(pExpr, EP_xIsSelect) ); - - pSel = pExpr->x.pSelect; - ExplainQueryPlan((pParse, 1, "%sSCALAR SUBQUERY", - jmpIfDynamic>=0?"":"CORRELATED ")); - nReg = pExpr->op==TK_SELECT ? pSel->pEList->nExpr : 1; - sqlite3SelectDestInit(&dest, 0, pParse->nMem+1); - pParse->nMem += nReg; - if( pExpr->op==TK_SELECT ){ - dest.eDest = SRT_Mem; - dest.iSdst = dest.iSDParm; - dest.nSdst = nReg; - sqlite3VdbeAddOp3(v, OP_Null, 0, dest.iSDParm, dest.iSDParm+nReg-1); - VdbeComment((v, "Init subquery result")); - }else{ - dest.eDest = SRT_Exists; - sqlite3VdbeAddOp2(v, OP_Integer, 0, dest.iSDParm); - VdbeComment((v, "Init EXISTS result")); - } - pLimit = sqlite3ExprAlloc(pParse->db, TK_INTEGER,&sqlite3IntTokens[1], 0); - if( pSel->pLimit ){ - sqlite3ExprDelete(pParse->db, pSel->pLimit->pLeft); - pSel->pLimit->pLeft = pLimit; - }else{ - pSel->pLimit = sqlite3PExpr(pParse, TK_LIMIT, pLimit, 0); + if( addrOnce && !sqlite3ExprIsConstant(pE2) ){ + sqlite3VdbeChangeToNoop(v, addrOnce); + ExprClearProperty(pExpr, EP_Subrtn); + addrOnce = 0; } - pSel->iLimit = 0; - if( sqlite3Select(pParse, pSel, &dest) ){ - return 0; - } - rReg = dest.iSDParm; - ExprSetVVAProperty(pExpr, EP_NoReduce); - break; + + /* Evaluate the expression and insert it into the temp table */ + sqlite3ExprCode(pParse, pE2, r1); + sqlite3VdbeAddOp4(v, OP_MakeRecord, r1, 1, r2, &affinity, 1); + sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iTab, r2, r1, 1); } + sqlite3ReleaseTempReg(pParse, r1); + sqlite3ReleaseTempReg(pParse, r2); + } + if( pKeyInfo ){ + sqlite3VdbeChangeP4(v, addr, (void *)pKeyInfo, P4_KEYINFO); } + if( addrOnce ){ + sqlite3VdbeJumpHere(v, addrOnce); + /* Subroutine return */ + sqlite3VdbeAddOp1(v, OP_Return, pExpr->y.sub.regReturn); + sqlite3VdbeChangeP1(v, pExpr->y.sub.iAddr-1, sqlite3VdbeCurrentAddr(v)-1); + sqlite3ClearTempRegCache(pParse); + } +} +#endif /* SQLITE_OMIT_SUBQUERY */ - if( rHasNullFlag ){ - sqlite3SetHasNullFlag(v, pExpr->iTable, rHasNullFlag); +/* +** Generate code for scalar subqueries used as a subquery expression +** or EXISTS operator: +** +** (SELECT a FROM b) -- subquery +** EXISTS (SELECT a FROM b) -- EXISTS subquery +** +** The pExpr parameter is the SELECT or EXISTS operator to be coded. +** +** Return the register that holds the result. For a multi-column SELECT, +** the result is stored in a contiguous array of registers and the +** return value is the register of the left-most result column. +** Return 0 if an error occurs. +*/ +#ifndef SQLITE_OMIT_SUBQUERY +SQLITE_PRIVATE int sqlite3CodeSubselect(Parse *pParse, Expr *pExpr){ + int addrOnce = 0; /* Address of OP_Once at top of subroutine */ + int rReg = 0; /* Register storing resulting */ + Select *pSel; /* SELECT statement to encode */ + SelectDest dest; /* How to deal with SELECT result */ + int nReg; /* Registers to allocate */ + Expr *pLimit; /* New limit expression */ + + Vdbe *v = pParse->pVdbe; + assert( v!=0 ); + testcase( pExpr->op==TK_EXISTS ); + testcase( pExpr->op==TK_SELECT ); + assert( pExpr->op==TK_EXISTS || pExpr->op==TK_SELECT ); + assert( ExprHasProperty(pExpr, EP_xIsSelect) ); + pSel = pExpr->x.pSelect; + + /* The evaluation of the EXISTS/SELECT must be repeated every time it + ** is encountered if any of the following is true: + ** + ** * The right-hand side is a correlated subquery + ** * The right-hand side is an expression list containing variables + ** * We are inside a trigger + ** + ** If all of the above are false, then we can run this code just once + ** save the results, and reuse the same result on subsequent invocations. + */ + if( !ExprHasProperty(pExpr, EP_VarSelect) ){ + /* If this routine has already been coded, then invoke it as a + ** subroutine. */ + if( ExprHasProperty(pExpr, EP_Subrtn) ){ + ExplainQueryPlan((pParse, 0, "REUSE SUBQUERY %d", pSel->selId)); + sqlite3VdbeAddOp2(v, OP_Gosub, pExpr->y.sub.regReturn, + pExpr->y.sub.iAddr); + return pExpr->iTable; + } + + /* Begin coding the subroutine */ + ExprSetProperty(pExpr, EP_Subrtn); + pExpr->y.sub.regReturn = ++pParse->nMem; + pExpr->y.sub.iAddr = + sqlite3VdbeAddOp2(v, OP_Integer, 0, pExpr->y.sub.regReturn) + 1; + VdbeComment((v, "return address")); + + addrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); + } + + /* For a SELECT, generate code to put the values for all columns of + ** the first row into an array of registers and return the index of + ** the first register. + ** + ** If this is an EXISTS, write an integer 0 (not exists) or 1 (exists) + ** into a register and return that register number. + ** + ** In both cases, the query is augmented with "LIMIT 1". Any + ** preexisting limit is discarded in place of the new LIMIT 1. + */ + ExplainQueryPlan((pParse, 1, "%sSCALAR SUBQUERY %d", + addrOnce?"":"CORRELATED ", pSel->selId)); + nReg = pExpr->op==TK_SELECT ? pSel->pEList->nExpr : 1; + sqlite3SelectDestInit(&dest, 0, pParse->nMem+1); + pParse->nMem += nReg; + if( pExpr->op==TK_SELECT ){ + dest.eDest = SRT_Mem; + dest.iSdst = dest.iSDParm; + dest.nSdst = nReg; + sqlite3VdbeAddOp3(v, OP_Null, 0, dest.iSDParm, dest.iSDParm+nReg-1); + VdbeComment((v, "Init subquery result")); + }else{ + dest.eDest = SRT_Exists; + sqlite3VdbeAddOp2(v, OP_Integer, 0, dest.iSDParm); + VdbeComment((v, "Init EXISTS result")); + } + if( pSel->pLimit ){ + /* The subquery already has a limit. If the pre-existing limit is X + ** then make the new limit X<>0 so that the new limit is either 1 or 0 */ + sqlite3 *db = pParse->db; + pLimit = sqlite3Expr(db, TK_INTEGER, "0"); + if( pLimit ){ + pLimit->affExpr = SQLITE_AFF_NUMERIC; + pLimit = sqlite3PExpr(pParse, TK_NE, + sqlite3ExprDup(db, pSel->pLimit->pLeft, 0), pLimit); + } + sqlite3ExprDelete(db, pSel->pLimit->pLeft); + pSel->pLimit->pLeft = pLimit; + }else{ + /* If there is no pre-existing limit add a limit of 1 */ + pLimit = sqlite3Expr(pParse->db, TK_INTEGER, "1"); + pSel->pLimit = sqlite3PExpr(pParse, TK_LIMIT, pLimit, 0); } + pSel->iLimit = 0; + if( sqlite3Select(pParse, pSel, &dest) ){ + return 0; + } + pExpr->iTable = rReg = dest.iSDParm; + ExprSetVVAProperty(pExpr, EP_NoReduce); + if( addrOnce ){ + sqlite3VdbeJumpHere(v, addrOnce); - if( jmpIfDynamic>=0 ){ - sqlite3VdbeJumpHere(v, jmpIfDynamic); + /* Subroutine return */ + sqlite3VdbeAddOp1(v, OP_Return, pExpr->y.sub.regReturn); + sqlite3VdbeChangeP1(v, pExpr->y.sub.iAddr-1, sqlite3VdbeCurrentAddr(v)-1); + sqlite3ClearTempRegCache(pParse); } - sqlite3ExprCachePop(pParse); return rReg; } @@ -97632,7 +102671,10 @@ static void sqlite3ExprCodeIN( int addrTruthOp; /* Address of opcode that determines the IN is true */ int destNotNull; /* Jump here if a comparison is not true in step 6 */ int addrTop; /* Top of the step-6 loop */ + int iTab = 0; /* Index to use */ + u8 okConstFactor = pParse->okConstFactor; + assert( !ExprHasVVAProperty(pExpr,EP_Immutable) ); pLeft = pExpr->pLeft; if( sqlite3ExprCheckIN(pParse, pExpr) ) return; zAff = exprINAffinity(pParse, pExpr); @@ -97643,7 +102685,7 @@ static void sqlite3ExprCodeIN( if( pParse->db->mallocFailed ) goto sqlite3ExprCodeIN_oom_error; /* Attempt to compute the RHS. After this step, if anything other than - ** IN_INDEX_NOOP is returned, the table opened ith cursor pExpr->iTable + ** IN_INDEX_NOOP is returned, the table opened with cursor iTab ** contains the values that make up the RHS. If IN_INDEX_NOOP is returned, ** the RHS has not yet been coded. */ v = pParse->pVdbe; @@ -97651,7 +102693,8 @@ static void sqlite3ExprCodeIN( VdbeNoopComment((v, "begin IN expr")); eType = sqlite3FindInIndex(pParse, pExpr, IN_INDEX_MEMBERSHIP | IN_INDEX_NOOP_OK, - destIfFalse==destIfNull ? 0 : &rRhsHasNull, aiMap); + destIfFalse==destIfNull ? 0 : &rRhsHasNull, + aiMap, &iTab); assert( pParse->nErr || nVector==1 || eType==IN_INDEX_EPH || eType==IN_INDEX_INDEX_ASC || eType==IN_INDEX_INDEX_DESC @@ -97674,9 +102717,14 @@ static void sqlite3ExprCodeIN( ** so that the fields are in the same order as an existing index. The ** aiMap[] array contains a mapping from the original LHS field order to ** the field order that matches the RHS index. - */ - sqlite3ExprCachePush(pParse); + ** + ** Avoid factoring the LHS of the IN(...) expression out of the loop, + ** even if it is constant, as OP_Affinity may be used on the register + ** by code generated below. */ + assert( pParse->okConstFactor==okConstFactor ); + pParse->okConstFactor = 0; rLhsOrig = exprCodeVector(pParse, pLeft, &iDummy); + pParse->okConstFactor = okConstFactor; for(i=0; ix.pList; CollSeq *pColl = sqlite3ExprCollSeq(pParse, pExpr->pLeft); - int labelOk = sqlite3VdbeMakeLabel(v); + int labelOk = sqlite3VdbeMakeLabel(pParse); int r2, regToFree; int regCkNull = 0; int ii; @@ -97712,19 +102760,25 @@ static void sqlite3ExprCodeIN( if( regCkNull && sqlite3ExprCanBeNull(pList->a[ii].pExpr) ){ sqlite3VdbeAddOp3(v, OP_BitAnd, regCkNull, r2, regCkNull); } + sqlite3ReleaseTempReg(pParse, regToFree); if( iinExpr-1 || destIfNull!=destIfFalse ){ - sqlite3VdbeAddOp4(v, OP_Eq, rLhs, labelOk, r2, + int op = rLhs!=r2 ? OP_Eq : OP_NotNull; + sqlite3VdbeAddOp4(v, op, rLhs, labelOk, r2, (void*)pColl, P4_COLLSEQ); - VdbeCoverageIf(v, iinExpr-1); - VdbeCoverageIf(v, ii==pList->nExpr-1); + VdbeCoverageIf(v, iinExpr-1 && op==OP_Eq); + VdbeCoverageIf(v, ii==pList->nExpr-1 && op==OP_Eq); + VdbeCoverageIf(v, iinExpr-1 && op==OP_NotNull); + VdbeCoverageIf(v, ii==pList->nExpr-1 && op==OP_NotNull); sqlite3VdbeChangeP5(v, zAff[0]); }else{ + int op = rLhs!=r2 ? OP_Ne : OP_IsNull; assert( destIfNull==destIfFalse ); - sqlite3VdbeAddOp4(v, OP_Ne, rLhs, destIfFalse, r2, - (void*)pColl, P4_COLLSEQ); VdbeCoverage(v); + sqlite3VdbeAddOp4(v, op, rLhs, destIfFalse, r2, + (void*)pColl, P4_COLLSEQ); + VdbeCoverageIf(v, op==OP_Ne); + VdbeCoverageIf(v, op==OP_IsNull); sqlite3VdbeChangeP5(v, zAff[0] | SQLITE_JUMPIFNULL); } - sqlite3ReleaseTempReg(pParse, regToFree); } if( regCkNull ){ sqlite3VdbeAddOp2(v, OP_IsNull, regCkNull, destIfNull); VdbeCoverage(v); @@ -97742,8 +102796,9 @@ static void sqlite3ExprCodeIN( if( destIfNull==destIfFalse ){ destStep2 = destIfFalse; }else{ - destStep2 = destStep6 = sqlite3VdbeMakeLabel(v); + destStep2 = destStep6 = sqlite3VdbeMakeLabel(pParse); } + if( pParse->nErr ) goto sqlite3ExprCodeIN_finished; for(i=0; ipLeft, i); if( sqlite3ExprCanBeNull(p) ){ @@ -97760,19 +102815,19 @@ static void sqlite3ExprCodeIN( /* In this case, the RHS is the ROWID of table b-tree and so we also ** know that the RHS is non-NULL. Hence, we combine steps 3 and 4 ** into a single opcode. */ - sqlite3VdbeAddOp3(v, OP_SeekRowid, pExpr->iTable, destIfFalse, rLhs); + sqlite3VdbeAddOp3(v, OP_SeekRowid, iTab, destIfFalse, rLhs); VdbeCoverage(v); addrTruthOp = sqlite3VdbeAddOp0(v, OP_Goto); /* Return True */ }else{ sqlite3VdbeAddOp4(v, OP_Affinity, rLhs, nVector, 0, zAff, nVector); if( destIfFalse==destIfNull ){ /* Combine Step 3 and Step 5 into a single opcode */ - sqlite3VdbeAddOp4Int(v, OP_NotFound, pExpr->iTable, destIfFalse, + sqlite3VdbeAddOp4Int(v, OP_NotFound, iTab, destIfFalse, rLhs, nVector); VdbeCoverage(v); goto sqlite3ExprCodeIN_finished; } /* Ordinary Step 3, for the case where FALSE and NULL are distinct */ - addrTruthOp = sqlite3VdbeAddOp4Int(v, OP_Found, pExpr->iTable, 0, + addrTruthOp = sqlite3VdbeAddOp4Int(v, OP_Found, iTab, 0, rLhs, nVector); VdbeCoverage(v); } @@ -97797,10 +102852,10 @@ static void sqlite3ExprCodeIN( ** of the RHS. */ if( destStep6 ) sqlite3VdbeResolveLabel(v, destStep6); - addrTop = sqlite3VdbeAddOp2(v, OP_Rewind, pExpr->iTable, destIfFalse); + addrTop = sqlite3VdbeAddOp2(v, OP_Rewind, iTab, destIfFalse); VdbeCoverage(v); if( nVector>1 ){ - destNotNull = sqlite3VdbeMakeLabel(v); + destNotNull = sqlite3VdbeMakeLabel(pParse); }else{ /* For nVector==1, combine steps 6 and 7 by immediately returning ** FALSE if the first comparison is not NULL */ @@ -97812,7 +102867,7 @@ static void sqlite3ExprCodeIN( int r3 = sqlite3GetTempReg(pParse); p = sqlite3VectorFieldSubexpr(pLeft, i); pColl = sqlite3ExprCollSeq(pParse, p); - sqlite3VdbeAddOp3(v, OP_Column, pExpr->iTable, i, r3); + sqlite3VdbeAddOp3(v, OP_Column, iTab, i, r3); sqlite3VdbeAddOp4(v, OP_Ne, rLhs+i, destNotNull, r3, (void*)pColl, P4_COLLSEQ); VdbeCoverage(v); @@ -97821,7 +102876,7 @@ static void sqlite3ExprCodeIN( sqlite3VdbeAddOp2(v, OP_Goto, 0, destIfNull); if( nVector>1 ){ sqlite3VdbeResolveLabel(v, destNotNull); - sqlite3VdbeAddOp2(v, OP_Next, pExpr->iTable, addrTop+1); + sqlite3VdbeAddOp2(v, OP_Next, iTab, addrTop+1); VdbeCoverage(v); /* Step 7: If we reach this point, we know that the result must @@ -97834,7 +102889,6 @@ static void sqlite3ExprCodeIN( sqlite3ExprCodeIN_finished: if( rLhs!=rLhsOrig ) sqlite3ReleaseTempReg(pParse, rLhs); - sqlite3ExprCachePop(pParse); VdbeComment((v, "end IN expr")); sqlite3ExprCodeIN_oom_error: sqlite3DbFree(pParse->db, aiMap); @@ -97902,145 +102956,6 @@ static void codeInteger(Parse *pParse, Expr *pExpr, int negFlag, int iMem){ } } -/* -** Erase column-cache entry number i -*/ -static void cacheEntryClear(Parse *pParse, int i){ - if( pParse->aColCache[i].tempReg ){ - if( pParse->nTempRegaTempReg) ){ - pParse->aTempReg[pParse->nTempReg++] = pParse->aColCache[i].iReg; - } - } - pParse->nColCache--; - if( inColCache ){ - pParse->aColCache[i] = pParse->aColCache[pParse->nColCache]; - } -} - - -/* -** Record in the column cache that a particular column from a -** particular table is stored in a particular register. -*/ -SQLITE_PRIVATE void sqlite3ExprCacheStore(Parse *pParse, int iTab, int iCol, int iReg){ - int i; - int minLru; - int idxLru; - struct yColCache *p; - - /* Unless an error has occurred, register numbers are always positive. */ - assert( iReg>0 || pParse->nErr || pParse->db->mallocFailed ); - assert( iCol>=-1 && iCol<32768 ); /* Finite column numbers */ - - /* The SQLITE_ColumnCache flag disables the column cache. This is used - ** for testing only - to verify that SQLite always gets the same answer - ** with and without the column cache. - */ - if( OptimizationDisabled(pParse->db, SQLITE_ColumnCache) ) return; - - /* First replace any existing entry. - ** - ** Actually, the way the column cache is currently used, we are guaranteed - ** that the object will never already be in cache. Verify this guarantee. - */ -#ifndef NDEBUG - for(i=0, p=pParse->aColCache; inColCache; i++, p++){ - assert( p->iTable!=iTab || p->iColumn!=iCol ); - } -#endif - - /* If the cache is already full, delete the least recently used entry */ - if( pParse->nColCache>=SQLITE_N_COLCACHE ){ - minLru = 0x7fffffff; - idxLru = -1; - for(i=0, p=pParse->aColCache; ilrulru; - } - } - p = &pParse->aColCache[idxLru]; - }else{ - p = &pParse->aColCache[pParse->nColCache++]; - } - - /* Add the new entry to the end of the cache */ - p->iLevel = pParse->iCacheLevel; - p->iTable = iTab; - p->iColumn = iCol; - p->iReg = iReg; - p->tempReg = 0; - p->lru = pParse->iCacheCnt++; -} - -/* -** Indicate that registers between iReg..iReg+nReg-1 are being overwritten. -** Purge the range of registers from the column cache. -*/ -SQLITE_PRIVATE void sqlite3ExprCacheRemove(Parse *pParse, int iReg, int nReg){ - int i = 0; - while( inColCache ){ - struct yColCache *p = &pParse->aColCache[i]; - if( p->iReg >= iReg && p->iReg < iReg+nReg ){ - cacheEntryClear(pParse, i); - }else{ - i++; - } - } -} - -/* -** Remember the current column cache context. Any new entries added -** added to the column cache after this call are removed when the -** corresponding pop occurs. -*/ -SQLITE_PRIVATE void sqlite3ExprCachePush(Parse *pParse){ - pParse->iCacheLevel++; -#ifdef SQLITE_DEBUG - if( pParse->db->flags & SQLITE_VdbeAddopTrace ){ - printf("PUSH to %d\n", pParse->iCacheLevel); - } -#endif -} - -/* -** Remove from the column cache any entries that were added since the -** the previous sqlite3ExprCachePush operation. In other words, restore -** the cache to the state it was in prior the most recent Push. -*/ -SQLITE_PRIVATE void sqlite3ExprCachePop(Parse *pParse){ - int i = 0; - assert( pParse->iCacheLevel>=1 ); - pParse->iCacheLevel--; -#ifdef SQLITE_DEBUG - if( pParse->db->flags & SQLITE_VdbeAddopTrace ){ - printf("POP to %d\n", pParse->iCacheLevel); - } -#endif - while( inColCache ){ - if( pParse->aColCache[i].iLevel>pParse->iCacheLevel ){ - cacheEntryClear(pParse, i); - }else{ - i++; - } - } -} - -/* -** When a cached column is reused, make sure that its register is -** no longer available as a temp register. ticket #3879: that same -** register might be in the cache in multiple places, so be sure to -** get them all. -*/ -static void sqlite3ExprCachePinRegister(Parse *pParse, int iReg){ - int i; - struct yColCache *p; - for(i=0, p=pParse->aColCache; inColCache; i++, p++){ - if( p->iReg==iReg ){ - p->tempReg = 0; - } - } -} /* Generate code that will load into register regOut a value that is ** appropriate for the iIdxCol-th column of index pIdx. @@ -98065,16 +102980,45 @@ SQLITE_PRIVATE void sqlite3ExprCodeLoadIndexColumn( } } +#ifndef SQLITE_OMIT_GENERATED_COLUMNS +/* +** Generate code that will compute the value of generated column pCol +** and store the result in register regOut +*/ +SQLITE_PRIVATE void sqlite3ExprCodeGeneratedColumn( + Parse *pParse, + Column *pCol, + int regOut +){ + int iAddr; + Vdbe *v = pParse->pVdbe; + assert( v!=0 ); + assert( pParse->iSelfTab!=0 ); + if( pParse->iSelfTab>0 ){ + iAddr = sqlite3VdbeAddOp3(v, OP_IfNullRow, pParse->iSelfTab-1, 0, regOut); + }else{ + iAddr = 0; + } + sqlite3ExprCodeCopy(pParse, pCol->pDflt, regOut); + if( pCol->affinity>=SQLITE_AFF_TEXT ){ + sqlite3VdbeAddOp4(v, OP_Affinity, regOut, 1, 0, &pCol->affinity, 1); + } + if( iAddr ) sqlite3VdbeJumpHere(v, iAddr); +} +#endif /* SQLITE_OMIT_GENERATED_COLUMNS */ + /* ** Generate code to extract the value of the iCol-th column of a table. */ SQLITE_PRIVATE void sqlite3ExprCodeGetColumnOfTable( - Vdbe *v, /* The VDBE under construction */ + Vdbe *v, /* Parsing context */ Table *pTab, /* The table containing the value */ int iTabCur, /* The table cursor. Or the PK cursor for WITHOUT ROWID */ int iCol, /* Index of the column to extract */ int regOut /* Extract the value into this register */ ){ + Column *pCol; + assert( v!=0 ); if( pTab==0 ){ sqlite3VdbeAddOp3(v, OP_Column, iTabCur, iCol, regOut); return; @@ -98082,26 +103026,43 @@ SQLITE_PRIVATE void sqlite3ExprCodeGetColumnOfTable( if( iCol<0 || iCol==pTab->iPKey ){ sqlite3VdbeAddOp2(v, OP_Rowid, iTabCur, regOut); }else{ - int op = IsVirtual(pTab) ? OP_VColumn : OP_Column; - int x = iCol; - if( !HasRowid(pTab) && !IsVirtual(pTab) ){ - x = sqlite3ColumnOfIndex(sqlite3PrimaryKeyIndex(pTab), iCol); + int op; + int x; + if( IsVirtual(pTab) ){ + op = OP_VColumn; + x = iCol; +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + }else if( (pCol = &pTab->aCol[iCol])->colFlags & COLFLAG_VIRTUAL ){ + Parse *pParse = sqlite3VdbeParser(v); + if( pCol->colFlags & COLFLAG_BUSY ){ + sqlite3ErrorMsg(pParse, "generated column loop on \"%s\"", pCol->zName); + }else{ + int savedSelfTab = pParse->iSelfTab; + pCol->colFlags |= COLFLAG_BUSY; + pParse->iSelfTab = iTabCur+1; + sqlite3ExprCodeGeneratedColumn(pParse, pCol, regOut); + pParse->iSelfTab = savedSelfTab; + pCol->colFlags &= ~COLFLAG_BUSY; + } + return; +#endif + }else if( !HasRowid(pTab) ){ + testcase( iCol!=sqlite3TableColumnToStorage(pTab, iCol) ); + x = sqlite3TableColumnToIndex(sqlite3PrimaryKeyIndex(pTab), iCol); + op = OP_Column; + }else{ + x = sqlite3TableColumnToStorage(pTab,iCol); + testcase( x!=iCol ); + op = OP_Column; } sqlite3VdbeAddOp3(v, op, iTabCur, x, regOut); - } - if( iCol>=0 ){ sqlite3ColumnDefault(v, pTab, iCol, regOut); } } /* ** Generate code that will extract the iColumn-th column from -** table pTab and store the column value in a register. -** -** An effort is made to store the column value in register iReg. This -** is not garanteeed for GetColumn() - the result can be stored in -** any register. But the result is guaranteed to land in register iReg -** for GetColumnToReg(). +** table pTab and store the column value in register iReg. ** ** There must be an open cursor to pTab in iTable when this routine ** is called. If iColumn<0 then code is generated that extracts the rowid. @@ -98114,103 +103075,30 @@ SQLITE_PRIVATE int sqlite3ExprCodeGetColumn( int iReg, /* Store results here */ u8 p5 /* P5 value for OP_Column + FLAGS */ ){ - Vdbe *v = pParse->pVdbe; - int i; - struct yColCache *p; - - for(i=0, p=pParse->aColCache; inColCache; i++, p++){ - if( p->iTable==iTable && p->iColumn==iColumn ){ - p->lru = pParse->iCacheCnt++; - sqlite3ExprCachePinRegister(pParse, p->iReg); - return p->iReg; - } - } - assert( v!=0 ); - sqlite3ExprCodeGetColumnOfTable(v, pTab, iTable, iColumn, iReg); + assert( pParse->pVdbe!=0 ); + sqlite3ExprCodeGetColumnOfTable(pParse->pVdbe, pTab, iTable, iColumn, iReg); if( p5 ){ - sqlite3VdbeChangeP5(v, p5); - }else{ - sqlite3ExprCacheStore(pParse, iTable, iColumn, iReg); + VdbeOp *pOp = sqlite3VdbeGetOp(pParse->pVdbe,-1); + if( pOp->opcode==OP_Column ) pOp->p5 = p5; } return iReg; } -SQLITE_PRIVATE void sqlite3ExprCodeGetColumnToReg( - Parse *pParse, /* Parsing and code generating context */ - Table *pTab, /* Description of the table we are reading from */ - int iColumn, /* Index of the table column */ - int iTable, /* The cursor pointing to the table */ - int iReg /* Store results here */ -){ - int r1 = sqlite3ExprCodeGetColumn(pParse, pTab, iColumn, iTable, iReg, 0); - if( r1!=iReg ) sqlite3VdbeAddOp2(pParse->pVdbe, OP_SCopy, r1, iReg); -} - - -/* -** Clear all column cache entries. -*/ -SQLITE_PRIVATE void sqlite3ExprCacheClear(Parse *pParse){ - int i; - -#ifdef SQLITE_DEBUG - if( pParse->db->flags & SQLITE_VdbeAddopTrace ){ - printf("CLEAR\n"); - } -#endif - for(i=0; inColCache; i++){ - if( pParse->aColCache[i].tempReg - && pParse->nTempRegaTempReg) - ){ - pParse->aTempReg[pParse->nTempReg++] = pParse->aColCache[i].iReg; - } - } - pParse->nColCache = 0; -} - -/* -** Record the fact that an affinity change has occurred on iCount -** registers starting with iStart. -*/ -SQLITE_PRIVATE void sqlite3ExprCacheAffinityChange(Parse *pParse, int iStart, int iCount){ - sqlite3ExprCacheRemove(pParse, iStart, iCount); -} /* ** Generate code to move content from registers iFrom...iFrom+nReg-1 -** over to iTo..iTo+nReg-1. Keep the column cache up-to-date. +** over to iTo..iTo+nReg-1. */ SQLITE_PRIVATE void sqlite3ExprCodeMove(Parse *pParse, int iFrom, int iTo, int nReg){ - assert( iFrom>=iTo+nReg || iFrom+nReg<=iTo ); sqlite3VdbeAddOp3(pParse->pVdbe, OP_Move, iFrom, iTo, nReg); - sqlite3ExprCacheRemove(pParse, iFrom, nReg); } -#if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST) -/* -** Return true if any register in the range iFrom..iTo (inclusive) -** is used as part of the column cache. -** -** This routine is used within assert() and testcase() macros only -** and does not appear in a normal build. -*/ -static int usedAsColumnCache(Parse *pParse, int iFrom, int iTo){ - int i; - struct yColCache *p; - for(i=0, p=pParse->aColCache; inColCache; i++, p++){ - int r = p->iReg; - if( r>=iFrom && r<=iTo ) return 1; /*NO_TEST*/ - } - return 0; -} -#endif /* SQLITE_DEBUG || SQLITE_COVERAGE_TEST */ - - /* ** Convert a scalar expression node to a TK_REGISTER referencing ** register iReg. The caller must ensure that iReg already contains ** the correct value for the expression. */ -static void exprToRegister(Expr *p, int iReg){ +static void exprToRegister(Expr *pExpr, int iReg){ + Expr *p = sqlite3ExprSkipCollateAndLikely(pExpr); p->op2 = p->op; p->op = TK_REGISTER; p->iTable = iReg; @@ -98238,7 +103126,7 @@ static int exprCodeVector(Parse *pParse, Expr *p, int *piFreeable){ #if SQLITE_OMIT_SUBQUERY iResult = 0; #else - iResult = sqlite3CodeSubselect(pParse, p, 0, 0); + iResult = sqlite3CodeSubselect(pParse, p); #endif }else{ int i; @@ -98252,6 +103140,124 @@ static int exprCodeVector(Parse *pParse, Expr *p, int *piFreeable){ return iResult; } +/* +** If the last opcode is a OP_Copy, then set the do-not-merge flag (p5) +** so that a subsequent copy will not be merged into this one. +*/ +static void setDoNotMergeFlagOnCopy(Vdbe *v){ + if( sqlite3VdbeGetOp(v, -1)->opcode==OP_Copy ){ + sqlite3VdbeChangeP5(v, 1); /* Tag trailing OP_Copy as not mergable */ + } +} + +/* +** Generate code to implement special SQL functions that are implemented +** in-line rather than by using the usual callbacks. +*/ +static int exprCodeInlineFunction( + Parse *pParse, /* Parsing context */ + ExprList *pFarg, /* List of function arguments */ + int iFuncId, /* Function ID. One of the INTFUNC_... values */ + int target /* Store function result in this register */ +){ + int nFarg; + Vdbe *v = pParse->pVdbe; + assert( v!=0 ); + assert( pFarg!=0 ); + nFarg = pFarg->nExpr; + assert( nFarg>0 ); /* All in-line functions have at least one argument */ + switch( iFuncId ){ + case INLINEFUNC_coalesce: { + /* Attempt a direct implementation of the built-in COALESCE() and + ** IFNULL() functions. This avoids unnecessary evaluation of + ** arguments past the first non-NULL argument. + */ + int endCoalesce = sqlite3VdbeMakeLabel(pParse); + int i; + assert( nFarg>=2 ); + sqlite3ExprCode(pParse, pFarg->a[0].pExpr, target); + for(i=1; ia[i].pExpr, target); + } + setDoNotMergeFlagOnCopy(v); + sqlite3VdbeResolveLabel(v, endCoalesce); + break; + } + case INLINEFUNC_iif: { + Expr caseExpr; + memset(&caseExpr, 0, sizeof(caseExpr)); + caseExpr.op = TK_CASE; + caseExpr.x.pList = pFarg; + return sqlite3ExprCodeTarget(pParse, &caseExpr, target); + } + + default: { + /* The UNLIKELY() function is a no-op. The result is the value + ** of the first argument. + */ + assert( nFarg==1 || nFarg==2 ); + target = sqlite3ExprCodeTarget(pParse, pFarg->a[0].pExpr, target); + break; + } + + /*********************************************************************** + ** Test-only SQL functions that are only usable if enabled + ** via SQLITE_TESTCTRL_INTERNAL_FUNCTIONS + */ + case INLINEFUNC_expr_compare: { + /* Compare two expressions using sqlite3ExprCompare() */ + assert( nFarg==2 ); + sqlite3VdbeAddOp2(v, OP_Integer, + sqlite3ExprCompare(0,pFarg->a[0].pExpr, pFarg->a[1].pExpr,-1), + target); + break; + } + + case INLINEFUNC_expr_implies_expr: { + /* Compare two expressions using sqlite3ExprImpliesExpr() */ + assert( nFarg==2 ); + sqlite3VdbeAddOp2(v, OP_Integer, + sqlite3ExprImpliesExpr(pParse,pFarg->a[0].pExpr, pFarg->a[1].pExpr,-1), + target); + break; + } + + case INLINEFUNC_implies_nonnull_row: { + /* REsult of sqlite3ExprImpliesNonNullRow() */ + Expr *pA1; + assert( nFarg==2 ); + pA1 = pFarg->a[1].pExpr; + if( pA1->op==TK_COLUMN ){ + sqlite3VdbeAddOp2(v, OP_Integer, + sqlite3ExprImpliesNonNullRow(pFarg->a[0].pExpr,pA1->iTable), + target); + }else{ + sqlite3VdbeAddOp2(v, OP_Null, 0, target); + } + break; + } + +#ifdef SQLITE_DEBUG + case INLINEFUNC_affinity: { + /* The AFFINITY() function evaluates to a string that describes + ** the type affinity of the argument. This is used for testing of + ** the SQLite type logic. + */ + const char *azAff[] = { "blob", "text", "numeric", "integer", "real" }; + char aff; + assert( nFarg==1 ); + aff = sqlite3ExprAffinity(pFarg->a[0].pExpr); + sqlite3VdbeLoadString(v, target, + (aff<=SQLITE_AFF_NONE) ? "none" : azAff[aff-SQLITE_AFF_BLOB]); + break; + } +#endif + } + return target; +} + /* ** Generate code into the current Vdbe to evaluate the given @@ -98284,37 +103290,117 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) if( pExpr==0 ){ op = TK_NULL; }else{ + assert( !ExprHasVVAProperty(pExpr,EP_Immutable) ); op = pExpr->op; } switch( op ){ case TK_AGG_COLUMN: { AggInfo *pAggInfo = pExpr->pAggInfo; - struct AggInfo_col *pCol = &pAggInfo->aCol[pExpr->iAgg]; + struct AggInfo_col *pCol; + assert( pAggInfo!=0 ); + assert( pExpr->iAgg>=0 && pExpr->iAggnColumn ); + pCol = &pAggInfo->aCol[pExpr->iAgg]; if( !pAggInfo->directMode ){ assert( pCol->iMem>0 ); return pCol->iMem; }else if( pAggInfo->useSortingIdx ){ + Table *pTab = pCol->pTab; sqlite3VdbeAddOp3(v, OP_Column, pAggInfo->sortingIdxPTab, pCol->iSorterColumn, target); + if( pCol->iColumn<0 ){ + VdbeComment((v,"%s.rowid",pTab->zName)); + }else{ + VdbeComment((v,"%s.%s",pTab->zName,pTab->aCol[pCol->iColumn].zName)); + if( pTab->aCol[pCol->iColumn].affinity==SQLITE_AFF_REAL ){ + sqlite3VdbeAddOp1(v, OP_RealAffinity, target); + } + } return target; } /* Otherwise, fall thru into the TK_COLUMN case */ } case TK_COLUMN: { int iTab = pExpr->iTable; + int iReg; + if( ExprHasProperty(pExpr, EP_FixedCol) ){ + /* This COLUMN expression is really a constant due to WHERE clause + ** constraints, and that constant is coded by the pExpr->pLeft + ** expresssion. However, make sure the constant has the correct + ** datatype by applying the Affinity of the table column to the + ** constant. + */ + int aff; + iReg = sqlite3ExprCodeTarget(pParse, pExpr->pLeft,target); + if( pExpr->y.pTab ){ + aff = sqlite3TableColumnAffinity(pExpr->y.pTab, pExpr->iColumn); + }else{ + aff = pExpr->affExpr; + } + if( aff>SQLITE_AFF_BLOB ){ + static const char zAff[] = "B\000C\000D\000E"; + assert( SQLITE_AFF_BLOB=='A' ); + assert( SQLITE_AFF_TEXT=='B' ); + sqlite3VdbeAddOp4(v, OP_Affinity, iReg, 1, 0, + &zAff[(aff-'B')*2], P4_STATIC); + } + return iReg; + } if( iTab<0 ){ if( pParse->iSelfTab<0 ){ - /* Generating CHECK constraints or inserting into partial index */ - return pExpr->iColumn - pParse->iSelfTab; + /* Other columns in the same row for CHECK constraints or + ** generated columns or for inserting into partial index. + ** The row is unpacked into registers beginning at + ** 0-(pParse->iSelfTab). The rowid (if any) is in a register + ** immediately prior to the first column. + */ + Column *pCol; + Table *pTab = pExpr->y.pTab; + int iSrc; + int iCol = pExpr->iColumn; + assert( pTab!=0 ); + assert( iCol>=XN_ROWID ); + assert( iColnCol ); + if( iCol<0 ){ + return -1-pParse->iSelfTab; + } + pCol = pTab->aCol + iCol; + testcase( iCol!=sqlite3TableColumnToStorage(pTab,iCol) ); + iSrc = sqlite3TableColumnToStorage(pTab, iCol) - pParse->iSelfTab; +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + if( pCol->colFlags & COLFLAG_GENERATED ){ + if( pCol->colFlags & COLFLAG_BUSY ){ + sqlite3ErrorMsg(pParse, "generated column loop on \"%s\"", + pCol->zName); + return 0; + } + pCol->colFlags |= COLFLAG_BUSY; + if( pCol->colFlags & COLFLAG_NOTAVAIL ){ + sqlite3ExprCodeGeneratedColumn(pParse, pCol, iSrc); + } + pCol->colFlags &= ~(COLFLAG_BUSY|COLFLAG_NOTAVAIL); + return iSrc; + }else +#endif /* SQLITE_OMIT_GENERATED_COLUMNS */ + if( pCol->affinity==SQLITE_AFF_REAL ){ + sqlite3VdbeAddOp2(v, OP_SCopy, iSrc, target); + sqlite3VdbeAddOp1(v, OP_RealAffinity, target); + return target; + }else{ + return iSrc; + } }else{ /* Coding an expression that is part of an index where column names ** in the index refer to the table to which the index belongs */ iTab = pParse->iSelfTab - 1; } } - return sqlite3ExprCodeGetColumn(pParse, pExpr->pTab, + iReg = sqlite3ExprCodeGetColumn(pParse, pExpr->y.pTab, pExpr->iColumn, iTab, target, pExpr->op2); + if( pExpr->y.pTab==0 && pExpr->affExpr==SQLITE_AFF_REAL ){ + sqlite3VdbeAddOp1(v, OP_RealAffinity, iReg); + } + return iReg; } case TK_INTEGER: { codeInteger(pParse, pExpr, 0, target); @@ -98336,7 +103422,12 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) sqlite3VdbeLoadString(v, target, pExpr->u.zToken); return target; } - case TK_NULL: { + default: { + /* Make NULL the default case so that if a bug causes an illegal + ** Expr node to be passed into this function, it will be handled + ** sanely and not crash. But keep the assert() to bring the problem + ** to the attention of the developers. */ + assert( op==TK_NULL ); sqlite3VdbeAddOp2(v, OP_Null, 0, target); return target; } @@ -98363,7 +103454,7 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) sqlite3VdbeAddOp2(v, OP_Variable, pExpr->iColumn, target); if( pExpr->u.zToken[1]!=0 ){ const char *z = sqlite3VListNumToName(pParse->pVList, pExpr->iColumn); - assert( pExpr->u.zToken[0]=='?' || strcmp(pExpr->u.zToken, z)==0 ); + assert( pExpr->u.zToken[0]=='?' || (z && !strcmp(pExpr->u.zToken, z)) ); pParse->pVList[0] = 0; /* Indicate VList may no longer be enlarged */ sqlite3VdbeAppendP4(v, (char*)z, P4_STATIC); } @@ -98382,8 +103473,6 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) } sqlite3VdbeAddOp2(v, OP_Cast, target, sqlite3AffinityType(pExpr->u.zToken, 0)); - testcase( usedAsColumnCache(pParse, inReg, inReg) ); - sqlite3ExprCacheAffinityChange(pParse, inReg, 1); return inReg; } #endif /* SQLITE_OMIT_CAST */ @@ -98405,7 +103494,8 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) r1 = sqlite3ExprCodeTemp(pParse, pLeft, ®Free1); r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); codeCompare(pParse, pLeft, pExpr->pRight, op, - r1, r2, inReg, SQLITE_STOREP2 | p5); + r1, r2, inReg, SQLITE_STOREP2 | p5, + ExprHasProperty(pExpr,EP_Commuted)); assert(TK_LT==OP_Lt); testcase(op==OP_Lt); VdbeCoverageIf(v,op==OP_Lt); assert(TK_LE==OP_Le); testcase(op==OP_Le); VdbeCoverageIf(v,op==OP_Le); assert(TK_GT==OP_Gt); testcase(op==OP_Gt); VdbeCoverageIf(v,op==OP_Gt); @@ -98463,6 +103553,7 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) tempX.op = TK_INTEGER; tempX.flags = EP_IntValue|EP_TokenOnly; tempX.u.iValue = 0; + ExprClearVVAProperties(&tempX); r1 = sqlite3ExprCodeTemp(pParse, &tempX, ®Free1); r2 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free2); sqlite3VdbeAddOp3(v, OP_Subtract, r2, r1, target); @@ -98508,7 +103599,10 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) } case TK_AGG_FUNCTION: { AggInfo *pInfo = pExpr->pAggInfo; - if( pInfo==0 ){ + if( pInfo==0 + || NEVER(pExpr->iAgg<0) + || NEVER(pExpr->iAgg>=pInfo->nFunc) + ){ assert( !ExprHasProperty(pExpr, EP_IntValue) ); sqlite3ErrorMsg(pParse, "misuse of aggregate: %s()", pExpr->u.zToken); }else{ @@ -98527,17 +103621,20 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) u8 enc = ENC(db); /* The text encoding used by this database */ CollSeq *pColl = 0; /* A collating sequence */ +#ifndef SQLITE_OMIT_WINDOWFUNC + if( ExprHasProperty(pExpr, EP_WinFunc) ){ + return pExpr->y.pWin->regResult; + } +#endif + if( ConstFactorOk(pParse) && sqlite3ExprIsConstantNotJoin(pExpr) ){ - /* SQL functions can be expensive. So try to move constant functions - ** out of the inner loop, even if that means an extra OP_Copy. */ - return sqlite3ExprCodeAtInit(pParse, pExpr, -1); + /* SQL functions can be expensive. So try to avoid running them + ** multiple times if we know they always give the same result */ + return sqlite3ExprCodeRunJustOnce(pParse, pExpr, -1); } assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); - if( ExprHasProperty(pExpr, EP_TokenOnly) ){ - pFarg = 0; - }else{ - pFarg = pExpr->x.pList; - } + assert( !ExprHasProperty(pExpr, EP_TokenOnly) ); + pFarg = pExpr->x.pList; nFarg = pFarg ? pFarg->nExpr : 0; assert( !ExprHasProperty(pExpr, EP_IntValue) ); zId = pExpr->u.zToken; @@ -98551,51 +103648,15 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) sqlite3ErrorMsg(pParse, "unknown function: %s()", zId); break; } - - /* Attempt a direct implementation of the built-in COALESCE() and - ** IFNULL() functions. This avoids unnecessary evaluation of - ** arguments past the first non-NULL argument. - */ - if( pDef->funcFlags & SQLITE_FUNC_COALESCE ){ - int endCoalesce = sqlite3VdbeMakeLabel(v); - assert( nFarg>=2 ); - sqlite3ExprCode(pParse, pFarg->a[0].pExpr, target); - for(i=1; ia[i].pExpr, target); - sqlite3ExprCachePop(pParse); - } - sqlite3VdbeResolveLabel(v, endCoalesce); - break; - } - - /* The UNLIKELY() function is a no-op. The result is the value - ** of the first argument. - */ - if( pDef->funcFlags & SQLITE_FUNC_UNLIKELY ){ - assert( nFarg>=1 ); - return sqlite3ExprCodeTarget(pParse, pFarg->a[0].pExpr, target); + if( pDef->funcFlags & SQLITE_FUNC_INLINE ){ + assert( (pDef->funcFlags & SQLITE_FUNC_UNSAFE)==0 ); + assert( (pDef->funcFlags & SQLITE_FUNC_DIRECT)==0 ); + return exprCodeInlineFunction(pParse, pFarg, + SQLITE_PTR_TO_INT(pDef->pUserData), target); + }else if( pDef->funcFlags & (SQLITE_FUNC_DIRECT|SQLITE_FUNC_UNSAFE) ){ + sqlite3ExprFunctionUsable(pParse, pExpr, pDef); } -#ifdef SQLITE_DEBUG - /* The AFFINITY() function evaluates to a string that describes - ** the type affinity of the argument. This is used for testing of - ** the SQLite type logic. - */ - if( pDef->funcFlags & SQLITE_FUNC_AFFINITY ){ - const char *azAff[] = { "blob", "text", "numeric", "integer", "real" }; - char aff; - assert( nFarg==1 ); - aff = sqlite3ExprAffinity(pFarg->a[0].pExpr); - sqlite3VdbeLoadString(v, target, - aff ? azAff[aff-SQLITE_AFF_BLOB] : "none"); - return target; - } -#endif - for(i=0; ia[i].pExpr) ){ testcase( i==31 ); @@ -98632,10 +103693,8 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) } } - sqlite3ExprCachePush(pParse); /* Ticket 2ea2425d34be */ sqlite3ExprCodeExprList(pParse, pFarg, r1, 0, SQLITE_ECEL_DUP|SQLITE_ECEL_FACTOR); - sqlite3ExprCachePop(pParse); /* Ticket 2ea2425d34be */ }else{ r1 = 0; } @@ -98652,7 +103711,7 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) ** "glob(B,A). We want to use the A in "A glob B" to test ** for function overloading. But we use the B term in "glob(B,A)". */ - if( nFarg>=2 && (pExpr->flags & EP_InfixFunc) ){ + if( nFarg>=2 && ExprHasProperty(pExpr, EP_InfixFunc) ){ pDef = sqlite3VtabOverloadFunction(db, pDef, nFarg, pFarg->a[1].pExpr); }else if( nFarg>0 ){ pDef = sqlite3VtabOverloadFunction(db, pDef, nFarg, pFarg->a[0].pExpr); @@ -98673,12 +103732,15 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) }else #endif { - sqlite3VdbeAddOp4(v, pParse->iSelfTab ? OP_PureFunc0 : OP_Function0, - constMask, r1, target, (char*)pDef, P4_FUNCDEF); - sqlite3VdbeChangeP5(v, (u8)nFarg); + sqlite3VdbeAddFunctionCall(pParse, constMask, r1, target, nFarg, + pDef, pExpr->op2); } - if( nFarg && constMask==0 ){ - sqlite3ReleaseTempRange(pParse, r1, nFarg); + if( nFarg ){ + if( constMask==0 ){ + sqlite3ReleaseTempRange(pParse, r1, nFarg); + }else{ + sqlite3VdbeReleaseRegisters(pParse, r1, nFarg, constMask, 1); + } } return target; } @@ -98691,18 +103753,18 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) if( op==TK_SELECT && (nCol = pExpr->x.pSelect->pEList->nExpr)!=1 ){ sqlite3SubselectError(pParse, nCol, 1); }else{ - return sqlite3CodeSubselect(pParse, pExpr, 0, 0); + return sqlite3CodeSubselect(pParse, pExpr); } break; } case TK_SELECT_COLUMN: { int n; if( pExpr->pLeft->iTable==0 ){ - pExpr->pLeft->iTable = sqlite3CodeSubselect(pParse, pExpr->pLeft, 0, 0); + pExpr->pLeft->iTable = sqlite3CodeSubselect(pParse, pExpr->pLeft); } assert( pExpr->iTable==0 || pExpr->pLeft->op==TK_SELECT ); - if( pExpr->iTable - && pExpr->iTable!=(n = sqlite3ExprVectorSize(pExpr->pLeft)) + if( pExpr->iTable!=0 + && pExpr->iTable!=(n = sqlite3ExprVectorSize(pExpr->pLeft)) ){ sqlite3ErrorMsg(pParse, "%d columns assigned %d values", pExpr->iTable, n); @@ -98710,8 +103772,8 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) return pExpr->pLeft->iTable + pExpr->iColumn; } case TK_IN: { - int destIfFalse = sqlite3VdbeMakeLabel(v); - int destIfNull = sqlite3VdbeMakeLabel(v); + int destIfFalse = sqlite3VdbeMakeLabel(pParse); + int destIfNull = sqlite3VdbeMakeLabel(pParse); sqlite3VdbeAddOp2(v, OP_Null, 0, target); sqlite3ExprCodeIN(pParse, pExpr, destIfFalse, destIfNull); sqlite3VdbeAddOp2(v, OP_Integer, 1, target); @@ -98771,18 +103833,20 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) ** p1==1 -> old.a p1==4 -> new.a ** p1==2 -> old.b p1==5 -> new.b */ - Table *pTab = pExpr->pTab; - int p1 = pExpr->iTable * (pTab->nCol+1) + 1 + pExpr->iColumn; + Table *pTab = pExpr->y.pTab; + int iCol = pExpr->iColumn; + int p1 = pExpr->iTable * (pTab->nCol+1) + 1 + + sqlite3TableColumnToStorage(pTab, iCol); assert( pExpr->iTable==0 || pExpr->iTable==1 ); - assert( pExpr->iColumn>=-1 && pExpr->iColumnnCol ); - assert( pTab->iPKey<0 || pExpr->iColumn!=pTab->iPKey ); + assert( iCol>=-1 && iColnCol ); + assert( pTab->iPKey<0 || iCol!=pTab->iPKey ); assert( p1>=0 && p1<(pTab->nCol*2+2) ); sqlite3VdbeAddOp2(v, OP_Param, p1, target); VdbeComment((v, "r[%d]=%s.%s", target, (pExpr->iTable ? "new" : "old"), - (pExpr->iColumn<0 ? "rowid" : pExpr->pTab->aCol[pExpr->iColumn].zName) + (pExpr->iColumn<0 ? "rowid" : pExpr->y.pTab->aCol[iCol].zName) )); #ifndef SQLITE_OMIT_FLOATING_POINT @@ -98791,9 +103855,7 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) ** ** EVIDENCE-OF: R-60985-57662 SQLite will convert the value back to ** floating point when extracting it from the record. */ - if( pExpr->iColumn>=0 - && pTab->aCol[pExpr->iColumn].affinity==SQLITE_AFF_REAL - ){ + if( iCol>=0 && pTab->aCol[iCol].affinity==SQLITE_AFF_REAL ){ sqlite3VdbeAddOp1(v, OP_RealAffinity, target); } #endif @@ -98805,12 +103867,23 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) break; } + /* TK_IF_NULL_ROW Expr nodes are inserted ahead of expressions + ** that derive from the right-hand table of a LEFT JOIN. The + ** Expr.iTable value is the table number for the right-hand table. + ** The expression is only evaluated if that table is not currently + ** on a LEFT JOIN NULL row. + */ case TK_IF_NULL_ROW: { int addrINR; + u8 okConstFactor = pParse->okConstFactor; addrINR = sqlite3VdbeAddOp1(v, OP_IfNullRow, pExpr->iTable); - sqlite3ExprCachePush(pParse); + /* Temporarily disable factoring of constant expressions, since + ** even though expressions may appear to be constant, they are not + ** really constant because they originate from the right-hand side + ** of a LEFT JOIN. */ + pParse->okConstFactor = 0; inReg = sqlite3ExprCodeTarget(pParse, pExpr->pLeft, target); - sqlite3ExprCachePop(pParse); + pParse->okConstFactor = okConstFactor; sqlite3VdbeJumpHere(v, addrINR); sqlite3VdbeChangeP3(v, addrINR, inReg); break; @@ -98837,7 +103910,7 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) ** or if there is no matching Ei, the ELSE term Y, or if there is ** no ELSE term, NULL. */ - default: assert( op==TK_CASE ); { + case TK_CASE: { int endLabel; /* GOTO label for end of CASE stmt */ int nextCase; /* GOTO label for next WHEN clause */ int nExpr; /* 2x number of WHEN terms */ @@ -98847,22 +103920,27 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) Expr opCompare; /* The X==Ei expression */ Expr *pX; /* The X expression */ Expr *pTest = 0; /* X==Ei (form A) or just Ei (form B) */ - VVA_ONLY( int iCacheLevel = pParse->iCacheLevel; ) + Expr *pDel = 0; + sqlite3 *db = pParse->db; assert( !ExprHasProperty(pExpr, EP_xIsSelect) && pExpr->x.pList ); assert(pExpr->x.pList->nExpr > 0); pEList = pExpr->x.pList; aListelem = pEList->a; nExpr = pEList->nExpr; - endLabel = sqlite3VdbeMakeLabel(v); + endLabel = sqlite3VdbeMakeLabel(pParse); if( (pX = pExpr->pLeft)!=0 ){ - tempX = *pX; + pDel = sqlite3ExprDup(db, pX, 0); + if( db->mallocFailed ){ + sqlite3ExprDelete(db, pDel); + break; + } testcase( pX->op==TK_COLUMN ); - exprToRegister(&tempX, exprCodeVector(pParse, &tempX, ®Free1)); + exprToRegister(pDel, exprCodeVector(pParse, pDel, ®Free1)); testcase( regFree1==0 ); memset(&opCompare, 0, sizeof(opCompare)); opCompare.op = TK_EQ; - opCompare.pLeft = &tempX; + opCompare.pLeft = pDel; pTest = &opCompare; /* Ticket b351d95f9cd5ef17e9d9dbae18f5ca8611190001: ** The value in regFree1 might get SCopy-ed into the file result. @@ -98871,57 +103949,54 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) regFree1 = 0; } for(i=0; iop==TK_COLUMN ); sqlite3ExprIfFalse(pParse, pTest, nextCase, SQLITE_JUMPIFNULL); testcase( aListelem[i+1].pExpr->op==TK_COLUMN ); sqlite3ExprCode(pParse, aListelem[i+1].pExpr, target); sqlite3VdbeGoto(v, endLabel); - sqlite3ExprCachePop(pParse); sqlite3VdbeResolveLabel(v, nextCase); } if( (nExpr&1)!=0 ){ - sqlite3ExprCachePush(pParse); sqlite3ExprCode(pParse, pEList->a[nExpr-1].pExpr, target); - sqlite3ExprCachePop(pParse); }else{ sqlite3VdbeAddOp2(v, OP_Null, 0, target); } - assert( pParse->db->mallocFailed || pParse->nErr>0 - || pParse->iCacheLevel==iCacheLevel ); + sqlite3ExprDelete(db, pDel); + setDoNotMergeFlagOnCopy(v); sqlite3VdbeResolveLabel(v, endLabel); break; } #ifndef SQLITE_OMIT_TRIGGER case TK_RAISE: { - assert( pExpr->affinity==OE_Rollback - || pExpr->affinity==OE_Abort - || pExpr->affinity==OE_Fail - || pExpr->affinity==OE_Ignore + assert( pExpr->affExpr==OE_Rollback + || pExpr->affExpr==OE_Abort + || pExpr->affExpr==OE_Fail + || pExpr->affExpr==OE_Ignore ); - if( !pParse->pTriggerTab ){ + if( !pParse->pTriggerTab && !pParse->nested ){ sqlite3ErrorMsg(pParse, "RAISE() may only be used within a trigger-program"); return 0; } - if( pExpr->affinity==OE_Abort ){ + if( pExpr->affExpr==OE_Abort ){ sqlite3MayAbort(pParse); } assert( !ExprHasProperty(pExpr, EP_IntValue) ); - if( pExpr->affinity==OE_Ignore ){ + if( pExpr->affExpr==OE_Ignore ){ sqlite3VdbeAddOp4( v, OP_Halt, SQLITE_OK, OE_Ignore, 0, pExpr->u.zToken,0); VdbeCoverage(v); }else{ - sqlite3HaltConstraint(pParse, SQLITE_CONSTRAINT_TRIGGER, - pExpr->affinity, pExpr->u.zToken, 0, 0); + sqlite3HaltConstraint(pParse, + pParse->pTriggerTab ? SQLITE_CONSTRAINT_TRIGGER : SQLITE_ERROR, + pExpr->affExpr, pExpr->u.zToken, 0, 0); } break; @@ -98934,15 +104009,23 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) } /* -** Factor out the code of the given expression to initialization time. +** Generate code that will evaluate expression pExpr just one time +** per prepared statement execution. +** +** If the expression uses functions (that might throw an exception) then +** guard them with an OP_Once opcode to ensure that the code is only executed +** once. If no functions are involved, then factor the code out and put it at +** the end of the prepared statement in the initialization section. ** ** If regDest>=0 then the result is always stored in that register and the ** result is not reusable. If regDest<0 then this routine is free to ** store the value whereever it wants. The register where the expression -** is stored is returned. When regDest<0, two identical expressions will -** code to the same register. +** is stored is returned. When regDest<0, two identical expressions might +** code to the same register, if they do not contain function calls and hence +** are factored out into the initialization section at the end of the +** prepared statement. */ -SQLITE_PRIVATE int sqlite3ExprCodeAtInit( +SQLITE_PRIVATE int sqlite3ExprCodeRunJustOnce( Parse *pParse, /* Parsing context */ Expr *pExpr, /* The expression to code when the VDBE initializes */ int regDest /* Store the value in this register */ @@ -98960,14 +104043,29 @@ SQLITE_PRIVATE int sqlite3ExprCodeAtInit( } } pExpr = sqlite3ExprDup(pParse->db, pExpr, 0); - p = sqlite3ExprListAppend(pParse, p, pExpr); - if( p ){ - struct ExprList_item *pItem = &p->a[p->nExpr-1]; - pItem->reusable = regDest<0; - if( regDest<0 ) regDest = ++pParse->nMem; - pItem->u.iConstExprReg = regDest; + if( pExpr!=0 && ExprHasProperty(pExpr, EP_HasFunc) ){ + Vdbe *v = pParse->pVdbe; + int addr; + assert( v ); + addr = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); + pParse->okConstFactor = 0; + if( !pParse->db->mallocFailed ){ + if( regDest<0 ) regDest = ++pParse->nMem; + sqlite3ExprCode(pParse, pExpr, regDest); + } + pParse->okConstFactor = 1; + sqlite3ExprDelete(pParse->db, pExpr); + sqlite3VdbeJumpHere(v, addr); + }else{ + p = sqlite3ExprListAppend(pParse, p, pExpr); + if( p ){ + struct ExprList_item *pItem = &p->a[p->nExpr-1]; + pItem->reusable = regDest<0; + if( regDest<0 ) regDest = ++pParse->nMem; + pItem->u.iConstExprReg = regDest; + } + pParse->pConstExpr = p; } - pParse->pConstExpr = p; return regDest; } @@ -98986,13 +104084,13 @@ SQLITE_PRIVATE int sqlite3ExprCodeAtInit( */ SQLITE_PRIVATE int sqlite3ExprCodeTemp(Parse *pParse, Expr *pExpr, int *pReg){ int r2; - pExpr = sqlite3ExprSkipCollate(pExpr); + pExpr = sqlite3ExprSkipCollateAndLikely(pExpr); if( ConstFactorOk(pParse) && pExpr->op!=TK_REGISTER && sqlite3ExprIsConstantNotJoin(pExpr) ){ *pReg = 0; - r2 = sqlite3ExprCodeAtInit(pParse, pExpr, -1); + r2 = sqlite3ExprCodeRunJustOnce(pParse, pExpr, -1); }else{ int r1 = sqlite3GetTempReg(pParse); r2 = sqlite3ExprCodeTarget(pParse, pExpr, r1); @@ -99014,15 +104112,18 @@ SQLITE_PRIVATE int sqlite3ExprCodeTemp(Parse *pParse, Expr *pExpr, int *pReg){ SQLITE_PRIVATE void sqlite3ExprCode(Parse *pParse, Expr *pExpr, int target){ int inReg; + assert( pExpr==0 || !ExprHasVVAProperty(pExpr,EP_Immutable) ); assert( target>0 && target<=pParse->nMem ); - if( pExpr && pExpr->op==TK_REGISTER ){ - sqlite3VdbeAddOp2(pParse->pVdbe, OP_Copy, pExpr->iTable, target); - }else{ - inReg = sqlite3ExprCodeTarget(pParse, pExpr, target); - assert( pParse->pVdbe!=0 || pParse->db->mallocFailed ); - if( inReg!=target && pParse->pVdbe ){ - sqlite3VdbeAddOp2(pParse->pVdbe, OP_SCopy, inReg, target); + inReg = sqlite3ExprCodeTarget(pParse, pExpr, target); + assert( pParse->pVdbe!=0 || pParse->db->mallocFailed ); + if( inReg!=target && pParse->pVdbe ){ + u8 op; + if( ExprHasProperty(pExpr,EP_Subquery) ){ + op = OP_Copy; + }else{ + op = OP_SCopy; } + sqlite3VdbeAddOp2(pParse->pVdbe, op, inReg, target); } } @@ -99045,37 +104146,13 @@ SQLITE_PRIVATE void sqlite3ExprCodeCopy(Parse *pParse, Expr *pExpr, int target){ ** might choose to code the expression at initialization time. */ SQLITE_PRIVATE void sqlite3ExprCodeFactorable(Parse *pParse, Expr *pExpr, int target){ - if( pParse->okConstFactor && sqlite3ExprIsConstant(pExpr) ){ - sqlite3ExprCodeAtInit(pParse, pExpr, target); + if( pParse->okConstFactor && sqlite3ExprIsConstantNotJoin(pExpr) ){ + sqlite3ExprCodeRunJustOnce(pParse, pExpr, target); }else{ - sqlite3ExprCode(pParse, pExpr, target); + sqlite3ExprCodeCopy(pParse, pExpr, target); } } -/* -** Generate code that evaluates the given expression and puts the result -** in register target. -** -** Also make a copy of the expression results into another "cache" register -** and modify the expression so that the next time it is evaluated, -** the result is a copy of the cache register. -** -** This routine is used for expressions that are used multiple -** times. They are evaluated once and the results of the expression -** are reused. -*/ -SQLITE_PRIVATE void sqlite3ExprCodeAndCache(Parse *pParse, Expr *pExpr, int target){ - Vdbe *v = pParse->pVdbe; - int iMem; - - assert( target>0 ); - assert( pExpr->op!=TK_REGISTER ); - sqlite3ExprCode(pParse, pExpr, target); - iMem = ++pParse->nMem; - sqlite3VdbeAddOp2(v, OP_Copy, target, iMem); - exprToRegister(pExpr, iMem); -} - /* ** Generate code that pushes the value of every element of the given ** expression list into a sequence of registers beginning at target. @@ -99127,8 +104204,10 @@ SQLITE_PRIVATE int sqlite3ExprCodeExprList( }else{ sqlite3VdbeAddOp2(v, copyOp, j+srcReg-1, target+i); } - }else if( (flags & SQLITE_ECEL_FACTOR)!=0 && sqlite3ExprIsConstant(pExpr) ){ - sqlite3ExprCodeAtInit(pParse, pExpr, target+i); + }else if( (flags & SQLITE_ECEL_FACTOR)!=0 + && sqlite3ExprIsConstantNotJoin(pExpr) + ){ + sqlite3ExprCodeRunJustOnce(pParse, pExpr, target+i); }else{ int inReg = sqlite3ExprCodeTarget(pParse, pExpr, target+i); if( inReg!=target+i ){ @@ -99137,6 +104216,7 @@ SQLITE_PRIVATE int sqlite3ExprCodeExprList( && (pOp=sqlite3VdbeGetOp(v, -1))->opcode==OP_Copy && pOp->p1+pOp->p3+1==inReg && pOp->p2+pOp->p3+1==target+i + && pOp->p5==0 /* The do-not-merge flag must be clear */ ){ pOp->p3++; }else{ @@ -99175,41 +104255,44 @@ static void exprCodeBetween( void (*xJump)(Parse*,Expr*,int,int), /* Action to take */ int jumpIfNull /* Take the jump if the BETWEEN is NULL */ ){ - Expr exprAnd; /* The AND operator in x>=y AND x<=z */ + Expr exprAnd; /* The AND operator in x>=y AND x<=z */ Expr compLeft; /* The x>=y term */ Expr compRight; /* The x<=z term */ - Expr exprX; /* The x subexpression */ int regFree1 = 0; /* Temporary use register */ - + Expr *pDel = 0; + sqlite3 *db = pParse->db; memset(&compLeft, 0, sizeof(Expr)); memset(&compRight, 0, sizeof(Expr)); memset(&exprAnd, 0, sizeof(Expr)); assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); - exprX = *pExpr->pLeft; - exprAnd.op = TK_AND; - exprAnd.pLeft = &compLeft; - exprAnd.pRight = &compRight; - compLeft.op = TK_GE; - compLeft.pLeft = &exprX; - compLeft.pRight = pExpr->x.pList->a[0].pExpr; - compRight.op = TK_LE; - compRight.pLeft = &exprX; - compRight.pRight = pExpr->x.pList->a[1].pExpr; - exprToRegister(&exprX, exprCodeVector(pParse, &exprX, ®Free1)); - if( xJump ){ - xJump(pParse, &exprAnd, dest, jumpIfNull); - }else{ - /* Mark the expression is being from the ON or USING clause of a join - ** so that the sqlite3ExprCodeTarget() routine will not attempt to move - ** it into the Parse.pConstExpr list. We should use a new bit for this, - ** for clarity, but we are out of bits in the Expr.flags field so we - ** have to reuse the EP_FromJoin bit. Bummer. */ - exprX.flags |= EP_FromJoin; - sqlite3ExprCodeTarget(pParse, &exprAnd, dest); + pDel = sqlite3ExprDup(db, pExpr->pLeft, 0); + if( db->mallocFailed==0 ){ + exprAnd.op = TK_AND; + exprAnd.pLeft = &compLeft; + exprAnd.pRight = &compRight; + compLeft.op = TK_GE; + compLeft.pLeft = pDel; + compLeft.pRight = pExpr->x.pList->a[0].pExpr; + compRight.op = TK_LE; + compRight.pLeft = pDel; + compRight.pRight = pExpr->x.pList->a[1].pExpr; + exprToRegister(pDel, exprCodeVector(pParse, pDel, ®Free1)); + if( xJump ){ + xJump(pParse, &exprAnd, dest, jumpIfNull); + }else{ + /* Mark the expression is being from the ON or USING clause of a join + ** so that the sqlite3ExprCodeTarget() routine will not attempt to move + ** it into the Parse.pConstExpr list. We should use a new bit for this, + ** for clarity, but we are out of bits in the Expr.flags field so we + ** have to reuse the EP_FromJoin bit. Bummer. */ + pDel->flags |= EP_FromJoin; + sqlite3ExprCodeTarget(pParse, &exprAnd, dest); + } + sqlite3ReleaseTempReg(pParse, regFree1); } - sqlite3ReleaseTempReg(pParse, regFree1); + sqlite3ExprDelete(db, pDel); /* Ensure adequate test coverage */ testcase( xJump==sqlite3ExprIfTrue && jumpIfNull==0 && regFree1==0 ); @@ -99247,24 +104330,26 @@ SQLITE_PRIVATE void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int assert( jumpIfNull==SQLITE_JUMPIFNULL || jumpIfNull==0 ); if( NEVER(v==0) ) return; /* Existence of VDBE checked by caller */ if( NEVER(pExpr==0) ) return; /* No way this can happen */ + assert( !ExprHasVVAProperty(pExpr, EP_Immutable) ); op = pExpr->op; switch( op ){ - case TK_AND: { - int d2 = sqlite3VdbeMakeLabel(v); - testcase( jumpIfNull==0 ); - sqlite3ExprIfFalse(pParse, pExpr->pLeft, d2,jumpIfNull^SQLITE_JUMPIFNULL); - sqlite3ExprCachePush(pParse); - sqlite3ExprIfTrue(pParse, pExpr->pRight, dest, jumpIfNull); - sqlite3VdbeResolveLabel(v, d2); - sqlite3ExprCachePop(pParse); - break; - } + case TK_AND: case TK_OR: { - testcase( jumpIfNull==0 ); - sqlite3ExprIfTrue(pParse, pExpr->pLeft, dest, jumpIfNull); - sqlite3ExprCachePush(pParse); - sqlite3ExprIfTrue(pParse, pExpr->pRight, dest, jumpIfNull); - sqlite3ExprCachePop(pParse); + Expr *pAlt = sqlite3ExprSimplifiedAndOr(pExpr); + if( pAlt!=pExpr ){ + sqlite3ExprIfTrue(pParse, pAlt, dest, jumpIfNull); + }else if( op==TK_AND ){ + int d2 = sqlite3VdbeMakeLabel(pParse); + testcase( jumpIfNull==0 ); + sqlite3ExprIfFalse(pParse, pExpr->pLeft, d2, + jumpIfNull^SQLITE_JUMPIFNULL); + sqlite3ExprIfTrue(pParse, pExpr->pRight, dest, jumpIfNull); + sqlite3VdbeResolveLabel(v, d2); + }else{ + testcase( jumpIfNull==0 ); + sqlite3ExprIfTrue(pParse, pExpr->pLeft, dest, jumpIfNull); + sqlite3ExprIfTrue(pParse, pExpr->pRight, dest, jumpIfNull); + } break; } case TK_NOT: { @@ -99307,7 +104392,7 @@ SQLITE_PRIVATE void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); codeCompare(pParse, pExpr->pLeft, pExpr->pRight, op, - r1, r2, dest, jumpIfNull); + r1, r2, dest, jumpIfNull, ExprHasProperty(pExpr,EP_Commuted)); assert(TK_LT==OP_Lt); testcase(op==OP_Lt); VdbeCoverageIf(v,op==OP_Lt); assert(TK_LE==OP_Le); testcase(op==OP_Le); VdbeCoverageIf(v,op==OP_Le); assert(TK_GT==OP_Gt); testcase(op==OP_Gt); VdbeCoverageIf(v,op==OP_Gt); @@ -99340,7 +104425,7 @@ SQLITE_PRIVATE void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int } #ifndef SQLITE_OMIT_SUBQUERY case TK_IN: { - int destIfFalse = sqlite3VdbeMakeLabel(v); + int destIfFalse = sqlite3VdbeMakeLabel(pParse); int destIfNull = jumpIfNull ? dest : destIfFalse; sqlite3ExprCodeIN(pParse, pExpr, destIfFalse, destIfNull); sqlite3VdbeGoto(v, dest); @@ -99350,9 +104435,9 @@ SQLITE_PRIVATE void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int #endif default: { default_expr: - if( exprAlwaysTrue(pExpr) ){ + if( ExprAlwaysTrue(pExpr) ){ sqlite3VdbeGoto(v, dest); - }else if( exprAlwaysFalse(pExpr) ){ + }else if( ExprAlwaysFalse(pExpr) ){ /* No-op */ }else{ r1 = sqlite3ExprCodeTemp(pParse, pExpr, ®Free1); @@ -99387,6 +104472,7 @@ SQLITE_PRIVATE void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int assert( jumpIfNull==SQLITE_JUMPIFNULL || jumpIfNull==0 ); if( NEVER(v==0) ) return; /* Existence of VDBE checked by caller */ if( pExpr==0 ) return; + assert( !ExprHasVVAProperty(pExpr,EP_Immutable) ); /* The value of pExpr->op and op are related as follows: ** @@ -99420,22 +104506,23 @@ SQLITE_PRIVATE void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int assert( pExpr->op!=TK_GE || op==OP_Lt ); switch( pExpr->op ){ - case TK_AND: { - testcase( jumpIfNull==0 ); - sqlite3ExprIfFalse(pParse, pExpr->pLeft, dest, jumpIfNull); - sqlite3ExprCachePush(pParse); - sqlite3ExprIfFalse(pParse, pExpr->pRight, dest, jumpIfNull); - sqlite3ExprCachePop(pParse); - break; - } + case TK_AND: case TK_OR: { - int d2 = sqlite3VdbeMakeLabel(v); - testcase( jumpIfNull==0 ); - sqlite3ExprIfTrue(pParse, pExpr->pLeft, d2, jumpIfNull^SQLITE_JUMPIFNULL); - sqlite3ExprCachePush(pParse); - sqlite3ExprIfFalse(pParse, pExpr->pRight, dest, jumpIfNull); - sqlite3VdbeResolveLabel(v, d2); - sqlite3ExprCachePop(pParse); + Expr *pAlt = sqlite3ExprSimplifiedAndOr(pExpr); + if( pAlt!=pExpr ){ + sqlite3ExprIfFalse(pParse, pAlt, dest, jumpIfNull); + }else if( pExpr->op==TK_AND ){ + testcase( jumpIfNull==0 ); + sqlite3ExprIfFalse(pParse, pExpr->pLeft, dest, jumpIfNull); + sqlite3ExprIfFalse(pParse, pExpr->pRight, dest, jumpIfNull); + }else{ + int d2 = sqlite3VdbeMakeLabel(pParse); + testcase( jumpIfNull==0 ); + sqlite3ExprIfTrue(pParse, pExpr->pLeft, d2, + jumpIfNull^SQLITE_JUMPIFNULL); + sqlite3ExprIfFalse(pParse, pExpr->pRight, dest, jumpIfNull); + sqlite3VdbeResolveLabel(v, d2); + } break; } case TK_NOT: { @@ -99481,7 +104568,7 @@ SQLITE_PRIVATE void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); codeCompare(pParse, pExpr->pLeft, pExpr->pRight, op, - r1, r2, dest, jumpIfNull); + r1, r2, dest, jumpIfNull,ExprHasProperty(pExpr,EP_Commuted)); assert(TK_LT==OP_Lt); testcase(op==OP_Lt); VdbeCoverageIf(v,op==OP_Lt); assert(TK_LE==OP_Le); testcase(op==OP_Le); VdbeCoverageIf(v,op==OP_Le); assert(TK_GT==OP_Gt); testcase(op==OP_Gt); VdbeCoverageIf(v,op==OP_Gt); @@ -99515,7 +104602,7 @@ SQLITE_PRIVATE void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int if( jumpIfNull ){ sqlite3ExprCodeIN(pParse, pExpr, dest, dest); }else{ - int destIfNull = sqlite3VdbeMakeLabel(v); + int destIfNull = sqlite3VdbeMakeLabel(pParse); sqlite3ExprCodeIN(pParse, pExpr, dest, destIfNull); sqlite3VdbeResolveLabel(v, destIfNull); } @@ -99524,9 +104611,9 @@ SQLITE_PRIVATE void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int #endif default: { default_expr: - if( exprAlwaysFalse(pExpr) ){ + if( ExprAlwaysFalse(pExpr) ){ sqlite3VdbeGoto(v, dest); - }else if( exprAlwaysTrue(pExpr) ){ + }else if( ExprAlwaysTrue(pExpr) ){ /* no-op */ }else{ r1 = sqlite3ExprCodeTemp(pParse, pExpr, ®Free1); @@ -99636,7 +104723,7 @@ SQLITE_PRIVATE int sqlite3ExprCompare(Parse *pParse, Expr *pA, Expr *pB, int iTa } return 2; } - if( pA->op!=pB->op ){ + if( pA->op!=pB->op || pA->op==TK_RAISE ){ if( pA->op==TK_COLLATE && sqlite3ExprCompare(pParse, pA->pLeft,pB,iTab)<2 ){ return 1; } @@ -99646,33 +104733,53 @@ SQLITE_PRIVATE int sqlite3ExprCompare(Parse *pParse, Expr *pA, Expr *pB, int iTa return 2; } if( pA->op!=TK_COLUMN && pA->op!=TK_AGG_COLUMN && pA->u.zToken ){ - if( pA->op==TK_FUNCTION ){ + if( pA->op==TK_FUNCTION || pA->op==TK_AGG_FUNCTION ){ if( sqlite3StrICmp(pA->u.zToken,pB->u.zToken)!=0 ) return 2; +#ifndef SQLITE_OMIT_WINDOWFUNC + assert( pA->op==pB->op ); + if( ExprHasProperty(pA,EP_WinFunc)!=ExprHasProperty(pB,EP_WinFunc) ){ + return 2; + } + if( ExprHasProperty(pA,EP_WinFunc) ){ + if( sqlite3WindowCompare(pParse, pA->y.pWin, pB->y.pWin, 1)!=0 ){ + return 2; + } + } +#endif + }else if( pA->op==TK_NULL ){ + return 0; }else if( pA->op==TK_COLLATE ){ if( sqlite3_stricmp(pA->u.zToken,pB->u.zToken)!=0 ) return 2; - }else if( strcmp(pA->u.zToken,pB->u.zToken)!=0 ){ + }else if( ALWAYS(pB->u.zToken!=0) && strcmp(pA->u.zToken,pB->u.zToken)!=0 ){ return 2; } } - if( (pA->flags & EP_Distinct)!=(pB->flags & EP_Distinct) ) return 2; + if( (pA->flags & (EP_Distinct|EP_Commuted)) + != (pB->flags & (EP_Distinct|EP_Commuted)) ) return 2; if( ALWAYS((combinedFlags & EP_TokenOnly)==0) ){ if( combinedFlags & EP_xIsSelect ) return 2; - if( sqlite3ExprCompare(pParse, pA->pLeft, pB->pLeft, iTab) ) return 2; + if( (combinedFlags & EP_FixedCol)==0 + && sqlite3ExprCompare(pParse, pA->pLeft, pB->pLeft, iTab) ) return 2; if( sqlite3ExprCompare(pParse, pA->pRight, pB->pRight, iTab) ) return 2; if( sqlite3ExprListCompare(pA->x.pList, pB->x.pList, iTab) ) return 2; - assert( (combinedFlags & EP_Reduced)==0 ); - if( pA->op!=TK_STRING && pA->op!=TK_TRUEFALSE ){ + if( pA->op!=TK_STRING + && pA->op!=TK_TRUEFALSE + && ALWAYS((combinedFlags & EP_Reduced)==0) + ){ if( pA->iColumn!=pB->iColumn ) return 2; - if( pA->iTable!=pB->iTable - && (pA->iTable!=iTab || NEVER(pB->iTable>=0)) ) return 2; + if( pA->op2!=pB->op2 && pA->op==TK_TRUTH ) return 2; + if( pA->op!=TK_IN && pA->iTable!=pB->iTable && pA->iTable!=iTab ){ + return 2; + } } } return 0; } /* -** Compare two ExprList objects. Return 0 if they are identical and -** non-zero if they differ in any way. +** Compare two ExprList objects. Return 0 if they are identical, 1 +** if they are certainly different, or 2 if it is not possible to +** determine if they are identical or not. ** ** If any subelement of pB has Expr.iTable==(-1) then it is allowed ** to compare equal to an equivalent element in pA with Expr.iTable==iTab. @@ -99691,10 +104798,11 @@ SQLITE_PRIVATE int sqlite3ExprListCompare(ExprList *pA, ExprList *pB, int iTab){ if( pA==0 || pB==0 ) return 1; if( pA->nExpr!=pB->nExpr ) return 1; for(i=0; inExpr; i++){ + int res; Expr *pExprA = pA->a[i].pExpr; Expr *pExprB = pB->a[i].pExpr; - if( pA->a[i].sortOrder!=pB->a[i].sortOrder ) return 1; - if( sqlite3ExprCompare(0, pExprA, pExprB, iTab) ) return 1; + if( pA->a[i].sortFlags!=pB->a[i].sortFlags ) return 1; + if( (res = sqlite3ExprCompare(0, pExprA, pExprB, iTab)) ) return res; } return 0; } @@ -99705,11 +104813,88 @@ SQLITE_PRIVATE int sqlite3ExprListCompare(ExprList *pA, ExprList *pB, int iTab){ */ SQLITE_PRIVATE int sqlite3ExprCompareSkip(Expr *pA, Expr *pB, int iTab){ return sqlite3ExprCompare(0, - sqlite3ExprSkipCollate(pA), - sqlite3ExprSkipCollate(pB), + sqlite3ExprSkipCollateAndLikely(pA), + sqlite3ExprSkipCollateAndLikely(pB), iTab); } +/* +** Return non-zero if Expr p can only be true if pNN is not NULL. +** +** Or if seenNot is true, return non-zero if Expr p can only be +** non-NULL if pNN is not NULL +*/ +static int exprImpliesNotNull( + Parse *pParse, /* Parsing context */ + Expr *p, /* The expression to be checked */ + Expr *pNN, /* The expression that is NOT NULL */ + int iTab, /* Table being evaluated */ + int seenNot /* Return true only if p can be any non-NULL value */ +){ + assert( p ); + assert( pNN ); + if( sqlite3ExprCompare(pParse, p, pNN, iTab)==0 ){ + return pNN->op!=TK_NULL; + } + switch( p->op ){ + case TK_IN: { + if( seenNot && ExprHasProperty(p, EP_xIsSelect) ) return 0; + assert( ExprHasProperty(p,EP_xIsSelect) + || (p->x.pList!=0 && p->x.pList->nExpr>0) ); + return exprImpliesNotNull(pParse, p->pLeft, pNN, iTab, 1); + } + case TK_BETWEEN: { + ExprList *pList = p->x.pList; + assert( pList!=0 ); + assert( pList->nExpr==2 ); + if( seenNot ) return 0; + if( exprImpliesNotNull(pParse, pList->a[0].pExpr, pNN, iTab, 1) + || exprImpliesNotNull(pParse, pList->a[1].pExpr, pNN, iTab, 1) + ){ + return 1; + } + return exprImpliesNotNull(pParse, p->pLeft, pNN, iTab, 1); + } + case TK_EQ: + case TK_NE: + case TK_LT: + case TK_LE: + case TK_GT: + case TK_GE: + case TK_PLUS: + case TK_MINUS: + case TK_BITOR: + case TK_LSHIFT: + case TK_RSHIFT: + case TK_CONCAT: + seenNot = 1; + /* Fall thru */ + case TK_STAR: + case TK_REM: + case TK_BITAND: + case TK_SLASH: { + if( exprImpliesNotNull(pParse, p->pRight, pNN, iTab, seenNot) ) return 1; + /* Fall thru into the next case */ + } + case TK_SPAN: + case TK_COLLATE: + case TK_UPLUS: + case TK_UMINUS: { + return exprImpliesNotNull(pParse, p->pLeft, pNN, iTab, seenNot); + } + case TK_TRUTH: { + if( seenNot ) return 0; + if( p->op2!=TK_IS ) return 0; + return exprImpliesNotNull(pParse, p->pLeft, pNN, iTab, 1); + } + case TK_BITNOT: + case TK_NOT: { + return exprImpliesNotNull(pParse, p->pLeft, pNN, iTab, 1); + } + } + return 0; +} + /* ** Return true if we can prove the pE2 will always be true if pE1 is ** true. Return false if we cannot complete the proof or if pE2 might @@ -99745,47 +104930,48 @@ SQLITE_PRIVATE int sqlite3ExprImpliesExpr(Parse *pParse, Expr *pE1, Expr *pE2, i ){ return 1; } - if( pE2->op==TK_NOTNULL && pE1->op!=TK_ISNULL && pE1->op!=TK_IS ){ - Expr *pX = sqlite3ExprSkipCollate(pE1->pLeft); - testcase( pX!=pE1->pLeft ); - if( sqlite3ExprCompare(pParse, pX, pE2->pLeft, iTab)==0 ) return 1; + if( pE2->op==TK_NOTNULL + && exprImpliesNotNull(pParse, pE1, pE2->pLeft, iTab, 0) + ){ + return 1; } return 0; } /* -** This is the Expr node callback for sqlite3ExprImpliesNotNullRow(). +** This is the Expr node callback for sqlite3ExprImpliesNonNullRow(). ** If the expression node requires that the table at pWalker->iCur -** have a non-NULL column, then set pWalker->eCode to 1 and abort. +** have one or more non-NULL column, then set pWalker->eCode to 1 and abort. +** +** This routine controls an optimization. False positives (setting +** pWalker->eCode to 1 when it should not be) are deadly, but false-negatives +** (never setting pWalker->eCode) is a harmless missed optimization. */ static int impliesNotNullRow(Walker *pWalker, Expr *pExpr){ - /* This routine is only called for WHERE clause expressions and so it - ** cannot have any TK_AGG_COLUMN entries because those are only found - ** in HAVING clauses. We can get a TK_AGG_FUNCTION in a WHERE clause, - ** but that is an illegal construct and the query will be rejected at - ** a later stage of processing, so the TK_AGG_FUNCTION case does not - ** need to be considered here. */ - assert( pExpr->op!=TK_AGG_COLUMN ); + testcase( pExpr->op==TK_AGG_COLUMN ); testcase( pExpr->op==TK_AGG_FUNCTION ); - if( ExprHasProperty(pExpr, EP_FromJoin) ) return WRC_Prune; switch( pExpr->op ){ case TK_ISNOT: - case TK_NOT: case TK_ISNULL: + case TK_NOTNULL: case TK_IS: case TK_OR: + case TK_VECTOR: case TK_CASE: case TK_IN: case TK_FUNCTION: + case TK_TRUTH: testcase( pExpr->op==TK_ISNOT ); - testcase( pExpr->op==TK_NOT ); testcase( pExpr->op==TK_ISNULL ); + testcase( pExpr->op==TK_NOTNULL ); testcase( pExpr->op==TK_IS ); testcase( pExpr->op==TK_OR ); + testcase( pExpr->op==TK_VECTOR ); testcase( pExpr->op==TK_CASE ); testcase( pExpr->op==TK_IN ); testcase( pExpr->op==TK_FUNCTION ); + testcase( pExpr->op==TK_TRUTH ); return WRC_Prune; case TK_COLUMN: if( pWalker->u.iCur==pExpr->iTable ){ @@ -99794,6 +104980,23 @@ static int impliesNotNullRow(Walker *pWalker, Expr *pExpr){ } return WRC_Prune; + case TK_AND: + if( pWalker->eCode==0 ){ + sqlite3WalkExpr(pWalker, pExpr->pLeft); + if( pWalker->eCode ){ + pWalker->eCode = 0; + sqlite3WalkExpr(pWalker, pExpr->pRight); + } + } + return WRC_Prune; + + case TK_BETWEEN: + if( sqlite3WalkExpr(pWalker, pExpr->pLeft)==WRC_Abort ){ + assert( pWalker->eCode ); + return WRC_Abort; + } + return WRC_Prune; + /* Virtual tables are allowed to use constraints like x=NULL. So ** a term of the form x=y does not prove that y is not null if x ** is the column of a virtual table */ @@ -99802,18 +105005,25 @@ static int impliesNotNullRow(Walker *pWalker, Expr *pExpr){ case TK_LT: case TK_LE: case TK_GT: - case TK_GE: + case TK_GE: { + Expr *pLeft = pExpr->pLeft; + Expr *pRight = pExpr->pRight; testcase( pExpr->op==TK_EQ ); testcase( pExpr->op==TK_NE ); testcase( pExpr->op==TK_LT ); testcase( pExpr->op==TK_LE ); testcase( pExpr->op==TK_GT ); testcase( pExpr->op==TK_GE ); - if( (pExpr->pLeft->op==TK_COLUMN && IsVirtual(pExpr->pLeft->pTab)) - || (pExpr->pRight->op==TK_COLUMN && IsVirtual(pExpr->pRight->pTab)) + /* The y.pTab=0 assignment in wherecode.c always happens after the + ** impliesNotNullRow() test */ + if( (pLeft->op==TK_COLUMN && ALWAYS(pLeft->y.pTab!=0) + && IsVirtual(pLeft->y.pTab)) + || (pRight->op==TK_COLUMN && ALWAYS(pRight->y.pTab!=0) + && IsVirtual(pRight->y.pTab)) ){ - return WRC_Prune; + return WRC_Prune; } + } default: return WRC_Continue; } @@ -99843,6 +105053,16 @@ static int impliesNotNullRow(Walker *pWalker, Expr *pExpr){ */ SQLITE_PRIVATE int sqlite3ExprImpliesNonNullRow(Expr *p, int iTab){ Walker w; + p = sqlite3ExprSkipCollateAndLikely(p); + if( p==0 ) return 0; + if( p->op==TK_NOTNULL ){ + p = p->pLeft; + }else{ + while( p->op==TK_AND ){ + if( sqlite3ExprImpliesNonNullRow(p->pLeft, iTab) ) return 1; + p = p->pRight; + } + } w.xExprCallback = impliesNotNullRow; w.xSelectCallback = 0; w.xSelectCallback2 = 0; @@ -99872,7 +105092,7 @@ struct IdxCover { static int exprIdxCover(Walker *pWalker, Expr *pExpr){ if( pExpr->op==TK_COLUMN && pExpr->iTable==pWalker->u.pIdxCover->iCur - && sqlite3ColumnOfIndex(pWalker->u.pIdxCover->pIdx, pExpr->iColumn)<0 + && sqlite3TableColumnToIndex(pWalker->u.pIdxCover->pIdx, pExpr->iColumn)<0 ){ pWalker->eCode = 1; return WRC_Abort; @@ -99923,12 +105143,13 @@ struct SrcCount { ** Count the number of references to columns. */ static int exprSrcCount(Walker *pWalker, Expr *pExpr){ - /* The NEVER() on the second term is because sqlite3FunctionUsesThisSrc() - ** is always called before sqlite3ExprAnalyzeAggregates() and so the - ** TK_COLUMNs have not yet been converted into TK_AGG_COLUMN. If - ** sqlite3FunctionUsesThisSrc() is used differently in the future, the - ** NEVER() will need to be removed. */ - if( pExpr->op==TK_COLUMN || NEVER(pExpr->op==TK_AGG_COLUMN) ){ + /* There was once a NEVER() on the second term on the grounds that + ** sqlite3FunctionUsesThisSrc() was always called before + ** sqlite3ExprAnalyzeAggregates() and so the TK_COLUMNs have not yet + ** been converted into TK_AGG_COLUMN. But this is no longer true due + ** to window functions - sqlite3WindowRewrite() may now indirectly call + ** FunctionUsesThisSrc() when creating a new sub-select. */ + if( pExpr->op==TK_COLUMN || pExpr->op==TK_AGG_COLUMN ){ int i; struct SrcCount *p = pWalker->u.pSrcCount; SrcList *pSrc = p->pSrc; @@ -99938,7 +105159,10 @@ static int exprSrcCount(Walker *pWalker, Expr *pExpr){ } if( inThis++; - }else{ + }else if( nSrc==0 || pExpr->iTablea[0].iCursor ){ + /* In a well-formed parse tree (no name resolution errors), + ** TK_COLUMN nodes with smaller Expr.iTable values are in an + ** outer context. Those are the only ones to count as "other" */ p->nOther++; } } @@ -99955,13 +105179,19 @@ SQLITE_PRIVATE int sqlite3FunctionUsesThisSrc(Expr *pExpr, SrcList *pSrcList){ Walker w; struct SrcCount cnt; assert( pExpr->op==TK_AGG_FUNCTION ); + memset(&w, 0, sizeof(w)); w.xExprCallback = exprSrcCount; - w.xSelectCallback = 0; + w.xSelectCallback = sqlite3SelectWalkNoop; w.u.pSrcCount = &cnt; cnt.pSrc = pSrcList; cnt.nThis = 0; cnt.nOther = 0; sqlite3WalkExprList(&w, pExpr->x.pList); +#ifndef SQLITE_OMIT_WINDOWFUNC + if( ExprHasProperty(pExpr, EP_WinFunc) ){ + sqlite3WalkExpr(&w, pExpr->y.pWin->pFilter); + } +#endif return cnt.nThis>0 || cnt.nOther==0; } @@ -100041,7 +105271,7 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ && (k = addAggInfoColumn(pParse->db, pAggInfo))>=0 ){ pCol = &pAggInfo->aCol[k]; - pCol->pTab = pExpr->pTab; + pCol->pTab = pExpr->y.pTab; pCol->iTable = pExpr->iTable; pCol->iColumn = pExpr->iColumn; pCol->iMem = ++pParse->nMem; @@ -100128,15 +105358,6 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ } return WRC_Continue; } -static int analyzeAggregatesInSelect(Walker *pWalker, Select *pSelect){ - UNUSED_PARAMETER(pSelect); - pWalker->walkerDepth++; - return WRC_Continue; -} -static void analyzeAggregatesInSelectEnd(Walker *pWalker, Select *pSelect){ - UNUSED_PARAMETER(pSelect); - pWalker->walkerDepth--; -} /* ** Analyze the pExpr expression looking for aggregate functions and @@ -100150,10 +105371,11 @@ static void analyzeAggregatesInSelectEnd(Walker *pWalker, Select *pSelect){ SQLITE_PRIVATE void sqlite3ExprAnalyzeAggregates(NameContext *pNC, Expr *pExpr){ Walker w; w.xExprCallback = analyzeAggregate; - w.xSelectCallback = analyzeAggregatesInSelect; - w.xSelectCallback2 = analyzeAggregatesInSelectEnd; + w.xSelectCallback = sqlite3WalkerDepthIncrease; + w.xSelectCallback2 = sqlite3WalkerDepthDecrease; w.walkerDepth = 0; w.u.pNC = pNC; + w.pParse = 0; assert( pNC->pSrcList!=0 ); sqlite3WalkExpr(&w, pExpr); } @@ -100187,22 +105409,13 @@ SQLITE_PRIVATE int sqlite3GetTempReg(Parse *pParse){ /* ** Deallocate a register, making available for reuse for some other ** purpose. -** -** If a register is currently being used by the column cache, then -** the deallocation is deferred until the column cache line that uses -** the register becomes stale. */ SQLITE_PRIVATE void sqlite3ReleaseTempReg(Parse *pParse, int iReg){ - if( iReg && pParse->nTempRegaTempReg) ){ - int i; - struct yColCache *p; - for(i=0, p=pParse->aColCache; inColCache; i++, p++){ - if( p->iReg==iReg ){ - p->tempReg = 1; - return; - } + if( iReg ){ + sqlite3VdbeReleaseRegisters(pParse, iReg, 1, 0, 0); + if( pParse->nTempRegaTempReg) ){ + pParse->aTempReg[pParse->nTempReg++] = iReg; } - pParse->aTempReg[pParse->nTempReg++] = iReg; } } @@ -100215,7 +105428,6 @@ SQLITE_PRIVATE int sqlite3GetTempRange(Parse *pParse, int nReg){ i = pParse->iRangeReg; n = pParse->nRangeReg; if( nReg<=n ){ - assert( !usedAsColumnCache(pParse, i, i+n-1) ); pParse->iRangeReg += nReg; pParse->nRangeReg -= nReg; }else{ @@ -100229,7 +105441,7 @@ SQLITE_PRIVATE void sqlite3ReleaseTempRange(Parse *pParse, int iReg, int nReg){ sqlite3ReleaseTempReg(pParse, iReg); return; } - sqlite3ExprCacheRemove(pParse, iReg, nReg); + sqlite3VdbeReleaseRegisters(pParse, iReg, nReg, 0, 0); if( nReg>pParse->nRangeReg ){ pParse->nRangeReg = nReg; pParse->iRangeReg = iReg; @@ -100238,6 +105450,11 @@ SQLITE_PRIVATE void sqlite3ReleaseTempRange(Parse *pParse, int iReg, int nReg){ /* ** Mark all temporary registers as being unavailable for reuse. +** +** Always invoke this procedure after coding a subroutine or co-routine +** that might be invoked from other parts of the code, to ensure that +** the sub/co-routine does not use registers in common with the code that +** invokes the sub/co-routine. */ SQLITE_PRIVATE void sqlite3ClearTempRegCache(Parse *pParse){ pParse->nTempReg = 0; @@ -100291,366 +105508,69 @@ SQLITE_PRIVATE int sqlite3NoTempsInRange(Parse *pParse, int iFirst, int iLast){ */ #ifndef SQLITE_OMIT_ALTERTABLE - -/* -** This function is used by SQL generated to implement the -** ALTER TABLE command. The first argument is the text of a CREATE TABLE or -** CREATE INDEX command. The second is a table name. The table name in -** the CREATE TABLE or CREATE INDEX statement is replaced with the third -** argument and the result returned. Examples: -** -** sqlite_rename_table('CREATE TABLE abc(a, b, c)', 'def') -** -> 'CREATE TABLE def(a, b, c)' -** -** sqlite_rename_table('CREATE INDEX i ON abc(a)', 'def') -** -> 'CREATE INDEX i ON def(a, b, c)' -*/ -static void renameTableFunc( - sqlite3_context *context, - int NotUsed, - sqlite3_value **argv -){ - unsigned char const *zSql = sqlite3_value_text(argv[0]); - unsigned char const *zTableName = sqlite3_value_text(argv[1]); - - int token; - Token tname; - unsigned char const *zCsr = zSql; - int len = 0; - char *zRet; - - sqlite3 *db = sqlite3_context_db_handle(context); - - UNUSED_PARAMETER(NotUsed); - - /* The principle used to locate the table name in the CREATE TABLE - ** statement is that the table name is the first non-space token that - ** is immediately followed by a TK_LP or TK_USING token. - */ - if( zSql ){ - do { - if( !*zCsr ){ - /* Ran out of input before finding an opening bracket. Return NULL. */ - return; - } - - /* Store the token that zCsr points to in tname. */ - tname.z = (char*)zCsr; - tname.n = len; - - /* Advance zCsr to the next token. Store that token type in 'token', - ** and its length in 'len' (to be used next iteration of this loop). - */ - do { - zCsr += len; - len = sqlite3GetToken(zCsr, &token); - } while( token==TK_SPACE ); - assert( len>0 ); - } while( token!=TK_LP && token!=TK_USING ); - - zRet = sqlite3MPrintf(db, "%.*s\"%w\"%s", (int)(((u8*)tname.z) - zSql), - zSql, zTableName, tname.z+tname.n); - sqlite3_result_text(context, zRet, -1, SQLITE_DYNAMIC); - } -} - -/* -** This C function implements an SQL user function that is used by SQL code -** generated by the ALTER TABLE ... RENAME command to modify the definition -** of any foreign key constraints that use the table being renamed as the -** parent table. It is passed three arguments: -** -** 1) The complete text of the CREATE TABLE statement being modified, -** 2) The old name of the table being renamed, and -** 3) The new name of the table being renamed. -** -** It returns the new CREATE TABLE statement. For example: -** -** sqlite_rename_parent('CREATE TABLE t1(a REFERENCES t2)', 't2', 't3') -** -> 'CREATE TABLE t1(a REFERENCES t3)' -*/ -#ifndef SQLITE_OMIT_FOREIGN_KEY -static void renameParentFunc( - sqlite3_context *context, - int NotUsed, - sqlite3_value **argv -){ - sqlite3 *db = sqlite3_context_db_handle(context); - char *zOutput = 0; - char *zResult; - unsigned char const *zInput = sqlite3_value_text(argv[0]); - unsigned char const *zOld = sqlite3_value_text(argv[1]); - unsigned char const *zNew = sqlite3_value_text(argv[2]); - - unsigned const char *z; /* Pointer to token */ - int n; /* Length of token z */ - int token; /* Type of token */ - - UNUSED_PARAMETER(NotUsed); - if( zInput==0 || zOld==0 ) return; - for(z=zInput; *z; z=z+n){ - n = sqlite3GetToken(z, &token); - if( token==TK_REFERENCES ){ - char *zParent; - do { - z += n; - n = sqlite3GetToken(z, &token); - }while( token==TK_SPACE ); - - if( token==TK_ILLEGAL ) break; - zParent = sqlite3DbStrNDup(db, (const char *)z, n); - if( zParent==0 ) break; - sqlite3Dequote(zParent); - if( 0==sqlite3StrICmp((const char *)zOld, zParent) ){ - char *zOut = sqlite3MPrintf(db, "%s%.*s\"%w\"", - (zOutput?zOutput:""), (int)(z-zInput), zInput, (const char *)zNew - ); - sqlite3DbFree(db, zOutput); - zOutput = zOut; - zInput = &z[n]; - } - sqlite3DbFree(db, zParent); - } - } - - zResult = sqlite3MPrintf(db, "%s%s", (zOutput?zOutput:""), zInput), - sqlite3_result_text(context, zResult, -1, SQLITE_DYNAMIC); - sqlite3DbFree(db, zOutput); -} -#endif - -#ifndef SQLITE_OMIT_TRIGGER -/* This function is used by SQL generated to implement the -** ALTER TABLE command. The first argument is the text of a CREATE TRIGGER -** statement. The second is a table name. The table name in the CREATE -** TRIGGER statement is replaced with the third argument and the result -** returned. This is analagous to renameTableFunc() above, except for CREATE -** TRIGGER, not CREATE INDEX and CREATE TABLE. -*/ -static void renameTriggerFunc( - sqlite3_context *context, - int NotUsed, - sqlite3_value **argv -){ - unsigned char const *zSql = sqlite3_value_text(argv[0]); - unsigned char const *zTableName = sqlite3_value_text(argv[1]); - - int token; - Token tname; - int dist = 3; - unsigned char const *zCsr = zSql; - int len = 0; - char *zRet; - sqlite3 *db = sqlite3_context_db_handle(context); - - UNUSED_PARAMETER(NotUsed); - - /* The principle used to locate the table name in the CREATE TRIGGER - ** statement is that the table name is the first token that is immediately - ** preceded by either TK_ON or TK_DOT and immediately followed by one - ** of TK_WHEN, TK_BEGIN or TK_FOR. - */ - if( zSql ){ - do { - - if( !*zCsr ){ - /* Ran out of input before finding the table name. Return NULL. */ - return; - } - - /* Store the token that zCsr points to in tname. */ - tname.z = (char*)zCsr; - tname.n = len; - - /* Advance zCsr to the next token. Store that token type in 'token', - ** and its length in 'len' (to be used next iteration of this loop). - */ - do { - zCsr += len; - len = sqlite3GetToken(zCsr, &token); - }while( token==TK_SPACE ); - assert( len>0 ); - - /* Variable 'dist' stores the number of tokens read since the most - ** recent TK_DOT or TK_ON. This means that when a WHEN, FOR or BEGIN - ** token is read and 'dist' equals 2, the condition stated above - ** to be met. - ** - ** Note that ON cannot be a database, table or column name, so - ** there is no need to worry about syntax like - ** "CREATE TRIGGER ... ON ON.ON BEGIN ..." etc. - */ - dist++; - if( token==TK_DOT || token==TK_ON ){ - dist = 0; - } - } while( dist!=2 || (token!=TK_WHEN && token!=TK_FOR && token!=TK_BEGIN) ); - - /* Variable tname now contains the token that is the old table-name - ** in the CREATE TRIGGER statement. - */ - zRet = sqlite3MPrintf(db, "%.*s\"%w\"%s", (int)(((u8*)tname.z) - zSql), - zSql, zTableName, tname.z+tname.n); - sqlite3_result_text(context, zRet, -1, SQLITE_DYNAMIC); - } -} -#endif /* !SQLITE_OMIT_TRIGGER */ - -/* -** Register built-in functions used to help implement ALTER TABLE -*/ -SQLITE_PRIVATE void sqlite3AlterFunctions(void){ - static FuncDef aAlterTableFuncs[] = { - FUNCTION(sqlite_rename_table, 2, 0, 0, renameTableFunc), -#ifndef SQLITE_OMIT_TRIGGER - FUNCTION(sqlite_rename_trigger, 2, 0, 0, renameTriggerFunc), -#endif -#ifndef SQLITE_OMIT_FOREIGN_KEY - FUNCTION(sqlite_rename_parent, 3, 0, 0, renameParentFunc), -#endif - }; - sqlite3InsertBuiltinFuncs(aAlterTableFuncs, ArraySize(aAlterTableFuncs)); -} - /* -** This function is used to create the text of expressions of the form: -** -** name= OR name= OR ... -** -** If argument zWhere is NULL, then a pointer string containing the text -** "name=" is returned, where is the quoted version -** of the string passed as argument zConstant. The returned buffer is -** allocated using sqlite3DbMalloc(). It is the responsibility of the -** caller to ensure that it is eventually freed. +** Parameter zName is the name of a table that is about to be altered +** (either with ALTER TABLE ... RENAME TO or ALTER TABLE ... ADD COLUMN). +** If the table is a system table, this function leaves an error message +** in pParse->zErr (system tables may not be altered) and returns non-zero. ** -** If argument zWhere is not NULL, then the string returned is -** " OR name=", where is the contents of zWhere. -** In this case zWhere is passed to sqlite3DbFree() before returning. -** -*/ -static char *whereOrName(sqlite3 *db, char *zWhere, char *zConstant){ - char *zNew; - if( !zWhere ){ - zNew = sqlite3MPrintf(db, "name=%Q", zConstant); - }else{ - zNew = sqlite3MPrintf(db, "%s OR name=%Q", zWhere, zConstant); - sqlite3DbFree(db, zWhere); - } - return zNew; -} - -#if !defined(SQLITE_OMIT_FOREIGN_KEY) && !defined(SQLITE_OMIT_TRIGGER) -/* -** Generate the text of a WHERE expression which can be used to select all -** tables that have foreign key constraints that refer to table pTab (i.e. -** constraints for which pTab is the parent table) from the sqlite_master -** table. +** Or, if zName is not a system table, zero is returned. */ -static char *whereForeignKeys(Parse *pParse, Table *pTab){ - FKey *p; - char *zWhere = 0; - for(p=sqlite3FkReferences(pTab); p; p=p->pNextTo){ - zWhere = whereOrName(pParse->db, zWhere, p->pFrom->zName); - } - return zWhere; -} +static int isAlterableTable(Parse *pParse, Table *pTab){ + if( 0==sqlite3StrNICmp(pTab->zName, "sqlite_", 7) +#ifndef SQLITE_OMIT_VIRTUALTABLE + || ( (pTab->tabFlags & TF_Shadow)!=0 + && sqlite3ReadOnlyShadowTables(pParse->db) + ) #endif - -/* -** Generate the text of a WHERE expression which can be used to select all -** temporary triggers on table pTab from the sqlite_temp_master table. If -** table pTab has no temporary triggers, or is itself stored in the -** temporary database, NULL is returned. -*/ -static char *whereTempTriggers(Parse *pParse, Table *pTab){ - Trigger *pTrig; - char *zWhere = 0; - const Schema *pTempSchema = pParse->db->aDb[1].pSchema; /* Temp db schema */ - - /* If the table is not located in the temp-db (in which case NULL is - ** returned, loop through the tables list of triggers. For each trigger - ** that is not part of the temp-db schema, add a clause to the WHERE - ** expression being built up in zWhere. - */ - if( pTab->pSchema!=pTempSchema ){ - sqlite3 *db = pParse->db; - for(pTrig=sqlite3TriggerList(pParse, pTab); pTrig; pTrig=pTrig->pNext){ - if( pTrig->pSchema==pTempSchema ){ - zWhere = whereOrName(db, zWhere, pTrig->zName); - } - } - } - if( zWhere ){ - char *zNew = sqlite3MPrintf(pParse->db, "type='trigger' AND (%s)", zWhere); - sqlite3DbFree(pParse->db, zWhere); - zWhere = zNew; + ){ + sqlite3ErrorMsg(pParse, "table %s may not be altered", pTab->zName); + return 1; } - return zWhere; + return 0; } /* -** Generate code to drop and reload the internal representation of table -** pTab from the database, including triggers and temporary triggers. -** Argument zName is the name of the table in the database schema at -** the time the generated code is executed. This can be different from -** pTab->zName if this function is being called to code part of an -** "ALTER TABLE RENAME TO" statement. +** Generate code to verify that the schemas of database zDb and, if +** bTemp is not true, database "temp", can still be parsed. This is +** called at the end of the generation of an ALTER TABLE ... RENAME ... +** statement to ensure that the operation has not rendered any schema +** objects unusable. */ -static void reloadTableSchema(Parse *pParse, Table *pTab, const char *zName){ - Vdbe *v; - char *zWhere; - int iDb; /* Index of database containing pTab */ -#ifndef SQLITE_OMIT_TRIGGER - Trigger *pTrig; -#endif - - v = sqlite3GetVdbe(pParse); - if( NEVER(v==0) ) return; - assert( sqlite3BtreeHoldsAllMutexes(pParse->db) ); - iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema); - assert( iDb>=0 ); - -#ifndef SQLITE_OMIT_TRIGGER - /* Drop any table triggers from the internal schema. */ - for(pTrig=sqlite3TriggerList(pParse, pTab); pTrig; pTrig=pTrig->pNext){ - int iTrigDb = sqlite3SchemaToIndex(pParse->db, pTrig->pSchema); - assert( iTrigDb==iDb || iTrigDb==1 ); - sqlite3VdbeAddOp4(v, OP_DropTrigger, iTrigDb, 0, 0, pTrig->zName, 0); - } -#endif - - /* Drop the table and index from the internal schema. */ - sqlite3VdbeAddOp4(v, OP_DropTable, iDb, 0, 0, pTab->zName, 0); - - /* Reload the table, index and permanent trigger schemas. */ - zWhere = sqlite3MPrintf(pParse->db, "tbl_name=%Q", zName); - if( !zWhere ) return; - sqlite3VdbeAddParseSchemaOp(v, iDb, zWhere); +static void renameTestSchema(Parse *pParse, const char *zDb, int bTemp){ + sqlite3NestedParse(pParse, + "SELECT 1 " + "FROM \"%w\".%s " + "WHERE name NOT LIKE 'sqliteX_%%' ESCAPE 'X'" + " AND sql NOT LIKE 'create virtual%%'" + " AND sqlite_rename_test(%Q, sql, type, name, %d)=NULL ", + zDb, MASTER_NAME, + zDb, bTemp + ); -#ifndef SQLITE_OMIT_TRIGGER - /* Now, if the table is not stored in the temp database, reload any temp - ** triggers. Don't use IN(...) in case SQLITE_OMIT_SUBQUERY is defined. - */ - if( (zWhere=whereTempTriggers(pParse, pTab))!=0 ){ - sqlite3VdbeAddParseSchemaOp(v, 1, zWhere); + if( bTemp==0 ){ + sqlite3NestedParse(pParse, + "SELECT 1 " + "FROM temp.%s " + "WHERE name NOT LIKE 'sqliteX_%%' ESCAPE 'X'" + " AND sql NOT LIKE 'create virtual%%'" + " AND sqlite_rename_test(%Q, sql, type, name, 1)=NULL ", + MASTER_NAME, zDb + ); } -#endif } /* -** Parameter zName is the name of a table that is about to be altered -** (either with ALTER TABLE ... RENAME TO or ALTER TABLE ... ADD COLUMN). -** If the table is a system table, this function leaves an error message -** in pParse->zErr (system tables may not be altered) and returns non-zero. -** -** Or, if zName is not a system table, zero is returned. +** Generate code to reload the schema for database iDb. And, if iDb!=1, for +** the temp database as well. */ -static int isSystemTable(Parse *pParse, const char *zName){ - if( 0==sqlite3StrNICmp(zName, "sqlite_", 7) ){ - sqlite3ErrorMsg(pParse, "table %s may not be altered", zName); - return 1; +static void renameReloadSchema(Parse *pParse, int iDb){ + Vdbe *v = pParse->pVdbe; + if( v ){ + sqlite3ChangeCookie(pParse, iDb); + sqlite3VdbeAddParseSchemaOp(pParse->pVdbe, iDb, 0); + if( iDb!=1 ) sqlite3VdbeAddParseSchemaOp(pParse->pVdbe, 1, 0); } - return 0; } /* @@ -100670,9 +105590,6 @@ SQLITE_PRIVATE void sqlite3AlterRenameTable( int nTabName; /* Number of UTF-8 characters in zTabName */ const char *zTabName; /* Original name of the table */ Vdbe *v; -#ifndef SQLITE_OMIT_TRIGGER - char *zWhere = 0; /* Where clause to locate temp triggers */ -#endif VTable *pVTab = 0; /* Non-zero if this is a v-tab with an xRename() */ u32 savedDbFlags; /* Saved value of db->mDbFlags */ @@ -100694,7 +105611,10 @@ SQLITE_PRIVATE void sqlite3AlterRenameTable( /* Check that a table or index named 'zName' does not already exist ** in database iDb. If so, this is an error. */ - if( sqlite3FindTable(db, zName, zDb) || sqlite3FindIndex(db, zName, zDb) ){ + if( sqlite3FindTable(db, zName, zDb) + || sqlite3FindIndex(db, zName, zDb) + || sqlite3IsShadowTableOf(db, pTab, zName) + ){ sqlite3ErrorMsg(pParse, "there is already another table or index with this name: %s", zName); goto exit_rename_table; @@ -100703,11 +105623,11 @@ SQLITE_PRIVATE void sqlite3AlterRenameTable( /* Make sure it is not a system table being altered, or a reserved name ** that the table is being renamed to. */ - if( SQLITE_OK!=isSystemTable(pParse, pTab->zName) ){ + if( SQLITE_OK!=isAlterableTable(pParse, pTab) ){ goto exit_rename_table; } - if( SQLITE_OK!=sqlite3CheckObjectName(pParse, zName) ){ goto - exit_rename_table; + if( SQLITE_OK!=sqlite3CheckObjectName(pParse,zName,"table",zName) ){ + goto exit_rename_table; } #ifndef SQLITE_OMIT_VIEW @@ -100736,74 +105656,46 @@ SQLITE_PRIVATE void sqlite3AlterRenameTable( } #endif - /* Begin a transaction for database iDb. - ** Then modify the schema cookie (since the ALTER TABLE modifies the - ** schema). Open a statement transaction if the table is a virtual - ** table. - */ + /* Begin a transaction for database iDb. Then modify the schema cookie + ** (since the ALTER TABLE modifies the schema). Call sqlite3MayAbort(), + ** as the scalar functions (e.g. sqlite_rename_table()) invoked by the + ** nested SQL may raise an exception. */ v = sqlite3GetVdbe(pParse); if( v==0 ){ goto exit_rename_table; } - sqlite3BeginWriteOperation(pParse, pVTab!=0, iDb); - sqlite3ChangeCookie(pParse, iDb); - - /* If this is a virtual table, invoke the xRename() function if - ** one is defined. The xRename() callback will modify the names - ** of any resources used by the v-table implementation (including other - ** SQLite tables) that are identified by the name of the virtual table. - */ -#ifndef SQLITE_OMIT_VIRTUALTABLE - if( pVTab ){ - int i = ++pParse->nMem; - sqlite3VdbeLoadString(v, i, zName); - sqlite3VdbeAddOp4(v, OP_VRename, i, 0, 0,(const char*)pVTab, P4_VTAB); - sqlite3MayAbort(pParse); - } -#endif + sqlite3MayAbort(pParse); /* figure out how many UTF-8 characters are in zName */ zTabName = pTab->zName; nTabName = sqlite3Utf8CharLen(zTabName, -1); -#if !defined(SQLITE_OMIT_FOREIGN_KEY) && !defined(SQLITE_OMIT_TRIGGER) - if( db->flags&SQLITE_ForeignKeys ){ - /* If foreign-key support is enabled, rewrite the CREATE TABLE - ** statements corresponding to all child tables of foreign key constraints - ** for which the renamed table is the parent table. */ - if( (zWhere=whereForeignKeys(pParse, pTab))!=0 ){ - sqlite3NestedParse(pParse, - "UPDATE \"%w\".%s SET " - "sql = sqlite_rename_parent(sql, %Q, %Q) " - "WHERE %s;", zDb, MASTER_NAME, zTabName, zName, zWhere); - sqlite3DbFree(db, zWhere); - } - } -#endif + /* Rewrite all CREATE TABLE, INDEX, TRIGGER or VIEW statements in + ** the schema to use the new table name. */ + sqlite3NestedParse(pParse, + "UPDATE \"%w\".%s SET " + "sql = sqlite_rename_table(%Q, type, name, sql, %Q, %Q, %d) " + "WHERE (type!='index' OR tbl_name=%Q COLLATE nocase)" + "AND name NOT LIKE 'sqliteX_%%' ESCAPE 'X'" + , zDb, MASTER_NAME, zDb, zTabName, zName, (iDb==1), zTabName + ); - /* Modify the sqlite_master table to use the new table name. */ + /* Update the tbl_name and name columns of the sqlite_master table + ** as required. */ sqlite3NestedParse(pParse, "UPDATE %Q.%s SET " -#ifdef SQLITE_OMIT_TRIGGER - "sql = sqlite_rename_table(sql, %Q), " -#else - "sql = CASE " - "WHEN type = 'trigger' THEN sqlite_rename_trigger(sql, %Q)" - "ELSE sqlite_rename_table(sql, %Q) END, " -#endif "tbl_name = %Q, " "name = CASE " "WHEN type='table' THEN %Q " - "WHEN name LIKE 'sqlite_autoindex%%' AND type='index' THEN " + "WHEN name LIKE 'sqliteX_autoindex%%' ESCAPE 'X' " + " AND type='index' THEN " "'sqlite_autoindex_' || %Q || substr(name,%d+18) " "ELSE name END " "WHERE tbl_name=%Q COLLATE nocase AND " "(type='table' OR type='index' OR type='trigger');", - zDb, MASTER_NAME, zName, zName, zName, -#ifndef SQLITE_OMIT_TRIGGER - zName, -#endif - zName, nTabName, zTabName + zDb, MASTER_NAME, + zName, zName, zName, + nTabName, zTabName ); #ifndef SQLITE_OMIT_AUTOINCREMENT @@ -100817,35 +105709,36 @@ SQLITE_PRIVATE void sqlite3AlterRenameTable( } #endif -#ifndef SQLITE_OMIT_TRIGGER - /* If there are TEMP triggers on this table, modify the sqlite_temp_master - ** table. Don't do this if the table being ALTERed is itself located in - ** the temp database. - */ - if( (zWhere=whereTempTriggers(pParse, pTab))!=0 ){ + /* If the table being renamed is not itself part of the temp database, + ** edit view and trigger definitions within the temp database + ** as required. */ + if( iDb!=1 ){ sqlite3NestedParse(pParse, "UPDATE sqlite_temp_master SET " - "sql = sqlite_rename_trigger(sql, %Q), " - "tbl_name = %Q " - "WHERE %s;", zName, zName, zWhere); - sqlite3DbFree(db, zWhere); + "sql = sqlite_rename_table(%Q, type, name, sql, %Q, %Q, 1), " + "tbl_name = " + "CASE WHEN tbl_name=%Q COLLATE nocase AND " + " sqlite_rename_test(%Q, sql, type, name, 1) " + "THEN %Q ELSE tbl_name END " + "WHERE type IN ('view', 'trigger')" + , zDb, zTabName, zName, zTabName, zDb, zName); } -#endif -#if !defined(SQLITE_OMIT_FOREIGN_KEY) && !defined(SQLITE_OMIT_TRIGGER) - if( db->flags&SQLITE_ForeignKeys ){ - FKey *p; - for(p=sqlite3FkReferences(pTab); p; p=p->pNextTo){ - Table *pFrom = p->pFrom; - if( pFrom!=pTab ){ - reloadTableSchema(pParse, p->pFrom, pFrom->zName); - } - } + /* If this is a virtual table, invoke the xRename() function if + ** one is defined. The xRename() callback will modify the names + ** of any resources used by the v-table implementation (including other + ** SQLite tables) that are identified by the name of the virtual table. + */ +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( pVTab ){ + int i = ++pParse->nMem; + sqlite3VdbeLoadString(v, i, zName); + sqlite3VdbeAddOp4(v, OP_VRename, i, 0, 0,(const char*)pVTab, P4_VTAB); } #endif - /* Drop and reload the internal table schema. */ - reloadTableSchema(pParse, pTab, zName); + renameReloadSchema(pParse, iDb); + renameTestSchema(pParse, zDb, iDb==1); exit_rename_table: sqlite3SrcListDelete(db, pSrc); @@ -100853,6 +105746,22 @@ SQLITE_PRIVATE void sqlite3AlterRenameTable( db->mDbFlags = savedDbFlags; } +/* +** Write code that will raise an error if the table described by +** zDb and zTab is not empty. +*/ +static void sqlite3ErrorIfNotEmpty( + Parse *pParse, /* Parsing context */ + const char *zDb, /* Schema holding the table */ + const char *zTab, /* Table to check for empty */ + const char *zErr /* Error message text */ +){ + sqlite3NestedParse(pParse, + "SELECT raise(ABORT,%Q) FROM \"%w\".\"%w\"", + zErr, zDb, zTab + ); +} + /* ** This function is called after an "ALTER TABLE ... ADD" statement ** has been parsed. Argument pColDef contains the text of the new @@ -100871,12 +105780,11 @@ SQLITE_PRIVATE void sqlite3AlterFinishAddColumn(Parse *pParse, Token *pColDef){ Column *pCol; /* The new column */ Expr *pDflt; /* Default value for the new column */ sqlite3 *db; /* The database connection; */ - Vdbe *v = pParse->pVdbe; /* The prepared statement under construction */ + Vdbe *v; /* The prepared statement under construction */ int r1; /* Temporary registers */ db = pParse->db; if( pParse->nErr || db->mallocFailed ) return; - assert( v!=0 ); pNew = pParse->pNewTable; assert( pNew ); @@ -100896,14 +105804,6 @@ SQLITE_PRIVATE void sqlite3AlterFinishAddColumn(Parse *pParse, Token *pColDef){ } #endif - /* If the default value for the new column was specified with a - ** literal NULL, then set pDflt to 0. This simplifies checking - ** for an SQL NULL default below. - */ - assert( pDflt==0 || pDflt->op==TK_SPAN ); - if( pDflt && pDflt->pLeft->op==TK_NULL ){ - pDflt = 0; - } /* Check that the new column is not specified as PRIMARY KEY or UNIQUE. ** If there is a NOT NULL constraint, then the default value for the @@ -100914,39 +105814,52 @@ SQLITE_PRIVATE void sqlite3AlterFinishAddColumn(Parse *pParse, Token *pColDef){ return; } if( pNew->pIndex ){ - sqlite3ErrorMsg(pParse, "Cannot add a UNIQUE column"); - return; - } - if( (db->flags&SQLITE_ForeignKeys) && pNew->pFKey && pDflt ){ - sqlite3ErrorMsg(pParse, - "Cannot add a REFERENCES column with non-NULL default value"); - return; - } - if( pCol->notNull && !pDflt ){ - sqlite3ErrorMsg(pParse, - "Cannot add a NOT NULL column with default value NULL"); + sqlite3ErrorMsg(pParse, + "Cannot add a UNIQUE column"); return; } - - /* Ensure the default expression is something that sqlite3ValueFromExpr() - ** can handle (i.e. not CURRENT_TIME etc.) - */ - if( pDflt ){ - sqlite3_value *pVal = 0; - int rc; - rc = sqlite3ValueFromExpr(db, pDflt, SQLITE_UTF8, SQLITE_AFF_BLOB, &pVal); - assert( rc==SQLITE_OK || rc==SQLITE_NOMEM ); - if( rc!=SQLITE_OK ){ - assert( db->mallocFailed == 1 ); - return; + if( (pCol->colFlags & COLFLAG_GENERATED)==0 ){ + /* If the default value for the new column was specified with a + ** literal NULL, then set pDflt to 0. This simplifies checking + ** for an SQL NULL default below. + */ + assert( pDflt==0 || pDflt->op==TK_SPAN ); + if( pDflt && pDflt->pLeft->op==TK_NULL ){ + pDflt = 0; } - if( !pVal ){ - sqlite3ErrorMsg(pParse, "Cannot add a column with non-constant default"); - return; + if( (db->flags&SQLITE_ForeignKeys) && pNew->pFKey && pDflt ){ + sqlite3ErrorIfNotEmpty(pParse, zDb, zTab, + "Cannot add a REFERENCES column with non-NULL default value"); + } + if( pCol->notNull && !pDflt ){ + sqlite3ErrorIfNotEmpty(pParse, zDb, zTab, + "Cannot add a NOT NULL column with default value NULL"); + } + + + /* Ensure the default expression is something that sqlite3ValueFromExpr() + ** can handle (i.e. not CURRENT_TIME etc.) + */ + if( pDflt ){ + sqlite3_value *pVal = 0; + int rc; + rc = sqlite3ValueFromExpr(db, pDflt, SQLITE_UTF8, SQLITE_AFF_BLOB, &pVal); + assert( rc==SQLITE_OK || rc==SQLITE_NOMEM ); + if( rc!=SQLITE_OK ){ + assert( db->mallocFailed == 1 ); + return; + } + if( !pVal ){ + sqlite3ErrorIfNotEmpty(pParse, zDb, zTab, + "Cannot add a column with non-constant default"); + } + sqlite3ValueFree(pVal); } - sqlite3ValueFree(pVal); + }else if( pCol->colFlags & COLFLAG_STORED ){ + sqlite3ErrorIfNotEmpty(pParse, zDb, zTab, "cannot add a STORED column"); } + /* Modify the CREATE TABLE statement. */ zCol = sqlite3DbStrNDup(db, (char*)pColDef->z, pColDef->n); if( zCol ){ @@ -100971,17 +105884,20 @@ SQLITE_PRIVATE void sqlite3AlterFinishAddColumn(Parse *pParse, Token *pColDef){ ** from less than 3 to 4, as that will corrupt any preexisting DESC ** index. */ - r1 = sqlite3GetTempReg(pParse); - sqlite3VdbeAddOp3(v, OP_ReadCookie, iDb, r1, BTREE_FILE_FORMAT); - sqlite3VdbeUsesBtree(v, iDb); - sqlite3VdbeAddOp2(v, OP_AddImm, r1, -2); - sqlite3VdbeAddOp2(v, OP_IfPos, r1, sqlite3VdbeCurrentAddr(v)+2); - VdbeCoverage(v); - sqlite3VdbeAddOp3(v, OP_SetCookie, iDb, BTREE_FILE_FORMAT, 3); - sqlite3ReleaseTempReg(pParse, r1); + v = sqlite3GetVdbe(pParse); + if( v ){ + r1 = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp3(v, OP_ReadCookie, iDb, r1, BTREE_FILE_FORMAT); + sqlite3VdbeUsesBtree(v, iDb); + sqlite3VdbeAddOp2(v, OP_AddImm, r1, -2); + sqlite3VdbeAddOp2(v, OP_IfPos, r1, sqlite3VdbeCurrentAddr(v)+2); + VdbeCoverage(v); + sqlite3VdbeAddOp3(v, OP_SetCookie, iDb, BTREE_FILE_FORMAT, 3); + sqlite3ReleaseTempReg(pParse, r1); + } - /* Reload the schema of the modified table. */ - reloadTableSchema(pParse, pTab, pTab->zName); + /* Reload the table definition */ + renameReloadSchema(pParse, iDb); } /* @@ -101002,7 +105918,6 @@ SQLITE_PRIVATE void sqlite3AlterFinishAddColumn(Parse *pParse, Token *pColDef){ SQLITE_PRIVATE void sqlite3AlterBeginAddColumn(Parse *pParse, SrcList *pSrc){ Table *pNew; Table *pTab; - Vdbe *v; int iDb; int i; int nAlloc; @@ -101027,10 +105942,11 @@ SQLITE_PRIVATE void sqlite3AlterBeginAddColumn(Parse *pParse, SrcList *pSrc){ sqlite3ErrorMsg(pParse, "Cannot add a column to a view"); goto exit_begin_add_column; } - if( SQLITE_OK!=isSystemTable(pParse, pTab->zName) ){ + if( SQLITE_OK!=isAlterableTable(pParse, pTab) ){ goto exit_begin_add_column; } + sqlite3MayAbort(pParse); assert( pTab->addColOffset>0 ); iDb = sqlite3SchemaToIndex(db, pTab->pSchema); @@ -101059,6 +105975,7 @@ SQLITE_PRIVATE void sqlite3AlterBeginAddColumn(Parse *pParse, SrcList *pSrc){ for(i=0; inCol; i++){ Column *pCol = &pNew->aCol[i]; pCol->zName = sqlite3DbStrDup(db, pCol->zName); + pCol->hName = sqlite3StrIHash(pCol->zName); pCol->zColl = 0; pCol->pDflt = 0; } @@ -101066,16 +105983,1248 @@ SQLITE_PRIVATE void sqlite3AlterBeginAddColumn(Parse *pParse, SrcList *pSrc){ pNew->addColOffset = pTab->addColOffset; pNew->nTabRef = 1; - /* Begin a transaction and increment the schema cookie. */ - sqlite3BeginWriteOperation(pParse, 0, iDb); - v = sqlite3GetVdbe(pParse); - if( !v ) goto exit_begin_add_column; - sqlite3ChangeCookie(pParse, iDb); - exit_begin_add_column: sqlite3SrcListDelete(db, pSrc); return; } + +/* +** Parameter pTab is the subject of an ALTER TABLE ... RENAME COLUMN +** command. This function checks if the table is a view or virtual +** table (columns of views or virtual tables may not be renamed). If so, +** it loads an error message into pParse and returns non-zero. +** +** Or, if pTab is not a view or virtual table, zero is returned. +*/ +#if !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_VIRTUALTABLE) +static int isRealTable(Parse *pParse, Table *pTab){ + const char *zType = 0; +#ifndef SQLITE_OMIT_VIEW + if( pTab->pSelect ){ + zType = "view"; + } +#endif +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( IsVirtual(pTab) ){ + zType = "virtual table"; + } +#endif + if( zType ){ + sqlite3ErrorMsg( + pParse, "cannot rename columns of %s \"%s\"", zType, pTab->zName + ); + return 1; + } + return 0; +} +#else /* !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_VIRTUALTABLE) */ +# define isRealTable(x,y) (0) +#endif + +/* +** Handles the following parser reduction: +** +** cmd ::= ALTER TABLE pSrc RENAME COLUMN pOld TO pNew +*/ +SQLITE_PRIVATE void sqlite3AlterRenameColumn( + Parse *pParse, /* Parsing context */ + SrcList *pSrc, /* Table being altered. pSrc->nSrc==1 */ + Token *pOld, /* Name of column being changed */ + Token *pNew /* New column name */ +){ + sqlite3 *db = pParse->db; /* Database connection */ + Table *pTab; /* Table being updated */ + int iCol; /* Index of column being renamed */ + char *zOld = 0; /* Old column name */ + char *zNew = 0; /* New column name */ + const char *zDb; /* Name of schema containing the table */ + int iSchema; /* Index of the schema */ + int bQuote; /* True to quote the new name */ + + /* Locate the table to be altered */ + pTab = sqlite3LocateTableItem(pParse, 0, &pSrc->a[0]); + if( !pTab ) goto exit_rename_column; + + /* Cannot alter a system table */ + if( SQLITE_OK!=isAlterableTable(pParse, pTab) ) goto exit_rename_column; + if( SQLITE_OK!=isRealTable(pParse, pTab) ) goto exit_rename_column; + + /* Which schema holds the table to be altered */ + iSchema = sqlite3SchemaToIndex(db, pTab->pSchema); + assert( iSchema>=0 ); + zDb = db->aDb[iSchema].zDbSName; + +#ifndef SQLITE_OMIT_AUTHORIZATION + /* Invoke the authorization callback. */ + if( sqlite3AuthCheck(pParse, SQLITE_ALTER_TABLE, zDb, pTab->zName, 0) ){ + goto exit_rename_column; + } +#endif + + /* Make sure the old name really is a column name in the table to be + ** altered. Set iCol to be the index of the column being renamed */ + zOld = sqlite3NameFromToken(db, pOld); + if( !zOld ) goto exit_rename_column; + for(iCol=0; iColnCol; iCol++){ + if( 0==sqlite3StrICmp(pTab->aCol[iCol].zName, zOld) ) break; + } + if( iCol==pTab->nCol ){ + sqlite3ErrorMsg(pParse, "no such column: \"%s\"", zOld); + goto exit_rename_column; + } + + /* Do the rename operation using a recursive UPDATE statement that + ** uses the sqlite_rename_column() SQL function to compute the new + ** CREATE statement text for the sqlite_master table. + */ + sqlite3MayAbort(pParse); + zNew = sqlite3NameFromToken(db, pNew); + if( !zNew ) goto exit_rename_column; + assert( pNew->n>0 ); + bQuote = sqlite3Isquote(pNew->z[0]); + sqlite3NestedParse(pParse, + "UPDATE \"%w\".%s SET " + "sql = sqlite_rename_column(sql, type, name, %Q, %Q, %d, %Q, %d, %d) " + "WHERE name NOT LIKE 'sqliteX_%%' ESCAPE 'X' " + " AND (type != 'index' OR tbl_name = %Q)" + " AND sql NOT LIKE 'create virtual%%'", + zDb, MASTER_NAME, + zDb, pTab->zName, iCol, zNew, bQuote, iSchema==1, + pTab->zName + ); + + sqlite3NestedParse(pParse, + "UPDATE temp.%s SET " + "sql = sqlite_rename_column(sql, type, name, %Q, %Q, %d, %Q, %d, 1) " + "WHERE type IN ('trigger', 'view')", + MASTER_NAME, + zDb, pTab->zName, iCol, zNew, bQuote + ); + + /* Drop and reload the database schema. */ + renameReloadSchema(pParse, iSchema); + renameTestSchema(pParse, zDb, iSchema==1); + + exit_rename_column: + sqlite3SrcListDelete(db, pSrc); + sqlite3DbFree(db, zOld); + sqlite3DbFree(db, zNew); + return; +} + +/* +** Each RenameToken object maps an element of the parse tree into +** the token that generated that element. The parse tree element +** might be one of: +** +** * A pointer to an Expr that represents an ID +** * The name of a table column in Column.zName +** +** A list of RenameToken objects can be constructed during parsing. +** Each new object is created by sqlite3RenameTokenMap(). +** As the parse tree is transformed, the sqlite3RenameTokenRemap() +** routine is used to keep the mapping current. +** +** After the parse finishes, renameTokenFind() routine can be used +** to look up the actual token value that created some element in +** the parse tree. +*/ +struct RenameToken { + void *p; /* Parse tree element created by token t */ + Token t; /* The token that created parse tree element p */ + RenameToken *pNext; /* Next is a list of all RenameToken objects */ +}; + +/* +** The context of an ALTER TABLE RENAME COLUMN operation that gets passed +** down into the Walker. +*/ +typedef struct RenameCtx RenameCtx; +struct RenameCtx { + RenameToken *pList; /* List of tokens to overwrite */ + int nList; /* Number of tokens in pList */ + int iCol; /* Index of column being renamed */ + Table *pTab; /* Table being ALTERed */ + const char *zOld; /* Old column name */ +}; + +#ifdef SQLITE_DEBUG +/* +** This function is only for debugging. It performs two tasks: +** +** 1. Checks that pointer pPtr does not already appear in the +** rename-token list. +** +** 2. Dereferences each pointer in the rename-token list. +** +** The second is most effective when debugging under valgrind or +** address-sanitizer or similar. If any of these pointers no longer +** point to valid objects, an exception is raised by the memory-checking +** tool. +** +** The point of this is to prevent comparisons of invalid pointer values. +** Even though this always seems to work, it is undefined according to the +** C standard. Example of undefined comparison: +** +** sqlite3_free(x); +** if( x==y ) ... +** +** Technically, as x no longer points into a valid object or to the byte +** following a valid object, it may not be used in comparison operations. +*/ +static void renameTokenCheckAll(Parse *pParse, void *pPtr){ + if( pParse->nErr==0 && pParse->db->mallocFailed==0 ){ + RenameToken *p; + u8 i = 0; + for(p=pParse->pRename; p; p=p->pNext){ + if( p->p ){ + assert( p->p!=pPtr ); + i += *(u8*)(p->p); + } + } + } +} +#else +# define renameTokenCheckAll(x,y) +#endif + +/* +** Remember that the parser tree element pPtr was created using +** the token pToken. +** +** In other words, construct a new RenameToken object and add it +** to the list of RenameToken objects currently being built up +** in pParse->pRename. +** +** The pPtr argument is returned so that this routine can be used +** with tail recursion in tokenExpr() routine, for a small performance +** improvement. +*/ +SQLITE_PRIVATE void *sqlite3RenameTokenMap(Parse *pParse, void *pPtr, Token *pToken){ + RenameToken *pNew; + assert( pPtr || pParse->db->mallocFailed ); + renameTokenCheckAll(pParse, pPtr); + if( ALWAYS(pParse->eParseMode!=PARSE_MODE_UNMAP) ){ + pNew = sqlite3DbMallocZero(pParse->db, sizeof(RenameToken)); + if( pNew ){ + pNew->p = pPtr; + pNew->t = *pToken; + pNew->pNext = pParse->pRename; + pParse->pRename = pNew; + } + } + + return pPtr; +} + +/* +** It is assumed that there is already a RenameToken object associated +** with parse tree element pFrom. This function remaps the associated token +** to parse tree element pTo. +*/ +SQLITE_PRIVATE void sqlite3RenameTokenRemap(Parse *pParse, void *pTo, void *pFrom){ + RenameToken *p; + renameTokenCheckAll(pParse, pTo); + for(p=pParse->pRename; p; p=p->pNext){ + if( p->p==pFrom ){ + p->p = pTo; + break; + } + } +} + +/* +** Walker callback used by sqlite3RenameExprUnmap(). +*/ +static int renameUnmapExprCb(Walker *pWalker, Expr *pExpr){ + Parse *pParse = pWalker->pParse; + sqlite3RenameTokenRemap(pParse, 0, (void*)pExpr); + return WRC_Continue; +} + +/* +** Iterate through the Select objects that are part of WITH clauses attached +** to select statement pSelect. +*/ +static void renameWalkWith(Walker *pWalker, Select *pSelect){ + With *pWith = pSelect->pWith; + if( pWith ){ + int i; + for(i=0; inCte; i++){ + Select *p = pWith->a[i].pSelect; + NameContext sNC; + memset(&sNC, 0, sizeof(sNC)); + sNC.pParse = pWalker->pParse; + sqlite3SelectPrep(sNC.pParse, p, &sNC); + sqlite3WalkSelect(pWalker, p); + sqlite3RenameExprlistUnmap(pWalker->pParse, pWith->a[i].pCols); + } + } +} + +/* +** Unmap all tokens in the IdList object passed as the second argument. +*/ +static void unmapColumnIdlistNames( + Parse *pParse, + IdList *pIdList +){ + if( pIdList ){ + int ii; + for(ii=0; iinId; ii++){ + sqlite3RenameTokenRemap(pParse, 0, (void*)pIdList->a[ii].zName); + } + } +} + +/* +** Walker callback used by sqlite3RenameExprUnmap(). +*/ +static int renameUnmapSelectCb(Walker *pWalker, Select *p){ + Parse *pParse = pWalker->pParse; + int i; + if( pParse->nErr ) return WRC_Abort; + if( NEVER(p->selFlags & SF_View) ) return WRC_Prune; + if( ALWAYS(p->pEList) ){ + ExprList *pList = p->pEList; + for(i=0; inExpr; i++){ + if( pList->a[i].zEName && pList->a[i].eEName==ENAME_NAME ){ + sqlite3RenameTokenRemap(pParse, 0, (void*)pList->a[i].zEName); + } + } + } + if( ALWAYS(p->pSrc) ){ /* Every Select as a SrcList, even if it is empty */ + SrcList *pSrc = p->pSrc; + for(i=0; inSrc; i++){ + sqlite3RenameTokenRemap(pParse, 0, (void*)pSrc->a[i].zName); + if( sqlite3WalkExpr(pWalker, pSrc->a[i].pOn) ) return WRC_Abort; + unmapColumnIdlistNames(pParse, pSrc->a[i].pUsing); + } + } + + renameWalkWith(pWalker, p); + return WRC_Continue; +} + +/* +** Remove all nodes that are part of expression pExpr from the rename list. +*/ +SQLITE_PRIVATE void sqlite3RenameExprUnmap(Parse *pParse, Expr *pExpr){ + u8 eMode = pParse->eParseMode; + Walker sWalker; + memset(&sWalker, 0, sizeof(Walker)); + sWalker.pParse = pParse; + sWalker.xExprCallback = renameUnmapExprCb; + sWalker.xSelectCallback = renameUnmapSelectCb; + pParse->eParseMode = PARSE_MODE_UNMAP; + sqlite3WalkExpr(&sWalker, pExpr); + pParse->eParseMode = eMode; +} + +/* +** Remove all nodes that are part of expression-list pEList from the +** rename list. +*/ +SQLITE_PRIVATE void sqlite3RenameExprlistUnmap(Parse *pParse, ExprList *pEList){ + if( pEList ){ + int i; + Walker sWalker; + memset(&sWalker, 0, sizeof(Walker)); + sWalker.pParse = pParse; + sWalker.xExprCallback = renameUnmapExprCb; + sqlite3WalkExprList(&sWalker, pEList); + for(i=0; inExpr; i++){ + if( ALWAYS(pEList->a[i].eEName==ENAME_NAME) ){ + sqlite3RenameTokenRemap(pParse, 0, (void*)pEList->a[i].zEName); + } + } + } +} + +/* +** Free the list of RenameToken objects given in the second argument +*/ +static void renameTokenFree(sqlite3 *db, RenameToken *pToken){ + RenameToken *pNext; + RenameToken *p; + for(p=pToken; p; p=pNext){ + pNext = p->pNext; + sqlite3DbFree(db, p); + } +} + +/* +** Search the Parse object passed as the first argument for a RenameToken +** object associated with parse tree element pPtr. If found, remove it +** from the Parse object and add it to the list maintained by the +** RenameCtx object passed as the second argument. +*/ +static void renameTokenFind(Parse *pParse, struct RenameCtx *pCtx, void *pPtr){ + RenameToken **pp; + assert( pPtr!=0 ); + for(pp=&pParse->pRename; (*pp); pp=&(*pp)->pNext){ + if( (*pp)->p==pPtr ){ + RenameToken *pToken = *pp; + *pp = pToken->pNext; + pToken->pNext = pCtx->pList; + pCtx->pList = pToken; + pCtx->nList++; + break; + } + } +} + +/* +** This is a Walker select callback. It does nothing. It is only required +** because without a dummy callback, sqlite3WalkExpr() and similar do not +** descend into sub-select statements. +*/ +static int renameColumnSelectCb(Walker *pWalker, Select *p){ + if( p->selFlags & SF_View ) return WRC_Prune; + renameWalkWith(pWalker, p); + return WRC_Continue; +} + +/* +** This is a Walker expression callback. +** +** For every TK_COLUMN node in the expression tree, search to see +** if the column being references is the column being renamed by an +** ALTER TABLE statement. If it is, then attach its associated +** RenameToken object to the list of RenameToken objects being +** constructed in RenameCtx object at pWalker->u.pRename. +*/ +static int renameColumnExprCb(Walker *pWalker, Expr *pExpr){ + RenameCtx *p = pWalker->u.pRename; + if( pExpr->op==TK_TRIGGER + && pExpr->iColumn==p->iCol + && pWalker->pParse->pTriggerTab==p->pTab + ){ + renameTokenFind(pWalker->pParse, p, (void*)pExpr); + }else if( pExpr->op==TK_COLUMN + && pExpr->iColumn==p->iCol + && p->pTab==pExpr->y.pTab + ){ + renameTokenFind(pWalker->pParse, p, (void*)pExpr); + } + return WRC_Continue; +} + +/* +** The RenameCtx contains a list of tokens that reference a column that +** is being renamed by an ALTER TABLE statement. Return the "last" +** RenameToken in the RenameCtx and remove that RenameToken from the +** RenameContext. "Last" means the last RenameToken encountered when +** the input SQL is parsed from left to right. Repeated calls to this routine +** return all column name tokens in the order that they are encountered +** in the SQL statement. +*/ +static RenameToken *renameColumnTokenNext(RenameCtx *pCtx){ + RenameToken *pBest = pCtx->pList; + RenameToken *pToken; + RenameToken **pp; + + for(pToken=pBest->pNext; pToken; pToken=pToken->pNext){ + if( pToken->t.z>pBest->t.z ) pBest = pToken; + } + for(pp=&pCtx->pList; *pp!=pBest; pp=&(*pp)->pNext); + *pp = pBest->pNext; + + return pBest; +} + +/* +** An error occured while parsing or otherwise processing a database +** object (either pParse->pNewTable, pNewIndex or pNewTrigger) as part of an +** ALTER TABLE RENAME COLUMN program. The error message emitted by the +** sub-routine is currently stored in pParse->zErrMsg. This function +** adds context to the error message and then stores it in pCtx. +*/ +static void renameColumnParseError( + sqlite3_context *pCtx, + int bPost, + sqlite3_value *pType, + sqlite3_value *pObject, + Parse *pParse +){ + const char *zT = (const char*)sqlite3_value_text(pType); + const char *zN = (const char*)sqlite3_value_text(pObject); + char *zErr; + + zErr = sqlite3_mprintf("error in %s %s%s: %s", + zT, zN, (bPost ? " after rename" : ""), + pParse->zErrMsg + ); + sqlite3_result_error(pCtx, zErr, -1); + sqlite3_free(zErr); +} + +/* +** For each name in the the expression-list pEList (i.e. each +** pEList->a[i].zName) that matches the string in zOld, extract the +** corresponding rename-token from Parse object pParse and add it +** to the RenameCtx pCtx. +*/ +static void renameColumnElistNames( + Parse *pParse, + RenameCtx *pCtx, + ExprList *pEList, + const char *zOld +){ + if( pEList ){ + int i; + for(i=0; inExpr; i++){ + char *zName = pEList->a[i].zEName; + if( ALWAYS(pEList->a[i].eEName==ENAME_NAME) + && ALWAYS(zName!=0) + && 0==sqlite3_stricmp(zName, zOld) + ){ + renameTokenFind(pParse, pCtx, (void*)zName); + } + } + } +} + +/* +** For each name in the the id-list pIdList (i.e. each pIdList->a[i].zName) +** that matches the string in zOld, extract the corresponding rename-token +** from Parse object pParse and add it to the RenameCtx pCtx. +*/ +static void renameColumnIdlistNames( + Parse *pParse, + RenameCtx *pCtx, + IdList *pIdList, + const char *zOld +){ + if( pIdList ){ + int i; + for(i=0; inId; i++){ + char *zName = pIdList->a[i].zName; + if( 0==sqlite3_stricmp(zName, zOld) ){ + renameTokenFind(pParse, pCtx, (void*)zName); + } + } + } +} + + +/* +** Parse the SQL statement zSql using Parse object (*p). The Parse object +** is initialized by this function before it is used. +*/ +static int renameParseSql( + Parse *p, /* Memory to use for Parse object */ + const char *zDb, /* Name of schema SQL belongs to */ + sqlite3 *db, /* Database handle */ + const char *zSql, /* SQL to parse */ + int bTemp /* True if SQL is from temp schema */ +){ + int rc; + char *zErr = 0; + + db->init.iDb = bTemp ? 1 : sqlite3FindDbName(db, zDb); + + /* Parse the SQL statement passed as the first argument. If no error + ** occurs and the parse does not result in a new table, index or + ** trigger object, the database must be corrupt. */ + memset(p, 0, sizeof(Parse)); + p->eParseMode = PARSE_MODE_RENAME; + p->db = db; + p->nQueryLoop = 1; + rc = sqlite3RunParser(p, zSql, &zErr); + assert( p->zErrMsg==0 ); + assert( rc!=SQLITE_OK || zErr==0 ); + p->zErrMsg = zErr; + if( db->mallocFailed ) rc = SQLITE_NOMEM; + if( rc==SQLITE_OK + && p->pNewTable==0 && p->pNewIndex==0 && p->pNewTrigger==0 + ){ + rc = SQLITE_CORRUPT_BKPT; + } + +#ifdef SQLITE_DEBUG + /* Ensure that all mappings in the Parse.pRename list really do map to + ** a part of the input string. */ + if( rc==SQLITE_OK ){ + int nSql = sqlite3Strlen30(zSql); + RenameToken *pToken; + for(pToken=p->pRename; pToken; pToken=pToken->pNext){ + assert( pToken->t.z>=zSql && &pToken->t.z[pToken->t.n]<=&zSql[nSql] ); + } + } +#endif + + db->init.iDb = 0; + return rc; +} + +/* +** This function edits SQL statement zSql, replacing each token identified +** by the linked list pRename with the text of zNew. If argument bQuote is +** true, then zNew is always quoted first. If no error occurs, the result +** is loaded into context object pCtx as the result. +** +** Or, if an error occurs (i.e. an OOM condition), an error is left in +** pCtx and an SQLite error code returned. +*/ +static int renameEditSql( + sqlite3_context *pCtx, /* Return result here */ + RenameCtx *pRename, /* Rename context */ + const char *zSql, /* SQL statement to edit */ + const char *zNew, /* New token text */ + int bQuote /* True to always quote token */ +){ + int nNew = sqlite3Strlen30(zNew); + int nSql = sqlite3Strlen30(zSql); + sqlite3 *db = sqlite3_context_db_handle(pCtx); + int rc = SQLITE_OK; + char *zQuot; + char *zOut; + int nQuot; + + /* Set zQuot to point to a buffer containing a quoted copy of the + ** identifier zNew. If the corresponding identifier in the original + ** ALTER TABLE statement was quoted (bQuote==1), then set zNew to + ** point to zQuot so that all substitutions are made using the + ** quoted version of the new column name. */ + zQuot = sqlite3MPrintf(db, "\"%w\"", zNew); + if( zQuot==0 ){ + return SQLITE_NOMEM; + }else{ + nQuot = sqlite3Strlen30(zQuot); + } + if( bQuote ){ + zNew = zQuot; + nNew = nQuot; + } + + /* At this point pRename->pList contains a list of RenameToken objects + ** corresponding to all tokens in the input SQL that must be replaced + ** with the new column name. All that remains is to construct and + ** return the edited SQL string. */ + assert( nQuot>=nNew ); + zOut = sqlite3DbMallocZero(db, nSql + pRename->nList*nQuot + 1); + if( zOut ){ + int nOut = nSql; + memcpy(zOut, zSql, nSql); + while( pRename->pList ){ + int iOff; /* Offset of token to replace in zOut */ + RenameToken *pBest = renameColumnTokenNext(pRename); + + u32 nReplace; + const char *zReplace; + if( sqlite3IsIdChar(*pBest->t.z) ){ + nReplace = nNew; + zReplace = zNew; + }else{ + nReplace = nQuot; + zReplace = zQuot; + } + + iOff = pBest->t.z - zSql; + if( pBest->t.n!=nReplace ){ + memmove(&zOut[iOff + nReplace], &zOut[iOff + pBest->t.n], + nOut - (iOff + pBest->t.n) + ); + nOut += nReplace - pBest->t.n; + zOut[nOut] = '\0'; + } + memcpy(&zOut[iOff], zReplace, nReplace); + sqlite3DbFree(db, pBest); + } + + sqlite3_result_text(pCtx, zOut, -1, SQLITE_TRANSIENT); + sqlite3DbFree(db, zOut); + }else{ + rc = SQLITE_NOMEM; + } + + sqlite3_free(zQuot); + return rc; +} + +/* +** Resolve all symbols in the trigger at pParse->pNewTrigger, assuming +** it was read from the schema of database zDb. Return SQLITE_OK if +** successful. Otherwise, return an SQLite error code and leave an error +** message in the Parse object. +*/ +static int renameResolveTrigger(Parse *pParse, const char *zDb){ + sqlite3 *db = pParse->db; + Trigger *pNew = pParse->pNewTrigger; + TriggerStep *pStep; + NameContext sNC; + int rc = SQLITE_OK; + + memset(&sNC, 0, sizeof(sNC)); + sNC.pParse = pParse; + assert( pNew->pTabSchema ); + pParse->pTriggerTab = sqlite3FindTable(db, pNew->table, + db->aDb[sqlite3SchemaToIndex(db, pNew->pTabSchema)].zDbSName + ); + pParse->eTriggerOp = pNew->op; + /* ALWAYS() because if the table of the trigger does not exist, the + ** error would have been hit before this point */ + if( ALWAYS(pParse->pTriggerTab) ){ + rc = sqlite3ViewGetColumnNames(pParse, pParse->pTriggerTab); + } + + /* Resolve symbols in WHEN clause */ + if( rc==SQLITE_OK && pNew->pWhen ){ + rc = sqlite3ResolveExprNames(&sNC, pNew->pWhen); + } + + for(pStep=pNew->step_list; rc==SQLITE_OK && pStep; pStep=pStep->pNext){ + if( pStep->pSelect ){ + sqlite3SelectPrep(pParse, pStep->pSelect, &sNC); + if( pParse->nErr ) rc = pParse->rc; + } + if( rc==SQLITE_OK && pStep->zTarget ){ + Table *pTarget = sqlite3LocateTable(pParse, 0, pStep->zTarget, zDb); + if( pTarget==0 ){ + rc = SQLITE_ERROR; + }else if( SQLITE_OK==(rc = sqlite3ViewGetColumnNames(pParse, pTarget)) ){ + SrcList sSrc; + memset(&sSrc, 0, sizeof(sSrc)); + sSrc.nSrc = 1; + sSrc.a[0].zName = pStep->zTarget; + sSrc.a[0].pTab = pTarget; + sNC.pSrcList = &sSrc; + if( pStep->pWhere ){ + rc = sqlite3ResolveExprNames(&sNC, pStep->pWhere); + } + if( rc==SQLITE_OK ){ + rc = sqlite3ResolveExprListNames(&sNC, pStep->pExprList); + } + assert( !pStep->pUpsert || (!pStep->pWhere && !pStep->pExprList) ); + if( pStep->pUpsert ){ + Upsert *pUpsert = pStep->pUpsert; + assert( rc==SQLITE_OK ); + pUpsert->pUpsertSrc = &sSrc; + sNC.uNC.pUpsert = pUpsert; + sNC.ncFlags = NC_UUpsert; + rc = sqlite3ResolveExprListNames(&sNC, pUpsert->pUpsertTarget); + if( rc==SQLITE_OK ){ + ExprList *pUpsertSet = pUpsert->pUpsertSet; + rc = sqlite3ResolveExprListNames(&sNC, pUpsertSet); + } + if( rc==SQLITE_OK ){ + rc = sqlite3ResolveExprNames(&sNC, pUpsert->pUpsertWhere); + } + if( rc==SQLITE_OK ){ + rc = sqlite3ResolveExprNames(&sNC, pUpsert->pUpsertTargetWhere); + } + sNC.ncFlags = 0; + } + sNC.pSrcList = 0; + } + } + } + return rc; +} + +/* +** Invoke sqlite3WalkExpr() or sqlite3WalkSelect() on all Select or Expr +** objects that are part of the trigger passed as the second argument. +*/ +static void renameWalkTrigger(Walker *pWalker, Trigger *pTrigger){ + TriggerStep *pStep; + + /* Find tokens to edit in WHEN clause */ + sqlite3WalkExpr(pWalker, pTrigger->pWhen); + + /* Find tokens to edit in trigger steps */ + for(pStep=pTrigger->step_list; pStep; pStep=pStep->pNext){ + sqlite3WalkSelect(pWalker, pStep->pSelect); + sqlite3WalkExpr(pWalker, pStep->pWhere); + sqlite3WalkExprList(pWalker, pStep->pExprList); + if( pStep->pUpsert ){ + Upsert *pUpsert = pStep->pUpsert; + sqlite3WalkExprList(pWalker, pUpsert->pUpsertTarget); + sqlite3WalkExprList(pWalker, pUpsert->pUpsertSet); + sqlite3WalkExpr(pWalker, pUpsert->pUpsertWhere); + sqlite3WalkExpr(pWalker, pUpsert->pUpsertTargetWhere); + } + } +} + +/* +** Free the contents of Parse object (*pParse). Do not free the memory +** occupied by the Parse object itself. +*/ +static void renameParseCleanup(Parse *pParse){ + sqlite3 *db = pParse->db; + Index *pIdx; + if( pParse->pVdbe ){ + sqlite3VdbeFinalize(pParse->pVdbe); + } + sqlite3DeleteTable(db, pParse->pNewTable); + while( (pIdx = pParse->pNewIndex)!=0 ){ + pParse->pNewIndex = pIdx->pNext; + sqlite3FreeIndex(db, pIdx); + } + sqlite3DeleteTrigger(db, pParse->pNewTrigger); + sqlite3DbFree(db, pParse->zErrMsg); + renameTokenFree(db, pParse->pRename); + sqlite3ParserReset(pParse); +} + +/* +** SQL function: +** +** sqlite_rename_column(zSql, iCol, bQuote, zNew, zTable, zOld) +** +** 0. zSql: SQL statement to rewrite +** 1. type: Type of object ("table", "view" etc.) +** 2. object: Name of object +** 3. Database: Database name (e.g. "main") +** 4. Table: Table name +** 5. iCol: Index of column to rename +** 6. zNew: New column name +** 7. bQuote: Non-zero if the new column name should be quoted. +** 8. bTemp: True if zSql comes from temp schema +** +** Do a column rename operation on the CREATE statement given in zSql. +** The iCol-th column (left-most is 0) of table zTable is renamed from zCol +** into zNew. The name should be quoted if bQuote is true. +** +** This function is used internally by the ALTER TABLE RENAME COLUMN command. +** It is only accessible to SQL created using sqlite3NestedParse(). It is +** not reachable from ordinary SQL passed into sqlite3_prepare(). +*/ +static void renameColumnFunc( + sqlite3_context *context, + int NotUsed, + sqlite3_value **argv +){ + sqlite3 *db = sqlite3_context_db_handle(context); + RenameCtx sCtx; + const char *zSql = (const char*)sqlite3_value_text(argv[0]); + const char *zDb = (const char*)sqlite3_value_text(argv[3]); + const char *zTable = (const char*)sqlite3_value_text(argv[4]); + int iCol = sqlite3_value_int(argv[5]); + const char *zNew = (const char*)sqlite3_value_text(argv[6]); + int bQuote = sqlite3_value_int(argv[7]); + int bTemp = sqlite3_value_int(argv[8]); + const char *zOld; + int rc; + Parse sParse; + Walker sWalker; + Index *pIdx; + int i; + Table *pTab; +#ifndef SQLITE_OMIT_AUTHORIZATION + sqlite3_xauth xAuth = db->xAuth; +#endif + + UNUSED_PARAMETER(NotUsed); + if( zSql==0 ) return; + if( zTable==0 ) return; + if( zNew==0 ) return; + if( iCol<0 ) return; + sqlite3BtreeEnterAll(db); + pTab = sqlite3FindTable(db, zTable, zDb); + if( pTab==0 || iCol>=pTab->nCol ){ + sqlite3BtreeLeaveAll(db); + return; + } + zOld = pTab->aCol[iCol].zName; + memset(&sCtx, 0, sizeof(sCtx)); + sCtx.iCol = ((iCol==pTab->iPKey) ? -1 : iCol); + +#ifndef SQLITE_OMIT_AUTHORIZATION + db->xAuth = 0; +#endif + rc = renameParseSql(&sParse, zDb, db, zSql, bTemp); + + /* Find tokens that need to be replaced. */ + memset(&sWalker, 0, sizeof(Walker)); + sWalker.pParse = &sParse; + sWalker.xExprCallback = renameColumnExprCb; + sWalker.xSelectCallback = renameColumnSelectCb; + sWalker.u.pRename = &sCtx; + + sCtx.pTab = pTab; + if( rc!=SQLITE_OK ) goto renameColumnFunc_done; + if( sParse.pNewTable ){ + Select *pSelect = sParse.pNewTable->pSelect; + if( pSelect ){ + pSelect->selFlags &= ~SF_View; + sParse.rc = SQLITE_OK; + sqlite3SelectPrep(&sParse, pSelect, 0); + rc = (db->mallocFailed ? SQLITE_NOMEM : sParse.rc); + if( rc==SQLITE_OK ){ + sqlite3WalkSelect(&sWalker, pSelect); + } + if( rc!=SQLITE_OK ) goto renameColumnFunc_done; + }else{ + /* A regular table */ + int bFKOnly = sqlite3_stricmp(zTable, sParse.pNewTable->zName); + FKey *pFKey; + assert( sParse.pNewTable->pSelect==0 ); + sCtx.pTab = sParse.pNewTable; + if( bFKOnly==0 ){ + renameTokenFind( + &sParse, &sCtx, (void*)sParse.pNewTable->aCol[iCol].zName + ); + if( sCtx.iCol<0 ){ + renameTokenFind(&sParse, &sCtx, (void*)&sParse.pNewTable->iPKey); + } + sqlite3WalkExprList(&sWalker, sParse.pNewTable->pCheck); + for(pIdx=sParse.pNewTable->pIndex; pIdx; pIdx=pIdx->pNext){ + sqlite3WalkExprList(&sWalker, pIdx->aColExpr); + } + for(pIdx=sParse.pNewIndex; pIdx; pIdx=pIdx->pNext){ + sqlite3WalkExprList(&sWalker, pIdx->aColExpr); + } + } +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + for(i=0; inCol; i++){ + sqlite3WalkExpr(&sWalker, sParse.pNewTable->aCol[i].pDflt); + } +#endif + + for(pFKey=sParse.pNewTable->pFKey; pFKey; pFKey=pFKey->pNextFrom){ + for(i=0; inCol; i++){ + if( bFKOnly==0 && pFKey->aCol[i].iFrom==iCol ){ + renameTokenFind(&sParse, &sCtx, (void*)&pFKey->aCol[i]); + } + if( 0==sqlite3_stricmp(pFKey->zTo, zTable) + && 0==sqlite3_stricmp(pFKey->aCol[i].zCol, zOld) + ){ + renameTokenFind(&sParse, &sCtx, (void*)pFKey->aCol[i].zCol); + } + } + } + } + }else if( sParse.pNewIndex ){ + sqlite3WalkExprList(&sWalker, sParse.pNewIndex->aColExpr); + sqlite3WalkExpr(&sWalker, sParse.pNewIndex->pPartIdxWhere); + }else{ + /* A trigger */ + TriggerStep *pStep; + rc = renameResolveTrigger(&sParse, (bTemp ? 0 : zDb)); + if( rc!=SQLITE_OK ) goto renameColumnFunc_done; + + for(pStep=sParse.pNewTrigger->step_list; pStep; pStep=pStep->pNext){ + if( pStep->zTarget ){ + Table *pTarget = sqlite3LocateTable(&sParse, 0, pStep->zTarget, zDb); + if( pTarget==pTab ){ + if( pStep->pUpsert ){ + ExprList *pUpsertSet = pStep->pUpsert->pUpsertSet; + renameColumnElistNames(&sParse, &sCtx, pUpsertSet, zOld); + } + renameColumnIdlistNames(&sParse, &sCtx, pStep->pIdList, zOld); + renameColumnElistNames(&sParse, &sCtx, pStep->pExprList, zOld); + } + } + } + + + /* Find tokens to edit in UPDATE OF clause */ + if( sParse.pTriggerTab==pTab ){ + renameColumnIdlistNames(&sParse, &sCtx,sParse.pNewTrigger->pColumns,zOld); + } + + /* Find tokens to edit in various expressions and selects */ + renameWalkTrigger(&sWalker, sParse.pNewTrigger); + } + + assert( rc==SQLITE_OK ); + rc = renameEditSql(context, &sCtx, zSql, zNew, bQuote); + +renameColumnFunc_done: + if( rc!=SQLITE_OK ){ + if( sParse.zErrMsg ){ + renameColumnParseError(context, 0, argv[1], argv[2], &sParse); + }else{ + sqlite3_result_error_code(context, rc); + } + } + + renameParseCleanup(&sParse); + renameTokenFree(db, sCtx.pList); +#ifndef SQLITE_OMIT_AUTHORIZATION + db->xAuth = xAuth; +#endif + sqlite3BtreeLeaveAll(db); +} + +/* +** Walker expression callback used by "RENAME TABLE". +*/ +static int renameTableExprCb(Walker *pWalker, Expr *pExpr){ + RenameCtx *p = pWalker->u.pRename; + if( pExpr->op==TK_COLUMN && p->pTab==pExpr->y.pTab ){ + renameTokenFind(pWalker->pParse, p, (void*)&pExpr->y.pTab); + } + return WRC_Continue; +} + +/* +** Walker select callback used by "RENAME TABLE". +*/ +static int renameTableSelectCb(Walker *pWalker, Select *pSelect){ + int i; + RenameCtx *p = pWalker->u.pRename; + SrcList *pSrc = pSelect->pSrc; + if( pSelect->selFlags & SF_View ) return WRC_Prune; + if( pSrc==0 ){ + assert( pWalker->pParse->db->mallocFailed ); + return WRC_Abort; + } + for(i=0; inSrc; i++){ + struct SrcList_item *pItem = &pSrc->a[i]; + if( pItem->pTab==p->pTab ){ + renameTokenFind(pWalker->pParse, p, pItem->zName); + } + } + renameWalkWith(pWalker, pSelect); + + return WRC_Continue; +} + + +/* +** This C function implements an SQL user function that is used by SQL code +** generated by the ALTER TABLE ... RENAME command to modify the definition +** of any foreign key constraints that use the table being renamed as the +** parent table. It is passed three arguments: +** +** 0: The database containing the table being renamed. +** 1. type: Type of object ("table", "view" etc.) +** 2. object: Name of object +** 3: The complete text of the schema statement being modified, +** 4: The old name of the table being renamed, and +** 5: The new name of the table being renamed. +** 6: True if the schema statement comes from the temp db. +** +** It returns the new schema statement. For example: +** +** sqlite_rename_table('main', 'CREATE TABLE t1(a REFERENCES t2)','t2','t3',0) +** -> 'CREATE TABLE t1(a REFERENCES t3)' +*/ +static void renameTableFunc( + sqlite3_context *context, + int NotUsed, + sqlite3_value **argv +){ + sqlite3 *db = sqlite3_context_db_handle(context); + const char *zDb = (const char*)sqlite3_value_text(argv[0]); + const char *zInput = (const char*)sqlite3_value_text(argv[3]); + const char *zOld = (const char*)sqlite3_value_text(argv[4]); + const char *zNew = (const char*)sqlite3_value_text(argv[5]); + int bTemp = sqlite3_value_int(argv[6]); + UNUSED_PARAMETER(NotUsed); + + if( zInput && zOld && zNew ){ + Parse sParse; + int rc; + int bQuote = 1; + RenameCtx sCtx; + Walker sWalker; + +#ifndef SQLITE_OMIT_AUTHORIZATION + sqlite3_xauth xAuth = db->xAuth; + db->xAuth = 0; +#endif + + sqlite3BtreeEnterAll(db); + + memset(&sCtx, 0, sizeof(RenameCtx)); + sCtx.pTab = sqlite3FindTable(db, zOld, zDb); + memset(&sWalker, 0, sizeof(Walker)); + sWalker.pParse = &sParse; + sWalker.xExprCallback = renameTableExprCb; + sWalker.xSelectCallback = renameTableSelectCb; + sWalker.u.pRename = &sCtx; + + rc = renameParseSql(&sParse, zDb, db, zInput, bTemp); + + if( rc==SQLITE_OK ){ + int isLegacy = (db->flags & SQLITE_LegacyAlter); + if( sParse.pNewTable ){ + Table *pTab = sParse.pNewTable; + + if( pTab->pSelect ){ + if( isLegacy==0 ){ + Select *pSelect = pTab->pSelect; + NameContext sNC; + memset(&sNC, 0, sizeof(sNC)); + sNC.pParse = &sParse; + + assert( pSelect->selFlags & SF_View ); + pSelect->selFlags &= ~SF_View; + sqlite3SelectPrep(&sParse, pTab->pSelect, &sNC); + if( sParse.nErr ){ + rc = sParse.rc; + }else{ + sqlite3WalkSelect(&sWalker, pTab->pSelect); + } + } + }else{ + /* Modify any FK definitions to point to the new table. */ +#ifndef SQLITE_OMIT_FOREIGN_KEY + if( isLegacy==0 || (db->flags & SQLITE_ForeignKeys) ){ + FKey *pFKey; + for(pFKey=pTab->pFKey; pFKey; pFKey=pFKey->pNextFrom){ + if( sqlite3_stricmp(pFKey->zTo, zOld)==0 ){ + renameTokenFind(&sParse, &sCtx, (void*)pFKey->zTo); + } + } + } +#endif + + /* If this is the table being altered, fix any table refs in CHECK + ** expressions. Also update the name that appears right after the + ** "CREATE [VIRTUAL] TABLE" bit. */ + if( sqlite3_stricmp(zOld, pTab->zName)==0 ){ + sCtx.pTab = pTab; + if( isLegacy==0 ){ + sqlite3WalkExprList(&sWalker, pTab->pCheck); + } + renameTokenFind(&sParse, &sCtx, pTab->zName); + } + } + } + + else if( sParse.pNewIndex ){ + renameTokenFind(&sParse, &sCtx, sParse.pNewIndex->zName); + if( isLegacy==0 ){ + sqlite3WalkExpr(&sWalker, sParse.pNewIndex->pPartIdxWhere); + } + } + +#ifndef SQLITE_OMIT_TRIGGER + else{ + Trigger *pTrigger = sParse.pNewTrigger; + TriggerStep *pStep; + if( 0==sqlite3_stricmp(sParse.pNewTrigger->table, zOld) + && sCtx.pTab->pSchema==pTrigger->pTabSchema + ){ + renameTokenFind(&sParse, &sCtx, sParse.pNewTrigger->table); + } + + if( isLegacy==0 ){ + rc = renameResolveTrigger(&sParse, bTemp ? 0 : zDb); + if( rc==SQLITE_OK ){ + renameWalkTrigger(&sWalker, pTrigger); + for(pStep=pTrigger->step_list; pStep; pStep=pStep->pNext){ + if( pStep->zTarget && 0==sqlite3_stricmp(pStep->zTarget, zOld) ){ + renameTokenFind(&sParse, &sCtx, pStep->zTarget); + } + } + } + } + } +#endif + } + + if( rc==SQLITE_OK ){ + rc = renameEditSql(context, &sCtx, zInput, zNew, bQuote); + } + if( rc!=SQLITE_OK ){ + if( sParse.zErrMsg ){ + renameColumnParseError(context, 0, argv[1], argv[2], &sParse); + }else{ + sqlite3_result_error_code(context, rc); + } + } + + renameParseCleanup(&sParse); + renameTokenFree(db, sCtx.pList); + sqlite3BtreeLeaveAll(db); +#ifndef SQLITE_OMIT_AUTHORIZATION + db->xAuth = xAuth; +#endif + } + + return; +} + +/* +** An SQL user function that checks that there are no parse or symbol +** resolution problems in a CREATE TRIGGER|TABLE|VIEW|INDEX statement. +** After an ALTER TABLE .. RENAME operation is performed and the schema +** reloaded, this function is called on each SQL statement in the schema +** to ensure that it is still usable. +** +** 0: Database name ("main", "temp" etc.). +** 1: SQL statement. +** 2: Object type ("view", "table", "trigger" or "index"). +** 3: Object name. +** 4: True if object is from temp schema. +** +** Unless it finds an error, this function normally returns NULL. However, it +** returns integer value 1 if: +** +** * the SQL argument creates a trigger, and +** * the table that the trigger is attached to is in database zDb. +*/ +static void renameTableTest( + sqlite3_context *context, + int NotUsed, + sqlite3_value **argv +){ + sqlite3 *db = sqlite3_context_db_handle(context); + char const *zDb = (const char*)sqlite3_value_text(argv[0]); + char const *zInput = (const char*)sqlite3_value_text(argv[1]); + int bTemp = sqlite3_value_int(argv[4]); + int isLegacy = (db->flags & SQLITE_LegacyAlter); + +#ifndef SQLITE_OMIT_AUTHORIZATION + sqlite3_xauth xAuth = db->xAuth; + db->xAuth = 0; +#endif + + UNUSED_PARAMETER(NotUsed); + if( zDb && zInput ){ + int rc; + Parse sParse; + rc = renameParseSql(&sParse, zDb, db, zInput, bTemp); + if( rc==SQLITE_OK ){ + if( isLegacy==0 && sParse.pNewTable && sParse.pNewTable->pSelect ){ + NameContext sNC; + memset(&sNC, 0, sizeof(sNC)); + sNC.pParse = &sParse; + sqlite3SelectPrep(&sParse, sParse.pNewTable->pSelect, &sNC); + if( sParse.nErr ) rc = sParse.rc; + } + + else if( sParse.pNewTrigger ){ + if( isLegacy==0 ){ + rc = renameResolveTrigger(&sParse, bTemp ? 0 : zDb); + } + if( rc==SQLITE_OK ){ + int i1 = sqlite3SchemaToIndex(db, sParse.pNewTrigger->pTabSchema); + int i2 = sqlite3FindDbName(db, zDb); + if( i1==i2 ) sqlite3_result_int(context, 1); + } + } + } + + if( rc!=SQLITE_OK ){ + renameColumnParseError(context, 1, argv[2], argv[3], &sParse); + } + renameParseCleanup(&sParse); + } + +#ifndef SQLITE_OMIT_AUTHORIZATION + db->xAuth = xAuth; +#endif +} + +/* +** Register built-in functions used to help implement ALTER TABLE +*/ +SQLITE_PRIVATE void sqlite3AlterFunctions(void){ + static FuncDef aAlterTableFuncs[] = { + INTERNAL_FUNCTION(sqlite_rename_column, 9, renameColumnFunc), + INTERNAL_FUNCTION(sqlite_rename_table, 7, renameTableFunc), + INTERNAL_FUNCTION(sqlite_rename_test, 5, renameTableTest), + }; + sqlite3InsertBuiltinFuncs(aAlterTableFuncs, ArraySize(aAlterTableFuncs)); +} #endif /* SQLITE_ALTER_TABLE */ /************** End of alter.c ***********************************************/ @@ -101109,13 +107258,13 @@ SQLITE_PRIVATE void sqlite3AlterBeginAddColumn(Parse *pParse, SrcList *pSrc){ ** is between 3.6.18 and 3.7.8, inclusive, and unless SQLite is compiled ** with SQLITE_ENABLE_STAT2. The sqlite_stat2 table is deprecated. ** The sqlite_stat2 table is superseded by sqlite_stat3, which is only -** created and used by SQLite versions 3.7.9 and later and with +** created and used by SQLite versions 3.7.9 through 3.29.0 when ** SQLITE_ENABLE_STAT3 defined. The functionality of sqlite_stat3 -** is a superset of sqlite_stat2. The sqlite_stat4 is an enhanced -** version of sqlite_stat3 and is only available when compiled with -** SQLITE_ENABLE_STAT4 and in SQLite versions 3.8.1 and later. It is -** not possible to enable both STAT3 and STAT4 at the same time. If they -** are both enabled, then STAT4 takes precedence. +** is a superset of sqlite_stat2 and is also now deprecated. The +** sqlite_stat4 is an enhanced version of sqlite_stat3 and is only +** available when compiled with SQLITE_ENABLE_STAT4 and in SQLite +** versions 3.8.1 and later. STAT4 is the only variant that is still +** supported. ** ** For most applications, sqlite_stat1 provides all the statistics required ** for the query planner to make good choices. @@ -101226,17 +107375,11 @@ SQLITE_PRIVATE void sqlite3AlterBeginAddColumn(Parse *pParse, SrcList *pSrc){ #if defined(SQLITE_ENABLE_STAT4) # define IsStat4 1 -# define IsStat3 0 -#elif defined(SQLITE_ENABLE_STAT3) -# define IsStat4 0 -# define IsStat3 1 #else # define IsStat4 0 -# define IsStat3 0 # undef SQLITE_STAT4_SAMPLES # define SQLITE_STAT4_SAMPLES 1 #endif -#define IsStat34 (IsStat3+IsStat4) /* 1 for STAT3 or STAT4. 0 otherwise */ /* ** This routine generates code that opens the sqlite_statN tables. @@ -101265,14 +107408,10 @@ static void openStatTable( { "sqlite_stat1", "tbl,idx,stat" }, #if defined(SQLITE_ENABLE_STAT4) { "sqlite_stat4", "tbl,idx,neq,nlt,ndlt,sample" }, - { "sqlite_stat3", 0 }, -#elif defined(SQLITE_ENABLE_STAT3) - { "sqlite_stat3", "tbl,idx,neq,nlt,ndlt,sample" }, - { "sqlite_stat4", 0 }, #else - { "sqlite_stat3", 0 }, { "sqlite_stat4", 0 }, #endif + { "sqlite_stat3", 0 }, }; int i; sqlite3 *db = pParse->db; @@ -101280,6 +107419,11 @@ static void openStatTable( Vdbe *v = sqlite3GetVdbe(pParse); int aRoot[ArraySize(aTable)]; u8 aCreateTbl[ArraySize(aTable)]; +#ifdef SQLITE_ENABLE_STAT4 + const int nToOpen = OptimizationEnabled(db,SQLITE_Stat4) ? 2 : 1; +#else + const int nToOpen = 1; +#endif if( v==0 ) return; assert( sqlite3BtreeHoldsAllMutexes(db) ); @@ -101292,8 +107436,9 @@ static void openStatTable( for(i=0; izDbSName))==0 ){ - if( aTable[i].zCols ){ + if( iregRoot. This is important @@ -101309,7 +107454,6 @@ static void openStatTable( ** associated with the table zWhere. If zWhere is NULL, delete the ** entire contents of the table. */ aRoot[i] = pStat->tnum; - aCreateTbl[i] = 0; sqlite3TableLock(pParse, iDb, aRoot[i], 1, zTab); if( zWhere ){ sqlite3NestedParse(pParse, @@ -101328,7 +107472,7 @@ static void openStatTable( } /* Open the sqlite_stat[134] tables for writing. */ - for(i=0; aTable[i].zCols; i++){ + for(i=0; inRowid ){ sqlite3DbFree(db, p->u.aRowid); @@ -101396,8 +107545,8 @@ static void sampleClear(sqlite3 *db, Stat4Sample *p){ /* Initialize the BLOB value of a ROWID */ -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 -static void sampleSetRowid(sqlite3 *db, Stat4Sample *p, int n, const u8 *pData){ +#ifdef SQLITE_ENABLE_STAT4 +static void sampleSetRowid(sqlite3 *db, StatSample *p, int n, const u8 *pData){ assert( db!=0 ); if( p->nRowid ) sqlite3DbFree(db, p->u.aRowid); p->u.aRowid = sqlite3DbMallocRawNN(db, n); @@ -101412,8 +107561,8 @@ static void sampleSetRowid(sqlite3 *db, Stat4Sample *p, int n, const u8 *pData){ /* Initialize the INTEGER value of a ROWID. */ -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 -static void sampleSetRowidInt64(sqlite3 *db, Stat4Sample *p, i64 iRowid){ +#ifdef SQLITE_ENABLE_STAT4 +static void sampleSetRowidInt64(sqlite3 *db, StatSample *p, i64 iRowid){ assert( db!=0 ); if( p->nRowid ) sqlite3DbFree(db, p->u.aRowid); p->nRowid = 0; @@ -101425,8 +107574,8 @@ static void sampleSetRowidInt64(sqlite3 *db, Stat4Sample *p, i64 iRowid){ /* ** Copy the contents of object (*pFrom) into (*pTo). */ -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 -static void sampleCopy(Stat4Accum *p, Stat4Sample *pTo, Stat4Sample *pFrom){ +#ifdef SQLITE_ENABLE_STAT4 +static void sampleCopy(StatAccum *p, StatSample *pTo, StatSample *pFrom){ pTo->isPSample = pFrom->isPSample; pTo->iCol = pFrom->iCol; pTo->iHash = pFrom->iHash; @@ -101442,40 +107591,41 @@ static void sampleCopy(Stat4Accum *p, Stat4Sample *pTo, Stat4Sample *pFrom){ #endif /* -** Reclaim all memory of a Stat4Accum structure. +** Reclaim all memory of a StatAccum structure. */ -static void stat4Destructor(void *pOld){ - Stat4Accum *p = (Stat4Accum*)pOld; -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 - int i; - for(i=0; inCol; i++) sampleClear(p->db, p->aBest+i); - for(i=0; imxSample; i++) sampleClear(p->db, p->a+i); - sampleClear(p->db, &p->current); +static void statAccumDestructor(void *pOld){ + StatAccum *p = (StatAccum*)pOld; +#ifdef SQLITE_ENABLE_STAT4 + if( p->mxSample ){ + int i; + for(i=0; inCol; i++) sampleClear(p->db, p->aBest+i); + for(i=0; imxSample; i++) sampleClear(p->db, p->a+i); + sampleClear(p->db, &p->current); + } #endif sqlite3DbFree(p->db, p); } /* -** Implementation of the stat_init(N,K,C) SQL function. The three parameters +** Implementation of the stat_init(N,K,C,L) SQL function. The four parameters ** are: ** N: The number of columns in the index including the rowid/pk (note 1) ** K: The number of columns in the index excluding the rowid/pk. -** C: The number of rows in the index (note 2) +** C: Estimated number of rows in the index +** L: A limit on the number of rows to scan, or 0 for no-limit ** ** Note 1: In the special case of the covering index that implements a ** WITHOUT ROWID table, N is the number of PRIMARY KEY columns, not the ** total number of columns in the table. ** -** Note 2: C is only used for STAT3 and STAT4. -** ** For indexes on ordinary rowid tables, N==K+1. But for indexes on ** WITHOUT ROWID tables, N=K+P where P is the number of columns in the ** PRIMARY KEY of the table. The covering index that implements the ** original WITHOUT ROWID table as N==K as a special case. ** -** This routine allocates the Stat4Accum object in heap memory. The return -** value is a pointer to the Stat4Accum object. The datatype of the -** return value is BLOB, but it is really just a pointer to the Stat4Accum +** This routine allocates the StatAccum object in heap memory. The return +** value is a pointer to the StatAccum object. The datatype of the +** return value is BLOB, but it is really just a pointer to the StatAccum ** object. */ static void statInit( @@ -101483,14 +107633,15 @@ static void statInit( int argc, sqlite3_value **argv ){ - Stat4Accum *p; + StatAccum *p; int nCol; /* Number of columns in index being sampled */ int nKeyCol; /* Number of key columns */ int nColUp; /* nCol rounded up for alignment */ int n; /* Bytes of space to allocate */ - sqlite3 *db; /* Database connection */ -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 - int mxSample = SQLITE_STAT4_SAMPLES; + sqlite3 *db = sqlite3_context_db_handle(context); /* Database connection */ +#ifdef SQLITE_ENABLE_STAT4 + /* Maximum number of samples. 0 if STAT4 data is not collected */ + int mxSample = OptimizationEnabled(db,SQLITE_Stat4) ?SQLITE_STAT4_SAMPLES :0; #endif /* Decode the three function arguments */ @@ -101502,16 +107653,17 @@ static void statInit( assert( nKeyCol<=nCol ); assert( nKeyCol>0 ); - /* Allocate the space required for the Stat4Accum object */ + /* Allocate the space required for the StatAccum object */ n = sizeof(*p) - + sizeof(tRowcnt)*nColUp /* Stat4Accum.anEq */ - + sizeof(tRowcnt)*nColUp /* Stat4Accum.anDLt */ -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 - + sizeof(tRowcnt)*nColUp /* Stat4Accum.anLt */ - + sizeof(Stat4Sample)*(nCol+mxSample) /* Stat4Accum.aBest[], a[] */ - + sizeof(tRowcnt)*3*nColUp*(nCol+mxSample) -#endif - ; + + sizeof(tRowcnt)*nColUp /* StatAccum.anEq */ + + sizeof(tRowcnt)*nColUp; /* StatAccum.anDLt */ +#ifdef SQLITE_ENABLE_STAT4 + if( mxSample ){ + n += sizeof(tRowcnt)*nColUp /* StatAccum.anLt */ + + sizeof(StatSample)*(nCol+mxSample) /* StatAccum.aBest[], a[] */ + + sizeof(tRowcnt)*3*nColUp*(nCol+mxSample); + } +#endif db = sqlite3_context_db_handle(context); p = sqlite3DbMallocZero(db, n); if( p==0 ){ @@ -101520,25 +107672,28 @@ static void statInit( } p->db = db; + p->nEst = sqlite3_value_int64(argv[2]); p->nRow = 0; + p->nLimit = sqlite3_value_int64(argv[3]); p->nCol = nCol; p->nKeyCol = nKeyCol; + p->nSkipAhead = 0; p->current.anDLt = (tRowcnt*)&p[1]; p->current.anEq = &p->current.anDLt[nColUp]; -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 - { +#ifdef SQLITE_ENABLE_STAT4 + p->mxSample = p->nLimit==0 ? mxSample : 0; + if( mxSample ){ u8 *pSpace; /* Allocated space not yet assigned */ int i; /* Used to iterate through p->aSample[] */ p->iGet = -1; - p->mxSample = mxSample; - p->nPSample = (tRowcnt)(sqlite3_value_int64(argv[2])/(mxSample/3+1) + 1); + p->nPSample = (tRowcnt)(p->nEst/(mxSample/3+1) + 1); p->current.anLt = &p->current.anEq[nColUp]; p->iPrn = 0x689e962d*(u32)nCol ^ 0xd0944565*(u32)sqlite3_value_int(argv[2]); - /* Set up the Stat4Accum.a[] and aBest[] arrays */ - p->a = (struct Stat4Sample*)&p->current.anLt[nColUp]; + /* Set up the StatAccum.a[] and aBest[] arrays */ + p->a = (struct StatSample*)&p->current.anLt[nColUp]; p->aBest = &p->a[mxSample]; pSpace = (u8*)(&p->a[mxSample+nCol]); for(i=0; i<(mxSample+nCol); i++){ @@ -101558,15 +107713,16 @@ static void statInit( ** only the pointer (the 2nd parameter) matters. The size of the object ** (given by the 3rd parameter) is never used and can be any positive ** value. */ - sqlite3_result_blob(context, p, sizeof(*p), stat4Destructor); + sqlite3_result_blob(context, p, sizeof(*p), statAccumDestructor); } static const FuncDef statInitFuncdef = { - 2+IsStat34, /* nArg */ + 4, /* nArg */ SQLITE_UTF8, /* funcFlags */ 0, /* pUserData */ 0, /* pNext */ statInit, /* xSFunc */ 0, /* xFinalize */ + 0, 0, /* xValue, xInverse */ "stat_init", /* zName */ {0} }; @@ -101584,9 +107740,9 @@ static const FuncDef statInitFuncdef = { ** the anEq[] array from pSample->anEq[pSample->iCol+1] onwards are valid. */ static int sampleIsBetterPost( - Stat4Accum *pAccum, - Stat4Sample *pNew, - Stat4Sample *pOld + StatAccum *pAccum, + StatSample *pNew, + StatSample *pOld ){ int nCol = pAccum->nCol; int i; @@ -101600,7 +107756,7 @@ static int sampleIsBetterPost( } #endif -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +#ifdef SQLITE_ENABLE_STAT4 /* ** Return true if pNew is to be preferred over pOld. ** @@ -101608,9 +107764,9 @@ static int sampleIsBetterPost( ** the anEq[] array from pSample->anEq[pSample->iCol] onwards are valid. */ static int sampleIsBetter( - Stat4Accum *pAccum, - Stat4Sample *pNew, - Stat4Sample *pOld + StatAccum *pAccum, + StatSample *pNew, + StatSample *pOld ){ tRowcnt nEqNew = pNew->anEq[pNew->iCol]; tRowcnt nEqOld = pOld->anEq[pOld->iCol]; @@ -101619,37 +107775,32 @@ static int sampleIsBetter( assert( IsStat4 || (pNew->iCol==0 && pOld->iCol==0) ); if( (nEqNew>nEqOld) ) return 1; -#ifdef SQLITE_ENABLE_STAT4 if( nEqNew==nEqOld ){ if( pNew->iColiCol ) return 1; return (pNew->iCol==pOld->iCol && sampleIsBetterPost(pAccum, pNew, pOld)); } return 0; -#else - return (nEqNew==nEqOld && pNew->iHash>pOld->iHash); -#endif } /* ** Copy the contents of sample *pNew into the p->a[] array. If necessary, ** remove the least desirable sample from p->a[] to make room. */ -static void sampleInsert(Stat4Accum *p, Stat4Sample *pNew, int nEqZero){ - Stat4Sample *pSample = 0; +static void sampleInsert(StatAccum *p, StatSample *pNew, int nEqZero){ + StatSample *pSample = 0; int i; assert( IsStat4 || nEqZero==0 ); -#ifdef SQLITE_ENABLE_STAT4 - /* Stat4Accum.nMaxEqZero is set to the maximum number of leading 0 - ** values in the anEq[] array of any sample in Stat4Accum.a[]. In + /* StatAccum.nMaxEqZero is set to the maximum number of leading 0 + ** values in the anEq[] array of any sample in StatAccum.a[]. In ** other words, if nMaxEqZero is n, then it is guaranteed that there - ** are no samples with Stat4Sample.anEq[m]==0 for (m>=n). */ + ** are no samples with StatSample.anEq[m]==0 for (m>=n). */ if( nEqZero>p->nMaxEqZero ){ p->nMaxEqZero = nEqZero; } if( pNew->isPSample==0 ){ - Stat4Sample *pUpgrade = 0; + StatSample *pUpgrade = 0; assert( pNew->anEq[pNew->iCol]>0 ); /* This sample is being added because the prefix that ends in column @@ -101658,7 +107809,7 @@ static void sampleInsert(Stat4Accum *p, Stat4Sample *pNew, int nEqZero){ ** this one. Instead, upgrade the priority of the highest priority ** existing sample that shares this prefix. */ for(i=p->nSample-1; i>=0; i--){ - Stat4Sample *pOld = &p->a[i]; + StatSample *pOld = &p->a[i]; if( pOld->anEq[pNew->iCol]==0 ){ if( pOld->isPSample ) return; assert( pOld->iCol>pNew->iCol ); @@ -101674,11 +107825,10 @@ static void sampleInsert(Stat4Accum *p, Stat4Sample *pNew, int nEqZero){ goto find_new_min; } } -#endif /* If necessary, remove sample iMin to make room for the new sample. */ if( p->nSample>=p->mxSample ){ - Stat4Sample *pMin = &p->a[p->iMin]; + StatSample *pMin = &p->a[p->iMin]; tRowcnt *anEq = pMin->anEq; tRowcnt *anLt = pMin->anLt; tRowcnt *anDLt = pMin->anDLt; @@ -101695,10 +107845,8 @@ static void sampleInsert(Stat4Accum *p, Stat4Sample *pNew, int nEqZero){ /* The "rows less-than" for the rowid column must be greater than that ** for the last sample in the p->a[] array. Otherwise, the samples would ** be out of order. */ -#ifdef SQLITE_ENABLE_STAT4 assert( p->nSample==0 || pNew->anLt[p->nCol-1] > p->a[p->nSample-1].anLt[p->nCol-1] ); -#endif /* Insert the new sample */ pSample = &p->a[p->nSample]; @@ -101708,9 +107856,7 @@ static void sampleInsert(Stat4Accum *p, Stat4Sample *pNew, int nEqZero){ /* Zero the first nEqZero entries in the anEq[] array. */ memset(pSample->anEq, 0, sizeof(tRowcnt)*nEqZero); -#ifdef SQLITE_ENABLE_STAT4 - find_new_min: -#endif +find_new_min: if( p->nSample>=p->mxSample ){ int iMin = -1; for(i=0; imxSample; i++){ @@ -101723,22 +107869,22 @@ static void sampleInsert(Stat4Accum *p, Stat4Sample *pNew, int nEqZero){ p->iMin = iMin; } } -#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ +#endif /* SQLITE_ENABLE_STAT4 */ +#ifdef SQLITE_ENABLE_STAT4 /* ** Field iChng of the index being scanned has changed. So at this point ** p->current contains a sample that reflects the previous row of the ** index. The value of anEq[iChng] and subsequent anEq[] elements are ** correct at this point. */ -static void samplePushPrevious(Stat4Accum *p, int iChng){ -#ifdef SQLITE_ENABLE_STAT4 +static void samplePushPrevious(StatAccum *p, int iChng){ int i; /* Check if any samples from the aBest[] array should be pushed ** into IndexSample.a[] at this point. */ for(i=(p->nCol-2); i>=iChng; i--){ - Stat4Sample *pBest = &p->aBest[i]; + StatSample *pBest = &p->aBest[i]; pBest->anEq[i] = p->current.anEq[i]; if( p->nSamplemxSample || sampleIsBetter(p, pBest, &p->a[p->iMin]) ){ sampleInsert(p, pBest, i); @@ -101762,50 +107908,27 @@ static void samplePushPrevious(Stat4Accum *p, int iChng){ } p->nMaxEqZero = iChng; } -#endif - -#if defined(SQLITE_ENABLE_STAT3) && !defined(SQLITE_ENABLE_STAT4) - if( iChng==0 ){ - tRowcnt nLt = p->current.anLt[0]; - tRowcnt nEq = p->current.anEq[0]; - - /* Check if this is to be a periodic sample. If so, add it. */ - if( (nLt/p->nPSample)!=(nLt+nEq)/p->nPSample ){ - p->current.isPSample = 1; - sampleInsert(p, &p->current, 0); - p->current.isPSample = 0; - }else - - /* Or if it is a non-periodic sample. Add it in this case too. */ - if( p->nSamplemxSample - || sampleIsBetter(p, &p->current, &p->a[p->iMin]) - ){ - sampleInsert(p, &p->current, 0); - } - } -#endif - -#ifndef SQLITE_ENABLE_STAT3_OR_STAT4 - UNUSED_PARAMETER( p ); - UNUSED_PARAMETER( iChng ); -#endif } +#endif /* SQLITE_ENABLE_STAT4 */ /* ** Implementation of the stat_push SQL function: stat_push(P,C,R) ** Arguments: ** -** P Pointer to the Stat4Accum object created by stat_init() +** P Pointer to the StatAccum object created by stat_init() ** C Index of left-most column to differ from previous row ** R Rowid for the current row. Might be a key record for ** WITHOUT ROWID tables. ** -** This SQL function always returns NULL. It's purpose it to accumulate -** statistical data and/or samples in the Stat4Accum object about the -** index being analyzed. The stat_get() SQL function will later be used to -** extract relevant information for constructing the sqlite_statN tables. +** The purpose of this routine is to collect statistical data and/or +** samples from the index being analyzed into the StatAccum object. +** The stat_get() SQL function will be used afterwards to +** retrieve the information gathered. ** -** The R parameter is only used for STAT3 and STAT4 +** This SQL function usually returns NULL, but might return an integer +** if it wants the byte-code to do special processing. +** +** The R parameter is only used for STAT4 */ static void statPush( sqlite3_context *context, @@ -101815,7 +107938,7 @@ static void statPush( int i; /* The three function arguments */ - Stat4Accum *p = (Stat4Accum*)sqlite3_value_blob(argv[0]); + StatAccum *p = (StatAccum*)sqlite3_value_blob(argv[0]); int iChng = sqlite3_value_int(argv[1]); UNUSED_PARAMETER( argc ); @@ -101828,7 +107951,9 @@ static void statPush( for(i=0; inCol; i++) p->current.anEq[i] = 1; }else{ /* Second and subsequent calls get processed here */ - samplePushPrevious(p, iChng); +#ifdef SQLITE_ENABLE_STAT4 + if( p->mxSample ) samplePushPrevious(p, iChng); +#endif /* Update anDLt[], anLt[] and anEq[] to reflect the values that apply ** to the current row of the index. */ @@ -101837,27 +107962,26 @@ static void statPush( } for(i=iChng; inCol; i++){ p->current.anDLt[i]++; -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 - p->current.anLt[i] += p->current.anEq[i]; +#ifdef SQLITE_ENABLE_STAT4 + if( p->mxSample ) p->current.anLt[i] += p->current.anEq[i]; #endif p->current.anEq[i] = 1; } } - p->nRow++; -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 - if( sqlite3_value_type(argv[2])==SQLITE_INTEGER ){ - sampleSetRowidInt64(p->db, &p->current, sqlite3_value_int64(argv[2])); - }else{ - sampleSetRowid(p->db, &p->current, sqlite3_value_bytes(argv[2]), - sqlite3_value_blob(argv[2])); - } - p->current.iHash = p->iPrn = p->iPrn*1103515245 + 12345; -#endif + p->nRow++; #ifdef SQLITE_ENABLE_STAT4 - { - tRowcnt nLt = p->current.anLt[p->nCol-1]; + if( p->mxSample ){ + tRowcnt nLt; + if( sqlite3_value_type(argv[2])==SQLITE_INTEGER ){ + sampleSetRowidInt64(p->db, &p->current, sqlite3_value_int64(argv[2])); + }else{ + sampleSetRowid(p->db, &p->current, sqlite3_value_bytes(argv[2]), + sqlite3_value_blob(argv[2])); + } + p->current.iHash = p->iPrn = p->iPrn*1103515245 + 12345; + nLt = p->current.anLt[p->nCol-1]; /* Check if this is to be a periodic sample. If so, add it. */ if( (nLt/p->nPSample)!=(nLt+1)/p->nPSample ){ p->current.isPSample = 1; @@ -101873,16 +107997,22 @@ static void statPush( sampleCopy(p, &p->aBest[i], &p->current); } } - } + }else #endif + if( p->nLimit && p->nRow>(tRowcnt)p->nLimit*(p->nSkipAhead+1) ){ + p->nSkipAhead++; + sqlite3_result_int(context, p->current.anDLt[0]>0); + } } + static const FuncDef statPushFuncdef = { - 2+IsStat34, /* nArg */ + 2+IsStat4, /* nArg */ SQLITE_UTF8, /* funcFlags */ 0, /* pUserData */ 0, /* pNext */ statPush, /* xSFunc */ 0, /* xFinalize */ + 0, 0, /* xValue, xInverse */ "stat_push", /* zName */ {0} }; @@ -101896,18 +108026,18 @@ static const FuncDef statPushFuncdef = { /* ** Implementation of the stat_get(P,J) SQL function. This routine is ** used to query statistical information that has been gathered into -** the Stat4Accum object by prior calls to stat_push(). The P parameter -** has type BLOB but it is really just a pointer to the Stat4Accum object. +** the StatAccum object by prior calls to stat_push(). The P parameter +** has type BLOB but it is really just a pointer to the StatAccum object. ** The content to returned is determined by the parameter J ** which is one of the STAT_GET_xxxx values defined above. ** ** The stat_get(P,J) function is not available to generic SQL. It is ** inserted as part of a manually constructed bytecode program. (See ** the callStatGet() routine below.) It is guaranteed that the P -** parameter will always be a poiner to a Stat4Accum object, never a +** parameter will always be a pointer to a StatAccum object, never a ** NULL. ** -** If neither STAT3 nor STAT4 are enabled, then J is always +** If STAT4 is not enabled, then J is always ** STAT_GET_STAT1 and is hence omitted and this routine becomes ** a one-parameter function, stat_get(P), that always returns the ** stat1 table entry information. @@ -101917,15 +108047,16 @@ static void statGet( int argc, sqlite3_value **argv ){ - Stat4Accum *p = (Stat4Accum*)sqlite3_value_blob(argv[0]); -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 - /* STAT3 and STAT4 have a parameter on this routine. */ + StatAccum *p = (StatAccum*)sqlite3_value_blob(argv[0]); +#ifdef SQLITE_ENABLE_STAT4 + /* STAT4 has a parameter on this routine. */ int eCall = sqlite3_value_int(argv[1]); assert( argc==2 ); assert( eCall==STAT_GET_STAT1 || eCall==STAT_GET_NEQ || eCall==STAT_GET_ROWID || eCall==STAT_GET_NLT || eCall==STAT_GET_NDLT ); + assert( eCall==STAT_GET_STAT1 || p->mxSample ); if( eCall==STAT_GET_STAT1 ) #else assert( argc==1 ); @@ -101938,7 +108069,7 @@ static void statGet( ** the index. The first integer in the list is the total number of ** entries in the index. There is one additional integer in the list ** for each indexed column. This additional integer is an estimate of - ** the number of rows matched by a stabbing query on the index using + ** the number of rows matched by a equality query on the index using ** a key with the corresponding number of fields. In other words, ** if the index is on columns (a,b) and the sqlite_stat1 value is ** "100 10 2", then SQLite estimates that: @@ -101961,7 +108092,8 @@ static void statGet( return; } - sqlite3_snprintf(24, zRet, "%llu", (u64)p->nRow); + sqlite3_snprintf(24, zRet, "%llu", + p->nSkipAhead ? (u64)p->nEst : (u64)p->nRow); z = zRet + sqlite3Strlen30(zRet); for(i=0; inKeyCol; i++){ u64 nDistinct = p->current.anDLt[i] + 1; @@ -101974,14 +108106,14 @@ static void statGet( sqlite3_result_text(context, zRet, -1, sqlite3_free); } -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +#ifdef SQLITE_ENABLE_STAT4 else if( eCall==STAT_GET_ROWID ){ if( p->iGet<0 ){ samplePushPrevious(p, 0); p->iGet = 0; } if( p->iGetnSample ){ - Stat4Sample *pS = p->a + p->iGet; + StatSample *pS = p->a + p->iGet; if( pS->nRowid==0 ){ sqlite3_result_int64(context, pS->u.iRowid); }else{ @@ -102003,9 +108135,7 @@ static void statGet( } } - if( IsStat3 ){ - sqlite3_result_int64(context, (i64)aCnt[0]); - }else{ + { char *zRet = sqlite3MallocZero(p->nCol * 25); if( zRet==0 ){ sqlite3_result_error_nomem(context); @@ -102022,34 +108152,34 @@ static void statGet( } } } -#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ +#endif /* SQLITE_ENABLE_STAT4 */ #ifndef SQLITE_DEBUG UNUSED_PARAMETER( argc ); #endif } static const FuncDef statGetFuncdef = { - 1+IsStat34, /* nArg */ + 1+IsStat4, /* nArg */ SQLITE_UTF8, /* funcFlags */ 0, /* pUserData */ 0, /* pNext */ statGet, /* xSFunc */ 0, /* xFinalize */ + 0, 0, /* xValue, xInverse */ "stat_get", /* zName */ {0} }; -static void callStatGet(Vdbe *v, int regStat4, int iParam, int regOut){ - assert( regOut!=regStat4 && regOut!=regStat4+1 ); -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 - sqlite3VdbeAddOp2(v, OP_Integer, iParam, regStat4+1); +static void callStatGet(Parse *pParse, int regStat, int iParam, int regOut){ +#ifdef SQLITE_ENABLE_STAT4 + sqlite3VdbeAddOp2(pParse->pVdbe, OP_Integer, iParam, regStat+1); #elif SQLITE_DEBUG assert( iParam==STAT_GET_STAT1 ); #else UNUSED_PARAMETER( iParam ); #endif - sqlite3VdbeAddOp4(v, OP_Function0, 0, regStat4, regOut, - (char*)&statGetFuncdef, P4_FUNCDEF); - sqlite3VdbeChangeP5(v, 1 + IsStat34); + assert( regOut!=regStat && regOut!=regStat+1 ); + sqlite3VdbeAddFunctionCall(pParse, 0, regStat, regOut, 1+IsStat4, + &statGetFuncdef, 0); } /* @@ -102074,12 +108204,11 @@ static void analyzeOneTable( int iDb; /* Index of database containing pTab */ u8 needTableCnt = 1; /* True to count the table */ int regNewRowid = iMem++; /* Rowid for the inserted record */ - int regStat4 = iMem++; /* Register to hold Stat4Accum object */ + int regStat = iMem++; /* Register to hold StatAccum object */ int regChng = iMem++; /* Index of changed index field */ -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 int regRowid = iMem++; /* Rowid argument passed to stat_push() */ -#endif int regTemp = iMem++; /* Temporary use register */ + int regTemp2 = iMem++; /* Second temporary use register */ int regTabname = iMem++; /* Register containing table name */ int regIdxname = iMem++; /* Register containing index name */ int regStat1 = iMem++; /* Value for the stat column of sqlite_stat1 */ @@ -102207,19 +108336,27 @@ static void analyzeOneTable( ** (1) the number of columns in the index including the rowid ** (or for a WITHOUT ROWID table, the number of PK columns), ** (2) the number of columns in the key without the rowid/pk - ** (3) the number of rows in the index, - ** - ** - ** The third argument is only used for STAT3 and STAT4 + ** (3) estimated number of rows in the index, */ -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 - sqlite3VdbeAddOp2(v, OP_Count, iIdxCur, regStat4+3); + sqlite3VdbeAddOp2(v, OP_Integer, nCol, regStat+1); + assert( regRowid==regStat+2 ); + sqlite3VdbeAddOp2(v, OP_Integer, pIdx->nKeyCol, regRowid); +#ifdef SQLITE_ENABLE_STAT4 + if( OptimizationEnabled(db, SQLITE_Stat4) ){ + sqlite3VdbeAddOp2(v, OP_Count, iIdxCur, regTemp); + addrRewind = sqlite3VdbeAddOp1(v, OP_Rewind, iIdxCur); + VdbeCoverage(v); + }else #endif - sqlite3VdbeAddOp2(v, OP_Integer, nCol, regStat4+1); - sqlite3VdbeAddOp2(v, OP_Integer, pIdx->nKeyCol, regStat4+2); - sqlite3VdbeAddOp4(v, OP_Function0, 0, regStat4+1, regStat4, - (char*)&statInitFuncdef, P4_FUNCDEF); - sqlite3VdbeChangeP5(v, 2+IsStat34); + { + addrRewind = sqlite3VdbeAddOp1(v, OP_Rewind, iIdxCur); + VdbeCoverage(v); + sqlite3VdbeAddOp3(v, OP_Count, iIdxCur, regTemp, 1); + } + assert( regTemp2==regStat+4 ); + sqlite3VdbeAddOp2(v, OP_Integer, db->nAnalysisLimit, regTemp2); + sqlite3VdbeAddFunctionCall(pParse, 0, regStat+1, regStat, 4, + &statInitFuncdef, 0); /* Implementation of the following: ** @@ -102229,13 +108366,11 @@ static void analyzeOneTable( ** goto next_push_0; ** */ - addrRewind = sqlite3VdbeAddOp1(v, OP_Rewind, iIdxCur); - VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_Integer, 0, regChng); addrNextRow = sqlite3VdbeCurrentAddr(v); if( nColTest>0 ){ - int endDistinctTest = sqlite3VdbeMakeLabel(v); + int endDistinctTest = sqlite3VdbeMakeLabel(pParse); int *aGotoChng; /* Array of jump instruction addresses */ aGotoChng = sqlite3DbMallocRawNN(db, sizeof(int)*nColTest); if( aGotoChng==0 ) continue; @@ -102263,6 +108398,7 @@ static void analyzeOneTable( char *pColl = (char*)sqlite3LocateCollSeq(pParse, pIdx->azColl[i]); sqlite3VdbeAddOp2(v, OP_Integer, i, regChng); sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, i, regTemp); + VdbeComment((v, "%s.column(%d)", pIdx->zName, i)); aGotoChng[i] = sqlite3VdbeAddOp4(v, OP_Ne, regTemp, 0, regPrev+i, pColl, P4_COLLSEQ); sqlite3VdbeChangeP5(v, SQLITE_NULLEQ); @@ -102283,6 +108419,7 @@ static void analyzeOneTable( for(i=0; izName, i)); } sqlite3VdbeResolveLabel(v, endDistinctTest); sqlite3DbFree(db, aGotoChng); @@ -102290,37 +108427,52 @@ static void analyzeOneTable( /* ** chng_addr_N: - ** regRowid = idx(rowid) // STAT34 only - ** stat_push(P, regChng, regRowid) // 3rd parameter STAT34 only + ** regRowid = idx(rowid) // STAT4 only + ** stat_push(P, regChng, regRowid) // 3rd parameter STAT4 only ** Next csr ** if !eof(csr) goto next_row; */ -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 - assert( regRowid==(regStat4+2) ); - if( HasRowid(pTab) ){ - sqlite3VdbeAddOp2(v, OP_IdxRowid, iIdxCur, regRowid); - }else{ - Index *pPk = sqlite3PrimaryKeyIndex(pIdx->pTable); - int j, k, regKey; - regKey = sqlite3GetTempRange(pParse, pPk->nKeyCol); - for(j=0; jnKeyCol; j++){ - k = sqlite3ColumnOfIndex(pIdx, pPk->aiColumn[j]); - assert( k>=0 && knColumn ); - sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, k, regKey+j); - VdbeComment((v, "%s", pTab->aCol[pPk->aiColumn[j]].zName)); +#ifdef SQLITE_ENABLE_STAT4 + if( OptimizationEnabled(db, SQLITE_Stat4) ){ + assert( regRowid==(regStat+2) ); + if( HasRowid(pTab) ){ + sqlite3VdbeAddOp2(v, OP_IdxRowid, iIdxCur, regRowid); + }else{ + Index *pPk = sqlite3PrimaryKeyIndex(pIdx->pTable); + int j, k, regKey; + regKey = sqlite3GetTempRange(pParse, pPk->nKeyCol); + for(j=0; jnKeyCol; j++){ + k = sqlite3TableColumnToIndex(pIdx, pPk->aiColumn[j]); + assert( k>=0 && knColumn ); + sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, k, regKey+j); + VdbeComment((v, "%s.column(%d)", pIdx->zName, i)); + } + sqlite3VdbeAddOp3(v, OP_MakeRecord, regKey, pPk->nKeyCol, regRowid); + sqlite3ReleaseTempRange(pParse, regKey, pPk->nKeyCol); } - sqlite3VdbeAddOp3(v, OP_MakeRecord, regKey, pPk->nKeyCol, regRowid); - sqlite3ReleaseTempRange(pParse, regKey, pPk->nKeyCol); } #endif - assert( regChng==(regStat4+1) ); - sqlite3VdbeAddOp4(v, OP_Function0, 1, regStat4, regTemp, - (char*)&statPushFuncdef, P4_FUNCDEF); - sqlite3VdbeChangeP5(v, 2+IsStat34); - sqlite3VdbeAddOp2(v, OP_Next, iIdxCur, addrNextRow); VdbeCoverage(v); + assert( regChng==(regStat+1) ); + { + sqlite3VdbeAddFunctionCall(pParse, 1, regStat, regTemp, 2+IsStat4, + &statPushFuncdef, 0); + if( db->nAnalysisLimit ){ + int j1, j2, j3; + j1 = sqlite3VdbeAddOp1(v, OP_IsNull, regTemp); VdbeCoverage(v); + j2 = sqlite3VdbeAddOp1(v, OP_If, regTemp); VdbeCoverage(v); + j3 = sqlite3VdbeAddOp4Int(v, OP_SeekGT, iIdxCur, 0, regPrev, 1); + VdbeCoverage(v); + sqlite3VdbeJumpHere(v, j1); + sqlite3VdbeAddOp2(v, OP_Next, iIdxCur, addrNextRow); VdbeCoverage(v); + sqlite3VdbeJumpHere(v, j2); + sqlite3VdbeJumpHere(v, j3); + }else{ + sqlite3VdbeAddOp2(v, OP_Next, iIdxCur, addrNextRow); VdbeCoverage(v); + } + } /* Add the entry to the stat1 table. */ - callStatGet(v, regStat4, STAT_GET_STAT1, regStat1); + callStatGet(pParse, regStat, STAT_GET_STAT1, regStat1); assert( "BBB"[0]==SQLITE_AFF_TEXT ); sqlite3VdbeAddOp4(v, OP_MakeRecord, regTabname, 3, regTemp, "BBB", 0); sqlite3VdbeAddOp2(v, OP_NewRowid, iStatCur, regNewRowid); @@ -102330,9 +108482,9 @@ static void analyzeOneTable( #endif sqlite3VdbeChangeP5(v, OPFLAG_APPEND); - /* Add the entries to the stat3 or stat4 table. */ -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 - { + /* Add the entries to the stat4 table. */ +#ifdef SQLITE_ENABLE_STAT4 + if( OptimizationEnabled(db, SQLITE_Stat4) && db->nAnalysisLimit==0 ){ int regEq = regStat1; int regLt = regStat1+1; int regDLt = regStat1+2; @@ -102346,32 +108498,25 @@ static void analyzeOneTable( pParse->nMem = MAX(pParse->nMem, regCol+nCol); addrNext = sqlite3VdbeCurrentAddr(v); - callStatGet(v, regStat4, STAT_GET_ROWID, regSampleRowid); + callStatGet(pParse, regStat, STAT_GET_ROWID, regSampleRowid); addrIsNull = sqlite3VdbeAddOp1(v, OP_IsNull, regSampleRowid); VdbeCoverage(v); - callStatGet(v, regStat4, STAT_GET_NEQ, regEq); - callStatGet(v, regStat4, STAT_GET_NLT, regLt); - callStatGet(v, regStat4, STAT_GET_NDLT, regDLt); + callStatGet(pParse, regStat, STAT_GET_NEQ, regEq); + callStatGet(pParse, regStat, STAT_GET_NLT, regLt); + callStatGet(pParse, regStat, STAT_GET_NDLT, regDLt); sqlite3VdbeAddOp4Int(v, seekOp, iTabCur, addrNext, regSampleRowid, 0); - /* We know that the regSampleRowid row exists because it was read by - ** the previous loop. Thus the not-found jump of seekOp will never - ** be taken */ - VdbeCoverageNeverTaken(v); -#ifdef SQLITE_ENABLE_STAT3 - sqlite3ExprCodeLoadIndexColumn(pParse, pIdx, iTabCur, 0, regSample); -#else + VdbeCoverage(v); for(i=0; ibUnordered = 1; }else if( sqlite3_strglob("sz=[0-9]*", z)==0 ){ - pIndex->szIdxRow = sqlite3LogEst(sqlite3Atoi(z+3)); + int sz = sqlite3Atoi(z+3); + if( sz<2 ) sz = 2; + pIndex->szIdxRow = sqlite3LogEst(sz); }else if( sqlite3_strglob("noskipscan*", z)==0 ){ pIndex->noSkipScan = 1; } @@ -102633,7 +108780,7 @@ static int analysisLoader(void *pData, int argc, char **argv, char **NotUsed){ if( pIndex ){ tRowcnt *aiRowEst = 0; int nCol = pIndex->nKeyCol+1; -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +#ifdef SQLITE_ENABLE_STAT4 /* Index.aiRowEst may already be set here if there are duplicate ** sqlite_stat1 entries for this index. In that case just clobber ** the old data with the new instead of allocating a new array. */ @@ -102669,7 +108816,7 @@ static int analysisLoader(void *pData, int argc, char **argv, char **NotUsed){ ** and its contents. */ SQLITE_PRIVATE void sqlite3DeleteIndexSamples(sqlite3 *db, Index *pIdx){ -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +#ifdef SQLITE_ENABLE_STAT4 if( pIdx->aSample ){ int j; for(j=0; jnSample; j++){ @@ -102685,10 +108832,10 @@ SQLITE_PRIVATE void sqlite3DeleteIndexSamples(sqlite3 *db, Index *pIdx){ #else UNUSED_PARAMETER(db); UNUSED_PARAMETER(pIdx); -#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ +#endif /* SQLITE_ENABLE_STAT4 */ } -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +#ifdef SQLITE_ENABLE_STAT4 /* ** Populate the pIdx->aAvgEq[] array based on the samples currently ** stored in pIdx->aSample[]. @@ -102766,12 +108913,11 @@ static Index *findIndexOrPrimaryKey( } /* -** Load the content from either the sqlite_stat4 or sqlite_stat3 table +** Load the content from either the sqlite_stat4 ** into the relevant Index.aSample[] arrays. ** ** Arguments zSql1 and zSql2 must point to SQL statements that return -** data equivalent to the following (statements are different for stat3, -** see the caller of this function for details): +** data equivalent to the following: ** ** zSql1: SELECT idx,count(*) FROM %Q.sqlite_stat4 GROUP BY idx ** zSql2: SELECT idx,neq,nlt,ndlt,sample FROM %Q.sqlite_stat4 @@ -102780,7 +108926,6 @@ static Index *findIndexOrPrimaryKey( */ static int loadStatTbl( sqlite3 *db, /* Database handle */ - int bStat3, /* Assume single column records only */ const char *zSql1, /* SQL statement 1 (see above) */ const char *zSql2, /* SQL statement 2 (see above) */ const char *zDb /* Database name (e.g. "main") */ @@ -102814,17 +108959,13 @@ static int loadStatTbl( if( zIndex==0 ) continue; nSample = sqlite3_column_int(pStmt, 1); pIdx = findIndexOrPrimaryKey(db, zIndex, zDb); - assert( pIdx==0 || bStat3 || pIdx->nSample==0 ); - /* Index.nSample is non-zero at this point if data has already been - ** loaded from the stat4 table. In this case ignore stat3 data. */ - if( pIdx==0 || pIdx->nSample ) continue; - if( bStat3==0 ){ - assert( !HasRowid(pIdx->pTable) || pIdx->nColumn==pIdx->nKeyCol+1 ); - if( !HasRowid(pIdx->pTable) && IsPrimaryKeyIndex(pIdx) ){ - nIdxCol = pIdx->nKeyCol; - }else{ - nIdxCol = pIdx->nColumn; - } + assert( pIdx==0 || pIdx->nSample==0 ); + if( pIdx==0 ) continue; + assert( !HasRowid(pIdx->pTable) || pIdx->nColumn==pIdx->nKeyCol+1 ); + if( !HasRowid(pIdx->pTable) && IsPrimaryKeyIndex(pIdx) ){ + nIdxCol = pIdx->nKeyCol; + }else{ + nIdxCol = pIdx->nColumn; } pIdx->nSampleCol = nIdxCol; nByte = sizeof(IndexSample) * nSample; @@ -102866,9 +109007,8 @@ static int loadStatTbl( pIdx = findIndexOrPrimaryKey(db, zIndex, zDb); if( pIdx==0 ) continue; /* This next condition is true if data has already been loaded from - ** the sqlite_stat4 table. In this case ignore stat3 data. */ + ** the sqlite_stat4 table. */ nCol = pIdx->nSampleCol; - if( bStat3 && nCol>1 ) continue; if( pIdx!=pPrevIdx ){ initAvgEq(pPrevIdx); pPrevIdx = pIdx; @@ -102901,7 +109041,7 @@ static int loadStatTbl( } /* -** Load content from the sqlite_stat4 and sqlite_stat3 tables into +** Load content from the sqlite_stat4 table into ** the Index.aSample[] arrays of all indices. */ static int loadStat4(sqlite3 *db, const char *zDb){ @@ -102909,37 +109049,28 @@ static int loadStat4(sqlite3 *db, const char *zDb){ assert( db->lookaside.bDisable ); if( sqlite3FindTable(db, "sqlite_stat4", zDb) ){ - rc = loadStatTbl(db, 0, + rc = loadStatTbl(db, "SELECT idx,count(*) FROM %Q.sqlite_stat4 GROUP BY idx", "SELECT idx,neq,nlt,ndlt,sample FROM %Q.sqlite_stat4", zDb ); } - - if( rc==SQLITE_OK && sqlite3FindTable(db, "sqlite_stat3", zDb) ){ - rc = loadStatTbl(db, 1, - "SELECT idx,count(*) FROM %Q.sqlite_stat3 GROUP BY idx", - "SELECT idx,neq,nlt,ndlt,sqlite_record(sample) FROM %Q.sqlite_stat3", - zDb - ); - } - return rc; } -#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ +#endif /* SQLITE_ENABLE_STAT4 */ /* -** Load the content of the sqlite_stat1 and sqlite_stat3/4 tables. The +** Load the content of the sqlite_stat1 and sqlite_stat4 tables. The ** contents of sqlite_stat1 are used to populate the Index.aiRowEst[] -** arrays. The contents of sqlite_stat3/4 are used to populate the +** arrays. The contents of sqlite_stat4 are used to populate the ** Index.aSample[] arrays. ** ** If the sqlite_stat1 table is not present in the database, SQLITE_ERROR -** is returned. In this case, even if SQLITE_ENABLE_STAT3/4 was defined -** during compilation and the sqlite_stat3/4 table is present, no data is +** is returned. In this case, even if SQLITE_ENABLE_STAT4 was defined +** during compilation and the sqlite_stat4 table is present, no data is ** read from it. ** -** If SQLITE_ENABLE_STAT3/4 was defined during compilation and the +** If SQLITE_ENABLE_STAT4 was defined during compilation and the ** sqlite_stat4 table is not present in the database, SQLITE_ERROR is ** returned. However, in this case, data is read from the sqlite_stat1 ** table (if it is present) before returning. @@ -102967,7 +109098,7 @@ SQLITE_PRIVATE int sqlite3AnalysisLoad(sqlite3 *db, int iDb){ for(i=sqliteHashFirst(&pSchema->idxHash); i; i=sqliteHashNext(i)){ Index *pIdx = sqliteHashData(i); pIdx->hasStat1 = 0; -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +#ifdef SQLITE_ENABLE_STAT4 sqlite3DeleteIndexSamples(db, pIdx); pIdx->aSample = 0; #endif @@ -102995,11 +109126,11 @@ SQLITE_PRIVATE int sqlite3AnalysisLoad(sqlite3 *db, int iDb){ } /* Load the statistics from the sqlite_stat4 table. */ -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 - if( rc==SQLITE_OK && OptimizationEnabled(db, SQLITE_Stat34) ){ - db->lookaside.bDisable++; +#ifdef SQLITE_ENABLE_STAT4 + if( rc==SQLITE_OK ){ + DisableLookaside; rc = loadStat4(db, sInfo.zDatabase); - db->lookaside.bDisable--; + EnableLookaside; } for(i=sqliteHashFirst(&pSchema->idxHash); i; i=sqliteHashNext(i)){ Index *pIdx = sqliteHashData(i); @@ -103066,6 +109197,17 @@ static int resolveAttachExpr(NameContext *pName, Expr *pExpr) return rc; } +/* +** Return true if zName points to a name that may be used to refer to +** database iDb attached to handle db. +*/ +SQLITE_PRIVATE int sqlite3DbIsNamed(sqlite3 *db, int iDb, const char *zName){ + return ( + sqlite3StrICmp(db->aDb[iDb].zDbSName, zName)==0 + || (iDb==0 && sqlite3StrICmp("main", zName)==0) + ); +} + /* ** An SQL user-function registered to do the work of an ATTACH statement. The ** three arguments to the function come directly from an attach statement: @@ -103121,7 +109263,7 @@ static void attachFunc( if( pNew->pBt ) sqlite3BtreeClose(pNew->pBt); pNew->pBt = 0; pNew->pSchema = 0; - rc = sqlite3BtreeOpen(pVfs, "x", db, &pNew->pBt, 0, SQLITE_OPEN_MAIN_DB); + rc = sqlite3BtreeOpen(pVfs, "x\0", db, &pNew->pBt, 0, SQLITE_OPEN_MAIN_DB); }else{ /* This is a real ATTACH ** @@ -103138,9 +109280,8 @@ static void attachFunc( goto attach_error; } for(i=0; inDb; i++){ - char *z = db->aDb[i].zDbSName; - assert( z && zName ); - if( sqlite3StrICmp(z, zName)==0 ){ + assert( zName ); + if( sqlite3DbIsNamed(db, i, zName) ){ zErrDyn = sqlite3MPrintf(db, "database %s is already in use", zName); goto attach_error; } @@ -103176,8 +109317,8 @@ static void attachFunc( assert( pVfs ); flags |= SQLITE_OPEN_MAIN_DB; rc = sqlite3BtreeOpen(pVfs, zPath, db, &pNew->pBt, 0, flags); - sqlite3_free( zPath ); db->nDb++; + pNew->zDbSName = sqlite3DbStrDup(db, zName); } db->noSharedCache = 0; if( rc==SQLITE_CONSTRAINT ){ @@ -103205,43 +109346,10 @@ static void attachFunc( sqlite3BtreeLeave(pNew->pBt); } pNew->safety_level = SQLITE_DEFAULT_SYNCHRONOUS+1; - if( !REOPEN_AS_MEMDB(db) ) pNew->zDbSName = sqlite3DbStrDup(db, zName); if( rc==SQLITE_OK && pNew->zDbSName==0 ){ rc = SQLITE_NOMEM_BKPT; } - - -#ifdef SQLITE_HAS_CODEC - if( rc==SQLITE_OK ){ - extern int sqlite3CodecAttach(sqlite3*, int, const void*, int); - extern void sqlite3CodecGetKey(sqlite3*, int, void**, int*); - int nKey; - char *zKey; - int t = sqlite3_value_type(argv[2]); - switch( t ){ - case SQLITE_INTEGER: - case SQLITE_FLOAT: - zErrDyn = sqlite3DbStrDup(db, "Invalid key value"); - rc = SQLITE_ERROR; - break; - - case SQLITE_TEXT: - case SQLITE_BLOB: - nKey = sqlite3_value_bytes(argv[2]); - zKey = (char *)sqlite3_value_blob(argv[2]); - rc = sqlite3CodecAttach(db, db->nDb-1, zKey, nKey); - break; - - case SQLITE_NULL: - /* No key specified. Use the key from the main database */ - sqlite3CodecGetKey(db, 0, (void**)&zKey, &nKey); - if( nKey || sqlite3BtreeGetOptimalReserve(db->aDb[0].pBt)>0 ){ - rc = sqlite3CodecAttach(db, db->nDb-1, zKey, nKey); - } - break; - } - } -#endif + sqlite3_free_filename( zPath ); /* If the file was opened successfully, read the schema for the new database. ** If this fails, or if opening the file failed, then close the file and @@ -103252,12 +109360,14 @@ static void attachFunc( sqlite3BtreeEnterAll(db); db->init.iDb = 0; db->mDbFlags &= ~(DBFLAG_SchemaKnownOk); - rc = sqlite3Init(db, &zErrDyn); + if( !REOPEN_AS_MEMDB(db) ){ + rc = sqlite3Init(db, &zErrDyn); + } sqlite3BtreeLeaveAll(db); assert( zErrDyn==0 || rc!=SQLITE_OK ); } #ifdef SQLITE_USER_AUTHENTICATION - if( rc==SQLITE_OK ){ + if( rc==SQLITE_OK && !REOPEN_AS_MEMDB(db) ){ u8 newAuth = 0; rc = sqlite3UserAuthCheckLogin(db, zName, &newAuth); if( newAuthauth.authLevel ){ @@ -103315,6 +109425,7 @@ static void detachFunc( sqlite3 *db = sqlite3_context_db_handle(context); int i; Db *pDb = 0; + HashElem *pEntry; char zErr[128]; UNUSED_PARAMETER(NotUsed); @@ -103323,7 +109434,7 @@ static void detachFunc( for(i=0; inDb; i++){ pDb = &db->aDb[i]; if( pDb->pBt==0 ) continue; - if( sqlite3StrICmp(pDb->zDbSName, zName)==0 ) break; + if( sqlite3DbIsNamed(db, i, zName) ) break; } if( i>=db->nDb ){ @@ -103339,6 +109450,18 @@ static void detachFunc( goto detach_error; } + /* If any TEMP triggers reference the schema being detached, move those + ** triggers to reference the TEMP schema itself. */ + assert( db->aDb[1].pSchema ); + pEntry = sqliteHashFirst(&db->aDb[1].pSchema->trigHash); + while( pEntry ){ + Trigger *pTrig = (Trigger*)sqliteHashData(pEntry); + if( pTrig->pTabSchema==pDb->pSchema ){ + pTrig->pTabSchema = pTrig->pSchema; + } + pEntry = sqliteHashNext(pEntry); + } + sqlite3BtreeClose(pDb->pBt); pDb->pBt = 0; pDb->pSchema = 0; @@ -103404,11 +109527,8 @@ static void codeAttach( assert( v || db->mallocFailed ); if( v ){ - sqlite3VdbeAddOp4(v, OP_Function0, 0, regArgs+3-pFunc->nArg, regArgs+3, - (char *)pFunc, P4_FUNCDEF); - assert( pFunc->nArg==-1 || (pFunc->nArg&0xff)==pFunc->nArg ); - sqlite3VdbeChangeP5(v, (u8)(pFunc->nArg)); - + sqlite3VdbeAddFunctionCall(pParse, 0, regArgs+3-pFunc->nArg, regArgs+3, + pFunc->nArg, pFunc, 0); /* Code an OP_Expire. For an ATTACH statement, set P1 to true (expire this ** statement only). For DETACH, set it to false (expire all existing ** statements). @@ -103435,6 +109555,7 @@ SQLITE_PRIVATE void sqlite3Detach(Parse *pParse, Expr *pDbname){ 0, /* pNext */ detachFunc, /* xSFunc */ 0, /* xFinalize */ + 0, 0, /* xValue, xInverse */ "sqlite_detach", /* zName */ {0} }; @@ -103454,6 +109575,7 @@ SQLITE_PRIVATE void sqlite3Attach(Parse *pParse, Expr *p, Expr *pDbname, Expr *p 0, /* pNext */ attachFunc, /* xSFunc */ 0, /* xFinalize */ + 0, 0, /* xValue, xInverse */ "sqlite_attach", /* zName */ {0} }; @@ -103481,7 +109603,7 @@ SQLITE_PRIVATE void sqlite3FixInit( pFix->pSchema = db->aDb[iDb].pSchema; pFix->zType = zType; pFix->pName = pName; - pFix->bVarOnly = (iDb==1); + pFix->bTemp = (iDb==1); } /* @@ -103503,22 +109625,24 @@ SQLITE_PRIVATE int sqlite3FixSrcList( SrcList *pList /* The Source list to check and modify */ ){ int i; - const char *zDb; struct SrcList_item *pItem; + sqlite3 *db = pFix->pParse->db; + int iDb = sqlite3FindDbName(db, pFix->zDb); if( NEVER(pList==0) ) return 0; - zDb = pFix->zDb; + for(i=0, pItem=pList->a; inSrc; i++, pItem++){ - if( pFix->bVarOnly==0 ){ - if( pItem->zDatabase && sqlite3StrICmp(pItem->zDatabase, zDb) ){ + if( pFix->bTemp==0 ){ + if( pItem->zDatabase && iDb!=sqlite3FindDbName(db, pItem->zDatabase) ){ sqlite3ErrorMsg(pFix->pParse, "%s %T cannot reference objects in database %s", pFix->zType, pFix->pName, pItem->zDatabase); return 1; } - sqlite3DbFree(pFix->pParse->db, pItem->zDatabase); + sqlite3DbFree(db, pItem->zDatabase); pItem->zDatabase = 0; pItem->pSchema = pFix->pSchema; + pItem->fg.fromDDL = 1; } #if !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_TRIGGER) if( sqlite3FixSelect(pFix, pItem->pSelect) ) return 1; @@ -103574,6 +109698,7 @@ SQLITE_PRIVATE int sqlite3FixExpr( Expr *pExpr /* The expression to be fixed to one database */ ){ while( pExpr ){ + if( !pFix->bTemp ) ExprSetProperty(pExpr, EP_FromDDL); if( pExpr->op==TK_VARIABLE ){ if( pFix->pParse->db->init.busy ){ pExpr->op = TK_NULL; @@ -103726,7 +109851,7 @@ SQLITE_API int sqlite3_set_authorizer( sqlite3_mutex_enter(db->mutex); db->xAuth = (sqlite3_xauth)xAuth; db->pAuthArg = pArg; - sqlite3ExpirePreparedStatements(db); + if( db->xAuth ) sqlite3ExpirePreparedStatements(db, 1); sqlite3_mutex_leave(db->mutex); return SQLITE_OK; } @@ -103799,6 +109924,7 @@ SQLITE_PRIVATE void sqlite3AuthRead( int iCol; /* Index of column in table */ assert( pExpr->op==TK_COLUMN || pExpr->op==TK_TRIGGER ); + assert( !IN_RENAME_OBJECT || db->xAuth==0 ); if( db->xAuth==0 ) return; iDb = sqlite3SchemaToIndex(pParse->db, pSchema); if( iDb<0 ){ @@ -103855,7 +109981,8 @@ SQLITE_PRIVATE int sqlite3AuthCheck( /* Don't do any authorization checks if the database is initialising ** or if the parser is being invoked from within sqlite3_declare_vtab. */ - if( db->init.busy || IN_DECLARE_VTAB ){ + assert( !IN_RENAME_OBJECT || db->xAuth==0 ); + if( db->init.busy || IN_SPECIAL_PARSE ){ return SQLITE_OK; } @@ -104147,10 +110274,9 @@ SQLITE_PRIVATE void sqlite3FinishCoding(Parse *pParse){ /* Get the VDBE program ready for execution */ if( v && pParse->nErr==0 && !db->mallocFailed ){ - assert( pParse->iCacheLevel==0 ); /* Disables and re-enables match */ /* A minimum of one cursor is required if autoincrement is used * See ticket [a696379c1f08866] */ - if( pParse->pAinc!=0 && pParse->nTab==0 ) pParse->nTab = 1; + assert( pParse->pAinc==0 || pParse->nTab>0 ); sqlite3VdbeMakeReady(v, pParse); pParse->rc = SQLITE_DONE; }else{ @@ -104183,7 +110309,12 @@ SQLITE_PRIVATE void sqlite3NestedParse(Parse *pParse, const char *zFormat, ...){ zSql = sqlite3VMPrintf(db, zFormat, ap); va_end(ap); if( zSql==0 ){ - return; /* A malloc must have failed */ + /* This can result either from an OOM or because the formatted string + ** exceeds SQLITE_LIMIT_LENGTH. In the latter case, we need to set + ** an error */ + if( !db->mallocFailed ) pParse->rc = SQLITE_TOOBIG; + pParse->nErr++; + return; } pParse->nested++; memcpy(saveBuf, PARSE_TAIL(pParse), PARSE_TAIL_SZ); @@ -104230,22 +110361,39 @@ SQLITE_PRIVATE Table *sqlite3FindTable(sqlite3 *db, const char *zName, const cha return 0; } #endif - while(1){ - for(i=OMIT_TEMPDB; inDb; i++){ - int j = (i<2) ? i^1 : i; /* Search TEMP before MAIN */ - if( zDatabase==0 || sqlite3StrICmp(zDatabase, db->aDb[j].zDbSName)==0 ){ - assert( sqlite3SchemaMutexHeld(db, j, 0) ); - p = sqlite3HashFind(&db->aDb[j].pSchema->tblHash, zName); - if( p ) return p; + if( zDatabase ){ + for(i=0; inDb; i++){ + if( sqlite3StrICmp(zDatabase, db->aDb[i].zDbSName)==0 ) break; + } + if( i>=db->nDb ){ + /* No match against the official names. But always match "main" + ** to schema 0 as a legacy fallback. */ + if( sqlite3StrICmp(zDatabase,"main")==0 ){ + i = 0; + }else{ + return 0; } } - /* Not found. If the name we were looking for was temp.sqlite_master - ** then change the name to sqlite_temp_master and try again. */ - if( sqlite3StrICmp(zName, MASTER_NAME)!=0 ) break; - if( sqlite3_stricmp(zDatabase, db->aDb[1].zDbSName)!=0 ) break; - zName = TEMP_MASTER_NAME; + p = sqlite3HashFind(&db->aDb[i].pSchema->tblHash, zName); + if( p==0 && i==1 && sqlite3StrICmp(zName, MASTER_NAME)==0 ){ + /* All temp.sqlite_master to be an alias for sqlite_temp_master */ + p = sqlite3HashFind(&db->aDb[1].pSchema->tblHash, TEMP_MASTER_NAME); + } + }else{ + /* Match against TEMP first */ + p = sqlite3HashFind(&db->aDb[1].pSchema->tblHash, zName); + if( p ) return p; + /* The main database is second */ + p = sqlite3HashFind(&db->aDb[0].pSchema->tblHash, zName); + if( p ) return p; + /* Attached databases are in order of attachment */ + for(i=2; inDb; i++){ + assert( sqlite3SchemaMutexHeld(db, i, 0) ); + p = sqlite3HashFind(&db->aDb[i].pSchema->tblHash, zName); + if( p ) break; + } } - return 0; + return p; } /* @@ -104277,12 +110425,11 @@ SQLITE_PRIVATE Table *sqlite3LocateTable( p = sqlite3FindTable(db, zName, zDbase); if( p==0 ){ - const char *zMsg = flags & LOCATE_VIEW ? "no such view" : "no such table"; #ifndef SQLITE_OMIT_VIRTUALTABLE - if( sqlite3FindDbName(db, zDbase)<1 ){ - /* If zName is the not the name of a table in the schema created using - ** CREATE, then check to see if it is the name of an virtual table that - ** can be an eponymous virtual table. */ + /* If zName is the not the name of a table in the schema created using + ** CREATE, then check to see if it is the name of an virtual table that + ** can be an eponymous virtual table. */ + if( pParse->disableVtab==0 ){ Module *pMod = (Module*)sqlite3HashFind(&db->aModule, zName); if( pMod==0 && sqlite3_strnicmp(zName, "pragma_", 7)==0 ){ pMod = sqlite3PragmaVtabRegister(db, zName); @@ -104292,13 +110439,18 @@ SQLITE_PRIVATE Table *sqlite3LocateTable( } } #endif - if( (flags & LOCATE_NOERR)==0 ){ - if( zDbase ){ - sqlite3ErrorMsg(pParse, "%s: %s.%s", zMsg, zDbase, zName); - }else{ - sqlite3ErrorMsg(pParse, "%s: %s", zMsg, zName); - } - pParse->checkSchema = 1; + if( flags & LOCATE_NOERR ) return 0; + pParse->checkSchema = 1; + }else if( IsVirtual(p) && pParse->disableVtab ){ + p = 0; + } + + if( p==0 ){ + const char *zMsg = flags & LOCATE_VIEW ? "no such view" : "no such table"; + if( zDbase ){ + sqlite3ErrorMsg(pParse, "%s: %s.%s", zMsg, zDbase, zName); + }else{ + sqlite3ErrorMsg(pParse, "%s: %s", zMsg, zName); } } @@ -104351,7 +110503,7 @@ SQLITE_PRIVATE Index *sqlite3FindIndex(sqlite3 *db, const char *zName, const cha int j = (i<2) ? i^1 : i; /* Search TEMP before MAIN */ Schema *pSchema = db->aDb[j].pSchema; assert( pSchema ); - if( zDb && sqlite3StrICmp(zDb, db->aDb[j].zDbSName) ) continue; + if( zDb && sqlite3DbIsNamed(db, j, zDb)==0 ) continue; assert( sqlite3SchemaMutexHeld(db, j, 0) ); p = sqlite3HashFind(&pSchema->idxHash, zName); if( p ) break; @@ -104362,7 +110514,7 @@ SQLITE_PRIVATE Index *sqlite3FindIndex(sqlite3 *db, const char *zName, const cha /* ** Reclaim the memory used by an index */ -static void freeIndex(sqlite3 *db, Index *p){ +SQLITE_PRIVATE void sqlite3FreeIndex(sqlite3 *db, Index *p){ #ifndef SQLITE_OMIT_ANALYZE sqlite3DeleteIndexSamples(db, p); #endif @@ -104370,7 +110522,7 @@ static void freeIndex(sqlite3 *db, Index *p){ sqlite3ExprListDelete(db, p->aColExpr); sqlite3DbFree(db, p->zColAff); if( p->isResized ) sqlite3DbFree(db, (void *)p->azColl); -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +#ifdef SQLITE_ENABLE_STAT4 sqlite3_free(p->aiRowEst); #endif sqlite3DbFree(db, p); @@ -104402,7 +110554,7 @@ SQLITE_PRIVATE void sqlite3UnlinkAndDeleteIndex(sqlite3 *db, int iDb, const char p->pNext = pIndex->pNext; } } - freeIndex(db, pIndex); + sqlite3FreeIndex(db, pIndex); } db->mDbFlags |= DBFLAG_SchemaChange; } @@ -104469,17 +110621,22 @@ SQLITE_PRIVATE void sqlite3ResetOneSchema(sqlite3 *db, int iDb){ SQLITE_PRIVATE void sqlite3ResetAllSchemasOfConnection(sqlite3 *db){ int i; sqlite3BtreeEnterAll(db); - assert( db->nSchemaLock==0 ); for(i=0; inDb; i++){ Db *pDb = &db->aDb[i]; if( pDb->pSchema ){ - sqlite3SchemaClear(pDb->pSchema); + if( db->nSchemaLock==0 ){ + sqlite3SchemaClear(pDb->pSchema); + }else{ + DbSetProperty(db, i, DB_ResetWanted); + } } } db->mDbFlags &= ~(DBFLAG_SchemaChange|DBFLAG_SchemaKnownOk); sqlite3VtabUnlockList(db); sqlite3BtreeLeaveAll(db); - sqlite3CollapseDatabaseArray(db); + if( db->nSchemaLock==0 ){ + sqlite3CollapseDatabaseArray(db); + } } /* @@ -104499,6 +110656,7 @@ SQLITE_PRIVATE void sqlite3DeleteColumnNames(sqlite3 *db, Table *pTable){ assert( pTable!=0 ); if( (pCol = pTable->aCol)!=0 ){ for(i=0; inCol; i++, pCol++){ + assert( pCol->zName==0 || pCol->hName==sqlite3StrIHash(pCol->zName) ); sqlite3DbFree(db, pCol->zName); sqlite3ExprDelete(db, pCol->pDflt); sqlite3DbFree(db, pCol->zColl); @@ -104527,10 +110685,14 @@ static void SQLITE_NOINLINE deleteTable(sqlite3 *db, Table *pTable){ #ifdef SQLITE_DEBUG /* Record the number of outstanding lookaside allocations in schema Tables - ** prior to doing any free() operations. Since schema Tables do not use - ** lookaside, this number should not change. */ + ** prior to doing any free() operations. Since schema Tables do not use + ** lookaside, this number should not change. + ** + ** If malloc has already failed, it may be that it failed while allocating + ** a Table object that was going to be marked ephemeral. So do not check + ** that no lookaside memory is used in this case either. */ int nLookaside = 0; - if( db && (pTable->tabFlags & TF_Ephemeral)==0 ){ + if( db && !db->mallocFailed && (pTable->tabFlags & TF_Ephemeral)==0 ){ nLookaside = sqlite3LookasideUsed(db, 0); } #endif @@ -104548,7 +110710,7 @@ static void SQLITE_NOINLINE deleteTable(sqlite3 *db, Table *pTable){ assert( db==0 || sqlite3SchemaMutexHeld(db, 0, pIndex->pSchema) ); assert( pOld==pIndex || pOld==0 ); } - freeIndex(db, pIndex); + sqlite3FreeIndex(db, pIndex); } /* Delete any foreign keys attached to this table. */ @@ -104706,7 +110868,7 @@ SQLITE_PRIVATE int sqlite3TwoPartName( return -1; } }else{ - assert( db->init.iDb==0 || db->init.busy + assert( db->init.iDb==0 || db->init.busy || IN_RENAME_OBJECT || (db->mDbFlags & DBFLAG_Vacuum)!=0); iDb = db->init.iDb; *pUnqual = pName1; @@ -104714,19 +110876,61 @@ SQLITE_PRIVATE int sqlite3TwoPartName( return iDb; } +/* +** True if PRAGMA writable_schema is ON +*/ +SQLITE_PRIVATE int sqlite3WritableSchema(sqlite3 *db){ + testcase( (db->flags&(SQLITE_WriteSchema|SQLITE_Defensive))==0 ); + testcase( (db->flags&(SQLITE_WriteSchema|SQLITE_Defensive))== + SQLITE_WriteSchema ); + testcase( (db->flags&(SQLITE_WriteSchema|SQLITE_Defensive))== + SQLITE_Defensive ); + testcase( (db->flags&(SQLITE_WriteSchema|SQLITE_Defensive))== + (SQLITE_WriteSchema|SQLITE_Defensive) ); + return (db->flags&(SQLITE_WriteSchema|SQLITE_Defensive))==SQLITE_WriteSchema; +} + /* ** This routine is used to check if the UTF-8 string zName is a legal ** unqualified name for a new schema object (table, index, view or ** trigger). All names are legal except those that begin with the string ** "sqlite_" (in upper, lower or mixed case). This portion of the namespace ** is reserved for internal use. +** +** When parsing the sqlite_master table, this routine also checks to +** make sure the "type", "name", and "tbl_name" columns are consistent +** with the SQL. */ -SQLITE_PRIVATE int sqlite3CheckObjectName(Parse *pParse, const char *zName){ - if( !pParse->db->init.busy && pParse->nested==0 - && (pParse->db->flags & SQLITE_WriteSchema)==0 - && 0==sqlite3StrNICmp(zName, "sqlite_", 7) ){ - sqlite3ErrorMsg(pParse, "object name reserved for internal use: %s", zName); - return SQLITE_ERROR; +SQLITE_PRIVATE int sqlite3CheckObjectName( + Parse *pParse, /* Parsing context */ + const char *zName, /* Name of the object to check */ + const char *zType, /* Type of this object */ + const char *zTblName /* Parent table name for triggers and indexes */ +){ + sqlite3 *db = pParse->db; + if( sqlite3WritableSchema(db) || db->init.imposterTable ){ + /* Skip these error checks for writable_schema=ON */ + return SQLITE_OK; + } + if( db->init.busy ){ + if( sqlite3_stricmp(zType, db->init.azInit[0]) + || sqlite3_stricmp(zName, db->init.azInit[1]) + || sqlite3_stricmp(zTblName, db->init.azInit[2]) + ){ + if( sqlite3Config.bExtraSchemaChecks ){ + sqlite3ErrorMsg(pParse, ""); /* corruptSchema() will supply the error */ + return SQLITE_ERROR; + } + } + }else{ + if( (pParse->nested==0 && 0==sqlite3StrNICmp(zName, "sqlite_", 7)) + || (sqlite3ReadOnlyShadowTables(db) && sqlite3ShadowTableName(db, zName)) + ){ + sqlite3ErrorMsg(pParse, "object name reserved for internal use: %s", + zName); + return SQLITE_ERROR; + } + } return SQLITE_OK; } @@ -104741,10 +110945,12 @@ SQLITE_PRIVATE Index *sqlite3PrimaryKeyIndex(Table *pTab){ } /* -** Return the column of index pIdx that corresponds to table -** column iCol. Return -1 if not found. +** Convert an table column number into a index column number. That is, +** for the column iCol in the table (as defined by the CREATE TABLE statement) +** find the (first) offset of that column in index pIdx. Or return -1 +** if column iCol is not used in index pIdx. */ -SQLITE_PRIVATE i16 sqlite3ColumnOfIndex(Index *pIdx, i16 iCol){ +SQLITE_PRIVATE i16 sqlite3TableColumnToIndex(Index *pIdx, i16 iCol){ int i; for(i=0; inColumn; i++){ if( iCol==pIdx->aiColumn[i] ) return i; @@ -104752,6 +110958,84 @@ SQLITE_PRIVATE i16 sqlite3ColumnOfIndex(Index *pIdx, i16 iCol){ return -1; } +#ifndef SQLITE_OMIT_GENERATED_COLUMNS +/* Convert a storage column number into a table column number. +** +** The storage column number (0,1,2,....) is the index of the value +** as it appears in the record on disk. The true column number +** is the index (0,1,2,...) of the column in the CREATE TABLE statement. +** +** The storage column number is less than the table column number if +** and only there are VIRTUAL columns to the left. +** +** If SQLITE_OMIT_GENERATED_COLUMNS, this routine is a no-op macro. +*/ +SQLITE_PRIVATE i16 sqlite3StorageColumnToTable(Table *pTab, i16 iCol){ + if( pTab->tabFlags & TF_HasVirtual ){ + int i; + for(i=0; i<=iCol; i++){ + if( pTab->aCol[i].colFlags & COLFLAG_VIRTUAL ) iCol++; + } + } + return iCol; +} +#endif + +#ifndef SQLITE_OMIT_GENERATED_COLUMNS +/* Convert a table column number into a storage column number. +** +** The storage column number (0,1,2,....) is the index of the value +** as it appears in the record on disk. Or, if the input column is +** the N-th virtual column (zero-based) then the storage number is +** the number of non-virtual columns in the table plus N. +** +** The true column number is the index (0,1,2,...) of the column in +** the CREATE TABLE statement. +** +** If the input column is a VIRTUAL column, then it should not appear +** in storage. But the value sometimes is cached in registers that +** follow the range of registers used to construct storage. This +** avoids computing the same VIRTUAL column multiple times, and provides +** values for use by OP_Param opcodes in triggers. Hence, if the +** input column is a VIRTUAL table, put it after all the other columns. +** +** In the following, N means "normal column", S means STORED, and +** V means VIRTUAL. Suppose the CREATE TABLE has columns like this: +** +** CREATE TABLE ex(N,S,V,N,S,V,N,S,V); +** -- 0 1 2 3 4 5 6 7 8 +** +** Then the mapping from this function is as follows: +** +** INPUTS: 0 1 2 3 4 5 6 7 8 +** OUTPUTS: 0 1 6 2 3 7 4 5 8 +** +** So, in other words, this routine shifts all the virtual columns to +** the end. +** +** If SQLITE_OMIT_GENERATED_COLUMNS then there are no virtual columns and +** this routine is a no-op macro. If the pTab does not have any virtual +** columns, then this routine is no-op that always return iCol. If iCol +** is negative (indicating the ROWID column) then this routine return iCol. +*/ +SQLITE_PRIVATE i16 sqlite3TableColumnToStorage(Table *pTab, i16 iCol){ + int i; + i16 n; + assert( iColnCol ); + if( (pTab->tabFlags & TF_HasVirtual)==0 || iCol<0 ) return iCol; + for(i=0, n=0; iaCol[i].colFlags & COLFLAG_VIRTUAL)==0 ) n++; + } + if( pTab->aCol[i].colFlags & COLFLAG_VIRTUAL ){ + /* iCol is a virtual column itself */ + return pTab->nNVCol + i - n; + }else{ + /* iCol is a normal or stored column */ + return n; + } +} +#endif + /* ** Begin constructing a new table representation in memory. This is ** the first of several action routines that get called in response @@ -104801,10 +111085,13 @@ SQLITE_PRIVATE void sqlite3StartTable( } if( !OMIT_TEMPDB && isTemp ) iDb = 1; zName = sqlite3NameFromToken(db, pName); + if( IN_RENAME_OBJECT ){ + sqlite3RenameTokenMap(pParse, (void*)zName, pName); + } } pParse->sNameToken = *pName; if( zName==0 ) return; - if( SQLITE_OK!=sqlite3CheckObjectName(pParse, zName) ){ + if( sqlite3CheckObjectName(pParse, zName, isView?"view":"table", zName) ){ goto begin_table_error; } if( db->init.iDb==1 ) isTemp = 1; @@ -104836,7 +111123,7 @@ SQLITE_PRIVATE void sqlite3StartTable( ** and types will be used, so there is no need to test for namespace ** collisions. */ - if( !IN_DECLARE_VTAB ){ + if( !IN_SPECIAL_PARSE ){ char *zDb = db->aDb[iDb].zDbSName; if( SQLITE_OK!=sqlite3ReadSchema(pParse) ){ goto begin_table_error; @@ -104995,6 +111282,7 @@ SQLITE_PRIVATE void sqlite3AddColumn(Parse *pParse, Token *pName, Token *pType){ } z = sqlite3DbMallocRaw(db, pName->n + pType->n + 2); if( z==0 ) return; + if( IN_RENAME_OBJECT ) sqlite3RenameTokenMap(pParse, (void*)z, pName); memcpy(z, pName->z, pName->n); z[pName->n] = 0; sqlite3Dequote(z); @@ -105017,6 +111305,7 @@ SQLITE_PRIVATE void sqlite3AddColumn(Parse *pParse, Token *pName, Token *pType){ pCol = &p->aCol[p->nCol]; memset(pCol, 0, sizeof(p->aCol[0])); pCol->zName = z; + pCol->hName = sqlite3StrIHash(z); sqlite3ColumnPropertiesFromName(p, pCol); if( pType->n==0 ){ @@ -105038,6 +111327,7 @@ SQLITE_PRIVATE void sqlite3AddColumn(Parse *pParse, Token *pName, Token *pType){ pCol->colFlags |= COLFLAG_HASTYPE; } p->nCol++; + p->nNVCol++; pParse->constraintName.n = 0; } @@ -105182,10 +111472,17 @@ SQLITE_PRIVATE void sqlite3AddDefaultValue( sqlite3 *db = pParse->db; p = pParse->pNewTable; if( p!=0 ){ + int isInit = db->init.busy && db->init.iDb!=1; pCol = &(p->aCol[p->nCol-1]); - if( !sqlite3ExprIsConstantOrFunction(pExpr, db->init.busy) ){ + if( !sqlite3ExprIsConstantOrFunction(pExpr, isInit) ){ sqlite3ErrorMsg(pParse, "default value of column [%s] is not constant", pCol->zName); +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + }else if( pCol->colFlags & COLFLAG_GENERATED ){ + testcase( pCol->colFlags & COLFLAG_VIRTUAL ); + testcase( pCol->colFlags & COLFLAG_STORED ); + sqlite3ErrorMsg(pParse, "cannot use DEFAULT on a generated column"); +#endif }else{ /* A copy of pExpr is used instead of the original, as pExpr contains ** tokens that point to volatile memory. @@ -105201,6 +111498,9 @@ SQLITE_PRIVATE void sqlite3AddDefaultValue( sqlite3DbFree(db, x.u.zToken); } } + if( IN_RENAME_OBJECT ){ + sqlite3RenameExprUnmap(pParse, pExpr); + } sqlite3ExprDelete(db, pExpr); } @@ -105217,7 +111517,7 @@ SQLITE_PRIVATE void sqlite3AddDefaultValue( ** accept it. This routine does the necessary conversion. It converts ** the expression given in its argument from a TK_STRING into a TK_ID ** if the expression is just a TK_STRING with an optional COLLATE clause. -** If the epxression is anything other than TK_STRING, the expression is +** If the expression is anything other than TK_STRING, the expression is ** unchanged. */ static void sqlite3StringToId(Expr *p){ @@ -105228,6 +111528,21 @@ static void sqlite3StringToId(Expr *p){ } } +/* +** Tag the given column as being part of the PRIMARY KEY +*/ +static void makeColumnPartOfPrimaryKey(Parse *pParse, Column *pCol){ + pCol->colFlags |= COLFLAG_PRIMKEY; +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + if( pCol->colFlags & COLFLAG_GENERATED ){ + testcase( pCol->colFlags & COLFLAG_VIRTUAL ); + testcase( pCol->colFlags & COLFLAG_STORED ); + sqlite3ErrorMsg(pParse, + "generated columns cannot be part of the PRIMARY KEY"); + } +#endif +} + /* ** Designate the PRIMARY KEY for the table. pList is a list of names ** of columns that form the primary key. If pList is NULL, then the @@ -105267,7 +111582,7 @@ SQLITE_PRIVATE void sqlite3AddPrimaryKey( if( pList==0 ){ iCol = pTab->nCol - 1; pCol = &pTab->aCol[iCol]; - pCol->colFlags |= COLFLAG_PRIMKEY; + makeColumnPartOfPrimaryKey(pParse, pCol); nTerm = 1; }else{ nTerm = pList->nExpr; @@ -105280,7 +111595,7 @@ SQLITE_PRIVATE void sqlite3AddPrimaryKey( for(iCol=0; iColnCol; iCol++){ if( sqlite3StrICmp(zCName, pTab->aCol[iCol].zName)==0 ){ pCol = &pTab->aCol[iCol]; - pCol->colFlags |= COLFLAG_PRIMKEY; + makeColumnPartOfPrimaryKey(pParse, pCol); break; } } @@ -105292,11 +111607,16 @@ SQLITE_PRIVATE void sqlite3AddPrimaryKey( && sqlite3StrICmp(sqlite3ColumnType(pCol,""), "INTEGER")==0 && sortOrder!=SQLITE_SO_DESC ){ + if( IN_RENAME_OBJECT && pList ){ + Expr *pCExpr = sqlite3ExprSkipCollate(pList->a[0].pExpr); + sqlite3RenameTokenRemap(pParse, &pTab->iPKey, pCExpr); + } pTab->iPKey = iCol; pTab->keyConf = (u8)onError; assert( autoInc==0 || autoInc==1 ); pTab->tabFlags |= autoInc*TF_Autoincrement; - if( pList ) pParse->iPkSortOrder = pList->a[0].sortOrder; + if( pList ) pParse->iPkSortOrder = pList->a[0].sortFlags; + (void)sqlite3HasExplicitNulls(pParse, pList); }else if( autoInc ){ #ifndef SQLITE_OMIT_AUTOINCREMENT sqlite3ErrorMsg(pParse, "AUTOINCREMENT is only allowed on an " @@ -105373,41 +111693,58 @@ SQLITE_PRIVATE void sqlite3AddCollateType(Parse *pParse, Token *pToken){ } } -/* -** This function returns the collation sequence for database native text -** encoding identified by the string zName, length nName. -** -** If the requested collation sequence is not available, or not available -** in the database native encoding, the collation factory is invoked to -** request it. If the collation factory does not supply such a sequence, -** and the sequence is available in another text encoding, then that is -** returned instead. -** -** If no versions of the requested collations sequence are available, or -** another error occurs, NULL is returned and an error message written into -** pParse. -** -** This routine is a wrapper around sqlite3FindCollSeq(). This routine -** invokes the collation factory if the named collation cannot be found -** and generates an error message. -** -** See also: sqlite3FindCollSeq(), sqlite3GetCollSeq() +/* Change the most recently parsed column to be a GENERATED ALWAYS AS +** column. */ -SQLITE_PRIVATE CollSeq *sqlite3LocateCollSeq(Parse *pParse, const char *zName){ - sqlite3 *db = pParse->db; - u8 enc = ENC(db); - u8 initbusy = db->init.busy; - CollSeq *pColl; - - pColl = sqlite3FindCollSeq(db, enc, zName, initbusy); - if( !initbusy && (!pColl || !pColl->xCmp) ){ - pColl = sqlite3GetCollSeq(pParse, enc, pColl, zName); +SQLITE_PRIVATE void sqlite3AddGenerated(Parse *pParse, Expr *pExpr, Token *pType){ +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + u8 eType = COLFLAG_VIRTUAL; + Table *pTab = pParse->pNewTable; + Column *pCol; + if( pTab==0 ){ + /* generated column in an CREATE TABLE IF NOT EXISTS that already exists */ + goto generated_done; + } + pCol = &(pTab->aCol[pTab->nCol-1]); + if( IN_DECLARE_VTAB ){ + sqlite3ErrorMsg(pParse, "virtual tables cannot use computed columns"); + goto generated_done; } + if( pCol->pDflt ) goto generated_error; + if( pType ){ + if( pType->n==7 && sqlite3StrNICmp("virtual",pType->z,7)==0 ){ + /* no-op */ + }else if( pType->n==6 && sqlite3StrNICmp("stored",pType->z,6)==0 ){ + eType = COLFLAG_STORED; + }else{ + goto generated_error; + } + } + if( eType==COLFLAG_VIRTUAL ) pTab->nNVCol--; + pCol->colFlags |= eType; + assert( TF_HasVirtual==COLFLAG_VIRTUAL ); + assert( TF_HasStored==COLFLAG_STORED ); + pTab->tabFlags |= eType; + if( pCol->colFlags & COLFLAG_PRIMKEY ){ + makeColumnPartOfPrimaryKey(pParse, pCol); /* For the error message */ + } + pCol->pDflt = pExpr; + pExpr = 0; + goto generated_done; - return pColl; +generated_error: + sqlite3ErrorMsg(pParse, "error in generated column \"%s\"", + pCol->zName); +generated_done: + sqlite3ExprDelete(pParse->db, pExpr); +#else + /* Throw and error for the GENERATED ALWAYS AS clause if the + ** SQLITE_OMIT_GENERATED_COLUMNS compile-time option is used. */ + sqlite3ErrorMsg(pParse, "generated columns not supported"); + sqlite3ExprDelete(pParse->db, pExpr); +#endif } - /* ** Generate code that will increment the schema cookie. ** @@ -105610,13 +111947,88 @@ static void estimateIndexWidth(Index *pIdx){ pIdx->szIdxRow = sqlite3LogEst(wIndex*4); } -/* Return true if value x is found any of the first nCol entries of aiCol[] +/* Return true if column number x is any of the first nCol entries of aiCol[]. +** This is used to determine if the column number x appears in any of the +** first nCol entries of an index. */ static int hasColumn(const i16 *aiCol, int nCol, int x){ - while( nCol-- > 0 ) if( x==*(aiCol++) ) return 1; + while( nCol-- > 0 ){ + assert( aiCol[0]>=0 ); + if( x==*(aiCol++) ){ + return 1; + } + } return 0; } +/* +** Return true if any of the first nKey entries of index pIdx exactly +** match the iCol-th entry of pPk. pPk is always a WITHOUT ROWID +** PRIMARY KEY index. pIdx is an index on the same table. pIdx may +** or may not be the same index as pPk. +** +** The first nKey entries of pIdx are guaranteed to be ordinary columns, +** not a rowid or expression. +** +** This routine differs from hasColumn() in that both the column and the +** collating sequence must match for this routine, but for hasColumn() only +** the column name must match. +*/ +static int isDupColumn(Index *pIdx, int nKey, Index *pPk, int iCol){ + int i, j; + assert( nKey<=pIdx->nColumn ); + assert( iColnColumn,pPk->nKeyCol) ); + assert( pPk->idxType==SQLITE_IDXTYPE_PRIMARYKEY ); + assert( pPk->pTable->tabFlags & TF_WithoutRowid ); + assert( pPk->pTable==pIdx->pTable ); + testcase( pPk==pIdx ); + j = pPk->aiColumn[iCol]; + assert( j!=XN_ROWID && j!=XN_EXPR ); + for(i=0; iaiColumn[i]>=0 || j>=0 ); + if( pIdx->aiColumn[i]==j + && sqlite3StrICmp(pIdx->azColl[i], pPk->azColl[iCol])==0 + ){ + return 1; + } + } + return 0; +} + +/* Recompute the colNotIdxed field of the Index. +** +** colNotIdxed is a bitmask that has a 0 bit representing each indexed +** columns that are within the first 63 columns of the table. The +** high-order bit of colNotIdxed is always 1. All unindexed columns +** of the table have a 1. +** +** 2019-10-24: For the purpose of this computation, virtual columns are +** not considered to be covered by the index, even if they are in the +** index, because we do not trust the logic in whereIndexExprTrans() to be +** able to find all instances of a reference to the indexed table column +** and convert them into references to the index. Hence we always want +** the actual table at hand in order to recompute the virtual column, if +** necessary. +** +** The colNotIdxed mask is AND-ed with the SrcList.a[].colUsed mask +** to determine if the index is covering index. +*/ +static void recomputeColumnsNotIndexed(Index *pIdx){ + Bitmask m = 0; + int j; + Table *pTab = pIdx->pTable; + for(j=pIdx->nColumn-1; j>=0; j--){ + int x = pIdx->aiColumn[j]; + if( x>=0 && (pTab->aCol[x].colFlags & COLFLAG_VIRTUAL)==0 ){ + testcase( x==BMS-1 ); + testcase( x==BMS-2 ); + if( xcolNotIdxed = ~m; + assert( (pIdx->colNotIdxed>>63)==1 ); +} + /* ** This routine runs at the end of parsing a CREATE TABLE statement that ** has a WITHOUT ROWID clause. The job of this routine is to convert both @@ -105645,6 +112057,7 @@ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){ Index *pIdx; Index *pPk; int nPk; + int nExtra; int i, j; sqlite3 *db = pParse->db; Vdbe *v = pParse->pVdbe; @@ -105657,12 +112070,9 @@ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){ pTab->aCol[i].notNull = OE_Abort; } } + pTab->tabFlags |= TF_HasNotNull; } - /* The remaining transformations only apply to b-tree tables, not to - ** virtual tables */ - if( IN_DECLARE_VTAB ) return; - /* Convert the P3 operand of the OP_CreateBtree opcode from BTREE_INTKEY ** into BTREE_BLOBKEY. */ @@ -105681,15 +112091,20 @@ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){ pList = sqlite3ExprListAppend(pParse, 0, sqlite3ExprAlloc(db, TK_ID, &ipkToken, 0)); if( pList==0 ) return; - pList->a[0].sortOrder = pParse->iPkSortOrder; + if( IN_RENAME_OBJECT ){ + sqlite3RenameTokenRemap(pParse, pList->a[0].pExpr, &pTab->iPKey); + } + pList->a[0].sortFlags = pParse->iPkSortOrder; assert( pParse->pNewTable==pTab ); + pTab->iPKey = -1; sqlite3CreateIndex(pParse, 0, 0, 0, pList, pTab->keyConf, 0, 0, 0, 0, SQLITE_IDXTYPE_PRIMARYKEY); - if( db->mallocFailed ) return; + if( db->mallocFailed || pParse->nErr ) return; pPk = sqlite3PrimaryKeyIndex(pTab); - pTab->iPKey = -1; + assert( pPk->nKeyCol==1 ); }else{ pPk = sqlite3PrimaryKeyIndex(pTab); + assert( pPk!=0 ); /* ** Remove all redundant columns from the PRIMARY KEY. For example, change @@ -105697,9 +112112,12 @@ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){ ** code assumes the PRIMARY KEY contains no repeated columns. */ for(i=j=1; inKeyCol; i++){ - if( hasColumn(pPk->aiColumn, j, pPk->aiColumn[i]) ){ + if( isDupColumn(pPk, j, pPk, i) ){ pPk->nColumn--; }else{ + testcase( hasColumn(pPk->aiColumn, j, pPk->aiColumn[i]) ); + pPk->azColl[j] = pPk->azColl[i]; + pPk->aSortOrder[j] = pPk->aSortOrder[i]; pPk->aiColumn[j++] = pPk->aiColumn[i]; } } @@ -105708,7 +112126,7 @@ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){ assert( pPk!=0 ); pPk->isCovering = 1; if( !db->init.imposterTable ) pPk->uniqNotNull = 1; - nPk = pPk->nKeyCol; + nPk = pPk->nColumn = pPk->nKeyCol; /* Bypass the creation of the PRIMARY KEY btree and the sqlite_master ** table entry. This is only required if currently generating VDBE @@ -105729,7 +112147,10 @@ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){ int n; if( IsPrimaryKeyIndex(pIdx) ) continue; for(i=n=0; iaiColumn, pIdx->nKeyCol, pPk->aiColumn[i]) ) n++; + if( !isDupColumn(pIdx, pIdx->nKeyCol, pPk, i) ){ + testcase( hasColumn(pIdx->aiColumn, pIdx->nKeyCol, pPk->aiColumn[i]) ); + n++; + } } if( n==0 ){ /* This index is a superset of the primary key */ @@ -105738,9 +112159,14 @@ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){ } if( resizeIndexObject(db, pIdx, pIdx->nKeyCol+n) ) return; for(i=0, j=pIdx->nKeyCol; iaiColumn, pIdx->nKeyCol, pPk->aiColumn[i]) ){ + if( !isDupColumn(pIdx, pIdx->nKeyCol, pPk, i) ){ + testcase( hasColumn(pIdx->aiColumn, pIdx->nKeyCol, pPk->aiColumn[i]) ); pIdx->aiColumn[j] = pPk->aiColumn[i]; pIdx->azColl[j] = pPk->azColl[i]; + if( pPk->aSortOrder[i] ){ + /* See ticket https://www.sqlite.org/src/info/bba7b69f9849b5bf */ + pIdx->bAscKeyBug = 1; + } j++; } } @@ -105750,22 +112176,97 @@ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){ /* Add all table columns to the PRIMARY KEY index */ - if( nPknCol ){ - if( resizeIndexObject(db, pPk, pTab->nCol) ) return; - for(i=0, j=nPk; inCol; i++){ - if( !hasColumn(pPk->aiColumn, j, i) ){ - assert( jnColumn ); - pPk->aiColumn[j] = i; - pPk->azColl[j] = sqlite3StrBINARY; - j++; - } + nExtra = 0; + for(i=0; inCol; i++){ + if( !hasColumn(pPk->aiColumn, nPk, i) + && (pTab->aCol[i].colFlags & COLFLAG_VIRTUAL)==0 ) nExtra++; + } + if( resizeIndexObject(db, pPk, nPk+nExtra) ) return; + for(i=0, j=nPk; inCol; i++){ + if( !hasColumn(pPk->aiColumn, j, i) + && (pTab->aCol[i].colFlags & COLFLAG_VIRTUAL)==0 + ){ + assert( jnColumn ); + pPk->aiColumn[j] = i; + pPk->azColl[j] = sqlite3StrBINARY; + j++; } - assert( pPk->nColumn==j ); - assert( pTab->nCol==j ); - }else{ - pPk->nColumn = pTab->nCol; + } + assert( pPk->nColumn==j ); + assert( pTab->nNVCol<=j ); + recomputeColumnsNotIndexed(pPk); +} + + +#ifndef SQLITE_OMIT_VIRTUALTABLE +/* +** Return true if pTab is a virtual table and zName is a shadow table name +** for that virtual table. +*/ +SQLITE_PRIVATE int sqlite3IsShadowTableOf(sqlite3 *db, Table *pTab, const char *zName){ + int nName; /* Length of zName */ + Module *pMod; /* Module for the virtual table */ + + if( !IsVirtual(pTab) ) return 0; + nName = sqlite3Strlen30(pTab->zName); + if( sqlite3_strnicmp(zName, pTab->zName, nName)!=0 ) return 0; + if( zName[nName]!='_' ) return 0; + pMod = (Module*)sqlite3HashFind(&db->aModule, pTab->azModuleArg[0]); + if( pMod==0 ) return 0; + if( pMod->pModule->iVersion<3 ) return 0; + if( pMod->pModule->xShadowName==0 ) return 0; + return pMod->pModule->xShadowName(zName+nName+1); +} +#endif /* ifndef SQLITE_OMIT_VIRTUALTABLE */ + +#ifndef SQLITE_OMIT_VIRTUALTABLE +/* +** Return true if zName is a shadow table name in the current database +** connection. +** +** zName is temporarily modified while this routine is running, but is +** restored to its original value prior to this routine returning. +*/ +SQLITE_PRIVATE int sqlite3ShadowTableName(sqlite3 *db, const char *zName){ + char *zTail; /* Pointer to the last "_" in zName */ + Table *pTab; /* Table that zName is a shadow of */ + zTail = strrchr(zName, '_'); + if( zTail==0 ) return 0; + *zTail = 0; + pTab = sqlite3FindTable(db, zName, 0); + *zTail = '_'; + if( pTab==0 ) return 0; + if( !IsVirtual(pTab) ) return 0; + return sqlite3IsShadowTableOf(db, pTab, zName); +} +#endif /* ifndef SQLITE_OMIT_VIRTUALTABLE */ + + +#ifdef SQLITE_DEBUG +/* +** Mark all nodes of an expression as EP_Immutable, indicating that +** they should not be changed. Expressions attached to a table or +** index definition are tagged this way to help ensure that we do +** not pass them into code generator routines by mistake. +*/ +static int markImmutableExprStep(Walker *pWalker, Expr *pExpr){ + ExprSetVVAProperty(pExpr, EP_Immutable); + return WRC_Continue; +} +static void markExprListImmutable(ExprList *pList){ + if( pList ){ + Walker w; + memset(&w, 0, sizeof(w)); + w.xExprCallback = markImmutableExprStep; + w.xSelectCallback = sqlite3SelectWalkNoop; + w.xSelectCallback2 = 0; + sqlite3WalkExprList(&w, pList); } } +#else +#define markExprListImmutable(X) /* no-op */ +#endif /* SQLITE_DEBUG */ + /* ** This routine is called to report the final ")" that terminates @@ -105806,6 +112307,10 @@ SQLITE_PRIVATE void sqlite3EndTable( p = pParse->pNewTable; if( p==0 ) return; + if( pSelect==0 && sqlite3ShadowTableName(db, p->zName) ){ + p->tabFlags |= TF_Shadow; + } + /* If the db->init.busy is 1 it means we are reading the SQL off the ** "sqlite_master" or "sqlite_temp_master" table on the disk. ** So do not write to the disk again. Extract the root page number @@ -105824,6 +112329,11 @@ SQLITE_PRIVATE void sqlite3EndTable( if( p->tnum==1 ) p->tabFlags |= TF_Readonly; } + assert( (p->tabFlags & TF_HasPrimaryKey)==0 + || p->iPKey>=0 || sqlite3PrimaryKeyIndex(p)!=0 ); + assert( (p->tabFlags & TF_HasPrimaryKey)!=0 + || (p->iPKey<0 && sqlite3PrimaryKeyIndex(p)==0) ); + /* Special processing for WITHOUT ROWID Tables */ if( tabOpts & TF_WithoutRowid ){ if( (p->tabFlags & TF_Autoincrement) ){ @@ -105833,12 +112343,11 @@ SQLITE_PRIVATE void sqlite3EndTable( } if( (p->tabFlags & TF_HasPrimaryKey)==0 ){ sqlite3ErrorMsg(pParse, "PRIMARY KEY missing on table %s", p->zName); - }else{ - p->tabFlags |= TF_WithoutRowid | TF_NoVisibleRowid; - convertToWithoutRowidTable(pParse, p); + return; } + p->tabFlags |= TF_WithoutRowid | TF_NoVisibleRowid; + convertToWithoutRowidTable(pParse, p); } - iDb = sqlite3SchemaToIndex(db, p->pSchema); #ifndef SQLITE_OMIT_CHECK @@ -105846,8 +112355,47 @@ SQLITE_PRIVATE void sqlite3EndTable( */ if( p->pCheck ){ sqlite3ResolveSelfReference(pParse, p, NC_IsCheck, 0, p->pCheck); + if( pParse->nErr ){ + /* If errors are seen, delete the CHECK constraints now, else they might + ** actually be used if PRAGMA writable_schema=ON is set. */ + sqlite3ExprListDelete(db, p->pCheck); + p->pCheck = 0; + }else{ + markExprListImmutable(p->pCheck); + } } #endif /* !defined(SQLITE_OMIT_CHECK) */ +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + if( p->tabFlags & TF_HasGenerated ){ + int ii, nNG = 0; + testcase( p->tabFlags & TF_HasVirtual ); + testcase( p->tabFlags & TF_HasStored ); + for(ii=0; iinCol; ii++){ + u32 colFlags = p->aCol[ii].colFlags; + if( (colFlags & COLFLAG_GENERATED)!=0 ){ + Expr *pX = p->aCol[ii].pDflt; + testcase( colFlags & COLFLAG_VIRTUAL ); + testcase( colFlags & COLFLAG_STORED ); + if( sqlite3ResolveSelfReference(pParse, p, NC_GenCol, pX, 0) ){ + /* If there are errors in resolving the expression, change the + ** expression to a NULL. This prevents code generators that operate + ** on the expression from inserting extra parts into the expression + ** tree that have been allocated from lookaside memory, which is + ** illegal in a schema and will lead to errors or heap corruption + ** when the database connection closes. */ + sqlite3ExprDelete(db, pX); + p->aCol[ii].pDflt = sqlite3ExprAlloc(db, TK_NULL, 0, 0); + } + }else{ + nNG++; + } + } + if( nNG==0 ){ + sqlite3ErrorMsg(pParse, "must have at least one non-generated column"); + return; + } + } +#endif /* Estimate the average row size for the table and for all implied indices */ estimateTableWidth(p); @@ -105921,10 +112469,10 @@ SQLITE_PRIVATE void sqlite3EndTable( addrTop = sqlite3VdbeCurrentAddr(v) + 1; sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, addrTop); if( pParse->nErr ) return; - pSelTab = sqlite3ResultSetOfSelect(pParse, pSelect); + pSelTab = sqlite3ResultSetOfSelect(pParse, pSelect, SQLITE_AFF_BLOB); if( pSelTab==0 ) return; assert( p->aCol==0 ); - p->nCol = pSelTab->nCol; + p->nCol = p->nNVCol = pSelTab->nCol; p->aCol = pSelTab->aCol; pSelTab->nCol = 0; pSelTab->aCol = 0; @@ -105997,7 +112545,6 @@ SQLITE_PRIVATE void sqlite3EndTable( sqlite3MPrintf(db, "tbl_name='%q' AND type!='trigger'", p->zName)); } - /* Add the table to the in-memory representation of the database. */ if( db->init.busy ){ @@ -106068,7 +112615,13 @@ SQLITE_PRIVATE void sqlite3CreateView( ** allocated rather than point to the input string - which means that ** they will persist after the current sqlite3_exec() call returns. */ - p->pSelect = sqlite3SelectDup(db, pSelect, EXPRDUP_REDUCE); + pSelect->selFlags |= SF_View; + if( IN_RENAME_OBJECT ){ + p->pSelect = pSelect; + pSelect = 0; + }else{ + p->pSelect = sqlite3SelectDup(db, pSelect, EXPRDUP_REDUCE); + } p->pCheck = sqlite3ExprListDup(db, pCNames, EXPRDUP_REDUCE); if( db->mallocFailed ) goto create_view_fail; @@ -106093,6 +112646,9 @@ SQLITE_PRIVATE void sqlite3CreateView( create_view_fail: sqlite3SelectDelete(db, pSelect); + if( IN_RENAME_OBJECT ){ + sqlite3RenameExprlistUnmap(pParse, pCNames); + } sqlite3ExprListDelete(db, pCNames); return; } @@ -106166,20 +112722,27 @@ SQLITE_PRIVATE int sqlite3ViewGetColumnNames(Parse *pParse, Table *pTable){ assert( pTable->pSelect ); pSel = sqlite3SelectDup(db, pTable->pSelect, 0); if( pSel ){ +#ifndef SQLITE_OMIT_ALTERTABLE + u8 eParseMode = pParse->eParseMode; + pParse->eParseMode = PARSE_MODE_NORMAL; +#endif n = pParse->nTab; sqlite3SrcListAssignCursors(pParse, pSel->pSrc); pTable->nCol = -1; - db->lookaside.bDisable++; + DisableLookaside; #ifndef SQLITE_OMIT_AUTHORIZATION xAuth = db->xAuth; db->xAuth = 0; - pSelTab = sqlite3ResultSetOfSelect(pParse, pSel); + pSelTab = sqlite3ResultSetOfSelect(pParse, pSel, SQLITE_AFF_NONE); db->xAuth = xAuth; #else - pSelTab = sqlite3ResultSetOfSelect(pParse, pSel); + pSelTab = sqlite3ResultSetOfSelect(pParse, pSel, SQLITE_AFF_NONE); #endif pParse->nTab = n; - if( pTable->pCheck ){ + if( pSelTab==0 ){ + pTable->nCol = 0; + nErr++; + }else if( pTable->pCheck ){ /* CREATE VIEW name(arglist) AS ... ** The names of the columns in the table are taken from ** arglist which is stored in pTable->pCheck. The pCheck field @@ -106192,9 +112755,10 @@ SQLITE_PRIVATE int sqlite3ViewGetColumnNames(Parse *pParse, Table *pTable){ && pParse->nErr==0 && pTable->nCol==pSel->pEList->nExpr ){ - sqlite3SelectAddColumnTypeAndCollation(pParse, pTable, pSel); + sqlite3SelectAddColumnTypeAndCollation(pParse, pTable, pSel, + SQLITE_AFF_NONE); } - }else if( pSelTab ){ + }else{ /* CREATE VIEW name AS... without an argument list. Construct ** the column names from the SELECT statement that defines the view. */ @@ -106204,17 +112768,23 @@ SQLITE_PRIVATE int sqlite3ViewGetColumnNames(Parse *pParse, Table *pTable){ pSelTab->nCol = 0; pSelTab->aCol = 0; assert( sqlite3SchemaMutexHeld(db, 0, pTable->pSchema) ); - }else{ - pTable->nCol = 0; - nErr++; } + pTable->nNVCol = pTable->nCol; sqlite3DeleteTable(db, pSelTab); sqlite3SelectDelete(db, pSel); - db->lookaside.bDisable--; + EnableLookaside; +#ifndef SQLITE_OMIT_ALTERTABLE + pParse->eParseMode = eParseMode; +#endif } else { nErr++; } pTable->pSchema->schemaFlags |= DB_UnresetViews; + if( db->mallocFailed ){ + sqlite3DeleteColumnNames(db, pTable); + pTable->aCol = 0; + pTable->nCol = 0; + } #endif /* SQLITE_OMIT_VIEW */ return nErr; } @@ -106293,7 +112863,7 @@ SQLITE_PRIVATE void sqlite3RootPageMoved(sqlite3 *db, int iDb, int iFrom, int iT static void destroyRootPage(Parse *pParse, int iTable, int iDb){ Vdbe *v = sqlite3GetVdbe(pParse); int r1 = sqlite3GetTempReg(pParse); - assert( iTable>1 ); + if( iTable<2 ) sqlite3ErrorMsg(pParse, "corrupt schema"); sqlite3VdbeAddOp3(v, OP_Destroy, iTable, r1, iDb); sqlite3MayAbort(pParse); #ifndef SQLITE_OMIT_AUTOVACUUM @@ -106452,12 +113022,44 @@ SQLITE_PRIVATE void sqlite3CodeDropTable(Parse *pParse, Table *pTab, int iDb, in */ if( IsVirtual(pTab) ){ sqlite3VdbeAddOp4(v, OP_VDestroy, iDb, 0, 0, pTab->zName, 0); + sqlite3MayAbort(pParse); } sqlite3VdbeAddOp4(v, OP_DropTable, iDb, 0, 0, pTab->zName, 0); sqlite3ChangeCookie(pParse, iDb); sqliteViewResetAll(db, iDb); } +/* +** Return TRUE if shadow tables should be read-only in the current +** context. +*/ +SQLITE_PRIVATE int sqlite3ReadOnlyShadowTables(sqlite3 *db){ +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( (db->flags & SQLITE_Defensive)!=0 + && db->pVtabCtx==0 + && db->nVdbeExec==0 + ){ + return 1; + } +#endif + return 0; +} + +/* +** Return true if it is not allowed to drop the given table +*/ +static int tableMayNotBeDropped(sqlite3 *db, Table *pTab){ + if( sqlite3StrNICmp(pTab->zName, "sqlite_", 7)==0 ){ + if( sqlite3StrNICmp(pTab->zName+7, "stat", 4)==0 ) return 0; + if( sqlite3StrNICmp(pTab->zName+7, "parameters", 10)==0 ) return 0; + return 1; + } + if( (pTab->tabFlags & TF_Shadow)!=0 && sqlite3ReadOnlyShadowTables(db) ){ + return 1; + } + return 0; +} + /* ** This routine is called to do the work of a DROP TABLE statement. ** pName is the name of the table to be dropped. @@ -106527,8 +113129,7 @@ SQLITE_PRIVATE void sqlite3DropTable(Parse *pParse, SrcList *pName, int isView, } } #endif - if( sqlite3StrNICmp(pTab->zName, "sqlite_", 7)==0 - && sqlite3StrNICmp(pTab->zName, "sqlite_stat", 11)!=0 ){ + if( tableMayNotBeDropped(db, pTab) ){ sqlite3ErrorMsg(pParse, "table %s may not be dropped", pTab->zName); goto exit_drop_table; } @@ -106553,8 +113154,10 @@ SQLITE_PRIVATE void sqlite3DropTable(Parse *pParse, SrcList *pName, int isView, v = sqlite3GetVdbe(pParse); if( v ){ sqlite3BeginWriteOperation(pParse, 1, iDb); - sqlite3ClearStatTables(pParse, iDb, "tbl", pTab->zName); - sqlite3FkDropTable(pParse, pName, pTab); + if( !isView ){ + sqlite3ClearStatTables(pParse, iDb, "tbl", pTab->zName); + sqlite3FkDropTable(pParse, pName, pTab); + } sqlite3CodeDropTable(pParse, pTab, iDb, isView); } @@ -106618,7 +113221,7 @@ SQLITE_PRIVATE void sqlite3CreateForeignKey( nByte = sizeof(*pFKey) + (nCol-1)*sizeof(pFKey->aCol[0]) + pTo->n + 1; if( pToCol ){ for(i=0; inExpr; i++){ - nByte += sqlite3Strlen30(pToCol->a[i].zName) + 1; + nByte += sqlite3Strlen30(pToCol->a[i].zEName) + 1; } } pFKey = sqlite3DbMallocZero(db, nByte ); @@ -106629,6 +113232,9 @@ SQLITE_PRIVATE void sqlite3CreateForeignKey( pFKey->pNextFrom = p->pFKey; z = (char*)&pFKey->aCol[nCol]; pFKey->zTo = z; + if( IN_RENAME_OBJECT ){ + sqlite3RenameTokenMap(pParse, (void*)z, pTo); + } memcpy(z, pTo->z, pTo->n); z[pTo->n] = 0; sqlite3Dequote(z); @@ -106640,7 +113246,7 @@ SQLITE_PRIVATE void sqlite3CreateForeignKey( for(i=0; inCol; j++){ - if( sqlite3StrICmp(p->aCol[j].zName, pFromCol->a[i].zName)==0 ){ + if( sqlite3StrICmp(p->aCol[j].zName, pFromCol->a[i].zEName)==0 ){ pFKey->aCol[i].iFrom = j; break; } @@ -106648,16 +113254,22 @@ SQLITE_PRIVATE void sqlite3CreateForeignKey( if( j>=p->nCol ){ sqlite3ErrorMsg(pParse, "unknown column \"%s\" in foreign key definition", - pFromCol->a[i].zName); + pFromCol->a[i].zEName); goto fk_end; } + if( IN_RENAME_OBJECT ){ + sqlite3RenameTokenRemap(pParse, &pFKey->aCol[i], pFromCol->a[i].zEName); + } } } if( pToCol ){ for(i=0; ia[i].zName); + int n = sqlite3Strlen30(pToCol->a[i].zEName); pFKey->aCol[i].zCol = z; - memcpy(z, pToCol->a[i].zName, n); + if( IN_RENAME_OBJECT ){ + sqlite3RenameTokenRemap(pParse, z, pToCol->a[i].zEName); + } + memcpy(z, pToCol->a[i].zEName, n); z[n] = 0; z += n+1; } @@ -106787,10 +113399,27 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ sqlite3UniqueConstraint(pParse, OE_Abort, pIndex); sqlite3VdbeJumpHere(v, j2); }else{ + /* Most CREATE INDEX and REINDEX statements that are not UNIQUE can not + ** abort. The exception is if one of the indexed expressions contains a + ** user function that throws an exception when it is evaluated. But the + ** overhead of adding a statement journal to a CREATE INDEX statement is + ** very small (since most of the pages written do not contain content that + ** needs to be restored if the statement aborts), so we call + ** sqlite3MayAbort() for all CREATE INDEX statements. */ + sqlite3MayAbort(pParse); addr2 = sqlite3VdbeCurrentAddr(v); } sqlite3VdbeAddOp3(v, OP_SorterData, iSorter, regRecord, iIdx); - sqlite3VdbeAddOp1(v, OP_SeekEnd, iIdx); + if( !pIndex->bAscKeyBug ){ + /* This OP_SeekEnd opcode makes index insert for a REINDEX go much + ** faster by avoiding unnecessary seeks. But the optimization does + ** not work for UNIQUE constraint indexes on WITHOUT ROWID tables + ** with DESC primary keys, since those indexes have there keys in + ** a different order from the main table. + ** See ticket: https://www.sqlite.org/src/info/bba7b69f9849b5bf + */ + sqlite3VdbeAddOp1(v, OP_SeekEnd, iIdx); + } sqlite3VdbeAddOp2(v, OP_IdxInsert, iIdx, regRecord); sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); sqlite3ReleaseTempReg(pParse, regRecord); @@ -106837,6 +113466,27 @@ SQLITE_PRIVATE Index *sqlite3AllocateIndexObject( return p; } +/* +** If expression list pList contains an expression that was parsed with +** an explicit "NULLS FIRST" or "NULLS LAST" clause, leave an error in +** pParse and return non-zero. Otherwise, return zero. +*/ +SQLITE_PRIVATE int sqlite3HasExplicitNulls(Parse *pParse, ExprList *pList){ + if( pList ){ + int i; + for(i=0; inExpr; i++){ + if( pList->a[i].bNulls ){ + u8 sf = pList->a[i].sortFlags; + sqlite3ErrorMsg(pParse, "unsupported use of NULLS %s", + (sf==0 || sf==3) ? "FIRST" : "LAST" + ); + return 1; + } + } + } + return 0; +} + /* ** Create a new index for an SQL table. pName1.pName2 is the name of the index ** and pTblList is the name of the table that is to be indexed. Both will @@ -106888,6 +113538,9 @@ SQLITE_PRIVATE void sqlite3CreateIndex( if( SQLITE_OK!=sqlite3ReadSchema(pParse) ){ goto exit_create_index; } + if( sqlite3HasExplicitNulls(pParse, pList) ){ + goto exit_create_index; + } /* ** Find the table that is to be indexed. Return early if not found. @@ -106945,13 +113598,13 @@ SQLITE_PRIVATE void sqlite3CreateIndex( assert( pParse->nErr==0 ); if( sqlite3StrNICmp(pTab->zName, "sqlite_", 7)==0 && db->init.busy==0 + && pTblName!=0 #if SQLITE_USER_AUTHENTICATION && sqlite3UserAuthTable(pTab->zName)==0 #endif #ifdef SQLITE_ALLOW_SQLITE_MASTER_INDEX && sqlite3StrICmp(&pTab->zName[7],"master")!=0 #endif - && sqlite3StrNICmp(&pTab->zName[7],"altertab_",9)!=0 ){ sqlite3ErrorMsg(pParse, "table %s may not be indexed", pTab->zName); goto exit_create_index; @@ -106986,23 +113639,25 @@ SQLITE_PRIVATE void sqlite3CreateIndex( zName = sqlite3NameFromToken(db, pName); if( zName==0 ) goto exit_create_index; assert( pName->z!=0 ); - if( SQLITE_OK!=sqlite3CheckObjectName(pParse, zName) ){ + if( SQLITE_OK!=sqlite3CheckObjectName(pParse, zName,"index",pTab->zName) ){ goto exit_create_index; } - if( !db->init.busy ){ - if( sqlite3FindTable(db, zName, 0)!=0 ){ - sqlite3ErrorMsg(pParse, "there is already a table named %s", zName); - goto exit_create_index; + if( !IN_RENAME_OBJECT ){ + if( !db->init.busy ){ + if( sqlite3FindTable(db, zName, 0)!=0 ){ + sqlite3ErrorMsg(pParse, "there is already a table named %s", zName); + goto exit_create_index; + } } - } - if( sqlite3FindIndex(db, zName, pDb->zDbSName)!=0 ){ - if( !ifNotExist ){ - sqlite3ErrorMsg(pParse, "index %s already exists", zName); - }else{ - assert( !db->init.busy ); - sqlite3CodeVerifySchema(pParse, iDb); + if( sqlite3FindIndex(db, zName, pDb->zDbSName)!=0 ){ + if( !ifNotExist ){ + sqlite3ErrorMsg(pParse, "index %s already exists", zName); + }else{ + assert( !db->init.busy ); + sqlite3CodeVerifySchema(pParse, iDb); + } + goto exit_create_index; } - goto exit_create_index; } }else{ int n; @@ -107018,13 +113673,13 @@ SQLITE_PRIVATE void sqlite3CreateIndex( ** The following statement converts "sqlite3_autoindex..." into ** "sqlite3_butoindex..." in order to make the names distinct. ** The "vtab_err.test" test demonstrates the need of this statement. */ - if( IN_DECLARE_VTAB ) zName[7]++; + if( IN_SPECIAL_PARSE ) zName[7]++; } /* Check for authorization to create an index. */ #ifndef SQLITE_OMIT_AUTHORIZATION - { + if( !IN_RENAME_OBJECT ){ const char *zDb = pDb->zDbSName; if( sqlite3AuthCheck(pParse, SQLITE_INSERT, SCHEMA_TABLE(iDb), 0, zDb) ){ goto exit_create_index; @@ -107050,9 +113705,10 @@ SQLITE_PRIVATE void sqlite3CreateIndex( sqlite3ExprAlloc(db, TK_ID, &prevCol, 0)); if( pList==0 ) goto exit_create_index; assert( pList->nExpr==1 ); - sqlite3ExprListSetSortOrder(pList, sortOrder); + sqlite3ExprListSetSortOrder(pList, sortOrder, SQLITE_SO_UNDEFINED); }else{ sqlite3ExprListCheckLength(pParse, pList, "index"); + if( pParse->nErr ) goto exit_create_index; } /* Figure out how many bytes of space are required to store explicitly @@ -107071,6 +113727,7 @@ SQLITE_PRIVATE void sqlite3CreateIndex( */ nName = sqlite3Strlen30(zName); nExtraCol = pPk ? pPk->nKeyCol : 1; + assert( pList->nExpr + nExtraCol <= 32767 /* Fits in i16 */ ); pIndex = sqlite3AllocateIndexObject(db, pList->nExpr + nExtraCol, nName + nExtra + 1, &zExtra); if( db->mallocFailed ){ @@ -107111,7 +113768,12 @@ SQLITE_PRIVATE void sqlite3CreateIndex( ** TODO: Issue a warning if the table primary key is used as part of the ** index key. */ - for(i=0, pListItem=pList->a; inExpr; i++, pListItem++){ + pListItem = pList->a; + if( IN_RENAME_OBJECT ){ + pIndex->aColExpr = pList; + pList = 0; + } + for(i=0; inKeyCol; i++, pListItem++){ Expr *pCExpr; /* The i-th index expression */ int requestedSortOrder; /* ASC or DESC on the i-th expression */ const char *zColl; /* Collation sequence name */ @@ -107127,12 +113789,8 @@ SQLITE_PRIVATE void sqlite3CreateIndex( goto exit_create_index; } if( pIndex->aColExpr==0 ){ - ExprList *pCopy = sqlite3ExprListDup(db, pList, 0); - pIndex->aColExpr = pCopy; - if( !db->mallocFailed ){ - assert( pCopy!=0 ); - pListItem = &pCopy->a[i]; - } + pIndex->aColExpr = pList; + pList = 0; } j = XN_EXPR; pIndex->aiColumn[i] = XN_EXPR; @@ -107142,8 +113800,13 @@ SQLITE_PRIVATE void sqlite3CreateIndex( assert( j<=0x7fff ); if( j<0 ){ j = pTab->iPKey; - }else if( pTab->aCol[j].notNull==0 ){ - pIndex->uniqNotNull = 0; + }else{ + if( pTab->aCol[j].notNull==0 ){ + pIndex->uniqNotNull = 0; + } + if( pTab->aCol[j].colFlags & COLFLAG_VIRTUAL ){ + pIndex->bHasVCol = 1; + } } pIndex->aiColumn[i] = (i16)j; } @@ -107165,7 +113828,7 @@ SQLITE_PRIVATE void sqlite3CreateIndex( goto exit_create_index; } pIndex->azColl[i] = zColl; - requestedSortOrder = pListItem->sortOrder & sortOrderMask; + requestedSortOrder = pListItem->sortFlags & sortOrderMask; pIndex->aSortOrder[i] = (u8)requestedSortOrder; } @@ -107177,9 +113840,10 @@ SQLITE_PRIVATE void sqlite3CreateIndex( for(j=0; jnKeyCol; j++){ int x = pPk->aiColumn[j]; assert( x>=0 ); - if( hasColumn(pIndex->aiColumn, pIndex->nKeyCol, x) ){ + if( isDupColumn(pIndex, pIndex->nKeyCol, pPk, j) ){ pIndex->nColumn--; }else{ + testcase( hasColumn(pIndex->aiColumn,pIndex->nKeyCol,x) ); pIndex->aiColumn[i] = x; pIndex->azColl[i] = pPk->azColl[j]; pIndex->aSortOrder[i] = pPk->aSortOrder[j]; @@ -107197,12 +113861,13 @@ SQLITE_PRIVATE void sqlite3CreateIndex( /* If this index contains every column of its table, then mark ** it as a covering index */ assert( HasRowid(pTab) - || pTab->iPKey<0 || sqlite3ColumnOfIndex(pIndex, pTab->iPKey)>=0 ); + || pTab->iPKey<0 || sqlite3TableColumnToIndex(pIndex, pTab->iPKey)>=0 ); + recomputeColumnsNotIndexed(pIndex); if( pTblName!=0 && pIndex->nColumn>=pTab->nCol ){ pIndex->isCovering = 1; for(j=0; jnCol; j++){ if( j==pTab->iPKey ) continue; - if( sqlite3ColumnOfIndex(pIndex,j)>=0 ) continue; + if( sqlite3TableColumnToIndex(pIndex,j)>=0 ) continue; pIndex->isCovering = 0; break; } @@ -107265,130 +113930,147 @@ SQLITE_PRIVATE void sqlite3CreateIndex( } } if( idxType==SQLITE_IDXTYPE_PRIMARYKEY ) pIdx->idxType = idxType; + if( IN_RENAME_OBJECT ){ + pIndex->pNext = pParse->pNewIndex; + pParse->pNewIndex = pIndex; + pIndex = 0; + } goto exit_create_index; } } } - /* Link the new Index structure to its table and to the other - ** in-memory database structures. - */ - assert( pParse->nErr==0 ); - if( db->init.busy ){ - Index *p; - assert( !IN_DECLARE_VTAB ); - assert( sqlite3SchemaMutexHeld(db, 0, pIndex->pSchema) ); - p = sqlite3HashInsert(&pIndex->pSchema->idxHash, - pIndex->zName, pIndex); - if( p ){ - assert( p==pIndex ); /* Malloc must have failed */ - sqlite3OomFault(db); - goto exit_create_index; - } - db->mDbFlags |= DBFLAG_SchemaChange; - if( pTblName!=0 ){ - pIndex->tnum = db->init.newTnum; - } - } - - /* If this is the initial CREATE INDEX statement (or CREATE TABLE if the - ** index is an implied index for a UNIQUE or PRIMARY KEY constraint) then - ** emit code to allocate the index rootpage on disk and make an entry for - ** the index in the sqlite_master table and populate the index with - ** content. But, do not do this if we are simply reading the sqlite_master - ** table to parse the schema, or if this index is the PRIMARY KEY index - ** of a WITHOUT ROWID table. - ** - ** If pTblName==0 it means this index is generated as an implied PRIMARY KEY - ** or UNIQUE index in a CREATE TABLE statement. Since the table - ** has just been created, it contains no data and the index initialization - ** step can be skipped. - */ - else if( HasRowid(pTab) || pTblName!=0 ){ - Vdbe *v; - char *zStmt; - int iMem = ++pParse->nMem; - - v = sqlite3GetVdbe(pParse); - if( v==0 ) goto exit_create_index; - - sqlite3BeginWriteOperation(pParse, 1, iDb); + if( !IN_RENAME_OBJECT ){ - /* Create the rootpage for the index using CreateIndex. But before - ** doing so, code a Noop instruction and store its address in - ** Index.tnum. This is required in case this index is actually a - ** PRIMARY KEY and the table is actually a WITHOUT ROWID table. In - ** that case the convertToWithoutRowidTable() routine will replace - ** the Noop with a Goto to jump over the VDBE code generated below. */ - pIndex->tnum = sqlite3VdbeAddOp0(v, OP_Noop); - sqlite3VdbeAddOp3(v, OP_CreateBtree, iDb, iMem, BTREE_BLOBKEY); - - /* Gather the complete text of the CREATE INDEX statement into - ** the zStmt variable + /* Link the new Index structure to its table and to the other + ** in-memory database structures. */ - if( pStart ){ - int n = (int)(pParse->sLastToken.z - pName->z) + pParse->sLastToken.n; - if( pName->z[n-1]==';' ) n--; - /* A named index with an explicit CREATE INDEX statement */ - zStmt = sqlite3MPrintf(db, "CREATE%s INDEX %.*s", - onError==OE_None ? "" : " UNIQUE", n, pName->z); - }else{ - /* An automatic index created by a PRIMARY KEY or UNIQUE constraint */ - /* zStmt = sqlite3MPrintf(""); */ - zStmt = 0; + assert( pParse->nErr==0 ); + if( db->init.busy ){ + Index *p; + assert( !IN_SPECIAL_PARSE ); + assert( sqlite3SchemaMutexHeld(db, 0, pIndex->pSchema) ); + if( pTblName!=0 ){ + pIndex->tnum = db->init.newTnum; + if( sqlite3IndexHasDuplicateRootPage(pIndex) ){ + sqlite3ErrorMsg(pParse, "invalid rootpage"); + pParse->rc = SQLITE_CORRUPT_BKPT; + goto exit_create_index; + } + } + p = sqlite3HashInsert(&pIndex->pSchema->idxHash, + pIndex->zName, pIndex); + if( p ){ + assert( p==pIndex ); /* Malloc must have failed */ + sqlite3OomFault(db); + goto exit_create_index; + } + db->mDbFlags |= DBFLAG_SchemaChange; } - /* Add an entry in sqlite_master for this index - */ - sqlite3NestedParse(pParse, - "INSERT INTO %Q.%s VALUES('index',%Q,%Q,#%d,%Q);", - db->aDb[iDb].zDbSName, MASTER_NAME, - pIndex->zName, - pTab->zName, - iMem, - zStmt - ); - sqlite3DbFree(db, zStmt); - - /* Fill the index with data and reparse the schema. Code an OP_Expire - ** to invalidate all pre-compiled statements. + /* If this is the initial CREATE INDEX statement (or CREATE TABLE if the + ** index is an implied index for a UNIQUE or PRIMARY KEY constraint) then + ** emit code to allocate the index rootpage on disk and make an entry for + ** the index in the sqlite_master table and populate the index with + ** content. But, do not do this if we are simply reading the sqlite_master + ** table to parse the schema, or if this index is the PRIMARY KEY index + ** of a WITHOUT ROWID table. + ** + ** If pTblName==0 it means this index is generated as an implied PRIMARY KEY + ** or UNIQUE index in a CREATE TABLE statement. Since the table + ** has just been created, it contains no data and the index initialization + ** step can be skipped. */ - if( pTblName ){ - sqlite3RefillIndex(pParse, pIndex, iMem); - sqlite3ChangeCookie(pParse, iDb); - sqlite3VdbeAddParseSchemaOp(v, iDb, - sqlite3MPrintf(db, "name='%q' AND type='index'", pIndex->zName)); - sqlite3VdbeAddOp0(v, OP_Expire); - } + else if( HasRowid(pTab) || pTblName!=0 ){ + Vdbe *v; + char *zStmt; + int iMem = ++pParse->nMem; + + v = sqlite3GetVdbe(pParse); + if( v==0 ) goto exit_create_index; + + sqlite3BeginWriteOperation(pParse, 1, iDb); + + /* Create the rootpage for the index using CreateIndex. But before + ** doing so, code a Noop instruction and store its address in + ** Index.tnum. This is required in case this index is actually a + ** PRIMARY KEY and the table is actually a WITHOUT ROWID table. In + ** that case the convertToWithoutRowidTable() routine will replace + ** the Noop with a Goto to jump over the VDBE code generated below. */ + pIndex->tnum = sqlite3VdbeAddOp0(v, OP_Noop); + sqlite3VdbeAddOp3(v, OP_CreateBtree, iDb, iMem, BTREE_BLOBKEY); + + /* Gather the complete text of the CREATE INDEX statement into + ** the zStmt variable + */ + assert( pName!=0 || pStart==0 ); + if( pStart ){ + int n = (int)(pParse->sLastToken.z - pName->z) + pParse->sLastToken.n; + if( pName->z[n-1]==';' ) n--; + /* A named index with an explicit CREATE INDEX statement */ + zStmt = sqlite3MPrintf(db, "CREATE%s INDEX %.*s", + onError==OE_None ? "" : " UNIQUE", n, pName->z); + }else{ + /* An automatic index created by a PRIMARY KEY or UNIQUE constraint */ + /* zStmt = sqlite3MPrintf(""); */ + zStmt = 0; + } - sqlite3VdbeJumpHere(v, pIndex->tnum); - } + /* Add an entry in sqlite_master for this index + */ + sqlite3NestedParse(pParse, + "INSERT INTO %Q.%s VALUES('index',%Q,%Q,#%d,%Q);", + db->aDb[iDb].zDbSName, MASTER_NAME, + pIndex->zName, + pTab->zName, + iMem, + zStmt + ); + sqlite3DbFree(db, zStmt); - /* When adding an index to the list of indices for a table, make - ** sure all indices labeled OE_Replace come after all those labeled - ** OE_Ignore. This is necessary for the correct constraint check - ** processing (in sqlite3GenerateConstraintChecks()) as part of - ** UPDATE and INSERT statements. - */ - if( db->init.busy || pTblName==0 ){ - if( onError!=OE_Replace || pTab->pIndex==0 - || pTab->pIndex->onError==OE_Replace){ - pIndex->pNext = pTab->pIndex; - pTab->pIndex = pIndex; - }else{ - Index *pOther = pTab->pIndex; - while( pOther->pNext && pOther->pNext->onError!=OE_Replace ){ - pOther = pOther->pNext; + /* Fill the index with data and reparse the schema. Code an OP_Expire + ** to invalidate all pre-compiled statements. + */ + if( pTblName ){ + sqlite3RefillIndex(pParse, pIndex, iMem); + sqlite3ChangeCookie(pParse, iDb); + sqlite3VdbeAddParseSchemaOp(v, iDb, + sqlite3MPrintf(db, "name='%q' AND type='index'", pIndex->zName)); + sqlite3VdbeAddOp2(v, OP_Expire, 0, 1); } - pIndex->pNext = pOther->pNext; - pOther->pNext = pIndex; + + sqlite3VdbeJumpHere(v, pIndex->tnum); } + } + if( db->init.busy || pTblName==0 ){ + pIndex->pNext = pTab->pIndex; + pTab->pIndex = pIndex; + pIndex = 0; + } + else if( IN_RENAME_OBJECT ){ + assert( pParse->pNewIndex==0 ); + pParse->pNewIndex = pIndex; pIndex = 0; } /* Clean up before exiting */ exit_create_index: - if( pIndex ) freeIndex(db, pIndex); + if( pIndex ) sqlite3FreeIndex(db, pIndex); + if( pTab ){ /* Ensure all REPLACE indexes are at the end of the list */ + Index **ppFrom = &pTab->pIndex; + Index *pThis; + for(ppFrom=&pTab->pIndex; (pThis = *ppFrom)!=0; ppFrom=&pThis->pNext){ + Index *pNext; + if( pThis->onError!=OE_Replace ) continue; + while( (pNext = pThis->pNext)!=0 && pNext->onError!=OE_Replace ){ + *ppFrom = pNext; + pThis->pNext = pNext->pNext; + pNext->pNext = pThis; + ppFrom = &pNext->pNext; + } + break; + } + } sqlite3ExprDelete(db, pPIWhere); sqlite3ExprListDelete(db, pList); sqlite3SrcListDelete(db, pTblName); @@ -107534,9 +114216,9 @@ SQLITE_PRIVATE void *sqlite3ArrayAllocate( int *pIdx /* Write the index of a new slot here */ ){ char *z; - int n = *pnEntry; + sqlite3_int64 n = *pIdx = *pnEntry; if( (n & (n-1))==0 ){ - int sz = (n==0) ? 1 : 2*n; + sqlite3_int64 sz = (n==0) ? 1 : 2*n; void *pNew = sqlite3DbRealloc(db, pArray, sz*szEntry); if( pNew==0 ){ *pIdx = -1; @@ -107546,7 +114228,6 @@ SQLITE_PRIVATE void *sqlite3ArrayAllocate( } z = (char*)pArray; memset(&z[n * szEntry], 0, szEntry); - *pIdx = n; ++*pnEntry; return pArray; } @@ -107557,7 +114238,8 @@ SQLITE_PRIVATE void *sqlite3ArrayAllocate( ** ** A new IdList is returned, or NULL if malloc() fails. */ -SQLITE_PRIVATE IdList *sqlite3IdListAppend(sqlite3 *db, IdList *pList, Token *pToken){ +SQLITE_PRIVATE IdList *sqlite3IdListAppend(Parse *pParse, IdList *pList, Token *pToken){ + sqlite3 *db = pParse->db; int i; if( pList==0 ){ pList = sqlite3DbMallocZero(db, sizeof(IdList) ); @@ -107575,6 +114257,9 @@ SQLITE_PRIVATE IdList *sqlite3IdListAppend(sqlite3 *db, IdList *pList, Token *pT return 0; } pList->a[i].zName = sqlite3NameFromToken(db, pToken); + if( IN_RENAME_OBJECT && pList->a[i].zName ){ + sqlite3RenameTokenMap(pParse, (void*)pList->a[i].zName, pToken); + } return pList; } @@ -107604,6 +114289,18 @@ SQLITE_PRIVATE int sqlite3IdListIndex(IdList *pList, const char *zName){ return -1; } +/* +** Maximum size of a SrcList object. +** The SrcList object is used to represent the FROM clause of a +** SELECT statement, and the query planner cannot deal with more +** than 64 tables in a join. So any value larger than 64 here +** is sufficient for most uses. Smaller values, like say 10, are +** appropriate for small and memory-limited applications. +*/ +#ifndef SQLITE_MAX_SRCLIST +# define SQLITE_MAX_SRCLIST 200 +#endif + /* ** Expand the space allocated for the given SrcList object by ** creating nExtra new slots beginning at iStart. iStart is zero based. @@ -107620,11 +114317,12 @@ SQLITE_PRIVATE int sqlite3IdListIndex(IdList *pList, const char *zName){ ** the iStart value would be 0. The result then would ** be: nil, nil, nil, A, B. ** -** If a memory allocation fails the SrcList is unchanged. The -** db->mallocFailed flag will be set to true. +** If a memory allocation fails or the SrcList becomes too large, leave +** the original SrcList unchanged, return NULL, and leave an error message +** in pParse. */ SQLITE_PRIVATE SrcList *sqlite3SrcListEnlarge( - sqlite3 *db, /* Database connection to notify of OOM errors */ + Parse *pParse, /* Parsing context into which errors are reported */ SrcList *pSrc, /* The SrcList to be enlarged */ int nExtra, /* Number of new slots to add to pSrc->a[] */ int iStart /* Index in pSrc->a[] of first new slot */ @@ -107640,17 +114338,23 @@ SQLITE_PRIVATE SrcList *sqlite3SrcListEnlarge( /* Allocate additional space if needed */ if( (u32)pSrc->nSrc+nExtra>pSrc->nAlloc ){ SrcList *pNew; - int nAlloc = pSrc->nSrc*2+nExtra; - int nGot; + sqlite3_int64 nAlloc = 2*(sqlite3_int64)pSrc->nSrc+nExtra; + sqlite3 *db = pParse->db; + + if( pSrc->nSrc+nExtra>=SQLITE_MAX_SRCLIST ){ + sqlite3ErrorMsg(pParse, "too many FROM clause terms, max: %d", + SQLITE_MAX_SRCLIST); + return 0; + } + if( nAlloc>SQLITE_MAX_SRCLIST ) nAlloc = SQLITE_MAX_SRCLIST; pNew = sqlite3DbRealloc(db, pSrc, sizeof(*pSrc) + (nAlloc-1)*sizeof(pSrc->a[0]) ); if( pNew==0 ){ assert( db->mallocFailed ); - return pSrc; + return 0; } pSrc = pNew; - nGot = (sqlite3DbMallocSize(db, pNew) - sizeof(*pSrc))/sizeof(pSrc->a[0])+1; - pSrc->nAlloc = nGot; + pSrc->nAlloc = nAlloc; } /* Move existing slots that come after the newly inserted slots @@ -107675,7 +114379,8 @@ SQLITE_PRIVATE SrcList *sqlite3SrcListEnlarge( ** Append a new table name to the given SrcList. Create a new SrcList if ** need be. A new entry is created in the SrcList even if pTable is NULL. ** -** A SrcList is returned, or NULL if there is an OOM error. The returned +** A SrcList is returned, or NULL if there is an OOM error or if the +** SrcList grows to large. The returned ** SrcList might be the same as the SrcList that was input or it might be ** a new one. If an OOM error does occurs, then the prior value of pList ** that is input to this routine is automatically freed. @@ -107706,27 +114411,32 @@ SQLITE_PRIVATE SrcList *sqlite3SrcListEnlarge( ** before being added to the SrcList. */ SQLITE_PRIVATE SrcList *sqlite3SrcListAppend( - sqlite3 *db, /* Connection to notify of malloc failures */ + Parse *pParse, /* Parsing context, in which errors are reported */ SrcList *pList, /* Append to this SrcList. NULL creates a new SrcList */ Token *pTable, /* Table to append */ Token *pDatabase /* Database of the table */ ){ struct SrcList_item *pItem; + sqlite3 *db; assert( pDatabase==0 || pTable!=0 ); /* Cannot have C without B */ - assert( db!=0 ); + assert( pParse!=0 ); + assert( pParse->db!=0 ); + db = pParse->db; if( pList==0 ){ - pList = sqlite3DbMallocRawNN(db, sizeof(SrcList) ); + pList = sqlite3DbMallocRawNN(pParse->db, sizeof(SrcList) ); if( pList==0 ) return 0; pList->nAlloc = 1; pList->nSrc = 1; memset(&pList->a[0], 0, sizeof(pList->a[0])); pList->a[0].iCursor = -1; }else{ - pList = sqlite3SrcListEnlarge(db, pList, 1, pList->nSrc); - } - if( db->mallocFailed ){ - sqlite3SrcListDelete(db, pList); - return 0; + SrcList *pNew = sqlite3SrcListEnlarge(pParse, pList, 1, pList->nSrc); + if( pNew==0 ){ + sqlite3SrcListDelete(db, pList); + return 0; + }else{ + pList = pNew; + } } pItem = &pList->a[pList->nSrc-1]; if( pDatabase && pDatabase->z==0 ){ @@ -107815,12 +114525,18 @@ SQLITE_PRIVATE SrcList *sqlite3SrcListAppendFromTerm( ); goto append_from_error; } - p = sqlite3SrcListAppend(db, p, pTable, pDatabase); + p = sqlite3SrcListAppend(pParse, p, pTable, pDatabase); if( p==0 ){ goto append_from_error; } assert( p->nSrc>0 ); pItem = &p->a[p->nSrc-1]; + assert( (pTable==0)==(pDatabase==0) ); + assert( pItem->zName==0 || pDatabase!=0 ); + if( IN_RENAME_OBJECT && pItem->zName ){ + Token *pToken = (ALWAYS(pDatabase) && pDatabase->z) ? pDatabase : pTable; + sqlite3RenameTokenMap(pParse, pItem->zName, pToken); + } assert( pAlias!=0 ); if( pAlias->n ){ pItem->zAlias = sqlite3NameFromToken(db, pAlias); @@ -107997,7 +114713,7 @@ SQLITE_PRIVATE int sqlite3OpenTempDatabase(Parse *pParse){ } db->aDb[1].pBt = pBt; assert( db->aDb[1].pSchema ); - if( SQLITE_NOMEM==sqlite3BtreeSetPageSize(pBt, db->nextPagesize, -1, 0) ){ + if( SQLITE_NOMEM==sqlite3BtreeSetPageSize(pBt, db->nextPagesize, 0, 0) ){ sqlite3OomFault(db); return 1; } @@ -108108,7 +114824,7 @@ SQLITE_PRIVATE void sqlite3HaltConstraint( u8 p5Errmsg /* P5_ErrMsg type */ ){ Vdbe *v = sqlite3GetVdbe(pParse); - assert( (errCode&0xff)==SQLITE_CONSTRAINT ); + assert( (errCode&0xff)==SQLITE_CONSTRAINT || pParse->nested ); if( onError==OE_Abort ){ sqlite3MayAbort(pParse); } @@ -108129,7 +114845,8 @@ SQLITE_PRIVATE void sqlite3UniqueConstraint( StrAccum errMsg; Table *pTab = pIdx->pTable; - sqlite3StrAccumInit(&errMsg, pParse->db, 0, 0, 200); + sqlite3StrAccumInit(&errMsg, pParse->db, 0, 0, + pParse->db->aLimit[SQLITE_LIMIT_LENGTH]); if( pIdx->aColExpr ){ sqlite3_str_appendf(&errMsg, "index '%q'", pIdx->zName); }else{ @@ -108198,13 +114915,15 @@ static int collationMatch(const char *zColl, Index *pIndex){ */ #ifndef SQLITE_OMIT_REINDEX static void reindexTable(Parse *pParse, Table *pTab, char const *zColl){ - Index *pIndex; /* An index associated with pTab */ + if( !IsVirtual(pTab) ){ + Index *pIndex; /* An index associated with pTab */ - for(pIndex=pTab->pIndex; pIndex; pIndex=pIndex->pNext){ - if( zColl==0 || collationMatch(zColl, pIndex) ){ - int iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema); - sqlite3BeginWriteOperation(pParse, 0, iDb); - sqlite3RefillIndex(pParse, pIndex, -1); + for(pIndex=pTab->pIndex; pIndex; pIndex=pIndex->pNext){ + if( zColl==0 || collationMatch(zColl, pIndex) ){ + int iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema); + sqlite3BeginWriteOperation(pParse, 0, iDb); + sqlite3RefillIndex(pParse, pIndex, -1); + } } } } @@ -108325,7 +115044,8 @@ SQLITE_PRIVATE KeyInfo *sqlite3KeyInfoOfIndex(Parse *pParse, Index *pIdx){ const char *zColl = pIdx->azColl[i]; pKey->aColl[i] = zColl==sqlite3StrBINARY ? 0 : sqlite3LocateCollSeq(pParse, zColl); - pKey->aSortOrder[i] = pIdx->aSortOrder[i]; + pKey->aSortFlags[i] = pIdx->aSortOrder[i]; + assert( 0==(pKey->aSortFlags[i] & KEYINFO_ORDER_BIGNULL) ); } if( pParse->nErr ){ assert( pParse->rc==SQLITE_ERROR_MISSING_COLLSEQ ); @@ -108376,7 +115096,7 @@ SQLITE_PRIVATE With *sqlite3WithAdd( } if( pWith ){ - int nByte = sizeof(*pWith) + (sizeof(pWith->a[1]) * pWith->nCte); + sqlite3_int64 nByte = sizeof(*pWith) + (sizeof(pWith->a[1]) * pWith->nCte); pNew = sqlite3DbRealloc(db, pWith, nByte); }else{ pNew = sqlite3DbMallocZero(db, sizeof(*pWith)); @@ -108485,6 +115205,128 @@ static int synthCollSeq(sqlite3 *db, CollSeq *pColl){ return SQLITE_ERROR; } +/* +** This routine is called on a collation sequence before it is used to +** check that it is defined. An undefined collation sequence exists when +** a database is loaded that contains references to collation sequences +** that have not been defined by sqlite3_create_collation() etc. +** +** If required, this routine calls the 'collation needed' callback to +** request a definition of the collating sequence. If this doesn't work, +** an equivalent collating sequence that uses a text encoding different +** from the main database is substituted, if one is available. +*/ +SQLITE_PRIVATE int sqlite3CheckCollSeq(Parse *pParse, CollSeq *pColl){ + if( pColl && pColl->xCmp==0 ){ + const char *zName = pColl->zName; + sqlite3 *db = pParse->db; + CollSeq *p = sqlite3GetCollSeq(pParse, ENC(db), pColl, zName); + if( !p ){ + return SQLITE_ERROR; + } + assert( p==pColl ); + } + return SQLITE_OK; +} + + + +/* +** Locate and return an entry from the db.aCollSeq hash table. If the entry +** specified by zName and nName is not found and parameter 'create' is +** true, then create a new entry. Otherwise return NULL. +** +** Each pointer stored in the sqlite3.aCollSeq hash table contains an +** array of three CollSeq structures. The first is the collation sequence +** preferred for UTF-8, the second UTF-16le, and the third UTF-16be. +** +** Stored immediately after the three collation sequences is a copy of +** the collation sequence name. A pointer to this string is stored in +** each collation sequence structure. +*/ +static CollSeq *findCollSeqEntry( + sqlite3 *db, /* Database connection */ + const char *zName, /* Name of the collating sequence */ + int create /* Create a new entry if true */ +){ + CollSeq *pColl; + pColl = sqlite3HashFind(&db->aCollSeq, zName); + + if( 0==pColl && create ){ + int nName = sqlite3Strlen30(zName) + 1; + pColl = sqlite3DbMallocZero(db, 3*sizeof(*pColl) + nName); + if( pColl ){ + CollSeq *pDel = 0; + pColl[0].zName = (char*)&pColl[3]; + pColl[0].enc = SQLITE_UTF8; + pColl[1].zName = (char*)&pColl[3]; + pColl[1].enc = SQLITE_UTF16LE; + pColl[2].zName = (char*)&pColl[3]; + pColl[2].enc = SQLITE_UTF16BE; + memcpy(pColl[0].zName, zName, nName); + pDel = sqlite3HashInsert(&db->aCollSeq, pColl[0].zName, pColl); + + /* If a malloc() failure occurred in sqlite3HashInsert(), it will + ** return the pColl pointer to be deleted (because it wasn't added + ** to the hash table). + */ + assert( pDel==0 || pDel==pColl ); + if( pDel!=0 ){ + sqlite3OomFault(db); + sqlite3DbFree(db, pDel); + pColl = 0; + } + } + } + return pColl; +} + +/* +** Parameter zName points to a UTF-8 encoded string nName bytes long. +** Return the CollSeq* pointer for the collation sequence named zName +** for the encoding 'enc' from the database 'db'. +** +** If the entry specified is not found and 'create' is true, then create a +** new entry. Otherwise return NULL. +** +** A separate function sqlite3LocateCollSeq() is a wrapper around +** this routine. sqlite3LocateCollSeq() invokes the collation factory +** if necessary and generates an error message if the collating sequence +** cannot be found. +** +** See also: sqlite3LocateCollSeq(), sqlite3GetCollSeq() +*/ +SQLITE_PRIVATE CollSeq *sqlite3FindCollSeq( + sqlite3 *db, /* Database connection to search */ + u8 enc, /* Desired text encoding */ + const char *zName, /* Name of the collating sequence. Might be NULL */ + int create /* True to create CollSeq if doesn't already exist */ +){ + CollSeq *pColl; + assert( SQLITE_UTF8==1 && SQLITE_UTF16LE==2 && SQLITE_UTF16BE==3 ); + assert( enc>=SQLITE_UTF8 && enc<=SQLITE_UTF16BE ); + if( zName ){ + pColl = findCollSeqEntry(db, zName, create); + if( pColl ) pColl += enc-1; + }else{ + pColl = db->pDfltColl; + } + return pColl; +} + +/* +** Change the text encoding for a database connection. This means that +** the pDfltColl must change as well. +*/ +SQLITE_PRIVATE void sqlite3SetTextEncoding(sqlite3 *db, u8 enc){ + assert( enc==SQLITE_UTF8 || enc==SQLITE_UTF16LE || enc==SQLITE_UTF16BE ); + db->enc = enc; + /* EVIDENCE-OF: R-08308-17224 The default collating function for all + ** strings is BINARY. + */ + db->pDfltColl = sqlite3FindCollSeq(db, enc, sqlite3StrBINARY, 0); +} + /* ** This function is responsible for invoking the collation factory callback ** or substituting a collation sequence of a different encoding when the @@ -108531,111 +115373,36 @@ SQLITE_PRIVATE CollSeq *sqlite3GetCollSeq( } /* -** This routine is called on a collation sequence before it is used to -** check that it is defined. An undefined collation sequence exists when -** a database is loaded that contains references to collation sequences -** that have not been defined by sqlite3_create_collation() etc. +** This function returns the collation sequence for database native text +** encoding identified by the string zName. ** -** If required, this routine calls the 'collation needed' callback to -** request a definition of the collating sequence. If this doesn't work, -** an equivalent collating sequence that uses a text encoding different -** from the main database is substituted, if one is available. -*/ -SQLITE_PRIVATE int sqlite3CheckCollSeq(Parse *pParse, CollSeq *pColl){ - if( pColl && pColl->xCmp==0 ){ - const char *zName = pColl->zName; - sqlite3 *db = pParse->db; - CollSeq *p = sqlite3GetCollSeq(pParse, ENC(db), pColl, zName); - if( !p ){ - return SQLITE_ERROR; - } - assert( p==pColl ); - } - return SQLITE_OK; -} - - - -/* -** Locate and return an entry from the db.aCollSeq hash table. If the entry -** specified by zName and nName is not found and parameter 'create' is -** true, then create a new entry. Otherwise return NULL. +** If the requested collation sequence is not available, or not available +** in the database native encoding, the collation factory is invoked to +** request it. If the collation factory does not supply such a sequence, +** and the sequence is available in another text encoding, then that is +** returned instead. ** -** Each pointer stored in the sqlite3.aCollSeq hash table contains an -** array of three CollSeq structures. The first is the collation sequence -** preferred for UTF-8, the second UTF-16le, and the third UTF-16be. +** If no versions of the requested collations sequence are available, or +** another error occurs, NULL is returned and an error message written into +** pParse. ** -** Stored immediately after the three collation sequences is a copy of -** the collation sequence name. A pointer to this string is stored in -** each collation sequence structure. +** This routine is a wrapper around sqlite3FindCollSeq(). This routine +** invokes the collation factory if the named collation cannot be found +** and generates an error message. +** +** See also: sqlite3FindCollSeq(), sqlite3GetCollSeq() */ -static CollSeq *findCollSeqEntry( - sqlite3 *db, /* Database connection */ - const char *zName, /* Name of the collating sequence */ - int create /* Create a new entry if true */ -){ +SQLITE_PRIVATE CollSeq *sqlite3LocateCollSeq(Parse *pParse, const char *zName){ + sqlite3 *db = pParse->db; + u8 enc = ENC(db); + u8 initbusy = db->init.busy; CollSeq *pColl; - pColl = sqlite3HashFind(&db->aCollSeq, zName); - - if( 0==pColl && create ){ - int nName = sqlite3Strlen30(zName) + 1; - pColl = sqlite3DbMallocZero(db, 3*sizeof(*pColl) + nName); - if( pColl ){ - CollSeq *pDel = 0; - pColl[0].zName = (char*)&pColl[3]; - pColl[0].enc = SQLITE_UTF8; - pColl[1].zName = (char*)&pColl[3]; - pColl[1].enc = SQLITE_UTF16LE; - pColl[2].zName = (char*)&pColl[3]; - pColl[2].enc = SQLITE_UTF16BE; - memcpy(pColl[0].zName, zName, nName); - pDel = sqlite3HashInsert(&db->aCollSeq, pColl[0].zName, pColl); - /* If a malloc() failure occurred in sqlite3HashInsert(), it will - ** return the pColl pointer to be deleted (because it wasn't added - ** to the hash table). - */ - assert( pDel==0 || pDel==pColl ); - if( pDel!=0 ){ - sqlite3OomFault(db); - sqlite3DbFree(db, pDel); - pColl = 0; - } - } + pColl = sqlite3FindCollSeq(db, enc, zName, initbusy); + if( !initbusy && (!pColl || !pColl->xCmp) ){ + pColl = sqlite3GetCollSeq(pParse, enc, pColl, zName); } - return pColl; -} -/* -** Parameter zName points to a UTF-8 encoded string nName bytes long. -** Return the CollSeq* pointer for the collation sequence named zName -** for the encoding 'enc' from the database 'db'. -** -** If the entry specified is not found and 'create' is true, then create a -** new entry. Otherwise return NULL. -** -** A separate function sqlite3LocateCollSeq() is a wrapper around -** this routine. sqlite3LocateCollSeq() invokes the collation factory -** if necessary and generates an error message if the collating sequence -** cannot be found. -** -** See also: sqlite3LocateCollSeq(), sqlite3GetCollSeq() -*/ -SQLITE_PRIVATE CollSeq *sqlite3FindCollSeq( - sqlite3 *db, - u8 enc, - const char *zName, - int create -){ - CollSeq *pColl; - if( zName ){ - pColl = findCollSeqEntry(db, zName, create); - }else{ - pColl = db->pDfltColl; - } - assert( SQLITE_UTF8==1 && SQLITE_UTF16LE==2 && SQLITE_UTF16BE==3 ); - assert( enc>=SQLITE_UTF8 && enc<=SQLITE_UTF16BE ); - if( pColl ) pColl += enc-1; return pColl; } @@ -108674,12 +115441,13 @@ static int matchQuality( u8 enc /* Desired text encoding */ ){ int match; - - /* nArg of -2 is a special case */ - if( nArg==(-2) ) return (p->xSFunc==0) ? 0 : FUNC_PERFECT_MATCH; + assert( p->nArg>=-1 ); /* Wrong number of arguments means "no match" */ - if( p->nArg!=nArg && p->nArg>=0 ) return 0; + if( p->nArg!=nArg ){ + if( nArg==(-2) ) return (p->xSFunc==0) ? 0 : FUNC_PERFECT_MATCH; + if( p->nArg>=0 ) return 0; + } /* Give a better score to a function with a specific number of arguments ** than to function that accepts any number of arguments. */ @@ -108703,7 +115471,7 @@ static int matchQuality( ** Search a FuncDefHash for a function with the given name. Return ** a pointer to the matching FuncDef if found, or 0 if there is no match. */ -static FuncDef *functionSearch( +SQLITE_PRIVATE FuncDef *sqlite3FunctionSearch( int h, /* Hash of the name */ const char *zFunc /* Name of function */ ){ @@ -108728,9 +115496,9 @@ SQLITE_PRIVATE void sqlite3InsertBuiltinFuncs( FuncDef *pOther; const char *zName = aDef[i].zName; int nName = sqlite3Strlen30(zName); - int h = (zName[0] + nName) % SQLITE_FUNC_HASH_SZ; + int h = SQLITE_FUNC_HASH(zName[0], nName); assert( zName[0]>='a' && zName[0]<='z' ); - pOther = functionSearch(h, zName); + pOther = sqlite3FunctionSearch(h, zName); if( pOther ){ assert( pOther!=&aDef[i] && pOther->pNext!=&aDef[i] ); aDef[i].pNext = pOther->pNext; @@ -108807,8 +115575,8 @@ SQLITE_PRIVATE FuncDef *sqlite3FindFunction( */ if( !createFlag && (pBest==0 || (db->mDbFlags & DBFLAG_PreferBuiltin)!=0) ){ bestScore = 0; - h = (sqlite3UpperToLower[(u8)zName[0]] + nName) % SQLITE_FUNC_HASH_SZ; - p = functionSearch(h, zName); + h = SQLITE_FUNC_HASH(sqlite3UpperToLower[(u8)zName[0]], nName); + p = sqlite3FunctionSearch(h, zName); while( p ){ int score = matchQuality(p, nArg, enc); if( score>bestScore ){ @@ -108955,32 +115723,45 @@ SQLITE_PRIVATE Table *sqlite3SrcListLookup(Parse *pParse, SrcList *pSrc){ return pTab; } +/* Return true if table pTab is read-only. +** +** A table is read-only if any of the following are true: +** +** 1) It is a virtual table and no implementation of the xUpdate method +** has been provided +** +** 2) It is a system table (i.e. sqlite_master), this call is not +** part of a nested parse and writable_schema pragma has not +** been specified +** +** 3) The table is a shadow table, the database connection is in +** defensive mode, and the current sqlite3_prepare() +** is for a top-level SQL statement. +*/ +static int tabIsReadOnly(Parse *pParse, Table *pTab){ + sqlite3 *db; + if( IsVirtual(pTab) ){ + return sqlite3GetVTable(pParse->db, pTab)->pMod->pModule->xUpdate==0; + } + if( (pTab->tabFlags & (TF_Readonly|TF_Shadow))==0 ) return 0; + db = pParse->db; + if( (pTab->tabFlags & TF_Readonly)!=0 ){ + return sqlite3WritableSchema(db)==0 && pParse->nested==0; + } + assert( pTab->tabFlags & TF_Shadow ); + return sqlite3ReadOnlyShadowTables(db); +} + /* ** Check to make sure the given table is writable. If it is not ** writable, generate an error message and return 1. If it is ** writable return 0; */ SQLITE_PRIVATE int sqlite3IsReadOnly(Parse *pParse, Table *pTab, int viewOk){ - /* A table is not writable under the following circumstances: - ** - ** 1) It is a virtual table and no implementation of the xUpdate method - ** has been provided, or - ** 2) It is a system table (i.e. sqlite_master), this call is not - ** part of a nested parse and writable_schema pragma has not - ** been specified. - ** - ** In either case leave an error message in pParse and return non-zero. - */ - if( ( IsVirtual(pTab) - && sqlite3GetVTable(pParse->db, pTab)->pMod->pModule->xUpdate==0 ) - || ( (pTab->tabFlags & TF_Readonly)!=0 - && (pParse->db->flags & SQLITE_WriteSchema)==0 - && pParse->nested==0 ) - ){ + if( tabIsReadOnly(pParse, pTab) ){ sqlite3ErrorMsg(pParse, "table %s may not be modified", pTab->zName); return 1; } - #ifndef SQLITE_OMIT_VIEW if( !viewOk && pTab->pSelect ){ sqlite3ErrorMsg(pParse,"cannot modify %s because it is a view",pTab->zName); @@ -109011,7 +115792,7 @@ SQLITE_PRIVATE void sqlite3MaterializeView( sqlite3 *db = pParse->db; int iDb = sqlite3SchemaToIndex(db, pView->pSchema); pWhere = sqlite3ExprDup(db, pWhere, 0); - pFrom = sqlite3SrcListAppend(db, 0, 0, 0); + pFrom = sqlite3SrcListAppend(pParse, 0, 0, 0); if( pFrom ){ assert( pFrom->nSrc==1 ); pFrom->a[0].zName = sqlite3DbStrDup(db, pView->zName); @@ -109374,9 +116155,8 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( } iKey = iPk; }else{ - iKey = pParse->nMem + 1; - iKey = sqlite3ExprCodeGetColumn(pParse, pTab, -1, iTabCur, iKey, 0); - if( iKey>pParse->nMem ) pParse->nMem = iKey; + iKey = ++pParse->nMem; + sqlite3ExprCodeGetColumnOfTable(v, pTab, iTabCur, -1, iKey); } if( eOnePass!=ONEPASS_OFF ){ @@ -109412,7 +116192,7 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( /* If this DELETE cannot use the ONEPASS strategy, this is the ** end of the WHERE loop */ if( eOnePass!=ONEPASS_OFF ){ - addrBypass = sqlite3VdbeMakeLabel(v); + addrBypass = sqlite3VdbeMakeLabel(pParse); }else{ sqlite3WhereEnd(pWInfo); } @@ -109432,7 +116212,9 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( iTabCur, aToOpen, &iDataCur, &iIdxCur); assert( pPk || IsVirtual(pTab) || iDataCur==iTabCur ); assert( pPk || IsVirtual(pTab) || iIdxCur==iDataCur+1 ); - if( eOnePass==ONEPASS_MULTI ) sqlite3VdbeJumpHere(v, iAddrOnce); + if( eOnePass==ONEPASS_MULTI ){ + sqlite3VdbeJumpHereOrPopInst(v, iAddrOnce); + } } /* Set up a loop over the rowids/primary-keys that were found in the @@ -109601,7 +116383,7 @@ SQLITE_PRIVATE void sqlite3GenerateRowDelete( /* Seek cursor iCur to the row to delete. If this row no longer exists ** (this can happen if a trigger program has already deleted it), do ** not attempt to delete it or fire any DELETE triggers. */ - iLabel = sqlite3VdbeMakeLabel(v); + iLabel = sqlite3VdbeMakeLabel(pParse); opSeek = HasRowid(pTab) ? OP_NotExists : OP_NotFound; if( eMode==ONEPASS_OFF ){ sqlite3VdbeAddOp4Int(v, opSeek, iDataCur, iLabel, iPk, nPk); @@ -109632,7 +116414,8 @@ SQLITE_PRIVATE void sqlite3GenerateRowDelete( testcase( mask!=0xffffffff && iCol==31 ); testcase( mask!=0xffffffff && iCol==32 ); if( mask==0xffffffff || (iCol<=31 && (mask & MASKBIT32(iCol))!=0) ){ - sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, iCol, iOld+iCol+1); + int kk = sqlite3TableColumnToStorage(pTab, iCol); + sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, iCol, iOld+kk+1); } } @@ -109754,6 +116537,7 @@ SQLITE_PRIVATE void sqlite3GenerateRowIndexDelete( &iPartIdxLabel, pPrior, r1); sqlite3VdbeAddOp3(v, OP_IdxDelete, iIdxCur+i, r1, pIdx->uniqNotNull ? pIdx->nKeyCol : pIdx->nColumn); + sqlite3VdbeChangeP5(v, 1); /* Cause IdxDelete to error if no entry found */ sqlite3ResolvePartIdxLabel(pParse, iPartIdxLabel); pPrior = pIdx; } @@ -109807,12 +116591,13 @@ SQLITE_PRIVATE int sqlite3GenerateIndexKey( if( piPartIdxLabel ){ if( pIdx->pPartIdxWhere ){ - *piPartIdxLabel = sqlite3VdbeMakeLabel(v); + *piPartIdxLabel = sqlite3VdbeMakeLabel(pParse); pParse->iSelfTab = iDataCur + 1; - sqlite3ExprCachePush(pParse); sqlite3ExprIfFalseDup(pParse, pIdx->pPartIdxWhere, *piPartIdxLabel, SQLITE_JUMPIFNULL); pParse->iSelfTab = 0; + pPrior = 0; /* Ticket a9efb42811fa41ee 2019-11-02; + ** pPartIdxWhere may have corrupted regPrior registers */ }else{ *piPartIdxLabel = 0; } @@ -109856,7 +116641,6 @@ SQLITE_PRIVATE int sqlite3GenerateIndexKey( SQLITE_PRIVATE void sqlite3ResolvePartIdxLabel(Parse *pParse, int iLabel){ if( iLabel ){ sqlite3VdbeResolveLabel(pParse->pVdbe, iLabel); - sqlite3ExprCachePop(pParse); } } @@ -109880,6 +116664,9 @@ SQLITE_PRIVATE void sqlite3ResolvePartIdxLabel(Parse *pParse, int iLabel){ /* #include "sqliteInt.h" */ /* #include */ /* #include */ +#ifndef SQLITE_OMIT_FLOATING_POINT +/* #include */ +#endif /* #include "vdbeInt.h" */ /* @@ -110065,6 +116852,9 @@ static void instrFunc( int typeHaystack, typeNeedle; int N = 1; int isText; + unsigned char firstChar; + sqlite3_value *pC1 = 0; + sqlite3_value *pC2 = 0; UNUSED_PARAMETER(argc); typeHaystack = sqlite3_value_type(argv[0]); @@ -110077,13 +116867,26 @@ static void instrFunc( zHaystack = sqlite3_value_blob(argv[0]); zNeedle = sqlite3_value_blob(argv[1]); isText = 0; - }else{ + }else if( typeHaystack!=SQLITE_BLOB && typeNeedle!=SQLITE_BLOB ){ zHaystack = sqlite3_value_text(argv[0]); zNeedle = sqlite3_value_text(argv[1]); isText = 1; + }else{ + pC1 = sqlite3_value_dup(argv[0]); + zHaystack = sqlite3_value_text(pC1); + if( zHaystack==0 ) goto endInstrOOM; + nHaystack = sqlite3_value_bytes(pC1); + pC2 = sqlite3_value_dup(argv[1]); + zNeedle = sqlite3_value_text(pC2); + if( zNeedle==0 ) goto endInstrOOM; + nNeedle = sqlite3_value_bytes(pC2); + isText = 1; } - if( zNeedle==0 || (nHaystack && zHaystack==0) ) return; - while( nNeedle<=nHaystack && memcmp(zHaystack, zNeedle, nNeedle)!=0 ){ + if( zNeedle==0 || (nHaystack && zHaystack==0) ) goto endInstrOOM; + firstChar = zNeedle[0]; + while( nNeedle<=nHaystack + && (zHaystack[0]!=firstChar || memcmp(zHaystack, zNeedle, nNeedle)!=0) + ){ N++; do{ nHaystack--; @@ -110093,6 +116896,13 @@ static void instrFunc( if( nNeedle>nHaystack ) N = 0; } sqlite3_result_int(context, N); +endInstr: + sqlite3_value_free(pC1); + sqlite3_value_free(pC2); + return; +endInstrOOM: + sqlite3_result_error_nomem(context); + goto endInstr; } /* @@ -110246,10 +117056,10 @@ static void roundFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ ** handle the rounding directly, ** otherwise use printf. */ - if( n==0 && r>=0 && r+4503599627370496.0 ){ + /* The value has no fractional part so there is nothing to round */ + }else if( n==0 ){ + r = (double)((sqlite_int64)(r+(r<0?-0.5:+0.5))); }else{ zBuf = sqlite3_mprintf("%.*f",n,r); if( zBuf==0 ){ @@ -110374,11 +117184,11 @@ static void randomBlob( int argc, sqlite3_value **argv ){ - int n; + sqlite3_int64 n; unsigned char *p; assert( argc==1 ); UNUSED_PARAMETER(argc); - n = sqlite3_value_int(argv[0]); + n = sqlite3_value_int64(argv[0]); if( n<1 ){ n = 1; } @@ -110691,6 +117501,7 @@ static void likeFunc( int nPat; sqlite3 *db = sqlite3_context_db_handle(context); struct compareInfo *pInfo = sqlite3_user_data(context); + struct compareInfo backupInfo; #ifdef SQLITE_LIKE_DOESNT_MATCH_BLOBS if( sqlite3_value_type(argv[0])==SQLITE_BLOB @@ -110703,8 +117514,6 @@ static void likeFunc( return; } #endif - zB = sqlite3_value_text(argv[0]); - zA = sqlite3_value_text(argv[1]); /* Limit the length of the LIKE or GLOB pattern to avoid problems ** of deep recursion and N*N behavior in patternCompare(). @@ -110716,8 +117525,6 @@ static void likeFunc( sqlite3_result_error(context, "LIKE or GLOB pattern too complex", -1); return; } - assert( zB==sqlite3_value_text(argv[0]) ); /* Encoding did not change */ - if( argc==3 ){ /* The escape character string must consist of a single UTF-8 character. ** Otherwise, return an error. @@ -110730,9 +117537,17 @@ static void likeFunc( return; } escape = sqlite3Utf8Read(&zEsc); + if( escape==pInfo->matchAll || escape==pInfo->matchOne ){ + memcpy(&backupInfo, pInfo, sizeof(backupInfo)); + pInfo = &backupInfo; + if( escape==pInfo->matchAll ) pInfo->matchAll = 0; + if( escape==pInfo->matchOne ) pInfo->matchOne = 0; + } }else{ escape = pInfo->matchSet; } + zB = sqlite3_value_text(argv[0]); + zA = sqlite3_value_text(argv[1]); if( zA && zB ){ #ifdef SQLITE_TEST sqlite3_like_count++; @@ -111116,7 +117931,7 @@ static void replaceFunc( ** whose index is a power of two: 1, 2, 4, 8, 16, 32, ... */ u8 *zOld; zOld = zOut; - zOut = sqlite3_realloc64(zOut, (int)nOut + (nOut - nStr - 1)); + zOut = sqlite3Realloc(zOut, (int)nOut + (nOut - nStr - 1)); if( zOut==0 ){ sqlite3_result_error_nomem(context); sqlite3_free(zOld); @@ -111369,7 +118184,7 @@ static void sumStep(sqlite3_context *context, int argc, sqlite3_value **argv){ i64 v = sqlite3_value_int64(argv[0]); p->rSum += v; if( (p->approx|p->overflow)==0 && sqlite3AddInt64(&p->iSum, v) ){ - p->overflow = 1; + p->approx = p->overflow = 1; } }else{ p->rSum += sqlite3_value_double(argv[0]); @@ -111377,6 +118192,32 @@ static void sumStep(sqlite3_context *context, int argc, sqlite3_value **argv){ } } } +#ifndef SQLITE_OMIT_WINDOWFUNC +static void sumInverse(sqlite3_context *context, int argc, sqlite3_value**argv){ + SumCtx *p; + int type; + assert( argc==1 ); + UNUSED_PARAMETER(argc); + p = sqlite3_aggregate_context(context, sizeof(*p)); + type = sqlite3_value_numeric_type(argv[0]); + /* p is always non-NULL because sumStep() will have been called first + ** to initialize it */ + if( ALWAYS(p) && type!=SQLITE_NULL ){ + assert( p->cnt>0 ); + p->cnt--; + assert( type==SQLITE_INTEGER || p->approx ); + if( type==SQLITE_INTEGER && p->approx==0 ){ + i64 v = sqlite3_value_int64(argv[0]); + p->rSum -= v; + p->iSum -= v; + }else{ + p->rSum -= sqlite3_value_double(argv[0]); + } + } +} +#else +# define sumInverse 0 +#endif /* SQLITE_OMIT_WINDOWFUNC */ static void sumFinalize(sqlite3_context *context){ SumCtx *p; p = sqlite3_aggregate_context(context, 0); @@ -111411,6 +118252,9 @@ static void totalFinalize(sqlite3_context *context){ typedef struct CountCtx CountCtx; struct CountCtx { i64 n; +#ifdef SQLITE_DEBUG + int bInverse; /* True if xInverse() ever called */ +#endif }; /* @@ -111428,7 +118272,7 @@ static void countStep(sqlite3_context *context, int argc, sqlite3_value **argv){ ** sure it still operates correctly, verify that its count agrees with our ** internal count when using count(*) and when the total count can be ** expressed as a 32-bit integer. */ - assert( argc==1 || p==0 || p->n>0x7fffffff + assert( argc==1 || p==0 || p->n>0x7fffffff || p->bInverse || p->n==sqlite3_aggregate_count(context) ); #endif } @@ -111437,6 +118281,21 @@ static void countFinalize(sqlite3_context *context){ p = sqlite3_aggregate_context(context, 0); sqlite3_result_int64(context, p ? p->n : 0); } +#ifndef SQLITE_OMIT_WINDOWFUNC +static void countInverse(sqlite3_context *ctx, int argc, sqlite3_value **argv){ + CountCtx *p; + p = sqlite3_aggregate_context(ctx, sizeof(*p)); + /* p is always non-NULL since countStep() will have been called first */ + if( (argc==0 || SQLITE_NULL!=sqlite3_value_type(argv[0])) && ALWAYS(p) ){ + p->n--; +#ifdef SQLITE_DEBUG + p->bInverse = 1; +#endif + } +} +#else +# define countInverse 0 +#endif /* SQLITE_OMIT_WINDOWFUNC */ /* ** Routines to implement min() and max() aggregate functions. @@ -111453,7 +118312,7 @@ static void minmaxStep( pBest = (Mem *)sqlite3_aggregate_context(context, sizeof(*pBest)); if( !pBest ) return; - if( sqlite3_value_type(argv[0])==SQLITE_NULL ){ + if( sqlite3_value_type(pArg)==SQLITE_NULL ){ if( pBest->flags ) sqlite3SkipAccumulatorLoad(context); }else if( pBest->flags ){ int max; @@ -111479,16 +118338,26 @@ static void minmaxStep( sqlite3VdbeMemCopy(pBest, pArg); } } -static void minMaxFinalize(sqlite3_context *context){ +static void minMaxValueFinalize(sqlite3_context *context, int bValue){ sqlite3_value *pRes; pRes = (sqlite3_value *)sqlite3_aggregate_context(context, 0); if( pRes ){ if( pRes->flags ){ sqlite3_result_value(context, pRes); } - sqlite3VdbeMemRelease(pRes); + if( bValue==0 ) sqlite3VdbeMemRelease(pRes); } } +#ifndef SQLITE_OMIT_WINDOWFUNC +static void minMaxValue(sqlite3_context *context){ + minMaxValueFinalize(context, 1); +} +#else +# define minMaxValue 0 +#endif /* SQLITE_OMIT_WINDOWFUNC */ +static void minMaxFinalize(sqlite3_context *context){ + minMaxValueFinalize(context, 0); +} /* ** group_concat(EXPR, ?SEPARATOR?) @@ -111525,6 +118394,38 @@ static void groupConcatStep( if( zVal ) sqlite3_str_append(pAccum, zVal, nVal); } } +#ifndef SQLITE_OMIT_WINDOWFUNC +static void groupConcatInverse( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + int n; + StrAccum *pAccum; + assert( argc==1 || argc==2 ); + if( sqlite3_value_type(argv[0])==SQLITE_NULL ) return; + pAccum = (StrAccum*)sqlite3_aggregate_context(context, sizeof(*pAccum)); + /* pAccum is always non-NULL since groupConcatStep() will have always + ** run frist to initialize it */ + if( ALWAYS(pAccum) ){ + n = sqlite3_value_bytes(argv[0]); + if( argc==2 ){ + n += sqlite3_value_bytes(argv[1]); + }else{ + n++; + } + if( n>=(int)pAccum->nChar ){ + pAccum->nChar = 0; + }else{ + pAccum->nChar -= n; + memmove(pAccum->zText, &pAccum->zText[n], pAccum->nChar); + } + if( pAccum->nChar==0 ) pAccum->mxAlloc = 0; + } +} +#else +# define groupConcatInverse 0 +#endif /* SQLITE_OMIT_WINDOWFUNC */ static void groupConcatFinalize(sqlite3_context *context){ StrAccum *pAccum; pAccum = sqlite3_aggregate_context(context, 0); @@ -111539,6 +118440,24 @@ static void groupConcatFinalize(sqlite3_context *context){ } } } +#ifndef SQLITE_OMIT_WINDOWFUNC +static void groupConcatValue(sqlite3_context *context){ + sqlite3_str *pAccum; + pAccum = (sqlite3_str*)sqlite3_aggregate_context(context, 0); + if( pAccum ){ + if( pAccum->accError==SQLITE_TOOBIG ){ + sqlite3_result_error_toobig(context); + }else if( pAccum->accError==SQLITE_NOMEM ){ + sqlite3_result_error_nomem(context); + }else{ + const char *zText = sqlite3_str_value(pAccum); + sqlite3_result_text(context, zText, -1, SQLITE_TRANSIENT); + } + } +} +#else +# define groupConcatValue 0 +#endif /* SQLITE_OMIT_WINDOWFUNC */ /* ** This routine does per-connection function registration. Most @@ -111554,35 +118473,24 @@ SQLITE_PRIVATE void sqlite3RegisterPerConnectionBuiltinFunctions(sqlite3 *db){ } /* -** Set the LIKEOPT flag on the 2-argument function with the given name. -*/ -static void setLikeOptFlag(sqlite3 *db, const char *zName, u8 flagVal){ - FuncDef *pDef; - pDef = sqlite3FindFunction(db, zName, 2, SQLITE_UTF8, 0); - if( ALWAYS(pDef) ){ - pDef->funcFlags |= flagVal; - } -} - -/* -** Register the built-in LIKE and GLOB functions. The caseSensitive +** Re-register the built-in LIKE functions. The caseSensitive ** parameter determines whether or not the LIKE operator is case -** sensitive. GLOB is always case sensitive. +** sensitive. */ SQLITE_PRIVATE void sqlite3RegisterLikeFunctions(sqlite3 *db, int caseSensitive){ struct compareInfo *pInfo; + int flags; if( caseSensitive ){ pInfo = (struct compareInfo*)&likeInfoAlt; + flags = SQLITE_FUNC_LIKE | SQLITE_FUNC_CASE; }else{ pInfo = (struct compareInfo*)&likeInfoNorm; + flags = SQLITE_FUNC_LIKE; } - sqlite3CreateFunc(db, "like", 2, SQLITE_UTF8, pInfo, likeFunc, 0, 0, 0); - sqlite3CreateFunc(db, "like", 3, SQLITE_UTF8, pInfo, likeFunc, 0, 0, 0); - sqlite3CreateFunc(db, "glob", 2, SQLITE_UTF8, - (struct compareInfo*)&globInfo, likeFunc, 0, 0, 0); - setLikeOptFlag(db, "glob", SQLITE_FUNC_LIKE | SQLITE_FUNC_CASE); - setLikeOptFlag(db, "like", - caseSensitive ? (SQLITE_FUNC_LIKE | SQLITE_FUNC_CASE) : SQLITE_FUNC_LIKE); + sqlite3CreateFunc(db, "like", 2, SQLITE_UTF8, pInfo, likeFunc, 0, 0, 0, 0, 0); + sqlite3CreateFunc(db, "like", 3, SQLITE_UTF8, pInfo, likeFunc, 0, 0, 0, 0, 0); + sqlite3FindFunction(db, "like", 2, SQLITE_UTF8, 0)->funcFlags |= flags; + sqlite3FindFunction(db, "like", 3, SQLITE_UTF8, 0)->funcFlags |= flags; } /* @@ -111611,9 +118519,22 @@ SQLITE_PRIVATE int sqlite3IsLikeFunction(sqlite3 *db, Expr *pExpr, int *pIsNocas assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); nExpr = pExpr->x.pList->nExpr; pDef = sqlite3FindFunction(db, pExpr->u.zToken, nExpr, SQLITE_UTF8, 0); +#ifdef SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION + if( pDef==0 ) return 0; +#endif if( NEVER(pDef==0) || (pDef->funcFlags & SQLITE_FUNC_LIKE)==0 ){ return 0; } + + /* The memcpy() statement assumes that the wildcard characters are + ** the first three statements in the compareInfo structure. The + ** asserts() that follow verify that assumption + */ + memcpy(aWc, pDef->pUserData, 3); + assert( (char*)&likeInfoAlt == (char*)&likeInfoAlt.matchAll ); + assert( &((char*)&likeInfoAlt)[1] == (char*)&likeInfoAlt.matchOne ); + assert( &((char*)&likeInfoAlt)[2] == (char*)&likeInfoAlt.matchSet ); + if( nExpr<3 ){ aWc[3] = 0; }else{ @@ -111622,17 +118543,11 @@ SQLITE_PRIVATE int sqlite3IsLikeFunction(sqlite3 *db, Expr *pExpr, int *pIsNocas if( pEscape->op!=TK_STRING ) return 0; zEscape = pEscape->u.zToken; if( zEscape[0]==0 || zEscape[1]!=0 ) return 0; + if( zEscape[0]==aWc[0] ) return 0; + if( zEscape[0]==aWc[1] ) return 0; aWc[3] = zEscape[0]; } - /* The memcpy() statement assumes that the wildcard characters are - ** the first three statements in the compareInfo structure. The - ** asserts() that follow verify that assumption - */ - memcpy(aWc, pDef->pUserData, 3); - assert( (char*)&likeInfoAlt == (char*)&likeInfoAlt.matchAll ); - assert( &((char*)&likeInfoAlt)[1] == (char*)&likeInfoAlt.matchOne ); - assert( &((char*)&likeInfoAlt)[2] == (char*)&likeInfoAlt.matchSet ); *pIsNocase = (pDef->funcFlags & SQLITE_FUNC_CASE)==0; return 1; } @@ -111656,12 +118571,20 @@ SQLITE_PRIVATE void sqlite3RegisterBuiltinFunctions(void){ ** For peak efficiency, put the most frequently used function last. */ static FuncDef aBuiltinFunc[] = { +/***** Functions only available with SQLITE_TESTCTRL_INTERNAL_FUNCTIONS *****/ + TEST_FUNC(implies_nonnull_row, 2, INLINEFUNC_implies_nonnull_row, 0), + TEST_FUNC(expr_compare, 2, INLINEFUNC_expr_compare, 0), + TEST_FUNC(expr_implies_expr, 2, INLINEFUNC_expr_implies_expr, 0), +#ifdef SQLITE_DEBUG + TEST_FUNC(affinity, 1, INLINEFUNC_affinity, 0), +#endif +/***** Regular functions *****/ #ifdef SQLITE_SOUNDEX FUNCTION(soundex, 1, 0, 0, soundexFunc ), #endif #ifndef SQLITE_OMIT_LOAD_EXTENSION - VFUNCTION(load_extension, 1, 0, 0, loadExt ), - VFUNCTION(load_extension, 2, 0, 0, loadExt ), + SFUNCTION(load_extension, 1, 0, 0, loadExt ), + SFUNCTION(load_extension, 2, 0, 0, loadExt ), #endif #if SQLITE_USER_AUTHENTICATION FUNCTION(sqlite_crypt, 2, 0, 0, sqlite3CryptFunc ), @@ -111670,12 +118593,9 @@ SQLITE_PRIVATE void sqlite3RegisterBuiltinFunctions(void){ DFUNCTION(sqlite_compileoption_used,1, 0, 0, compileoptionusedFunc ), DFUNCTION(sqlite_compileoption_get, 1, 0, 0, compileoptiongetFunc ), #endif /* SQLITE_OMIT_COMPILEOPTION_DIAGS */ - FUNCTION2(unlikely, 1, 0, 0, noopFunc, SQLITE_FUNC_UNLIKELY), - FUNCTION2(likelihood, 2, 0, 0, noopFunc, SQLITE_FUNC_UNLIKELY), - FUNCTION2(likely, 1, 0, 0, noopFunc, SQLITE_FUNC_UNLIKELY), -#ifdef SQLITE_DEBUG - FUNCTION2(affinity, 1, 0, 0, noopFunc, SQLITE_FUNC_AFFINITY), -#endif + INLINE_FUNC(unlikely, 1, INLINEFUNC_unlikely, SQLITE_FUNC_UNLIKELY), + INLINE_FUNC(likelihood, 2, INLINEFUNC_unlikely, SQLITE_FUNC_UNLIKELY), + INLINE_FUNC(likely, 1, INLINEFUNC_unlikely, SQLITE_FUNC_UNLIKELY), #ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC FUNCTION2(sqlite_offset, 1, 0, 0, noopFunc, SQLITE_FUNC_OFFSET| SQLITE_FUNC_TYPEOF), @@ -111688,11 +118608,11 @@ SQLITE_PRIVATE void sqlite3RegisterBuiltinFunctions(void){ FUNCTION(trim, 2, 3, 0, trimFunc ), FUNCTION(min, -1, 0, 1, minmaxFunc ), FUNCTION(min, 0, 0, 1, 0 ), - AGGREGATE2(min, 1, 0, 1, minmaxStep, minMaxFinalize, + WAGGREGATE(min, 1, 0, 1, minmaxStep, minMaxFinalize, minMaxValue, 0, SQLITE_FUNC_MINMAX ), FUNCTION(max, -1, 1, 1, minmaxFunc ), FUNCTION(max, 0, 1, 1, 0 ), - AGGREGATE2(max, 1, 1, 1, minmaxStep, minMaxFinalize, + WAGGREGATE(max, 1, 1, 1, minmaxStep, minMaxFinalize, minMaxValue, 0, SQLITE_FUNC_MINMAX ), FUNCTION2(typeof, 1, 0, 0, typeofFunc, SQLITE_FUNC_TYPEOF), FUNCTION2(length, 1, 0, 0, lengthFunc, SQLITE_FUNC_LENGTH), @@ -111708,7 +118628,7 @@ SQLITE_PRIVATE void sqlite3RegisterBuiltinFunctions(void){ FUNCTION(upper, 1, 0, 0, upperFunc ), FUNCTION(lower, 1, 0, 0, lowerFunc ), FUNCTION(hex, 1, 0, 0, hexFunc ), - FUNCTION2(ifnull, 2, 0, 0, noopFunc, SQLITE_FUNC_COALESCE), + INLINE_FUNC(ifnull, 2, INLINEFUNC_coalesce, 0 ), VFUNCTION(random, 0, 0, 0, randomFunc ), VFUNCTION(randomblob, 1, 0, 0, randomBlob ), FUNCTION(nullif, 2, 0, 1, nullifFunc ), @@ -111723,14 +118643,17 @@ SQLITE_PRIVATE void sqlite3RegisterBuiltinFunctions(void){ FUNCTION(zeroblob, 1, 0, 0, zeroblobFunc ), FUNCTION(substr, 2, 0, 0, substrFunc ), FUNCTION(substr, 3, 0, 0, substrFunc ), - AGGREGATE(sum, 1, 0, 0, sumStep, sumFinalize ), - AGGREGATE(total, 1, 0, 0, sumStep, totalFinalize ), - AGGREGATE(avg, 1, 0, 0, sumStep, avgFinalize ), - AGGREGATE2(count, 0, 0, 0, countStep, countFinalize, - SQLITE_FUNC_COUNT ), - AGGREGATE(count, 1, 0, 0, countStep, countFinalize ), - AGGREGATE(group_concat, 1, 0, 0, groupConcatStep, groupConcatFinalize), - AGGREGATE(group_concat, 2, 0, 0, groupConcatStep, groupConcatFinalize), + WAGGREGATE(sum, 1,0,0, sumStep, sumFinalize, sumFinalize, sumInverse, 0), + WAGGREGATE(total, 1,0,0, sumStep,totalFinalize,totalFinalize,sumInverse, 0), + WAGGREGATE(avg, 1,0,0, sumStep, avgFinalize, avgFinalize, sumInverse, 0), + WAGGREGATE(count, 0,0,0, countStep, + countFinalize, countFinalize, countInverse, SQLITE_FUNC_COUNT ), + WAGGREGATE(count, 1,0,0, countStep, + countFinalize, countFinalize, countInverse, 0 ), + WAGGREGATE(group_concat, 1, 0, 0, groupConcatStep, + groupConcatFinalize, groupConcatValue, groupConcatInverse, 0), + WAGGREGATE(group_concat, 2, 0, 0, groupConcatStep, + groupConcatFinalize, groupConcatValue, groupConcatInverse, 0), LIKEFUNC(glob, 2, &globInfo, SQLITE_FUNC_LIKE|SQLITE_FUNC_CASE), #ifdef SQLITE_CASE_SENSITIVE_LIKE @@ -111745,14 +118668,13 @@ SQLITE_PRIVATE void sqlite3RegisterBuiltinFunctions(void){ #endif FUNCTION(coalesce, 1, 0, 0, 0 ), FUNCTION(coalesce, 0, 0, 0, 0 ), - FUNCTION2(coalesce, -1, 0, 0, noopFunc, SQLITE_FUNC_COALESCE), + INLINE_FUNC(coalesce, -1, INLINEFUNC_coalesce, 0 ), + INLINE_FUNC(iif, 3, INLINEFUNC_iif, 0 ), }; #ifndef SQLITE_OMIT_ALTERTABLE sqlite3AlterFunctions(); #endif -#if defined(SQLITE_ENABLE_STAT3) || defined(SQLITE_ENABLE_STAT4) - sqlite3AnalyzeFunctions(); -#endif + sqlite3WindowFunctions(); sqlite3RegisterDateTimeFunctions(); sqlite3InsertBuiltinFuncs(aBuiltinFunc, ArraySize(aBuiltinFunc)); @@ -112106,7 +119028,7 @@ static void fkLookupParent( int i; /* Iterator variable */ Vdbe *v = sqlite3GetVdbe(pParse); /* Vdbe to add code to */ int iCur = pParse->nTab - 1; /* Cursor number to use */ - int iOk = sqlite3VdbeMakeLabel(v); /* jump here if parent key found */ + int iOk = sqlite3VdbeMakeLabel(pParse); /* jump here if parent key found */ sqlite3VdbeVerifyAbortable(v, (!pFKey->isDeferred @@ -112126,7 +119048,7 @@ static void fkLookupParent( VdbeCoverage(v); } for(i=0; inCol; i++){ - int iReg = aiCol[i] + regData + 1; + int iReg = sqlite3TableColumnToStorage(pFKey->pFrom,aiCol[i]) + regData + 1; sqlite3VdbeAddOp2(v, OP_IsNull, iReg, iOk); VdbeCoverage(v); } @@ -112142,7 +119064,8 @@ static void fkLookupParent( ** is no matching parent key. Before using MustBeInt, make a copy of ** the value. Otherwise, the value inserted into the child key column ** will have INTEGER affinity applied to it, which may not be correct. */ - sqlite3VdbeAddOp2(v, OP_SCopy, aiCol[0]+1+regData, regTemp); + sqlite3VdbeAddOp2(v, OP_SCopy, + sqlite3TableColumnToStorage(pFKey->pFrom,aiCol[0])+1+regData, regTemp); iMustBeInt = sqlite3VdbeAddOp2(v, OP_MustBeInt, regTemp, 0); VdbeCoverage(v); @@ -112169,7 +119092,9 @@ static void fkLookupParent( sqlite3VdbeAddOp3(v, OP_OpenRead, iCur, pIdx->tnum, iDb); sqlite3VdbeSetP4KeyInfo(pParse, pIdx); for(i=0; ipFrom, aiCol[i])+1+regData, + regTemp+i); } /* If the parent table is the same as the child table, and we are about @@ -112185,8 +119110,11 @@ static void fkLookupParent( if( pTab==pFKey->pFrom && nIncr==1 ){ int iJump = sqlite3VdbeCurrentAddr(v) + nCol + 1; for(i=0; iaiColumn[i]+1+regData; + int iChild = sqlite3TableColumnToStorage(pFKey->pFrom,aiCol[i]) + +1+regData; + int iParent = 1+regData; + iParent += sqlite3TableColumnToStorage(pIdx->pTable, + pIdx->aiColumn[i]); assert( pIdx->aiColumn[i]>=0 ); assert( aiCol[i]!=pTab->iPKey ); if( pIdx->aiColumn[i]==pTab->iPKey ){ @@ -112254,14 +119182,14 @@ static Expr *exprTableRegister( if( pExpr ){ if( iCol>=0 && iCol!=pTab->iPKey ){ pCol = &pTab->aCol[iCol]; - pExpr->iTable = regBase + iCol + 1; - pExpr->affinity = pCol->affinity; + pExpr->iTable = regBase + sqlite3TableColumnToStorage(pTab,iCol) + 1; + pExpr->affExpr = pCol->affinity; zColl = pCol->zColl; if( zColl==0 ) zColl = db->pDfltColl->zName; pExpr = sqlite3ExprAddCollateString(pParse, pExpr, zColl); }else{ pExpr->iTable = regBase; - pExpr->affinity = SQLITE_AFF_INTEGER; + pExpr->affExpr = SQLITE_AFF_INTEGER; } } return pExpr; @@ -112279,7 +119207,7 @@ static Expr *exprTableColumn( ){ Expr *pExpr = sqlite3Expr(db, TK_COLUMN, 0); if( pExpr ){ - pExpr->pTab = pTab; + pExpr->y.pTab = pTab; pExpr->iTable = iCursor; pExpr->iColumn = iCol; } @@ -112368,7 +119296,7 @@ static void fkScanChildren( zCol = pFKey->pFrom->aCol[iCol].zName; pRight = sqlite3Expr(db, TK_ID, zCol); pEq = sqlite3PExpr(pParse, TK_EQ, pLeft, pRight); - pWhere = sqlite3ExprAnd(db, pWhere, pEq); + pWhere = sqlite3ExprAnd(pParse, pWhere, pEq); } /* If the child table is the same as the parent table, then add terms @@ -112379,8 +119307,11 @@ static void fkScanChildren( ** NOT( $current_a==a AND $current_b==b AND ... ) ** ** The first form is used for rowid tables. The second form is used - ** for WITHOUT ROWID tables. In the second form, the primary key is - ** (a,b,...) + ** for WITHOUT ROWID tables. In the second form, the *parent* key is + ** (a,b,...). Either the parent or primary key could be used to + ** uniquely identify the current row, but the parent key is more convenient + ** as the required values have already been loaded into registers + ** by the caller. */ if( pTab==pFKey->pFrom && nIncr>0 ){ Expr *pNe; /* Expression (pLeft != pRight) */ @@ -112392,19 +119323,18 @@ static void fkScanChildren( pNe = sqlite3PExpr(pParse, TK_NE, pLeft, pRight); }else{ Expr *pEq, *pAll = 0; - Index *pPk = sqlite3PrimaryKeyIndex(pTab); assert( pIdx!=0 ); - for(i=0; inKeyCol; i++){ + for(i=0; inKeyCol; i++){ i16 iCol = pIdx->aiColumn[i]; assert( iCol>=0 ); pLeft = exprTableRegister(pParse, pTab, regData, iCol); - pRight = exprTableColumn(db, pTab, pSrc->a[0].iCursor, iCol); - pEq = sqlite3PExpr(pParse, TK_EQ, pLeft, pRight); - pAll = sqlite3ExprAnd(db, pAll, pEq); + pRight = sqlite3Expr(db, TK_ID, pTab->aCol[iCol].zName); + pEq = sqlite3PExpr(pParse, TK_IS, pLeft, pRight); + pAll = sqlite3ExprAnd(pParse, pAll, pEq); } pNe = sqlite3PExpr(pParse, TK_NOT, pAll, 0); } - pWhere = sqlite3ExprAnd(db, pWhere, pNe); + pWhere = sqlite3ExprAnd(pParse, pWhere, pNe); } /* Resolve the references in the WHERE clause. */ @@ -112427,7 +119357,7 @@ static void fkScanChildren( /* Clean up the WHERE clause constructed above. */ sqlite3ExprDelete(db, pWhere); if( iFkIfZero ){ - sqlite3VdbeJumpHere(v, iFkIfZero); + sqlite3VdbeJumpHereOrPopInst(v, iFkIfZero); } } @@ -112487,11 +119417,12 @@ static void fkTriggerDelete(sqlite3 *dbMem, Trigger *p){ */ SQLITE_PRIVATE void sqlite3FkDropTable(Parse *pParse, SrcList *pName, Table *pTab){ sqlite3 *db = pParse->db; - if( (db->flags&SQLITE_ForeignKeys) && !IsVirtual(pTab) && !pTab->pSelect ){ + if( (db->flags&SQLITE_ForeignKeys) && !IsVirtual(pTab) ){ int iSkip = 0; Vdbe *v = sqlite3GetVdbe(pParse); assert( v ); /* VDBE has already been allocated */ + assert( pTab->pSelect==0 ); /* Not a view */ if( sqlite3FkReferences(pTab)==0 ){ /* Search for a deferred foreign key constraint for which this table ** is the child table. If one cannot be found, return without @@ -112503,7 +119434,7 @@ SQLITE_PRIVATE void sqlite3FkDropTable(Parse *pParse, SrcList *pName, Table *pTa if( p->isDeferred || (db->flags & SQLITE_DeferFKs) ) break; } if( !p ) return; - iSkip = sqlite3VdbeMakeLabel(v); + iSkip = sqlite3VdbeMakeLabel(pParse); sqlite3VdbeAddOp2(v, OP_FkIfZero, 1, iSkip); VdbeCoverage(v); } @@ -112700,7 +119631,9 @@ SQLITE_PRIVATE void sqlite3FkCheck( Vdbe *v = sqlite3GetVdbe(pParse); int iJump = sqlite3VdbeCurrentAddr(v) + pFKey->nCol + 1; for(i=0; inCol; i++){ - int iReg = pFKey->aCol[i].iFrom + regOld + 1; + int iFromCol, iReg; + iFromCol = pFKey->aCol[i].iFrom; + iReg = sqlite3TableColumnToStorage(pFKey->pFrom,iFromCol) + regOld+1; sqlite3VdbeAddOp2(v, OP_IsNull, iReg, iJump); VdbeCoverage(v); } sqlite3VdbeAddOp2(v, OP_FkCounter, pFKey->isDeferred, -1); @@ -112788,7 +119721,7 @@ SQLITE_PRIVATE void sqlite3FkCheck( /* Create a SrcList structure containing the child table. We need the ** child table as a SrcList for sqlite3WhereBegin() */ - pSrc = sqlite3SrcListAppend(db, 0, 0, 0); + pSrc = sqlite3SrcListAppend(pParse, 0, 0, 0); if( pSrc ){ struct SrcList_item *pItem = pSrc->a; pItem->pTab = pFKey->pFrom; @@ -113009,7 +119942,7 @@ static Trigger *fkActionTrigger( sqlite3ExprAlloc(db, TK_ID, &tToCol, 0)), sqlite3ExprAlloc(db, TK_ID, &tFromCol, 0) ); - pWhere = sqlite3ExprAnd(db, pWhere, pEq); + pWhere = sqlite3ExprAnd(pParse, pWhere, pEq); /* For ON UPDATE, construct the next term of the WHEN clause. ** The final WHEN clause will be like this: @@ -113025,7 +119958,7 @@ static Trigger *fkActionTrigger( sqlite3ExprAlloc(db, TK_ID, &tNew, 0), sqlite3ExprAlloc(db, TK_ID, &tToCol, 0)) ); - pWhen = sqlite3ExprAnd(db, pWhen, pEq); + pWhen = sqlite3ExprAnd(pParse, pWhen, pEq); } if( action!=OE_Restrict && (action!=OE_Cascade || pChanges) ){ @@ -113035,7 +119968,15 @@ static Trigger *fkActionTrigger( sqlite3ExprAlloc(db, TK_ID, &tNew, 0), sqlite3ExprAlloc(db, TK_ID, &tToCol, 0)); }else if( action==OE_SetDflt ){ - Expr *pDflt = pFKey->pFrom->aCol[iFromCol].pDflt; + Column *pCol = pFKey->pFrom->aCol + iFromCol; + Expr *pDflt; + if( pCol->colFlags & COLFLAG_GENERATED ){ + testcase( pCol->colFlags & COLFLAG_VIRTUAL ); + testcase( pCol->colFlags & COLFLAG_STORED ); + pDflt = 0; + }else{ + pDflt = pCol->pDflt; + } if( pDflt ){ pNew = sqlite3ExprDup(db, pDflt, 0); }else{ @@ -113061,11 +120002,11 @@ static Trigger *fkActionTrigger( tFrom.n = nFrom; pRaise = sqlite3Expr(db, TK_RAISE, "FOREIGN KEY constraint failed"); if( pRaise ){ - pRaise->affinity = OE_Abort; + pRaise->affExpr = OE_Abort; } pSelect = sqlite3SelectNew(pParse, sqlite3ExprListAppend(pParse, 0, pRaise), - sqlite3SrcListAppend(db, 0, &tFrom, 0), + sqlite3SrcListAppend(pParse, 0, &tFrom, 0), pWhere, 0, 0, 0, 0, 0 ); @@ -113073,7 +120014,7 @@ static Trigger *fkActionTrigger( } /* Disable lookaside memory allocation */ - db->lookaside.bDisable++; + DisableLookaside; pTrigger = (Trigger *)sqlite3DbMallocZero(db, sizeof(Trigger) + /* struct Trigger */ @@ -113095,7 +120036,7 @@ static Trigger *fkActionTrigger( } /* Re-enable the lookaside buffer, if it was disabled earlier. */ - db->lookaside.bDisable--; + EnableLookaside; sqlite3ExprDelete(db, pWhere); sqlite3ExprDelete(db, pWhen); @@ -113106,6 +120047,7 @@ static Trigger *fkActionTrigger( return 0; } assert( pStep!=0 ); + assert( pTrigger!=0 ); switch( action ){ case OE_Restrict: @@ -113245,7 +120187,7 @@ SQLITE_PRIVATE void sqlite3OpenTable( sqlite3TableLock(pParse, iDb, pTab->tnum, (opcode==OP_OpenWrite)?1:0, pTab->zName); if( HasRowid(pTab) ){ - sqlite3VdbeAddOp4Int(v, opcode, iCur, pTab->tnum, iDb, pTab->nCol); + sqlite3VdbeAddOp4Int(v, opcode, iCur, pTab->tnum, iDb, pTab->nNVCol); VdbeComment((v, "%s", pTab->zName)); }else{ Index *pPk = sqlite3PrimaryKeyIndex(pTab); @@ -113296,18 +120238,19 @@ SQLITE_PRIVATE const char *sqlite3IndexAffinityStr(sqlite3 *db, Index *pIdx){ } for(n=0; nnColumn; n++){ i16 x = pIdx->aiColumn[n]; + char aff; if( x>=0 ){ - pIdx->zColAff[n] = pTab->aCol[x].affinity; + aff = pTab->aCol[x].affinity; }else if( x==XN_ROWID ){ - pIdx->zColAff[n] = SQLITE_AFF_INTEGER; + aff = SQLITE_AFF_INTEGER; }else{ - char aff; assert( x==XN_EXPR ); assert( pIdx->aColExpr!=0 ); aff = sqlite3ExprAffinity(pIdx->aColExpr->a[n].pExpr); - if( aff==0 ) aff = SQLITE_AFF_BLOB; - pIdx->zColAff[n] = aff; } + if( affSQLITE_AFF_NUMERIC) aff = SQLITE_AFF_NUMERIC; + pIdx->zColAff[n] = aff; } pIdx->zColAff[n] = 0; } @@ -113336,7 +120279,7 @@ SQLITE_PRIVATE const char *sqlite3IndexAffinityStr(sqlite3 *db, Index *pIdx){ ** 'E' REAL */ SQLITE_PRIVATE void sqlite3TableAffinity(Vdbe *v, Table *pTab, int iReg){ - int i; + int i, j; char *zColAff = pTab->zColAff; if( zColAff==0 ){ sqlite3 *db = sqlite3VdbeDb(v); @@ -113346,15 +120289,19 @@ SQLITE_PRIVATE void sqlite3TableAffinity(Vdbe *v, Table *pTab, int iReg){ return; } - for(i=0; inCol; i++){ - zColAff[i] = pTab->aCol[i].affinity; + for(i=j=0; inCol; i++){ + assert( pTab->aCol[i].affinity!=0 ); + if( (pTab->aCol[i].colFlags & COLFLAG_VIRTUAL)==0 ){ + zColAff[j++] = pTab->aCol[i].affinity; + } } do{ - zColAff[i--] = 0; - }while( i>=0 && zColAff[i]==SQLITE_AFF_BLOB ); + zColAff[j--] = 0; + }while( j>=0 && zColAff[j]<=SQLITE_AFF_BLOB ); pTab->zColAff = zColAff; } - i = sqlite3Strlen30(zColAff); + assert( zColAff!=0 ); + i = sqlite3Strlen30NN(zColAff); if( i ){ if( iReg ){ sqlite3VdbeAddOp4(v, OP_Affinity, iReg, i, 0, zColAff, i); @@ -113404,6 +120351,119 @@ static int readsTable(Parse *p, int iDb, Table *pTab){ return 0; } +/* This walker callback will compute the union of colFlags flags for all +** referenced columns in a CHECK constraint or generated column expression. +*/ +static int exprColumnFlagUnion(Walker *pWalker, Expr *pExpr){ + if( pExpr->op==TK_COLUMN && pExpr->iColumn>=0 ){ + assert( pExpr->iColumn < pWalker->u.pTab->nCol ); + pWalker->eCode |= pWalker->u.pTab->aCol[pExpr->iColumn].colFlags; + } + return WRC_Continue; +} + +#ifndef SQLITE_OMIT_GENERATED_COLUMNS +/* +** All regular columns for table pTab have been puts into registers +** starting with iRegStore. The registers that correspond to STORED +** or VIRTUAL columns have not yet been initialized. This routine goes +** back and computes the values for those columns based on the previously +** computed normal columns. +*/ +SQLITE_PRIVATE void sqlite3ComputeGeneratedColumns( + Parse *pParse, /* Parsing context */ + int iRegStore, /* Register holding the first column */ + Table *pTab /* The table */ +){ + int i; + Walker w; + Column *pRedo; + int eProgress; + VdbeOp *pOp; + + assert( pTab->tabFlags & TF_HasGenerated ); + testcase( pTab->tabFlags & TF_HasVirtual ); + testcase( pTab->tabFlags & TF_HasStored ); + + /* Before computing generated columns, first go through and make sure + ** that appropriate affinity has been applied to the regular columns + */ + sqlite3TableAffinity(pParse->pVdbe, pTab, iRegStore); + if( (pTab->tabFlags & TF_HasStored)!=0 + && (pOp = sqlite3VdbeGetOp(pParse->pVdbe,-1))->opcode==OP_Affinity + ){ + /* Change the OP_Affinity argument to '@' (NONE) for all stored + ** columns. '@' is the no-op affinity and those columns have not + ** yet been computed. */ + int ii, jj; + char *zP4 = pOp->p4.z; + assert( zP4!=0 ); + assert( pOp->p4type==P4_DYNAMIC ); + for(ii=jj=0; zP4[jj]; ii++){ + if( pTab->aCol[ii].colFlags & COLFLAG_VIRTUAL ){ + continue; + } + if( pTab->aCol[ii].colFlags & COLFLAG_STORED ){ + zP4[jj] = SQLITE_AFF_NONE; + } + jj++; + } + } + + /* Because there can be multiple generated columns that refer to one another, + ** this is a two-pass algorithm. On the first pass, mark all generated + ** columns as "not available". + */ + for(i=0; inCol; i++){ + if( pTab->aCol[i].colFlags & COLFLAG_GENERATED ){ + testcase( pTab->aCol[i].colFlags & COLFLAG_VIRTUAL ); + testcase( pTab->aCol[i].colFlags & COLFLAG_STORED ); + pTab->aCol[i].colFlags |= COLFLAG_NOTAVAIL; + } + } + + w.u.pTab = pTab; + w.xExprCallback = exprColumnFlagUnion; + w.xSelectCallback = 0; + w.xSelectCallback2 = 0; + + /* On the second pass, compute the value of each NOT-AVAILABLE column. + ** Companion code in the TK_COLUMN case of sqlite3ExprCodeTarget() will + ** compute dependencies and mark remove the COLSPAN_NOTAVAIL mark, as + ** they are needed. + */ + pParse->iSelfTab = -iRegStore; + do{ + eProgress = 0; + pRedo = 0; + for(i=0; inCol; i++){ + Column *pCol = pTab->aCol + i; + if( (pCol->colFlags & COLFLAG_NOTAVAIL)!=0 ){ + int x; + pCol->colFlags |= COLFLAG_BUSY; + w.eCode = 0; + sqlite3WalkExpr(&w, pCol->pDflt); + pCol->colFlags &= ~COLFLAG_BUSY; + if( w.eCode & COLFLAG_NOTAVAIL ){ + pRedo = pCol; + continue; + } + eProgress = 1; + assert( pCol->colFlags & COLFLAG_GENERATED ); + x = sqlite3TableColumnToStorage(pTab, i) + iRegStore; + sqlite3ExprCodeGeneratedColumn(pParse, pCol, x); + pCol->colFlags &= ~COLFLAG_NOTAVAIL; + } + } + }while( pRedo && eProgress ); + if( pRedo ){ + sqlite3ErrorMsg(pParse, "generated column loop on \"%s\"", pRedo->zName); + } + pParse->iSelfTab = 0; +} +#endif /* SQLITE_OMIT_GENERATED_COLUMNS */ + + #ifndef SQLITE_OMIT_AUTOINCREMENT /* ** Locate or create an AutoincInfo structure associated with table pTab @@ -113526,6 +120586,7 @@ SQLITE_PRIVATE void sqlite3AutoincrementBegin(Parse *pParse){ aOp[7].p2 = memId+2; aOp[7].p1 = memId; aOp[10].p2 = memId; + if( pParse->nTab==0 ) pParse->nTab = 1; } } @@ -113710,7 +120771,7 @@ SQLITE_PRIVATE void sqlite3Insert( Parse *pParse, /* Parser context */ SrcList *pTabList, /* Name of table into which we are inserting */ Select *pSelect, /* A SELECT statement to use as the data source */ - IdList *pColumn, /* Column names corresponding to IDLIST. */ + IdList *pColumn, /* Column names corresponding to IDLIST, or NULL. */ int onError, /* How to handle constraint errors */ Upsert *pUpsert /* ON CONFLICT clauses for upsert, or NULL */ ){ @@ -113735,6 +120796,7 @@ SQLITE_PRIVATE void sqlite3Insert( u8 withoutRowid; /* 0 for normal table. 1 for WITHOUT ROWID table */ u8 bIdListInOrder; /* True if IDLIST is in table order */ ExprList *pList = 0; /* List of VALUES() to be inserted */ + int iRegStore; /* Register in which to store next column */ /* Register allocations */ int regFromSelect = 0;/* Base register for data coming from SELECT */ @@ -113842,8 +120904,8 @@ SQLITE_PRIVATE void sqlite3Insert( */ regAutoinc = autoIncBegin(pParse, iDb, pTab); - /* Allocate registers for holding the rowid of the new row, - ** the content of the new row, and the assembled row record. + /* Allocate a block registers to hold the rowid and the values + ** for all columns of the new row. */ regRowid = regIns = pParse->nMem+1; pParse->nMem += pTab->nCol + 1; @@ -113862,9 +120924,17 @@ SQLITE_PRIVATE void sqlite3Insert( ** the index into IDLIST of the primary key column. ipkColumn is ** the index of the primary key as it appears in IDLIST, not as ** is appears in the original table. (The index of the INTEGER - ** PRIMARY KEY in the original table is pTab->iPKey.) + ** PRIMARY KEY in the original table is pTab->iPKey.) After this + ** loop, if ipkColumn==(-1), that means that integer primary key + ** is unspecified, and hence the table is either WITHOUT ROWID or + ** it will automatically generated an integer primary key. + ** + ** bIdListInOrder is true if the columns in IDLIST are in storage + ** order. This enables an optimization that avoids shuffling the + ** columns into storage order. False negatives are harmless, + ** but false positives will cause database corruption. */ - bIdListInOrder = (pTab->tabFlags & TF_OOOHidden)==0; + bIdListInOrder = (pTab->tabFlags & (TF_OOOHidden|TF_HasStored))==0; if( pColumn ){ for(i=0; inId; i++){ pColumn->a[i].idx = -1; @@ -113877,6 +120947,14 @@ SQLITE_PRIVATE void sqlite3Insert( if( j==pTab->iPKey ){ ipkColumn = i; assert( !withoutRowid ); } +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + if( pTab->aCol[j].colFlags & (COLFLAG_STORED|COLFLAG_VIRTUAL) ){ + sqlite3ErrorMsg(pParse, + "cannot INSERT into generated column \"%s\"", + pTab->aCol[j].zName); + goto insert_cleanup; + } +#endif break; } } @@ -113986,13 +121064,26 @@ SQLITE_PRIVATE void sqlite3Insert( */ if( pColumn==0 && nColumn>0 ){ ipkColumn = pTab->iPKey; +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + if( ipkColumn>=0 && (pTab->tabFlags & TF_HasGenerated)!=0 ){ + testcase( pTab->tabFlags & TF_HasVirtual ); + testcase( pTab->tabFlags & TF_HasStored ); + for(i=ipkColumn-1; i>=0; i--){ + if( pTab->aCol[i].colFlags & COLFLAG_GENERATED ){ + testcase( pTab->aCol[i].colFlags & COLFLAG_VIRTUAL ); + testcase( pTab->aCol[i].colFlags & COLFLAG_STORED ); + ipkColumn--; + } + } + } +#endif } /* Make sure the number of columns in the source data matches the number ** of columns to be inserted into the table. */ for(i=0; inCol; i++){ - nHidden += (IsHiddenColumn(&pTab->aCol[i]) ? 1 : 0); + if( pTab->aCol[i].colFlags & COLFLAG_NOINSERT ) nHidden++; } if( pColumn==0 && nColumn && nColumn!=(pTab->nCol-nHidden) ){ sqlite3ErrorMsg(pParse, @@ -114020,7 +121111,7 @@ SQLITE_PRIVATE void sqlite3Insert( int nIdx; nIdx = sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenWrite, 0, -1, 0, &iDataCur, &iIdxCur); - aRegIdx = sqlite3DbMallocRawNN(db, sizeof(int)*(nIdx+1)); + aRegIdx = sqlite3DbMallocRawNN(db, sizeof(int)*(nIdx+2)); if( aRegIdx==0 ){ goto insert_cleanup; } @@ -114029,9 +121120,22 @@ SQLITE_PRIVATE void sqlite3Insert( aRegIdx[i] = ++pParse->nMem; pParse->nMem += pIdx->nColumn; } + aRegIdx[i] = ++pParse->nMem; /* Register to store the table record */ } #ifndef SQLITE_OMIT_UPSERT if( pUpsert ){ + if( IsVirtual(pTab) ){ + sqlite3ErrorMsg(pParse, "UPSERT not implemented for virtual table \"%s\"", + pTab->zName); + goto insert_cleanup; + } + if( pTab->pSelect ){ + sqlite3ErrorMsg(pParse, "cannot UPSERT a view"); + goto insert_cleanup; + } + if( sqlite3HasExplicitNulls(pParse, pUpsert->pUpsertTarget) ){ + goto insert_cleanup; + } pTabList->a[0].iCursor = iDataCur; pUpsert->pUpsertSrc = pTabList; pUpsert->regData = regData; @@ -114066,13 +121170,94 @@ SQLITE_PRIVATE void sqlite3Insert( ** goto C ** D: ... */ + sqlite3VdbeReleaseRegisters(pParse, regData, pTab->nCol, 0, 0); addrInsTop = addrCont = sqlite3VdbeAddOp1(v, OP_Yield, dest.iSDParm); VdbeCoverage(v); + if( ipkColumn>=0 ){ + /* tag-20191021-001: If the INTEGER PRIMARY KEY is being generated by the + ** SELECT, go ahead and copy the value into the rowid slot now, so that + ** the value does not get overwritten by a NULL at tag-20191021-002. */ + sqlite3VdbeAddOp2(v, OP_Copy, regFromSelect+ipkColumn, regRowid); + } } + /* Compute data for ordinary columns of the new entry. Values + ** are written in storage order into registers starting with regData. + ** Only ordinary columns are computed in this loop. The rowid + ** (if there is one) is computed later and generated columns are + ** computed after the rowid since they might depend on the value + ** of the rowid. + */ + nHidden = 0; + iRegStore = regData; assert( regData==regRowid+1 ); + for(i=0; inCol; i++, iRegStore++){ + int k; + u32 colFlags; + assert( i>=nHidden ); + if( i==pTab->iPKey ){ + /* tag-20191021-002: References to the INTEGER PRIMARY KEY are filled + ** using the rowid. So put a NULL in the IPK slot of the record to avoid + ** using excess space. The file format definition requires this extra + ** NULL - we cannot optimize further by skipping the column completely */ + sqlite3VdbeAddOp1(v, OP_SoftNull, iRegStore); + continue; + } + if( ((colFlags = pTab->aCol[i].colFlags) & COLFLAG_NOINSERT)!=0 ){ + nHidden++; + if( (colFlags & COLFLAG_VIRTUAL)!=0 ){ + /* Virtual columns do not participate in OP_MakeRecord. So back up + ** iRegStore by one slot to compensate for the iRegStore++ in the + ** outer for() loop */ + iRegStore--; + continue; + }else if( (colFlags & COLFLAG_STORED)!=0 ){ + /* Stored columns are computed later. But if there are BEFORE + ** triggers, the slots used for stored columns will be OP_Copy-ed + ** to a second block of registers, so the register needs to be + ** initialized to NULL to avoid an uninitialized register read */ + if( tmask & TRIGGER_BEFORE ){ + sqlite3VdbeAddOp1(v, OP_SoftNull, iRegStore); + } + continue; + }else if( pColumn==0 ){ + /* Hidden columns that are not explicitly named in the INSERT + ** get there default value */ + sqlite3ExprCodeFactorable(pParse, pTab->aCol[i].pDflt, iRegStore); + continue; + } + } + if( pColumn ){ + for(j=0; jnId && pColumn->a[j].idx!=i; j++){} + if( j>=pColumn->nId ){ + /* A column not named in the insert column list gets its + ** default value */ + sqlite3ExprCodeFactorable(pParse, pTab->aCol[i].pDflt, iRegStore); + continue; + } + k = j; + }else if( nColumn==0 ){ + /* This is INSERT INTO ... DEFAULT VALUES. Load the default value. */ + sqlite3ExprCodeFactorable(pParse, pTab->aCol[i].pDflt, iRegStore); + continue; + }else{ + k = i - nHidden; + } + + if( useTempTable ){ + sqlite3VdbeAddOp3(v, OP_Column, srcTab, k, iRegStore); + }else if( pSelect ){ + if( regFromSelect!=regData ){ + sqlite3VdbeAddOp2(v, OP_SCopy, regFromSelect+k, iRegStore); + } + }else{ + sqlite3ExprCode(pParse, pList->a[k].pExpr, iRegStore); + } + } + + /* Run the BEFORE and INSTEAD OF triggers, if there are any */ - endOfLoop = sqlite3VdbeMakeLabel(v); + endOfLoop = sqlite3VdbeMakeLabel(pParse); if( tmask & TRIGGER_BEFORE ){ int regCols = sqlite3GetTempRange(pParse, pTab->nCol+1); @@ -114104,25 +121289,21 @@ SQLITE_PRIVATE void sqlite3Insert( */ assert( !IsVirtual(pTab) ); - /* Create the new column data - */ - for(i=j=0; inCol; i++){ - if( pColumn ){ - for(j=0; jnId; j++){ - if( pColumn->a[j].idx==i ) break; - } - } - if( (!useTempTable && !pList) || (pColumn && j>=pColumn->nId) - || (pColumn==0 && IsOrdinaryHiddenColumn(&pTab->aCol[i])) ){ - sqlite3ExprCode(pParse, pTab->aCol[i].pDflt, regCols+i+1); - }else if( useTempTable ){ - sqlite3VdbeAddOp3(v, OP_Column, srcTab, j, regCols+i+1); - }else{ - assert( pSelect==0 ); /* Otherwise useTempTable is true */ - sqlite3ExprCodeAndCache(pParse, pList->a[j].pExpr, regCols+i+1); - } - if( pColumn==0 && !IsOrdinaryHiddenColumn(&pTab->aCol[i]) ) j++; + /* Copy the new data already generated. */ + assert( pTab->nNVCol>0 ); + sqlite3VdbeAddOp3(v, OP_Copy, regRowid+1, regCols+1, pTab->nNVCol-1); + +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + /* Compute the new value for generated columns after all other + ** columns have already been computed. This must be done after + ** computing the ROWID in case one of the generated columns + ** refers to the ROWID. */ + if( pTab->tabFlags & TF_HasGenerated ){ + testcase( pTab->tabFlags & TF_HasVirtual ); + testcase( pTab->tabFlags & TF_HasStored ); + sqlite3ComputeGeneratedColumns(pParse, regCols+1, pTab); } +#endif /* If this is an INSERT on a view with an INSTEAD OF INSERT trigger, ** do not attempt any conversions before assembling the record. @@ -114140,30 +121321,24 @@ SQLITE_PRIVATE void sqlite3Insert( sqlite3ReleaseTempRange(pParse, regCols, pTab->nCol+1); } - /* Compute the content of the next row to insert into a range of - ** registers beginning at regIns. - */ if( !isView ){ if( IsVirtual(pTab) ){ /* The row that the VUpdate opcode will delete: none */ sqlite3VdbeAddOp2(v, OP_Null, 0, regIns); } if( ipkColumn>=0 ){ + /* Compute the new rowid */ if( useTempTable ){ sqlite3VdbeAddOp3(v, OP_Column, srcTab, ipkColumn, regRowid); }else if( pSelect ){ - sqlite3VdbeAddOp2(v, OP_Copy, regFromSelect+ipkColumn, regRowid); + /* Rowid already initialized at tag-20191021-001 */ }else{ - VdbeOp *pOp; - sqlite3ExprCode(pParse, pList->a[ipkColumn].pExpr, regRowid); - pOp = sqlite3VdbeGetOp(v, -1); - assert( pOp!=0 ); - if( pOp->opcode==OP_Null && !IsVirtual(pTab) ){ + Expr *pIpk = pList->a[ipkColumn].pExpr; + if( pIpk->op==TK_NULL && !IsVirtual(pTab) ){ + sqlite3VdbeAddOp3(v, OP_NewRowid, iDataCur, regRowid, regAutoinc); appendFlag = 1; - pOp->opcode = OP_NewRowid; - pOp->p1 = iDataCur; - pOp->p2 = regRowid; - pOp->p3 = regAutoinc; + }else{ + sqlite3ExprCode(pParse, pList->a[ipkColumn].pExpr, regRowid); } } /* If the PRIMARY KEY expression is NULL, then use OP_NewRowid @@ -114189,45 +121364,15 @@ SQLITE_PRIVATE void sqlite3Insert( } autoIncStep(pParse, regAutoinc, regRowid); - /* Compute data for all columns of the new entry, beginning - ** with the first column. - */ - nHidden = 0; - for(i=0; inCol; i++){ - int iRegStore = regRowid+1+i; - if( i==pTab->iPKey ){ - /* The value of the INTEGER PRIMARY KEY column is always a NULL. - ** Whenever this column is read, the rowid will be substituted - ** in its place. Hence, fill this column with a NULL to avoid - ** taking up data space with information that will never be used. - ** As there may be shallow copies of this value, make it a soft-NULL */ - sqlite3VdbeAddOp1(v, OP_SoftNull, iRegStore); - continue; - } - if( pColumn==0 ){ - if( IsHiddenColumn(&pTab->aCol[i]) ){ - j = -1; - nHidden++; - }else{ - j = i - nHidden; - } - }else{ - for(j=0; jnId; j++){ - if( pColumn->a[j].idx==i ) break; - } - } - if( j<0 || nColumn==0 || (pColumn && j>=pColumn->nId) ){ - sqlite3ExprCodeFactorable(pParse, pTab->aCol[i].pDflt, iRegStore); - }else if( useTempTable ){ - sqlite3VdbeAddOp3(v, OP_Column, srcTab, j, iRegStore); - }else if( pSelect ){ - if( regFromSelect!=regData ){ - sqlite3VdbeAddOp2(v, OP_SCopy, regFromSelect+j, iRegStore); - } - }else{ - sqlite3ExprCode(pParse, pList->a[j].pExpr, iRegStore); - } +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + /* Compute the new value for generated columns after all other + ** columns have already been computed. This must be done after + ** computing the ROWID in case one of the generated columns + ** is derived from the INTEGER PRIMARY KEY. */ + if( pTab->tabFlags & TF_HasGenerated ){ + sqlite3ComputeGeneratedColumns(pParse, regRowid+1, pTab); } +#endif /* Generate code to check constraints and generate index keys and ** do the insertion. @@ -114257,9 +121402,7 @@ SQLITE_PRIVATE void sqlite3Insert( ** cursor that is disturbed. And these instructions both clear the ** VdbeCursor.seekResult variable, disabling the OPFLAG_USESEEKRESULT ** functionality. */ - bUseSeek = (isReplace==0 || (pTrigger==0 && - ((db->flags & SQLITE_ForeignKeys)==0 || sqlite3FkReferences(pTab)==0) - )); + bUseSeek = (isReplace==0 || !sqlite3VdbeHasSubProgram(v)); sqlite3CompleteInsertion(pParse, pTab, iDataCur, iIdxCur, regIns, aRegIdx, 0, appendFlag, bUseSeek ); @@ -114288,6 +121431,15 @@ SQLITE_PRIVATE void sqlite3Insert( sqlite3VdbeAddOp1(v, OP_Close, srcTab); }else if( pSelect ){ sqlite3VdbeGoto(v, addrCont); +#ifdef SQLITE_DEBUG + /* If we are jumping back to an OP_Yield that is preceded by an + ** OP_ReleaseReg, set the p5 flag on the OP_Goto so that the + ** OP_ReleaseReg will be included in the loop. */ + if( sqlite3VdbeGetOp(v, addrCont-1)->opcode==OP_ReleaseReg ){ + assert( sqlite3VdbeGetOp(v, addrCont)->opcode==OP_Yield ); + sqlite3VdbeChangeP5(v, 1); + } +#endif sqlite3VdbeJumpHere(v, addrInsTop); } @@ -114334,14 +121486,15 @@ SQLITE_PRIVATE void sqlite3Insert( #endif /* -** Meanings of bits in of pWalker->eCode for checkConstraintUnchanged() +** Meanings of bits in of pWalker->eCode for +** sqlite3ExprReferencesUpdatedColumn() */ #define CKCNSTRNT_COLUMN 0x01 /* CHECK constraint uses a changing column */ #define CKCNSTRNT_ROWID 0x02 /* CHECK constraint references the ROWID */ -/* This is the Walker callback from checkConstraintUnchanged(). Set -** bit 0x01 of pWalker->eCode if -** pWalker->eCode to 0 if this expression node references any of the +/* This is the Walker callback from sqlite3ExprReferencesUpdatedColumn(). +* Set bit 0x01 of pWalker->eCode if pWalker->eCode to 0 and if this +** expression node references any of the ** columns that are being modifed by an UPDATE statement. */ static int checkConstraintExprNode(Walker *pWalker, Expr *pExpr){ @@ -114363,12 +121516,21 @@ static int checkConstraintExprNode(Walker *pWalker, Expr *pExpr){ ** only columns that are modified by the UPDATE are those for which ** aiChng[i]>=0, and also the ROWID is modified if chngRowid is true. ** -** Return true if CHECK constraint pExpr does not use any of the +** Return true if CHECK constraint pExpr uses any of the ** changing columns (or the rowid if it is changing). In other words, -** return true if this CHECK constraint can be skipped when validating +** return true if this CHECK constraint must be validated for ** the new row in the UPDATE statement. +** +** 2018-09-15: pExpr might also be an expression for an index-on-expressions. +** The operation of this routine is the same - return true if an only if +** the expression uses one or more of columns identified by the second and +** third arguments. */ -static int checkConstraintUnchanged(Expr *pExpr, int *aiChng, int chngRowid){ +SQLITE_PRIVATE int sqlite3ExprReferencesUpdatedColumn( + Expr *pExpr, /* The expression to be checked */ + int *aiChng, /* aiChng[x]>=0 if column x changed by the UPDATE */ + int chngRowid /* True if UPDATE changes the rowid */ +){ Walker w; memset(&w, 0, sizeof(w)); w.eCode = 0; @@ -114383,45 +121545,7 @@ static int checkConstraintUnchanged(Expr *pExpr, int *aiChng, int chngRowid){ testcase( w.eCode==CKCNSTRNT_COLUMN ); testcase( w.eCode==CKCNSTRNT_ROWID ); testcase( w.eCode==(CKCNSTRNT_ROWID|CKCNSTRNT_COLUMN) ); - return !w.eCode; -} - -/* -** An instance of the ConstraintAddr object remembers the byte-code addresses -** for sections of the constraint checks that deal with uniqueness constraints -** on the rowid and on the upsert constraint. -** -** This information is passed into checkReorderConstraintChecks() to insert -** some OP_Goto operations so that the rowid and upsert constraints occur -** in the correct order relative to other constraints. -*/ -typedef struct ConstraintAddr ConstraintAddr; -struct ConstraintAddr { - int ipkTop; /* Subroutine for rowid constraint check */ - int upsertTop; /* Label for upsert constraint check subroutine */ - int upsertTop2; /* Copy of upsertTop not cleared by the call */ - int upsertBtm; /* upsert constraint returns to this label */ - int ipkBtm; /* Return opcode rowid constraint check */ -}; - -/* -** Generate any OP_Goto operations needed to cause constraints to be -** run that haven't already been run. -*/ -static void reorderConstraintChecks(Vdbe *v, ConstraintAddr *p){ - if( p->upsertTop ){ - testcase( sqlite3VdbeLabelHasBeenResolved(v, p->upsertTop) ); - sqlite3VdbeGoto(v, p->upsertTop); - VdbeComment((v, "call upsert subroutine")); - sqlite3VdbeResolveLabel(v, p->upsertBtm); - p->upsertTop = 0; - } - if( p->ipkTop ){ - sqlite3VdbeGoto(v, p->ipkTop); - VdbeComment((v, "call rowid unique-check subroutine")); - sqlite3VdbeJumpHere(v, p->ipkBtm); - p->ipkTop = 0; - } + return w.eCode!=0; } /* @@ -114459,6 +121583,14 @@ static void reorderConstraintChecks(Vdbe *v, ConstraintAddr *p){ ** the same as the order of indices on the linked list of indices ** at pTab->pIndex. ** +** (2019-05-07) The generated code also creates a new record for the +** main table, if pTab is a rowid table, and stores that record in the +** register identified by aRegIdx[nIdx] - in other words in the first +** entry of aRegIdx[] past the last index. It is important that the +** record be generated during constraint checks to avoid affinity changes +** to the register content that occur after constraint checks but before +** the new record is inserted. +** ** The caller must have already opened writeable cursors on the main ** table and all applicable indices (that is to say, all indices for which ** aRegIdx[] is not zero). iDataCur is the cursor for the main table when @@ -114530,14 +121662,22 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( int ix; /* Index loop counter */ int nCol; /* Number of columns */ int onError; /* Conflict resolution strategy */ - int addr1; /* Address of jump instruction */ int seenReplace = 0; /* True if REPLACE is used to resolve INT PK conflict */ int nPkField; /* Number of fields in PRIMARY KEY. 1 for ROWID tables */ - ConstraintAddr sAddr;/* Address information for constraint reordering */ Index *pUpIdx = 0; /* Index to which to apply the upsert */ u8 isUpdate; /* True if this is an UPDATE operation */ u8 bAffinityDone = 0; /* True if the OP_Affinity operation has been run */ int upsertBypass = 0; /* Address of Goto to bypass upsert subroutine */ + int upsertJump = 0; /* Address of Goto that jumps into upsert subroutine */ + int ipkTop = 0; /* Top of the IPK uniqueness check */ + int ipkBottom = 0; /* OP_Goto at the end of the IPK uniqueness check */ + /* Variables associated with retesting uniqueness constraints after + ** replace triggers fire have run */ + int regTrigCnt; /* Register used to count replace trigger invocations */ + int addrRecheck = 0; /* Jump here to recheck all uniqueness constraints */ + int lblRecheckOk = 0; /* Each recheck jumps to this label if it passes */ + Trigger *pTrigger; /* List of DELETE triggers on the table pTab */ + int nReplaceTrig = 0; /* Number of replace triggers coded */ isUpdate = regOldData!=0; db = pParse->db; @@ -114545,7 +121685,6 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( assert( v!=0 ); assert( pTab->pSelect==0 ); /* This table is not a VIEW */ nCol = pTab->nCol; - memset(&sAddr, 0, sizeof(sAddr)); /* pPk is the PRIMARY KEY index for WITHOUT ROWID tables and NULL for ** normal rowid tables. nPkField is the number of key fields in the @@ -114565,56 +121704,103 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( /* Test all NOT NULL constraints. */ - for(i=0; iiPKey ){ - continue; /* ROWID is never NULL */ - } - if( aiChng && aiChng[i]<0 ){ - /* Don't bother checking for NOT NULL on columns that do not change */ - continue; - } - onError = pTab->aCol[i].notNull; - if( onError==OE_None ) continue; /* This column is allowed to be NULL */ - if( overrideError!=OE_Default ){ - onError = overrideError; - }else if( onError==OE_Default ){ - onError = OE_Abort; - } - if( onError==OE_Replace && pTab->aCol[i].pDflt==0 ){ - onError = OE_Abort; - } - assert( onError==OE_Rollback || onError==OE_Abort || onError==OE_Fail - || onError==OE_Ignore || onError==OE_Replace ); - switch( onError ){ - case OE_Abort: - sqlite3MayAbort(pParse); - /* Fall through */ - case OE_Rollback: - case OE_Fail: { - char *zMsg = sqlite3MPrintf(db, "%s.%s", pTab->zName, - pTab->aCol[i].zName); - sqlite3VdbeAddOp3(v, OP_HaltIfNull, SQLITE_CONSTRAINT_NOTNULL, onError, - regNewData+1+i); - sqlite3VdbeAppendP4(v, zMsg, P4_DYNAMIC); - sqlite3VdbeChangeP5(v, P5_ConstraintNotNull); - VdbeCoverage(v); - break; - } - case OE_Ignore: { - sqlite3VdbeAddOp2(v, OP_IsNull, regNewData+1+i, ignoreDest); - VdbeCoverage(v); + if( pTab->tabFlags & TF_HasNotNull ){ + int b2ndPass = 0; /* True if currently running 2nd pass */ + int nSeenReplace = 0; /* Number of ON CONFLICT REPLACE operations */ + int nGenerated = 0; /* Number of generated columns with NOT NULL */ + while(1){ /* Make 2 passes over columns. Exit loop via "break" */ + for(i=0; iaCol[i]; /* The column to check for NOT NULL */ + int isGenerated; /* non-zero if column is generated */ + onError = pCol->notNull; + if( onError==OE_None ) continue; /* No NOT NULL on this column */ + if( i==pTab->iPKey ){ + continue; /* ROWID is never NULL */ + } + isGenerated = pCol->colFlags & COLFLAG_GENERATED; + if( isGenerated && !b2ndPass ){ + nGenerated++; + continue; /* Generated columns processed on 2nd pass */ + } + if( aiChng && aiChng[i]<0 && !isGenerated ){ + /* Do not check NOT NULL on columns that do not change */ + continue; + } + if( overrideError!=OE_Default ){ + onError = overrideError; + }else if( onError==OE_Default ){ + onError = OE_Abort; + } + if( onError==OE_Replace ){ + if( b2ndPass /* REPLACE becomes ABORT on the 2nd pass */ + || pCol->pDflt==0 /* REPLACE is ABORT if no DEFAULT value */ + ){ + testcase( pCol->colFlags & COLFLAG_VIRTUAL ); + testcase( pCol->colFlags & COLFLAG_STORED ); + testcase( pCol->colFlags & COLFLAG_GENERATED ); + onError = OE_Abort; + }else{ + assert( !isGenerated ); + } + }else if( b2ndPass && !isGenerated ){ + continue; + } + assert( onError==OE_Rollback || onError==OE_Abort || onError==OE_Fail + || onError==OE_Ignore || onError==OE_Replace ); + testcase( i!=sqlite3TableColumnToStorage(pTab, i) ); + iReg = sqlite3TableColumnToStorage(pTab, i) + regNewData + 1; + switch( onError ){ + case OE_Replace: { + int addr1 = sqlite3VdbeAddOp1(v, OP_NotNull, iReg); + VdbeCoverage(v); + assert( (pCol->colFlags & COLFLAG_GENERATED)==0 ); + nSeenReplace++; + sqlite3ExprCodeCopy(pParse, pCol->pDflt, iReg); + sqlite3VdbeJumpHere(v, addr1); + break; + } + case OE_Abort: + sqlite3MayAbort(pParse); + /* Fall through */ + case OE_Rollback: + case OE_Fail: { + char *zMsg = sqlite3MPrintf(db, "%s.%s", pTab->zName, + pCol->zName); + sqlite3VdbeAddOp3(v, OP_HaltIfNull, SQLITE_CONSTRAINT_NOTNULL, + onError, iReg); + sqlite3VdbeAppendP4(v, zMsg, P4_DYNAMIC); + sqlite3VdbeChangeP5(v, P5_ConstraintNotNull); + VdbeCoverage(v); + break; + } + default: { + assert( onError==OE_Ignore ); + sqlite3VdbeAddOp2(v, OP_IsNull, iReg, ignoreDest); + VdbeCoverage(v); + break; + } + } /* end switch(onError) */ + } /* end loop i over columns */ + if( nGenerated==0 && nSeenReplace==0 ){ + /* If there are no generated columns with NOT NULL constraints + ** and no NOT NULL ON CONFLICT REPLACE constraints, then a single + ** pass is sufficient */ break; } - default: { - assert( onError==OE_Replace ); - addr1 = sqlite3VdbeAddOp1(v, OP_NotNull, regNewData+1+i); - VdbeCoverage(v); - sqlite3ExprCode(pParse, pTab->aCol[i].pDflt, regNewData+1+i); - sqlite3VdbeJumpHere(v, addr1); - break; + if( b2ndPass ) break; /* Never need more than 2 passes */ + b2ndPass = 1; +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + if( nSeenReplace>0 && (pTab->tabFlags & TF_HasGenerated)!=0 ){ + /* If any NOT NULL ON CONFLICT REPLACE constraints fired on the + ** first pass, recomputed values for all generated columns, as + ** those values might depend on columns affected by the REPLACE. + */ + sqlite3ComputeGeneratedColumns(pParse, regNewData+1, pTab); } - } - } +#endif + } /* end of 2-pass loop */ + } /* end if( has-not-null-constraints ) */ /* Test all CHECK constraints */ @@ -114625,17 +121811,32 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( onError = overrideError!=OE_Default ? overrideError : OE_Abort; for(i=0; inExpr; i++){ int allOk; + Expr *pCopy; Expr *pExpr = pCheck->a[i].pExpr; - if( aiChng && checkConstraintUnchanged(pExpr, aiChng, pkChng) ) continue; - allOk = sqlite3VdbeMakeLabel(v); + if( aiChng + && !sqlite3ExprReferencesUpdatedColumn(pExpr, aiChng, pkChng) + ){ + /* The check constraints do not reference any of the columns being + ** updated so there is no point it verifying the check constraint */ + continue; + } + if( bAffinityDone==0 ){ + sqlite3TableAffinity(v, pTab, regNewData+1); + bAffinityDone = 1; + } + allOk = sqlite3VdbeMakeLabel(pParse); sqlite3VdbeVerifyAbortable(v, onError); - sqlite3ExprIfTrue(pParse, pExpr, allOk, SQLITE_JUMPIFNULL); + pCopy = sqlite3ExprDup(db, pExpr, 0); + if( !db->mallocFailed ){ + sqlite3ExprIfTrue(pParse, pCopy, allOk, SQLITE_JUMPIFNULL); + } + sqlite3ExprDelete(db, pCopy); if( onError==OE_Ignore ){ sqlite3VdbeGoto(v, ignoreDest); }else{ - char *zName = pCheck->a[i].zName; + char *zName = pCheck->a[i].zEName; if( zName==0 ) zName = pTab->zName; - if( onError==OE_Replace ) onError = OE_Abort; /* IMP: R-15569-63625 */ + if( onError==OE_Replace ) onError = OE_Abort; /* IMP: R-26383-51744 */ sqlite3HaltConstraint(pParse, SQLITE_CONSTRAINT_CHECK, onError, zName, P4_TRANSIENT, P5_ConstraintCheck); @@ -114649,8 +121850,8 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( /* UNIQUE and PRIMARY KEY constraints should be handled in the following ** order: ** - ** (1) OE_Abort, OE_Fail, OE_Rollback, OE_Ignore - ** (2) OE_Update + ** (1) OE_Update + ** (2) OE_Abort, OE_Fail, OE_Rollback, OE_Ignore ** (3) OE_Replace ** ** OE_Fail and OE_Ignore must happen before any changes are made. @@ -114659,6 +121860,11 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( ** could happen in any order, but they are grouped up front for ** convenience. ** + ** 2018-08-14: Ticket https://www.sqlite.org/src/info/908f001483982c43 + ** The order of constraints used to have OE_Update as (2) and OE_Abort + ** and so forth as (1). But apparently PostgreSQL checks the OE_Update + ** constraint before any others, so it had to be moved. + ** ** Constraint checking code is generated in this order: ** (A) The rowid constraint ** (B) Unique index constraints that do not have OE_Replace as their @@ -114678,11 +121884,54 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( overrideError = OE_Ignore; pUpsert = 0; }else if( (pUpIdx = pUpsert->pUpsertIdx)!=0 ){ - /* If the constraint-target is on some column other than - ** then ROWID, then we might need to move the UPSERT around - ** so that it occurs in the correct order. */ - sAddr.upsertTop = sAddr.upsertTop2 = sqlite3VdbeMakeLabel(v); - sAddr.upsertBtm = sqlite3VdbeMakeLabel(v); + /* If the constraint-target uniqueness check must be run first. + ** Jump to that uniqueness check now */ + upsertJump = sqlite3VdbeAddOp0(v, OP_Goto); + VdbeComment((v, "UPSERT constraint goes first")); + } + } + + /* Determine if it is possible that triggers (either explicitly coded + ** triggers or FK resolution actions) might run as a result of deletes + ** that happen when OE_Replace conflict resolution occurs. (Call these + ** "replace triggers".) If any replace triggers run, we will need to + ** recheck all of the uniqueness constraints after they have all run. + ** But on the recheck, the resolution is OE_Abort instead of OE_Replace. + ** + ** If replace triggers are a possibility, then + ** + ** (1) Allocate register regTrigCnt and initialize it to zero. + ** That register will count the number of replace triggers that + ** fire. Constraint recheck only occurs if the number is positive. + ** (2) Initialize pTrigger to the list of all DELETE triggers on pTab. + ** (3) Initialize addrRecheck and lblRecheckOk + ** + ** The uniqueness rechecking code will create a series of tests to run + ** in a second pass. The addrRecheck and lblRecheckOk variables are + ** used to link together these tests which are separated from each other + ** in the generate bytecode. + */ + if( (db->flags & (SQLITE_RecTriggers|SQLITE_ForeignKeys))==0 ){ + /* There are not DELETE triggers nor FK constraints. No constraint + ** rechecks are needed. */ + pTrigger = 0; + regTrigCnt = 0; + }else{ + if( db->flags&SQLITE_RecTriggers ){ + pTrigger = sqlite3TriggersExist(pParse, pTab, TK_DELETE, 0, 0); + regTrigCnt = pTrigger!=0 || sqlite3FkRequired(pParse, pTab, 0, 0); + }else{ + pTrigger = 0; + regTrigCnt = sqlite3FkRequired(pParse, pTab, 0, 0); + } + if( regTrigCnt ){ + /* Replace triggers might exist. Allocate the counter and + ** initialize it to zero. */ + regTrigCnt = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Integer, 0, regTrigCnt); + VdbeComment((v, "trigger count")); + lblRecheckOk = sqlite3VdbeMakeLabel(pParse); + addrRecheck = lblRecheckOk; } } @@ -114690,7 +121939,7 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( ** exist in the table. */ if( pkChng && pPk==0 ){ - int addrRowidOk = sqlite3VdbeMakeLabel(v); + int addrRowidOk = sqlite3VdbeMakeLabel(pParse); /* Figure out what action to take in case of a rowid collision */ onError = pTab->keyConf; @@ -114714,16 +121963,12 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( ** to defer the running of the rowid conflict checking until after ** the UNIQUE constraints have run. */ - assert( OE_Update>OE_Replace ); - assert( OE_Ignore=OE_Replace - && (pUpsert || onError!=overrideError) - && pTab->pIndex + if( onError==OE_Replace /* IPK rule is REPLACE */ + && onError!=overrideError /* Rules for other contraints are different */ + && pTab->pIndex /* There exist other constraints */ ){ - sAddr.ipkTop = sqlite3VdbeAddOp0(v, OP_Goto)+1; + ipkTop = sqlite3VdbeAddOp0(v, OP_Goto)+1; + VdbeComment((v, "defer IPK REPLACE until last")); } if( isUpdate ){ @@ -114779,14 +122024,12 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( ** to run without a statement journal if there are no indexes on the ** table. */ - Trigger *pTrigger = 0; - if( db->flags&SQLITE_RecTriggers ){ - pTrigger = sqlite3TriggersExist(pParse, pTab, TK_DELETE, 0, 0); - } - if( pTrigger || sqlite3FkRequired(pParse, pTab, 0, 0) ){ + if( regTrigCnt ){ sqlite3MultiWrite(pParse); sqlite3GenerateRowDelete(pParse, pTab, pTrigger, iDataCur, iIdxCur, regNewData, 1, 0, OE_Replace, 1, -1); + sqlite3VdbeAddOp2(v, OP_AddImm, regTrigCnt, 1); /* incr trigger cnt */ + nReplaceTrig++; }else{ #ifdef SQLITE_ENABLE_PREUPDATE_HOOK assert( HasRowid(pTab) ); @@ -114818,9 +122061,9 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( } } sqlite3VdbeResolveLabel(v, addrRowidOk); - if( sAddr.ipkTop ){ - sAddr.ipkBtm = sqlite3VdbeAddOp0(v, OP_Goto); - sqlite3VdbeJumpHere(v, sAddr.ipkTop-1); + if( ipkTop ){ + ipkBottom = sqlite3VdbeAddOp0(v, OP_Goto); + sqlite3VdbeJumpHere(v, ipkTop-1); } } @@ -114836,21 +122079,22 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( int regR; /* Range of registers holding conflicting PK */ int iThisCur; /* Cursor for this UNIQUE index */ int addrUniqueOk; /* Jump here if the UNIQUE constraint is satisfied */ + int addrConflictCk; /* First opcode in the conflict check logic */ if( aRegIdx[ix]==0 ) continue; /* Skip indices that do not change */ if( pUpIdx==pIdx ){ - addrUniqueOk = sAddr.upsertBtm; + addrUniqueOk = upsertJump+1; upsertBypass = sqlite3VdbeGoto(v, 0); VdbeComment((v, "Skip upsert subroutine")); - sqlite3VdbeResolveLabel(v, sAddr.upsertTop2); + sqlite3VdbeJumpHere(v, upsertJump); }else{ - addrUniqueOk = sqlite3VdbeMakeLabel(v); + addrUniqueOk = sqlite3VdbeMakeLabel(pParse); } - VdbeNoopComment((v, "uniqueness check for %s", pIdx->zName)); - if( bAffinityDone==0 ){ + if( bAffinityDone==0 && (pUpIdx==0 || pUpIdx==pIdx) ){ sqlite3TableAffinity(v, pTab, regNewData+1); bAffinityDone = 1; } + VdbeNoopComment((v, "prep index %s", pIdx->zName)); iThisCur = iIdxCur+ix; @@ -114875,21 +122119,25 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( sqlite3ExprCodeCopy(pParse, pIdx->aColExpr->a[i].pExpr, regIdx+i); pParse->iSelfTab = 0; VdbeComment((v, "%s column %d", pIdx->zName, i)); + }else if( iField==XN_ROWID || iField==pTab->iPKey ){ + x = regNewData; + sqlite3VdbeAddOp2(v, OP_IntCopy, x, regIdx+i); + VdbeComment((v, "rowid")); }else{ - if( iField==XN_ROWID || iField==pTab->iPKey ){ - x = regNewData; - }else{ - x = iField + regNewData + 1; - } - sqlite3VdbeAddOp2(v, iField<0 ? OP_IntCopy : OP_SCopy, x, regIdx+i); - VdbeComment((v, "%s", iField<0 ? "rowid" : pTab->aCol[iField].zName)); + testcase( sqlite3TableColumnToStorage(pTab, iField)!=iField ); + x = sqlite3TableColumnToStorage(pTab, iField) + regNewData + 1; + sqlite3VdbeAddOp2(v, OP_SCopy, x, regIdx+i); + VdbeComment((v, "%s", pTab->aCol[iField].zName)); } } sqlite3VdbeAddOp3(v, OP_MakeRecord, regIdx, pIdx->nColumn, aRegIdx[ix]); VdbeComment((v, "for %s", pIdx->zName)); #ifdef SQLITE_ENABLE_NULL_TRIM - if( pIdx->idxType==2 ) sqlite3SetMakeRecordP5(v, pIdx->pTable); + if( pIdx->idxType==SQLITE_IDXTYPE_PRIMARYKEY ){ + sqlite3SetMakeRecordP5(v, pIdx->pTable); + } #endif + sqlite3VdbeReleaseRegisters(pParse, regIdx, pIdx->nColumn, 0, 0); /* In an UPDATE operation, if this index is the PRIMARY KEY index ** of a WITHOUT ROWID table and there has been no change the @@ -114921,22 +122169,17 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( } } - /* Invoke subroutines to handle IPK replace and upsert prior to running - ** the first REPLACE constraint check. */ - if( onError==OE_Replace ){ - testcase( sAddr.ipkTop ); - testcase( sAddr.upsertTop - && sqlite3VdbeLabelHasBeenResolved(v,sAddr.upsertTop) ); - reorderConstraintChecks(v, &sAddr); - } - /* Collision detection may be omitted if all of the following are true: ** (1) The conflict resolution algorithm is REPLACE ** (2) The table is a WITHOUT ROWID table ** (3) There are no secondary indexes on the table ** (4) No delete triggers need to be fired if there is a conflict ** (5) No FK constraint counters need to be updated if a conflict occurs. - */ + ** + ** This is not possible for ENABLE_PREUPDATE_HOOK builds, as the row + ** must be explicitly deleted in order to ensure any pre-update hook + ** is invoked. */ +#ifndef SQLITE_ENABLE_PREUPDATE_HOOK if( (ix==0 && pIdx->pNext==0) /* Condition 3 */ && pPk==pIdx /* Condition 2 */ && onError==OE_Replace /* Condition 1 */ @@ -114948,12 +122191,13 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( sqlite3VdbeResolveLabel(v, addrUniqueOk); continue; } +#endif /* ifndef SQLITE_ENABLE_PREUPDATE_HOOK */ /* Check to see if the new index entry will be unique */ - sqlite3ExprCachePush(pParse); sqlite3VdbeVerifyAbortable(v, onError); - sqlite3VdbeAddOp4Int(v, OP_NoConflict, iThisCur, addrUniqueOk, - regIdx, pIdx->nKeyCol); VdbeCoverage(v); + addrConflictCk = + sqlite3VdbeAddOp4Int(v, OP_NoConflict, iThisCur, addrUniqueOk, + regIdx, pIdx->nKeyCol); VdbeCoverage(v); /* Generate code to handle collisions */ regR = (pIdx==pPk) ? regIdx : sqlite3GetTempRange(pParse, nPkField); @@ -114974,7 +122218,7 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( if( pIdx!=pPk ){ for(i=0; inKeyCol; i++){ assert( pPk->aiColumn[i]>=0 ); - x = sqlite3ColumnOfIndex(pIdx, pPk->aiColumn[i]); + x = sqlite3TableColumnToIndex(pIdx, pPk->aiColumn[i]); sqlite3VdbeAddOp3(v, OP_Column, iThisCur, x, regR+i); VdbeComment((v, "%s.%s", pTab->zName, pTab->aCol[pPk->aiColumn[i]].zName)); @@ -115000,6 +122244,7 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( addrJump = addrUniqueOk; op = OP_Eq; } + x = sqlite3TableColumnToStorage(pTab, x); sqlite3VdbeAddOp4(v, op, regOldData+1+x, addrJump, regCmp+i, p4, P4_COLLSEQ ); @@ -115036,35 +122281,123 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( break; } default: { - Trigger *pTrigger = 0; + int nConflictCk; /* Number of opcodes in conflict check logic */ + assert( onError==OE_Replace ); - if( db->flags&SQLITE_RecTriggers ){ - pTrigger = sqlite3TriggersExist(pParse, pTab, TK_DELETE, 0, 0); - } - if( pTrigger || sqlite3FkRequired(pParse, pTab, 0, 0) ){ + nConflictCk = sqlite3VdbeCurrentAddr(v) - addrConflictCk; + assert( nConflictCk>0 ); + testcase( nConflictCk>1 ); + if( regTrigCnt ){ sqlite3MultiWrite(pParse); + nReplaceTrig++; + } + if( pTrigger && isUpdate ){ + sqlite3VdbeAddOp1(v, OP_CursorLock, iDataCur); } sqlite3GenerateRowDelete(pParse, pTab, pTrigger, iDataCur, iIdxCur, regR, nPkField, 0, OE_Replace, (pIdx==pPk ? ONEPASS_SINGLE : ONEPASS_OFF), iThisCur); + if( pTrigger && isUpdate ){ + sqlite3VdbeAddOp1(v, OP_CursorUnlock, iDataCur); + } + if( regTrigCnt ){ + int addrBypass; /* Jump destination to bypass recheck logic */ + + sqlite3VdbeAddOp2(v, OP_AddImm, regTrigCnt, 1); /* incr trigger cnt */ + addrBypass = sqlite3VdbeAddOp0(v, OP_Goto); /* Bypass recheck */ + VdbeComment((v, "bypass recheck")); + + /* Here we insert code that will be invoked after all constraint + ** checks have run, if and only if one or more replace triggers + ** fired. */ + sqlite3VdbeResolveLabel(v, lblRecheckOk); + lblRecheckOk = sqlite3VdbeMakeLabel(pParse); + if( pIdx->pPartIdxWhere ){ + /* Bypass the recheck if this partial index is not defined + ** for the current row */ + sqlite3VdbeAddOp2(v, OP_IsNull, regIdx-1, lblRecheckOk); + VdbeCoverage(v); + } + /* Copy the constraint check code from above, except change + ** the constraint-ok jump destination to be the address of + ** the next retest block */ + while( nConflictCk>0 ){ + VdbeOp x; /* Conflict check opcode to copy */ + /* The sqlite3VdbeAddOp4() call might reallocate the opcode array. + ** Hence, make a complete copy of the opcode, rather than using + ** a pointer to the opcode. */ + x = *sqlite3VdbeGetOp(v, addrConflictCk); + if( x.opcode!=OP_IdxRowid ){ + int p2; /* New P2 value for copied conflict check opcode */ + const char *zP4; + if( sqlite3OpcodeProperty[x.opcode]&OPFLG_JUMP ){ + p2 = lblRecheckOk; + }else{ + p2 = x.p2; + } + zP4 = x.p4type==P4_INT32 ? SQLITE_INT_TO_PTR(x.p4.i) : x.p4.z; + sqlite3VdbeAddOp4(v, x.opcode, x.p1, p2, x.p3, zP4, x.p4type); + sqlite3VdbeChangeP5(v, x.p5); + VdbeCoverageIf(v, p2!=x.p2); + } + nConflictCk--; + addrConflictCk++; + } + /* If the retest fails, issue an abort */ + sqlite3UniqueConstraint(pParse, OE_Abort, pIdx); + + sqlite3VdbeJumpHere(v, addrBypass); /* Terminate the recheck bypass */ + } seenReplace = 1; break; } } if( pUpIdx==pIdx ){ + sqlite3VdbeGoto(v, upsertJump+1); sqlite3VdbeJumpHere(v, upsertBypass); }else{ sqlite3VdbeResolveLabel(v, addrUniqueOk); } - sqlite3ExprCachePop(pParse); if( regR!=regIdx ) sqlite3ReleaseTempRange(pParse, regR, nPkField); + } + /* If the IPK constraint is a REPLACE, run it last */ + if( ipkTop ){ + sqlite3VdbeGoto(v, ipkTop); + VdbeComment((v, "Do IPK REPLACE")); + sqlite3VdbeJumpHere(v, ipkBottom); } - testcase( sAddr.ipkTop!=0 ); - testcase( sAddr.upsertTop - && sqlite3VdbeLabelHasBeenResolved(v,sAddr.upsertTop) ); - reorderConstraintChecks(v, &sAddr); - + + /* Recheck all uniqueness constraints after replace triggers have run */ + testcase( regTrigCnt!=0 && nReplaceTrig==0 ); + assert( regTrigCnt!=0 || nReplaceTrig==0 ); + if( nReplaceTrig ){ + sqlite3VdbeAddOp2(v, OP_IfNot, regTrigCnt, lblRecheckOk);VdbeCoverage(v); + if( !pPk ){ + if( isUpdate ){ + sqlite3VdbeAddOp3(v, OP_Eq, regNewData, addrRecheck, regOldData); + sqlite3VdbeChangeP5(v, SQLITE_NOTNULL); + VdbeCoverage(v); + } + sqlite3VdbeAddOp3(v, OP_NotExists, iDataCur, addrRecheck, regNewData); + VdbeCoverage(v); + sqlite3RowidConstraint(pParse, OE_Abort, pTab); + }else{ + sqlite3VdbeGoto(v, addrRecheck); + } + sqlite3VdbeResolveLabel(v, lblRecheckOk); + } + + /* Generate the table record */ + if( HasRowid(pTab) ){ + int regRec = aRegIdx[ix]; + sqlite3VdbeAddOp3(v, OP_MakeRecord, regNewData+1, pTab->nNVCol, regRec); + sqlite3SetMakeRecordP5(v, pTab); + if( !bAffinityDone ){ + sqlite3TableAffinity(v, pTab, 0); + } + } + *pbMayReplace = seenReplace; VdbeModuleComment((v, "END: GenCnstCks(%d)", seenReplace)); } @@ -115114,10 +122447,7 @@ SQLITE_PRIVATE void sqlite3CompleteInsertion( Vdbe *v; /* Prepared statements under construction */ Index *pIdx; /* An index being inserted or updated */ u8 pik_flags; /* flag values passed to the btree insert */ - int regData; /* Content registers (after the rowid) */ - int regRec; /* Register holding assembled record for the table */ int i; /* Loop counter */ - u8 bAffinityDone = 0; /* True if OP_Affinity has been run already */ assert( update_flags==0 || update_flags==OPFLAG_ISUPDATE @@ -115128,8 +122458,11 @@ SQLITE_PRIVATE void sqlite3CompleteInsertion( assert( v!=0 ); assert( pTab->pSelect==0 ); /* This table is not a VIEW */ for(i=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, i++){ + /* All REPLACE indexes are at the end of the list */ + assert( pIdx->onError!=OE_Replace + || pIdx->pNext==0 + || pIdx->pNext->onError==OE_Replace ); if( aRegIdx[i]==0 ) continue; - bAffinityDone = 1; if( pIdx->pPartIdxWhere ){ sqlite3VdbeAddOp2(v, OP_IsNull, aRegIdx[i], sqlite3VdbeCurrentAddr(v)+2); VdbeCoverage(v); @@ -115141,10 +122474,13 @@ SQLITE_PRIVATE void sqlite3CompleteInsertion( pik_flags |= (update_flags & OPFLAG_SAVEPOSITION); #ifdef SQLITE_ENABLE_PREUPDATE_HOOK if( update_flags==0 ){ - sqlite3VdbeAddOp4(v, OP_InsertInt, - iIdxCur+i, aRegIdx[i], 0, (char*)pTab, P4_TABLE + int r = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp2(v, OP_Integer, 0, r); + sqlite3VdbeAddOp4(v, OP_Insert, + iIdxCur+i, aRegIdx[i], r, (char*)pTab, P4_TABLE ); sqlite3VdbeChangeP5(v, OPFLAG_ISNOOP); + sqlite3ReleaseTempReg(pParse, r); } #endif } @@ -115154,14 +122490,6 @@ SQLITE_PRIVATE void sqlite3CompleteInsertion( sqlite3VdbeChangeP5(v, pik_flags); } if( !HasRowid(pTab) ) return; - regData = regNewData + 1; - regRec = sqlite3GetTempReg(pParse); - sqlite3VdbeAddOp3(v, OP_MakeRecord, regData, pTab->nCol, regRec); - sqlite3SetMakeRecordP5(v, pTab); - if( !bAffinityDone ){ - sqlite3TableAffinity(v, pTab, 0); - sqlite3ExprCacheAffinityChange(pParse, regData, pTab->nCol); - } if( pParse->nested ){ pik_flags = 0; }else{ @@ -115174,7 +122502,7 @@ SQLITE_PRIVATE void sqlite3CompleteInsertion( if( useSeekResult ){ pik_flags |= OPFLAG_USESEEKRESULT; } - sqlite3VdbeAddOp3(v, OP_Insert, iDataCur, regRec, regNewData); + sqlite3VdbeAddOp3(v, OP_Insert, iDataCur, aRegIdx[i], regNewData); if( !pParse->nested ){ sqlite3VdbeAppendP4(v, pTab, P4_TABLE); } @@ -115284,7 +122612,7 @@ static int xferCompatibleIndex(Index *pDest, Index *pSrc){ int i; assert( pDest && pSrc ); assert( pDest->pTable!=pSrc->pTable ); - if( pDest->nKeyCol!=pSrc->nKeyCol ){ + if( pDest->nKeyCol!=pSrc->nKeyCol || pDest->nColumn!=pSrc->nColumn ){ return 0; /* Different number of columns */ } if( pDest->onError!=pSrc->onError ){ @@ -115431,7 +122759,8 @@ static int xferOptimization( if( pSrc==0 ){ return 0; /* FROM clause does not contain a real table */ } - if( pSrc==pDest ){ + if( pSrc->tnum==pDest->tnum && pSrc->pSchema==pDest->pSchema ){ + testcase( pSrc!=pDest ); /* Possible due to bad sqlite_master.rootpage */ return 0; /* tab1 and tab2 may not be the same table */ } if( HasRowid(pDest)!=HasRowid(pSrc) ){ @@ -115460,6 +122789,39 @@ static int xferOptimization( ){ return 0; /* Neither table may have __hidden__ columns */ } +#endif +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + /* Even if tables t1 and t2 have identical schemas, if they contain + ** generated columns, then this statement is semantically incorrect: + ** + ** INSERT INTO t2 SELECT * FROM t1; + ** + ** The reason is that generated column values are returned by the + ** the SELECT statement on the right but the INSERT statement on the + ** left wants them to be omitted. + ** + ** Nevertheless, this is a useful notational shorthand to tell SQLite + ** to do a bulk transfer all of the content from t1 over to t2. + ** + ** We could, in theory, disable this (except for internal use by the + ** VACUUM command where it is actually needed). But why do that? It + ** seems harmless enough, and provides a useful service. + */ + if( (pDestCol->colFlags & COLFLAG_GENERATED) != + (pSrcCol->colFlags & COLFLAG_GENERATED) ){ + return 0; /* Both columns have the same generated-column type */ + } + /* But the transfer is only allowed if both the source and destination + ** tables have the exact same expressions for generated columns. + ** This requirement could be relaxed for VIRTUAL columns, I suppose. + */ + if( (pDestCol->colFlags & COLFLAG_GENERATED)!=0 ){ + if( sqlite3ExprCompare(0, pSrcCol->pDflt, pDestCol->pDflt, -1)!=0 ){ + testcase( pDestCol->colFlags & COLFLAG_VIRTUAL ); + testcase( pDestCol->colFlags & COLFLAG_STORED ); + return 0; /* Different generator expressions */ + } + } #endif if( pDestCol->affinity!=pSrcCol->affinity ){ return 0; /* Affinity must be the same on all columns */ @@ -115471,7 +122833,7 @@ static int xferOptimization( return 0; /* tab2 must be NOT NULL if tab1 is */ } /* Default values for second and subsequent columns need to match. */ - if( i>0 ){ + if( (pDestCol->colFlags & COLFLAG_GENERATED)==0 && i>0 ){ assert( pDestCol->pDflt==0 || pDestCol->pDflt->op==TK_SPAN ); assert( pSrcCol->pDflt==0 || pSrcCol->pDflt->op==TK_SPAN ); if( (pDestCol->pDflt==0)!=(pSrcCol->pDflt==0) @@ -115492,6 +122854,13 @@ static int xferOptimization( if( pSrcIdx==0 ){ return 0; /* pDestIdx has no corresponding index in pSrc */ } + if( pSrcIdx->tnum==pDestIdx->tnum && pSrc->pSchema==pDest->pSchema + && sqlite3FaultSim(411)==SQLITE_OK ){ + /* The sqlite3FaultSim() call allows this corruption test to be + ** bypassed during testing, in order to exercise other corruption tests + ** further downstream. */ + return 0; /* Corrupt schema - two indexes on the same btree */ + } } #ifndef SQLITE_OMIT_CHECK if( pDest->pCheck && sqlite3ExprListCompare(pSrc->pCheck,pDest->pCheck,-1) ){ @@ -115569,20 +122938,19 @@ static int xferOptimization( sqlite3RowidConstraint(pParse, onError, pDest); sqlite3VdbeJumpHere(v, addr2); autoIncStep(pParse, regAutoinc, regRowid); - }else if( pDest->pIndex==0 ){ + }else if( pDest->pIndex==0 && !(db->mDbFlags & DBFLAG_VacuumInto) ){ addr1 = sqlite3VdbeAddOp2(v, OP_NewRowid, iDest, regRowid); }else{ addr1 = sqlite3VdbeAddOp2(v, OP_Rowid, iSrc, regRowid); assert( (pDest->tabFlags & TF_Autoincrement)==0 ); } - sqlite3VdbeAddOp3(v, OP_RowData, iSrc, regData, 1); if( db->mDbFlags & DBFLAG_Vacuum ){ sqlite3VdbeAddOp1(v, OP_SeekEnd, iDest); - insFlags = OPFLAG_NCHANGE|OPFLAG_LASTROWID| - OPFLAG_APPEND|OPFLAG_USESEEKRESULT; + insFlags = OPFLAG_APPEND|OPFLAG_USESEEKRESULT; }else{ insFlags = OPFLAG_NCHANGE|OPFLAG_LASTROWID|OPFLAG_APPEND; } + sqlite3VdbeAddOp3(v, OP_RowData, iSrc, regData, 1); sqlite3VdbeAddOp4(v, OP_Insert, iDest, regData, regRowid, (char*)pDest, P4_TABLE); sqlite3VdbeChangeP5(v, insFlags); @@ -115607,7 +122975,6 @@ static int xferOptimization( sqlite3VdbeChangeP5(v, OPFLAG_BULKCSR); VdbeComment((v, "%s", pDestIdx->zName)); addr1 = sqlite3VdbeAddOp2(v, OP_Rewind, iSrc, 0); VdbeCoverage(v); - sqlite3VdbeAddOp3(v, OP_RowData, iSrc, regData, 1); if( db->mDbFlags & DBFLAG_Vacuum ){ /* This INSERT command is part of a VACUUM operation, which guarantees ** that the destination table is empty. If all indexed columns use @@ -115631,10 +122998,10 @@ static int xferOptimization( idxInsFlags = OPFLAG_USESEEKRESULT; sqlite3VdbeAddOp1(v, OP_SeekEnd, iDest); } - } - if( !HasRowid(pSrc) && pDestIdx->idxType==2 ){ + }else if( !HasRowid(pSrc) && pDestIdx->idxType==SQLITE_IDXTYPE_PRIMARYKEY ){ idxInsFlags |= OPFLAG_NCHANGE; } + sqlite3VdbeAddOp3(v, OP_RowData, iSrc, regData, 1); sqlite3VdbeAddOp2(v, OP_IdxInsert, iDest, regData); sqlite3VdbeChangeP5(v, idxInsFlags|OPFLAG_APPEND); sqlite3VdbeAddOp2(v, OP_Next, iSrc, addr1+1); VdbeCoverage(v); @@ -115707,7 +123074,7 @@ SQLITE_API int sqlite3_exec( sqlite3_mutex_enter(db->mutex); sqlite3Error(db, SQLITE_OK); while( rc==SQLITE_OK && zSql[0] ){ - int nCol; + int nCol = 0; char **azVals = 0; pStmt = 0; @@ -115721,9 +123088,7 @@ SQLITE_API int sqlite3_exec( zSql = zLeftover; continue; } - callbackIsInit = 0; - nCol = sqlite3_column_count(pStmt); while( 1 ){ int i; @@ -115734,6 +123099,7 @@ SQLITE_API int sqlite3_exec( (SQLITE_DONE==rc && !callbackIsInit && db->flags&SQLITE_NullCallback)) ){ if( !callbackIsInit ){ + nCol = sqlite3_column_count(pStmt); azCols = sqlite3DbMallocRaw(db, (2*nCol+1)*sizeof(const char*)); if( azCols==0 ){ goto exec_out; @@ -116136,6 +123502,31 @@ struct sqlite3_api_routines { int (*str_errcode)(sqlite3_str*); int (*str_length)(sqlite3_str*); char *(*str_value)(sqlite3_str*); + /* Version 3.25.0 and later */ + int (*create_window_function)(sqlite3*,const char*,int,int,void*, + void (*xStep)(sqlite3_context*,int,sqlite3_value**), + void (*xFinal)(sqlite3_context*), + void (*xValue)(sqlite3_context*), + void (*xInv)(sqlite3_context*,int,sqlite3_value**), + void(*xDestroy)(void*)); + /* Version 3.26.0 and later */ + const char *(*normalized_sql)(sqlite3_stmt*); + /* Version 3.28.0 and later */ + int (*stmt_isexplain)(sqlite3_stmt*); + int (*value_frombind)(sqlite3_value*); + /* Version 3.30.0 and later */ + int (*drop_modules)(sqlite3*,const char**); + /* Version 3.31.0 and later */ + sqlite3_int64 (*hard_heap_limit64)(sqlite3_int64); + const char *(*uri_key)(const char*,int); + const char *(*filename_database)(const char*); + const char *(*filename_journal)(const char*); + const char *(*filename_wal)(const char*); + /* Version 3.32.0 and later */ + char *(*create_filename)(const char*,const char*,const char*, + int,const char**); + void (*free_filename)(char*); + sqlite3_file *(*database_file_object)(const char*); }; /* @@ -116421,6 +123812,25 @@ typedef int (*sqlite3_loadext_entry)( #define sqlite3_str_errcode sqlite3_api->str_errcode #define sqlite3_str_length sqlite3_api->str_length #define sqlite3_str_value sqlite3_api->str_value +/* Version 3.25.0 and later */ +#define sqlite3_create_window_function sqlite3_api->create_window_function +/* Version 3.26.0 and later */ +#define sqlite3_normalized_sql sqlite3_api->normalized_sql +/* Version 3.28.0 and later */ +#define sqlite3_stmt_isexplain sqlite3_api->stmt_isexplain +#define sqlite3_value_frombind sqlite3_api->value_frombind +/* Version 3.30.0 and later */ +#define sqlite3_drop_modules sqlite3_api->drop_modules +/* Version 3.31.0 and later */ +#define sqlite3_hard_heap_limit64 sqlite3_api->hard_heap_limit64 +#define sqlite3_uri_key sqlite3_api->uri_key +#define sqlite3_filename_database sqlite3_api->filename_database +#define sqlite3_filename_journal sqlite3_api->filename_journal +#define sqlite3_filename_wal sqlite3_api->filename_wal +/* Version 3.32.0 and later */ +#define sqlite3_create_filename sqlite3_api->create_filename +#define sqlite3_free_filename sqlite3_api->free_filename +#define sqlite3_database_file_object sqlite3_api->database_file_object #endif /* !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) */ #if !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) @@ -116509,6 +123919,7 @@ typedef int (*sqlite3_loadext_entry)( # define sqlite3_declare_vtab 0 # define sqlite3_vtab_config 0 # define sqlite3_vtab_on_conflict 0 +# define sqlite3_vtab_collation 0 #endif #ifdef SQLITE_OMIT_SHARED_CACHE @@ -116874,9 +124285,44 @@ static const sqlite3_api_routines sqlite3Apis = { sqlite3_str_reset, sqlite3_str_errcode, sqlite3_str_length, - sqlite3_str_value + sqlite3_str_value, + /* Version 3.25.0 and later */ + sqlite3_create_window_function, + /* Version 3.26.0 and later */ +#ifdef SQLITE_ENABLE_NORMALIZE + sqlite3_normalized_sql, +#else + 0, +#endif + /* Version 3.28.0 and later */ + sqlite3_stmt_isexplain, + sqlite3_value_frombind, + /* Version 3.30.0 and later */ +#ifndef SQLITE_OMIT_VIRTUALTABLE + sqlite3_drop_modules, +#else + 0, +#endif + /* Version 3.31.0 and later */ + sqlite3_hard_heap_limit64, + sqlite3_uri_key, + sqlite3_filename_database, + sqlite3_filename_journal, + sqlite3_filename_wal, + /* Version 3.32.0 and later */ + sqlite3_create_filename, + sqlite3_free_filename, + sqlite3_database_file_object, }; +/* True if x is the directory separator character +*/ +#if SQLITE_OS_WIN +# define DirSep(X) ((X)=='/'||(X)=='\\') +#else +# define DirSep(X) ((X)=='/') +#endif + /* ** Attempt to load an SQLite extension library contained in the file ** zFile. The entry point is zProc. zProc may be 0 in which case a @@ -116978,7 +124424,7 @@ static int sqlite3LoadExtension( return SQLITE_NOMEM_BKPT; } memcpy(zAltEntry, "sqlite3_", 8); - for(iFile=ncFile-1; iFile>=0 && zFile[iFile]!='/'; iFile--){} + for(iFile=ncFile-1; iFile>=0 && !DirSep(zFile[iFile]); iFile--){} iFile++; if( sqlite3_strnicmp(zFile+iFile, "lib", 3)==0 ) iFile += 3; for(iEntry=8; (c = zFile[iFile])!=0 && c!='.'; iFile++){ @@ -117066,7 +124512,7 @@ SQLITE_API int sqlite3_enable_load_extension(sqlite3 *db, int onoff){ if( onoff ){ db->flags |= SQLITE_LoadExtension|SQLITE_LoadExtFunc; }else{ - db->flags &= ~(SQLITE_LoadExtension|SQLITE_LoadExtFunc); + db->flags &= ~(u64)(SQLITE_LoadExtension|SQLITE_LoadExtFunc); } sqlite3_mutex_leave(db->mutex); return SQLITE_OK; @@ -117281,53 +124727,51 @@ SQLITE_PRIVATE void sqlite3AutoLoadExtensions(sqlite3 *db){ */ /* The various pragma types */ -#define PragTyp_HEADER_VALUE 0 -#define PragTyp_AUTO_VACUUM 1 -#define PragTyp_FLAG 2 -#define PragTyp_BUSY_TIMEOUT 3 -#define PragTyp_CACHE_SIZE 4 -#define PragTyp_CACHE_SPILL 5 -#define PragTyp_CASE_SENSITIVE_LIKE 6 -#define PragTyp_COLLATION_LIST 7 -#define PragTyp_COMPILE_OPTIONS 8 -#define PragTyp_DATA_STORE_DIRECTORY 9 -#define PragTyp_DATABASE_LIST 10 -#define PragTyp_DEFAULT_CACHE_SIZE 11 -#define PragTyp_ENCODING 12 -#define PragTyp_FOREIGN_KEY_CHECK 13 -#define PragTyp_FOREIGN_KEY_LIST 14 -#define PragTyp_FUNCTION_LIST 15 -#define PragTyp_INCREMENTAL_VACUUM 16 -#define PragTyp_INDEX_INFO 17 -#define PragTyp_INDEX_LIST 18 -#define PragTyp_INTEGRITY_CHECK 19 -#define PragTyp_JOURNAL_MODE 20 -#define PragTyp_JOURNAL_SIZE_LIMIT 21 -#define PragTyp_LOCK_PROXY_FILE 22 -#define PragTyp_LOCKING_MODE 23 -#define PragTyp_PAGE_COUNT 24 -#define PragTyp_MMAP_SIZE 25 -#define PragTyp_MODULE_LIST 26 -#define PragTyp_OPTIMIZE 27 -#define PragTyp_PAGE_SIZE 28 -#define PragTyp_PRAGMA_LIST 29 -#define PragTyp_SECURE_DELETE 30 -#define PragTyp_SHRINK_MEMORY 31 -#define PragTyp_SOFT_HEAP_LIMIT 32 -#define PragTyp_SYNCHRONOUS 33 -#define PragTyp_TABLE_INFO 34 -#define PragTyp_TEMP_STORE 35 -#define PragTyp_TEMP_STORE_DIRECTORY 36 -#define PragTyp_THREADS 37 -#define PragTyp_WAL_AUTOCHECKPOINT 38 -#define PragTyp_WAL_CHECKPOINT 39 -#define PragTyp_ACTIVATE_EXTENSIONS 40 -#define PragTyp_HEXKEY 41 -#define PragTyp_KEY 42 -#define PragTyp_REKEY 43 -#define PragTyp_LOCK_STATUS 44 -#define PragTyp_PARSER_TRACE 45 -#define PragTyp_STATS 46 +#define PragTyp_ACTIVATE_EXTENSIONS 0 +#define PragTyp_ANALYSIS_LIMIT 1 +#define PragTyp_HEADER_VALUE 2 +#define PragTyp_AUTO_VACUUM 3 +#define PragTyp_FLAG 4 +#define PragTyp_BUSY_TIMEOUT 5 +#define PragTyp_CACHE_SIZE 6 +#define PragTyp_CACHE_SPILL 7 +#define PragTyp_CASE_SENSITIVE_LIKE 8 +#define PragTyp_COLLATION_LIST 9 +#define PragTyp_COMPILE_OPTIONS 10 +#define PragTyp_DATA_STORE_DIRECTORY 11 +#define PragTyp_DATABASE_LIST 12 +#define PragTyp_DEFAULT_CACHE_SIZE 13 +#define PragTyp_ENCODING 14 +#define PragTyp_FOREIGN_KEY_CHECK 15 +#define PragTyp_FOREIGN_KEY_LIST 16 +#define PragTyp_FUNCTION_LIST 17 +#define PragTyp_HARD_HEAP_LIMIT 18 +#define PragTyp_INCREMENTAL_VACUUM 19 +#define PragTyp_INDEX_INFO 20 +#define PragTyp_INDEX_LIST 21 +#define PragTyp_INTEGRITY_CHECK 22 +#define PragTyp_JOURNAL_MODE 23 +#define PragTyp_JOURNAL_SIZE_LIMIT 24 +#define PragTyp_LOCK_PROXY_FILE 25 +#define PragTyp_LOCKING_MODE 26 +#define PragTyp_PAGE_COUNT 27 +#define PragTyp_MMAP_SIZE 28 +#define PragTyp_MODULE_LIST 29 +#define PragTyp_OPTIMIZE 30 +#define PragTyp_PAGE_SIZE 31 +#define PragTyp_PRAGMA_LIST 32 +#define PragTyp_SECURE_DELETE 33 +#define PragTyp_SHRINK_MEMORY 34 +#define PragTyp_SOFT_HEAP_LIMIT 35 +#define PragTyp_SYNCHRONOUS 36 +#define PragTyp_TABLE_INFO 37 +#define PragTyp_TEMP_STORE 38 +#define PragTyp_TEMP_STORE_DIRECTORY 39 +#define PragTyp_THREADS 40 +#define PragTyp_WAL_AUTOCHECKPOINT 41 +#define PragTyp_WAL_CHECKPOINT 42 +#define PragTyp_LOCK_STATUS 43 +#define PragTyp_STATS 44 /* Property flags associated with various pragma. */ #define PragFlg_NeedSchema 0x01 /* Force schema load before running */ @@ -117344,58 +124788,61 @@ SQLITE_PRIVATE void sqlite3AutoLoadExtensions(sqlite3 *db){ ** result column is different from the name of the pragma */ static const char *const pragCName[] = { - /* 0 */ "cache_size", /* Used by: default_cache_size */ - /* 1 */ "cid", /* Used by: table_info */ - /* 2 */ "name", - /* 3 */ "type", - /* 4 */ "notnull", - /* 5 */ "dflt_value", - /* 6 */ "pk", - /* 7 */ "tbl", /* Used by: stats */ - /* 8 */ "idx", - /* 9 */ "wdth", - /* 10 */ "hght", - /* 11 */ "flgs", - /* 12 */ "seqno", /* Used by: index_info */ - /* 13 */ "cid", - /* 14 */ "name", + /* 0 */ "id", /* Used by: foreign_key_list */ + /* 1 */ "seq", + /* 2 */ "table", + /* 3 */ "from", + /* 4 */ "to", + /* 5 */ "on_update", + /* 6 */ "on_delete", + /* 7 */ "match", + /* 8 */ "cid", /* Used by: table_xinfo */ + /* 9 */ "name", + /* 10 */ "type", + /* 11 */ "notnull", + /* 12 */ "dflt_value", + /* 13 */ "pk", + /* 14 */ "hidden", + /* table_info reuses 8 */ /* 15 */ "seqno", /* Used by: index_xinfo */ /* 16 */ "cid", /* 17 */ "name", /* 18 */ "desc", /* 19 */ "coll", /* 20 */ "key", - /* 21 */ "seq", /* Used by: index_list */ - /* 22 */ "name", - /* 23 */ "unique", - /* 24 */ "origin", - /* 25 */ "partial", - /* 26 */ "seq", /* Used by: database_list */ - /* 27 */ "name", - /* 28 */ "file", - /* 29 */ "name", /* Used by: function_list */ - /* 30 */ "builtin", - /* 31 */ "name", /* Used by: module_list pragma_list */ - /* 32 */ "seq", /* Used by: collation_list */ + /* 21 */ "name", /* Used by: function_list */ + /* 22 */ "builtin", + /* 23 */ "type", + /* 24 */ "enc", + /* 25 */ "narg", + /* 26 */ "flags", + /* 27 */ "tbl", /* Used by: stats */ + /* 28 */ "idx", + /* 29 */ "wdth", + /* 30 */ "hght", + /* 31 */ "flgs", + /* 32 */ "seq", /* Used by: index_list */ /* 33 */ "name", - /* 34 */ "id", /* Used by: foreign_key_list */ - /* 35 */ "seq", - /* 36 */ "table", - /* 37 */ "from", - /* 38 */ "to", - /* 39 */ "on_update", - /* 40 */ "on_delete", - /* 41 */ "match", - /* 42 */ "table", /* Used by: foreign_key_check */ - /* 43 */ "rowid", - /* 44 */ "parent", - /* 45 */ "fkid", - /* 46 */ "busy", /* Used by: wal_checkpoint */ - /* 47 */ "log", - /* 48 */ "checkpointed", - /* 49 */ "timeout", /* Used by: busy_timeout */ - /* 50 */ "database", /* Used by: lock_status */ - /* 51 */ "status", + /* 34 */ "unique", + /* 35 */ "origin", + /* 36 */ "partial", + /* 37 */ "table", /* Used by: foreign_key_check */ + /* 38 */ "rowid", + /* 39 */ "parent", + /* 40 */ "fkid", + /* index_info reuses 15 */ + /* 41 */ "seq", /* Used by: database_list */ + /* 42 */ "name", + /* 43 */ "file", + /* 44 */ "busy", /* Used by: wal_checkpoint */ + /* 45 */ "log", + /* 46 */ "checkpointed", + /* collation_list reuses 32 */ + /* 47 */ "database", /* Used by: lock_status */ + /* 48 */ "status", + /* 49 */ "cache_size", /* Used by: default_cache_size */ + /* module_list pragma_list reuses 9 */ + /* 50 */ "timeout", /* Used by: busy_timeout */ }; /* Definitions of all built-in pragmas */ @@ -117405,16 +124852,21 @@ typedef struct PragmaName { u8 mPragFlg; /* Zero or more PragFlg_XXX values */ u8 iPragCName; /* Start of column names in pragCName[] */ u8 nPragCName; /* Num of col names. 0 means use pragma name */ - u32 iArg; /* Extra argument */ + u64 iArg; /* Extra argument */ } PragmaName; static const PragmaName aPragmaName[] = { -#if defined(SQLITE_HAS_CODEC) || defined(SQLITE_ENABLE_CEROD) +#if defined(SQLITE_ENABLE_CEROD) {/* zName: */ "activate_extensions", /* ePragTyp: */ PragTyp_ACTIVATE_EXTENSIONS, /* ePragFlg: */ 0, /* ColNames: */ 0, 0, /* iArg: */ 0 }, #endif + {/* zName: */ "analysis_limit", + /* ePragTyp: */ PragTyp_ANALYSIS_LIMIT, + /* ePragFlg: */ PragFlg_Result0, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, #if !defined(SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS) {/* zName: */ "application_id", /* ePragTyp: */ PragTyp_HEADER_VALUE, @@ -117441,7 +124893,7 @@ static const PragmaName aPragmaName[] = { {/* zName: */ "busy_timeout", /* ePragTyp: */ PragTyp_BUSY_TIMEOUT, /* ePragFlg: */ PragFlg_Result0, - /* ColNames: */ 49, 1, + /* ColNames: */ 50, 1, /* iArg: */ 0 }, #if !defined(SQLITE_OMIT_PAGER_PRAGMAS) {/* zName: */ "cache_size", @@ -117457,11 +124909,13 @@ static const PragmaName aPragmaName[] = { /* ColNames: */ 0, 0, /* iArg: */ 0 }, #endif +#if !defined(SQLITE_OMIT_CASE_SENSITIVE_LIKE_PRAGMA) {/* zName: */ "case_sensitive_like", /* ePragTyp: */ PragTyp_CASE_SENSITIVE_LIKE, /* ePragFlg: */ PragFlg_NoColumns, /* ColNames: */ 0, 0, /* iArg: */ 0 }, +#endif {/* zName: */ "cell_size_check", /* ePragTyp: */ PragTyp_FLAG, /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, @@ -117513,14 +124967,14 @@ static const PragmaName aPragmaName[] = { {/* zName: */ "database_list", /* ePragTyp: */ PragTyp_DATABASE_LIST, /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result0, - /* ColNames: */ 26, 3, + /* ColNames: */ 41, 3, /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_PAGER_PRAGMAS) && !defined(SQLITE_OMIT_DEPRECATED) {/* zName: */ "default_cache_size", /* ePragTyp: */ PragTyp_DEFAULT_CACHE_SIZE, /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result0|PragFlg_SchemaReq|PragFlg_NoColumns1, - /* ColNames: */ 0, 1, + /* ColNames: */ 49, 1, /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_FLAG_PRAGMAS) @@ -117550,14 +125004,14 @@ static const PragmaName aPragmaName[] = { {/* zName: */ "foreign_key_check", /* ePragTyp: */ PragTyp_FOREIGN_KEY_CHECK, /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result0, - /* ColNames: */ 42, 4, + /* ColNames: */ 37, 4, /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_FOREIGN_KEY) {/* zName: */ "foreign_key_list", /* ePragTyp: */ PragTyp_FOREIGN_KEY_LIST, /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result1|PragFlg_SchemaOpt, - /* ColNames: */ 34, 8, + /* ColNames: */ 0, 8, /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_FLAG_PRAGMAS) @@ -117589,26 +125043,19 @@ static const PragmaName aPragmaName[] = { /* iArg: */ SQLITE_FullFSync }, #endif #if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) -#if defined(SQLITE_INTROSPECTION_PRAGMAS) +#if !defined(SQLITE_OMIT_INTROSPECTION_PRAGMAS) {/* zName: */ "function_list", /* ePragTyp: */ PragTyp_FUNCTION_LIST, /* ePragFlg: */ PragFlg_Result0, - /* ColNames: */ 29, 2, + /* ColNames: */ 21, 6, /* iArg: */ 0 }, #endif #endif -#if defined(SQLITE_HAS_CODEC) - {/* zName: */ "hexkey", - /* ePragTyp: */ PragTyp_HEXKEY, - /* ePragFlg: */ 0, - /* ColNames: */ 0, 0, - /* iArg: */ 0 }, - {/* zName: */ "hexrekey", - /* ePragTyp: */ PragTyp_HEXKEY, - /* ePragFlg: */ 0, + {/* zName: */ "hard_heap_limit", + /* ePragTyp: */ PragTyp_HARD_HEAP_LIMIT, + /* ePragFlg: */ PragFlg_Result0, /* ColNames: */ 0, 0, /* iArg: */ 0 }, -#endif #if !defined(SQLITE_OMIT_FLAG_PRAGMAS) #if !defined(SQLITE_OMIT_CHECK) {/* zName: */ "ignore_check_constraints", @@ -117629,12 +125076,12 @@ static const PragmaName aPragmaName[] = { {/* zName: */ "index_info", /* ePragTyp: */ PragTyp_INDEX_INFO, /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result1|PragFlg_SchemaOpt, - /* ColNames: */ 12, 3, + /* ColNames: */ 15, 3, /* iArg: */ 0 }, {/* zName: */ "index_list", /* ePragTyp: */ PragTyp_INDEX_LIST, /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result1|PragFlg_SchemaOpt, - /* ColNames: */ 21, 5, + /* ColNames: */ 32, 5, /* iArg: */ 0 }, {/* zName: */ "index_xinfo", /* ePragTyp: */ PragTyp_INDEX_INFO, @@ -117661,19 +125108,12 @@ static const PragmaName aPragmaName[] = { /* ColNames: */ 0, 0, /* iArg: */ 0 }, #endif -#if defined(SQLITE_HAS_CODEC) - {/* zName: */ "key", - /* ePragTyp: */ PragTyp_KEY, - /* ePragFlg: */ 0, - /* ColNames: */ 0, 0, - /* iArg: */ 0 }, -#endif #if !defined(SQLITE_OMIT_FLAG_PRAGMAS) - {/* zName: */ "legacy_file_format", + {/* zName: */ "legacy_alter_table", /* ePragTyp: */ PragTyp_FLAG, /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, /* ColNames: */ 0, 0, - /* iArg: */ SQLITE_LegacyFileFmt }, + /* iArg: */ SQLITE_LegacyAlter }, #endif #if !defined(SQLITE_OMIT_PAGER_PRAGMAS) && SQLITE_ENABLE_LOCKING_STYLE {/* zName: */ "lock_proxy_file", @@ -117686,7 +125126,7 @@ static const PragmaName aPragmaName[] = { {/* zName: */ "lock_status", /* ePragTyp: */ PragTyp_LOCK_STATUS, /* ePragFlg: */ PragFlg_Result0, - /* ColNames: */ 50, 2, + /* ColNames: */ 47, 2, /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_PAGER_PRAGMAS) @@ -117708,11 +125148,11 @@ static const PragmaName aPragmaName[] = { #endif #if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) #if !defined(SQLITE_OMIT_VIRTUALTABLE) -#if defined(SQLITE_INTROSPECTION_PRAGMAS) +#if !defined(SQLITE_OMIT_INTROSPECTION_PRAGMAS) {/* zName: */ "module_list", /* ePragTyp: */ PragTyp_MODULE_LIST, /* ePragFlg: */ PragFlg_Result0, - /* ColNames: */ 31, 1, + /* ColNames: */ 9, 1, /* iArg: */ 0 }, #endif #endif @@ -117734,18 +125174,20 @@ static const PragmaName aPragmaName[] = { /* ColNames: */ 0, 0, /* iArg: */ 0 }, #endif -#if defined(SQLITE_DEBUG) && !defined(SQLITE_OMIT_PARSER_TRACE) +#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) +#if defined(SQLITE_DEBUG) {/* zName: */ "parser_trace", - /* ePragTyp: */ PragTyp_PARSER_TRACE, - /* ePragFlg: */ 0, + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, /* ColNames: */ 0, 0, - /* iArg: */ 0 }, + /* iArg: */ SQLITE_ParserTrace }, +#endif #endif -#if defined(SQLITE_INTROSPECTION_PRAGMAS) +#if !defined(SQLITE_OMIT_INTROSPECTION_PRAGMAS) {/* zName: */ "pragma_list", /* ePragTyp: */ PragTyp_PRAGMA_LIST, /* ePragFlg: */ PragFlg_Result0, - /* ColNames: */ 31, 1, + /* ColNames: */ 9, 1, /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_FLAG_PRAGMAS) @@ -117773,15 +125215,6 @@ static const PragmaName aPragmaName[] = { /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, /* ColNames: */ 0, 0, /* iArg: */ SQLITE_RecTriggers }, -#endif -#if defined(SQLITE_HAS_CODEC) - {/* zName: */ "rekey", - /* ePragTyp: */ PragTyp_REKEY, - /* ePragFlg: */ 0, - /* ColNames: */ 0, 0, - /* iArg: */ 0 }, -#endif -#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) {/* zName: */ "reverse_unordered_selects", /* ePragTyp: */ PragTyp_FLAG, /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, @@ -117832,7 +125265,7 @@ static const PragmaName aPragmaName[] = { {/* zName: */ "stats", /* ePragTyp: */ PragTyp_STATS, /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result0|PragFlg_SchemaReq, - /* ColNames: */ 7, 5, + /* ColNames: */ 27, 5, /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_PAGER_PRAGMAS) @@ -117846,8 +125279,13 @@ static const PragmaName aPragmaName[] = { {/* zName: */ "table_info", /* ePragTyp: */ PragTyp_TABLE_INFO, /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result1|PragFlg_SchemaOpt, - /* ColNames: */ 1, 6, + /* ColNames: */ 8, 6, /* iArg: */ 0 }, + {/* zName: */ "table_xinfo", + /* ePragTyp: */ PragTyp_TABLE_INFO, + /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result1|PragFlg_SchemaOpt, + /* ColNames: */ 8, 7, + /* iArg: */ 1 }, #endif #if !defined(SQLITE_OMIT_PAGER_PRAGMAS) {/* zName: */ "temp_store", @@ -117866,6 +125304,13 @@ static const PragmaName aPragmaName[] = { /* ePragFlg: */ PragFlg_Result0, /* ColNames: */ 0, 0, /* iArg: */ 0 }, +#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) + {/* zName: */ "trusted_schema", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_TrustedSchema }, +#endif #if !defined(SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS) {/* zName: */ "user_version", /* ePragTyp: */ PragTyp_HEADER_VALUE, @@ -117911,7 +125356,7 @@ static const PragmaName aPragmaName[] = { {/* zName: */ "wal_checkpoint", /* ePragTyp: */ PragTyp_WAL_CHECKPOINT, /* ePragFlg: */ PragFlg_NeedSchema, - /* ColNames: */ 46, 3, + /* ColNames: */ 44, 3, /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_FLAG_PRAGMAS) @@ -117919,10 +125364,10 @@ static const PragmaName aPragmaName[] = { /* ePragTyp: */ PragTyp_FLAG, /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, /* ColNames: */ 0, 0, - /* iArg: */ SQLITE_WriteSchema }, + /* iArg: */ SQLITE_WriteSchema|SQLITE_NoSchemaError }, #endif }; -/* Number of pragmas: 60 on by default, 77 total. */ +/* Number of pragmas: 67 on by default, 77 total. */ /************** End of pragma.h **********************************************/ /************** Continuing where we left off in pragma.c *********************/ @@ -118192,6 +125637,55 @@ static const PragmaName *pragmaLocate(const char *zName){ return lwr>upr ? 0 : &aPragmaName[mid]; } +/* +** Create zero or more entries in the output for the SQL functions +** defined by FuncDef p. +*/ +static void pragmaFunclistLine( + Vdbe *v, /* The prepared statement being created */ + FuncDef *p, /* A particular function definition */ + int isBuiltin, /* True if this is a built-in function */ + int showInternFuncs /* True if showing internal functions */ +){ + for(; p; p=p->pNext){ + const char *zType; + static const u32 mask = + SQLITE_DETERMINISTIC | + SQLITE_DIRECTONLY | + SQLITE_SUBTYPE | + SQLITE_INNOCUOUS | + SQLITE_FUNC_INTERNAL + ; + static const char *azEnc[] = { 0, "utf8", "utf16le", "utf16be" }; + + assert( SQLITE_FUNC_ENCMASK==0x3 ); + assert( strcmp(azEnc[SQLITE_UTF8],"utf8")==0 ); + assert( strcmp(azEnc[SQLITE_UTF16LE],"utf16le")==0 ); + assert( strcmp(azEnc[SQLITE_UTF16BE],"utf16be")==0 ); + + if( p->xSFunc==0 ) continue; + if( (p->funcFlags & SQLITE_FUNC_INTERNAL)!=0 + && showInternFuncs==0 + ){ + continue; + } + if( p->xValue!=0 ){ + zType = "w"; + }else if( p->xFinalize!=0 ){ + zType = "a"; + }else{ + zType = "s"; + } + sqlite3VdbeMultiLoad(v, 1, "sissii", + p->zName, isBuiltin, + zType, azEnc[p->funcFlags&SQLITE_FUNC_ENCMASK], + p->nArg, + (p->funcFlags & mask) ^ SQLITE_INNOCUOUS + ); + } +} + + /* ** Helper subroutine for PRAGMA integrity_check: ** @@ -118403,7 +125897,7 @@ SQLITE_PRIVATE void sqlite3Pragma( ** buffer that the pager module resizes using sqlite3_realloc(). */ db->nextPagesize = sqlite3Atoi(zRight); - if( SQLITE_NOMEM==sqlite3BtreeSetPageSize(pBt, db->nextPagesize,-1,0) ){ + if( SQLITE_NOMEM==sqlite3BtreeSetPageSize(pBt, db->nextPagesize,0,0) ){ sqlite3OomFault(db); } } @@ -118541,6 +126035,11 @@ SQLITE_PRIVATE void sqlite3Pragma( ** then do a query */ eMode = PAGER_JOURNALMODE_QUERY; } + if( eMode==PAGER_JOURNALMODE_OFF && (db->flags & SQLITE_Defensive)!=0 ){ + /* Do not allow journal-mode "OFF" in defensive since the database + ** can become corrupted using ordinary SQL when the journal is off */ + eMode = PAGER_JOURNALMODE_QUERY; + } } if( eMode==PAGER_JOURNALMODE_QUERY && pId2->n==0 ){ /* Convert "PRAGMA journal_mode" into "PRAGMA main.journal_mode" */ @@ -118713,7 +126212,7 @@ SQLITE_PRIVATE void sqlite3Pragma( if( sqlite3GetBoolean(zRight, size!=0) ){ db->flags |= SQLITE_CacheSpill; }else{ - db->flags &= ~SQLITE_CacheSpill; + db->flags &= ~(u64)SQLITE_CacheSpill; } setAllPagerFlags(db); } @@ -118934,7 +126433,7 @@ SQLITE_PRIVATE void sqlite3Pragma( setPragmaResultColumnNames(v, pPragma); returnSingleInt(v, (db->flags & pPragma->iArg)!=0 ); }else{ - int mask = pPragma->iArg; /* Mask of bits to set or clear. */ + u64 mask = pPragma->iArg; /* Mask of bits to set or clear. */ if( db->autoCommit==0 ){ /* Foreign key support may not be enabled or disabled while not ** in auto-commit mode. */ @@ -118983,17 +126482,28 @@ SQLITE_PRIVATE void sqlite3Pragma( Table *pTab; pTab = sqlite3LocateTable(pParse, LOCATE_NOERR, zRight, zDb); if( pTab ){ + int iTabDb = sqlite3SchemaToIndex(db, pTab->pSchema); int i, k; int nHidden = 0; Column *pCol; Index *pPk = sqlite3PrimaryKeyIndex(pTab); - pParse->nMem = 6; - sqlite3CodeVerifySchema(pParse, iDb); + pParse->nMem = 7; + sqlite3CodeVerifySchema(pParse, iTabDb); sqlite3ViewGetColumnNames(pParse, pTab); for(i=0, pCol=pTab->aCol; inCol; i++, pCol++){ - if( IsHiddenColumn(pCol) ){ - nHidden++; - continue; + int isHidden = 0; + if( pCol->colFlags & COLFLAG_NOINSERT ){ + if( pPragma->iArg==0 ){ + nHidden++; + continue; + } + if( pCol->colFlags & COLFLAG_VIRTUAL ){ + isHidden = 2; /* GENERATED ALWAYS AS ... VIRTUAL */ + }else if( pCol->colFlags & COLFLAG_STORED ){ + isHidden = 3; /* GENERATED ALWAYS AS ... STORED */ + }else{ assert( pCol->colFlags & COLFLAG_HIDDEN ); + isHidden = 1; /* HIDDEN */ + } } if( (pCol->colFlags & COLFLAG_PRIMKEY)==0 ){ k = 0; @@ -119002,14 +126512,15 @@ SQLITE_PRIVATE void sqlite3Pragma( }else{ for(k=1; k<=pTab->nCol && pPk->aiColumn[k-1]!=i; k++){} } - assert( pCol->pDflt==0 || pCol->pDflt->op==TK_SPAN ); - sqlite3VdbeMultiLoad(v, 1, "issisi", + assert( pCol->pDflt==0 || pCol->pDflt->op==TK_SPAN || isHidden>=2 ); + sqlite3VdbeMultiLoad(v, 1, pPragma->iArg ? "issisii" : "issisi", i-nHidden, pCol->zName, sqlite3ColumnType(pCol,""), pCol->notNull ? 1 : 0, - pCol->pDflt ? pCol->pDflt->u.zToken : 0, - k); + pCol->pDflt && isHidden<2 ? pCol->pDflt->u.zToken : 0, + k, + isHidden); } } } @@ -119046,7 +126557,17 @@ SQLITE_PRIVATE void sqlite3Pragma( Index *pIdx; Table *pTab; pIdx = sqlite3FindIndex(db, zRight, zDb); + if( pIdx==0 ){ + /* If there is no index named zRight, check to see if there is a + ** WITHOUT ROWID table named zRight, and if there is, show the + ** structure of the PRIMARY KEY index for that table. */ + pTab = sqlite3LocateTable(pParse, LOCATE_NOERR, zRight, zDb); + if( pTab && !HasRowid(pTab) ){ + pIdx = sqlite3PrimaryKeyIndex(pTab); + } + } if( pIdx ){ + int iIdxDb = sqlite3SchemaToIndex(db, pIdx->pSchema); int i; int mx; if( pPragma->iArg ){ @@ -119059,7 +126580,7 @@ SQLITE_PRIVATE void sqlite3Pragma( pParse->nMem = 3; } pTab = pIdx->pTable; - sqlite3CodeVerifySchema(pParse, iDb); + sqlite3CodeVerifySchema(pParse, iIdxDb); assert( pParse->nMem<=pPragma->nPragCName ); for(i=0; iaiColumn[i]; @@ -119083,8 +126604,9 @@ SQLITE_PRIVATE void sqlite3Pragma( int i; pTab = sqlite3FindTable(db, zRight, zDb); if( pTab ){ + int iTabDb = sqlite3SchemaToIndex(db, pTab->pSchema); pParse->nMem = 5; - sqlite3CodeVerifySchema(pParse, iDb); + sqlite3CodeVerifySchema(pParse, iTabDb); for(pIdx=pTab->pIndex, i=0; pIdx; pIdx=pIdx->pNext, i++){ const char *azOrigin[] = { "c", "u", "pk" }; sqlite3VdbeMultiLoad(v, 1, "isisi", @@ -119123,20 +126645,21 @@ SQLITE_PRIVATE void sqlite3Pragma( } break; -#ifdef SQLITE_INTROSPECTION_PRAGMAS +#ifndef SQLITE_OMIT_INTROSPECTION_PRAGMAS case PragTyp_FUNCTION_LIST: { int i; HashElem *j; FuncDef *p; - pParse->nMem = 2; + int showInternFunc = (db->mDbFlags & DBFLAG_InternalFunc)!=0; + pParse->nMem = 6; for(i=0; iu.pHash ){ - sqlite3VdbeMultiLoad(v, 1, "si", p->zName, 1); + pragmaFunclistLine(v, p, 1, showInternFunc); } } for(j=sqliteHashFirst(&db->aFunc); j; j=sqliteHashNext(j)){ p = (FuncDef*)sqliteHashData(j); - sqlite3VdbeMultiLoad(v, 1, "si", p->zName, 0); + pragmaFunclistLine(v, p, 0, showInternFunc); } } break; @@ -119172,9 +126695,10 @@ SQLITE_PRIVATE void sqlite3Pragma( if( pTab ){ pFK = pTab->pFKey; if( pFK ){ + int iTabDb = sqlite3SchemaToIndex(db, pTab->pSchema); int i = 0; pParse->nMem = 8; - sqlite3CodeVerifySchema(pParse, iDb); + sqlite3CodeVerifySchema(pParse, iTabDb); while(pFK){ int j; for(j=0; jnCol; j++){ @@ -119219,9 +126743,9 @@ SQLITE_PRIVATE void sqlite3Pragma( pParse->nMem += 4; regKey = ++pParse->nMem; regRow = ++pParse->nMem; - sqlite3CodeVerifySchema(pParse, iDb); k = sqliteHashFirst(&db->aDb[iDb].pSchema->tblHash); while( k ){ + int iTabDb; if( zRight ){ pTab = sqlite3LocateTable(pParse, 0, zRight, zDb); k = 0; @@ -119230,21 +126754,23 @@ SQLITE_PRIVATE void sqlite3Pragma( k = sqliteHashNext(k); } if( pTab==0 || pTab->pFKey==0 ) continue; - sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName); + iTabDb = sqlite3SchemaToIndex(db, pTab->pSchema); + sqlite3CodeVerifySchema(pParse, iTabDb); + sqlite3TableLock(pParse, iTabDb, pTab->tnum, 0, pTab->zName); if( pTab->nCol+regRow>pParse->nMem ) pParse->nMem = pTab->nCol + regRow; - sqlite3OpenTable(pParse, 0, iDb, pTab, OP_OpenRead); + sqlite3OpenTable(pParse, 0, iTabDb, pTab, OP_OpenRead); sqlite3VdbeLoadString(v, regResult, pTab->zName); for(i=1, pFK=pTab->pFKey; pFK; i++, pFK=pFK->pNextFrom){ pParent = sqlite3FindTable(db, pFK->zTo, zDb); if( pParent==0 ) continue; pIdx = 0; - sqlite3TableLock(pParse, iDb, pParent->tnum, 0, pParent->zName); + sqlite3TableLock(pParse, iTabDb, pParent->tnum, 0, pParent->zName); x = sqlite3FkLocateIndex(pParse, pParent, pFK, &pIdx, 0); if( x==0 ){ if( pIdx==0 ){ - sqlite3OpenTable(pParse, i, iDb, pParent, OP_OpenRead); + sqlite3OpenTable(pParse, i, iTabDb, pParent, OP_OpenRead); }else{ - sqlite3VdbeAddOp3(v, OP_OpenRead, i, pIdx->tnum, iDb); + sqlite3VdbeAddOp3(v, OP_OpenRead, i, pIdx->tnum, iTabDb); sqlite3VdbeSetP4KeyInfo(pParse, pIdx); } }else{ @@ -119264,7 +126790,7 @@ SQLITE_PRIVATE void sqlite3Pragma( x = sqlite3FkLocateIndex(pParse, pParent, pFK, &pIdx, &aiCols); assert( x==0 ); } - addrOk = sqlite3VdbeMakeLabel(v); + addrOk = sqlite3VdbeMakeLabel(pParse); /* Generate code to read the child key values into registers ** regRow..regRow+n. If any of the child key values are NULL, this @@ -119309,19 +126835,7 @@ SQLITE_PRIVATE void sqlite3Pragma( #endif /* !defined(SQLITE_OMIT_TRIGGER) */ #endif /* !defined(SQLITE_OMIT_FOREIGN_KEY) */ -#ifndef NDEBUG - case PragTyp_PARSER_TRACE: { - if( zRight ){ - if( sqlite3GetBoolean(zRight, 0) ){ - sqlite3ParserTrace(stdout, "parser: "); - }else{ - sqlite3ParserTrace(0, 0); - } - } - } - break; -#endif - +#ifndef SQLITE_OMIT_CASE_SENSITIVE_LIKE_PRAGMA /* Reinstall the LIKE and GLOB functions. The variant of LIKE ** used will be case sensitive or not depending on the RHS. */ @@ -119331,6 +126845,7 @@ SQLITE_PRIVATE void sqlite3Pragma( } } break; +#endif /* SQLITE_OMIT_CASE_SENSITIVE_LIKE_PRAGMA */ #ifndef SQLITE_INTEGRITY_CHECK_ERROR_MAX # define SQLITE_INTEGRITY_CHECK_ERROR_MAX 100 @@ -119447,7 +126962,6 @@ SQLITE_PRIVATE void sqlite3Pragma( if( pTab->tnum<1 ) continue; /* Skip VIEWs or VIRTUAL TABLEs */ pPk = HasRowid(pTab) ? 0 : sqlite3PrimaryKeyIndex(pTab); - sqlite3ExprCacheClear(pParse); sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenRead, 0, 1, 0, &iDataCur, &iIdxCur); /* reg[7] counts the number of entries in the table. @@ -119461,6 +126975,11 @@ SQLITE_PRIVATE void sqlite3Pragma( assert( sqlite3NoTempsInRange(pParse,1,7+j) ); sqlite3VdbeAddOp2(v, OP_Rewind, iDataCur, 0); VdbeCoverage(v); loopTop = sqlite3VdbeAddOp2(v, OP_AddImm, 7, 1); + if( !isQuick ){ + /* Sanity check on record header decoding */ + sqlite3VdbeAddOp3(v, OP_Column, iDataCur, pTab->nNVCol-1,3); + sqlite3VdbeChangeP5(v, OPFLAG_TYPEOFARG); + } /* Verify that all NOT NULL columns really are NOT NULL */ for(j=0; jnCol; j++){ char *zErr; @@ -119468,7 +126987,9 @@ SQLITE_PRIVATE void sqlite3Pragma( if( j==pTab->iPKey ) continue; if( pTab->aCol[j].notNull==0 ) continue; sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, j, 3); - sqlite3VdbeChangeP5(v, OPFLAG_TYPEOFARG); + if( sqlite3VdbeGetOp(v,-1)->opcode==OP_Column ){ + sqlite3VdbeChangeP5(v, OPFLAG_TYPEOFARG); + } jmp2 = sqlite3VdbeAddOp1(v, OP_NotNull, 3); VdbeCoverage(v); zErr = sqlite3MPrintf(db, "NULL value in %s.%s", pTab->zName, pTab->aCol[j].zName); @@ -119480,12 +127001,11 @@ SQLITE_PRIVATE void sqlite3Pragma( if( pTab->pCheck && (db->flags & SQLITE_IgnoreChecks)==0 ){ ExprList *pCheck = sqlite3ExprListDup(db, pTab->pCheck, 0); if( db->mallocFailed==0 ){ - int addrCkFault = sqlite3VdbeMakeLabel(v); - int addrCkOk = sqlite3VdbeMakeLabel(v); + int addrCkFault = sqlite3VdbeMakeLabel(pParse); + int addrCkOk = sqlite3VdbeMakeLabel(pParse); char *zErr; int k; pParse->iSelfTab = iDataCur + 1; - sqlite3ExprCachePush(pParse); for(k=pCheck->nExpr-1; k>0; k--){ sqlite3ExprIfFalse(pParse, pCheck->a[k].pExpr, addrCkFault, 0); } @@ -119498,18 +127018,14 @@ SQLITE_PRIVATE void sqlite3Pragma( sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, zErr, P4_DYNAMIC); integrityCheckResultRow(v); sqlite3VdbeResolveLabel(v, addrCkOk); - sqlite3ExprCachePop(pParse); } sqlite3ExprListDelete(db, pCheck); } if( !isQuick ){ /* Omit the remaining tests for quick_check */ - /* Sanity check on record header decoding */ - sqlite3VdbeAddOp3(v, OP_Column, iDataCur, pTab->nCol-1, 3); - sqlite3VdbeChangeP5(v, OPFLAG_TYPEOFARG); /* Validate index entries for the current row */ for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){ int jmp2, jmp3, jmp4, jmp5; - int ckUniq = sqlite3VdbeMakeLabel(v); + int ckUniq = sqlite3VdbeMakeLabel(pParse); if( pPk==pIdx ) continue; r1 = sqlite3GenerateIndexKey(pParse, pIdx, iDataCur, 0, 0, &jmp3, pPrior, r1); @@ -119530,7 +127046,7 @@ SQLITE_PRIVATE void sqlite3Pragma( ** current key. The entry is unique if (1) any column is NULL ** or (2) the next entry has a different key */ if( IsUniqueIndex(pIdx) ){ - int uniqOk = sqlite3VdbeMakeLabel(v); + int uniqOk = sqlite3VdbeMakeLabel(pParse); int jmp6; int kk; for(kk=0; kknKeyCol; kk++){ @@ -119555,7 +127071,6 @@ SQLITE_PRIVATE void sqlite3Pragma( } sqlite3VdbeAddOp2(v, OP_Next, iDataCur, loopTop); VdbeCoverage(v); sqlite3VdbeJumpHere(v, loopTop-1); -#ifndef SQLITE_OMIT_BTREECOUNT if( !isQuick ){ sqlite3VdbeLoadString(v, 2, "wrong # of entries in index "); for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){ @@ -119569,7 +127084,6 @@ SQLITE_PRIVATE void sqlite3Pragma( sqlite3VdbeJumpHere(v, addr); } } -#endif /* SQLITE_OMIT_BTREECOUNT */ } } { @@ -119650,14 +127164,12 @@ SQLITE_PRIVATE void sqlite3Pragma( ** will be overwritten when the schema is next loaded. If it does not ** already exists, it will be created to use the new encoding value. */ - if( - !(DbHasProperty(db, 0, DB_SchemaLoaded)) || - DbHasProperty(db, 0, DB_Empty) - ){ + if( (db->mDbFlags & DBFLAG_EncodingFixed)==0 ){ for(pEnc=&encnames[0]; pEnc->zName; pEnc++){ if( 0==sqlite3StrICmp(zRight, pEnc->zName) ){ - SCHEMA_ENC(db) = ENC(db) = - pEnc->enc ? pEnc->enc : SQLITE_UTF16NATIVE; + u8 enc = pEnc->enc ? pEnc->enc : SQLITE_UTF16NATIVE; + SCHEMA_ENC(db) = enc; + sqlite3SetTextEncoding(db, enc); break; } } @@ -119966,6 +127478,27 @@ SQLITE_PRIVATE void sqlite3Pragma( break; } + /* + ** PRAGMA hard_heap_limit + ** PRAGMA hard_heap_limit = N + ** + ** Invoke sqlite3_hard_heap_limit64() to query or set the hard heap + ** limit. The hard heap limit can be activated or lowered by this + ** pragma, but not raised or deactivated. Only the + ** sqlite3_hard_heap_limit64() C-language API can raise or deactivate + ** the hard heap limit. This allows an application to set a heap limit + ** constraint that cannot be relaxed by an untrusted SQL script. + */ + case PragTyp_HARD_HEAP_LIMIT: { + sqlite3_int64 N; + if( zRight && sqlite3DecOrHexToI64(zRight, &N)==SQLITE_OK ){ + sqlite3_int64 iPrior = sqlite3_hard_heap_limit64(-1); + if( N>0 && (iPrior==0 || iPrior>N) ) sqlite3_hard_heap_limit64(N); + } + returnSingleInt(v, sqlite3_hard_heap_limit64(-1)); + break; + } + /* ** PRAGMA threads ** PRAGMA threads = N @@ -119985,6 +127518,25 @@ SQLITE_PRIVATE void sqlite3Pragma( break; } + /* + ** PRAGMA analysis_limit + ** PRAGMA analysis_limit = N + ** + ** Configure the maximum number of rows that ANALYZE will examine + ** in each index that it looks at. Return the new limit. + */ + case PragTyp_ANALYSIS_LIMIT: { + sqlite3_int64 N; + if( zRight + && sqlite3DecOrHexToI64(zRight, &N)==SQLITE_OK + && N>=0 + ){ + db->nAnalysisLimit = (int)(N&0x7fffffff); + } + returnSingleInt(v, db->nAnalysisLimit); + break; + } + #if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) /* ** Report the current state of file logs for all databases @@ -120013,45 +127565,11 @@ SQLITE_PRIVATE void sqlite3Pragma( } #endif -#ifdef SQLITE_HAS_CODEC - case PragTyp_KEY: { - if( zRight ) sqlite3_key_v2(db, zDb, zRight, sqlite3Strlen30(zRight)); - break; - } - case PragTyp_REKEY: { - if( zRight ) sqlite3_rekey_v2(db, zDb, zRight, sqlite3Strlen30(zRight)); - break; - } - case PragTyp_HEXKEY: { - if( zRight ){ - u8 iByte; - int i; - char zKey[40]; - for(i=0, iByte=0; izName); - cSep = ','; i++; } j = 0; @@ -120362,7 +127879,8 @@ static const sqlite3_module pragmaVtabModule = { 0, /* xRename - rename the table */ 0, /* xSavepoint */ 0, /* xRelease */ - 0 /* xRollbackTo */ + 0, /* xRollbackTo */ + 0 /* xShadowName */ }; /* @@ -120413,17 +127931,50 @@ static void corruptSchema( const char *zExtra /* Error information */ ){ sqlite3 *db = pData->db; - if( !db->mallocFailed && (db->flags & SQLITE_WriteSchema)==0 ){ + if( db->mallocFailed ){ + pData->rc = SQLITE_NOMEM_BKPT; + }else if( pData->pzErrMsg[0]!=0 ){ + /* A error message has already been generated. Do not overwrite it */ + }else if( pData->mInitFlags & INITFLAG_AlterTable ){ + *pData->pzErrMsg = sqlite3DbStrDup(db, zExtra); + pData->rc = SQLITE_ERROR; + }else if( db->flags & SQLITE_WriteSchema ){ + pData->rc = SQLITE_CORRUPT_BKPT; + }else{ char *z; if( zObj==0 ) zObj = "?"; z = sqlite3MPrintf(db, "malformed database schema (%s)", zObj); if( zExtra && zExtra[0] ) z = sqlite3MPrintf(db, "%z - %s", z, zExtra); - sqlite3DbFree(db, *pData->pzErrMsg); *pData->pzErrMsg = z; + pData->rc = SQLITE_CORRUPT_BKPT; } - pData->rc = db->mallocFailed ? SQLITE_NOMEM_BKPT : SQLITE_CORRUPT_BKPT; } +/* +** Check to see if any sibling index (another index on the same table) +** of pIndex has the same root page number, and if it does, return true. +** This would indicate a corrupt schema. +*/ +SQLITE_PRIVATE int sqlite3IndexHasDuplicateRootPage(Index *pIndex){ + Index *p; + for(p=pIndex->pTable->pIndex; p; p=p->pNext){ + if( p->tnum==pIndex->tnum && p!=pIndex ) return 1; + } + return 0; +} + +/* forward declaration */ +static int sqlite3Prepare( + sqlite3 *db, /* Database handle. */ + const char *zSql, /* UTF-8 encoded SQL statement. */ + int nBytes, /* Length of zSql in bytes. */ + u32 prepFlags, /* Zero or more SQLITE_PREPARE_* flags */ + Vdbe *pReprepare, /* VM being reprepared */ + sqlite3_stmt **ppStmt, /* OUT: A pointer to the prepared statement */ + const char **pzTail /* OUT: End of parsed string */ +); + + /* ** This is the callback routine for the code that initializes the ** database. See sqlite3Init() below for additional information. @@ -120431,9 +127982,11 @@ static void corruptSchema( ** ** Each callback contains the following information: ** -** argv[0] = name of thing being created -** argv[1] = root page number for table or index. 0 for trigger or view. -** argv[2] = SQL text for the CREATE statement. +** argv[0] = type of object: "table", "index", "trigger", or "view". +** argv[1] = name of thing being created +** argv[2] = associated table if an index or trigger +** argv[3] = root page number for table or index. 0 for trigger or view. +** argv[4] = SQL text for the CREATE statement. ** */ SQLITE_PRIVATE int sqlite3InitCallback(void *pInit, int argc, char **argv, char **NotUsed){ @@ -120441,20 +127994,21 @@ SQLITE_PRIVATE int sqlite3InitCallback(void *pInit, int argc, char **argv, char sqlite3 *db = pData->db; int iDb = pData->iDb; - assert( argc==3 ); + assert( argc==5 ); UNUSED_PARAMETER2(NotUsed, argc); assert( sqlite3_mutex_held(db->mutex) ); - DbClearProperty(db, iDb, DB_Empty); + db->mDbFlags |= DBFLAG_EncodingFixed; + pData->nInitRow++; if( db->mallocFailed ){ - corruptSchema(pData, argv[0], 0); + corruptSchema(pData, argv[1], 0); return 1; } assert( iDb>=0 && iDbnDb ); if( argv==0 ) return 0; /* Might happen if EMPTY_RESULT_CALLBACKS are on */ - if( argv[1]==0 ){ - corruptSchema(pData, argv[0], 0); - }else if( sqlite3_strnicmp(argv[2],"create ",7)==0 ){ + if( argv[3]==0 ){ + corruptSchema(pData, argv[1], 0); + }else if( sqlite3_strnicmp(argv[4],"create ",7)==0 ){ /* Call the parser to process a CREATE TABLE, INDEX or VIEW. ** But because db->init.busy is set to 1, no VDBE code is generated ** or executed. All the parser does is build the internal data @@ -120467,28 +128021,30 @@ SQLITE_PRIVATE int sqlite3InitCallback(void *pInit, int argc, char **argv, char assert( db->init.busy ); db->init.iDb = iDb; - db->init.newTnum = sqlite3Atoi(argv[1]); + db->init.newTnum = sqlite3Atoi(argv[3]); db->init.orphanTrigger = 0; - TESTONLY(rcp = ) sqlite3_prepare(db, argv[2], -1, &pStmt, 0); + db->init.azInit = argv; + pStmt = 0; + TESTONLY(rcp = ) sqlite3Prepare(db, argv[4], -1, 0, 0, &pStmt, 0); rc = db->errCode; assert( (rc&0xFF)==(rcp&0xFF) ); db->init.iDb = saved_iDb; - assert( saved_iDb==0 || (db->mDbFlags & DBFLAG_Vacuum)!=0 ); + /* assert( saved_iDb==0 || (db->mDbFlags & DBFLAG_Vacuum)!=0 ); */ if( SQLITE_OK!=rc ){ if( db->init.orphanTrigger ){ assert( iDb==1 ); }else{ - pData->rc = rc; + if( rc > pData->rc ) pData->rc = rc; if( rc==SQLITE_NOMEM ){ sqlite3OomFault(db); }else if( rc!=SQLITE_INTERRUPT && (rc&0xFF)!=SQLITE_LOCKED ){ - corruptSchema(pData, argv[0], sqlite3_errmsg(db)); + corruptSchema(pData, argv[1], sqlite3_errmsg(db)); } } } sqlite3_finalize(pStmt); - }else if( argv[0]==0 || (argv[2]!=0 && argv[2][0]!=0) ){ - corruptSchema(pData, argv[0], 0); + }else if( argv[1]==0 || (argv[4]!=0 && argv[4][0]!=0) ){ + corruptSchema(pData, argv[1], 0); }else{ /* If the SQL column is blank it means this is an index that ** was created to be the PRIMARY KEY or to fulfill a UNIQUE @@ -120497,16 +128053,13 @@ SQLITE_PRIVATE int sqlite3InitCallback(void *pInit, int argc, char **argv, char ** to do here is record the root page number for that index. */ Index *pIndex; - pIndex = sqlite3FindIndex(db, argv[0], db->aDb[iDb].zDbSName); - if( pIndex==0 ){ - /* This can occur if there exists an index on a TEMP table which - ** has the same name as another index on a permanent index. Since - ** the permanent table is hidden by the TEMP table, we can also - ** safely ignore the index on the permanent table. - */ - /* Do Nothing */; - }else if( sqlite3GetInt32(argv[1], &pIndex->tnum)==0 ){ - corruptSchema(pData, argv[0], "invalid rootpage"); + pIndex = sqlite3FindIndex(db, argv[1], db->aDb[iDb].zDbSName); + if( pIndex==0 + || sqlite3GetInt32(argv[3],&pIndex->tnum)==0 + || pIndex->tnum<2 + || sqlite3IndexHasDuplicateRootPage(pIndex) + ){ + corruptSchema(pData, argv[1], pIndex?"invalid rootpage":"orphan index"); } } return 0; @@ -120520,18 +128073,19 @@ SQLITE_PRIVATE int sqlite3InitCallback(void *pInit, int argc, char **argv, char ** auxiliary databases. Return one of the SQLITE_ error codes to ** indicate success or failure. */ -static int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg){ +SQLITE_PRIVATE int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg, u32 mFlags){ int rc; int i; #ifndef SQLITE_OMIT_DEPRECATED int size; #endif Db *pDb; - char const *azArg[4]; + char const *azArg[6]; int meta[5]; InitData initData; const char *zMasterName; int openedTransaction = 0; + int mask = ((db->mDbFlags & DBFLAG_EncodingFixed) | ~DBFLAG_EncodingFixed); assert( (db->mDbFlags & DBFLAG_SchemaKnownOk)==0 ); assert( iDb>=0 && iDbnDb ); @@ -120546,16 +128100,21 @@ static int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg){ ** table name will be inserted automatically by the parser so we can just ** use the abbreviation "x" here. The parser will also automatically tag ** the schema table as read-only. */ - azArg[0] = zMasterName = SCHEMA_TABLE(iDb); - azArg[1] = "1"; - azArg[2] = "CREATE TABLE x(type text,name text,tbl_name text," + azArg[0] = "table"; + azArg[1] = zMasterName = SCHEMA_TABLE(iDb); + azArg[2] = azArg[1]; + azArg[3] = "1"; + azArg[4] = "CREATE TABLE x(type text,name text,tbl_name text," "rootpage int,sql text)"; - azArg[3] = 0; + azArg[5] = 0; initData.db = db; initData.iDb = iDb; initData.rc = SQLITE_OK; initData.pzErrMsg = pzErrMsg; - sqlite3InitCallback(&initData, 3, (char **)azArg, 0); + initData.mInitFlags = mFlags; + initData.nInitRow = 0; + sqlite3InitCallback(&initData, 5, (char **)azArg, 0); + db->mDbFlags &= mask; if( initData.rc ){ rc = initData.rc; goto error_out; @@ -120576,7 +128135,7 @@ static int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg){ ** will be closed before this function returns. */ sqlite3BtreeEnter(pDb->pBt); if( !sqlite3BtreeIsInReadTrans(pDb->pBt) ){ - rc = sqlite3BtreeBeginTrans(pDb->pBt, 0); + rc = sqlite3BtreeBeginTrans(pDb->pBt, 0, 0); if( rc!=SQLITE_OK ){ sqlite3SetString(pzErrMsg, db, sqlite3ErrStr(rc)); goto initone_error_out; @@ -120615,27 +128174,25 @@ static int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg){ ** as sqlite3.enc. */ if( meta[BTREE_TEXT_ENCODING-1] ){ /* text encoding */ - if( iDb==0 ){ -#ifndef SQLITE_OMIT_UTF16 + if( iDb==0 && (db->mDbFlags & DBFLAG_EncodingFixed)==0 ){ u8 encoding; +#ifndef SQLITE_OMIT_UTF16 /* If opening the main database, set ENC(db). */ encoding = (u8)meta[BTREE_TEXT_ENCODING-1] & 3; if( encoding==0 ) encoding = SQLITE_UTF8; - ENC(db) = encoding; #else - ENC(db) = SQLITE_UTF8; + encoding = SQLITE_UTF8; #endif + sqlite3SetTextEncoding(db, encoding); }else{ /* If opening an attached database, the encoding much match ENC(db) */ - if( meta[BTREE_TEXT_ENCODING-1]!=ENC(db) ){ + if( (meta[BTREE_TEXT_ENCODING-1] & 3)!=ENC(db) ){ sqlite3SetString(pzErrMsg, db, "attached databases must use the same" " text encoding as main database"); rc = SQLITE_ERROR; goto initone_error_out; } } - }else{ - DbSetProperty(db, iDb, DB_Empty); } pDb->pSchema->enc = ENC(db); @@ -120672,7 +128229,7 @@ static int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg){ ** indices that the user might have created. */ if( iDb==0 && meta[BTREE_FILE_FORMAT-1]>=4 ){ - db->flags &= ~SQLITE_LegacyFileFmt; + db->flags &= ~(u64)SQLITE_LegacyFileFmt; } /* Read the schema information out of the schema tables @@ -120681,7 +128238,7 @@ static int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg){ { char *zSql; zSql = sqlite3MPrintf(db, - "SELECT name, rootpage, sql FROM \"%w\".%s ORDER BY rowid", + "SELECT*FROM\"%w\".%s ORDER BY rowid", db->aDb[iDb].zDbSName, zMasterName); #ifndef SQLITE_OMIT_AUTHORIZATION { @@ -120706,8 +128263,8 @@ static int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg){ rc = SQLITE_NOMEM_BKPT; sqlite3ResetAllSchemasOfConnection(db); } - if( rc==SQLITE_OK || (db->flags&SQLITE_WriteSchema)){ - /* Black magic: If the SQLITE_WriteSchema flag is set, then consider + if( rc==SQLITE_OK || (db->flags&SQLITE_NoSchemaError)){ + /* Black magic: If the SQLITE_NoSchemaError flag is set, then consider ** the schema loaded, even if errors occurred. In this situation the ** current sqlite3_prepare() operation will fail, but the following one ** will attempt to compile the supplied statement against whatever subset @@ -120747,8 +128304,7 @@ static int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg){ ** error occurs, write an error message into *pzErrMsg. ** ** After a database is initialized, the DB_SchemaLoaded bit is set -** bit is set in the flags field of the Db structure. If the database -** file was of zero-length, then the DB_Empty flag is also set. +** bit is set in the flags field of the Db structure. */ SQLITE_PRIVATE int sqlite3Init(sqlite3 *db, char **pzErrMsg){ int i, rc; @@ -120761,14 +128317,14 @@ SQLITE_PRIVATE int sqlite3Init(sqlite3 *db, char **pzErrMsg){ assert( db->nDb>0 ); /* Do the main schema first */ if( !DbHasProperty(db, 0, DB_SchemaLoaded) ){ - rc = sqlite3InitOne(db, 0, pzErrMsg); + rc = sqlite3InitOne(db, 0, pzErrMsg, 0); if( rc ) return rc; } /* All other schemas after the main schema. The "temp" schema must be last */ for(i=db->nDb-1; i>0; i--){ assert( i==1 || sqlite3BtreeHoldsMutex(db->aDb[i].pBt) ); if( !DbHasProperty(db, i, DB_SchemaLoaded) ){ - rc = sqlite3InitOne(db, i, pzErrMsg); + rc = sqlite3InitOne(db, i, pzErrMsg, 0); if( rc ) return rc; } } @@ -120821,7 +128377,7 @@ static void schemaIsValid(Parse *pParse){ ** on the b-tree database, open one now. If a transaction is opened, it ** will be closed immediately after reading the meta-value. */ if( !sqlite3BtreeIsInReadTrans(pBt) ){ - rc = sqlite3BtreeBeginTrans(pBt, 0); + rc = sqlite3BtreeBeginTrans(pBt, 0, 0); if( rc==SQLITE_NOMEM || rc==SQLITE_IOERR_NOMEM ){ sqlite3OomFault(db); } @@ -120889,6 +128445,7 @@ SQLITE_PRIVATE void sqlite3ParserReset(Parse *pParse){ if( db ){ assert( db->lookaside.bDisable >= pParse->disableLookaside ); db->lookaside.bDisable -= pParse->disableLookaside; + db->lookaside.sz = db->lookaside.bDisable ? 0 : db->lookaside.szTrue; } pParse->disableLookaside = 0; } @@ -120922,8 +128479,9 @@ static int sqlite3Prepare( */ if( prepFlags & SQLITE_PREPARE_PERSISTENT ){ sParse.disableLookaside++; - db->lookaside.bDisable++; + DisableLookaside; } + sParse.disableVtab = (prepFlags & SQLITE_PREPARE_NO_VTAB)!=0; /* Check to verify that it is possible to get a read lock on all ** database schemas. The inability to get a read lock indicates that @@ -120948,16 +128506,18 @@ static int sqlite3Prepare( ** but it does *not* override schema lock detection, so this all still ** works even if READ_UNCOMMITTED is set. */ - for(i=0; inDb; i++) { - Btree *pBt = db->aDb[i].pBt; - if( pBt ){ - assert( sqlite3BtreeHoldsMutex(pBt) ); - rc = sqlite3BtreeSchemaLocked(pBt); - if( rc ){ - const char *zDb = db->aDb[i].zDbSName; - sqlite3ErrorWithMsg(db, rc, "database schema is locked: %s", zDb); - testcase( db->flags & SQLITE_ReadUncommit ); - goto end_prepare; + if( !db->noSharedCache ){ + for(i=0; inDb; i++) { + Btree *pBt = db->aDb[i].pBt; + if( pBt ){ + assert( sqlite3BtreeHoldsMutex(pBt) ); + rc = sqlite3BtreeSchemaLocked(pBt); + if( rc ){ + const char *zDb = db->aDb[i].zDbSName; + sqlite3ErrorWithMsg(db, rc, "database schema is locked: %s", zDb); + testcase( db->flags & SQLITE_ReadUncommit ); + goto end_prepare; + } } } } @@ -120988,46 +128548,25 @@ static int sqlite3Prepare( } assert( 0==sParse.nQueryLoop ); - if( sParse.rc==SQLITE_DONE ) sParse.rc = SQLITE_OK; + if( sParse.rc==SQLITE_DONE ){ + sParse.rc = SQLITE_OK; + } if( sParse.checkSchema ){ schemaIsValid(&sParse); } - if( db->mallocFailed ){ - sParse.rc = SQLITE_NOMEM_BKPT; - } if( pzTail ){ *pzTail = sParse.zTail; } - rc = sParse.rc; - -#ifndef SQLITE_OMIT_EXPLAIN - if( rc==SQLITE_OK && sParse.pVdbe && sParse.explain ){ - static const char * const azColName[] = { - "addr", "opcode", "p1", "p2", "p3", "p4", "p5", "comment", - "id", "parent", "notused", "detail" - }; - int iFirst, mx; - if( sParse.explain==2 ){ - sqlite3VdbeSetNumCols(sParse.pVdbe, 4); - iFirst = 8; - mx = 12; - }else{ - sqlite3VdbeSetNumCols(sParse.pVdbe, 8); - iFirst = 0; - mx = 8; - } - for(i=iFirst; iinit.busy==0 ){ sqlite3VdbeSetSql(sParse.pVdbe, zSql, (int)(sParse.zTail-zSql), prepFlags); } - if( sParse.pVdbe && (rc!=SQLITE_OK || db->mallocFailed) ){ - sqlite3VdbeFinalize(sParse.pVdbe); + if( db->mallocFailed ){ + sParse.rc = SQLITE_NOMEM_BKPT; + } + rc = sParse.rc; + if( rc!=SQLITE_OK ){ + if( sParse.pVdbe ) sqlite3VdbeFinalize(sParse.pVdbe); assert(!(*ppStmt)); }else{ *ppStmt = (sqlite3_stmt*)sParse.pVdbe; @@ -121088,6 +128627,7 @@ static int sqlite3LockAndPrepare( return rc; } + /* ** Rerun the compilation of a statement after a schema change. ** @@ -121318,7 +128858,7 @@ SQLITE_API int sqlite3_prepare16_v3( /***/ int sqlite3SelectTrace = 0; # define SELECTTRACE(K,P,S,X) \ if(sqlite3SelectTrace&(K)) \ - sqlite3DebugPrintf("%s/%d/%p: ",(S)->zSelName,(P)->addrExplain,(S)),\ + sqlite3DebugPrintf("%u/%d/%p: ",(S)->selId,(P)->addrExplain,(S)),\ sqlite3DebugPrintf X #else # define SELECTTRACE(K,P,S,X) @@ -121365,8 +128905,8 @@ struct SortCtx { int labelBkOut; /* Start label for the block-output subroutine */ int addrSortIndex; /* Address of the OP_SorterOpen or OP_OpenEphemeral */ int labelDone; /* Jump here when done, ex: LIMIT reached */ + int labelOBLopt; /* Jump here when sorter is full */ u8 sortFlags; /* Zero or more SORTFLAG_* bits */ - u8 bOrderedInnerLoop; /* ORDER BY correctly sorts the inner loop */ #ifdef SQLITE_ENABLE_SORTER_REFERENCES u8 nDefer; /* Number of valid entries in aDefer[] */ struct DeferredCsr { @@ -121381,7 +128921,10 @@ struct SortCtx { /* ** Delete all the content of a Select structure. Deallocate the structure -** itself only if bFree is true. +** itself depending on the value of bFree +** +** If bFree==1, call sqlite3DbFree() on the p object. +** If bFree==0, Leave the first Select object unfreed */ static void clearSelect(sqlite3 *db, Select *p, int bFree){ while( p ){ @@ -121393,6 +128936,11 @@ static void clearSelect(sqlite3 *db, Select *p, int bFree){ sqlite3ExprDelete(db, p->pHaving); sqlite3ExprListDelete(db, p->pOrderBy); sqlite3ExprDelete(db, p->pLimit); +#ifndef SQLITE_OMIT_WINDOWFUNC + if( OK_IF_ALWAYS_TRUE(p->pWinDefn) ){ + sqlite3WindowListDelete(db, p->pWinDefn); + } +#endif if( OK_IF_ALWAYS_TRUE(p->pWith) ) sqlite3WithDelete(db, p->pWith); if( bFree ) sqlite3DbFreeNN(db, p); p = pPrior; @@ -121443,9 +128991,7 @@ SQLITE_PRIVATE Select *sqlite3SelectNew( pNew->selFlags = selFlags; pNew->iLimit = 0; pNew->iOffset = 0; -#if SELECTTRACE_ENABLED - pNew->zSelName[0] = 0; -#endif + pNew->selId = ++pParse->nSelect; pNew->addrOpenEphm[0] = -1; pNew->addrOpenEphm[1] = -1; pNew->nSelectRow = 0; @@ -121459,6 +129005,10 @@ SQLITE_PRIVATE Select *sqlite3SelectNew( pNew->pNext = 0; pNew->pLimit = pLimit; pNew->pWith = 0; +#ifndef SQLITE_OMIT_WINDOWFUNC + pNew->pWin = 0; + pNew->pWinDefn = 0; +#endif if( pParse->db->mallocFailed ) { clearSelect(pParse->db, pNew, pNew!=&standin); pNew = 0; @@ -121469,17 +129019,6 @@ SQLITE_PRIVATE Select *sqlite3SelectNew( return pNew; } -#if SELECTTRACE_ENABLED -/* -** Set the name of a Select object -*/ -SQLITE_PRIVATE void sqlite3SelectSetName(Select *p, const char *zName){ - if( p && zName ){ - sqlite3_snprintf(sizeof(p->zSelName), p->zSelName, "%s", zName); - } -} -#endif - /* ** Delete the given Select structure and all of its substructures. @@ -121488,6 +129027,21 @@ SQLITE_PRIVATE void sqlite3SelectDelete(sqlite3 *db, Select *p){ if( OK_IF_ALWAYS_TRUE(p) ) clearSelect(db, p, 1); } +/* +** Delete all the substructure for p, but keep p allocated. Redefine +** p to be a single SELECT where every column of the result set has a +** value of NULL. +*/ +SQLITE_PRIVATE void sqlite3SelectReset(Parse *pParse, Select *p){ + if( ALWAYS(p) ){ + clearSelect(pParse->db, p, 0); + memset(&p->iLimit, 0, sizeof(Select) - offsetof(Select,iLimit)); + p->pEList = sqlite3ExprListAppend(pParse, 0, + sqlite3ExprAlloc(pParse->db,TK_NULL,0,0)); + p->pSrc = sqlite3DbMallocZero(pParse->db, sizeof(SrcList)); + } +} + /* ** Return a pointer to the right-most SELECT statement in a compound. */ @@ -121596,7 +129150,8 @@ static int tableAndColumnIndex( int N, /* Number of tables in pSrc->a[] to search */ const char *zCol, /* Name of the column we are looking for */ int *piTab, /* Write index of pSrc->a[] here */ - int *piCol /* Write index of pSrc->a[*piTab].pTab->aCol[] here */ + int *piCol, /* Write index of pSrc->a[*piTab].pTab->aCol[] here */ + int bIgnoreHidden /* True to ignore hidden columns */ ){ int i; /* For looping over tables in pSrc */ int iCol; /* Index of column matching zCol */ @@ -121604,7 +129159,9 @@ static int tableAndColumnIndex( assert( (piTab==0)==(piCol==0) ); /* Both or neither are NULL */ for(i=0; ia[i].pTab, zCol); - if( iCol>=0 ){ + if( iCol>=0 + && (bIgnoreHidden==0 || IsHiddenColumn(&pSrc->a[i].pTab->aCol[iCol])==0) + ){ if( piTab ){ *piTab = i; *piCol = iCol; @@ -121656,7 +129213,7 @@ static void addWhereTerm( ExprSetVVAProperty(pEq, EP_NoReduce); pEq->iRightJoinTable = (i16)pE2->iTable; } - *ppWhere = sqlite3ExprAnd(db, *ppWhere, pEq); + *ppWhere = sqlite3ExprAnd(pParse, *ppWhere, pEq); } /* @@ -121685,7 +129242,7 @@ static void addWhereTerm( ** after the t1 loop and rows with t1.x!=5 will never appear in ** the output, which is incorrect. */ -static void setJoinExpr(Expr *p, int iTable){ +SQLITE_PRIVATE void sqlite3SetJoinExpr(Expr *p, int iTable){ while( p ){ ExprSetProperty(p, EP_FromJoin); assert( !ExprHasProperty(p, EP_TokenOnly|EP_Reduced) ); @@ -121694,15 +129251,15 @@ static void setJoinExpr(Expr *p, int iTable){ if( p->op==TK_FUNCTION && p->x.pList ){ int i; for(i=0; ix.pList->nExpr; i++){ - setJoinExpr(p->x.pList->a[i].pExpr, iTable); + sqlite3SetJoinExpr(p->x.pList->a[i].pExpr, iTable); } } - setJoinExpr(p->pLeft, iTable); + sqlite3SetJoinExpr(p->pLeft, iTable); p = p->pRight; } } -/* Undo the work of setJoinExpr(). In the expression tree p, convert every +/* Undo the work of sqlite3SetJoinExpr(). In the expression p, convert every ** term that is marked with EP_FromJoin and iRightJoinTable==iTable into ** an ordinary term that omits the EP_FromJoin mark. ** @@ -121769,10 +129326,11 @@ static int sqliteProcessJoin(Parse *pParse, Select *p){ int iLeft; /* Matching left table */ int iLeftCol; /* Matching column in the left table */ + if( IsHiddenColumn(&pRightTab->aCol[j]) ) continue; zName = pRightTab->aCol[j].zName; - if( tableAndColumnIndex(pSrc, i+1, zName, &iLeft, &iLeftCol) ){ + if( tableAndColumnIndex(pSrc, i+1, zName, &iLeft, &iLeftCol, 1) ){ addWhereTerm(pParse, pSrc, iLeft, iLeftCol, i+1, j, - isOuter, &p->pWhere); + isOuter, &p->pWhere); } } } @@ -121789,8 +129347,8 @@ static int sqliteProcessJoin(Parse *pParse, Select *p){ ** an AND operator. */ if( pRight->pOn ){ - if( isOuter ) setJoinExpr(pRight->pOn, pRight->iCursor); - p->pWhere = sqlite3ExprAnd(pParse->db, p->pWhere, pRight->pOn); + if( isOuter ) sqlite3SetJoinExpr(pRight->pOn, pRight->iCursor); + p->pWhere = sqlite3ExprAnd(pParse, p->pWhere, pRight->pOn); pRight->pOn = 0; } @@ -121812,7 +129370,7 @@ static int sqliteProcessJoin(Parse *pParse, Select *p){ zName = pList->a[j].zName; iRightCol = columnIndex(pRightTab, zName); if( iRightCol<0 - || !tableAndColumnIndex(pSrc, i+1, zName, &iLeft, &iLeftCol) + || !tableAndColumnIndex(pSrc, i+1, zName, &iLeft, &iLeftCol, 0) ){ sqlite3ErrorMsg(pParse, "cannot join using column %s - column " "not present in both tables", zName); @@ -121826,14 +129384,6 @@ static int sqliteProcessJoin(Parse *pParse, Select *p){ return 0; } -/* Forward reference */ -static KeyInfo *keyInfoFromExprList( - Parse *pParse, /* Parsing context */ - ExprList *pList, /* Form the KeyInfo object from this ExprList */ - int iStart, /* Begin with this column of pList */ - int nExtra /* Add this many extra columns to the end */ -); - /* ** An instance of this object holds information (beyond pParse and pSelect) ** needed to load the next result row that is to be added to the sorter. @@ -121940,7 +129490,7 @@ static void pushOntoSorter( } assert( pSelect->iOffset==0 || pSelect->iLimit!=0 ); iLimit = pSelect->iOffset ? pSelect->iOffset+1 : pSelect->iLimit; - pSort->labelDone = sqlite3VdbeMakeLabel(v); + pSort->labelDone = sqlite3VdbeMakeLabel(pParse); sqlite3ExprCodeExprList(pParse, pSort->pOrderBy, regBase, regOrigData, SQLITE_ECEL_DUP | (regOrigData? SQLITE_ECEL_REF : 0)); if( bSeq ){ @@ -121972,14 +129522,15 @@ static void pushOntoSorter( if( pParse->db->mallocFailed ) return; pOp->p2 = nKey + nData; pKI = pOp->p4.pKeyInfo; - memset(pKI->aSortOrder, 0, pKI->nKeyField); /* Makes OP_Jump testable */ + memset(pKI->aSortFlags, 0, pKI->nKeyField); /* Makes OP_Jump testable */ sqlite3VdbeChangeP4(v, -1, (char*)pKI, P4_KEYINFO); testcase( pKI->nAllField > pKI->nKeyField+2 ); - pOp->p4.pKeyInfo = keyInfoFromExprList(pParse, pSort->pOrderBy, nOBSat, + pOp->p4.pKeyInfo = sqlite3KeyInfoFromExprList(pParse,pSort->pOrderBy,nOBSat, pKI->nAllField-pKI->nKeyField-1); + pOp = 0; /* Ensure pOp not used after sqltie3VdbeAddOp3() */ addrJmp = sqlite3VdbeCurrentAddr(v); sqlite3VdbeAddOp3(v, OP_Jump, addrJmp+1, 0, addrJmp+1); VdbeCoverage(v); - pSort->labelBkOut = sqlite3VdbeMakeLabel(v); + pSort->labelBkOut = sqlite3VdbeMakeLabel(pParse); pSort->regReturn = ++pParse->nMem; sqlite3VdbeAddOp2(v, OP_Gosub, pSort->regReturn, pSort->labelBkOut); sqlite3VdbeAddOp1(v, OP_ResetSorter, pSort->iECursor); @@ -122002,10 +129553,10 @@ static void pushOntoSorter( ** than LIMIT+OFFSET items in the sorter. ** ** If the new record does not need to be inserted into the sorter, - ** jump to the next iteration of the loop. Or, if the - ** pSort->bOrderedInnerLoop flag is set to indicate that the inner - ** loop delivers items in sorted order, jump to the next iteration - ** of the outer loop. + ** jump to the next iteration of the loop. If the pSort->labelOBLopt + ** value is not zero, then it is a label of where to jump. Otherwise, + ** just bypass the row insert logic. See the header comment on the + ** sqlite3WhereOrderByLimitOptLabel() function for additional info. */ int iCsr = pSort->iECursor; sqlite3VdbeAddOp2(v, OP_IfNotZero, iLimit, sqlite3VdbeCurrentAddr(v)+4); @@ -122027,9 +129578,8 @@ static void pushOntoSorter( sqlite3VdbeAddOp4Int(v, op, pSort->iECursor, regRecord, regBase+nOBSat, nBase-nOBSat); if( iSkip ){ - assert( pSort->bOrderedInnerLoop==0 || pSort->bOrderedInnerLoop==1 ); sqlite3VdbeChangeP2(v, iSkip, - sqlite3VdbeCurrentAddr(v) + pSort->bOrderedInnerLoop); + pSort->labelOBLopt ? pSort->labelOBLopt : sqlite3VdbeCurrentAddr(v)); } } @@ -122113,7 +129663,7 @@ static void selectExprDefer( struct ExprList_item *pItem = &pEList->a[i]; if( pItem->u.x.iOrderByCol==0 ){ Expr *pExpr = pItem->pExpr; - Table *pTab = pExpr->pTab; + Table *pTab = pExpr->y.pTab; if( pExpr->op==TK_COLUMN && pExpr->iColumn>=0 && pTab && !IsVirtual(pTab) && (pTab->aCol[pExpr->iColumn].colFlags & COLFLAG_SORTERREF) ){ @@ -122136,12 +129686,12 @@ static void selectExprDefer( Expr *pNew = sqlite3PExpr(pParse, TK_COLUMN, 0, 0); if( pNew ){ pNew->iTable = pExpr->iTable; - pNew->pTab = pExpr->pTab; + pNew->y.pTab = pExpr->y.pTab; pNew->iColumn = pPk ? pPk->aiColumn[k] : -1; pExtra = sqlite3ExprListAppend(pParse, pExtra, pNew); } } - pSort->aDefer[nDefer].pTab = pExpr->pTab; + pSort->aDefer[nDefer].pTab = pExpr->y.pTab; pSort->aDefer[nDefer].iCsr = pExpr->iTable; pSort->aDefer[nDefer].nKey = nKey; nDefer++; @@ -122226,7 +129776,7 @@ static void selectInnerLoop( if( srcTab>=0 ){ for(i=0; ipEList->a[i].zName)); + VdbeComment((v, "%s", p->pEList->a[i].zEName)); } }else if( eDest!=SRT_Exists ){ #ifdef SQLITE_ENABLE_SORTER_REFERENCES @@ -122340,6 +129890,7 @@ static void selectInnerLoop( pOp->opcode = OP_Null; pOp->p1 = 1; pOp->p2 = regPrev; + pOp = 0; /* Ensure pOp is not used after sqlite3VdbeAddOp() */ iJump = sqlite3VdbeCurrentAddr(v) + nResultCol; for(i=0; izAffSdst)==nResultCol ); sqlite3VdbeAddOp4(v, OP_MakeRecord, regResult, nResultCol, r1, pDest->zAffSdst, nResultCol); - sqlite3ExprCacheAffinityChange(pParse, regResult, nResultCol); sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iParm, r1, regResult, nResultCol); sqlite3ReleaseTempReg(pParse, r1); } @@ -122502,7 +130052,6 @@ static void selectInnerLoop( sqlite3VdbeAddOp1(v, OP_Yield, pDest->iSDParm); }else{ sqlite3VdbeAddOp2(v, OP_ResultRow, regResult, nResultCol); - sqlite3ExprCacheAffinityChange(pParse, regResult, nResultCol); } break; } @@ -122586,7 +130135,7 @@ SQLITE_PRIVATE KeyInfo *sqlite3KeyInfoAlloc(sqlite3 *db, int N, int X){ int nExtra = (N+X)*(sizeof(CollSeq*)+1) - sizeof(CollSeq*); KeyInfo *p = sqlite3DbMallocRawNN(db, sizeof(KeyInfo) + nExtra); if( p ){ - p->aSortOrder = (u8*)&p->aColl[N+X]; + p->aSortFlags = (u8*)&p->aColl[N+X]; p->nKeyField = (u16)N; p->nAllField = (u16)(N+X); p->enc = ENC(db); @@ -122645,7 +130194,7 @@ SQLITE_PRIVATE int sqlite3KeyInfoIsWriteable(KeyInfo *p){ return p->nRef==1; } ** function is responsible for seeing that this structure is eventually ** freed. */ -static KeyInfo *keyInfoFromExprList( +SQLITE_PRIVATE KeyInfo *sqlite3KeyInfoFromExprList( Parse *pParse, /* Parsing context */ ExprList *pList, /* Form the KeyInfo object from this ExprList */ int iStart, /* Begin with this column of pList */ @@ -122663,7 +130212,7 @@ static KeyInfo *keyInfoFromExprList( assert( sqlite3KeyInfoIsWriteable(pInfo) ); for(i=iStart, pItem=pList->a+iStart; iaColl[i-iStart] = sqlite3ExprNNCollSeq(pParse, pItem->pExpr); - pInfo->aSortOrder[i-iStart] = pItem->sortOrder; + pInfo->aSortFlags[i-iStart] = pItem->sortFlags; } } return pInfo; @@ -122729,7 +130278,7 @@ static void generateSortTail( ){ Vdbe *v = pParse->pVdbe; /* The prepared statement */ int addrBreak = pSort->labelDone; /* Jump here to exit loop */ - int addrContinue = sqlite3VdbeMakeLabel(v); /* Jump here for next cycle */ + int addrContinue = sqlite3VdbeMakeLabel(pParse);/* Jump here for next cycle */ int addr; /* Top of output loop. Jump for Next. */ int addrOnce = 0; int iTab; @@ -122769,7 +130318,12 @@ static void generateSortTail( regRow = pDest->iSdst; }else{ regRowid = sqlite3GetTempReg(pParse); - regRow = sqlite3GetTempRange(pParse, nColumn); + if( eDest==SRT_EphemTab || eDest==SRT_Table ){ + regRow = sqlite3GetTempReg(pParse); + nColumn = 0; + }else{ + regRow = sqlite3GetTempRange(pParse, nColumn); + } } nKey = pOrderBy->nExpr - pSort->nOBSat; if( pSort->sortFlags & SORTFLAG_UseSorter ){ @@ -122843,12 +130397,13 @@ static void generateSortTail( iRead = iCol--; } sqlite3VdbeAddOp3(v, OP_Column, iSortTab, iRead, regRow+i); - VdbeComment((v, "%s", aOutEx[i].zName?aOutEx[i].zName : aOutEx[i].zSpan)); + VdbeComment((v, "%s", aOutEx[i].zEName)); } } switch( eDest ){ case SRT_Table: case SRT_EphemTab: { + sqlite3VdbeAddOp3(v, OP_Column, iSortTab, nKey+bSeq, regRow); sqlite3VdbeAddOp2(v, OP_NewRowid, iParm, regRowid); sqlite3VdbeAddOp3(v, OP_Insert, iParm, regRow, regRowid); sqlite3VdbeChangeP5(v, OPFLAG_APPEND); @@ -122859,7 +130414,6 @@ static void generateSortTail( assert( nColumn==sqlite3Strlen30(pDest->zAffSdst) ); sqlite3VdbeAddOp4(v, OP_MakeRecord, regRow, nColumn, regRowid, pDest->zAffSdst, nColumn); - sqlite3ExprCacheAffinityChange(pParse, regRow, nColumn); sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iParm, regRowid, regRow, nColumn); break; } @@ -122874,7 +130428,6 @@ static void generateSortTail( testcase( eDest==SRT_Coroutine ); if( eDest==SRT_Output ){ sqlite3VdbeAddOp2(v, OP_ResultRow, pDest->iSdst, nColumn); - sqlite3ExprCacheAffinityChange(pParse, pDest->iSdst, nColumn); }else{ sqlite3VdbeAddOp1(v, OP_Yield, pDest->iSDParm); } @@ -122951,8 +130504,6 @@ static const char *columnTypeImpl( assert( pExpr!=0 ); assert( pNC->pSrcList!=0 ); - assert( pExpr->op!=TK_AGG_COLUMN ); /* This routine runes before aggregates - ** are processed */ switch( pExpr->op ){ case TK_COLUMN: { /* The expression is a column. Locate the table the column is being @@ -122994,7 +130545,7 @@ static const char *columnTypeImpl( break; } - assert( pTab && pExpr->pTab==pTab ); + assert( pTab && pExpr->y.pTab==pTab ); if( pS ){ /* The "table" is actually a sub-select or a view in the FROM clause ** of the SELECT statement. Return the declaration type and origin @@ -123179,15 +130730,15 @@ static void generateColumnNames( assert( p!=0 ); assert( p->op!=TK_AGG_COLUMN ); /* Agg processing has not run yet */ - assert( p->op!=TK_COLUMN || p->pTab!=0 ); /* Covering idx not yet coded */ - if( pEList->a[i].zName ){ + assert( p->op!=TK_COLUMN || p->y.pTab!=0 ); /* Covering idx not yet coded */ + if( pEList->a[i].zEName && pEList->a[i].eEName==ENAME_NAME ){ /* An AS clause always takes first priority */ - char *zName = pEList->a[i].zName; + char *zName = pEList->a[i].zEName; sqlite3VdbeSetColName(v, i, COLNAME_NAME, zName, SQLITE_TRANSIENT); }else if( srcName && p->op==TK_COLUMN ){ char *zCol; int iCol = p->iColumn; - pTab = p->pTab; + pTab = p->y.pTab; assert( pTab!=0 ); if( iCol<0 ) iCol = pTab->iPKey; assert( iCol==-1 || (iCol>=0 && iColnCol) ); @@ -123204,7 +130755,7 @@ static void generateColumnNames( sqlite3VdbeSetColName(v, i, COLNAME_NAME, zCol, SQLITE_TRANSIENT); } }else{ - const char *z = pEList->a[i].zSpan; + const char *z = pEList->a[i].zEName; z = z==0 ? sqlite3MPrintf(db, "column%d", i+1) : sqlite3DbStrDup(db, z); sqlite3VdbeSetColName(v, i, COLNAME_NAME, z, SQLITE_DYNAMIC); } @@ -123266,19 +130817,18 @@ SQLITE_PRIVATE int sqlite3ColumnsFromExprList( for(i=0, pCol=aCol; imallocFailed; i++, pCol++){ /* Get an appropriate name for the column */ - if( (zName = pEList->a[i].zName)!=0 ){ + if( (zName = pEList->a[i].zEName)!=0 && pEList->a[i].eEName==ENAME_NAME ){ /* If the column contains an "AS " phrase, use as the name */ }else{ - Expr *pColExpr = sqlite3ExprSkipCollate(pEList->a[i].pExpr); + Expr *pColExpr = sqlite3ExprSkipCollateAndLikely(pEList->a[i].pExpr); while( pColExpr->op==TK_DOT ){ pColExpr = pColExpr->pRight; assert( pColExpr!=0 ); } - assert( pColExpr->op!=TK_AGG_COLUMN ); if( pColExpr->op==TK_COLUMN ){ /* For columns use the column name name */ int iCol = pColExpr->iColumn; - Table *pTab = pColExpr->pTab; + Table *pTab = pColExpr->y.pTab; assert( pTab!=0 ); if( iCol<0 ) iCol = pTab->iPKey; zName = iCol>=0 ? pTab->aCol[iCol].zName : "rowid"; @@ -123287,10 +130837,10 @@ SQLITE_PRIVATE int sqlite3ColumnsFromExprList( zName = pColExpr->u.zToken; }else{ /* Use the original text of the column expression as its name */ - zName = pEList->a[i].zSpan; + zName = pEList->a[i].zEName; } } - if( zName ){ + if( zName && !sqlite3IsTrueOrFalse(zName) ){ zName = sqlite3DbStrDup(db, zName); }else{ zName = sqlite3MPrintf(db,"column%d",i+1); @@ -123310,6 +130860,7 @@ SQLITE_PRIVATE int sqlite3ColumnsFromExprList( if( cnt>3 ) sqlite3_randomness(sizeof(cnt), &cnt); } pCol->zName = zName; + pCol->hName = sqlite3StrIHash(zName); sqlite3ColumnPropertiesFromName(0, pCol); if( zName && sqlite3HashInsert(&ht, zName, pCol)==pCol ){ sqlite3OomFault(db); @@ -123342,7 +130893,8 @@ SQLITE_PRIVATE int sqlite3ColumnsFromExprList( SQLITE_PRIVATE void sqlite3SelectAddColumnTypeAndCollation( Parse *pParse, /* Parsing contexts */ Table *pTab, /* Add column type information to this table */ - Select *pSelect /* SELECT used to determine types and collations */ + Select *pSelect, /* SELECT used to determine types and collations */ + char aff /* Default affinity for columns */ ){ sqlite3 *db = pParse->db; NameContext sNC; @@ -123375,7 +130927,7 @@ SQLITE_PRIVATE void sqlite3SelectAddColumnTypeAndCollation( pCol->colFlags |= COLFLAG_HASTYPE; } } - if( pCol->affinity==0 ) pCol->affinity = SQLITE_AFF_BLOB; + if( pCol->affinity<=SQLITE_AFF_NONE ) pCol->affinity = aff; pColl = sqlite3ExprCollSeq(pParse, p); if( pColl && pCol->zColl==0 ){ pCol->zColl = sqlite3DbStrDup(db, pColl->zName); @@ -123388,30 +130940,27 @@ SQLITE_PRIVATE void sqlite3SelectAddColumnTypeAndCollation( ** Given a SELECT statement, generate a Table structure that describes ** the result set of that SELECT. */ -SQLITE_PRIVATE Table *sqlite3ResultSetOfSelect(Parse *pParse, Select *pSelect){ +SQLITE_PRIVATE Table *sqlite3ResultSetOfSelect(Parse *pParse, Select *pSelect, char aff){ Table *pTab; sqlite3 *db = pParse->db; - int savedFlags; + u64 savedFlags; savedFlags = db->flags; - db->flags &= ~SQLITE_FullColNames; + db->flags &= ~(u64)SQLITE_FullColNames; db->flags |= SQLITE_ShortColNames; sqlite3SelectPrep(pParse, pSelect, 0); + db->flags = savedFlags; if( pParse->nErr ) return 0; while( pSelect->pPrior ) pSelect = pSelect->pPrior; - db->flags = savedFlags; pTab = sqlite3DbMallocZero(db, sizeof(Table) ); if( pTab==0 ){ return 0; } - /* The sqlite3ResultSetOfSelect() is only used n contexts where lookaside - ** is disabled */ - assert( db->lookaside.bDisable ); pTab->nTabRef = 1; pTab->zName = 0; pTab->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); sqlite3ColumnsFromExprList(pParse, pSelect->pEList, &pTab->nCol, &pTab->aCol); - sqlite3SelectAddColumnTypeAndCollation(pParse, pTab, pSelect); + sqlite3SelectAddColumnTypeAndCollation(pParse, pTab, pSelect, aff); pTab->iPKey = -1; if( db->mallocFailed ){ sqlite3DeleteTable(db, pTab); @@ -123475,7 +131024,6 @@ static void computeLimitRegisters(Parse *pParse, Select *p, int iBreak){ ** The current implementation interprets "LIMIT 0" to mean ** no rows. */ - sqlite3ExprCacheClear(pParse); if( pLimit ){ assert( pLimit->op==TK_LIMIT ); assert( pLimit->pLeft!=0 ); @@ -123566,7 +131114,7 @@ static KeyInfo *multiSelectOrderByKeyInfo(Parse *pParse, Select *p, int nExtra){ } assert( sqlite3KeyInfoIsWriteable(pRet) ); pRet->aColl[i] = pColl; - pRet->aSortOrder[i] = pOrderBy->a[i].sortOrder; + pRet->aSortFlags[i] = pOrderBy->a[i].sortFlags; } } @@ -123633,11 +131181,18 @@ static void generateWithRecursiveQuery( Expr *pLimit; /* Saved LIMIT and OFFSET */ int regLimit, regOffset; /* Registers used by LIMIT and OFFSET */ +#ifndef SQLITE_OMIT_WINDOWFUNC + if( p->pWin ){ + sqlite3ErrorMsg(pParse, "cannot use window functions in recursive queries"); + return; + } +#endif + /* Obtain authorization to do a recursive query */ if( sqlite3AuthCheck(pParse, SQLITE_RECURSIVE, 0, 0, 0) ) return; /* Process the LIMIT and OFFSET clauses, if they exist */ - addrBreak = sqlite3VdbeMakeLabel(v); + addrBreak = sqlite3VdbeMakeLabel(pParse); p->nSelectRow = 320; /* 4 billion rows */ computeLimitRegisters(pParse, p, addrBreak); pLimit = p->pLimit; @@ -123707,7 +131262,7 @@ static void generateWithRecursiveQuery( sqlite3VdbeAddOp1(v, OP_Delete, iQueue); /* Output the single row in Current */ - addrCont = sqlite3VdbeMakeLabel(v); + addrCont = sqlite3VdbeMakeLabel(pParse); codeOffset(v, regOffset, addrCont); selectInnerLoop(pParse, p, iCurrent, 0, 0, pDest, addrCont, addrBreak); @@ -123778,6 +131333,9 @@ static int multiSelectValues( assert( p->selFlags & SF_Values ); assert( p->op==TK_ALL || (p->op==TK_SELECT && p->pPrior==0) ); assert( p->pNext==0 || p->pEList->nExpr==p->pNext->pEList->nExpr ); +#ifndef SQLITE_OMIT_WINDOWFUNC + if( p->pWin ) return -1; +#endif if( p->pPrior==0 ) break; assert( p->pPrior->pNext==p ); p = p->pPrior; @@ -123842,6 +131400,7 @@ static int multiSelect( */ assert( p && p->pPrior ); /* Calling function guarantees this much */ assert( (p->selFlags & SF_Recursive)==0 || p->op==TK_ALL || p->op==TK_UNION ); + assert( p->selFlags & SF_Compound ); db = pParse->db; pPrior = p->pPrior; dest = *pDest; @@ -123867,7 +131426,8 @@ static int multiSelect( */ if( p->selFlags & SF_MultiValue ){ rc = multiSelectValues(pParse, p, &dest); - goto multi_select_end; + if( rc>=0 ) goto multi_select_end; + rc = SQLITE_OK; } /* Make sure all SELECTs in the statement have the same number of elements @@ -124012,11 +131572,11 @@ static int multiSelect( ** it is that we currently need. */ assert( unionTab==dest.iSDParm || dest.eDest!=priorOp ); - if( dest.eDest!=priorOp ){ + assert( p->pEList || db->mallocFailed ); + if( dest.eDest!=priorOp && db->mallocFailed==0 ){ int iCont, iBreak, iStart; - assert( p->pEList ); - iBreak = sqlite3VdbeMakeLabel(v); - iCont = sqlite3VdbeMakeLabel(v); + iBreak = sqlite3VdbeMakeLabel(pParse); + iCont = sqlite3VdbeMakeLabel(pParse); computeLimitRegisters(pParse, p, iBreak); sqlite3VdbeAddOp2(v, OP_Rewind, unionTab, iBreak); VdbeCoverage(v); iStart = sqlite3VdbeCurrentAddr(v); @@ -124083,9 +131643,10 @@ static int multiSelect( /* Generate code to take the intersection of the two temporary ** tables. */ + if( rc ) break; assert( p->pEList ); - iBreak = sqlite3VdbeMakeLabel(v); - iCont = sqlite3VdbeMakeLabel(v); + iBreak = sqlite3VdbeMakeLabel(pParse); + iCont = sqlite3VdbeMakeLabel(pParse); computeLimitRegisters(pParse, p, iBreak); sqlite3VdbeAddOp2(v, OP_Rewind, tab1, iBreak); VdbeCoverage(v); r1 = sqlite3GetTempReg(pParse); @@ -124110,6 +131671,7 @@ static int multiSelect( } #endif } + if( pParse->nErr ) goto multi_select_end; /* Compute collating sequences used by ** temporary tables needed to implement the compound select. @@ -124215,7 +131777,7 @@ static int generateOutputSubroutine( int addr; addr = sqlite3VdbeCurrentAddr(v); - iContinue = sqlite3VdbeMakeLabel(v); + iContinue = sqlite3VdbeMakeLabel(pParse); /* Suppress duplicates for UNION, EXCEPT, and INTERSECT */ @@ -124261,7 +131823,6 @@ static int generateOutputSubroutine( r1 = sqlite3GetTempReg(pParse); sqlite3VdbeAddOp4(v, OP_MakeRecord, pIn->iSdst, pIn->nSdst, r1, pDest->zAffSdst, pIn->nSdst); - sqlite3ExprCacheAffinityChange(pParse, pIn->iSdst, pIn->nSdst); sqlite3VdbeAddOp4Int(v, OP_IdxInsert, pDest->iSDParm, r1, pIn->iSdst, pIn->nSdst); sqlite3ReleaseTempReg(pParse, r1); @@ -124270,11 +131831,14 @@ static int generateOutputSubroutine( /* If this is a scalar select that is part of an expression, then ** store the results in the appropriate memory cell and break out - ** of the scan loop. + ** of the scan loop. Note that the select might return multiple columns + ** if it is the RHS of a row-value IN operator. */ case SRT_Mem: { - assert( pIn->nSdst==1 || pParse->nErr>0 ); testcase( pIn->nSdst!=1 ); - sqlite3ExprCodeMove(pParse, pIn->iSdst, pDest->iSDParm, 1); + if( pParse->nErr==0 ){ + testcase( pIn->nSdst>1 ); + sqlite3ExprCodeMove(pParse, pIn->iSdst, pDest->iSDParm, pIn->nSdst); + } /* The LIMIT clause will jump out of the loop for us */ break; } @@ -124304,7 +131868,6 @@ static int generateOutputSubroutine( default: { assert( pDest->eDest==SRT_Output ); sqlite3VdbeAddOp2(v, OP_ResultRow, pIn->iSdst, pIn->nSdst); - sqlite3ExprCacheAffinityChange(pParse, pIn->iSdst, pIn->nSdst); break; } } @@ -124454,8 +132017,8 @@ static int multiSelectOrderBy( db = pParse->db; v = pParse->pVdbe; assert( v!=0 ); /* Already thrown the error if VDBE alloc failed */ - labelEnd = sqlite3VdbeMakeLabel(v); - labelCmpr = sqlite3VdbeMakeLabel(v); + labelEnd = sqlite3VdbeMakeLabel(pParse); + labelCmpr = sqlite3VdbeMakeLabel(pParse); /* Patch up the ORDER BY clause @@ -124532,7 +132095,7 @@ static int multiSelectOrderBy( assert( sqlite3KeyInfoIsWriteable(pKeyDup) ); for(i=0; iaColl[i] = multiSelectCollSeq(pParse, p, i); - pKeyDup->aSortOrder[i] = 0; + pKeyDup->aSortFlags[i] = 0; } } } @@ -124751,7 +132314,10 @@ static Expr *substExpr( ){ pExpr->iRightJoinTable = pSubst->iNewTable; } - if( pExpr->op==TK_COLUMN && pExpr->iTable==pSubst->iTable ){ + if( pExpr->op==TK_COLUMN + && pExpr->iTable==pSubst->iTable + && !ExprHasProperty(pExpr, EP_FixedCol) + ){ if( pExpr->iColumn<0 ){ pExpr->op = TK_NULL; }else{ @@ -124759,7 +132325,7 @@ static Expr *substExpr( Expr *pCopy = pSubst->pEList->a[pExpr->iColumn].pExpr; Expr ifNullRow; assert( pSubst->pEList!=0 && pExpr->iColumnpEList->nExpr ); - assert( pExpr->pLeft==0 && pExpr->pRight==0 ); + assert( pExpr->pRight==0 ); if( sqlite3ExprIsVector(pCopy) ){ sqlite3VectorErrorMsg(pSubst->pParse, pCopy); }else{ @@ -124769,8 +132335,10 @@ static Expr *substExpr( ifNullRow.op = TK_IF_NULL_ROW; ifNullRow.pLeft = pCopy; ifNullRow.iTable = pSubst->iNewTable; + ifNullRow.flags = EP_Skip; pCopy = &ifNullRow; } + testcase( ExprHasProperty(pCopy, EP_Subquery) ); pNew = sqlite3ExprDup(db, pCopy, 0); if( pNew && pSubst->isLeftJoin ){ ExprSetProperty(pNew, EP_CanBeNull); @@ -124781,6 +132349,18 @@ static Expr *substExpr( } sqlite3ExprDelete(db, pExpr); pExpr = pNew; + + /* Ensure that the expression now has an implicit collation sequence, + ** just as it did when it was a column of a view or sub-query. */ + if( pExpr ){ + if( pExpr->op!=TK_COLUMN && pExpr->op!=TK_COLLATE ){ + CollSeq *pColl = sqlite3ExprCollSeq(pSubst->pParse, pExpr); + pExpr = sqlite3ExprAddCollateString(pSubst->pParse, pExpr, + (pColl ? pColl->zName : "BINARY") + ); + } + ExprClearProperty(pExpr, EP_Collate); + } } } }else{ @@ -124794,6 +132374,14 @@ static Expr *substExpr( }else{ substExprList(pSubst, pExpr->x.pList); } +#ifndef SQLITE_OMIT_WINDOWFUNC + if( ExprHasProperty(pExpr, EP_WinFunc) ){ + Window *pWin = pExpr->y.pWin; + pWin->pFilter = substExpr(pSubst, pWin->pFilter); + substExprList(pSubst, pWin->pPartition); + substExprList(pSubst, pWin->pOrderBy); + } +#endif } return pExpr; } @@ -124834,6 +132422,38 @@ static void substSelect( } #endif /* !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) */ +#if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) +/* +** pSelect is a SELECT statement and pSrcItem is one item in the FROM +** clause of that SELECT. +** +** This routine scans the entire SELECT statement and recomputes the +** pSrcItem->colUsed mask. +*/ +static int recomputeColumnsUsedExpr(Walker *pWalker, Expr *pExpr){ + struct SrcList_item *pItem; + if( pExpr->op!=TK_COLUMN ) return WRC_Continue; + pItem = pWalker->u.pSrcItem; + if( pItem->iCursor!=pExpr->iTable ) return WRC_Continue; + if( pExpr->iColumn<0 ) return WRC_Continue; + pItem->colUsed |= sqlite3ExprColUsed(pExpr); + return WRC_Continue; +} +static void recomputeColumnsUsed( + Select *pSelect, /* The complete SELECT statement */ + struct SrcList_item *pSrcItem /* Which FROM clause item to recompute */ +){ + Walker w; + if( NEVER(pSrcItem->pTab==0) ) return; + memset(&w, 0, sizeof(w)); + w.xExprCallback = recomputeColumnsUsedExpr; + w.xSelectCallback = sqlite3SelectWalkNoop; + w.u.pSrcItem = pSrcItem; + pSrcItem->colUsed = 0; + sqlite3WalkSelect(&w, pSelect); +} +#endif /* !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) */ + #if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) /* ** This routine attempts to flatten subqueries as a performance optimization. @@ -124879,6 +132499,7 @@ static void substSelect( ** (3b) the FROM clause of the subquery may not contain a virtual ** table and ** (3c) the outer query may not be an aggregate. +** (3d) the outer query may not be DISTINCT. ** ** (4) The subquery can not be DISTINCT. ** @@ -124929,6 +132550,7 @@ static void substSelect( ** (17d1) aggregate, or ** (17d2) DISTINCT, or ** (17d3) a join. +** (17e) the subquery may not contain window functions ** ** The parent and sub-query may contain WHERE clauses. Subject to ** rules (11), (13) and (14), they may also contain ORDER BY, @@ -124973,6 +132595,10 @@ static void substSelect( ** "SELECT x FROM (SELECT max(y), x FROM t1)" would not necessarily ** return the value X for which Y was maximal.) ** +** (25) If either the subquery or the parent query contains a window +** function in the select list or ORDER BY clause, flattening +** is not attempted. +** ** ** In this routine, the "p" parameter is a pointer to the outer query. ** The subquery is p->pSrc->a[iFrom]. isAgg is true if the outer query @@ -125016,6 +132642,10 @@ static int flattenSubquery( pSub = pSubitem->pSelect; assert( pSub!=0 ); +#ifndef SQLITE_OMIT_WINDOWFUNC + if( p->pWin || pSub->pWin ) return 0; /* Restriction (25) */ +#endif + pSubSrc = pSub->pSrc; assert( pSubSrc ); /* Prior to version 3.1.2, when LIMIT and OFFSET had to be simple constants, @@ -125067,8 +132697,11 @@ static int flattenSubquery( */ if( (pSubitem->fg.jointype & JT_OUTER)!=0 ){ isLeftJoin = 1; - if( pSubSrc->nSrc>1 || isAgg || IsVirtual(pSubSrc->a[0].pTab) ){ - /* (3a) (3c) (3b) */ + if( pSubSrc->nSrc>1 /* (3a) */ + || isAgg /* (3b) */ + || IsVirtual(pSubSrc->a[0].pTab) /* (3c) */ + || (p->selFlags & SF_Distinct)!=0 /* (3d) */ + ){ return 0; } } @@ -125102,6 +132735,9 @@ static int flattenSubquery( if( (pSub1->selFlags & (SF_Distinct|SF_Aggregate))!=0 /* (17b) */ || (pSub1->pPrior && pSub1->op!=TK_ALL) /* (17a) */ || pSub1->pSrc->nSrc<1 /* (17c) */ +#ifndef SQLITE_OMIT_WINDOWFUNC + || pSub1->pWin /* (17e) */ +#endif ){ return 0; } @@ -125126,8 +132762,8 @@ static int flattenSubquery( assert( (p->selFlags & SF_Recursive)==0 || pSub->pPrior==0 ); /***** If we reach this point, flattening is permitted. *****/ - SELECTTRACE(1,pParse,p,("flatten %s.%p from term %d\n", - pSub->zSelName, pSub, iFrom)); + SELECTTRACE(1,pParse,p,("flatten %u.%p from term %d\n", + pSub->selId, pSub, iFrom)); /* Authorize the subquery */ pParse->zAuthContext = pSubitem->zName; @@ -125178,7 +132814,6 @@ static int flattenSubquery( p->pPrior = 0; p->pLimit = 0; pNew = sqlite3SelectDup(db, p, 0); - sqlite3SelectSetName(pNew, pSub->zSelName); p->pLimit = pLimit; p->pOrderBy = pOrderBy; p->pSrc = pSrc; @@ -125191,7 +132826,7 @@ static int flattenSubquery( pNew->pNext = p; p->pPrior = pNew; SELECTTRACE(2,pParse,p,("compound-subquery flattener" - " creates %s.%p as peer\n",pNew->zSelName, pNew)); + " creates %u as peer\n",pNew->selId)); } if( db->mallocFailed ) return 1; } @@ -125247,6 +132882,7 @@ static int flattenSubquery( for(pParent=p; pParent; pParent=pParent->pPrior, pSub=pSub->pPrior){ int nSubSrc; u8 jointype = 0; + assert( pSub!=0 ); pSubSrc = pSub->pSrc; /* FROM clause of subquery */ nSubSrc = pSubSrc->nSrc; /* Number of terms in subquery FROM clause */ pSrc = pParent->pSrc; /* FROM clause of the outer query */ @@ -125256,11 +132892,9 @@ static int flattenSubquery( jointype = pSubitem->fg.jointype; }else{ assert( pParent!=p ); /* 2nd and subsequent times through the loop */ - pSrc = pParent->pSrc = sqlite3SrcListAppend(db, 0, 0, 0); - if( pSrc==0 ){ - assert( db->mallocFailed ); - break; - } + pSrc = sqlite3SrcListAppend(pParse, 0, 0, 0); + if( pSrc==0 ) break; + pParent->pSrc = pSrc; } /* The subquery uses a single slot of the FROM clause of the outer @@ -125279,10 +132913,9 @@ static int flattenSubquery( ** for the two elements in the FROM clause of the subquery. */ if( nSubSrc>1 ){ - pParent->pSrc = pSrc = sqlite3SrcListEnlarge(db, pSrc, nSubSrc-1,iFrom+1); - if( db->mallocFailed ){ - break; - } + pSrc = sqlite3SrcListEnlarge(pParse, pSrc, nSubSrc-1,iFrom+1); + if( pSrc==0 ) break; + pParent->pSrc = pSrc; } /* Transfer the FROM clause terms from the subquery into the @@ -125328,11 +132961,12 @@ static int flattenSubquery( pParent->pOrderBy = pOrderBy; pSub->pOrderBy = 0; } - pWhere = sqlite3ExprDup(db, pSub->pWhere, 0); + pWhere = pSub->pWhere; + pSub->pWhere = 0; if( isLeftJoin>0 ){ - setJoinExpr(pWhere, iNewParent); + sqlite3SetJoinExpr(pWhere, iNewParent); } - pParent->pWhere = sqlite3ExprAnd(db, pWhere, pParent->pWhere); + pParent->pWhere = sqlite3ExprAnd(pParse, pWhere, pParent->pWhere); if( db->mallocFailed==0 ){ SubstContext x; x.pParse = pParse; @@ -125343,10 +132977,10 @@ static int flattenSubquery( substSelect(&x, pParent, 0); } - /* The flattened query is distinct if either the inner or the - ** outer query is distinct. - */ - pParent->selFlags |= pSub->selFlags & SF_Distinct; + /* The flattened query is a compound if either the inner or the + ** outer query is a compound. */ + pParent->selFlags |= pSub->selFlags & SF_Compound; + assert( (pSub->selFlags & SF_Distinct)==0 ); /* restriction (17b) */ /* ** SELECT ... FROM (SELECT ... LIMIT a OFFSET b) LIMIT x OFFSET y; @@ -125358,6 +132992,12 @@ static int flattenSubquery( pParent->pLimit = pSub->pLimit; pSub->pLimit = 0; } + + /* Recompute the SrcList_item.colUsed masks for the flattened + ** tables. */ + for(i=0; ia[i+iFrom]); + } } /* Finially, delete what is left of the subquery and return @@ -125376,7 +133016,189 @@ static int flattenSubquery( } #endif /* !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) */ +/* +** A structure to keep track of all of the column values that are fixed to +** a known value due to WHERE clause constraints of the form COLUMN=VALUE. +*/ +typedef struct WhereConst WhereConst; +struct WhereConst { + Parse *pParse; /* Parsing context */ + int nConst; /* Number for COLUMN=CONSTANT terms */ + int nChng; /* Number of times a constant is propagated */ + Expr **apExpr; /* [i*2] is COLUMN and [i*2+1] is VALUE */ +}; + +/* +** Add a new entry to the pConst object. Except, do not add duplicate +** pColumn entires. Also, do not add if doing so would not be appropriate. +** +** The caller guarantees the pColumn is a column and pValue is a constant. +** This routine has to do some additional checks before completing the +** insert. +*/ +static void constInsert( + WhereConst *pConst, /* The WhereConst into which we are inserting */ + Expr *pColumn, /* The COLUMN part of the constraint */ + Expr *pValue, /* The VALUE part of the constraint */ + Expr *pExpr /* Overall expression: COLUMN=VALUE or VALUE=COLUMN */ +){ + int i; + assert( pColumn->op==TK_COLUMN ); + assert( sqlite3ExprIsConstant(pValue) ); + + if( ExprHasProperty(pColumn, EP_FixedCol) ) return; + if( sqlite3ExprAffinity(pValue)!=0 ) return; + if( !sqlite3IsBinary(sqlite3ExprCompareCollSeq(pConst->pParse,pExpr)) ){ + return; + } + + /* 2018-10-25 ticket [cf5ed20f] + ** Make sure the same pColumn is not inserted more than once */ + for(i=0; inConst; i++){ + const Expr *pE2 = pConst->apExpr[i*2]; + assert( pE2->op==TK_COLUMN ); + if( pE2->iTable==pColumn->iTable + && pE2->iColumn==pColumn->iColumn + ){ + return; /* Already present. Return without doing anything. */ + } + } + + pConst->nConst++; + pConst->apExpr = sqlite3DbReallocOrFree(pConst->pParse->db, pConst->apExpr, + pConst->nConst*2*sizeof(Expr*)); + if( pConst->apExpr==0 ){ + pConst->nConst = 0; + }else{ + pConst->apExpr[pConst->nConst*2-2] = pColumn; + pConst->apExpr[pConst->nConst*2-1] = pValue; + } +} + +/* +** Find all terms of COLUMN=VALUE or VALUE=COLUMN in pExpr where VALUE +** is a constant expression and where the term must be true because it +** is part of the AND-connected terms of the expression. For each term +** found, add it to the pConst structure. +*/ +static void findConstInWhere(WhereConst *pConst, Expr *pExpr){ + Expr *pRight, *pLeft; + if( pExpr==0 ) return; + if( ExprHasProperty(pExpr, EP_FromJoin) ) return; + if( pExpr->op==TK_AND ){ + findConstInWhere(pConst, pExpr->pRight); + findConstInWhere(pConst, pExpr->pLeft); + return; + } + if( pExpr->op!=TK_EQ ) return; + pRight = pExpr->pRight; + pLeft = pExpr->pLeft; + assert( pRight!=0 ); + assert( pLeft!=0 ); + if( pRight->op==TK_COLUMN && sqlite3ExprIsConstant(pLeft) ){ + constInsert(pConst,pRight,pLeft,pExpr); + } + if( pLeft->op==TK_COLUMN && sqlite3ExprIsConstant(pRight) ){ + constInsert(pConst,pLeft,pRight,pExpr); + } +} +/* +** This is a Walker expression callback. pExpr is a candidate expression +** to be replaced by a value. If pExpr is equivalent to one of the +** columns named in pWalker->u.pConst, then overwrite it with its +** corresponding value. +*/ +static int propagateConstantExprRewrite(Walker *pWalker, Expr *pExpr){ + int i; + WhereConst *pConst; + if( pExpr->op!=TK_COLUMN ) return WRC_Continue; + if( ExprHasProperty(pExpr, EP_FixedCol|EP_FromJoin) ){ + testcase( ExprHasProperty(pExpr, EP_FixedCol) ); + testcase( ExprHasProperty(pExpr, EP_FromJoin) ); + return WRC_Continue; + } + pConst = pWalker->u.pConst; + for(i=0; inConst; i++){ + Expr *pColumn = pConst->apExpr[i*2]; + if( pColumn==pExpr ) continue; + if( pColumn->iTable!=pExpr->iTable ) continue; + if( pColumn->iColumn!=pExpr->iColumn ) continue; + /* A match is found. Add the EP_FixedCol property */ + pConst->nChng++; + ExprClearProperty(pExpr, EP_Leaf); + ExprSetProperty(pExpr, EP_FixedCol); + assert( pExpr->pLeft==0 ); + pExpr->pLeft = sqlite3ExprDup(pConst->pParse->db, pConst->apExpr[i*2+1], 0); + break; + } + return WRC_Prune; +} + +/* +** The WHERE-clause constant propagation optimization. +** +** If the WHERE clause contains terms of the form COLUMN=CONSTANT or +** CONSTANT=COLUMN that are top-level AND-connected terms that are not +** part of a ON clause from a LEFT JOIN, then throughout the query +** replace all other occurrences of COLUMN with CONSTANT. +** +** For example, the query: +** +** SELECT * FROM t1, t2, t3 WHERE t1.a=39 AND t2.b=t1.a AND t3.c=t2.b +** +** Is transformed into +** +** SELECT * FROM t1, t2, t3 WHERE t1.a=39 AND t2.b=39 AND t3.c=39 +** +** Return true if any transformations where made and false if not. +** +** Implementation note: Constant propagation is tricky due to affinity +** and collating sequence interactions. Consider this example: +** +** CREATE TABLE t1(a INT,b TEXT); +** INSERT INTO t1 VALUES(123,'0123'); +** SELECT * FROM t1 WHERE a=123 AND b=a; +** SELECT * FROM t1 WHERE a=123 AND b=123; +** +** The two SELECT statements above should return different answers. b=a +** is alway true because the comparison uses numeric affinity, but b=123 +** is false because it uses text affinity and '0123' is not the same as '123'. +** To work around this, the expression tree is not actually changed from +** "b=a" to "b=123" but rather the "a" in "b=a" is tagged with EP_FixedCol +** and the "123" value is hung off of the pLeft pointer. Code generator +** routines know to generate the constant "123" instead of looking up the +** column value. Also, to avoid collation problems, this optimization is +** only attempted if the "a=123" term uses the default BINARY collation. +*/ +static int propagateConstants( + Parse *pParse, /* The parsing context */ + Select *p /* The query in which to propagate constants */ +){ + WhereConst x; + Walker w; + int nChng = 0; + x.pParse = pParse; + do{ + x.nConst = 0; + x.nChng = 0; + x.apExpr = 0; + findConstInWhere(&x, p->pWhere); + if( x.nConst ){ + memset(&w, 0, sizeof(w)); + w.pParse = pParse; + w.xExprCallback = propagateConstantExprRewrite; + w.xSelectCallback = sqlite3SelectWalkNoop; + w.xSelectCallback2 = 0; + w.walkerDepth = 0; + w.u.pConst = &x; + sqlite3WalkExpr(&w, p->pWhere); + sqlite3DbFree(x.pParse->db, x.apExpr); + nChng += x.nChng; + } + }while( x.nChng ); + return nChng; +} #if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) /* @@ -125406,7 +133228,7 @@ static int flattenSubquery( ** (2) The inner query is the recursive part of a common table expression. ** ** (3) The inner query has a LIMIT clause (since the changes to the WHERE -** close would change the meaning of the LIMIT). +** clause would change the meaning of the LIMIT). ** ** (4) The inner query is the right operand of a LEFT JOIN and the ** expression to be pushed down does not come from the ON clause @@ -125425,6 +133247,10 @@ static int flattenSubquery( ** But if the (b2=2) term were to be pushed down into the bb subquery, ** then the (1,1,NULL) row would be suppressed. ** +** (6) The inner query features one or more window-functions (since +** changes to the WHERE clause of the inner query could change the +** window over which window functions are calculated). +** ** Return 0 if no changes are made and non-zero if one or more WHERE clause ** terms are duplicated into the subquery. */ @@ -125440,6 +133266,10 @@ static int pushDownWhereTerms( if( pWhere==0 ) return 0; if( pSubq->selFlags & SF_Recursive ) return 0; /* restriction (2) */ +#ifndef SQLITE_OMIT_WINDOWFUNC + if( pSubq->pWin ) return 0; /* restriction (6) */ +#endif + #ifdef SQLITE_DEBUG /* Only the first term of a compound can have a WITH clause. But make ** sure no other terms are marked SF_Recursive in case something changes @@ -125483,9 +133313,9 @@ static int pushDownWhereTerms( x.pEList = pSubq->pEList; pNew = substExpr(&x, pNew); if( pSubq->selFlags & SF_Aggregate ){ - pSubq->pHaving = sqlite3ExprAnd(pParse->db, pSubq->pHaving, pNew); + pSubq->pHaving = sqlite3ExprAnd(pParse, pSubq->pHaving, pNew); }else{ - pSubq->pWhere = sqlite3ExprAnd(pParse->db, pSubq->pWhere, pNew); + pSubq->pWhere = sqlite3ExprAnd(pParse, pSubq->pWhere, pNew); } pSubq = pSubq->pPrior; } @@ -125515,24 +133345,29 @@ static u8 minMaxQuery(sqlite3 *db, Expr *pFunc, ExprList **ppMinMax){ ExprList *pEList = pFunc->x.pList; /* Arguments to agg function */ const char *zFunc; /* Name of aggregate function pFunc */ ExprList *pOrderBy; - u8 sortOrder; + u8 sortFlags = 0; assert( *ppMinMax==0 ); assert( pFunc->op==TK_AGG_FUNCTION ); - if( pEList==0 || pEList->nExpr!=1 ) return eRet; + assert( !IsWindowFunc(pFunc) ); + if( pEList==0 || pEList->nExpr!=1 || ExprHasProperty(pFunc, EP_WinFunc) ){ + return eRet; + } zFunc = pFunc->u.zToken; if( sqlite3StrICmp(zFunc, "min")==0 ){ eRet = WHERE_ORDERBY_MIN; - sortOrder = SQLITE_SO_ASC; + if( sqlite3ExprCanBeNull(pEList->a[0].pExpr) ){ + sortFlags = KEYINFO_ORDER_BIGNULL; + } }else if( sqlite3StrICmp(zFunc, "max")==0 ){ eRet = WHERE_ORDERBY_MAX; - sortOrder = SQLITE_SO_DESC; + sortFlags = KEYINFO_ORDER_DESC; }else{ return eRet; } *ppMinMax = pOrderBy = sqlite3ExprListDup(db, pEList, 0); assert( pOrderBy!=0 || db->mallocFailed ); - if( pOrderBy ) pOrderBy->a[0].sortOrder = sortOrder; + if( pOrderBy ) pOrderBy->a[0].sortFlags = sortFlags; return eRet; } @@ -125566,7 +133401,7 @@ static Table *isSimpleCount(Select *p, AggInfo *pAggInfo){ if( pExpr->op!=TK_AGG_FUNCTION ) return 0; if( NEVER(pAggInfo->nFunc==0) ) return 0; if( (pAggInfo->aFunc[0].pFunc->funcFlags&SQLITE_FUNC_COUNT)==0 ) return 0; - if( pExpr->flags&EP_Distinct ) return 0; + if( ExprHasProperty(pExpr, EP_Distinct|EP_WinFunc) ) return 0; return pTab; } @@ -125657,6 +133492,9 @@ static int convertCompoundSelectToSubquery(Walker *pWalker, Select *p){ p->pPrior = 0; p->pNext = 0; p->pWith = 0; +#ifndef SQLITE_OMIT_WINDOWFUNC + p->pWinDefn = 0; +#endif p->selFlags &= ~SF_Compound; assert( (p->selFlags & SF_Converted)==0 ); p->selFlags |= SF_Converted; @@ -125756,6 +133594,9 @@ static int withExpand( With *pWith; /* WITH clause that pCte belongs to */ assert( pFrom->pTab==0 ); + if( pParse->nErr ){ + return SQLITE_ERROR; + } pCte = searchWith(pParse->pWith, pFrom, &pWith); if( pCte ){ @@ -125876,7 +133717,7 @@ static void selectPopWith(Walker *pWalker, Select *p){ if( OK_IF_ALWAYS_TRUE(pParse->pWith) && p->pPrior==0 ){ With *pWith = findRightmost(p)->pWith; if( pWith!=0 ){ - assert( pParse->pWith==pWith ); + assert( pParse->pWith==pWith || pParse->nErr ); pParse->pWith = pWith->pOuter; } } @@ -125885,6 +133726,35 @@ static void selectPopWith(Walker *pWalker, Select *p){ #define selectPopWith 0 #endif +/* +** The SrcList_item structure passed as the second argument represents a +** sub-query in the FROM clause of a SELECT statement. This function +** allocates and populates the SrcList_item.pTab object. If successful, +** SQLITE_OK is returned. Otherwise, if an OOM error is encountered, +** SQLITE_NOMEM. +*/ +SQLITE_PRIVATE int sqlite3ExpandSubquery(Parse *pParse, struct SrcList_item *pFrom){ + Select *pSel = pFrom->pSelect; + Table *pTab; + + assert( pSel ); + pFrom->pTab = pTab = sqlite3DbMallocZero(pParse->db, sizeof(Table)); + if( pTab==0 ) return SQLITE_NOMEM; + pTab->nTabRef = 1; + if( pFrom->zAlias ){ + pTab->zName = sqlite3DbStrDup(pParse->db, pFrom->zAlias); + }else{ + pTab->zName = sqlite3MPrintf(pParse->db, "subquery_%u", pSel->selId); + } + while( pSel->pPrior ){ pSel = pSel->pPrior; } + sqlite3ColumnsFromExprList(pParse, pSel->pEList,&pTab->nCol,&pTab->aCol); + pTab->iPKey = -1; + pTab->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); + pTab->tabFlags |= TF_Ephemeral; + + return pParse->nErr ? SQLITE_ERROR : SQLITE_OK; +} + /* ** This routine is a Walker callback for "expanding" a SELECT statement. ** "Expanding" means to do the following: @@ -125928,6 +133798,10 @@ static int selectExpander(Walker *pWalker, Select *p){ if( (selFlags & SF_Expanded)!=0 ){ return WRC_Prune; } + if( pWalker->eCode ){ + /* Renumber selId because it has been copied from a view */ + p->selId = ++pParse->nSelect; + } pTabList = p->pSrc; pEList = p->pEList; sqlite3WithPush(pParse, p->pWith, 0); @@ -125957,19 +133831,7 @@ static int selectExpander(Walker *pWalker, Select *p){ assert( pSel!=0 ); assert( pFrom->pTab==0 ); if( sqlite3WalkSelect(pWalker, pSel) ) return WRC_Abort; - pFrom->pTab = pTab = sqlite3DbMallocZero(db, sizeof(Table)); - if( pTab==0 ) return WRC_Abort; - pTab->nTabRef = 1; - if( pFrom->zAlias ){ - pTab->zName = sqlite3DbStrDup(db, pFrom->zAlias); - }else{ - pTab->zName = sqlite3MPrintf(db, "subquery_%p", (void*)pTab); - } - while( pSel->pPrior ){ pSel = pSel->pPrior; } - sqlite3ColumnsFromExprList(pParse, pSel->pEList,&pTab->nCol,&pTab->aCol); - pTab->iPKey = -1; - pTab->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); - pTab->tabFlags |= TF_Ephemeral; + if( sqlite3ExpandSubquery(pParse, pFrom) ) return WRC_Abort; #endif }else{ /* An ordinary table or view name in the FROM clause */ @@ -125986,16 +133848,32 @@ static int selectExpander(Walker *pWalker, Select *p){ if( !IsVirtual(pTab) && cannotBeFunction(pParse, pFrom) ){ return WRC_Abort; } -#if !defined(SQLITE_OMIT_VIEW) || !defined (SQLITE_OMIT_VIRTUALTABLE) +#if !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_VIRTUALTABLE) if( IsVirtual(pTab) || pTab->pSelect ){ i16 nCol; + u8 eCodeOrig = pWalker->eCode; if( sqlite3ViewGetColumnNames(pParse, pTab) ) return WRC_Abort; assert( pFrom->pSelect==0 ); + if( pTab->pSelect && (db->flags & SQLITE_EnableView)==0 ){ + sqlite3ErrorMsg(pParse, "access to view \"%s\" prohibited", + pTab->zName); + } +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( IsVirtual(pTab) + && pFrom->fg.fromDDL + && ALWAYS(pTab->pVTable!=0) + && pTab->pVTable->eVtabRisk > ((db->flags & SQLITE_TrustedSchema)!=0) + ){ + sqlite3ErrorMsg(pParse, "unsafe use of virtual table \"%s\"", + pTab->zName); + } +#endif pFrom->pSelect = sqlite3SelectDup(db, pTab->pSelect, 0); - sqlite3SelectSetName(pFrom->pSelect, pTab->zName); nCol = pTab->nCol; pTab->nCol = -1; + pWalker->eCode = 1; /* Turn on Select.selId renumbering */ sqlite3WalkSelect(pWalker, pFrom->pSelect); + pWalker->eCode = eCodeOrig; pTab->nCol = nCol; } #endif @@ -126009,7 +133887,7 @@ static int selectExpander(Walker *pWalker, Select *p){ /* Process NATURAL keywords, and ON and USING clauses of joins. */ - if( db->mallocFailed || sqliteProcessJoin(pParse, p) ){ + if( pParse->nErr || db->mallocFailed || sqliteProcessJoin(pParse, p) ){ return WRC_Abort; } @@ -126056,10 +133934,9 @@ static int selectExpander(Walker *pWalker, Select *p){ */ pNew = sqlite3ExprListAppend(pParse, pNew, a[k].pExpr); if( pNew ){ - pNew->a[pNew->nExpr-1].zName = a[k].zName; - pNew->a[pNew->nExpr-1].zSpan = a[k].zSpan; - a[k].zName = 0; - a[k].zSpan = 0; + pNew->a[pNew->nExpr-1].zEName = a[k].zEName; + pNew->a[pNew->nExpr-1].eEName = a[k].eEName; + a[k].zEName = 0; } a[k].pExpr = 0; }else{ @@ -126098,7 +133975,7 @@ static int selectExpander(Walker *pWalker, Select *p){ assert( zName ); if( zTName && pSub - && sqlite3MatchSpanName(pSub->pEList->a[j].zSpan, 0, zTName, 0)==0 + && sqlite3MatchEName(&pSub->pEList->a[j], 0, zTName, 0)==0 ){ continue; } @@ -126116,7 +133993,7 @@ static int selectExpander(Walker *pWalker, Select *p){ if( i>0 && zTName==0 ){ if( (pFrom->fg.jointype & JT_NATURAL)!=0 - && tableAndColumnIndex(pTabList, i, zName, 0, 0) + && tableAndColumnIndex(pTabList, i, zName, 0, 0, 1) ){ /* In a NATURAL join, omit the join columns from the ** table to the right of the join */ @@ -126149,17 +134026,18 @@ static int selectExpander(Walker *pWalker, Select *p){ pNew = sqlite3ExprListAppend(pParse, pNew, pExpr); sqlite3TokenInit(&sColname, zColname); sqlite3ExprListSetName(pParse, pNew, &sColname, 0); - if( pNew && (p->selFlags & SF_NestedFrom)!=0 ){ + if( pNew && (p->selFlags & SF_NestedFrom)!=0 && !IN_RENAME_OBJECT ){ struct ExprList_item *pX = &pNew->a[pNew->nExpr-1]; + sqlite3DbFree(db, pX->zEName); if( pSub ){ - pX->zSpan = sqlite3DbStrDup(db, pSub->pEList->a[j].zSpan); - testcase( pX->zSpan==0 ); + pX->zEName = sqlite3DbStrDup(db, pSub->pEList->a[j].zEName); + testcase( pX->zEName==0 ); }else{ - pX->zSpan = sqlite3MPrintf(db, "%s.%s.%s", + pX->zEName = sqlite3MPrintf(db, "%s.%s.%s", zSchemaName, zTabName, zColname); - testcase( pX->zSpan==0 ); + testcase( pX->zEName==0 ); } - pX->bSpanIsTab = 1; + pX->eEName = ENAME_TAB; } sqlite3DbFree(db, zToFree); } @@ -126188,29 +134066,6 @@ static int selectExpander(Walker *pWalker, Select *p){ return WRC_Continue; } -/* -** No-op routine for the parse-tree walker. -** -** When this routine is the Walker.xExprCallback then expression trees -** are walked without any actions being taken at each node. Presumably, -** when this routine is used for Walker.xExprCallback then -** Walker.xSelectCallback is set to do something useful for every -** subquery in the parser tree. -*/ -SQLITE_PRIVATE int sqlite3ExprWalkNoop(Walker *NotUsed, Expr *NotUsed2){ - UNUSED_PARAMETER2(NotUsed, NotUsed2); - return WRC_Continue; -} - -/* -** No-op routine for the parse-tree walker for SELECT statements. -** subquery in the parser tree. -*/ -SQLITE_PRIVATE int sqlite3SelectWalkNoop(Walker *NotUsed, Select *NotUsed2){ - UNUSED_PARAMETER2(NotUsed, NotUsed2); - return WRC_Continue; -} - #if SQLITE_DEBUG /* ** Always assert. This xSelectCallback2 implementation proves that the @@ -126245,6 +134100,7 @@ static void sqlite3SelectExpand(Parse *pParse, Select *pSelect){ } w.xSelectCallback = selectExpander; w.xSelectCallback2 = selectPopWith; + w.eCode = 0; sqlite3WalkSelect(&w, pSelect); } @@ -126270,7 +134126,7 @@ static void selectAddSubqueryTypeInfo(Walker *pWalker, Select *p){ struct SrcList_item *pFrom; assert( p->selFlags & SF_Resolved ); - assert( (p->selFlags & SF_HasTypeInfo)==0 ); + if( p->selFlags & SF_HasTypeInfo ) return; p->selFlags |= SF_HasTypeInfo; pParse = pWalker->pParse; pTabList = p->pSrc; @@ -126282,7 +134138,8 @@ static void selectAddSubqueryTypeInfo(Walker *pWalker, Select *p){ Select *pSel = pFrom->pSelect; if( pSel ){ while( pSel->pPrior ) pSel = pSel->pPrior; - sqlite3SelectAddColumnTypeAndCollation(pParse, pTab, pSel); + sqlite3SelectAddColumnTypeAndCollation(pParse, pTab, pSel, + SQLITE_AFF_NONE); } } } @@ -126350,6 +134207,7 @@ static void resetAccumulator(Parse *pParse, AggInfo *pAggInfo){ struct AggInfo_func *pFunc; int nReg = pAggInfo->nFunc + pAggInfo->nColumn; if( nReg==0 ) return; + if( pParse->nErr ) return; #ifdef SQLITE_DEBUG /* Verify that all AggInfo registers are within the range specified by ** AggInfo.mnReg..AggInfo.mxReg */ @@ -126373,7 +134231,7 @@ static void resetAccumulator(Parse *pParse, AggInfo *pAggInfo){ "argument"); pFunc->iDistinct = -1; }else{ - KeyInfo *pKeyInfo = keyInfoFromExprList(pParse, pE->x.pList, 0, 0); + KeyInfo *pKeyInfo = sqlite3KeyInfoFromExprList(pParse, pE->x.pList,0,0); sqlite3VdbeAddOp4(v, OP_OpenEphemeral, pFunc->iDistinct, 0, 0, (char*)pKeyInfo, P4_KEYINFO); } @@ -126397,11 +134255,17 @@ static void finalizeAggFunctions(Parse *pParse, AggInfo *pAggInfo){ } } + /* ** Update the accumulator memory cells for an aggregate based on ** the current cursor position. +** +** If regAcc is non-zero and there are no min() or max() aggregates +** in pAggInfo, then only populate the pAggInfo->nAccumulator accumulator +** registers if register regAcc contains 0. The caller will take care +** of setting and clearing regAcc. */ -static void updateAccumulator(Parse *pParse, AggInfo *pAggInfo){ +static void updateAccumulator(Parse *pParse, int regAcc, AggInfo *pAggInfo){ Vdbe *v = pParse->pVdbe; int i; int regHit = 0; @@ -126416,6 +134280,25 @@ static void updateAccumulator(Parse *pParse, AggInfo *pAggInfo){ int regAgg; ExprList *pList = pF->pExpr->x.pList; assert( !ExprHasProperty(pF->pExpr, EP_xIsSelect) ); + assert( !IsWindowFunc(pF->pExpr) ); + if( ExprHasProperty(pF->pExpr, EP_WinFunc) ){ + Expr *pFilter = pF->pExpr->y.pWin->pFilter; + if( pAggInfo->nAccumulator + && (pF->pFunc->funcFlags & SQLITE_FUNC_NEEDCOLL) + ){ + if( regHit==0 ) regHit = ++pParse->nMem; + /* If this is the first row of the group (regAcc==0), clear the + ** "magnet" register regHit so that the accumulator registers + ** are populated if the FILTER clause jumps over the the + ** invocation of min() or max() altogether. Or, if this is not + ** the first row (regAcc==1), set the magnet register so that the + ** accumulators are not populated unless the min()/max() is invoked and + ** indicates that they should be. */ + sqlite3VdbeAddOp2(v, OP_Copy, regAcc, regHit); + } + addrNext = sqlite3VdbeMakeLabel(pParse); + sqlite3ExprIfFalse(pParse, pFilter, addrNext, SQLITE_JUMPIFNULL); + } if( pList ){ nArg = pList->nExpr; regAgg = sqlite3GetTempRange(pParse, nArg); @@ -126425,7 +134308,9 @@ static void updateAccumulator(Parse *pParse, AggInfo *pAggInfo){ regAgg = 0; } if( pF->iDistinct>=0 ){ - addrNext = sqlite3VdbeMakeLabel(v); + if( addrNext==0 ){ + addrNext = sqlite3VdbeMakeLabel(pParse); + } testcase( nArg==0 ); /* Error condition */ testcase( nArg>1 ); /* Also an error */ codeDistinct(pParse, pF->iDistinct, addrNext, 1, regAgg); @@ -126444,38 +134329,27 @@ static void updateAccumulator(Parse *pParse, AggInfo *pAggInfo){ if( regHit==0 && pAggInfo->nAccumulator ) regHit = ++pParse->nMem; sqlite3VdbeAddOp4(v, OP_CollSeq, regHit, 0, 0, (char *)pColl, P4_COLLSEQ); } - sqlite3VdbeAddOp3(v, OP_AggStep0, 0, regAgg, pF->iMem); + sqlite3VdbeAddOp3(v, OP_AggStep, 0, regAgg, pF->iMem); sqlite3VdbeAppendP4(v, pF->pFunc, P4_FUNCDEF); sqlite3VdbeChangeP5(v, (u8)nArg); - sqlite3ExprCacheAffinityChange(pParse, regAgg, nArg); sqlite3ReleaseTempRange(pParse, regAgg, nArg); if( addrNext ){ sqlite3VdbeResolveLabel(v, addrNext); - sqlite3ExprCacheClear(pParse); } } - - /* Before populating the accumulator registers, clear the column cache. - ** Otherwise, if any of the required column values are already present - ** in registers, sqlite3ExprCode() may use OP_SCopy to copy the value - ** to pC->iMem. But by the time the value is used, the original register - ** may have been used, invalidating the underlying buffer holding the - ** text or blob value. See ticket [883034dcb5]. - ** - ** Another solution would be to change the OP_SCopy used to copy cached - ** values to an OP_Copy. - */ + if( regHit==0 && pAggInfo->nAccumulator ){ + regHit = regAcc; + } if( regHit ){ addrHitTest = sqlite3VdbeAddOp1(v, OP_If, regHit); VdbeCoverage(v); } - sqlite3ExprCacheClear(pParse); for(i=0, pC=pAggInfo->aCol; inAccumulator; i++, pC++){ sqlite3ExprCode(pParse, pC->pExpr, pC->iMem); } + pAggInfo->directMode = 0; - sqlite3ExprCacheClear(pParse); if( addrHitTest ){ - sqlite3VdbeJumpHere(v, addrHitTest); + sqlite3VdbeJumpHereOrPopInst(v, addrHitTest); } } @@ -126518,11 +134392,11 @@ static int havingToWhereExprCb(Walker *pWalker, Expr *pExpr){ Select *pS = pWalker->u.pSelect; if( sqlite3ExprIsConstantOrGroupBy(pWalker->pParse, pExpr, pS->pGroupBy) ){ sqlite3 *db = pWalker->pParse->db; - Expr *pNew = sqlite3ExprAlloc(db, TK_INTEGER, &sqlite3IntTokens[1], 0); + Expr *pNew = sqlite3Expr(db, TK_INTEGER, "1"); if( pNew ){ Expr *pWhere = pS->pWhere; SWAP(Expr, *pNew, *pExpr); - pNew = sqlite3ExprAnd(db, pWhere, pNew); + pNew = sqlite3ExprAnd(pWalker->pParse, pWhere, pNew); pS->pWhere = pNew; pWalker->eCode = 1; } @@ -126573,13 +134447,22 @@ static struct SrcList_item *isSelfJoinView( ){ struct SrcList_item *pItem; for(pItem = pTabList->a; pItempSelect==0 ) continue; if( pItem->fg.viaCoroutine ) continue; if( pItem->zName==0 ) continue; - if( sqlite3_stricmp(pItem->zDatabase, pThis->zDatabase)!=0 ) continue; + assert( pItem->pTab!=0 ); + assert( pThis->pTab!=0 ); + if( pItem->pTab->pSchema!=pThis->pTab->pSchema ) continue; if( sqlite3_stricmp(pItem->zName, pThis->zName)!=0 ) continue; - if( sqlite3ExprCompare(0, - pThis->pSelect->pWhere, pItem->pSelect->pWhere, -1) + pS1 = pItem->pSelect; + if( pItem->pTab->pSchema==0 && pThis->pSelect->selId!=pS1->selId ){ + /* The query flattener left two different CTE tables with identical + ** names in the same FROM clause. */ + continue; + } + if( sqlite3ExprCompare(0, pThis->pSelect->pWhere, pS1->pWhere, -1) + || sqlite3ExprCompare(0, pThis->pSelect->pHaving, pS1->pHaving, -1) ){ /* The view was modified by some other optimization such as ** pushDownWhereTerms() */ @@ -126603,8 +134486,10 @@ static struct SrcList_item *isSelfJoinView( ** The transformation only works if all of the following are true: ** ** * The subquery is a UNION ALL of two or more terms +** * The subquery does not have a LIMIT clause ** * There is no WHERE or GROUP BY or HAVING clauses on the subqueries -** * The outer query is a simple count(*) +** * The outer query is a simple count(*) with no WHERE clause or other +** extraneous syntax. ** ** Return TRUE if the optimization is undertaken. */ @@ -126615,6 +134500,8 @@ static int countOfViewOptimization(Parse *pParse, Select *p){ sqlite3 *db; if( (p->selFlags & SF_Aggregate)==0 ) return 0; /* This is an aggregate */ if( p->pEList->nExpr!=1 ) return 0; /* Single result column */ + if( p->pWhere ) return 0; + if( p->pGroupBy ) return 0; pExpr = p->pEList->a[0].pExpr; if( pExpr->op!=TK_AGG_FUNCTION ) return 0; /* Result is an aggregate */ if( sqlite3_stricmp(pExpr->u.zToken,"count") ) return 0; /* Is count() */ @@ -126626,6 +134513,7 @@ static int countOfViewOptimization(Parse *pParse, Select *p){ do{ if( pSub->op!=TK_ALL && pSub->pPrior ) return 0; /* Must be UNION ALL */ if( pSub->pWhere ) return 0; /* No WHERE clause */ + if( pSub->pLimit ) return 0; /* No LIMIT clause */ if( pSub->selFlags & SF_Aggregate ) return 0; /* Not an aggregate */ pSub = pSub->pPrior; /* Repeat over compound */ }while( pSub ); @@ -126738,14 +134626,10 @@ SQLITE_PRIVATE int sqlite3Select( p->selFlags &= ~SF_Distinct; } sqlite3SelectPrep(pParse, p, 0); - memset(&sSort, 0, sizeof(sSort)); - sSort.pOrderBy = p->pOrderBy; - pTabList = p->pSrc; if( pParse->nErr || db->mallocFailed ){ goto select_end; } assert( p->pEList!=0 ); - isAgg = (p->selFlags & SF_Aggregate)!=0; #if SELECTTRACE_ENABLED if( sqlite3SelectTrace & 0x104 ){ SELECTTRACE(0x104,pParse,p, ("after name resolution:\n")); @@ -126757,6 +134641,24 @@ SQLITE_PRIVATE int sqlite3Select( generateColumnNames(pParse, p); } +#ifndef SQLITE_OMIT_WINDOWFUNC + rc = sqlite3WindowRewrite(pParse, p); + if( rc ){ + assert( db->mallocFailed || pParse->nErr>0 ); + goto select_end; + } +#if SELECTTRACE_ENABLED + if( p->pWin && (sqlite3SelectTrace & 0x108)!=0 ){ + SELECTTRACE(0x104,pParse,p, ("after window rewrite:\n")); + sqlite3TreeViewSelect(0, p, 0); + } +#endif +#endif /* SQLITE_OMIT_WINDOWFUNC */ + pTabList = p->pSrc; + isAgg = (p->selFlags & SF_Aggregate)!=0; + memset(&sSort, 0, sizeof(sSort)); + sSort.pOrderBy = p->pOrderBy; + /* Try to various optimizations (flattening subqueries, and strength ** reduction of join operators) in the FROM clause up into the main query */ @@ -126828,6 +134730,7 @@ SQLITE_PRIVATE int sqlite3Select( } if( flattenSubquery(pParse, p, i, isAgg) ){ + if( pParse->nErr ) goto select_end; /* This subquery can be absorbed into its parent. */ i = -1; } @@ -126856,6 +134759,35 @@ SQLITE_PRIVATE int sqlite3Select( } #endif + /* Do the WHERE-clause constant propagation optimization if this is + ** a join. No need to speed time on this operation for non-join queries + ** as the equivalent optimization will be handled by query planner in + ** sqlite3WhereBegin(). + */ + if( pTabList->nSrc>1 + && OptimizationEnabled(db, SQLITE_PropagateConst) + && propagateConstants(pParse, p) + ){ +#if SELECTTRACE_ENABLED + if( sqlite3SelectTrace & 0x100 ){ + SELECTTRACE(0x100,pParse,p,("After constant propagation:\n")); + sqlite3TreeViewSelect(0, p, 0); + } +#endif + }else{ + SELECTTRACE(0x100,pParse,p,("Constant propagation not helpful\n")); + } + +#ifdef SQLITE_COUNTOFVIEW_OPTIMIZATION + if( OptimizationEnabled(db, SQLITE_QueryFlattener|SQLITE_CountOfView) + && countOfViewOptimization(pParse, p) + ){ + if( db->mallocFailed ) goto select_end; + pEList = p->pEList; + pTabList = p->pSrc; + } +#endif + /* For each term in the FROM clause, do two things: ** (1) Authorized unreferenced tables ** (2) Generate code for all sub-queries @@ -126884,7 +134816,7 @@ SQLITE_PRIVATE int sqlite3Select( ** assume the column name is non-NULL and segfault. The use of an empty ** string for the fake column name seems safer. */ - if( pItem->colUsed==0 ){ + if( pItem->colUsed==0 && pItem->zName!=0 ){ sqlite3AuthCheck(pParse, SQLITE_READ, pItem->zName, "", pItem->zDatabase); } @@ -126894,22 +134826,19 @@ SQLITE_PRIVATE int sqlite3Select( pSub = pItem->pSelect; if( pSub==0 ) continue; - /* Sometimes the code for a subquery will be generated more than - ** once, if the subquery is part of the WHERE clause in a LEFT JOIN, - ** for example. In that case, do not regenerate the code to manifest - ** a view or the co-routine to implement a view. The first instance - ** is sufficient, though the subroutine to manifest the view does need - ** to be invoked again. */ - if( pItem->addrFillSub ){ - if( pItem->fg.viaCoroutine==0 ){ - /* The subroutine that manifests the view might be a one-time routine, - ** or it might need to be rerun on each iteration because it - ** encodes a correlated subquery. */ - testcase( sqlite3VdbeGetOp(v, pItem->addrFillSub)->opcode==OP_Once ); - sqlite3VdbeAddOp2(v, OP_Gosub, pItem->regReturn, pItem->addrFillSub); - } - continue; - } + /* The code for a subquery should only be generated once, though it is + ** technically harmless for it to be generated multiple times. The + ** following assert() will detect if something changes to cause + ** the same subquery to be coded multiple times, as a signal to the + ** developers to try to optimize the situation. + ** + ** Update 2019-07-24: + ** See ticket https://sqlite.org/src/tktview/c52b09c7f38903b1311cec40. + ** The dbsqlfuzz fuzzer found a case where the same subquery gets + ** coded twice. So this assert() now becomes a testcase(). It should + ** be very rare, though. + */ + testcase( pItem->addrFillSub!=0 ); /* Increment Parse.nHeight by the height of the largest expression ** tree referred to by this, the parent select. The child select @@ -126929,7 +134858,8 @@ SQLITE_PRIVATE int sqlite3Select( ){ #if SELECTTRACE_ENABLED if( sqlite3SelectTrace & 0x100 ){ - SELECTTRACE(0x100,pParse,p,("After WHERE-clause push-down:\n")); + SELECTTRACE(0x100,pParse,p, + ("After WHERE-clause push-down into subquery %d:\n", pSub->selId)); sqlite3TreeViewSelect(0, p, 0); } #endif @@ -126963,7 +134893,7 @@ SQLITE_PRIVATE int sqlite3Select( VdbeComment((v, "%s", pItem->pTab->zName)); pItem->addrFillSub = addrTop; sqlite3SelectDestInit(&dest, SRT_Coroutine, pItem->regReturn); - ExplainQueryPlan((pParse, 1, "CO-ROUTINE 0x%p", pSub)); + ExplainQueryPlan((pParse, 1, "CO-ROUTINE %u", pSub->selId)); sqlite3Select(pParse, pSub, &dest); pItem->pTab->nRowLogEst = pSub->nSelectRow; pItem->fg.viaCoroutine = 1; @@ -126982,7 +134912,7 @@ SQLITE_PRIVATE int sqlite3Select( int retAddr; struct SrcList_item *pPrior; - assert( pItem->addrFillSub==0 ); + testcase( pItem->addrFillSub==0 ); /* Ticket c52b09c7f38903b1311 */ pItem->regReturn = ++pParse->nMem; topAddr = sqlite3VdbeAddOp2(v, OP_Integer, 0, pItem->regReturn); pItem->addrFillSub = topAddr+1; @@ -127002,7 +134932,7 @@ SQLITE_PRIVATE int sqlite3Select( pSub->nSelectRow = pPrior->pSelect->nSelectRow; }else{ sqlite3SelectDestInit(&dest, SRT_EphemTab, pItem->iCursor); - ExplainQueryPlan((pParse, 1, "MATERIALIZE 0x%p", pSub)); + ExplainQueryPlan((pParse, 1, "MATERIALIZE %u", pSub->selId)); sqlite3Select(pParse, pSub, &dest); } pItem->pTab->nRowLogEst = pSub->nSelectRow; @@ -127033,16 +134963,6 @@ SQLITE_PRIVATE int sqlite3Select( } #endif -#ifdef SQLITE_COUNTOFVIEW_OPTIMIZATION - if( OptimizationEnabled(db, SQLITE_QueryFlattener|SQLITE_CountOfView) - && countOfViewOptimization(pParse, p) - ){ - if( db->mallocFailed ) goto select_end; - pEList = p->pEList; - pTabList = p->pSrc; - } -#endif - /* If the query is DISTINCT with an ORDER BY but is not an aggregate, and ** if the select-list is the same as the ORDER BY list, then this query ** can be rewritten as a GROUP BY. In other words, this: @@ -127060,9 +134980,13 @@ SQLITE_PRIVATE int sqlite3Select( */ if( (p->selFlags & (SF_Distinct|SF_Aggregate))==SF_Distinct && sqlite3ExprListCompare(sSort.pOrderBy, pEList, -1)==0 +#ifndef SQLITE_OMIT_WINDOWFUNC + && p->pWin==0 +#endif ){ p->selFlags &= ~SF_Distinct; pGroupBy = p->pGroupBy = sqlite3ExprListDup(db, pEList, 0); + p->selFlags |= SF_Aggregate; /* Notice that even thought SF_Distinct has been cleared from p->selFlags, ** the sDistinct.isTnct is still set. Hence, isTnct represents the ** original setting of the SF_Distinct flag, not the current setting */ @@ -127086,7 +135010,8 @@ SQLITE_PRIVATE int sqlite3Select( */ if( sSort.pOrderBy ){ KeyInfo *pKeyInfo; - pKeyInfo = keyInfoFromExprList(pParse, sSort.pOrderBy, 0, pEList->nExpr); + pKeyInfo = sqlite3KeyInfoFromExprList( + pParse, sSort.pOrderBy, 0, pEList->nExpr); sSort.iECursor = pParse->nTab++; sSort.addrSortIndex = sqlite3VdbeAddOp4(v, OP_OpenEphemeral, @@ -127105,7 +135030,7 @@ SQLITE_PRIVATE int sqlite3Select( /* Set the limiter. */ - iEnd = sqlite3VdbeMakeLabel(v); + iEnd = sqlite3VdbeMakeLabel(pParse); if( (p->selFlags & SF_FixedLimit)==0 ){ p->nSelectRow = 320; /* 4 billion rows */ } @@ -127120,9 +135045,9 @@ SQLITE_PRIVATE int sqlite3Select( if( p->selFlags & SF_Distinct ){ sDistinct.tabTnct = pParse->nTab++; sDistinct.addrTnct = sqlite3VdbeAddOp4(v, OP_OpenEphemeral, - sDistinct.tabTnct, 0, 0, - (char*)keyInfoFromExprList(pParse, p->pEList,0,0), - P4_KEYINFO); + sDistinct.tabTnct, 0, 0, + (char*)sqlite3KeyInfoFromExprList(pParse, p->pEList,0,0), + P4_KEYINFO); sqlite3VdbeChangeP5(v, BTREE_UNORDERED); sDistinct.eTnctType = WHERE_DISTINCT_UNORDERED; }else{ @@ -127131,9 +135056,16 @@ SQLITE_PRIVATE int sqlite3Select( if( !isAgg && pGroupBy==0 ){ /* No aggregate functions and no GROUP BY clause */ - u16 wctrlFlags = (sDistinct.isTnct ? WHERE_WANT_DISTINCT : 0); + u16 wctrlFlags = (sDistinct.isTnct ? WHERE_WANT_DISTINCT : 0) + | (p->selFlags & SF_FixedLimit); +#ifndef SQLITE_OMIT_WINDOWFUNC + Window *pWin = p->pWin; /* Master window object (or NULL) */ + if( pWin ){ + sqlite3WindowCodeInit(pParse, p); + } +#endif assert( WHERE_USE_LIMIT==SF_FixedLimit ); - wctrlFlags |= p->selFlags & SF_FixedLimit; + /* Begin the database scan. */ SELECTTRACE(1,pParse,p,("WhereBegin\n")); @@ -127148,7 +135080,7 @@ SQLITE_PRIVATE int sqlite3Select( } if( sSort.pOrderBy ){ sSort.nOBSat = sqlite3WhereIsOrdered(pWInfo); - sSort.bOrderedInnerLoop = sqlite3WhereOrderedInnerLoop(pWInfo); + sSort.labelOBLopt = sqlite3WhereOrderByLimitOptLabel(pWInfo); if( sSort.nOBSat==sSort.pOrderBy->nExpr ){ sSort.pOrderBy = 0; } @@ -127162,15 +135094,37 @@ SQLITE_PRIVATE int sqlite3Select( sqlite3VdbeChangeToNoop(v, sSort.addrSortIndex); } - /* Use the standard inner loop. */ assert( p->pEList==pEList ); - selectInnerLoop(pParse, p, -1, &sSort, &sDistinct, pDest, - sqlite3WhereContinueLabel(pWInfo), - sqlite3WhereBreakLabel(pWInfo)); +#ifndef SQLITE_OMIT_WINDOWFUNC + if( pWin ){ + int addrGosub = sqlite3VdbeMakeLabel(pParse); + int iCont = sqlite3VdbeMakeLabel(pParse); + int iBreak = sqlite3VdbeMakeLabel(pParse); + int regGosub = ++pParse->nMem; + + sqlite3WindowCodeStep(pParse, p, pWInfo, regGosub, addrGosub); + + sqlite3VdbeAddOp2(v, OP_Goto, 0, iBreak); + sqlite3VdbeResolveLabel(v, addrGosub); + VdbeNoopComment((v, "inner-loop subroutine")); + sSort.labelOBLopt = 0; + selectInnerLoop(pParse, p, -1, &sSort, &sDistinct, pDest, iCont, iBreak); + sqlite3VdbeResolveLabel(v, iCont); + sqlite3VdbeAddOp1(v, OP_Return, regGosub); + VdbeComment((v, "end inner-loop subroutine")); + sqlite3VdbeResolveLabel(v, iBreak); + }else +#endif /* SQLITE_OMIT_WINDOWFUNC */ + { + /* Use the standard inner loop. */ + selectInnerLoop(pParse, p, -1, &sSort, &sDistinct, pDest, + sqlite3WhereContinueLabel(pWInfo), + sqlite3WhereBreakLabel(pWInfo)); - /* End the database scan loop. - */ - sqlite3WhereEnd(pWInfo); + /* End the database scan loop. + */ + sqlite3WhereEnd(pWInfo); + } }else{ /* This case when there exist aggregate functions or a GROUP BY clause ** or both */ @@ -127202,25 +135156,37 @@ SQLITE_PRIVATE int sqlite3Select( } assert( 66==sqlite3LogEst(100) ); if( p->nSelectRow>66 ) p->nSelectRow = 66; + + /* If there is both a GROUP BY and an ORDER BY clause and they are + ** identical, then it may be possible to disable the ORDER BY clause + ** on the grounds that the GROUP BY will cause elements to come out + ** in the correct order. It also may not - the GROUP BY might use a + ** database index that causes rows to be grouped together as required + ** but not actually sorted. Either way, record the fact that the + ** ORDER BY and GROUP BY clauses are the same by setting the orderByGrp + ** variable. */ + if( sSort.pOrderBy && pGroupBy->nExpr==sSort.pOrderBy->nExpr ){ + int ii; + /* The GROUP BY processing doesn't care whether rows are delivered in + ** ASC or DESC order - only that each group is returned contiguously. + ** So set the ASC/DESC flags in the GROUP BY to match those in the + ** ORDER BY to maximize the chances of rows being delivered in an + ** order that makes the ORDER BY redundant. */ + for(ii=0; iinExpr; ii++){ + u8 sortFlags = sSort.pOrderBy->a[ii].sortFlags & KEYINFO_ORDER_DESC; + pGroupBy->a[ii].sortFlags = sortFlags; + } + if( sqlite3ExprListCompare(pGroupBy, sSort.pOrderBy, -1)==0 ){ + orderByGrp = 1; + } + } }else{ assert( 0==sqlite3LogEst(1) ); p->nSelectRow = 0; } - /* If there is both a GROUP BY and an ORDER BY clause and they are - ** identical, then it may be possible to disable the ORDER BY clause - ** on the grounds that the GROUP BY will cause elements to come out - ** in the correct order. It also may not - the GROUP BY might use a - ** database index that causes rows to be grouped together as required - ** but not actually sorted. Either way, record the fact that the - ** ORDER BY and GROUP BY clauses are the same by setting the orderByGrp - ** variable. */ - if( sqlite3ExprListCompare(pGroupBy, sSort.pOrderBy, -1)==0 ){ - orderByGrp = 1; - } - /* Create a label to jump to when we want to abort the query */ - addrEnd = sqlite3VdbeMakeLabel(v); + addrEnd = sqlite3VdbeMakeLabel(pParse); /* Convert TK_COLUMN nodes into TK_AGG_COLUMN and make entries in ** sAggInfo for all TK_AGG_FUNCTION nodes in expressions of the @@ -127253,9 +135219,16 @@ SQLITE_PRIVATE int sqlite3Select( minMaxFlag = WHERE_ORDERBY_NORMAL; } for(i=0; ix.pList); + sqlite3ExprAnalyzeAggList(&sNC, pExpr->x.pList); +#ifndef SQLITE_OMIT_WINDOWFUNC + assert( !IsWindowFunc(pExpr) ); + if( ExprHasProperty(pExpr, EP_WinFunc) ){ + sqlite3ExprAnalyzeAggregates(&sNC, pExpr->y.pWin->pFilter); + } +#endif sNC.ncFlags &= ~NC_InAggFunc; } sAggInfo.mxReg = pParse->nMem; @@ -127263,7 +135236,7 @@ SQLITE_PRIVATE int sqlite3Select( #if SELECTTRACE_ENABLED if( sqlite3SelectTrace & 0x400 ){ int ii; - SELECTTRACE(0x400,pParse,p,("After aggregate analysis:\n")); + SELECTTRACE(0x400,pParse,p,("After aggregate analysis %p:\n", &sAggInfo)); sqlite3TreeViewSelect(0, p, 0); for(ii=0; iinTab++; - pKeyInfo = keyInfoFromExprList(pParse, pGroupBy, 0, sAggInfo.nColumn); + pKeyInfo = sqlite3KeyInfoFromExprList(pParse,pGroupBy,0,sAggInfo.nColumn); addrSortingIdx = sqlite3VdbeAddOp4(v, OP_SorterOpen, sAggInfo.sortingIdx, sAggInfo.nSortingColumn, 0, (char*)pKeyInfo, P4_KEYINFO); @@ -127309,17 +135282,15 @@ SQLITE_PRIVATE int sqlite3Select( iUseFlag = ++pParse->nMem; iAbortFlag = ++pParse->nMem; regOutputRow = ++pParse->nMem; - addrOutputRow = sqlite3VdbeMakeLabel(v); + addrOutputRow = sqlite3VdbeMakeLabel(pParse); regReset = ++pParse->nMem; - addrReset = sqlite3VdbeMakeLabel(v); + addrReset = sqlite3VdbeMakeLabel(pParse); iAMem = pParse->nMem + 1; pParse->nMem += pGroupBy->nExpr; iBMem = pParse->nMem + 1; pParse->nMem += pGroupBy->nExpr; sqlite3VdbeAddOp2(v, OP_Integer, 0, iAbortFlag); VdbeComment((v, "clear abort flag")); - sqlite3VdbeAddOp2(v, OP_Integer, 0, iUseFlag); - VdbeComment((v, "indicate accumulator empty")); sqlite3VdbeAddOp3(v, OP_Null, 0, iAMem, iAMem+pGroupBy->nExpr-1); /* Begin a loop that will extract all source rows in GROUP BY order. @@ -127365,15 +135336,14 @@ SQLITE_PRIVATE int sqlite3Select( } } regBase = sqlite3GetTempRange(pParse, nCol); - sqlite3ExprCacheClear(pParse); sqlite3ExprCodeExprList(pParse, pGroupBy, regBase, 0, 0); j = nGroupBy; for(i=0; iiSorterColumn>=j ){ int r1 = j + regBase; - sqlite3ExprCodeGetColumnToReg(pParse, - pCol->pTab, pCol->iColumn, pCol->iTable, r1); + sqlite3ExprCodeGetColumnOfTable(v, + pCol->pTab, pCol->iTable, pCol->iColumn, r1); j++; } } @@ -127389,8 +135359,6 @@ SQLITE_PRIVATE int sqlite3Select( sqlite3VdbeAddOp2(v, OP_SorterSort, sAggInfo.sortingIdx, addrEnd); VdbeComment((v, "GROUP BY sort")); VdbeCoverage(v); sAggInfo.useSortingIdx = 1; - sqlite3ExprCacheClear(pParse); - } /* If the index or temporary table used by the GROUP BY sort @@ -127413,7 +135381,6 @@ SQLITE_PRIVATE int sqlite3Select( ** from the previous row currently stored in a0, a1, a2... */ addrTopOfLoop = sqlite3VdbeCurrentAddr(v); - sqlite3ExprCacheClear(pParse); if( groupBySort ){ sqlite3VdbeAddOp3(v, OP_SorterData, sAggInfo.sortingIdx, sortOut, sortPTab); @@ -127452,7 +135419,7 @@ SQLITE_PRIVATE int sqlite3Select( ** the current row */ sqlite3VdbeJumpHere(v, addr1); - updateAccumulator(pParse, &sAggInfo); + updateAccumulator(pParse, iUseFlag, &sAggInfo); sqlite3VdbeAddOp2(v, OP_Integer, 1, iUseFlag); VdbeComment((v, "indicate data in accumulator")); @@ -127504,11 +135471,12 @@ SQLITE_PRIVATE int sqlite3Select( */ sqlite3VdbeResolveLabel(v, addrReset); resetAccumulator(pParse, &sAggInfo); + sqlite3VdbeAddOp2(v, OP_Integer, 0, iUseFlag); + VdbeComment((v, "indicate accumulator empty")); sqlite3VdbeAddOp1(v, OP_Return, regReset); } /* endif pGroupBy. Begin aggregate queries without GROUP BY: */ else { -#ifndef SQLITE_OMIT_BTREECOUNT Table *pTab; if( (pTab = isSimpleCount(p, &sAggInfo))!=0 ){ /* If isSimpleCount() returns a pointer to a Table structure, then @@ -127544,13 +135512,15 @@ SQLITE_PRIVATE int sqlite3Select( ** passed to keep OP_OpenRead happy. */ if( !HasRowid(pTab) ) pBest = sqlite3PrimaryKeyIndex(pTab); - for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ - if( pIdx->bUnordered==0 - && pIdx->szIdxRowszTabRow - && pIdx->pPartIdxWhere==0 - && (!pBest || pIdx->szIdxRowszIdxRow) - ){ - pBest = pIdx; + if( !p->pSrc->a[0].fg.notIndexed ){ + for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + if( pIdx->bUnordered==0 + && pIdx->szIdxRowszTabRow + && pIdx->pPartIdxWhere==0 + && (!pBest || pIdx->szIdxRowszIdxRow) + ){ + pBest = pIdx; + } } } if( pBest ){ @@ -127566,9 +135536,29 @@ SQLITE_PRIVATE int sqlite3Select( sqlite3VdbeAddOp2(v, OP_Count, iCsr, sAggInfo.aFunc[0].iMem); sqlite3VdbeAddOp1(v, OP_Close, iCsr); explainSimpleCount(pParse, pTab, pBest); - }else -#endif /* SQLITE_OMIT_BTREECOUNT */ - { + }else{ + int regAcc = 0; /* "populate accumulators" flag */ + + /* If there are accumulator registers but no min() or max() functions + ** without FILTER clauses, allocate register regAcc. Register regAcc + ** will contain 0 the first time the inner loop runs, and 1 thereafter. + ** The code generated by updateAccumulator() uses this to ensure + ** that the accumulator registers are (a) updated only once if + ** there are no min() or max functions or (b) always updated for the + ** first row visited by the aggregate, so that they are updated at + ** least once even if the FILTER clause means the min() or max() + ** function visits zero rows. */ + if( sAggInfo.nAccumulator ){ + for(i=0; ifuncFlags&SQLITE_FUNC_NEEDCOLL ) break; + } + if( i==sAggInfo.nFunc ){ + regAcc = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Integer, 0, regAcc); + } + } + /* This case runs if the aggregate has no GROUP BY clause. The ** processing is much simpler since there is only a single row ** of output. @@ -127590,7 +135580,8 @@ SQLITE_PRIVATE int sqlite3Select( if( pWInfo==0 ){ goto select_end; } - updateAccumulator(pParse, &sAggInfo); + updateAccumulator(pParse, regAcc, &sAggInfo); + if( regAcc ) sqlite3VdbeAddOp2(v, OP_Integer, 1, regAcc); if( sqlite3WhereIsOrdered(pWInfo)>0 ){ sqlite3VdbeGoto(v, sqlite3WhereBreakLabel(pWInfo)); VdbeComment((v, "%s() by index", @@ -127708,7 +135699,7 @@ static int sqlite3_get_table_cb(void *pArg, int nCol, char **argv, char **colv){ if( p->nData + need > p->nAlloc ){ char **azNew; p->nAlloc = p->nAlloc*2 + need; - azNew = sqlite3_realloc64( p->azResult, sizeof(char*)*p->nAlloc ); + azNew = sqlite3Realloc( p->azResult, sizeof(char*)*p->nAlloc ); if( azNew==0 ) goto malloc_failed; p->azResult = azNew; } @@ -127817,7 +135808,7 @@ SQLITE_API int sqlite3_get_table( } if( res.nAlloc>res.nData ){ char **azNew; - azNew = sqlite3_realloc64( res.azResult, sizeof(char*)*res.nData ); + azNew = sqlite3Realloc( res.azResult, sizeof(char*)*res.nData ); if( azNew==0 ){ sqlite3_free_table(&res.azResult[1]); db->errCode = SQLITE_NOMEM; @@ -128030,18 +136021,24 @@ SQLITE_PRIVATE void sqlite3BeginTrigger( /* Check that the trigger name is not reserved and that no trigger of the ** specified name exists */ zName = sqlite3NameFromToken(db, pName); - if( !zName || SQLITE_OK!=sqlite3CheckObjectName(pParse, zName) ){ + if( zName==0 ){ + assert( db->mallocFailed ); + goto trigger_cleanup; + } + if( sqlite3CheckObjectName(pParse, zName, "trigger", pTab->zName) ){ goto trigger_cleanup; } assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); - if( sqlite3HashFind(&(db->aDb[iDb].pSchema->trigHash),zName) ){ - if( !noErr ){ - sqlite3ErrorMsg(pParse, "trigger %T already exists", pName); - }else{ - assert( !db->init.busy ); - sqlite3CodeVerifySchema(pParse, iDb); + if( !IN_RENAME_OBJECT ){ + if( sqlite3HashFind(&(db->aDb[iDb].pSchema->trigHash),zName) ){ + if( !noErr ){ + sqlite3ErrorMsg(pParse, "trigger %T already exists", pName); + }else{ + assert( !db->init.busy ); + sqlite3CodeVerifySchema(pParse, iDb); + } + goto trigger_cleanup; } - goto trigger_cleanup; } /* Do not create a trigger on a system table */ @@ -128065,7 +136062,7 @@ SQLITE_PRIVATE void sqlite3BeginTrigger( } #ifndef SQLITE_OMIT_AUTHORIZATION - { + if( !IN_RENAME_OBJECT ){ int iTabDb = sqlite3SchemaToIndex(db, pTab->pSchema); int code = SQLITE_CREATE_TRIGGER; const char *zDb = db->aDb[iTabDb].zDbSName; @@ -128099,8 +136096,15 @@ SQLITE_PRIVATE void sqlite3BeginTrigger( pTrigger->pTabSchema = pTab->pSchema; pTrigger->op = (u8)op; pTrigger->tr_tm = tr_tm==TK_BEFORE ? TRIGGER_BEFORE : TRIGGER_AFTER; - pTrigger->pWhen = sqlite3ExprDup(db, pWhen, EXPRDUP_REDUCE); - pTrigger->pColumns = sqlite3IdListDup(db, pColumns); + if( IN_RENAME_OBJECT ){ + sqlite3RenameTokenRemap(pParse, pTrigger->table, pTableName->a[0].zName); + pTrigger->pWhen = pWhen; + pWhen = 0; + }else{ + pTrigger->pWhen = sqlite3ExprDup(db, pWhen, EXPRDUP_REDUCE); + } + pTrigger->pColumns = pColumns; + pColumns = 0; assert( pParse->pNewTrigger==0 ); pParse->pNewTrigger = pTrigger; @@ -128149,6 +136153,14 @@ SQLITE_PRIVATE void sqlite3FinishTrigger( goto triggerfinish_cleanup; } +#ifndef SQLITE_OMIT_ALTERTABLE + if( IN_RENAME_OBJECT ){ + assert( !db->init.busy ); + pParse->pNewTrigger = pTrig; + pTrig = 0; + }else +#endif + /* if we are not initializing, ** build the sqlite_master entry */ @@ -128176,6 +136188,7 @@ SQLITE_PRIVATE void sqlite3FinishTrigger( Trigger *pLink = pTrig; Hash *pHash = &db->aDb[iDb].pSchema->trigHash; assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + assert( pLink!=0 ); pTrig = sqlite3HashInsert(pHash, zName, pTrig); if( pTrig ){ sqlite3OomFault(db); @@ -128190,7 +136203,7 @@ SQLITE_PRIVATE void sqlite3FinishTrigger( triggerfinish_cleanup: sqlite3DeleteTrigger(db, pTrig); - assert( !pParse->pNewTrigger ); + assert( IN_RENAME_OBJECT || !pParse->pNewTrigger ); sqlite3DeleteTriggerStep(db, pStepList); } @@ -128237,12 +136250,13 @@ SQLITE_PRIVATE TriggerStep *sqlite3TriggerSelectStep( ** If an OOM error occurs, NULL is returned and db->mallocFailed is set. */ static TriggerStep *triggerStepAllocate( - sqlite3 *db, /* Database connection */ + Parse *pParse, /* Parser context */ u8 op, /* Trigger opcode */ Token *pName, /* The target name */ const char *zStart, /* Start of SQL text */ const char *zEnd /* End of SQL text */ ){ + sqlite3 *db = pParse->db; TriggerStep *pTriggerStep; pTriggerStep = sqlite3DbMallocZero(db, sizeof(TriggerStep) + pName->n + 1); @@ -128253,6 +136267,9 @@ static TriggerStep *triggerStepAllocate( pTriggerStep->zTarget = z; pTriggerStep->op = op; pTriggerStep->zSpan = triggerSpanDup(db, zStart, zEnd); + if( IN_RENAME_OBJECT ){ + sqlite3RenameTokenMap(pParse, pTriggerStep->zTarget, pName); + } } return pTriggerStep; } @@ -128265,7 +136282,7 @@ static TriggerStep *triggerStepAllocate( ** body of a trigger. */ SQLITE_PRIVATE TriggerStep *sqlite3TriggerInsertStep( - sqlite3 *db, /* The database connection */ + Parse *pParse, /* Parser */ Token *pTableName, /* Name of the table into which we insert */ IdList *pColumn, /* List of columns in pTableName to insert into */ Select *pSelect, /* A SELECT statement that supplies values */ @@ -128274,16 +136291,25 @@ SQLITE_PRIVATE TriggerStep *sqlite3TriggerInsertStep( const char *zStart, /* Start of SQL text */ const char *zEnd /* End of SQL text */ ){ + sqlite3 *db = pParse->db; TriggerStep *pTriggerStep; assert(pSelect != 0 || db->mallocFailed); - pTriggerStep = triggerStepAllocate(db, TK_INSERT, pTableName, zStart, zEnd); + pTriggerStep = triggerStepAllocate(pParse, TK_INSERT, pTableName,zStart,zEnd); if( pTriggerStep ){ - pTriggerStep->pSelect = sqlite3SelectDup(db, pSelect, EXPRDUP_REDUCE); + if( IN_RENAME_OBJECT ){ + pTriggerStep->pSelect = pSelect; + pSelect = 0; + }else{ + pTriggerStep->pSelect = sqlite3SelectDup(db, pSelect, EXPRDUP_REDUCE); + } pTriggerStep->pIdList = pColumn; pTriggerStep->pUpsert = pUpsert; pTriggerStep->orconf = orconf; + if( pUpsert ){ + sqlite3HasExplicitNulls(pParse, pUpsert->pUpsertTarget); + } }else{ testcase( pColumn ); sqlite3IdListDelete(db, pColumn); @@ -128301,7 +136327,7 @@ SQLITE_PRIVATE TriggerStep *sqlite3TriggerInsertStep( ** sees an UPDATE statement inside the body of a CREATE TRIGGER. */ SQLITE_PRIVATE TriggerStep *sqlite3TriggerUpdateStep( - sqlite3 *db, /* The database connection */ + Parse *pParse, /* Parser */ Token *pTableName, /* Name of the table to be updated */ ExprList *pEList, /* The SET clause: list of column and new values */ Expr *pWhere, /* The WHERE clause */ @@ -128309,12 +136335,20 @@ SQLITE_PRIVATE TriggerStep *sqlite3TriggerUpdateStep( const char *zStart, /* Start of SQL text */ const char *zEnd /* End of SQL text */ ){ + sqlite3 *db = pParse->db; TriggerStep *pTriggerStep; - pTriggerStep = triggerStepAllocate(db, TK_UPDATE, pTableName, zStart, zEnd); + pTriggerStep = triggerStepAllocate(pParse, TK_UPDATE, pTableName,zStart,zEnd); if( pTriggerStep ){ - pTriggerStep->pExprList = sqlite3ExprListDup(db, pEList, EXPRDUP_REDUCE); - pTriggerStep->pWhere = sqlite3ExprDup(db, pWhere, EXPRDUP_REDUCE); + if( IN_RENAME_OBJECT ){ + pTriggerStep->pExprList = pEList; + pTriggerStep->pWhere = pWhere; + pEList = 0; + pWhere = 0; + }else{ + pTriggerStep->pExprList = sqlite3ExprListDup(db, pEList, EXPRDUP_REDUCE); + pTriggerStep->pWhere = sqlite3ExprDup(db, pWhere, EXPRDUP_REDUCE); + } pTriggerStep->orconf = orconf; } sqlite3ExprListDelete(db, pEList); @@ -128328,17 +136362,23 @@ SQLITE_PRIVATE TriggerStep *sqlite3TriggerUpdateStep( ** sees a DELETE statement inside the body of a CREATE TRIGGER. */ SQLITE_PRIVATE TriggerStep *sqlite3TriggerDeleteStep( - sqlite3 *db, /* Database connection */ + Parse *pParse, /* Parser */ Token *pTableName, /* The table from which rows are deleted */ Expr *pWhere, /* The WHERE clause */ const char *zStart, /* Start of SQL text */ const char *zEnd /* End of SQL text */ ){ + sqlite3 *db = pParse->db; TriggerStep *pTriggerStep; - pTriggerStep = triggerStepAllocate(db, TK_DELETE, pTableName, zStart, zEnd); + pTriggerStep = triggerStepAllocate(pParse, TK_DELETE, pTableName,zStart,zEnd); if( pTriggerStep ){ - pTriggerStep->pWhere = sqlite3ExprDup(db, pWhere, EXPRDUP_REDUCE); + if( IN_RENAME_OBJECT ){ + pTriggerStep->pWhere = pWhere; + pWhere = 0; + }else{ + pTriggerStep->pWhere = sqlite3ExprDup(db, pWhere, EXPRDUP_REDUCE); + } pTriggerStep->orconf = OE_Default; } sqlite3ExprDelete(db, pWhere); @@ -128384,7 +136424,7 @@ SQLITE_PRIVATE void sqlite3DropTrigger(Parse *pParse, SrcList *pName, int noErr) assert( zDb!=0 || sqlite3BtreeHoldsAllMutexes(db) ); for(i=OMIT_TEMPDB; inDb; i++){ int j = (i<2) ? i^1 : i; /* Search TEMP before MAIN */ - if( zDb && sqlite3StrICmp(db->aDb[j].zDbSName, zDb) ) continue; + if( zDb && sqlite3DbIsNamed(db, j, zDb)==0 ) continue; assert( sqlite3SchemaMutexHeld(db, j, 0) ); pTrigger = sqlite3HashFind(&(db->aDb[j].pSchema->trigHash), zName); if( pTrigger ) break; @@ -128425,10 +136465,9 @@ SQLITE_PRIVATE void sqlite3DropTriggerPtr(Parse *pParse, Trigger *pTrigger){ iDb = sqlite3SchemaToIndex(pParse->db, pTrigger->pSchema); assert( iDb>=0 && iDbnDb ); pTable = tableOfTrigger(pTrigger); - assert( pTable ); - assert( pTable->pSchema==pTrigger->pSchema || iDb==1 ); + assert( (pTable && pTable->pSchema==pTrigger->pSchema) || iDb==1 ); #ifndef SQLITE_OMIT_AUTHORIZATION - { + if( pTable ){ int code = SQLITE_DROP_TRIGGER; const char *zDb = db->aDb[iDb].zDbSName; const char *zTab = SCHEMA_TABLE(iDb); @@ -128442,7 +136481,6 @@ SQLITE_PRIVATE void sqlite3DropTriggerPtr(Parse *pParse, Trigger *pTrigger){ /* Generate code to destroy the database record of the trigger. */ - assert( pTable!=0 ); if( (v = sqlite3GetVdbe(pParse))!=0 ){ sqlite3NestedParse(pParse, "DELETE FROM %Q.%s WHERE name=%Q AND type='trigger'", @@ -128466,9 +136504,15 @@ SQLITE_PRIVATE void sqlite3UnlinkAndDeleteTrigger(sqlite3 *db, int iDb, const ch if( ALWAYS(pTrigger) ){ if( pTrigger->pSchema==pTrigger->pTabSchema ){ Table *pTab = tableOfTrigger(pTrigger); - Trigger **pp; - for(pp=&pTab->pTrigger; *pp!=pTrigger; pp=&((*pp)->pNext)); - *pp = (*pp)->pNext; + if( pTab ){ + Trigger **pp; + for(pp=&pTab->pTrigger; *pp; pp=&((*pp)->pNext)){ + if( *pp==pTrigger ){ + *pp = (*pp)->pNext; + break; + } + } + } } sqlite3DeleteTrigger(db, pTrigger); db->mDbFlags |= DBFLAG_SchemaChange; @@ -128488,7 +136532,7 @@ static int checkColumnOverlap(IdList *pIdList, ExprList *pEList){ int e; if( pIdList==0 || NEVER(pEList==0) ) return 1; for(e=0; enExpr; e++){ - if( sqlite3IdListIndex(pIdList, pEList->a[e].zName)>=0 ) return 1; + if( sqlite3IdListIndex(pIdList, pEList->a[e].zEName)>=0 ) return 1; } return 0; } @@ -128543,7 +136587,7 @@ static SrcList *targetSrcList( int iDb; /* Index of the database to use */ SrcList *pSrc; /* SrcList to be returned */ - pSrc = sqlite3SrcListAppend(db, 0, 0, 0); + pSrc = sqlite3SrcListAppend(pParse, 0, 0, 0); if( pSrc ){ assert( pSrc->nSrc>0 ); pSrc->a[pSrc->nSrc-1].zName = sqlite3DbStrDup(db, pStep->zTarget); @@ -128728,6 +136772,7 @@ static TriggerPrg *codeRowTrigger( pSubParse->zAuthContext = pTrigger->zName; pSubParse->eTriggerOp = pTrigger->op; pSubParse->nQueryLoop = pParse->nQueryLoop; + pSubParse->disableVtab = pParse->disableVtab; v = sqlite3GetVdbe(pSubParse); if( v ){ @@ -128755,7 +136800,7 @@ static TriggerPrg *codeRowTrigger( if( SQLITE_OK==sqlite3ResolveExprNames(&sNC, pWhen) && db->mallocFailed==0 ){ - iEndTrigger = sqlite3VdbeMakeLabel(v); + iEndTrigger = sqlite3VdbeMakeLabel(pSubParse); sqlite3ExprIfFalse(pSubParse, pWhen, iEndTrigger, SQLITE_JUMPIFNULL); } sqlite3ExprDelete(db, pWhen); @@ -129053,10 +137098,10 @@ static void updateVirtualTable( ** function is capable of transforming these types of expressions into ** sqlite3_value objects. ** -** If parameter iReg is not negative, code an OP_RealAffinity instruction -** on register iReg. This is used when an equivalent integer value is -** stored in place of an 8-byte floating point value in order to save -** space. +** If column as REAL affinity and the table is an ordinary b-tree table +** (not a virtual table) then the value might have been stored as an +** integer. In that case, add an OP_RealAffinity opcode to make sure +** it has been converted into REAL. */ SQLITE_PRIVATE void sqlite3ColumnDefault(Vdbe *v, Table *pTab, int i, int iReg){ assert( pTab!=0 ); @@ -129073,12 +137118,63 @@ SQLITE_PRIVATE void sqlite3ColumnDefault(Vdbe *v, Table *pTab, int i, int iReg){ } } #ifndef SQLITE_OMIT_FLOATING_POINT - if( pTab->aCol[i].affinity==SQLITE_AFF_REAL ){ + if( pTab->aCol[i].affinity==SQLITE_AFF_REAL && !IsVirtual(pTab) ){ sqlite3VdbeAddOp1(v, OP_RealAffinity, iReg); } #endif } +/* +** Check to see if column iCol of index pIdx references any of the +** columns defined by aXRef and chngRowid. Return true if it does +** and false if not. This is an optimization. False-positives are a +** performance degradation, but false-negatives can result in a corrupt +** index and incorrect answers. +** +** aXRef[j] will be non-negative if column j of the original table is +** being updated. chngRowid will be true if the rowid of the table is +** being updated. +*/ +static int indexColumnIsBeingUpdated( + Index *pIdx, /* The index to check */ + int iCol, /* Which column of the index to check */ + int *aXRef, /* aXRef[j]>=0 if column j is being updated */ + int chngRowid /* true if the rowid is being updated */ +){ + i16 iIdxCol = pIdx->aiColumn[iCol]; + assert( iIdxCol!=XN_ROWID ); /* Cannot index rowid */ + if( iIdxCol>=0 ){ + return aXRef[iIdxCol]>=0; + } + assert( iIdxCol==XN_EXPR ); + assert( pIdx->aColExpr!=0 ); + assert( pIdx->aColExpr->a[iCol].pExpr!=0 ); + return sqlite3ExprReferencesUpdatedColumn(pIdx->aColExpr->a[iCol].pExpr, + aXRef,chngRowid); +} + +/* +** Check to see if index pIdx is a partial index whose conditional +** expression might change values due to an UPDATE. Return true if +** the index is subject to change and false if the index is guaranteed +** to be unchanged. This is an optimization. False-positives are a +** performance degradation, but false-negatives can result in a corrupt +** index and incorrect answers. +** +** aXRef[j] will be non-negative if column j of the original table is +** being updated. chngRowid will be true if the rowid of the table is +** being updated. +*/ +static int indexWhereClauseMightChange( + Index *pIdx, /* The index to check */ + int *aXRef, /* aXRef[j]>=0 if column j is being updated */ + int chngRowid /* true if the rowid is being updated */ +){ + if( pIdx->pPartIdxWhere==0 ) return 0; + return sqlite3ExprReferencesUpdatedColumn(pIdx->pPartIdxWhere, + aXRef, chngRowid); +} + /* ** Process an UPDATE statement. ** @@ -129096,7 +137192,7 @@ SQLITE_PRIVATE void sqlite3Update( Expr *pLimit, /* LIMIT clause. May be null */ Upsert *pUpsert /* ON CONFLICT clause, or null */ ){ - int i, j; /* Loop counters */ + int i, j, k; /* Loop counters */ Table *pTab; /* The table to be updated */ int addrTop = 0; /* VDBE instruction address of the start of the loop */ WhereInfo *pWInfo; /* Information about the WHERE clause */ @@ -129104,11 +137200,12 @@ SQLITE_PRIVATE void sqlite3Update( Index *pIdx; /* For looping over indices */ Index *pPk; /* The PRIMARY KEY index for WITHOUT ROWID tables */ int nIdx; /* Number of indices that need updating */ + int nAllIdx; /* Total number of indexes */ int iBaseCur; /* Base cursor number */ int iDataCur; /* Cursor for the canonical data btree */ int iIdxCur; /* Cursor for the first index */ sqlite3 *db; /* The database structure */ - int *aRegIdx = 0; /* First register in array assigned to each index */ + int *aRegIdx = 0; /* Registers for to each index and the main table */ int *aXRef = 0; /* aXRef[i] is the index in pChanges->a[] of the ** an expression for the i-th column of the table. ** aXRef[i]==-1 if the i-th column is not changed. */ @@ -129139,6 +137236,7 @@ SQLITE_PRIVATE void sqlite3Update( int iPk = 0; /* First of nPk cells holding PRIMARY KEY value */ i16 nPk = 0; /* Number of components of the PRIMARY KEY */ int bReplace = 0; /* True if REPLACE conflict resolution might happen */ + int bFinishSeek = 1; /* The OP_FinishSeek opcode is needed */ /* Register Allocations */ int regRowCount = 0; /* A count of rows changed */ @@ -129222,10 +137320,10 @@ SQLITE_PRIVATE void sqlite3Update( /* Allocate space for aXRef[], aRegIdx[], and aToOpen[]. ** Initialize aXRef[] and aToOpen[] to their default values. */ - aXRef = sqlite3DbMallocRawNN(db, sizeof(int) * (pTab->nCol+nIdx) + nIdx+2 ); + aXRef = sqlite3DbMallocRawNN(db, sizeof(int) * (pTab->nCol+nIdx+1) + nIdx+2 ); if( aXRef==0 ) goto update_cleanup; aRegIdx = aXRef+pTab->nCol; - aToOpen = (u8*)(aRegIdx+nIdx); + aToOpen = (u8*)(aRegIdx+nIdx+1); memset(aToOpen, 1, nIdx+1); aToOpen[nIdx+1] = 0; for(i=0; inCol; i++) aXRef[i] = -1; @@ -129237,6 +137335,10 @@ SQLITE_PRIVATE void sqlite3Update( sNC.uNC.pUpsert = pUpsert; sNC.ncFlags = NC_UUpsert; + /* Begin generating code. */ + v = sqlite3GetVdbe(pParse); + if( v==0 ) goto update_cleanup; + /* Resolve the column names in all the expressions of the ** of the UPDATE statement. Also find the column index ** for each column to be updated in the pChanges array. For each @@ -129249,24 +137351,34 @@ SQLITE_PRIVATE void sqlite3Update( goto update_cleanup; } for(j=0; jnCol; j++){ - if( sqlite3StrICmp(pTab->aCol[j].zName, pChanges->a[i].zName)==0 ){ + if( sqlite3StrICmp(pTab->aCol[j].zName, pChanges->a[i].zEName)==0 ){ if( j==pTab->iPKey ){ chngRowid = 1; pRowidExpr = pChanges->a[i].pExpr; }else if( pPk && (pTab->aCol[j].colFlags & COLFLAG_PRIMKEY)!=0 ){ chngPk = 1; } +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + else if( pTab->aCol[j].colFlags & COLFLAG_GENERATED ){ + testcase( pTab->aCol[j].colFlags & COLFLAG_VIRTUAL ); + testcase( pTab->aCol[j].colFlags & COLFLAG_STORED ); + sqlite3ErrorMsg(pParse, + "cannot UPDATE generated column \"%s\"", + pTab->aCol[j].zName); + goto update_cleanup; + } +#endif aXRef[j] = i; break; } } if( j>=pTab->nCol ){ - if( pPk==0 && sqlite3IsRowid(pChanges->a[i].zName) ){ + if( pPk==0 && sqlite3IsRowid(pChanges->a[i].zEName) ){ j = -1; chngRowid = 1; pRowidExpr = pChanges->a[i].pExpr; }else{ - sqlite3ErrorMsg(pParse, "no such column: %s", pChanges->a[i].zName); + sqlite3ErrorMsg(pParse, "no such column: %s", pChanges->a[i].zEName); pParse->checkSchema = 1; goto update_cleanup; } @@ -129290,6 +137402,33 @@ SQLITE_PRIVATE void sqlite3Update( assert( chngPk==0 || chngPk==1 ); chngKey = chngRowid + chngPk; +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + /* Mark generated columns as changing if their generator expressions + ** reference any changing column. The actual aXRef[] value for + ** generated expressions is not used, other than to check to see that it + ** is non-negative, so the value of aXRef[] for generated columns can be + ** set to any non-negative number. We use 99999 so that the value is + ** obvious when looking at aXRef[] in a symbolic debugger. + */ + if( pTab->tabFlags & TF_HasGenerated ){ + int bProgress; + testcase( pTab->tabFlags & TF_HasVirtual ); + testcase( pTab->tabFlags & TF_HasStored ); + do{ + bProgress = 0; + for(i=0; inCol; i++){ + if( aXRef[i]>=0 ) continue; + if( (pTab->aCol[i].colFlags & COLFLAG_GENERATED)==0 ) continue; + if( sqlite3ExprReferencesUpdatedColumn(pTab->aCol[i].pDflt, + aXRef, chngRowid) ){ + aXRef[i] = 99999; + bProgress = 1; + } + } + }while( bProgress ); + } +#endif + /* The SET expressions are not actually used inside the WHERE loop. ** So reset the colUsed mask. Unless this is a virtual table. In that ** case, set all bits of the colUsed mask (to ensure that the virtual @@ -129302,48 +137441,50 @@ SQLITE_PRIVATE void sqlite3Update( /* There is one entry in the aRegIdx[] array for each index on the table ** being updated. Fill in aRegIdx[] with a register number that will hold ** the key for accessing each index. - ** - ** FIXME: Be smarter about omitting indexes that use expressions. */ - for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){ + if( onError==OE_Replace ) bReplace = 1; + for(nAllIdx=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, nAllIdx++){ int reg; - if( chngKey || hasFK>1 || pIdx->pPartIdxWhere || pIdx==pPk ){ + if( chngKey || hasFK>1 || pIdx==pPk + || indexWhereClauseMightChange(pIdx,aXRef,chngRowid) + ){ reg = ++pParse->nMem; pParse->nMem += pIdx->nColumn; }else{ reg = 0; for(i=0; inKeyCol; i++){ - i16 iIdxCol = pIdx->aiColumn[i]; - if( iIdxCol<0 || aXRef[iIdxCol]>=0 ){ + if( indexColumnIsBeingUpdated(pIdx, i, aXRef, chngRowid) ){ reg = ++pParse->nMem; pParse->nMem += pIdx->nColumn; - if( (onError==OE_Replace) - || (onError==OE_Default && pIdx->onError==OE_Replace) - ){ + if( onError==OE_Default && pIdx->onError==OE_Replace ){ bReplace = 1; } break; } } } - if( reg==0 ) aToOpen[j+1] = 0; - aRegIdx[j] = reg; + if( reg==0 ) aToOpen[nAllIdx+1] = 0; + aRegIdx[nAllIdx] = reg; } + aRegIdx[nAllIdx] = ++pParse->nMem; /* Register storing the table record */ if( bReplace ){ /* If REPLACE conflict resolution might be invoked, open cursors on all ** indexes in case they are needed to delete records. */ memset(aToOpen, 1, nIdx+1); } - /* Begin generating code. */ - v = sqlite3GetVdbe(pParse); - if( v==0 ) goto update_cleanup; if( pParse->nested==0 ) sqlite3VdbeCountChanges(v); sqlite3BeginWriteOperation(pParse, pTrigger || hasFK, iDb); /* Allocate required registers. */ if( !IsVirtual(pTab) ){ - regRowSet = ++pParse->nMem; + /* For now, regRowSet and aRegIdx[nAllIdx] share the same register. + ** If regRowSet turns out to be needed, then aRegIdx[nAllIdx] will be + ** reallocated. aRegIdx[nAllIdx] is the register in which the main + ** table record is written. regRowSet holds the RowSet for the + ** two-pass update algorithm. */ + assert( aRegIdx[nAllIdx]==pParse->nMem ); + regRowSet = aRegIdx[nAllIdx]; regOldRowid = regNewRowid = ++pParse->nMem; if( chngPk || pTrigger || hasFK ){ regOld = pParse->nMem + 1; @@ -129391,7 +137532,7 @@ SQLITE_PRIVATE void sqlite3Update( #endif /* Jump to labelBreak to abandon further processing of this UPDATE */ - labelContinue = labelBreak = sqlite3VdbeMakeLabel(v); + labelContinue = labelBreak = sqlite3VdbeMakeLabel(pParse); /* Not an UPSERT. Normal processing. Begin by ** initialize the count of updated rows */ @@ -129429,6 +137570,7 @@ SQLITE_PRIVATE void sqlite3Update( pWInfo = 0; eOnePass = ONEPASS_SINGLE; sqlite3ExprIfFalse(pParse, pWhere, labelBreak, SQLITE_JUMPIFNULL); + bFinishSeek = 0; }else{ /* Begin the database scan. ** @@ -129455,6 +137597,7 @@ SQLITE_PRIVATE void sqlite3Update( ** strategy that uses an index for which one or more columns are being ** updated. */ eOnePass = sqlite3WhereOkOnePass(pWInfo, aiCurOnePass); + bFinishSeek = sqlite3WhereUsesDeferredSeek(pWInfo); if( eOnePass!=ONEPASS_SINGLE ){ sqlite3MultiWrite(pParse); if( eOnePass==ONEPASS_MULTI ){ @@ -129473,6 +137616,8 @@ SQLITE_PRIVATE void sqlite3Update( ** leave it in register regOldRowid. */ sqlite3VdbeAddOp2(v, OP_Rowid, iDataCur, regOldRowid); if( eOnePass==ONEPASS_OFF ){ + /* We need to use regRowSet, so reallocate aRegIdx[nAllIdx] */ + aRegIdx[nAllIdx] = ++pParse->nMem; sqlite3VdbeAddOp2(v, OP_RowSetAdd, regRowSet, regOldRowid); } }else{ @@ -129483,7 +137628,8 @@ SQLITE_PRIVATE void sqlite3Update( ** is not required) and leave the PK fields in the array of registers. */ for(i=0; iaiColumn[i]>=0 ); - sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur,pPk->aiColumn[i],iPk+i); + sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, + pPk->aiColumn[i], iPk+i); } if( eOnePass ){ if( addrOpen ) sqlite3VdbeChangeToNoop(v, addrOpen); @@ -129515,7 +137661,9 @@ SQLITE_PRIVATE void sqlite3Update( } sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenWrite, 0, iBaseCur, aToOpen, 0, 0); - if( addrOnce ) sqlite3VdbeJumpHere(v, addrOnce); + if( addrOnce ){ + sqlite3VdbeJumpHereOrPopInst(v, addrOnce); + } } /* Top of the update loop */ @@ -129523,16 +137671,16 @@ SQLITE_PRIVATE void sqlite3Update( if( !isView && aiCurOnePass[0]!=iDataCur && aiCurOnePass[1]!=iDataCur ){ assert( pPk ); sqlite3VdbeAddOp4Int(v, OP_NotFound, iDataCur, labelBreak, regKey,nKey); - VdbeCoverageNeverTaken(v); + VdbeCoverage(v); } if( eOnePass!=ONEPASS_SINGLE ){ - labelContinue = sqlite3VdbeMakeLabel(v); + labelContinue = sqlite3VdbeMakeLabel(pParse); } sqlite3VdbeAddOp2(v, OP_IsNull, pPk ? regKey : regOldRowid, labelBreak); VdbeCoverageIf(v, pPk==0); VdbeCoverageIf(v, pPk!=0); }else if( pPk ){ - labelContinue = sqlite3VdbeMakeLabel(v); + labelContinue = sqlite3VdbeMakeLabel(pParse); sqlite3VdbeAddOp2(v, OP_Rewind, iEph, labelBreak); VdbeCoverage(v); addrTop = sqlite3VdbeAddOp2(v, OP_RowData, iEph, regKey); sqlite3VdbeAddOp4Int(v, OP_NotFound, iDataCur, labelContinue, regKey, 0); @@ -129564,14 +137712,16 @@ SQLITE_PRIVATE void sqlite3Update( pTrigger, pChanges, 0, TRIGGER_BEFORE|TRIGGER_AFTER, pTab, onError ); for(i=0; inCol; i++){ + u32 colFlags = pTab->aCol[i].colFlags; + k = sqlite3TableColumnToStorage(pTab, i) + regOld; if( oldmask==0xffffffff || (i<32 && (oldmask & MASKBIT32(i))!=0) - || (pTab->aCol[i].colFlags & COLFLAG_PRIMKEY)!=0 + || (colFlags & COLFLAG_PRIMKEY)!=0 ){ testcase( oldmask!=0xffffffff && i==31 ); - sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, i, regOld+i); + sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, i, k); }else{ - sqlite3VdbeAddOp2(v, OP_Null, 0, regOld+i); + sqlite3VdbeAddOp2(v, OP_Null, 0, k); } } if( chngRowid==0 && pPk==0 ){ @@ -129595,13 +137745,15 @@ SQLITE_PRIVATE void sqlite3Update( newmask = sqlite3TriggerColmask( pParse, pTrigger, pChanges, 1, TRIGGER_BEFORE, pTab, onError ); - for(i=0; inCol; i++){ + for(i=0, k=regNew; inCol; i++, k++){ if( i==pTab->iPKey ){ - sqlite3VdbeAddOp2(v, OP_Null, 0, regNew+i); + sqlite3VdbeAddOp2(v, OP_Null, 0, k); + }else if( (pTab->aCol[i].colFlags & COLFLAG_GENERATED)!=0 ){ + if( pTab->aCol[i].colFlags & COLFLAG_VIRTUAL ) k--; }else{ j = aXRef[i]; if( j>=0 ){ - sqlite3ExprCode(pParse, pChanges->a[j].pExpr, regNew+i); + sqlite3ExprCode(pParse, pChanges->a[j].pExpr, k); }else if( 0==(tmask&TRIGGER_BEFORE) || i>31 || (newmask & MASKBIT32(i)) ){ /* This branch loads the value of a column that will not be changed ** into a register. This is done if there are no BEFORE triggers, or @@ -129610,18 +137762,20 @@ SQLITE_PRIVATE void sqlite3Update( */ testcase( i==31 ); testcase( i==32 ); - sqlite3ExprCodeGetColumnToReg(pParse, pTab, i, iDataCur, regNew+i); - if( tmask & TRIGGER_BEFORE ){ - /* This value will be recomputed in After-BEFORE-trigger-reload-loop - ** below, so make sure that it is not cached and reused. - ** Ticket d85fffd6ffe856092ed8daefa811b1e399706b28. */ - sqlite3ExprCacheRemove(pParse, regNew+i, 1); - } + sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, i, k); + bFinishSeek = 0; }else{ - sqlite3VdbeAddOp2(v, OP_Null, 0, regNew+i); + sqlite3VdbeAddOp2(v, OP_Null, 0, k); } } } +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + if( pTab->tabFlags & TF_HasGenerated ){ + testcase( pTab->tabFlags & TF_HasVirtual ); + testcase( pTab->tabFlags & TF_HasStored ); + sqlite3ComputeGeneratedColumns(pParse, regNew, pTab); + } +#endif /* Fire any BEFORE UPDATE triggers. This happens before constraints are ** verified. One could argue that this is wrong. @@ -129654,38 +137808,58 @@ SQLITE_PRIVATE void sqlite3Update( ** BEFORE trigger runs. See test case trigger1-18.0 (added 2018-04-26) ** for an example. */ - for(i=0; inCol; i++){ - if( aXRef[i]<0 && i!=pTab->iPKey ){ - sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, i, regNew+i); + for(i=0, k=regNew; inCol; i++, k++){ + if( pTab->aCol[i].colFlags & COLFLAG_GENERATED ){ + if( pTab->aCol[i].colFlags & COLFLAG_VIRTUAL ) k--; + }else if( aXRef[i]<0 && i!=pTab->iPKey ){ + sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, i, k); } } +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + if( pTab->tabFlags & TF_HasGenerated ){ + testcase( pTab->tabFlags & TF_HasVirtual ); + testcase( pTab->tabFlags & TF_HasStored ); + sqlite3ComputeGeneratedColumns(pParse, regNew, pTab); + } +#endif } if( !isView ){ - int addr1 = 0; /* Address of jump instruction */ - /* Do constraint checks. */ assert( regOldRowid>0 ); sqlite3GenerateConstraintChecks(pParse, pTab, aRegIdx, iDataCur, iIdxCur, regNewRowid, regOldRowid, chngKey, onError, labelContinue, &bReplace, aXRef, 0); - /* Do FK constraint checks. */ - if( hasFK ){ - sqlite3FkCheck(pParse, pTab, regOldRowid, 0, aXRef, chngKey); - } - - /* Delete the index entries associated with the current record. */ + /* If REPLACE conflict handling may have been used, or if the PK of the + ** row is changing, then the GenerateConstraintChecks() above may have + ** moved cursor iDataCur. Reseek it. */ if( bReplace || chngKey ){ if( pPk ){ - addr1 = sqlite3VdbeAddOp4Int(v, OP_NotFound, iDataCur, 0, regKey, nKey); + sqlite3VdbeAddOp4Int(v, OP_NotFound,iDataCur,labelContinue,regKey,nKey); }else{ - addr1 = sqlite3VdbeAddOp3(v, OP_NotExists, iDataCur, 0, regOldRowid); + sqlite3VdbeAddOp3(v, OP_NotExists, iDataCur, labelContinue,regOldRowid); } VdbeCoverageNeverTaken(v); } + + /* Do FK constraint checks. */ + if( hasFK ){ + sqlite3FkCheck(pParse, pTab, regOldRowid, 0, aXRef, chngKey); + } + + /* Delete the index entries associated with the current record. */ sqlite3GenerateRowIndexDelete(pParse, pTab, iDataCur, iIdxCur, aRegIdx, -1); + /* We must run the OP_FinishSeek opcode to resolve a prior + ** OP_DeferredSeek if there is any possibility that there have been + ** no OP_Column opcodes since the OP_DeferredSeek was issued. But + ** we want to avoid the OP_FinishSeek if possible, as running it + ** costs CPU cycles. */ + if( bFinishSeek ){ + sqlite3VdbeAddOp1(v, OP_FinishSeek, iDataCur); + } + /* If changing the rowid value, or if there are foreign key constraints ** to process, delete the old record. Otherwise, add a noop OP_Delete ** to invoke the pre-update hook. @@ -129713,9 +137887,6 @@ SQLITE_PRIVATE void sqlite3Update( sqlite3VdbeAddOp2(v, OP_Delete, iDataCur, 0); } #endif - if( bReplace || chngKey ){ - sqlite3VdbeJumpHere(v, addr1); - } if( hasFK ){ sqlite3FkCheck(pParse, pTab, 0, regNewRowid, aXRef, chngKey); @@ -129865,11 +138036,12 @@ static void updateVirtualTable( /* Populate the argument registers. */ for(i=0; inCol; i++){ + assert( (pTab->aCol[i].colFlags & COLFLAG_GENERATED)==0 ); if( aXRef[i]>=0 ){ sqlite3ExprCode(pParse, pChanges->a[aXRef[i]].pExpr, regArg+2+i); }else{ sqlite3VdbeAddOp3(v, OP_VColumn, iCsr, i, regArg+2+i); - sqlite3VdbeChangeP5(v, 1); /* Enable sqlite3_vtab_nochange() */ + sqlite3VdbeChangeP5(v, OPFLAG_NOCHNG);/* Enable sqlite3_vtab_nochange() */ } } if( HasRowid(pTab) ){ @@ -130153,10 +138325,13 @@ SQLITE_PRIVATE void sqlite3UpsertDoUpdate( Vdbe *v = pParse->pVdbe; sqlite3 *db = pParse->db; SrcList *pSrc; /* FROM clause for the UPDATE */ - int iDataCur = pUpsert->iDataCur; + int iDataCur; + int i; assert( v!=0 ); + assert( pUpsert!=0 ); VdbeNoopComment((v, "Begin DO UPDATE of UPSERT")); + iDataCur = pUpsert->iDataCur; if( pIdx && iCur!=iDataCur ){ if( HasRowid(pTab) ){ int regRowid = sqlite3GetTempReg(pParse); @@ -130168,12 +138343,11 @@ SQLITE_PRIVATE void sqlite3UpsertDoUpdate( Index *pPk = sqlite3PrimaryKeyIndex(pTab); int nPk = pPk->nKeyCol; int iPk = pParse->nMem+1; - int i; pParse->nMem += nPk; for(i=0; iaiColumn[i]>=0 ); - k = sqlite3ColumnOfIndex(pIdx, pPk->aiColumn[i]); + k = sqlite3TableColumnToIndex(pIdx, pPk->aiColumn[i]); sqlite3VdbeAddOp3(v, OP_Column, iCur, k, iPk+i); VdbeComment((v, "%s.%s", pIdx->zName, pTab->aCol[pPk->aiColumn[i]].zName)); @@ -130183,12 +138357,19 @@ SQLITE_PRIVATE void sqlite3UpsertDoUpdate( VdbeCoverage(v); sqlite3VdbeAddOp4(v, OP_Halt, SQLITE_CORRUPT, OE_Abort, 0, "corrupt database", P4_STATIC); + sqlite3MayAbort(pParse); sqlite3VdbeJumpHere(v, i); } } /* pUpsert does not own pUpsertSrc - the outer INSERT statement does. So ** we have to make a copy before passing it down into sqlite3Update() */ pSrc = sqlite3SrcListDup(db, pUpsert->pUpsertSrc, 0); + /* excluded.* columns of type REAL need to be converted to a hard real */ + for(i=0; inCol; i++){ + if( pTab->aCol[i].affinity==SQLITE_AFF_REAL ){ + sqlite3VdbeAddOp1(v, OP_RealAffinity, pUpsert->regData+i); + } + } sqlite3Update(pParse, pSrc, pUpsert->pUpsertSet, pUpsert->pUpsertWhere, OE_Abort, 0, 0, pUpsert); pUpsert->pUpsertSet = 0; /* Will have been deleted by sqlite3Update() */ @@ -130304,16 +138485,17 @@ static int execSqlF(sqlite3 *db, char **pzErrMsg, const char *zSql, ...){ ** transient would cause the database file to appear to be deleted ** following reboot. */ -SQLITE_PRIVATE void sqlite3Vacuum(Parse *pParse, Token *pNm){ +SQLITE_PRIVATE void sqlite3Vacuum(Parse *pParse, Token *pNm, Expr *pInto){ Vdbe *v = sqlite3GetVdbe(pParse); int iDb = 0; - if( v==0 ) return; + if( v==0 ) goto build_vacuum_end; + if( pParse->nErr ) goto build_vacuum_end; if( pNm ){ #ifndef SQLITE_BUG_COMPATIBLE_20160819 /* Default behavior: Report an error if the argument to VACUUM is ** not recognized */ iDb = sqlite3TwoPartName(pParse, pNm, pNm, &pNm); - if( iDb<0 ) return; + if( iDb<0 ) goto build_vacuum_end; #else /* When SQLITE_BUG_COMPATIBLE_20160819 is defined, unrecognized arguments ** to VACUUM are silently ignored. This is a back-out of a bug fix that @@ -130325,37 +138507,63 @@ SQLITE_PRIVATE void sqlite3Vacuum(Parse *pParse, Token *pNm){ #endif } if( iDb!=1 ){ - sqlite3VdbeAddOp1(v, OP_Vacuum, iDb); + int iIntoReg = 0; + if( pInto && sqlite3ResolveSelfReference(pParse,0,0,pInto,0)==0 ){ + iIntoReg = ++pParse->nMem; + sqlite3ExprCode(pParse, pInto, iIntoReg); + } + sqlite3VdbeAddOp2(v, OP_Vacuum, iDb, iIntoReg); sqlite3VdbeUsesBtree(v, iDb); } +build_vacuum_end: + sqlite3ExprDelete(pParse->db, pInto); return; } /* ** This routine implements the OP_Vacuum opcode of the VDBE. */ -SQLITE_PRIVATE int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db, int iDb){ +SQLITE_PRIVATE SQLITE_NOINLINE int sqlite3RunVacuum( + char **pzErrMsg, /* Write error message here */ + sqlite3 *db, /* Database connection */ + int iDb, /* Which attached DB to vacuum */ + sqlite3_value *pOut /* Write results here, if not NULL. VACUUM INTO */ +){ int rc = SQLITE_OK; /* Return code from service routines */ Btree *pMain; /* The database being vacuumed */ Btree *pTemp; /* The temporary database we vacuum into */ - u16 saved_mDbFlags; /* Saved value of db->mDbFlags */ - u32 saved_flags; /* Saved value of db->flags */ + u32 saved_mDbFlags; /* Saved value of db->mDbFlags */ + u64 saved_flags; /* Saved value of db->flags */ int saved_nChange; /* Saved value of db->nChange */ int saved_nTotalChange; /* Saved value of db->nTotalChange */ + u32 saved_openFlags; /* Saved value of db->openFlags */ u8 saved_mTrace; /* Saved trace settings */ Db *pDb = 0; /* Database to detach at end of vacuum */ int isMemDb; /* True if vacuuming a :memory: database */ int nRes; /* Bytes of reserved space at the end of each page */ int nDb; /* Number of attached databases */ const char *zDbMain; /* Schema name of database to vacuum */ + const char *zOut; /* Name of output file */ if( !db->autoCommit ){ sqlite3SetString(pzErrMsg, db, "cannot VACUUM from within a transaction"); - return SQLITE_ERROR; + return SQLITE_ERROR; /* IMP: R-12218-18073 */ } if( db->nVdbeActive>1 ){ sqlite3SetString(pzErrMsg, db,"cannot VACUUM - SQL statements in progress"); - return SQLITE_ERROR; + return SQLITE_ERROR; /* IMP: R-15610-35227 */ + } + saved_openFlags = db->openFlags; + if( pOut ){ + if( sqlite3_value_type(pOut)!=SQLITE_TEXT ){ + sqlite3SetString(pzErrMsg, db, "non-text filename"); + return SQLITE_ERROR; + } + zOut = (const char*)sqlite3_value_text(pOut); + db->openFlags &= ~SQLITE_OPEN_READONLY; + db->openFlags |= SQLITE_OPEN_CREATE|SQLITE_OPEN_READWRITE; + }else{ + zOut = ""; } /* Save the current value of the database flags so that it can be @@ -130368,7 +138576,8 @@ SQLITE_PRIVATE int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db, int iDb){ saved_mTrace = db->mTrace; db->flags |= SQLITE_WriteSchema | SQLITE_IgnoreChecks; db->mDbFlags |= DBFLAG_PreferBuiltin | DBFLAG_Vacuum; - db->flags &= ~(SQLITE_ForeignKeys | SQLITE_ReverseOrder | SQLITE_CountRows); + db->flags &= ~(u64)(SQLITE_ForeignKeys | SQLITE_ReverseOrder + | SQLITE_Defensive | SQLITE_CountRows); db->mTrace = 0; zDbMain = db->aDb[iDb].zDbSName; @@ -130390,31 +138599,24 @@ SQLITE_PRIVATE int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db, int iDb){ ** to write the journal header file. */ nDb = db->nDb; - rc = execSql(db, pzErrMsg, "ATTACH''AS vacuum_db"); + rc = execSqlF(db, pzErrMsg, "ATTACH %Q AS vacuum_db", zOut); + db->openFlags = saved_openFlags; if( rc!=SQLITE_OK ) goto end_of_vacuum; assert( (db->nDb-1)==nDb ); pDb = &db->aDb[nDb]; assert( strcmp(pDb->zDbSName,"vacuum_db")==0 ); pTemp = pDb->pBt; - - /* The call to execSql() to attach the temp database has left the file - ** locked (as there was more than one active statement when the transaction - ** to read the schema was concluded. Unlock it here so that this doesn't - ** cause problems for the call to BtreeSetPageSize() below. */ - sqlite3BtreeCommit(pTemp); - - nRes = sqlite3BtreeGetOptimalReserve(pMain); - - /* A VACUUM cannot change the pagesize of an encrypted database. */ -#ifdef SQLITE_HAS_CODEC - if( db->nextPagesize ){ - extern void sqlite3CodecGetKey(sqlite3*, int, void**, int*); - int nKey; - char *zKey; - sqlite3CodecGetKey(db, iDb, (void**)&zKey, &nKey); - if( nKey ) db->nextPagesize = 0; + if( pOut ){ + sqlite3_file *id = sqlite3PagerFile(sqlite3BtreePager(pTemp)); + i64 sz = 0; + if( id->pMethods!=0 && (sqlite3OsFileSize(id, &sz)!=SQLITE_OK || sz>0) ){ + rc = SQLITE_ERROR; + sqlite3SetString(pzErrMsg, db, "output file already exists"); + goto end_of_vacuum; + } + db->mDbFlags |= DBFLAG_VacuumInto; } -#endif + nRes = sqlite3BtreeGetRequestedReserve(pMain); sqlite3BtreeSetCacheSize(pTemp, db->aDb[iDb].pSchema->cache_size); sqlite3BtreeSetSpillSize(pTemp, sqlite3BtreeSetSpillSize(pMain,0)); @@ -130426,7 +138628,7 @@ SQLITE_PRIVATE int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db, int iDb){ */ rc = execSql(db, pzErrMsg, "BEGIN"); if( rc!=SQLITE_OK ) goto end_of_vacuum; - rc = sqlite3BtreeBeginTrans(pMain, 2); + rc = sqlite3BtreeBeginTrans(pMain, pOut==0 ? 2 : 0, 0); if( rc!=SQLITE_OK ) goto end_of_vacuum; /* Do not attempt to change the page size for a WAL database */ @@ -130521,7 +138723,7 @@ SQLITE_PRIVATE int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db, int iDb){ }; assert( 1==sqlite3BtreeIsInTrans(pTemp) ); - assert( 1==sqlite3BtreeIsInTrans(pMain) ); + assert( pOut!=0 || 1==sqlite3BtreeIsInTrans(pMain) ); /* Copy Btree meta values */ for(i=0; iflags */ @@ -130552,7 +138760,7 @@ SQLITE_PRIVATE int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db, int iDb){ db->nChange = saved_nChange; db->nTotalChange = saved_nTotalChange; db->mTrace = saved_mTrace; - sqlite3BtreeSetPageSize(pMain, -1, -1, 1); + sqlite3BtreeSetPageSize(pMain, -1, 0, 1); /* Currently there is an SQL level transaction open on the vacuum ** database. No locks are held on any other files (since the main file @@ -130614,6 +138822,9 @@ struct VtabCtx { ** Construct and install a Module object for a virtual table. When this ** routine is called, it is guaranteed that all appropriate locks are held ** and the module is not already part of the connection. +** +** If there already exists a module with zName, replace it with the new one. +** If pModule==0, then delete the module zName if it exists. */ SQLITE_PRIVATE Module *sqlite3VtabCreateModule( sqlite3 *db, /* Database in which module is registered */ @@ -130623,25 +138834,36 @@ SQLITE_PRIVATE Module *sqlite3VtabCreateModule( void (*xDestroy)(void *) /* Module destructor function */ ){ Module *pMod; - int nName = sqlite3Strlen30(zName); - pMod = (Module *)sqlite3Malloc(sizeof(Module) + nName + 1); - if( pMod==0 ){ - sqlite3OomFault(db); + Module *pDel; + char *zCopy; + if( pModule==0 ){ + zCopy = (char*)zName; + pMod = 0; }else{ - Module *pDel; - char *zCopy = (char *)(&pMod[1]); + int nName = sqlite3Strlen30(zName); + pMod = (Module *)sqlite3Malloc(sizeof(Module) + nName + 1); + if( pMod==0 ){ + sqlite3OomFault(db); + return 0; + } + zCopy = (char *)(&pMod[1]); memcpy(zCopy, zName, nName+1); pMod->zName = zCopy; pMod->pModule = pModule; pMod->pAux = pAux; pMod->xDestroy = xDestroy; pMod->pEpoTab = 0; - pDel = (Module *)sqlite3HashInsert(&db->aModule,zCopy,(void*)pMod); - assert( pDel==0 || pDel==pMod ); - if( pDel ){ + pMod->nRefModule = 1; + } + pDel = (Module *)sqlite3HashInsert(&db->aModule,zCopy,(void*)pMod); + if( pDel ){ + if( pDel==pMod ){ sqlite3OomFault(db); sqlite3DbFree(db, pDel); pMod = 0; + }else{ + sqlite3VtabEponymousTableClear(db, pDel); + sqlite3VtabModuleUnref(db, pDel); } } return pMod; @@ -130662,11 +138884,7 @@ static int createModule( int rc = SQLITE_OK; sqlite3_mutex_enter(db->mutex); - if( sqlite3HashFind(&db->aModule, zName) ){ - rc = SQLITE_MISUSE_BKPT; - }else{ - (void)sqlite3VtabCreateModule(db, zName, pModule, pAux, xDestroy); - } + (void)sqlite3VtabCreateModule(db, zName, pModule, pAux, xDestroy); rc = sqlite3ApiExit(db, rc); if( rc!=SQLITE_OK && xDestroy ) xDestroy(pAux); sqlite3_mutex_leave(db->mutex); @@ -130705,6 +138923,44 @@ SQLITE_API int sqlite3_create_module_v2( return createModule(db, zName, pModule, pAux, xDestroy); } +/* +** External API to drop all virtual-table modules, except those named +** on the azNames list. +*/ +SQLITE_API int sqlite3_drop_modules(sqlite3 *db, const char** azNames){ + HashElem *pThis, *pNext; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif + for(pThis=sqliteHashFirst(&db->aModule); pThis; pThis=pNext){ + Module *pMod = (Module*)sqliteHashData(pThis); + pNext = sqliteHashNext(pThis); + if( azNames ){ + int ii; + for(ii=0; azNames[ii]!=0 && strcmp(azNames[ii],pMod->zName)!=0; ii++){} + if( azNames[ii]!=0 ) continue; + } + createModule(db, pMod->zName, 0, 0, 0); + } + return SQLITE_OK; +} + +/* +** Decrement the reference count on a Module object. Destroy the +** module when the reference count reaches zero. +*/ +SQLITE_PRIVATE void sqlite3VtabModuleUnref(sqlite3 *db, Module *pMod){ + assert( pMod->nRefModule>0 ); + pMod->nRefModule--; + if( pMod->nRefModule==0 ){ + if( pMod->xDestroy ){ + pMod->xDestroy(pMod->pAux); + } + assert( pMod->pEpoTab==0 ); + sqlite3DbFree(db, pMod); + } +} + /* ** Lock the virtual table so that it cannot be disconnected. ** Locks nest. Every lock should have a corresponding unlock. @@ -130744,6 +139000,7 @@ SQLITE_PRIVATE void sqlite3VtabUnlock(VTable *pVTab){ pVTab->nRef--; if( pVTab->nRef==0 ){ sqlite3_vtab *p = pVTab->pVtab; + sqlite3VtabModuleUnref(pVTab->db, pVTab->pMod); if( p ){ p->pModule->xDisconnect(p); } @@ -130838,13 +139095,13 @@ SQLITE_PRIVATE void sqlite3VtabDisconnect(sqlite3 *db, Table *p){ */ SQLITE_PRIVATE void sqlite3VtabUnlockList(sqlite3 *db){ VTable *p = db->pDisconnect; - db->pDisconnect = 0; assert( sqlite3BtreeHoldsAllMutexes(db) ); assert( sqlite3_mutex_held(db->mutex) ); if( p ){ - sqlite3ExpirePreparedStatements(db); + db->pDisconnect = 0; + sqlite3ExpirePreparedStatements(db, 0); do { VTable *pNext = p->pNext; sqlite3VtabUnlock(p); @@ -130884,9 +139141,13 @@ SQLITE_PRIVATE void sqlite3VtabClear(sqlite3 *db, Table *p){ ** string will be freed automatically when the table is ** deleted. */ -static void addModuleArgument(sqlite3 *db, Table *pTable, char *zArg){ - int nBytes = sizeof(char *)*(2+pTable->nModuleArg); +static void addModuleArgument(Parse *pParse, Table *pTable, char *zArg){ + sqlite3_int64 nBytes = sizeof(char *)*(2+pTable->nModuleArg); char **azModuleArg; + sqlite3 *db = pParse->db; + if( pTable->nModuleArg+3>=db->aLimit[SQLITE_LIMIT_COLUMN] ){ + sqlite3ErrorMsg(pParse, "too many columns on %s", pTable->zName); + } azModuleArg = sqlite3DbRealloc(db, pTable->azModuleArg, nBytes); if( azModuleArg==0 ){ sqlite3DbFree(db, zArg); @@ -130910,7 +139171,6 @@ SQLITE_PRIVATE void sqlite3VtabBeginParse( Token *pModuleName, /* Name of the module for the virtual table */ int ifNotExists /* No error if the table already exists */ ){ - int iDb; /* The database the table is being created in */ Table *pTable; /* The new virtual table */ sqlite3 *db; /* Database connection */ @@ -130920,13 +139180,11 @@ SQLITE_PRIVATE void sqlite3VtabBeginParse( assert( 0==pTable->pIndex ); db = pParse->db; - iDb = sqlite3SchemaToIndex(db, pTable->pSchema); - assert( iDb>=0 ); assert( pTable->nModuleArg==0 ); - addModuleArgument(db, pTable, sqlite3NameFromToken(db, pModuleName)); - addModuleArgument(db, pTable, 0); - addModuleArgument(db, pTable, sqlite3DbStrDup(db, pTable->zName)); + addModuleArgument(pParse, pTable, sqlite3NameFromToken(db, pModuleName)); + addModuleArgument(pParse, pTable, 0); + addModuleArgument(pParse, pTable, sqlite3DbStrDup(db, pTable->zName)); assert( (pParse->sNameToken.z==pName2->z && pName2->z!=0) || (pParse->sNameToken.z==pName1->z && pName2->z==0) ); @@ -130941,6 +139199,8 @@ SQLITE_PRIVATE void sqlite3VtabBeginParse( ** The second call, to obtain permission to create the table, is made now. */ if( pTable->azModuleArg ){ + int iDb = sqlite3SchemaToIndex(db, pTable->pSchema); + assert( iDb>=0 ); /* The database the table is being created in */ sqlite3AuthCheck(pParse, SQLITE_CREATE_VTABLE, pTable->zName, pTable->azModuleArg[0], pParse->db->aDb[iDb].zDbSName); } @@ -130957,7 +139217,7 @@ static void addArgumentToVtab(Parse *pParse){ const char *z = (const char*)pParse->sArg.z; int n = pParse->sArg.n; sqlite3 *db = pParse->db; - addModuleArgument(db, pParse->pNewTable, sqlite3DbStrNDup(db, z, n)); + addModuleArgument(pParse, pParse->pNewTable, sqlite3DbStrNDup(db, z, n)); } } @@ -130987,6 +139247,8 @@ SQLITE_PRIVATE void sqlite3VtabFinishParse(Parse *pParse, Token *pEnd){ int iReg; Vdbe *v; + sqlite3MayAbort(pParse); + /* Compute the complete text of the CREATE VIRTUAL TABLE statement */ if( pEnd ){ pParse->sNameToken.n = (int)(pEnd->z - pParse->sNameToken.z) + pEnd->n; @@ -131012,13 +139274,13 @@ SQLITE_PRIVATE void sqlite3VtabFinishParse(Parse *pParse, Token *pEnd){ zStmt, pParse->regRowid ); - sqlite3DbFree(db, zStmt); v = sqlite3GetVdbe(pParse); sqlite3ChangeCookie(pParse, iDb); sqlite3VdbeAddOp0(v, OP_Expire); - zWhere = sqlite3MPrintf(db, "name='%q' AND type='table'", pTab->zName); + zWhere = sqlite3MPrintf(db, "name=%Q AND sql=%Q", pTab->zName, zStmt); sqlite3VdbeAddParseSchemaOp(v, iDb, zWhere); + sqlite3DbFree(db, zStmt); iReg = ++pParse->nMem; sqlite3VdbeLoadString(v, iReg, pTab->zName); @@ -131115,6 +139377,7 @@ static int vtabCallConstructor( } pVTable->db = db; pVTable->pMod = pMod; + pVTable->eVtabRisk = SQLITE_VTABRISK_Normal; iDb = sqlite3SchemaToIndex(db, pTab->pSchema); pTab->azModuleArg[1] = db->aDb[iDb].zDbSName; @@ -131145,6 +139408,7 @@ static int vtabCallConstructor( ** the sqlite3_vtab object if successful. */ memset(pVTable->pVtab, 0, sizeof(pVTable->pVtab[0])); pVTable->pVtab->pModule = pMod->pModule; + pMod->nRefModule++; pVTable->nRef = 1; if( sCtx.bDeclared==0 ){ const char *zFormat = "vtable constructor did not declare schema: %s"; @@ -131153,7 +139417,7 @@ static int vtabCallConstructor( rc = SQLITE_ERROR; }else{ int iCol; - u8 oooHidden = 0; + u16 oooHidden = 0; /* If everything went according to plan, link the new VTable structure ** into the linked list headed by pTab->pVTable. Then loop through the ** columns of the table to see if any of them contain the token "hidden". @@ -131246,7 +139510,8 @@ static int growVTrans(sqlite3 *db){ /* Grow the sqlite3.aVTrans array if required */ if( (db->nVTrans%ARRAY_INCR)==0 ){ VTable **aVTrans; - int nBytes = sizeof(sqlite3_vtab *) * (db->nVTrans + ARRAY_INCR); + sqlite3_int64 nBytes = sizeof(sqlite3_vtab*)* + ((sqlite3_int64)db->nVTrans + ARRAY_INCR); aVTrans = sqlite3DbRealloc(db, (void *)db->aVTrans, nBytes); if( !aVTrans ){ return SQLITE_NOMEM_BKPT; @@ -131340,7 +139605,7 @@ SQLITE_API int sqlite3_declare_vtab(sqlite3 *db, const char *zCreateTable){ assert( IsVirtual(pTab) ); memset(&sParse, 0, sizeof(sParse)); - sParse.declareVtab = 1; + sParse.eParseMode = PARSE_MODE_DECLARE_VTAB; sParse.db = db; sParse.nQueryLoop = 1; if( SQLITE_OK==sqlite3RunParser(&sParse, zCreateTable, &zErr) @@ -131381,7 +139646,7 @@ SQLITE_API int sqlite3_declare_vtab(sqlite3 *db, const char *zCreateTable){ sqlite3DbFree(db, zErr); rc = SQLITE_ERROR; } - sParse.declareVtab = 0; + sParse.eParseMode = PARSE_MODE_NORMAL; if( sParse.pVdbe ){ sqlite3VdbeFinalize(sParse.pVdbe); @@ -131418,7 +139683,9 @@ SQLITE_PRIVATE int sqlite3VtabCallDestroy(sqlite3 *db, int iDb, const char *zTab } p = vtabDisconnectAll(db, pTab); xDestroy = p->pMod->pModule->xDestroy; - assert( xDestroy!=0 ); /* Checked before the virtual table is created */ + if( xDestroy==0 ) xDestroy = p->pMod->pModule->xDisconnect; + assert( xDestroy!=0 ); + pTab->nTabRef++; rc = xDestroy(p->pVtab); /* Remove the sqlite3_vtab* from the aVTrans[] array, if applicable */ if( rc==SQLITE_OK ){ @@ -131427,6 +139694,7 @@ SQLITE_PRIVATE int sqlite3VtabCallDestroy(sqlite3 *db, int iDb, const char *zTab pTab->pVTable = 0; sqlite3VtabUnlock(p); } + sqlite3DeleteTable(db, pTab); } return rc; @@ -131584,6 +139852,7 @@ SQLITE_PRIVATE int sqlite3VtabSavepoint(sqlite3 *db, int op, int iSavepoint){ const sqlite3_module *pMod = pVTab->pMod->pModule; if( pVTab->pVtab && pMod->iVersion>=2 ){ int (*xMethod)(sqlite3_vtab *, int); + sqlite3VtabLock(pVTab); switch( op ){ case SAVEPOINT_BEGIN: xMethod = pMod->xSavepoint; @@ -131599,6 +139868,7 @@ SQLITE_PRIVATE int sqlite3VtabSavepoint(sqlite3 *db, int op, int iSavepoint){ if( xMethod && pVTab->iSavepoint>iSavepoint ){ rc = xMethod(pVTab->pVtab, iSavepoint); } + sqlite3VtabUnlock(pVTab); } } } @@ -131635,7 +139905,7 @@ SQLITE_PRIVATE FuncDef *sqlite3VtabOverloadFunction( /* Check to see the left operand is a column in a virtual table */ if( NEVER(pExpr==0) ) return pDef; if( pExpr->op!=TK_COLUMN ) return pDef; - pTab = pExpr->pTab; + pTab = pExpr->y.pTab; if( pTab==0 ) return pDef; if( !IsVirtual(pTab) ) return pDef; pVtab = sqlite3GetVTable(db, pTab)->pVtab; @@ -131697,7 +139967,7 @@ SQLITE_PRIVATE void sqlite3VtabMakeWritable(Parse *pParse, Table *pTab){ if( pTab==pToplevel->apVtabLock[i] ) return; } n = (pToplevel->nVtabLock+1)*sizeof(pToplevel->apVtabLock[0]); - apVtabLock = sqlite3_realloc64(pToplevel->apVtabLock, n); + apVtabLock = sqlite3Realloc(pToplevel->apVtabLock, n); if( apVtabLock ){ pToplevel->apVtabLock = apVtabLock; pToplevel->apVtabLock[pToplevel->nVtabLock++] = pTab; @@ -131740,9 +140010,9 @@ SQLITE_PRIVATE int sqlite3VtabEponymousTableInit(Parse *pParse, Module *pMod){ pTab->pSchema = db->aDb[0].pSchema; assert( pTab->nModuleArg==0 ); pTab->iPKey = -1; - addModuleArgument(db, pTab, sqlite3DbStrDup(db, pTab->zName)); - addModuleArgument(db, pTab, 0); - addModuleArgument(db, pTab, sqlite3DbStrDup(db, pTab->zName)); + addModuleArgument(pParse, pTab, sqlite3DbStrDup(db, pTab->zName)); + addModuleArgument(pParse, pTab, 0); + addModuleArgument(pParse, pTab, sqlite3DbStrDup(db, pTab->zName)); rc = vtabCallConstructor(db, pTab, pMod, pModule->xConnect, &zErr); if( rc ){ sqlite3ErrorMsg(pParse, "%s", zErr); @@ -131797,28 +140067,38 @@ SQLITE_API int sqlite3_vtab_on_conflict(sqlite3 *db){ SQLITE_API int sqlite3_vtab_config(sqlite3 *db, int op, ...){ va_list ap; int rc = SQLITE_OK; + VtabCtx *p; #ifdef SQLITE_ENABLE_API_ARMOR if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; #endif sqlite3_mutex_enter(db->mutex); - va_start(ap, op); - switch( op ){ - case SQLITE_VTAB_CONSTRAINT_SUPPORT: { - VtabCtx *p = db->pVtabCtx; - if( !p ){ - rc = SQLITE_MISUSE_BKPT; - }else{ - assert( p->pTab==0 || IsVirtual(p->pTab) ); + p = db->pVtabCtx; + if( !p ){ + rc = SQLITE_MISUSE_BKPT; + }else{ + assert( p->pTab==0 || IsVirtual(p->pTab) ); + va_start(ap, op); + switch( op ){ + case SQLITE_VTAB_CONSTRAINT_SUPPORT: { p->pVTable->bConstraint = (u8)va_arg(ap, int); + break; + } + case SQLITE_VTAB_INNOCUOUS: { + p->pVTable->eVtabRisk = SQLITE_VTABRISK_Low; + break; + } + case SQLITE_VTAB_DIRECTONLY: { + p->pVTable->eVtabRisk = SQLITE_VTABRISK_High; + break; + } + default: { + rc = SQLITE_MISUSE_BKPT; + break; } - break; } - default: - rc = SQLITE_MISUSE_BKPT; - break; + va_end(ap); } - va_end(ap); if( rc!=SQLITE_OK ) sqlite3Error(db, rc); sqlite3_mutex_leave(db->mutex); @@ -131867,6 +140147,8 @@ SQLITE_API int sqlite3_vtab_config(sqlite3 *db, int op, ...){ ** planner logic in "where.c". These definitions are broken out into ** a separate source file for easier editing. */ +#ifndef SQLITE_WHEREINT_H +#define SQLITE_WHEREINT_H /* ** Trace output macros @@ -131922,19 +140204,23 @@ struct WhereLevel { int addrCont; /* Jump here to continue with the next loop cycle */ int addrFirst; /* First instruction of interior of the loop */ int addrBody; /* Beginning of the body of this loop */ + int regBignull; /* big-null flag reg. True if a NULL-scan is needed */ + int addrBignull; /* Jump here for next part of big-null scan */ #ifndef SQLITE_LIKE_DOESNT_MATCH_BLOBS u32 iLikeRepCntr; /* LIKE range processing counter register (times 2) */ int addrLikeRep; /* LIKE range processing address */ #endif u8 iFrom; /* Which entry in the FROM clause */ u8 op, p3, p5; /* Opcode, P3 & P5 of the opcode that ends the loop */ - int p1, p2; /* Operands of the opcode used to ends the loop */ + int p1, p2; /* Operands of the opcode used to end the loop */ union { /* Information that depends on pWLoop->wsFlags */ struct { int nIn; /* Number of entries in aInLoop[] */ struct InLoop { int iCur; /* The VDBE cursor used by this IN operator */ int addrInTop; /* Top of the IN loop */ + int iBase; /* Base register of multi-key index record */ + int nPrefix; /* Number of prior entires in the key */ u8 eEndLoopOp; /* IN Loop terminator. OP_Next or OP_Prev */ } *aInLoop; /* Information about each nested IN operator */ } in; /* Used when pWLoop->wsFlags&WHERE_IN_ABLE */ @@ -131977,7 +140263,7 @@ struct WhereLoop { u16 nEq; /* Number of equality constraints */ u16 nBtm; /* Size of BTM vector */ u16 nTop; /* Size of TOP vector */ - u16 nIdxCol; /* Index column used for ORDER BY */ + u16 nDistinctCol; /* Index columns used to sort for DISTINCT */ Index *pIndex; /* Index used, or NULL */ } btree; struct { /* Information for virtual tables */ @@ -132121,23 +140407,29 @@ struct WhereTerm { /* ** Allowed values of WhereTerm.wtFlags */ -#define TERM_DYNAMIC 0x01 /* Need to call sqlite3ExprDelete(db, pExpr) */ -#define TERM_VIRTUAL 0x02 /* Added by the optimizer. Do not code */ -#define TERM_CODED 0x04 /* This term is already coded */ -#define TERM_COPIED 0x08 /* Has a child */ -#define TERM_ORINFO 0x10 /* Need to free the WhereTerm.u.pOrInfo object */ -#define TERM_ANDINFO 0x20 /* Need to free the WhereTerm.u.pAndInfo obj */ -#define TERM_OR_OK 0x40 /* Used during OR-clause processing */ -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 -# define TERM_VNULL 0x80 /* Manufactured x>NULL or x<=NULL term */ -#else -# define TERM_VNULL 0x00 /* Disabled if not using stat3 */ -#endif -#define TERM_LIKEOPT 0x100 /* Virtual terms from the LIKE optimization */ -#define TERM_LIKECOND 0x200 /* Conditionally this LIKE operator term */ -#define TERM_LIKE 0x400 /* The original LIKE operator */ -#define TERM_IS 0x800 /* Term.pExpr is an IS operator */ +#define TERM_DYNAMIC 0x0001 /* Need to call sqlite3ExprDelete(db, pExpr) */ +#define TERM_VIRTUAL 0x0002 /* Added by the optimizer. Do not code */ +#define TERM_CODED 0x0004 /* This term is already coded */ +#define TERM_COPIED 0x0008 /* Has a child */ +#define TERM_ORINFO 0x0010 /* Need to free the WhereTerm.u.pOrInfo object */ +#define TERM_ANDINFO 0x0020 /* Need to free the WhereTerm.u.pAndInfo obj */ +#define TERM_OR_OK 0x0040 /* Used during OR-clause processing */ +#ifdef SQLITE_ENABLE_STAT4 +# define TERM_VNULL 0x0080 /* Manufactured x>NULL or x<=NULL term */ +#else +# define TERM_VNULL 0x0000 /* Disabled if not using stat4 */ +#endif +#define TERM_LIKEOPT 0x0100 /* Virtual terms from the LIKE optimization */ +#define TERM_LIKECOND 0x0200 /* Conditionally this LIKE operator term */ +#define TERM_LIKE 0x0400 /* The original LIKE operator */ +#define TERM_IS 0x0800 /* Term.pExpr is an IS operator */ #define TERM_VARSELECT 0x1000 /* Term.pExpr contains a correlated sub-query */ +#define TERM_HEURTRUTH 0x2000 /* Heuristic truthProb used */ +#ifdef SQLITE_ENABLE_STAT4 +# define TERM_HIGHTRUTH 0x4000 /* Term excludes few rows */ +#else +# define TERM_HIGHTRUTH 0 /* Only used with STAT4 */ +#endif /* ** An instance of the WhereScan object is used as an iterator for locating @@ -132173,6 +140465,7 @@ struct WhereClause { WhereInfo *pWInfo; /* WHERE clause processing context */ WhereClause *pOuter; /* Outer conjunction */ u8 op; /* Split operator. TK_AND or TK_OR */ + u8 hasOr; /* True if any a[].eOperator is WO_OR */ int nTerm; /* Number of terms */ int nSlot; /* Number of entries in a[] */ WhereTerm *a; /* Each a[] describes a term of the WHERE cluase */ @@ -132247,16 +140540,54 @@ struct WhereLoopBuilder { ExprList *pOrderBy; /* ORDER BY clause */ WhereLoop *pNew; /* Template WhereLoop */ WhereOrSet *pOrSet; /* Record best loops here, if not NULL */ -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +#ifdef SQLITE_ENABLE_STAT4 UnpackedRecord *pRec; /* Probe for stat4 (if required) */ int nRecValid; /* Number of valid fields currently in pRec */ #endif - unsigned int bldFlags; /* SQLITE_BLDF_* flags */ + unsigned char bldFlags1; /* First set of SQLITE_BLDF_* flags */ + unsigned char bldFlags2; /* Second set of SQLITE_BLDF_* flags */ + unsigned int iPlanLimit; /* Search limiter */ }; /* Allowed values for WhereLoopBuider.bldFlags */ -#define SQLITE_BLDF_INDEXED 0x0001 /* An index is used */ -#define SQLITE_BLDF_UNIQUE 0x0002 /* All keys of a UNIQUE index used */ +#define SQLITE_BLDF1_INDEXED 0x0001 /* An index is used */ +#define SQLITE_BLDF1_UNIQUE 0x0002 /* All keys of a UNIQUE index used */ + +#define SQLITE_BLDF2_2NDPASS 0x0004 /* Second builder pass needed */ + +/* The WhereLoopBuilder.iPlanLimit is used to limit the number of +** index+constraint combinations the query planner will consider for a +** particular query. If this parameter is unlimited, then certain +** pathological queries can spend excess time in the sqlite3WhereBegin() +** routine. The limit is high enough that is should not impact real-world +** queries. +** +** SQLITE_QUERY_PLANNER_LIMIT is the baseline limit. The limit is +** increased by SQLITE_QUERY_PLANNER_LIMIT_INCR before each term of the FROM +** clause is processed, so that every table in a join is guaranteed to be +** able to propose a some index+constraint combinations even if the initial +** baseline limit was exhausted by prior tables of the join. +*/ +#ifndef SQLITE_QUERY_PLANNER_LIMIT +# define SQLITE_QUERY_PLANNER_LIMIT 20000 +#endif +#ifndef SQLITE_QUERY_PLANNER_LIMIT_INCR +# define SQLITE_QUERY_PLANNER_LIMIT_INCR 1000 +#endif + +/* +** Each instance of this object records a change to a single node +** in an expression tree to cause that node to point to a column +** of an index rather than an expression or a virtual column. All +** such transformations need to be undone at the end of WHERE clause +** processing. +*/ +typedef struct WhereExprMod WhereExprMod; +struct WhereExprMod { + WhereExprMod *pNext; /* Next translation on a list of them all */ + Expr *pExpr; /* The Expr node that was transformed */ + Expr orig; /* Original value of the Expr node */ +}; /* ** The WHERE clause processing routine has two halves. The @@ -132274,23 +140605,25 @@ struct WhereInfo { ExprList *pOrderBy; /* The ORDER BY clause or NULL */ ExprList *pResultSet; /* Result set of the query */ Expr *pWhere; /* The complete WHERE clause */ - LogEst iLimit; /* LIMIT if wctrlFlags has WHERE_USE_LIMIT */ int aiCurOnePass[2]; /* OP_OpenWrite cursors for the ONEPASS opt */ int iContinue; /* Jump here to continue with next record */ int iBreak; /* Jump here to break out of the loop */ int savedNQueryLoop; /* pParse->nQueryLoop outside the WHERE loop */ u16 wctrlFlags; /* Flags originally passed to sqlite3WhereBegin() */ + LogEst iLimit; /* LIMIT if wctrlFlags has WHERE_USE_LIMIT */ u8 nLevel; /* Number of nested loop */ i8 nOBSat; /* Number of ORDER BY terms satisfied by indices */ - u8 sorted; /* True if really sorted (not just grouped) */ u8 eOnePass; /* ONEPASS_OFF, or _SINGLE, or _MULTI */ - u8 untestedTerms; /* Not all WHERE terms resolved by outer loop */ u8 eDistinct; /* One of the WHERE_DISTINCT_* values */ - u8 bOrderedInnerLoop; /* True if only the inner-most loop is ordered */ + unsigned bDeferredSeek :1; /* Uses OP_DeferredSeek */ + unsigned untestedTerms :1; /* Not all WHERE terms resolved by outer loop */ + unsigned bOrderedInnerLoop:1;/* True if only the inner-most loop is ordered */ + unsigned sorted :1; /* True if really sorted (not just grouped) */ + LogEst nRowOut; /* Estimated number of output rows */ int iTop; /* The very beginning of the WHERE loop */ WhereLoop *pLoops; /* List of all WhereLoop objects */ + WhereExprMod *pExprMods; /* Expression modifications */ Bitmask revMask; /* Mask of ORDER BY terms that need reversing */ - LogEst nRowOut; /* Estimated number of output rows */ WhereClause sWC; /* Decomposition of the WHERE clause */ WhereMaskSet sMaskSet; /* Map cursor numbers to bitmasks */ WhereLevel a[1]; /* Information about each nest loop in WHERE */ @@ -132304,6 +140637,8 @@ struct WhereInfo { SQLITE_PRIVATE Bitmask sqlite3WhereGetMask(WhereMaskSet*,int); #ifdef WHERETRACE_ENABLED SQLITE_PRIVATE void sqlite3WhereClausePrint(WhereClause *pWC); +SQLITE_PRIVATE void sqlite3WhereTermPrint(WhereTerm *pTerm, int iTerm); +SQLITE_PRIVATE void sqlite3WhereLoopPrint(WhereLoop *p, WhereClause *pWC); #endif SQLITE_PRIVATE WhereTerm *sqlite3WhereFindTerm( WhereClause *pWC, /* The WHERE clause to be searched */ @@ -132336,8 +140671,11 @@ SQLITE_PRIVATE void sqlite3WhereAddScanStatus( # define sqlite3WhereAddScanStatus(a, b, c, d) ((void)d) #endif SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( + Parse *pParse, /* Parsing context */ + Vdbe *v, /* Prepared statement under construction */ WhereInfo *pWInfo, /* Complete information about the WHERE clause */ int iLevel, /* Which level of pWInfo->a[] should be coded */ + WhereLevel *pLevel, /* The current level pointer */ Bitmask notReady /* Which tables are currently available */ ); @@ -132346,6 +140684,7 @@ SQLITE_PRIVATE void sqlite3WhereClauseInit(WhereClause*,WhereInfo*); SQLITE_PRIVATE void sqlite3WhereClauseClear(WhereClause*); SQLITE_PRIVATE void sqlite3WhereSplit(WhereClause*,Expr*,u8); SQLITE_PRIVATE Bitmask sqlite3WhereExprUsage(WhereMaskSet*, Expr*); +SQLITE_PRIVATE Bitmask sqlite3WhereExprUsageNN(WhereMaskSet*, Expr*); SQLITE_PRIVATE Bitmask sqlite3WhereExprListUsage(WhereMaskSet*, ExprList*); SQLITE_PRIVATE void sqlite3WhereExprAnalyze(SrcList*, WhereClause*); SQLITE_PRIVATE void sqlite3WhereTabFuncArgs(Parse*, struct SrcList_item*, WhereClause*); @@ -132408,6 +140747,10 @@ SQLITE_PRIVATE void sqlite3WhereTabFuncArgs(Parse*, struct SrcList_item*, WhereC #define WHERE_SKIPSCAN 0x00008000 /* Uses the skip-scan algorithm */ #define WHERE_UNQ_WANTED 0x00010000 /* WHERE_ONEROW would have been helpful*/ #define WHERE_PARTIALIDX 0x00020000 /* The automatic index is partial */ +#define WHERE_IN_EARLYOUT 0x00040000 /* Perhaps quit IN loops early */ +#define WHERE_BIGNULL_SORT 0x00080000 /* Column nEq of index is BIGNULL */ + +#endif /* !defined(SQLITE_WHEREINT_H) */ /************** End of whereInt.h ********************************************/ /************** Continuing where we left off in wherecode.c ******************/ @@ -132542,7 +140885,7 @@ SQLITE_PRIVATE int sqlite3WhereExplainOneScan( sqlite3StrAccumInit(&str, db, zBuf, sizeof(zBuf), SQLITE_MAX_LENGTH); sqlite3_str_appendall(&str, isSearch ? "SEARCH" : "SCAN"); if( pItem->pSelect ){ - sqlite3_str_appendf(&str, " SUBQUERY 0x%p", pItem->pSelect); + sqlite3_str_appendf(&str, " SUBQUERY %u", pItem->pSelect->selId); }else{ sqlite3_str_appendf(&str, " TABLE %s", pItem->zName); } @@ -132605,6 +140948,7 @@ SQLITE_PRIVATE int sqlite3WhereExplainOneScan( } #endif zMsg = sqlite3StrAccumFinish(&str); + sqlite3ExplainBreakpoint("",zMsg); ret = sqlite3VdbeAddOp4(v, OP_Explain, sqlite3VdbeCurrentAddr(v), pParse->addrExplain, 0, zMsg,P4_DYNAMIC); } @@ -132709,9 +141053,9 @@ static void disableTerm(WhereLevel *pLevel, WhereTerm *pTerm){ ** Code an OP_Affinity opcode to apply the column affinity string zAff ** to the n registers starting at base. ** -** As an optimization, SQLITE_AFF_BLOB entries (which are no-ops) at the -** beginning and end of zAff are ignored. If all entries in zAff are -** SQLITE_AFF_BLOB, then no code gets generated. +** As an optimization, SQLITE_AFF_BLOB and SQLITE_AFF_NONE entries (which +** are no-ops) at the beginning and end of zAff are ignored. If all entries +** in zAff are SQLITE_AFF_BLOB or SQLITE_AFF_NONE, then no code gets generated. ** ** This routine makes its own copy of zAff so that the caller is free ** to modify zAff after this routine returns. @@ -132724,22 +141068,22 @@ static void codeApplyAffinity(Parse *pParse, int base, int n, char *zAff){ } assert( v!=0 ); - /* Adjust base and n to skip over SQLITE_AFF_BLOB entries at the beginning - ** and end of the affinity string. + /* Adjust base and n to skip over SQLITE_AFF_BLOB and SQLITE_AFF_NONE + ** entries at the beginning and end of the affinity string. */ - while( n>0 && zAff[0]==SQLITE_AFF_BLOB ){ + assert( SQLITE_AFF_NONE0 && zAff[0]<=SQLITE_AFF_BLOB ){ n--; base++; zAff++; } - while( n>1 && zAff[n-1]==SQLITE_AFF_BLOB ){ + while( n>1 && zAff[n-1]<=SQLITE_AFF_BLOB ){ n--; } /* Code the OP_Affinity opcode if there is anything left to do. */ if( n>0 ){ sqlite3VdbeAddOp4(v, OP_Affinity, base, n, 0, zAff, n); - sqlite3ExprCacheAffinityChange(pParse, base, n); } } @@ -132806,7 +141150,8 @@ static Expr *removeUnindexableInClauseTerms( Expr *pX /* The IN expression to be reduced */ ){ sqlite3 *db = pParse->db; - Expr *pNew = sqlite3ExprDup(db, pX, 0); + Expr *pNew; + pNew = sqlite3ExprDup(db, pX, 0); if( db->mallocFailed==0 ){ ExprList *pOrigRhs = pNew->x.pSelect->pEList; /* Original unmodified RHS */ ExprList *pOrigLhs = pNew->pLeft->x.pList; /* Original unmodified LHS */ @@ -132818,7 +141163,7 @@ static Expr *removeUnindexableInClauseTerms( for(i=iEq; inLTerm; i++){ if( pLoop->aLTerm[i]->pExpr==pX ){ int iField = pLoop->aLTerm[i]->iField - 1; - assert( pOrigRhs->a[iField].pExpr!=0 ); + if( pOrigRhs->a[iField].pExpr==0 ) continue; /* Duplicate PK column */ pRhs = sqlite3ExprListAppend(pParse, pRhs, pOrigRhs->a[iField].pExpr); pOrigRhs->a[iField].pExpr = 0; assert( pOrigLhs->a[iField].pExpr!=0 ); @@ -132931,16 +141276,17 @@ static int codeEqualityTerm( if( pLoop->aLTerm[i]->pExpr==pX ) nEq++; } + iTab = 0; if( (pX->flags & EP_xIsSelect)==0 || pX->x.pSelect->pEList->nExpr==1 ){ - eType = sqlite3FindInIndex(pParse, pX, IN_INDEX_LOOP, 0, 0); + eType = sqlite3FindInIndex(pParse, pX, IN_INDEX_LOOP, 0, 0, &iTab); }else{ sqlite3 *db = pParse->db; pX = removeUnindexableInClauseTerms(pParse, iEq, pLoop, pX); if( !db->mallocFailed ){ aiMap = (int*)sqlite3DbMallocZero(pParse->db, sizeof(int)*nEq); - eType = sqlite3FindInIndex(pParse, pX, IN_INDEX_LOOP, 0, aiMap); - pTerm->pExpr->iTable = pX->iTable; + eType = sqlite3FindInIndex(pParse, pX, IN_INDEX_LOOP, 0, aiMap, &iTab); + pTerm->pExpr->iTable = iTab; } sqlite3ExprDelete(db, pX); pX = pTerm->pExpr; @@ -132950,7 +141296,6 @@ static int codeEqualityTerm( testcase( bRev ); bRev = !bRev; } - iTab = pX->iTable; sqlite3VdbeAddOp2(v, bRev ? OP_Last : OP_Rewind, iTab, 0); VdbeCoverageIf(v, bRev); VdbeCoverageIf(v, !bRev); @@ -132958,7 +141303,7 @@ static int codeEqualityTerm( pLoop->wsFlags |= WHERE_IN_ABLE; if( pLevel->u.in.nIn==0 ){ - pLevel->addrNxt = sqlite3VdbeMakeLabel(v); + pLevel->addrNxt = sqlite3VdbeMakeLabel(pParse); } i = pLevel->u.in.nIn; @@ -132974,7 +141319,6 @@ static int codeEqualityTerm( if( pLoop->aLTerm[i]->pExpr==pX ){ int iOut = iReg + i - iEq; if( eType==IN_INDEX_ROWID ){ - testcase( nEq>1 ); /* Happens with a UNIQUE index on ROWID */ pIn->addrInTop = sqlite3VdbeAddOp2(v, OP_Rowid, iTab, iOut); }else{ int iCol = aiMap ? aiMap[iMap++] : 0; @@ -132983,7 +141327,14 @@ static int codeEqualityTerm( sqlite3VdbeAddOp1(v, OP_IsNull, iOut); VdbeCoverage(v); if( i==iEq ){ pIn->iCur = iTab; - pIn->eEndLoopOp = bRev ? OP_PrevIfOpen : OP_NextIfOpen; + pIn->eEndLoopOp = bRev ? OP_Prev : OP_Next; + if( iEq>0 ){ + pIn->iBase = iReg - i; + pIn->nPrefix = i; + pLoop->wsFlags |= WHERE_IN_EARLYOUT; + }else{ + pIn->nPrefix = 0; + } }else{ pIn->eEndLoopOp = OP_Noop; } @@ -133208,7 +141559,7 @@ static int codeCursorHintCheckExpr(Walker *pWalker, Expr *pExpr){ assert( pHint->pIdx!=0 ); if( pExpr->op==TK_COLUMN && pExpr->iTable==pHint->iTabCur - && sqlite3ColumnOfIndex(pHint->pIdx, pExpr->iColumn)<0 + && sqlite3TableColumnToIndex(pHint->pIdx, pExpr->iColumn)<0 ){ pWalker->eCode = 1; } @@ -133270,16 +141621,13 @@ static int codeCursorHintFixExpr(Walker *pWalker, Expr *pExpr){ struct CCurHint *pHint = pWalker->u.pCCurHint; if( pExpr->op==TK_COLUMN ){ if( pExpr->iTable!=pHint->iTabCur ){ - Vdbe *v = pWalker->pParse->pVdbe; int reg = ++pWalker->pParse->nMem; /* Register for column value */ - sqlite3ExprCodeGetColumnOfTable( - v, pExpr->pTab, pExpr->iTable, pExpr->iColumn, reg - ); + sqlite3ExprCode(pWalker->pParse, pExpr, reg); pExpr->op = TK_REGISTER; pExpr->iTable = reg; }else if( pHint->pIdx!=0 ){ pExpr->iTable = pHint->iIdxCur; - pExpr->iColumn = sqlite3ColumnOfIndex(pHint->pIdx, pExpr->iColumn); + pExpr->iColumn = sqlite3TableColumnToIndex(pHint->pIdx, pExpr->iColumn); assert( pExpr->iColumn>=0 ); } }else if( pExpr->op==TK_AGG_FUNCTION ){ @@ -133388,7 +141736,7 @@ static void codeCursorHint( } /* If we survive all prior tests, that means this term is worth hinting */ - pExpr = sqlite3ExprAnd(db, pExpr, sqlite3ExprDup(db, pTerm->pExpr, 0)); + pExpr = sqlite3ExprAnd(pParse, pExpr, sqlite3ExprDup(db, pTerm->pExpr, 0)); } if( pExpr!=0 ){ sWalker.xExprCallback = codeCursorHintFixExpr; @@ -133432,6 +141780,7 @@ static void codeDeferredSeek( assert( iIdxCur>0 ); assert( pIdx->aiColumn[pIdx->nColumn-1]==-1 ); + pWInfo->bDeferredSeek = 1; sqlite3VdbeAddOp3(v, OP_DeferredSeek, iIdxCur, 0, iCur); if( (pWInfo->wctrlFlags & WHERE_OR_SUBCLAUSE) && DbMaskAllZero(sqlite3ParseToplevel(pParse)->writeMask) @@ -133442,8 +141791,12 @@ static void codeDeferredSeek( if( ai ){ ai[0] = pTab->nCol; for(i=0; inColumn-1; i++){ + int x1, x2; assert( pIdx->aiColumn[i]nCol ); - if( pIdx->aiColumn[i]>=0 ) ai[pIdx->aiColumn[i]+1] = i+1; + x1 = pIdx->aiColumn[i]; + x2 = sqlite3TableColumnToStorage(pTab, x1); + testcase( x1!=x2 ); + if( x1>=0 ) ai[x2+1] = i+1; } sqlite3VdbeChangeP4(v, -1, (char*)ai, P4_INTARRAY); } @@ -133465,7 +141818,9 @@ static void codeExprOrVector(Parse *pParse, Expr *p, int iReg, int nReg){ #ifndef SQLITE_OMIT_SUBQUERY if( (p->flags & EP_xIsSelect) ){ Vdbe *v = pParse->pVdbe; - int iSelect = sqlite3CodeSubselect(pParse, p, 0, 0); + int iSelect; + assert( p->op==TK_SELECT ); + iSelect = sqlite3CodeSubselect(pParse, p); sqlite3VdbeAddOp3(v, OP_Copy, iSelect, iReg, nReg-1); }else #endif @@ -133492,8 +141847,24 @@ typedef struct IdxExprTrans { int iTabCur; /* The cursor of the corresponding table */ int iIdxCur; /* The cursor for the index */ int iIdxCol; /* The column for the index */ + int iTabCol; /* The column for the table */ + WhereInfo *pWInfo; /* Complete WHERE clause information */ + sqlite3 *db; /* Database connection (for malloc()) */ } IdxExprTrans; +/* +** Preserve pExpr on the WhereETrans list of the WhereInfo. +*/ +static void preserveExpr(IdxExprTrans *pTrans, Expr *pExpr){ + WhereExprMod *pNew; + pNew = sqlite3DbMallocRaw(pTrans->db, sizeof(*pNew)); + if( pNew==0 ) return; + pNew->pNext = pTrans->pWInfo->pExprMods; + pTrans->pWInfo->pExprMods = pNew; + pNew->pExpr = pExpr; + memcpy(&pNew->orig, pExpr, sizeof(*pExpr)); +} + /* The walker node callback used to transform matching expressions into ** a reference to an index column for an index on an expression. ** @@ -133503,20 +141874,49 @@ typedef struct IdxExprTrans { static int whereIndexExprTransNode(Walker *p, Expr *pExpr){ IdxExprTrans *pX = p->u.pIdxTrans; if( sqlite3ExprCompare(0, pExpr, pX->pIdxExpr, pX->iTabCur)==0 ){ + preserveExpr(pX, pExpr); + pExpr->affExpr = sqlite3ExprAffinity(pExpr); pExpr->op = TK_COLUMN; pExpr->iTable = pX->iIdxCur; pExpr->iColumn = pX->iIdxCol; - pExpr->pTab = 0; + pExpr->y.pTab = 0; + testcase( ExprHasProperty(pExpr, EP_Skip) ); + testcase( ExprHasProperty(pExpr, EP_Unlikely) ); + ExprClearProperty(pExpr, EP_Skip|EP_Unlikely); return WRC_Prune; }else{ return WRC_Continue; } } +#ifndef SQLITE_OMIT_GENERATED_COLUMNS +/* A walker node callback that translates a column reference to a table +** into a corresponding column reference of an index. +*/ +static int whereIndexExprTransColumn(Walker *p, Expr *pExpr){ + if( pExpr->op==TK_COLUMN ){ + IdxExprTrans *pX = p->u.pIdxTrans; + if( pExpr->iTable==pX->iTabCur && pExpr->iColumn==pX->iTabCol ){ + assert( pExpr->y.pTab!=0 ); + preserveExpr(pX, pExpr); + pExpr->affExpr = sqlite3TableColumnAffinity(pExpr->y.pTab,pExpr->iColumn); + pExpr->iTable = pX->iIdxCur; + pExpr->iColumn = pX->iIdxCol; + pExpr->y.pTab = 0; + } + } + return WRC_Continue; +} +#endif /* SQLITE_OMIT_GENERATED_COLUMNS */ + /* ** For an indexes on expression X, locate every instance of expression X ** in pExpr and change that subexpression into a reference to the appropriate ** column of the index. +** +** 2019-10-24: Updated to also translate references to a VIRTUAL column in +** the table into references to the corresponding (stored) column of the +** index. */ static void whereIndexExprTrans( Index *pIdx, /* The Index */ @@ -133526,47 +141926,102 @@ static void whereIndexExprTrans( ){ int iIdxCol; /* Column number of the index */ ExprList *aColExpr; /* Expressions that are indexed */ + Table *pTab; Walker w; IdxExprTrans x; aColExpr = pIdx->aColExpr; - if( aColExpr==0 ) return; /* Not an index on expressions */ + if( aColExpr==0 && !pIdx->bHasVCol ){ + /* The index does not reference any expressions or virtual columns + ** so no translations are needed. */ + return; + } + pTab = pIdx->pTable; memset(&w, 0, sizeof(w)); - w.xExprCallback = whereIndexExprTransNode; w.u.pIdxTrans = &x; x.iTabCur = iTabCur; x.iIdxCur = iIdxCur; - for(iIdxCol=0; iIdxColnExpr; iIdxCol++){ - if( pIdx->aiColumn[iIdxCol]!=XN_EXPR ) continue; - assert( aColExpr->a[iIdxCol].pExpr!=0 ); + x.pWInfo = pWInfo; + x.db = pWInfo->pParse->db; + for(iIdxCol=0; iIdxColnColumn; iIdxCol++){ + i16 iRef = pIdx->aiColumn[iIdxCol]; + if( iRef==XN_EXPR ){ + assert( aColExpr->a[iIdxCol].pExpr!=0 ); + x.pIdxExpr = aColExpr->a[iIdxCol].pExpr; + if( sqlite3ExprIsConstant(x.pIdxExpr) ) continue; + w.xExprCallback = whereIndexExprTransNode; +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + }else if( iRef>=0 + && (pTab->aCol[iRef].colFlags & COLFLAG_VIRTUAL)!=0 + && (pTab->aCol[iRef].zColl==0 + || sqlite3StrICmp(pTab->aCol[iRef].zColl, sqlite3StrBINARY)==0) + ){ + /* Check to see if there are direct references to generated columns + ** that are contained in the index. Pulling the generated column + ** out of the index is an optimization only - the main table is always + ** available if the index cannot be used. To avoid unnecessary + ** complication, omit this optimization if the collating sequence for + ** the column is non-standard */ + x.iTabCol = iRef; + w.xExprCallback = whereIndexExprTransColumn; +#endif /* SQLITE_OMIT_GENERATED_COLUMNS */ + }else{ + continue; + } x.iIdxCol = iIdxCol; - x.pIdxExpr = aColExpr->a[iIdxCol].pExpr; sqlite3WalkExpr(&w, pWInfo->pWhere); sqlite3WalkExprList(&w, pWInfo->pOrderBy); sqlite3WalkExprList(&w, pWInfo->pResultSet); } } +/* +** The pTruth expression is always true because it is the WHERE clause +** a partial index that is driving a query loop. Look through all of the +** WHERE clause terms on the query, and if any of those terms must be +** true because pTruth is true, then mark those WHERE clause terms as +** coded. +*/ +static void whereApplyPartialIndexConstraints( + Expr *pTruth, + int iTabCur, + WhereClause *pWC +){ + int i; + WhereTerm *pTerm; + while( pTruth->op==TK_AND ){ + whereApplyPartialIndexConstraints(pTruth->pLeft, iTabCur, pWC); + pTruth = pTruth->pRight; + } + for(i=0, pTerm=pWC->a; inTerm; i++, pTerm++){ + Expr *pExpr; + if( pTerm->wtFlags & TERM_CODED ) continue; + pExpr = pTerm->pExpr; + if( sqlite3ExprCompare(0, pExpr, pTruth, iTabCur)==0 ){ + pTerm->wtFlags |= TERM_CODED; + } + } +} + /* ** Generate code for the start of the iLevel-th loop in the WHERE clause ** implementation described by pWInfo. */ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( + Parse *pParse, /* Parsing context */ + Vdbe *v, /* Prepared statement under construction */ WhereInfo *pWInfo, /* Complete information about the WHERE clause */ int iLevel, /* Which level of pWInfo->a[] should be coded */ + WhereLevel *pLevel, /* The current level pointer */ Bitmask notReady /* Which tables are currently available */ ){ int j, k; /* Loop counters */ int iCur; /* The VDBE cursor for the table */ int addrNxt; /* Where to jump to continue with the next IN case */ - int omitTable; /* True if we use the index only */ int bRev; /* True if we need to scan in reverse order */ - WhereLevel *pLevel; /* The where level to be coded */ WhereLoop *pLoop; /* The WhereLoop object being coded */ WhereClause *pWC; /* Decomposition of the entire WHERE clause */ WhereTerm *pTerm; /* A WHERE clause term */ - Parse *pParse; /* Parsing context */ sqlite3 *db; /* Database connection */ - Vdbe *v; /* The prepared stmt under constructions */ struct SrcList_item *pTabItem; /* FROM clause term being coded */ int addrBrk; /* Jump here to break out of the loop */ int addrHalt; /* addrBrk for the outermost loop */ @@ -133576,19 +142031,29 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( Index *pIdx = 0; /* Index used by loop (if any) */ int iLoop; /* Iteration of constraint generator loop */ - pParse = pWInfo->pParse; - v = pParse->pVdbe; pWC = &pWInfo->sWC; db = pParse->db; - pLevel = &pWInfo->a[iLevel]; pLoop = pLevel->pWLoop; pTabItem = &pWInfo->pTabList->a[pLevel->iFrom]; iCur = pTabItem->iCursor; pLevel->notReady = notReady & ~sqlite3WhereGetMask(&pWInfo->sMaskSet, iCur); bRev = (pWInfo->revMask>>iLevel)&1; - omitTable = (pLoop->wsFlags & WHERE_IDX_ONLY)!=0 - && (pWInfo->wctrlFlags & WHERE_OR_SUBCLAUSE)==0; VdbeModuleComment((v, "Begin WHERE-loop%d: %s",iLevel,pTabItem->pTab->zName)); +#if WHERETRACE_ENABLED /* 0x20800 */ + if( sqlite3WhereTrace & 0x800 ){ + sqlite3DebugPrintf("Coding level %d of %d: notReady=%llx iFrom=%d\n", + iLevel, pWInfo->nLevel, (u64)notReady, pLevel->iFrom); + sqlite3WhereLoopPrint(pLoop, pWC); + } + if( sqlite3WhereTrace & 0x20000 ){ + if( iLevel==0 ){ + sqlite3DebugPrintf("WHERE clause being coded:\n"); + sqlite3TreeViewExpr(0, pWInfo->pWhere, 0); + } + sqlite3DebugPrintf("All WHERE-clause terms before coding:\n"); + sqlite3WhereClausePrint(pWC); + } +#endif /* Create labels for the "break" and "continue" instructions ** for the current loop. Jump to addrBrk to break out of a loop. @@ -133600,8 +142065,8 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( ** there are no IN operators in the constraints, the "addrNxt" label ** is the same as "addrBrk". */ - addrBrk = pLevel->addrBrk = pLevel->addrNxt = sqlite3VdbeMakeLabel(v); - addrCont = pLevel->addrCont = sqlite3VdbeMakeLabel(v); + addrBrk = pLevel->addrBrk = pLevel->addrNxt = sqlite3VdbeMakeLabel(pParse); + addrCont = pLevel->addrCont = sqlite3VdbeMakeLabel(pParse); /* If this is the right table of a LEFT OUTER JOIN, allocate and ** initialize a memory cell that records if this table matches any @@ -133627,7 +142092,7 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, pTabItem->addrFillSub); pLevel->p2 = sqlite3VdbeAddOp2(v, OP_Yield, regYield, addrBrk); VdbeCoverage(v); - VdbeComment((v, "next row of \"%s\"", pTabItem->pTab->zName)); + VdbeComment((v, "next row of %s", pTabItem->pTab->zName)); pLevel->op = OP_Goto; }else @@ -133641,7 +142106,6 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( int nConstraint = pLoop->nLTerm; int iIn; /* Counter for IN constraints */ - sqlite3ExprCachePush(pParse); iReg = sqlite3GetTempRange(pParse, nConstraint+2); addrNotFound = pLevel->addrBrk; for(j=0; ju.in.nIn; for(j=nConstraint-1; j>=0; j--){ pTerm = pLoop->aLTerm[j]; + if( (pTerm->eOperator & WO_IN)!=0 ) iIn--; if( j<16 && (pLoop->u.vtab.omitMask>>j)&1 ){ disableTerm(pLevel, pTerm); - }else if( (pTerm->eOperator & WO_IN)!=0 ){ + }else if( (pTerm->eOperator & WO_IN)!=0 + && sqlite3ExprVectorSize(pTerm->pExpr->pLeft)==1 + ){ Expr *pCompare; /* The comparison operator */ Expr *pRight; /* RHS of the comparison */ VdbeOp *pOp; /* Opcode to access the value of the IN constraint */ @@ -133682,8 +142149,8 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( ** encoding of the value in the register, so it *must* be reloaded. */ assert( pLevel->u.in.aInLoop!=0 || db->mallocFailed ); if( !db->mallocFailed ){ - assert( iIn>0 ); - pOp = sqlite3VdbeGetOp(v, pLevel->u.in.aInLoop[--iIn].addrInTop); + assert( iIn>=0 && iInu.in.nIn ); + pOp = sqlite3VdbeGetOp(v, pLevel->u.in.aInLoop[iIn].addrInTop); assert( pOp->opcode==OP_Column || pOp->opcode==OP_Rowid ); assert( pOp->opcode!=OP_Column || pOp->p3==iReg+j+2 ); assert( pOp->opcode!=OP_Rowid || pOp->p2==iReg+j+2 ); @@ -133700,13 +142167,16 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( pCompare->pRight = pRight = sqlite3Expr(db, TK_REGISTER, 0); if( pRight ){ pRight->iTable = iReg+j+2; - sqlite3ExprIfFalse(pParse, pCompare, pLevel->addrCont, 0); + sqlite3ExprIfFalse( + pParse, pCompare, pLevel->addrCont, SQLITE_JUMPIFNULL + ); } pCompare->pLeft = 0; sqlite3ExprDelete(db, pCompare); } } } + assert( iIn==0 || db->mallocFailed ); /* These registers need to be preserved in case there is an IN operator ** loop. So we could deallocate the registers here (and potentially ** reuse them later) if (pLoop->wsFlags & WHERE_IN_ABLE)==0. But it seems @@ -133714,7 +142184,6 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( ** ** sqlite3ReleaseTempRange(pParse, iReg, nConstraint+2); */ - sqlite3ExprCachePop(pParse); }else #endif /* SQLITE_OMIT_VIRTUALTABLE */ @@ -133730,7 +142199,6 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( pTerm = pLoop->aLTerm[0]; assert( pTerm!=0 ); assert( pTerm->pExpr!=0 ); - assert( omitTable==0 ); testcase( pTerm->wtFlags & TERM_VIRTUAL ); iReleaseReg = ++pParse->nMem; iRowidReg = codeEqualityTerm(pParse, pTerm, pLevel, 0, bRev, iReleaseReg); @@ -133738,10 +142206,10 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( addrNxt = pLevel->addrNxt; sqlite3VdbeAddOp3(v, OP_SeekRowid, iCur, addrNxt, iRowidReg); VdbeCoverage(v); - sqlite3ExprCacheAffinityChange(pParse, iRowidReg, 1); - sqlite3ExprCacheStore(pParse, iCur, -1, iRowidReg); - VdbeComment((v, "pk")); pLevel->op = OP_Noop; + if( (pTerm->prereqAll & pLevel->notReady)==0 ){ + pTerm->wtFlags |= TERM_CODED; + } }else if( (pLoop->wsFlags & WHERE_IPK)!=0 && (pLoop->wsFlags & WHERE_COLUMN_RANGE)!=0 ){ @@ -133752,7 +142220,6 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( int memEndValue = 0; WhereTerm *pStart, *pEnd; - assert( omitTable==0 ); j = 0; pStart = pEnd = 0; if( pLoop->wsFlags & WHERE_BTM_LIMIT ) pStart = pLoop->aLTerm[j++]; @@ -133810,7 +142277,6 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( VdbeCoverageIf(v, pX->op==TK_LE); VdbeCoverageIf(v, pX->op==TK_LT); VdbeCoverageIf(v, pX->op==TK_GE); - sqlite3ExprCacheAffinityChange(pParse, r1, 1); sqlite3ReleaseTempReg(pParse, rTemp); }else{ sqlite3VdbeAddOp2(v, bRev ? OP_Last : OP_Rewind, iCur, addrHalt); @@ -133845,7 +142311,6 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( if( testOp!=OP_Noop ){ iRowidReg = ++pParse->nMem; sqlite3VdbeAddOp2(v, OP_Rowid, iCur, iRowidReg); - sqlite3ExprCacheStore(pParse, iCur, -1, iRowidReg); sqlite3VdbeAddOp3(v, testOp, memEndValue, addrBrk, iRowidReg); VdbeCoverageIf(v, testOp==OP_Le); VdbeCoverageIf(v, testOp==OP_Lt); @@ -133918,31 +142383,13 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( char *zEndAff = 0; /* Affinity for end of range constraint */ u8 bSeekPastNull = 0; /* True to seek past initial nulls */ u8 bStopAtNull = 0; /* Add condition to terminate at NULLs */ + int omitTable; /* True if we use the index only */ + int regBignull = 0; /* big-null flag register */ pIdx = pLoop->u.btree.pIndex; iIdxCur = pLevel->iIdxCur; assert( nEq>=pLoop->nSkip ); - /* If this loop satisfies a sort order (pOrderBy) request that - ** was passed to this function to implement a "SELECT min(x) ..." - ** query, then the caller will only allow the loop to run for - ** a single iteration. This means that the first row returned - ** should not have a NULL value stored in 'x'. If column 'x' is - ** the first one after the nEq equality constraints in the index, - ** this requires some special handling. - */ - assert( pWInfo->pOrderBy==0 - || pWInfo->pOrderBy->nExpr==1 - || (pWInfo->wctrlFlags&WHERE_ORDERBY_MIN)==0 ); - if( (pWInfo->wctrlFlags&WHERE_ORDERBY_MIN)!=0 - && pWInfo->nOBSat>0 - && (pIdx->nKeyCol>nEq) - ){ - assert( pLoop->nSkip==0 ); - bSeekPastNull = 1; - nExtraReg = 1; - } - /* Find any inequality constraint terms for the start and end ** of the range. */ @@ -133983,6 +142430,28 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( } assert( pRangeEnd==0 || (pRangeEnd->wtFlags & TERM_VNULL)==0 ); + /* If the WHERE_BIGNULL_SORT flag is set, then index column nEq uses + ** a non-default "big-null" sort (either ASC NULLS LAST or DESC NULLS + ** FIRST). In both cases separate ordered scans are made of those + ** index entries for which the column is null and for those for which + ** it is not. For an ASC sort, the non-NULL entries are scanned first. + ** For DESC, NULL entries are scanned first. + */ + if( (pLoop->wsFlags & (WHERE_TOP_LIMIT|WHERE_BTM_LIMIT))==0 + && (pLoop->wsFlags & WHERE_BIGNULL_SORT)!=0 + ){ + assert( bSeekPastNull==0 && nExtraReg==0 && nBtm==0 && nTop==0 ); + assert( pRangeEnd==0 && pRangeStart==0 ); + testcase( pLoop->nSkip>0 ); + nExtraReg = 1; + bSeekPastNull = 1; + pLevel->regBignull = regBignull = ++pParse->nMem; + if( pLevel->iLeftJoin ){ + sqlite3VdbeAddOp2(v, OP_Integer, 0, regBignull); + } + pLevel->addrBignull = sqlite3VdbeMakeLabel(pParse); + } + /* If we are doing a reverse order scan on an ascending index, or ** a forward order scan on a descending index, interchange the ** start and end terms (pRangeStart and pRangeEnd). @@ -134005,7 +142474,7 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( if( zStartAff && nTop ){ zEndAff = sqlite3DbStrDup(db, &zStartAff[nEq]); } - addrNxt = pLevel->addrNxt; + addrNxt = (regBignull ? pLevel->addrBignull : pLevel->addrNxt); testcase( pRangeStart && (pRangeStart->eOperator & WO_LE)!=0 ); testcase( pRangeStart && (pRangeStart->eOperator & WO_GE)!=0 ); @@ -134039,10 +142508,14 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( } bSeekPastNull = 0; }else if( bSeekPastNull ){ + startEq = 0; sqlite3VdbeAddOp2(v, OP_Null, 0, regBase+nEq); + start_constraints = 1; nConstraint++; - startEq = 0; + }else if( regBignull ){ + sqlite3VdbeAddOp2(v, OP_Null, 0, regBase+nEq); start_constraints = 1; + nConstraint++; } codeApplyAffinity(pParse, regBase, nConstraint - bSeekPastNull, zStartAff); if( pLoop->nSkip>0 && nConstraint==pLoop->nSkip ){ @@ -134050,6 +142523,14 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( ** above has already left the cursor sitting on the correct row, ** so no further seeking is needed */ }else{ + if( pLoop->wsFlags & WHERE_IN_EARLYOUT ){ + sqlite3VdbeAddOp1(v, OP_SeekHit, iIdxCur); + } + if( regBignull ){ + sqlite3VdbeAddOp2(v, OP_Integer, 1, regBignull); + VdbeComment((v, "NULL-scan pass ctr")); + } + op = aStartOp[(start_constraints<<2) + (startEq<<1) + bRev]; assert( op!=0 ); sqlite3VdbeAddOp4Int(v, op, iIdxCur, addrNxt, regBase, nConstraint); @@ -134060,6 +142541,23 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( VdbeCoverageIf(v, op==OP_SeekGE); testcase( op==OP_SeekGE ); VdbeCoverageIf(v, op==OP_SeekLE); testcase( op==OP_SeekLE ); VdbeCoverageIf(v, op==OP_SeekLT); testcase( op==OP_SeekLT ); + + assert( bSeekPastNull==0 || bStopAtNull==0 ); + if( regBignull ){ + assert( bSeekPastNull==1 || bStopAtNull==1 ); + assert( bSeekPastNull==!bStopAtNull ); + assert( bStopAtNull==startEq ); + sqlite3VdbeAddOp2(v, OP_Goto, 0, sqlite3VdbeCurrentAddr(v)+2); + op = aStartOp[(nConstraint>1)*4 + 2 + bRev]; + sqlite3VdbeAddOp4Int(v, op, iIdxCur, addrNxt, regBase, + nConstraint-startEq); + VdbeCoverage(v); + VdbeCoverageIf(v, op==OP_Rewind); testcase( op==OP_Rewind ); + VdbeCoverageIf(v, op==OP_Last); testcase( op==OP_Last ); + VdbeCoverageIf(v, op==OP_SeekGE); testcase( op==OP_SeekGE ); + VdbeCoverageIf(v, op==OP_SeekLE); testcase( op==OP_SeekLE ); + assert( op==OP_Rewind || op==OP_Last || op==OP_SeekGE || op==OP_SeekLE); + } } /* Load the value for the inequality constraint at the end of the @@ -134068,7 +142566,6 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( nConstraint = nEq; if( pRangeEnd ){ Expr *pRight = pRangeEnd->pExpr->pRight; - sqlite3ExprCacheRemove(pParse, regBase+nEq, 1); codeExprOrVector(pParse, pRight, regBase+nEq, nTop); whereLikeOptimizationStringFixup(v, pLevel, pRangeEnd); if( (pRangeEnd->wtFlags & TERM_VNULL)==0 @@ -134092,9 +142589,10 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( endEq = 1; } }else if( bStopAtNull ){ - sqlite3VdbeAddOp2(v, OP_Null, 0, regBase+nEq); - sqlite3ExprCacheRemove(pParse, regBase+nEq, 1); - endEq = 0; + if( regBignull==0 ){ + sqlite3VdbeAddOp2(v, OP_Null, 0, regBase+nEq); + endEq = 0; + } nConstraint++; } sqlite3DbFree(db, zStartAff); @@ -134105,6 +142603,12 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( /* Check if the index cursor is past the end of the range. */ if( nConstraint ){ + if( regBignull ){ + /* Except, skip the end-of-range check while doing the NULL-scan */ + sqlite3VdbeAddOp2(v, OP_IfNot, regBignull, sqlite3VdbeCurrentAddr(v)+3); + VdbeComment((v, "If NULL-scan 2nd pass")); + VdbeCoverage(v); + } op = aEndOp[bRev*2 + endEq]; sqlite3VdbeAddOp4Int(v, op, iIdxCur, addrNxt, regBase, nConstraint); testcase( op==OP_IdxGT ); VdbeCoverageIf(v, op==OP_IdxGT ); @@ -134112,18 +142616,40 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( testcase( op==OP_IdxLT ); VdbeCoverageIf(v, op==OP_IdxLT ); testcase( op==OP_IdxLE ); VdbeCoverageIf(v, op==OP_IdxLE ); } + if( regBignull ){ + /* During a NULL-scan, check to see if we have reached the end of + ** the NULLs */ + assert( bSeekPastNull==!bStopAtNull ); + assert( bSeekPastNull+bStopAtNull==1 ); + assert( nConstraint+bSeekPastNull>0 ); + sqlite3VdbeAddOp2(v, OP_If, regBignull, sqlite3VdbeCurrentAddr(v)+2); + VdbeComment((v, "If NULL-scan 1st pass")); + VdbeCoverage(v); + op = aEndOp[bRev*2 + bSeekPastNull]; + sqlite3VdbeAddOp4Int(v, op, iIdxCur, addrNxt, regBase, + nConstraint+bSeekPastNull); + testcase( op==OP_IdxGT ); VdbeCoverageIf(v, op==OP_IdxGT ); + testcase( op==OP_IdxGE ); VdbeCoverageIf(v, op==OP_IdxGE ); + testcase( op==OP_IdxLT ); VdbeCoverageIf(v, op==OP_IdxLT ); + testcase( op==OP_IdxLE ); VdbeCoverageIf(v, op==OP_IdxLE ); + } + + if( pLoop->wsFlags & WHERE_IN_EARLYOUT ){ + sqlite3VdbeAddOp2(v, OP_SeekHit, iIdxCur, 1); + } /* Seek the table cursor, if required */ + omitTable = (pLoop->wsFlags & WHERE_IDX_ONLY)!=0 + && (pWInfo->wctrlFlags & WHERE_OR_SUBCLAUSE)==0; if( omitTable ){ /* pIdx is a covering index. No need to access the main table. */ }else if( HasRowid(pIdx->pTable) ){ - if( (pWInfo->wctrlFlags & WHERE_SEEK_TABLE) || ( - (pWInfo->wctrlFlags & WHERE_SEEK_UNIQ_TABLE) - && (pWInfo->eOnePass==ONEPASS_SINGLE) - )){ + if( (pWInfo->wctrlFlags & WHERE_SEEK_TABLE) + || ( (pWInfo->wctrlFlags & WHERE_SEEK_UNIQ_TABLE)!=0 + && (pWInfo->eOnePass==ONEPASS_SINGLE || pLoop->nLTerm==0) ) + ){ iRowidReg = ++pParse->nMem; sqlite3VdbeAddOp2(v, OP_IdxRowid, iIdxCur, iRowidReg); - sqlite3ExprCacheStore(pParse, iCur, -1, iRowidReg); sqlite3VdbeAddOp3(v, OP_NotExists, iCur, 0, iRowidReg); VdbeCoverage(v); }else{ @@ -134133,27 +142659,53 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( Index *pPk = sqlite3PrimaryKeyIndex(pIdx->pTable); iRowidReg = sqlite3GetTempRange(pParse, pPk->nKeyCol); for(j=0; jnKeyCol; j++){ - k = sqlite3ColumnOfIndex(pIdx, pPk->aiColumn[j]); + k = sqlite3TableColumnToIndex(pIdx, pPk->aiColumn[j]); sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, k, iRowidReg+j); } sqlite3VdbeAddOp4Int(v, OP_NotFound, iCur, addrCont, iRowidReg, pPk->nKeyCol); VdbeCoverage(v); } - /* If pIdx is an index on one or more expressions, then look through - ** all the expressions in pWInfo and try to transform matching expressions - ** into reference to index columns. - ** - ** Do not do this for the RHS of a LEFT JOIN. This is because the - ** expression may be evaluated after OP_NullRow has been executed on - ** the cursor. In this case it is important to do the full evaluation, - ** as the result of the expression may not be NULL, even if all table - ** column values are. https://www.sqlite.org/src/info/7fa8049685b50b5a - */ if( pLevel->iLeftJoin==0 ){ - whereIndexExprTrans(pIdx, iCur, iIdxCur, pWInfo); + /* If pIdx is an index on one or more expressions, then look through + ** all the expressions in pWInfo and try to transform matching expressions + ** into reference to index columns. Also attempt to translate references + ** to virtual columns in the table into references to (stored) columns + ** of the index. + ** + ** Do not do this for the RHS of a LEFT JOIN. This is because the + ** expression may be evaluated after OP_NullRow has been executed on + ** the cursor. In this case it is important to do the full evaluation, + ** as the result of the expression may not be NULL, even if all table + ** column values are. https://www.sqlite.org/src/info/7fa8049685b50b5a + ** + ** Also, do not do this when processing one index an a multi-index + ** OR clause, since the transformation will become invalid once we + ** move forward to the next index. + ** https://sqlite.org/src/info/4e8e4857d32d401f + */ + if( (pWInfo->wctrlFlags & WHERE_OR_SUBCLAUSE)==0 ){ + whereIndexExprTrans(pIdx, iCur, iIdxCur, pWInfo); + } + + /* If a partial index is driving the loop, try to eliminate WHERE clause + ** terms from the query that must be true due to the WHERE clause of + ** the partial index. + ** + ** 2019-11-02 ticket 623eff57e76d45f6: This optimization does not work + ** for a LEFT JOIN. + */ + if( pIdx->pPartIdxWhere ){ + whereApplyPartialIndexConstraints(pIdx->pPartIdxWhere, iCur, pWC); + } + }else{ + testcase( pIdx->pPartIdxWhere ); + /* The following assert() is not a requirement, merely an observation: + ** The OR-optimization doesn't work for the right hand table of + ** a LEFT JOIN: */ + assert( (pWInfo->wctrlFlags & WHERE_OR_SUBCLAUSE)==0 ); } - + /* Record the instruction used to terminate the loop. */ if( pLoop->wsFlags & WHERE_ONEROW ){ pLevel->op = OP_Noop; @@ -134225,7 +142777,7 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( int regReturn = ++pParse->nMem; /* Register used with OP_Gosub */ int regRowset = 0; /* Register for RowSet object */ int regRowid = 0; /* Register holding rowid */ - int iLoopBody = sqlite3VdbeMakeLabel(v); /* Start of loop body */ + int iLoopBody = sqlite3VdbeMakeLabel(pParse);/* Start of loop body */ int iRetInit; /* Address of regReturn init */ int untestedTerms = 0; /* Some terms not completely tested */ int ii; /* Loop counter */ @@ -134314,10 +142866,15 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( if( (pWC->a[iTerm].eOperator & WO_ALL)==0 ) continue; testcase( pWC->a[iTerm].wtFlags & TERM_ORINFO ); pExpr = sqlite3ExprDup(db, pExpr, 0); - pAndExpr = sqlite3ExprAnd(db, pAndExpr, pExpr); + pAndExpr = sqlite3ExprAnd(pParse, pAndExpr, pExpr); } if( pAndExpr ){ - pAndExpr = sqlite3PExpr(pParse, TK_AND|TKFLG_DONTFOLD, 0, pAndExpr); + /* The extra 0x10000 bit on the opcode is masked off and does not + ** become part of the new Expr.op. However, it does make the + ** op==TK_AND comparison inside of sqlite3PExpr() false, and this + ** prevents sqlite3PExpr() from implementing AND short-circuit + ** optimization, which we do not want here. */ + pAndExpr = sqlite3PExpr(pParse, TK_AND|0x10000, 0, pAndExpr); } } @@ -134333,14 +142890,15 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( WhereInfo *pSubWInfo; /* Info for single OR-term scan */ Expr *pOrExpr = pOrTerm->pExpr; /* Current OR clause term */ int jmp1 = 0; /* Address of jump operation */ - assert( (pTabItem[0].fg.jointype & JT_LEFT)==0 - || ExprHasProperty(pOrExpr, EP_FromJoin) - ); + testcase( (pTabItem[0].fg.jointype & JT_LEFT)!=0 + && !ExprHasProperty(pOrExpr, EP_FromJoin) + ); /* See TH3 vtab25.400 and ticket 614b25314c766238 */ if( pAndExpr ){ pAndExpr->pLeft = pOrExpr; pOrExpr = pAndExpr; } /* Loop through table entries that match term pOrTerm. */ + ExplainQueryPlan((pParse, 1, "INDEX %d", ii+1)); WHERETRACE(0xffff, ("Subplan for OR-clause:\n")); pSubWInfo = sqlite3WhereBegin(pParse, pOrTab, pOrExpr, 0, 0, wctrlFlags, iCovCur); @@ -134358,23 +142916,23 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( ** row will be skipped in subsequent sub-WHERE clauses. */ if( (pWInfo->wctrlFlags & WHERE_DUPLICATES_OK)==0 ){ - int r; int iSet = ((ii==pOrWc->nTerm-1)?-1:ii); if( HasRowid(pTab) ){ - r = sqlite3ExprCodeGetColumn(pParse, pTab, -1, iCur, regRowid, 0); + sqlite3ExprCodeGetColumnOfTable(v, pTab, iCur, -1, regRowid); jmp1 = sqlite3VdbeAddOp4Int(v, OP_RowSetTest, regRowset, 0, - r,iSet); + regRowid, iSet); VdbeCoverage(v); }else{ Index *pPk = sqlite3PrimaryKeyIndex(pTab); int nPk = pPk->nKeyCol; int iPk; + int r; /* Read the PK into an array of temp registers. */ r = sqlite3GetTempRange(pParse, nPk); for(iPk=0; iPkaiColumn[iPk]; - sqlite3ExprCodeGetColumnToReg(pParse, pTab, iCol, iCur, r+iPk); + sqlite3ExprCodeGetColumnOfTable(v, pTab, iCur, iCol,r+iPk); } /* Check if the temp table already contains this key. If so, @@ -134444,6 +143002,7 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( /* Finish the loop through table entries that match term pOrTerm. */ sqlite3WhereEnd(pSubWInfo); + ExplainQueryPlanPop(pParse); } } } @@ -134458,7 +143017,7 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( sqlite3VdbeGoto(v, pLevel->addrBrk); sqlite3VdbeResolveLabel(v, iLoopBody); - if( pWInfo->nLevel>1 ) sqlite3StackFree(db, pOrTab); + if( pWInfo->nLevel>1 ){ sqlite3StackFree(db, pOrTab); } if( !untestedTerms ) disableTerm(pLevel, pTerm); }else #endif /* SQLITE_OMIT_OR_OPTIMIZATION */ @@ -134545,8 +143104,9 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( u32 x = pLevel->iLikeRepCntr; if( x>0 ){ skipLikeAddr = sqlite3VdbeAddOp1(v, (x&1)?OP_IfNot:OP_If,(int)(x>>1)); + VdbeCoverageIf(v, (x&1)==1); + VdbeCoverageIf(v, (x&1)==0); } - VdbeCoverage(v); #endif } #ifdef WHERETRACE_ENABLED /* 0xffff */ @@ -134554,6 +143114,10 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( VdbeNoopComment((v, "WhereTerm[%d] (%p) priority=%d", pWC->nTerm-j, pTerm, iLoop)); } + if( sqlite3WhereTrace & 0x800 ){ + sqlite3DebugPrintf("Coding auxiliary constraint:\n"); + sqlite3WhereTermPrint(pTerm, pWC->nTerm-j); + } #endif sqlite3ExprIfFalse(pParse, pE, addrCont, SQLITE_JUMPIFNULL); if( skipLikeAddr ) sqlite3VdbeJumpHere(v, skipLikeAddr); @@ -134577,8 +143141,14 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( if( (pTerm->eOperator & (WO_EQ|WO_IS))==0 ) continue; if( (pTerm->eOperator & WO_EQUIV)==0 ) continue; if( pTerm->leftCursor!=iCur ) continue; - if( pLevel->iLeftJoin ) continue; + if( pTabItem->fg.jointype & JT_LEFT ) continue; pE = pTerm->pExpr; +#ifdef WHERETRACE_ENABLED /* 0x800 */ + if( sqlite3WhereTrace & 0x800 ){ + sqlite3DebugPrintf("Coding transitive constraint:\n"); + sqlite3WhereTermPrint(pTerm, pWC->nTerm-j); + } +#endif assert( !ExprHasProperty(pE, EP_FromJoin) ); assert( (pTerm->prereqRight & pLevel->notReady)!=0 ); pAlt = sqlite3WhereFindTerm(pWC, iCur, pTerm->u.leftColumn, notReady, @@ -134607,7 +143177,6 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( pLevel->addrFirst = sqlite3VdbeCurrentAddr(v); sqlite3VdbeAddOp2(v, OP_Integer, 1, pLevel->iLeftJoin); VdbeComment((v, "record LEFT JOIN hit")); - sqlite3ExprCacheClear(pParse); for(pTerm=pWC->a, j=0; jnTerm; j++, pTerm++){ testcase( pTerm->wtFlags & TERM_VIRTUAL ); testcase( pTerm->wtFlags & TERM_CODED ); @@ -134622,6 +143191,17 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( } } +#if WHERETRACE_ENABLED /* 0x20800 */ + if( sqlite3WhereTrace & 0x20000 ){ + sqlite3DebugPrintf("All WHERE-clause terms after coding level %d:\n", + iLevel); + sqlite3WhereClausePrint(pWC); + } + if( sqlite3WhereTrace & 0x800 ){ + sqlite3DebugPrintf("End Coding level %d: notReady=%llx\n", + iLevel, (u64)pLevel->notReady); + } +#endif return pLevel->notReady; } @@ -134713,7 +143293,7 @@ static int whereClauseInsert(WhereClause *pWC, Expr *p, u16 wtFlags){ }else{ pTerm->truthProb = 1; } - pTerm->pExpr = sqlite3ExprSkipCollate(p); + pTerm->pExpr = sqlite3ExprSkipCollateAndLikely(p); pTerm->wtFlags = wtFlags; pTerm->pWC = pWC; pTerm->iParent = -1; @@ -134738,31 +143318,14 @@ static int allowedOp(int op){ /* ** Commute a comparison operator. Expressions of the form "X op Y" ** are converted into "Y op X". -** -** If left/right precedence rules come into play when determining the -** collating sequence, then COLLATE operators are adjusted to ensure -** that the collating sequence does not change. For example: -** "Y collate NOCASE op X" becomes "X op Y" because any collation sequence on -** the left hand side of a comparison overrides any collation sequence -** attached to the right. For the same reason the EP_Collate flag -** is not commuted. -*/ -static void exprCommute(Parse *pParse, Expr *pExpr){ - u16 expRight = (pExpr->pRight->flags & EP_Collate); - u16 expLeft = (pExpr->pLeft->flags & EP_Collate); - assert( allowedOp(pExpr->op) && pExpr->op!=TK_IN ); - if( expRight==expLeft ){ - /* Either X and Y both have COLLATE operator or neither do */ - if( expRight ){ - /* Both X and Y have COLLATE operators. Make sure X is always - ** used by clearing the EP_Collate flag from Y. */ - pExpr->pRight->flags &= ~EP_Collate; - }else if( sqlite3ExprCollSeq(pParse, pExpr->pLeft)!=0 ){ - /* Neither X nor Y have COLLATE operators, but X has a non-default - ** collating sequence. So add the EP_Collate marker on X to cause - ** it to be searched first. */ - pExpr->pLeft->flags |= EP_Collate; - } +*/ +static u16 exprCommute(Parse *pParse, Expr *pExpr){ + if( pExpr->pLeft->op==TK_VECTOR + || pExpr->pRight->op==TK_VECTOR + || sqlite3BinaryCompareCollSeq(pParse, pExpr->pLeft, pExpr->pRight) != + sqlite3BinaryCompareCollSeq(pParse, pExpr->pRight, pExpr->pLeft) + ){ + pExpr->flags ^= EP_Commuted; } SWAP(Expr*,pExpr->pRight,pExpr->pLeft); if( pExpr->op>=TK_GT ){ @@ -134773,6 +143336,7 @@ static void exprCommute(Parse *pParse, Expr *pExpr){ assert( pExpr->op>=TK_GT && pExpr->op<=TK_GE ); pExpr->op = ((pExpr->op-TK_GT)^2)+TK_GT; } + return 0; } /* @@ -134823,18 +143387,18 @@ static int isLikeOrGlob( int *pisComplete, /* True if the only wildcard is % in the last character */ int *pnoCase /* True if uppercase is equivalent to lowercase */ ){ - const u8 *z = 0; /* String on RHS of LIKE operator */ + const u8 *z = 0; /* String on RHS of LIKE operator */ Expr *pRight, *pLeft; /* Right and left size of LIKE operator */ ExprList *pList; /* List of operands to the LIKE operator */ - int c; /* One character in z[] */ + u8 c; /* One character in z[] */ int cnt; /* Number of non-wildcard prefix characters */ - char wc[4]; /* Wildcard characters */ + u8 wc[4]; /* Wildcard characters */ sqlite3 *db = pParse->db; /* Database connection */ sqlite3_value *pVal = 0; int op; /* Opcode of pRight */ int rc; /* Result code to return */ - if( !sqlite3IsLikeFunction(db, pExpr, pnoCase, wc) ){ + if( !sqlite3IsLikeFunction(db, pExpr, pnoCase, (char*)wc) ){ return 0; } #ifdef SQLITE_EBCDIC @@ -134859,23 +143423,6 @@ static int isLikeOrGlob( } if( z ){ - /* If the RHS begins with a digit or a minus sign, then the LHS must - ** be an ordinary column (not a virtual table column) with TEXT affinity. - ** Otherwise the LHS might be numeric and "lhs >= rhs" would be false - ** even though "lhs LIKE rhs" is true. But if the RHS does not start - ** with a digit or '-', then "lhs LIKE rhs" will always be false if - ** the LHS is numeric and so the optimization still works. - */ - if( sqlite3Isdigit(z[0]) || z[0]=='-' ){ - if( pLeft->op!=TK_COLUMN - || sqlite3ExprAffinity(pLeft)!=SQLITE_AFF_TEXT - || IsVirtual(pLeft->pTab) /* Value might be numeric */ - ){ - sqlite3ValueFree(pVal); - return 0; - } - } - /* Count the number of prefix characters prior to the first wildcard */ cnt = 0; while( (c=z[cnt])!=0 && c!=wc[0] && c!=wc[1] && c!=wc[2] ){ @@ -134885,11 +143432,13 @@ static int isLikeOrGlob( /* The optimization is possible only if (1) the pattern does not begin ** with a wildcard and if (2) the non-wildcard prefix does not end with - ** an (illegal 0xff) character. The second condition is necessary so + ** an (illegal 0xff) character, or (3) the pattern does not consist of + ** a single escape character. The second condition is necessary so ** that we can increment the prefix key to find an upper bound for the - ** range search. - */ - if( cnt!=0 && 255!=(u8)z[cnt-1] ){ + ** range search. The third is because the caller assumes that the pattern + ** consists of at least one character after all escapes have been + ** removed. */ + if( cnt!=0 && 255!=(u8)z[cnt-1] && (cnt>1 || z[0]!=wc[3]) ){ Expr *pPrefix; /* A "complete" match if the pattern ends with "*" or "%" */ @@ -134906,6 +143455,43 @@ static int isLikeOrGlob( zNew[iTo++] = zNew[iFrom]; } zNew[iTo] = 0; + assert( iTo>0 ); + + /* If the LHS is not an ordinary column with TEXT affinity, then the + ** pattern prefix boundaries (both the start and end boundaries) must + ** not look like a number. Otherwise the pattern might be treated as + ** a number, which will invalidate the LIKE optimization. + ** + ** Getting this right has been a persistent source of bugs in the + ** LIKE optimization. See, for example: + ** 2018-09-10 https://sqlite.org/src/info/c94369cae9b561b1 + ** 2019-05-02 https://sqlite.org/src/info/b043a54c3de54b28 + ** 2019-06-10 https://sqlite.org/src/info/fd76310a5e843e07 + ** 2019-06-14 https://sqlite.org/src/info/ce8717f0885af975 + ** 2019-09-03 https://sqlite.org/src/info/0f0428096f17252a + */ + if( pLeft->op!=TK_COLUMN + || sqlite3ExprAffinity(pLeft)!=SQLITE_AFF_TEXT + || IsVirtual(pLeft->y.pTab) /* Value might be numeric */ + ){ + int isNum; + double rDummy; + isNum = sqlite3AtoF(zNew, &rDummy, iTo, SQLITE_UTF8); + if( isNum<=0 ){ + if( iTo==1 && zNew[0]=='-' ){ + isNum = +1; + }else{ + zNew[iTo-1]++; + isNum = sqlite3AtoF(zNew, &rDummy, iTo, SQLITE_UTF8); + zNew[iTo-1]--; + } + } + if( isNum>0 ){ + sqlite3ExprDelete(db, pPrefix); + sqlite3ValueFree(pVal); + return 0; + } + } } *ppPrefix = pPrefix; @@ -134967,6 +143553,7 @@ static int isLikeOrGlob( ** If the expression matches none of the patterns above, return 0. */ static int isAuxiliaryVtabOperator( + sqlite3 *db, /* Parsing context */ Expr *pExpr, /* Test this expression */ unsigned char *peOp2, /* OUT: 0 for MATCH, or else an op2 value */ Expr **ppLeft, /* Column expression to left of MATCH/op2 */ @@ -134990,26 +143577,68 @@ static int isAuxiliaryVtabOperator( if( pList==0 || pList->nExpr!=2 ){ return 0; } + + /* Built-in operators MATCH, GLOB, LIKE, and REGEXP attach to a + ** virtual table on their second argument, which is the same as + ** the left-hand side operand in their in-fix form. + ** + ** vtab_column MATCH expression + ** MATCH(expression,vtab_column) + */ pCol = pList->a[1].pExpr; - if( pCol->op!=TK_COLUMN || !IsVirtual(pCol->pTab) ){ - return 0; + testcase( pCol->op==TK_COLUMN && pCol->y.pTab==0 ); + if( ExprIsVtab(pCol) ){ + for(i=0; iu.zToken, aOp[i].zOp)==0 ){ + *peOp2 = aOp[i].eOp2; + *ppRight = pList->a[0].pExpr; + *ppLeft = pCol; + return 1; + } + } } - for(i=0; iu.zToken, aOp[i].zOp)==0 ){ - *peOp2 = aOp[i].eOp2; - *ppRight = pList->a[0].pExpr; - *ppLeft = pCol; - return 1; + + /* We can also match against the first column of overloaded + ** functions where xFindFunction returns a value of at least + ** SQLITE_INDEX_CONSTRAINT_FUNCTION. + ** + ** OVERLOADED(vtab_column,expression) + ** + ** Historically, xFindFunction expected to see lower-case function + ** names. But for this use case, xFindFunction is expected to deal + ** with function names in an arbitrary case. + */ + pCol = pList->a[0].pExpr; + testcase( pCol->op==TK_COLUMN && pCol->y.pTab==0 ); + if( ExprIsVtab(pCol) ){ + sqlite3_vtab *pVtab; + sqlite3_module *pMod; + void (*xNotUsed)(sqlite3_context*,int,sqlite3_value**); + void *pNotUsed; + pVtab = sqlite3GetVTable(db, pCol->y.pTab)->pVtab; + assert( pVtab!=0 ); + assert( pVtab->pModule!=0 ); + pMod = (sqlite3_module *)pVtab->pModule; + if( pMod->xFindFunction!=0 ){ + i = pMod->xFindFunction(pVtab,2, pExpr->u.zToken, &xNotUsed, &pNotUsed); + if( i>=SQLITE_INDEX_CONSTRAINT_FUNCTION ){ + *peOp2 = i; + *ppRight = pList->a[1].pExpr; + *ppLeft = pCol; + return 1; + } } } }else if( pExpr->op==TK_NE || pExpr->op==TK_ISNOT || pExpr->op==TK_NOTNULL ){ int res = 0; Expr *pLeft = pExpr->pLeft; Expr *pRight = pExpr->pRight; - if( pLeft->op==TK_COLUMN && IsVirtual(pLeft->pTab) ){ + testcase( pLeft->op==TK_COLUMN && pLeft->y.pTab==0 ); + if( ExprIsVtab(pLeft) ){ res++; } - if( pRight && pRight->op==TK_COLUMN && IsVirtual(pRight->pTab) ){ + testcase( pRight && pRight->op==TK_COLUMN && pRight->y.pTab==0 ); + if( pRight && ExprIsVtab(pRight) ){ res++; SWAP(Expr*, pLeft, pRight); } @@ -135301,7 +143930,12 @@ static void exprAnalyzeOrTerm( ** empty. */ pOrInfo->indexable = indexable; - pTerm->eOperator = indexable==0 ? 0 : WO_OR; + if( indexable ){ + pTerm->eOperator = WO_OR; + pWC->hasOr = 1; + }else{ + pTerm->eOperator = WO_OR; + } /* For a two-way OR, attempt to implementation case 2. */ @@ -135351,6 +143985,7 @@ static void exprAnalyzeOrTerm( ** and column is found but leave okToChngToIN false if not found. */ for(j=0; j<2 && !okToChngToIN; j++){ + Expr *pLeft = 0; pOrTerm = pOrWc->a; for(i=pOrWc->nTerm-1; i>=0; i--, pOrTerm++){ assert( pOrTerm->eOperator & WO_EQ ); @@ -135374,6 +144009,7 @@ static void exprAnalyzeOrTerm( } iColumn = pOrTerm->u.leftColumn; iCursor = pOrTerm->leftCursor; + pLeft = pOrTerm->pExpr->pLeft; break; } if( i<0 ){ @@ -135393,7 +144029,9 @@ static void exprAnalyzeOrTerm( assert( pOrTerm->eOperator & WO_EQ ); if( pOrTerm->leftCursor!=iCursor ){ pOrTerm->wtFlags &= ~TERM_OR_OK; - }else if( pOrTerm->u.leftColumn!=iColumn ){ + }else if( pOrTerm->u.leftColumn!=iColumn || (iColumn==XN_EXPR + && sqlite3ExprCompare(pParse, pOrTerm->pExpr->pLeft, pLeft, -1) + )){ okToChngToIN = 0; }else{ int affLeft, affRight; @@ -135442,7 +144080,7 @@ static void exprAnalyzeOrTerm( idxNew = whereClauseInsert(pWC, pNew, TERM_VIRTUAL|TERM_DYNAMIC); testcase( idxNew==0 ); exprAnalyze(pSrc, pWC, idxNew); - pTerm = &pWC->a[idxTerm]; + /* pTerm = &pWC->a[idxTerm]; // would be needed if pTerm where used again */ markTermAsChild(pWC, idxNew, idxTerm); }else{ sqlite3ExprListDelete(db, pList); @@ -135480,8 +144118,8 @@ static int termIsEquivalence(Parse *pParse, Expr *pExpr){ ){ return 0; } - pColl = sqlite3BinaryCompareCollSeq(pParse, pExpr->pLeft, pExpr->pRight); - if( pColl==0 || sqlite3StrICmp(pColl->zName, "BINARY")==0 ) return 1; + pColl = sqlite3ExprCompareCollSeq(pParse, pExpr); + if( sqlite3IsBinary(pColl) ) return 1; return sqlite3ExprCollSeqMatch(pParse, pExpr->pLeft, pExpr->pRight); } @@ -135640,7 +144278,7 @@ static void exprAnalyze( pTerm->prereqRight = sqlite3WhereExprUsage(pMaskSet, pExpr->pRight); } pMaskSet->bVarSelect = 0; - prereqAll = sqlite3WhereExprUsage(pMaskSet, pExpr); + prereqAll = sqlite3WhereExprUsageNN(pMaskSet, pExpr); if( pMaskSet->bVarSelect ) pTerm->wtFlags |= TERM_VARSELECT; if( ExprHasProperty(pExpr, EP_FromJoin) ){ Bitmask x = sqlite3WhereGetMask(pMaskSet, pExpr->iRightJoinTable); @@ -135704,7 +144342,7 @@ static void exprAnalyze( pDup = pExpr; pNew = pTerm; } - exprCommute(pParse, pDup); + pNew->wtFlags |= exprCommute(pParse, pDup); pNew->leftCursor = aiCurCol[0]; pNew->u.leftColumn = aiCurCol[1]; testcase( (prereqLeft | extraRight) != prereqLeft ); @@ -135822,7 +144460,7 @@ static void exprAnalyze( } *pC = c + 1; } - zCollSeqName = noCase ? "NOCASE" : "BINARY"; + zCollSeqName = noCase ? "NOCASE" : sqlite3StrBINARY; pNewExpr1 = sqlite3ExprDup(db, pLeft, 0); pNewExpr1 = sqlite3PExpr(pParse, TK_GE, sqlite3ExprAddCollateString(pParse,pNewExpr1,zCollSeqName), @@ -135859,7 +144497,7 @@ static void exprAnalyze( */ if( pWC->op==TK_AND ){ Expr *pRight = 0, *pLeft = 0; - int res = isAuxiliaryVtabOperator(pExpr, &eOp2, &pLeft, &pRight); + int res = isAuxiliaryVtabOperator(db, pExpr, &eOp2, &pLeft, &pRight); while( res-- > 0 ){ int idxNew; WhereTerm *pNewTerm; @@ -135873,6 +144511,7 @@ static void exprAnalyze( 0, sqlite3ExprDup(db, pRight, 0)); if( ExprHasProperty(pExpr, EP_FromJoin) && pNewExpr ){ ExprSetProperty(pNewExpr, EP_FromJoin); + pNewExpr->iRightJoinTable = pExpr->iRightJoinTable; } idxNew = whereClauseInsert(pWC, pNewExpr, TERM_VIRTUAL|TERM_DYNAMIC); testcase( idxNew==0 ); @@ -135929,11 +144568,15 @@ static void exprAnalyze( ** expression). The WhereTerm.iField variable identifies the index within ** the vector on the LHS that the virtual term represents. ** - ** This only works if the RHS is a simple SELECT, not a compound + ** This only works if the RHS is a simple SELECT (not a compound) that does + ** not use window functions. */ if( pWC->op==TK_AND && pExpr->op==TK_IN && pTerm->iField==0 && pExpr->pLeft->op==TK_VECTOR && pExpr->x.pSelect->pPrior==0 +#ifndef SQLITE_OMIT_WINDOWFUNC + && pExpr->x.pSelect->pWin==0 +#endif ){ int i; for(i=0; ipLeft); i++){ @@ -135945,8 +144588,8 @@ static void exprAnalyze( } } -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 - /* When sqlite_stat3 histogram data is available an operator of the +#ifdef SQLITE_ENABLE_STAT4 + /* When sqlite_stat4 histogram data is available an operator of the ** form "x IS NOT NULL" can sometimes be evaluated more efficiently ** as "x>NULL" if x is not an INTEGER PRIMARY KEY. So construct a ** virtual term of that form. @@ -135956,7 +144599,8 @@ static void exprAnalyze( if( pExpr->op==TK_NOTNULL && pExpr->pLeft->op==TK_COLUMN && pExpr->pLeft->iColumn>=0 - && OptimizationEnabled(db, SQLITE_Stat34) + && !ExprHasProperty(pExpr, EP_FromJoin) + && OptimizationEnabled(db, SQLITE_Stat4) ){ Expr *pNewExpr; Expr *pLeft = pExpr->pLeft; @@ -135981,7 +144625,7 @@ static void exprAnalyze( pNewTerm->prereqAll = pTerm->prereqAll; } } -#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ +#endif /* SQLITE_ENABLE_STAT4 */ /* Prevent ON clause terms of a LEFT JOIN from being used to drive ** an index for tables to the left of the join. @@ -136014,7 +144658,7 @@ static void exprAnalyze( ** all terms of the WHERE clause. */ SQLITE_PRIVATE void sqlite3WhereSplit(WhereClause *pWC, Expr *pExpr, u8 op){ - Expr *pE2 = sqlite3ExprSkipCollate(pExpr); + Expr *pE2 = sqlite3ExprSkipCollateAndLikely(pExpr); pWC->op = op; if( pE2==0 ) return; if( pE2->op!=op ){ @@ -136033,6 +144677,7 @@ SQLITE_PRIVATE void sqlite3WhereClauseInit( WhereInfo *pWInfo /* The WHERE processing context */ ){ pWC->pWInfo = pWInfo; + pWC->hasOr = 0; pWC->pOuter = 0; pWC->nTerm = 0; pWC->nSlot = ArraySize(pWC->aStatic); @@ -136069,17 +144714,18 @@ SQLITE_PRIVATE void sqlite3WhereClauseClear(WhereClause *pWC){ ** a bitmask indicating which tables are used in that expression ** tree. */ -SQLITE_PRIVATE Bitmask sqlite3WhereExprUsage(WhereMaskSet *pMaskSet, Expr *p){ +SQLITE_PRIVATE Bitmask sqlite3WhereExprUsageNN(WhereMaskSet *pMaskSet, Expr *p){ Bitmask mask; - if( p==0 ) return 0; - if( p->op==TK_COLUMN ){ + if( p->op==TK_COLUMN && !ExprHasProperty(p, EP_FixedCol) ){ return sqlite3WhereGetMask(pMaskSet, p->iTable); + }else if( ExprHasProperty(p, EP_TokenOnly|EP_Leaf) ){ + assert( p->op!=TK_IF_NULL_ROW ); + return 0; } mask = (p->op==TK_IF_NULL_ROW) ? sqlite3WhereGetMask(pMaskSet, p->iTable) : 0; - assert( !ExprHasProperty(p, EP_TokenOnly) ); - if( p->pLeft ) mask |= sqlite3WhereExprUsage(pMaskSet, p->pLeft); + if( p->pLeft ) mask |= sqlite3WhereExprUsageNN(pMaskSet, p->pLeft); if( p->pRight ){ - mask |= sqlite3WhereExprUsage(pMaskSet, p->pRight); + mask |= sqlite3WhereExprUsageNN(pMaskSet, p->pRight); assert( p->x.pList==0 ); }else if( ExprHasProperty(p, EP_xIsSelect) ){ if( ExprHasProperty(p, EP_VarSelect) ) pMaskSet->bVarSelect = 1; @@ -136087,8 +144733,18 @@ SQLITE_PRIVATE Bitmask sqlite3WhereExprUsage(WhereMaskSet *pMaskSet, Expr *p){ }else if( p->x.pList ){ mask |= sqlite3WhereExprListUsage(pMaskSet, p->x.pList); } +#ifndef SQLITE_OMIT_WINDOWFUNC + if( (p->op==TK_FUNCTION || p->op==TK_AGG_FUNCTION) && p->y.pWin ){ + mask |= sqlite3WhereExprListUsage(pMaskSet, p->y.pWin->pPartition); + mask |= sqlite3WhereExprListUsage(pMaskSet, p->y.pWin->pOrderBy); + mask |= sqlite3WhereExprUsage(pMaskSet, p->y.pWin->pFilter); + } +#endif return mask; } +SQLITE_PRIVATE Bitmask sqlite3WhereExprUsage(WhereMaskSet *pMaskSet, Expr *p){ + return p ? sqlite3WhereExprUsageNN(pMaskSet,p) : 0; +} SQLITE_PRIVATE Bitmask sqlite3WhereExprListUsage(WhereMaskSet *pMaskSet, ExprList *pList){ int i; Bitmask mask = 0; @@ -136142,6 +144798,7 @@ SQLITE_PRIVATE void sqlite3WhereTabFuncArgs( pArgs = pItem->u1.pFuncArg; if( pArgs==0 ) return; for(j=k=0; jnExpr; j++){ + Expr *pRhs; while( knCol && (pTab->aCol[k].colFlags & COLFLAG_HIDDEN)==0 ){k++;} if( k>=pTab->nCol ){ sqlite3ErrorMsg(pParse, "too many arguments on %s() - max %d", @@ -136152,9 +144809,13 @@ SQLITE_PRIVATE void sqlite3WhereTabFuncArgs( if( pColRef==0 ) return; pColRef->iTable = pItem->iCursor; pColRef->iColumn = k++; - pColRef->pTab = pTab; - pTerm = sqlite3PExpr(pParse, TK_EQ, pColRef, - sqlite3ExprDup(pParse->db, pArgs->a[j].pExpr, 0)); + pColRef->y.pTab = pTab; + pRhs = sqlite3PExpr(pParse, TK_UPLUS, + sqlite3ExprDup(pParse->db, pArgs->a[j].pExpr, 0), 0); + pTerm = sqlite3PExpr(pParse, TK_EQ, pColRef, pRhs); + if( pItem->fg.jointype & JT_LEFT ){ + sqlite3SetJoinExpr(pTerm, pItem->iCursor); + } whereClauseInsert(pWC, pTerm, TERM_DYNAMIC); } } @@ -136230,15 +144891,38 @@ SQLITE_PRIVATE int sqlite3WhereIsOrdered(WhereInfo *pWInfo){ } /* -** Return TRUE if the innermost loop of the WHERE clause implementation -** returns rows in ORDER BY order for complete run of the inner loop. +** In the ORDER BY LIMIT optimization, if the inner-most loop is known +** to emit rows in increasing order, and if the last row emitted by the +** inner-most loop did not fit within the sorter, then we can skip all +** subsequent rows for the current iteration of the inner loop (because they +** will not fit in the sorter either) and continue with the second inner +** loop - the loop immediately outside the inner-most. +** +** When a row does not fit in the sorter (because the sorter already +** holds LIMIT+OFFSET rows that are smaller), then a jump is made to the +** label returned by this function. +** +** If the ORDER BY LIMIT optimization applies, the jump destination should +** be the continuation for the second-inner-most loop. If the ORDER BY +** LIMIT optimization does not apply, then the jump destination should +** be the continuation for the inner-most loop. ** -** Across multiple iterations of outer loops, the output rows need not be -** sorted. As long as rows are sorted for just the innermost loop, this -** routine can return TRUE. +** It is always safe for this routine to return the continuation of the +** inner-most loop, in the sense that a correct answer will result. +** Returning the continuation the second inner loop is an optimization +** that might make the code run a little faster, but should not change +** the final answer. */ -SQLITE_PRIVATE int sqlite3WhereOrderedInnerLoop(WhereInfo *pWInfo){ - return pWInfo->bOrderedInnerLoop; +SQLITE_PRIVATE int sqlite3WhereOrderByLimitOptLabel(WhereInfo *pWInfo){ + WhereLevel *pInner; + if( !pWInfo->bOrderedInnerLoop ){ + /* The ORDER BY LIMIT optimization does not apply. Jump to the + ** continuation of the inner-most loop. */ + return pWInfo->iContinue; + } + pInner = &pWInfo->a[pWInfo->nLevel-1]; + assert( pInner->addrNxt!=0 ); + return pInner->addrNxt; } /* @@ -136260,7 +144944,7 @@ SQLITE_PRIVATE int sqlite3WhereBreakLabel(WhereInfo *pWInfo){ /* ** Return ONEPASS_OFF (0) if an UPDATE or DELETE statement is unable to -** operate directly on the rowis returned by a WHERE clause. Return +** operate directly on the rowids returned by a WHERE clause. Return ** ONEPASS_SINGLE (1) if the statement can operation directly because only ** a single row is to be changed. Return ONEPASS_MULTI (2) if the one-pass ** optimization can be used on multiple @@ -136287,6 +144971,14 @@ SQLITE_PRIVATE int sqlite3WhereOkOnePass(WhereInfo *pWInfo, int *aiCur){ return pWInfo->eOnePass; } +/* +** Return TRUE if the WHERE loop uses the OP_DeferredSeek opcode to move +** the data cursor to the row selected by the index cursor. +*/ +SQLITE_PRIVATE int sqlite3WhereUsesDeferredSeek(WhereInfo *pWInfo){ + return pWInfo->bDeferredSeek; +} + /* ** Move the content of pSrc into pDest */ @@ -136393,7 +145085,8 @@ static WhereTerm *whereScanNext(WhereScan *pScan){ ){ if( (pTerm->eOperator & WO_EQUIV)!=0 && pScan->nEquivaiCur) - && (pX = sqlite3ExprSkipCollate(pTerm->pExpr->pRight))->op==TK_COLUMN + && (pX = sqlite3ExprSkipCollateAndLikely(pTerm->pExpr->pRight))->op + ==TK_COLUMN ){ int j; for(j=0; jnEquiv; j++){ @@ -136418,8 +145111,7 @@ static WhereTerm *whereScanNext(WhereScan *pScan){ continue; } assert(pX->pLeft); - pColl = sqlite3BinaryCompareCollSeq(pParse, - pX->pLeft, pX->pRight); + pColl = sqlite3ExprCompareCollSeq(pParse, pX); if( pColl==0 ) pColl = pParse->db->pDfltColl; if( sqlite3StrICmp(pColl->zName, pScan->zCollName) ){ continue; @@ -136450,6 +145142,17 @@ static WhereTerm *whereScanNext(WhereScan *pScan){ return 0; } +/* +** This is whereScanInit() for the case of an index on an expression. +** It is factored out into a separate tail-recursion subroutine so that +** the normal whereScanInit() routine, which is a high-runner, does not +** need to push registers onto the stack as part of its prologue. +*/ +static SQLITE_NOINLINE WhereTerm *whereScanInitIndexExpr(WhereScan *pScan){ + pScan->idxaff = sqlite3ExprAffinity(pScan->pIdxExpr); + return whereScanNext(pScan); +} + /* ** Initialize a WHERE clause scanner object. Return a pointer to the ** first match. Return NULL if there are no matches. @@ -136482,12 +145185,19 @@ static WhereTerm *whereScanInit( pScan->pIdxExpr = 0; pScan->idxaff = 0; pScan->zCollName = 0; + pScan->opMask = opMask; + pScan->k = 0; + pScan->aiCur[0] = iCur; + pScan->nEquiv = 1; + pScan->iEquiv = 1; if( pIdx ){ int j = iColumn; iColumn = pIdx->aiColumn[j]; if( iColumn==XN_EXPR ){ pScan->pIdxExpr = pIdx->aColExpr->a[j].pExpr; pScan->zCollName = pIdx->azColl[j]; + pScan->aiColumn[0] = XN_EXPR; + return whereScanInitIndexExpr(pScan); }else if( iColumn==pIdx->pTable->iPKey ){ iColumn = XN_ROWID; }else if( iColumn>=0 ){ @@ -136497,12 +145207,7 @@ static WhereTerm *whereScanInit( }else if( iColumn==XN_EXPR ){ return 0; } - pScan->opMask = opMask; - pScan->k = 0; - pScan->aiCur[0] = iCur; pScan->aiColumn[0] = iColumn; - pScan->nEquiv = 1; - pScan->iEquiv = 1; return whereScanNext(pScan); } @@ -136576,7 +145281,7 @@ static int findIndexCol( const char *zColl = pIdx->azColl[iCol]; for(i=0; inExpr; i++){ - Expr *p = sqlite3ExprSkipCollate(pList->a[i].pExpr); + Expr *p = sqlite3ExprSkipCollateAndLikely(pList->a[i].pExpr); if( p->op==TK_COLUMN && p->iColumn==pIdx->aiColumn[iCol] && p->iTable==iBase @@ -136640,7 +145345,7 @@ static int isDistinctRedundant( ** current SELECT is a correlated sub-query. */ for(i=0; inExpr; i++){ - Expr *p = sqlite3ExprSkipCollate(pDistinct->a[i].pExpr); + Expr *p = sqlite3ExprSkipCollateAndLikely(pDistinct->a[i].pExpr); if( p->op==TK_COLUMN && p->iTable==iBase && p->iColumn<0 ) return 1; } @@ -136689,17 +145394,17 @@ static LogEst estLog(LogEst N){ ** opcodes into OP_Copy when the table is being accessed via co-routine ** instead of via table lookup. ** -** If the bIncrRowid parameter is 0, then any OP_Rowid instructions on -** cursor iTabCur are transformed into OP_Null. Or, if bIncrRowid is non-zero, -** then each OP_Rowid is transformed into an instruction to increment the -** value stored in its output register. +** If the iAutoidxCur is not zero, then any OP_Rowid instructions on +** cursor iTabCur are transformed into OP_Sequence opcode for the +** iAutoidxCur cursor, in order to generate unique rowids for the +** automatic index being generated. */ static void translateColumnToCopy( Parse *pParse, /* Parsing context */ int iStart, /* Translate from this opcode to the end */ int iTabCur, /* OP_Column/OP_Rowid references to this table */ int iRegister, /* The first column is in this register */ - int bIncrRowid /* If non-zero, transform OP_rowid to OP_AddImm(1) */ + int iAutoidxCur /* If non-zero, cursor of autoindex being generated */ ){ Vdbe *v = pParse->pVdbe; VdbeOp *pOp = sqlite3VdbeGetOp(v, iStart); @@ -136713,11 +145418,9 @@ static void translateColumnToCopy( pOp->p2 = pOp->p3; pOp->p3 = 0; }else if( pOp->opcode==OP_Rowid ){ - if( bIncrRowid ){ - /* Increment the value stored in the P2 operand of the OP_Rowid. */ - pOp->opcode = OP_AddImm; - pOp->p1 = pOp->p2; - pOp->p2 = 1; + if( iAutoidxCur ){ + pOp->opcode = OP_Sequence; + pOp->p1 = iAutoidxCur; }else{ pOp->opcode = OP_Null; pOp->p1 = 0; @@ -136734,7 +145437,7 @@ static void translateColumnToCopy( ** are no-ops. */ #if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(WHERETRACE_ENABLED) -static void TRACE_IDX_INPUTS(sqlite3_index_info *p){ +static void whereTraceIndexInfoInputs(sqlite3_index_info *p){ int i; if( !sqlite3WhereTrace ) return; for(i=0; inConstraint; i++){ @@ -136752,7 +145455,7 @@ static void TRACE_IDX_INPUTS(sqlite3_index_info *p){ p->aOrderBy[i].desc); } } -static void TRACE_IDX_OUTPUTS(sqlite3_index_info *p){ +static void whereTraceIndexInfoOutputs(sqlite3_index_info *p){ int i; if( !sqlite3WhereTrace ) return; for(i=0; inConstraint; i++){ @@ -136768,8 +145471,8 @@ static void TRACE_IDX_OUTPUTS(sqlite3_index_info *p){ sqlite3DebugPrintf(" estimatedRows=%lld\n", p->estimatedRows); } #else -#define TRACE_IDX_INPUTS(A) -#define TRACE_IDX_OUTPUTS(A) +#define whereTraceIndexInfoInputs(A) +#define whereTraceIndexInfoOutputs(A) #endif #ifndef SQLITE_OMIT_AUTOMATIC_INDEX @@ -136864,7 +145567,7 @@ static void constructAutomaticIndex( && (pTerm->wtFlags & TERM_VIRTUAL)==0 && !ExprHasProperty(pExpr, EP_FromJoin) && sqlite3ExprIsTableConstant(pExpr, pSrc->iCursor) ){ - pPartial = sqlite3ExprAnd(pParse->db, pPartial, + pPartial = sqlite3ExprAnd(pParse, pPartial, sqlite3ExprDup(pParse->db, pExpr, 0)); } if( termCanDriveIndex(pTerm, pSrc, notReady) ){ @@ -136929,7 +145632,8 @@ static void constructAutomaticIndex( Expr *pX = pTerm->pExpr; idxCols |= cMask; pIdx->aiColumn[n] = pTerm->u.leftColumn; - pColl = sqlite3BinaryCompareCollSeq(pParse, pX->pLeft, pX->pRight); + pColl = sqlite3ExprCompareCollSeq(pParse, pX); + assert( pColl!=0 || pParse->nErr>0 ); /* TH3 collate01.800 */ pIdx->azColl[n] = pColl ? pColl->zName : sqlite3StrBINARY; n++; } @@ -136965,7 +145669,6 @@ static void constructAutomaticIndex( VdbeComment((v, "for %s", pTable->zName)); /* Fill the automatic index with content */ - sqlite3ExprCachePush(pParse); pTabItem = &pWC->pWInfo->pTabList->a[pLevel->iFrom]; if( pTabItem->fg.viaCoroutine ){ int regYield = pTabItem->regReturn; @@ -136973,12 +145676,12 @@ static void constructAutomaticIndex( sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, pTabItem->addrFillSub); addrTop = sqlite3VdbeAddOp1(v, OP_Yield, regYield); VdbeCoverage(v); - VdbeComment((v, "next row of \"%s\"", pTabItem->pTab->zName)); + VdbeComment((v, "next row of %s", pTabItem->pTab->zName)); }else{ addrTop = sqlite3VdbeAddOp1(v, OP_Rewind, pLevel->iTabCur); VdbeCoverage(v); } if( pPartial ){ - iContinue = sqlite3VdbeMakeLabel(v); + iContinue = sqlite3VdbeMakeLabel(pParse); sqlite3ExprIfFalse(pParse, pPartial, iContinue, SQLITE_JUMPIFNULL); pLoop->wsFlags |= WHERE_PARTIALIDX; } @@ -136992,17 +145695,17 @@ static void constructAutomaticIndex( if( pTabItem->fg.viaCoroutine ){ sqlite3VdbeChangeP2(v, addrCounter, regBase+n); testcase( pParse->db->mallocFailed ); + assert( pLevel->iIdxCur>0 ); translateColumnToCopy(pParse, addrTop, pLevel->iTabCur, - pTabItem->regResult, 1); + pTabItem->regResult, pLevel->iIdxCur); sqlite3VdbeGoto(v, addrTop); pTabItem->fg.viaCoroutine = 0; }else{ sqlite3VdbeAddOp2(v, OP_Next, pLevel->iTabCur, addrTop+1); VdbeCoverage(v); + sqlite3VdbeChangeP5(v, SQLITE_STMTSTATUS_AUTOINDEX); } - sqlite3VdbeChangeP5(v, SQLITE_STMTSTATUS_AUTOINDEX); sqlite3VdbeJumpHere(v, addrTop); sqlite3ReleaseTempReg(pParse, regRecord); - sqlite3ExprCachePop(pParse); /* Jump here when skipping the initialization */ sqlite3VdbeJumpHere(v, addrInit); @@ -137063,6 +145766,7 @@ static sqlite3_index_info *allocateIndexInfo( for(i=0; ia[i].pExpr; if( pExpr->op!=TK_COLUMN || pExpr->iTable!=pSrc->iCursor ) break; + if( pOrderBy->a[i].sortFlags & KEYINFO_ORDER_BIGNULL ) break; } if( i==n){ nOrderBy = n; @@ -137078,23 +145782,14 @@ static sqlite3_index_info *allocateIndexInfo( sqlite3ErrorMsg(pParse, "out of memory"); return 0; } - - /* Initialize the structure. The sqlite3_index_info structure contains - ** many fields that are declared "const" to prevent xBestIndex from - ** changing them. We have to do some funky casting in order to - ** initialize those fields. - */ pHidden = (struct HiddenIndexInfo*)&pIdxInfo[1]; pIdxCons = (struct sqlite3_index_constraint*)&pHidden[1]; pIdxOrderBy = (struct sqlite3_index_orderby*)&pIdxCons[nTerm]; pUsage = (struct sqlite3_index_constraint_usage*)&pIdxOrderBy[nOrderBy]; - *(int*)&pIdxInfo->nConstraint = nTerm; - *(int*)&pIdxInfo->nOrderBy = nOrderBy; - *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint = pIdxCons; - *(struct sqlite3_index_orderby**)&pIdxInfo->aOrderBy = pIdxOrderBy; - *(struct sqlite3_index_constraint_usage**)&pIdxInfo->aConstraintUsage = - pUsage; - + pIdxInfo->nOrderBy = nOrderBy; + pIdxInfo->aConstraint = pIdxCons; + pIdxInfo->aOrderBy = pIdxOrderBy; + pIdxInfo->aConstraintUsage = pUsage; pHidden->pWC = pWC; pHidden->pParse = pParse; for(i=j=0, pTerm=pWC->a; inTerm; i++, pTerm++){ @@ -137108,6 +145803,15 @@ static sqlite3_index_info *allocateIndexInfo( testcase( pTerm->eOperator & WO_ALL ); if( (pTerm->eOperator & ~(WO_EQUIV))==0 ) continue; if( pTerm->wtFlags & TERM_VNULL ) continue; + + /* tag-20191211-002: WHERE-clause constraints are not useful to the + ** right-hand table of a LEFT JOIN. See tag-20191211-001 for the + ** equivalent restriction for ordinary tables. */ + if( (pSrc->fg.jointype & JT_LEFT)!=0 + && !ExprHasProperty(pTerm->pExpr, EP_FromJoin) + ){ + continue; + } assert( pTerm->u.leftColumn>=(-1) ); pIdxCons[j].iColumn = pTerm->u.leftColumn; pIdxCons[j].iTermOffset = i; @@ -137136,7 +145840,8 @@ static sqlite3_index_info *allocateIndexInfo( if( op & (WO_LT|WO_LE|WO_GT|WO_GE) && sqlite3ExprIsVector(pTerm->pExpr->pRight) ){ - if( i<16 ) mNoOmit |= (1 << i); + testcase( j!=i ); + if( j<16 ) mNoOmit |= (1 << j); if( op==WO_LT ) pIdxCons[j].op = WO_LE; if( op==WO_GT ) pIdxCons[j].op = WO_GE; } @@ -137144,10 +145849,11 @@ static sqlite3_index_info *allocateIndexInfo( j++; } + pIdxInfo->nConstraint = j; for(i=0; ia[i].pExpr; pIdxOrderBy[i].iColumn = pExpr->iColumn; - pIdxOrderBy[i].desc = pOrderBy->a[i].sortOrder; + pIdxOrderBy[i].desc = pOrderBy->a[i].sortFlags & KEYINFO_ORDER_DESC; } *pmNoOmit = mNoOmit; @@ -137160,9 +145866,11 @@ static sqlite3_index_info *allocateIndexInfo( ** method of the virtual table with the sqlite3_index_info object that ** comes in as the 3rd argument to this function. ** -** If an error occurs, pParse is populated with an error message and a -** non-zero value is returned. Otherwise, 0 is returned and the output -** part of the sqlite3_index_info structure is left populated. +** If an error occurs, pParse is populated with an error message and an +** appropriate error code is returned. A return of SQLITE_CONSTRAINT from +** xBestIndex is not considered an error. SQLITE_CONSTRAINT indicates that +** the current configuration of "unusable" flags in sqlite3_index_info can +** not result in a valid plan. ** ** Whether or not an error is returned, it is the responsibility of the ** caller to eventually free p->idxStr if p->needToFreeIdxStr indicates @@ -137172,11 +145880,11 @@ static int vtabBestIndex(Parse *pParse, Table *pTab, sqlite3_index_info *p){ sqlite3_vtab *pVtab = sqlite3GetVTable(pParse->db, pTab)->pVtab; int rc; - TRACE_IDX_INPUTS(p); + whereTraceIndexInfoInputs(p); rc = pVtab->pModule->xBestIndex(pVtab, p); - TRACE_IDX_OUTPUTS(p); + whereTraceIndexInfoOutputs(p); - if( rc!=SQLITE_OK ){ + if( rc!=SQLITE_OK && rc!=SQLITE_CONSTRAINT ){ if( rc==SQLITE_NOMEM ){ sqlite3OomFault(pParse->db); }else if( !pVtab->zErrMsg ){ @@ -137187,23 +145895,11 @@ static int vtabBestIndex(Parse *pParse, Table *pTab, sqlite3_index_info *p){ } sqlite3_free(pVtab->zErrMsg); pVtab->zErrMsg = 0; - -#if 0 - /* This error is now caught by the caller. - ** Search for "xBestIndex malfunction" below */ - for(i=0; inConstraint; i++){ - if( !p->aConstraint[i].usable && p->aConstraintUsage[i].argvIndex>0 ){ - sqlite3ErrorMsg(pParse, - "table %s: xBestIndex returned an invalid plan", pTab->zName); - } - } -#endif - - return pParse->nErr; + return rc; } #endif /* !defined(SQLITE_OMIT_VIRTUALTABLE) */ -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +#ifdef SQLITE_ENABLE_STAT4 /* ** Estimate the location of a particular key among all keys in an ** index. Store the results in aStat as follows: @@ -137396,7 +146092,7 @@ static int whereKeyStats( pRec->nField = nField; return i; } -#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ +#endif /* SQLITE_ENABLE_STAT4 */ /* ** If it is not NULL, pTerm is a term that provides an upper or lower @@ -137422,7 +146118,7 @@ static LogEst whereRangeAdjust(WhereTerm *pTerm, LogEst nNew){ } -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +#ifdef SQLITE_ENABLE_STAT4 /* ** Return the affinity for a single column of an index. */ @@ -137431,12 +146127,13 @@ SQLITE_PRIVATE char sqlite3IndexColumnAffinity(sqlite3 *db, Index *pIdx, int iCo if( !pIdx->zColAff ){ if( sqlite3IndexAffinityStr(db, pIdx)==0 ) return SQLITE_AFF_BLOB; } + assert( pIdx->zColAff[iCol]!=0 ); return pIdx->zColAff[iCol]; } #endif -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +#ifdef SQLITE_ENABLE_STAT4 /* ** This function is called to estimate the number of rows visited by a ** range-scan on a skip-scan index. For example: @@ -137542,7 +146239,7 @@ static int whereRangeSkipScanEst( return rc; } -#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ +#endif /* SQLITE_ENABLE_STAT4 */ /* ** This function is used to estimate the number of rows that will be visited @@ -137595,11 +146292,13 @@ static int whereRangeScanEst( int nOut = pLoop->nOut; LogEst nNew; -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +#ifdef SQLITE_ENABLE_STAT4 Index *p = pLoop->u.btree.pIndex; int nEq = pLoop->u.btree.nEq; - if( p->nSample>0 && nEqnSampleCol ){ + if( p->nSample>0 && ALWAYS(nEqnSampleCol) + && OptimizationEnabled(pParse->db, SQLITE_Stat4) + ){ if( nEq==pBuilder->nRecValid ){ UnpackedRecord *pRec = pBuilder->pRec; tRowcnt a[2]; @@ -137696,7 +146395,7 @@ static int whereRangeScanEst( /* TUNING: If both iUpper and iLower are derived from the same ** sample, then assume they are 4x more selective. This brings ** the estimated selectivity more in line with what it would be - ** if estimated without the use of STAT3/4 tables. */ + ** if estimated without the use of STAT4 tables. */ if( iLwrIdx==iUprIdx ) nNew -= 20; assert( 20==sqlite3LogEst(4) ); }else{ nNew = 10; assert( 10==sqlite3LogEst(2) ); @@ -137745,12 +146444,12 @@ static int whereRangeScanEst( return rc; } -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +#ifdef SQLITE_ENABLE_STAT4 /* ** Estimate the number of rows that will be returned based on ** an equality constraint x=VALUE and where that VALUE occurs in ** the histogram data. This only works when x is the left-most -** column of an index and sqlite_stat3 histogram data is available +** column of an index and sqlite_stat4 histogram data is available ** for that index. When pExpr==NULL that means the constraint is ** "x IS NULL" instead of "x=VALUE". ** @@ -137808,9 +146507,9 @@ static int whereEqualScanEst( return rc; } -#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ +#endif /* SQLITE_ENABLE_STAT4 */ -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +#ifdef SQLITE_ENABLE_STAT4 /* ** Estimate the number of rows that will be returned based on ** an IN constraint where the right-hand side of the IN operator @@ -137857,23 +146556,24 @@ static int whereInScanEst( assert( pBuilder->nRecValid==nRecValid ); return rc; } -#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ +#endif /* SQLITE_ENABLE_STAT4 */ #ifdef WHERETRACE_ENABLED /* ** Print the content of a WhereTerm object */ -static void whereTermPrint(WhereTerm *pTerm, int iTerm){ +SQLITE_PRIVATE void sqlite3WhereTermPrint(WhereTerm *pTerm, int iTerm){ if( pTerm==0 ){ sqlite3DebugPrintf("TERM-%-3d NULL\n", iTerm); }else{ - char zType[4]; + char zType[8]; char zLeft[50]; - memcpy(zType, "...", 4); + memcpy(zType, "....", 5); if( pTerm->wtFlags & TERM_VIRTUAL ) zType[0] = 'V'; if( pTerm->eOperator & WO_EQUIV ) zType[1] = 'E'; if( ExprHasProperty(pTerm->pExpr, EP_FromJoin) ) zType[2] = 'L'; + if( pTerm->wtFlags & TERM_CODED ) zType[3] = 'C'; if( pTerm->eOperator & WO_SINGLE ){ sqlite3_snprintf(sizeof(zLeft),zLeft,"left={%d:%d}", pTerm->leftCursor, pTerm->u.leftColumn); @@ -137884,14 +146584,21 @@ static void whereTermPrint(WhereTerm *pTerm, int iTerm){ sqlite3_snprintf(sizeof(zLeft),zLeft,"left=%d", pTerm->leftCursor); } sqlite3DebugPrintf( - "TERM-%-3d %p %s %-12s prob=%-3d op=0x%03x wtFlags=0x%04x", - iTerm, pTerm, zType, zLeft, pTerm->truthProb, - pTerm->eOperator, pTerm->wtFlags); + "TERM-%-3d %p %s %-12s op=%03x wtFlags=%04x", + iTerm, pTerm, zType, zLeft, pTerm->eOperator, pTerm->wtFlags); + /* The 0x10000 .wheretrace flag causes extra information to be + ** shown about each Term */ + if( sqlite3WhereTrace & 0x10000 ){ + sqlite3DebugPrintf(" prob=%-3d prereq=%llx,%llx", + pTerm->truthProb, (u64)pTerm->prereqAll, (u64)pTerm->prereqRight); + } if( pTerm->iField ){ - sqlite3DebugPrintf(" iField=%d\n", pTerm->iField); - }else{ - sqlite3DebugPrintf("\n"); + sqlite3DebugPrintf(" iField=%d", pTerm->iField); } + if( pTerm->iParent>=0 ){ + sqlite3DebugPrintf(" iParent=%d", pTerm->iParent); + } + sqlite3DebugPrintf("\n"); sqlite3TreeViewExpr(0, pTerm->pExpr, 0); } } @@ -137904,7 +146611,7 @@ static void whereTermPrint(WhereTerm *pTerm, int iTerm){ SQLITE_PRIVATE void sqlite3WhereClausePrint(WhereClause *pWC){ int i; for(i=0; inTerm; i++){ - whereTermPrint(&pWC->a[i], i); + sqlite3WhereTermPrint(&pWC->a[i], i); } } #endif @@ -137913,7 +146620,7 @@ SQLITE_PRIVATE void sqlite3WhereClausePrint(WhereClause *pWC){ /* ** Print a WhereLoop object for debugging purposes */ -static void whereLoopPrint(WhereLoop *p, WhereClause *pWC){ +SQLITE_PRIVATE void sqlite3WhereLoopPrint(WhereLoop *p, WhereClause *pWC){ WhereInfo *pWInfo = pWC->pWInfo; int nb = 1+(pWInfo->pTabList->nSrc+3)/4; struct SrcList_item *pItem = pWInfo->pTabList->a + p->iTab; @@ -137938,7 +146645,7 @@ static void whereLoopPrint(WhereLoop *p, WhereClause *pWC){ }else{ char *z; if( p->u.vtab.idxStr ){ - z = sqlite3_mprintf("(%d,\"%s\",%x)", + z = sqlite3_mprintf("(%d,\"%s\",%#x)", p->u.vtab.idxNum, p->u.vtab.idxStr, p->u.vtab.omitMask); }else{ z = sqlite3_mprintf("(%d,%x)", p->u.vtab.idxNum, p->u.vtab.omitMask); @@ -137955,7 +146662,7 @@ static void whereLoopPrint(WhereLoop *p, WhereClause *pWC){ if( p->nLTerm && (sqlite3WhereTrace & 0x100)!=0 ){ int i; for(i=0; inLTerm; i++){ - whereTermPrint(p->aLTerm[i], i); + sqlite3WhereTermPrint(p->aLTerm[i], i); } } } @@ -138059,6 +146766,7 @@ static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){ pWInfo->pLoops = p->pNextLoop; whereLoopDelete(db, p); } + assert( pWInfo->pExprMods==0 ); sqlite3DbFreeNN(db, pWInfo); } @@ -138252,6 +146960,16 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ sqlite3 *db = pWInfo->pParse->db; int rc; + /* Stop the search once we hit the query planner search limit */ + if( pBuilder->iPlanLimit==0 ){ + WHERETRACE(0xffffffff,("=== query planner search limit reached ===\n")); + if( pBuilder->pOrSet ) pBuilder->pOrSet->n = 0; + return SQLITE_DONE; + } + pBuilder->iPlanLimit--; + + whereLoopAdjustCost(pWInfo->pLoops, pTemplate); + /* If pBuilder->pOrSet is defined, then only keep track of the costs ** and prereqs. */ @@ -138266,7 +146984,7 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ #if WHERETRACE_ENABLED /* 0x8 */ if( sqlite3WhereTrace & 0x8 ){ sqlite3DebugPrintf(x?" or-%d: ":" or-X: ", n); - whereLoopPrint(pTemplate, pBuilder->pWC); + sqlite3WhereLoopPrint(pTemplate, pBuilder->pWC); } #endif } @@ -138275,7 +146993,6 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ /* Look for an existing WhereLoop to replace with pTemplate */ - whereLoopAdjustCost(pWInfo->pLoops, pTemplate); ppPrev = whereLoopFindLesser(&pWInfo->pLoops, pTemplate); if( ppPrev==0 ){ @@ -138284,7 +147001,7 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ #if WHERETRACE_ENABLED /* 0x8 */ if( sqlite3WhereTrace & 0x8 ){ sqlite3DebugPrintf(" skip: "); - whereLoopPrint(pTemplate, pBuilder->pWC); + sqlite3WhereLoopPrint(pTemplate, pBuilder->pWC); } #endif return SQLITE_OK; @@ -138300,12 +147017,12 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ if( sqlite3WhereTrace & 0x8 ){ if( p!=0 ){ sqlite3DebugPrintf("replace: "); - whereLoopPrint(p, pBuilder->pWC); + sqlite3WhereLoopPrint(p, pBuilder->pWC); sqlite3DebugPrintf(" with: "); }else{ sqlite3DebugPrintf(" add: "); } - whereLoopPrint(pTemplate, pBuilder->pWC); + sqlite3WhereLoopPrint(pTemplate, pBuilder->pWC); } #endif if( p==0 ){ @@ -138329,7 +147046,7 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ #if WHERETRACE_ENABLED /* 0x8 */ if( sqlite3WhereTrace & 0x8 ){ sqlite3DebugPrintf(" delete: "); - whereLoopPrint(pToDel, pBuilder->pWC); + sqlite3WhereLoopPrint(pToDel, pBuilder->pWC); } #endif whereLoopDelete(db, pToDel); @@ -138338,7 +147055,7 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ rc = whereLoopXfer(db, p, pTemplate); if( (p->wsFlags & WHERE_VIRTUALTABLE)==0 ){ Index *pIndex = p->u.btree.pIndex; - if( pIndex && pIndex->tnum==0 ){ + if( pIndex && pIndex->idxType==SQLITE_IDXTYPE_IPK ){ p->u.btree.pIndex = 0; } } @@ -138381,11 +147098,12 @@ static void whereLoopOutputAdjust( ){ WhereTerm *pTerm, *pX; Bitmask notAllowed = ~(pLoop->prereq|pLoop->maskSelf); - int i, j, k; + int i, j; LogEst iReduce = 0; /* pLoop->nOut should not exceed nRow-iReduce */ assert( (pLoop->wsFlags & WHERE_AUTO_INDEX)==0 ); for(i=pWC->nTerm, pTerm=pWC->a; i>0; i--, pTerm++){ + assert( pTerm!=0 ); if( (pTerm->wtFlags & TERM_VIRTUAL)!=0 ) break; if( (pTerm->prereqAll & pLoop->maskSelf)==0 ) continue; if( (pTerm->prereqAll & notAllowed)!=0 ) continue; @@ -138404,15 +147122,21 @@ static void whereLoopOutputAdjust( /* In the absence of explicit truth probabilities, use heuristics to ** guess a reasonable truth probability. */ pLoop->nOut--; - if( pTerm->eOperator&(WO_EQ|WO_IS) ){ + if( (pTerm->eOperator&(WO_EQ|WO_IS))!=0 + && (pTerm->wtFlags & TERM_HIGHTRUTH)==0 /* tag-20200224-1 */ + ){ Expr *pRight = pTerm->pExpr->pRight; + int k = 0; testcase( pTerm->pExpr->op==TK_IS ); if( sqlite3ExprIsInteger(pRight, &k) && k>=(-1) && k<=1 ){ k = 10; }else{ k = 20; } - if( iReducewtFlags |= TERM_HEURTRUTH; + iReduce = k; + } } } } @@ -138505,8 +147229,8 @@ static int whereRangeVectorLen( ** terms only. If it is modified, this value is restored before this ** function returns. ** -** If pProbe->tnum==0, that means pIndex is a fake index used for the -** INTEGER PRIMARY KEY. +** If pProbe->idxType==SQLITE_IDXTYPE_IPK, that means pIndex is +** a fake index used for the INTEGER PRIMARY KEY. */ static int whereLoopAddBtreeIndex( WhereLoopBuilder *pBuilder, /* The WhereLoop factory */ @@ -138536,8 +147260,9 @@ static int whereLoopAddBtreeIndex( pNew = pBuilder->pNew; if( db->mallocFailed ) return SQLITE_NOMEM_BKPT; - WHERETRACE(0x800, ("BEGIN %s.addBtreeIdx(%s), nEq=%d\n", - pProbe->pTable->zName,pProbe->zName, pNew->u.btree.nEq)); + WHERETRACE(0x800, ("BEGIN %s.addBtreeIdx(%s), nEq=%d, nSkip=%d\n", + pProbe->pTable->zName,pProbe->zName, + pNew->u.btree.nEq, pNew->nSkip)); assert( (pNew->wsFlags & WHERE_VIRTUALTABLE)==0 ); assert( (pNew->wsFlags & WHERE_TOP_LIMIT)==0 ); @@ -138569,7 +147294,7 @@ static int whereLoopAddBtreeIndex( LogEst rCostIdx; LogEst nOutUnadjusted; /* nOut before IN() and WHERE adjustments */ int nIn = 0; -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +#ifdef SQLITE_ENABLE_STAT4 int nRecValid = pBuilder->nRecValid; #endif if( (eOp==WO_ISNULL || (pTerm->wtFlags&TERM_VNULL)!=0) @@ -138583,9 +147308,9 @@ static int whereLoopAddBtreeIndex( ** to mix with a lower range bound from some other source */ if( pTerm->wtFlags & TERM_LIKEOPT && pTerm->eOperator==WO_LT ) continue; - /* Do not allow constraints from the WHERE clause to be used by the - ** right table of a LEFT JOIN. Only constraints in the ON clause are - ** allowed */ + /* tag-20191211-001: Do not allow constraints from the WHERE clause to + ** be used by the right table of a LEFT JOIN. Only constraints in the + ** ON clause are allowed. See tag-20191211-002 for the vtab equivalent. */ if( (pSrc->fg.jointype & JT_LEFT)!=0 && !ExprHasProperty(pTerm->pExpr, EP_FromJoin) ){ @@ -138593,9 +147318,9 @@ static int whereLoopAddBtreeIndex( } if( IsUniqueIndex(pProbe) && saved_nEq==pProbe->nKeyCol-1 ){ - pBuilder->bldFlags |= SQLITE_BLDF_UNIQUE; + pBuilder->bldFlags1 |= SQLITE_BLDF1_UNIQUE; }else{ - pBuilder->bldFlags |= SQLITE_BLDF_INDEXED; + pBuilder->bldFlags1 |= SQLITE_BLDF1_INDEXED; } pNew->wsFlags = saved_wsFlags; pNew->u.btree.nEq = saved_nEq; @@ -138614,7 +147339,6 @@ static int whereLoopAddBtreeIndex( if( eOp & WO_IN ){ Expr *pExpr = pTerm->pExpr; - pNew->wsFlags |= WHERE_COLUMN_IN; if( ExprHasProperty(pExpr, EP_xIsSelect) ){ /* "x IN (SELECT ...)": TUNING: the SELECT returns 25 rows */ int i; @@ -138631,9 +147355,43 @@ static int whereLoopAddBtreeIndex( }else if( ALWAYS(pExpr->x.pList && pExpr->x.pList->nExpr) ){ /* "x IN (value, value, ...)" */ nIn = sqlite3LogEst(pExpr->x.pList->nExpr); - assert( nIn>0 ); /* RHS always has 2 or more terms... The parser - ** changes "x IN (?)" into "x=?". */ } + if( pProbe->hasStat1 ){ + LogEst M, logK, safetyMargin; + /* Let: + ** N = the total number of rows in the table + ** K = the number of entries on the RHS of the IN operator + ** M = the number of rows in the table that match terms to the + ** to the left in the same index. If the IN operator is on + ** the left-most index column, M==N. + ** + ** Given the definitions above, it is better to omit the IN operator + ** from the index lookup and instead do a scan of the M elements, + ** testing each scanned row against the IN operator separately, if: + ** + ** M*log(K) < K*log(N) + ** + ** Our estimates for M, K, and N might be inaccurate, so we build in + ** a safety margin of 2 (LogEst: 10) that favors using the IN operator + ** with the index, as using an index has better worst-case behavior. + ** If we do not have real sqlite_stat1 data, always prefer to use + ** the index. + */ + M = pProbe->aiRowLogEst[saved_nEq]; + logK = estLog(nIn); + safetyMargin = 10; /* TUNING: extra weight for indexed IN */ + if( M + logK + safetyMargin < nIn + rLogSize ){ + WHERETRACE(0x40, + ("Scan preferred over IN operator on column %d of \"%s\" (%d<%d)\n", + saved_nEq, pProbe->zName, M+logK+10, nIn+rLogSize)); + continue; + }else{ + WHERETRACE(0x40, + ("IN operator preferred on column %d of \"%s\" (%d>=%d)\n", + saved_nEq, pProbe->zName, M+logK+10, nIn+rLogSize)); + } + } + pNew->wsFlags |= WHERE_COLUMN_IN; }else if( eOp & (WO_EQ|WO_IS) ){ int iCol = pProbe->aiColumn[saved_nEq]; pNew->wsFlags |= WHERE_COLUMN_EQ; @@ -138692,7 +147450,7 @@ static int whereLoopAddBtreeIndex( ** the value of pNew->nOut to account for pTerm (but not nIn/nInMul). */ assert( pNew->nOut==saved_nOut ); if( pNew->wsFlags & WHERE_COLUMN_RANGE ){ - /* Adjust nOut using stat3/stat4 data. Or, if there is no stat3/stat4 + /* Adjust nOut using stat4 data. Or, if there is no stat4 ** data, using some other estimate. */ whereRangeScanEst(pParse, pBuilder, pBtm, pTop, pNew); }else{ @@ -138706,12 +147464,13 @@ static int whereLoopAddBtreeIndex( pNew->nOut += pTerm->truthProb; pNew->nOut -= nIn; }else{ -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +#ifdef SQLITE_ENABLE_STAT4 tRowcnt nOut = 0; if( nInMul==0 && pProbe->nSample && pNew->u.btree.nEq<=pProbe->nSampleCol && ((eOp & WO_IN)==0 || !ExprHasProperty(pTerm->pExpr, EP_xIsSelect)) + && OptimizationEnabled(db, SQLITE_Stat4) ){ Expr *pExpr = pTerm->pExpr; if( (eOp & (WO_EQ|WO_ISNULL|WO_IS))!=0 ){ @@ -138726,6 +147485,27 @@ static int whereLoopAddBtreeIndex( if( rc!=SQLITE_OK ) break; /* Jump out of the pTerm loop */ if( nOut ){ pNew->nOut = sqlite3LogEst(nOut); + if( nEq==1 + /* TUNING: Mark terms as "low selectivity" if they seem likely + ** to be true for half or more of the rows in the table. + ** See tag-202002240-1 */ + && pNew->nOut+10 > pProbe->aiRowLogEst[0] + ){ +#if WHERETRACE_ENABLED /* 0x01 */ + if( sqlite3WhereTrace & 0x01 ){ + sqlite3DebugPrintf( + "STAT4 determines term has low selectivity:\n"); + sqlite3WhereTermPrint(pTerm, 999); + } +#endif + pTerm->wtFlags |= TERM_HIGHTRUTH; + if( pTerm->wtFlags & TERM_HEURTRUTH ){ + /* If the term has previously been used with an assumption of + ** higher selectivity, then set the flag to rerun the + ** loop computations. */ + pBuilder->bldFlags2 |= SQLITE_BLDF2_2NDPASS; + } + } if( pNew->nOut>saved_nOut ) pNew->nOut = saved_nOut; pNew->nOut -= nIn; } @@ -138748,6 +147528,7 @@ static int whereLoopAddBtreeIndex( ** it to pNew->rRun, which is currently set to the cost of the index ** seek only. Then, if this is a non-covering index, add the cost of ** visiting the rows in the main table. */ + assert( pSrc->pTab->szTabRow>0 ); rCostIdx = pNew->nOut + 1 + (15*pProbe->szIdxRow)/pSrc->pTab->szTabRow; pNew->rRun = sqlite3LogEstAdd(rLogSize, rCostIdx); if( (pNew->wsFlags & (WHERE_IDX_ONLY|WHERE_IPK))==0 ){ @@ -138773,7 +147554,7 @@ static int whereLoopAddBtreeIndex( whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nInMul+nIn); } pNew->nOut = saved_nOut; -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +#ifdef SQLITE_ENABLE_STAT4 pBuilder->nRecValid = nRecValid; #endif } @@ -138799,7 +147580,10 @@ static int whereLoopAddBtreeIndex( assert( 42==sqlite3LogEst(18) ); if( saved_nEq==saved_nSkip && saved_nEq+1nKeyCol + && saved_nEq==pNew->nLTerm && pProbe->noSkipScan==0 + && pProbe->hasStat1!=0 + && OptimizationEnabled(db, SQLITE_SkipScan) && pProbe->aiRowLogEst[saved_nEq+1]>=42 /* TUNING: Minimum for skip-scan */ && (rc = whereLoopResize(db, pNew, pNew->nLTerm+1))==SQLITE_OK ){ @@ -138845,7 +147629,7 @@ static int indexMightHelpWithOrderBy( if( pIndex->bUnordered ) return 0; if( (pOB = pBuilder->pWInfo->pOrderBy)==0 ) return 0; for(ii=0; iinExpr; ii++){ - Expr *pExpr = sqlite3ExprSkipCollate(pOB->a[ii].pExpr); + Expr *pExpr = sqlite3ExprSkipCollateAndLikely(pOB->a[ii].pExpr); if( pExpr->op==TK_COLUMN && pExpr->iTable==iCursor ){ if( pExpr->iColumn<0 ) return 1; for(jj=0; jjnKeyCol; jj++){ @@ -138863,39 +147647,28 @@ static int indexMightHelpWithOrderBy( return 0; } -/* -** Return a bitmask where 1s indicate that the corresponding column of -** the table is used by an index. Only the first 63 columns are considered. -*/ -static Bitmask columnsInIndex(Index *pIdx){ - Bitmask m = 0; - int j; - for(j=pIdx->nColumn-1; j>=0; j--){ - int x = pIdx->aiColumn[j]; - if( x>=0 ){ - testcase( x==BMS-1 ); - testcase( x==BMS-2 ); - if( xpWInfo->pParse; while( pWhere->op==TK_AND ){ - if( !whereUsablePartialIndex(iTab,pWC,pWhere->pLeft) ) return 0; + if( !whereUsablePartialIndex(iTab,isLeft,pWC,pWhere->pLeft) ) return 0; pWhere = pWhere->pRight; } if( pParse->db->flags & SQLITE_EnableQPSG ) pParse = 0; for(i=0, pTerm=pWC->a; inTerm; i++, pTerm++){ - Expr *pExpr = pTerm->pExpr; + Expr *pExpr; + pExpr = pTerm->pExpr; if( (!ExprHasProperty(pExpr, EP_FromJoin) || pExpr->iRightJoinTable==iTab) + && (isLeft==0 || ExprHasProperty(pExpr, EP_FromJoin)) && sqlite3ExprImpliesExpr(pParse, pExpr, pWhere, iTab) ){ return 1; @@ -138987,6 +147760,7 @@ static int whereLoopAddBtree( sPk.onError = OE_Replace; sPk.pTable = pTab; sPk.szIdxRow = pTab->szTabRow; + sPk.idxType = SQLITE_IDXTYPE_IPK; aiRowEstPk[0] = pTab->nRowLogEst; aiRowEstPk[1] = 0; pFirst = pSrc->pTab->pIndex; @@ -139057,8 +147831,11 @@ static int whereLoopAddBtree( for(; rc==SQLITE_OK && pProbe; pProbe=(pSrc->pIBIndex ? 0 : pProbe->pNext), iSortIdx++ ){ + int isLeft = (pSrc->fg.jointype & JT_OUTER)!=0; if( pProbe->pPartIdxWhere!=0 - && !whereUsablePartialIndex(pSrc->iCursor, pWC, pProbe->pPartIdxWhere) ){ + && !whereUsablePartialIndex(pSrc->iCursor, isLeft, pWC, + pProbe->pPartIdxWhere) + ){ testcase( pNew->iTab!=pSrc->iCursor ); /* See ticket [98d973b8f5] */ continue; /* Partial index inappropriate for this query */ } @@ -139077,7 +147854,7 @@ static int whereLoopAddBtree( b = indexMightHelpWithOrderBy(pBuilder, pProbe, pSrc->iCursor); /* The ONEPASS_DESIRED flags never occurs together with ORDER BY */ assert( (pWInfo->wctrlFlags & WHERE_ONEPASS_DESIRED)==0 || b==0 ); - if( pProbe->tnum<=0 ){ + if( pProbe->idxType==SQLITE_IDXTYPE_IPK ){ /* Integer primary key index */ pNew->wsFlags = WHERE_IPK; @@ -139096,7 +147873,7 @@ static int whereLoopAddBtree( pNew->wsFlags = WHERE_IDX_ONLY | WHERE_INDEXED; m = 0; }else{ - m = pSrc->colUsed & ~columnsInIndex(pProbe); + m = pSrc->colUsed & pProbe->colNotIdxed; pNew->wsFlags = (m==0) ? (WHERE_IDX_ONLY|WHERE_INDEXED) : WHERE_INDEXED; } @@ -139153,16 +147930,16 @@ static int whereLoopAddBtree( } } - pBuilder->bldFlags = 0; + pBuilder->bldFlags1 = 0; rc = whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, 0); - if( pBuilder->bldFlags==SQLITE_BLDF_INDEXED ){ + if( pBuilder->bldFlags1==SQLITE_BLDF1_INDEXED ){ /* If a non-unique index is used, or if a prefix of the key for ** unique index is used (making the index functionally non-unique) ** then the sqlite_stat1 data becomes important for scoring the ** plan */ pTab->tabFlags |= TF_StatsUsed; } -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +#ifdef SQLITE_ENABLE_STAT4 sqlite3Stat4ProbeFree(pBuilder->pRec); pBuilder->nRecValid = 0; pBuilder->pRec = 0; @@ -139243,7 +148020,17 @@ static int whereLoopAddVirtualOne( /* Invoke the virtual table xBestIndex() method */ rc = vtabBestIndex(pParse, pSrc->pTab, pIdxInfo); - if( rc ) return rc; + if( rc ){ + if( rc==SQLITE_CONSTRAINT ){ + /* If the xBestIndex method returns SQLITE_CONSTRAINT, that means + ** that the particular combination of parameters provided is unusable. + ** Make no entries in the loop table. + */ + WHERETRACE(0xffff, (" ^^^^--- non-viable plan rejected!\n")); + return SQLITE_OK; + } + return rc; + } mxTerm = -1; assert( pNew->nLSlot>=nConstraint ); @@ -139275,7 +148062,14 @@ static int whereLoopAddVirtualOne( if( iTerm>mxTerm ) mxTerm = iTerm; testcase( iTerm==15 ); testcase( iTerm==16 ); - if( iTerm<16 && pUsage[i].omit ) pNew->u.vtab.omitMask |= 1<u.vtab.omitMask |= 1<eOperator & WO_IN)!=0 ){ /* A virtual table that is constrained by an IN clause may not ** consume the ORDER BY clause because (1) the order of IN terms @@ -139288,7 +148082,6 @@ static int whereLoopAddVirtualOne( } } } - pNew->u.vtab.omitMask &= ~mNoOmit; pNew->nLTerm = mxTerm+1; for(i=0; i<=mxTerm; i++){ @@ -139345,9 +148138,9 @@ SQLITE_API const char *sqlite3_vtab_collation(sqlite3_index_info *pIdxInfo, int int iTerm = pIdxInfo->aConstraint[iCons].iTermOffset; Expr *pX = pHidden->pWC->a[iTerm].pExpr; if( pX->pLeft ){ - pC = sqlite3BinaryCompareCollSeq(pHidden->pParse, pX->pLeft, pX->pRight); + pC = sqlite3ExprCompareCollSeq(pHidden->pParse, pX); } - zRet = (pC ? pC->zName : "BINARY"); + zRet = (pC ? pC->zName : sqlite3StrBINARY); } return zRet; } @@ -139420,11 +148213,11 @@ static int whereLoopAddVirtual( rc = whereLoopAddVirtualOne(pBuilder, mPrereq, ALLBITS, 0, p, mNoOmit, &bIn); /* If the call to xBestIndex() with all terms enabled produced a plan - ** that does not require any source tables (IOW: a plan with mBest==0), - ** then there is no point in making any further calls to xBestIndex() - ** since they will all return the same result (if the xBestIndex() - ** implementation is sane). */ - if( rc==SQLITE_OK && (mBest = (pNew->prereq & ~mPrereq))!=0 ){ + ** that does not require any source tables (IOW: a plan with mBest==0) + ** and does not use an IN(...) operator, then there is no point in making + ** any further calls to xBestIndex() since they will all return the same + ** result (if the xBestIndex() implementation is sane). */ + if( rc==SQLITE_OK && ((mBest = (pNew->prereq & ~mPrereq))!=0 || bIn) ){ int seenZero = 0; /* True if a plan with no prereqs seen */ int seenZeroNoIN = 0; /* Plan with no prereqs and no IN(...) seen */ Bitmask mPrev = 0; @@ -139570,7 +148363,8 @@ static int whereLoopAddOr( if( rc==SQLITE_OK ){ rc = whereLoopAddOr(&sSubBuild, mPrereq, mUnusable); } - assert( rc==SQLITE_OK || sCur.n==0 ); + assert( rc==SQLITE_OK || rc==SQLITE_DONE || sCur.n==0 ); + testcase( rc==SQLITE_DONE ); if( sCur.n==0 ){ sSum.n = 0; break; @@ -139639,9 +148433,11 @@ static int whereLoopAddAll(WhereLoopBuilder *pBuilder){ /* Loop over the tables in the join, from left to right */ pNew = pBuilder->pNew; whereLoopInit(pNew); + pBuilder->iPlanLimit = SQLITE_QUERY_PLANNER_LIMIT; for(iTab=0, pItem=pTabList->a; pItemiTab = iTab; + pBuilder->iPlanLimit += SQLITE_QUERY_PLANNER_LIMIT_INCR; pNew->maskSelf = sqlite3WhereGetMask(&pWInfo->sMaskSet, pItem->iCursor); if( ((pItem->fg.jointype|priorJointype) & (JT_LEFT|JT_CROSS))!=0 ){ /* This condition is true when pItem is the FROM clause term on the @@ -139663,11 +148459,19 @@ static int whereLoopAddAll(WhereLoopBuilder *pBuilder){ { rc = whereLoopAddBtree(pBuilder, mPrereq); } - if( rc==SQLITE_OK ){ + if( rc==SQLITE_OK && pBuilder->pWC->hasOr ){ rc = whereLoopAddOr(pBuilder, mPrereq, mUnusable); } mPrior |= pNew->maskSelf; - if( rc || db->mallocFailed ) break; + if( rc || db->mallocFailed ){ + if( rc==SQLITE_DONE ){ + /* We hit the query planner search limit set by iPlanLimit */ + sqlite3_log(SQLITE_WARNING, "abbreviated query algorithm search"); + rc = SQLITE_OK; + }else{ + break; + } + } } whereLoopClear(db, pNew); @@ -139768,10 +148572,12 @@ static i8 wherePathSatisfiesOrderBy( pLoop = pLast; } if( pLoop->wsFlags & WHERE_VIRTUALTABLE ){ - if( pLoop->u.vtab.isOrdered ) obSat = obDone; + if( pLoop->u.vtab.isOrdered && (wctrlFlags & WHERE_DISTINCTBY)==0 ){ + obSat = obDone; + } break; - }else{ - pLoop->u.btree.nIdxCol = 0; + }else if( wctrlFlags & WHERE_DISTINCTBY ){ + pLoop->u.btree.nDistinctCol = 0; } iCur = pWInfo->pTabList->a[pLoop->iTab].iCursor; @@ -139782,7 +148588,7 @@ static i8 wherePathSatisfiesOrderBy( */ for(i=0; ia[i].pExpr); + pOBExpr = sqlite3ExprSkipCollateAndLikely(pOrderBy->a[i].pExpr); if( pOBExpr->op!=TK_COLUMN ) continue; if( pOBExpr->iTable!=iCur ) continue; pTerm = sqlite3WhereFindTerm(&pWInfo->sWC, iCur, pOBExpr->iColumn, @@ -139797,8 +148603,11 @@ static i8 wherePathSatisfiesOrderBy( if( j>=pLoop->nLTerm ) continue; } if( (pTerm->eOperator&(WO_EQ|WO_IS))!=0 && pOBExpr->iColumn>=0 ){ - if( sqlite3ExprCollSeqMatch(pWInfo->pParse, - pOrderBy->a[i].pExpr, pTerm->pExpr)==0 ){ + Parse *pParse = pWInfo->pParse; + CollSeq *pColl1 = sqlite3ExprNNCollSeq(pParse, pOrderBy->a[i].pExpr); + CollSeq *pColl2 = sqlite3ExprCompareCollSeq(pParse, pTerm->pExpr); + assert( pColl1 ); + if( pColl2==0 || sqlite3StrICmp(pColl1->zName, pColl2->zName) ){ continue; } testcase( pTerm->pExpr->op==TK_IS ); @@ -139819,7 +148628,8 @@ static i8 wherePathSatisfiesOrderBy( assert( nColumn==nKeyCol+1 || !HasRowid(pIndex->pTable) ); assert( pIndex->aiColumn[nColumn-1]==XN_ROWID || !HasRowid(pIndex->pTable)); - isOrderDistinct = IsUniqueIndex(pIndex); + isOrderDistinct = IsUniqueIndex(pIndex) + && (pLoop->wsFlags & WHERE_SKIPSCAN)==0; } /* Loop through all columns of the index and deal with the ones @@ -139837,15 +148647,21 @@ static i8 wherePathSatisfiesOrderBy( u16 eOp = pLoop->aLTerm[j]->eOperator; /* Skip over == and IS and ISNULL terms. (Also skip IN terms when - ** doing WHERE_ORDERBY_LIMIT processing). + ** doing WHERE_ORDERBY_LIMIT processing). Except, IS and ISNULL + ** terms imply that the index is not UNIQUE NOT NULL in which case + ** the loop need to be marked as not order-distinct because it can + ** have repeated NULL rows. ** ** If the current term is a column of an ((?,?) IN (SELECT...)) ** expression for which the SELECT returns more than one column, ** check that it is the only column used by this loop. Otherwise, ** if it is one of two or more, none of the columns can be - ** considered to match an ORDER BY term. */ + ** considered to match an ORDER BY term. + */ if( (eOp & eqOpMask)!=0 ){ - if( eOp & WO_ISNULL ){ + if( eOp & (WO_ISNULL|WO_IS) ){ + testcase( eOp & WO_ISNULL ); + testcase( eOp & WO_IS ); testcase( isOrderDistinct ); isOrderDistinct = 0; } @@ -139871,7 +148687,7 @@ static i8 wherePathSatisfiesOrderBy( */ if( pIndex ){ iColumn = pIndex->aiColumn[j]; - revIdx = pIndex->aSortOrder[j]; + revIdx = pIndex->aSortOrder[j] & KEYINFO_ORDER_DESC; if( iColumn==pIndex->pTable->iPKey ) iColumn = XN_ROWID; }else{ iColumn = XN_ROWID; @@ -139895,7 +148711,7 @@ static i8 wherePathSatisfiesOrderBy( isMatch = 0; for(i=0; bOnce && ia[i].pExpr); + pOBExpr = sqlite3ExprSkipCollateAndLikely(pOrderBy->a[i].pExpr); testcase( wctrlFlags & WHERE_GROUPBY ); testcase( wctrlFlags & WHERE_DISTINCTBY ); if( (wctrlFlags & (WHERE_GROUPBY|WHERE_DISTINCTBY))==0 ) bOnce = 0; @@ -139913,7 +148729,9 @@ static i8 wherePathSatisfiesOrderBy( pColl = sqlite3ExprNNCollSeq(pWInfo->pParse, pOrderBy->a[i].pExpr); if( sqlite3StrICmp(pColl->zName, pIndex->azColl[j])!=0 ) continue; } - pLoop->u.btree.nIdxCol = j+1; + if( wctrlFlags & WHERE_DISTINCTBY ){ + pLoop->u.btree.nDistinctCol = j+1; + } isMatch = 1; break; } @@ -139921,13 +148739,22 @@ static i8 wherePathSatisfiesOrderBy( /* Make sure the sort order is compatible in an ORDER BY clause. ** Sort order is irrelevant for a GROUP BY clause. */ if( revSet ){ - if( (rev ^ revIdx)!=pOrderBy->a[i].sortOrder ) isMatch = 0; + if( (rev ^ revIdx)!=(pOrderBy->a[i].sortFlags&KEYINFO_ORDER_DESC) ){ + isMatch = 0; + } }else{ - rev = revIdx ^ pOrderBy->a[i].sortOrder; + rev = revIdx ^ (pOrderBy->a[i].sortFlags & KEYINFO_ORDER_DESC); if( rev ) *pRevMask |= MASKBIT(iLoop); revSet = 1; } } + if( isMatch && (pOrderBy->a[i].sortFlags & KEYINFO_ORDER_BIGNULL) ){ + if( j==pLoop->u.btree.nEq ){ + pLoop->wsFlags |= WHERE_BIGNULL_SORT; + }else{ + isMatch = 0; + } + } if( isMatch ){ if( iColumn==XN_ROWID ){ testcase( distinctColumns==0 ); @@ -140198,7 +149025,11 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ pWInfo, nRowEst, nOrderBy, isOrdered ); } - rCost = sqlite3LogEstAdd(rUnsorted, aSortCost[isOrdered]); + /* TUNING: Add a small extra penalty (5) to sorting as an + ** extra encouragment to the query planner to select a plan + ** where the rows emerge in the correct order without any sorting + ** required. */ + rCost = sqlite3LogEstAdd(rUnsorted, aSortCost[isOrdered]) + 5; WHERETRACE(0x002, ("---- sort cost=%-3d (%d/%d) increases cost %3d to %-3d\n", @@ -140388,6 +149219,7 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ pWInfo->eDistinct = WHERE_DISTINCT_ORDERED; } } + pWInfo->bOrderedInnerLoop = 0; if( pWInfo->pOrderBy ){ if( pWInfo->wctrlFlags & WHERE_DISTINCTBY ){ if( pFrom->isOrdered==pWInfo->pOrderBy->nExpr ){ @@ -140499,7 +149331,7 @@ static int whereShortCut(WhereLoopBuilder *pBuilder){ } if( j!=pIdx->nKeyCol ) continue; pLoop->wsFlags = WHERE_COLUMN_EQ|WHERE_ONEROW|WHERE_INDEXED; - if( pIdx->isCovering || (pItem->colUsed & ~columnsInIndex(pIdx))==0 ){ + if( pIdx->isCovering || (pItem->colUsed & pIdx->colNotIdxed)==0 ){ pLoop->wsFlags |= WHERE_IDX_ONLY; } pLoop->nLTerm = j; @@ -140555,6 +149387,28 @@ static int exprIsDeterministic(Expr *p){ return w.eCode; } + +#ifdef WHERETRACE_ENABLED +/* +** Display all WhereLoops in pWInfo +*/ +static void showAllWhereLoops(WhereInfo *pWInfo, WhereClause *pWC){ + if( sqlite3WhereTrace ){ /* Display all of the WhereLoop objects */ + WhereLoop *p; + int i; + static const char zLabel[] = "0123456789abcdefghijklmnopqrstuvwyxz" + "ABCDEFGHIJKLMNOPQRSTUVWYXZ"; + for(p=pWInfo->pLoops, i=0; p; p=p->pNextLoop, i++){ + p->cId = zLabel[i%(sizeof(zLabel)-1)]; + sqlite3WhereLoopPrint(p, pWC); + } + } +} +# define WHERETRACE_ALL_LOOPS(W,C) showAllWhereLoops(W,C) +#else +# define WHERETRACE_ALL_LOOPS(W,C) +#endif + /* ** Generate the beginning of the loop used for WHERE clause processing. ** The return value is a pointer to an opaque structure that contains @@ -140728,7 +149582,7 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( pWInfo->pResultSet = pResultSet; pWInfo->aiCurOnePass[0] = pWInfo->aiCurOnePass[1] = -1; pWInfo->nLevel = nTabList; - pWInfo->iBreak = pWInfo->iContinue = sqlite3VdbeMakeLabel(v); + pWInfo->iBreak = pWInfo->iContinue = sqlite3VdbeMakeLabel(pParse); pWInfo->wctrlFlags = wctrlFlags; pWInfo->iLimit = iAuxArg; pWInfo->savedNQueryLoop = pParse->nQueryLoop; @@ -140836,8 +149690,19 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( sqlite3DebugPrintf(", limit: %d", iAuxArg); } sqlite3DebugPrintf(")\n"); + if( sqlite3WhereTrace & 0x100 ){ + Select sSelect; + memset(&sSelect, 0, sizeof(sSelect)); + sSelect.selFlags = SF_WhereBegin; + sSelect.pSrc = pTabList; + sSelect.pWhere = pWhere; + sSelect.pOrderBy = pOrderBy; + sSelect.pEList = pResultSet; + sqlite3TreeViewSelect(0, &sSelect, 0); + } } if( sqlite3WhereTrace & 0x100 ){ /* Display all terms of the WHERE clause */ + sqlite3DebugPrintf("---- WHERE clause at start of analysis:\n"); sqlite3WhereClausePrint(sWLB.pWC); } #endif @@ -140845,19 +149710,28 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( if( nTabList!=1 || whereShortCut(&sWLB)==0 ){ rc = whereLoopAddAll(&sWLB); if( rc ) goto whereBeginError; - -#ifdef WHERETRACE_ENABLED - if( sqlite3WhereTrace ){ /* Display all of the WhereLoop objects */ - WhereLoop *p; - int i; - static const char zLabel[] = "0123456789abcdefghijklmnopqrstuvwyxz" - "ABCDEFGHIJKLMNOPQRSTUVWYXZ"; - for(p=pWInfo->pLoops, i=0; p; p=p->pNextLoop, i++){ - p->cId = zLabel[i%(sizeof(zLabel)-1)]; - whereLoopPrint(p, sWLB.pWC); - } - } -#endif + +#ifdef SQLITE_ENABLE_STAT4 + /* If one or more WhereTerm.truthProb values were used in estimating + ** loop parameters, but then those truthProb values were subsequently + ** changed based on STAT4 information while computing subsequent loops, + ** then we need to rerun the whole loop building process so that all + ** loops will be built using the revised truthProb values. */ + if( sWLB.bldFlags2 & SQLITE_BLDF2_2NDPASS ){ + WHERETRACE_ALL_LOOPS(pWInfo, sWLB.pWC); + WHERETRACE(0xffff, + ("**** Redo all loop computations due to" + " TERM_HIGHTRUTH changes ****\n")); + while( pWInfo->pLoops ){ + WhereLoop *p = pWInfo->pLoops; + pWInfo->pLoops = p->pNextLoop; + whereLoopDelete(db, p); + } + rc = whereLoopAddAll(&sWLB); + if( rc ) goto whereBeginError; + } +#endif + WHERETRACE_ALL_LOOPS(pWInfo, sWLB.pWC); wherePathSolver(pWInfo, 0); if( db->mallocFailed ) goto whereBeginError; @@ -140894,7 +149768,7 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( } sqlite3DebugPrintf("\n"); for(ii=0; iinLevel; ii++){ - whereLoopPrint(pWInfo->a[ii].pWLoop, sWLB.pWC); + sqlite3WhereLoopPrint(pWInfo->a[ii].pWLoop, sWLB.pWC); } } #endif @@ -140919,14 +149793,14 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( ** then table t2 can be omitted from the following: ** ** SELECT v1, v3 FROM t1 - ** LEFT JOIN t2 USING (t1.ipk=t2.ipk) - ** LEFT JOIN t3 USING (t1.ipk=t3.ipk) + ** LEFT JOIN t2 ON (t1.ipk=t2.ipk) + ** LEFT JOIN t3 ON (t1.ipk=t3.ipk) ** ** or from: ** ** SELECT DISTINCT v1, v3 FROM t1 ** LEFT JOIN t2 - ** LEFT JOIN t3 USING (t1.ipk=t3.ipk) + ** LEFT JOIN t3 ON (t1.ipk=t3.ipk) */ notReady = ~(Bitmask)0; if( pWInfo->nLevel>=2 @@ -140976,7 +149850,13 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( nTabList--; } } +#if defined(WHERETRACE_ENABLED) + if( sqlite3WhereTrace & 0x100 ){ /* Display all terms of the WHERE clause */ + sqlite3DebugPrintf("---- WHERE clause at end of analysis:\n"); + sqlite3WhereClausePrint(sWLB.pWC); + } WHERETRACE(0xffff,("*** Optimizer Finished ***\n")); +#endif pWInfo->pParse->nQueryLoop += pWInfo->nRowOut; /* If the caller is an UPDATE or DELETE statement that is requesting @@ -141002,9 +149882,10 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( if( (wctrlFlags & WHERE_ONEPASS_DESIRED)!=0 ){ int wsFlags = pWInfo->a[0].pWLoop->wsFlags; int bOnerow = (wsFlags & WHERE_ONEROW)!=0; + assert( !(wsFlags & WHERE_VIRTUALTABLE) || IsVirtual(pTabList->a[0].pTab) ); if( bOnerow || ( 0!=(wctrlFlags & WHERE_ONEPASS_MULTIROW) - && 0==(wsFlags & WHERE_VIRTUALTABLE) + && !IsVirtual(pTabList->a[0].pTab) && (0==(wsFlags & WHERE_MULTI_OR) || (wctrlFlags & WHERE_DUPLICATES_OK)) )){ pWInfo->eOnePass = bOnerow ? ONEPASS_SINGLE : ONEPASS_MULTI; @@ -141052,7 +149933,13 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( assert( pTabItem->iCursor==pLevel->iTabCur ); testcase( pWInfo->eOnePass==ONEPASS_OFF && pTab->nCol==BMS-1 ); testcase( pWInfo->eOnePass==ONEPASS_OFF && pTab->nCol==BMS ); - if( pWInfo->eOnePass==ONEPASS_OFF && pTab->nColeOnePass==ONEPASS_OFF + && pTab->nColtabFlags & (TF_HasGenerated|TF_WithoutRowid))==0 + ){ + /* If we know that only a prefix of the record will be used, + ** it is advantageous to reduce the "column count" field in + ** the P4 operand of the OP_OpenRead/Write opcode. */ Bitmask b = pTabItem->colUsed; int n = 0; for(; b; b=b>>1, n++){} @@ -141111,10 +149998,11 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( sqlite3VdbeSetP4KeyInfo(pParse, pIx); if( (pLoop->wsFlags & WHERE_CONSTRAINT)!=0 && (pLoop->wsFlags & (WHERE_COLUMN_RANGE|WHERE_SKIPSCAN))==0 + && (pLoop->wsFlags & WHERE_BIGNULL_SORT)==0 && (pWInfo->wctrlFlags&WHERE_ORDERBY_MIN)==0 && pWInfo->eDistinct!=WHERE_DISTINCT_ORDERED ){ - sqlite3VdbeChangeP5(v, OPFLAG_SEEKEQ); /* Hint to COMDB2 */ + sqlite3VdbeChangeP5(v, OPFLAG_SEEKEQ); } VdbeComment((v, "%s", pIx->zName)); #ifdef SQLITE_ENABLE_COLUMN_USED_MASK @@ -141159,7 +150047,7 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( pParse, pTabList, pLevel, wctrlFlags ); pLevel->addrBody = sqlite3VdbeCurrentAddr(v); - notReady = sqlite3WhereCodeOneLoopStart(pWInfo, ii, notReady); + notReady = sqlite3WhereCodeOneLoopStart(pParse,v,pWInfo,ii,pLevel,notReady); pWInfo->iContinue = pLevel->addrCont; if( (wsFlags&WHERE_MULTI_OR)==0 && (wctrlFlags&WHERE_OR_SUBCLAUSE)==0 ){ sqlite3WhereAddScanStatus(v, pTabList, pLevel, addrExplain); @@ -141179,6 +150067,26 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( return 0; } +/* +** Part of sqlite3WhereEnd() will rewrite opcodes to reference the +** index rather than the main table. In SQLITE_DEBUG mode, we want +** to trace those changes if PRAGMA vdbe_addoptrace=on. This routine +** does that. +*/ +#ifndef SQLITE_DEBUG +# define OpcodeRewriteTrace(D,K,P) /* no-op */ +#else +# define OpcodeRewriteTrace(D,K,P) sqlite3WhereOpcodeRewriteTrace(D,K,P) + static void sqlite3WhereOpcodeRewriteTrace( + sqlite3 *db, + int pc, + VdbeOp *pOp + ){ + if( (db->flags & SQLITE_VdbeAddopTrace)==0 ) return; + sqlite3VdbePrintOp(0, pc, pOp); + } +#endif + /* ** Generate the end of the WHERE loop. See comments on ** sqlite3WhereBegin() for additional information. @@ -141195,7 +150103,6 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ /* Generate loop termination code. */ VdbeModuleComment((v, "End WHERE-core")); - sqlite3ExprCacheClear(pParse); for(i=pWInfo->nLevel-1; i>=0; i--){ int addr; pLevel = &pWInfo->a[i]; @@ -141209,7 +150116,7 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ && i==pWInfo->nLevel-1 /* Ticket [ef9318757b152e3] 2017-10-21 */ && (pLoop->wsFlags & WHERE_INDEXED)!=0 && (pIdx = pLoop->u.btree.pIndex)->hasStat1 - && (n = pLoop->u.btree.nIdxCol)>0 + && (n = pLoop->u.btree.nDistinctCol)>0 && pIdx->aiRowLogEst[n]>=36 ){ int r1 = pParse->nMem+1; @@ -141233,6 +150140,11 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ VdbeCoverageIf(v, pLevel->op==OP_Next); VdbeCoverageIf(v, pLevel->op==OP_Prev); VdbeCoverageIf(v, pLevel->op==OP_VNext); + if( pLevel->regBignull ){ + sqlite3VdbeResolveLabel(v, pLevel->addrBignull); + sqlite3VdbeAddOp2(v, OP_DecrJumpZero, pLevel->regBignull, pLevel->p2-1); + VdbeCoverage(v); + } #ifndef SQLITE_DISABLE_SKIPAHEAD_DISTINCT if( addrSeek ) sqlite3VdbeJumpHere(v, addrSeek); #endif @@ -141246,10 +150158,34 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ for(j=pLevel->u.in.nIn, pIn=&pLevel->u.in.aInLoop[j-1]; j>0; j--, pIn--){ sqlite3VdbeJumpHere(v, pIn->addrInTop+1); if( pIn->eEndLoopOp!=OP_Noop ){ + if( pIn->nPrefix ){ + assert( pLoop->wsFlags & WHERE_IN_EARLYOUT ); + if( pLevel->iLeftJoin ){ + /* For LEFT JOIN queries, cursor pIn->iCur may not have been + ** opened yet. This occurs for WHERE clauses such as + ** "a = ? AND b IN (...)", where the index is on (a, b). If + ** the RHS of the (a=?) is NULL, then the "b IN (...)" may + ** never have been coded, but the body of the loop run to + ** return the null-row. So, if the cursor is not open yet, + ** jump over the OP_Next or OP_Prev instruction about to + ** be coded. */ + sqlite3VdbeAddOp2(v, OP_IfNotOpen, pIn->iCur, + sqlite3VdbeCurrentAddr(v) + 2 + + ((pLoop->wsFlags & WHERE_VIRTUALTABLE)==0) + ); + VdbeCoverage(v); + } + if( (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0 ){ + sqlite3VdbeAddOp4Int(v, OP_IfNoHope, pLevel->iIdxCur, + sqlite3VdbeCurrentAddr(v)+2, + pIn->iBase, pIn->nPrefix); + VdbeCoverage(v); + } + } sqlite3VdbeAddOp2(v, pIn->eEndLoopOp, pIn->iCur, pIn->addrInTop); VdbeCoverage(v); - VdbeCoverageIf(v, pIn->eEndLoopOp==OP_PrevIfOpen); - VdbeCoverageIf(v, pIn->eEndLoopOp==OP_NextIfOpen); + VdbeCoverageIf(v, pIn->eEndLoopOp==OP_Prev); + VdbeCoverageIf(v, pIn->eEndLoopOp==OP_Next); } sqlite3VdbeJumpHere(v, pIn->addrInTop-1); } @@ -141318,6 +150254,29 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ continue; } +#ifdef SQLITE_ENABLE_EARLY_CURSOR_CLOSE + /* Close all of the cursors that were opened by sqlite3WhereBegin. + ** Except, do not close cursors that will be reused by the OR optimization + ** (WHERE_OR_SUBCLAUSE). And do not close the OP_OpenWrite cursors + ** created for the ONEPASS optimization. + */ + if( (pTab->tabFlags & TF_Ephemeral)==0 + && pTab->pSelect==0 + && (pWInfo->wctrlFlags & WHERE_OR_SUBCLAUSE)==0 + ){ + int ws = pLoop->wsFlags; + if( pWInfo->eOnePass==ONEPASS_OFF && (ws & WHERE_IDX_ONLY)==0 ){ + sqlite3VdbeAddOp1(v, OP_Close, pTabItem->iCursor); + } + if( (ws & WHERE_INDEXED)!=0 + && (ws & (WHERE_IPK|WHERE_AUTO_INDEX))==0 + && pLevel->iIdxCur!=pWInfo->aiCurOnePass[1] + ){ + sqlite3VdbeAddOp1(v, OP_Close, pLevel->iIdxCur); + } + } +#endif + /* If this scan uses an index, make VDBE code substitutions to read data ** from the index instead of from the table where possible. In some cases ** this optimization prevents the table from ever being read, which can @@ -141340,6 +150299,11 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ ){ last = sqlite3VdbeCurrentAddr(v); k = pLevel->addrBody; +#ifdef SQLITE_DEBUG + if( db->flags & SQLITE_VdbeAddopTrace ){ + printf("TRANSLATE opcodes in range %d..%d\n", k, last-1); + } +#endif pOp = sqlite3VdbeGetOp(v, k); for(; kp1!=pLevel->iTabCur ) continue; @@ -141354,24 +150318,41 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ Index *pPk = sqlite3PrimaryKeyIndex(pTab); x = pPk->aiColumn[x]; assert( x>=0 ); + }else{ + testcase( x!=sqlite3StorageColumnToTable(pTab,x) ); + x = sqlite3StorageColumnToTable(pTab,x); } - x = sqlite3ColumnOfIndex(pIdx, x); + x = sqlite3TableColumnToIndex(pIdx, x); if( x>=0 ){ pOp->p2 = x; pOp->p1 = pLevel->iIdxCur; + OpcodeRewriteTrace(db, k, pOp); } assert( (pLoop->wsFlags & WHERE_IDX_ONLY)==0 || x>=0 || pWInfo->eOnePass ); }else if( pOp->opcode==OP_Rowid ){ pOp->p1 = pLevel->iIdxCur; pOp->opcode = OP_IdxRowid; + OpcodeRewriteTrace(db, k, pOp); }else if( pOp->opcode==OP_IfNullRow ){ pOp->p1 = pLevel->iIdxCur; + OpcodeRewriteTrace(db, k, pOp); } } +#ifdef SQLITE_DEBUG + if( db->flags & SQLITE_VdbeAddopTrace ) printf("TRANSLATE complete\n"); +#endif } } + /* Undo all Expr node modifications */ + while( pWInfo->pExprMods ){ + WhereExprMod *p = pWInfo->pExprMods; + pWInfo->pExprMods = p->pNext; + memcpy(p->pExpr, &p->orig, sizeof(p->orig)); + sqlite3DbFree(db, p); + } + /* Final cleanup */ pParse->nQueryLoop = pWInfo->savedNQueryLoop; @@ -141380,9 +150361,9 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ } /************** End of where.c ***********************************************/ -/************** Begin file parse.c *******************************************/ +/************** Begin file window.c ******************************************/ /* -** 2000-05-29 +** 2018 May 08 ** ** The author disclaims copyright to this source code. In place of ** a legal notice, here is a blessing: @@ -141392,4080 +150373,3035 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ ** May you share freely, never taking more than you give. ** ************************************************************************* -** Driver template for the LEMON parser generator. -** -** The "lemon" program processes an LALR(1) input grammar file, then uses -** this template to construct a parser. The "lemon" program inserts text -** at each "%%" line. Also, any "P-a-r-s-e" identifer prefix (without the -** interstitial "-" characters) contained in this template is changed into -** the value of the %name directive from the grammar. Otherwise, the content -** of this template is copied straight through into the generate parser -** source file. -** -** The following is the concatenation of all %include directives from the -** input grammar file: */ -/* #include */ -/************ Begin %include sections from the grammar ************************/ - /* #include "sqliteInt.h" */ -/* -** Disable all error recovery processing in the parser push-down -** automaton. -*/ -#define YYNOERRORRECOVERY 1 - -/* -** Make yytestcase() the same as testcase() -*/ -#define yytestcase(X) testcase(X) - -/* -** Indicate that sqlite3ParserFree() will never be called with a null -** pointer. -*/ -#define YYPARSEFREENEVERNULL 1 - -/* -** In the amalgamation, the parse.c file generated by lemon and the -** tokenize.c file are concatenated. In that case, sqlite3RunParser() -** has access to the the size of the yyParser object and so the parser -** engine can be allocated from stack. In that case, only the -** sqlite3ParserInit() and sqlite3ParserFinalize() routines are invoked -** and the sqlite3ParserAlloc() and sqlite3ParserFree() routines can be -** omitted. -*/ -#ifdef SQLITE_AMALGAMATION -# define sqlite3Parser_ENGINEALWAYSONSTACK 1 -#endif - -/* -** Alternative datatype for the argument to the malloc() routine passed -** into sqlite3ParserAlloc(). The default is size_t. -*/ -#define YYMALLOCARGTYPE u64 +#ifndef SQLITE_OMIT_WINDOWFUNC /* -** An instance of the following structure describes the event of a -** TRIGGER. "a" is the event type, one of TK_UPDATE, TK_INSERT, -** TK_DELETE, or TK_INSTEAD. If the event is of the form +** SELECT REWRITING ** -** UPDATE ON (a,b,c) +** Any SELECT statement that contains one or more window functions in +** either the select list or ORDER BY clause (the only two places window +** functions may be used) is transformed by function sqlite3WindowRewrite() +** in order to support window function processing. For example, with the +** schema: ** -** Then the "b" IdList records the list "a,b,c". -*/ -struct TrigEvent { int a; IdList * b; }; - -/* -** Disable lookaside memory allocation for objects that might be -** shared across database connections. -*/ -static void disableLookaside(Parse *pParse){ - pParse->disableLookaside++; - pParse->db->lookaside.bDisable++; -} - - - /* - ** For a compound SELECT statement, make sure p->pPrior->pNext==p for - ** all elements in the list. And make sure list length does not exceed - ** SQLITE_LIMIT_COMPOUND_SELECT. - */ - static void parserDoubleLinkSelect(Parse *pParse, Select *p){ - if( p->pPrior ){ - Select *pNext = 0, *pLoop; - int mxSelect, cnt = 0; - for(pLoop=p; pLoop; pNext=pLoop, pLoop=pLoop->pPrior, cnt++){ - pLoop->pNext = pNext; - pLoop->selFlags |= SF_Compound; - } - if( (p->selFlags & SF_MultiValue)==0 && - (mxSelect = pParse->db->aLimit[SQLITE_LIMIT_COMPOUND_SELECT])>0 && - cnt>mxSelect - ){ - sqlite3ErrorMsg(pParse, "too many terms in compound SELECT"); - } - } - } - - - /* Construct a new Expr object from a single identifier. Use the - ** new Expr to populate pOut. Set the span of pOut to be the identifier - ** that created the expression. - */ - static Expr *tokenExpr(Parse *pParse, int op, Token t){ - Expr *p = sqlite3DbMallocRawNN(pParse->db, sizeof(Expr)+t.n+1); - if( p ){ - memset(p, 0, sizeof(Expr)); - p->op = (u8)op; - p->flags = EP_Leaf; - p->iAgg = -1; - p->u.zToken = (char*)&p[1]; - memcpy(p->u.zToken, t.z, t.n); - p->u.zToken[t.n] = 0; - if( sqlite3Isquote(p->u.zToken[0]) ){ - if( p->u.zToken[0]=='"' ) p->flags |= EP_DblQuoted; - sqlite3Dequote(p->u.zToken); - } -#if SQLITE_MAX_EXPR_DEPTH>0 - p->nHeight = 1; -#endif - } - return p; - } - - /* A routine to convert a binary TK_IS or TK_ISNOT expression into a - ** unary TK_ISNULL or TK_NOTNULL expression. */ - static void binaryToUnaryIfNull(Parse *pParse, Expr *pY, Expr *pA, int op){ - sqlite3 *db = pParse->db; - if( pA && pY && pY->op==TK_NULL ){ - pA->op = (u8)op; - sqlite3ExprDelete(db, pA->pRight); - pA->pRight = 0; - } - } - - /* Add a single new term to an ExprList that is used to store a - ** list of identifiers. Report an error if the ID list contains - ** a COLLATE clause or an ASC or DESC keyword, except ignore the - ** error while parsing a legacy schema. - */ - static ExprList *parserAddExprIdListTerm( - Parse *pParse, - ExprList *pPrior, - Token *pIdToken, - int hasCollate, - int sortOrder - ){ - ExprList *p = sqlite3ExprListAppend(pParse, pPrior, 0); - if( (hasCollate || sortOrder!=SQLITE_SO_UNDEFINED) - && pParse->db->init.busy==0 - ){ - sqlite3ErrorMsg(pParse, "syntax error after column name \"%.*s\"", - pIdToken->n, pIdToken->z); - } - sqlite3ExprListSetName(pParse, p, pIdToken, 1); - return p; - } -/**************** End of %include directives **********************************/ -/* These constants specify the various numeric values for terminal symbols -** in a format understandable to "makeheaders". This section is blank unless -** "lemon" is run with the "-m" command-line option. -***************** Begin makeheaders token definitions *************************/ -/**************** End makeheaders token definitions ***************************/ - -/* The next sections is a series of control #defines. -** various aspects of the generated parser. -** YYCODETYPE is the data type used to store the integer codes -** that represent terminal and non-terminal symbols. -** "unsigned char" is used if there are fewer than -** 256 symbols. Larger types otherwise. -** YYNOCODE is a number of type YYCODETYPE that is not used for -** any terminal or nonterminal symbol. -** YYFALLBACK If defined, this indicates that one or more tokens -** (also known as: "terminal symbols") have fall-back -** values which should be used if the original symbol -** would not parse. This permits keywords to sometimes -** be used as identifiers, for example. -** YYACTIONTYPE is the data type used for "action codes" - numbers -** that indicate what to do in response to the next -** token. -** sqlite3ParserTOKENTYPE is the data type used for minor type for terminal -** symbols. Background: A "minor type" is a semantic -** value associated with a terminal or non-terminal -** symbols. For example, for an "ID" terminal symbol, -** the minor type might be the name of the identifier. -** Each non-terminal can have a different minor type. -** Terminal symbols all have the same minor type, though. -** This macros defines the minor type for terminal -** symbols. -** YYMINORTYPE is the data type used for all minor types. -** This is typically a union of many types, one of -** which is sqlite3ParserTOKENTYPE. The entry in the union -** for terminal symbols is called "yy0". -** YYSTACKDEPTH is the maximum depth of the parser's stack. If -** zero the stack is dynamically sized using realloc() -** sqlite3ParserARG_SDECL A static variable declaration for the %extra_argument -** sqlite3ParserARG_PDECL A parameter declaration for the %extra_argument -** sqlite3ParserARG_PARAM Code to pass %extra_argument as a subroutine parameter -** sqlite3ParserARG_STORE Code to store %extra_argument into yypParser -** sqlite3ParserARG_FETCH Code to extract %extra_argument from yypParser -** sqlite3ParserCTX_* As sqlite3ParserARG_ except for %extra_context -** YYERRORSYMBOL is the code number of the error symbol. If not -** defined, then do no error processing. -** YYNSTATE the combined number of states. -** YYNRULE the number of rules in the grammar -** YYNTOKEN Number of terminal symbols -** YY_MAX_SHIFT Maximum value for shift actions -** YY_MIN_SHIFTREDUCE Minimum value for shift-reduce actions -** YY_MAX_SHIFTREDUCE Maximum value for shift-reduce actions -** YY_ERROR_ACTION The yy_action[] code for syntax error -** YY_ACCEPT_ACTION The yy_action[] code for accept -** YY_NO_ACTION The yy_action[] code for no-op -** YY_MIN_REDUCE Minimum value for reduce actions -** YY_MAX_REDUCE Maximum value for reduce actions -*/ -#ifndef INTERFACE -# define INTERFACE 1 -#endif -/************* Begin control #defines *****************************************/ -#define YYCODETYPE unsigned char -#define YYNOCODE 255 -#define YYACTIONTYPE unsigned short int -#define YYWILDCARD 84 -#define sqlite3ParserTOKENTYPE Token -typedef union { - int yyinit; - sqlite3ParserTOKENTYPE yy0; - const char* yy36; - TriggerStep* yy47; - With* yy91; - struct {int value; int mask;} yy107; - Expr* yy182; - Upsert* yy198; - ExprList* yy232; - struct TrigEvent yy300; - Select* yy399; - SrcList* yy427; - int yy502; - IdList* yy510; -} YYMINORTYPE; -#ifndef YYSTACKDEPTH -#define YYSTACKDEPTH 100 -#endif -#define sqlite3ParserARG_SDECL -#define sqlite3ParserARG_PDECL -#define sqlite3ParserARG_PARAM -#define sqlite3ParserARG_FETCH -#define sqlite3ParserARG_STORE -#define sqlite3ParserCTX_SDECL Parse *pParse; -#define sqlite3ParserCTX_PDECL ,Parse *pParse -#define sqlite3ParserCTX_PARAM ,pParse -#define sqlite3ParserCTX_FETCH Parse *pParse=yypParser->pParse; -#define sqlite3ParserCTX_STORE yypParser->pParse=pParse; -#define YYFALLBACK 1 -#define YYNSTATE 490 -#define YYNRULE 341 -#define YYNTOKEN 145 -#define YY_MAX_SHIFT 489 -#define YY_MIN_SHIFTREDUCE 705 -#define YY_MAX_SHIFTREDUCE 1045 -#define YY_ERROR_ACTION 1046 -#define YY_ACCEPT_ACTION 1047 -#define YY_NO_ACTION 1048 -#define YY_MIN_REDUCE 1049 -#define YY_MAX_REDUCE 1389 -/************* End control #defines *******************************************/ - -/* Define the yytestcase() macro to be a no-op if is not already defined -** otherwise. +** CREATE TABLE t1(a, b, c, d, e, f, g); ** -** Applications can choose to define yytestcase() in the %include section -** to a macro that can assist in verifying code coverage. For production -** code the yytestcase() macro should be turned off. But it is useful -** for testing. -*/ -#ifndef yytestcase -# define yytestcase(X) -#endif - - -/* Next are the tables used to determine what action to take based on the -** current state and lookahead token. These tables are used to implement -** functions that take a state number and lookahead value and return an -** action integer. +** the statement: ** -** Suppose the action integer is N. Then the action is determined as -** follows +** SELECT a+1, max(b) OVER (PARTITION BY c ORDER BY d) FROM t1 ORDER BY e; ** -** 0 <= N <= YY_MAX_SHIFT Shift N. That is, push the lookahead -** token onto the stack and goto state N. +** is transformed to: ** -** N between YY_MIN_SHIFTREDUCE Shift to an arbitrary state then -** and YY_MAX_SHIFTREDUCE reduce by rule N-YY_MIN_SHIFTREDUCE. +** SELECT a+1, max(b) OVER (PARTITION BY c ORDER BY d) FROM ( +** SELECT a, e, c, d, b FROM t1 ORDER BY c, d +** ) ORDER BY e; ** -** N == YY_ERROR_ACTION A syntax error has occurred. +** The flattening optimization is disabled when processing this transformed +** SELECT statement. This allows the implementation of the window function +** (in this case max()) to process rows sorted in order of (c, d), which +** makes things easier for obvious reasons. More generally: ** -** N == YY_ACCEPT_ACTION The parser accepts its input. +** * FROM, WHERE, GROUP BY and HAVING clauses are all moved to +** the sub-query. ** -** N == YY_NO_ACTION No such action. Denotes unused -** slots in the yy_action[] table. +** * ORDER BY, LIMIT and OFFSET remain part of the parent query. ** -** N between YY_MIN_REDUCE Reduce by rule N-YY_MIN_REDUCE -** and YY_MAX_REDUCE +** * Terminals from each of the expression trees that make up the +** select-list and ORDER BY expressions in the parent query are +** selected by the sub-query. For the purposes of the transformation, +** terminals are column references and aggregate functions. ** -** The action table is constructed as a single large table named yy_action[]. -** Given state S and lookahead X, the action is computed as either: +** If there is more than one window function in the SELECT that uses +** the same window declaration (the OVER bit), then a single scan may +** be used to process more than one window function. For example: ** -** (A) N = yy_action[ yy_shift_ofst[S] + X ] -** (B) N = yy_default[S] +** SELECT max(b) OVER (PARTITION BY c ORDER BY d), +** min(e) OVER (PARTITION BY c ORDER BY d) +** FROM t1; ** -** The (A) formula is preferred. The B formula is used instead if -** yy_lookahead[yy_shift_ofst[S]+X] is not equal to X. +** is transformed in the same way as the example above. However: ** -** The formulas above are for computing the action when the lookahead is -** a terminal symbol. If the lookahead is a non-terminal (as occurs after -** a reduce action) then the yy_reduce_ofst[] array is used in place of -** the yy_shift_ofst[] array. +** SELECT max(b) OVER (PARTITION BY c ORDER BY d), +** min(e) OVER (PARTITION BY a ORDER BY b) +** FROM t1; ** -** The following are the tables generated in this section: +** Must be transformed to: ** -** yy_action[] A single table containing all actions. -** yy_lookahead[] A table containing the lookahead for each entry in -** yy_action. Used to detect hash collisions. -** yy_shift_ofst[] For each state, the offset into yy_action for -** shifting terminals. -** yy_reduce_ofst[] For each state, the offset into yy_action for -** shifting non-terminals after a reduce. -** yy_default[] Default action for each state. +** SELECT max(b) OVER (PARTITION BY c ORDER BY d) FROM ( +** SELECT e, min(e) OVER (PARTITION BY a ORDER BY b), c, d, b FROM +** SELECT a, e, c, d, b FROM t1 ORDER BY a, b +** ) ORDER BY c, d +** ) ORDER BY e; ** -*********** Begin parsing tables **********************************************/ -#define YY_ACTTAB_COUNT (1657) -static const YYACTIONTYPE yy_action[] = { - /* 0 */ 349, 99, 96, 185, 99, 96, 185, 233, 1047, 1, - /* 10 */ 1, 489, 2, 1051, 484, 477, 477, 477, 260, 351, - /* 20 */ 121, 1310, 1120, 1120, 1178, 1115, 1094, 1128, 380, 380, - /* 30 */ 380, 835, 454, 410, 1115, 59, 59, 1357, 425, 836, - /* 40 */ 710, 711, 712, 106, 107, 97, 1023, 1023, 900, 903, - /* 50 */ 892, 892, 104, 104, 105, 105, 105, 105, 346, 238, - /* 60 */ 238, 99, 96, 185, 238, 238, 889, 889, 901, 904, - /* 70 */ 460, 481, 351, 99, 96, 185, 481, 347, 1177, 82, - /* 80 */ 388, 214, 182, 23, 194, 103, 103, 103, 103, 102, - /* 90 */ 102, 101, 101, 101, 100, 381, 106, 107, 97, 1023, - /* 100 */ 1023, 900, 903, 892, 892, 104, 104, 105, 105, 105, - /* 110 */ 105, 10, 385, 484, 24, 484, 1333, 489, 2, 1051, - /* 120 */ 335, 1043, 108, 893, 260, 351, 121, 99, 96, 185, - /* 130 */ 100, 381, 386, 1128, 59, 59, 59, 59, 103, 103, - /* 140 */ 103, 103, 102, 102, 101, 101, 101, 100, 381, 106, - /* 150 */ 107, 97, 1023, 1023, 900, 903, 892, 892, 104, 104, - /* 160 */ 105, 105, 105, 105, 360, 238, 238, 170, 170, 467, - /* 170 */ 455, 467, 464, 67, 381, 329, 169, 481, 351, 343, - /* 180 */ 338, 400, 1044, 68, 101, 101, 101, 100, 381, 393, - /* 190 */ 194, 103, 103, 103, 103, 102, 102, 101, 101, 101, - /* 200 */ 100, 381, 106, 107, 97, 1023, 1023, 900, 903, 892, - /* 210 */ 892, 104, 104, 105, 105, 105, 105, 483, 385, 103, - /* 220 */ 103, 103, 103, 102, 102, 101, 101, 101, 100, 381, - /* 230 */ 268, 351, 946, 946, 422, 296, 102, 102, 101, 101, - /* 240 */ 101, 100, 381, 861, 103, 103, 103, 103, 102, 102, - /* 250 */ 101, 101, 101, 100, 381, 106, 107, 97, 1023, 1023, - /* 260 */ 900, 903, 892, 892, 104, 104, 105, 105, 105, 105, - /* 270 */ 484, 983, 1383, 206, 1353, 1383, 438, 435, 434, 281, - /* 280 */ 396, 269, 1089, 941, 351, 1002, 433, 861, 743, 401, - /* 290 */ 282, 57, 57, 482, 145, 791, 791, 103, 103, 103, - /* 300 */ 103, 102, 102, 101, 101, 101, 100, 381, 106, 107, - /* 310 */ 97, 1023, 1023, 900, 903, 892, 892, 104, 104, 105, - /* 320 */ 105, 105, 105, 281, 1002, 1003, 1004, 206, 879, 319, - /* 330 */ 438, 435, 434, 981, 259, 474, 360, 351, 1118, 1118, - /* 340 */ 433, 736, 379, 378, 872, 1002, 1356, 322, 871, 766, - /* 350 */ 103, 103, 103, 103, 102, 102, 101, 101, 101, 100, - /* 360 */ 381, 106, 107, 97, 1023, 1023, 900, 903, 892, 892, - /* 370 */ 104, 104, 105, 105, 105, 105, 484, 801, 484, 871, - /* 380 */ 871, 873, 401, 282, 1002, 1003, 1004, 1030, 360, 1030, - /* 390 */ 351, 983, 1384, 213, 880, 1384, 145, 59, 59, 59, - /* 400 */ 59, 1002, 244, 103, 103, 103, 103, 102, 102, 101, - /* 410 */ 101, 101, 100, 381, 106, 107, 97, 1023, 1023, 900, - /* 420 */ 903, 892, 892, 104, 104, 105, 105, 105, 105, 274, - /* 430 */ 484, 110, 467, 479, 467, 444, 259, 474, 232, 232, - /* 440 */ 1002, 1003, 1004, 351, 210, 335, 982, 866, 1385, 336, - /* 450 */ 481, 59, 59, 981, 245, 307, 103, 103, 103, 103, - /* 460 */ 102, 102, 101, 101, 101, 100, 381, 106, 107, 97, - /* 470 */ 1023, 1023, 900, 903, 892, 892, 104, 104, 105, 105, - /* 480 */ 105, 105, 453, 459, 484, 408, 377, 259, 474, 271, - /* 490 */ 183, 273, 209, 208, 207, 356, 351, 307, 178, 177, - /* 500 */ 127, 1006, 1098, 14, 14, 43, 43, 1044, 425, 103, - /* 510 */ 103, 103, 103, 102, 102, 101, 101, 101, 100, 381, - /* 520 */ 106, 107, 97, 1023, 1023, 900, 903, 892, 892, 104, - /* 530 */ 104, 105, 105, 105, 105, 294, 1132, 408, 160, 484, - /* 540 */ 408, 1006, 129, 962, 1209, 239, 239, 481, 307, 425, - /* 550 */ 1309, 1097, 351, 235, 243, 272, 820, 481, 963, 425, - /* 560 */ 11, 11, 103, 103, 103, 103, 102, 102, 101, 101, - /* 570 */ 101, 100, 381, 964, 362, 1002, 106, 107, 97, 1023, - /* 580 */ 1023, 900, 903, 892, 892, 104, 104, 105, 105, 105, - /* 590 */ 105, 1275, 161, 126, 777, 289, 1209, 292, 1072, 357, - /* 600 */ 1209, 1127, 476, 357, 778, 425, 247, 425, 351, 248, - /* 610 */ 414, 364, 414, 171, 1002, 1003, 1004, 84, 103, 103, - /* 620 */ 103, 103, 102, 102, 101, 101, 101, 100, 381, 1002, - /* 630 */ 184, 484, 106, 107, 97, 1023, 1023, 900, 903, 892, - /* 640 */ 892, 104, 104, 105, 105, 105, 105, 1123, 1209, 287, - /* 650 */ 484, 1209, 11, 11, 179, 820, 259, 474, 307, 237, - /* 660 */ 182, 351, 321, 365, 414, 308, 367, 366, 1002, 1003, - /* 670 */ 1004, 44, 44, 87, 103, 103, 103, 103, 102, 102, - /* 680 */ 101, 101, 101, 100, 381, 106, 107, 97, 1023, 1023, - /* 690 */ 900, 903, 892, 892, 104, 104, 105, 105, 105, 105, - /* 700 */ 246, 368, 280, 128, 10, 358, 146, 796, 835, 258, - /* 710 */ 1020, 88, 795, 86, 351, 421, 836, 943, 376, 348, - /* 720 */ 191, 943, 1318, 267, 308, 279, 456, 103, 103, 103, - /* 730 */ 103, 102, 102, 101, 101, 101, 100, 381, 106, 95, - /* 740 */ 97, 1023, 1023, 900, 903, 892, 892, 104, 104, 105, - /* 750 */ 105, 105, 105, 420, 249, 238, 238, 238, 238, 79, - /* 760 */ 375, 125, 305, 29, 262, 978, 351, 481, 337, 481, - /* 770 */ 756, 755, 304, 278, 415, 15, 81, 940, 1126, 940, - /* 780 */ 103, 103, 103, 103, 102, 102, 101, 101, 101, 100, - /* 790 */ 381, 107, 97, 1023, 1023, 900, 903, 892, 892, 104, - /* 800 */ 104, 105, 105, 105, 105, 457, 263, 484, 174, 484, - /* 810 */ 238, 238, 863, 407, 402, 216, 216, 351, 409, 193, - /* 820 */ 283, 216, 481, 81, 763, 764, 266, 5, 13, 13, - /* 830 */ 34, 34, 103, 103, 103, 103, 102, 102, 101, 101, - /* 840 */ 101, 100, 381, 97, 1023, 1023, 900, 903, 892, 892, - /* 850 */ 104, 104, 105, 105, 105, 105, 93, 475, 1002, 4, - /* 860 */ 403, 1002, 340, 431, 1002, 297, 212, 1277, 81, 746, - /* 870 */ 1163, 152, 926, 478, 166, 212, 757, 829, 930, 939, - /* 880 */ 216, 939, 858, 103, 103, 103, 103, 102, 102, 101, - /* 890 */ 101, 101, 100, 381, 238, 238, 382, 1002, 1003, 1004, - /* 900 */ 1002, 1003, 1004, 1002, 1003, 1004, 481, 439, 472, 746, - /* 910 */ 105, 105, 105, 105, 98, 758, 1162, 145, 930, 412, - /* 920 */ 879, 406, 793, 81, 395, 89, 90, 91, 105, 105, - /* 930 */ 105, 105, 1323, 92, 484, 382, 486, 485, 240, 275, - /* 940 */ 871, 103, 103, 103, 103, 102, 102, 101, 101, 101, - /* 950 */ 100, 381, 1096, 371, 355, 45, 45, 259, 474, 103, - /* 960 */ 103, 103, 103, 102, 102, 101, 101, 101, 100, 381, - /* 970 */ 1150, 871, 871, 873, 874, 21, 1332, 991, 384, 730, - /* 980 */ 722, 242, 123, 1298, 124, 875, 333, 333, 332, 227, - /* 990 */ 330, 991, 384, 719, 256, 242, 484, 391, 413, 1297, - /* 1000 */ 333, 333, 332, 227, 330, 748, 187, 719, 265, 470, - /* 1010 */ 1279, 1002, 484, 417, 391, 390, 264, 11, 11, 284, - /* 1020 */ 187, 732, 265, 93, 475, 875, 4, 1279, 1281, 419, - /* 1030 */ 264, 369, 416, 11, 11, 1159, 288, 484, 399, 1346, - /* 1040 */ 478, 379, 378, 291, 484, 293, 189, 250, 295, 1027, - /* 1050 */ 1002, 1003, 1004, 190, 1029, 1111, 140, 188, 11, 11, - /* 1060 */ 189, 732, 1028, 382, 923, 46, 46, 190, 1095, 230, - /* 1070 */ 140, 188, 462, 93, 475, 472, 4, 300, 309, 391, - /* 1080 */ 373, 6, 1069, 217, 739, 310, 1030, 879, 1030, 1171, - /* 1090 */ 478, 352, 1279, 90, 91, 800, 259, 474, 1208, 484, - /* 1100 */ 92, 1268, 382, 486, 485, 352, 1002, 871, 879, 426, - /* 1110 */ 259, 474, 172, 382, 238, 238, 1146, 170, 1021, 389, - /* 1120 */ 47, 47, 1157, 739, 872, 472, 481, 469, 871, 350, - /* 1130 */ 1214, 83, 475, 389, 4, 1078, 1071, 879, 871, 871, - /* 1140 */ 873, 874, 21, 90, 91, 1002, 1003, 1004, 478, 251, - /* 1150 */ 92, 251, 382, 486, 485, 443, 370, 871, 1021, 871, - /* 1160 */ 871, 873, 224, 241, 306, 441, 301, 440, 211, 1060, - /* 1170 */ 820, 382, 822, 447, 299, 1059, 484, 1061, 1143, 962, - /* 1180 */ 430, 796, 484, 472, 1340, 312, 795, 465, 871, 871, - /* 1190 */ 873, 874, 21, 314, 963, 879, 316, 59, 59, 1002, - /* 1200 */ 9, 90, 91, 48, 48, 238, 238, 210, 92, 964, - /* 1210 */ 382, 486, 485, 176, 334, 871, 242, 481, 1193, 238, - /* 1220 */ 238, 333, 333, 332, 227, 330, 394, 270, 719, 277, - /* 1230 */ 471, 481, 467, 466, 484, 145, 217, 1201, 1002, 1003, - /* 1240 */ 1004, 187, 3, 265, 184, 445, 871, 871, 873, 874, - /* 1250 */ 21, 264, 1337, 450, 1051, 39, 39, 392, 356, 260, - /* 1260 */ 342, 121, 468, 411, 436, 821, 180, 1094, 1128, 820, - /* 1270 */ 303, 1021, 1272, 1271, 299, 259, 474, 238, 238, 1002, - /* 1280 */ 473, 189, 484, 318, 327, 238, 238, 484, 190, 481, - /* 1290 */ 446, 140, 188, 1343, 238, 238, 1038, 481, 148, 175, - /* 1300 */ 238, 238, 484, 49, 49, 219, 481, 484, 35, 35, - /* 1310 */ 1317, 1021, 481, 484, 1035, 484, 1315, 484, 1002, 1003, - /* 1320 */ 1004, 484, 66, 36, 36, 194, 352, 484, 38, 38, - /* 1330 */ 484, 259, 474, 69, 50, 50, 51, 51, 52, 52, - /* 1340 */ 359, 484, 12, 12, 484, 1198, 484, 158, 53, 53, - /* 1350 */ 405, 112, 112, 385, 389, 484, 26, 484, 143, 484, - /* 1360 */ 150, 484, 54, 54, 397, 40, 40, 55, 55, 484, - /* 1370 */ 79, 484, 153, 1190, 484, 154, 56, 56, 41, 41, - /* 1380 */ 58, 58, 133, 133, 484, 398, 484, 429, 484, 155, - /* 1390 */ 134, 134, 135, 135, 484, 63, 63, 484, 341, 484, - /* 1400 */ 339, 484, 196, 484, 156, 42, 42, 113, 113, 60, - /* 1410 */ 60, 484, 404, 484, 27, 114, 114, 1204, 115, 115, - /* 1420 */ 111, 111, 132, 132, 131, 131, 1266, 418, 484, 162, - /* 1430 */ 484, 200, 119, 119, 118, 118, 484, 74, 424, 484, - /* 1440 */ 1286, 484, 231, 484, 202, 484, 167, 286, 427, 116, - /* 1450 */ 116, 117, 117, 290, 203, 442, 1062, 62, 62, 204, - /* 1460 */ 64, 64, 61, 61, 33, 33, 37, 37, 344, 372, - /* 1470 */ 1114, 1105, 748, 1113, 374, 1112, 254, 458, 1086, 255, - /* 1480 */ 345, 1085, 302, 1084, 1355, 78, 1154, 311, 1104, 449, - /* 1490 */ 452, 1155, 1153, 218, 7, 313, 315, 320, 1152, 85, - /* 1500 */ 1252, 317, 109, 80, 463, 225, 461, 1068, 25, 487, - /* 1510 */ 997, 323, 257, 226, 229, 228, 1136, 324, 325, 326, - /* 1520 */ 488, 136, 1057, 1052, 1302, 1303, 1301, 706, 1300, 137, - /* 1530 */ 122, 138, 383, 173, 1082, 261, 186, 252, 1081, 65, - /* 1540 */ 387, 120, 938, 936, 855, 353, 149, 1079, 139, 151, - /* 1550 */ 192, 780, 195, 276, 952, 157, 141, 361, 70, 363, - /* 1560 */ 859, 159, 71, 72, 142, 73, 955, 354, 147, 197, - /* 1570 */ 198, 951, 130, 16, 199, 285, 216, 1032, 201, 423, - /* 1580 */ 164, 944, 163, 28, 721, 428, 304, 165, 205, 759, - /* 1590 */ 75, 432, 298, 17, 18, 437, 76, 253, 878, 144, - /* 1600 */ 877, 906, 77, 986, 30, 448, 987, 31, 451, 181, - /* 1610 */ 234, 236, 168, 828, 823, 89, 910, 921, 81, 907, - /* 1620 */ 215, 905, 909, 961, 960, 19, 221, 20, 220, 22, - /* 1630 */ 32, 331, 876, 731, 94, 790, 794, 8, 992, 222, - /* 1640 */ 480, 328, 1048, 1048, 1048, 1048, 1048, 1048, 1048, 1048, - /* 1650 */ 223, 1048, 1048, 1048, 1048, 1348, 1347, -}; -static const YYCODETYPE yy_lookahead[] = { - /* 0 */ 174, 226, 227, 228, 226, 227, 228, 172, 145, 146, - /* 10 */ 147, 148, 149, 150, 153, 169, 170, 171, 155, 19, - /* 20 */ 157, 246, 192, 193, 177, 181, 182, 164, 169, 170, - /* 30 */ 171, 31, 164, 153, 190, 174, 175, 187, 153, 39, - /* 40 */ 7, 8, 9, 43, 44, 45, 46, 47, 48, 49, - /* 50 */ 50, 51, 52, 53, 54, 55, 56, 57, 174, 196, - /* 60 */ 197, 226, 227, 228, 196, 197, 46, 47, 48, 49, - /* 70 */ 209, 208, 19, 226, 227, 228, 208, 174, 177, 26, - /* 80 */ 195, 213, 214, 22, 221, 85, 86, 87, 88, 89, - /* 90 */ 90, 91, 92, 93, 94, 95, 43, 44, 45, 46, - /* 100 */ 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, - /* 110 */ 57, 172, 249, 153, 53, 153, 147, 148, 149, 150, - /* 120 */ 22, 23, 69, 103, 155, 19, 157, 226, 227, 228, - /* 130 */ 94, 95, 247, 164, 174, 175, 174, 175, 85, 86, - /* 140 */ 87, 88, 89, 90, 91, 92, 93, 94, 95, 43, - /* 150 */ 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, - /* 160 */ 54, 55, 56, 57, 153, 196, 197, 153, 153, 209, - /* 170 */ 210, 209, 210, 67, 95, 161, 237, 208, 19, 165, - /* 180 */ 165, 242, 84, 24, 91, 92, 93, 94, 95, 223, - /* 190 */ 221, 85, 86, 87, 88, 89, 90, 91, 92, 93, - /* 200 */ 94, 95, 43, 44, 45, 46, 47, 48, 49, 50, - /* 210 */ 51, 52, 53, 54, 55, 56, 57, 153, 249, 85, - /* 220 */ 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, - /* 230 */ 219, 19, 109, 110, 111, 23, 89, 90, 91, 92, - /* 240 */ 93, 94, 95, 73, 85, 86, 87, 88, 89, 90, - /* 250 */ 91, 92, 93, 94, 95, 43, 44, 45, 46, 47, - /* 260 */ 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, - /* 270 */ 153, 22, 23, 101, 173, 26, 104, 105, 106, 109, - /* 280 */ 110, 111, 181, 11, 19, 59, 114, 73, 23, 110, - /* 290 */ 111, 174, 175, 116, 80, 118, 119, 85, 86, 87, - /* 300 */ 88, 89, 90, 91, 92, 93, 94, 95, 43, 44, - /* 310 */ 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, - /* 320 */ 55, 56, 57, 109, 98, 99, 100, 101, 83, 153, - /* 330 */ 104, 105, 106, 84, 120, 121, 153, 19, 192, 193, - /* 340 */ 114, 23, 89, 90, 99, 59, 23, 230, 103, 26, - /* 350 */ 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, - /* 360 */ 95, 43, 44, 45, 46, 47, 48, 49, 50, 51, - /* 370 */ 52, 53, 54, 55, 56, 57, 153, 91, 153, 134, - /* 380 */ 135, 136, 110, 111, 98, 99, 100, 134, 153, 136, - /* 390 */ 19, 22, 23, 26, 23, 26, 80, 174, 175, 174, - /* 400 */ 175, 59, 219, 85, 86, 87, 88, 89, 90, 91, - /* 410 */ 92, 93, 94, 95, 43, 44, 45, 46, 47, 48, - /* 420 */ 49, 50, 51, 52, 53, 54, 55, 56, 57, 16, - /* 430 */ 153, 22, 209, 210, 209, 210, 120, 121, 196, 197, - /* 440 */ 98, 99, 100, 19, 46, 22, 23, 23, 252, 253, - /* 450 */ 208, 174, 175, 84, 219, 153, 85, 86, 87, 88, - /* 460 */ 89, 90, 91, 92, 93, 94, 95, 43, 44, 45, - /* 470 */ 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, - /* 480 */ 56, 57, 153, 153, 153, 153, 209, 120, 121, 76, - /* 490 */ 153, 78, 109, 110, 111, 97, 19, 153, 89, 90, - /* 500 */ 198, 59, 183, 174, 175, 174, 175, 84, 153, 85, - /* 510 */ 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, - /* 520 */ 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, - /* 530 */ 53, 54, 55, 56, 57, 16, 197, 153, 22, 153, - /* 540 */ 153, 99, 198, 12, 153, 196, 197, 208, 153, 153, - /* 550 */ 195, 183, 19, 23, 222, 142, 26, 208, 27, 153, - /* 560 */ 174, 175, 85, 86, 87, 88, 89, 90, 91, 92, - /* 570 */ 93, 94, 95, 42, 188, 59, 43, 44, 45, 46, - /* 580 */ 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, - /* 590 */ 57, 195, 22, 198, 63, 76, 153, 78, 167, 168, - /* 600 */ 153, 195, 167, 168, 73, 153, 222, 153, 19, 222, - /* 610 */ 153, 220, 153, 24, 98, 99, 100, 140, 85, 86, - /* 620 */ 87, 88, 89, 90, 91, 92, 93, 94, 95, 59, - /* 630 */ 100, 153, 43, 44, 45, 46, 47, 48, 49, 50, - /* 640 */ 51, 52, 53, 54, 55, 56, 57, 195, 153, 195, - /* 650 */ 153, 153, 174, 175, 26, 125, 120, 121, 153, 213, - /* 660 */ 214, 19, 153, 220, 153, 153, 188, 220, 98, 99, - /* 670 */ 100, 174, 175, 140, 85, 86, 87, 88, 89, 90, - /* 680 */ 91, 92, 93, 94, 95, 43, 44, 45, 46, 47, - /* 690 */ 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, - /* 700 */ 243, 189, 243, 198, 172, 250, 251, 117, 31, 201, - /* 710 */ 26, 139, 122, 141, 19, 220, 39, 29, 220, 211, - /* 720 */ 24, 33, 153, 164, 153, 164, 19, 85, 86, 87, - /* 730 */ 88, 89, 90, 91, 92, 93, 94, 95, 43, 44, - /* 740 */ 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, - /* 750 */ 55, 56, 57, 65, 243, 196, 197, 196, 197, 131, - /* 760 */ 189, 22, 103, 24, 153, 23, 19, 208, 26, 208, - /* 770 */ 102, 103, 113, 23, 242, 22, 26, 134, 164, 136, - /* 780 */ 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, - /* 790 */ 95, 44, 45, 46, 47, 48, 49, 50, 51, 52, - /* 800 */ 53, 54, 55, 56, 57, 98, 153, 153, 124, 153, - /* 810 */ 196, 197, 23, 23, 61, 26, 26, 19, 23, 123, - /* 820 */ 23, 26, 208, 26, 7, 8, 153, 22, 174, 175, - /* 830 */ 174, 175, 85, 86, 87, 88, 89, 90, 91, 92, - /* 840 */ 93, 94, 95, 45, 46, 47, 48, 49, 50, 51, - /* 850 */ 52, 53, 54, 55, 56, 57, 19, 20, 59, 22, - /* 860 */ 111, 59, 164, 23, 59, 23, 26, 153, 26, 59, - /* 870 */ 153, 72, 23, 36, 72, 26, 35, 23, 59, 134, - /* 880 */ 26, 136, 133, 85, 86, 87, 88, 89, 90, 91, - /* 890 */ 92, 93, 94, 95, 196, 197, 59, 98, 99, 100, - /* 900 */ 98, 99, 100, 98, 99, 100, 208, 66, 71, 99, - /* 910 */ 54, 55, 56, 57, 58, 74, 153, 80, 99, 19, - /* 920 */ 83, 223, 23, 26, 153, 26, 89, 90, 54, 55, - /* 930 */ 56, 57, 153, 96, 153, 98, 99, 100, 22, 153, - /* 940 */ 103, 85, 86, 87, 88, 89, 90, 91, 92, 93, - /* 950 */ 94, 95, 183, 112, 158, 174, 175, 120, 121, 85, - /* 960 */ 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, - /* 970 */ 215, 134, 135, 136, 137, 138, 0, 1, 2, 23, - /* 980 */ 21, 5, 26, 153, 22, 59, 10, 11, 12, 13, - /* 990 */ 14, 1, 2, 17, 212, 5, 153, 153, 98, 153, - /* 1000 */ 10, 11, 12, 13, 14, 108, 30, 17, 32, 193, - /* 1010 */ 153, 59, 153, 153, 170, 171, 40, 174, 175, 153, - /* 1020 */ 30, 59, 32, 19, 20, 99, 22, 170, 171, 233, - /* 1030 */ 40, 188, 236, 174, 175, 153, 153, 153, 79, 123, - /* 1040 */ 36, 89, 90, 153, 153, 153, 70, 188, 153, 97, - /* 1050 */ 98, 99, 100, 77, 102, 153, 80, 81, 174, 175, - /* 1060 */ 70, 99, 110, 59, 105, 174, 175, 77, 153, 238, - /* 1070 */ 80, 81, 188, 19, 20, 71, 22, 153, 153, 235, - /* 1080 */ 19, 22, 164, 24, 59, 153, 134, 83, 136, 153, - /* 1090 */ 36, 115, 235, 89, 90, 91, 120, 121, 153, 153, - /* 1100 */ 96, 142, 98, 99, 100, 115, 59, 103, 83, 239, - /* 1110 */ 120, 121, 199, 59, 196, 197, 153, 153, 59, 143, - /* 1120 */ 174, 175, 153, 98, 99, 71, 208, 153, 103, 165, - /* 1130 */ 153, 19, 20, 143, 22, 153, 153, 83, 134, 135, - /* 1140 */ 136, 137, 138, 89, 90, 98, 99, 100, 36, 185, - /* 1150 */ 96, 187, 98, 99, 100, 91, 95, 103, 99, 134, - /* 1160 */ 135, 136, 101, 102, 103, 104, 105, 106, 107, 153, - /* 1170 */ 26, 59, 125, 164, 113, 153, 153, 153, 212, 12, - /* 1180 */ 19, 117, 153, 71, 153, 212, 122, 164, 134, 135, - /* 1190 */ 136, 137, 138, 212, 27, 83, 212, 174, 175, 59, - /* 1200 */ 200, 89, 90, 174, 175, 196, 197, 46, 96, 42, - /* 1210 */ 98, 99, 100, 172, 151, 103, 5, 208, 203, 196, - /* 1220 */ 197, 10, 11, 12, 13, 14, 216, 216, 17, 244, - /* 1230 */ 63, 208, 209, 210, 153, 80, 24, 203, 98, 99, - /* 1240 */ 100, 30, 22, 32, 100, 164, 134, 135, 136, 137, - /* 1250 */ 138, 40, 148, 164, 150, 174, 175, 102, 97, 155, - /* 1260 */ 203, 157, 164, 244, 178, 125, 186, 182, 164, 125, - /* 1270 */ 177, 59, 177, 177, 113, 120, 121, 196, 197, 59, - /* 1280 */ 232, 70, 153, 216, 202, 196, 197, 153, 77, 208, - /* 1290 */ 209, 80, 81, 156, 196, 197, 60, 208, 248, 200, - /* 1300 */ 196, 197, 153, 174, 175, 123, 208, 153, 174, 175, - /* 1310 */ 160, 99, 208, 153, 38, 153, 160, 153, 98, 99, - /* 1320 */ 100, 153, 245, 174, 175, 221, 115, 153, 174, 175, - /* 1330 */ 153, 120, 121, 245, 174, 175, 174, 175, 174, 175, - /* 1340 */ 160, 153, 174, 175, 153, 225, 153, 22, 174, 175, - /* 1350 */ 97, 174, 175, 249, 143, 153, 224, 153, 43, 153, - /* 1360 */ 191, 153, 174, 175, 18, 174, 175, 174, 175, 153, - /* 1370 */ 131, 153, 194, 203, 153, 194, 174, 175, 174, 175, - /* 1380 */ 174, 175, 174, 175, 153, 160, 153, 18, 153, 194, - /* 1390 */ 174, 175, 174, 175, 153, 174, 175, 153, 225, 153, - /* 1400 */ 203, 153, 159, 153, 194, 174, 175, 174, 175, 174, - /* 1410 */ 175, 153, 203, 153, 224, 174, 175, 191, 174, 175, - /* 1420 */ 174, 175, 174, 175, 174, 175, 203, 160, 153, 191, - /* 1430 */ 153, 159, 174, 175, 174, 175, 153, 139, 62, 153, - /* 1440 */ 241, 153, 160, 153, 159, 153, 22, 240, 179, 174, - /* 1450 */ 175, 174, 175, 160, 159, 97, 160, 174, 175, 159, - /* 1460 */ 174, 175, 174, 175, 174, 175, 174, 175, 179, 64, - /* 1470 */ 176, 184, 108, 176, 95, 176, 234, 126, 176, 234, - /* 1480 */ 179, 178, 176, 176, 176, 97, 218, 217, 184, 179, - /* 1490 */ 179, 218, 218, 160, 22, 217, 217, 160, 218, 139, - /* 1500 */ 229, 217, 130, 129, 127, 25, 128, 163, 26, 162, - /* 1510 */ 13, 206, 231, 154, 6, 154, 207, 205, 204, 203, - /* 1520 */ 152, 166, 152, 152, 172, 172, 172, 4, 172, 166, - /* 1530 */ 180, 166, 3, 22, 172, 144, 15, 180, 172, 172, - /* 1540 */ 82, 16, 23, 23, 121, 254, 132, 172, 112, 124, - /* 1550 */ 24, 20, 126, 16, 1, 124, 112, 61, 53, 37, - /* 1560 */ 133, 132, 53, 53, 112, 53, 98, 254, 251, 34, - /* 1570 */ 123, 1, 5, 22, 97, 142, 26, 75, 123, 41, - /* 1580 */ 97, 68, 68, 24, 20, 19, 113, 22, 107, 28, - /* 1590 */ 22, 67, 23, 22, 22, 67, 22, 67, 23, 37, - /* 1600 */ 23, 23, 26, 23, 22, 24, 23, 22, 24, 123, - /* 1610 */ 23, 23, 22, 98, 125, 26, 11, 23, 26, 23, - /* 1620 */ 34, 23, 23, 23, 23, 34, 22, 34, 26, 22, - /* 1630 */ 22, 15, 23, 23, 22, 117, 23, 22, 1, 123, - /* 1640 */ 26, 23, 255, 255, 255, 255, 255, 255, 255, 255, - /* 1650 */ 123, 255, 255, 255, 255, 123, 123, 255, 255, 255, - /* 1660 */ 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - /* 1670 */ 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - /* 1680 */ 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - /* 1690 */ 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - /* 1700 */ 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - /* 1710 */ 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - /* 1720 */ 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - /* 1730 */ 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - /* 1740 */ 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - /* 1750 */ 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - /* 1760 */ 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - /* 1770 */ 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - /* 1780 */ 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - /* 1790 */ 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - /* 1800 */ 255, 255, -}; -#define YY_SHIFT_COUNT (489) -#define YY_SHIFT_MIN (0) -#define YY_SHIFT_MAX (1637) -static const unsigned short int yy_shift_ofst[] = { - /* 0 */ 990, 976, 1211, 837, 837, 316, 1054, 1054, 1054, 1054, - /* 10 */ 214, 0, 0, 106, 642, 1054, 1054, 1054, 1054, 1054, - /* 20 */ 1054, 1054, 1054, 952, 952, 226, 1155, 316, 316, 316, - /* 30 */ 316, 316, 316, 53, 159, 212, 265, 318, 371, 424, - /* 40 */ 477, 533, 589, 642, 642, 642, 642, 642, 642, 642, - /* 50 */ 642, 642, 642, 642, 642, 642, 642, 642, 642, 642, - /* 60 */ 695, 642, 747, 798, 798, 1004, 1054, 1054, 1054, 1054, - /* 70 */ 1054, 1054, 1054, 1054, 1054, 1054, 1054, 1054, 1054, 1054, - /* 80 */ 1054, 1054, 1054, 1054, 1054, 1054, 1054, 1054, 1054, 1054, - /* 90 */ 1054, 1054, 1054, 1054, 1054, 1054, 1054, 1112, 1054, 1054, - /* 100 */ 1054, 1054, 1054, 1054, 1054, 1054, 1054, 1054, 1054, 1054, - /* 110 */ 1054, 856, 874, 874, 874, 874, 874, 134, 147, 93, - /* 120 */ 342, 959, 1161, 253, 253, 342, 367, 367, 367, 367, - /* 130 */ 179, 36, 79, 1657, 1657, 1657, 1061, 1061, 1061, 516, - /* 140 */ 799, 516, 516, 531, 531, 802, 249, 369, 342, 342, - /* 150 */ 342, 342, 342, 342, 342, 342, 342, 342, 342, 342, - /* 160 */ 342, 342, 342, 342, 342, 342, 342, 342, 342, 272, - /* 170 */ 442, 442, 536, 1657, 1657, 1657, 1025, 245, 245, 570, - /* 180 */ 172, 286, 805, 1047, 1140, 1220, 342, 342, 342, 342, - /* 190 */ 342, 342, 342, 342, 170, 342, 342, 342, 342, 342, - /* 200 */ 342, 342, 342, 342, 342, 342, 342, 841, 841, 841, - /* 210 */ 342, 342, 342, 342, 530, 342, 342, 342, 1059, 342, - /* 220 */ 342, 1167, 342, 342, 342, 342, 342, 342, 342, 342, - /* 230 */ 123, 688, 177, 1212, 1212, 1212, 1212, 1144, 177, 177, - /* 240 */ 1064, 409, 33, 628, 707, 707, 900, 628, 628, 900, - /* 250 */ 897, 323, 398, 677, 677, 677, 707, 572, 684, 590, - /* 260 */ 739, 1236, 1182, 1182, 1276, 1276, 1182, 1253, 1325, 1315, - /* 270 */ 1239, 1346, 1346, 1346, 1346, 1182, 1369, 1239, 1239, 1253, - /* 280 */ 1325, 1315, 1315, 1239, 1182, 1369, 1298, 1376, 1182, 1369, - /* 290 */ 1424, 1182, 1369, 1182, 1369, 1424, 1358, 1358, 1358, 1405, - /* 300 */ 1424, 1358, 1364, 1358, 1405, 1358, 1358, 1424, 1379, 1379, - /* 310 */ 1424, 1351, 1388, 1351, 1388, 1351, 1388, 1351, 1388, 1182, - /* 320 */ 1472, 1182, 1360, 1372, 1377, 1374, 1378, 1239, 1480, 1482, - /* 330 */ 1497, 1497, 1508, 1508, 1508, 1657, 1657, 1657, 1657, 1657, - /* 340 */ 1657, 1657, 1657, 1657, 1657, 1657, 1657, 1657, 1657, 1657, - /* 350 */ 1657, 20, 413, 98, 423, 519, 383, 962, 742, 61, - /* 360 */ 696, 749, 750, 753, 789, 790, 795, 797, 840, 842, - /* 370 */ 810, 668, 817, 659, 819, 849, 854, 899, 643, 745, - /* 380 */ 956, 926, 916, 1523, 1529, 1511, 1391, 1521, 1458, 1525, - /* 390 */ 1519, 1520, 1423, 1414, 1436, 1526, 1425, 1531, 1426, 1537, - /* 400 */ 1553, 1431, 1427, 1444, 1496, 1522, 1429, 1505, 1509, 1510, - /* 410 */ 1512, 1452, 1468, 1535, 1447, 1570, 1567, 1551, 1477, 1433, - /* 420 */ 1513, 1550, 1514, 1502, 1538, 1455, 1483, 1559, 1564, 1566, - /* 430 */ 1473, 1481, 1565, 1524, 1568, 1571, 1569, 1572, 1528, 1561, - /* 440 */ 1574, 1530, 1562, 1575, 1577, 1578, 1576, 1580, 1582, 1581, - /* 450 */ 1583, 1585, 1584, 1486, 1587, 1588, 1515, 1586, 1590, 1489, - /* 460 */ 1589, 1591, 1592, 1593, 1594, 1596, 1598, 1589, 1599, 1600, - /* 470 */ 1602, 1601, 1604, 1605, 1607, 1608, 1609, 1610, 1612, 1613, - /* 480 */ 1615, 1614, 1518, 1516, 1527, 1532, 1533, 1618, 1616, 1637, -}; -#define YY_REDUCE_COUNT (350) -#define YY_REDUCE_MIN (-225) -#define YY_REDUCE_MAX (1375) -static const short yy_reduce_ofst[] = { - /* 0 */ -137, -31, 1104, 1023, 1081, -132, -40, -38, 223, 225, - /* 10 */ 698, -153, -99, -225, -165, 386, 478, 843, 859, -139, - /* 20 */ 884, 117, 277, 844, 857, 964, 559, 561, 614, 918, - /* 30 */ 1009, 1089, 1098, -222, -222, -222, -222, -222, -222, -222, - /* 40 */ -222, -222, -222, -222, -222, -222, -222, -222, -222, -222, - /* 50 */ -222, -222, -222, -222, -222, -222, -222, -222, -222, -222, - /* 60 */ -222, -222, -222, -222, -222, 329, 331, 497, 654, 656, - /* 70 */ 781, 891, 946, 1029, 1129, 1134, 1149, 1154, 1160, 1162, - /* 80 */ 1164, 1168, 1174, 1177, 1188, 1191, 1193, 1202, 1204, 1206, - /* 90 */ 1208, 1216, 1218, 1221, 1231, 1233, 1235, 1241, 1244, 1246, - /* 100 */ 1248, 1250, 1258, 1260, 1275, 1277, 1283, 1286, 1288, 1290, - /* 110 */ 1292, -222, -222, -222, -222, -222, -222, -222, -222, -222, - /* 120 */ -115, 796, -156, -154, -141, 14, 242, 349, 242, 349, - /* 130 */ -61, -222, -222, -222, -222, -222, 101, 101, 101, 332, - /* 140 */ 302, 384, 387, -170, 146, 344, 196, 196, 15, 11, - /* 150 */ 183, 235, 395, 355, 396, 406, 452, 457, 391, 459, - /* 160 */ 443, 447, 511, 495, 454, 512, 505, 571, 498, 532, - /* 170 */ 431, 435, 339, 455, 446, 508, -174, -116, -97, -120, - /* 180 */ -150, 64, 176, 330, 337, 509, 569, 611, 653, 673, - /* 190 */ 714, 717, 763, 771, -34, 779, 786, 830, 846, 860, - /* 200 */ 866, 882, 883, 890, 892, 895, 902, 319, 368, 769, - /* 210 */ 915, 924, 925, 932, 755, 936, 945, 963, 782, 969, - /* 220 */ 974, 816, 977, 64, 982, 983, 1016, 1022, 1024, 1031, - /* 230 */ 870, 831, 913, 966, 973, 981, 984, 755, 913, 913, - /* 240 */ 1000, 1041, 1063, 1015, 1010, 1011, 985, 1034, 1057, 1019, - /* 250 */ 1086, 1080, 1085, 1093, 1095, 1096, 1067, 1048, 1082, 1099, - /* 260 */ 1137, 1050, 1150, 1156, 1077, 1088, 1180, 1120, 1132, 1169, - /* 270 */ 1170, 1178, 1181, 1195, 1210, 1225, 1243, 1197, 1209, 1173, - /* 280 */ 1190, 1226, 1238, 1223, 1267, 1272, 1199, 1207, 1282, 1285, - /* 290 */ 1269, 1293, 1295, 1296, 1300, 1289, 1294, 1297, 1299, 1287, - /* 300 */ 1301, 1302, 1303, 1306, 1304, 1307, 1308, 1310, 1242, 1245, - /* 310 */ 1311, 1268, 1270, 1273, 1278, 1274, 1279, 1280, 1284, 1333, - /* 320 */ 1271, 1337, 1281, 1309, 1305, 1312, 1314, 1316, 1344, 1347, - /* 330 */ 1359, 1361, 1368, 1370, 1371, 1291, 1313, 1317, 1355, 1352, - /* 340 */ 1353, 1354, 1356, 1363, 1350, 1357, 1362, 1366, 1367, 1375, - /* 350 */ 1365, -}; -static const YYACTIONTYPE yy_default[] = { - /* 0 */ 1389, 1389, 1389, 1261, 1046, 1151, 1261, 1261, 1261, 1261, - /* 10 */ 1046, 1181, 1181, 1312, 1077, 1046, 1046, 1046, 1046, 1046, - /* 20 */ 1046, 1260, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, - /* 30 */ 1046, 1046, 1046, 1187, 1046, 1046, 1046, 1046, 1262, 1263, - /* 40 */ 1046, 1046, 1046, 1311, 1313, 1197, 1196, 1195, 1194, 1294, - /* 50 */ 1168, 1192, 1185, 1189, 1256, 1257, 1255, 1259, 1262, 1263, - /* 60 */ 1046, 1188, 1226, 1240, 1225, 1046, 1046, 1046, 1046, 1046, - /* 70 */ 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, - /* 80 */ 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, - /* 90 */ 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, - /* 100 */ 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, - /* 110 */ 1046, 1234, 1239, 1246, 1238, 1235, 1228, 1227, 1229, 1230, - /* 120 */ 1046, 1067, 1116, 1046, 1046, 1046, 1329, 1328, 1046, 1046, - /* 130 */ 1077, 1231, 1232, 1243, 1242, 1241, 1319, 1345, 1344, 1046, - /* 140 */ 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, - /* 150 */ 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, - /* 160 */ 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1077, - /* 170 */ 1073, 1073, 1046, 1324, 1151, 1142, 1046, 1046, 1046, 1046, - /* 180 */ 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1316, 1314, 1046, - /* 190 */ 1276, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, - /* 200 */ 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, - /* 210 */ 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1147, 1046, - /* 220 */ 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1339, - /* 230 */ 1046, 1289, 1130, 1147, 1147, 1147, 1147, 1149, 1131, 1129, - /* 240 */ 1141, 1077, 1053, 1191, 1170, 1170, 1378, 1191, 1191, 1378, - /* 250 */ 1091, 1359, 1088, 1181, 1181, 1181, 1170, 1258, 1148, 1141, - /* 260 */ 1046, 1381, 1156, 1156, 1380, 1380, 1156, 1200, 1206, 1119, - /* 270 */ 1191, 1125, 1125, 1125, 1125, 1156, 1064, 1191, 1191, 1200, - /* 280 */ 1206, 1119, 1119, 1191, 1156, 1064, 1293, 1375, 1156, 1064, - /* 290 */ 1269, 1156, 1064, 1156, 1064, 1269, 1117, 1117, 1117, 1106, - /* 300 */ 1269, 1117, 1091, 1117, 1106, 1117, 1117, 1269, 1273, 1273, - /* 310 */ 1269, 1174, 1169, 1174, 1169, 1174, 1169, 1174, 1169, 1156, - /* 320 */ 1264, 1156, 1046, 1186, 1175, 1184, 1182, 1191, 1070, 1109, - /* 330 */ 1342, 1342, 1338, 1338, 1338, 1386, 1386, 1324, 1354, 1077, - /* 340 */ 1077, 1077, 1077, 1354, 1093, 1093, 1077, 1077, 1077, 1077, - /* 350 */ 1354, 1046, 1046, 1046, 1046, 1046, 1046, 1349, 1046, 1278, - /* 360 */ 1160, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, - /* 370 */ 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, - /* 380 */ 1046, 1046, 1211, 1046, 1049, 1321, 1046, 1046, 1320, 1046, - /* 390 */ 1046, 1046, 1046, 1046, 1046, 1161, 1046, 1046, 1046, 1046, - /* 400 */ 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, - /* 410 */ 1046, 1046, 1046, 1046, 1377, 1046, 1046, 1046, 1046, 1046, - /* 420 */ 1046, 1292, 1291, 1046, 1046, 1158, 1046, 1046, 1046, 1046, - /* 430 */ 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, - /* 440 */ 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, - /* 450 */ 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, - /* 460 */ 1183, 1046, 1176, 1046, 1046, 1046, 1046, 1368, 1046, 1046, - /* 470 */ 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, - /* 480 */ 1046, 1363, 1133, 1213, 1046, 1212, 1216, 1046, 1058, 1046, -}; -/********** End of lemon-generated parsing tables *****************************/ - -/* The next table maps tokens (terminal symbols) into fallback tokens. -** If a construct like the following: -** -** %fallback ID X Y Z. +** so that both min() and max() may process rows in the order defined by +** their respective window declarations. ** -** appears in the grammar, then ID becomes a fallback token for X, Y, -** and Z. Whenever one of the tokens X, Y, or Z is input to the parser -** but it does not parse, the type of the token is changed to ID and -** the parse is retried before an error is thrown. +** INTERFACE WITH SELECT.C ** -** This feature can be used, for example, to cause some keywords in a language -** to revert to identifiers if they keyword does not apply in the context where -** it appears. -*/ -#ifdef YYFALLBACK -static const YYCODETYPE yyFallback[] = { - 0, /* $ => nothing */ - 0, /* SEMI => nothing */ - 59, /* EXPLAIN => ID */ - 59, /* QUERY => ID */ - 59, /* PLAN => ID */ - 59, /* BEGIN => ID */ - 0, /* TRANSACTION => nothing */ - 59, /* DEFERRED => ID */ - 59, /* IMMEDIATE => ID */ - 59, /* EXCLUSIVE => ID */ - 0, /* COMMIT => nothing */ - 59, /* END => ID */ - 59, /* ROLLBACK => ID */ - 59, /* SAVEPOINT => ID */ - 59, /* RELEASE => ID */ - 0, /* TO => nothing */ - 0, /* TABLE => nothing */ - 0, /* CREATE => nothing */ - 59, /* IF => ID */ - 0, /* NOT => nothing */ - 0, /* EXISTS => nothing */ - 59, /* TEMP => ID */ - 0, /* LP => nothing */ - 0, /* RP => nothing */ - 0, /* AS => nothing */ - 59, /* WITHOUT => ID */ - 0, /* COMMA => nothing */ - 59, /* ABORT => ID */ - 59, /* ACTION => ID */ - 59, /* AFTER => ID */ - 59, /* ANALYZE => ID */ - 59, /* ASC => ID */ - 59, /* ATTACH => ID */ - 59, /* BEFORE => ID */ - 59, /* BY => ID */ - 59, /* CASCADE => ID */ - 59, /* CAST => ID */ - 59, /* CONFLICT => ID */ - 59, /* DATABASE => ID */ - 59, /* DESC => ID */ - 59, /* DETACH => ID */ - 59, /* EACH => ID */ - 59, /* FAIL => ID */ - 0, /* OR => nothing */ - 0, /* AND => nothing */ - 0, /* IS => nothing */ - 59, /* MATCH => ID */ - 59, /* LIKE_KW => ID */ - 0, /* BETWEEN => nothing */ - 0, /* IN => nothing */ - 0, /* ISNULL => nothing */ - 0, /* NOTNULL => nothing */ - 0, /* NE => nothing */ - 0, /* EQ => nothing */ - 0, /* GT => nothing */ - 0, /* LE => nothing */ - 0, /* LT => nothing */ - 0, /* GE => nothing */ - 0, /* ESCAPE => nothing */ - 0, /* ID => nothing */ - 59, /* COLUMNKW => ID */ - 59, /* DO => ID */ - 59, /* FOR => ID */ - 59, /* IGNORE => ID */ - 59, /* INITIALLY => ID */ - 59, /* INSTEAD => ID */ - 59, /* NO => ID */ - 59, /* KEY => ID */ - 59, /* OF => ID */ - 59, /* OFFSET => ID */ - 59, /* PRAGMA => ID */ - 59, /* RAISE => ID */ - 59, /* RECURSIVE => ID */ - 59, /* REPLACE => ID */ - 59, /* RESTRICT => ID */ - 59, /* ROW => ID */ - 59, /* TRIGGER => ID */ - 59, /* VACUUM => ID */ - 59, /* VIEW => ID */ - 59, /* VIRTUAL => ID */ - 59, /* WITH => ID */ - 59, /* REINDEX => ID */ - 59, /* RENAME => ID */ - 59, /* CTIME_KW => ID */ -}; -#endif /* YYFALLBACK */ - -/* The following structure represents a single element of the -** parser's stack. Information stored includes: +** When processing the rewritten SELECT statement, code in select.c calls +** sqlite3WhereBegin() to begin iterating through the results of the +** sub-query, which is always implemented as a co-routine. It then calls +** sqlite3WindowCodeStep() to process rows and finish the scan by calling +** sqlite3WhereEnd(). ** -** + The state number for the parser at this level of the stack. +** sqlite3WindowCodeStep() generates VM code so that, for each row returned +** by the sub-query a sub-routine (OP_Gosub) coded by select.c is invoked. +** When the sub-routine is invoked: ** -** + The value of the token stored at this level of the stack. -** (In other words, the "major" token.) +** * The results of all window-functions for the row are stored +** in the associated Window.regResult registers. ** -** + The semantic value stored at this level of the stack. This is -** the information used by the action routines in the grammar. -** It is sometimes called the "minor" token. +** * The required terminal values are stored in the current row of +** temp table Window.iEphCsr. ** -** After the "shift" half of a SHIFTREDUCE action, the stateno field -** actually contains the reduce action for the second half of the -** SHIFTREDUCE. +** In some cases, depending on the window frame and the specific window +** functions invoked, sqlite3WindowCodeStep() caches each entire partition +** in a temp table before returning any rows. In other cases it does not. +** This detail is encapsulated within this file, the code generated by +** select.c is the same in either case. +** +** BUILT-IN WINDOW FUNCTIONS +** +** This implementation features the following built-in window functions: +** +** row_number() +** rank() +** dense_rank() +** percent_rank() +** cume_dist() +** ntile(N) +** lead(expr [, offset [, default]]) +** lag(expr [, offset [, default]]) +** first_value(expr) +** last_value(expr) +** nth_value(expr, N) +** +** These are the same built-in window functions supported by Postgres. +** Although the behaviour of aggregate window functions (functions that +** can be used as either aggregates or window funtions) allows them to +** be implemented using an API, built-in window functions are much more +** esoteric. Additionally, some window functions (e.g. nth_value()) +** may only be implemented by caching the entire partition in memory. +** As such, some built-in window functions use the same API as aggregate +** window functions and some are implemented directly using VDBE +** instructions. Additionally, for those functions that use the API, the +** window frame is sometimes modified before the SELECT statement is +** rewritten. For example, regardless of the specified window frame, the +** row_number() function always uses: +** +** ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW +** +** See sqlite3WindowUpdate() for details. +** +** As well as some of the built-in window functions, aggregate window +** functions min() and max() are implemented using VDBE instructions if +** the start of the window frame is declared as anything other than +** UNBOUNDED PRECEDING. */ -struct yyStackEntry { - YYACTIONTYPE stateno; /* The state-number, or reduce action in SHIFTREDUCE */ - YYCODETYPE major; /* The major token value. This is the code - ** number for the token at this stack level */ - YYMINORTYPE minor; /* The user-supplied minor token value. This - ** is the value of the token */ -}; -typedef struct yyStackEntry yyStackEntry; - -/* The state of the parser is completely contained in an instance of -** the following structure */ -struct yyParser { - yyStackEntry *yytos; /* Pointer to top element of the stack */ -#ifdef YYTRACKMAXSTACKDEPTH - int yyhwm; /* High-water mark of the stack */ -#endif -#ifndef YYNOERRORRECOVERY - int yyerrcnt; /* Shifts left before out of the error */ -#endif - sqlite3ParserARG_SDECL /* A place to hold %extra_argument */ - sqlite3ParserCTX_SDECL /* A place to hold %extra_context */ -#if YYSTACKDEPTH<=0 - int yystksz; /* Current side of the stack */ - yyStackEntry *yystack; /* The parser's stack */ - yyStackEntry yystk0; /* First stack entry */ -#else - yyStackEntry yystack[YYSTACKDEPTH]; /* The parser's stack */ - yyStackEntry *yystackEnd; /* Last entry in the stack */ -#endif -}; -typedef struct yyParser yyParser; - -#ifndef NDEBUG -/* #include */ -static FILE *yyTraceFILE = 0; -static char *yyTracePrompt = 0; -#endif /* NDEBUG */ -#ifndef NDEBUG -/* -** Turn parser tracing on by giving a stream to which to write the trace -** and a prompt to preface each trace message. Tracing is turned off -** by making either argument NULL -** -** Inputs: -**
          -**
        • A FILE* to which trace output should be written. -** If NULL, then tracing is turned off. -**
        • A prefix string written at the beginning of every -** line of trace output. If NULL, then tracing is -** turned off. -**
        +/* +** Implementation of built-in window function row_number(). Assumes that the +** window frame has been coerced to: ** -** Outputs: -** None. +** ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW */ -SQLITE_PRIVATE void sqlite3ParserTrace(FILE *TraceFILE, char *zTracePrompt){ - yyTraceFILE = TraceFILE; - yyTracePrompt = zTracePrompt; - if( yyTraceFILE==0 ) yyTracePrompt = 0; - else if( yyTracePrompt==0 ) yyTraceFILE = 0; +static void row_numberStepFunc( + sqlite3_context *pCtx, + int nArg, + sqlite3_value **apArg +){ + i64 *p = (i64*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + if( p ) (*p)++; + UNUSED_PARAMETER(nArg); + UNUSED_PARAMETER(apArg); +} +static void row_numberValueFunc(sqlite3_context *pCtx){ + i64 *p = (i64*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + sqlite3_result_int64(pCtx, (p ? *p : 0)); } -#endif /* NDEBUG */ - -#if defined(YYCOVERAGE) || !defined(NDEBUG) -/* For tracing shifts, the names of all terminals and nonterminals -** are required. The following table supplies these names */ -static const char *const yyTokenName[] = { - /* 0 */ "$", - /* 1 */ "SEMI", - /* 2 */ "EXPLAIN", - /* 3 */ "QUERY", - /* 4 */ "PLAN", - /* 5 */ "BEGIN", - /* 6 */ "TRANSACTION", - /* 7 */ "DEFERRED", - /* 8 */ "IMMEDIATE", - /* 9 */ "EXCLUSIVE", - /* 10 */ "COMMIT", - /* 11 */ "END", - /* 12 */ "ROLLBACK", - /* 13 */ "SAVEPOINT", - /* 14 */ "RELEASE", - /* 15 */ "TO", - /* 16 */ "TABLE", - /* 17 */ "CREATE", - /* 18 */ "IF", - /* 19 */ "NOT", - /* 20 */ "EXISTS", - /* 21 */ "TEMP", - /* 22 */ "LP", - /* 23 */ "RP", - /* 24 */ "AS", - /* 25 */ "WITHOUT", - /* 26 */ "COMMA", - /* 27 */ "ABORT", - /* 28 */ "ACTION", - /* 29 */ "AFTER", - /* 30 */ "ANALYZE", - /* 31 */ "ASC", - /* 32 */ "ATTACH", - /* 33 */ "BEFORE", - /* 34 */ "BY", - /* 35 */ "CASCADE", - /* 36 */ "CAST", - /* 37 */ "CONFLICT", - /* 38 */ "DATABASE", - /* 39 */ "DESC", - /* 40 */ "DETACH", - /* 41 */ "EACH", - /* 42 */ "FAIL", - /* 43 */ "OR", - /* 44 */ "AND", - /* 45 */ "IS", - /* 46 */ "MATCH", - /* 47 */ "LIKE_KW", - /* 48 */ "BETWEEN", - /* 49 */ "IN", - /* 50 */ "ISNULL", - /* 51 */ "NOTNULL", - /* 52 */ "NE", - /* 53 */ "EQ", - /* 54 */ "GT", - /* 55 */ "LE", - /* 56 */ "LT", - /* 57 */ "GE", - /* 58 */ "ESCAPE", - /* 59 */ "ID", - /* 60 */ "COLUMNKW", - /* 61 */ "DO", - /* 62 */ "FOR", - /* 63 */ "IGNORE", - /* 64 */ "INITIALLY", - /* 65 */ "INSTEAD", - /* 66 */ "NO", - /* 67 */ "KEY", - /* 68 */ "OF", - /* 69 */ "OFFSET", - /* 70 */ "PRAGMA", - /* 71 */ "RAISE", - /* 72 */ "RECURSIVE", - /* 73 */ "REPLACE", - /* 74 */ "RESTRICT", - /* 75 */ "ROW", - /* 76 */ "TRIGGER", - /* 77 */ "VACUUM", - /* 78 */ "VIEW", - /* 79 */ "VIRTUAL", - /* 80 */ "WITH", - /* 81 */ "REINDEX", - /* 82 */ "RENAME", - /* 83 */ "CTIME_KW", - /* 84 */ "ANY", - /* 85 */ "BITAND", - /* 86 */ "BITOR", - /* 87 */ "LSHIFT", - /* 88 */ "RSHIFT", - /* 89 */ "PLUS", - /* 90 */ "MINUS", - /* 91 */ "STAR", - /* 92 */ "SLASH", - /* 93 */ "REM", - /* 94 */ "CONCAT", - /* 95 */ "COLLATE", - /* 96 */ "BITNOT", - /* 97 */ "ON", - /* 98 */ "INDEXED", - /* 99 */ "STRING", - /* 100 */ "JOIN_KW", - /* 101 */ "CONSTRAINT", - /* 102 */ "DEFAULT", - /* 103 */ "NULL", - /* 104 */ "PRIMARY", - /* 105 */ "UNIQUE", - /* 106 */ "CHECK", - /* 107 */ "REFERENCES", - /* 108 */ "AUTOINCR", - /* 109 */ "INSERT", - /* 110 */ "DELETE", - /* 111 */ "UPDATE", - /* 112 */ "SET", - /* 113 */ "DEFERRABLE", - /* 114 */ "FOREIGN", - /* 115 */ "DROP", - /* 116 */ "UNION", - /* 117 */ "ALL", - /* 118 */ "EXCEPT", - /* 119 */ "INTERSECT", - /* 120 */ "SELECT", - /* 121 */ "VALUES", - /* 122 */ "DISTINCT", - /* 123 */ "DOT", - /* 124 */ "FROM", - /* 125 */ "JOIN", - /* 126 */ "USING", - /* 127 */ "ORDER", - /* 128 */ "GROUP", - /* 129 */ "HAVING", - /* 130 */ "LIMIT", - /* 131 */ "WHERE", - /* 132 */ "INTO", - /* 133 */ "NOTHING", - /* 134 */ "FLOAT", - /* 135 */ "BLOB", - /* 136 */ "INTEGER", - /* 137 */ "VARIABLE", - /* 138 */ "CASE", - /* 139 */ "WHEN", - /* 140 */ "THEN", - /* 141 */ "ELSE", - /* 142 */ "INDEX", - /* 143 */ "ALTER", - /* 144 */ "ADD", - /* 145 */ "input", - /* 146 */ "cmdlist", - /* 147 */ "ecmd", - /* 148 */ "cmdx", - /* 149 */ "explain", - /* 150 */ "cmd", - /* 151 */ "transtype", - /* 152 */ "trans_opt", - /* 153 */ "nm", - /* 154 */ "savepoint_opt", - /* 155 */ "create_table", - /* 156 */ "create_table_args", - /* 157 */ "createkw", - /* 158 */ "temp", - /* 159 */ "ifnotexists", - /* 160 */ "dbnm", - /* 161 */ "columnlist", - /* 162 */ "conslist_opt", - /* 163 */ "table_options", - /* 164 */ "select", - /* 165 */ "columnname", - /* 166 */ "carglist", - /* 167 */ "typetoken", - /* 168 */ "typename", - /* 169 */ "signed", - /* 170 */ "plus_num", - /* 171 */ "minus_num", - /* 172 */ "scanpt", - /* 173 */ "ccons", - /* 174 */ "term", - /* 175 */ "expr", - /* 176 */ "onconf", - /* 177 */ "sortorder", - /* 178 */ "autoinc", - /* 179 */ "eidlist_opt", - /* 180 */ "refargs", - /* 181 */ "defer_subclause", - /* 182 */ "refarg", - /* 183 */ "refact", - /* 184 */ "init_deferred_pred_opt", - /* 185 */ "conslist", - /* 186 */ "tconscomma", - /* 187 */ "tcons", - /* 188 */ "sortlist", - /* 189 */ "eidlist", - /* 190 */ "defer_subclause_opt", - /* 191 */ "orconf", - /* 192 */ "resolvetype", - /* 193 */ "raisetype", - /* 194 */ "ifexists", - /* 195 */ "fullname", - /* 196 */ "selectnowith", - /* 197 */ "oneselect", - /* 198 */ "wqlist", - /* 199 */ "multiselect_op", - /* 200 */ "distinct", - /* 201 */ "selcollist", - /* 202 */ "from", - /* 203 */ "where_opt", - /* 204 */ "groupby_opt", - /* 205 */ "having_opt", - /* 206 */ "orderby_opt", - /* 207 */ "limit_opt", - /* 208 */ "values", - /* 209 */ "nexprlist", - /* 210 */ "exprlist", - /* 211 */ "sclp", - /* 212 */ "as", - /* 213 */ "seltablist", - /* 214 */ "stl_prefix", - /* 215 */ "joinop", - /* 216 */ "indexed_opt", - /* 217 */ "on_opt", - /* 218 */ "using_opt", - /* 219 */ "xfullname", - /* 220 */ "idlist", - /* 221 */ "with", - /* 222 */ "setlist", - /* 223 */ "insert_cmd", - /* 224 */ "idlist_opt", - /* 225 */ "upsert", - /* 226 */ "likeop", - /* 227 */ "between_op", - /* 228 */ "in_op", - /* 229 */ "paren_exprlist", - /* 230 */ "case_operand", - /* 231 */ "case_exprlist", - /* 232 */ "case_else", - /* 233 */ "uniqueflag", - /* 234 */ "collate", - /* 235 */ "nmnum", - /* 236 */ "trigger_decl", - /* 237 */ "trigger_cmd_list", - /* 238 */ "trigger_time", - /* 239 */ "trigger_event", - /* 240 */ "foreach_clause", - /* 241 */ "when_clause", - /* 242 */ "trigger_cmd", - /* 243 */ "trnm", - /* 244 */ "tridxby", - /* 245 */ "database_kw_opt", - /* 246 */ "key_opt", - /* 247 */ "add_column_fullname", - /* 248 */ "kwcolumn_opt", - /* 249 */ "create_vtab", - /* 250 */ "vtabarglist", - /* 251 */ "vtabarg", - /* 252 */ "vtabargtoken", - /* 253 */ "lp", - /* 254 */ "anylist", -}; -#endif /* defined(YYCOVERAGE) || !defined(NDEBUG) */ -#ifndef NDEBUG -/* For tracing reduce actions, the names of all rules are required. +/* +** Context object type used by rank(), dense_rank(), percent_rank() and +** cume_dist(). */ -static const char *const yyRuleName[] = { - /* 0 */ "explain ::= EXPLAIN", - /* 1 */ "explain ::= EXPLAIN QUERY PLAN", - /* 2 */ "cmdx ::= cmd", - /* 3 */ "cmd ::= BEGIN transtype trans_opt", - /* 4 */ "transtype ::=", - /* 5 */ "transtype ::= DEFERRED", - /* 6 */ "transtype ::= IMMEDIATE", - /* 7 */ "transtype ::= EXCLUSIVE", - /* 8 */ "cmd ::= COMMIT|END trans_opt", - /* 9 */ "cmd ::= ROLLBACK trans_opt", - /* 10 */ "cmd ::= SAVEPOINT nm", - /* 11 */ "cmd ::= RELEASE savepoint_opt nm", - /* 12 */ "cmd ::= ROLLBACK trans_opt TO savepoint_opt nm", - /* 13 */ "create_table ::= createkw temp TABLE ifnotexists nm dbnm", - /* 14 */ "createkw ::= CREATE", - /* 15 */ "ifnotexists ::=", - /* 16 */ "ifnotexists ::= IF NOT EXISTS", - /* 17 */ "temp ::= TEMP", - /* 18 */ "temp ::=", - /* 19 */ "create_table_args ::= LP columnlist conslist_opt RP table_options", - /* 20 */ "create_table_args ::= AS select", - /* 21 */ "table_options ::=", - /* 22 */ "table_options ::= WITHOUT nm", - /* 23 */ "columnname ::= nm typetoken", - /* 24 */ "typetoken ::=", - /* 25 */ "typetoken ::= typename LP signed RP", - /* 26 */ "typetoken ::= typename LP signed COMMA signed RP", - /* 27 */ "typename ::= typename ID|STRING", - /* 28 */ "scanpt ::=", - /* 29 */ "ccons ::= CONSTRAINT nm", - /* 30 */ "ccons ::= DEFAULT scanpt term scanpt", - /* 31 */ "ccons ::= DEFAULT LP expr RP", - /* 32 */ "ccons ::= DEFAULT PLUS term scanpt", - /* 33 */ "ccons ::= DEFAULT MINUS term scanpt", - /* 34 */ "ccons ::= DEFAULT scanpt ID|INDEXED", - /* 35 */ "ccons ::= NOT NULL onconf", - /* 36 */ "ccons ::= PRIMARY KEY sortorder onconf autoinc", - /* 37 */ "ccons ::= UNIQUE onconf", - /* 38 */ "ccons ::= CHECK LP expr RP", - /* 39 */ "ccons ::= REFERENCES nm eidlist_opt refargs", - /* 40 */ "ccons ::= defer_subclause", - /* 41 */ "ccons ::= COLLATE ID|STRING", - /* 42 */ "autoinc ::=", - /* 43 */ "autoinc ::= AUTOINCR", - /* 44 */ "refargs ::=", - /* 45 */ "refargs ::= refargs refarg", - /* 46 */ "refarg ::= MATCH nm", - /* 47 */ "refarg ::= ON INSERT refact", - /* 48 */ "refarg ::= ON DELETE refact", - /* 49 */ "refarg ::= ON UPDATE refact", - /* 50 */ "refact ::= SET NULL", - /* 51 */ "refact ::= SET DEFAULT", - /* 52 */ "refact ::= CASCADE", - /* 53 */ "refact ::= RESTRICT", - /* 54 */ "refact ::= NO ACTION", - /* 55 */ "defer_subclause ::= NOT DEFERRABLE init_deferred_pred_opt", - /* 56 */ "defer_subclause ::= DEFERRABLE init_deferred_pred_opt", - /* 57 */ "init_deferred_pred_opt ::=", - /* 58 */ "init_deferred_pred_opt ::= INITIALLY DEFERRED", - /* 59 */ "init_deferred_pred_opt ::= INITIALLY IMMEDIATE", - /* 60 */ "conslist_opt ::=", - /* 61 */ "tconscomma ::= COMMA", - /* 62 */ "tcons ::= CONSTRAINT nm", - /* 63 */ "tcons ::= PRIMARY KEY LP sortlist autoinc RP onconf", - /* 64 */ "tcons ::= UNIQUE LP sortlist RP onconf", - /* 65 */ "tcons ::= CHECK LP expr RP onconf", - /* 66 */ "tcons ::= FOREIGN KEY LP eidlist RP REFERENCES nm eidlist_opt refargs defer_subclause_opt", - /* 67 */ "defer_subclause_opt ::=", - /* 68 */ "onconf ::=", - /* 69 */ "onconf ::= ON CONFLICT resolvetype", - /* 70 */ "orconf ::=", - /* 71 */ "orconf ::= OR resolvetype", - /* 72 */ "resolvetype ::= IGNORE", - /* 73 */ "resolvetype ::= REPLACE", - /* 74 */ "cmd ::= DROP TABLE ifexists fullname", - /* 75 */ "ifexists ::= IF EXISTS", - /* 76 */ "ifexists ::=", - /* 77 */ "cmd ::= createkw temp VIEW ifnotexists nm dbnm eidlist_opt AS select", - /* 78 */ "cmd ::= DROP VIEW ifexists fullname", - /* 79 */ "cmd ::= select", - /* 80 */ "select ::= WITH wqlist selectnowith", - /* 81 */ "select ::= WITH RECURSIVE wqlist selectnowith", - /* 82 */ "select ::= selectnowith", - /* 83 */ "selectnowith ::= selectnowith multiselect_op oneselect", - /* 84 */ "multiselect_op ::= UNION", - /* 85 */ "multiselect_op ::= UNION ALL", - /* 86 */ "multiselect_op ::= EXCEPT|INTERSECT", - /* 87 */ "oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt orderby_opt limit_opt", - /* 88 */ "values ::= VALUES LP nexprlist RP", - /* 89 */ "values ::= values COMMA LP exprlist RP", - /* 90 */ "distinct ::= DISTINCT", - /* 91 */ "distinct ::= ALL", - /* 92 */ "distinct ::=", - /* 93 */ "sclp ::=", - /* 94 */ "selcollist ::= sclp scanpt expr scanpt as", - /* 95 */ "selcollist ::= sclp scanpt STAR", - /* 96 */ "selcollist ::= sclp scanpt nm DOT STAR", - /* 97 */ "as ::= AS nm", - /* 98 */ "as ::=", - /* 99 */ "from ::=", - /* 100 */ "from ::= FROM seltablist", - /* 101 */ "stl_prefix ::= seltablist joinop", - /* 102 */ "stl_prefix ::=", - /* 103 */ "seltablist ::= stl_prefix nm dbnm as indexed_opt on_opt using_opt", - /* 104 */ "seltablist ::= stl_prefix nm dbnm LP exprlist RP as on_opt using_opt", - /* 105 */ "seltablist ::= stl_prefix LP select RP as on_opt using_opt", - /* 106 */ "seltablist ::= stl_prefix LP seltablist RP as on_opt using_opt", - /* 107 */ "dbnm ::=", - /* 108 */ "dbnm ::= DOT nm", - /* 109 */ "fullname ::= nm", - /* 110 */ "fullname ::= nm DOT nm", - /* 111 */ "xfullname ::= nm", - /* 112 */ "xfullname ::= nm DOT nm", - /* 113 */ "xfullname ::= nm DOT nm AS nm", - /* 114 */ "xfullname ::= nm AS nm", - /* 115 */ "joinop ::= COMMA|JOIN", - /* 116 */ "joinop ::= JOIN_KW JOIN", - /* 117 */ "joinop ::= JOIN_KW nm JOIN", - /* 118 */ "joinop ::= JOIN_KW nm nm JOIN", - /* 119 */ "on_opt ::= ON expr", - /* 120 */ "on_opt ::=", - /* 121 */ "indexed_opt ::=", - /* 122 */ "indexed_opt ::= INDEXED BY nm", - /* 123 */ "indexed_opt ::= NOT INDEXED", - /* 124 */ "using_opt ::= USING LP idlist RP", - /* 125 */ "using_opt ::=", - /* 126 */ "orderby_opt ::=", - /* 127 */ "orderby_opt ::= ORDER BY sortlist", - /* 128 */ "sortlist ::= sortlist COMMA expr sortorder", - /* 129 */ "sortlist ::= expr sortorder", - /* 130 */ "sortorder ::= ASC", - /* 131 */ "sortorder ::= DESC", - /* 132 */ "sortorder ::=", - /* 133 */ "groupby_opt ::=", - /* 134 */ "groupby_opt ::= GROUP BY nexprlist", - /* 135 */ "having_opt ::=", - /* 136 */ "having_opt ::= HAVING expr", - /* 137 */ "limit_opt ::=", - /* 138 */ "limit_opt ::= LIMIT expr", - /* 139 */ "limit_opt ::= LIMIT expr OFFSET expr", - /* 140 */ "limit_opt ::= LIMIT expr COMMA expr", - /* 141 */ "cmd ::= with DELETE FROM xfullname indexed_opt where_opt", - /* 142 */ "where_opt ::=", - /* 143 */ "where_opt ::= WHERE expr", - /* 144 */ "cmd ::= with UPDATE orconf xfullname indexed_opt SET setlist where_opt", - /* 145 */ "setlist ::= setlist COMMA nm EQ expr", - /* 146 */ "setlist ::= setlist COMMA LP idlist RP EQ expr", - /* 147 */ "setlist ::= nm EQ expr", - /* 148 */ "setlist ::= LP idlist RP EQ expr", - /* 149 */ "cmd ::= with insert_cmd INTO xfullname idlist_opt select upsert", - /* 150 */ "cmd ::= with insert_cmd INTO xfullname idlist_opt DEFAULT VALUES", - /* 151 */ "upsert ::=", - /* 152 */ "upsert ::= ON CONFLICT LP sortlist RP where_opt DO UPDATE SET setlist where_opt", - /* 153 */ "upsert ::= ON CONFLICT LP sortlist RP where_opt DO NOTHING", - /* 154 */ "upsert ::= ON CONFLICT DO NOTHING", - /* 155 */ "insert_cmd ::= INSERT orconf", - /* 156 */ "insert_cmd ::= REPLACE", - /* 157 */ "idlist_opt ::=", - /* 158 */ "idlist_opt ::= LP idlist RP", - /* 159 */ "idlist ::= idlist COMMA nm", - /* 160 */ "idlist ::= nm", - /* 161 */ "expr ::= LP expr RP", - /* 162 */ "expr ::= ID|INDEXED", - /* 163 */ "expr ::= JOIN_KW", - /* 164 */ "expr ::= nm DOT nm", - /* 165 */ "expr ::= nm DOT nm DOT nm", - /* 166 */ "term ::= NULL|FLOAT|BLOB", - /* 167 */ "term ::= STRING", - /* 168 */ "term ::= INTEGER", - /* 169 */ "expr ::= VARIABLE", - /* 170 */ "expr ::= expr COLLATE ID|STRING", - /* 171 */ "expr ::= CAST LP expr AS typetoken RP", - /* 172 */ "expr ::= ID|INDEXED LP distinct exprlist RP", - /* 173 */ "expr ::= ID|INDEXED LP STAR RP", - /* 174 */ "term ::= CTIME_KW", - /* 175 */ "expr ::= LP nexprlist COMMA expr RP", - /* 176 */ "expr ::= expr AND expr", - /* 177 */ "expr ::= expr OR expr", - /* 178 */ "expr ::= expr LT|GT|GE|LE expr", - /* 179 */ "expr ::= expr EQ|NE expr", - /* 180 */ "expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr", - /* 181 */ "expr ::= expr PLUS|MINUS expr", - /* 182 */ "expr ::= expr STAR|SLASH|REM expr", - /* 183 */ "expr ::= expr CONCAT expr", - /* 184 */ "likeop ::= NOT LIKE_KW|MATCH", - /* 185 */ "expr ::= expr likeop expr", - /* 186 */ "expr ::= expr likeop expr ESCAPE expr", - /* 187 */ "expr ::= expr ISNULL|NOTNULL", - /* 188 */ "expr ::= expr NOT NULL", - /* 189 */ "expr ::= expr IS expr", - /* 190 */ "expr ::= expr IS NOT expr", - /* 191 */ "expr ::= NOT expr", - /* 192 */ "expr ::= BITNOT expr", - /* 193 */ "expr ::= MINUS expr", - /* 194 */ "expr ::= PLUS expr", - /* 195 */ "between_op ::= BETWEEN", - /* 196 */ "between_op ::= NOT BETWEEN", - /* 197 */ "expr ::= expr between_op expr AND expr", - /* 198 */ "in_op ::= IN", - /* 199 */ "in_op ::= NOT IN", - /* 200 */ "expr ::= expr in_op LP exprlist RP", - /* 201 */ "expr ::= LP select RP", - /* 202 */ "expr ::= expr in_op LP select RP", - /* 203 */ "expr ::= expr in_op nm dbnm paren_exprlist", - /* 204 */ "expr ::= EXISTS LP select RP", - /* 205 */ "expr ::= CASE case_operand case_exprlist case_else END", - /* 206 */ "case_exprlist ::= case_exprlist WHEN expr THEN expr", - /* 207 */ "case_exprlist ::= WHEN expr THEN expr", - /* 208 */ "case_else ::= ELSE expr", - /* 209 */ "case_else ::=", - /* 210 */ "case_operand ::= expr", - /* 211 */ "case_operand ::=", - /* 212 */ "exprlist ::=", - /* 213 */ "nexprlist ::= nexprlist COMMA expr", - /* 214 */ "nexprlist ::= expr", - /* 215 */ "paren_exprlist ::=", - /* 216 */ "paren_exprlist ::= LP exprlist RP", - /* 217 */ "cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt", - /* 218 */ "uniqueflag ::= UNIQUE", - /* 219 */ "uniqueflag ::=", - /* 220 */ "eidlist_opt ::=", - /* 221 */ "eidlist_opt ::= LP eidlist RP", - /* 222 */ "eidlist ::= eidlist COMMA nm collate sortorder", - /* 223 */ "eidlist ::= nm collate sortorder", - /* 224 */ "collate ::=", - /* 225 */ "collate ::= COLLATE ID|STRING", - /* 226 */ "cmd ::= DROP INDEX ifexists fullname", - /* 227 */ "cmd ::= VACUUM", - /* 228 */ "cmd ::= VACUUM nm", - /* 229 */ "cmd ::= PRAGMA nm dbnm", - /* 230 */ "cmd ::= PRAGMA nm dbnm EQ nmnum", - /* 231 */ "cmd ::= PRAGMA nm dbnm LP nmnum RP", - /* 232 */ "cmd ::= PRAGMA nm dbnm EQ minus_num", - /* 233 */ "cmd ::= PRAGMA nm dbnm LP minus_num RP", - /* 234 */ "plus_num ::= PLUS INTEGER|FLOAT", - /* 235 */ "minus_num ::= MINUS INTEGER|FLOAT", - /* 236 */ "cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END", - /* 237 */ "trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause", - /* 238 */ "trigger_time ::= BEFORE|AFTER", - /* 239 */ "trigger_time ::= INSTEAD OF", - /* 240 */ "trigger_time ::=", - /* 241 */ "trigger_event ::= DELETE|INSERT", - /* 242 */ "trigger_event ::= UPDATE", - /* 243 */ "trigger_event ::= UPDATE OF idlist", - /* 244 */ "when_clause ::=", - /* 245 */ "when_clause ::= WHEN expr", - /* 246 */ "trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI", - /* 247 */ "trigger_cmd_list ::= trigger_cmd SEMI", - /* 248 */ "trnm ::= nm DOT nm", - /* 249 */ "tridxby ::= INDEXED BY nm", - /* 250 */ "tridxby ::= NOT INDEXED", - /* 251 */ "trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist where_opt scanpt", - /* 252 */ "trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt", - /* 253 */ "trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt", - /* 254 */ "trigger_cmd ::= scanpt select scanpt", - /* 255 */ "expr ::= RAISE LP IGNORE RP", - /* 256 */ "expr ::= RAISE LP raisetype COMMA nm RP", - /* 257 */ "raisetype ::= ROLLBACK", - /* 258 */ "raisetype ::= ABORT", - /* 259 */ "raisetype ::= FAIL", - /* 260 */ "cmd ::= DROP TRIGGER ifexists fullname", - /* 261 */ "cmd ::= ATTACH database_kw_opt expr AS expr key_opt", - /* 262 */ "cmd ::= DETACH database_kw_opt expr", - /* 263 */ "key_opt ::=", - /* 264 */ "key_opt ::= KEY expr", - /* 265 */ "cmd ::= REINDEX", - /* 266 */ "cmd ::= REINDEX nm dbnm", - /* 267 */ "cmd ::= ANALYZE", - /* 268 */ "cmd ::= ANALYZE nm dbnm", - /* 269 */ "cmd ::= ALTER TABLE fullname RENAME TO nm", - /* 270 */ "cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist", - /* 271 */ "add_column_fullname ::= fullname", - /* 272 */ "cmd ::= create_vtab", - /* 273 */ "cmd ::= create_vtab LP vtabarglist RP", - /* 274 */ "create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm", - /* 275 */ "vtabarg ::=", - /* 276 */ "vtabargtoken ::= ANY", - /* 277 */ "vtabargtoken ::= lp anylist RP", - /* 278 */ "lp ::= LP", - /* 279 */ "with ::= WITH wqlist", - /* 280 */ "with ::= WITH RECURSIVE wqlist", - /* 281 */ "wqlist ::= nm eidlist_opt AS LP select RP", - /* 282 */ "wqlist ::= wqlist COMMA nm eidlist_opt AS LP select RP", - /* 283 */ "input ::= cmdlist", - /* 284 */ "cmdlist ::= cmdlist ecmd", - /* 285 */ "cmdlist ::= ecmd", - /* 286 */ "ecmd ::= SEMI", - /* 287 */ "ecmd ::= cmdx SEMI", - /* 288 */ "ecmd ::= explain cmdx", - /* 289 */ "trans_opt ::=", - /* 290 */ "trans_opt ::= TRANSACTION", - /* 291 */ "trans_opt ::= TRANSACTION nm", - /* 292 */ "savepoint_opt ::= SAVEPOINT", - /* 293 */ "savepoint_opt ::=", - /* 294 */ "cmd ::= create_table create_table_args", - /* 295 */ "columnlist ::= columnlist COMMA columnname carglist", - /* 296 */ "columnlist ::= columnname carglist", - /* 297 */ "nm ::= ID|INDEXED", - /* 298 */ "nm ::= STRING", - /* 299 */ "nm ::= JOIN_KW", - /* 300 */ "typetoken ::= typename", - /* 301 */ "typename ::= ID|STRING", - /* 302 */ "signed ::= plus_num", - /* 303 */ "signed ::= minus_num", - /* 304 */ "carglist ::= carglist ccons", - /* 305 */ "carglist ::=", - /* 306 */ "ccons ::= NULL onconf", - /* 307 */ "conslist_opt ::= COMMA conslist", - /* 308 */ "conslist ::= conslist tconscomma tcons", - /* 309 */ "conslist ::= tcons", - /* 310 */ "tconscomma ::=", - /* 311 */ "defer_subclause_opt ::= defer_subclause", - /* 312 */ "resolvetype ::= raisetype", - /* 313 */ "selectnowith ::= oneselect", - /* 314 */ "oneselect ::= values", - /* 315 */ "sclp ::= selcollist COMMA", - /* 316 */ "as ::= ID|STRING", - /* 317 */ "expr ::= term", - /* 318 */ "likeop ::= LIKE_KW|MATCH", - /* 319 */ "exprlist ::= nexprlist", - /* 320 */ "nmnum ::= plus_num", - /* 321 */ "nmnum ::= nm", - /* 322 */ "nmnum ::= ON", - /* 323 */ "nmnum ::= DELETE", - /* 324 */ "nmnum ::= DEFAULT", - /* 325 */ "plus_num ::= INTEGER|FLOAT", - /* 326 */ "foreach_clause ::=", - /* 327 */ "foreach_clause ::= FOR EACH ROW", - /* 328 */ "trnm ::= nm", - /* 329 */ "tridxby ::=", - /* 330 */ "database_kw_opt ::= DATABASE", - /* 331 */ "database_kw_opt ::=", - /* 332 */ "kwcolumn_opt ::=", - /* 333 */ "kwcolumn_opt ::= COLUMNKW", - /* 334 */ "vtabarglist ::= vtabarg", - /* 335 */ "vtabarglist ::= vtabarglist COMMA vtabarg", - /* 336 */ "vtabarg ::= vtabarg vtabargtoken", - /* 337 */ "anylist ::=", - /* 338 */ "anylist ::= anylist LP anylist RP", - /* 339 */ "anylist ::= anylist ANY", - /* 340 */ "with ::=", +struct CallCount { + i64 nValue; + i64 nStep; + i64 nTotal; }; -#endif /* NDEBUG */ - -#if YYSTACKDEPTH<=0 /* -** Try to increase the size of the parser stack. Return the number -** of errors. Return 0 on success. +** Implementation of built-in window function dense_rank(). Assumes that +** the window frame has been set to: +** +** RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW */ -static int yyGrowStack(yyParser *p){ - int newSize; - int idx; - yyStackEntry *pNew; - - newSize = p->yystksz*2 + 100; - idx = p->yytos ? (int)(p->yytos - p->yystack) : 0; - if( p->yystack==&p->yystk0 ){ - pNew = malloc(newSize*sizeof(pNew[0])); - if( pNew ) pNew[0] = p->yystk0; - }else{ - pNew = realloc(p->yystack, newSize*sizeof(pNew[0])); - } - if( pNew ){ - p->yystack = pNew; - p->yytos = &p->yystack[idx]; -#ifndef NDEBUG - if( yyTraceFILE ){ - fprintf(yyTraceFILE,"%sStack grows from %d to %d entries.\n", - yyTracePrompt, p->yystksz, newSize); +static void dense_rankStepFunc( + sqlite3_context *pCtx, + int nArg, + sqlite3_value **apArg +){ + struct CallCount *p; + p = (struct CallCount*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + if( p ) p->nStep = 1; + UNUSED_PARAMETER(nArg); + UNUSED_PARAMETER(apArg); +} +static void dense_rankValueFunc(sqlite3_context *pCtx){ + struct CallCount *p; + p = (struct CallCount*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + if( p ){ + if( p->nStep ){ + p->nValue++; + p->nStep = 0; } -#endif - p->yystksz = newSize; + sqlite3_result_int64(pCtx, p->nValue); } - return pNew==0; } -#endif -/* Datatype of the argument to the memory allocated passed as the -** second argument to sqlite3ParserAlloc() below. This can be changed by -** putting an appropriate #define in the %include section of the input -** grammar. +/* +** Implementation of built-in window function nth_value(). This +** implementation is used in "slow mode" only - when the EXCLUDE clause +** is not set to the default value "NO OTHERS". */ -#ifndef YYMALLOCARGTYPE -# define YYMALLOCARGTYPE size_t -#endif +struct NthValueCtx { + i64 nStep; + sqlite3_value *pValue; +}; +static void nth_valueStepFunc( + sqlite3_context *pCtx, + int nArg, + sqlite3_value **apArg +){ + struct NthValueCtx *p; + p = (struct NthValueCtx*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + if( p ){ + i64 iVal; + switch( sqlite3_value_numeric_type(apArg[1]) ){ + case SQLITE_INTEGER: + iVal = sqlite3_value_int64(apArg[1]); + break; + case SQLITE_FLOAT: { + double fVal = sqlite3_value_double(apArg[1]); + if( ((i64)fVal)!=fVal ) goto error_out; + iVal = (i64)fVal; + break; + } + default: + goto error_out; + } + if( iVal<=0 ) goto error_out; -/* Initialize a new parser that has already been allocated. -*/ -SQLITE_PRIVATE void sqlite3ParserInit(void *yypRawParser sqlite3ParserCTX_PDECL){ - yyParser *yypParser = (yyParser*)yypRawParser; - sqlite3ParserCTX_STORE -#ifdef YYTRACKMAXSTACKDEPTH - yypParser->yyhwm = 0; -#endif -#if YYSTACKDEPTH<=0 - yypParser->yytos = NULL; - yypParser->yystack = NULL; - yypParser->yystksz = 0; - if( yyGrowStack(yypParser) ){ - yypParser->yystack = &yypParser->yystk0; - yypParser->yystksz = 1; + p->nStep++; + if( iVal==p->nStep ){ + p->pValue = sqlite3_value_dup(apArg[0]); + if( !p->pValue ){ + sqlite3_result_error_nomem(pCtx); + } + } } -#endif -#ifndef YYNOERRORRECOVERY - yypParser->yyerrcnt = -1; -#endif - yypParser->yytos = yypParser->yystack; - yypParser->yystack[0].stateno = 0; - yypParser->yystack[0].major = 0; -#if YYSTACKDEPTH>0 - yypParser->yystackEnd = &yypParser->yystack[YYSTACKDEPTH-1]; -#endif -} + UNUSED_PARAMETER(nArg); + UNUSED_PARAMETER(apArg); + return; -#ifndef sqlite3Parser_ENGINEALWAYSONSTACK -/* -** This function allocates a new parser. -** The only argument is a pointer to a function which works like -** malloc. -** -** Inputs: -** A pointer to the function used to allocate memory. -** -** Outputs: -** A pointer to a parser. This pointer is used in subsequent calls -** to sqlite3Parser and sqlite3ParserFree. -*/ -SQLITE_PRIVATE void *sqlite3ParserAlloc(void *(*mallocProc)(YYMALLOCARGTYPE) sqlite3ParserCTX_PDECL){ - yyParser *yypParser; - yypParser = (yyParser*)(*mallocProc)( (YYMALLOCARGTYPE)sizeof(yyParser) ); - if( yypParser ){ - sqlite3ParserCTX_STORE - sqlite3ParserInit(yypParser sqlite3ParserCTX_PARAM); + error_out: + sqlite3_result_error( + pCtx, "second argument to nth_value must be a positive integer", -1 + ); +} +static void nth_valueFinalizeFunc(sqlite3_context *pCtx){ + struct NthValueCtx *p; + p = (struct NthValueCtx*)sqlite3_aggregate_context(pCtx, 0); + if( p && p->pValue ){ + sqlite3_result_value(pCtx, p->pValue); + sqlite3_value_free(p->pValue); + p->pValue = 0; } - return (void*)yypParser; } -#endif /* sqlite3Parser_ENGINEALWAYSONSTACK */ - +#define nth_valueInvFunc noopStepFunc +#define nth_valueValueFunc noopValueFunc -/* The following function deletes the "minor type" or semantic value -** associated with a symbol. The symbol can be either a terminal -** or nonterminal. "yymajor" is the symbol code, and "yypminor" is -** a pointer to the value to be deleted. The code used to do the -** deletions is derived from the %destructor and/or %token_destructor -** directives of the input grammar. -*/ -static void yy_destructor( - yyParser *yypParser, /* The parser */ - YYCODETYPE yymajor, /* Type code for object to destroy */ - YYMINORTYPE *yypminor /* The object to be destroyed */ +static void first_valueStepFunc( + sqlite3_context *pCtx, + int nArg, + sqlite3_value **apArg ){ - sqlite3ParserARG_FETCH - sqlite3ParserCTX_FETCH - switch( yymajor ){ - /* Here is inserted the actions which take place when a - ** terminal or non-terminal is destroyed. This can happen - ** when the symbol is popped from the stack during a - ** reduce or during error processing or when a parser is - ** being destroyed before it is finished parsing. - ** - ** Note: during a reduce, the only symbols destroyed are those - ** which appear on the RHS of the rule, but which are *not* used - ** inside the C code. - */ -/********* Begin destructor definitions ***************************************/ - case 164: /* select */ - case 196: /* selectnowith */ - case 197: /* oneselect */ - case 208: /* values */ -{ -sqlite3SelectDelete(pParse->db, (yypminor->yy399)); -} - break; - case 174: /* term */ - case 175: /* expr */ - case 203: /* where_opt */ - case 205: /* having_opt */ - case 217: /* on_opt */ - case 230: /* case_operand */ - case 232: /* case_else */ - case 241: /* when_clause */ - case 246: /* key_opt */ -{ -sqlite3ExprDelete(pParse->db, (yypminor->yy182)); -} - break; - case 179: /* eidlist_opt */ - case 188: /* sortlist */ - case 189: /* eidlist */ - case 201: /* selcollist */ - case 204: /* groupby_opt */ - case 206: /* orderby_opt */ - case 209: /* nexprlist */ - case 210: /* exprlist */ - case 211: /* sclp */ - case 222: /* setlist */ - case 229: /* paren_exprlist */ - case 231: /* case_exprlist */ -{ -sqlite3ExprListDelete(pParse->db, (yypminor->yy232)); + struct NthValueCtx *p; + p = (struct NthValueCtx*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + if( p && p->pValue==0 ){ + p->pValue = sqlite3_value_dup(apArg[0]); + if( !p->pValue ){ + sqlite3_result_error_nomem(pCtx); + } + } + UNUSED_PARAMETER(nArg); + UNUSED_PARAMETER(apArg); } - break; - case 195: /* fullname */ - case 202: /* from */ - case 213: /* seltablist */ - case 214: /* stl_prefix */ - case 219: /* xfullname */ -{ -sqlite3SrcListDelete(pParse->db, (yypminor->yy427)); +static void first_valueFinalizeFunc(sqlite3_context *pCtx){ + struct NthValueCtx *p; + p = (struct NthValueCtx*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + if( p && p->pValue ){ + sqlite3_result_value(pCtx, p->pValue); + sqlite3_value_free(p->pValue); + p->pValue = 0; + } } - break; - case 198: /* wqlist */ -{ -sqlite3WithDelete(pParse->db, (yypminor->yy91)); +#define first_valueInvFunc noopStepFunc +#define first_valueValueFunc noopValueFunc + +/* +** Implementation of built-in window function rank(). Assumes that +** the window frame has been set to: +** +** RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW +*/ +static void rankStepFunc( + sqlite3_context *pCtx, + int nArg, + sqlite3_value **apArg +){ + struct CallCount *p; + p = (struct CallCount*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + if( p ){ + p->nStep++; + if( p->nValue==0 ){ + p->nValue = p->nStep; + } + } + UNUSED_PARAMETER(nArg); + UNUSED_PARAMETER(apArg); } - break; - case 218: /* using_opt */ - case 220: /* idlist */ - case 224: /* idlist_opt */ -{ -sqlite3IdListDelete(pParse->db, (yypminor->yy510)); +static void rankValueFunc(sqlite3_context *pCtx){ + struct CallCount *p; + p = (struct CallCount*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + if( p ){ + sqlite3_result_int64(pCtx, p->nValue); + p->nValue = 0; + } } - break; - case 237: /* trigger_cmd_list */ - case 242: /* trigger_cmd */ -{ -sqlite3DeleteTriggerStep(pParse->db, (yypminor->yy47)); + +/* +** Implementation of built-in window function percent_rank(). Assumes that +** the window frame has been set to: +** +** GROUPS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING +*/ +static void percent_rankStepFunc( + sqlite3_context *pCtx, + int nArg, + sqlite3_value **apArg +){ + struct CallCount *p; + UNUSED_PARAMETER(nArg); assert( nArg==0 ); + UNUSED_PARAMETER(apArg); + p = (struct CallCount*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + if( p ){ + p->nTotal++; + } } - break; - case 239: /* trigger_event */ -{ -sqlite3IdListDelete(pParse->db, (yypminor->yy300).b); +static void percent_rankInvFunc( + sqlite3_context *pCtx, + int nArg, + sqlite3_value **apArg +){ + struct CallCount *p; + UNUSED_PARAMETER(nArg); assert( nArg==0 ); + UNUSED_PARAMETER(apArg); + p = (struct CallCount*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + p->nStep++; } - break; -/********* End destructor definitions *****************************************/ - default: break; /* If no destructor action specified: do nothing */ +static void percent_rankValueFunc(sqlite3_context *pCtx){ + struct CallCount *p; + p = (struct CallCount*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + if( p ){ + p->nValue = p->nStep; + if( p->nTotal>1 ){ + double r = (double)p->nValue / (double)(p->nTotal-1); + sqlite3_result_double(pCtx, r); + }else{ + sqlite3_result_double(pCtx, 0.0); + } } } +#define percent_rankFinalizeFunc percent_rankValueFunc /* -** Pop the parser's stack once. +** Implementation of built-in window function cume_dist(). Assumes that +** the window frame has been set to: ** -** If there is a destructor routine associated with the token which -** is popped from the stack, then call it. +** GROUPS BETWEEN 1 FOLLOWING AND UNBOUNDED FOLLOWING */ -static void yy_pop_parser_stack(yyParser *pParser){ - yyStackEntry *yytos; - assert( pParser->yytos!=0 ); - assert( pParser->yytos > pParser->yystack ); - yytos = pParser->yytos--; -#ifndef NDEBUG - if( yyTraceFILE ){ - fprintf(yyTraceFILE,"%sPopping %s\n", - yyTracePrompt, - yyTokenName[yytos->major]); +static void cume_distStepFunc( + sqlite3_context *pCtx, + int nArg, + sqlite3_value **apArg +){ + struct CallCount *p; + UNUSED_PARAMETER(nArg); assert( nArg==0 ); + UNUSED_PARAMETER(apArg); + p = (struct CallCount*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + if( p ){ + p->nTotal++; + } +} +static void cume_distInvFunc( + sqlite3_context *pCtx, + int nArg, + sqlite3_value **apArg +){ + struct CallCount *p; + UNUSED_PARAMETER(nArg); assert( nArg==0 ); + UNUSED_PARAMETER(apArg); + p = (struct CallCount*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + p->nStep++; +} +static void cume_distValueFunc(sqlite3_context *pCtx){ + struct CallCount *p; + p = (struct CallCount*)sqlite3_aggregate_context(pCtx, 0); + if( p ){ + double r = (double)(p->nStep) / (double)(p->nTotal); + sqlite3_result_double(pCtx, r); } -#endif - yy_destructor(pParser, yytos->major, &yytos->minor); } +#define cume_distFinalizeFunc cume_distValueFunc /* -** Clear all secondary memory allocations from the parser +** Context object for ntile() window function. */ -SQLITE_PRIVATE void sqlite3ParserFinalize(void *p){ - yyParser *pParser = (yyParser*)p; - while( pParser->yytos>pParser->yystack ) yy_pop_parser_stack(pParser); -#if YYSTACKDEPTH<=0 - if( pParser->yystack!=&pParser->yystk0 ) free(pParser->yystack); -#endif -} +struct NtileCtx { + i64 nTotal; /* Total rows in partition */ + i64 nParam; /* Parameter passed to ntile(N) */ + i64 iRow; /* Current row */ +}; -#ifndef sqlite3Parser_ENGINEALWAYSONSTACK -/* -** Deallocate and destroy a parser. Destructors are called for -** all stack elements before shutting the parser down. +/* +** Implementation of ntile(). This assumes that the window frame has +** been coerced to: ** -** If the YYPARSEFREENEVERNULL macro exists (for example because it -** is defined in a %include section of the input grammar) then it is -** assumed that the input pointer is never NULL. +** ROWS CURRENT ROW AND UNBOUNDED FOLLOWING */ -SQLITE_PRIVATE void sqlite3ParserFree( - void *p, /* The parser to be deleted */ - void (*freeProc)(void*) /* Function used to reclaim memory */ +static void ntileStepFunc( + sqlite3_context *pCtx, + int nArg, + sqlite3_value **apArg ){ -#ifndef YYPARSEFREENEVERNULL - if( p==0 ) return; -#endif - sqlite3ParserFinalize(p); - (*freeProc)(p); + struct NtileCtx *p; + assert( nArg==1 ); UNUSED_PARAMETER(nArg); + p = (struct NtileCtx*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + if( p ){ + if( p->nTotal==0 ){ + p->nParam = sqlite3_value_int64(apArg[0]); + if( p->nParam<=0 ){ + sqlite3_result_error( + pCtx, "argument of ntile must be a positive integer", -1 + ); + } + } + p->nTotal++; + } } -#endif /* sqlite3Parser_ENGINEALWAYSONSTACK */ +static void ntileInvFunc( + sqlite3_context *pCtx, + int nArg, + sqlite3_value **apArg +){ + struct NtileCtx *p; + assert( nArg==1 ); UNUSED_PARAMETER(nArg); + UNUSED_PARAMETER(apArg); + p = (struct NtileCtx*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + p->iRow++; +} +static void ntileValueFunc(sqlite3_context *pCtx){ + struct NtileCtx *p; + p = (struct NtileCtx*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + if( p && p->nParam>0 ){ + int nSize = (p->nTotal / p->nParam); + if( nSize==0 ){ + sqlite3_result_int64(pCtx, p->iRow+1); + }else{ + i64 nLarge = p->nTotal - p->nParam*nSize; + i64 iSmall = nLarge*(nSize+1); + i64 iRow = p->iRow; + + assert( (nLarge*(nSize+1) + (p->nParam-nLarge)*nSize)==p->nTotal ); + + if( iRowyyhwm; -} -#endif +struct LastValueCtx { + sqlite3_value *pVal; + int nVal; +}; -/* This array of booleans keeps track of the parser statement -** coverage. The element yycoverage[X][Y] is set when the parser -** is in state X and has a lookahead token Y. In a well-tested -** systems, every element of this matrix should end up being set. +/* +** Implementation of last_value(). */ -#if defined(YYCOVERAGE) -static unsigned char yycoverage[YYNSTATE][YYNTOKEN]; -#endif +static void last_valueStepFunc( + sqlite3_context *pCtx, + int nArg, + sqlite3_value **apArg +){ + struct LastValueCtx *p; + UNUSED_PARAMETER(nArg); + p = (struct LastValueCtx*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + if( p ){ + sqlite3_value_free(p->pVal); + p->pVal = sqlite3_value_dup(apArg[0]); + if( p->pVal==0 ){ + sqlite3_result_error_nomem(pCtx); + }else{ + p->nVal++; + } + } +} +static void last_valueInvFunc( + sqlite3_context *pCtx, + int nArg, + sqlite3_value **apArg +){ + struct LastValueCtx *p; + UNUSED_PARAMETER(nArg); + UNUSED_PARAMETER(apArg); + p = (struct LastValueCtx*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + if( ALWAYS(p) ){ + p->nVal--; + if( p->nVal==0 ){ + sqlite3_value_free(p->pVal); + p->pVal = 0; + } + } +} +static void last_valueValueFunc(sqlite3_context *pCtx){ + struct LastValueCtx *p; + p = (struct LastValueCtx*)sqlite3_aggregate_context(pCtx, 0); + if( p && p->pVal ){ + sqlite3_result_value(pCtx, p->pVal); + } +} +static void last_valueFinalizeFunc(sqlite3_context *pCtx){ + struct LastValueCtx *p; + p = (struct LastValueCtx*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + if( p && p->pVal ){ + sqlite3_result_value(pCtx, p->pVal); + sqlite3_value_free(p->pVal); + p->pVal = 0; + } +} + +/* +** Static names for the built-in window function names. These static +** names are used, rather than string literals, so that FuncDef objects +** can be associated with a particular window function by direct +** comparison of the zName pointer. Example: +** +** if( pFuncDef->zName==row_valueName ){ ... } +*/ +static const char row_numberName[] = "row_number"; +static const char dense_rankName[] = "dense_rank"; +static const char rankName[] = "rank"; +static const char percent_rankName[] = "percent_rank"; +static const char cume_distName[] = "cume_dist"; +static const char ntileName[] = "ntile"; +static const char last_valueName[] = "last_value"; +static const char nth_valueName[] = "nth_value"; +static const char first_valueName[] = "first_value"; +static const char leadName[] = "lead"; +static const char lagName[] = "lag"; + +/* +** No-op implementations of xStep() and xFinalize(). Used as place-holders +** for built-in window functions that never call those interfaces. +** +** The noopValueFunc() is called but is expected to do nothing. The +** noopStepFunc() is never called, and so it is marked with NO_TEST to +** let the test coverage routine know not to expect this function to be +** invoked. +*/ +static void noopStepFunc( /*NO_TEST*/ + sqlite3_context *p, /*NO_TEST*/ + int n, /*NO_TEST*/ + sqlite3_value **a /*NO_TEST*/ +){ /*NO_TEST*/ + UNUSED_PARAMETER(p); /*NO_TEST*/ + UNUSED_PARAMETER(n); /*NO_TEST*/ + UNUSED_PARAMETER(a); /*NO_TEST*/ + assert(0); /*NO_TEST*/ +} /*NO_TEST*/ +static void noopValueFunc(sqlite3_context *p){ UNUSED_PARAMETER(p); /*no-op*/ } + +/* Window functions that use all window interfaces: xStep, xFinal, +** xValue, and xInverse */ +#define WINDOWFUNCALL(name,nArg,extra) { \ + nArg, (SQLITE_UTF8|SQLITE_FUNC_WINDOW|extra), 0, 0, \ + name ## StepFunc, name ## FinalizeFunc, name ## ValueFunc, \ + name ## InvFunc, name ## Name, {0} \ +} + +/* Window functions that are implemented using bytecode and thus have +** no-op routines for their methods */ +#define WINDOWFUNCNOOP(name,nArg,extra) { \ + nArg, (SQLITE_UTF8|SQLITE_FUNC_WINDOW|extra), 0, 0, \ + noopStepFunc, noopValueFunc, noopValueFunc, \ + noopStepFunc, name ## Name, {0} \ +} + +/* Window functions that use all window interfaces: xStep, the +** same routine for xFinalize and xValue and which never call +** xInverse. */ +#define WINDOWFUNCX(name,nArg,extra) { \ + nArg, (SQLITE_UTF8|SQLITE_FUNC_WINDOW|extra), 0, 0, \ + name ## StepFunc, name ## ValueFunc, name ## ValueFunc, \ + noopStepFunc, name ## Name, {0} \ +} + + +/* +** Register those built-in window functions that are not also aggregates. +*/ +SQLITE_PRIVATE void sqlite3WindowFunctions(void){ + static FuncDef aWindowFuncs[] = { + WINDOWFUNCX(row_number, 0, 0), + WINDOWFUNCX(dense_rank, 0, 0), + WINDOWFUNCX(rank, 0, 0), + WINDOWFUNCALL(percent_rank, 0, 0), + WINDOWFUNCALL(cume_dist, 0, 0), + WINDOWFUNCALL(ntile, 1, 0), + WINDOWFUNCALL(last_value, 1, 0), + WINDOWFUNCALL(nth_value, 2, 0), + WINDOWFUNCALL(first_value, 1, 0), + WINDOWFUNCNOOP(lead, 1, 0), + WINDOWFUNCNOOP(lead, 2, 0), + WINDOWFUNCNOOP(lead, 3, 0), + WINDOWFUNCNOOP(lag, 1, 0), + WINDOWFUNCNOOP(lag, 2, 0), + WINDOWFUNCNOOP(lag, 3, 0), + }; + sqlite3InsertBuiltinFuncs(aWindowFuncs, ArraySize(aWindowFuncs)); +} + +static Window *windowFind(Parse *pParse, Window *pList, const char *zName){ + Window *p; + for(p=pList; p; p=p->pNextWin){ + if( sqlite3StrICmp(p->zName, zName)==0 ) break; + } + if( p==0 ){ + sqlite3ErrorMsg(pParse, "no such window: %s", zName); + } + return p; +} /* -** Write into out a description of every state/lookahead combination that +** This function is called immediately after resolving the function name +** for a window function within a SELECT statement. Argument pList is a +** linked list of WINDOW definitions for the current SELECT statement. +** Argument pFunc is the function definition just resolved and pWin +** is the Window object representing the associated OVER clause. This +** function updates the contents of pWin as follows: ** -** (1) has not been used by the parser, and -** (2) is not a syntax error. +** * If the OVER clause refered to a named window (as in "max(x) OVER win"), +** search list pList for a matching WINDOW definition, and update pWin +** accordingly. If no such WINDOW clause can be found, leave an error +** in pParse. ** -** Return the number of missed state/lookahead combinations. +** * If the function is a built-in window function that requires the +** window to be coerced (see "BUILT-IN WINDOW FUNCTIONS" at the top +** of this file), pWin is updated here. */ -#if defined(YYCOVERAGE) -SQLITE_PRIVATE int sqlite3ParserCoverage(FILE *out){ - int stateno, iLookAhead, i; - int nMissed = 0; - for(stateno=0; statenozName && pWin->eFrmType==0 ){ + Window *p = windowFind(pParse, pList, pWin->zName); + if( p==0 ) return; + pWin->pPartition = sqlite3ExprListDup(pParse->db, p->pPartition, 0); + pWin->pOrderBy = sqlite3ExprListDup(pParse->db, p->pOrderBy, 0); + pWin->pStart = sqlite3ExprDup(pParse->db, p->pStart, 0); + pWin->pEnd = sqlite3ExprDup(pParse->db, p->pEnd, 0); + pWin->eStart = p->eStart; + pWin->eEnd = p->eEnd; + pWin->eFrmType = p->eFrmType; + pWin->eExclude = p->eExclude; + }else{ + sqlite3WindowChain(pParse, pWin, pList); + } + if( (pWin->eFrmType==TK_RANGE) + && (pWin->pStart || pWin->pEnd) + && (pWin->pOrderBy==0 || pWin->pOrderBy->nExpr!=1) + ){ + sqlite3ErrorMsg(pParse, + "RANGE with offset PRECEDING/FOLLOWING requires one ORDER BY expression" + ); + }else + if( pFunc->funcFlags & SQLITE_FUNC_WINDOW ){ + sqlite3 *db = pParse->db; + if( pWin->pFilter ){ + sqlite3ErrorMsg(pParse, + "FILTER clause may only be used with aggregate window functions" + ); + }else{ + struct WindowUpdate { + const char *zFunc; + int eFrmType; + int eStart; + int eEnd; + } aUp[] = { + { row_numberName, TK_ROWS, TK_UNBOUNDED, TK_CURRENT }, + { dense_rankName, TK_RANGE, TK_UNBOUNDED, TK_CURRENT }, + { rankName, TK_RANGE, TK_UNBOUNDED, TK_CURRENT }, + { percent_rankName, TK_GROUPS, TK_CURRENT, TK_UNBOUNDED }, + { cume_distName, TK_GROUPS, TK_FOLLOWING, TK_UNBOUNDED }, + { ntileName, TK_ROWS, TK_CURRENT, TK_UNBOUNDED }, + { leadName, TK_ROWS, TK_UNBOUNDED, TK_UNBOUNDED }, + { lagName, TK_ROWS, TK_UNBOUNDED, TK_CURRENT }, + }; + int i; + for(i=0; izName==aUp[i].zFunc ){ + sqlite3ExprDelete(db, pWin->pStart); + sqlite3ExprDelete(db, pWin->pEnd); + pWin->pEnd = pWin->pStart = 0; + pWin->eFrmType = aUp[i].eFrmType; + pWin->eStart = aUp[i].eStart; + pWin->eEnd = aUp[i].eEnd; + pWin->eExclude = 0; + if( pWin->eStart==TK_FOLLOWING ){ + pWin->pStart = sqlite3Expr(db, TK_INTEGER, "1"); + } + break; + } } } } - return nMissed; + pWin->pFunc = pFunc; } -#endif /* -** Find the appropriate action for a parser given the terminal -** look-ahead token iLookAhead. +** Context object passed through sqlite3WalkExprList() to +** selectWindowRewriteExprCb() by selectWindowRewriteEList(). */ -static YYACTIONTYPE yy_find_shift_action( - YYCODETYPE iLookAhead, /* The look-ahead token */ - YYACTIONTYPE stateno /* Current state number */ -){ - int i; +typedef struct WindowRewrite WindowRewrite; +struct WindowRewrite { + Window *pWin; + SrcList *pSrc; + ExprList *pSub; + Table *pTab; + Select *pSubSelect; /* Current sub-select, if any */ +}; - if( stateno>YY_MAX_SHIFT ) return stateno; - assert( stateno <= YY_SHIFT_COUNT ); -#if defined(YYCOVERAGE) - yycoverage[stateno][iLookAhead] = 1; -#endif - do{ - i = yy_shift_ofst[stateno]; - assert( i>=0 ); - assert( i+YYNTOKEN<=(int)sizeof(yy_lookahead)/sizeof(yy_lookahead[0]) ); - assert( iLookAhead!=YYNOCODE ); - assert( iLookAhead < YYNTOKEN ); - i += iLookAhead; - if( yy_lookahead[i]!=iLookAhead ){ -#ifdef YYFALLBACK - YYCODETYPE iFallback; /* Fallback token */ - if( iLookAhead %s\n", - yyTracePrompt, yyTokenName[iLookAhead], yyTokenName[iFallback]); +/* +** Callback function used by selectWindowRewriteEList(). If necessary, +** this function appends to the output expression-list and updates +** expression (*ppExpr) in place. +*/ +static int selectWindowRewriteExprCb(Walker *pWalker, Expr *pExpr){ + struct WindowRewrite *p = pWalker->u.pRewrite; + Parse *pParse = pWalker->pParse; + assert( p!=0 ); + assert( p->pWin!=0 ); + + /* If this function is being called from within a scalar sub-select + ** that used by the SELECT statement being processed, only process + ** TK_COLUMN expressions that refer to it (the outer SELECT). Do + ** not process aggregates or window functions at all, as they belong + ** to the scalar sub-select. */ + if( p->pSubSelect ){ + if( pExpr->op!=TK_COLUMN ){ + return WRC_Continue; + }else{ + int nSrc = p->pSrc->nSrc; + int i; + for(i=0; iiTable==p->pSrc->a[i].iCursor ) break; + } + if( i==nSrc ) return WRC_Continue; + } + } + + switch( pExpr->op ){ + + case TK_FUNCTION: + if( !ExprHasProperty(pExpr, EP_WinFunc) ){ + break; + }else{ + Window *pWin; + for(pWin=p->pWin; pWin; pWin=pWin->pNextWin){ + if( pExpr->y.pWin==pWin ){ + assert( pWin->pOwner==pExpr ); + return WRC_Prune; + } } -#endif - assert( yyFallback[iFallback]==0 ); /* Fallback loop must terminate */ - iLookAhead = iFallback; - continue; } -#endif -#ifdef YYWILDCARD - { - int j = i - iLookAhead + YYWILDCARD; - if( -#if YY_SHIFT_MIN+YYWILDCARD<0 - j>=0 && -#endif -#if YY_SHIFT_MAX+YYWILDCARD>=YY_ACTTAB_COUNT - j0 - ){ -#ifndef NDEBUG - if( yyTraceFILE ){ - fprintf(yyTraceFILE, "%sWILDCARD %s => %s\n", - yyTracePrompt, yyTokenName[iLookAhead], - yyTokenName[YYWILDCARD]); + /* Fall through. */ + + case TK_AGG_FUNCTION: + case TK_COLUMN: { + int iCol = -1; + if( p->pSub ){ + int i; + for(i=0; ipSub->nExpr; i++){ + if( 0==sqlite3ExprCompare(0, p->pSub->a[i].pExpr, pExpr, -1) ){ + iCol = i; + break; } -#endif /* NDEBUG */ - return yy_action[j]; } } -#endif /* YYWILDCARD */ - return yy_default[stateno]; - }else{ - return yy_action[i]; + if( iCol<0 ){ + Expr *pDup = sqlite3ExprDup(pParse->db, pExpr, 0); + if( pDup && pDup->op==TK_AGG_FUNCTION ) pDup->op = TK_FUNCTION; + p->pSub = sqlite3ExprListAppend(pParse, p->pSub, pDup); + } + if( p->pSub ){ + assert( ExprHasProperty(pExpr, EP_Static)==0 ); + ExprSetProperty(pExpr, EP_Static); + sqlite3ExprDelete(pParse->db, pExpr); + ExprClearProperty(pExpr, EP_Static); + memset(pExpr, 0, sizeof(Expr)); + + pExpr->op = TK_COLUMN; + pExpr->iColumn = (iCol<0 ? p->pSub->nExpr-1: iCol); + pExpr->iTable = p->pWin->iEphCsr; + pExpr->y.pTab = p->pTab; + } + if( pParse->db->mallocFailed ) return WRC_Abort; + break; } - }while(1); -} -/* -** Find the appropriate action for a parser given the non-terminal -** look-ahead token iLookAhead. -*/ -static int yy_find_reduce_action( - YYACTIONTYPE stateno, /* Current state number */ - YYCODETYPE iLookAhead /* The look-ahead token */ -){ - int i; -#ifdef YYERRORSYMBOL - if( stateno>YY_REDUCE_COUNT ){ - return yy_default[stateno]; + default: /* no-op */ + break; } -#else - assert( stateno<=YY_REDUCE_COUNT ); -#endif - i = yy_reduce_ofst[stateno]; - assert( iLookAhead!=YYNOCODE ); - i += iLookAhead; -#ifdef YYERRORSYMBOL - if( i<0 || i>=YY_ACTTAB_COUNT || yy_lookahead[i]!=iLookAhead ){ - return yy_default[stateno]; + + return WRC_Continue; +} +static int selectWindowRewriteSelectCb(Walker *pWalker, Select *pSelect){ + struct WindowRewrite *p = pWalker->u.pRewrite; + Select *pSave = p->pSubSelect; + if( pSave==pSelect ){ + return WRC_Continue; + }else{ + p->pSubSelect = pSelect; + sqlite3WalkSelect(pWalker, pSelect); + p->pSubSelect = pSave; } -#else - assert( i>=0 && iiEphCsr, where N is the number of elements in (*ppSub) after +** appending the new one. */ -static void yyStackOverflow(yyParser *yypParser){ - sqlite3ParserARG_FETCH - sqlite3ParserCTX_FETCH -#ifndef NDEBUG - if( yyTraceFILE ){ - fprintf(yyTraceFILE,"%sStack Overflow!\n",yyTracePrompt); - } -#endif - while( yypParser->yytos>yypParser->yystack ) yy_pop_parser_stack(yypParser); - /* Here code is inserted which will execute if the parser - ** stack every overflows */ -/******** Begin %stack_overflow code ******************************************/ +static void selectWindowRewriteEList( + Parse *pParse, + Window *pWin, + SrcList *pSrc, + ExprList *pEList, /* Rewrite expressions in this list */ + Table *pTab, + ExprList **ppSub /* IN/OUT: Sub-select expression-list */ +){ + Walker sWalker; + WindowRewrite sRewrite; - sqlite3ErrorMsg(pParse, "parser stack overflow"); -/******** End %stack_overflow code ********************************************/ - sqlite3ParserARG_STORE /* Suppress warning about unused %extra_argument var */ - sqlite3ParserCTX_STORE + assert( pWin!=0 ); + memset(&sWalker, 0, sizeof(Walker)); + memset(&sRewrite, 0, sizeof(WindowRewrite)); + + sRewrite.pSub = *ppSub; + sRewrite.pWin = pWin; + sRewrite.pSrc = pSrc; + sRewrite.pTab = pTab; + + sWalker.pParse = pParse; + sWalker.xExprCallback = selectWindowRewriteExprCb; + sWalker.xSelectCallback = selectWindowRewriteSelectCb; + sWalker.u.pRewrite = &sRewrite; + + (void)sqlite3WalkExprList(&sWalker, pEList); + + *ppSub = sRewrite.pSub; } /* -** Print tracing information for a SHIFT action +** Append a copy of each expression in expression-list pAppend to +** expression list pList. Return a pointer to the result list. */ -#ifndef NDEBUG -static void yyTraceShift(yyParser *yypParser, int yyNewState, const char *zTag){ - if( yyTraceFILE ){ - if( yyNewStateyytos->major], - yyNewState); - }else{ - fprintf(yyTraceFILE,"%s%s '%s', pending reduce %d\n", - yyTracePrompt, zTag, yyTokenName[yypParser->yytos->major], - yyNewState - YY_MIN_REDUCE); +static ExprList *exprListAppendList( + Parse *pParse, /* Parsing context */ + ExprList *pList, /* List to which to append. Might be NULL */ + ExprList *pAppend, /* List of values to append. Might be NULL */ + int bIntToNull +){ + if( pAppend ){ + int i; + int nInit = pList ? pList->nExpr : 0; + for(i=0; inExpr; i++){ + Expr *pDup = sqlite3ExprDup(pParse->db, pAppend->a[i].pExpr, 0); + assert( pDup==0 || !ExprHasProperty(pDup, EP_MemToken) ); + if( bIntToNull && pDup ){ + int iDummy; + Expr *pSub; + for(pSub=pDup; ExprHasProperty(pSub, EP_Skip); pSub=pSub->pLeft){ + assert( pSub ); + } + if( sqlite3ExprIsInteger(pSub, &iDummy) ){ + pSub->op = TK_NULL; + pSub->flags &= ~(EP_IntValue|EP_IsTrue|EP_IsFalse); + pSub->u.zToken = 0; + } + } + pList = sqlite3ExprListAppend(pParse, pList, pDup); + if( pList ) pList->a[nInit+i].sortFlags = pAppend->a[i].sortFlags; } } + return pList; } -#else -# define yyTraceShift(X,Y,Z) -#endif /* -** Perform a shift action. +** When rewriting a query, if the new subquery in the FROM clause +** contains TK_AGG_FUNCTION nodes that refer to an outer query, +** then we have to increase the Expr->op2 values of those nodes +** due to the extra subquery layer that was added. +** +** See also the incrAggDepth() routine in resolve.c */ -static void yy_shift( - yyParser *yypParser, /* The parser to be shifted */ - YYACTIONTYPE yyNewState, /* The new state to shift in */ - YYCODETYPE yyMajor, /* The major token to shift in */ - sqlite3ParserTOKENTYPE yyMinor /* The minor token to shift in */ -){ - yyStackEntry *yytos; - yypParser->yytos++; -#ifdef YYTRACKMAXSTACKDEPTH - if( (int)(yypParser->yytos - yypParser->yystack)>yypParser->yyhwm ){ - yypParser->yyhwm++; - assert( yypParser->yyhwm == (int)(yypParser->yytos - yypParser->yystack) ); - } -#endif -#if YYSTACKDEPTH>0 - if( yypParser->yytos>yypParser->yystackEnd ){ - yypParser->yytos--; - yyStackOverflow(yypParser); - return; - } -#else - if( yypParser->yytos>=&yypParser->yystack[yypParser->yystksz] ){ - if( yyGrowStack(yypParser) ){ - yypParser->yytos--; - yyStackOverflow(yypParser); - return; - } - } -#endif - if( yyNewState > YY_MAX_SHIFT ){ - yyNewState += YY_MIN_REDUCE - YY_MIN_SHIFTREDUCE; +static int sqlite3WindowExtraAggFuncDepth(Walker *pWalker, Expr *pExpr){ + if( pExpr->op==TK_AGG_FUNCTION + && pExpr->op2>=pWalker->walkerDepth + ){ + pExpr->op2++; } - yytos = yypParser->yytos; - yytos->stateno = yyNewState; - yytos->major = yyMajor; - yytos->minor.yy0 = yyMinor; - yyTraceShift(yypParser, yyNewState, "Shift"); + return WRC_Continue; } -/* The following table contains information about every rule that -** is used during the reduce. -*/ -static const struct { - YYCODETYPE lhs; /* Symbol on the left-hand side of the rule */ - signed char nrhs; /* Negative of the number of RHS symbols in the rule */ -} yyRuleInfo[] = { - { 149, -1 }, /* (0) explain ::= EXPLAIN */ - { 149, -3 }, /* (1) explain ::= EXPLAIN QUERY PLAN */ - { 148, -1 }, /* (2) cmdx ::= cmd */ - { 150, -3 }, /* (3) cmd ::= BEGIN transtype trans_opt */ - { 151, 0 }, /* (4) transtype ::= */ - { 151, -1 }, /* (5) transtype ::= DEFERRED */ - { 151, -1 }, /* (6) transtype ::= IMMEDIATE */ - { 151, -1 }, /* (7) transtype ::= EXCLUSIVE */ - { 150, -2 }, /* (8) cmd ::= COMMIT|END trans_opt */ - { 150, -2 }, /* (9) cmd ::= ROLLBACK trans_opt */ - { 150, -2 }, /* (10) cmd ::= SAVEPOINT nm */ - { 150, -3 }, /* (11) cmd ::= RELEASE savepoint_opt nm */ - { 150, -5 }, /* (12) cmd ::= ROLLBACK trans_opt TO savepoint_opt nm */ - { 155, -6 }, /* (13) create_table ::= createkw temp TABLE ifnotexists nm dbnm */ - { 157, -1 }, /* (14) createkw ::= CREATE */ - { 159, 0 }, /* (15) ifnotexists ::= */ - { 159, -3 }, /* (16) ifnotexists ::= IF NOT EXISTS */ - { 158, -1 }, /* (17) temp ::= TEMP */ - { 158, 0 }, /* (18) temp ::= */ - { 156, -5 }, /* (19) create_table_args ::= LP columnlist conslist_opt RP table_options */ - { 156, -2 }, /* (20) create_table_args ::= AS select */ - { 163, 0 }, /* (21) table_options ::= */ - { 163, -2 }, /* (22) table_options ::= WITHOUT nm */ - { 165, -2 }, /* (23) columnname ::= nm typetoken */ - { 167, 0 }, /* (24) typetoken ::= */ - { 167, -4 }, /* (25) typetoken ::= typename LP signed RP */ - { 167, -6 }, /* (26) typetoken ::= typename LP signed COMMA signed RP */ - { 168, -2 }, /* (27) typename ::= typename ID|STRING */ - { 172, 0 }, /* (28) scanpt ::= */ - { 173, -2 }, /* (29) ccons ::= CONSTRAINT nm */ - { 173, -4 }, /* (30) ccons ::= DEFAULT scanpt term scanpt */ - { 173, -4 }, /* (31) ccons ::= DEFAULT LP expr RP */ - { 173, -4 }, /* (32) ccons ::= DEFAULT PLUS term scanpt */ - { 173, -4 }, /* (33) ccons ::= DEFAULT MINUS term scanpt */ - { 173, -3 }, /* (34) ccons ::= DEFAULT scanpt ID|INDEXED */ - { 173, -3 }, /* (35) ccons ::= NOT NULL onconf */ - { 173, -5 }, /* (36) ccons ::= PRIMARY KEY sortorder onconf autoinc */ - { 173, -2 }, /* (37) ccons ::= UNIQUE onconf */ - { 173, -4 }, /* (38) ccons ::= CHECK LP expr RP */ - { 173, -4 }, /* (39) ccons ::= REFERENCES nm eidlist_opt refargs */ - { 173, -1 }, /* (40) ccons ::= defer_subclause */ - { 173, -2 }, /* (41) ccons ::= COLLATE ID|STRING */ - { 178, 0 }, /* (42) autoinc ::= */ - { 178, -1 }, /* (43) autoinc ::= AUTOINCR */ - { 180, 0 }, /* (44) refargs ::= */ - { 180, -2 }, /* (45) refargs ::= refargs refarg */ - { 182, -2 }, /* (46) refarg ::= MATCH nm */ - { 182, -3 }, /* (47) refarg ::= ON INSERT refact */ - { 182, -3 }, /* (48) refarg ::= ON DELETE refact */ - { 182, -3 }, /* (49) refarg ::= ON UPDATE refact */ - { 183, -2 }, /* (50) refact ::= SET NULL */ - { 183, -2 }, /* (51) refact ::= SET DEFAULT */ - { 183, -1 }, /* (52) refact ::= CASCADE */ - { 183, -1 }, /* (53) refact ::= RESTRICT */ - { 183, -2 }, /* (54) refact ::= NO ACTION */ - { 181, -3 }, /* (55) defer_subclause ::= NOT DEFERRABLE init_deferred_pred_opt */ - { 181, -2 }, /* (56) defer_subclause ::= DEFERRABLE init_deferred_pred_opt */ - { 184, 0 }, /* (57) init_deferred_pred_opt ::= */ - { 184, -2 }, /* (58) init_deferred_pred_opt ::= INITIALLY DEFERRED */ - { 184, -2 }, /* (59) init_deferred_pred_opt ::= INITIALLY IMMEDIATE */ - { 162, 0 }, /* (60) conslist_opt ::= */ - { 186, -1 }, /* (61) tconscomma ::= COMMA */ - { 187, -2 }, /* (62) tcons ::= CONSTRAINT nm */ - { 187, -7 }, /* (63) tcons ::= PRIMARY KEY LP sortlist autoinc RP onconf */ - { 187, -5 }, /* (64) tcons ::= UNIQUE LP sortlist RP onconf */ - { 187, -5 }, /* (65) tcons ::= CHECK LP expr RP onconf */ - { 187, -10 }, /* (66) tcons ::= FOREIGN KEY LP eidlist RP REFERENCES nm eidlist_opt refargs defer_subclause_opt */ - { 190, 0 }, /* (67) defer_subclause_opt ::= */ - { 176, 0 }, /* (68) onconf ::= */ - { 176, -3 }, /* (69) onconf ::= ON CONFLICT resolvetype */ - { 191, 0 }, /* (70) orconf ::= */ - { 191, -2 }, /* (71) orconf ::= OR resolvetype */ - { 192, -1 }, /* (72) resolvetype ::= IGNORE */ - { 192, -1 }, /* (73) resolvetype ::= REPLACE */ - { 150, -4 }, /* (74) cmd ::= DROP TABLE ifexists fullname */ - { 194, -2 }, /* (75) ifexists ::= IF EXISTS */ - { 194, 0 }, /* (76) ifexists ::= */ - { 150, -9 }, /* (77) cmd ::= createkw temp VIEW ifnotexists nm dbnm eidlist_opt AS select */ - { 150, -4 }, /* (78) cmd ::= DROP VIEW ifexists fullname */ - { 150, -1 }, /* (79) cmd ::= select */ - { 164, -3 }, /* (80) select ::= WITH wqlist selectnowith */ - { 164, -4 }, /* (81) select ::= WITH RECURSIVE wqlist selectnowith */ - { 164, -1 }, /* (82) select ::= selectnowith */ - { 196, -3 }, /* (83) selectnowith ::= selectnowith multiselect_op oneselect */ - { 199, -1 }, /* (84) multiselect_op ::= UNION */ - { 199, -2 }, /* (85) multiselect_op ::= UNION ALL */ - { 199, -1 }, /* (86) multiselect_op ::= EXCEPT|INTERSECT */ - { 197, -9 }, /* (87) oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt orderby_opt limit_opt */ - { 208, -4 }, /* (88) values ::= VALUES LP nexprlist RP */ - { 208, -5 }, /* (89) values ::= values COMMA LP exprlist RP */ - { 200, -1 }, /* (90) distinct ::= DISTINCT */ - { 200, -1 }, /* (91) distinct ::= ALL */ - { 200, 0 }, /* (92) distinct ::= */ - { 211, 0 }, /* (93) sclp ::= */ - { 201, -5 }, /* (94) selcollist ::= sclp scanpt expr scanpt as */ - { 201, -3 }, /* (95) selcollist ::= sclp scanpt STAR */ - { 201, -5 }, /* (96) selcollist ::= sclp scanpt nm DOT STAR */ - { 212, -2 }, /* (97) as ::= AS nm */ - { 212, 0 }, /* (98) as ::= */ - { 202, 0 }, /* (99) from ::= */ - { 202, -2 }, /* (100) from ::= FROM seltablist */ - { 214, -2 }, /* (101) stl_prefix ::= seltablist joinop */ - { 214, 0 }, /* (102) stl_prefix ::= */ - { 213, -7 }, /* (103) seltablist ::= stl_prefix nm dbnm as indexed_opt on_opt using_opt */ - { 213, -9 }, /* (104) seltablist ::= stl_prefix nm dbnm LP exprlist RP as on_opt using_opt */ - { 213, -7 }, /* (105) seltablist ::= stl_prefix LP select RP as on_opt using_opt */ - { 213, -7 }, /* (106) seltablist ::= stl_prefix LP seltablist RP as on_opt using_opt */ - { 160, 0 }, /* (107) dbnm ::= */ - { 160, -2 }, /* (108) dbnm ::= DOT nm */ - { 195, -1 }, /* (109) fullname ::= nm */ - { 195, -3 }, /* (110) fullname ::= nm DOT nm */ - { 219, -1 }, /* (111) xfullname ::= nm */ - { 219, -3 }, /* (112) xfullname ::= nm DOT nm */ - { 219, -5 }, /* (113) xfullname ::= nm DOT nm AS nm */ - { 219, -3 }, /* (114) xfullname ::= nm AS nm */ - { 215, -1 }, /* (115) joinop ::= COMMA|JOIN */ - { 215, -2 }, /* (116) joinop ::= JOIN_KW JOIN */ - { 215, -3 }, /* (117) joinop ::= JOIN_KW nm JOIN */ - { 215, -4 }, /* (118) joinop ::= JOIN_KW nm nm JOIN */ - { 217, -2 }, /* (119) on_opt ::= ON expr */ - { 217, 0 }, /* (120) on_opt ::= */ - { 216, 0 }, /* (121) indexed_opt ::= */ - { 216, -3 }, /* (122) indexed_opt ::= INDEXED BY nm */ - { 216, -2 }, /* (123) indexed_opt ::= NOT INDEXED */ - { 218, -4 }, /* (124) using_opt ::= USING LP idlist RP */ - { 218, 0 }, /* (125) using_opt ::= */ - { 206, 0 }, /* (126) orderby_opt ::= */ - { 206, -3 }, /* (127) orderby_opt ::= ORDER BY sortlist */ - { 188, -4 }, /* (128) sortlist ::= sortlist COMMA expr sortorder */ - { 188, -2 }, /* (129) sortlist ::= expr sortorder */ - { 177, -1 }, /* (130) sortorder ::= ASC */ - { 177, -1 }, /* (131) sortorder ::= DESC */ - { 177, 0 }, /* (132) sortorder ::= */ - { 204, 0 }, /* (133) groupby_opt ::= */ - { 204, -3 }, /* (134) groupby_opt ::= GROUP BY nexprlist */ - { 205, 0 }, /* (135) having_opt ::= */ - { 205, -2 }, /* (136) having_opt ::= HAVING expr */ - { 207, 0 }, /* (137) limit_opt ::= */ - { 207, -2 }, /* (138) limit_opt ::= LIMIT expr */ - { 207, -4 }, /* (139) limit_opt ::= LIMIT expr OFFSET expr */ - { 207, -4 }, /* (140) limit_opt ::= LIMIT expr COMMA expr */ - { 150, -6 }, /* (141) cmd ::= with DELETE FROM xfullname indexed_opt where_opt */ - { 203, 0 }, /* (142) where_opt ::= */ - { 203, -2 }, /* (143) where_opt ::= WHERE expr */ - { 150, -8 }, /* (144) cmd ::= with UPDATE orconf xfullname indexed_opt SET setlist where_opt */ - { 222, -5 }, /* (145) setlist ::= setlist COMMA nm EQ expr */ - { 222, -7 }, /* (146) setlist ::= setlist COMMA LP idlist RP EQ expr */ - { 222, -3 }, /* (147) setlist ::= nm EQ expr */ - { 222, -5 }, /* (148) setlist ::= LP idlist RP EQ expr */ - { 150, -7 }, /* (149) cmd ::= with insert_cmd INTO xfullname idlist_opt select upsert */ - { 150, -7 }, /* (150) cmd ::= with insert_cmd INTO xfullname idlist_opt DEFAULT VALUES */ - { 225, 0 }, /* (151) upsert ::= */ - { 225, -11 }, /* (152) upsert ::= ON CONFLICT LP sortlist RP where_opt DO UPDATE SET setlist where_opt */ - { 225, -8 }, /* (153) upsert ::= ON CONFLICT LP sortlist RP where_opt DO NOTHING */ - { 225, -4 }, /* (154) upsert ::= ON CONFLICT DO NOTHING */ - { 223, -2 }, /* (155) insert_cmd ::= INSERT orconf */ - { 223, -1 }, /* (156) insert_cmd ::= REPLACE */ - { 224, 0 }, /* (157) idlist_opt ::= */ - { 224, -3 }, /* (158) idlist_opt ::= LP idlist RP */ - { 220, -3 }, /* (159) idlist ::= idlist COMMA nm */ - { 220, -1 }, /* (160) idlist ::= nm */ - { 175, -3 }, /* (161) expr ::= LP expr RP */ - { 175, -1 }, /* (162) expr ::= ID|INDEXED */ - { 175, -1 }, /* (163) expr ::= JOIN_KW */ - { 175, -3 }, /* (164) expr ::= nm DOT nm */ - { 175, -5 }, /* (165) expr ::= nm DOT nm DOT nm */ - { 174, -1 }, /* (166) term ::= NULL|FLOAT|BLOB */ - { 174, -1 }, /* (167) term ::= STRING */ - { 174, -1 }, /* (168) term ::= INTEGER */ - { 175, -1 }, /* (169) expr ::= VARIABLE */ - { 175, -3 }, /* (170) expr ::= expr COLLATE ID|STRING */ - { 175, -6 }, /* (171) expr ::= CAST LP expr AS typetoken RP */ - { 175, -5 }, /* (172) expr ::= ID|INDEXED LP distinct exprlist RP */ - { 175, -4 }, /* (173) expr ::= ID|INDEXED LP STAR RP */ - { 174, -1 }, /* (174) term ::= CTIME_KW */ - { 175, -5 }, /* (175) expr ::= LP nexprlist COMMA expr RP */ - { 175, -3 }, /* (176) expr ::= expr AND expr */ - { 175, -3 }, /* (177) expr ::= expr OR expr */ - { 175, -3 }, /* (178) expr ::= expr LT|GT|GE|LE expr */ - { 175, -3 }, /* (179) expr ::= expr EQ|NE expr */ - { 175, -3 }, /* (180) expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr */ - { 175, -3 }, /* (181) expr ::= expr PLUS|MINUS expr */ - { 175, -3 }, /* (182) expr ::= expr STAR|SLASH|REM expr */ - { 175, -3 }, /* (183) expr ::= expr CONCAT expr */ - { 226, -2 }, /* (184) likeop ::= NOT LIKE_KW|MATCH */ - { 175, -3 }, /* (185) expr ::= expr likeop expr */ - { 175, -5 }, /* (186) expr ::= expr likeop expr ESCAPE expr */ - { 175, -2 }, /* (187) expr ::= expr ISNULL|NOTNULL */ - { 175, -3 }, /* (188) expr ::= expr NOT NULL */ - { 175, -3 }, /* (189) expr ::= expr IS expr */ - { 175, -4 }, /* (190) expr ::= expr IS NOT expr */ - { 175, -2 }, /* (191) expr ::= NOT expr */ - { 175, -2 }, /* (192) expr ::= BITNOT expr */ - { 175, -2 }, /* (193) expr ::= MINUS expr */ - { 175, -2 }, /* (194) expr ::= PLUS expr */ - { 227, -1 }, /* (195) between_op ::= BETWEEN */ - { 227, -2 }, /* (196) between_op ::= NOT BETWEEN */ - { 175, -5 }, /* (197) expr ::= expr between_op expr AND expr */ - { 228, -1 }, /* (198) in_op ::= IN */ - { 228, -2 }, /* (199) in_op ::= NOT IN */ - { 175, -5 }, /* (200) expr ::= expr in_op LP exprlist RP */ - { 175, -3 }, /* (201) expr ::= LP select RP */ - { 175, -5 }, /* (202) expr ::= expr in_op LP select RP */ - { 175, -5 }, /* (203) expr ::= expr in_op nm dbnm paren_exprlist */ - { 175, -4 }, /* (204) expr ::= EXISTS LP select RP */ - { 175, -5 }, /* (205) expr ::= CASE case_operand case_exprlist case_else END */ - { 231, -5 }, /* (206) case_exprlist ::= case_exprlist WHEN expr THEN expr */ - { 231, -4 }, /* (207) case_exprlist ::= WHEN expr THEN expr */ - { 232, -2 }, /* (208) case_else ::= ELSE expr */ - { 232, 0 }, /* (209) case_else ::= */ - { 230, -1 }, /* (210) case_operand ::= expr */ - { 230, 0 }, /* (211) case_operand ::= */ - { 210, 0 }, /* (212) exprlist ::= */ - { 209, -3 }, /* (213) nexprlist ::= nexprlist COMMA expr */ - { 209, -1 }, /* (214) nexprlist ::= expr */ - { 229, 0 }, /* (215) paren_exprlist ::= */ - { 229, -3 }, /* (216) paren_exprlist ::= LP exprlist RP */ - { 150, -12 }, /* (217) cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt */ - { 233, -1 }, /* (218) uniqueflag ::= UNIQUE */ - { 233, 0 }, /* (219) uniqueflag ::= */ - { 179, 0 }, /* (220) eidlist_opt ::= */ - { 179, -3 }, /* (221) eidlist_opt ::= LP eidlist RP */ - { 189, -5 }, /* (222) eidlist ::= eidlist COMMA nm collate sortorder */ - { 189, -3 }, /* (223) eidlist ::= nm collate sortorder */ - { 234, 0 }, /* (224) collate ::= */ - { 234, -2 }, /* (225) collate ::= COLLATE ID|STRING */ - { 150, -4 }, /* (226) cmd ::= DROP INDEX ifexists fullname */ - { 150, -1 }, /* (227) cmd ::= VACUUM */ - { 150, -2 }, /* (228) cmd ::= VACUUM nm */ - { 150, -3 }, /* (229) cmd ::= PRAGMA nm dbnm */ - { 150, -5 }, /* (230) cmd ::= PRAGMA nm dbnm EQ nmnum */ - { 150, -6 }, /* (231) cmd ::= PRAGMA nm dbnm LP nmnum RP */ - { 150, -5 }, /* (232) cmd ::= PRAGMA nm dbnm EQ minus_num */ - { 150, -6 }, /* (233) cmd ::= PRAGMA nm dbnm LP minus_num RP */ - { 170, -2 }, /* (234) plus_num ::= PLUS INTEGER|FLOAT */ - { 171, -2 }, /* (235) minus_num ::= MINUS INTEGER|FLOAT */ - { 150, -5 }, /* (236) cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END */ - { 236, -11 }, /* (237) trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */ - { 238, -1 }, /* (238) trigger_time ::= BEFORE|AFTER */ - { 238, -2 }, /* (239) trigger_time ::= INSTEAD OF */ - { 238, 0 }, /* (240) trigger_time ::= */ - { 239, -1 }, /* (241) trigger_event ::= DELETE|INSERT */ - { 239, -1 }, /* (242) trigger_event ::= UPDATE */ - { 239, -3 }, /* (243) trigger_event ::= UPDATE OF idlist */ - { 241, 0 }, /* (244) when_clause ::= */ - { 241, -2 }, /* (245) when_clause ::= WHEN expr */ - { 237, -3 }, /* (246) trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */ - { 237, -2 }, /* (247) trigger_cmd_list ::= trigger_cmd SEMI */ - { 243, -3 }, /* (248) trnm ::= nm DOT nm */ - { 244, -3 }, /* (249) tridxby ::= INDEXED BY nm */ - { 244, -2 }, /* (250) tridxby ::= NOT INDEXED */ - { 242, -8 }, /* (251) trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist where_opt scanpt */ - { 242, -8 }, /* (252) trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt */ - { 242, -6 }, /* (253) trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt */ - { 242, -3 }, /* (254) trigger_cmd ::= scanpt select scanpt */ - { 175, -4 }, /* (255) expr ::= RAISE LP IGNORE RP */ - { 175, -6 }, /* (256) expr ::= RAISE LP raisetype COMMA nm RP */ - { 193, -1 }, /* (257) raisetype ::= ROLLBACK */ - { 193, -1 }, /* (258) raisetype ::= ABORT */ - { 193, -1 }, /* (259) raisetype ::= FAIL */ - { 150, -4 }, /* (260) cmd ::= DROP TRIGGER ifexists fullname */ - { 150, -6 }, /* (261) cmd ::= ATTACH database_kw_opt expr AS expr key_opt */ - { 150, -3 }, /* (262) cmd ::= DETACH database_kw_opt expr */ - { 246, 0 }, /* (263) key_opt ::= */ - { 246, -2 }, /* (264) key_opt ::= KEY expr */ - { 150, -1 }, /* (265) cmd ::= REINDEX */ - { 150, -3 }, /* (266) cmd ::= REINDEX nm dbnm */ - { 150, -1 }, /* (267) cmd ::= ANALYZE */ - { 150, -3 }, /* (268) cmd ::= ANALYZE nm dbnm */ - { 150, -6 }, /* (269) cmd ::= ALTER TABLE fullname RENAME TO nm */ - { 150, -7 }, /* (270) cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist */ - { 247, -1 }, /* (271) add_column_fullname ::= fullname */ - { 150, -1 }, /* (272) cmd ::= create_vtab */ - { 150, -4 }, /* (273) cmd ::= create_vtab LP vtabarglist RP */ - { 249, -8 }, /* (274) create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm */ - { 251, 0 }, /* (275) vtabarg ::= */ - { 252, -1 }, /* (276) vtabargtoken ::= ANY */ - { 252, -3 }, /* (277) vtabargtoken ::= lp anylist RP */ - { 253, -1 }, /* (278) lp ::= LP */ - { 221, -2 }, /* (279) with ::= WITH wqlist */ - { 221, -3 }, /* (280) with ::= WITH RECURSIVE wqlist */ - { 198, -6 }, /* (281) wqlist ::= nm eidlist_opt AS LP select RP */ - { 198, -8 }, /* (282) wqlist ::= wqlist COMMA nm eidlist_opt AS LP select RP */ - { 145, -1 }, /* (283) input ::= cmdlist */ - { 146, -2 }, /* (284) cmdlist ::= cmdlist ecmd */ - { 146, -1 }, /* (285) cmdlist ::= ecmd */ - { 147, -1 }, /* (286) ecmd ::= SEMI */ - { 147, -2 }, /* (287) ecmd ::= cmdx SEMI */ - { 147, -2 }, /* (288) ecmd ::= explain cmdx */ - { 152, 0 }, /* (289) trans_opt ::= */ - { 152, -1 }, /* (290) trans_opt ::= TRANSACTION */ - { 152, -2 }, /* (291) trans_opt ::= TRANSACTION nm */ - { 154, -1 }, /* (292) savepoint_opt ::= SAVEPOINT */ - { 154, 0 }, /* (293) savepoint_opt ::= */ - { 150, -2 }, /* (294) cmd ::= create_table create_table_args */ - { 161, -4 }, /* (295) columnlist ::= columnlist COMMA columnname carglist */ - { 161, -2 }, /* (296) columnlist ::= columnname carglist */ - { 153, -1 }, /* (297) nm ::= ID|INDEXED */ - { 153, -1 }, /* (298) nm ::= STRING */ - { 153, -1 }, /* (299) nm ::= JOIN_KW */ - { 167, -1 }, /* (300) typetoken ::= typename */ - { 168, -1 }, /* (301) typename ::= ID|STRING */ - { 169, -1 }, /* (302) signed ::= plus_num */ - { 169, -1 }, /* (303) signed ::= minus_num */ - { 166, -2 }, /* (304) carglist ::= carglist ccons */ - { 166, 0 }, /* (305) carglist ::= */ - { 173, -2 }, /* (306) ccons ::= NULL onconf */ - { 162, -2 }, /* (307) conslist_opt ::= COMMA conslist */ - { 185, -3 }, /* (308) conslist ::= conslist tconscomma tcons */ - { 185, -1 }, /* (309) conslist ::= tcons */ - { 186, 0 }, /* (310) tconscomma ::= */ - { 190, -1 }, /* (311) defer_subclause_opt ::= defer_subclause */ - { 192, -1 }, /* (312) resolvetype ::= raisetype */ - { 196, -1 }, /* (313) selectnowith ::= oneselect */ - { 197, -1 }, /* (314) oneselect ::= values */ - { 211, -2 }, /* (315) sclp ::= selcollist COMMA */ - { 212, -1 }, /* (316) as ::= ID|STRING */ - { 175, -1 }, /* (317) expr ::= term */ - { 226, -1 }, /* (318) likeop ::= LIKE_KW|MATCH */ - { 210, -1 }, /* (319) exprlist ::= nexprlist */ - { 235, -1 }, /* (320) nmnum ::= plus_num */ - { 235, -1 }, /* (321) nmnum ::= nm */ - { 235, -1 }, /* (322) nmnum ::= ON */ - { 235, -1 }, /* (323) nmnum ::= DELETE */ - { 235, -1 }, /* (324) nmnum ::= DEFAULT */ - { 170, -1 }, /* (325) plus_num ::= INTEGER|FLOAT */ - { 240, 0 }, /* (326) foreach_clause ::= */ - { 240, -3 }, /* (327) foreach_clause ::= FOR EACH ROW */ - { 243, -1 }, /* (328) trnm ::= nm */ - { 244, 0 }, /* (329) tridxby ::= */ - { 245, -1 }, /* (330) database_kw_opt ::= DATABASE */ - { 245, 0 }, /* (331) database_kw_opt ::= */ - { 248, 0 }, /* (332) kwcolumn_opt ::= */ - { 248, -1 }, /* (333) kwcolumn_opt ::= COLUMNKW */ - { 250, -1 }, /* (334) vtabarglist ::= vtabarg */ - { 250, -3 }, /* (335) vtabarglist ::= vtabarglist COMMA vtabarg */ - { 251, -2 }, /* (336) vtabarg ::= vtabarg vtabargtoken */ - { 254, 0 }, /* (337) anylist ::= */ - { 254, -4 }, /* (338) anylist ::= anylist LP anylist RP */ - { 254, -2 }, /* (339) anylist ::= anylist ANY */ - { 221, 0 }, /* (340) with ::= */ -}; - -static void yy_accept(yyParser*); /* Forward Declaration */ - /* -** Perform a reduce action and the shift that must immediately -** follow the reduce. -** -** The yyLookahead and yyLookaheadToken parameters provide reduce actions -** access to the lookahead token (if any). The yyLookahead will be YYNOCODE -** if the lookahead token has already been consumed. As this procedure is -** only called from one place, optimizing compilers will in-line it, which -** means that the extra parameters have no performance impact. +** If the SELECT statement passed as the second argument does not invoke +** any SQL window functions, this function is a no-op. Otherwise, it +** rewrites the SELECT statement so that window function xStep functions +** are invoked in the correct order as described under "SELECT REWRITING" +** at the top of this file. */ -static YYACTIONTYPE yy_reduce( - yyParser *yypParser, /* The parser */ - unsigned int yyruleno, /* Number of the rule by which to reduce */ - int yyLookahead, /* Lookahead token, or YYNOCODE if none */ - sqlite3ParserTOKENTYPE yyLookaheadToken /* Value of the lookahead token */ - sqlite3ParserCTX_PDECL /* %extra_context */ -){ - int yygoto; /* The next state */ - int yyact; /* The next action */ - yyStackEntry *yymsp; /* The top of the parser's stack */ - int yysize; /* Amount to pop the stack */ - sqlite3ParserARG_FETCH - (void)yyLookahead; - (void)yyLookaheadToken; - yymsp = yypParser->yytos; -#ifndef NDEBUG - if( yyTraceFILE && yyruleno<(int)(sizeof(yyRuleName)/sizeof(yyRuleName[0])) ){ - yysize = yyRuleInfo[yyruleno].nrhs; - if( yysize ){ - fprintf(yyTraceFILE, "%sReduce %d [%s], go to state %d.\n", - yyTracePrompt, - yyruleno, yyRuleName[yyruleno], yymsp[yysize].stateno); - }else{ - fprintf(yyTraceFILE, "%sReduce %d [%s].\n", - yyTracePrompt, yyruleno, yyRuleName[yyruleno]); +SQLITE_PRIVATE int sqlite3WindowRewrite(Parse *pParse, Select *p){ + int rc = SQLITE_OK; + if( p->pWin && p->pPrior==0 && (p->selFlags & SF_WinRewrite)==0 ){ + Vdbe *v = sqlite3GetVdbe(pParse); + sqlite3 *db = pParse->db; + Select *pSub = 0; /* The subquery */ + SrcList *pSrc = p->pSrc; + Expr *pWhere = p->pWhere; + ExprList *pGroupBy = p->pGroupBy; + Expr *pHaving = p->pHaving; + ExprList *pSort = 0; + + ExprList *pSublist = 0; /* Expression list for sub-query */ + Window *pMWin = p->pWin; /* Master window object */ + Window *pWin; /* Window object iterator */ + Table *pTab; + u32 selFlags = p->selFlags; + + pTab = sqlite3DbMallocZero(db, sizeof(Table)); + if( pTab==0 ){ + return sqlite3ErrorToParser(db, SQLITE_NOMEM); } - } -#endif /* NDEBUG */ - /* Check that the stack is large enough to grow by a single entry - ** if the RHS of the rule is empty. This ensures that there is room - ** enough on the stack to push the LHS value */ - if( yyRuleInfo[yyruleno].nrhs==0 ){ -#ifdef YYTRACKMAXSTACKDEPTH - if( (int)(yypParser->yytos - yypParser->yystack)>yypParser->yyhwm ){ - yypParser->yyhwm++; - assert( yypParser->yyhwm == (int)(yypParser->yytos - yypParser->yystack)); + p->pSrc = 0; + p->pWhere = 0; + p->pGroupBy = 0; + p->pHaving = 0; + p->selFlags &= ~SF_Aggregate; + p->selFlags |= SF_WinRewrite; + + /* Create the ORDER BY clause for the sub-select. This is the concatenation + ** of the window PARTITION and ORDER BY clauses. Then, if this makes it + ** redundant, remove the ORDER BY from the parent SELECT. */ + pSort = exprListAppendList(pParse, 0, pMWin->pPartition, 1); + pSort = exprListAppendList(pParse, pSort, pMWin->pOrderBy, 1); + if( pSort && p->pOrderBy && p->pOrderBy->nExpr<=pSort->nExpr ){ + int nSave = pSort->nExpr; + pSort->nExpr = p->pOrderBy->nExpr; + if( sqlite3ExprListCompare(pSort, p->pOrderBy, -1)==0 ){ + sqlite3ExprListDelete(db, p->pOrderBy); + p->pOrderBy = 0; + } + pSort->nExpr = nSave; + } + + /* Assign a cursor number for the ephemeral table used to buffer rows. + ** The OpenEphemeral instruction is coded later, after it is known how + ** many columns the table will have. */ + pMWin->iEphCsr = pParse->nTab++; + pParse->nTab += 3; + + selectWindowRewriteEList(pParse, pMWin, pSrc, p->pEList, pTab, &pSublist); + selectWindowRewriteEList(pParse, pMWin, pSrc, p->pOrderBy, pTab, &pSublist); + pMWin->nBufferCol = (pSublist ? pSublist->nExpr : 0); + + /* Append the PARTITION BY and ORDER BY expressions to the to the + ** sub-select expression list. They are required to figure out where + ** boundaries for partitions and sets of peer rows lie. */ + pSublist = exprListAppendList(pParse, pSublist, pMWin->pPartition, 0); + pSublist = exprListAppendList(pParse, pSublist, pMWin->pOrderBy, 0); + + /* Append the arguments passed to each window function to the + ** sub-select expression list. Also allocate two registers for each + ** window function - one for the accumulator, another for interim + ** results. */ + for(pWin=pMWin; pWin; pWin=pWin->pNextWin){ + ExprList *pArgs = pWin->pOwner->x.pList; + if( pWin->pFunc->funcFlags & SQLITE_FUNC_SUBTYPE ){ + selectWindowRewriteEList(pParse, pMWin, pSrc, pArgs, pTab, &pSublist); + pWin->iArgCol = (pSublist ? pSublist->nExpr : 0); + pWin->bExprArgs = 1; + }else{ + pWin->iArgCol = (pSublist ? pSublist->nExpr : 0); + pSublist = exprListAppendList(pParse, pSublist, pArgs, 0); + } + if( pWin->pFilter ){ + Expr *pFilter = sqlite3ExprDup(db, pWin->pFilter, 0); + pSublist = sqlite3ExprListAppend(pParse, pSublist, pFilter); + } + pWin->regAccum = ++pParse->nMem; + pWin->regResult = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Null, 0, pWin->regAccum); } -#endif -#if YYSTACKDEPTH>0 - if( yypParser->yytos>=yypParser->yystackEnd ){ - yyStackOverflow(yypParser); - /* The call to yyStackOverflow() above pops the stack until it is - ** empty, causing the main parser loop to exit. So the return value - ** is never used and does not matter. */ - return 0; + + /* If there is no ORDER BY or PARTITION BY clause, and the window + ** function accepts zero arguments, and there are no other columns + ** selected (e.g. "SELECT row_number() OVER () FROM t1"), it is possible + ** that pSublist is still NULL here. Add a constant expression here to + ** keep everything legal in this case. + */ + if( pSublist==0 ){ + pSublist = sqlite3ExprListAppend(pParse, 0, + sqlite3Expr(db, TK_INTEGER, "0") + ); } -#else - if( yypParser->yytos>=&yypParser->yystack[yypParser->yystksz-1] ){ - if( yyGrowStack(yypParser) ){ - yyStackOverflow(yypParser); - /* The call to yyStackOverflow() above pops the stack until it is - ** empty, causing the main parser loop to exit. So the return value - ** is never used and does not matter. */ - return 0; + + pSub = sqlite3SelectNew( + pParse, pSublist, pSrc, pWhere, pGroupBy, pHaving, pSort, 0, 0 + ); + p->pSrc = sqlite3SrcListAppend(pParse, 0, 0, 0); + if( p->pSrc ){ + Table *pTab2; + Walker w; + p->pSrc->a[0].pSelect = pSub; + sqlite3SrcListAssignCursors(pParse, p->pSrc); + pSub->selFlags |= SF_Expanded; + pTab2 = sqlite3ResultSetOfSelect(pParse, pSub, SQLITE_AFF_NONE); + pSub->selFlags |= (selFlags & SF_Aggregate); + if( pTab2==0 ){ + /* Might actually be some other kind of error, but in that case + ** pParse->nErr will be set, so if SQLITE_NOMEM is set, we will get + ** the correct error message regardless. */ + rc = SQLITE_NOMEM; + }else{ + memcpy(pTab, pTab2, sizeof(Table)); + pTab->tabFlags |= TF_Ephemeral; + p->pSrc->a[0].pTab = pTab; + pTab = pTab2; + memset(&w, 0, sizeof(w)); + w.xExprCallback = sqlite3WindowExtraAggFuncDepth; + w.xSelectCallback = sqlite3WalkerDepthIncrease; + w.xSelectCallback2 = sqlite3WalkerDepthDecrease; + sqlite3WalkSelect(&w, pSub); } - yymsp = yypParser->yytos; + }else{ + sqlite3SelectDelete(db, pSub); } -#endif + if( db->mallocFailed ) rc = SQLITE_NOMEM; + sqlite3DbFree(db, pTab); } - switch( yyruleno ){ - /* Beginning here are the reduction cases. A typical example - ** follows: - ** case 0: - ** #line - ** { ... } // User supplied code - ** #line - ** break; - */ -/********** Begin reduce actions **********************************************/ - YYMINORTYPE yylhsminor; - case 0: /* explain ::= EXPLAIN */ -{ pParse->explain = 1; } - break; - case 1: /* explain ::= EXPLAIN QUERY PLAN */ -{ pParse->explain = 2; } - break; - case 2: /* cmdx ::= cmd */ -{ sqlite3FinishCoding(pParse); } - break; - case 3: /* cmd ::= BEGIN transtype trans_opt */ -{sqlite3BeginTransaction(pParse, yymsp[-1].minor.yy502);} - break; - case 4: /* transtype ::= */ -{yymsp[1].minor.yy502 = TK_DEFERRED;} - break; - case 5: /* transtype ::= DEFERRED */ - case 6: /* transtype ::= IMMEDIATE */ yytestcase(yyruleno==6); - case 7: /* transtype ::= EXCLUSIVE */ yytestcase(yyruleno==7); -{yymsp[0].minor.yy502 = yymsp[0].major; /*A-overwrites-X*/} - break; - case 8: /* cmd ::= COMMIT|END trans_opt */ - case 9: /* cmd ::= ROLLBACK trans_opt */ yytestcase(yyruleno==9); -{sqlite3EndTransaction(pParse,yymsp[-1].major);} - break; - case 10: /* cmd ::= SAVEPOINT nm */ -{ - sqlite3Savepoint(pParse, SAVEPOINT_BEGIN, &yymsp[0].minor.yy0); -} - break; - case 11: /* cmd ::= RELEASE savepoint_opt nm */ -{ - sqlite3Savepoint(pParse, SAVEPOINT_RELEASE, &yymsp[0].minor.yy0); -} - break; - case 12: /* cmd ::= ROLLBACK trans_opt TO savepoint_opt nm */ -{ - sqlite3Savepoint(pParse, SAVEPOINT_ROLLBACK, &yymsp[0].minor.yy0); -} - break; - case 13: /* create_table ::= createkw temp TABLE ifnotexists nm dbnm */ -{ - sqlite3StartTable(pParse,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0,yymsp[-4].minor.yy502,0,0,yymsp[-2].minor.yy502); -} - break; - case 14: /* createkw ::= CREATE */ -{disableLookaside(pParse);} - break; - case 15: /* ifnotexists ::= */ - case 18: /* temp ::= */ yytestcase(yyruleno==18); - case 21: /* table_options ::= */ yytestcase(yyruleno==21); - case 42: /* autoinc ::= */ yytestcase(yyruleno==42); - case 57: /* init_deferred_pred_opt ::= */ yytestcase(yyruleno==57); - case 67: /* defer_subclause_opt ::= */ yytestcase(yyruleno==67); - case 76: /* ifexists ::= */ yytestcase(yyruleno==76); - case 92: /* distinct ::= */ yytestcase(yyruleno==92); - case 224: /* collate ::= */ yytestcase(yyruleno==224); -{yymsp[1].minor.yy502 = 0;} - break; - case 16: /* ifnotexists ::= IF NOT EXISTS */ -{yymsp[-2].minor.yy502 = 1;} - break; - case 17: /* temp ::= TEMP */ - case 43: /* autoinc ::= AUTOINCR */ yytestcase(yyruleno==43); -{yymsp[0].minor.yy502 = 1;} - break; - case 19: /* create_table_args ::= LP columnlist conslist_opt RP table_options */ -{ - sqlite3EndTable(pParse,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0,yymsp[0].minor.yy502,0); -} - break; - case 20: /* create_table_args ::= AS select */ -{ - sqlite3EndTable(pParse,0,0,0,yymsp[0].minor.yy399); - sqlite3SelectDelete(pParse->db, yymsp[0].minor.yy399); -} - break; - case 22: /* table_options ::= WITHOUT nm */ -{ - if( yymsp[0].minor.yy0.n==5 && sqlite3_strnicmp(yymsp[0].minor.yy0.z,"rowid",5)==0 ){ - yymsp[-1].minor.yy502 = TF_WithoutRowid | TF_NoVisibleRowid; - }else{ - yymsp[-1].minor.yy502 = 0; - sqlite3ErrorMsg(pParse, "unknown table option: %.*s", yymsp[0].minor.yy0.n, yymsp[0].minor.yy0.z); + if( rc ){ + if( pParse->nErr==0 ){ + assert( pParse->db->mallocFailed ); + sqlite3ErrorToParser(pParse->db, SQLITE_NOMEM); + } + sqlite3SelectReset(pParse, p); } + return rc; } - break; - case 23: /* columnname ::= nm typetoken */ -{sqlite3AddColumn(pParse,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0);} - break; - case 24: /* typetoken ::= */ - case 60: /* conslist_opt ::= */ yytestcase(yyruleno==60); - case 98: /* as ::= */ yytestcase(yyruleno==98); -{yymsp[1].minor.yy0.n = 0; yymsp[1].minor.yy0.z = 0;} - break; - case 25: /* typetoken ::= typename LP signed RP */ -{ - yymsp[-3].minor.yy0.n = (int)(&yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n] - yymsp[-3].minor.yy0.z); -} - break; - case 26: /* typetoken ::= typename LP signed COMMA signed RP */ -{ - yymsp[-5].minor.yy0.n = (int)(&yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n] - yymsp[-5].minor.yy0.z); -} - break; - case 27: /* typename ::= typename ID|STRING */ -{yymsp[-1].minor.yy0.n=yymsp[0].minor.yy0.n+(int)(yymsp[0].minor.yy0.z-yymsp[-1].minor.yy0.z);} - break; - case 28: /* scanpt ::= */ -{ - assert( yyLookahead!=YYNOCODE ); - yymsp[1].minor.yy36 = yyLookaheadToken.z; -} - break; - case 29: /* ccons ::= CONSTRAINT nm */ - case 62: /* tcons ::= CONSTRAINT nm */ yytestcase(yyruleno==62); -{pParse->constraintName = yymsp[0].minor.yy0;} - break; - case 30: /* ccons ::= DEFAULT scanpt term scanpt */ -{sqlite3AddDefaultValue(pParse,yymsp[-1].minor.yy182,yymsp[-2].minor.yy36,yymsp[0].minor.yy36);} - break; - case 31: /* ccons ::= DEFAULT LP expr RP */ -{sqlite3AddDefaultValue(pParse,yymsp[-1].minor.yy182,yymsp[-2].minor.yy0.z+1,yymsp[0].minor.yy0.z);} - break; - case 32: /* ccons ::= DEFAULT PLUS term scanpt */ -{sqlite3AddDefaultValue(pParse,yymsp[-1].minor.yy182,yymsp[-2].minor.yy0.z,yymsp[0].minor.yy36);} - break; - case 33: /* ccons ::= DEFAULT MINUS term scanpt */ -{ - Expr *p = sqlite3PExpr(pParse, TK_UMINUS, yymsp[-1].minor.yy182, 0); - sqlite3AddDefaultValue(pParse,p,yymsp[-2].minor.yy0.z,yymsp[0].minor.yy36); + +/* +** Unlink the Window object from the Select to which it is attached, +** if it is attached. +*/ +SQLITE_PRIVATE void sqlite3WindowUnlinkFromSelect(Window *p){ + if( p->ppThis ){ + *p->ppThis = p->pNextWin; + if( p->pNextWin ) p->pNextWin->ppThis = p->ppThis; + p->ppThis = 0; + } } - break; - case 34: /* ccons ::= DEFAULT scanpt ID|INDEXED */ -{ - Expr *p = tokenExpr(pParse, TK_STRING, yymsp[0].minor.yy0); + +/* +** Free the Window object passed as the second argument. +*/ +SQLITE_PRIVATE void sqlite3WindowDelete(sqlite3 *db, Window *p){ if( p ){ - sqlite3ExprIdToTrueFalse(p); - testcase( p->op==TK_TRUEFALSE && sqlite3ExprTruthValue(p) ); + sqlite3WindowUnlinkFromSelect(p); + sqlite3ExprDelete(db, p->pFilter); + sqlite3ExprListDelete(db, p->pPartition); + sqlite3ExprListDelete(db, p->pOrderBy); + sqlite3ExprDelete(db, p->pEnd); + sqlite3ExprDelete(db, p->pStart); + sqlite3DbFree(db, p->zName); + sqlite3DbFree(db, p->zBase); + sqlite3DbFree(db, p); } - sqlite3AddDefaultValue(pParse,p,yymsp[0].minor.yy0.z,yymsp[0].minor.yy0.z+yymsp[0].minor.yy0.n); -} - break; - case 35: /* ccons ::= NOT NULL onconf */ -{sqlite3AddNotNull(pParse, yymsp[0].minor.yy502);} - break; - case 36: /* ccons ::= PRIMARY KEY sortorder onconf autoinc */ -{sqlite3AddPrimaryKey(pParse,0,yymsp[-1].minor.yy502,yymsp[0].minor.yy502,yymsp[-2].minor.yy502);} - break; - case 37: /* ccons ::= UNIQUE onconf */ -{sqlite3CreateIndex(pParse,0,0,0,0,yymsp[0].minor.yy502,0,0,0,0, - SQLITE_IDXTYPE_UNIQUE);} - break; - case 38: /* ccons ::= CHECK LP expr RP */ -{sqlite3AddCheckConstraint(pParse,yymsp[-1].minor.yy182);} - break; - case 39: /* ccons ::= REFERENCES nm eidlist_opt refargs */ -{sqlite3CreateForeignKey(pParse,0,&yymsp[-2].minor.yy0,yymsp[-1].minor.yy232,yymsp[0].minor.yy502);} - break; - case 40: /* ccons ::= defer_subclause */ -{sqlite3DeferForeignKey(pParse,yymsp[0].minor.yy502);} - break; - case 41: /* ccons ::= COLLATE ID|STRING */ -{sqlite3AddCollateType(pParse, &yymsp[0].minor.yy0);} - break; - case 44: /* refargs ::= */ -{ yymsp[1].minor.yy502 = OE_None*0x0101; /* EV: R-19803-45884 */} - break; - case 45: /* refargs ::= refargs refarg */ -{ yymsp[-1].minor.yy502 = (yymsp[-1].minor.yy502 & ~yymsp[0].minor.yy107.mask) | yymsp[0].minor.yy107.value; } - break; - case 46: /* refarg ::= MATCH nm */ -{ yymsp[-1].minor.yy107.value = 0; yymsp[-1].minor.yy107.mask = 0x000000; } - break; - case 47: /* refarg ::= ON INSERT refact */ -{ yymsp[-2].minor.yy107.value = 0; yymsp[-2].minor.yy107.mask = 0x000000; } - break; - case 48: /* refarg ::= ON DELETE refact */ -{ yymsp[-2].minor.yy107.value = yymsp[0].minor.yy502; yymsp[-2].minor.yy107.mask = 0x0000ff; } - break; - case 49: /* refarg ::= ON UPDATE refact */ -{ yymsp[-2].minor.yy107.value = yymsp[0].minor.yy502<<8; yymsp[-2].minor.yy107.mask = 0x00ff00; } - break; - case 50: /* refact ::= SET NULL */ -{ yymsp[-1].minor.yy502 = OE_SetNull; /* EV: R-33326-45252 */} - break; - case 51: /* refact ::= SET DEFAULT */ -{ yymsp[-1].minor.yy502 = OE_SetDflt; /* EV: R-33326-45252 */} - break; - case 52: /* refact ::= CASCADE */ -{ yymsp[0].minor.yy502 = OE_Cascade; /* EV: R-33326-45252 */} - break; - case 53: /* refact ::= RESTRICT */ -{ yymsp[0].minor.yy502 = OE_Restrict; /* EV: R-33326-45252 */} - break; - case 54: /* refact ::= NO ACTION */ -{ yymsp[-1].minor.yy502 = OE_None; /* EV: R-33326-45252 */} - break; - case 55: /* defer_subclause ::= NOT DEFERRABLE init_deferred_pred_opt */ -{yymsp[-2].minor.yy502 = 0;} - break; - case 56: /* defer_subclause ::= DEFERRABLE init_deferred_pred_opt */ - case 71: /* orconf ::= OR resolvetype */ yytestcase(yyruleno==71); - case 155: /* insert_cmd ::= INSERT orconf */ yytestcase(yyruleno==155); -{yymsp[-1].minor.yy502 = yymsp[0].minor.yy502;} - break; - case 58: /* init_deferred_pred_opt ::= INITIALLY DEFERRED */ - case 75: /* ifexists ::= IF EXISTS */ yytestcase(yyruleno==75); - case 196: /* between_op ::= NOT BETWEEN */ yytestcase(yyruleno==196); - case 199: /* in_op ::= NOT IN */ yytestcase(yyruleno==199); - case 225: /* collate ::= COLLATE ID|STRING */ yytestcase(yyruleno==225); -{yymsp[-1].minor.yy502 = 1;} - break; - case 59: /* init_deferred_pred_opt ::= INITIALLY IMMEDIATE */ -{yymsp[-1].minor.yy502 = 0;} - break; - case 61: /* tconscomma ::= COMMA */ -{pParse->constraintName.n = 0;} - break; - case 63: /* tcons ::= PRIMARY KEY LP sortlist autoinc RP onconf */ -{sqlite3AddPrimaryKey(pParse,yymsp[-3].minor.yy232,yymsp[0].minor.yy502,yymsp[-2].minor.yy502,0);} - break; - case 64: /* tcons ::= UNIQUE LP sortlist RP onconf */ -{sqlite3CreateIndex(pParse,0,0,0,yymsp[-2].minor.yy232,yymsp[0].minor.yy502,0,0,0,0, - SQLITE_IDXTYPE_UNIQUE);} - break; - case 65: /* tcons ::= CHECK LP expr RP onconf */ -{sqlite3AddCheckConstraint(pParse,yymsp[-2].minor.yy182);} - break; - case 66: /* tcons ::= FOREIGN KEY LP eidlist RP REFERENCES nm eidlist_opt refargs defer_subclause_opt */ -{ - sqlite3CreateForeignKey(pParse, yymsp[-6].minor.yy232, &yymsp[-3].minor.yy0, yymsp[-2].minor.yy232, yymsp[-1].minor.yy502); - sqlite3DeferForeignKey(pParse, yymsp[0].minor.yy502); -} - break; - case 68: /* onconf ::= */ - case 70: /* orconf ::= */ yytestcase(yyruleno==70); -{yymsp[1].minor.yy502 = OE_Default;} - break; - case 69: /* onconf ::= ON CONFLICT resolvetype */ -{yymsp[-2].minor.yy502 = yymsp[0].minor.yy502;} - break; - case 72: /* resolvetype ::= IGNORE */ -{yymsp[0].minor.yy502 = OE_Ignore;} - break; - case 73: /* resolvetype ::= REPLACE */ - case 156: /* insert_cmd ::= REPLACE */ yytestcase(yyruleno==156); -{yymsp[0].minor.yy502 = OE_Replace;} - break; - case 74: /* cmd ::= DROP TABLE ifexists fullname */ -{ - sqlite3DropTable(pParse, yymsp[0].minor.yy427, 0, yymsp[-1].minor.yy502); } - break; - case 77: /* cmd ::= createkw temp VIEW ifnotexists nm dbnm eidlist_opt AS select */ -{ - sqlite3CreateView(pParse, &yymsp[-8].minor.yy0, &yymsp[-4].minor.yy0, &yymsp[-3].minor.yy0, yymsp[-2].minor.yy232, yymsp[0].minor.yy399, yymsp[-7].minor.yy502, yymsp[-5].minor.yy502); + +/* +** Free the linked list of Window objects starting at the second argument. +*/ +SQLITE_PRIVATE void sqlite3WindowListDelete(sqlite3 *db, Window *p){ + while( p ){ + Window *pNext = p->pNextWin; + sqlite3WindowDelete(db, p); + p = pNext; + } } - break; - case 78: /* cmd ::= DROP VIEW ifexists fullname */ -{ - sqlite3DropTable(pParse, yymsp[0].minor.yy427, 1, yymsp[-1].minor.yy502); + +/* +** The argument expression is an PRECEDING or FOLLOWING offset. The +** value should be a non-negative integer. If the value is not a +** constant, change it to NULL. The fact that it is then a non-negative +** integer will be caught later. But it is important not to leave +** variable values in the expression tree. +*/ +static Expr *sqlite3WindowOffsetExpr(Parse *pParse, Expr *pExpr){ + if( 0==sqlite3ExprIsConstant(pExpr) ){ + if( IN_RENAME_OBJECT ) sqlite3RenameExprUnmap(pParse, pExpr); + sqlite3ExprDelete(pParse->db, pExpr); + pExpr = sqlite3ExprAlloc(pParse->db, TK_NULL, 0, 0); + } + return pExpr; } - break; - case 79: /* cmd ::= select */ -{ - SelectDest dest = {SRT_Output, 0, 0, 0, 0, 0}; - sqlite3Select(pParse, yymsp[0].minor.yy399, &dest); - sqlite3SelectDelete(pParse->db, yymsp[0].minor.yy399); + +/* +** Allocate and return a new Window object describing a Window Definition. +*/ +SQLITE_PRIVATE Window *sqlite3WindowAlloc( + Parse *pParse, /* Parsing context */ + int eType, /* Frame type. TK_RANGE, TK_ROWS, TK_GROUPS, or 0 */ + int eStart, /* Start type: CURRENT, PRECEDING, FOLLOWING, UNBOUNDED */ + Expr *pStart, /* Start window size if TK_PRECEDING or FOLLOWING */ + int eEnd, /* End type: CURRENT, FOLLOWING, TK_UNBOUNDED, PRECEDING */ + Expr *pEnd, /* End window size if TK_FOLLOWING or PRECEDING */ + u8 eExclude /* EXCLUDE clause */ +){ + Window *pWin = 0; + int bImplicitFrame = 0; + + /* Parser assures the following: */ + assert( eType==0 || eType==TK_RANGE || eType==TK_ROWS || eType==TK_GROUPS ); + assert( eStart==TK_CURRENT || eStart==TK_PRECEDING + || eStart==TK_UNBOUNDED || eStart==TK_FOLLOWING ); + assert( eEnd==TK_CURRENT || eEnd==TK_FOLLOWING + || eEnd==TK_UNBOUNDED || eEnd==TK_PRECEDING ); + assert( (eStart==TK_PRECEDING || eStart==TK_FOLLOWING)==(pStart!=0) ); + assert( (eEnd==TK_FOLLOWING || eEnd==TK_PRECEDING)==(pEnd!=0) ); + + if( eType==0 ){ + bImplicitFrame = 1; + eType = TK_RANGE; + } + + /* Additionally, the + ** starting boundary type may not occur earlier in the following list than + ** the ending boundary type: + ** + ** UNBOUNDED PRECEDING + ** PRECEDING + ** CURRENT ROW + ** FOLLOWING + ** UNBOUNDED FOLLOWING + ** + ** The parser ensures that "UNBOUNDED PRECEDING" cannot be used as an ending + ** boundary, and than "UNBOUNDED FOLLOWING" cannot be used as a starting + ** frame boundary. + */ + if( (eStart==TK_CURRENT && eEnd==TK_PRECEDING) + || (eStart==TK_FOLLOWING && (eEnd==TK_PRECEDING || eEnd==TK_CURRENT)) + ){ + sqlite3ErrorMsg(pParse, "unsupported frame specification"); + goto windowAllocErr; + } + + pWin = (Window*)sqlite3DbMallocZero(pParse->db, sizeof(Window)); + if( pWin==0 ) goto windowAllocErr; + pWin->eFrmType = eType; + pWin->eStart = eStart; + pWin->eEnd = eEnd; + if( eExclude==0 && OptimizationDisabled(pParse->db, SQLITE_WindowFunc) ){ + eExclude = TK_NO; + } + pWin->eExclude = eExclude; + pWin->bImplicitFrame = bImplicitFrame; + pWin->pEnd = sqlite3WindowOffsetExpr(pParse, pEnd); + pWin->pStart = sqlite3WindowOffsetExpr(pParse, pStart); + return pWin; + +windowAllocErr: + sqlite3ExprDelete(pParse->db, pEnd); + sqlite3ExprDelete(pParse->db, pStart); + return 0; } - break; - case 80: /* select ::= WITH wqlist selectnowith */ -{ - Select *p = yymsp[0].minor.yy399; - if( p ){ - p->pWith = yymsp[-1].minor.yy91; - parserDoubleLinkSelect(pParse, p); + +/* +** Attach PARTITION and ORDER BY clauses pPartition and pOrderBy to window +** pWin. Also, if parameter pBase is not NULL, set pWin->zBase to the +** equivalent nul-terminated string. +*/ +SQLITE_PRIVATE Window *sqlite3WindowAssemble( + Parse *pParse, + Window *pWin, + ExprList *pPartition, + ExprList *pOrderBy, + Token *pBase +){ + if( pWin ){ + pWin->pPartition = pPartition; + pWin->pOrderBy = pOrderBy; + if( pBase ){ + pWin->zBase = sqlite3DbStrNDup(pParse->db, pBase->z, pBase->n); + } }else{ - sqlite3WithDelete(pParse->db, yymsp[-1].minor.yy91); + sqlite3ExprListDelete(pParse->db, pPartition); + sqlite3ExprListDelete(pParse->db, pOrderBy); } - yymsp[-2].minor.yy399 = p; + return pWin; } - break; - case 81: /* select ::= WITH RECURSIVE wqlist selectnowith */ -{ - Select *p = yymsp[0].minor.yy399; + +/* +** Window *pWin has just been created from a WINDOW clause. Tokne pBase +** is the base window. Earlier windows from the same WINDOW clause are +** stored in the linked list starting at pWin->pNextWin. This function +** either updates *pWin according to the base specification, or else +** leaves an error in pParse. +*/ +SQLITE_PRIVATE void sqlite3WindowChain(Parse *pParse, Window *pWin, Window *pList){ + if( pWin->zBase ){ + sqlite3 *db = pParse->db; + Window *pExist = windowFind(pParse, pList, pWin->zBase); + if( pExist ){ + const char *zErr = 0; + /* Check for errors */ + if( pWin->pPartition ){ + zErr = "PARTITION clause"; + }else if( pExist->pOrderBy && pWin->pOrderBy ){ + zErr = "ORDER BY clause"; + }else if( pExist->bImplicitFrame==0 ){ + zErr = "frame specification"; + } + if( zErr ){ + sqlite3ErrorMsg(pParse, + "cannot override %s of window: %s", zErr, pWin->zBase + ); + }else{ + pWin->pPartition = sqlite3ExprListDup(db, pExist->pPartition, 0); + if( pExist->pOrderBy ){ + assert( pWin->pOrderBy==0 ); + pWin->pOrderBy = sqlite3ExprListDup(db, pExist->pOrderBy, 0); + } + sqlite3DbFree(db, pWin->zBase); + pWin->zBase = 0; + } + } + } +} + +/* +** Attach window object pWin to expression p. +*/ +SQLITE_PRIVATE void sqlite3WindowAttach(Parse *pParse, Expr *p, Window *pWin){ if( p ){ - p->pWith = yymsp[-1].minor.yy91; - parserDoubleLinkSelect(pParse, p); + assert( p->op==TK_FUNCTION ); + assert( pWin ); + p->y.pWin = pWin; + ExprSetProperty(p, EP_WinFunc); + pWin->pOwner = p; + if( (p->flags & EP_Distinct) && pWin->eFrmType!=TK_FILTER ){ + sqlite3ErrorMsg(pParse, + "DISTINCT is not supported for window functions" + ); + } }else{ - sqlite3WithDelete(pParse->db, yymsp[-1].minor.yy91); + sqlite3WindowDelete(pParse->db, pWin); } - yymsp[-3].minor.yy399 = p; } - break; - case 82: /* select ::= selectnowith */ -{ - Select *p = yymsp[0].minor.yy399; - if( p ){ - parserDoubleLinkSelect(pParse, p); + +/* +** Possibly link window pWin into the list at pSel->pWin (window functions +** to be processed as part of SELECT statement pSel). The window is linked +** in if either (a) there are no other windows already linked to this +** SELECT, or (b) the windows already linked use a compatible window frame. +*/ +SQLITE_PRIVATE void sqlite3WindowLink(Select *pSel, Window *pWin){ + if( pSel!=0 + && (0==pSel->pWin || 0==sqlite3WindowCompare(0, pSel->pWin, pWin, 0)) + ){ + pWin->pNextWin = pSel->pWin; + if( pSel->pWin ){ + pSel->pWin->ppThis = &pWin->pNextWin; + } + pSel->pWin = pWin; + pWin->ppThis = &pSel->pWin; } - yymsp[0].minor.yy399 = p; /*A-overwrites-X*/ } - break; - case 83: /* selectnowith ::= selectnowith multiselect_op oneselect */ -{ - Select *pRhs = yymsp[0].minor.yy399; - Select *pLhs = yymsp[-2].minor.yy399; - if( pRhs && pRhs->pPrior ){ - SrcList *pFrom; - Token x; - x.n = 0; - parserDoubleLinkSelect(pParse, pRhs); - pFrom = sqlite3SrcListAppendFromTerm(pParse,0,0,0,&x,pRhs,0,0); - pRhs = sqlite3SelectNew(pParse,0,pFrom,0,0,0,0,0,0); + +/* +** Return 0 if the two window objects are identical, 1 if they are +** different, or 2 if it cannot be determined if the objects are identical +** or not. Identical window objects can be processed in a single scan. +*/ +SQLITE_PRIVATE int sqlite3WindowCompare(Parse *pParse, Window *p1, Window *p2, int bFilter){ + int res; + if( NEVER(p1==0) || NEVER(p2==0) ) return 1; + if( p1->eFrmType!=p2->eFrmType ) return 1; + if( p1->eStart!=p2->eStart ) return 1; + if( p1->eEnd!=p2->eEnd ) return 1; + if( p1->eExclude!=p2->eExclude ) return 1; + if( sqlite3ExprCompare(pParse, p1->pStart, p2->pStart, -1) ) return 1; + if( sqlite3ExprCompare(pParse, p1->pEnd, p2->pEnd, -1) ) return 1; + if( (res = sqlite3ExprListCompare(p1->pPartition, p2->pPartition, -1)) ){ + return res; } - if( pRhs ){ - pRhs->op = (u8)yymsp[-1].minor.yy502; - pRhs->pPrior = pLhs; - if( ALWAYS(pLhs) ) pLhs->selFlags &= ~SF_MultiValue; - pRhs->selFlags &= ~SF_MultiValue; - if( yymsp[-1].minor.yy502!=TK_ALL ) pParse->hasCompound = 1; - }else{ - sqlite3SelectDelete(pParse->db, pLhs); + if( (res = sqlite3ExprListCompare(p1->pOrderBy, p2->pOrderBy, -1)) ){ + return res; } - yymsp[-2].minor.yy399 = pRhs; -} - break; - case 84: /* multiselect_op ::= UNION */ - case 86: /* multiselect_op ::= EXCEPT|INTERSECT */ yytestcase(yyruleno==86); -{yymsp[0].minor.yy502 = yymsp[0].major; /*A-overwrites-OP*/} - break; - case 85: /* multiselect_op ::= UNION ALL */ -{yymsp[-1].minor.yy502 = TK_ALL;} - break; - case 87: /* oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt orderby_opt limit_opt */ -{ -#if SELECTTRACE_ENABLED - Token s = yymsp[-8].minor.yy0; /*A-overwrites-S*/ -#endif - yymsp[-8].minor.yy399 = sqlite3SelectNew(pParse,yymsp[-6].minor.yy232,yymsp[-5].minor.yy427,yymsp[-4].minor.yy182,yymsp[-3].minor.yy232,yymsp[-2].minor.yy182,yymsp[-1].minor.yy232,yymsp[-7].minor.yy502,yymsp[0].minor.yy182); -#if SELECTTRACE_ENABLED - /* Populate the Select.zSelName[] string that is used to help with - ** query planner debugging, to differentiate between multiple Select - ** objects in a complex query. - ** - ** If the SELECT keyword is immediately followed by a C-style comment - ** then extract the first few alphanumeric characters from within that - ** comment to be the zSelName value. Otherwise, the label is #N where - ** is an integer that is incremented with each SELECT statement seen. - */ - if( yymsp[-8].minor.yy399!=0 ){ - const char *z = s.z+6; - int i; - sqlite3_snprintf(sizeof(yymsp[-8].minor.yy399->zSelName), yymsp[-8].minor.yy399->zSelName,"#%d",++pParse->nSelect); - while( z[0]==' ' ) z++; - if( z[0]=='/' && z[1]=='*' ){ - z += 2; - while( z[0]==' ' ) z++; - for(i=0; sqlite3Isalnum(z[i]); i++){} - sqlite3_snprintf(sizeof(yymsp[-8].minor.yy399->zSelName), yymsp[-8].minor.yy399->zSelName, "%.*s", i, z); + if( bFilter ){ + if( (res = sqlite3ExprCompare(pParse, p1->pFilter, p2->pFilter, -1)) ){ + return res; } } -#endif /* SELECTRACE_ENABLED */ -} - break; - case 88: /* values ::= VALUES LP nexprlist RP */ -{ - yymsp[-3].minor.yy399 = sqlite3SelectNew(pParse,yymsp[-1].minor.yy232,0,0,0,0,0,SF_Values,0); + return 0; } - break; - case 89: /* values ::= values COMMA LP exprlist RP */ -{ - Select *pRight, *pLeft = yymsp[-4].minor.yy399; - pRight = sqlite3SelectNew(pParse,yymsp[-1].minor.yy232,0,0,0,0,0,SF_Values|SF_MultiValue,0); - if( ALWAYS(pLeft) ) pLeft->selFlags &= ~SF_MultiValue; - if( pRight ){ - pRight->op = TK_ALL; - pRight->pPrior = pLeft; - yymsp[-4].minor.yy399 = pRight; - }else{ - yymsp[-4].minor.yy399 = pLeft; + + +/* +** This is called by code in select.c before it calls sqlite3WhereBegin() +** to begin iterating through the sub-query results. It is used to allocate +** and initialize registers and cursors used by sqlite3WindowCodeStep(). +*/ +SQLITE_PRIVATE void sqlite3WindowCodeInit(Parse *pParse, Select *pSelect){ + int nEphExpr = pSelect->pSrc->a[0].pSelect->pEList->nExpr; + Window *pMWin = pSelect->pWin; + Window *pWin; + Vdbe *v = sqlite3GetVdbe(pParse); + + sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pMWin->iEphCsr, nEphExpr); + sqlite3VdbeAddOp2(v, OP_OpenDup, pMWin->iEphCsr+1, pMWin->iEphCsr); + sqlite3VdbeAddOp2(v, OP_OpenDup, pMWin->iEphCsr+2, pMWin->iEphCsr); + sqlite3VdbeAddOp2(v, OP_OpenDup, pMWin->iEphCsr+3, pMWin->iEphCsr); + + /* Allocate registers to use for PARTITION BY values, if any. Initialize + ** said registers to NULL. */ + if( pMWin->pPartition ){ + int nExpr = pMWin->pPartition->nExpr; + pMWin->regPart = pParse->nMem+1; + pParse->nMem += nExpr; + sqlite3VdbeAddOp3(v, OP_Null, 0, pMWin->regPart, pMWin->regPart+nExpr-1); + } + + pMWin->regOne = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Integer, 1, pMWin->regOne); + + if( pMWin->eExclude ){ + pMWin->regStartRowid = ++pParse->nMem; + pMWin->regEndRowid = ++pParse->nMem; + pMWin->csrApp = pParse->nTab++; + sqlite3VdbeAddOp2(v, OP_Integer, 1, pMWin->regStartRowid); + sqlite3VdbeAddOp2(v, OP_Integer, 0, pMWin->regEndRowid); + sqlite3VdbeAddOp2(v, OP_OpenDup, pMWin->csrApp, pMWin->iEphCsr); + return; + } + + for(pWin=pMWin; pWin; pWin=pWin->pNextWin){ + FuncDef *p = pWin->pFunc; + if( (p->funcFlags & SQLITE_FUNC_MINMAX) && pWin->eStart!=TK_UNBOUNDED ){ + /* The inline versions of min() and max() require a single ephemeral + ** table and 3 registers. The registers are used as follows: + ** + ** regApp+0: slot to copy min()/max() argument to for MakeRecord + ** regApp+1: integer value used to ensure keys are unique + ** regApp+2: output of MakeRecord + */ + ExprList *pList = pWin->pOwner->x.pList; + KeyInfo *pKeyInfo = sqlite3KeyInfoFromExprList(pParse, pList, 0, 0); + pWin->csrApp = pParse->nTab++; + pWin->regApp = pParse->nMem+1; + pParse->nMem += 3; + if( pKeyInfo && pWin->pFunc->zName[1]=='i' ){ + assert( pKeyInfo->aSortFlags[0]==0 ); + pKeyInfo->aSortFlags[0] = KEYINFO_ORDER_DESC; + } + sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pWin->csrApp, 2); + sqlite3VdbeAppendP4(v, pKeyInfo, P4_KEYINFO); + sqlite3VdbeAddOp2(v, OP_Integer, 0, pWin->regApp+1); + } + else if( p->zName==nth_valueName || p->zName==first_valueName ){ + /* Allocate two registers at pWin->regApp. These will be used to + ** store the start and end index of the current frame. */ + pWin->regApp = pParse->nMem+1; + pWin->csrApp = pParse->nTab++; + pParse->nMem += 2; + sqlite3VdbeAddOp2(v, OP_OpenDup, pWin->csrApp, pMWin->iEphCsr); + } + else if( p->zName==leadName || p->zName==lagName ){ + pWin->csrApp = pParse->nTab++; + sqlite3VdbeAddOp2(v, OP_OpenDup, pWin->csrApp, pMWin->iEphCsr); + } } } - break; - case 90: /* distinct ::= DISTINCT */ -{yymsp[0].minor.yy502 = SF_Distinct;} - break; - case 91: /* distinct ::= ALL */ -{yymsp[0].minor.yy502 = SF_All;} - break; - case 93: /* sclp ::= */ - case 126: /* orderby_opt ::= */ yytestcase(yyruleno==126); - case 133: /* groupby_opt ::= */ yytestcase(yyruleno==133); - case 212: /* exprlist ::= */ yytestcase(yyruleno==212); - case 215: /* paren_exprlist ::= */ yytestcase(yyruleno==215); - case 220: /* eidlist_opt ::= */ yytestcase(yyruleno==220); -{yymsp[1].minor.yy232 = 0;} - break; - case 94: /* selcollist ::= sclp scanpt expr scanpt as */ -{ - yymsp[-4].minor.yy232 = sqlite3ExprListAppend(pParse, yymsp[-4].minor.yy232, yymsp[-2].minor.yy182); - if( yymsp[0].minor.yy0.n>0 ) sqlite3ExprListSetName(pParse, yymsp[-4].minor.yy232, &yymsp[0].minor.yy0, 1); - sqlite3ExprListSetSpan(pParse,yymsp[-4].minor.yy232,yymsp[-3].minor.yy36,yymsp[-1].minor.yy36); -} - break; - case 95: /* selcollist ::= sclp scanpt STAR */ -{ - Expr *p = sqlite3Expr(pParse->db, TK_ASTERISK, 0); - yymsp[-2].minor.yy232 = sqlite3ExprListAppend(pParse, yymsp[-2].minor.yy232, p); -} - break; - case 96: /* selcollist ::= sclp scanpt nm DOT STAR */ -{ - Expr *pRight = sqlite3PExpr(pParse, TK_ASTERISK, 0, 0); - Expr *pLeft = sqlite3ExprAlloc(pParse->db, TK_ID, &yymsp[-2].minor.yy0, 1); - Expr *pDot = sqlite3PExpr(pParse, TK_DOT, pLeft, pRight); - yymsp[-4].minor.yy232 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy232, pDot); + +#define WINDOW_STARTING_INT 0 +#define WINDOW_ENDING_INT 1 +#define WINDOW_NTH_VALUE_INT 2 +#define WINDOW_STARTING_NUM 3 +#define WINDOW_ENDING_NUM 4 + +/* +** A "PRECEDING " (eCond==0) or "FOLLOWING " (eCond==1) or the +** value of the second argument to nth_value() (eCond==2) has just been +** evaluated and the result left in register reg. This function generates VM +** code to check that the value is a non-negative integer and throws an +** exception if it is not. +*/ +static void windowCheckValue(Parse *pParse, int reg, int eCond){ + static const char *azErr[] = { + "frame starting offset must be a non-negative integer", + "frame ending offset must be a non-negative integer", + "second argument to nth_value must be a positive integer", + "frame starting offset must be a non-negative number", + "frame ending offset must be a non-negative number", + }; + static int aOp[] = { OP_Ge, OP_Ge, OP_Gt, OP_Ge, OP_Ge }; + Vdbe *v = sqlite3GetVdbe(pParse); + int regZero = sqlite3GetTempReg(pParse); + assert( eCond>=0 && eCond=WINDOW_STARTING_NUM ){ + int regString = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp4(v, OP_String8, 0, regString, 0, "", P4_STATIC); + sqlite3VdbeAddOp3(v, OP_Ge, regString, sqlite3VdbeCurrentAddr(v)+2, reg); + sqlite3VdbeChangeP5(v, SQLITE_AFF_NUMERIC|SQLITE_JUMPIFNULL); + VdbeCoverage(v); + assert( eCond==3 || eCond==4 ); + VdbeCoverageIf(v, eCond==3); + VdbeCoverageIf(v, eCond==4); + }else{ + sqlite3VdbeAddOp2(v, OP_MustBeInt, reg, sqlite3VdbeCurrentAddr(v)+2); + VdbeCoverage(v); + assert( eCond==0 || eCond==1 || eCond==2 ); + VdbeCoverageIf(v, eCond==0); + VdbeCoverageIf(v, eCond==1); + VdbeCoverageIf(v, eCond==2); + } + sqlite3VdbeAddOp3(v, aOp[eCond], regZero, sqlite3VdbeCurrentAddr(v)+2, reg); + VdbeCoverageNeverNullIf(v, eCond==0); /* NULL case captured by */ + VdbeCoverageNeverNullIf(v, eCond==1); /* the OP_MustBeInt */ + VdbeCoverageNeverNullIf(v, eCond==2); + VdbeCoverageNeverNullIf(v, eCond==3); /* NULL case caught by */ + VdbeCoverageNeverNullIf(v, eCond==4); /* the OP_Ge */ + sqlite3MayAbort(pParse); + sqlite3VdbeAddOp2(v, OP_Halt, SQLITE_ERROR, OE_Abort); + sqlite3VdbeAppendP4(v, (void*)azErr[eCond], P4_STATIC); + sqlite3ReleaseTempReg(pParse, regZero); } - break; - case 97: /* as ::= AS nm */ - case 108: /* dbnm ::= DOT nm */ yytestcase(yyruleno==108); - case 234: /* plus_num ::= PLUS INTEGER|FLOAT */ yytestcase(yyruleno==234); - case 235: /* minus_num ::= MINUS INTEGER|FLOAT */ yytestcase(yyruleno==235); -{yymsp[-1].minor.yy0 = yymsp[0].minor.yy0;} - break; - case 99: /* from ::= */ -{yymsp[1].minor.yy427 = sqlite3DbMallocZero(pParse->db, sizeof(*yymsp[1].minor.yy427));} - break; - case 100: /* from ::= FROM seltablist */ -{ - yymsp[-1].minor.yy427 = yymsp[0].minor.yy427; - sqlite3SrcListShiftJoinType(yymsp[-1].minor.yy427); + +/* +** Return the number of arguments passed to the window-function associated +** with the object passed as the only argument to this function. +*/ +static int windowArgCount(Window *pWin){ + ExprList *pList = pWin->pOwner->x.pList; + return (pList ? pList->nExpr : 0); } - break; - case 101: /* stl_prefix ::= seltablist joinop */ -{ - if( ALWAYS(yymsp[-1].minor.yy427 && yymsp[-1].minor.yy427->nSrc>0) ) yymsp[-1].minor.yy427->a[yymsp[-1].minor.yy427->nSrc-1].fg.jointype = (u8)yymsp[0].minor.yy502; + +typedef struct WindowCodeArg WindowCodeArg; +typedef struct WindowCsrAndReg WindowCsrAndReg; + +/* +** See comments above struct WindowCodeArg. +*/ +struct WindowCsrAndReg { + int csr; /* Cursor number */ + int reg; /* First in array of peer values */ +}; + +/* +** A single instance of this structure is allocated on the stack by +** sqlite3WindowCodeStep() and a pointer to it passed to the various helper +** routines. This is to reduce the number of arguments required by each +** helper function. +** +** regArg: +** Each window function requires an accumulator register (just as an +** ordinary aggregate function does). This variable is set to the first +** in an array of accumulator registers - one for each window function +** in the WindowCodeArg.pMWin list. +** +** eDelete: +** The window functions implementation sometimes caches the input rows +** that it processes in a temporary table. If it is not zero, this +** variable indicates when rows may be removed from the temp table (in +** order to reduce memory requirements - it would always be safe just +** to leave them there). Possible values for eDelete are: +** +** WINDOW_RETURN_ROW: +** An input row can be discarded after it is returned to the caller. +** +** WINDOW_AGGINVERSE: +** An input row can be discarded after the window functions xInverse() +** callbacks have been invoked in it. +** +** WINDOW_AGGSTEP: +** An input row can be discarded after the window functions xStep() +** callbacks have been invoked in it. +** +** start,current,end +** Consider a window-frame similar to the following: +** +** (ORDER BY a, b GROUPS BETWEEN 2 PRECEDING AND 2 FOLLOWING) +** +** The windows functions implmentation caches the input rows in a temp +** table, sorted by "a, b" (it actually populates the cache lazily, and +** aggressively removes rows once they are no longer required, but that's +** a mere detail). It keeps three cursors open on the temp table. One +** (current) that points to the next row to return to the query engine +** once its window function values have been calculated. Another (end) +** points to the next row to call the xStep() method of each window function +** on (so that it is 2 groups ahead of current). And a third (start) that +** points to the next row to call the xInverse() method of each window +** function on. +** +** Each cursor (start, current and end) consists of a VDBE cursor +** (WindowCsrAndReg.csr) and an array of registers (starting at +** WindowCodeArg.reg) that always contains a copy of the peer values +** read from the corresponding cursor. +** +** Depending on the window-frame in question, all three cursors may not +** be required. In this case both WindowCodeArg.csr and reg are set to +** 0. +*/ +struct WindowCodeArg { + Parse *pParse; /* Parse context */ + Window *pMWin; /* First in list of functions being processed */ + Vdbe *pVdbe; /* VDBE object */ + int addrGosub; /* OP_Gosub to this address to return one row */ + int regGosub; /* Register used with OP_Gosub(addrGosub) */ + int regArg; /* First in array of accumulator registers */ + int eDelete; /* See above */ + + WindowCsrAndReg start; + WindowCsrAndReg current; + WindowCsrAndReg end; +}; + +/* +** Generate VM code to read the window frames peer values from cursor csr into +** an array of registers starting at reg. +*/ +static void windowReadPeerValues( + WindowCodeArg *p, + int csr, + int reg +){ + Window *pMWin = p->pMWin; + ExprList *pOrderBy = pMWin->pOrderBy; + if( pOrderBy ){ + Vdbe *v = sqlite3GetVdbe(p->pParse); + ExprList *pPart = pMWin->pPartition; + int iColOff = pMWin->nBufferCol + (pPart ? pPart->nExpr : 0); + int i; + for(i=0; inExpr; i++){ + sqlite3VdbeAddOp3(v, OP_Column, csr, iColOff+i, reg+i); + } + } } - break; - case 102: /* stl_prefix ::= */ -{yymsp[1].minor.yy427 = 0;} - break; - case 103: /* seltablist ::= stl_prefix nm dbnm as indexed_opt on_opt using_opt */ -{ - yymsp[-6].minor.yy427 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy427,&yymsp[-5].minor.yy0,&yymsp[-4].minor.yy0,&yymsp[-3].minor.yy0,0,yymsp[-1].minor.yy182,yymsp[0].minor.yy510); - sqlite3SrcListIndexedBy(pParse, yymsp[-6].minor.yy427, &yymsp[-2].minor.yy0); + +/* +** Generate VM code to invoke either xStep() (if bInverse is 0) or +** xInverse (if bInverse is non-zero) for each window function in the +** linked list starting at pMWin. Or, for built-in window functions +** that do not use the standard function API, generate the required +** inline VM code. +** +** If argument csr is greater than or equal to 0, then argument reg is +** the first register in an array of registers guaranteed to be large +** enough to hold the array of arguments for each function. In this case +** the arguments are extracted from the current row of csr into the +** array of registers before invoking OP_AggStep or OP_AggInverse +** +** Or, if csr is less than zero, then the array of registers at reg is +** already populated with all columns from the current row of the sub-query. +** +** If argument regPartSize is non-zero, then it is a register containing the +** number of rows in the current partition. +*/ +static void windowAggStep( + WindowCodeArg *p, + Window *pMWin, /* Linked list of window functions */ + int csr, /* Read arguments from this cursor */ + int bInverse, /* True to invoke xInverse instead of xStep */ + int reg /* Array of registers */ +){ + Parse *pParse = p->pParse; + Vdbe *v = sqlite3GetVdbe(pParse); + Window *pWin; + for(pWin=pMWin; pWin; pWin=pWin->pNextWin){ + FuncDef *pFunc = pWin->pFunc; + int regArg; + int nArg = pWin->bExprArgs ? 0 : windowArgCount(pWin); + int i; + + assert( bInverse==0 || pWin->eStart!=TK_UNBOUNDED ); + + /* All OVER clauses in the same window function aggregate step must + ** be the same. */ + assert( pWin==pMWin || sqlite3WindowCompare(pParse,pWin,pMWin,0)!=1 ); + + for(i=0; izName!=nth_valueName ){ + sqlite3VdbeAddOp3(v, OP_Column, csr, pWin->iArgCol+i, reg+i); + }else{ + sqlite3VdbeAddOp3(v, OP_Column, pMWin->iEphCsr, pWin->iArgCol+i, reg+i); + } + } + regArg = reg; + + if( pMWin->regStartRowid==0 + && (pFunc->funcFlags & SQLITE_FUNC_MINMAX) + && (pWin->eStart!=TK_UNBOUNDED) + ){ + int addrIsNull = sqlite3VdbeAddOp1(v, OP_IsNull, regArg); + VdbeCoverage(v); + if( bInverse==0 ){ + sqlite3VdbeAddOp2(v, OP_AddImm, pWin->regApp+1, 1); + sqlite3VdbeAddOp2(v, OP_SCopy, regArg, pWin->regApp); + sqlite3VdbeAddOp3(v, OP_MakeRecord, pWin->regApp, 2, pWin->regApp+2); + sqlite3VdbeAddOp2(v, OP_IdxInsert, pWin->csrApp, pWin->regApp+2); + }else{ + sqlite3VdbeAddOp4Int(v, OP_SeekGE, pWin->csrApp, 0, regArg, 1); + VdbeCoverageNeverTaken(v); + sqlite3VdbeAddOp1(v, OP_Delete, pWin->csrApp); + sqlite3VdbeJumpHere(v, sqlite3VdbeCurrentAddr(v)-2); + } + sqlite3VdbeJumpHere(v, addrIsNull); + }else if( pWin->regApp ){ + assert( pFunc->zName==nth_valueName + || pFunc->zName==first_valueName + ); + assert( bInverse==0 || bInverse==1 ); + sqlite3VdbeAddOp2(v, OP_AddImm, pWin->regApp+1-bInverse, 1); + }else if( pFunc->xSFunc!=noopStepFunc ){ + int addrIf = 0; + if( pWin->pFilter ){ + int regTmp; + assert( pWin->bExprArgs || !nArg ||nArg==pWin->pOwner->x.pList->nExpr ); + assert( pWin->bExprArgs || nArg ||pWin->pOwner->x.pList==0 ); + regTmp = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp3(v, OP_Column, csr, pWin->iArgCol+nArg,regTmp); + addrIf = sqlite3VdbeAddOp3(v, OP_IfNot, regTmp, 0, 1); + VdbeCoverage(v); + sqlite3ReleaseTempReg(pParse, regTmp); + } + + if( pWin->bExprArgs ){ + int iStart = sqlite3VdbeCurrentAddr(v); + VdbeOp *pOp, *pEnd; + + nArg = pWin->pOwner->x.pList->nExpr; + regArg = sqlite3GetTempRange(pParse, nArg); + sqlite3ExprCodeExprList(pParse, pWin->pOwner->x.pList, regArg, 0, 0); + + pEnd = sqlite3VdbeGetOp(v, -1); + for(pOp=sqlite3VdbeGetOp(v, iStart); pOp<=pEnd; pOp++){ + if( pOp->opcode==OP_Column && pOp->p1==pWin->iEphCsr ){ + pOp->p1 = csr; + } + } + } + if( pFunc->funcFlags & SQLITE_FUNC_NEEDCOLL ){ + CollSeq *pColl; + assert( nArg>0 ); + pColl = sqlite3ExprNNCollSeq(pParse, pWin->pOwner->x.pList->a[0].pExpr); + sqlite3VdbeAddOp4(v, OP_CollSeq, 0,0,0, (const char*)pColl, P4_COLLSEQ); + } + sqlite3VdbeAddOp3(v, bInverse? OP_AggInverse : OP_AggStep, + bInverse, regArg, pWin->regAccum); + sqlite3VdbeAppendP4(v, pFunc, P4_FUNCDEF); + sqlite3VdbeChangeP5(v, (u8)nArg); + if( pWin->bExprArgs ){ + sqlite3ReleaseTempRange(pParse, regArg, nArg); + } + if( addrIf ) sqlite3VdbeJumpHere(v, addrIf); + } + } } - break; - case 104: /* seltablist ::= stl_prefix nm dbnm LP exprlist RP as on_opt using_opt */ -{ - yymsp[-8].minor.yy427 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-8].minor.yy427,&yymsp[-7].minor.yy0,&yymsp[-6].minor.yy0,&yymsp[-2].minor.yy0,0,yymsp[-1].minor.yy182,yymsp[0].minor.yy510); - sqlite3SrcListFuncArgs(pParse, yymsp[-8].minor.yy427, yymsp[-4].minor.yy232); + +/* +** Values that may be passed as the second argument to windowCodeOp(). +*/ +#define WINDOW_RETURN_ROW 1 +#define WINDOW_AGGINVERSE 2 +#define WINDOW_AGGSTEP 3 + +/* +** Generate VM code to invoke either xValue() (bFin==0) or xFinalize() +** (bFin==1) for each window function in the linked list starting at +** pMWin. Or, for built-in window-functions that do not use the standard +** API, generate the equivalent VM code. +*/ +static void windowAggFinal(WindowCodeArg *p, int bFin){ + Parse *pParse = p->pParse; + Window *pMWin = p->pMWin; + Vdbe *v = sqlite3GetVdbe(pParse); + Window *pWin; + + for(pWin=pMWin; pWin; pWin=pWin->pNextWin){ + if( pMWin->regStartRowid==0 + && (pWin->pFunc->funcFlags & SQLITE_FUNC_MINMAX) + && (pWin->eStart!=TK_UNBOUNDED) + ){ + sqlite3VdbeAddOp2(v, OP_Null, 0, pWin->regResult); + sqlite3VdbeAddOp1(v, OP_Last, pWin->csrApp); + VdbeCoverage(v); + sqlite3VdbeAddOp3(v, OP_Column, pWin->csrApp, 0, pWin->regResult); + sqlite3VdbeJumpHere(v, sqlite3VdbeCurrentAddr(v)-2); + }else if( pWin->regApp ){ + assert( pMWin->regStartRowid==0 ); + }else{ + int nArg = windowArgCount(pWin); + if( bFin ){ + sqlite3VdbeAddOp2(v, OP_AggFinal, pWin->regAccum, nArg); + sqlite3VdbeAppendP4(v, pWin->pFunc, P4_FUNCDEF); + sqlite3VdbeAddOp2(v, OP_Copy, pWin->regAccum, pWin->regResult); + sqlite3VdbeAddOp2(v, OP_Null, 0, pWin->regAccum); + }else{ + sqlite3VdbeAddOp3(v, OP_AggValue,pWin->regAccum,nArg,pWin->regResult); + sqlite3VdbeAppendP4(v, pWin->pFunc, P4_FUNCDEF); + } + } + } } - break; - case 105: /* seltablist ::= stl_prefix LP select RP as on_opt using_opt */ -{ - yymsp[-6].minor.yy427 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy427,0,0,&yymsp[-2].minor.yy0,yymsp[-4].minor.yy399,yymsp[-1].minor.yy182,yymsp[0].minor.yy510); + +/* +** Generate code to calculate the current values of all window functions in the +** p->pMWin list by doing a full scan of the current window frame. Store the +** results in the Window.regResult registers, ready to return the upper +** layer. +*/ +static void windowFullScan(WindowCodeArg *p){ + Window *pWin; + Parse *pParse = p->pParse; + Window *pMWin = p->pMWin; + Vdbe *v = p->pVdbe; + + int regCRowid = 0; /* Current rowid value */ + int regCPeer = 0; /* Current peer values */ + int regRowid = 0; /* AggStep rowid value */ + int regPeer = 0; /* AggStep peer values */ + + int nPeer; + int lblNext; + int lblBrk; + int addrNext; + int csr; + + VdbeModuleComment((v, "windowFullScan begin")); + + assert( pMWin!=0 ); + csr = pMWin->csrApp; + nPeer = (pMWin->pOrderBy ? pMWin->pOrderBy->nExpr : 0); + + lblNext = sqlite3VdbeMakeLabel(pParse); + lblBrk = sqlite3VdbeMakeLabel(pParse); + + regCRowid = sqlite3GetTempReg(pParse); + regRowid = sqlite3GetTempReg(pParse); + if( nPeer ){ + regCPeer = sqlite3GetTempRange(pParse, nPeer); + regPeer = sqlite3GetTempRange(pParse, nPeer); } - break; - case 106: /* seltablist ::= stl_prefix LP seltablist RP as on_opt using_opt */ -{ - if( yymsp[-6].minor.yy427==0 && yymsp[-2].minor.yy0.n==0 && yymsp[-1].minor.yy182==0 && yymsp[0].minor.yy510==0 ){ - yymsp[-6].minor.yy427 = yymsp[-4].minor.yy427; - }else if( yymsp[-4].minor.yy427->nSrc==1 ){ - yymsp[-6].minor.yy427 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy427,0,0,&yymsp[-2].minor.yy0,0,yymsp[-1].minor.yy182,yymsp[0].minor.yy510); - if( yymsp[-6].minor.yy427 ){ - struct SrcList_item *pNew = &yymsp[-6].minor.yy427->a[yymsp[-6].minor.yy427->nSrc-1]; - struct SrcList_item *pOld = yymsp[-4].minor.yy427->a; - pNew->zName = pOld->zName; - pNew->zDatabase = pOld->zDatabase; - pNew->pSelect = pOld->pSelect; - pOld->zName = pOld->zDatabase = 0; - pOld->pSelect = 0; - } - sqlite3SrcListDelete(pParse->db, yymsp[-4].minor.yy427); + + sqlite3VdbeAddOp2(v, OP_Rowid, pMWin->iEphCsr, regCRowid); + windowReadPeerValues(p, pMWin->iEphCsr, regCPeer); + + for(pWin=pMWin; pWin; pWin=pWin->pNextWin){ + sqlite3VdbeAddOp2(v, OP_Null, 0, pWin->regAccum); + } + + sqlite3VdbeAddOp3(v, OP_SeekGE, csr, lblBrk, pMWin->regStartRowid); + VdbeCoverage(v); + addrNext = sqlite3VdbeCurrentAddr(v); + sqlite3VdbeAddOp2(v, OP_Rowid, csr, regRowid); + sqlite3VdbeAddOp3(v, OP_Gt, pMWin->regEndRowid, lblBrk, regRowid); + VdbeCoverageNeverNull(v); + + if( pMWin->eExclude==TK_CURRENT ){ + sqlite3VdbeAddOp3(v, OP_Eq, regCRowid, lblNext, regRowid); + VdbeCoverageNeverNull(v); + }else if( pMWin->eExclude!=TK_NO ){ + int addr; + int addrEq = 0; + KeyInfo *pKeyInfo = 0; + + if( pMWin->pOrderBy ){ + pKeyInfo = sqlite3KeyInfoFromExprList(pParse, pMWin->pOrderBy, 0, 0); + } + if( pMWin->eExclude==TK_TIES ){ + addrEq = sqlite3VdbeAddOp3(v, OP_Eq, regCRowid, 0, regRowid); + VdbeCoverageNeverNull(v); + } + if( pKeyInfo ){ + windowReadPeerValues(p, csr, regPeer); + sqlite3VdbeAddOp3(v, OP_Compare, regPeer, regCPeer, nPeer); + sqlite3VdbeAppendP4(v, (void*)pKeyInfo, P4_KEYINFO); + addr = sqlite3VdbeCurrentAddr(v)+1; + sqlite3VdbeAddOp3(v, OP_Jump, addr, lblNext, addr); + VdbeCoverageEqNe(v); }else{ - Select *pSubquery; - sqlite3SrcListShiftJoinType(yymsp[-4].minor.yy427); - pSubquery = sqlite3SelectNew(pParse,0,yymsp[-4].minor.yy427,0,0,0,0,SF_NestedFrom,0); - yymsp[-6].minor.yy427 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy427,0,0,&yymsp[-2].minor.yy0,pSubquery,yymsp[-1].minor.yy182,yymsp[0].minor.yy510); + sqlite3VdbeAddOp2(v, OP_Goto, 0, lblNext); } + if( addrEq ) sqlite3VdbeJumpHere(v, addrEq); } - break; - case 107: /* dbnm ::= */ - case 121: /* indexed_opt ::= */ yytestcase(yyruleno==121); -{yymsp[1].minor.yy0.z=0; yymsp[1].minor.yy0.n=0;} - break; - case 109: /* fullname ::= nm */ - case 111: /* xfullname ::= nm */ yytestcase(yyruleno==111); -{yymsp[0].minor.yy427 = sqlite3SrcListAppend(pParse->db,0,&yymsp[0].minor.yy0,0); /*A-overwrites-X*/} - break; - case 110: /* fullname ::= nm DOT nm */ - case 112: /* xfullname ::= nm DOT nm */ yytestcase(yyruleno==112); -{yymsp[-2].minor.yy427 = sqlite3SrcListAppend(pParse->db,0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0); /*A-overwrites-X*/} - break; - case 113: /* xfullname ::= nm DOT nm AS nm */ -{ - yymsp[-4].minor.yy427 = sqlite3SrcListAppend(pParse->db,0,&yymsp[-4].minor.yy0,&yymsp[-2].minor.yy0); /*A-overwrites-X*/ - if( yymsp[-4].minor.yy427 ) yymsp[-4].minor.yy427->a[0].zAlias = sqlite3NameFromToken(pParse->db, &yymsp[0].minor.yy0); -} - break; - case 114: /* xfullname ::= nm AS nm */ -{ - yymsp[-2].minor.yy427 = sqlite3SrcListAppend(pParse->db,0,&yymsp[-2].minor.yy0,0); /*A-overwrites-X*/ - if( yymsp[-2].minor.yy427 ) yymsp[-2].minor.yy427->a[0].zAlias = sqlite3NameFromToken(pParse->db, &yymsp[0].minor.yy0); -} - break; - case 115: /* joinop ::= COMMA|JOIN */ -{ yymsp[0].minor.yy502 = JT_INNER; } - break; - case 116: /* joinop ::= JOIN_KW JOIN */ -{yymsp[-1].minor.yy502 = sqlite3JoinType(pParse,&yymsp[-1].minor.yy0,0,0); /*X-overwrites-A*/} - break; - case 117: /* joinop ::= JOIN_KW nm JOIN */ -{yymsp[-2].minor.yy502 = sqlite3JoinType(pParse,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0,0); /*X-overwrites-A*/} - break; - case 118: /* joinop ::= JOIN_KW nm nm JOIN */ -{yymsp[-3].minor.yy502 = sqlite3JoinType(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0);/*X-overwrites-A*/} - break; - case 119: /* on_opt ::= ON expr */ - case 136: /* having_opt ::= HAVING expr */ yytestcase(yyruleno==136); - case 143: /* where_opt ::= WHERE expr */ yytestcase(yyruleno==143); - case 208: /* case_else ::= ELSE expr */ yytestcase(yyruleno==208); -{yymsp[-1].minor.yy182 = yymsp[0].minor.yy182;} - break; - case 120: /* on_opt ::= */ - case 135: /* having_opt ::= */ yytestcase(yyruleno==135); - case 137: /* limit_opt ::= */ yytestcase(yyruleno==137); - case 142: /* where_opt ::= */ yytestcase(yyruleno==142); - case 209: /* case_else ::= */ yytestcase(yyruleno==209); - case 211: /* case_operand ::= */ yytestcase(yyruleno==211); -{yymsp[1].minor.yy182 = 0;} - break; - case 122: /* indexed_opt ::= INDEXED BY nm */ -{yymsp[-2].minor.yy0 = yymsp[0].minor.yy0;} - break; - case 123: /* indexed_opt ::= NOT INDEXED */ -{yymsp[-1].minor.yy0.z=0; yymsp[-1].minor.yy0.n=1;} - break; - case 124: /* using_opt ::= USING LP idlist RP */ -{yymsp[-3].minor.yy510 = yymsp[-1].minor.yy510;} - break; - case 125: /* using_opt ::= */ - case 157: /* idlist_opt ::= */ yytestcase(yyruleno==157); -{yymsp[1].minor.yy510 = 0;} - break; - case 127: /* orderby_opt ::= ORDER BY sortlist */ - case 134: /* groupby_opt ::= GROUP BY nexprlist */ yytestcase(yyruleno==134); -{yymsp[-2].minor.yy232 = yymsp[0].minor.yy232;} - break; - case 128: /* sortlist ::= sortlist COMMA expr sortorder */ -{ - yymsp[-3].minor.yy232 = sqlite3ExprListAppend(pParse,yymsp[-3].minor.yy232,yymsp[-1].minor.yy182); - sqlite3ExprListSetSortOrder(yymsp[-3].minor.yy232,yymsp[0].minor.yy502); -} - break; - case 129: /* sortlist ::= expr sortorder */ -{ - yymsp[-1].minor.yy232 = sqlite3ExprListAppend(pParse,0,yymsp[-1].minor.yy182); /*A-overwrites-Y*/ - sqlite3ExprListSetSortOrder(yymsp[-1].minor.yy232,yymsp[0].minor.yy502); -} - break; - case 130: /* sortorder ::= ASC */ -{yymsp[0].minor.yy502 = SQLITE_SO_ASC;} - break; - case 131: /* sortorder ::= DESC */ -{yymsp[0].minor.yy502 = SQLITE_SO_DESC;} - break; - case 132: /* sortorder ::= */ -{yymsp[1].minor.yy502 = SQLITE_SO_UNDEFINED;} - break; - case 138: /* limit_opt ::= LIMIT expr */ -{yymsp[-1].minor.yy182 = sqlite3PExpr(pParse,TK_LIMIT,yymsp[0].minor.yy182,0);} - break; - case 139: /* limit_opt ::= LIMIT expr OFFSET expr */ -{yymsp[-3].minor.yy182 = sqlite3PExpr(pParse,TK_LIMIT,yymsp[-2].minor.yy182,yymsp[0].minor.yy182);} - break; - case 140: /* limit_opt ::= LIMIT expr COMMA expr */ -{yymsp[-3].minor.yy182 = sqlite3PExpr(pParse,TK_LIMIT,yymsp[0].minor.yy182,yymsp[-2].minor.yy182);} - break; - case 141: /* cmd ::= with DELETE FROM xfullname indexed_opt where_opt */ -{ - sqlite3SrcListIndexedBy(pParse, yymsp[-2].minor.yy427, &yymsp[-1].minor.yy0); - sqlite3DeleteFrom(pParse,yymsp[-2].minor.yy427,yymsp[0].minor.yy182,0,0); -} - break; - case 144: /* cmd ::= with UPDATE orconf xfullname indexed_opt SET setlist where_opt */ -{ - sqlite3SrcListIndexedBy(pParse, yymsp[-4].minor.yy427, &yymsp[-3].minor.yy0); - sqlite3ExprListCheckLength(pParse,yymsp[-1].minor.yy232,"set list"); - sqlite3Update(pParse,yymsp[-4].minor.yy427,yymsp[-1].minor.yy232,yymsp[0].minor.yy182,yymsp[-5].minor.yy502,0,0,0); -} - break; - case 145: /* setlist ::= setlist COMMA nm EQ expr */ -{ - yymsp[-4].minor.yy232 = sqlite3ExprListAppend(pParse, yymsp[-4].minor.yy232, yymsp[0].minor.yy182); - sqlite3ExprListSetName(pParse, yymsp[-4].minor.yy232, &yymsp[-2].minor.yy0, 1); -} - break; - case 146: /* setlist ::= setlist COMMA LP idlist RP EQ expr */ -{ - yymsp[-6].minor.yy232 = sqlite3ExprListAppendVector(pParse, yymsp[-6].minor.yy232, yymsp[-3].minor.yy510, yymsp[0].minor.yy182); -} - break; - case 147: /* setlist ::= nm EQ expr */ -{ - yylhsminor.yy232 = sqlite3ExprListAppend(pParse, 0, yymsp[0].minor.yy182); - sqlite3ExprListSetName(pParse, yylhsminor.yy232, &yymsp[-2].minor.yy0, 1); -} - yymsp[-2].minor.yy232 = yylhsminor.yy232; - break; - case 148: /* setlist ::= LP idlist RP EQ expr */ -{ - yymsp[-4].minor.yy232 = sqlite3ExprListAppendVector(pParse, 0, yymsp[-3].minor.yy510, yymsp[0].minor.yy182); -} - break; - case 149: /* cmd ::= with insert_cmd INTO xfullname idlist_opt select upsert */ -{ - sqlite3Insert(pParse, yymsp[-3].minor.yy427, yymsp[-1].minor.yy399, yymsp[-2].minor.yy510, yymsp[-5].minor.yy502, yymsp[0].minor.yy198); -} - break; - case 150: /* cmd ::= with insert_cmd INTO xfullname idlist_opt DEFAULT VALUES */ -{ - sqlite3Insert(pParse, yymsp[-3].minor.yy427, 0, yymsp[-2].minor.yy510, yymsp[-5].minor.yy502, 0); -} - break; - case 151: /* upsert ::= */ -{ yymsp[1].minor.yy198 = 0; } - break; - case 152: /* upsert ::= ON CONFLICT LP sortlist RP where_opt DO UPDATE SET setlist where_opt */ -{ yymsp[-10].minor.yy198 = sqlite3UpsertNew(pParse->db,yymsp[-7].minor.yy232,yymsp[-5].minor.yy182,yymsp[-1].minor.yy232,yymsp[0].minor.yy182);} - break; - case 153: /* upsert ::= ON CONFLICT LP sortlist RP where_opt DO NOTHING */ -{ yymsp[-7].minor.yy198 = sqlite3UpsertNew(pParse->db,yymsp[-4].minor.yy232,yymsp[-2].minor.yy182,0,0); } - break; - case 154: /* upsert ::= ON CONFLICT DO NOTHING */ -{ yymsp[-3].minor.yy198 = sqlite3UpsertNew(pParse->db,0,0,0,0); } - break; - case 158: /* idlist_opt ::= LP idlist RP */ -{yymsp[-2].minor.yy510 = yymsp[-1].minor.yy510;} - break; - case 159: /* idlist ::= idlist COMMA nm */ -{yymsp[-2].minor.yy510 = sqlite3IdListAppend(pParse->db,yymsp[-2].minor.yy510,&yymsp[0].minor.yy0);} - break; - case 160: /* idlist ::= nm */ -{yymsp[0].minor.yy510 = sqlite3IdListAppend(pParse->db,0,&yymsp[0].minor.yy0); /*A-overwrites-Y*/} - break; - case 161: /* expr ::= LP expr RP */ -{yymsp[-2].minor.yy182 = yymsp[-1].minor.yy182;} - break; - case 162: /* expr ::= ID|INDEXED */ - case 163: /* expr ::= JOIN_KW */ yytestcase(yyruleno==163); -{yymsp[0].minor.yy182=tokenExpr(pParse,TK_ID,yymsp[0].minor.yy0); /*A-overwrites-X*/} - break; - case 164: /* expr ::= nm DOT nm */ -{ - Expr *temp1 = sqlite3ExprAlloc(pParse->db, TK_ID, &yymsp[-2].minor.yy0, 1); - Expr *temp2 = sqlite3ExprAlloc(pParse->db, TK_ID, &yymsp[0].minor.yy0, 1); - yylhsminor.yy182 = sqlite3PExpr(pParse, TK_DOT, temp1, temp2); -} - yymsp[-2].minor.yy182 = yylhsminor.yy182; - break; - case 165: /* expr ::= nm DOT nm DOT nm */ -{ - Expr *temp1 = sqlite3ExprAlloc(pParse->db, TK_ID, &yymsp[-4].minor.yy0, 1); - Expr *temp2 = sqlite3ExprAlloc(pParse->db, TK_ID, &yymsp[-2].minor.yy0, 1); - Expr *temp3 = sqlite3ExprAlloc(pParse->db, TK_ID, &yymsp[0].minor.yy0, 1); - Expr *temp4 = sqlite3PExpr(pParse, TK_DOT, temp2, temp3); - yylhsminor.yy182 = sqlite3PExpr(pParse, TK_DOT, temp1, temp4); -} - yymsp[-4].minor.yy182 = yylhsminor.yy182; - break; - case 166: /* term ::= NULL|FLOAT|BLOB */ - case 167: /* term ::= STRING */ yytestcase(yyruleno==167); -{yymsp[0].minor.yy182=tokenExpr(pParse,yymsp[0].major,yymsp[0].minor.yy0); /*A-overwrites-X*/} - break; - case 168: /* term ::= INTEGER */ -{ - yylhsminor.yy182 = sqlite3ExprAlloc(pParse->db, TK_INTEGER, &yymsp[0].minor.yy0, 1); + + windowAggStep(p, pMWin, csr, 0, p->regArg); + + sqlite3VdbeResolveLabel(v, lblNext); + sqlite3VdbeAddOp2(v, OP_Next, csr, addrNext); + VdbeCoverage(v); + sqlite3VdbeJumpHere(v, addrNext-1); + sqlite3VdbeJumpHere(v, addrNext+1); + sqlite3ReleaseTempReg(pParse, regRowid); + sqlite3ReleaseTempReg(pParse, regCRowid); + if( nPeer ){ + sqlite3ReleaseTempRange(pParse, regPeer, nPeer); + sqlite3ReleaseTempRange(pParse, regCPeer, nPeer); + } + + windowAggFinal(p, 1); + VdbeModuleComment((v, "windowFullScan end")); } - yymsp[0].minor.yy182 = yylhsminor.yy182; - break; - case 169: /* expr ::= VARIABLE */ -{ - if( !(yymsp[0].minor.yy0.z[0]=='#' && sqlite3Isdigit(yymsp[0].minor.yy0.z[1])) ){ - u32 n = yymsp[0].minor.yy0.n; - yymsp[0].minor.yy182 = tokenExpr(pParse, TK_VARIABLE, yymsp[0].minor.yy0); - sqlite3ExprAssignVarNumber(pParse, yymsp[0].minor.yy182, n); + +/* +** Invoke the sub-routine at regGosub (generated by code in select.c) to +** return the current row of Window.iEphCsr. If all window functions are +** aggregate window functions that use the standard API, a single +** OP_Gosub instruction is all that this routine generates. Extra VM code +** for per-row processing is only generated for the following built-in window +** functions: +** +** nth_value() +** first_value() +** lag() +** lead() +*/ +static void windowReturnOneRow(WindowCodeArg *p){ + Window *pMWin = p->pMWin; + Vdbe *v = p->pVdbe; + + if( pMWin->regStartRowid ){ + windowFullScan(p); }else{ - /* When doing a nested parse, one can include terms in an expression - ** that look like this: #1 #2 ... These terms refer to registers - ** in the virtual machine. #N is the N-th register. */ - Token t = yymsp[0].minor.yy0; /*A-overwrites-X*/ - assert( t.n>=2 ); - if( pParse->nested==0 ){ - sqlite3ErrorMsg(pParse, "near \"%T\": syntax error", &t); - yymsp[0].minor.yy182 = 0; - }else{ - yymsp[0].minor.yy182 = sqlite3PExpr(pParse, TK_REGISTER, 0, 0); - if( yymsp[0].minor.yy182 ) sqlite3GetInt32(&t.z[1], &yymsp[0].minor.yy182->iTable); + Parse *pParse = p->pParse; + Window *pWin; + + for(pWin=pMWin; pWin; pWin=pWin->pNextWin){ + FuncDef *pFunc = pWin->pFunc; + if( pFunc->zName==nth_valueName + || pFunc->zName==first_valueName + ){ + int csr = pWin->csrApp; + int lbl = sqlite3VdbeMakeLabel(pParse); + int tmpReg = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp2(v, OP_Null, 0, pWin->regResult); + + if( pFunc->zName==nth_valueName ){ + sqlite3VdbeAddOp3(v, OP_Column,pMWin->iEphCsr,pWin->iArgCol+1,tmpReg); + windowCheckValue(pParse, tmpReg, 2); + }else{ + sqlite3VdbeAddOp2(v, OP_Integer, 1, tmpReg); + } + sqlite3VdbeAddOp3(v, OP_Add, tmpReg, pWin->regApp, tmpReg); + sqlite3VdbeAddOp3(v, OP_Gt, pWin->regApp+1, lbl, tmpReg); + VdbeCoverageNeverNull(v); + sqlite3VdbeAddOp3(v, OP_SeekRowid, csr, 0, tmpReg); + VdbeCoverageNeverTaken(v); + sqlite3VdbeAddOp3(v, OP_Column, csr, pWin->iArgCol, pWin->regResult); + sqlite3VdbeResolveLabel(v, lbl); + sqlite3ReleaseTempReg(pParse, tmpReg); + } + else if( pFunc->zName==leadName || pFunc->zName==lagName ){ + int nArg = pWin->pOwner->x.pList->nExpr; + int csr = pWin->csrApp; + int lbl = sqlite3VdbeMakeLabel(pParse); + int tmpReg = sqlite3GetTempReg(pParse); + int iEph = pMWin->iEphCsr; + + if( nArg<3 ){ + sqlite3VdbeAddOp2(v, OP_Null, 0, pWin->regResult); + }else{ + sqlite3VdbeAddOp3(v, OP_Column, iEph,pWin->iArgCol+2,pWin->regResult); + } + sqlite3VdbeAddOp2(v, OP_Rowid, iEph, tmpReg); + if( nArg<2 ){ + int val = (pFunc->zName==leadName ? 1 : -1); + sqlite3VdbeAddOp2(v, OP_AddImm, tmpReg, val); + }else{ + int op = (pFunc->zName==leadName ? OP_Add : OP_Subtract); + int tmpReg2 = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp3(v, OP_Column, iEph, pWin->iArgCol+1, tmpReg2); + sqlite3VdbeAddOp3(v, op, tmpReg2, tmpReg, tmpReg); + sqlite3ReleaseTempReg(pParse, tmpReg2); + } + + sqlite3VdbeAddOp3(v, OP_SeekRowid, csr, lbl, tmpReg); + VdbeCoverage(v); + sqlite3VdbeAddOp3(v, OP_Column, csr, pWin->iArgCol, pWin->regResult); + sqlite3VdbeResolveLabel(v, lbl); + sqlite3ReleaseTempReg(pParse, tmpReg); + } } } + sqlite3VdbeAddOp2(v, OP_Gosub, p->regGosub, p->addrGosub); } - break; - case 170: /* expr ::= expr COLLATE ID|STRING */ -{ - yymsp[-2].minor.yy182 = sqlite3ExprAddCollateToken(pParse, yymsp[-2].minor.yy182, &yymsp[0].minor.yy0, 1); -} - break; - case 171: /* expr ::= CAST LP expr AS typetoken RP */ -{ - yymsp[-5].minor.yy182 = sqlite3ExprAlloc(pParse->db, TK_CAST, &yymsp[-1].minor.yy0, 1); - sqlite3ExprAttachSubtrees(pParse->db, yymsp[-5].minor.yy182, yymsp[-3].minor.yy182, 0); -} - break; - case 172: /* expr ::= ID|INDEXED LP distinct exprlist RP */ -{ - if( yymsp[-1].minor.yy232 && yymsp[-1].minor.yy232->nExpr>pParse->db->aLimit[SQLITE_LIMIT_FUNCTION_ARG] ){ - sqlite3ErrorMsg(pParse, "too many arguments on function %T", &yymsp[-4].minor.yy0); - } - yylhsminor.yy182 = sqlite3ExprFunction(pParse, yymsp[-1].minor.yy232, &yymsp[-4].minor.yy0); - if( yymsp[-2].minor.yy502==SF_Distinct && yylhsminor.yy182 ){ - yylhsminor.yy182->flags |= EP_Distinct; + +/* +** Generate code to set the accumulator register for each window function +** in the linked list passed as the second argument to NULL. And perform +** any equivalent initialization required by any built-in window functions +** in the list. +*/ +static int windowInitAccum(Parse *pParse, Window *pMWin){ + Vdbe *v = sqlite3GetVdbe(pParse); + int regArg; + int nArg = 0; + Window *pWin; + for(pWin=pMWin; pWin; pWin=pWin->pNextWin){ + FuncDef *pFunc = pWin->pFunc; + assert( pWin->regAccum ); + sqlite3VdbeAddOp2(v, OP_Null, 0, pWin->regAccum); + nArg = MAX(nArg, windowArgCount(pWin)); + if( pMWin->regStartRowid==0 ){ + if( pFunc->zName==nth_valueName || pFunc->zName==first_valueName ){ + sqlite3VdbeAddOp2(v, OP_Integer, 0, pWin->regApp); + sqlite3VdbeAddOp2(v, OP_Integer, 0, pWin->regApp+1); + } + + if( (pFunc->funcFlags & SQLITE_FUNC_MINMAX) && pWin->csrApp ){ + assert( pWin->eStart!=TK_UNBOUNDED ); + sqlite3VdbeAddOp1(v, OP_ResetSorter, pWin->csrApp); + sqlite3VdbeAddOp2(v, OP_Integer, 0, pWin->regApp+1); + } + } } + regArg = pParse->nMem+1; + pParse->nMem += nArg; + return regArg; } - yymsp[-4].minor.yy182 = yylhsminor.yy182; - break; - case 173: /* expr ::= ID|INDEXED LP STAR RP */ -{ - yylhsminor.yy182 = sqlite3ExprFunction(pParse, 0, &yymsp[-3].minor.yy0); -} - yymsp[-3].minor.yy182 = yylhsminor.yy182; - break; - case 174: /* term ::= CTIME_KW */ -{ - yylhsminor.yy182 = sqlite3ExprFunction(pParse, 0, &yymsp[0].minor.yy0); + +/* +** Return true if the current frame should be cached in the ephemeral table, +** even if there are no xInverse() calls required. +*/ +static int windowCacheFrame(Window *pMWin){ + Window *pWin; + if( pMWin->regStartRowid ) return 1; + for(pWin=pMWin; pWin; pWin=pWin->pNextWin){ + FuncDef *pFunc = pWin->pFunc; + if( (pFunc->zName==nth_valueName) + || (pFunc->zName==first_valueName) + || (pFunc->zName==leadName) + || (pFunc->zName==lagName) + ){ + return 1; + } + } + return 0; } - yymsp[0].minor.yy182 = yylhsminor.yy182; - break; - case 175: /* expr ::= LP nexprlist COMMA expr RP */ -{ - ExprList *pList = sqlite3ExprListAppend(pParse, yymsp[-3].minor.yy232, yymsp[-1].minor.yy182); - yymsp[-4].minor.yy182 = sqlite3PExpr(pParse, TK_VECTOR, 0, 0); - if( yymsp[-4].minor.yy182 ){ - yymsp[-4].minor.yy182->x.pList = pList; + +/* +** regOld and regNew are each the first register in an array of size +** pOrderBy->nExpr. This function generates code to compare the two +** arrays of registers using the collation sequences and other comparison +** parameters specified by pOrderBy. +** +** If the two arrays are not equal, the contents of regNew is copied to +** regOld and control falls through. Otherwise, if the contents of the arrays +** are equal, an OP_Goto is executed. The address of the OP_Goto is returned. +*/ +static void windowIfNewPeer( + Parse *pParse, + ExprList *pOrderBy, + int regNew, /* First in array of new values */ + int regOld, /* First in array of old values */ + int addr /* Jump here */ +){ + Vdbe *v = sqlite3GetVdbe(pParse); + if( pOrderBy ){ + int nVal = pOrderBy->nExpr; + KeyInfo *pKeyInfo = sqlite3KeyInfoFromExprList(pParse, pOrderBy, 0, 0); + sqlite3VdbeAddOp3(v, OP_Compare, regOld, regNew, nVal); + sqlite3VdbeAppendP4(v, (void*)pKeyInfo, P4_KEYINFO); + sqlite3VdbeAddOp3(v, OP_Jump, + sqlite3VdbeCurrentAddr(v)+1, addr, sqlite3VdbeCurrentAddr(v)+1 + ); + VdbeCoverageEqNe(v); + sqlite3VdbeAddOp3(v, OP_Copy, regNew, regOld, nVal-1); }else{ - sqlite3ExprListDelete(pParse->db, pList); + sqlite3VdbeAddOp2(v, OP_Goto, 0, addr); } } + +/* +** This function is called as part of generating VM programs for RANGE +** offset PRECEDING/FOLLOWING frame boundaries. Assuming "ASC" order for +** the ORDER BY term in the window, and that argument op is OP_Ge, it generates +** code equivalent to: +** +** if( csr1.peerVal + regVal >= csr2.peerVal ) goto lbl; +** +** The value of parameter op may also be OP_Gt or OP_Le. In these cases the +** operator in the above pseudo-code is replaced with ">" or "<=", respectively. +** +** If the sort-order for the ORDER BY term in the window is DESC, then the +** comparison is reversed. Instead of adding regVal to csr1.peerVal, it is +** subtracted. And the comparison operator is inverted to - ">=" becomes "<=", +** ">" becomes "<", and so on. So, with DESC sort order, if the argument op +** is OP_Ge, the generated code is equivalent to: +** +** if( csr1.peerVal - regVal <= csr2.peerVal ) goto lbl; +** +** A special type of arithmetic is used such that if csr1.peerVal is not +** a numeric type (real or integer), then the result of the addition addition +** or subtraction is a a copy of csr1.peerVal. +*/ +static void windowCodeRangeTest( + WindowCodeArg *p, + int op, /* OP_Ge, OP_Gt, or OP_Le */ + int csr1, /* Cursor number for cursor 1 */ + int regVal, /* Register containing non-negative number */ + int csr2, /* Cursor number for cursor 2 */ + int lbl /* Jump destination if condition is true */ +){ + Parse *pParse = p->pParse; + Vdbe *v = sqlite3GetVdbe(pParse); + ExprList *pOrderBy = p->pMWin->pOrderBy; /* ORDER BY clause for window */ + int reg1 = sqlite3GetTempReg(pParse); /* Reg. for csr1.peerVal+regVal */ + int reg2 = sqlite3GetTempReg(pParse); /* Reg. for csr2.peerVal */ + int regString = ++pParse->nMem; /* Reg. for constant value '' */ + int arith = OP_Add; /* OP_Add or OP_Subtract */ + int addrGe; /* Jump destination */ + + assert( op==OP_Ge || op==OP_Gt || op==OP_Le ); + assert( pOrderBy && pOrderBy->nExpr==1 ); + if( pOrderBy->a[0].sortFlags & KEYINFO_ORDER_DESC ){ + switch( op ){ + case OP_Ge: op = OP_Le; break; + case OP_Gt: op = OP_Lt; break; + default: assert( op==OP_Le ); op = OP_Ge; break; + } + arith = OP_Subtract; + } + + /* Read the peer-value from each cursor into a register */ + windowReadPeerValues(p, csr1, reg1); + windowReadPeerValues(p, csr2, reg2); + + VdbeModuleComment((v, "CodeRangeTest: if( R%d %s R%d %s R%d ) goto lbl", + reg1, (arith==OP_Add ? "+" : "-"), regVal, + ((op==OP_Ge) ? ">=" : (op==OP_Le) ? "<=" : (op==OP_Gt) ? ">" : "<"), reg2 + )); + + /* Register reg1 currently contains csr1.peerVal (the peer-value from csr1). + ** This block adds (or subtracts for DESC) the numeric value in regVal + ** from it. Or, if reg1 is not numeric (it is a NULL, a text value or a blob), + ** then leave reg1 as it is. In pseudo-code, this is implemented as: + ** + ** if( reg1>='' ) goto addrGe; + ** reg1 = reg1 +/- regVal + ** addrGe: + ** + ** Since all strings and blobs are greater-than-or-equal-to an empty string, + ** the add/subtract is skipped for these, as required. If reg1 is a NULL, + ** then the arithmetic is performed, but since adding or subtracting from + ** NULL is always NULL anyway, this case is handled as required too. */ + sqlite3VdbeAddOp4(v, OP_String8, 0, regString, 0, "", P4_STATIC); + addrGe = sqlite3VdbeAddOp3(v, OP_Ge, regString, 0, reg1); + VdbeCoverage(v); + sqlite3VdbeAddOp3(v, arith, regVal, reg1, reg1); + sqlite3VdbeJumpHere(v, addrGe); + + /* If the BIGNULL flag is set for the ORDER BY, then it is required to + ** consider NULL values to be larger than all other values, instead of + ** the usual smaller. The VDBE opcodes OP_Ge and so on do not handle this + ** (and adding that capability causes a performance regression), so + ** instead if the BIGNULL flag is set then cases where either reg1 or + ** reg2 are NULL are handled separately in the following block. The code + ** generated is equivalent to: + ** + ** if( reg1 IS NULL ){ + ** if( op==OP_Ge ) goto lbl; + ** if( op==OP_Gt && reg2 IS NOT NULL ) goto lbl; + ** if( op==OP_Le && reg2 IS NULL ) goto lbl; + ** }else if( reg2 IS NULL ){ + ** if( op==OP_Le ) goto lbl; + ** } + ** + ** Additionally, if either reg1 or reg2 are NULL but the jump to lbl is + ** not taken, control jumps over the comparison operator coded below this + ** block. */ + if( pOrderBy->a[0].sortFlags & KEYINFO_ORDER_BIGNULL ){ + /* This block runs if reg1 contains a NULL. */ + int addr = sqlite3VdbeAddOp1(v, OP_NotNull, reg1); VdbeCoverage(v); + switch( op ){ + case OP_Ge: + sqlite3VdbeAddOp2(v, OP_Goto, 0, lbl); break; - case 176: /* expr ::= expr AND expr */ - case 177: /* expr ::= expr OR expr */ yytestcase(yyruleno==177); - case 178: /* expr ::= expr LT|GT|GE|LE expr */ yytestcase(yyruleno==178); - case 179: /* expr ::= expr EQ|NE expr */ yytestcase(yyruleno==179); - case 180: /* expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr */ yytestcase(yyruleno==180); - case 181: /* expr ::= expr PLUS|MINUS expr */ yytestcase(yyruleno==181); - case 182: /* expr ::= expr STAR|SLASH|REM expr */ yytestcase(yyruleno==182); - case 183: /* expr ::= expr CONCAT expr */ yytestcase(yyruleno==183); -{yymsp[-2].minor.yy182=sqlite3PExpr(pParse,yymsp[-1].major,yymsp[-2].minor.yy182,yymsp[0].minor.yy182);} + case OP_Gt: + sqlite3VdbeAddOp2(v, OP_NotNull, reg2, lbl); + VdbeCoverage(v); break; - case 184: /* likeop ::= NOT LIKE_KW|MATCH */ -{yymsp[-1].minor.yy0=yymsp[0].minor.yy0; yymsp[-1].minor.yy0.n|=0x80000000; /*yymsp[-1].minor.yy0-overwrite-yymsp[0].minor.yy0*/} + case OP_Le: + sqlite3VdbeAddOp2(v, OP_IsNull, reg2, lbl); + VdbeCoverage(v); break; - case 185: /* expr ::= expr likeop expr */ -{ - ExprList *pList; - int bNot = yymsp[-1].minor.yy0.n & 0x80000000; - yymsp[-1].minor.yy0.n &= 0x7fffffff; - pList = sqlite3ExprListAppend(pParse,0, yymsp[0].minor.yy182); - pList = sqlite3ExprListAppend(pParse,pList, yymsp[-2].minor.yy182); - yymsp[-2].minor.yy182 = sqlite3ExprFunction(pParse, pList, &yymsp[-1].minor.yy0); - if( bNot ) yymsp[-2].minor.yy182 = sqlite3PExpr(pParse, TK_NOT, yymsp[-2].minor.yy182, 0); - if( yymsp[-2].minor.yy182 ) yymsp[-2].minor.yy182->flags |= EP_InfixFunc; -} - break; - case 186: /* expr ::= expr likeop expr ESCAPE expr */ -{ - ExprList *pList; - int bNot = yymsp[-3].minor.yy0.n & 0x80000000; - yymsp[-3].minor.yy0.n &= 0x7fffffff; - pList = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy182); - pList = sqlite3ExprListAppend(pParse,pList, yymsp[-4].minor.yy182); - pList = sqlite3ExprListAppend(pParse,pList, yymsp[0].minor.yy182); - yymsp[-4].minor.yy182 = sqlite3ExprFunction(pParse, pList, &yymsp[-3].minor.yy0); - if( bNot ) yymsp[-4].minor.yy182 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy182, 0); - if( yymsp[-4].minor.yy182 ) yymsp[-4].minor.yy182->flags |= EP_InfixFunc; -} - break; - case 187: /* expr ::= expr ISNULL|NOTNULL */ -{yymsp[-1].minor.yy182 = sqlite3PExpr(pParse,yymsp[0].major,yymsp[-1].minor.yy182,0);} - break; - case 188: /* expr ::= expr NOT NULL */ -{yymsp[-2].minor.yy182 = sqlite3PExpr(pParse,TK_NOTNULL,yymsp[-2].minor.yy182,0);} - break; - case 189: /* expr ::= expr IS expr */ -{ - yymsp[-2].minor.yy182 = sqlite3PExpr(pParse,TK_IS,yymsp[-2].minor.yy182,yymsp[0].minor.yy182); - binaryToUnaryIfNull(pParse, yymsp[0].minor.yy182, yymsp[-2].minor.yy182, TK_ISNULL); -} - break; - case 190: /* expr ::= expr IS NOT expr */ -{ - yymsp[-3].minor.yy182 = sqlite3PExpr(pParse,TK_ISNOT,yymsp[-3].minor.yy182,yymsp[0].minor.yy182); - binaryToUnaryIfNull(pParse, yymsp[0].minor.yy182, yymsp[-3].minor.yy182, TK_NOTNULL); -} - break; - case 191: /* expr ::= NOT expr */ - case 192: /* expr ::= BITNOT expr */ yytestcase(yyruleno==192); -{yymsp[-1].minor.yy182 = sqlite3PExpr(pParse, yymsp[-1].major, yymsp[0].minor.yy182, 0);/*A-overwrites-B*/} - break; - case 193: /* expr ::= MINUS expr */ -{yymsp[-1].minor.yy182 = sqlite3PExpr(pParse, TK_UMINUS, yymsp[0].minor.yy182, 0);} - break; - case 194: /* expr ::= PLUS expr */ -{yymsp[-1].minor.yy182 = sqlite3PExpr(pParse, TK_UPLUS, yymsp[0].minor.yy182, 0);} - break; - case 195: /* between_op ::= BETWEEN */ - case 198: /* in_op ::= IN */ yytestcase(yyruleno==198); -{yymsp[0].minor.yy502 = 0;} - break; - case 197: /* expr ::= expr between_op expr AND expr */ -{ - ExprList *pList = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy182); - pList = sqlite3ExprListAppend(pParse,pList, yymsp[0].minor.yy182); - yymsp[-4].minor.yy182 = sqlite3PExpr(pParse, TK_BETWEEN, yymsp[-4].minor.yy182, 0); - if( yymsp[-4].minor.yy182 ){ - yymsp[-4].minor.yy182->x.pList = pList; - }else{ - sqlite3ExprListDelete(pParse->db, pList); - } - if( yymsp[-3].minor.yy502 ) yymsp[-4].minor.yy182 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy182, 0); + default: assert( op==OP_Lt ); /* no-op */ break; + } + sqlite3VdbeAddOp2(v, OP_Goto, 0, sqlite3VdbeCurrentAddr(v)+3); + + /* This block runs if reg1 is not NULL, but reg2 is. */ + sqlite3VdbeJumpHere(v, addr); + sqlite3VdbeAddOp2(v, OP_IsNull, reg2, lbl); VdbeCoverage(v); + if( op==OP_Gt || op==OP_Ge ){ + sqlite3VdbeChangeP2(v, -1, sqlite3VdbeCurrentAddr(v)+1); + } + } + + /* Compare registers reg2 and reg1, taking the jump if required. Note that + ** control skips over this test if the BIGNULL flag is set and either + ** reg1 or reg2 contain a NULL value. */ + sqlite3VdbeAddOp3(v, op, reg2, lbl, reg1); VdbeCoverage(v); + sqlite3VdbeChangeP5(v, SQLITE_NULLEQ); + + assert( op==OP_Ge || op==OP_Gt || op==OP_Lt || op==OP_Le ); + testcase(op==OP_Ge); VdbeCoverageIf(v, op==OP_Ge); + testcase(op==OP_Lt); VdbeCoverageIf(v, op==OP_Lt); + testcase(op==OP_Le); VdbeCoverageIf(v, op==OP_Le); + testcase(op==OP_Gt); VdbeCoverageIf(v, op==OP_Gt); + sqlite3ReleaseTempReg(pParse, reg1); + sqlite3ReleaseTempReg(pParse, reg2); + + VdbeModuleComment((v, "CodeRangeTest: end")); } - break; - case 200: /* expr ::= expr in_op LP exprlist RP */ -{ - if( yymsp[-1].minor.yy232==0 ){ - /* Expressions of the form - ** - ** expr1 IN () - ** expr1 NOT IN () - ** - ** simplify to constants 0 (false) and 1 (true), respectively, - ** regardless of the value of expr1. - */ - sqlite3ExprDelete(pParse->db, yymsp[-4].minor.yy182); - yymsp[-4].minor.yy182 = sqlite3ExprAlloc(pParse->db, TK_INTEGER,&sqlite3IntTokens[yymsp[-3].minor.yy502],1); - }else if( yymsp[-1].minor.yy232->nExpr==1 ){ - /* Expressions of the form: - ** - ** expr1 IN (?1) - ** expr1 NOT IN (?2) - ** - ** with exactly one value on the RHS can be simplified to something - ** like this: - ** - ** expr1 == ?1 - ** expr1 <> ?2 - ** - ** But, the RHS of the == or <> is marked with the EP_Generic flag - ** so that it may not contribute to the computation of comparison - ** affinity or the collating sequence to use for comparison. Otherwise, - ** the semantics would be subtly different from IN or NOT IN. - */ - Expr *pRHS = yymsp[-1].minor.yy232->a[0].pExpr; - yymsp[-1].minor.yy232->a[0].pExpr = 0; - sqlite3ExprListDelete(pParse->db, yymsp[-1].minor.yy232); - /* pRHS cannot be NULL because a malloc error would have been detected - ** before now and control would have never reached this point */ - if( ALWAYS(pRHS) ){ - pRHS->flags &= ~EP_Collate; - pRHS->flags |= EP_Generic; - } - yymsp[-4].minor.yy182 = sqlite3PExpr(pParse, yymsp[-3].minor.yy502 ? TK_NE : TK_EQ, yymsp[-4].minor.yy182, pRHS); - }else{ - yymsp[-4].minor.yy182 = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy182, 0); - if( yymsp[-4].minor.yy182 ){ - yymsp[-4].minor.yy182->x.pList = yymsp[-1].minor.yy232; - sqlite3ExprSetHeightAndFlags(pParse, yymsp[-4].minor.yy182); + +/* +** Helper function for sqlite3WindowCodeStep(). Each call to this function +** generates VM code for a single RETURN_ROW, AGGSTEP or AGGINVERSE +** operation. Refer to the header comment for sqlite3WindowCodeStep() for +** details. +*/ +static int windowCodeOp( + WindowCodeArg *p, /* Context object */ + int op, /* WINDOW_RETURN_ROW, AGGSTEP or AGGINVERSE */ + int regCountdown, /* Register for OP_IfPos countdown */ + int jumpOnEof /* Jump here if stepped cursor reaches EOF */ +){ + int csr, reg; + Parse *pParse = p->pParse; + Window *pMWin = p->pMWin; + int ret = 0; + Vdbe *v = p->pVdbe; + int addrContinue = 0; + int bPeer = (pMWin->eFrmType!=TK_ROWS); + + int lblDone = sqlite3VdbeMakeLabel(pParse); + int addrNextRange = 0; + + /* Special case - WINDOW_AGGINVERSE is always a no-op if the frame + ** starts with UNBOUNDED PRECEDING. */ + if( op==WINDOW_AGGINVERSE && pMWin->eStart==TK_UNBOUNDED ){ + assert( regCountdown==0 && jumpOnEof==0 ); + return 0; + } + + if( regCountdown>0 ){ + if( pMWin->eFrmType==TK_RANGE ){ + addrNextRange = sqlite3VdbeCurrentAddr(v); + assert( op==WINDOW_AGGINVERSE || op==WINDOW_AGGSTEP ); + if( op==WINDOW_AGGINVERSE ){ + if( pMWin->eStart==TK_FOLLOWING ){ + windowCodeRangeTest( + p, OP_Le, p->current.csr, regCountdown, p->start.csr, lblDone + ); + }else{ + windowCodeRangeTest( + p, OP_Ge, p->start.csr, regCountdown, p->current.csr, lblDone + ); + } }else{ - sqlite3ExprListDelete(pParse->db, yymsp[-1].minor.yy232); + windowCodeRangeTest( + p, OP_Gt, p->end.csr, regCountdown, p->current.csr, lblDone + ); } - if( yymsp[-3].minor.yy502 ) yymsp[-4].minor.yy182 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy182, 0); + }else{ + sqlite3VdbeAddOp3(v, OP_IfPos, regCountdown, lblDone, 1); + VdbeCoverage(v); } } - break; - case 201: /* expr ::= LP select RP */ -{ - yymsp[-2].minor.yy182 = sqlite3PExpr(pParse, TK_SELECT, 0, 0); - sqlite3PExprAddSelect(pParse, yymsp[-2].minor.yy182, yymsp[-1].minor.yy399); + + if( op==WINDOW_RETURN_ROW && pMWin->regStartRowid==0 ){ + windowAggFinal(p, 0); } - break; - case 202: /* expr ::= expr in_op LP select RP */ -{ - yymsp[-4].minor.yy182 = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy182, 0); - sqlite3PExprAddSelect(pParse, yymsp[-4].minor.yy182, yymsp[-1].minor.yy399); - if( yymsp[-3].minor.yy502 ) yymsp[-4].minor.yy182 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy182, 0); + addrContinue = sqlite3VdbeCurrentAddr(v); + + /* If this is a (RANGE BETWEEN a FOLLOWING AND b FOLLOWING) or + ** (RANGE BETWEEN b PRECEDING AND a PRECEDING) frame, ensure the + ** start cursor does not advance past the end cursor within the + ** temporary table. It otherwise might, if (a>b). */ + if( pMWin->eStart==pMWin->eEnd && regCountdown + && pMWin->eFrmType==TK_RANGE && op==WINDOW_AGGINVERSE + ){ + int regRowid1 = sqlite3GetTempReg(pParse); + int regRowid2 = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp2(v, OP_Rowid, p->start.csr, regRowid1); + sqlite3VdbeAddOp2(v, OP_Rowid, p->end.csr, regRowid2); + sqlite3VdbeAddOp3(v, OP_Ge, regRowid2, lblDone, regRowid1); + VdbeCoverage(v); + sqlite3ReleaseTempReg(pParse, regRowid1); + sqlite3ReleaseTempReg(pParse, regRowid2); + assert( pMWin->eStart==TK_PRECEDING || pMWin->eStart==TK_FOLLOWING ); } - break; - case 203: /* expr ::= expr in_op nm dbnm paren_exprlist */ -{ - SrcList *pSrc = sqlite3SrcListAppend(pParse->db, 0,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0); - Select *pSelect = sqlite3SelectNew(pParse, 0,pSrc,0,0,0,0,0,0); - if( yymsp[0].minor.yy232 ) sqlite3SrcListFuncArgs(pParse, pSelect ? pSrc : 0, yymsp[0].minor.yy232); - yymsp[-4].minor.yy182 = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy182, 0); - sqlite3PExprAddSelect(pParse, yymsp[-4].minor.yy182, pSelect); - if( yymsp[-3].minor.yy502 ) yymsp[-4].minor.yy182 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy182, 0); + + switch( op ){ + case WINDOW_RETURN_ROW: + csr = p->current.csr; + reg = p->current.reg; + windowReturnOneRow(p); + break; + + case WINDOW_AGGINVERSE: + csr = p->start.csr; + reg = p->start.reg; + if( pMWin->regStartRowid ){ + assert( pMWin->regEndRowid ); + sqlite3VdbeAddOp2(v, OP_AddImm, pMWin->regStartRowid, 1); + }else{ + windowAggStep(p, pMWin, csr, 1, p->regArg); + } + break; + + default: + assert( op==WINDOW_AGGSTEP ); + csr = p->end.csr; + reg = p->end.reg; + if( pMWin->regStartRowid ){ + assert( pMWin->regEndRowid ); + sqlite3VdbeAddOp2(v, OP_AddImm, pMWin->regEndRowid, 1); + }else{ + windowAggStep(p, pMWin, csr, 0, p->regArg); + } + break; } - break; - case 204: /* expr ::= EXISTS LP select RP */ -{ - Expr *p; - p = yymsp[-3].minor.yy182 = sqlite3PExpr(pParse, TK_EXISTS, 0, 0); - sqlite3PExprAddSelect(pParse, p, yymsp[-1].minor.yy399); + + if( op==p->eDelete ){ + sqlite3VdbeAddOp1(v, OP_Delete, csr); + sqlite3VdbeChangeP5(v, OPFLAG_SAVEPOSITION); } - break; - case 205: /* expr ::= CASE case_operand case_exprlist case_else END */ -{ - yymsp[-4].minor.yy182 = sqlite3PExpr(pParse, TK_CASE, yymsp[-3].minor.yy182, 0); - if( yymsp[-4].minor.yy182 ){ - yymsp[-4].minor.yy182->x.pList = yymsp[-1].minor.yy182 ? sqlite3ExprListAppend(pParse,yymsp[-2].minor.yy232,yymsp[-1].minor.yy182) : yymsp[-2].minor.yy232; - sqlite3ExprSetHeightAndFlags(pParse, yymsp[-4].minor.yy182); + + if( jumpOnEof ){ + sqlite3VdbeAddOp2(v, OP_Next, csr, sqlite3VdbeCurrentAddr(v)+2); + VdbeCoverage(v); + ret = sqlite3VdbeAddOp0(v, OP_Goto); }else{ - sqlite3ExprListDelete(pParse->db, yymsp[-2].minor.yy232); - sqlite3ExprDelete(pParse->db, yymsp[-1].minor.yy182); + sqlite3VdbeAddOp2(v, OP_Next, csr, sqlite3VdbeCurrentAddr(v)+1+bPeer); + VdbeCoverage(v); + if( bPeer ){ + sqlite3VdbeAddOp2(v, OP_Goto, 0, lblDone); + } } -} - break; - case 206: /* case_exprlist ::= case_exprlist WHEN expr THEN expr */ -{ - yymsp[-4].minor.yy232 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy232, yymsp[-2].minor.yy182); - yymsp[-4].minor.yy232 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy232, yymsp[0].minor.yy182); -} - break; - case 207: /* case_exprlist ::= WHEN expr THEN expr */ -{ - yymsp[-3].minor.yy232 = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy182); - yymsp[-3].minor.yy232 = sqlite3ExprListAppend(pParse,yymsp[-3].minor.yy232, yymsp[0].minor.yy182); -} - break; - case 210: /* case_operand ::= expr */ -{yymsp[0].minor.yy182 = yymsp[0].minor.yy182; /*A-overwrites-X*/} - break; - case 213: /* nexprlist ::= nexprlist COMMA expr */ -{yymsp[-2].minor.yy232 = sqlite3ExprListAppend(pParse,yymsp[-2].minor.yy232,yymsp[0].minor.yy182);} - break; - case 214: /* nexprlist ::= expr */ -{yymsp[0].minor.yy232 = sqlite3ExprListAppend(pParse,0,yymsp[0].minor.yy182); /*A-overwrites-Y*/} - break; - case 216: /* paren_exprlist ::= LP exprlist RP */ - case 221: /* eidlist_opt ::= LP eidlist RP */ yytestcase(yyruleno==221); -{yymsp[-2].minor.yy232 = yymsp[-1].minor.yy232;} - break; - case 217: /* cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt */ -{ - sqlite3CreateIndex(pParse, &yymsp[-7].minor.yy0, &yymsp[-6].minor.yy0, - sqlite3SrcListAppend(pParse->db,0,&yymsp[-4].minor.yy0,0), yymsp[-2].minor.yy232, yymsp[-10].minor.yy502, - &yymsp[-11].minor.yy0, yymsp[0].minor.yy182, SQLITE_SO_ASC, yymsp[-8].minor.yy502, SQLITE_IDXTYPE_APPDEF); -} - break; - case 218: /* uniqueflag ::= UNIQUE */ - case 258: /* raisetype ::= ABORT */ yytestcase(yyruleno==258); -{yymsp[0].minor.yy502 = OE_Abort;} - break; - case 219: /* uniqueflag ::= */ -{yymsp[1].minor.yy502 = OE_None;} - break; - case 222: /* eidlist ::= eidlist COMMA nm collate sortorder */ -{ - yymsp[-4].minor.yy232 = parserAddExprIdListTerm(pParse, yymsp[-4].minor.yy232, &yymsp[-2].minor.yy0, yymsp[-1].minor.yy502, yymsp[0].minor.yy502); -} - break; - case 223: /* eidlist ::= nm collate sortorder */ -{ - yymsp[-2].minor.yy232 = parserAddExprIdListTerm(pParse, 0, &yymsp[-2].minor.yy0, yymsp[-1].minor.yy502, yymsp[0].minor.yy502); /*A-overwrites-Y*/ -} - break; - case 226: /* cmd ::= DROP INDEX ifexists fullname */ -{sqlite3DropIndex(pParse, yymsp[0].minor.yy427, yymsp[-1].minor.yy502);} - break; - case 227: /* cmd ::= VACUUM */ -{sqlite3Vacuum(pParse,0);} - break; - case 228: /* cmd ::= VACUUM nm */ -{sqlite3Vacuum(pParse,&yymsp[0].minor.yy0);} - break; - case 229: /* cmd ::= PRAGMA nm dbnm */ -{sqlite3Pragma(pParse,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0,0,0);} - break; - case 230: /* cmd ::= PRAGMA nm dbnm EQ nmnum */ -{sqlite3Pragma(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0,0);} - break; - case 231: /* cmd ::= PRAGMA nm dbnm LP nmnum RP */ -{sqlite3Pragma(pParse,&yymsp[-4].minor.yy0,&yymsp[-3].minor.yy0,&yymsp[-1].minor.yy0,0);} - break; - case 232: /* cmd ::= PRAGMA nm dbnm EQ minus_num */ -{sqlite3Pragma(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0,1);} - break; - case 233: /* cmd ::= PRAGMA nm dbnm LP minus_num RP */ -{sqlite3Pragma(pParse,&yymsp[-4].minor.yy0,&yymsp[-3].minor.yy0,&yymsp[-1].minor.yy0,1);} - break; - case 236: /* cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END */ -{ - Token all; - all.z = yymsp[-3].minor.yy0.z; - all.n = (int)(yymsp[0].minor.yy0.z - yymsp[-3].minor.yy0.z) + yymsp[0].minor.yy0.n; - sqlite3FinishTrigger(pParse, yymsp[-1].minor.yy47, &all); -} - break; - case 237: /* trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */ -{ - sqlite3BeginTrigger(pParse, &yymsp[-7].minor.yy0, &yymsp[-6].minor.yy0, yymsp[-5].minor.yy502, yymsp[-4].minor.yy300.a, yymsp[-4].minor.yy300.b, yymsp[-2].minor.yy427, yymsp[0].minor.yy182, yymsp[-10].minor.yy502, yymsp[-8].minor.yy502); - yymsp[-10].minor.yy0 = (yymsp[-6].minor.yy0.n==0?yymsp[-7].minor.yy0:yymsp[-6].minor.yy0); /*A-overwrites-T*/ -} - break; - case 238: /* trigger_time ::= BEFORE|AFTER */ -{ yymsp[0].minor.yy502 = yymsp[0].major; /*A-overwrites-X*/ } - break; - case 239: /* trigger_time ::= INSTEAD OF */ -{ yymsp[-1].minor.yy502 = TK_INSTEAD;} - break; - case 240: /* trigger_time ::= */ -{ yymsp[1].minor.yy502 = TK_BEFORE; } - break; - case 241: /* trigger_event ::= DELETE|INSERT */ - case 242: /* trigger_event ::= UPDATE */ yytestcase(yyruleno==242); -{yymsp[0].minor.yy300.a = yymsp[0].major; /*A-overwrites-X*/ yymsp[0].minor.yy300.b = 0;} - break; - case 243: /* trigger_event ::= UPDATE OF idlist */ -{yymsp[-2].minor.yy300.a = TK_UPDATE; yymsp[-2].minor.yy300.b = yymsp[0].minor.yy510;} - break; - case 244: /* when_clause ::= */ - case 263: /* key_opt ::= */ yytestcase(yyruleno==263); -{ yymsp[1].minor.yy182 = 0; } - break; - case 245: /* when_clause ::= WHEN expr */ - case 264: /* key_opt ::= KEY expr */ yytestcase(yyruleno==264); -{ yymsp[-1].minor.yy182 = yymsp[0].minor.yy182; } - break; - case 246: /* trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */ -{ - assert( yymsp[-2].minor.yy47!=0 ); - yymsp[-2].minor.yy47->pLast->pNext = yymsp[-1].minor.yy47; - yymsp[-2].minor.yy47->pLast = yymsp[-1].minor.yy47; -} - break; - case 247: /* trigger_cmd_list ::= trigger_cmd SEMI */ -{ - assert( yymsp[-1].minor.yy47!=0 ); - yymsp[-1].minor.yy47->pLast = yymsp[-1].minor.yy47; -} - break; - case 248: /* trnm ::= nm DOT nm */ -{ - yymsp[-2].minor.yy0 = yymsp[0].minor.yy0; - sqlite3ErrorMsg(pParse, - "qualified table names are not allowed on INSERT, UPDATE, and DELETE " - "statements within triggers"); -} - break; - case 249: /* tridxby ::= INDEXED BY nm */ -{ - sqlite3ErrorMsg(pParse, - "the INDEXED BY clause is not allowed on UPDATE or DELETE statements " - "within triggers"); -} - break; - case 250: /* tridxby ::= NOT INDEXED */ -{ - sqlite3ErrorMsg(pParse, - "the NOT INDEXED clause is not allowed on UPDATE or DELETE statements " - "within triggers"); -} - break; - case 251: /* trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist where_opt scanpt */ -{yylhsminor.yy47 = sqlite3TriggerUpdateStep(pParse->db, &yymsp[-5].minor.yy0, yymsp[-2].minor.yy232, yymsp[-1].minor.yy182, yymsp[-6].minor.yy502, yymsp[-7].minor.yy0.z, yymsp[0].minor.yy36);} - yymsp[-7].minor.yy47 = yylhsminor.yy47; - break; - case 252: /* trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt */ -{ - yylhsminor.yy47 = sqlite3TriggerInsertStep(pParse->db,&yymsp[-4].minor.yy0,yymsp[-3].minor.yy510,yymsp[-2].minor.yy399,yymsp[-6].minor.yy502,yymsp[-1].minor.yy198,yymsp[-7].minor.yy36,yymsp[0].minor.yy36);/*yylhsminor.yy47-overwrites-yymsp[-6].minor.yy502*/ -} - yymsp[-7].minor.yy47 = yylhsminor.yy47; - break; - case 253: /* trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt */ -{yylhsminor.yy47 = sqlite3TriggerDeleteStep(pParse->db, &yymsp[-3].minor.yy0, yymsp[-1].minor.yy182, yymsp[-5].minor.yy0.z, yymsp[0].minor.yy36);} - yymsp[-5].minor.yy47 = yylhsminor.yy47; - break; - case 254: /* trigger_cmd ::= scanpt select scanpt */ -{yylhsminor.yy47 = sqlite3TriggerSelectStep(pParse->db, yymsp[-1].minor.yy399, yymsp[-2].minor.yy36, yymsp[0].minor.yy36); /*yylhsminor.yy47-overwrites-yymsp[-1].minor.yy399*/} - yymsp[-2].minor.yy47 = yylhsminor.yy47; - break; - case 255: /* expr ::= RAISE LP IGNORE RP */ -{ - yymsp[-3].minor.yy182 = sqlite3PExpr(pParse, TK_RAISE, 0, 0); - if( yymsp[-3].minor.yy182 ){ - yymsp[-3].minor.yy182->affinity = OE_Ignore; + + if( bPeer ){ + int nReg = (pMWin->pOrderBy ? pMWin->pOrderBy->nExpr : 0); + int regTmp = (nReg ? sqlite3GetTempRange(pParse, nReg) : 0); + windowReadPeerValues(p, csr, regTmp); + windowIfNewPeer(pParse, pMWin->pOrderBy, regTmp, reg, addrContinue); + sqlite3ReleaseTempRange(pParse, regTmp, nReg); } -} - break; - case 256: /* expr ::= RAISE LP raisetype COMMA nm RP */ -{ - yymsp[-5].minor.yy182 = sqlite3ExprAlloc(pParse->db, TK_RAISE, &yymsp[-1].minor.yy0, 1); - if( yymsp[-5].minor.yy182 ) { - yymsp[-5].minor.yy182->affinity = (char)yymsp[-3].minor.yy502; + + if( addrNextRange ){ + sqlite3VdbeAddOp2(v, OP_Goto, 0, addrNextRange); } + sqlite3VdbeResolveLabel(v, lblDone); + return ret; } - break; - case 257: /* raisetype ::= ROLLBACK */ -{yymsp[0].minor.yy502 = OE_Rollback;} - break; - case 259: /* raisetype ::= FAIL */ -{yymsp[0].minor.yy502 = OE_Fail;} - break; - case 260: /* cmd ::= DROP TRIGGER ifexists fullname */ -{ - sqlite3DropTrigger(pParse,yymsp[0].minor.yy427,yymsp[-1].minor.yy502); -} - break; - case 261: /* cmd ::= ATTACH database_kw_opt expr AS expr key_opt */ -{ - sqlite3Attach(pParse, yymsp[-3].minor.yy182, yymsp[-1].minor.yy182, yymsp[0].minor.yy182); -} - break; - case 262: /* cmd ::= DETACH database_kw_opt expr */ -{ - sqlite3Detach(pParse, yymsp[0].minor.yy182); -} - break; - case 265: /* cmd ::= REINDEX */ -{sqlite3Reindex(pParse, 0, 0);} - break; - case 266: /* cmd ::= REINDEX nm dbnm */ -{sqlite3Reindex(pParse, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0);} - break; - case 267: /* cmd ::= ANALYZE */ -{sqlite3Analyze(pParse, 0, 0);} - break; - case 268: /* cmd ::= ANALYZE nm dbnm */ -{sqlite3Analyze(pParse, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0);} - break; - case 269: /* cmd ::= ALTER TABLE fullname RENAME TO nm */ -{ - sqlite3AlterRenameTable(pParse,yymsp[-3].minor.yy427,&yymsp[0].minor.yy0); -} - break; - case 270: /* cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist */ -{ - yymsp[-1].minor.yy0.n = (int)(pParse->sLastToken.z-yymsp[-1].minor.yy0.z) + pParse->sLastToken.n; - sqlite3AlterFinishAddColumn(pParse, &yymsp[-1].minor.yy0); -} - break; - case 271: /* add_column_fullname ::= fullname */ -{ - disableLookaside(pParse); - sqlite3AlterBeginAddColumn(pParse, yymsp[0].minor.yy427); -} - break; - case 272: /* cmd ::= create_vtab */ -{sqlite3VtabFinishParse(pParse,0);} - break; - case 273: /* cmd ::= create_vtab LP vtabarglist RP */ -{sqlite3VtabFinishParse(pParse,&yymsp[0].minor.yy0);} - break; - case 274: /* create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm */ -{ - sqlite3VtabBeginParse(pParse, &yymsp[-3].minor.yy0, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-4].minor.yy502); -} - break; - case 275: /* vtabarg ::= */ -{sqlite3VtabArgInit(pParse);} - break; - case 276: /* vtabargtoken ::= ANY */ - case 277: /* vtabargtoken ::= lp anylist RP */ yytestcase(yyruleno==277); - case 278: /* lp ::= LP */ yytestcase(yyruleno==278); -{sqlite3VtabArgExtend(pParse,&yymsp[0].minor.yy0);} - break; - case 279: /* with ::= WITH wqlist */ - case 280: /* with ::= WITH RECURSIVE wqlist */ yytestcase(yyruleno==280); -{ sqlite3WithPush(pParse, yymsp[0].minor.yy91, 1); } - break; - case 281: /* wqlist ::= nm eidlist_opt AS LP select RP */ -{ - yymsp[-5].minor.yy91 = sqlite3WithAdd(pParse, 0, &yymsp[-5].minor.yy0, yymsp[-4].minor.yy232, yymsp[-1].minor.yy399); /*A-overwrites-X*/ -} - break; - case 282: /* wqlist ::= wqlist COMMA nm eidlist_opt AS LP select RP */ -{ - yymsp[-7].minor.yy91 = sqlite3WithAdd(pParse, yymsp[-7].minor.yy91, &yymsp[-5].minor.yy0, yymsp[-4].minor.yy232, yymsp[-1].minor.yy399); -} - break; - default: - /* (283) input ::= cmdlist */ yytestcase(yyruleno==283); - /* (284) cmdlist ::= cmdlist ecmd */ yytestcase(yyruleno==284); - /* (285) cmdlist ::= ecmd (OPTIMIZED OUT) */ assert(yyruleno!=285); - /* (286) ecmd ::= SEMI */ yytestcase(yyruleno==286); - /* (287) ecmd ::= cmdx SEMI */ yytestcase(yyruleno==287); - /* (288) ecmd ::= explain cmdx */ yytestcase(yyruleno==288); - /* (289) trans_opt ::= */ yytestcase(yyruleno==289); - /* (290) trans_opt ::= TRANSACTION */ yytestcase(yyruleno==290); - /* (291) trans_opt ::= TRANSACTION nm */ yytestcase(yyruleno==291); - /* (292) savepoint_opt ::= SAVEPOINT */ yytestcase(yyruleno==292); - /* (293) savepoint_opt ::= */ yytestcase(yyruleno==293); - /* (294) cmd ::= create_table create_table_args */ yytestcase(yyruleno==294); - /* (295) columnlist ::= columnlist COMMA columnname carglist */ yytestcase(yyruleno==295); - /* (296) columnlist ::= columnname carglist */ yytestcase(yyruleno==296); - /* (297) nm ::= ID|INDEXED */ yytestcase(yyruleno==297); - /* (298) nm ::= STRING */ yytestcase(yyruleno==298); - /* (299) nm ::= JOIN_KW */ yytestcase(yyruleno==299); - /* (300) typetoken ::= typename */ yytestcase(yyruleno==300); - /* (301) typename ::= ID|STRING */ yytestcase(yyruleno==301); - /* (302) signed ::= plus_num (OPTIMIZED OUT) */ assert(yyruleno!=302); - /* (303) signed ::= minus_num (OPTIMIZED OUT) */ assert(yyruleno!=303); - /* (304) carglist ::= carglist ccons */ yytestcase(yyruleno==304); - /* (305) carglist ::= */ yytestcase(yyruleno==305); - /* (306) ccons ::= NULL onconf */ yytestcase(yyruleno==306); - /* (307) conslist_opt ::= COMMA conslist */ yytestcase(yyruleno==307); - /* (308) conslist ::= conslist tconscomma tcons */ yytestcase(yyruleno==308); - /* (309) conslist ::= tcons (OPTIMIZED OUT) */ assert(yyruleno!=309); - /* (310) tconscomma ::= */ yytestcase(yyruleno==310); - /* (311) defer_subclause_opt ::= defer_subclause (OPTIMIZED OUT) */ assert(yyruleno!=311); - /* (312) resolvetype ::= raisetype (OPTIMIZED OUT) */ assert(yyruleno!=312); - /* (313) selectnowith ::= oneselect (OPTIMIZED OUT) */ assert(yyruleno!=313); - /* (314) oneselect ::= values */ yytestcase(yyruleno==314); - /* (315) sclp ::= selcollist COMMA */ yytestcase(yyruleno==315); - /* (316) as ::= ID|STRING */ yytestcase(yyruleno==316); - /* (317) expr ::= term (OPTIMIZED OUT) */ assert(yyruleno!=317); - /* (318) likeop ::= LIKE_KW|MATCH */ yytestcase(yyruleno==318); - /* (319) exprlist ::= nexprlist */ yytestcase(yyruleno==319); - /* (320) nmnum ::= plus_num (OPTIMIZED OUT) */ assert(yyruleno!=320); - /* (321) nmnum ::= nm (OPTIMIZED OUT) */ assert(yyruleno!=321); - /* (322) nmnum ::= ON */ yytestcase(yyruleno==322); - /* (323) nmnum ::= DELETE */ yytestcase(yyruleno==323); - /* (324) nmnum ::= DEFAULT */ yytestcase(yyruleno==324); - /* (325) plus_num ::= INTEGER|FLOAT */ yytestcase(yyruleno==325); - /* (326) foreach_clause ::= */ yytestcase(yyruleno==326); - /* (327) foreach_clause ::= FOR EACH ROW */ yytestcase(yyruleno==327); - /* (328) trnm ::= nm */ yytestcase(yyruleno==328); - /* (329) tridxby ::= */ yytestcase(yyruleno==329); - /* (330) database_kw_opt ::= DATABASE */ yytestcase(yyruleno==330); - /* (331) database_kw_opt ::= */ yytestcase(yyruleno==331); - /* (332) kwcolumn_opt ::= */ yytestcase(yyruleno==332); - /* (333) kwcolumn_opt ::= COLUMNKW */ yytestcase(yyruleno==333); - /* (334) vtabarglist ::= vtabarg */ yytestcase(yyruleno==334); - /* (335) vtabarglist ::= vtabarglist COMMA vtabarg */ yytestcase(yyruleno==335); - /* (336) vtabarg ::= vtabarg vtabargtoken */ yytestcase(yyruleno==336); - /* (337) anylist ::= */ yytestcase(yyruleno==337); - /* (338) anylist ::= anylist LP anylist RP */ yytestcase(yyruleno==338); - /* (339) anylist ::= anylist ANY */ yytestcase(yyruleno==339); - /* (340) with ::= */ yytestcase(yyruleno==340); - break; -/********** End reduce actions ************************************************/ - }; - assert( yyrulenoYY_MAX_SHIFT && yyact<=YY_MAX_SHIFTREDUCE) ); - - /* It is not possible for a REDUCE to be followed by an error */ - assert( yyact!=YY_ERROR_ACTION ); - yymsp += yysize+1; - yypParser->yytos = yymsp; - yymsp->stateno = (YYACTIONTYPE)yyact; - yymsp->major = (YYCODETYPE)yygoto; - yyTraceShift(yypParser, yyact, "... then shift"); - return yyact; -} /* -** The following code executes when the parse fails +** Allocate and return a duplicate of the Window object indicated by the +** third argument. Set the Window.pOwner field of the new object to +** pOwner. */ -#ifndef YYNOERRORRECOVERY -static void yy_parse_failed( - yyParser *yypParser /* The parser */ -){ - sqlite3ParserARG_FETCH - sqlite3ParserCTX_FETCH -#ifndef NDEBUG - if( yyTraceFILE ){ - fprintf(yyTraceFILE,"%sFail!\n",yyTracePrompt); +SQLITE_PRIVATE Window *sqlite3WindowDup(sqlite3 *db, Expr *pOwner, Window *p){ + Window *pNew = 0; + if( ALWAYS(p) ){ + pNew = sqlite3DbMallocZero(db, sizeof(Window)); + if( pNew ){ + pNew->zName = sqlite3DbStrDup(db, p->zName); + pNew->zBase = sqlite3DbStrDup(db, p->zBase); + pNew->pFilter = sqlite3ExprDup(db, p->pFilter, 0); + pNew->pFunc = p->pFunc; + pNew->pPartition = sqlite3ExprListDup(db, p->pPartition, 0); + pNew->pOrderBy = sqlite3ExprListDup(db, p->pOrderBy, 0); + pNew->eFrmType = p->eFrmType; + pNew->eEnd = p->eEnd; + pNew->eStart = p->eStart; + pNew->eExclude = p->eExclude; + pNew->regResult = p->regResult; + pNew->regAccum = p->regAccum; + pNew->iArgCol = p->iArgCol; + pNew->iEphCsr = p->iEphCsr; + pNew->bExprArgs = p->bExprArgs; + pNew->pStart = sqlite3ExprDup(db, p->pStart, 0); + pNew->pEnd = sqlite3ExprDup(db, p->pEnd, 0); + pNew->pOwner = pOwner; + pNew->bImplicitFrame = p->bImplicitFrame; + } } -#endif - while( yypParser->yytos>yypParser->yystack ) yy_pop_parser_stack(yypParser); - /* Here code is inserted which will be executed whenever the - ** parser fails */ -/************ Begin %parse_failure code ***************************************/ -/************ End %parse_failure code *****************************************/ - sqlite3ParserARG_STORE /* Suppress warning about unused %extra_argument variable */ - sqlite3ParserCTX_STORE + return pNew; } -#endif /* YYNOERRORRECOVERY */ /* -** The following code executes when a syntax error first occurs. +** Return a copy of the linked list of Window objects passed as the +** second argument. */ -static void yy_syntax_error( - yyParser *yypParser, /* The parser */ - int yymajor, /* The major type of the error token */ - sqlite3ParserTOKENTYPE yyminor /* The minor type of the error token */ -){ - sqlite3ParserARG_FETCH - sqlite3ParserCTX_FETCH -#define TOKEN yyminor -/************ Begin %syntax_error code ****************************************/ +SQLITE_PRIVATE Window *sqlite3WindowListDup(sqlite3 *db, Window *p){ + Window *pWin; + Window *pRet = 0; + Window **pp = &pRet; - UNUSED_PARAMETER(yymajor); /* Silence some compiler warnings */ - if( TOKEN.z[0] ){ - sqlite3ErrorMsg(pParse, "near \"%T\": syntax error", &TOKEN); - }else{ - sqlite3ErrorMsg(pParse, "incomplete input"); + for(pWin=p; pWin; pWin=pWin->pNextWin){ + *pp = sqlite3WindowDup(db, 0, pWin); + if( *pp==0 ) break; + pp = &((*pp)->pNextWin); } -/************ End %syntax_error code ******************************************/ - sqlite3ParserARG_STORE /* Suppress warning about unused %extra_argument variable */ - sqlite3ParserCTX_STORE + + return pRet; } /* -** The following is executed when the parser accepts +** Return true if it can be determined at compile time that expression +** pExpr evaluates to a value that, when cast to an integer, is greater +** than zero. False otherwise. +** +** If an OOM error occurs, this function sets the Parse.db.mallocFailed +** flag and returns zero. */ -static void yy_accept( - yyParser *yypParser /* The parser */ -){ - sqlite3ParserARG_FETCH - sqlite3ParserCTX_FETCH -#ifndef NDEBUG - if( yyTraceFILE ){ - fprintf(yyTraceFILE,"%sAccept!\n",yyTracePrompt); +static int windowExprGtZero(Parse *pParse, Expr *pExpr){ + int ret = 0; + sqlite3 *db = pParse->db; + sqlite3_value *pVal = 0; + sqlite3ValueFromExpr(db, pExpr, db->enc, SQLITE_AFF_NUMERIC, &pVal); + if( pVal && sqlite3_value_int(pVal)>0 ){ + ret = 1; } -#endif -#ifndef YYNOERRORRECOVERY - yypParser->yyerrcnt = -1; -#endif - assert( yypParser->yytos==yypParser->yystack ); - /* Here code is inserted which will be executed whenever the - ** parser accepts */ -/*********** Begin %parse_accept code *****************************************/ -/*********** End %parse_accept code *******************************************/ - sqlite3ParserARG_STORE /* Suppress warning about unused %extra_argument variable */ - sqlite3ParserCTX_STORE + sqlite3ValueFree(pVal); + return ret; } -/* The main parser program. -** The first argument is a pointer to a structure obtained from -** "sqlite3ParserAlloc" which describes the current state of the parser. -** The second argument is the major token number. The third is -** the minor token. The fourth optional argument is whatever the -** user wants (and specified in the grammar) and is available for -** use by the action routines. +/* +** sqlite3WhereBegin() has already been called for the SELECT statement +** passed as the second argument when this function is invoked. It generates +** code to populate the Window.regResult register for each window function +** and invoke the sub-routine at instruction addrGosub once for each row. +** sqlite3WhereEnd() is always called before returning. ** -** Inputs: -**
          -**
        • A pointer to the parser (an opaque structure.) -**
        • The major token number. -**
        • The minor token number. -**
        • An option argument of a grammar-specified type. -**
        +** This function handles several different types of window frames, which +** require slightly different processing. The following pseudo code is +** used to implement window frames of the form: ** -** Outputs: -** None. +** ROWS BETWEEN PRECEDING AND FOLLOWING +** +** Other window frame types use variants of the following: +** +** ... loop started by sqlite3WhereBegin() ... +** if( new partition ){ +** Gosub flush +** } +** Insert new row into eph table. +** +** if( first row of partition ){ +** // Rewind three cursors, all open on the eph table. +** Rewind(csrEnd); +** Rewind(csrStart); +** Rewind(csrCurrent); +** +** regEnd = // FOLLOWING expression +** regStart = // PRECEDING expression +** }else{ +** // First time this branch is taken, the eph table contains two +** // rows. The first row in the partition, which all three cursors +** // currently point to, and the following row. +** AGGSTEP +** if( (regEnd--)<=0 ){ +** RETURN_ROW +** if( (regStart--)<=0 ){ +** AGGINVERSE +** } +** } +** } +** } +** flush: +** AGGSTEP +** while( 1 ){ +** RETURN ROW +** if( csrCurrent is EOF ) break; +** if( (regStart--)<=0 ){ +** AggInverse(csrStart) +** Next(csrStart) +** } +** } +** +** The pseudo-code above uses the following shorthand: +** +** AGGSTEP: invoke the aggregate xStep() function for each window function +** with arguments read from the current row of cursor csrEnd, then +** step cursor csrEnd forward one row (i.e. sqlite3BtreeNext()). +** +** RETURN_ROW: return a row to the caller based on the contents of the +** current row of csrCurrent and the current state of all +** aggregates. Then step cursor csrCurrent forward one row. +** +** AGGINVERSE: invoke the aggregate xInverse() function for each window +** functions with arguments read from the current row of cursor +** csrStart. Then step csrStart forward one row. +** +** There are two other ROWS window frames that are handled significantly +** differently from the above - "BETWEEN PRECEDING AND PRECEDING" +** and "BETWEEN FOLLOWING AND FOLLOWING". These are special +** cases because they change the order in which the three cursors (csrStart, +** csrCurrent and csrEnd) iterate through the ephemeral table. Cases that +** use UNBOUNDED or CURRENT ROW are much simpler variations on one of these +** three. +** +** ROWS BETWEEN PRECEDING AND PRECEDING +** +** ... loop started by sqlite3WhereBegin() ... +** if( new partition ){ +** Gosub flush +** } +** Insert new row into eph table. +** if( first row of partition ){ +** Rewind(csrEnd) ; Rewind(csrStart) ; Rewind(csrCurrent) +** regEnd = +** regStart = +** }else{ +** if( (regEnd--)<=0 ){ +** AGGSTEP +** } +** RETURN_ROW +** if( (regStart--)<=0 ){ +** AGGINVERSE +** } +** } +** } +** flush: +** if( (regEnd--)<=0 ){ +** AGGSTEP +** } +** RETURN_ROW +** +** +** ROWS BETWEEN FOLLOWING AND FOLLOWING +** +** ... loop started by sqlite3WhereBegin() ... +** if( new partition ){ +** Gosub flush +** } +** Insert new row into eph table. +** if( first row of partition ){ +** Rewind(csrEnd) ; Rewind(csrStart) ; Rewind(csrCurrent) +** regEnd = +** regStart = regEnd - +** }else{ +** AGGSTEP +** if( (regEnd--)<=0 ){ +** RETURN_ROW +** } +** if( (regStart--)<=0 ){ +** AGGINVERSE +** } +** } +** } +** flush: +** AGGSTEP +** while( 1 ){ +** if( (regEnd--)<=0 ){ +** RETURN_ROW +** if( eof ) break; +** } +** if( (regStart--)<=0 ){ +** AGGINVERSE +** if( eof ) break +** } +** } +** while( !eof csrCurrent ){ +** RETURN_ROW +** } +** +** For the most part, the patterns above are adapted to support UNBOUNDED by +** assuming that it is equivalent to "infinity PRECEDING/FOLLOWING" and +** CURRENT ROW by assuming that it is equivilent to "0 PRECEDING/FOLLOWING". +** This is optimized of course - branches that will never be taken and +** conditions that are always true are omitted from the VM code. The only +** exceptional case is: +** +** ROWS BETWEEN FOLLOWING AND UNBOUNDED FOLLOWING +** +** ... loop started by sqlite3WhereBegin() ... +** if( new partition ){ +** Gosub flush +** } +** Insert new row into eph table. +** if( first row of partition ){ +** Rewind(csrEnd) ; Rewind(csrStart) ; Rewind(csrCurrent) +** regStart = +** }else{ +** AGGSTEP +** } +** } +** flush: +** AGGSTEP +** while( 1 ){ +** if( (regStart--)<=0 ){ +** AGGINVERSE +** if( eof ) break +** } +** RETURN_ROW +** } +** while( !eof csrCurrent ){ +** RETURN_ROW +** } +** +** Also requiring special handling are the cases: +** +** ROWS BETWEEN PRECEDING AND PRECEDING +** ROWS BETWEEN FOLLOWING AND FOLLOWING +** +** when (expr1 < expr2). This is detected at runtime, not by this function. +** To handle this case, the pseudo-code programs depicted above are modified +** slightly to be: +** +** ... loop started by sqlite3WhereBegin() ... +** if( new partition ){ +** Gosub flush +** } +** Insert new row into eph table. +** if( first row of partition ){ +** Rewind(csrEnd) ; Rewind(csrStart) ; Rewind(csrCurrent) +** regEnd = +** regStart = +** if( regEnd < regStart ){ +** RETURN_ROW +** delete eph table contents +** continue +** } +** ... +** +** The new "continue" statement in the above jumps to the next iteration +** of the outer loop - the one started by sqlite3WhereBegin(). +** +** The various GROUPS cases are implemented using the same patterns as +** ROWS. The VM code is modified slightly so that: +** +** 1. The else branch in the main loop is only taken if the row just +** added to the ephemeral table is the start of a new group. In +** other words, it becomes: +** +** ... loop started by sqlite3WhereBegin() ... +** if( new partition ){ +** Gosub flush +** } +** Insert new row into eph table. +** if( first row of partition ){ +** Rewind(csrEnd) ; Rewind(csrStart) ; Rewind(csrCurrent) +** regEnd = +** regStart = +** }else if( new group ){ +** ... +** } +** } +** +** 2. Instead of processing a single row, each RETURN_ROW, AGGSTEP or +** AGGINVERSE step processes the current row of the relevant cursor and +** all subsequent rows belonging to the same group. +** +** RANGE window frames are a little different again. As for GROUPS, the +** main loop runs once per group only. And RETURN_ROW, AGGSTEP and AGGINVERSE +** deal in groups instead of rows. As for ROWS and GROUPS, there are three +** basic cases: +** +** RANGE BETWEEN PRECEDING AND FOLLOWING +** +** ... loop started by sqlite3WhereBegin() ... +** if( new partition ){ +** Gosub flush +** } +** Insert new row into eph table. +** if( first row of partition ){ +** Rewind(csrEnd) ; Rewind(csrStart) ; Rewind(csrCurrent) +** regEnd = +** regStart = +** }else{ +** AGGSTEP +** while( (csrCurrent.key + regEnd) < csrEnd.key ){ +** RETURN_ROW +** while( csrStart.key + regStart) < csrCurrent.key ){ +** AGGINVERSE +** } +** } +** } +** } +** flush: +** AGGSTEP +** while( 1 ){ +** RETURN ROW +** if( csrCurrent is EOF ) break; +** while( csrStart.key + regStart) < csrCurrent.key ){ +** AGGINVERSE +** } +** } +** } +** +** In the above notation, "csr.key" means the current value of the ORDER BY +** expression (there is only ever 1 for a RANGE that uses an FOLLOWING +** or PRECEDING AND PRECEDING +** +** ... loop started by sqlite3WhereBegin() ... +** if( new partition ){ +** Gosub flush +** } +** Insert new row into eph table. +** if( first row of partition ){ +** Rewind(csrEnd) ; Rewind(csrStart) ; Rewind(csrCurrent) +** regEnd = +** regStart = +** }else{ +** while( (csrEnd.key + regEnd) <= csrCurrent.key ){ +** AGGSTEP +** } +** while( (csrStart.key + regStart) < csrCurrent.key ){ +** AGGINVERSE +** } +** RETURN_ROW +** } +** } +** flush: +** while( (csrEnd.key + regEnd) <= csrCurrent.key ){ +** AGGSTEP +** } +** while( (csrStart.key + regStart) < csrCurrent.key ){ +** AGGINVERSE +** } +** RETURN_ROW +** +** RANGE BETWEEN FOLLOWING AND FOLLOWING +** +** ... loop started by sqlite3WhereBegin() ... +** if( new partition ){ +** Gosub flush +** } +** Insert new row into eph table. +** if( first row of partition ){ +** Rewind(csrEnd) ; Rewind(csrStart) ; Rewind(csrCurrent) +** regEnd = +** regStart = +** }else{ +** AGGSTEP +** while( (csrCurrent.key + regEnd) < csrEnd.key ){ +** while( (csrCurrent.key + regStart) > csrStart.key ){ +** AGGINVERSE +** } +** RETURN_ROW +** } +** } +** } +** flush: +** AGGSTEP +** while( 1 ){ +** while( (csrCurrent.key + regStart) > csrStart.key ){ +** AGGINVERSE +** if( eof ) break "while( 1 )" loop. +** } +** RETURN_ROW +** } +** while( !eof csrCurrent ){ +** RETURN_ROW +** } +** +** The text above leaves out many details. Refer to the code and comments +** below for a more complete picture. */ -SQLITE_PRIVATE void sqlite3Parser( - void *yyp, /* The parser */ - int yymajor, /* The major token code number */ - sqlite3ParserTOKENTYPE yyminor /* The value for the token */ - sqlite3ParserARG_PDECL /* Optional %extra_argument parameter */ +SQLITE_PRIVATE void sqlite3WindowCodeStep( + Parse *pParse, /* Parse context */ + Select *p, /* Rewritten SELECT statement */ + WhereInfo *pWInfo, /* Context returned by sqlite3WhereBegin() */ + int regGosub, /* Register for OP_Gosub */ + int addrGosub /* OP_Gosub here to return each row */ ){ - YYMINORTYPE yyminorunion; - YYACTIONTYPE yyact; /* The parser action. */ -#if !defined(YYERRORSYMBOL) && !defined(YYNOERRORRECOVERY) - int yyendofinput; /* True if we are at the end of input */ -#endif -#ifdef YYERRORSYMBOL - int yyerrorhit = 0; /* True if yymajor has invoked an error */ -#endif - yyParser *yypParser = (yyParser*)yyp; /* The parser */ - sqlite3ParserCTX_FETCH - sqlite3ParserARG_STORE - - assert( yypParser->yytos!=0 ); -#if !defined(YYERRORSYMBOL) && !defined(YYNOERRORRECOVERY) - yyendofinput = (yymajor==0); -#endif - - yyact = yypParser->yytos->stateno; -#ifndef NDEBUG - if( yyTraceFILE ){ - if( yyact < YY_MIN_REDUCE ){ - fprintf(yyTraceFILE,"%sInput '%s' in state %d\n", - yyTracePrompt,yyTokenName[yymajor],yyact); - }else{ - fprintf(yyTraceFILE,"%sInput '%s' with pending reduce %d\n", - yyTracePrompt,yyTokenName[yymajor],yyact-YY_MIN_REDUCE); - } - } -#endif + Window *pMWin = p->pWin; + ExprList *pOrderBy = pMWin->pOrderBy; + Vdbe *v = sqlite3GetVdbe(pParse); + int csrWrite; /* Cursor used to write to eph. table */ + int csrInput = p->pSrc->a[0].iCursor; /* Cursor of sub-select */ + int nInput = p->pSrc->a[0].pTab->nCol; /* Number of cols returned by sub */ + int iInput; /* To iterate through sub cols */ + int addrNe; /* Address of OP_Ne */ + int addrGosubFlush = 0; /* Address of OP_Gosub to flush: */ + int addrInteger = 0; /* Address of OP_Integer */ + int addrEmpty; /* Address of OP_Rewind in flush: */ + int regNew; /* Array of registers holding new input row */ + int regRecord; /* regNew array in record form */ + int regRowid; /* Rowid for regRecord in eph table */ + int regNewPeer = 0; /* Peer values for new row (part of regNew) */ + int regPeer = 0; /* Peer values for current row */ + int regFlushPart = 0; /* Register for "Gosub flush_partition" */ + WindowCodeArg s; /* Context object for sub-routines */ + int lblWhereEnd; /* Label just before sqlite3WhereEnd() code */ + int regStart = 0; /* Value of PRECEDING */ + int regEnd = 0; /* Value of FOLLOWING */ + + assert( pMWin->eStart==TK_PRECEDING || pMWin->eStart==TK_CURRENT + || pMWin->eStart==TK_FOLLOWING || pMWin->eStart==TK_UNBOUNDED + ); + assert( pMWin->eEnd==TK_FOLLOWING || pMWin->eEnd==TK_CURRENT + || pMWin->eEnd==TK_UNBOUNDED || pMWin->eEnd==TK_PRECEDING + ); + assert( pMWin->eExclude==0 || pMWin->eExclude==TK_CURRENT + || pMWin->eExclude==TK_GROUP || pMWin->eExclude==TK_TIES + || pMWin->eExclude==TK_NO + ); - do{ - assert( yyact==yypParser->yytos->stateno ); - yyact = yy_find_shift_action(yymajor,yyact); - if( yyact >= YY_MIN_REDUCE ){ - yyact = yy_reduce(yypParser,yyact-YY_MIN_REDUCE,yymajor, - yyminor sqlite3ParserCTX_PARAM); - }else if( yyact <= YY_MAX_SHIFTREDUCE ){ - yy_shift(yypParser,yyact,yymajor,yyminor); -#ifndef YYNOERRORRECOVERY - yypParser->yyerrcnt--; -#endif + lblWhereEnd = sqlite3VdbeMakeLabel(pParse); + + /* Fill in the context object */ + memset(&s, 0, sizeof(WindowCodeArg)); + s.pParse = pParse; + s.pMWin = pMWin; + s.pVdbe = v; + s.regGosub = regGosub; + s.addrGosub = addrGosub; + s.current.csr = pMWin->iEphCsr; + csrWrite = s.current.csr+1; + s.start.csr = s.current.csr+2; + s.end.csr = s.current.csr+3; + + /* Figure out when rows may be deleted from the ephemeral table. There + ** are four options - they may never be deleted (eDelete==0), they may + ** be deleted as soon as they are no longer part of the window frame + ** (eDelete==WINDOW_AGGINVERSE), they may be deleted as after the row + ** has been returned to the caller (WINDOW_RETURN_ROW), or they may + ** be deleted after they enter the frame (WINDOW_AGGSTEP). */ + switch( pMWin->eStart ){ + case TK_FOLLOWING: + if( pMWin->eFrmType!=TK_RANGE + && windowExprGtZero(pParse, pMWin->pStart) + ){ + s.eDelete = WINDOW_RETURN_ROW; + } break; - }else if( yyact==YY_ACCEPT_ACTION ){ - yypParser->yytos--; - yy_accept(yypParser); - return; - }else{ - assert( yyact == YY_ERROR_ACTION ); - yyminorunion.yy0 = yyminor; -#ifdef YYERRORSYMBOL - int yymx; -#endif -#ifndef NDEBUG - if( yyTraceFILE ){ - fprintf(yyTraceFILE,"%sSyntax Error!\n",yyTracePrompt); + case TK_UNBOUNDED: + if( windowCacheFrame(pMWin)==0 ){ + if( pMWin->eEnd==TK_PRECEDING ){ + if( pMWin->eFrmType!=TK_RANGE + && windowExprGtZero(pParse, pMWin->pEnd) + ){ + s.eDelete = WINDOW_AGGSTEP; + } + }else{ + s.eDelete = WINDOW_RETURN_ROW; + } } -#endif -#ifdef YYERRORSYMBOL - /* A syntax error has occurred. - ** The response to an error depends upon whether or not the - ** grammar defines an error token "ERROR". - ** - ** This is what we do if the grammar does define ERROR: - ** - ** * Call the %syntax_error function. - ** - ** * Begin popping the stack until we enter a state where - ** it is legal to shift the error symbol, then shift - ** the error symbol. - ** - ** * Set the error count to three. - ** - ** * Begin accepting and shifting new tokens. No new error - ** processing will occur until three tokens have been - ** shifted successfully. - ** - */ - if( yypParser->yyerrcnt<0 ){ - yy_syntax_error(yypParser,yymajor,yyminor); + break; + default: + s.eDelete = WINDOW_AGGINVERSE; + break; + } + + /* Allocate registers for the array of values from the sub-query, the + ** samve values in record form, and the rowid used to insert said record + ** into the ephemeral table. */ + regNew = pParse->nMem+1; + pParse->nMem += nInput; + regRecord = ++pParse->nMem; + regRowid = ++pParse->nMem; + + /* If the window frame contains an " PRECEDING" or " FOLLOWING" + ** clause, allocate registers to store the results of evaluating each + ** . */ + if( pMWin->eStart==TK_PRECEDING || pMWin->eStart==TK_FOLLOWING ){ + regStart = ++pParse->nMem; + } + if( pMWin->eEnd==TK_PRECEDING || pMWin->eEnd==TK_FOLLOWING ){ + regEnd = ++pParse->nMem; + } + + /* If this is not a "ROWS BETWEEN ..." frame, then allocate arrays of + ** registers to store copies of the ORDER BY expressions (peer values) + ** for the main loop, and for each cursor (start, current and end). */ + if( pMWin->eFrmType!=TK_ROWS ){ + int nPeer = (pOrderBy ? pOrderBy->nExpr : 0); + regNewPeer = regNew + pMWin->nBufferCol; + if( pMWin->pPartition ) regNewPeer += pMWin->pPartition->nExpr; + regPeer = pParse->nMem+1; pParse->nMem += nPeer; + s.start.reg = pParse->nMem+1; pParse->nMem += nPeer; + s.current.reg = pParse->nMem+1; pParse->nMem += nPeer; + s.end.reg = pParse->nMem+1; pParse->nMem += nPeer; + } + + /* Load the column values for the row returned by the sub-select + ** into an array of registers starting at regNew. Assemble them into + ** a record in register regRecord. */ + for(iInput=0; iInputpPartition ){ + int addr; + ExprList *pPart = pMWin->pPartition; + int nPart = pPart->nExpr; + int regNewPart = regNew + pMWin->nBufferCol; + KeyInfo *pKeyInfo = sqlite3KeyInfoFromExprList(pParse, pPart, 0, 0); + + regFlushPart = ++pParse->nMem; + addr = sqlite3VdbeAddOp3(v, OP_Compare, regNewPart, pMWin->regPart, nPart); + sqlite3VdbeAppendP4(v, (void*)pKeyInfo, P4_KEYINFO); + sqlite3VdbeAddOp3(v, OP_Jump, addr+2, addr+4, addr+2); + VdbeCoverageEqNe(v); + addrGosubFlush = sqlite3VdbeAddOp1(v, OP_Gosub, regFlushPart); + VdbeComment((v, "call flush_partition")); + sqlite3VdbeAddOp3(v, OP_Copy, regNewPart, pMWin->regPart, nPart-1); + } + + /* Insert the new row into the ephemeral table */ + sqlite3VdbeAddOp2(v, OP_NewRowid, csrWrite, regRowid); + sqlite3VdbeAddOp3(v, OP_Insert, csrWrite, regRecord, regRowid); + addrNe = sqlite3VdbeAddOp3(v, OP_Ne, pMWin->regOne, 0, regRowid); + VdbeCoverageNeverNull(v); + + /* This block is run for the first row of each partition */ + s.regArg = windowInitAccum(pParse, pMWin); + + if( regStart ){ + sqlite3ExprCode(pParse, pMWin->pStart, regStart); + windowCheckValue(pParse, regStart, 0 + (pMWin->eFrmType==TK_RANGE?3:0)); + } + if( regEnd ){ + sqlite3ExprCode(pParse, pMWin->pEnd, regEnd); + windowCheckValue(pParse, regEnd, 1 + (pMWin->eFrmType==TK_RANGE?3:0)); + } + + if( pMWin->eFrmType!=TK_RANGE && pMWin->eStart==pMWin->eEnd && regStart ){ + int op = ((pMWin->eStart==TK_FOLLOWING) ? OP_Ge : OP_Le); + int addrGe = sqlite3VdbeAddOp3(v, op, regStart, 0, regEnd); + VdbeCoverageNeverNullIf(v, op==OP_Ge); /* NeverNull because bound */ + VdbeCoverageNeverNullIf(v, op==OP_Le); /* values previously checked */ + windowAggFinal(&s, 0); + sqlite3VdbeAddOp2(v, OP_Rewind, s.current.csr, 1); + VdbeCoverageNeverTaken(v); + windowReturnOneRow(&s); + sqlite3VdbeAddOp1(v, OP_ResetSorter, s.current.csr); + sqlite3VdbeAddOp2(v, OP_Goto, 0, lblWhereEnd); + sqlite3VdbeJumpHere(v, addrGe); + } + if( pMWin->eStart==TK_FOLLOWING && pMWin->eFrmType!=TK_RANGE && regEnd ){ + assert( pMWin->eEnd==TK_FOLLOWING ); + sqlite3VdbeAddOp3(v, OP_Subtract, regStart, regEnd, regStart); + } + + if( pMWin->eStart!=TK_UNBOUNDED ){ + sqlite3VdbeAddOp2(v, OP_Rewind, s.start.csr, 1); + VdbeCoverageNeverTaken(v); + } + sqlite3VdbeAddOp2(v, OP_Rewind, s.current.csr, 1); + VdbeCoverageNeverTaken(v); + sqlite3VdbeAddOp2(v, OP_Rewind, s.end.csr, 1); + VdbeCoverageNeverTaken(v); + if( regPeer && pOrderBy ){ + sqlite3VdbeAddOp3(v, OP_Copy, regNewPeer, regPeer, pOrderBy->nExpr-1); + sqlite3VdbeAddOp3(v, OP_Copy, regPeer, s.start.reg, pOrderBy->nExpr-1); + sqlite3VdbeAddOp3(v, OP_Copy, regPeer, s.current.reg, pOrderBy->nExpr-1); + sqlite3VdbeAddOp3(v, OP_Copy, regPeer, s.end.reg, pOrderBy->nExpr-1); + } + + sqlite3VdbeAddOp2(v, OP_Goto, 0, lblWhereEnd); + + sqlite3VdbeJumpHere(v, addrNe); + + /* Beginning of the block executed for the second and subsequent rows. */ + if( regPeer ){ + windowIfNewPeer(pParse, pOrderBy, regNewPeer, regPeer, lblWhereEnd); + } + if( pMWin->eStart==TK_FOLLOWING ){ + windowCodeOp(&s, WINDOW_AGGSTEP, 0, 0); + if( pMWin->eEnd!=TK_UNBOUNDED ){ + if( pMWin->eFrmType==TK_RANGE ){ + int lbl = sqlite3VdbeMakeLabel(pParse); + int addrNext = sqlite3VdbeCurrentAddr(v); + windowCodeRangeTest(&s, OP_Ge, s.current.csr, regEnd, s.end.csr, lbl); + windowCodeOp(&s, WINDOW_AGGINVERSE, regStart, 0); + windowCodeOp(&s, WINDOW_RETURN_ROW, 0, 0); + sqlite3VdbeAddOp2(v, OP_Goto, 0, addrNext); + sqlite3VdbeResolveLabel(v, lbl); + }else{ + windowCodeOp(&s, WINDOW_RETURN_ROW, regEnd, 0); + windowCodeOp(&s, WINDOW_AGGINVERSE, regStart, 0); } - yymx = yypParser->yytos->major; - if( yymx==YYERRORSYMBOL || yyerrorhit ){ -#ifndef NDEBUG - if( yyTraceFILE ){ - fprintf(yyTraceFILE,"%sDiscard input token %s\n", - yyTracePrompt,yyTokenName[yymajor]); + } + }else + if( pMWin->eEnd==TK_PRECEDING ){ + int bRPS = (pMWin->eStart==TK_PRECEDING && pMWin->eFrmType==TK_RANGE); + windowCodeOp(&s, WINDOW_AGGSTEP, regEnd, 0); + if( bRPS ) windowCodeOp(&s, WINDOW_AGGINVERSE, regStart, 0); + windowCodeOp(&s, WINDOW_RETURN_ROW, 0, 0); + if( !bRPS ) windowCodeOp(&s, WINDOW_AGGINVERSE, regStart, 0); + }else{ + int addr = 0; + windowCodeOp(&s, WINDOW_AGGSTEP, 0, 0); + if( pMWin->eEnd!=TK_UNBOUNDED ){ + if( pMWin->eFrmType==TK_RANGE ){ + int lbl = 0; + addr = sqlite3VdbeCurrentAddr(v); + if( regEnd ){ + lbl = sqlite3VdbeMakeLabel(pParse); + windowCodeRangeTest(&s, OP_Ge, s.current.csr, regEnd, s.end.csr, lbl); + } + windowCodeOp(&s, WINDOW_RETURN_ROW, 0, 0); + windowCodeOp(&s, WINDOW_AGGINVERSE, regStart, 0); + if( regEnd ){ + sqlite3VdbeAddOp2(v, OP_Goto, 0, addr); + sqlite3VdbeResolveLabel(v, lbl); } -#endif - yy_destructor(yypParser, (YYCODETYPE)yymajor, &yyminorunion); - yymajor = YYNOCODE; }else{ - while( yypParser->yytos >= yypParser->yystack - && yymx != YYERRORSYMBOL - && (yyact = yy_find_reduce_action( - yypParser->yytos->stateno, - YYERRORSYMBOL)) >= YY_MIN_REDUCE - ){ - yy_pop_parser_stack(yypParser); - } - if( yypParser->yytos < yypParser->yystack || yymajor==0 ){ - yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion); - yy_parse_failed(yypParser); -#ifndef YYNOERRORRECOVERY - yypParser->yyerrcnt = -1; -#endif - yymajor = YYNOCODE; - }else if( yymx!=YYERRORSYMBOL ){ - yy_shift(yypParser,yyact,YYERRORSYMBOL,yyminor); + if( regEnd ){ + addr = sqlite3VdbeAddOp3(v, OP_IfPos, regEnd, 0, 1); + VdbeCoverage(v); } + windowCodeOp(&s, WINDOW_RETURN_ROW, 0, 0); + windowCodeOp(&s, WINDOW_AGGINVERSE, regStart, 0); + if( regEnd ) sqlite3VdbeJumpHere(v, addr); } - yypParser->yyerrcnt = 3; - yyerrorhit = 1; - if( yymajor==YYNOCODE ) break; - yyact = yypParser->yytos->stateno; -#elif defined(YYNOERRORRECOVERY) - /* If the YYNOERRORRECOVERY macro is defined, then do not attempt to - ** do any kind of error recovery. Instead, simply invoke the syntax - ** error routine and continue going as if nothing had happened. - ** - ** Applications can set this macro (for example inside %include) if - ** they intend to abandon the parse upon the first syntax error seen. - */ - yy_syntax_error(yypParser,yymajor, yyminor); - yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion); - break; -#else /* YYERRORSYMBOL is not defined */ - /* This is what we do if the grammar does not define ERROR: - ** - ** * Report an error message, and throw away the input token. - ** - ** * If the input token is $, then fail the parse. - ** - ** As before, subsequent error messages are suppressed until - ** three input tokens have been successfully shifted. - */ - if( yypParser->yyerrcnt<=0 ){ - yy_syntax_error(yypParser,yymajor, yyminor); - } - yypParser->yyerrcnt = 3; - yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion); - if( yyendofinput ){ - yy_parse_failed(yypParser); -#ifndef YYNOERRORRECOVERY - yypParser->yyerrcnt = -1; -#endif - } - break; -#endif } - }while( yypParser->yytos>yypParser->yystack ); -#ifndef NDEBUG - if( yyTraceFILE ){ - yyStackEntry *i; - char cDiv = '['; - fprintf(yyTraceFILE,"%sReturn. Stack=",yyTracePrompt); - for(i=&yypParser->yystack[1]; i<=yypParser->yytos; i++){ - fprintf(yyTraceFILE,"%c%s", cDiv, yyTokenName[i->major]); - cDiv = ' '; + } + + /* End of the main input loop */ + sqlite3VdbeResolveLabel(v, lblWhereEnd); + sqlite3WhereEnd(pWInfo); + + /* Fall through */ + if( pMWin->pPartition ){ + addrInteger = sqlite3VdbeAddOp2(v, OP_Integer, 0, regFlushPart); + sqlite3VdbeJumpHere(v, addrGosubFlush); + } + + addrEmpty = sqlite3VdbeAddOp1(v, OP_Rewind, csrWrite); + VdbeCoverage(v); + if( pMWin->eEnd==TK_PRECEDING ){ + int bRPS = (pMWin->eStart==TK_PRECEDING && pMWin->eFrmType==TK_RANGE); + windowCodeOp(&s, WINDOW_AGGSTEP, regEnd, 0); + if( bRPS ) windowCodeOp(&s, WINDOW_AGGINVERSE, regStart, 0); + windowCodeOp(&s, WINDOW_RETURN_ROW, 0, 0); + }else if( pMWin->eStart==TK_FOLLOWING ){ + int addrStart; + int addrBreak1; + int addrBreak2; + int addrBreak3; + windowCodeOp(&s, WINDOW_AGGSTEP, 0, 0); + if( pMWin->eFrmType==TK_RANGE ){ + addrStart = sqlite3VdbeCurrentAddr(v); + addrBreak2 = windowCodeOp(&s, WINDOW_AGGINVERSE, regStart, 1); + addrBreak1 = windowCodeOp(&s, WINDOW_RETURN_ROW, 0, 1); + }else + if( pMWin->eEnd==TK_UNBOUNDED ){ + addrStart = sqlite3VdbeCurrentAddr(v); + addrBreak1 = windowCodeOp(&s, WINDOW_RETURN_ROW, regStart, 1); + addrBreak2 = windowCodeOp(&s, WINDOW_AGGINVERSE, 0, 1); + }else{ + assert( pMWin->eEnd==TK_FOLLOWING ); + addrStart = sqlite3VdbeCurrentAddr(v); + addrBreak1 = windowCodeOp(&s, WINDOW_RETURN_ROW, regEnd, 1); + addrBreak2 = windowCodeOp(&s, WINDOW_AGGINVERSE, regStart, 1); + } + sqlite3VdbeAddOp2(v, OP_Goto, 0, addrStart); + sqlite3VdbeJumpHere(v, addrBreak2); + addrStart = sqlite3VdbeCurrentAddr(v); + addrBreak3 = windowCodeOp(&s, WINDOW_RETURN_ROW, 0, 1); + sqlite3VdbeAddOp2(v, OP_Goto, 0, addrStart); + sqlite3VdbeJumpHere(v, addrBreak1); + sqlite3VdbeJumpHere(v, addrBreak3); + }else{ + int addrBreak; + int addrStart; + windowCodeOp(&s, WINDOW_AGGSTEP, 0, 0); + addrStart = sqlite3VdbeCurrentAddr(v); + addrBreak = windowCodeOp(&s, WINDOW_RETURN_ROW, 0, 1); + windowCodeOp(&s, WINDOW_AGGINVERSE, regStart, 0); + sqlite3VdbeAddOp2(v, OP_Goto, 0, addrStart); + sqlite3VdbeJumpHere(v, addrBreak); + } + sqlite3VdbeJumpHere(v, addrEmpty); + + sqlite3VdbeAddOp1(v, OP_ResetSorter, s.current.csr); + if( pMWin->pPartition ){ + if( pMWin->regStartRowid ){ + sqlite3VdbeAddOp2(v, OP_Integer, 1, pMWin->regStartRowid); + sqlite3VdbeAddOp2(v, OP_Integer, 0, pMWin->regEndRowid); } - fprintf(yyTraceFILE,"]\n"); + sqlite3VdbeChangeP1(v, addrInteger, sqlite3VdbeCurrentAddr(v)); + sqlite3VdbeAddOp1(v, OP_Return, regFlushPart); } -#endif - return; } -/************** End of parse.c ***********************************************/ -/************** Begin file tokenize.c ****************************************/ +#endif /* SQLITE_OMIT_WINDOWFUNC */ + +/************** End of window.c **********************************************/ +/************** Begin file parse.c *******************************************/ /* -** 2001 September 15 +** 2000-05-29 ** ** The author disclaims copyright to this source code. In place of ** a legal notice, here is a blessing: @@ -145475,12780 +153411,20950 @@ SQLITE_PRIVATE void sqlite3Parser( ** May you share freely, never taking more than you give. ** ************************************************************************* -** An tokenizer for SQL +** Driver template for the LEMON parser generator. ** -** This file contains C code that splits an SQL input string up into -** individual tokens and sends those tokens one-by-one over to the -** parser for analysis. +** The "lemon" program processes an LALR(1) input grammar file, then uses +** this template to construct a parser. The "lemon" program inserts text +** at each "%%" line. Also, any "P-a-r-s-e" identifer prefix (without the +** interstitial "-" characters) contained in this template is changed into +** the value of the %name directive from the grammar. Otherwise, the content +** of this template is copied straight through into the generate parser +** source file. +** +** The following is the concatenation of all %include directives from the +** input grammar file: */ +/* #include */ +/* #include */ +/************ Begin %include sections from the grammar ************************/ + /* #include "sqliteInt.h" */ -/* #include */ -/* Character classes for tokenizing -** -** In the sqlite3GetToken() function, a switch() on aiClass[c] is implemented -** using a lookup table, whereas a switch() directly on c uses a binary search. -** The lookup table is much faster. To maximize speed, and to ensure that -** a lookup table is used, all of the classes need to be small integers and -** all of them need to be used within the switch. +/* +** Disable all error recovery processing in the parser push-down +** automaton. */ -#define CC_X 0 /* The letter 'x', or start of BLOB literal */ -#define CC_KYWD 1 /* Alphabetics or '_'. Usable in a keyword */ -#define CC_ID 2 /* unicode characters usable in IDs */ -#define CC_DIGIT 3 /* Digits */ -#define CC_DOLLAR 4 /* '$' */ -#define CC_VARALPHA 5 /* '@', '#', ':'. Alphabetic SQL variables */ -#define CC_VARNUM 6 /* '?'. Numeric SQL variables */ -#define CC_SPACE 7 /* Space characters */ -#define CC_QUOTE 8 /* '"', '\'', or '`'. String literals, quoted ids */ -#define CC_QUOTE2 9 /* '['. [...] style quoted ids */ -#define CC_PIPE 10 /* '|'. Bitwise OR or concatenate */ -#define CC_MINUS 11 /* '-'. Minus or SQL-style comment */ -#define CC_LT 12 /* '<'. Part of < or <= or <> */ -#define CC_GT 13 /* '>'. Part of > or >= */ -#define CC_EQ 14 /* '='. Part of = or == */ -#define CC_BANG 15 /* '!'. Part of != */ -#define CC_SLASH 16 /* '/'. / or c-style comment */ -#define CC_LP 17 /* '(' */ -#define CC_RP 18 /* ')' */ -#define CC_SEMI 19 /* ';' */ -#define CC_PLUS 20 /* '+' */ -#define CC_STAR 21 /* '*' */ -#define CC_PERCENT 22 /* '%' */ -#define CC_COMMA 23 /* ',' */ -#define CC_AND 24 /* '&' */ -#define CC_TILDA 25 /* '~' */ -#define CC_DOT 26 /* '.' */ -#define CC_ILLEGAL 27 /* Illegal character */ - -static const unsigned char aiClass[] = { -#ifdef SQLITE_ASCII -/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xa xb xc xd xe xf */ -/* 0x */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 7, 7, 27, 7, 7, 27, 27, -/* 1x */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, -/* 2x */ 7, 15, 8, 5, 4, 22, 24, 8, 17, 18, 21, 20, 23, 11, 26, 16, -/* 3x */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 5, 19, 12, 14, 13, 6, -/* 4x */ 5, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, -/* 5x */ 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 9, 27, 27, 27, 1, -/* 6x */ 8, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, -/* 7x */ 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 27, 10, 27, 25, 27, -/* 8x */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, -/* 9x */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, -/* Ax */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, -/* Bx */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, -/* Cx */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, -/* Dx */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, -/* Ex */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, -/* Fx */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 -#endif -#ifdef SQLITE_EBCDIC -/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xa xb xc xd xe xf */ -/* 0x */ 27, 27, 27, 27, 27, 7, 27, 27, 27, 27, 27, 27, 7, 7, 27, 27, -/* 1x */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, -/* 2x */ 27, 27, 27, 27, 27, 7, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, -/* 3x */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, -/* 4x */ 7, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 26, 12, 17, 20, 10, -/* 5x */ 24, 27, 27, 27, 27, 27, 27, 27, 27, 27, 15, 4, 21, 18, 19, 27, -/* 6x */ 11, 16, 27, 27, 27, 27, 27, 27, 27, 27, 27, 23, 22, 1, 13, 6, -/* 7x */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 8, 5, 5, 5, 8, 14, 8, -/* 8x */ 27, 1, 1, 1, 1, 1, 1, 1, 1, 1, 27, 27, 27, 27, 27, 27, -/* 9x */ 27, 1, 1, 1, 1, 1, 1, 1, 1, 1, 27, 27, 27, 27, 27, 27, -/* Ax */ 27, 25, 1, 1, 1, 1, 1, 0, 1, 1, 27, 27, 27, 27, 27, 27, -/* Bx */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 9, 27, 27, 27, 27, 27, -/* Cx */ 27, 1, 1, 1, 1, 1, 1, 1, 1, 1, 27, 27, 27, 27, 27, 27, -/* Dx */ 27, 1, 1, 1, 1, 1, 1, 1, 1, 1, 27, 27, 27, 27, 27, 27, -/* Ex */ 27, 27, 1, 1, 1, 1, 1, 0, 1, 1, 27, 27, 27, 27, 27, 27, -/* Fx */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 27, 27, 27, 27, 27, 27, -#endif -}; +#define YYNOERRORRECOVERY 1 /* -** The charMap() macro maps alphabetic characters (only) into their -** lower-case ASCII equivalent. On ASCII machines, this is just -** an upper-to-lower case map. On EBCDIC machines we also need -** to adjust the encoding. The mapping is only valid for alphabetics -** which are the only characters for which this feature is used. -** -** Used by keywordhash.h +** Make yytestcase() the same as testcase() */ -#ifdef SQLITE_ASCII -# define charMap(X) sqlite3UpperToLower[(unsigned char)X] -#endif -#ifdef SQLITE_EBCDIC -# define charMap(X) ebcdicToAscii[(unsigned char)X] -const unsigned char ebcdicToAscii[] = { -/* 0 1 2 3 4 5 6 7 8 9 A B C D E F */ - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x */ - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 1x */ - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 2x */ - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 3x */ - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 4x */ - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 5x */ - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 0, 0, /* 6x */ - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 7x */ - 0, 97, 98, 99,100,101,102,103,104,105, 0, 0, 0, 0, 0, 0, /* 8x */ - 0,106,107,108,109,110,111,112,113,114, 0, 0, 0, 0, 0, 0, /* 9x */ - 0, 0,115,116,117,118,119,120,121,122, 0, 0, 0, 0, 0, 0, /* Ax */ - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* Bx */ - 0, 97, 98, 99,100,101,102,103,104,105, 0, 0, 0, 0, 0, 0, /* Cx */ - 0,106,107,108,109,110,111,112,113,114, 0, 0, 0, 0, 0, 0, /* Dx */ - 0, 0,115,116,117,118,119,120,121,122, 0, 0, 0, 0, 0, 0, /* Ex */ - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* Fx */ -}; -#endif +#define yytestcase(X) testcase(X) /* -** The sqlite3KeywordCode function looks up an identifier to determine if -** it is a keyword. If it is a keyword, the token code of that keyword is -** returned. If the input is not a keyword, TK_ID is returned. -** -** The implementation of this routine was generated by a program, -** mkkeywordhash.c, located in the tool subdirectory of the distribution. -** The output of the mkkeywordhash.c program is written into a file -** named keywordhash.h and then included into this source file by -** the #include below. +** Indicate that sqlite3ParserFree() will never be called with a null +** pointer. */ -/************** Include keywordhash.h in the middle of tokenize.c ************/ -/************** Begin file keywordhash.h *************************************/ -/***** This file contains automatically generated code ****** -** -** The code in this file has been automatically generated by -** -** sqlite/tool/mkkeywordhash.c -** -** The code in this file implements a function that determines whether -** or not a given identifier is really an SQL keyword. The same thing -** might be implemented more directly using a hand-written hash table. -** But by using this automatically generated code, the size of the code -** is substantially reduced. This is important for embedded applications -** on platforms with limited memory. +#define YYPARSEFREENEVERNULL 1 + +/* +** In the amalgamation, the parse.c file generated by lemon and the +** tokenize.c file are concatenated. In that case, sqlite3RunParser() +** has access to the the size of the yyParser object and so the parser +** engine can be allocated from stack. In that case, only the +** sqlite3ParserInit() and sqlite3ParserFinalize() routines are invoked +** and the sqlite3ParserAlloc() and sqlite3ParserFree() routines can be +** omitted. */ -/* Hash score: 185 */ -/* zKWText[] encodes 845 bytes of keyword text in 561 bytes */ -/* REINDEXEDESCAPEACHECKEYBEFOREIGNOREGEXPLAINSTEADDATABASELECT */ -/* ABLEFTHENDEFERRABLELSEXCEPTRANSACTIONATURALTERAISEXCLUSIVE */ -/* XISTSAVEPOINTERSECTRIGGEREFERENCESCONSTRAINTOFFSETEMPORARY */ -/* UNIQUERYWITHOUTERELEASEATTACHAVINGROUPDATEBEGINNERECURSIVE */ -/* BETWEENOTHINGLOBYCASCADELETECASECOLLATECREATECURRENT_DATE */ -/* DETACHIMMEDIATEJOINSERTLIKEMATCHPLANALYZEPRAGMABORTVALUES */ -/* VIRTUALIMITWHENOTNULLWHERENAMEAFTEREPLACEANDEFAULT */ -/* AUTOINCREMENTCASTCOLUMNCOMMITCONFLICTCROSSCURRENT_TIMESTAMP */ -/* RIMARYDEFERREDISTINCTDORDERESTRICTDROPFAILFROMFULLIFISNULL */ -/* RIGHTROLLBACKROWUNIONUSINGVACUUMVIEWINITIALLY */ -static const char zKWText[560] = { - 'R','E','I','N','D','E','X','E','D','E','S','C','A','P','E','A','C','H', - 'E','C','K','E','Y','B','E','F','O','R','E','I','G','N','O','R','E','G', - 'E','X','P','L','A','I','N','S','T','E','A','D','D','A','T','A','B','A', - 'S','E','L','E','C','T','A','B','L','E','F','T','H','E','N','D','E','F', - 'E','R','R','A','B','L','E','L','S','E','X','C','E','P','T','R','A','N', - 'S','A','C','T','I','O','N','A','T','U','R','A','L','T','E','R','A','I', - 'S','E','X','C','L','U','S','I','V','E','X','I','S','T','S','A','V','E', - 'P','O','I','N','T','E','R','S','E','C','T','R','I','G','G','E','R','E', - 'F','E','R','E','N','C','E','S','C','O','N','S','T','R','A','I','N','T', - 'O','F','F','S','E','T','E','M','P','O','R','A','R','Y','U','N','I','Q', - 'U','E','R','Y','W','I','T','H','O','U','T','E','R','E','L','E','A','S', - 'E','A','T','T','A','C','H','A','V','I','N','G','R','O','U','P','D','A', - 'T','E','B','E','G','I','N','N','E','R','E','C','U','R','S','I','V','E', - 'B','E','T','W','E','E','N','O','T','H','I','N','G','L','O','B','Y','C', - 'A','S','C','A','D','E','L','E','T','E','C','A','S','E','C','O','L','L', - 'A','T','E','C','R','E','A','T','E','C','U','R','R','E','N','T','_','D', - 'A','T','E','D','E','T','A','C','H','I','M','M','E','D','I','A','T','E', - 'J','O','I','N','S','E','R','T','L','I','K','E','M','A','T','C','H','P', - 'L','A','N','A','L','Y','Z','E','P','R','A','G','M','A','B','O','R','T', - 'V','A','L','U','E','S','V','I','R','T','U','A','L','I','M','I','T','W', - 'H','E','N','O','T','N','U','L','L','W','H','E','R','E','N','A','M','E', - 'A','F','T','E','R','E','P','L','A','C','E','A','N','D','E','F','A','U', - 'L','T','A','U','T','O','I','N','C','R','E','M','E','N','T','C','A','S', - 'T','C','O','L','U','M','N','C','O','M','M','I','T','C','O','N','F','L', - 'I','C','T','C','R','O','S','S','C','U','R','R','E','N','T','_','T','I', - 'M','E','S','T','A','M','P','R','I','M','A','R','Y','D','E','F','E','R', - 'R','E','D','I','S','T','I','N','C','T','D','O','R','D','E','R','E','S', - 'T','R','I','C','T','D','R','O','P','F','A','I','L','F','R','O','M','F', - 'U','L','L','I','F','I','S','N','U','L','L','R','I','G','H','T','R','O', - 'L','L','B','A','C','K','R','O','W','U','N','I','O','N','U','S','I','N', - 'G','V','A','C','U','U','M','V','I','E','W','I','N','I','T','I','A','L', - 'L','Y', -}; -/* aKWHash[i] is the hash value for the i-th keyword */ -static const unsigned char aKWHash[127] = { - 74, 108, 119, 72, 0, 45, 0, 0, 81, 0, 76, 61, 0, - 42, 12, 77, 15, 0, 118, 84, 54, 116, 0, 19, 0, 0, - 123, 0, 121, 111, 0, 22, 96, 0, 9, 0, 0, 68, 69, - 0, 67, 6, 0, 48, 93, 105, 0, 120, 104, 0, 0, 44, - 0, 106, 24, 0, 17, 0, 124, 53, 23, 0, 5, 62, 25, - 99, 0, 0, 126, 112, 60, 125, 57, 28, 55, 0, 94, 0, - 103, 26, 0, 102, 0, 0, 0, 98, 95, 100, 91, 115, 14, - 39, 114, 0, 80, 0, 109, 92, 90, 32, 0, 122, 79, 117, - 86, 46, 83, 0, 0, 97, 40, 59, 110, 0, 36, 0, 0, - 29, 0, 89, 87, 88, 0, 20, 85, 0, 56, -}; -/* aKWNext[] forms the hash collision chain. If aKWHash[i]==0 -** then the i-th keyword has no more hash collisions. Otherwise, -** the next keyword with the same hash is aKWHash[i]-1. */ -static const unsigned char aKWNext[126] = { - 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 2, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, - 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 33, 0, 21, 0, 0, 0, 0, 0, 50, - 0, 43, 3, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 1, 64, 0, 0, 65, 0, 41, 0, 38, 0, 0, 0, - 0, 0, 49, 75, 0, 0, 30, 0, 58, 0, 0, 63, 31, - 52, 16, 34, 10, 0, 0, 0, 0, 0, 0, 0, 11, 70, - 78, 0, 8, 0, 18, 51, 0, 107, 101, 0, 113, 0, 73, - 27, 37, 71, 82, 0, 35, 66, 0, 0, -}; -/* aKWLen[i] is the length (in bytes) of the i-th keyword */ -static const unsigned char aKWLen[126] = { - 7, 7, 5, 4, 6, 4, 5, 3, 6, 7, 3, 6, 6, - 7, 7, 3, 8, 2, 6, 5, 4, 4, 3, 10, 4, 6, - 11, 6, 2, 7, 5, 5, 9, 6, 9, 9, 7, 10, 10, - 4, 6, 2, 3, 9, 4, 2, 6, 5, 7, 4, 5, 7, - 6, 6, 5, 6, 5, 5, 9, 7, 7, 4, 2, 7, 3, - 6, 4, 7, 6, 12, 6, 9, 4, 6, 4, 5, 4, 7, - 6, 5, 6, 7, 5, 4, 7, 3, 2, 4, 5, 6, 5, - 7, 3, 7, 13, 2, 2, 4, 6, 6, 8, 5, 17, 12, - 7, 8, 8, 2, 2, 5, 8, 4, 4, 4, 4, 2, 6, - 5, 8, 3, 5, 5, 6, 4, 9, 3, -}; -/* aKWOffset[i] is the index into zKWText[] of the start of -** the text for the i-th keyword. */ -static const unsigned short int aKWOffset[126] = { - 0, 2, 2, 8, 9, 14, 16, 20, 23, 25, 25, 29, 33, - 36, 41, 46, 48, 53, 54, 59, 62, 65, 67, 69, 78, 81, - 86, 91, 95, 96, 101, 105, 109, 117, 122, 128, 136, 142, 152, - 159, 162, 162, 165, 167, 167, 171, 176, 179, 184, 184, 188, 192, - 199, 204, 209, 212, 218, 221, 225, 234, 240, 246, 249, 251, 252, - 256, 262, 266, 273, 279, 291, 297, 306, 308, 314, 318, 323, 325, - 332, 337, 342, 348, 354, 359, 362, 362, 362, 365, 369, 372, 378, - 382, 389, 391, 398, 400, 402, 411, 415, 421, 427, 435, 440, 440, - 456, 463, 470, 471, 478, 479, 483, 491, 495, 499, 503, 507, 509, - 515, 520, 528, 531, 536, 541, 547, 551, 556, -}; -/* aKWCode[i] is the parser symbol code for the i-th keyword */ -static const unsigned char aKWCode[126] = { - TK_REINDEX, TK_INDEXED, TK_INDEX, TK_DESC, TK_ESCAPE, - TK_EACH, TK_CHECK, TK_KEY, TK_BEFORE, TK_FOREIGN, - TK_FOR, TK_IGNORE, TK_LIKE_KW, TK_EXPLAIN, TK_INSTEAD, - TK_ADD, TK_DATABASE, TK_AS, TK_SELECT, TK_TABLE, - TK_JOIN_KW, TK_THEN, TK_END, TK_DEFERRABLE, TK_ELSE, - TK_EXCEPT, TK_TRANSACTION,TK_ACTION, TK_ON, TK_JOIN_KW, - TK_ALTER, TK_RAISE, TK_EXCLUSIVE, TK_EXISTS, TK_SAVEPOINT, - TK_INTERSECT, TK_TRIGGER, TK_REFERENCES, TK_CONSTRAINT, TK_INTO, - TK_OFFSET, TK_OF, TK_SET, TK_TEMP, TK_TEMP, - TK_OR, TK_UNIQUE, TK_QUERY, TK_WITHOUT, TK_WITH, - TK_JOIN_KW, TK_RELEASE, TK_ATTACH, TK_HAVING, TK_GROUP, - TK_UPDATE, TK_BEGIN, TK_JOIN_KW, TK_RECURSIVE, TK_BETWEEN, - TK_NOTHING, TK_LIKE_KW, TK_BY, TK_CASCADE, TK_ASC, - TK_DELETE, TK_CASE, TK_COLLATE, TK_CREATE, TK_CTIME_KW, - TK_DETACH, TK_IMMEDIATE, TK_JOIN, TK_INSERT, TK_LIKE_KW, - TK_MATCH, TK_PLAN, TK_ANALYZE, TK_PRAGMA, TK_ABORT, - TK_VALUES, TK_VIRTUAL, TK_LIMIT, TK_WHEN, TK_NOTNULL, - TK_NOT, TK_NO, TK_NULL, TK_WHERE, TK_RENAME, - TK_AFTER, TK_REPLACE, TK_AND, TK_DEFAULT, TK_AUTOINCR, - TK_TO, TK_IN, TK_CAST, TK_COLUMNKW, TK_COMMIT, - TK_CONFLICT, TK_JOIN_KW, TK_CTIME_KW, TK_CTIME_KW, TK_PRIMARY, - TK_DEFERRED, TK_DISTINCT, TK_IS, TK_DO, TK_ORDER, - TK_RESTRICT, TK_DROP, TK_FAIL, TK_FROM, TK_JOIN_KW, - TK_IF, TK_ISNULL, TK_JOIN_KW, TK_ROLLBACK, TK_ROW, - TK_UNION, TK_USING, TK_VACUUM, TK_VIEW, TK_INITIALLY, - TK_ALL, -}; -/* Check to see if z[0..n-1] is a keyword. If it is, write the -** parser symbol code for that keyword into *pType. Always -** return the integer n (the length of the token). */ -static int keywordCode(const char *z, int n, int *pType){ - int i, j; - const char *zKW; - if( n>=2 ){ - i = ((charMap(z[0])*4) ^ (charMap(z[n-1])*3) ^ n) % 127; - for(i=((int)aKWHash[i])-1; i>=0; i=((int)aKWNext[i])-1){ - if( aKWLen[i]!=n ) continue; - j = 0; - zKW = &zKWText[aKWOffset[i]]; -#ifdef SQLITE_ASCII - while( j=SQLITE_N_KEYWORD ) return SQLITE_ERROR; - *pzName = zKWText + aKWOffset[i]; - *pnName = aKWLen[i]; - return SQLITE_OK; -} -SQLITE_API int sqlite3_keyword_count(void){ return SQLITE_N_KEYWORD; } -SQLITE_API int sqlite3_keyword_check(const char *zName, int nName){ - return TK_ID!=sqlite3KeywordCode((const u8*)zName, nName); -} - -/************** End of keywordhash.h *****************************************/ -/************** Continuing where we left off in tokenize.c *******************/ +/* +** Alternative datatype for the argument to the malloc() routine passed +** into sqlite3ParserAlloc(). The default is size_t. +*/ +#define YYMALLOCARGTYPE u64 /* -** If X is a character that can be used in an identifier then -** IdChar(X) will be true. Otherwise it is false. -** -** For ASCII, any character with the high-order bit set is -** allowed in an identifier. For 7-bit characters, -** sqlite3IsIdChar[X] must be 1. +** An instance of the following structure describes the event of a +** TRIGGER. "a" is the event type, one of TK_UPDATE, TK_INSERT, +** TK_DELETE, or TK_INSTEAD. If the event is of the form ** -** For EBCDIC, the rules are more complex but have the same -** end result. +** UPDATE ON (a,b,c) ** -** Ticket #1066. the SQL standard does not allow '$' in the -** middle of identifiers. But many SQL implementations do. -** SQLite will allow '$' in identifiers for compatibility. -** But the feature is undocumented. +** Then the "b" IdList records the list "a,b,c". */ -#ifdef SQLITE_ASCII -#define IdChar(C) ((sqlite3CtypeMap[(unsigned char)C]&0x46)!=0) -#endif -#ifdef SQLITE_EBCDIC -SQLITE_PRIVATE const char sqlite3IsEbcdicIdChar[] = { -/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */ - 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 4x */ - 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 0, 0, /* 5x */ - 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 0, 0, /* 6x */ - 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, /* 7x */ - 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 0, /* 8x */ - 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 1, 0, /* 9x */ - 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 0, /* Ax */ - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* Bx */ - 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, /* Cx */ - 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, /* Dx */ - 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, /* Ex */ - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0, /* Fx */ -}; -#define IdChar(C) (((c=C)>=0x42 && sqlite3IsEbcdicIdChar[c-0x40])) -#endif - -/* Make the IdChar function accessible from ctime.c */ -#ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS -SQLITE_PRIVATE int sqlite3IsIdChar(u8 c){ return IdChar(c); } -#endif +struct TrigEvent { int a; IdList * b; }; +struct FrameBound { int eType; Expr *pExpr; }; /* -** Return the length (in bytes) of the token that begins at z[0]. -** Store the token type in *tokenType before returning. +** Disable lookaside memory allocation for objects that might be +** shared across database connections. */ -SQLITE_PRIVATE int sqlite3GetToken(const unsigned char *z, int *tokenType){ - int i, c; - switch( aiClass[*z] ){ /* Switch on the character-class of the first byte - ** of the token. See the comment on the CC_ defines - ** above. */ - case CC_SPACE: { - testcase( z[0]==' ' ); - testcase( z[0]=='\t' ); - testcase( z[0]=='\n' ); - testcase( z[0]=='\f' ); - testcase( z[0]=='\r' ); - for(i=1; sqlite3Isspace(z[i]); i++){} - *tokenType = TK_SPACE; - return i; - } - case CC_MINUS: { - if( z[1]=='-' ){ - for(i=2; (c=z[i])!=0 && c!='\n'; i++){} - *tokenType = TK_SPACE; /* IMP: R-22934-25134 */ - return i; +static void disableLookaside(Parse *pParse){ + sqlite3 *db = pParse->db; + pParse->disableLookaside++; + DisableLookaside; +} + + + /* + ** For a compound SELECT statement, make sure p->pPrior->pNext==p for + ** all elements in the list. And make sure list length does not exceed + ** SQLITE_LIMIT_COMPOUND_SELECT. + */ + static void parserDoubleLinkSelect(Parse *pParse, Select *p){ + assert( p!=0 ); + if( p->pPrior ){ + Select *pNext = 0, *pLoop; + int mxSelect, cnt = 0; + for(pLoop=p; pLoop; pNext=pLoop, pLoop=pLoop->pPrior, cnt++){ + pLoop->pNext = pNext; + pLoop->selFlags |= SF_Compound; } - *tokenType = TK_MINUS; - return 1; - } - case CC_LP: { - *tokenType = TK_LP; - return 1; - } - case CC_RP: { - *tokenType = TK_RP; - return 1; - } - case CC_SEMI: { - *tokenType = TK_SEMI; - return 1; - } - case CC_PLUS: { - *tokenType = TK_PLUS; - return 1; - } - case CC_STAR: { - *tokenType = TK_STAR; - return 1; - } - case CC_SLASH: { - if( z[1]!='*' || z[2]==0 ){ - *tokenType = TK_SLASH; - return 1; + if( (p->selFlags & SF_MultiValue)==0 && + (mxSelect = pParse->db->aLimit[SQLITE_LIMIT_COMPOUND_SELECT])>0 && + cnt>mxSelect + ){ + sqlite3ErrorMsg(pParse, "too many terms in compound SELECT"); } - for(i=3, c=z[2]; (c!='*' || z[i]!='/') && (c=z[i])!=0; i++){} - if( c ) i++; - *tokenType = TK_SPACE; /* IMP: R-22934-25134 */ - return i; + } + } + + + /* Construct a new Expr object from a single identifier. Use the + ** new Expr to populate pOut. Set the span of pOut to be the identifier + ** that created the expression. + */ + static Expr *tokenExpr(Parse *pParse, int op, Token t){ + Expr *p = sqlite3DbMallocRawNN(pParse->db, sizeof(Expr)+t.n+1); + if( p ){ + /* memset(p, 0, sizeof(Expr)); */ + p->op = (u8)op; + p->affExpr = 0; + p->flags = EP_Leaf; + ExprClearVVAProperties(p); + p->iAgg = -1; + p->pLeft = p->pRight = 0; + p->x.pList = 0; + p->pAggInfo = 0; + p->y.pTab = 0; + p->op2 = 0; + p->iTable = 0; + p->iColumn = 0; + p->u.zToken = (char*)&p[1]; + memcpy(p->u.zToken, t.z, t.n); + p->u.zToken[t.n] = 0; + if( sqlite3Isquote(p->u.zToken[0]) ){ + sqlite3DequoteExpr(p); + } +#if SQLITE_MAX_EXPR_DEPTH>0 + p->nHeight = 1; +#endif + if( IN_RENAME_OBJECT ){ + return (Expr*)sqlite3RenameTokenMap(pParse, (void*)p, &t); + } + } + return p; + } + + + /* A routine to convert a binary TK_IS or TK_ISNOT expression into a + ** unary TK_ISNULL or TK_NOTNULL expression. */ + static void binaryToUnaryIfNull(Parse *pParse, Expr *pY, Expr *pA, int op){ + sqlite3 *db = pParse->db; + if( pA && pY && pY->op==TK_NULL && !IN_RENAME_OBJECT ){ + pA->op = (u8)op; + sqlite3ExprDelete(db, pA->pRight); + pA->pRight = 0; + } + } + + /* Add a single new term to an ExprList that is used to store a + ** list of identifiers. Report an error if the ID list contains + ** a COLLATE clause or an ASC or DESC keyword, except ignore the + ** error while parsing a legacy schema. + */ + static ExprList *parserAddExprIdListTerm( + Parse *pParse, + ExprList *pPrior, + Token *pIdToken, + int hasCollate, + int sortOrder + ){ + ExprList *p = sqlite3ExprListAppend(pParse, pPrior, 0); + if( (hasCollate || sortOrder!=SQLITE_SO_UNDEFINED) + && pParse->db->init.busy==0 + ){ + sqlite3ErrorMsg(pParse, "syntax error after column name \"%.*s\"", + pIdToken->n, pIdToken->z); + } + sqlite3ExprListSetName(pParse, p, pIdToken, 1); + return p; + } + +#if TK_SPAN>255 +# error too many tokens in the grammar +#endif +/**************** End of %include directives **********************************/ +/* These constants specify the various numeric values for terminal symbols +** in a format understandable to "makeheaders". This section is blank unless +** "lemon" is run with the "-m" command-line option. +***************** Begin makeheaders token definitions *************************/ +/**************** End makeheaders token definitions ***************************/ + +/* The next sections is a series of control #defines. +** various aspects of the generated parser. +** YYCODETYPE is the data type used to store the integer codes +** that represent terminal and non-terminal symbols. +** "unsigned char" is used if there are fewer than +** 256 symbols. Larger types otherwise. +** YYNOCODE is a number of type YYCODETYPE that is not used for +** any terminal or nonterminal symbol. +** YYFALLBACK If defined, this indicates that one or more tokens +** (also known as: "terminal symbols") have fall-back +** values which should be used if the original symbol +** would not parse. This permits keywords to sometimes +** be used as identifiers, for example. +** YYACTIONTYPE is the data type used for "action codes" - numbers +** that indicate what to do in response to the next +** token. +** sqlite3ParserTOKENTYPE is the data type used for minor type for terminal +** symbols. Background: A "minor type" is a semantic +** value associated with a terminal or non-terminal +** symbols. For example, for an "ID" terminal symbol, +** the minor type might be the name of the identifier. +** Each non-terminal can have a different minor type. +** Terminal symbols all have the same minor type, though. +** This macros defines the minor type for terminal +** symbols. +** YYMINORTYPE is the data type used for all minor types. +** This is typically a union of many types, one of +** which is sqlite3ParserTOKENTYPE. The entry in the union +** for terminal symbols is called "yy0". +** YYSTACKDEPTH is the maximum depth of the parser's stack. If +** zero the stack is dynamically sized using realloc() +** sqlite3ParserARG_SDECL A static variable declaration for the %extra_argument +** sqlite3ParserARG_PDECL A parameter declaration for the %extra_argument +** sqlite3ParserARG_PARAM Code to pass %extra_argument as a subroutine parameter +** sqlite3ParserARG_STORE Code to store %extra_argument into yypParser +** sqlite3ParserARG_FETCH Code to extract %extra_argument from yypParser +** sqlite3ParserCTX_* As sqlite3ParserARG_ except for %extra_context +** YYERRORSYMBOL is the code number of the error symbol. If not +** defined, then do no error processing. +** YYNSTATE the combined number of states. +** YYNRULE the number of rules in the grammar +** YYNTOKEN Number of terminal symbols +** YY_MAX_SHIFT Maximum value for shift actions +** YY_MIN_SHIFTREDUCE Minimum value for shift-reduce actions +** YY_MAX_SHIFTREDUCE Maximum value for shift-reduce actions +** YY_ERROR_ACTION The yy_action[] code for syntax error +** YY_ACCEPT_ACTION The yy_action[] code for accept +** YY_NO_ACTION The yy_action[] code for no-op +** YY_MIN_REDUCE Minimum value for reduce actions +** YY_MAX_REDUCE Maximum value for reduce actions +*/ +#ifndef INTERFACE +# define INTERFACE 1 +#endif +/************* Begin control #defines *****************************************/ +#define YYCODETYPE unsigned short int +#define YYNOCODE 310 +#define YYACTIONTYPE unsigned short int +#define YYWILDCARD 100 +#define sqlite3ParserTOKENTYPE Token +typedef union { + int yyinit; + sqlite3ParserTOKENTYPE yy0; + SrcList* yy47; + u8 yy58; + struct FrameBound yy77; + With* yy131; + int yy192; + Expr* yy202; + struct {int value; int mask;} yy207; + struct TrigEvent yy230; + ExprList* yy242; + Window* yy303; + Upsert* yy318; + const char* yy436; + TriggerStep* yy447; + Select* yy539; + IdList* yy600; +} YYMINORTYPE; +#ifndef YYSTACKDEPTH +#define YYSTACKDEPTH 100 +#endif +#define sqlite3ParserARG_SDECL +#define sqlite3ParserARG_PDECL +#define sqlite3ParserARG_PARAM +#define sqlite3ParserARG_FETCH +#define sqlite3ParserARG_STORE +#define sqlite3ParserCTX_SDECL Parse *pParse; +#define sqlite3ParserCTX_PDECL ,Parse *pParse +#define sqlite3ParserCTX_PARAM ,pParse +#define sqlite3ParserCTX_FETCH Parse *pParse=yypParser->pParse; +#define sqlite3ParserCTX_STORE yypParser->pParse=pParse; +#define YYFALLBACK 1 +#define YYNSTATE 551 +#define YYNRULE 385 +#define YYNRULE_WITH_ACTION 325 +#define YYNTOKEN 181 +#define YY_MAX_SHIFT 550 +#define YY_MIN_SHIFTREDUCE 801 +#define YY_MAX_SHIFTREDUCE 1185 +#define YY_ERROR_ACTION 1186 +#define YY_ACCEPT_ACTION 1187 +#define YY_NO_ACTION 1188 +#define YY_MIN_REDUCE 1189 +#define YY_MAX_REDUCE 1573 +/************* End control #defines *******************************************/ +#define YY_NLOOKAHEAD ((int)(sizeof(yy_lookahead)/sizeof(yy_lookahead[0]))) + +/* Define the yytestcase() macro to be a no-op if is not already defined +** otherwise. +** +** Applications can choose to define yytestcase() in the %include section +** to a macro that can assist in verifying code coverage. For production +** code the yytestcase() macro should be turned off. But it is useful +** for testing. +*/ +#ifndef yytestcase +# define yytestcase(X) +#endif + + +/* Next are the tables used to determine what action to take based on the +** current state and lookahead token. These tables are used to implement +** functions that take a state number and lookahead value and return an +** action integer. +** +** Suppose the action integer is N. Then the action is determined as +** follows +** +** 0 <= N <= YY_MAX_SHIFT Shift N. That is, push the lookahead +** token onto the stack and goto state N. +** +** N between YY_MIN_SHIFTREDUCE Shift to an arbitrary state then +** and YY_MAX_SHIFTREDUCE reduce by rule N-YY_MIN_SHIFTREDUCE. +** +** N == YY_ERROR_ACTION A syntax error has occurred. +** +** N == YY_ACCEPT_ACTION The parser accepts its input. +** +** N == YY_NO_ACTION No such action. Denotes unused +** slots in the yy_action[] table. +** +** N between YY_MIN_REDUCE Reduce by rule N-YY_MIN_REDUCE +** and YY_MAX_REDUCE +** +** The action table is constructed as a single large table named yy_action[]. +** Given state S and lookahead X, the action is computed as either: +** +** (A) N = yy_action[ yy_shift_ofst[S] + X ] +** (B) N = yy_default[S] +** +** The (A) formula is preferred. The B formula is used instead if +** yy_lookahead[yy_shift_ofst[S]+X] is not equal to X. +** +** The formulas above are for computing the action when the lookahead is +** a terminal symbol. If the lookahead is a non-terminal (as occurs after +** a reduce action) then the yy_reduce_ofst[] array is used in place of +** the yy_shift_ofst[] array. +** +** The following are the tables generated in this section: +** +** yy_action[] A single table containing all actions. +** yy_lookahead[] A table containing the lookahead for each entry in +** yy_action. Used to detect hash collisions. +** yy_shift_ofst[] For each state, the offset into yy_action for +** shifting terminals. +** yy_reduce_ofst[] For each state, the offset into yy_action for +** shifting non-terminals after a reduce. +** yy_default[] Default action for each state. +** +*********** Begin parsing tables **********************************************/ +#define YY_ACTTAB_COUNT (1958) +static const YYACTIONTYPE yy_action[] = { + /* 0 */ 544, 1220, 544, 449, 1258, 544, 1237, 544, 114, 111, + /* 10 */ 211, 544, 1535, 544, 1258, 521, 114, 111, 211, 390, + /* 20 */ 1230, 342, 42, 42, 42, 42, 1223, 42, 42, 71, + /* 30 */ 71, 935, 1222, 71, 71, 71, 71, 1460, 1491, 936, + /* 40 */ 818, 451, 6, 121, 122, 112, 1163, 1163, 1004, 1007, + /* 50 */ 997, 997, 119, 119, 120, 120, 120, 120, 1541, 390, + /* 60 */ 1356, 1515, 550, 2, 1191, 194, 526, 434, 143, 291, + /* 70 */ 526, 136, 526, 369, 261, 502, 272, 383, 1271, 525, + /* 80 */ 501, 491, 164, 121, 122, 112, 1163, 1163, 1004, 1007, + /* 90 */ 997, 997, 119, 119, 120, 120, 120, 120, 1356, 440, + /* 100 */ 1512, 118, 118, 118, 118, 117, 117, 116, 116, 116, + /* 110 */ 115, 422, 266, 266, 266, 266, 1496, 356, 1498, 433, + /* 120 */ 355, 1496, 515, 522, 1483, 541, 1112, 541, 1112, 390, + /* 130 */ 403, 241, 208, 114, 111, 211, 98, 290, 535, 221, + /* 140 */ 1027, 118, 118, 118, 118, 117, 117, 116, 116, 116, + /* 150 */ 115, 422, 1140, 121, 122, 112, 1163, 1163, 1004, 1007, + /* 160 */ 997, 997, 119, 119, 120, 120, 120, 120, 404, 426, + /* 170 */ 117, 117, 116, 116, 116, 115, 422, 1416, 466, 123, + /* 180 */ 118, 118, 118, 118, 117, 117, 116, 116, 116, 115, + /* 190 */ 422, 116, 116, 116, 115, 422, 538, 538, 538, 390, + /* 200 */ 503, 120, 120, 120, 120, 113, 1049, 1140, 1141, 1142, + /* 210 */ 1049, 118, 118, 118, 118, 117, 117, 116, 116, 116, + /* 220 */ 115, 422, 1459, 121, 122, 112, 1163, 1163, 1004, 1007, + /* 230 */ 997, 997, 119, 119, 120, 120, 120, 120, 390, 442, + /* 240 */ 314, 83, 461, 81, 357, 380, 1140, 80, 118, 118, + /* 250 */ 118, 118, 117, 117, 116, 116, 116, 115, 422, 179, + /* 260 */ 432, 422, 121, 122, 112, 1163, 1163, 1004, 1007, 997, + /* 270 */ 997, 119, 119, 120, 120, 120, 120, 432, 431, 266, + /* 280 */ 266, 118, 118, 118, 118, 117, 117, 116, 116, 116, + /* 290 */ 115, 422, 541, 1107, 901, 504, 1140, 114, 111, 211, + /* 300 */ 1429, 1140, 1141, 1142, 206, 489, 1107, 390, 447, 1107, + /* 310 */ 543, 328, 120, 120, 120, 120, 298, 1429, 1431, 17, + /* 320 */ 118, 118, 118, 118, 117, 117, 116, 116, 116, 115, + /* 330 */ 422, 121, 122, 112, 1163, 1163, 1004, 1007, 997, 997, + /* 340 */ 119, 119, 120, 120, 120, 120, 390, 1356, 432, 1140, + /* 350 */ 480, 1140, 1141, 1142, 994, 994, 1005, 1008, 443, 118, + /* 360 */ 118, 118, 118, 117, 117, 116, 116, 116, 115, 422, + /* 370 */ 121, 122, 112, 1163, 1163, 1004, 1007, 997, 997, 119, + /* 380 */ 119, 120, 120, 120, 120, 1052, 1052, 463, 1429, 118, + /* 390 */ 118, 118, 118, 117, 117, 116, 116, 116, 115, 422, + /* 400 */ 1140, 449, 544, 1424, 1140, 1141, 1142, 233, 964, 1140, + /* 410 */ 479, 476, 475, 171, 358, 390, 164, 405, 412, 840, + /* 420 */ 474, 164, 185, 332, 71, 71, 1241, 998, 118, 118, + /* 430 */ 118, 118, 117, 117, 116, 116, 116, 115, 422, 121, + /* 440 */ 122, 112, 1163, 1163, 1004, 1007, 997, 997, 119, 119, + /* 450 */ 120, 120, 120, 120, 390, 1140, 1141, 1142, 833, 12, + /* 460 */ 313, 507, 163, 354, 1140, 1141, 1142, 114, 111, 211, + /* 470 */ 506, 290, 535, 544, 276, 180, 290, 535, 121, 122, + /* 480 */ 112, 1163, 1163, 1004, 1007, 997, 997, 119, 119, 120, + /* 490 */ 120, 120, 120, 343, 482, 71, 71, 118, 118, 118, + /* 500 */ 118, 117, 117, 116, 116, 116, 115, 422, 1140, 209, + /* 510 */ 409, 521, 1140, 1107, 1569, 376, 252, 269, 340, 485, + /* 520 */ 335, 484, 238, 390, 511, 362, 1107, 1125, 331, 1107, + /* 530 */ 191, 407, 286, 32, 455, 441, 118, 118, 118, 118, + /* 540 */ 117, 117, 116, 116, 116, 115, 422, 121, 122, 112, + /* 550 */ 1163, 1163, 1004, 1007, 997, 997, 119, 119, 120, 120, + /* 560 */ 120, 120, 390, 1140, 1141, 1142, 985, 1140, 1141, 1142, + /* 570 */ 1140, 233, 490, 1490, 479, 476, 475, 6, 163, 544, + /* 580 */ 510, 544, 115, 422, 474, 5, 121, 122, 112, 1163, + /* 590 */ 1163, 1004, 1007, 997, 997, 119, 119, 120, 120, 120, + /* 600 */ 120, 13, 13, 13, 13, 118, 118, 118, 118, 117, + /* 610 */ 117, 116, 116, 116, 115, 422, 401, 500, 406, 544, + /* 620 */ 1484, 542, 1140, 890, 890, 1140, 1141, 1142, 1471, 1140, + /* 630 */ 275, 390, 806, 807, 808, 969, 420, 420, 420, 16, + /* 640 */ 16, 55, 55, 1240, 118, 118, 118, 118, 117, 117, + /* 650 */ 116, 116, 116, 115, 422, 121, 122, 112, 1163, 1163, + /* 660 */ 1004, 1007, 997, 997, 119, 119, 120, 120, 120, 120, + /* 670 */ 390, 1187, 1, 1, 550, 2, 1191, 1140, 1141, 1142, + /* 680 */ 194, 291, 896, 136, 1140, 1141, 1142, 895, 519, 1490, + /* 690 */ 1271, 3, 378, 6, 121, 122, 112, 1163, 1163, 1004, + /* 700 */ 1007, 997, 997, 119, 119, 120, 120, 120, 120, 856, + /* 710 */ 544, 922, 544, 118, 118, 118, 118, 117, 117, 116, + /* 720 */ 116, 116, 115, 422, 266, 266, 1090, 1567, 1140, 549, + /* 730 */ 1567, 1191, 13, 13, 13, 13, 291, 541, 136, 390, + /* 740 */ 483, 419, 418, 964, 342, 1271, 466, 408, 857, 279, + /* 750 */ 140, 221, 118, 118, 118, 118, 117, 117, 116, 116, + /* 760 */ 116, 115, 422, 121, 122, 112, 1163, 1163, 1004, 1007, + /* 770 */ 997, 997, 119, 119, 120, 120, 120, 120, 544, 266, + /* 780 */ 266, 426, 390, 1140, 1141, 1142, 1170, 828, 1170, 466, + /* 790 */ 429, 145, 541, 1144, 399, 313, 437, 301, 836, 1488, + /* 800 */ 71, 71, 410, 6, 1088, 471, 221, 100, 112, 1163, + /* 810 */ 1163, 1004, 1007, 997, 997, 119, 119, 120, 120, 120, + /* 820 */ 120, 118, 118, 118, 118, 117, 117, 116, 116, 116, + /* 830 */ 115, 422, 237, 1423, 544, 449, 426, 287, 984, 544, + /* 840 */ 236, 235, 234, 828, 97, 527, 427, 1263, 1263, 1144, + /* 850 */ 492, 306, 428, 836, 975, 544, 71, 71, 974, 1239, + /* 860 */ 544, 51, 51, 300, 118, 118, 118, 118, 117, 117, + /* 870 */ 116, 116, 116, 115, 422, 194, 103, 70, 70, 266, + /* 880 */ 266, 544, 71, 71, 266, 266, 30, 389, 342, 974, + /* 890 */ 974, 976, 541, 526, 1107, 326, 390, 541, 493, 395, + /* 900 */ 1468, 195, 528, 13, 13, 1356, 240, 1107, 277, 280, + /* 910 */ 1107, 280, 303, 455, 305, 331, 390, 31, 188, 417, + /* 920 */ 121, 122, 112, 1163, 1163, 1004, 1007, 997, 997, 119, + /* 930 */ 119, 120, 120, 120, 120, 142, 390, 363, 455, 984, + /* 940 */ 121, 122, 112, 1163, 1163, 1004, 1007, 997, 997, 119, + /* 950 */ 119, 120, 120, 120, 120, 975, 321, 1140, 324, 974, + /* 960 */ 121, 110, 112, 1163, 1163, 1004, 1007, 997, 997, 119, + /* 970 */ 119, 120, 120, 120, 120, 462, 375, 1183, 118, 118, + /* 980 */ 118, 118, 117, 117, 116, 116, 116, 115, 422, 1140, + /* 990 */ 974, 974, 976, 304, 9, 364, 244, 360, 118, 118, + /* 1000 */ 118, 118, 117, 117, 116, 116, 116, 115, 422, 312, + /* 1010 */ 544, 342, 1140, 1141, 1142, 299, 290, 535, 118, 118, + /* 1020 */ 118, 118, 117, 117, 116, 116, 116, 115, 422, 1261, + /* 1030 */ 1261, 1161, 13, 13, 278, 419, 418, 466, 390, 921, + /* 1040 */ 260, 260, 289, 1167, 1140, 1141, 1142, 189, 1169, 266, + /* 1050 */ 266, 466, 388, 541, 1184, 544, 1168, 263, 144, 487, + /* 1060 */ 920, 544, 541, 122, 112, 1163, 1163, 1004, 1007, 997, + /* 1070 */ 997, 119, 119, 120, 120, 120, 120, 71, 71, 1140, + /* 1080 */ 1170, 1270, 1170, 13, 13, 896, 1068, 1161, 544, 466, + /* 1090 */ 895, 107, 536, 1489, 4, 1266, 1107, 6, 523, 1047, + /* 1100 */ 12, 1069, 1090, 1568, 311, 453, 1568, 518, 539, 1107, + /* 1110 */ 56, 56, 1107, 1487, 421, 1356, 1070, 6, 343, 285, + /* 1120 */ 118, 118, 118, 118, 117, 117, 116, 116, 116, 115, + /* 1130 */ 422, 423, 1269, 319, 1140, 1141, 1142, 876, 266, 266, + /* 1140 */ 1275, 107, 536, 533, 4, 1486, 293, 877, 1209, 6, + /* 1150 */ 210, 541, 541, 164, 1540, 494, 414, 865, 539, 267, + /* 1160 */ 267, 1212, 396, 509, 497, 204, 266, 266, 394, 529, + /* 1170 */ 8, 984, 541, 517, 544, 920, 456, 105, 105, 541, + /* 1180 */ 1088, 423, 266, 266, 106, 415, 423, 546, 545, 266, + /* 1190 */ 266, 974, 516, 533, 1371, 541, 15, 15, 266, 266, + /* 1200 */ 454, 1118, 541, 266, 266, 1068, 1370, 513, 290, 535, + /* 1210 */ 544, 541, 512, 97, 442, 314, 541, 544, 920, 125, + /* 1220 */ 1069, 984, 974, 974, 976, 977, 27, 105, 105, 399, + /* 1230 */ 341, 1509, 44, 44, 106, 1070, 423, 546, 545, 57, + /* 1240 */ 57, 974, 341, 1509, 107, 536, 544, 4, 460, 399, + /* 1250 */ 214, 1118, 457, 294, 375, 1089, 532, 297, 544, 537, + /* 1260 */ 396, 539, 290, 535, 104, 244, 102, 524, 58, 58, + /* 1270 */ 544, 109, 974, 974, 976, 977, 27, 1514, 1129, 425, + /* 1280 */ 59, 59, 270, 237, 423, 138, 95, 373, 373, 372, + /* 1290 */ 255, 370, 60, 60, 815, 1178, 533, 544, 273, 544, + /* 1300 */ 1161, 843, 387, 386, 544, 1307, 544, 215, 210, 296, + /* 1310 */ 513, 847, 544, 265, 208, 514, 1306, 295, 274, 61, + /* 1320 */ 61, 62, 62, 436, 984, 1160, 45, 45, 46, 46, + /* 1330 */ 105, 105, 1184, 920, 47, 47, 1474, 106, 544, 423, + /* 1340 */ 546, 545, 218, 544, 974, 935, 1085, 217, 544, 377, + /* 1350 */ 395, 107, 536, 936, 4, 156, 1161, 843, 158, 544, + /* 1360 */ 49, 49, 141, 544, 38, 50, 50, 544, 539, 307, + /* 1370 */ 63, 63, 544, 1448, 216, 974, 974, 976, 977, 27, + /* 1380 */ 444, 64, 64, 544, 1447, 65, 65, 544, 524, 14, + /* 1390 */ 14, 423, 458, 544, 66, 66, 310, 544, 316, 97, + /* 1400 */ 1034, 544, 961, 533, 268, 127, 127, 544, 391, 67, + /* 1410 */ 67, 544, 978, 290, 535, 52, 52, 513, 544, 68, + /* 1420 */ 68, 1294, 512, 69, 69, 397, 165, 855, 854, 53, + /* 1430 */ 53, 984, 966, 151, 151, 243, 430, 105, 105, 199, + /* 1440 */ 152, 152, 448, 1303, 106, 243, 423, 546, 545, 1129, + /* 1450 */ 425, 974, 320, 270, 862, 863, 1034, 220, 373, 373, + /* 1460 */ 372, 255, 370, 450, 323, 815, 243, 544, 978, 544, + /* 1470 */ 107, 536, 544, 4, 544, 938, 939, 325, 215, 1046, + /* 1480 */ 296, 1046, 974, 974, 976, 977, 27, 539, 295, 76, + /* 1490 */ 76, 54, 54, 327, 72, 72, 128, 128, 1503, 1254, + /* 1500 */ 107, 536, 544, 4, 1045, 544, 1045, 531, 1238, 544, + /* 1510 */ 423, 544, 315, 334, 544, 97, 544, 539, 217, 544, + /* 1520 */ 472, 1528, 533, 239, 73, 73, 156, 129, 129, 158, + /* 1530 */ 467, 130, 130, 126, 126, 344, 150, 150, 149, 149, + /* 1540 */ 423, 134, 134, 329, 1030, 216, 97, 239, 929, 345, + /* 1550 */ 984, 243, 533, 1315, 339, 544, 105, 105, 900, 1355, + /* 1560 */ 544, 1290, 258, 106, 338, 423, 546, 545, 544, 1301, + /* 1570 */ 974, 893, 99, 536, 109, 4, 544, 133, 133, 391, + /* 1580 */ 984, 197, 131, 131, 290, 535, 105, 105, 530, 539, + /* 1590 */ 132, 132, 1361, 106, 1219, 423, 546, 545, 75, 75, + /* 1600 */ 974, 974, 974, 976, 977, 27, 544, 430, 826, 1211, + /* 1610 */ 894, 139, 423, 109, 544, 1200, 1199, 1201, 1522, 544, + /* 1620 */ 201, 544, 11, 374, 533, 1287, 347, 349, 77, 77, + /* 1630 */ 1340, 974, 974, 976, 977, 27, 74, 74, 351, 213, + /* 1640 */ 435, 43, 43, 48, 48, 302, 477, 309, 1348, 382, + /* 1650 */ 353, 452, 984, 337, 1237, 1420, 1419, 205, 105, 105, + /* 1660 */ 192, 367, 193, 534, 1525, 106, 1178, 423, 546, 545, + /* 1670 */ 247, 167, 974, 270, 1467, 200, 1465, 1175, 373, 373, + /* 1680 */ 372, 255, 370, 398, 79, 815, 83, 82, 1425, 446, + /* 1690 */ 161, 177, 169, 95, 1337, 438, 172, 173, 215, 174, + /* 1700 */ 296, 175, 35, 974, 974, 976, 977, 27, 295, 1345, + /* 1710 */ 439, 470, 223, 36, 379, 445, 1414, 381, 459, 1351, + /* 1720 */ 181, 227, 88, 465, 259, 229, 1436, 318, 186, 468, + /* 1730 */ 322, 230, 384, 1202, 231, 486, 1257, 1256, 217, 411, + /* 1740 */ 1255, 1248, 90, 847, 206, 413, 156, 505, 1539, 158, + /* 1750 */ 1226, 1538, 283, 1508, 1227, 336, 385, 284, 1225, 496, + /* 1760 */ 1537, 1298, 94, 346, 348, 216, 1247, 499, 1299, 245, + /* 1770 */ 246, 1297, 416, 350, 1494, 124, 1493, 10, 524, 361, + /* 1780 */ 1400, 101, 96, 288, 508, 253, 1135, 1208, 34, 1296, + /* 1790 */ 547, 254, 256, 257, 392, 548, 1197, 1192, 359, 391, + /* 1800 */ 1280, 1279, 196, 365, 290, 535, 366, 352, 1452, 1322, + /* 1810 */ 1321, 1453, 153, 137, 281, 154, 802, 424, 155, 1451, + /* 1820 */ 1450, 198, 292, 202, 203, 78, 212, 430, 271, 135, + /* 1830 */ 1044, 1042, 958, 168, 219, 157, 170, 879, 308, 222, + /* 1840 */ 1058, 176, 159, 962, 400, 84, 402, 178, 85, 86, + /* 1850 */ 87, 166, 160, 393, 1061, 224, 225, 1057, 146, 18, + /* 1860 */ 226, 317, 1050, 1172, 243, 464, 182, 228, 37, 183, + /* 1870 */ 817, 469, 338, 232, 330, 481, 184, 89, 845, 19, + /* 1880 */ 20, 92, 473, 478, 333, 91, 162, 858, 147, 488, + /* 1890 */ 282, 1123, 148, 1010, 928, 1093, 39, 93, 40, 495, + /* 1900 */ 1094, 187, 498, 207, 262, 264, 923, 242, 1109, 109, + /* 1910 */ 1113, 1111, 1097, 33, 21, 1117, 520, 1025, 22, 23, + /* 1920 */ 24, 1116, 25, 190, 97, 1011, 1009, 26, 1013, 1067, + /* 1930 */ 248, 7, 1066, 249, 1014, 28, 41, 889, 979, 827, + /* 1940 */ 108, 29, 250, 540, 251, 1530, 371, 368, 1131, 1130, + /* 1950 */ 1188, 1188, 1188, 1188, 1188, 1188, 1188, 1529, +}; +static const YYCODETYPE yy_lookahead[] = { + /* 0 */ 189, 211, 189, 189, 218, 189, 220, 189, 267, 268, + /* 10 */ 269, 189, 210, 189, 228, 189, 267, 268, 269, 19, + /* 20 */ 218, 189, 211, 212, 211, 212, 211, 211, 212, 211, + /* 30 */ 212, 31, 211, 211, 212, 211, 212, 288, 300, 39, + /* 40 */ 21, 189, 304, 43, 44, 45, 46, 47, 48, 49, + /* 50 */ 50, 51, 52, 53, 54, 55, 56, 57, 225, 19, + /* 60 */ 189, 183, 184, 185, 186, 189, 248, 263, 236, 191, + /* 70 */ 248, 193, 248, 197, 208, 257, 262, 201, 200, 257, + /* 80 */ 200, 257, 81, 43, 44, 45, 46, 47, 48, 49, + /* 90 */ 50, 51, 52, 53, 54, 55, 56, 57, 189, 80, + /* 100 */ 189, 101, 102, 103, 104, 105, 106, 107, 108, 109, + /* 110 */ 110, 111, 234, 235, 234, 235, 305, 306, 305, 118, + /* 120 */ 307, 305, 306, 297, 298, 247, 86, 247, 88, 19, + /* 130 */ 259, 251, 252, 267, 268, 269, 26, 136, 137, 261, + /* 140 */ 121, 101, 102, 103, 104, 105, 106, 107, 108, 109, + /* 150 */ 110, 111, 59, 43, 44, 45, 46, 47, 48, 49, + /* 160 */ 50, 51, 52, 53, 54, 55, 56, 57, 259, 291, + /* 170 */ 105, 106, 107, 108, 109, 110, 111, 158, 189, 69, + /* 180 */ 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, + /* 190 */ 111, 107, 108, 109, 110, 111, 205, 206, 207, 19, + /* 200 */ 19, 54, 55, 56, 57, 58, 29, 114, 115, 116, + /* 210 */ 33, 101, 102, 103, 104, 105, 106, 107, 108, 109, + /* 220 */ 110, 111, 233, 43, 44, 45, 46, 47, 48, 49, + /* 230 */ 50, 51, 52, 53, 54, 55, 56, 57, 19, 126, + /* 240 */ 127, 148, 65, 24, 214, 200, 59, 67, 101, 102, + /* 250 */ 103, 104, 105, 106, 107, 108, 109, 110, 111, 22, + /* 260 */ 189, 111, 43, 44, 45, 46, 47, 48, 49, 50, + /* 270 */ 51, 52, 53, 54, 55, 56, 57, 206, 207, 234, + /* 280 */ 235, 101, 102, 103, 104, 105, 106, 107, 108, 109, + /* 290 */ 110, 111, 247, 76, 107, 114, 59, 267, 268, 269, + /* 300 */ 189, 114, 115, 116, 162, 163, 89, 19, 263, 92, + /* 310 */ 189, 23, 54, 55, 56, 57, 189, 206, 207, 22, + /* 320 */ 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, + /* 330 */ 111, 43, 44, 45, 46, 47, 48, 49, 50, 51, + /* 340 */ 52, 53, 54, 55, 56, 57, 19, 189, 277, 59, + /* 350 */ 23, 114, 115, 116, 46, 47, 48, 49, 61, 101, + /* 360 */ 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, + /* 370 */ 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, + /* 380 */ 53, 54, 55, 56, 57, 125, 126, 127, 277, 101, + /* 390 */ 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, + /* 400 */ 59, 189, 189, 276, 114, 115, 116, 117, 73, 59, + /* 410 */ 120, 121, 122, 72, 214, 19, 81, 259, 19, 23, + /* 420 */ 130, 81, 72, 24, 211, 212, 221, 119, 101, 102, + /* 430 */ 103, 104, 105, 106, 107, 108, 109, 110, 111, 43, + /* 440 */ 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, + /* 450 */ 54, 55, 56, 57, 19, 114, 115, 116, 23, 208, + /* 460 */ 125, 248, 189, 189, 114, 115, 116, 267, 268, 269, + /* 470 */ 189, 136, 137, 189, 262, 22, 136, 137, 43, 44, + /* 480 */ 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, + /* 490 */ 55, 56, 57, 189, 95, 211, 212, 101, 102, 103, + /* 500 */ 104, 105, 106, 107, 108, 109, 110, 111, 59, 189, + /* 510 */ 111, 189, 59, 76, 294, 295, 117, 118, 119, 120, + /* 520 */ 121, 122, 123, 19, 87, 189, 89, 23, 129, 92, + /* 530 */ 279, 227, 248, 22, 189, 284, 101, 102, 103, 104, + /* 540 */ 105, 106, 107, 108, 109, 110, 111, 43, 44, 45, + /* 550 */ 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, + /* 560 */ 56, 57, 19, 114, 115, 116, 23, 114, 115, 116, + /* 570 */ 59, 117, 299, 300, 120, 121, 122, 304, 189, 189, + /* 580 */ 143, 189, 110, 111, 130, 22, 43, 44, 45, 46, + /* 590 */ 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, + /* 600 */ 57, 211, 212, 211, 212, 101, 102, 103, 104, 105, + /* 610 */ 106, 107, 108, 109, 110, 111, 226, 189, 226, 189, + /* 620 */ 298, 132, 59, 134, 135, 114, 115, 116, 189, 59, + /* 630 */ 285, 19, 7, 8, 9, 23, 205, 206, 207, 211, + /* 640 */ 212, 211, 212, 221, 101, 102, 103, 104, 105, 106, + /* 650 */ 107, 108, 109, 110, 111, 43, 44, 45, 46, 47, + /* 660 */ 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, + /* 670 */ 19, 181, 182, 183, 184, 185, 186, 114, 115, 116, + /* 680 */ 189, 191, 133, 193, 114, 115, 116, 138, 299, 300, + /* 690 */ 200, 22, 201, 304, 43, 44, 45, 46, 47, 48, + /* 700 */ 49, 50, 51, 52, 53, 54, 55, 56, 57, 35, + /* 710 */ 189, 141, 189, 101, 102, 103, 104, 105, 106, 107, + /* 720 */ 108, 109, 110, 111, 234, 235, 22, 23, 59, 184, + /* 730 */ 26, 186, 211, 212, 211, 212, 191, 247, 193, 19, + /* 740 */ 66, 105, 106, 73, 189, 200, 189, 226, 74, 226, + /* 750 */ 22, 261, 101, 102, 103, 104, 105, 106, 107, 108, + /* 760 */ 109, 110, 111, 43, 44, 45, 46, 47, 48, 49, + /* 770 */ 50, 51, 52, 53, 54, 55, 56, 57, 189, 234, + /* 780 */ 235, 291, 19, 114, 115, 116, 150, 59, 152, 189, + /* 790 */ 233, 236, 247, 59, 189, 125, 126, 127, 59, 300, + /* 800 */ 211, 212, 128, 304, 100, 19, 261, 156, 45, 46, + /* 810 */ 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, + /* 820 */ 57, 101, 102, 103, 104, 105, 106, 107, 108, 109, + /* 830 */ 110, 111, 46, 233, 189, 189, 291, 248, 99, 189, + /* 840 */ 125, 126, 127, 115, 26, 200, 289, 230, 231, 115, + /* 850 */ 200, 16, 189, 114, 115, 189, 211, 212, 119, 221, + /* 860 */ 189, 211, 212, 258, 101, 102, 103, 104, 105, 106, + /* 870 */ 107, 108, 109, 110, 111, 189, 156, 211, 212, 234, + /* 880 */ 235, 189, 211, 212, 234, 235, 22, 201, 189, 150, + /* 890 */ 151, 152, 247, 248, 76, 16, 19, 247, 248, 113, + /* 900 */ 189, 24, 257, 211, 212, 189, 26, 89, 262, 223, + /* 910 */ 92, 225, 77, 189, 79, 129, 19, 53, 226, 248, + /* 920 */ 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, + /* 930 */ 53, 54, 55, 56, 57, 236, 19, 271, 189, 99, + /* 940 */ 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, + /* 950 */ 53, 54, 55, 56, 57, 115, 77, 59, 79, 119, + /* 960 */ 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, + /* 970 */ 53, 54, 55, 56, 57, 259, 22, 23, 101, 102, + /* 980 */ 103, 104, 105, 106, 107, 108, 109, 110, 111, 59, + /* 990 */ 150, 151, 152, 158, 22, 244, 24, 246, 101, 102, + /* 1000 */ 103, 104, 105, 106, 107, 108, 109, 110, 111, 285, + /* 1010 */ 189, 189, 114, 115, 116, 200, 136, 137, 101, 102, + /* 1020 */ 103, 104, 105, 106, 107, 108, 109, 110, 111, 230, + /* 1030 */ 231, 59, 211, 212, 285, 105, 106, 189, 19, 141, + /* 1040 */ 234, 235, 239, 113, 114, 115, 116, 226, 118, 234, + /* 1050 */ 235, 189, 249, 247, 100, 189, 126, 23, 236, 107, + /* 1060 */ 26, 189, 247, 44, 45, 46, 47, 48, 49, 50, + /* 1070 */ 51, 52, 53, 54, 55, 56, 57, 211, 212, 59, + /* 1080 */ 150, 233, 152, 211, 212, 133, 12, 115, 189, 189, + /* 1090 */ 138, 19, 20, 300, 22, 233, 76, 304, 226, 11, + /* 1100 */ 208, 27, 22, 23, 200, 19, 26, 87, 36, 89, + /* 1110 */ 211, 212, 92, 300, 248, 189, 42, 304, 189, 250, + /* 1120 */ 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, + /* 1130 */ 111, 59, 200, 233, 114, 115, 116, 63, 234, 235, + /* 1140 */ 235, 19, 20, 71, 22, 300, 189, 73, 200, 304, + /* 1150 */ 116, 247, 247, 81, 23, 200, 227, 26, 36, 234, + /* 1160 */ 235, 203, 204, 143, 200, 26, 234, 235, 194, 200, + /* 1170 */ 48, 99, 247, 66, 189, 141, 284, 105, 106, 247, + /* 1180 */ 100, 59, 234, 235, 112, 259, 114, 115, 116, 234, + /* 1190 */ 235, 119, 85, 71, 266, 247, 211, 212, 234, 235, + /* 1200 */ 114, 94, 247, 234, 235, 12, 266, 85, 136, 137, + /* 1210 */ 189, 247, 90, 26, 126, 127, 247, 189, 26, 22, + /* 1220 */ 27, 99, 150, 151, 152, 153, 154, 105, 106, 189, + /* 1230 */ 302, 303, 211, 212, 112, 42, 114, 115, 116, 211, + /* 1240 */ 212, 119, 302, 303, 19, 20, 189, 22, 274, 189, + /* 1250 */ 15, 144, 278, 189, 22, 23, 63, 189, 189, 203, + /* 1260 */ 204, 36, 136, 137, 155, 24, 157, 143, 211, 212, + /* 1270 */ 189, 26, 150, 151, 152, 153, 154, 0, 1, 2, + /* 1280 */ 211, 212, 5, 46, 59, 161, 147, 10, 11, 12, + /* 1290 */ 13, 14, 211, 212, 17, 60, 71, 189, 258, 189, + /* 1300 */ 59, 59, 105, 106, 189, 189, 189, 30, 116, 32, + /* 1310 */ 85, 124, 189, 251, 252, 90, 189, 40, 258, 211, + /* 1320 */ 212, 211, 212, 189, 99, 26, 211, 212, 211, 212, + /* 1330 */ 105, 106, 100, 141, 211, 212, 189, 112, 189, 114, + /* 1340 */ 115, 116, 24, 189, 119, 31, 23, 70, 189, 26, + /* 1350 */ 113, 19, 20, 39, 22, 78, 115, 115, 81, 189, + /* 1360 */ 211, 212, 22, 189, 24, 211, 212, 189, 36, 189, + /* 1370 */ 211, 212, 189, 189, 97, 150, 151, 152, 153, 154, + /* 1380 */ 127, 211, 212, 189, 189, 211, 212, 189, 143, 211, + /* 1390 */ 212, 59, 189, 189, 211, 212, 23, 189, 189, 26, + /* 1400 */ 59, 189, 149, 71, 22, 211, 212, 189, 131, 211, + /* 1410 */ 212, 189, 59, 136, 137, 211, 212, 85, 189, 211, + /* 1420 */ 212, 253, 90, 211, 212, 292, 293, 118, 119, 211, + /* 1430 */ 212, 99, 23, 211, 212, 26, 159, 105, 106, 140, + /* 1440 */ 211, 212, 23, 189, 112, 26, 114, 115, 116, 1, + /* 1450 */ 2, 119, 189, 5, 7, 8, 115, 139, 10, 11, + /* 1460 */ 12, 13, 14, 23, 189, 17, 26, 189, 115, 189, + /* 1470 */ 19, 20, 189, 22, 189, 83, 84, 189, 30, 150, + /* 1480 */ 32, 152, 150, 151, 152, 153, 154, 36, 40, 211, + /* 1490 */ 212, 211, 212, 189, 211, 212, 211, 212, 309, 189, + /* 1500 */ 19, 20, 189, 22, 150, 189, 152, 231, 189, 189, + /* 1510 */ 59, 189, 23, 189, 189, 26, 189, 36, 70, 189, + /* 1520 */ 23, 139, 71, 26, 211, 212, 78, 211, 212, 81, + /* 1530 */ 281, 211, 212, 211, 212, 189, 211, 212, 211, 212, + /* 1540 */ 59, 211, 212, 23, 23, 97, 26, 26, 23, 189, + /* 1550 */ 99, 26, 71, 189, 119, 189, 105, 106, 107, 189, + /* 1560 */ 189, 189, 280, 112, 129, 114, 115, 116, 189, 189, + /* 1570 */ 119, 23, 19, 20, 26, 22, 189, 211, 212, 131, + /* 1580 */ 99, 237, 211, 212, 136, 137, 105, 106, 189, 36, + /* 1590 */ 211, 212, 189, 112, 189, 114, 115, 116, 211, 212, + /* 1600 */ 119, 150, 151, 152, 153, 154, 189, 159, 23, 189, + /* 1610 */ 23, 26, 59, 26, 189, 189, 189, 189, 189, 189, + /* 1620 */ 209, 189, 238, 187, 71, 250, 250, 250, 211, 212, + /* 1630 */ 241, 150, 151, 152, 153, 154, 211, 212, 250, 290, + /* 1640 */ 254, 211, 212, 211, 212, 254, 215, 286, 241, 241, + /* 1650 */ 254, 286, 99, 214, 220, 214, 214, 224, 105, 106, + /* 1660 */ 244, 240, 244, 273, 192, 112, 60, 114, 115, 116, + /* 1670 */ 139, 290, 119, 5, 196, 238, 196, 38, 10, 11, + /* 1680 */ 12, 13, 14, 196, 287, 17, 148, 287, 276, 113, + /* 1690 */ 43, 22, 229, 147, 241, 18, 232, 232, 30, 232, + /* 1700 */ 32, 232, 264, 150, 151, 152, 153, 154, 40, 265, + /* 1710 */ 196, 18, 195, 264, 241, 241, 241, 265, 196, 229, + /* 1720 */ 229, 195, 155, 62, 196, 195, 283, 282, 22, 216, + /* 1730 */ 196, 195, 216, 196, 195, 113, 213, 213, 70, 64, + /* 1740 */ 213, 222, 22, 124, 162, 111, 78, 142, 219, 81, + /* 1750 */ 215, 219, 275, 303, 213, 213, 216, 275, 213, 216, + /* 1760 */ 213, 256, 113, 255, 255, 97, 222, 216, 256, 196, + /* 1770 */ 91, 256, 82, 255, 308, 146, 308, 22, 143, 196, + /* 1780 */ 270, 155, 145, 272, 144, 25, 13, 199, 26, 256, + /* 1790 */ 198, 190, 190, 6, 296, 188, 188, 188, 244, 131, + /* 1800 */ 245, 245, 243, 242, 136, 137, 241, 255, 208, 260, + /* 1810 */ 260, 208, 202, 217, 217, 202, 4, 3, 202, 208, + /* 1820 */ 208, 22, 160, 209, 209, 208, 15, 159, 98, 16, + /* 1830 */ 23, 23, 137, 148, 24, 128, 140, 20, 16, 142, + /* 1840 */ 1, 140, 128, 149, 61, 53, 37, 148, 53, 53, + /* 1850 */ 53, 293, 128, 296, 114, 34, 139, 1, 5, 22, + /* 1860 */ 113, 158, 68, 75, 26, 41, 68, 139, 24, 113, + /* 1870 */ 20, 19, 129, 123, 23, 96, 22, 22, 59, 22, + /* 1880 */ 22, 147, 67, 67, 24, 22, 37, 28, 23, 22, + /* 1890 */ 67, 23, 23, 23, 114, 23, 22, 26, 22, 24, + /* 1900 */ 23, 22, 24, 139, 23, 23, 141, 34, 88, 26, + /* 1910 */ 75, 86, 23, 22, 34, 75, 24, 23, 34, 34, + /* 1920 */ 34, 93, 34, 26, 26, 23, 23, 34, 23, 23, + /* 1930 */ 26, 44, 23, 22, 11, 22, 22, 133, 23, 23, + /* 1940 */ 22, 22, 139, 26, 139, 139, 15, 23, 1, 1, + /* 1950 */ 310, 310, 310, 310, 310, 310, 310, 139, 310, 310, + /* 1960 */ 310, 310, 310, 310, 310, 310, 310, 310, 310, 310, + /* 1970 */ 310, 310, 310, 310, 310, 310, 310, 310, 310, 310, + /* 1980 */ 310, 310, 310, 310, 310, 310, 310, 310, 310, 310, + /* 1990 */ 310, 310, 310, 310, 310, 310, 310, 310, 310, 310, + /* 2000 */ 310, 310, 310, 310, 310, 310, 310, 310, 310, 310, + /* 2010 */ 310, 310, 310, 310, 310, 310, 310, 310, 310, 310, + /* 2020 */ 310, 310, 310, 310, 310, 310, 310, 310, 310, 310, + /* 2030 */ 310, 310, 310, 310, 310, 310, 310, 310, 310, 310, + /* 2040 */ 310, 310, 310, 310, 310, 310, 310, 310, 310, 310, + /* 2050 */ 310, 310, 310, 310, 310, 310, 310, 310, 310, 310, + /* 2060 */ 310, 310, 310, 310, 310, 310, 310, 310, 310, 310, + /* 2070 */ 310, 310, 310, 310, 310, 310, 310, 310, 310, 310, + /* 2080 */ 310, 310, 310, 310, 310, 310, 310, 310, 310, 310, + /* 2090 */ 310, 310, 310, 310, 310, 310, 310, 310, 310, 310, + /* 2100 */ 310, 310, 310, 310, 310, 310, 310, 310, 310, 310, + /* 2110 */ 310, 310, 310, 310, 310, 310, 310, 310, 310, 310, + /* 2120 */ 310, 310, 310, 310, 310, 310, 310, 310, 310, 310, + /* 2130 */ 310, 310, 310, 310, 310, 310, 310, 310, 310, +}; +#define YY_SHIFT_COUNT (550) +#define YY_SHIFT_MIN (0) +#define YY_SHIFT_MAX (1948) +static const unsigned short int yy_shift_ofst[] = { + /* 0 */ 1448, 1277, 1668, 1072, 1072, 340, 1122, 1225, 1332, 1481, + /* 10 */ 1481, 1481, 335, 0, 0, 180, 897, 1481, 1481, 1481, + /* 20 */ 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, + /* 30 */ 930, 930, 1020, 1020, 290, 1, 340, 340, 340, 340, + /* 40 */ 340, 340, 40, 110, 219, 288, 327, 396, 435, 504, + /* 50 */ 543, 612, 651, 720, 877, 897, 897, 897, 897, 897, + /* 60 */ 897, 897, 897, 897, 897, 897, 897, 897, 897, 897, + /* 70 */ 897, 897, 897, 917, 897, 1019, 763, 763, 1451, 1481, + /* 80 */ 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, + /* 90 */ 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, + /* 100 */ 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, + /* 110 */ 1481, 1481, 1553, 1481, 1481, 1481, 1481, 1481, 1481, 1481, + /* 120 */ 1481, 1481, 1481, 1481, 1481, 1481, 147, 258, 258, 258, + /* 130 */ 258, 258, 79, 65, 84, 449, 19, 786, 449, 636, + /* 140 */ 636, 449, 880, 880, 880, 880, 113, 142, 142, 472, + /* 150 */ 150, 1958, 1958, 399, 399, 399, 93, 237, 341, 237, + /* 160 */ 237, 1074, 1074, 437, 350, 704, 1080, 449, 449, 449, + /* 170 */ 449, 449, 449, 449, 449, 449, 449, 449, 449, 449, + /* 180 */ 449, 449, 449, 449, 449, 449, 449, 449, 818, 818, + /* 190 */ 449, 1088, 217, 217, 734, 734, 1124, 1126, 1958, 1958, + /* 200 */ 1958, 739, 840, 840, 453, 454, 511, 187, 563, 570, + /* 210 */ 898, 669, 449, 449, 449, 449, 449, 449, 449, 449, + /* 220 */ 449, 670, 449, 449, 449, 449, 449, 449, 449, 449, + /* 230 */ 449, 449, 449, 449, 674, 674, 674, 449, 449, 449, + /* 240 */ 449, 1034, 449, 449, 449, 972, 1107, 449, 449, 1193, + /* 250 */ 449, 449, 449, 449, 449, 449, 449, 449, 260, 177, + /* 260 */ 489, 1241, 1241, 1241, 1241, 1192, 489, 489, 952, 1197, + /* 270 */ 625, 1235, 1139, 181, 181, 1086, 1139, 1139, 1086, 1187, + /* 280 */ 1131, 1237, 1314, 1314, 1314, 181, 1245, 1245, 1109, 1299, + /* 290 */ 549, 1340, 1606, 1531, 1531, 1639, 1639, 1531, 1538, 1576, + /* 300 */ 1669, 1647, 1546, 1677, 1677, 1677, 1677, 1531, 1693, 1546, + /* 310 */ 1546, 1576, 1669, 1647, 1647, 1546, 1531, 1693, 1567, 1661, + /* 320 */ 1531, 1693, 1706, 1531, 1693, 1531, 1693, 1706, 1622, 1622, + /* 330 */ 1622, 1675, 1720, 1720, 1706, 1622, 1619, 1622, 1675, 1622, + /* 340 */ 1622, 1582, 1706, 1634, 1634, 1706, 1605, 1649, 1605, 1649, + /* 350 */ 1605, 1649, 1605, 1649, 1531, 1679, 1679, 1690, 1690, 1629, + /* 360 */ 1635, 1755, 1531, 1626, 1629, 1637, 1640, 1546, 1760, 1762, + /* 370 */ 1773, 1773, 1787, 1787, 1787, 1958, 1958, 1958, 1958, 1958, + /* 380 */ 1958, 1958, 1958, 1958, 1958, 1958, 1958, 1958, 1958, 1958, + /* 390 */ 308, 835, 954, 1232, 879, 715, 728, 1323, 864, 1318, + /* 400 */ 1253, 1373, 297, 1409, 1419, 1440, 1489, 1497, 1520, 1242, + /* 410 */ 1309, 1447, 1435, 1341, 1521, 1525, 1392, 1548, 1329, 1354, + /* 420 */ 1585, 1587, 1353, 1382, 1812, 1814, 1799, 1662, 1811, 1730, + /* 430 */ 1813, 1807, 1808, 1695, 1685, 1707, 1810, 1696, 1817, 1697, + /* 440 */ 1822, 1839, 1701, 1694, 1714, 1783, 1809, 1699, 1792, 1795, + /* 450 */ 1796, 1797, 1724, 1740, 1821, 1717, 1856, 1853, 1837, 1747, + /* 460 */ 1703, 1794, 1838, 1798, 1788, 1824, 1728, 1756, 1844, 1850, + /* 470 */ 1852, 1743, 1750, 1854, 1815, 1855, 1857, 1851, 1858, 1816, + /* 480 */ 1819, 1860, 1779, 1859, 1863, 1823, 1849, 1865, 1734, 1867, + /* 490 */ 1868, 1869, 1870, 1871, 1872, 1874, 1875, 1877, 1876, 1878, + /* 500 */ 1764, 1881, 1882, 1780, 1873, 1879, 1765, 1883, 1880, 1884, + /* 510 */ 1885, 1886, 1820, 1835, 1825, 1887, 1840, 1828, 1888, 1889, + /* 520 */ 1891, 1892, 1897, 1898, 1893, 1894, 1883, 1902, 1903, 1905, + /* 530 */ 1906, 1904, 1909, 1911, 1923, 1913, 1914, 1915, 1916, 1918, + /* 540 */ 1919, 1917, 1804, 1803, 1805, 1806, 1818, 1924, 1931, 1947, + /* 550 */ 1948, +}; +#define YY_REDUCE_COUNT (389) +#define YY_REDUCE_MIN (-262) +#define YY_REDUCE_MAX (1617) +static const short yy_reduce_ofst[] = { + /* 0 */ 490, -122, 545, 645, 650, -120, -189, -187, -184, -182, + /* 10 */ -178, -176, 45, 30, 200, -251, -134, 390, 392, 521, + /* 20 */ 523, 213, 692, 821, 284, 589, 872, 666, 671, 866, + /* 30 */ 71, 111, 273, 389, 686, 815, 904, 932, 948, 955, + /* 40 */ 964, 969, -259, -259, -259, -259, -259, -259, -259, -259, + /* 50 */ -259, -259, -259, -259, -259, -259, -259, -259, -259, -259, + /* 60 */ -259, -259, -259, -259, -259, -259, -259, -259, -259, -259, + /* 70 */ -259, -259, -259, -259, -259, -259, -259, -259, 428, 430, + /* 80 */ 899, 985, 1021, 1028, 1057, 1069, 1081, 1108, 1110, 1115, + /* 90 */ 1117, 1123, 1149, 1154, 1159, 1170, 1174, 1178, 1183, 1194, + /* 100 */ 1198, 1204, 1208, 1212, 1218, 1222, 1229, 1278, 1280, 1283, + /* 110 */ 1285, 1313, 1316, 1320, 1322, 1325, 1327, 1330, 1366, 1371, + /* 120 */ 1379, 1387, 1417, 1425, 1430, 1432, -259, -259, -259, -259, + /* 130 */ -259, -259, -259, -259, -259, 557, 974, -214, -174, -9, + /* 140 */ 431, -124, 806, 925, 806, 925, 251, 928, 940, -259, + /* 150 */ -259, -259, -259, -198, -198, -198, 127, -186, -168, 212, + /* 160 */ 646, 617, 799, -262, 555, 220, 220, 491, 605, 1040, + /* 170 */ 1060, 699, -11, 600, 848, 862, 345, -129, 724, -91, + /* 180 */ 158, 749, 716, 900, 304, 822, 929, 926, 499, 793, + /* 190 */ 322, 892, 813, 845, 958, 1056, 751, 905, 1133, 1062, + /* 200 */ 803, -210, -185, -179, -148, -167, -89, 121, 274, 281, + /* 210 */ 320, 336, 439, 663, 711, 957, 1064, 1068, 1116, 1127, + /* 220 */ 1134, -196, 1147, 1180, 1184, 1195, 1203, 1209, 1254, 1263, + /* 230 */ 1275, 1288, 1304, 1310, 205, 422, 638, 1319, 1324, 1346, + /* 240 */ 1360, 1168, 1364, 1370, 1372, 869, 1189, 1380, 1399, 1276, + /* 250 */ 1403, 121, 1405, 1420, 1426, 1427, 1428, 1429, 1249, 1282, + /* 260 */ 1344, 1375, 1376, 1377, 1388, 1168, 1344, 1344, 1384, 1411, + /* 270 */ 1436, 1349, 1389, 1386, 1391, 1361, 1407, 1408, 1365, 1431, + /* 280 */ 1433, 1434, 1439, 1441, 1442, 1396, 1416, 1418, 1390, 1421, + /* 290 */ 1437, 1472, 1381, 1478, 1480, 1397, 1400, 1487, 1412, 1444, + /* 300 */ 1438, 1463, 1453, 1464, 1465, 1467, 1469, 1514, 1517, 1473, + /* 310 */ 1474, 1452, 1449, 1490, 1491, 1475, 1522, 1526, 1443, 1445, + /* 320 */ 1528, 1530, 1513, 1534, 1536, 1537, 1539, 1516, 1523, 1524, + /* 330 */ 1527, 1519, 1529, 1532, 1540, 1541, 1535, 1542, 1544, 1545, + /* 340 */ 1547, 1450, 1543, 1477, 1482, 1551, 1505, 1508, 1512, 1509, + /* 350 */ 1515, 1518, 1533, 1552, 1573, 1466, 1468, 1549, 1550, 1555, + /* 360 */ 1554, 1510, 1583, 1511, 1556, 1559, 1561, 1565, 1588, 1592, + /* 370 */ 1601, 1602, 1607, 1608, 1609, 1498, 1557, 1558, 1610, 1600, + /* 380 */ 1603, 1611, 1612, 1613, 1596, 1597, 1614, 1615, 1617, 1616, +}; +static const YYACTIONTYPE yy_default[] = { + /* 0 */ 1573, 1573, 1573, 1409, 1186, 1295, 1186, 1186, 1186, 1409, + /* 10 */ 1409, 1409, 1186, 1325, 1325, 1462, 1217, 1186, 1186, 1186, + /* 20 */ 1186, 1186, 1186, 1186, 1186, 1186, 1186, 1408, 1186, 1186, + /* 30 */ 1186, 1186, 1492, 1492, 1186, 1186, 1186, 1186, 1186, 1186, + /* 40 */ 1186, 1186, 1186, 1334, 1186, 1186, 1186, 1186, 1186, 1186, + /* 50 */ 1410, 1411, 1186, 1186, 1186, 1461, 1463, 1426, 1344, 1343, + /* 60 */ 1342, 1341, 1444, 1312, 1339, 1332, 1336, 1404, 1405, 1403, + /* 70 */ 1407, 1411, 1410, 1186, 1335, 1375, 1389, 1374, 1186, 1186, + /* 80 */ 1186, 1186, 1186, 1186, 1186, 1186, 1186, 1186, 1186, 1186, + /* 90 */ 1186, 1186, 1186, 1186, 1186, 1186, 1186, 1186, 1186, 1186, + /* 100 */ 1186, 1186, 1186, 1186, 1186, 1186, 1186, 1186, 1186, 1186, + /* 110 */ 1186, 1186, 1186, 1186, 1186, 1186, 1186, 1186, 1186, 1186, + /* 120 */ 1186, 1186, 1186, 1186, 1186, 1186, 1383, 1388, 1394, 1387, + /* 130 */ 1384, 1377, 1376, 1378, 1379, 1186, 1207, 1259, 1186, 1186, + /* 140 */ 1186, 1186, 1480, 1479, 1186, 1186, 1217, 1369, 1368, 1380, + /* 150 */ 1381, 1391, 1390, 1469, 1527, 1526, 1427, 1186, 1186, 1186, + /* 160 */ 1186, 1186, 1186, 1492, 1186, 1186, 1186, 1186, 1186, 1186, + /* 170 */ 1186, 1186, 1186, 1186, 1186, 1186, 1186, 1186, 1186, 1186, + /* 180 */ 1186, 1186, 1186, 1186, 1186, 1186, 1186, 1186, 1492, 1492, + /* 190 */ 1186, 1217, 1492, 1492, 1213, 1213, 1319, 1186, 1475, 1295, + /* 200 */ 1286, 1186, 1186, 1186, 1186, 1186, 1186, 1186, 1186, 1186, + /* 210 */ 1186, 1186, 1186, 1186, 1186, 1466, 1464, 1186, 1186, 1186, + /* 220 */ 1186, 1186, 1186, 1186, 1186, 1186, 1186, 1186, 1186, 1186, + /* 230 */ 1186, 1186, 1186, 1186, 1186, 1186, 1186, 1186, 1186, 1186, + /* 240 */ 1186, 1186, 1186, 1186, 1186, 1291, 1186, 1186, 1186, 1186, + /* 250 */ 1186, 1186, 1186, 1186, 1186, 1186, 1186, 1521, 1186, 1439, + /* 260 */ 1273, 1291, 1291, 1291, 1291, 1293, 1274, 1272, 1285, 1218, + /* 270 */ 1193, 1565, 1338, 1314, 1314, 1562, 1338, 1338, 1562, 1234, + /* 280 */ 1543, 1229, 1325, 1325, 1325, 1314, 1319, 1319, 1406, 1292, + /* 290 */ 1285, 1186, 1565, 1300, 1300, 1564, 1564, 1300, 1427, 1347, + /* 300 */ 1353, 1262, 1338, 1268, 1268, 1268, 1268, 1300, 1204, 1338, + /* 310 */ 1338, 1347, 1353, 1262, 1262, 1338, 1300, 1204, 1443, 1559, + /* 320 */ 1300, 1204, 1417, 1300, 1204, 1300, 1204, 1417, 1260, 1260, + /* 330 */ 1260, 1249, 1186, 1186, 1417, 1260, 1234, 1260, 1249, 1260, + /* 340 */ 1260, 1510, 1417, 1421, 1421, 1417, 1318, 1313, 1318, 1313, + /* 350 */ 1318, 1313, 1318, 1313, 1300, 1502, 1502, 1328, 1328, 1333, + /* 360 */ 1319, 1412, 1300, 1186, 1333, 1331, 1329, 1338, 1210, 1252, + /* 370 */ 1524, 1524, 1520, 1520, 1520, 1570, 1570, 1475, 1536, 1217, + /* 380 */ 1217, 1217, 1217, 1536, 1236, 1236, 1218, 1218, 1217, 1536, + /* 390 */ 1186, 1186, 1186, 1186, 1186, 1186, 1531, 1186, 1428, 1304, + /* 400 */ 1186, 1186, 1186, 1186, 1186, 1186, 1186, 1186, 1186, 1186, + /* 410 */ 1186, 1186, 1186, 1186, 1186, 1186, 1186, 1186, 1186, 1186, + /* 420 */ 1186, 1186, 1186, 1358, 1186, 1189, 1472, 1186, 1186, 1470, + /* 430 */ 1186, 1186, 1186, 1186, 1186, 1186, 1305, 1186, 1186, 1186, + /* 440 */ 1186, 1186, 1186, 1186, 1186, 1186, 1186, 1186, 1186, 1186, + /* 450 */ 1186, 1186, 1186, 1186, 1186, 1561, 1186, 1186, 1186, 1186, + /* 460 */ 1186, 1186, 1442, 1441, 1186, 1186, 1302, 1186, 1186, 1186, + /* 470 */ 1186, 1186, 1186, 1186, 1186, 1186, 1186, 1186, 1186, 1186, + /* 480 */ 1232, 1186, 1186, 1186, 1186, 1186, 1186, 1186, 1186, 1186, + /* 490 */ 1186, 1186, 1186, 1186, 1186, 1186, 1186, 1186, 1186, 1186, + /* 500 */ 1186, 1186, 1186, 1186, 1186, 1186, 1186, 1330, 1186, 1186, + /* 510 */ 1186, 1186, 1186, 1186, 1186, 1186, 1186, 1186, 1186, 1186, + /* 520 */ 1186, 1186, 1507, 1320, 1186, 1186, 1552, 1186, 1186, 1186, + /* 530 */ 1186, 1186, 1186, 1186, 1186, 1186, 1186, 1186, 1186, 1186, + /* 540 */ 1186, 1547, 1276, 1360, 1186, 1359, 1363, 1186, 1198, 1186, + /* 550 */ 1186, +}; +/********** End of lemon-generated parsing tables *****************************/ + +/* The next table maps tokens (terminal symbols) into fallback tokens. +** If a construct like the following: +** +** %fallback ID X Y Z. +** +** appears in the grammar, then ID becomes a fallback token for X, Y, +** and Z. Whenever one of the tokens X, Y, or Z is input to the parser +** but it does not parse, the type of the token is changed to ID and +** the parse is retried before an error is thrown. +** +** This feature can be used, for example, to cause some keywords in a language +** to revert to identifiers if they keyword does not apply in the context where +** it appears. +*/ +#ifdef YYFALLBACK +static const YYCODETYPE yyFallback[] = { + 0, /* $ => nothing */ + 0, /* SEMI => nothing */ + 59, /* EXPLAIN => ID */ + 59, /* QUERY => ID */ + 59, /* PLAN => ID */ + 59, /* BEGIN => ID */ + 0, /* TRANSACTION => nothing */ + 59, /* DEFERRED => ID */ + 59, /* IMMEDIATE => ID */ + 59, /* EXCLUSIVE => ID */ + 0, /* COMMIT => nothing */ + 59, /* END => ID */ + 59, /* ROLLBACK => ID */ + 59, /* SAVEPOINT => ID */ + 59, /* RELEASE => ID */ + 0, /* TO => nothing */ + 0, /* TABLE => nothing */ + 0, /* CREATE => nothing */ + 59, /* IF => ID */ + 0, /* NOT => nothing */ + 0, /* EXISTS => nothing */ + 59, /* TEMP => ID */ + 0, /* LP => nothing */ + 0, /* RP => nothing */ + 0, /* AS => nothing */ + 59, /* WITHOUT => ID */ + 0, /* COMMA => nothing */ + 59, /* ABORT => ID */ + 59, /* ACTION => ID */ + 59, /* AFTER => ID */ + 59, /* ANALYZE => ID */ + 59, /* ASC => ID */ + 59, /* ATTACH => ID */ + 59, /* BEFORE => ID */ + 59, /* BY => ID */ + 59, /* CASCADE => ID */ + 59, /* CAST => ID */ + 59, /* CONFLICT => ID */ + 59, /* DATABASE => ID */ + 59, /* DESC => ID */ + 59, /* DETACH => ID */ + 59, /* EACH => ID */ + 59, /* FAIL => ID */ + 0, /* OR => nothing */ + 0, /* AND => nothing */ + 0, /* IS => nothing */ + 59, /* MATCH => ID */ + 59, /* LIKE_KW => ID */ + 0, /* BETWEEN => nothing */ + 0, /* IN => nothing */ + 0, /* ISNULL => nothing */ + 0, /* NOTNULL => nothing */ + 0, /* NE => nothing */ + 0, /* EQ => nothing */ + 0, /* GT => nothing */ + 0, /* LE => nothing */ + 0, /* LT => nothing */ + 0, /* GE => nothing */ + 0, /* ESCAPE => nothing */ + 0, /* ID => nothing */ + 59, /* COLUMNKW => ID */ + 59, /* DO => ID */ + 59, /* FOR => ID */ + 59, /* IGNORE => ID */ + 59, /* INITIALLY => ID */ + 59, /* INSTEAD => ID */ + 59, /* NO => ID */ + 59, /* KEY => ID */ + 59, /* OF => ID */ + 59, /* OFFSET => ID */ + 59, /* PRAGMA => ID */ + 59, /* RAISE => ID */ + 59, /* RECURSIVE => ID */ + 59, /* REPLACE => ID */ + 59, /* RESTRICT => ID */ + 59, /* ROW => ID */ + 59, /* ROWS => ID */ + 59, /* TRIGGER => ID */ + 59, /* VACUUM => ID */ + 59, /* VIEW => ID */ + 59, /* VIRTUAL => ID */ + 59, /* WITH => ID */ + 59, /* NULLS => ID */ + 59, /* FIRST => ID */ + 59, /* LAST => ID */ + 59, /* CURRENT => ID */ + 59, /* FOLLOWING => ID */ + 59, /* PARTITION => ID */ + 59, /* PRECEDING => ID */ + 59, /* RANGE => ID */ + 59, /* UNBOUNDED => ID */ + 59, /* EXCLUDE => ID */ + 59, /* GROUPS => ID */ + 59, /* OTHERS => ID */ + 59, /* TIES => ID */ + 59, /* GENERATED => ID */ + 59, /* ALWAYS => ID */ + 59, /* REINDEX => ID */ + 59, /* RENAME => ID */ + 59, /* CTIME_KW => ID */ + 0, /* ANY => nothing */ + 0, /* BITAND => nothing */ + 0, /* BITOR => nothing */ + 0, /* LSHIFT => nothing */ + 0, /* RSHIFT => nothing */ + 0, /* PLUS => nothing */ + 0, /* MINUS => nothing */ + 0, /* STAR => nothing */ + 0, /* SLASH => nothing */ + 0, /* REM => nothing */ + 0, /* CONCAT => nothing */ + 0, /* COLLATE => nothing */ + 0, /* BITNOT => nothing */ + 0, /* ON => nothing */ + 0, /* INDEXED => nothing */ + 0, /* STRING => nothing */ + 0, /* JOIN_KW => nothing */ + 0, /* CONSTRAINT => nothing */ + 0, /* DEFAULT => nothing */ + 0, /* NULL => nothing */ + 0, /* PRIMARY => nothing */ + 0, /* UNIQUE => nothing */ + 0, /* CHECK => nothing */ + 0, /* REFERENCES => nothing */ + 0, /* AUTOINCR => nothing */ + 0, /* INSERT => nothing */ + 0, /* DELETE => nothing */ + 0, /* UPDATE => nothing */ + 0, /* SET => nothing */ + 0, /* DEFERRABLE => nothing */ + 0, /* FOREIGN => nothing */ + 0, /* DROP => nothing */ + 0, /* UNION => nothing */ + 0, /* ALL => nothing */ + 0, /* EXCEPT => nothing */ + 0, /* INTERSECT => nothing */ + 0, /* SELECT => nothing */ + 0, /* VALUES => nothing */ + 0, /* DISTINCT => nothing */ + 0, /* DOT => nothing */ + 0, /* FROM => nothing */ + 0, /* JOIN => nothing */ + 0, /* USING => nothing */ + 0, /* ORDER => nothing */ + 0, /* GROUP => nothing */ + 0, /* HAVING => nothing */ + 0, /* LIMIT => nothing */ + 0, /* WHERE => nothing */ + 0, /* INTO => nothing */ + 0, /* NOTHING => nothing */ + 0, /* FLOAT => nothing */ + 0, /* BLOB => nothing */ + 0, /* INTEGER => nothing */ + 0, /* VARIABLE => nothing */ + 0, /* CASE => nothing */ + 0, /* WHEN => nothing */ + 0, /* THEN => nothing */ + 0, /* ELSE => nothing */ + 0, /* INDEX => nothing */ + 0, /* ALTER => nothing */ + 0, /* ADD => nothing */ + 0, /* WINDOW => nothing */ + 0, /* OVER => nothing */ + 0, /* FILTER => nothing */ + 0, /* COLUMN => nothing */ + 0, /* AGG_FUNCTION => nothing */ + 0, /* AGG_COLUMN => nothing */ + 0, /* TRUEFALSE => nothing */ + 0, /* ISNOT => nothing */ + 0, /* FUNCTION => nothing */ + 0, /* UMINUS => nothing */ + 0, /* UPLUS => nothing */ + 0, /* TRUTH => nothing */ + 0, /* REGISTER => nothing */ + 0, /* VECTOR => nothing */ + 0, /* SELECT_COLUMN => nothing */ + 0, /* IF_NULL_ROW => nothing */ + 0, /* ASTERISK => nothing */ + 0, /* SPAN => nothing */ + 0, /* SPACE => nothing */ + 0, /* ILLEGAL => nothing */ +}; +#endif /* YYFALLBACK */ + +/* The following structure represents a single element of the +** parser's stack. Information stored includes: +** +** + The state number for the parser at this level of the stack. +** +** + The value of the token stored at this level of the stack. +** (In other words, the "major" token.) +** +** + The semantic value stored at this level of the stack. This is +** the information used by the action routines in the grammar. +** It is sometimes called the "minor" token. +** +** After the "shift" half of a SHIFTREDUCE action, the stateno field +** actually contains the reduce action for the second half of the +** SHIFTREDUCE. +*/ +struct yyStackEntry { + YYACTIONTYPE stateno; /* The state-number, or reduce action in SHIFTREDUCE */ + YYCODETYPE major; /* The major token value. This is the code + ** number for the token at this stack level */ + YYMINORTYPE minor; /* The user-supplied minor token value. This + ** is the value of the token */ +}; +typedef struct yyStackEntry yyStackEntry; + +/* The state of the parser is completely contained in an instance of +** the following structure */ +struct yyParser { + yyStackEntry *yytos; /* Pointer to top element of the stack */ +#ifdef YYTRACKMAXSTACKDEPTH + int yyhwm; /* High-water mark of the stack */ +#endif +#ifndef YYNOERRORRECOVERY + int yyerrcnt; /* Shifts left before out of the error */ +#endif + sqlite3ParserARG_SDECL /* A place to hold %extra_argument */ + sqlite3ParserCTX_SDECL /* A place to hold %extra_context */ +#if YYSTACKDEPTH<=0 + int yystksz; /* Current side of the stack */ + yyStackEntry *yystack; /* The parser's stack */ + yyStackEntry yystk0; /* First stack entry */ +#else + yyStackEntry yystack[YYSTACKDEPTH]; /* The parser's stack */ + yyStackEntry *yystackEnd; /* Last entry in the stack */ +#endif +}; +typedef struct yyParser yyParser; + +#ifndef NDEBUG +/* #include */ +static FILE *yyTraceFILE = 0; +static char *yyTracePrompt = 0; +#endif /* NDEBUG */ + +#ifndef NDEBUG +/* +** Turn parser tracing on by giving a stream to which to write the trace +** and a prompt to preface each trace message. Tracing is turned off +** by making either argument NULL +** +** Inputs: +**
          +**
        • A FILE* to which trace output should be written. +** If NULL, then tracing is turned off. +**
        • A prefix string written at the beginning of every +** line of trace output. If NULL, then tracing is +** turned off. +**
        +** +** Outputs: +** None. +*/ +SQLITE_PRIVATE void sqlite3ParserTrace(FILE *TraceFILE, char *zTracePrompt){ + yyTraceFILE = TraceFILE; + yyTracePrompt = zTracePrompt; + if( yyTraceFILE==0 ) yyTracePrompt = 0; + else if( yyTracePrompt==0 ) yyTraceFILE = 0; +} +#endif /* NDEBUG */ + +#if defined(YYCOVERAGE) || !defined(NDEBUG) +/* For tracing shifts, the names of all terminals and nonterminals +** are required. The following table supplies these names */ +static const char *const yyTokenName[] = { + /* 0 */ "$", + /* 1 */ "SEMI", + /* 2 */ "EXPLAIN", + /* 3 */ "QUERY", + /* 4 */ "PLAN", + /* 5 */ "BEGIN", + /* 6 */ "TRANSACTION", + /* 7 */ "DEFERRED", + /* 8 */ "IMMEDIATE", + /* 9 */ "EXCLUSIVE", + /* 10 */ "COMMIT", + /* 11 */ "END", + /* 12 */ "ROLLBACK", + /* 13 */ "SAVEPOINT", + /* 14 */ "RELEASE", + /* 15 */ "TO", + /* 16 */ "TABLE", + /* 17 */ "CREATE", + /* 18 */ "IF", + /* 19 */ "NOT", + /* 20 */ "EXISTS", + /* 21 */ "TEMP", + /* 22 */ "LP", + /* 23 */ "RP", + /* 24 */ "AS", + /* 25 */ "WITHOUT", + /* 26 */ "COMMA", + /* 27 */ "ABORT", + /* 28 */ "ACTION", + /* 29 */ "AFTER", + /* 30 */ "ANALYZE", + /* 31 */ "ASC", + /* 32 */ "ATTACH", + /* 33 */ "BEFORE", + /* 34 */ "BY", + /* 35 */ "CASCADE", + /* 36 */ "CAST", + /* 37 */ "CONFLICT", + /* 38 */ "DATABASE", + /* 39 */ "DESC", + /* 40 */ "DETACH", + /* 41 */ "EACH", + /* 42 */ "FAIL", + /* 43 */ "OR", + /* 44 */ "AND", + /* 45 */ "IS", + /* 46 */ "MATCH", + /* 47 */ "LIKE_KW", + /* 48 */ "BETWEEN", + /* 49 */ "IN", + /* 50 */ "ISNULL", + /* 51 */ "NOTNULL", + /* 52 */ "NE", + /* 53 */ "EQ", + /* 54 */ "GT", + /* 55 */ "LE", + /* 56 */ "LT", + /* 57 */ "GE", + /* 58 */ "ESCAPE", + /* 59 */ "ID", + /* 60 */ "COLUMNKW", + /* 61 */ "DO", + /* 62 */ "FOR", + /* 63 */ "IGNORE", + /* 64 */ "INITIALLY", + /* 65 */ "INSTEAD", + /* 66 */ "NO", + /* 67 */ "KEY", + /* 68 */ "OF", + /* 69 */ "OFFSET", + /* 70 */ "PRAGMA", + /* 71 */ "RAISE", + /* 72 */ "RECURSIVE", + /* 73 */ "REPLACE", + /* 74 */ "RESTRICT", + /* 75 */ "ROW", + /* 76 */ "ROWS", + /* 77 */ "TRIGGER", + /* 78 */ "VACUUM", + /* 79 */ "VIEW", + /* 80 */ "VIRTUAL", + /* 81 */ "WITH", + /* 82 */ "NULLS", + /* 83 */ "FIRST", + /* 84 */ "LAST", + /* 85 */ "CURRENT", + /* 86 */ "FOLLOWING", + /* 87 */ "PARTITION", + /* 88 */ "PRECEDING", + /* 89 */ "RANGE", + /* 90 */ "UNBOUNDED", + /* 91 */ "EXCLUDE", + /* 92 */ "GROUPS", + /* 93 */ "OTHERS", + /* 94 */ "TIES", + /* 95 */ "GENERATED", + /* 96 */ "ALWAYS", + /* 97 */ "REINDEX", + /* 98 */ "RENAME", + /* 99 */ "CTIME_KW", + /* 100 */ "ANY", + /* 101 */ "BITAND", + /* 102 */ "BITOR", + /* 103 */ "LSHIFT", + /* 104 */ "RSHIFT", + /* 105 */ "PLUS", + /* 106 */ "MINUS", + /* 107 */ "STAR", + /* 108 */ "SLASH", + /* 109 */ "REM", + /* 110 */ "CONCAT", + /* 111 */ "COLLATE", + /* 112 */ "BITNOT", + /* 113 */ "ON", + /* 114 */ "INDEXED", + /* 115 */ "STRING", + /* 116 */ "JOIN_KW", + /* 117 */ "CONSTRAINT", + /* 118 */ "DEFAULT", + /* 119 */ "NULL", + /* 120 */ "PRIMARY", + /* 121 */ "UNIQUE", + /* 122 */ "CHECK", + /* 123 */ "REFERENCES", + /* 124 */ "AUTOINCR", + /* 125 */ "INSERT", + /* 126 */ "DELETE", + /* 127 */ "UPDATE", + /* 128 */ "SET", + /* 129 */ "DEFERRABLE", + /* 130 */ "FOREIGN", + /* 131 */ "DROP", + /* 132 */ "UNION", + /* 133 */ "ALL", + /* 134 */ "EXCEPT", + /* 135 */ "INTERSECT", + /* 136 */ "SELECT", + /* 137 */ "VALUES", + /* 138 */ "DISTINCT", + /* 139 */ "DOT", + /* 140 */ "FROM", + /* 141 */ "JOIN", + /* 142 */ "USING", + /* 143 */ "ORDER", + /* 144 */ "GROUP", + /* 145 */ "HAVING", + /* 146 */ "LIMIT", + /* 147 */ "WHERE", + /* 148 */ "INTO", + /* 149 */ "NOTHING", + /* 150 */ "FLOAT", + /* 151 */ "BLOB", + /* 152 */ "INTEGER", + /* 153 */ "VARIABLE", + /* 154 */ "CASE", + /* 155 */ "WHEN", + /* 156 */ "THEN", + /* 157 */ "ELSE", + /* 158 */ "INDEX", + /* 159 */ "ALTER", + /* 160 */ "ADD", + /* 161 */ "WINDOW", + /* 162 */ "OVER", + /* 163 */ "FILTER", + /* 164 */ "COLUMN", + /* 165 */ "AGG_FUNCTION", + /* 166 */ "AGG_COLUMN", + /* 167 */ "TRUEFALSE", + /* 168 */ "ISNOT", + /* 169 */ "FUNCTION", + /* 170 */ "UMINUS", + /* 171 */ "UPLUS", + /* 172 */ "TRUTH", + /* 173 */ "REGISTER", + /* 174 */ "VECTOR", + /* 175 */ "SELECT_COLUMN", + /* 176 */ "IF_NULL_ROW", + /* 177 */ "ASTERISK", + /* 178 */ "SPAN", + /* 179 */ "SPACE", + /* 180 */ "ILLEGAL", + /* 181 */ "input", + /* 182 */ "cmdlist", + /* 183 */ "ecmd", + /* 184 */ "cmdx", + /* 185 */ "explain", + /* 186 */ "cmd", + /* 187 */ "transtype", + /* 188 */ "trans_opt", + /* 189 */ "nm", + /* 190 */ "savepoint_opt", + /* 191 */ "create_table", + /* 192 */ "create_table_args", + /* 193 */ "createkw", + /* 194 */ "temp", + /* 195 */ "ifnotexists", + /* 196 */ "dbnm", + /* 197 */ "columnlist", + /* 198 */ "conslist_opt", + /* 199 */ "table_options", + /* 200 */ "select", + /* 201 */ "columnname", + /* 202 */ "carglist", + /* 203 */ "typetoken", + /* 204 */ "typename", + /* 205 */ "signed", + /* 206 */ "plus_num", + /* 207 */ "minus_num", + /* 208 */ "scanpt", + /* 209 */ "scantok", + /* 210 */ "ccons", + /* 211 */ "term", + /* 212 */ "expr", + /* 213 */ "onconf", + /* 214 */ "sortorder", + /* 215 */ "autoinc", + /* 216 */ "eidlist_opt", + /* 217 */ "refargs", + /* 218 */ "defer_subclause", + /* 219 */ "generated", + /* 220 */ "refarg", + /* 221 */ "refact", + /* 222 */ "init_deferred_pred_opt", + /* 223 */ "conslist", + /* 224 */ "tconscomma", + /* 225 */ "tcons", + /* 226 */ "sortlist", + /* 227 */ "eidlist", + /* 228 */ "defer_subclause_opt", + /* 229 */ "orconf", + /* 230 */ "resolvetype", + /* 231 */ "raisetype", + /* 232 */ "ifexists", + /* 233 */ "fullname", + /* 234 */ "selectnowith", + /* 235 */ "oneselect", + /* 236 */ "wqlist", + /* 237 */ "multiselect_op", + /* 238 */ "distinct", + /* 239 */ "selcollist", + /* 240 */ "from", + /* 241 */ "where_opt", + /* 242 */ "groupby_opt", + /* 243 */ "having_opt", + /* 244 */ "orderby_opt", + /* 245 */ "limit_opt", + /* 246 */ "window_clause", + /* 247 */ "values", + /* 248 */ "nexprlist", + /* 249 */ "sclp", + /* 250 */ "as", + /* 251 */ "seltablist", + /* 252 */ "stl_prefix", + /* 253 */ "joinop", + /* 254 */ "indexed_opt", + /* 255 */ "on_opt", + /* 256 */ "using_opt", + /* 257 */ "exprlist", + /* 258 */ "xfullname", + /* 259 */ "idlist", + /* 260 */ "nulls", + /* 261 */ "with", + /* 262 */ "setlist", + /* 263 */ "insert_cmd", + /* 264 */ "idlist_opt", + /* 265 */ "upsert", + /* 266 */ "filter_over", + /* 267 */ "likeop", + /* 268 */ "between_op", + /* 269 */ "in_op", + /* 270 */ "paren_exprlist", + /* 271 */ "case_operand", + /* 272 */ "case_exprlist", + /* 273 */ "case_else", + /* 274 */ "uniqueflag", + /* 275 */ "collate", + /* 276 */ "vinto", + /* 277 */ "nmnum", + /* 278 */ "trigger_decl", + /* 279 */ "trigger_cmd_list", + /* 280 */ "trigger_time", + /* 281 */ "trigger_event", + /* 282 */ "foreach_clause", + /* 283 */ "when_clause", + /* 284 */ "trigger_cmd", + /* 285 */ "trnm", + /* 286 */ "tridxby", + /* 287 */ "database_kw_opt", + /* 288 */ "key_opt", + /* 289 */ "add_column_fullname", + /* 290 */ "kwcolumn_opt", + /* 291 */ "create_vtab", + /* 292 */ "vtabarglist", + /* 293 */ "vtabarg", + /* 294 */ "vtabargtoken", + /* 295 */ "lp", + /* 296 */ "anylist", + /* 297 */ "windowdefn_list", + /* 298 */ "windowdefn", + /* 299 */ "window", + /* 300 */ "frame_opt", + /* 301 */ "part_opt", + /* 302 */ "filter_clause", + /* 303 */ "over_clause", + /* 304 */ "range_or_rows", + /* 305 */ "frame_bound", + /* 306 */ "frame_bound_s", + /* 307 */ "frame_bound_e", + /* 308 */ "frame_exclude_opt", + /* 309 */ "frame_exclude", +}; +#endif /* defined(YYCOVERAGE) || !defined(NDEBUG) */ + +#ifndef NDEBUG +/* For tracing reduce actions, the names of all rules are required. +*/ +static const char *const yyRuleName[] = { + /* 0 */ "explain ::= EXPLAIN", + /* 1 */ "explain ::= EXPLAIN QUERY PLAN", + /* 2 */ "cmdx ::= cmd", + /* 3 */ "cmd ::= BEGIN transtype trans_opt", + /* 4 */ "transtype ::=", + /* 5 */ "transtype ::= DEFERRED", + /* 6 */ "transtype ::= IMMEDIATE", + /* 7 */ "transtype ::= EXCLUSIVE", + /* 8 */ "cmd ::= COMMIT|END trans_opt", + /* 9 */ "cmd ::= ROLLBACK trans_opt", + /* 10 */ "cmd ::= SAVEPOINT nm", + /* 11 */ "cmd ::= RELEASE savepoint_opt nm", + /* 12 */ "cmd ::= ROLLBACK trans_opt TO savepoint_opt nm", + /* 13 */ "create_table ::= createkw temp TABLE ifnotexists nm dbnm", + /* 14 */ "createkw ::= CREATE", + /* 15 */ "ifnotexists ::=", + /* 16 */ "ifnotexists ::= IF NOT EXISTS", + /* 17 */ "temp ::= TEMP", + /* 18 */ "temp ::=", + /* 19 */ "create_table_args ::= LP columnlist conslist_opt RP table_options", + /* 20 */ "create_table_args ::= AS select", + /* 21 */ "table_options ::=", + /* 22 */ "table_options ::= WITHOUT nm", + /* 23 */ "columnname ::= nm typetoken", + /* 24 */ "typetoken ::=", + /* 25 */ "typetoken ::= typename LP signed RP", + /* 26 */ "typetoken ::= typename LP signed COMMA signed RP", + /* 27 */ "typename ::= typename ID|STRING", + /* 28 */ "scanpt ::=", + /* 29 */ "scantok ::=", + /* 30 */ "ccons ::= CONSTRAINT nm", + /* 31 */ "ccons ::= DEFAULT scantok term", + /* 32 */ "ccons ::= DEFAULT LP expr RP", + /* 33 */ "ccons ::= DEFAULT PLUS scantok term", + /* 34 */ "ccons ::= DEFAULT MINUS scantok term", + /* 35 */ "ccons ::= DEFAULT scantok ID|INDEXED", + /* 36 */ "ccons ::= NOT NULL onconf", + /* 37 */ "ccons ::= PRIMARY KEY sortorder onconf autoinc", + /* 38 */ "ccons ::= UNIQUE onconf", + /* 39 */ "ccons ::= CHECK LP expr RP", + /* 40 */ "ccons ::= REFERENCES nm eidlist_opt refargs", + /* 41 */ "ccons ::= defer_subclause", + /* 42 */ "ccons ::= COLLATE ID|STRING", + /* 43 */ "generated ::= LP expr RP", + /* 44 */ "generated ::= LP expr RP ID", + /* 45 */ "autoinc ::=", + /* 46 */ "autoinc ::= AUTOINCR", + /* 47 */ "refargs ::=", + /* 48 */ "refargs ::= refargs refarg", + /* 49 */ "refarg ::= MATCH nm", + /* 50 */ "refarg ::= ON INSERT refact", + /* 51 */ "refarg ::= ON DELETE refact", + /* 52 */ "refarg ::= ON UPDATE refact", + /* 53 */ "refact ::= SET NULL", + /* 54 */ "refact ::= SET DEFAULT", + /* 55 */ "refact ::= CASCADE", + /* 56 */ "refact ::= RESTRICT", + /* 57 */ "refact ::= NO ACTION", + /* 58 */ "defer_subclause ::= NOT DEFERRABLE init_deferred_pred_opt", + /* 59 */ "defer_subclause ::= DEFERRABLE init_deferred_pred_opt", + /* 60 */ "init_deferred_pred_opt ::=", + /* 61 */ "init_deferred_pred_opt ::= INITIALLY DEFERRED", + /* 62 */ "init_deferred_pred_opt ::= INITIALLY IMMEDIATE", + /* 63 */ "conslist_opt ::=", + /* 64 */ "tconscomma ::= COMMA", + /* 65 */ "tcons ::= CONSTRAINT nm", + /* 66 */ "tcons ::= PRIMARY KEY LP sortlist autoinc RP onconf", + /* 67 */ "tcons ::= UNIQUE LP sortlist RP onconf", + /* 68 */ "tcons ::= CHECK LP expr RP onconf", + /* 69 */ "tcons ::= FOREIGN KEY LP eidlist RP REFERENCES nm eidlist_opt refargs defer_subclause_opt", + /* 70 */ "defer_subclause_opt ::=", + /* 71 */ "onconf ::=", + /* 72 */ "onconf ::= ON CONFLICT resolvetype", + /* 73 */ "orconf ::=", + /* 74 */ "orconf ::= OR resolvetype", + /* 75 */ "resolvetype ::= IGNORE", + /* 76 */ "resolvetype ::= REPLACE", + /* 77 */ "cmd ::= DROP TABLE ifexists fullname", + /* 78 */ "ifexists ::= IF EXISTS", + /* 79 */ "ifexists ::=", + /* 80 */ "cmd ::= createkw temp VIEW ifnotexists nm dbnm eidlist_opt AS select", + /* 81 */ "cmd ::= DROP VIEW ifexists fullname", + /* 82 */ "cmd ::= select", + /* 83 */ "select ::= WITH wqlist selectnowith", + /* 84 */ "select ::= WITH RECURSIVE wqlist selectnowith", + /* 85 */ "select ::= selectnowith", + /* 86 */ "selectnowith ::= selectnowith multiselect_op oneselect", + /* 87 */ "multiselect_op ::= UNION", + /* 88 */ "multiselect_op ::= UNION ALL", + /* 89 */ "multiselect_op ::= EXCEPT|INTERSECT", + /* 90 */ "oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt orderby_opt limit_opt", + /* 91 */ "oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt window_clause orderby_opt limit_opt", + /* 92 */ "values ::= VALUES LP nexprlist RP", + /* 93 */ "values ::= values COMMA LP nexprlist RP", + /* 94 */ "distinct ::= DISTINCT", + /* 95 */ "distinct ::= ALL", + /* 96 */ "distinct ::=", + /* 97 */ "sclp ::=", + /* 98 */ "selcollist ::= sclp scanpt expr scanpt as", + /* 99 */ "selcollist ::= sclp scanpt STAR", + /* 100 */ "selcollist ::= sclp scanpt nm DOT STAR", + /* 101 */ "as ::= AS nm", + /* 102 */ "as ::=", + /* 103 */ "from ::=", + /* 104 */ "from ::= FROM seltablist", + /* 105 */ "stl_prefix ::= seltablist joinop", + /* 106 */ "stl_prefix ::=", + /* 107 */ "seltablist ::= stl_prefix nm dbnm as indexed_opt on_opt using_opt", + /* 108 */ "seltablist ::= stl_prefix nm dbnm LP exprlist RP as on_opt using_opt", + /* 109 */ "seltablist ::= stl_prefix LP select RP as on_opt using_opt", + /* 110 */ "seltablist ::= stl_prefix LP seltablist RP as on_opt using_opt", + /* 111 */ "dbnm ::=", + /* 112 */ "dbnm ::= DOT nm", + /* 113 */ "fullname ::= nm", + /* 114 */ "fullname ::= nm DOT nm", + /* 115 */ "xfullname ::= nm", + /* 116 */ "xfullname ::= nm DOT nm", + /* 117 */ "xfullname ::= nm DOT nm AS nm", + /* 118 */ "xfullname ::= nm AS nm", + /* 119 */ "joinop ::= COMMA|JOIN", + /* 120 */ "joinop ::= JOIN_KW JOIN", + /* 121 */ "joinop ::= JOIN_KW nm JOIN", + /* 122 */ "joinop ::= JOIN_KW nm nm JOIN", + /* 123 */ "on_opt ::= ON expr", + /* 124 */ "on_opt ::=", + /* 125 */ "indexed_opt ::=", + /* 126 */ "indexed_opt ::= INDEXED BY nm", + /* 127 */ "indexed_opt ::= NOT INDEXED", + /* 128 */ "using_opt ::= USING LP idlist RP", + /* 129 */ "using_opt ::=", + /* 130 */ "orderby_opt ::=", + /* 131 */ "orderby_opt ::= ORDER BY sortlist", + /* 132 */ "sortlist ::= sortlist COMMA expr sortorder nulls", + /* 133 */ "sortlist ::= expr sortorder nulls", + /* 134 */ "sortorder ::= ASC", + /* 135 */ "sortorder ::= DESC", + /* 136 */ "sortorder ::=", + /* 137 */ "nulls ::= NULLS FIRST", + /* 138 */ "nulls ::= NULLS LAST", + /* 139 */ "nulls ::=", + /* 140 */ "groupby_opt ::=", + /* 141 */ "groupby_opt ::= GROUP BY nexprlist", + /* 142 */ "having_opt ::=", + /* 143 */ "having_opt ::= HAVING expr", + /* 144 */ "limit_opt ::=", + /* 145 */ "limit_opt ::= LIMIT expr", + /* 146 */ "limit_opt ::= LIMIT expr OFFSET expr", + /* 147 */ "limit_opt ::= LIMIT expr COMMA expr", + /* 148 */ "cmd ::= with DELETE FROM xfullname indexed_opt where_opt", + /* 149 */ "where_opt ::=", + /* 150 */ "where_opt ::= WHERE expr", + /* 151 */ "cmd ::= with UPDATE orconf xfullname indexed_opt SET setlist where_opt", + /* 152 */ "setlist ::= setlist COMMA nm EQ expr", + /* 153 */ "setlist ::= setlist COMMA LP idlist RP EQ expr", + /* 154 */ "setlist ::= nm EQ expr", + /* 155 */ "setlist ::= LP idlist RP EQ expr", + /* 156 */ "cmd ::= with insert_cmd INTO xfullname idlist_opt select upsert", + /* 157 */ "cmd ::= with insert_cmd INTO xfullname idlist_opt DEFAULT VALUES", + /* 158 */ "upsert ::=", + /* 159 */ "upsert ::= ON CONFLICT LP sortlist RP where_opt DO UPDATE SET setlist where_opt", + /* 160 */ "upsert ::= ON CONFLICT LP sortlist RP where_opt DO NOTHING", + /* 161 */ "upsert ::= ON CONFLICT DO NOTHING", + /* 162 */ "insert_cmd ::= INSERT orconf", + /* 163 */ "insert_cmd ::= REPLACE", + /* 164 */ "idlist_opt ::=", + /* 165 */ "idlist_opt ::= LP idlist RP", + /* 166 */ "idlist ::= idlist COMMA nm", + /* 167 */ "idlist ::= nm", + /* 168 */ "expr ::= LP expr RP", + /* 169 */ "expr ::= ID|INDEXED", + /* 170 */ "expr ::= JOIN_KW", + /* 171 */ "expr ::= nm DOT nm", + /* 172 */ "expr ::= nm DOT nm DOT nm", + /* 173 */ "term ::= NULL|FLOAT|BLOB", + /* 174 */ "term ::= STRING", + /* 175 */ "term ::= INTEGER", + /* 176 */ "expr ::= VARIABLE", + /* 177 */ "expr ::= expr COLLATE ID|STRING", + /* 178 */ "expr ::= CAST LP expr AS typetoken RP", + /* 179 */ "expr ::= ID|INDEXED LP distinct exprlist RP", + /* 180 */ "expr ::= ID|INDEXED LP STAR RP", + /* 181 */ "expr ::= ID|INDEXED LP distinct exprlist RP filter_over", + /* 182 */ "expr ::= ID|INDEXED LP STAR RP filter_over", + /* 183 */ "term ::= CTIME_KW", + /* 184 */ "expr ::= LP nexprlist COMMA expr RP", + /* 185 */ "expr ::= expr AND expr", + /* 186 */ "expr ::= expr OR expr", + /* 187 */ "expr ::= expr LT|GT|GE|LE expr", + /* 188 */ "expr ::= expr EQ|NE expr", + /* 189 */ "expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr", + /* 190 */ "expr ::= expr PLUS|MINUS expr", + /* 191 */ "expr ::= expr STAR|SLASH|REM expr", + /* 192 */ "expr ::= expr CONCAT expr", + /* 193 */ "likeop ::= NOT LIKE_KW|MATCH", + /* 194 */ "expr ::= expr likeop expr", + /* 195 */ "expr ::= expr likeop expr ESCAPE expr", + /* 196 */ "expr ::= expr ISNULL|NOTNULL", + /* 197 */ "expr ::= expr NOT NULL", + /* 198 */ "expr ::= expr IS expr", + /* 199 */ "expr ::= expr IS NOT expr", + /* 200 */ "expr ::= NOT expr", + /* 201 */ "expr ::= BITNOT expr", + /* 202 */ "expr ::= PLUS|MINUS expr", + /* 203 */ "between_op ::= BETWEEN", + /* 204 */ "between_op ::= NOT BETWEEN", + /* 205 */ "expr ::= expr between_op expr AND expr", + /* 206 */ "in_op ::= IN", + /* 207 */ "in_op ::= NOT IN", + /* 208 */ "expr ::= expr in_op LP exprlist RP", + /* 209 */ "expr ::= LP select RP", + /* 210 */ "expr ::= expr in_op LP select RP", + /* 211 */ "expr ::= expr in_op nm dbnm paren_exprlist", + /* 212 */ "expr ::= EXISTS LP select RP", + /* 213 */ "expr ::= CASE case_operand case_exprlist case_else END", + /* 214 */ "case_exprlist ::= case_exprlist WHEN expr THEN expr", + /* 215 */ "case_exprlist ::= WHEN expr THEN expr", + /* 216 */ "case_else ::= ELSE expr", + /* 217 */ "case_else ::=", + /* 218 */ "case_operand ::= expr", + /* 219 */ "case_operand ::=", + /* 220 */ "exprlist ::=", + /* 221 */ "nexprlist ::= nexprlist COMMA expr", + /* 222 */ "nexprlist ::= expr", + /* 223 */ "paren_exprlist ::=", + /* 224 */ "paren_exprlist ::= LP exprlist RP", + /* 225 */ "cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt", + /* 226 */ "uniqueflag ::= UNIQUE", + /* 227 */ "uniqueflag ::=", + /* 228 */ "eidlist_opt ::=", + /* 229 */ "eidlist_opt ::= LP eidlist RP", + /* 230 */ "eidlist ::= eidlist COMMA nm collate sortorder", + /* 231 */ "eidlist ::= nm collate sortorder", + /* 232 */ "collate ::=", + /* 233 */ "collate ::= COLLATE ID|STRING", + /* 234 */ "cmd ::= DROP INDEX ifexists fullname", + /* 235 */ "cmd ::= VACUUM vinto", + /* 236 */ "cmd ::= VACUUM nm vinto", + /* 237 */ "vinto ::= INTO expr", + /* 238 */ "vinto ::=", + /* 239 */ "cmd ::= PRAGMA nm dbnm", + /* 240 */ "cmd ::= PRAGMA nm dbnm EQ nmnum", + /* 241 */ "cmd ::= PRAGMA nm dbnm LP nmnum RP", + /* 242 */ "cmd ::= PRAGMA nm dbnm EQ minus_num", + /* 243 */ "cmd ::= PRAGMA nm dbnm LP minus_num RP", + /* 244 */ "plus_num ::= PLUS INTEGER|FLOAT", + /* 245 */ "minus_num ::= MINUS INTEGER|FLOAT", + /* 246 */ "cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END", + /* 247 */ "trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause", + /* 248 */ "trigger_time ::= BEFORE|AFTER", + /* 249 */ "trigger_time ::= INSTEAD OF", + /* 250 */ "trigger_time ::=", + /* 251 */ "trigger_event ::= DELETE|INSERT", + /* 252 */ "trigger_event ::= UPDATE", + /* 253 */ "trigger_event ::= UPDATE OF idlist", + /* 254 */ "when_clause ::=", + /* 255 */ "when_clause ::= WHEN expr", + /* 256 */ "trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI", + /* 257 */ "trigger_cmd_list ::= trigger_cmd SEMI", + /* 258 */ "trnm ::= nm DOT nm", + /* 259 */ "tridxby ::= INDEXED BY nm", + /* 260 */ "tridxby ::= NOT INDEXED", + /* 261 */ "trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist where_opt scanpt", + /* 262 */ "trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt", + /* 263 */ "trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt", + /* 264 */ "trigger_cmd ::= scanpt select scanpt", + /* 265 */ "expr ::= RAISE LP IGNORE RP", + /* 266 */ "expr ::= RAISE LP raisetype COMMA nm RP", + /* 267 */ "raisetype ::= ROLLBACK", + /* 268 */ "raisetype ::= ABORT", + /* 269 */ "raisetype ::= FAIL", + /* 270 */ "cmd ::= DROP TRIGGER ifexists fullname", + /* 271 */ "cmd ::= ATTACH database_kw_opt expr AS expr key_opt", + /* 272 */ "cmd ::= DETACH database_kw_opt expr", + /* 273 */ "key_opt ::=", + /* 274 */ "key_opt ::= KEY expr", + /* 275 */ "cmd ::= REINDEX", + /* 276 */ "cmd ::= REINDEX nm dbnm", + /* 277 */ "cmd ::= ANALYZE", + /* 278 */ "cmd ::= ANALYZE nm dbnm", + /* 279 */ "cmd ::= ALTER TABLE fullname RENAME TO nm", + /* 280 */ "cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist", + /* 281 */ "add_column_fullname ::= fullname", + /* 282 */ "cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm", + /* 283 */ "cmd ::= create_vtab", + /* 284 */ "cmd ::= create_vtab LP vtabarglist RP", + /* 285 */ "create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm", + /* 286 */ "vtabarg ::=", + /* 287 */ "vtabargtoken ::= ANY", + /* 288 */ "vtabargtoken ::= lp anylist RP", + /* 289 */ "lp ::= LP", + /* 290 */ "with ::= WITH wqlist", + /* 291 */ "with ::= WITH RECURSIVE wqlist", + /* 292 */ "wqlist ::= nm eidlist_opt AS LP select RP", + /* 293 */ "wqlist ::= wqlist COMMA nm eidlist_opt AS LP select RP", + /* 294 */ "windowdefn_list ::= windowdefn", + /* 295 */ "windowdefn_list ::= windowdefn_list COMMA windowdefn", + /* 296 */ "windowdefn ::= nm AS LP window RP", + /* 297 */ "window ::= PARTITION BY nexprlist orderby_opt frame_opt", + /* 298 */ "window ::= nm PARTITION BY nexprlist orderby_opt frame_opt", + /* 299 */ "window ::= ORDER BY sortlist frame_opt", + /* 300 */ "window ::= nm ORDER BY sortlist frame_opt", + /* 301 */ "window ::= frame_opt", + /* 302 */ "window ::= nm frame_opt", + /* 303 */ "frame_opt ::=", + /* 304 */ "frame_opt ::= range_or_rows frame_bound_s frame_exclude_opt", + /* 305 */ "frame_opt ::= range_or_rows BETWEEN frame_bound_s AND frame_bound_e frame_exclude_opt", + /* 306 */ "range_or_rows ::= RANGE|ROWS|GROUPS", + /* 307 */ "frame_bound_s ::= frame_bound", + /* 308 */ "frame_bound_s ::= UNBOUNDED PRECEDING", + /* 309 */ "frame_bound_e ::= frame_bound", + /* 310 */ "frame_bound_e ::= UNBOUNDED FOLLOWING", + /* 311 */ "frame_bound ::= expr PRECEDING|FOLLOWING", + /* 312 */ "frame_bound ::= CURRENT ROW", + /* 313 */ "frame_exclude_opt ::=", + /* 314 */ "frame_exclude_opt ::= EXCLUDE frame_exclude", + /* 315 */ "frame_exclude ::= NO OTHERS", + /* 316 */ "frame_exclude ::= CURRENT ROW", + /* 317 */ "frame_exclude ::= GROUP|TIES", + /* 318 */ "window_clause ::= WINDOW windowdefn_list", + /* 319 */ "filter_over ::= filter_clause over_clause", + /* 320 */ "filter_over ::= over_clause", + /* 321 */ "filter_over ::= filter_clause", + /* 322 */ "over_clause ::= OVER LP window RP", + /* 323 */ "over_clause ::= OVER nm", + /* 324 */ "filter_clause ::= FILTER LP WHERE expr RP", + /* 325 */ "input ::= cmdlist", + /* 326 */ "cmdlist ::= cmdlist ecmd", + /* 327 */ "cmdlist ::= ecmd", + /* 328 */ "ecmd ::= SEMI", + /* 329 */ "ecmd ::= cmdx SEMI", + /* 330 */ "ecmd ::= explain cmdx SEMI", + /* 331 */ "trans_opt ::=", + /* 332 */ "trans_opt ::= TRANSACTION", + /* 333 */ "trans_opt ::= TRANSACTION nm", + /* 334 */ "savepoint_opt ::= SAVEPOINT", + /* 335 */ "savepoint_opt ::=", + /* 336 */ "cmd ::= create_table create_table_args", + /* 337 */ "columnlist ::= columnlist COMMA columnname carglist", + /* 338 */ "columnlist ::= columnname carglist", + /* 339 */ "nm ::= ID|INDEXED", + /* 340 */ "nm ::= STRING", + /* 341 */ "nm ::= JOIN_KW", + /* 342 */ "typetoken ::= typename", + /* 343 */ "typename ::= ID|STRING", + /* 344 */ "signed ::= plus_num", + /* 345 */ "signed ::= minus_num", + /* 346 */ "carglist ::= carglist ccons", + /* 347 */ "carglist ::=", + /* 348 */ "ccons ::= NULL onconf", + /* 349 */ "ccons ::= GENERATED ALWAYS AS generated", + /* 350 */ "ccons ::= AS generated", + /* 351 */ "conslist_opt ::= COMMA conslist", + /* 352 */ "conslist ::= conslist tconscomma tcons", + /* 353 */ "conslist ::= tcons", + /* 354 */ "tconscomma ::=", + /* 355 */ "defer_subclause_opt ::= defer_subclause", + /* 356 */ "resolvetype ::= raisetype", + /* 357 */ "selectnowith ::= oneselect", + /* 358 */ "oneselect ::= values", + /* 359 */ "sclp ::= selcollist COMMA", + /* 360 */ "as ::= ID|STRING", + /* 361 */ "expr ::= term", + /* 362 */ "likeop ::= LIKE_KW|MATCH", + /* 363 */ "exprlist ::= nexprlist", + /* 364 */ "nmnum ::= plus_num", + /* 365 */ "nmnum ::= nm", + /* 366 */ "nmnum ::= ON", + /* 367 */ "nmnum ::= DELETE", + /* 368 */ "nmnum ::= DEFAULT", + /* 369 */ "plus_num ::= INTEGER|FLOAT", + /* 370 */ "foreach_clause ::=", + /* 371 */ "foreach_clause ::= FOR EACH ROW", + /* 372 */ "trnm ::= nm", + /* 373 */ "tridxby ::=", + /* 374 */ "database_kw_opt ::= DATABASE", + /* 375 */ "database_kw_opt ::=", + /* 376 */ "kwcolumn_opt ::=", + /* 377 */ "kwcolumn_opt ::= COLUMNKW", + /* 378 */ "vtabarglist ::= vtabarg", + /* 379 */ "vtabarglist ::= vtabarglist COMMA vtabarg", + /* 380 */ "vtabarg ::= vtabarg vtabargtoken", + /* 381 */ "anylist ::=", + /* 382 */ "anylist ::= anylist LP anylist RP", + /* 383 */ "anylist ::= anylist ANY", + /* 384 */ "with ::=", +}; +#endif /* NDEBUG */ + + +#if YYSTACKDEPTH<=0 +/* +** Try to increase the size of the parser stack. Return the number +** of errors. Return 0 on success. +*/ +static int yyGrowStack(yyParser *p){ + int newSize; + int idx; + yyStackEntry *pNew; + + newSize = p->yystksz*2 + 100; + idx = p->yytos ? (int)(p->yytos - p->yystack) : 0; + if( p->yystack==&p->yystk0 ){ + pNew = malloc(newSize*sizeof(pNew[0])); + if( pNew ) pNew[0] = p->yystk0; + }else{ + pNew = realloc(p->yystack, newSize*sizeof(pNew[0])); + } + if( pNew ){ + p->yystack = pNew; + p->yytos = &p->yystack[idx]; +#ifndef NDEBUG + if( yyTraceFILE ){ + fprintf(yyTraceFILE,"%sStack grows from %d to %d entries.\n", + yyTracePrompt, p->yystksz, newSize); + } +#endif + p->yystksz = newSize; + } + return pNew==0; +} +#endif + +/* Datatype of the argument to the memory allocated passed as the +** second argument to sqlite3ParserAlloc() below. This can be changed by +** putting an appropriate #define in the %include section of the input +** grammar. +*/ +#ifndef YYMALLOCARGTYPE +# define YYMALLOCARGTYPE size_t +#endif + +/* Initialize a new parser that has already been allocated. +*/ +SQLITE_PRIVATE void sqlite3ParserInit(void *yypRawParser sqlite3ParserCTX_PDECL){ + yyParser *yypParser = (yyParser*)yypRawParser; + sqlite3ParserCTX_STORE +#ifdef YYTRACKMAXSTACKDEPTH + yypParser->yyhwm = 0; +#endif +#if YYSTACKDEPTH<=0 + yypParser->yytos = NULL; + yypParser->yystack = NULL; + yypParser->yystksz = 0; + if( yyGrowStack(yypParser) ){ + yypParser->yystack = &yypParser->yystk0; + yypParser->yystksz = 1; + } +#endif +#ifndef YYNOERRORRECOVERY + yypParser->yyerrcnt = -1; +#endif + yypParser->yytos = yypParser->yystack; + yypParser->yystack[0].stateno = 0; + yypParser->yystack[0].major = 0; +#if YYSTACKDEPTH>0 + yypParser->yystackEnd = &yypParser->yystack[YYSTACKDEPTH-1]; +#endif +} + +#ifndef sqlite3Parser_ENGINEALWAYSONSTACK +/* +** This function allocates a new parser. +** The only argument is a pointer to a function which works like +** malloc. +** +** Inputs: +** A pointer to the function used to allocate memory. +** +** Outputs: +** A pointer to a parser. This pointer is used in subsequent calls +** to sqlite3Parser and sqlite3ParserFree. +*/ +SQLITE_PRIVATE void *sqlite3ParserAlloc(void *(*mallocProc)(YYMALLOCARGTYPE) sqlite3ParserCTX_PDECL){ + yyParser *yypParser; + yypParser = (yyParser*)(*mallocProc)( (YYMALLOCARGTYPE)sizeof(yyParser) ); + if( yypParser ){ + sqlite3ParserCTX_STORE + sqlite3ParserInit(yypParser sqlite3ParserCTX_PARAM); + } + return (void*)yypParser; +} +#endif /* sqlite3Parser_ENGINEALWAYSONSTACK */ + + +/* The following function deletes the "minor type" or semantic value +** associated with a symbol. The symbol can be either a terminal +** or nonterminal. "yymajor" is the symbol code, and "yypminor" is +** a pointer to the value to be deleted. The code used to do the +** deletions is derived from the %destructor and/or %token_destructor +** directives of the input grammar. +*/ +static void yy_destructor( + yyParser *yypParser, /* The parser */ + YYCODETYPE yymajor, /* Type code for object to destroy */ + YYMINORTYPE *yypminor /* The object to be destroyed */ +){ + sqlite3ParserARG_FETCH + sqlite3ParserCTX_FETCH + switch( yymajor ){ + /* Here is inserted the actions which take place when a + ** terminal or non-terminal is destroyed. This can happen + ** when the symbol is popped from the stack during a + ** reduce or during error processing or when a parser is + ** being destroyed before it is finished parsing. + ** + ** Note: during a reduce, the only symbols destroyed are those + ** which appear on the RHS of the rule, but which are *not* used + ** inside the C code. + */ +/********* Begin destructor definitions ***************************************/ + case 200: /* select */ + case 234: /* selectnowith */ + case 235: /* oneselect */ + case 247: /* values */ +{ +sqlite3SelectDelete(pParse->db, (yypminor->yy539)); +} + break; + case 211: /* term */ + case 212: /* expr */ + case 241: /* where_opt */ + case 243: /* having_opt */ + case 255: /* on_opt */ + case 271: /* case_operand */ + case 273: /* case_else */ + case 276: /* vinto */ + case 283: /* when_clause */ + case 288: /* key_opt */ + case 302: /* filter_clause */ +{ +sqlite3ExprDelete(pParse->db, (yypminor->yy202)); +} + break; + case 216: /* eidlist_opt */ + case 226: /* sortlist */ + case 227: /* eidlist */ + case 239: /* selcollist */ + case 242: /* groupby_opt */ + case 244: /* orderby_opt */ + case 248: /* nexprlist */ + case 249: /* sclp */ + case 257: /* exprlist */ + case 262: /* setlist */ + case 270: /* paren_exprlist */ + case 272: /* case_exprlist */ + case 301: /* part_opt */ +{ +sqlite3ExprListDelete(pParse->db, (yypminor->yy242)); +} + break; + case 233: /* fullname */ + case 240: /* from */ + case 251: /* seltablist */ + case 252: /* stl_prefix */ + case 258: /* xfullname */ +{ +sqlite3SrcListDelete(pParse->db, (yypminor->yy47)); +} + break; + case 236: /* wqlist */ +{ +sqlite3WithDelete(pParse->db, (yypminor->yy131)); +} + break; + case 246: /* window_clause */ + case 297: /* windowdefn_list */ +{ +sqlite3WindowListDelete(pParse->db, (yypminor->yy303)); +} + break; + case 256: /* using_opt */ + case 259: /* idlist */ + case 264: /* idlist_opt */ +{ +sqlite3IdListDelete(pParse->db, (yypminor->yy600)); +} + break; + case 266: /* filter_over */ + case 298: /* windowdefn */ + case 299: /* window */ + case 300: /* frame_opt */ + case 303: /* over_clause */ +{ +sqlite3WindowDelete(pParse->db, (yypminor->yy303)); +} + break; + case 279: /* trigger_cmd_list */ + case 284: /* trigger_cmd */ +{ +sqlite3DeleteTriggerStep(pParse->db, (yypminor->yy447)); +} + break; + case 281: /* trigger_event */ +{ +sqlite3IdListDelete(pParse->db, (yypminor->yy230).b); +} + break; + case 305: /* frame_bound */ + case 306: /* frame_bound_s */ + case 307: /* frame_bound_e */ +{ +sqlite3ExprDelete(pParse->db, (yypminor->yy77).pExpr); +} + break; +/********* End destructor definitions *****************************************/ + default: break; /* If no destructor action specified: do nothing */ + } +} + +/* +** Pop the parser's stack once. +** +** If there is a destructor routine associated with the token which +** is popped from the stack, then call it. +*/ +static void yy_pop_parser_stack(yyParser *pParser){ + yyStackEntry *yytos; + assert( pParser->yytos!=0 ); + assert( pParser->yytos > pParser->yystack ); + yytos = pParser->yytos--; +#ifndef NDEBUG + if( yyTraceFILE ){ + fprintf(yyTraceFILE,"%sPopping %s\n", + yyTracePrompt, + yyTokenName[yytos->major]); + } +#endif + yy_destructor(pParser, yytos->major, &yytos->minor); +} + +/* +** Clear all secondary memory allocations from the parser +*/ +SQLITE_PRIVATE void sqlite3ParserFinalize(void *p){ + yyParser *pParser = (yyParser*)p; + while( pParser->yytos>pParser->yystack ) yy_pop_parser_stack(pParser); +#if YYSTACKDEPTH<=0 + if( pParser->yystack!=&pParser->yystk0 ) free(pParser->yystack); +#endif +} + +#ifndef sqlite3Parser_ENGINEALWAYSONSTACK +/* +** Deallocate and destroy a parser. Destructors are called for +** all stack elements before shutting the parser down. +** +** If the YYPARSEFREENEVERNULL macro exists (for example because it +** is defined in a %include section of the input grammar) then it is +** assumed that the input pointer is never NULL. +*/ +SQLITE_PRIVATE void sqlite3ParserFree( + void *p, /* The parser to be deleted */ + void (*freeProc)(void*) /* Function used to reclaim memory */ +){ +#ifndef YYPARSEFREENEVERNULL + if( p==0 ) return; +#endif + sqlite3ParserFinalize(p); + (*freeProc)(p); +} +#endif /* sqlite3Parser_ENGINEALWAYSONSTACK */ + +/* +** Return the peak depth of the stack for a parser. +*/ +#ifdef YYTRACKMAXSTACKDEPTH +SQLITE_PRIVATE int sqlite3ParserStackPeak(void *p){ + yyParser *pParser = (yyParser*)p; + return pParser->yyhwm; +} +#endif + +/* This array of booleans keeps track of the parser statement +** coverage. The element yycoverage[X][Y] is set when the parser +** is in state X and has a lookahead token Y. In a well-tested +** systems, every element of this matrix should end up being set. +*/ +#if defined(YYCOVERAGE) +static unsigned char yycoverage[YYNSTATE][YYNTOKEN]; +#endif + +/* +** Write into out a description of every state/lookahead combination that +** +** (1) has not been used by the parser, and +** (2) is not a syntax error. +** +** Return the number of missed state/lookahead combinations. +*/ +#if defined(YYCOVERAGE) +SQLITE_PRIVATE int sqlite3ParserCoverage(FILE *out){ + int stateno, iLookAhead, i; + int nMissed = 0; + for(stateno=0; statenoYY_MAX_SHIFT ) return stateno; + assert( stateno <= YY_SHIFT_COUNT ); +#if defined(YYCOVERAGE) + yycoverage[stateno][iLookAhead] = 1; +#endif + do{ + i = yy_shift_ofst[stateno]; + assert( i>=0 ); + assert( i<=YY_ACTTAB_COUNT ); + assert( i+YYNTOKEN<=(int)YY_NLOOKAHEAD ); + assert( iLookAhead!=YYNOCODE ); + assert( iLookAhead < YYNTOKEN ); + i += iLookAhead; + assert( i<(int)YY_NLOOKAHEAD ); + if( yy_lookahead[i]!=iLookAhead ){ +#ifdef YYFALLBACK + YYCODETYPE iFallback; /* Fallback token */ + assert( iLookAhead %s\n", + yyTracePrompt, yyTokenName[iLookAhead], yyTokenName[iFallback]); + } +#endif + assert( yyFallback[iFallback]==0 ); /* Fallback loop must terminate */ + iLookAhead = iFallback; + continue; + } +#endif +#ifdef YYWILDCARD + { + int j = i - iLookAhead + YYWILDCARD; + assert( j<(int)(sizeof(yy_lookahead)/sizeof(yy_lookahead[0])) ); + if( yy_lookahead[j]==YYWILDCARD && iLookAhead>0 ){ +#ifndef NDEBUG + if( yyTraceFILE ){ + fprintf(yyTraceFILE, "%sWILDCARD %s => %s\n", + yyTracePrompt, yyTokenName[iLookAhead], + yyTokenName[YYWILDCARD]); + } +#endif /* NDEBUG */ + return yy_action[j]; + } + } +#endif /* YYWILDCARD */ + return yy_default[stateno]; + }else{ + assert( i>=0 && iYY_REDUCE_COUNT ){ + return yy_default[stateno]; + } +#else + assert( stateno<=YY_REDUCE_COUNT ); +#endif + i = yy_reduce_ofst[stateno]; + assert( iLookAhead!=YYNOCODE ); + i += iLookAhead; +#ifdef YYERRORSYMBOL + if( i<0 || i>=YY_ACTTAB_COUNT || yy_lookahead[i]!=iLookAhead ){ + return yy_default[stateno]; + } +#else + assert( i>=0 && iyytos>yypParser->yystack ) yy_pop_parser_stack(yypParser); + /* Here code is inserted which will execute if the parser + ** stack every overflows */ +/******** Begin %stack_overflow code ******************************************/ + + sqlite3ErrorMsg(pParse, "parser stack overflow"); +/******** End %stack_overflow code ********************************************/ + sqlite3ParserARG_STORE /* Suppress warning about unused %extra_argument var */ + sqlite3ParserCTX_STORE +} + +/* +** Print tracing information for a SHIFT action +*/ +#ifndef NDEBUG +static void yyTraceShift(yyParser *yypParser, int yyNewState, const char *zTag){ + if( yyTraceFILE ){ + if( yyNewStateyytos->major], + yyNewState); + }else{ + fprintf(yyTraceFILE,"%s%s '%s', pending reduce %d\n", + yyTracePrompt, zTag, yyTokenName[yypParser->yytos->major], + yyNewState - YY_MIN_REDUCE); + } + } +} +#else +# define yyTraceShift(X,Y,Z) +#endif + +/* +** Perform a shift action. +*/ +static void yy_shift( + yyParser *yypParser, /* The parser to be shifted */ + YYACTIONTYPE yyNewState, /* The new state to shift in */ + YYCODETYPE yyMajor, /* The major token to shift in */ + sqlite3ParserTOKENTYPE yyMinor /* The minor token to shift in */ +){ + yyStackEntry *yytos; + yypParser->yytos++; +#ifdef YYTRACKMAXSTACKDEPTH + if( (int)(yypParser->yytos - yypParser->yystack)>yypParser->yyhwm ){ + yypParser->yyhwm++; + assert( yypParser->yyhwm == (int)(yypParser->yytos - yypParser->yystack) ); + } +#endif +#if YYSTACKDEPTH>0 + if( yypParser->yytos>yypParser->yystackEnd ){ + yypParser->yytos--; + yyStackOverflow(yypParser); + return; + } +#else + if( yypParser->yytos>=&yypParser->yystack[yypParser->yystksz] ){ + if( yyGrowStack(yypParser) ){ + yypParser->yytos--; + yyStackOverflow(yypParser); + return; + } + } +#endif + if( yyNewState > YY_MAX_SHIFT ){ + yyNewState += YY_MIN_REDUCE - YY_MIN_SHIFTREDUCE; + } + yytos = yypParser->yytos; + yytos->stateno = yyNewState; + yytos->major = yyMajor; + yytos->minor.yy0 = yyMinor; + yyTraceShift(yypParser, yyNewState, "Shift"); +} + +/* For rule J, yyRuleInfoLhs[J] contains the symbol on the left-hand side +** of that rule */ +static const YYCODETYPE yyRuleInfoLhs[] = { + 185, /* (0) explain ::= EXPLAIN */ + 185, /* (1) explain ::= EXPLAIN QUERY PLAN */ + 184, /* (2) cmdx ::= cmd */ + 186, /* (3) cmd ::= BEGIN transtype trans_opt */ + 187, /* (4) transtype ::= */ + 187, /* (5) transtype ::= DEFERRED */ + 187, /* (6) transtype ::= IMMEDIATE */ + 187, /* (7) transtype ::= EXCLUSIVE */ + 186, /* (8) cmd ::= COMMIT|END trans_opt */ + 186, /* (9) cmd ::= ROLLBACK trans_opt */ + 186, /* (10) cmd ::= SAVEPOINT nm */ + 186, /* (11) cmd ::= RELEASE savepoint_opt nm */ + 186, /* (12) cmd ::= ROLLBACK trans_opt TO savepoint_opt nm */ + 191, /* (13) create_table ::= createkw temp TABLE ifnotexists nm dbnm */ + 193, /* (14) createkw ::= CREATE */ + 195, /* (15) ifnotexists ::= */ + 195, /* (16) ifnotexists ::= IF NOT EXISTS */ + 194, /* (17) temp ::= TEMP */ + 194, /* (18) temp ::= */ + 192, /* (19) create_table_args ::= LP columnlist conslist_opt RP table_options */ + 192, /* (20) create_table_args ::= AS select */ + 199, /* (21) table_options ::= */ + 199, /* (22) table_options ::= WITHOUT nm */ + 201, /* (23) columnname ::= nm typetoken */ + 203, /* (24) typetoken ::= */ + 203, /* (25) typetoken ::= typename LP signed RP */ + 203, /* (26) typetoken ::= typename LP signed COMMA signed RP */ + 204, /* (27) typename ::= typename ID|STRING */ + 208, /* (28) scanpt ::= */ + 209, /* (29) scantok ::= */ + 210, /* (30) ccons ::= CONSTRAINT nm */ + 210, /* (31) ccons ::= DEFAULT scantok term */ + 210, /* (32) ccons ::= DEFAULT LP expr RP */ + 210, /* (33) ccons ::= DEFAULT PLUS scantok term */ + 210, /* (34) ccons ::= DEFAULT MINUS scantok term */ + 210, /* (35) ccons ::= DEFAULT scantok ID|INDEXED */ + 210, /* (36) ccons ::= NOT NULL onconf */ + 210, /* (37) ccons ::= PRIMARY KEY sortorder onconf autoinc */ + 210, /* (38) ccons ::= UNIQUE onconf */ + 210, /* (39) ccons ::= CHECK LP expr RP */ + 210, /* (40) ccons ::= REFERENCES nm eidlist_opt refargs */ + 210, /* (41) ccons ::= defer_subclause */ + 210, /* (42) ccons ::= COLLATE ID|STRING */ + 219, /* (43) generated ::= LP expr RP */ + 219, /* (44) generated ::= LP expr RP ID */ + 215, /* (45) autoinc ::= */ + 215, /* (46) autoinc ::= AUTOINCR */ + 217, /* (47) refargs ::= */ + 217, /* (48) refargs ::= refargs refarg */ + 220, /* (49) refarg ::= MATCH nm */ + 220, /* (50) refarg ::= ON INSERT refact */ + 220, /* (51) refarg ::= ON DELETE refact */ + 220, /* (52) refarg ::= ON UPDATE refact */ + 221, /* (53) refact ::= SET NULL */ + 221, /* (54) refact ::= SET DEFAULT */ + 221, /* (55) refact ::= CASCADE */ + 221, /* (56) refact ::= RESTRICT */ + 221, /* (57) refact ::= NO ACTION */ + 218, /* (58) defer_subclause ::= NOT DEFERRABLE init_deferred_pred_opt */ + 218, /* (59) defer_subclause ::= DEFERRABLE init_deferred_pred_opt */ + 222, /* (60) init_deferred_pred_opt ::= */ + 222, /* (61) init_deferred_pred_opt ::= INITIALLY DEFERRED */ + 222, /* (62) init_deferred_pred_opt ::= INITIALLY IMMEDIATE */ + 198, /* (63) conslist_opt ::= */ + 224, /* (64) tconscomma ::= COMMA */ + 225, /* (65) tcons ::= CONSTRAINT nm */ + 225, /* (66) tcons ::= PRIMARY KEY LP sortlist autoinc RP onconf */ + 225, /* (67) tcons ::= UNIQUE LP sortlist RP onconf */ + 225, /* (68) tcons ::= CHECK LP expr RP onconf */ + 225, /* (69) tcons ::= FOREIGN KEY LP eidlist RP REFERENCES nm eidlist_opt refargs defer_subclause_opt */ + 228, /* (70) defer_subclause_opt ::= */ + 213, /* (71) onconf ::= */ + 213, /* (72) onconf ::= ON CONFLICT resolvetype */ + 229, /* (73) orconf ::= */ + 229, /* (74) orconf ::= OR resolvetype */ + 230, /* (75) resolvetype ::= IGNORE */ + 230, /* (76) resolvetype ::= REPLACE */ + 186, /* (77) cmd ::= DROP TABLE ifexists fullname */ + 232, /* (78) ifexists ::= IF EXISTS */ + 232, /* (79) ifexists ::= */ + 186, /* (80) cmd ::= createkw temp VIEW ifnotexists nm dbnm eidlist_opt AS select */ + 186, /* (81) cmd ::= DROP VIEW ifexists fullname */ + 186, /* (82) cmd ::= select */ + 200, /* (83) select ::= WITH wqlist selectnowith */ + 200, /* (84) select ::= WITH RECURSIVE wqlist selectnowith */ + 200, /* (85) select ::= selectnowith */ + 234, /* (86) selectnowith ::= selectnowith multiselect_op oneselect */ + 237, /* (87) multiselect_op ::= UNION */ + 237, /* (88) multiselect_op ::= UNION ALL */ + 237, /* (89) multiselect_op ::= EXCEPT|INTERSECT */ + 235, /* (90) oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt orderby_opt limit_opt */ + 235, /* (91) oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt window_clause orderby_opt limit_opt */ + 247, /* (92) values ::= VALUES LP nexprlist RP */ + 247, /* (93) values ::= values COMMA LP nexprlist RP */ + 238, /* (94) distinct ::= DISTINCT */ + 238, /* (95) distinct ::= ALL */ + 238, /* (96) distinct ::= */ + 249, /* (97) sclp ::= */ + 239, /* (98) selcollist ::= sclp scanpt expr scanpt as */ + 239, /* (99) selcollist ::= sclp scanpt STAR */ + 239, /* (100) selcollist ::= sclp scanpt nm DOT STAR */ + 250, /* (101) as ::= AS nm */ + 250, /* (102) as ::= */ + 240, /* (103) from ::= */ + 240, /* (104) from ::= FROM seltablist */ + 252, /* (105) stl_prefix ::= seltablist joinop */ + 252, /* (106) stl_prefix ::= */ + 251, /* (107) seltablist ::= stl_prefix nm dbnm as indexed_opt on_opt using_opt */ + 251, /* (108) seltablist ::= stl_prefix nm dbnm LP exprlist RP as on_opt using_opt */ + 251, /* (109) seltablist ::= stl_prefix LP select RP as on_opt using_opt */ + 251, /* (110) seltablist ::= stl_prefix LP seltablist RP as on_opt using_opt */ + 196, /* (111) dbnm ::= */ + 196, /* (112) dbnm ::= DOT nm */ + 233, /* (113) fullname ::= nm */ + 233, /* (114) fullname ::= nm DOT nm */ + 258, /* (115) xfullname ::= nm */ + 258, /* (116) xfullname ::= nm DOT nm */ + 258, /* (117) xfullname ::= nm DOT nm AS nm */ + 258, /* (118) xfullname ::= nm AS nm */ + 253, /* (119) joinop ::= COMMA|JOIN */ + 253, /* (120) joinop ::= JOIN_KW JOIN */ + 253, /* (121) joinop ::= JOIN_KW nm JOIN */ + 253, /* (122) joinop ::= JOIN_KW nm nm JOIN */ + 255, /* (123) on_opt ::= ON expr */ + 255, /* (124) on_opt ::= */ + 254, /* (125) indexed_opt ::= */ + 254, /* (126) indexed_opt ::= INDEXED BY nm */ + 254, /* (127) indexed_opt ::= NOT INDEXED */ + 256, /* (128) using_opt ::= USING LP idlist RP */ + 256, /* (129) using_opt ::= */ + 244, /* (130) orderby_opt ::= */ + 244, /* (131) orderby_opt ::= ORDER BY sortlist */ + 226, /* (132) sortlist ::= sortlist COMMA expr sortorder nulls */ + 226, /* (133) sortlist ::= expr sortorder nulls */ + 214, /* (134) sortorder ::= ASC */ + 214, /* (135) sortorder ::= DESC */ + 214, /* (136) sortorder ::= */ + 260, /* (137) nulls ::= NULLS FIRST */ + 260, /* (138) nulls ::= NULLS LAST */ + 260, /* (139) nulls ::= */ + 242, /* (140) groupby_opt ::= */ + 242, /* (141) groupby_opt ::= GROUP BY nexprlist */ + 243, /* (142) having_opt ::= */ + 243, /* (143) having_opt ::= HAVING expr */ + 245, /* (144) limit_opt ::= */ + 245, /* (145) limit_opt ::= LIMIT expr */ + 245, /* (146) limit_opt ::= LIMIT expr OFFSET expr */ + 245, /* (147) limit_opt ::= LIMIT expr COMMA expr */ + 186, /* (148) cmd ::= with DELETE FROM xfullname indexed_opt where_opt */ + 241, /* (149) where_opt ::= */ + 241, /* (150) where_opt ::= WHERE expr */ + 186, /* (151) cmd ::= with UPDATE orconf xfullname indexed_opt SET setlist where_opt */ + 262, /* (152) setlist ::= setlist COMMA nm EQ expr */ + 262, /* (153) setlist ::= setlist COMMA LP idlist RP EQ expr */ + 262, /* (154) setlist ::= nm EQ expr */ + 262, /* (155) setlist ::= LP idlist RP EQ expr */ + 186, /* (156) cmd ::= with insert_cmd INTO xfullname idlist_opt select upsert */ + 186, /* (157) cmd ::= with insert_cmd INTO xfullname idlist_opt DEFAULT VALUES */ + 265, /* (158) upsert ::= */ + 265, /* (159) upsert ::= ON CONFLICT LP sortlist RP where_opt DO UPDATE SET setlist where_opt */ + 265, /* (160) upsert ::= ON CONFLICT LP sortlist RP where_opt DO NOTHING */ + 265, /* (161) upsert ::= ON CONFLICT DO NOTHING */ + 263, /* (162) insert_cmd ::= INSERT orconf */ + 263, /* (163) insert_cmd ::= REPLACE */ + 264, /* (164) idlist_opt ::= */ + 264, /* (165) idlist_opt ::= LP idlist RP */ + 259, /* (166) idlist ::= idlist COMMA nm */ + 259, /* (167) idlist ::= nm */ + 212, /* (168) expr ::= LP expr RP */ + 212, /* (169) expr ::= ID|INDEXED */ + 212, /* (170) expr ::= JOIN_KW */ + 212, /* (171) expr ::= nm DOT nm */ + 212, /* (172) expr ::= nm DOT nm DOT nm */ + 211, /* (173) term ::= NULL|FLOAT|BLOB */ + 211, /* (174) term ::= STRING */ + 211, /* (175) term ::= INTEGER */ + 212, /* (176) expr ::= VARIABLE */ + 212, /* (177) expr ::= expr COLLATE ID|STRING */ + 212, /* (178) expr ::= CAST LP expr AS typetoken RP */ + 212, /* (179) expr ::= ID|INDEXED LP distinct exprlist RP */ + 212, /* (180) expr ::= ID|INDEXED LP STAR RP */ + 212, /* (181) expr ::= ID|INDEXED LP distinct exprlist RP filter_over */ + 212, /* (182) expr ::= ID|INDEXED LP STAR RP filter_over */ + 211, /* (183) term ::= CTIME_KW */ + 212, /* (184) expr ::= LP nexprlist COMMA expr RP */ + 212, /* (185) expr ::= expr AND expr */ + 212, /* (186) expr ::= expr OR expr */ + 212, /* (187) expr ::= expr LT|GT|GE|LE expr */ + 212, /* (188) expr ::= expr EQ|NE expr */ + 212, /* (189) expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr */ + 212, /* (190) expr ::= expr PLUS|MINUS expr */ + 212, /* (191) expr ::= expr STAR|SLASH|REM expr */ + 212, /* (192) expr ::= expr CONCAT expr */ + 267, /* (193) likeop ::= NOT LIKE_KW|MATCH */ + 212, /* (194) expr ::= expr likeop expr */ + 212, /* (195) expr ::= expr likeop expr ESCAPE expr */ + 212, /* (196) expr ::= expr ISNULL|NOTNULL */ + 212, /* (197) expr ::= expr NOT NULL */ + 212, /* (198) expr ::= expr IS expr */ + 212, /* (199) expr ::= expr IS NOT expr */ + 212, /* (200) expr ::= NOT expr */ + 212, /* (201) expr ::= BITNOT expr */ + 212, /* (202) expr ::= PLUS|MINUS expr */ + 268, /* (203) between_op ::= BETWEEN */ + 268, /* (204) between_op ::= NOT BETWEEN */ + 212, /* (205) expr ::= expr between_op expr AND expr */ + 269, /* (206) in_op ::= IN */ + 269, /* (207) in_op ::= NOT IN */ + 212, /* (208) expr ::= expr in_op LP exprlist RP */ + 212, /* (209) expr ::= LP select RP */ + 212, /* (210) expr ::= expr in_op LP select RP */ + 212, /* (211) expr ::= expr in_op nm dbnm paren_exprlist */ + 212, /* (212) expr ::= EXISTS LP select RP */ + 212, /* (213) expr ::= CASE case_operand case_exprlist case_else END */ + 272, /* (214) case_exprlist ::= case_exprlist WHEN expr THEN expr */ + 272, /* (215) case_exprlist ::= WHEN expr THEN expr */ + 273, /* (216) case_else ::= ELSE expr */ + 273, /* (217) case_else ::= */ + 271, /* (218) case_operand ::= expr */ + 271, /* (219) case_operand ::= */ + 257, /* (220) exprlist ::= */ + 248, /* (221) nexprlist ::= nexprlist COMMA expr */ + 248, /* (222) nexprlist ::= expr */ + 270, /* (223) paren_exprlist ::= */ + 270, /* (224) paren_exprlist ::= LP exprlist RP */ + 186, /* (225) cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt */ + 274, /* (226) uniqueflag ::= UNIQUE */ + 274, /* (227) uniqueflag ::= */ + 216, /* (228) eidlist_opt ::= */ + 216, /* (229) eidlist_opt ::= LP eidlist RP */ + 227, /* (230) eidlist ::= eidlist COMMA nm collate sortorder */ + 227, /* (231) eidlist ::= nm collate sortorder */ + 275, /* (232) collate ::= */ + 275, /* (233) collate ::= COLLATE ID|STRING */ + 186, /* (234) cmd ::= DROP INDEX ifexists fullname */ + 186, /* (235) cmd ::= VACUUM vinto */ + 186, /* (236) cmd ::= VACUUM nm vinto */ + 276, /* (237) vinto ::= INTO expr */ + 276, /* (238) vinto ::= */ + 186, /* (239) cmd ::= PRAGMA nm dbnm */ + 186, /* (240) cmd ::= PRAGMA nm dbnm EQ nmnum */ + 186, /* (241) cmd ::= PRAGMA nm dbnm LP nmnum RP */ + 186, /* (242) cmd ::= PRAGMA nm dbnm EQ minus_num */ + 186, /* (243) cmd ::= PRAGMA nm dbnm LP minus_num RP */ + 206, /* (244) plus_num ::= PLUS INTEGER|FLOAT */ + 207, /* (245) minus_num ::= MINUS INTEGER|FLOAT */ + 186, /* (246) cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END */ + 278, /* (247) trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */ + 280, /* (248) trigger_time ::= BEFORE|AFTER */ + 280, /* (249) trigger_time ::= INSTEAD OF */ + 280, /* (250) trigger_time ::= */ + 281, /* (251) trigger_event ::= DELETE|INSERT */ + 281, /* (252) trigger_event ::= UPDATE */ + 281, /* (253) trigger_event ::= UPDATE OF idlist */ + 283, /* (254) when_clause ::= */ + 283, /* (255) when_clause ::= WHEN expr */ + 279, /* (256) trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */ + 279, /* (257) trigger_cmd_list ::= trigger_cmd SEMI */ + 285, /* (258) trnm ::= nm DOT nm */ + 286, /* (259) tridxby ::= INDEXED BY nm */ + 286, /* (260) tridxby ::= NOT INDEXED */ + 284, /* (261) trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist where_opt scanpt */ + 284, /* (262) trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt */ + 284, /* (263) trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt */ + 284, /* (264) trigger_cmd ::= scanpt select scanpt */ + 212, /* (265) expr ::= RAISE LP IGNORE RP */ + 212, /* (266) expr ::= RAISE LP raisetype COMMA nm RP */ + 231, /* (267) raisetype ::= ROLLBACK */ + 231, /* (268) raisetype ::= ABORT */ + 231, /* (269) raisetype ::= FAIL */ + 186, /* (270) cmd ::= DROP TRIGGER ifexists fullname */ + 186, /* (271) cmd ::= ATTACH database_kw_opt expr AS expr key_opt */ + 186, /* (272) cmd ::= DETACH database_kw_opt expr */ + 288, /* (273) key_opt ::= */ + 288, /* (274) key_opt ::= KEY expr */ + 186, /* (275) cmd ::= REINDEX */ + 186, /* (276) cmd ::= REINDEX nm dbnm */ + 186, /* (277) cmd ::= ANALYZE */ + 186, /* (278) cmd ::= ANALYZE nm dbnm */ + 186, /* (279) cmd ::= ALTER TABLE fullname RENAME TO nm */ + 186, /* (280) cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist */ + 289, /* (281) add_column_fullname ::= fullname */ + 186, /* (282) cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm */ + 186, /* (283) cmd ::= create_vtab */ + 186, /* (284) cmd ::= create_vtab LP vtabarglist RP */ + 291, /* (285) create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm */ + 293, /* (286) vtabarg ::= */ + 294, /* (287) vtabargtoken ::= ANY */ + 294, /* (288) vtabargtoken ::= lp anylist RP */ + 295, /* (289) lp ::= LP */ + 261, /* (290) with ::= WITH wqlist */ + 261, /* (291) with ::= WITH RECURSIVE wqlist */ + 236, /* (292) wqlist ::= nm eidlist_opt AS LP select RP */ + 236, /* (293) wqlist ::= wqlist COMMA nm eidlist_opt AS LP select RP */ + 297, /* (294) windowdefn_list ::= windowdefn */ + 297, /* (295) windowdefn_list ::= windowdefn_list COMMA windowdefn */ + 298, /* (296) windowdefn ::= nm AS LP window RP */ + 299, /* (297) window ::= PARTITION BY nexprlist orderby_opt frame_opt */ + 299, /* (298) window ::= nm PARTITION BY nexprlist orderby_opt frame_opt */ + 299, /* (299) window ::= ORDER BY sortlist frame_opt */ + 299, /* (300) window ::= nm ORDER BY sortlist frame_opt */ + 299, /* (301) window ::= frame_opt */ + 299, /* (302) window ::= nm frame_opt */ + 300, /* (303) frame_opt ::= */ + 300, /* (304) frame_opt ::= range_or_rows frame_bound_s frame_exclude_opt */ + 300, /* (305) frame_opt ::= range_or_rows BETWEEN frame_bound_s AND frame_bound_e frame_exclude_opt */ + 304, /* (306) range_or_rows ::= RANGE|ROWS|GROUPS */ + 306, /* (307) frame_bound_s ::= frame_bound */ + 306, /* (308) frame_bound_s ::= UNBOUNDED PRECEDING */ + 307, /* (309) frame_bound_e ::= frame_bound */ + 307, /* (310) frame_bound_e ::= UNBOUNDED FOLLOWING */ + 305, /* (311) frame_bound ::= expr PRECEDING|FOLLOWING */ + 305, /* (312) frame_bound ::= CURRENT ROW */ + 308, /* (313) frame_exclude_opt ::= */ + 308, /* (314) frame_exclude_opt ::= EXCLUDE frame_exclude */ + 309, /* (315) frame_exclude ::= NO OTHERS */ + 309, /* (316) frame_exclude ::= CURRENT ROW */ + 309, /* (317) frame_exclude ::= GROUP|TIES */ + 246, /* (318) window_clause ::= WINDOW windowdefn_list */ + 266, /* (319) filter_over ::= filter_clause over_clause */ + 266, /* (320) filter_over ::= over_clause */ + 266, /* (321) filter_over ::= filter_clause */ + 303, /* (322) over_clause ::= OVER LP window RP */ + 303, /* (323) over_clause ::= OVER nm */ + 302, /* (324) filter_clause ::= FILTER LP WHERE expr RP */ + 181, /* (325) input ::= cmdlist */ + 182, /* (326) cmdlist ::= cmdlist ecmd */ + 182, /* (327) cmdlist ::= ecmd */ + 183, /* (328) ecmd ::= SEMI */ + 183, /* (329) ecmd ::= cmdx SEMI */ + 183, /* (330) ecmd ::= explain cmdx SEMI */ + 188, /* (331) trans_opt ::= */ + 188, /* (332) trans_opt ::= TRANSACTION */ + 188, /* (333) trans_opt ::= TRANSACTION nm */ + 190, /* (334) savepoint_opt ::= SAVEPOINT */ + 190, /* (335) savepoint_opt ::= */ + 186, /* (336) cmd ::= create_table create_table_args */ + 197, /* (337) columnlist ::= columnlist COMMA columnname carglist */ + 197, /* (338) columnlist ::= columnname carglist */ + 189, /* (339) nm ::= ID|INDEXED */ + 189, /* (340) nm ::= STRING */ + 189, /* (341) nm ::= JOIN_KW */ + 203, /* (342) typetoken ::= typename */ + 204, /* (343) typename ::= ID|STRING */ + 205, /* (344) signed ::= plus_num */ + 205, /* (345) signed ::= minus_num */ + 202, /* (346) carglist ::= carglist ccons */ + 202, /* (347) carglist ::= */ + 210, /* (348) ccons ::= NULL onconf */ + 210, /* (349) ccons ::= GENERATED ALWAYS AS generated */ + 210, /* (350) ccons ::= AS generated */ + 198, /* (351) conslist_opt ::= COMMA conslist */ + 223, /* (352) conslist ::= conslist tconscomma tcons */ + 223, /* (353) conslist ::= tcons */ + 224, /* (354) tconscomma ::= */ + 228, /* (355) defer_subclause_opt ::= defer_subclause */ + 230, /* (356) resolvetype ::= raisetype */ + 234, /* (357) selectnowith ::= oneselect */ + 235, /* (358) oneselect ::= values */ + 249, /* (359) sclp ::= selcollist COMMA */ + 250, /* (360) as ::= ID|STRING */ + 212, /* (361) expr ::= term */ + 267, /* (362) likeop ::= LIKE_KW|MATCH */ + 257, /* (363) exprlist ::= nexprlist */ + 277, /* (364) nmnum ::= plus_num */ + 277, /* (365) nmnum ::= nm */ + 277, /* (366) nmnum ::= ON */ + 277, /* (367) nmnum ::= DELETE */ + 277, /* (368) nmnum ::= DEFAULT */ + 206, /* (369) plus_num ::= INTEGER|FLOAT */ + 282, /* (370) foreach_clause ::= */ + 282, /* (371) foreach_clause ::= FOR EACH ROW */ + 285, /* (372) trnm ::= nm */ + 286, /* (373) tridxby ::= */ + 287, /* (374) database_kw_opt ::= DATABASE */ + 287, /* (375) database_kw_opt ::= */ + 290, /* (376) kwcolumn_opt ::= */ + 290, /* (377) kwcolumn_opt ::= COLUMNKW */ + 292, /* (378) vtabarglist ::= vtabarg */ + 292, /* (379) vtabarglist ::= vtabarglist COMMA vtabarg */ + 293, /* (380) vtabarg ::= vtabarg vtabargtoken */ + 296, /* (381) anylist ::= */ + 296, /* (382) anylist ::= anylist LP anylist RP */ + 296, /* (383) anylist ::= anylist ANY */ + 261, /* (384) with ::= */ +}; + +/* For rule J, yyRuleInfoNRhs[J] contains the negative of the number +** of symbols on the right-hand side of that rule. */ +static const signed char yyRuleInfoNRhs[] = { + -1, /* (0) explain ::= EXPLAIN */ + -3, /* (1) explain ::= EXPLAIN QUERY PLAN */ + -1, /* (2) cmdx ::= cmd */ + -3, /* (3) cmd ::= BEGIN transtype trans_opt */ + 0, /* (4) transtype ::= */ + -1, /* (5) transtype ::= DEFERRED */ + -1, /* (6) transtype ::= IMMEDIATE */ + -1, /* (7) transtype ::= EXCLUSIVE */ + -2, /* (8) cmd ::= COMMIT|END trans_opt */ + -2, /* (9) cmd ::= ROLLBACK trans_opt */ + -2, /* (10) cmd ::= SAVEPOINT nm */ + -3, /* (11) cmd ::= RELEASE savepoint_opt nm */ + -5, /* (12) cmd ::= ROLLBACK trans_opt TO savepoint_opt nm */ + -6, /* (13) create_table ::= createkw temp TABLE ifnotexists nm dbnm */ + -1, /* (14) createkw ::= CREATE */ + 0, /* (15) ifnotexists ::= */ + -3, /* (16) ifnotexists ::= IF NOT EXISTS */ + -1, /* (17) temp ::= TEMP */ + 0, /* (18) temp ::= */ + -5, /* (19) create_table_args ::= LP columnlist conslist_opt RP table_options */ + -2, /* (20) create_table_args ::= AS select */ + 0, /* (21) table_options ::= */ + -2, /* (22) table_options ::= WITHOUT nm */ + -2, /* (23) columnname ::= nm typetoken */ + 0, /* (24) typetoken ::= */ + -4, /* (25) typetoken ::= typename LP signed RP */ + -6, /* (26) typetoken ::= typename LP signed COMMA signed RP */ + -2, /* (27) typename ::= typename ID|STRING */ + 0, /* (28) scanpt ::= */ + 0, /* (29) scantok ::= */ + -2, /* (30) ccons ::= CONSTRAINT nm */ + -3, /* (31) ccons ::= DEFAULT scantok term */ + -4, /* (32) ccons ::= DEFAULT LP expr RP */ + -4, /* (33) ccons ::= DEFAULT PLUS scantok term */ + -4, /* (34) ccons ::= DEFAULT MINUS scantok term */ + -3, /* (35) ccons ::= DEFAULT scantok ID|INDEXED */ + -3, /* (36) ccons ::= NOT NULL onconf */ + -5, /* (37) ccons ::= PRIMARY KEY sortorder onconf autoinc */ + -2, /* (38) ccons ::= UNIQUE onconf */ + -4, /* (39) ccons ::= CHECK LP expr RP */ + -4, /* (40) ccons ::= REFERENCES nm eidlist_opt refargs */ + -1, /* (41) ccons ::= defer_subclause */ + -2, /* (42) ccons ::= COLLATE ID|STRING */ + -3, /* (43) generated ::= LP expr RP */ + -4, /* (44) generated ::= LP expr RP ID */ + 0, /* (45) autoinc ::= */ + -1, /* (46) autoinc ::= AUTOINCR */ + 0, /* (47) refargs ::= */ + -2, /* (48) refargs ::= refargs refarg */ + -2, /* (49) refarg ::= MATCH nm */ + -3, /* (50) refarg ::= ON INSERT refact */ + -3, /* (51) refarg ::= ON DELETE refact */ + -3, /* (52) refarg ::= ON UPDATE refact */ + -2, /* (53) refact ::= SET NULL */ + -2, /* (54) refact ::= SET DEFAULT */ + -1, /* (55) refact ::= CASCADE */ + -1, /* (56) refact ::= RESTRICT */ + -2, /* (57) refact ::= NO ACTION */ + -3, /* (58) defer_subclause ::= NOT DEFERRABLE init_deferred_pred_opt */ + -2, /* (59) defer_subclause ::= DEFERRABLE init_deferred_pred_opt */ + 0, /* (60) init_deferred_pred_opt ::= */ + -2, /* (61) init_deferred_pred_opt ::= INITIALLY DEFERRED */ + -2, /* (62) init_deferred_pred_opt ::= INITIALLY IMMEDIATE */ + 0, /* (63) conslist_opt ::= */ + -1, /* (64) tconscomma ::= COMMA */ + -2, /* (65) tcons ::= CONSTRAINT nm */ + -7, /* (66) tcons ::= PRIMARY KEY LP sortlist autoinc RP onconf */ + -5, /* (67) tcons ::= UNIQUE LP sortlist RP onconf */ + -5, /* (68) tcons ::= CHECK LP expr RP onconf */ + -10, /* (69) tcons ::= FOREIGN KEY LP eidlist RP REFERENCES nm eidlist_opt refargs defer_subclause_opt */ + 0, /* (70) defer_subclause_opt ::= */ + 0, /* (71) onconf ::= */ + -3, /* (72) onconf ::= ON CONFLICT resolvetype */ + 0, /* (73) orconf ::= */ + -2, /* (74) orconf ::= OR resolvetype */ + -1, /* (75) resolvetype ::= IGNORE */ + -1, /* (76) resolvetype ::= REPLACE */ + -4, /* (77) cmd ::= DROP TABLE ifexists fullname */ + -2, /* (78) ifexists ::= IF EXISTS */ + 0, /* (79) ifexists ::= */ + -9, /* (80) cmd ::= createkw temp VIEW ifnotexists nm dbnm eidlist_opt AS select */ + -4, /* (81) cmd ::= DROP VIEW ifexists fullname */ + -1, /* (82) cmd ::= select */ + -3, /* (83) select ::= WITH wqlist selectnowith */ + -4, /* (84) select ::= WITH RECURSIVE wqlist selectnowith */ + -1, /* (85) select ::= selectnowith */ + -3, /* (86) selectnowith ::= selectnowith multiselect_op oneselect */ + -1, /* (87) multiselect_op ::= UNION */ + -2, /* (88) multiselect_op ::= UNION ALL */ + -1, /* (89) multiselect_op ::= EXCEPT|INTERSECT */ + -9, /* (90) oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt orderby_opt limit_opt */ + -10, /* (91) oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt window_clause orderby_opt limit_opt */ + -4, /* (92) values ::= VALUES LP nexprlist RP */ + -5, /* (93) values ::= values COMMA LP nexprlist RP */ + -1, /* (94) distinct ::= DISTINCT */ + -1, /* (95) distinct ::= ALL */ + 0, /* (96) distinct ::= */ + 0, /* (97) sclp ::= */ + -5, /* (98) selcollist ::= sclp scanpt expr scanpt as */ + -3, /* (99) selcollist ::= sclp scanpt STAR */ + -5, /* (100) selcollist ::= sclp scanpt nm DOT STAR */ + -2, /* (101) as ::= AS nm */ + 0, /* (102) as ::= */ + 0, /* (103) from ::= */ + -2, /* (104) from ::= FROM seltablist */ + -2, /* (105) stl_prefix ::= seltablist joinop */ + 0, /* (106) stl_prefix ::= */ + -7, /* (107) seltablist ::= stl_prefix nm dbnm as indexed_opt on_opt using_opt */ + -9, /* (108) seltablist ::= stl_prefix nm dbnm LP exprlist RP as on_opt using_opt */ + -7, /* (109) seltablist ::= stl_prefix LP select RP as on_opt using_opt */ + -7, /* (110) seltablist ::= stl_prefix LP seltablist RP as on_opt using_opt */ + 0, /* (111) dbnm ::= */ + -2, /* (112) dbnm ::= DOT nm */ + -1, /* (113) fullname ::= nm */ + -3, /* (114) fullname ::= nm DOT nm */ + -1, /* (115) xfullname ::= nm */ + -3, /* (116) xfullname ::= nm DOT nm */ + -5, /* (117) xfullname ::= nm DOT nm AS nm */ + -3, /* (118) xfullname ::= nm AS nm */ + -1, /* (119) joinop ::= COMMA|JOIN */ + -2, /* (120) joinop ::= JOIN_KW JOIN */ + -3, /* (121) joinop ::= JOIN_KW nm JOIN */ + -4, /* (122) joinop ::= JOIN_KW nm nm JOIN */ + -2, /* (123) on_opt ::= ON expr */ + 0, /* (124) on_opt ::= */ + 0, /* (125) indexed_opt ::= */ + -3, /* (126) indexed_opt ::= INDEXED BY nm */ + -2, /* (127) indexed_opt ::= NOT INDEXED */ + -4, /* (128) using_opt ::= USING LP idlist RP */ + 0, /* (129) using_opt ::= */ + 0, /* (130) orderby_opt ::= */ + -3, /* (131) orderby_opt ::= ORDER BY sortlist */ + -5, /* (132) sortlist ::= sortlist COMMA expr sortorder nulls */ + -3, /* (133) sortlist ::= expr sortorder nulls */ + -1, /* (134) sortorder ::= ASC */ + -1, /* (135) sortorder ::= DESC */ + 0, /* (136) sortorder ::= */ + -2, /* (137) nulls ::= NULLS FIRST */ + -2, /* (138) nulls ::= NULLS LAST */ + 0, /* (139) nulls ::= */ + 0, /* (140) groupby_opt ::= */ + -3, /* (141) groupby_opt ::= GROUP BY nexprlist */ + 0, /* (142) having_opt ::= */ + -2, /* (143) having_opt ::= HAVING expr */ + 0, /* (144) limit_opt ::= */ + -2, /* (145) limit_opt ::= LIMIT expr */ + -4, /* (146) limit_opt ::= LIMIT expr OFFSET expr */ + -4, /* (147) limit_opt ::= LIMIT expr COMMA expr */ + -6, /* (148) cmd ::= with DELETE FROM xfullname indexed_opt where_opt */ + 0, /* (149) where_opt ::= */ + -2, /* (150) where_opt ::= WHERE expr */ + -8, /* (151) cmd ::= with UPDATE orconf xfullname indexed_opt SET setlist where_opt */ + -5, /* (152) setlist ::= setlist COMMA nm EQ expr */ + -7, /* (153) setlist ::= setlist COMMA LP idlist RP EQ expr */ + -3, /* (154) setlist ::= nm EQ expr */ + -5, /* (155) setlist ::= LP idlist RP EQ expr */ + -7, /* (156) cmd ::= with insert_cmd INTO xfullname idlist_opt select upsert */ + -7, /* (157) cmd ::= with insert_cmd INTO xfullname idlist_opt DEFAULT VALUES */ + 0, /* (158) upsert ::= */ + -11, /* (159) upsert ::= ON CONFLICT LP sortlist RP where_opt DO UPDATE SET setlist where_opt */ + -8, /* (160) upsert ::= ON CONFLICT LP sortlist RP where_opt DO NOTHING */ + -4, /* (161) upsert ::= ON CONFLICT DO NOTHING */ + -2, /* (162) insert_cmd ::= INSERT orconf */ + -1, /* (163) insert_cmd ::= REPLACE */ + 0, /* (164) idlist_opt ::= */ + -3, /* (165) idlist_opt ::= LP idlist RP */ + -3, /* (166) idlist ::= idlist COMMA nm */ + -1, /* (167) idlist ::= nm */ + -3, /* (168) expr ::= LP expr RP */ + -1, /* (169) expr ::= ID|INDEXED */ + -1, /* (170) expr ::= JOIN_KW */ + -3, /* (171) expr ::= nm DOT nm */ + -5, /* (172) expr ::= nm DOT nm DOT nm */ + -1, /* (173) term ::= NULL|FLOAT|BLOB */ + -1, /* (174) term ::= STRING */ + -1, /* (175) term ::= INTEGER */ + -1, /* (176) expr ::= VARIABLE */ + -3, /* (177) expr ::= expr COLLATE ID|STRING */ + -6, /* (178) expr ::= CAST LP expr AS typetoken RP */ + -5, /* (179) expr ::= ID|INDEXED LP distinct exprlist RP */ + -4, /* (180) expr ::= ID|INDEXED LP STAR RP */ + -6, /* (181) expr ::= ID|INDEXED LP distinct exprlist RP filter_over */ + -5, /* (182) expr ::= ID|INDEXED LP STAR RP filter_over */ + -1, /* (183) term ::= CTIME_KW */ + -5, /* (184) expr ::= LP nexprlist COMMA expr RP */ + -3, /* (185) expr ::= expr AND expr */ + -3, /* (186) expr ::= expr OR expr */ + -3, /* (187) expr ::= expr LT|GT|GE|LE expr */ + -3, /* (188) expr ::= expr EQ|NE expr */ + -3, /* (189) expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr */ + -3, /* (190) expr ::= expr PLUS|MINUS expr */ + -3, /* (191) expr ::= expr STAR|SLASH|REM expr */ + -3, /* (192) expr ::= expr CONCAT expr */ + -2, /* (193) likeop ::= NOT LIKE_KW|MATCH */ + -3, /* (194) expr ::= expr likeop expr */ + -5, /* (195) expr ::= expr likeop expr ESCAPE expr */ + -2, /* (196) expr ::= expr ISNULL|NOTNULL */ + -3, /* (197) expr ::= expr NOT NULL */ + -3, /* (198) expr ::= expr IS expr */ + -4, /* (199) expr ::= expr IS NOT expr */ + -2, /* (200) expr ::= NOT expr */ + -2, /* (201) expr ::= BITNOT expr */ + -2, /* (202) expr ::= PLUS|MINUS expr */ + -1, /* (203) between_op ::= BETWEEN */ + -2, /* (204) between_op ::= NOT BETWEEN */ + -5, /* (205) expr ::= expr between_op expr AND expr */ + -1, /* (206) in_op ::= IN */ + -2, /* (207) in_op ::= NOT IN */ + -5, /* (208) expr ::= expr in_op LP exprlist RP */ + -3, /* (209) expr ::= LP select RP */ + -5, /* (210) expr ::= expr in_op LP select RP */ + -5, /* (211) expr ::= expr in_op nm dbnm paren_exprlist */ + -4, /* (212) expr ::= EXISTS LP select RP */ + -5, /* (213) expr ::= CASE case_operand case_exprlist case_else END */ + -5, /* (214) case_exprlist ::= case_exprlist WHEN expr THEN expr */ + -4, /* (215) case_exprlist ::= WHEN expr THEN expr */ + -2, /* (216) case_else ::= ELSE expr */ + 0, /* (217) case_else ::= */ + -1, /* (218) case_operand ::= expr */ + 0, /* (219) case_operand ::= */ + 0, /* (220) exprlist ::= */ + -3, /* (221) nexprlist ::= nexprlist COMMA expr */ + -1, /* (222) nexprlist ::= expr */ + 0, /* (223) paren_exprlist ::= */ + -3, /* (224) paren_exprlist ::= LP exprlist RP */ + -12, /* (225) cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt */ + -1, /* (226) uniqueflag ::= UNIQUE */ + 0, /* (227) uniqueflag ::= */ + 0, /* (228) eidlist_opt ::= */ + -3, /* (229) eidlist_opt ::= LP eidlist RP */ + -5, /* (230) eidlist ::= eidlist COMMA nm collate sortorder */ + -3, /* (231) eidlist ::= nm collate sortorder */ + 0, /* (232) collate ::= */ + -2, /* (233) collate ::= COLLATE ID|STRING */ + -4, /* (234) cmd ::= DROP INDEX ifexists fullname */ + -2, /* (235) cmd ::= VACUUM vinto */ + -3, /* (236) cmd ::= VACUUM nm vinto */ + -2, /* (237) vinto ::= INTO expr */ + 0, /* (238) vinto ::= */ + -3, /* (239) cmd ::= PRAGMA nm dbnm */ + -5, /* (240) cmd ::= PRAGMA nm dbnm EQ nmnum */ + -6, /* (241) cmd ::= PRAGMA nm dbnm LP nmnum RP */ + -5, /* (242) cmd ::= PRAGMA nm dbnm EQ minus_num */ + -6, /* (243) cmd ::= PRAGMA nm dbnm LP minus_num RP */ + -2, /* (244) plus_num ::= PLUS INTEGER|FLOAT */ + -2, /* (245) minus_num ::= MINUS INTEGER|FLOAT */ + -5, /* (246) cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END */ + -11, /* (247) trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */ + -1, /* (248) trigger_time ::= BEFORE|AFTER */ + -2, /* (249) trigger_time ::= INSTEAD OF */ + 0, /* (250) trigger_time ::= */ + -1, /* (251) trigger_event ::= DELETE|INSERT */ + -1, /* (252) trigger_event ::= UPDATE */ + -3, /* (253) trigger_event ::= UPDATE OF idlist */ + 0, /* (254) when_clause ::= */ + -2, /* (255) when_clause ::= WHEN expr */ + -3, /* (256) trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */ + -2, /* (257) trigger_cmd_list ::= trigger_cmd SEMI */ + -3, /* (258) trnm ::= nm DOT nm */ + -3, /* (259) tridxby ::= INDEXED BY nm */ + -2, /* (260) tridxby ::= NOT INDEXED */ + -8, /* (261) trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist where_opt scanpt */ + -8, /* (262) trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt */ + -6, /* (263) trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt */ + -3, /* (264) trigger_cmd ::= scanpt select scanpt */ + -4, /* (265) expr ::= RAISE LP IGNORE RP */ + -6, /* (266) expr ::= RAISE LP raisetype COMMA nm RP */ + -1, /* (267) raisetype ::= ROLLBACK */ + -1, /* (268) raisetype ::= ABORT */ + -1, /* (269) raisetype ::= FAIL */ + -4, /* (270) cmd ::= DROP TRIGGER ifexists fullname */ + -6, /* (271) cmd ::= ATTACH database_kw_opt expr AS expr key_opt */ + -3, /* (272) cmd ::= DETACH database_kw_opt expr */ + 0, /* (273) key_opt ::= */ + -2, /* (274) key_opt ::= KEY expr */ + -1, /* (275) cmd ::= REINDEX */ + -3, /* (276) cmd ::= REINDEX nm dbnm */ + -1, /* (277) cmd ::= ANALYZE */ + -3, /* (278) cmd ::= ANALYZE nm dbnm */ + -6, /* (279) cmd ::= ALTER TABLE fullname RENAME TO nm */ + -7, /* (280) cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist */ + -1, /* (281) add_column_fullname ::= fullname */ + -8, /* (282) cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm */ + -1, /* (283) cmd ::= create_vtab */ + -4, /* (284) cmd ::= create_vtab LP vtabarglist RP */ + -8, /* (285) create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm */ + 0, /* (286) vtabarg ::= */ + -1, /* (287) vtabargtoken ::= ANY */ + -3, /* (288) vtabargtoken ::= lp anylist RP */ + -1, /* (289) lp ::= LP */ + -2, /* (290) with ::= WITH wqlist */ + -3, /* (291) with ::= WITH RECURSIVE wqlist */ + -6, /* (292) wqlist ::= nm eidlist_opt AS LP select RP */ + -8, /* (293) wqlist ::= wqlist COMMA nm eidlist_opt AS LP select RP */ + -1, /* (294) windowdefn_list ::= windowdefn */ + -3, /* (295) windowdefn_list ::= windowdefn_list COMMA windowdefn */ + -5, /* (296) windowdefn ::= nm AS LP window RP */ + -5, /* (297) window ::= PARTITION BY nexprlist orderby_opt frame_opt */ + -6, /* (298) window ::= nm PARTITION BY nexprlist orderby_opt frame_opt */ + -4, /* (299) window ::= ORDER BY sortlist frame_opt */ + -5, /* (300) window ::= nm ORDER BY sortlist frame_opt */ + -1, /* (301) window ::= frame_opt */ + -2, /* (302) window ::= nm frame_opt */ + 0, /* (303) frame_opt ::= */ + -3, /* (304) frame_opt ::= range_or_rows frame_bound_s frame_exclude_opt */ + -6, /* (305) frame_opt ::= range_or_rows BETWEEN frame_bound_s AND frame_bound_e frame_exclude_opt */ + -1, /* (306) range_or_rows ::= RANGE|ROWS|GROUPS */ + -1, /* (307) frame_bound_s ::= frame_bound */ + -2, /* (308) frame_bound_s ::= UNBOUNDED PRECEDING */ + -1, /* (309) frame_bound_e ::= frame_bound */ + -2, /* (310) frame_bound_e ::= UNBOUNDED FOLLOWING */ + -2, /* (311) frame_bound ::= expr PRECEDING|FOLLOWING */ + -2, /* (312) frame_bound ::= CURRENT ROW */ + 0, /* (313) frame_exclude_opt ::= */ + -2, /* (314) frame_exclude_opt ::= EXCLUDE frame_exclude */ + -2, /* (315) frame_exclude ::= NO OTHERS */ + -2, /* (316) frame_exclude ::= CURRENT ROW */ + -1, /* (317) frame_exclude ::= GROUP|TIES */ + -2, /* (318) window_clause ::= WINDOW windowdefn_list */ + -2, /* (319) filter_over ::= filter_clause over_clause */ + -1, /* (320) filter_over ::= over_clause */ + -1, /* (321) filter_over ::= filter_clause */ + -4, /* (322) over_clause ::= OVER LP window RP */ + -2, /* (323) over_clause ::= OVER nm */ + -5, /* (324) filter_clause ::= FILTER LP WHERE expr RP */ + -1, /* (325) input ::= cmdlist */ + -2, /* (326) cmdlist ::= cmdlist ecmd */ + -1, /* (327) cmdlist ::= ecmd */ + -1, /* (328) ecmd ::= SEMI */ + -2, /* (329) ecmd ::= cmdx SEMI */ + -3, /* (330) ecmd ::= explain cmdx SEMI */ + 0, /* (331) trans_opt ::= */ + -1, /* (332) trans_opt ::= TRANSACTION */ + -2, /* (333) trans_opt ::= TRANSACTION nm */ + -1, /* (334) savepoint_opt ::= SAVEPOINT */ + 0, /* (335) savepoint_opt ::= */ + -2, /* (336) cmd ::= create_table create_table_args */ + -4, /* (337) columnlist ::= columnlist COMMA columnname carglist */ + -2, /* (338) columnlist ::= columnname carglist */ + -1, /* (339) nm ::= ID|INDEXED */ + -1, /* (340) nm ::= STRING */ + -1, /* (341) nm ::= JOIN_KW */ + -1, /* (342) typetoken ::= typename */ + -1, /* (343) typename ::= ID|STRING */ + -1, /* (344) signed ::= plus_num */ + -1, /* (345) signed ::= minus_num */ + -2, /* (346) carglist ::= carglist ccons */ + 0, /* (347) carglist ::= */ + -2, /* (348) ccons ::= NULL onconf */ + -4, /* (349) ccons ::= GENERATED ALWAYS AS generated */ + -2, /* (350) ccons ::= AS generated */ + -2, /* (351) conslist_opt ::= COMMA conslist */ + -3, /* (352) conslist ::= conslist tconscomma tcons */ + -1, /* (353) conslist ::= tcons */ + 0, /* (354) tconscomma ::= */ + -1, /* (355) defer_subclause_opt ::= defer_subclause */ + -1, /* (356) resolvetype ::= raisetype */ + -1, /* (357) selectnowith ::= oneselect */ + -1, /* (358) oneselect ::= values */ + -2, /* (359) sclp ::= selcollist COMMA */ + -1, /* (360) as ::= ID|STRING */ + -1, /* (361) expr ::= term */ + -1, /* (362) likeop ::= LIKE_KW|MATCH */ + -1, /* (363) exprlist ::= nexprlist */ + -1, /* (364) nmnum ::= plus_num */ + -1, /* (365) nmnum ::= nm */ + -1, /* (366) nmnum ::= ON */ + -1, /* (367) nmnum ::= DELETE */ + -1, /* (368) nmnum ::= DEFAULT */ + -1, /* (369) plus_num ::= INTEGER|FLOAT */ + 0, /* (370) foreach_clause ::= */ + -3, /* (371) foreach_clause ::= FOR EACH ROW */ + -1, /* (372) trnm ::= nm */ + 0, /* (373) tridxby ::= */ + -1, /* (374) database_kw_opt ::= DATABASE */ + 0, /* (375) database_kw_opt ::= */ + 0, /* (376) kwcolumn_opt ::= */ + -1, /* (377) kwcolumn_opt ::= COLUMNKW */ + -1, /* (378) vtabarglist ::= vtabarg */ + -3, /* (379) vtabarglist ::= vtabarglist COMMA vtabarg */ + -2, /* (380) vtabarg ::= vtabarg vtabargtoken */ + 0, /* (381) anylist ::= */ + -4, /* (382) anylist ::= anylist LP anylist RP */ + -2, /* (383) anylist ::= anylist ANY */ + 0, /* (384) with ::= */ +}; + +static void yy_accept(yyParser*); /* Forward Declaration */ + +/* +** Perform a reduce action and the shift that must immediately +** follow the reduce. +** +** The yyLookahead and yyLookaheadToken parameters provide reduce actions +** access to the lookahead token (if any). The yyLookahead will be YYNOCODE +** if the lookahead token has already been consumed. As this procedure is +** only called from one place, optimizing compilers will in-line it, which +** means that the extra parameters have no performance impact. +*/ +static YYACTIONTYPE yy_reduce( + yyParser *yypParser, /* The parser */ + unsigned int yyruleno, /* Number of the rule by which to reduce */ + int yyLookahead, /* Lookahead token, or YYNOCODE if none */ + sqlite3ParserTOKENTYPE yyLookaheadToken /* Value of the lookahead token */ + sqlite3ParserCTX_PDECL /* %extra_context */ +){ + int yygoto; /* The next state */ + YYACTIONTYPE yyact; /* The next action */ + yyStackEntry *yymsp; /* The top of the parser's stack */ + int yysize; /* Amount to pop the stack */ + sqlite3ParserARG_FETCH + (void)yyLookahead; + (void)yyLookaheadToken; + yymsp = yypParser->yytos; +#ifndef NDEBUG + if( yyTraceFILE && yyruleno<(int)(sizeof(yyRuleName)/sizeof(yyRuleName[0])) ){ + yysize = yyRuleInfoNRhs[yyruleno]; + if( yysize ){ + fprintf(yyTraceFILE, "%sReduce %d [%s]%s, pop back to state %d.\n", + yyTracePrompt, + yyruleno, yyRuleName[yyruleno], + yyrulenoyytos - yypParser->yystack)>yypParser->yyhwm ){ + yypParser->yyhwm++; + assert( yypParser->yyhwm == (int)(yypParser->yytos - yypParser->yystack)); + } +#endif +#if YYSTACKDEPTH>0 + if( yypParser->yytos>=yypParser->yystackEnd ){ + yyStackOverflow(yypParser); + /* The call to yyStackOverflow() above pops the stack until it is + ** empty, causing the main parser loop to exit. So the return value + ** is never used and does not matter. */ + return 0; + } +#else + if( yypParser->yytos>=&yypParser->yystack[yypParser->yystksz-1] ){ + if( yyGrowStack(yypParser) ){ + yyStackOverflow(yypParser); + /* The call to yyStackOverflow() above pops the stack until it is + ** empty, causing the main parser loop to exit. So the return value + ** is never used and does not matter. */ + return 0; + } + yymsp = yypParser->yytos; + } +#endif + } + + switch( yyruleno ){ + /* Beginning here are the reduction cases. A typical example + ** follows: + ** case 0: + ** #line + ** { ... } // User supplied code + ** #line + ** break; + */ +/********** Begin reduce actions **********************************************/ + YYMINORTYPE yylhsminor; + case 0: /* explain ::= EXPLAIN */ +{ pParse->explain = 1; } + break; + case 1: /* explain ::= EXPLAIN QUERY PLAN */ +{ pParse->explain = 2; } + break; + case 2: /* cmdx ::= cmd */ +{ sqlite3FinishCoding(pParse); } + break; + case 3: /* cmd ::= BEGIN transtype trans_opt */ +{sqlite3BeginTransaction(pParse, yymsp[-1].minor.yy192);} + break; + case 4: /* transtype ::= */ +{yymsp[1].minor.yy192 = TK_DEFERRED;} + break; + case 5: /* transtype ::= DEFERRED */ + case 6: /* transtype ::= IMMEDIATE */ yytestcase(yyruleno==6); + case 7: /* transtype ::= EXCLUSIVE */ yytestcase(yyruleno==7); + case 306: /* range_or_rows ::= RANGE|ROWS|GROUPS */ yytestcase(yyruleno==306); +{yymsp[0].minor.yy192 = yymsp[0].major; /*A-overwrites-X*/} + break; + case 8: /* cmd ::= COMMIT|END trans_opt */ + case 9: /* cmd ::= ROLLBACK trans_opt */ yytestcase(yyruleno==9); +{sqlite3EndTransaction(pParse,yymsp[-1].major);} + break; + case 10: /* cmd ::= SAVEPOINT nm */ +{ + sqlite3Savepoint(pParse, SAVEPOINT_BEGIN, &yymsp[0].minor.yy0); +} + break; + case 11: /* cmd ::= RELEASE savepoint_opt nm */ +{ + sqlite3Savepoint(pParse, SAVEPOINT_RELEASE, &yymsp[0].minor.yy0); +} + break; + case 12: /* cmd ::= ROLLBACK trans_opt TO savepoint_opt nm */ +{ + sqlite3Savepoint(pParse, SAVEPOINT_ROLLBACK, &yymsp[0].minor.yy0); +} + break; + case 13: /* create_table ::= createkw temp TABLE ifnotexists nm dbnm */ +{ + sqlite3StartTable(pParse,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0,yymsp[-4].minor.yy192,0,0,yymsp[-2].minor.yy192); +} + break; + case 14: /* createkw ::= CREATE */ +{disableLookaside(pParse);} + break; + case 15: /* ifnotexists ::= */ + case 18: /* temp ::= */ yytestcase(yyruleno==18); + case 21: /* table_options ::= */ yytestcase(yyruleno==21); + case 45: /* autoinc ::= */ yytestcase(yyruleno==45); + case 60: /* init_deferred_pred_opt ::= */ yytestcase(yyruleno==60); + case 70: /* defer_subclause_opt ::= */ yytestcase(yyruleno==70); + case 79: /* ifexists ::= */ yytestcase(yyruleno==79); + case 96: /* distinct ::= */ yytestcase(yyruleno==96); + case 232: /* collate ::= */ yytestcase(yyruleno==232); +{yymsp[1].minor.yy192 = 0;} + break; + case 16: /* ifnotexists ::= IF NOT EXISTS */ +{yymsp[-2].minor.yy192 = 1;} + break; + case 17: /* temp ::= TEMP */ + case 46: /* autoinc ::= AUTOINCR */ yytestcase(yyruleno==46); +{yymsp[0].minor.yy192 = 1;} + break; + case 19: /* create_table_args ::= LP columnlist conslist_opt RP table_options */ +{ + sqlite3EndTable(pParse,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0,yymsp[0].minor.yy192,0); +} + break; + case 20: /* create_table_args ::= AS select */ +{ + sqlite3EndTable(pParse,0,0,0,yymsp[0].minor.yy539); + sqlite3SelectDelete(pParse->db, yymsp[0].minor.yy539); +} + break; + case 22: /* table_options ::= WITHOUT nm */ +{ + if( yymsp[0].minor.yy0.n==5 && sqlite3_strnicmp(yymsp[0].minor.yy0.z,"rowid",5)==0 ){ + yymsp[-1].minor.yy192 = TF_WithoutRowid | TF_NoVisibleRowid; + }else{ + yymsp[-1].minor.yy192 = 0; + sqlite3ErrorMsg(pParse, "unknown table option: %.*s", yymsp[0].minor.yy0.n, yymsp[0].minor.yy0.z); + } +} + break; + case 23: /* columnname ::= nm typetoken */ +{sqlite3AddColumn(pParse,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0);} + break; + case 24: /* typetoken ::= */ + case 63: /* conslist_opt ::= */ yytestcase(yyruleno==63); + case 102: /* as ::= */ yytestcase(yyruleno==102); +{yymsp[1].minor.yy0.n = 0; yymsp[1].minor.yy0.z = 0;} + break; + case 25: /* typetoken ::= typename LP signed RP */ +{ + yymsp[-3].minor.yy0.n = (int)(&yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n] - yymsp[-3].minor.yy0.z); +} + break; + case 26: /* typetoken ::= typename LP signed COMMA signed RP */ +{ + yymsp[-5].minor.yy0.n = (int)(&yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n] - yymsp[-5].minor.yy0.z); +} + break; + case 27: /* typename ::= typename ID|STRING */ +{yymsp[-1].minor.yy0.n=yymsp[0].minor.yy0.n+(int)(yymsp[0].minor.yy0.z-yymsp[-1].minor.yy0.z);} + break; + case 28: /* scanpt ::= */ +{ + assert( yyLookahead!=YYNOCODE ); + yymsp[1].minor.yy436 = yyLookaheadToken.z; +} + break; + case 29: /* scantok ::= */ +{ + assert( yyLookahead!=YYNOCODE ); + yymsp[1].minor.yy0 = yyLookaheadToken; +} + break; + case 30: /* ccons ::= CONSTRAINT nm */ + case 65: /* tcons ::= CONSTRAINT nm */ yytestcase(yyruleno==65); +{pParse->constraintName = yymsp[0].minor.yy0;} + break; + case 31: /* ccons ::= DEFAULT scantok term */ +{sqlite3AddDefaultValue(pParse,yymsp[0].minor.yy202,yymsp[-1].minor.yy0.z,&yymsp[-1].minor.yy0.z[yymsp[-1].minor.yy0.n]);} + break; + case 32: /* ccons ::= DEFAULT LP expr RP */ +{sqlite3AddDefaultValue(pParse,yymsp[-1].minor.yy202,yymsp[-2].minor.yy0.z+1,yymsp[0].minor.yy0.z);} + break; + case 33: /* ccons ::= DEFAULT PLUS scantok term */ +{sqlite3AddDefaultValue(pParse,yymsp[0].minor.yy202,yymsp[-2].minor.yy0.z,&yymsp[-1].minor.yy0.z[yymsp[-1].minor.yy0.n]);} + break; + case 34: /* ccons ::= DEFAULT MINUS scantok term */ +{ + Expr *p = sqlite3PExpr(pParse, TK_UMINUS, yymsp[0].minor.yy202, 0); + sqlite3AddDefaultValue(pParse,p,yymsp[-2].minor.yy0.z,&yymsp[-1].minor.yy0.z[yymsp[-1].minor.yy0.n]); +} + break; + case 35: /* ccons ::= DEFAULT scantok ID|INDEXED */ +{ + Expr *p = tokenExpr(pParse, TK_STRING, yymsp[0].minor.yy0); + if( p ){ + sqlite3ExprIdToTrueFalse(p); + testcase( p->op==TK_TRUEFALSE && sqlite3ExprTruthValue(p) ); + } + sqlite3AddDefaultValue(pParse,p,yymsp[0].minor.yy0.z,yymsp[0].minor.yy0.z+yymsp[0].minor.yy0.n); +} + break; + case 36: /* ccons ::= NOT NULL onconf */ +{sqlite3AddNotNull(pParse, yymsp[0].minor.yy192);} + break; + case 37: /* ccons ::= PRIMARY KEY sortorder onconf autoinc */ +{sqlite3AddPrimaryKey(pParse,0,yymsp[-1].minor.yy192,yymsp[0].minor.yy192,yymsp[-2].minor.yy192);} + break; + case 38: /* ccons ::= UNIQUE onconf */ +{sqlite3CreateIndex(pParse,0,0,0,0,yymsp[0].minor.yy192,0,0,0,0, + SQLITE_IDXTYPE_UNIQUE);} + break; + case 39: /* ccons ::= CHECK LP expr RP */ +{sqlite3AddCheckConstraint(pParse,yymsp[-1].minor.yy202);} + break; + case 40: /* ccons ::= REFERENCES nm eidlist_opt refargs */ +{sqlite3CreateForeignKey(pParse,0,&yymsp[-2].minor.yy0,yymsp[-1].minor.yy242,yymsp[0].minor.yy192);} + break; + case 41: /* ccons ::= defer_subclause */ +{sqlite3DeferForeignKey(pParse,yymsp[0].minor.yy192);} + break; + case 42: /* ccons ::= COLLATE ID|STRING */ +{sqlite3AddCollateType(pParse, &yymsp[0].minor.yy0);} + break; + case 43: /* generated ::= LP expr RP */ +{sqlite3AddGenerated(pParse,yymsp[-1].minor.yy202,0);} + break; + case 44: /* generated ::= LP expr RP ID */ +{sqlite3AddGenerated(pParse,yymsp[-2].minor.yy202,&yymsp[0].minor.yy0);} + break; + case 47: /* refargs ::= */ +{ yymsp[1].minor.yy192 = OE_None*0x0101; /* EV: R-19803-45884 */} + break; + case 48: /* refargs ::= refargs refarg */ +{ yymsp[-1].minor.yy192 = (yymsp[-1].minor.yy192 & ~yymsp[0].minor.yy207.mask) | yymsp[0].minor.yy207.value; } + break; + case 49: /* refarg ::= MATCH nm */ +{ yymsp[-1].minor.yy207.value = 0; yymsp[-1].minor.yy207.mask = 0x000000; } + break; + case 50: /* refarg ::= ON INSERT refact */ +{ yymsp[-2].minor.yy207.value = 0; yymsp[-2].minor.yy207.mask = 0x000000; } + break; + case 51: /* refarg ::= ON DELETE refact */ +{ yymsp[-2].minor.yy207.value = yymsp[0].minor.yy192; yymsp[-2].minor.yy207.mask = 0x0000ff; } + break; + case 52: /* refarg ::= ON UPDATE refact */ +{ yymsp[-2].minor.yy207.value = yymsp[0].minor.yy192<<8; yymsp[-2].minor.yy207.mask = 0x00ff00; } + break; + case 53: /* refact ::= SET NULL */ +{ yymsp[-1].minor.yy192 = OE_SetNull; /* EV: R-33326-45252 */} + break; + case 54: /* refact ::= SET DEFAULT */ +{ yymsp[-1].minor.yy192 = OE_SetDflt; /* EV: R-33326-45252 */} + break; + case 55: /* refact ::= CASCADE */ +{ yymsp[0].minor.yy192 = OE_Cascade; /* EV: R-33326-45252 */} + break; + case 56: /* refact ::= RESTRICT */ +{ yymsp[0].minor.yy192 = OE_Restrict; /* EV: R-33326-45252 */} + break; + case 57: /* refact ::= NO ACTION */ +{ yymsp[-1].minor.yy192 = OE_None; /* EV: R-33326-45252 */} + break; + case 58: /* defer_subclause ::= NOT DEFERRABLE init_deferred_pred_opt */ +{yymsp[-2].minor.yy192 = 0;} + break; + case 59: /* defer_subclause ::= DEFERRABLE init_deferred_pred_opt */ + case 74: /* orconf ::= OR resolvetype */ yytestcase(yyruleno==74); + case 162: /* insert_cmd ::= INSERT orconf */ yytestcase(yyruleno==162); +{yymsp[-1].minor.yy192 = yymsp[0].minor.yy192;} + break; + case 61: /* init_deferred_pred_opt ::= INITIALLY DEFERRED */ + case 78: /* ifexists ::= IF EXISTS */ yytestcase(yyruleno==78); + case 204: /* between_op ::= NOT BETWEEN */ yytestcase(yyruleno==204); + case 207: /* in_op ::= NOT IN */ yytestcase(yyruleno==207); + case 233: /* collate ::= COLLATE ID|STRING */ yytestcase(yyruleno==233); +{yymsp[-1].minor.yy192 = 1;} + break; + case 62: /* init_deferred_pred_opt ::= INITIALLY IMMEDIATE */ +{yymsp[-1].minor.yy192 = 0;} + break; + case 64: /* tconscomma ::= COMMA */ +{pParse->constraintName.n = 0;} + break; + case 66: /* tcons ::= PRIMARY KEY LP sortlist autoinc RP onconf */ +{sqlite3AddPrimaryKey(pParse,yymsp[-3].minor.yy242,yymsp[0].minor.yy192,yymsp[-2].minor.yy192,0);} + break; + case 67: /* tcons ::= UNIQUE LP sortlist RP onconf */ +{sqlite3CreateIndex(pParse,0,0,0,yymsp[-2].minor.yy242,yymsp[0].minor.yy192,0,0,0,0, + SQLITE_IDXTYPE_UNIQUE);} + break; + case 68: /* tcons ::= CHECK LP expr RP onconf */ +{sqlite3AddCheckConstraint(pParse,yymsp[-2].minor.yy202);} + break; + case 69: /* tcons ::= FOREIGN KEY LP eidlist RP REFERENCES nm eidlist_opt refargs defer_subclause_opt */ +{ + sqlite3CreateForeignKey(pParse, yymsp[-6].minor.yy242, &yymsp[-3].minor.yy0, yymsp[-2].minor.yy242, yymsp[-1].minor.yy192); + sqlite3DeferForeignKey(pParse, yymsp[0].minor.yy192); +} + break; + case 71: /* onconf ::= */ + case 73: /* orconf ::= */ yytestcase(yyruleno==73); +{yymsp[1].minor.yy192 = OE_Default;} + break; + case 72: /* onconf ::= ON CONFLICT resolvetype */ +{yymsp[-2].minor.yy192 = yymsp[0].minor.yy192;} + break; + case 75: /* resolvetype ::= IGNORE */ +{yymsp[0].minor.yy192 = OE_Ignore;} + break; + case 76: /* resolvetype ::= REPLACE */ + case 163: /* insert_cmd ::= REPLACE */ yytestcase(yyruleno==163); +{yymsp[0].minor.yy192 = OE_Replace;} + break; + case 77: /* cmd ::= DROP TABLE ifexists fullname */ +{ + sqlite3DropTable(pParse, yymsp[0].minor.yy47, 0, yymsp[-1].minor.yy192); +} + break; + case 80: /* cmd ::= createkw temp VIEW ifnotexists nm dbnm eidlist_opt AS select */ +{ + sqlite3CreateView(pParse, &yymsp[-8].minor.yy0, &yymsp[-4].minor.yy0, &yymsp[-3].minor.yy0, yymsp[-2].minor.yy242, yymsp[0].minor.yy539, yymsp[-7].minor.yy192, yymsp[-5].minor.yy192); +} + break; + case 81: /* cmd ::= DROP VIEW ifexists fullname */ +{ + sqlite3DropTable(pParse, yymsp[0].minor.yy47, 1, yymsp[-1].minor.yy192); +} + break; + case 82: /* cmd ::= select */ +{ + SelectDest dest = {SRT_Output, 0, 0, 0, 0, 0}; + sqlite3Select(pParse, yymsp[0].minor.yy539, &dest); + sqlite3SelectDelete(pParse->db, yymsp[0].minor.yy539); +} + break; + case 83: /* select ::= WITH wqlist selectnowith */ +{ + Select *p = yymsp[0].minor.yy539; + if( p ){ + p->pWith = yymsp[-1].minor.yy131; + parserDoubleLinkSelect(pParse, p); + }else{ + sqlite3WithDelete(pParse->db, yymsp[-1].minor.yy131); + } + yymsp[-2].minor.yy539 = p; +} + break; + case 84: /* select ::= WITH RECURSIVE wqlist selectnowith */ +{ + Select *p = yymsp[0].minor.yy539; + if( p ){ + p->pWith = yymsp[-1].minor.yy131; + parserDoubleLinkSelect(pParse, p); + }else{ + sqlite3WithDelete(pParse->db, yymsp[-1].minor.yy131); + } + yymsp[-3].minor.yy539 = p; +} + break; + case 85: /* select ::= selectnowith */ +{ + Select *p = yymsp[0].minor.yy539; + if( p ){ + parserDoubleLinkSelect(pParse, p); + } + yymsp[0].minor.yy539 = p; /*A-overwrites-X*/ +} + break; + case 86: /* selectnowith ::= selectnowith multiselect_op oneselect */ +{ + Select *pRhs = yymsp[0].minor.yy539; + Select *pLhs = yymsp[-2].minor.yy539; + if( pRhs && pRhs->pPrior ){ + SrcList *pFrom; + Token x; + x.n = 0; + parserDoubleLinkSelect(pParse, pRhs); + pFrom = sqlite3SrcListAppendFromTerm(pParse,0,0,0,&x,pRhs,0,0); + pRhs = sqlite3SelectNew(pParse,0,pFrom,0,0,0,0,0,0); + } + if( pRhs ){ + pRhs->op = (u8)yymsp[-1].minor.yy192; + pRhs->pPrior = pLhs; + if( ALWAYS(pLhs) ) pLhs->selFlags &= ~SF_MultiValue; + pRhs->selFlags &= ~SF_MultiValue; + if( yymsp[-1].minor.yy192!=TK_ALL ) pParse->hasCompound = 1; + }else{ + sqlite3SelectDelete(pParse->db, pLhs); + } + yymsp[-2].minor.yy539 = pRhs; +} + break; + case 87: /* multiselect_op ::= UNION */ + case 89: /* multiselect_op ::= EXCEPT|INTERSECT */ yytestcase(yyruleno==89); +{yymsp[0].minor.yy192 = yymsp[0].major; /*A-overwrites-OP*/} + break; + case 88: /* multiselect_op ::= UNION ALL */ +{yymsp[-1].minor.yy192 = TK_ALL;} + break; + case 90: /* oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt orderby_opt limit_opt */ +{ + yymsp[-8].minor.yy539 = sqlite3SelectNew(pParse,yymsp[-6].minor.yy242,yymsp[-5].minor.yy47,yymsp[-4].minor.yy202,yymsp[-3].minor.yy242,yymsp[-2].minor.yy202,yymsp[-1].minor.yy242,yymsp[-7].minor.yy192,yymsp[0].minor.yy202); +} + break; + case 91: /* oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt window_clause orderby_opt limit_opt */ +{ + yymsp[-9].minor.yy539 = sqlite3SelectNew(pParse,yymsp[-7].minor.yy242,yymsp[-6].minor.yy47,yymsp[-5].minor.yy202,yymsp[-4].minor.yy242,yymsp[-3].minor.yy202,yymsp[-1].minor.yy242,yymsp[-8].minor.yy192,yymsp[0].minor.yy202); + if( yymsp[-9].minor.yy539 ){ + yymsp[-9].minor.yy539->pWinDefn = yymsp[-2].minor.yy303; + }else{ + sqlite3WindowListDelete(pParse->db, yymsp[-2].minor.yy303); + } +} + break; + case 92: /* values ::= VALUES LP nexprlist RP */ +{ + yymsp[-3].minor.yy539 = sqlite3SelectNew(pParse,yymsp[-1].minor.yy242,0,0,0,0,0,SF_Values,0); +} + break; + case 93: /* values ::= values COMMA LP nexprlist RP */ +{ + Select *pRight, *pLeft = yymsp[-4].minor.yy539; + pRight = sqlite3SelectNew(pParse,yymsp[-1].minor.yy242,0,0,0,0,0,SF_Values|SF_MultiValue,0); + if( ALWAYS(pLeft) ) pLeft->selFlags &= ~SF_MultiValue; + if( pRight ){ + pRight->op = TK_ALL; + pRight->pPrior = pLeft; + yymsp[-4].minor.yy539 = pRight; + }else{ + yymsp[-4].minor.yy539 = pLeft; + } +} + break; + case 94: /* distinct ::= DISTINCT */ +{yymsp[0].minor.yy192 = SF_Distinct;} + break; + case 95: /* distinct ::= ALL */ +{yymsp[0].minor.yy192 = SF_All;} + break; + case 97: /* sclp ::= */ + case 130: /* orderby_opt ::= */ yytestcase(yyruleno==130); + case 140: /* groupby_opt ::= */ yytestcase(yyruleno==140); + case 220: /* exprlist ::= */ yytestcase(yyruleno==220); + case 223: /* paren_exprlist ::= */ yytestcase(yyruleno==223); + case 228: /* eidlist_opt ::= */ yytestcase(yyruleno==228); +{yymsp[1].minor.yy242 = 0;} + break; + case 98: /* selcollist ::= sclp scanpt expr scanpt as */ +{ + yymsp[-4].minor.yy242 = sqlite3ExprListAppend(pParse, yymsp[-4].minor.yy242, yymsp[-2].minor.yy202); + if( yymsp[0].minor.yy0.n>0 ) sqlite3ExprListSetName(pParse, yymsp[-4].minor.yy242, &yymsp[0].minor.yy0, 1); + sqlite3ExprListSetSpan(pParse,yymsp[-4].minor.yy242,yymsp[-3].minor.yy436,yymsp[-1].minor.yy436); +} + break; + case 99: /* selcollist ::= sclp scanpt STAR */ +{ + Expr *p = sqlite3Expr(pParse->db, TK_ASTERISK, 0); + yymsp[-2].minor.yy242 = sqlite3ExprListAppend(pParse, yymsp[-2].minor.yy242, p); +} + break; + case 100: /* selcollist ::= sclp scanpt nm DOT STAR */ +{ + Expr *pRight = sqlite3PExpr(pParse, TK_ASTERISK, 0, 0); + Expr *pLeft = sqlite3ExprAlloc(pParse->db, TK_ID, &yymsp[-2].minor.yy0, 1); + Expr *pDot = sqlite3PExpr(pParse, TK_DOT, pLeft, pRight); + yymsp[-4].minor.yy242 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy242, pDot); +} + break; + case 101: /* as ::= AS nm */ + case 112: /* dbnm ::= DOT nm */ yytestcase(yyruleno==112); + case 244: /* plus_num ::= PLUS INTEGER|FLOAT */ yytestcase(yyruleno==244); + case 245: /* minus_num ::= MINUS INTEGER|FLOAT */ yytestcase(yyruleno==245); +{yymsp[-1].minor.yy0 = yymsp[0].minor.yy0;} + break; + case 103: /* from ::= */ +{yymsp[1].minor.yy47 = sqlite3DbMallocZero(pParse->db, sizeof(*yymsp[1].minor.yy47));} + break; + case 104: /* from ::= FROM seltablist */ +{ + yymsp[-1].minor.yy47 = yymsp[0].minor.yy47; + sqlite3SrcListShiftJoinType(yymsp[-1].minor.yy47); +} + break; + case 105: /* stl_prefix ::= seltablist joinop */ +{ + if( ALWAYS(yymsp[-1].minor.yy47 && yymsp[-1].minor.yy47->nSrc>0) ) yymsp[-1].minor.yy47->a[yymsp[-1].minor.yy47->nSrc-1].fg.jointype = (u8)yymsp[0].minor.yy192; +} + break; + case 106: /* stl_prefix ::= */ +{yymsp[1].minor.yy47 = 0;} + break; + case 107: /* seltablist ::= stl_prefix nm dbnm as indexed_opt on_opt using_opt */ +{ + yymsp[-6].minor.yy47 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy47,&yymsp[-5].minor.yy0,&yymsp[-4].minor.yy0,&yymsp[-3].minor.yy0,0,yymsp[-1].minor.yy202,yymsp[0].minor.yy600); + sqlite3SrcListIndexedBy(pParse, yymsp[-6].minor.yy47, &yymsp[-2].minor.yy0); +} + break; + case 108: /* seltablist ::= stl_prefix nm dbnm LP exprlist RP as on_opt using_opt */ +{ + yymsp[-8].minor.yy47 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-8].minor.yy47,&yymsp[-7].minor.yy0,&yymsp[-6].minor.yy0,&yymsp[-2].minor.yy0,0,yymsp[-1].minor.yy202,yymsp[0].minor.yy600); + sqlite3SrcListFuncArgs(pParse, yymsp[-8].minor.yy47, yymsp[-4].minor.yy242); +} + break; + case 109: /* seltablist ::= stl_prefix LP select RP as on_opt using_opt */ +{ + yymsp[-6].minor.yy47 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy47,0,0,&yymsp[-2].minor.yy0,yymsp[-4].minor.yy539,yymsp[-1].minor.yy202,yymsp[0].minor.yy600); + } + break; + case 110: /* seltablist ::= stl_prefix LP seltablist RP as on_opt using_opt */ +{ + if( yymsp[-6].minor.yy47==0 && yymsp[-2].minor.yy0.n==0 && yymsp[-1].minor.yy202==0 && yymsp[0].minor.yy600==0 ){ + yymsp[-6].minor.yy47 = yymsp[-4].minor.yy47; + }else if( yymsp[-4].minor.yy47->nSrc==1 ){ + yymsp[-6].minor.yy47 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy47,0,0,&yymsp[-2].minor.yy0,0,yymsp[-1].minor.yy202,yymsp[0].minor.yy600); + if( yymsp[-6].minor.yy47 ){ + struct SrcList_item *pNew = &yymsp[-6].minor.yy47->a[yymsp[-6].minor.yy47->nSrc-1]; + struct SrcList_item *pOld = yymsp[-4].minor.yy47->a; + pNew->zName = pOld->zName; + pNew->zDatabase = pOld->zDatabase; + pNew->pSelect = pOld->pSelect; + if( pOld->fg.isTabFunc ){ + pNew->u1.pFuncArg = pOld->u1.pFuncArg; + pOld->u1.pFuncArg = 0; + pOld->fg.isTabFunc = 0; + pNew->fg.isTabFunc = 1; + } + pOld->zName = pOld->zDatabase = 0; + pOld->pSelect = 0; + } + sqlite3SrcListDelete(pParse->db, yymsp[-4].minor.yy47); + }else{ + Select *pSubquery; + sqlite3SrcListShiftJoinType(yymsp[-4].minor.yy47); + pSubquery = sqlite3SelectNew(pParse,0,yymsp[-4].minor.yy47,0,0,0,0,SF_NestedFrom,0); + yymsp[-6].minor.yy47 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy47,0,0,&yymsp[-2].minor.yy0,pSubquery,yymsp[-1].minor.yy202,yymsp[0].minor.yy600); + } + } + break; + case 111: /* dbnm ::= */ + case 125: /* indexed_opt ::= */ yytestcase(yyruleno==125); +{yymsp[1].minor.yy0.z=0; yymsp[1].minor.yy0.n=0;} + break; + case 113: /* fullname ::= nm */ +{ + yylhsminor.yy47 = sqlite3SrcListAppend(pParse,0,&yymsp[0].minor.yy0,0); + if( IN_RENAME_OBJECT && yylhsminor.yy47 ) sqlite3RenameTokenMap(pParse, yylhsminor.yy47->a[0].zName, &yymsp[0].minor.yy0); +} + yymsp[0].minor.yy47 = yylhsminor.yy47; + break; + case 114: /* fullname ::= nm DOT nm */ +{ + yylhsminor.yy47 = sqlite3SrcListAppend(pParse,0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0); + if( IN_RENAME_OBJECT && yylhsminor.yy47 ) sqlite3RenameTokenMap(pParse, yylhsminor.yy47->a[0].zName, &yymsp[0].minor.yy0); +} + yymsp[-2].minor.yy47 = yylhsminor.yy47; + break; + case 115: /* xfullname ::= nm */ +{yymsp[0].minor.yy47 = sqlite3SrcListAppend(pParse,0,&yymsp[0].minor.yy0,0); /*A-overwrites-X*/} + break; + case 116: /* xfullname ::= nm DOT nm */ +{yymsp[-2].minor.yy47 = sqlite3SrcListAppend(pParse,0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0); /*A-overwrites-X*/} + break; + case 117: /* xfullname ::= nm DOT nm AS nm */ +{ + yymsp[-4].minor.yy47 = sqlite3SrcListAppend(pParse,0,&yymsp[-4].minor.yy0,&yymsp[-2].minor.yy0); /*A-overwrites-X*/ + if( yymsp[-4].minor.yy47 ) yymsp[-4].minor.yy47->a[0].zAlias = sqlite3NameFromToken(pParse->db, &yymsp[0].minor.yy0); +} + break; + case 118: /* xfullname ::= nm AS nm */ +{ + yymsp[-2].minor.yy47 = sqlite3SrcListAppend(pParse,0,&yymsp[-2].minor.yy0,0); /*A-overwrites-X*/ + if( yymsp[-2].minor.yy47 ) yymsp[-2].minor.yy47->a[0].zAlias = sqlite3NameFromToken(pParse->db, &yymsp[0].minor.yy0); +} + break; + case 119: /* joinop ::= COMMA|JOIN */ +{ yymsp[0].minor.yy192 = JT_INNER; } + break; + case 120: /* joinop ::= JOIN_KW JOIN */ +{yymsp[-1].minor.yy192 = sqlite3JoinType(pParse,&yymsp[-1].minor.yy0,0,0); /*X-overwrites-A*/} + break; + case 121: /* joinop ::= JOIN_KW nm JOIN */ +{yymsp[-2].minor.yy192 = sqlite3JoinType(pParse,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0,0); /*X-overwrites-A*/} + break; + case 122: /* joinop ::= JOIN_KW nm nm JOIN */ +{yymsp[-3].minor.yy192 = sqlite3JoinType(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0);/*X-overwrites-A*/} + break; + case 123: /* on_opt ::= ON expr */ + case 143: /* having_opt ::= HAVING expr */ yytestcase(yyruleno==143); + case 150: /* where_opt ::= WHERE expr */ yytestcase(yyruleno==150); + case 216: /* case_else ::= ELSE expr */ yytestcase(yyruleno==216); + case 237: /* vinto ::= INTO expr */ yytestcase(yyruleno==237); +{yymsp[-1].minor.yy202 = yymsp[0].minor.yy202;} + break; + case 124: /* on_opt ::= */ + case 142: /* having_opt ::= */ yytestcase(yyruleno==142); + case 144: /* limit_opt ::= */ yytestcase(yyruleno==144); + case 149: /* where_opt ::= */ yytestcase(yyruleno==149); + case 217: /* case_else ::= */ yytestcase(yyruleno==217); + case 219: /* case_operand ::= */ yytestcase(yyruleno==219); + case 238: /* vinto ::= */ yytestcase(yyruleno==238); +{yymsp[1].minor.yy202 = 0;} + break; + case 126: /* indexed_opt ::= INDEXED BY nm */ +{yymsp[-2].minor.yy0 = yymsp[0].minor.yy0;} + break; + case 127: /* indexed_opt ::= NOT INDEXED */ +{yymsp[-1].minor.yy0.z=0; yymsp[-1].minor.yy0.n=1;} + break; + case 128: /* using_opt ::= USING LP idlist RP */ +{yymsp[-3].minor.yy600 = yymsp[-1].minor.yy600;} + break; + case 129: /* using_opt ::= */ + case 164: /* idlist_opt ::= */ yytestcase(yyruleno==164); +{yymsp[1].minor.yy600 = 0;} + break; + case 131: /* orderby_opt ::= ORDER BY sortlist */ + case 141: /* groupby_opt ::= GROUP BY nexprlist */ yytestcase(yyruleno==141); +{yymsp[-2].minor.yy242 = yymsp[0].minor.yy242;} + break; + case 132: /* sortlist ::= sortlist COMMA expr sortorder nulls */ +{ + yymsp[-4].minor.yy242 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy242,yymsp[-2].minor.yy202); + sqlite3ExprListSetSortOrder(yymsp[-4].minor.yy242,yymsp[-1].minor.yy192,yymsp[0].minor.yy192); +} + break; + case 133: /* sortlist ::= expr sortorder nulls */ +{ + yymsp[-2].minor.yy242 = sqlite3ExprListAppend(pParse,0,yymsp[-2].minor.yy202); /*A-overwrites-Y*/ + sqlite3ExprListSetSortOrder(yymsp[-2].minor.yy242,yymsp[-1].minor.yy192,yymsp[0].minor.yy192); +} + break; + case 134: /* sortorder ::= ASC */ +{yymsp[0].minor.yy192 = SQLITE_SO_ASC;} + break; + case 135: /* sortorder ::= DESC */ +{yymsp[0].minor.yy192 = SQLITE_SO_DESC;} + break; + case 136: /* sortorder ::= */ + case 139: /* nulls ::= */ yytestcase(yyruleno==139); +{yymsp[1].minor.yy192 = SQLITE_SO_UNDEFINED;} + break; + case 137: /* nulls ::= NULLS FIRST */ +{yymsp[-1].minor.yy192 = SQLITE_SO_ASC;} + break; + case 138: /* nulls ::= NULLS LAST */ +{yymsp[-1].minor.yy192 = SQLITE_SO_DESC;} + break; + case 145: /* limit_opt ::= LIMIT expr */ +{yymsp[-1].minor.yy202 = sqlite3PExpr(pParse,TK_LIMIT,yymsp[0].minor.yy202,0);} + break; + case 146: /* limit_opt ::= LIMIT expr OFFSET expr */ +{yymsp[-3].minor.yy202 = sqlite3PExpr(pParse,TK_LIMIT,yymsp[-2].minor.yy202,yymsp[0].minor.yy202);} + break; + case 147: /* limit_opt ::= LIMIT expr COMMA expr */ +{yymsp[-3].minor.yy202 = sqlite3PExpr(pParse,TK_LIMIT,yymsp[0].minor.yy202,yymsp[-2].minor.yy202);} + break; + case 148: /* cmd ::= with DELETE FROM xfullname indexed_opt where_opt */ +{ + sqlite3SrcListIndexedBy(pParse, yymsp[-2].minor.yy47, &yymsp[-1].minor.yy0); + sqlite3DeleteFrom(pParse,yymsp[-2].minor.yy47,yymsp[0].minor.yy202,0,0); +} + break; + case 151: /* cmd ::= with UPDATE orconf xfullname indexed_opt SET setlist where_opt */ +{ + sqlite3SrcListIndexedBy(pParse, yymsp[-4].minor.yy47, &yymsp[-3].minor.yy0); + sqlite3ExprListCheckLength(pParse,yymsp[-1].minor.yy242,"set list"); + sqlite3Update(pParse,yymsp[-4].minor.yy47,yymsp[-1].minor.yy242,yymsp[0].minor.yy202,yymsp[-5].minor.yy192,0,0,0); +} + break; + case 152: /* setlist ::= setlist COMMA nm EQ expr */ +{ + yymsp[-4].minor.yy242 = sqlite3ExprListAppend(pParse, yymsp[-4].minor.yy242, yymsp[0].minor.yy202); + sqlite3ExprListSetName(pParse, yymsp[-4].minor.yy242, &yymsp[-2].minor.yy0, 1); +} + break; + case 153: /* setlist ::= setlist COMMA LP idlist RP EQ expr */ +{ + yymsp[-6].minor.yy242 = sqlite3ExprListAppendVector(pParse, yymsp[-6].minor.yy242, yymsp[-3].minor.yy600, yymsp[0].minor.yy202); +} + break; + case 154: /* setlist ::= nm EQ expr */ +{ + yylhsminor.yy242 = sqlite3ExprListAppend(pParse, 0, yymsp[0].minor.yy202); + sqlite3ExprListSetName(pParse, yylhsminor.yy242, &yymsp[-2].minor.yy0, 1); +} + yymsp[-2].minor.yy242 = yylhsminor.yy242; + break; + case 155: /* setlist ::= LP idlist RP EQ expr */ +{ + yymsp[-4].minor.yy242 = sqlite3ExprListAppendVector(pParse, 0, yymsp[-3].minor.yy600, yymsp[0].minor.yy202); +} + break; + case 156: /* cmd ::= with insert_cmd INTO xfullname idlist_opt select upsert */ +{ + sqlite3Insert(pParse, yymsp[-3].minor.yy47, yymsp[-1].minor.yy539, yymsp[-2].minor.yy600, yymsp[-5].minor.yy192, yymsp[0].minor.yy318); +} + break; + case 157: /* cmd ::= with insert_cmd INTO xfullname idlist_opt DEFAULT VALUES */ +{ + sqlite3Insert(pParse, yymsp[-3].minor.yy47, 0, yymsp[-2].minor.yy600, yymsp[-5].minor.yy192, 0); +} + break; + case 158: /* upsert ::= */ +{ yymsp[1].minor.yy318 = 0; } + break; + case 159: /* upsert ::= ON CONFLICT LP sortlist RP where_opt DO UPDATE SET setlist where_opt */ +{ yymsp[-10].minor.yy318 = sqlite3UpsertNew(pParse->db,yymsp[-7].minor.yy242,yymsp[-5].minor.yy202,yymsp[-1].minor.yy242,yymsp[0].minor.yy202);} + break; + case 160: /* upsert ::= ON CONFLICT LP sortlist RP where_opt DO NOTHING */ +{ yymsp[-7].minor.yy318 = sqlite3UpsertNew(pParse->db,yymsp[-4].minor.yy242,yymsp[-2].minor.yy202,0,0); } + break; + case 161: /* upsert ::= ON CONFLICT DO NOTHING */ +{ yymsp[-3].minor.yy318 = sqlite3UpsertNew(pParse->db,0,0,0,0); } + break; + case 165: /* idlist_opt ::= LP idlist RP */ +{yymsp[-2].minor.yy600 = yymsp[-1].minor.yy600;} + break; + case 166: /* idlist ::= idlist COMMA nm */ +{yymsp[-2].minor.yy600 = sqlite3IdListAppend(pParse,yymsp[-2].minor.yy600,&yymsp[0].minor.yy0);} + break; + case 167: /* idlist ::= nm */ +{yymsp[0].minor.yy600 = sqlite3IdListAppend(pParse,0,&yymsp[0].minor.yy0); /*A-overwrites-Y*/} + break; + case 168: /* expr ::= LP expr RP */ +{yymsp[-2].minor.yy202 = yymsp[-1].minor.yy202;} + break; + case 169: /* expr ::= ID|INDEXED */ + case 170: /* expr ::= JOIN_KW */ yytestcase(yyruleno==170); +{yymsp[0].minor.yy202=tokenExpr(pParse,TK_ID,yymsp[0].minor.yy0); /*A-overwrites-X*/} + break; + case 171: /* expr ::= nm DOT nm */ +{ + Expr *temp1 = sqlite3ExprAlloc(pParse->db, TK_ID, &yymsp[-2].minor.yy0, 1); + Expr *temp2 = sqlite3ExprAlloc(pParse->db, TK_ID, &yymsp[0].minor.yy0, 1); + if( IN_RENAME_OBJECT ){ + sqlite3RenameTokenMap(pParse, (void*)temp2, &yymsp[0].minor.yy0); + sqlite3RenameTokenMap(pParse, (void*)temp1, &yymsp[-2].minor.yy0); + } + yylhsminor.yy202 = sqlite3PExpr(pParse, TK_DOT, temp1, temp2); +} + yymsp[-2].minor.yy202 = yylhsminor.yy202; + break; + case 172: /* expr ::= nm DOT nm DOT nm */ +{ + Expr *temp1 = sqlite3ExprAlloc(pParse->db, TK_ID, &yymsp[-4].minor.yy0, 1); + Expr *temp2 = sqlite3ExprAlloc(pParse->db, TK_ID, &yymsp[-2].minor.yy0, 1); + Expr *temp3 = sqlite3ExprAlloc(pParse->db, TK_ID, &yymsp[0].minor.yy0, 1); + Expr *temp4 = sqlite3PExpr(pParse, TK_DOT, temp2, temp3); + if( IN_RENAME_OBJECT ){ + sqlite3RenameTokenMap(pParse, (void*)temp3, &yymsp[0].minor.yy0); + sqlite3RenameTokenMap(pParse, (void*)temp2, &yymsp[-2].minor.yy0); + } + yylhsminor.yy202 = sqlite3PExpr(pParse, TK_DOT, temp1, temp4); +} + yymsp[-4].minor.yy202 = yylhsminor.yy202; + break; + case 173: /* term ::= NULL|FLOAT|BLOB */ + case 174: /* term ::= STRING */ yytestcase(yyruleno==174); +{yymsp[0].minor.yy202=tokenExpr(pParse,yymsp[0].major,yymsp[0].minor.yy0); /*A-overwrites-X*/} + break; + case 175: /* term ::= INTEGER */ +{ + yylhsminor.yy202 = sqlite3ExprAlloc(pParse->db, TK_INTEGER, &yymsp[0].minor.yy0, 1); +} + yymsp[0].minor.yy202 = yylhsminor.yy202; + break; + case 176: /* expr ::= VARIABLE */ +{ + if( !(yymsp[0].minor.yy0.z[0]=='#' && sqlite3Isdigit(yymsp[0].minor.yy0.z[1])) ){ + u32 n = yymsp[0].minor.yy0.n; + yymsp[0].minor.yy202 = tokenExpr(pParse, TK_VARIABLE, yymsp[0].minor.yy0); + sqlite3ExprAssignVarNumber(pParse, yymsp[0].minor.yy202, n); + }else{ + /* When doing a nested parse, one can include terms in an expression + ** that look like this: #1 #2 ... These terms refer to registers + ** in the virtual machine. #N is the N-th register. */ + Token t = yymsp[0].minor.yy0; /*A-overwrites-X*/ + assert( t.n>=2 ); + if( pParse->nested==0 ){ + sqlite3ErrorMsg(pParse, "near \"%T\": syntax error", &t); + yymsp[0].minor.yy202 = 0; + }else{ + yymsp[0].minor.yy202 = sqlite3PExpr(pParse, TK_REGISTER, 0, 0); + if( yymsp[0].minor.yy202 ) sqlite3GetInt32(&t.z[1], &yymsp[0].minor.yy202->iTable); + } + } +} + break; + case 177: /* expr ::= expr COLLATE ID|STRING */ +{ + yymsp[-2].minor.yy202 = sqlite3ExprAddCollateToken(pParse, yymsp[-2].minor.yy202, &yymsp[0].minor.yy0, 1); +} + break; + case 178: /* expr ::= CAST LP expr AS typetoken RP */ +{ + yymsp[-5].minor.yy202 = sqlite3ExprAlloc(pParse->db, TK_CAST, &yymsp[-1].minor.yy0, 1); + sqlite3ExprAttachSubtrees(pParse->db, yymsp[-5].minor.yy202, yymsp[-3].minor.yy202, 0); +} + break; + case 179: /* expr ::= ID|INDEXED LP distinct exprlist RP */ +{ + yylhsminor.yy202 = sqlite3ExprFunction(pParse, yymsp[-1].minor.yy242, &yymsp[-4].minor.yy0, yymsp[-2].minor.yy192); +} + yymsp[-4].minor.yy202 = yylhsminor.yy202; + break; + case 180: /* expr ::= ID|INDEXED LP STAR RP */ +{ + yylhsminor.yy202 = sqlite3ExprFunction(pParse, 0, &yymsp[-3].minor.yy0, 0); +} + yymsp[-3].minor.yy202 = yylhsminor.yy202; + break; + case 181: /* expr ::= ID|INDEXED LP distinct exprlist RP filter_over */ +{ + yylhsminor.yy202 = sqlite3ExprFunction(pParse, yymsp[-2].minor.yy242, &yymsp[-5].minor.yy0, yymsp[-3].minor.yy192); + sqlite3WindowAttach(pParse, yylhsminor.yy202, yymsp[0].minor.yy303); +} + yymsp[-5].minor.yy202 = yylhsminor.yy202; + break; + case 182: /* expr ::= ID|INDEXED LP STAR RP filter_over */ +{ + yylhsminor.yy202 = sqlite3ExprFunction(pParse, 0, &yymsp[-4].minor.yy0, 0); + sqlite3WindowAttach(pParse, yylhsminor.yy202, yymsp[0].minor.yy303); +} + yymsp[-4].minor.yy202 = yylhsminor.yy202; + break; + case 183: /* term ::= CTIME_KW */ +{ + yylhsminor.yy202 = sqlite3ExprFunction(pParse, 0, &yymsp[0].minor.yy0, 0); +} + yymsp[0].minor.yy202 = yylhsminor.yy202; + break; + case 184: /* expr ::= LP nexprlist COMMA expr RP */ +{ + ExprList *pList = sqlite3ExprListAppend(pParse, yymsp[-3].minor.yy242, yymsp[-1].minor.yy202); + yymsp[-4].minor.yy202 = sqlite3PExpr(pParse, TK_VECTOR, 0, 0); + if( yymsp[-4].minor.yy202 ){ + yymsp[-4].minor.yy202->x.pList = pList; + if( ALWAYS(pList->nExpr) ){ + yymsp[-4].minor.yy202->flags |= pList->a[0].pExpr->flags & EP_Propagate; + } + }else{ + sqlite3ExprListDelete(pParse->db, pList); + } +} + break; + case 185: /* expr ::= expr AND expr */ +{yymsp[-2].minor.yy202=sqlite3ExprAnd(pParse,yymsp[-2].minor.yy202,yymsp[0].minor.yy202);} + break; + case 186: /* expr ::= expr OR expr */ + case 187: /* expr ::= expr LT|GT|GE|LE expr */ yytestcase(yyruleno==187); + case 188: /* expr ::= expr EQ|NE expr */ yytestcase(yyruleno==188); + case 189: /* expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr */ yytestcase(yyruleno==189); + case 190: /* expr ::= expr PLUS|MINUS expr */ yytestcase(yyruleno==190); + case 191: /* expr ::= expr STAR|SLASH|REM expr */ yytestcase(yyruleno==191); + case 192: /* expr ::= expr CONCAT expr */ yytestcase(yyruleno==192); +{yymsp[-2].minor.yy202=sqlite3PExpr(pParse,yymsp[-1].major,yymsp[-2].minor.yy202,yymsp[0].minor.yy202);} + break; + case 193: /* likeop ::= NOT LIKE_KW|MATCH */ +{yymsp[-1].minor.yy0=yymsp[0].minor.yy0; yymsp[-1].minor.yy0.n|=0x80000000; /*yymsp[-1].minor.yy0-overwrite-yymsp[0].minor.yy0*/} + break; + case 194: /* expr ::= expr likeop expr */ +{ + ExprList *pList; + int bNot = yymsp[-1].minor.yy0.n & 0x80000000; + yymsp[-1].minor.yy0.n &= 0x7fffffff; + pList = sqlite3ExprListAppend(pParse,0, yymsp[0].minor.yy202); + pList = sqlite3ExprListAppend(pParse,pList, yymsp[-2].minor.yy202); + yymsp[-2].minor.yy202 = sqlite3ExprFunction(pParse, pList, &yymsp[-1].minor.yy0, 0); + if( bNot ) yymsp[-2].minor.yy202 = sqlite3PExpr(pParse, TK_NOT, yymsp[-2].minor.yy202, 0); + if( yymsp[-2].minor.yy202 ) yymsp[-2].minor.yy202->flags |= EP_InfixFunc; +} + break; + case 195: /* expr ::= expr likeop expr ESCAPE expr */ +{ + ExprList *pList; + int bNot = yymsp[-3].minor.yy0.n & 0x80000000; + yymsp[-3].minor.yy0.n &= 0x7fffffff; + pList = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy202); + pList = sqlite3ExprListAppend(pParse,pList, yymsp[-4].minor.yy202); + pList = sqlite3ExprListAppend(pParse,pList, yymsp[0].minor.yy202); + yymsp[-4].minor.yy202 = sqlite3ExprFunction(pParse, pList, &yymsp[-3].minor.yy0, 0); + if( bNot ) yymsp[-4].minor.yy202 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy202, 0); + if( yymsp[-4].minor.yy202 ) yymsp[-4].minor.yy202->flags |= EP_InfixFunc; +} + break; + case 196: /* expr ::= expr ISNULL|NOTNULL */ +{yymsp[-1].minor.yy202 = sqlite3PExpr(pParse,yymsp[0].major,yymsp[-1].minor.yy202,0);} + break; + case 197: /* expr ::= expr NOT NULL */ +{yymsp[-2].minor.yy202 = sqlite3PExpr(pParse,TK_NOTNULL,yymsp[-2].minor.yy202,0);} + break; + case 198: /* expr ::= expr IS expr */ +{ + yymsp[-2].minor.yy202 = sqlite3PExpr(pParse,TK_IS,yymsp[-2].minor.yy202,yymsp[0].minor.yy202); + binaryToUnaryIfNull(pParse, yymsp[0].minor.yy202, yymsp[-2].minor.yy202, TK_ISNULL); +} + break; + case 199: /* expr ::= expr IS NOT expr */ +{ + yymsp[-3].minor.yy202 = sqlite3PExpr(pParse,TK_ISNOT,yymsp[-3].minor.yy202,yymsp[0].minor.yy202); + binaryToUnaryIfNull(pParse, yymsp[0].minor.yy202, yymsp[-3].minor.yy202, TK_NOTNULL); +} + break; + case 200: /* expr ::= NOT expr */ + case 201: /* expr ::= BITNOT expr */ yytestcase(yyruleno==201); +{yymsp[-1].minor.yy202 = sqlite3PExpr(pParse, yymsp[-1].major, yymsp[0].minor.yy202, 0);/*A-overwrites-B*/} + break; + case 202: /* expr ::= PLUS|MINUS expr */ +{ + yymsp[-1].minor.yy202 = sqlite3PExpr(pParse, yymsp[-1].major==TK_PLUS ? TK_UPLUS : TK_UMINUS, yymsp[0].minor.yy202, 0); + /*A-overwrites-B*/ +} + break; + case 203: /* between_op ::= BETWEEN */ + case 206: /* in_op ::= IN */ yytestcase(yyruleno==206); +{yymsp[0].minor.yy192 = 0;} + break; + case 205: /* expr ::= expr between_op expr AND expr */ +{ + ExprList *pList = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy202); + pList = sqlite3ExprListAppend(pParse,pList, yymsp[0].minor.yy202); + yymsp[-4].minor.yy202 = sqlite3PExpr(pParse, TK_BETWEEN, yymsp[-4].minor.yy202, 0); + if( yymsp[-4].minor.yy202 ){ + yymsp[-4].minor.yy202->x.pList = pList; + }else{ + sqlite3ExprListDelete(pParse->db, pList); + } + if( yymsp[-3].minor.yy192 ) yymsp[-4].minor.yy202 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy202, 0); +} + break; + case 208: /* expr ::= expr in_op LP exprlist RP */ +{ + if( yymsp[-1].minor.yy242==0 ){ + /* Expressions of the form + ** + ** expr1 IN () + ** expr1 NOT IN () + ** + ** simplify to constants 0 (false) and 1 (true), respectively, + ** regardless of the value of expr1. + */ + sqlite3ExprUnmapAndDelete(pParse, yymsp[-4].minor.yy202); + yymsp[-4].minor.yy202 = sqlite3Expr(pParse->db, TK_INTEGER, yymsp[-3].minor.yy192 ? "1" : "0"); + }else if( yymsp[-1].minor.yy242->nExpr==1 && sqlite3ExprIsConstant(yymsp[-1].minor.yy242->a[0].pExpr) ){ + Expr *pRHS = yymsp[-1].minor.yy242->a[0].pExpr; + yymsp[-1].minor.yy242->a[0].pExpr = 0; + sqlite3ExprListDelete(pParse->db, yymsp[-1].minor.yy242); + pRHS = sqlite3PExpr(pParse, TK_UPLUS, pRHS, 0); + yymsp[-4].minor.yy202 = sqlite3PExpr(pParse, TK_EQ, yymsp[-4].minor.yy202, pRHS); + if( yymsp[-3].minor.yy192 ) yymsp[-4].minor.yy202 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy202, 0); + }else{ + yymsp[-4].minor.yy202 = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy202, 0); + if( yymsp[-4].minor.yy202 ){ + yymsp[-4].minor.yy202->x.pList = yymsp[-1].minor.yy242; + sqlite3ExprSetHeightAndFlags(pParse, yymsp[-4].minor.yy202); + }else{ + sqlite3ExprListDelete(pParse->db, yymsp[-1].minor.yy242); + } + if( yymsp[-3].minor.yy192 ) yymsp[-4].minor.yy202 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy202, 0); + } + } + break; + case 209: /* expr ::= LP select RP */ +{ + yymsp[-2].minor.yy202 = sqlite3PExpr(pParse, TK_SELECT, 0, 0); + sqlite3PExprAddSelect(pParse, yymsp[-2].minor.yy202, yymsp[-1].minor.yy539); + } + break; + case 210: /* expr ::= expr in_op LP select RP */ +{ + yymsp[-4].minor.yy202 = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy202, 0); + sqlite3PExprAddSelect(pParse, yymsp[-4].minor.yy202, yymsp[-1].minor.yy539); + if( yymsp[-3].minor.yy192 ) yymsp[-4].minor.yy202 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy202, 0); + } + break; + case 211: /* expr ::= expr in_op nm dbnm paren_exprlist */ +{ + SrcList *pSrc = sqlite3SrcListAppend(pParse, 0,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0); + Select *pSelect = sqlite3SelectNew(pParse, 0,pSrc,0,0,0,0,0,0); + if( yymsp[0].minor.yy242 ) sqlite3SrcListFuncArgs(pParse, pSelect ? pSrc : 0, yymsp[0].minor.yy242); + yymsp[-4].minor.yy202 = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy202, 0); + sqlite3PExprAddSelect(pParse, yymsp[-4].minor.yy202, pSelect); + if( yymsp[-3].minor.yy192 ) yymsp[-4].minor.yy202 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy202, 0); + } + break; + case 212: /* expr ::= EXISTS LP select RP */ +{ + Expr *p; + p = yymsp[-3].minor.yy202 = sqlite3PExpr(pParse, TK_EXISTS, 0, 0); + sqlite3PExprAddSelect(pParse, p, yymsp[-1].minor.yy539); + } + break; + case 213: /* expr ::= CASE case_operand case_exprlist case_else END */ +{ + yymsp[-4].minor.yy202 = sqlite3PExpr(pParse, TK_CASE, yymsp[-3].minor.yy202, 0); + if( yymsp[-4].minor.yy202 ){ + yymsp[-4].minor.yy202->x.pList = yymsp[-1].minor.yy202 ? sqlite3ExprListAppend(pParse,yymsp[-2].minor.yy242,yymsp[-1].minor.yy202) : yymsp[-2].minor.yy242; + sqlite3ExprSetHeightAndFlags(pParse, yymsp[-4].minor.yy202); + }else{ + sqlite3ExprListDelete(pParse->db, yymsp[-2].minor.yy242); + sqlite3ExprDelete(pParse->db, yymsp[-1].minor.yy202); + } +} + break; + case 214: /* case_exprlist ::= case_exprlist WHEN expr THEN expr */ +{ + yymsp[-4].minor.yy242 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy242, yymsp[-2].minor.yy202); + yymsp[-4].minor.yy242 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy242, yymsp[0].minor.yy202); +} + break; + case 215: /* case_exprlist ::= WHEN expr THEN expr */ +{ + yymsp[-3].minor.yy242 = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy202); + yymsp[-3].minor.yy242 = sqlite3ExprListAppend(pParse,yymsp[-3].minor.yy242, yymsp[0].minor.yy202); +} + break; + case 218: /* case_operand ::= expr */ +{yymsp[0].minor.yy202 = yymsp[0].minor.yy202; /*A-overwrites-X*/} + break; + case 221: /* nexprlist ::= nexprlist COMMA expr */ +{yymsp[-2].minor.yy242 = sqlite3ExprListAppend(pParse,yymsp[-2].minor.yy242,yymsp[0].minor.yy202);} + break; + case 222: /* nexprlist ::= expr */ +{yymsp[0].minor.yy242 = sqlite3ExprListAppend(pParse,0,yymsp[0].minor.yy202); /*A-overwrites-Y*/} + break; + case 224: /* paren_exprlist ::= LP exprlist RP */ + case 229: /* eidlist_opt ::= LP eidlist RP */ yytestcase(yyruleno==229); +{yymsp[-2].minor.yy242 = yymsp[-1].minor.yy242;} + break; + case 225: /* cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt */ +{ + sqlite3CreateIndex(pParse, &yymsp[-7].minor.yy0, &yymsp[-6].minor.yy0, + sqlite3SrcListAppend(pParse,0,&yymsp[-4].minor.yy0,0), yymsp[-2].minor.yy242, yymsp[-10].minor.yy192, + &yymsp[-11].minor.yy0, yymsp[0].minor.yy202, SQLITE_SO_ASC, yymsp[-8].minor.yy192, SQLITE_IDXTYPE_APPDEF); + if( IN_RENAME_OBJECT && pParse->pNewIndex ){ + sqlite3RenameTokenMap(pParse, pParse->pNewIndex->zName, &yymsp[-4].minor.yy0); + } +} + break; + case 226: /* uniqueflag ::= UNIQUE */ + case 268: /* raisetype ::= ABORT */ yytestcase(yyruleno==268); +{yymsp[0].minor.yy192 = OE_Abort;} + break; + case 227: /* uniqueflag ::= */ +{yymsp[1].minor.yy192 = OE_None;} + break; + case 230: /* eidlist ::= eidlist COMMA nm collate sortorder */ +{ + yymsp[-4].minor.yy242 = parserAddExprIdListTerm(pParse, yymsp[-4].minor.yy242, &yymsp[-2].minor.yy0, yymsp[-1].minor.yy192, yymsp[0].minor.yy192); +} + break; + case 231: /* eidlist ::= nm collate sortorder */ +{ + yymsp[-2].minor.yy242 = parserAddExprIdListTerm(pParse, 0, &yymsp[-2].minor.yy0, yymsp[-1].minor.yy192, yymsp[0].minor.yy192); /*A-overwrites-Y*/ +} + break; + case 234: /* cmd ::= DROP INDEX ifexists fullname */ +{sqlite3DropIndex(pParse, yymsp[0].minor.yy47, yymsp[-1].minor.yy192);} + break; + case 235: /* cmd ::= VACUUM vinto */ +{sqlite3Vacuum(pParse,0,yymsp[0].minor.yy202);} + break; + case 236: /* cmd ::= VACUUM nm vinto */ +{sqlite3Vacuum(pParse,&yymsp[-1].minor.yy0,yymsp[0].minor.yy202);} + break; + case 239: /* cmd ::= PRAGMA nm dbnm */ +{sqlite3Pragma(pParse,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0,0,0);} + break; + case 240: /* cmd ::= PRAGMA nm dbnm EQ nmnum */ +{sqlite3Pragma(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0,0);} + break; + case 241: /* cmd ::= PRAGMA nm dbnm LP nmnum RP */ +{sqlite3Pragma(pParse,&yymsp[-4].minor.yy0,&yymsp[-3].minor.yy0,&yymsp[-1].minor.yy0,0);} + break; + case 242: /* cmd ::= PRAGMA nm dbnm EQ minus_num */ +{sqlite3Pragma(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0,1);} + break; + case 243: /* cmd ::= PRAGMA nm dbnm LP minus_num RP */ +{sqlite3Pragma(pParse,&yymsp[-4].minor.yy0,&yymsp[-3].minor.yy0,&yymsp[-1].minor.yy0,1);} + break; + case 246: /* cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END */ +{ + Token all; + all.z = yymsp[-3].minor.yy0.z; + all.n = (int)(yymsp[0].minor.yy0.z - yymsp[-3].minor.yy0.z) + yymsp[0].minor.yy0.n; + sqlite3FinishTrigger(pParse, yymsp[-1].minor.yy447, &all); +} + break; + case 247: /* trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */ +{ + sqlite3BeginTrigger(pParse, &yymsp[-7].minor.yy0, &yymsp[-6].minor.yy0, yymsp[-5].minor.yy192, yymsp[-4].minor.yy230.a, yymsp[-4].minor.yy230.b, yymsp[-2].minor.yy47, yymsp[0].minor.yy202, yymsp[-10].minor.yy192, yymsp[-8].minor.yy192); + yymsp[-10].minor.yy0 = (yymsp[-6].minor.yy0.n==0?yymsp[-7].minor.yy0:yymsp[-6].minor.yy0); /*A-overwrites-T*/ +} + break; + case 248: /* trigger_time ::= BEFORE|AFTER */ +{ yymsp[0].minor.yy192 = yymsp[0].major; /*A-overwrites-X*/ } + break; + case 249: /* trigger_time ::= INSTEAD OF */ +{ yymsp[-1].minor.yy192 = TK_INSTEAD;} + break; + case 250: /* trigger_time ::= */ +{ yymsp[1].minor.yy192 = TK_BEFORE; } + break; + case 251: /* trigger_event ::= DELETE|INSERT */ + case 252: /* trigger_event ::= UPDATE */ yytestcase(yyruleno==252); +{yymsp[0].minor.yy230.a = yymsp[0].major; /*A-overwrites-X*/ yymsp[0].minor.yy230.b = 0;} + break; + case 253: /* trigger_event ::= UPDATE OF idlist */ +{yymsp[-2].minor.yy230.a = TK_UPDATE; yymsp[-2].minor.yy230.b = yymsp[0].minor.yy600;} + break; + case 254: /* when_clause ::= */ + case 273: /* key_opt ::= */ yytestcase(yyruleno==273); +{ yymsp[1].minor.yy202 = 0; } + break; + case 255: /* when_clause ::= WHEN expr */ + case 274: /* key_opt ::= KEY expr */ yytestcase(yyruleno==274); +{ yymsp[-1].minor.yy202 = yymsp[0].minor.yy202; } + break; + case 256: /* trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */ +{ + assert( yymsp[-2].minor.yy447!=0 ); + yymsp[-2].minor.yy447->pLast->pNext = yymsp[-1].minor.yy447; + yymsp[-2].minor.yy447->pLast = yymsp[-1].minor.yy447; +} + break; + case 257: /* trigger_cmd_list ::= trigger_cmd SEMI */ +{ + assert( yymsp[-1].minor.yy447!=0 ); + yymsp[-1].minor.yy447->pLast = yymsp[-1].minor.yy447; +} + break; + case 258: /* trnm ::= nm DOT nm */ +{ + yymsp[-2].minor.yy0 = yymsp[0].minor.yy0; + sqlite3ErrorMsg(pParse, + "qualified table names are not allowed on INSERT, UPDATE, and DELETE " + "statements within triggers"); +} + break; + case 259: /* tridxby ::= INDEXED BY nm */ +{ + sqlite3ErrorMsg(pParse, + "the INDEXED BY clause is not allowed on UPDATE or DELETE statements " + "within triggers"); +} + break; + case 260: /* tridxby ::= NOT INDEXED */ +{ + sqlite3ErrorMsg(pParse, + "the NOT INDEXED clause is not allowed on UPDATE or DELETE statements " + "within triggers"); +} + break; + case 261: /* trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist where_opt scanpt */ +{yylhsminor.yy447 = sqlite3TriggerUpdateStep(pParse, &yymsp[-5].minor.yy0, yymsp[-2].minor.yy242, yymsp[-1].minor.yy202, yymsp[-6].minor.yy192, yymsp[-7].minor.yy0.z, yymsp[0].minor.yy436);} + yymsp[-7].minor.yy447 = yylhsminor.yy447; + break; + case 262: /* trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt */ +{ + yylhsminor.yy447 = sqlite3TriggerInsertStep(pParse,&yymsp[-4].minor.yy0,yymsp[-3].minor.yy600,yymsp[-2].minor.yy539,yymsp[-6].minor.yy192,yymsp[-1].minor.yy318,yymsp[-7].minor.yy436,yymsp[0].minor.yy436);/*yylhsminor.yy447-overwrites-yymsp[-6].minor.yy192*/ +} + yymsp[-7].minor.yy447 = yylhsminor.yy447; + break; + case 263: /* trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt */ +{yylhsminor.yy447 = sqlite3TriggerDeleteStep(pParse, &yymsp[-3].minor.yy0, yymsp[-1].minor.yy202, yymsp[-5].minor.yy0.z, yymsp[0].minor.yy436);} + yymsp[-5].minor.yy447 = yylhsminor.yy447; + break; + case 264: /* trigger_cmd ::= scanpt select scanpt */ +{yylhsminor.yy447 = sqlite3TriggerSelectStep(pParse->db, yymsp[-1].minor.yy539, yymsp[-2].minor.yy436, yymsp[0].minor.yy436); /*yylhsminor.yy447-overwrites-yymsp[-1].minor.yy539*/} + yymsp[-2].minor.yy447 = yylhsminor.yy447; + break; + case 265: /* expr ::= RAISE LP IGNORE RP */ +{ + yymsp[-3].minor.yy202 = sqlite3PExpr(pParse, TK_RAISE, 0, 0); + if( yymsp[-3].minor.yy202 ){ + yymsp[-3].minor.yy202->affExpr = OE_Ignore; + } +} + break; + case 266: /* expr ::= RAISE LP raisetype COMMA nm RP */ +{ + yymsp[-5].minor.yy202 = sqlite3ExprAlloc(pParse->db, TK_RAISE, &yymsp[-1].minor.yy0, 1); + if( yymsp[-5].minor.yy202 ) { + yymsp[-5].minor.yy202->affExpr = (char)yymsp[-3].minor.yy192; + } +} + break; + case 267: /* raisetype ::= ROLLBACK */ +{yymsp[0].minor.yy192 = OE_Rollback;} + break; + case 269: /* raisetype ::= FAIL */ +{yymsp[0].minor.yy192 = OE_Fail;} + break; + case 270: /* cmd ::= DROP TRIGGER ifexists fullname */ +{ + sqlite3DropTrigger(pParse,yymsp[0].minor.yy47,yymsp[-1].minor.yy192); +} + break; + case 271: /* cmd ::= ATTACH database_kw_opt expr AS expr key_opt */ +{ + sqlite3Attach(pParse, yymsp[-3].minor.yy202, yymsp[-1].minor.yy202, yymsp[0].minor.yy202); +} + break; + case 272: /* cmd ::= DETACH database_kw_opt expr */ +{ + sqlite3Detach(pParse, yymsp[0].minor.yy202); +} + break; + case 275: /* cmd ::= REINDEX */ +{sqlite3Reindex(pParse, 0, 0);} + break; + case 276: /* cmd ::= REINDEX nm dbnm */ +{sqlite3Reindex(pParse, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0);} + break; + case 277: /* cmd ::= ANALYZE */ +{sqlite3Analyze(pParse, 0, 0);} + break; + case 278: /* cmd ::= ANALYZE nm dbnm */ +{sqlite3Analyze(pParse, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0);} + break; + case 279: /* cmd ::= ALTER TABLE fullname RENAME TO nm */ +{ + sqlite3AlterRenameTable(pParse,yymsp[-3].minor.yy47,&yymsp[0].minor.yy0); +} + break; + case 280: /* cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist */ +{ + yymsp[-1].minor.yy0.n = (int)(pParse->sLastToken.z-yymsp[-1].minor.yy0.z) + pParse->sLastToken.n; + sqlite3AlterFinishAddColumn(pParse, &yymsp[-1].minor.yy0); +} + break; + case 281: /* add_column_fullname ::= fullname */ +{ + disableLookaside(pParse); + sqlite3AlterBeginAddColumn(pParse, yymsp[0].minor.yy47); +} + break; + case 282: /* cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm */ +{ + sqlite3AlterRenameColumn(pParse, yymsp[-5].minor.yy47, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); +} + break; + case 283: /* cmd ::= create_vtab */ +{sqlite3VtabFinishParse(pParse,0);} + break; + case 284: /* cmd ::= create_vtab LP vtabarglist RP */ +{sqlite3VtabFinishParse(pParse,&yymsp[0].minor.yy0);} + break; + case 285: /* create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm */ +{ + sqlite3VtabBeginParse(pParse, &yymsp[-3].minor.yy0, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-4].minor.yy192); +} + break; + case 286: /* vtabarg ::= */ +{sqlite3VtabArgInit(pParse);} + break; + case 287: /* vtabargtoken ::= ANY */ + case 288: /* vtabargtoken ::= lp anylist RP */ yytestcase(yyruleno==288); + case 289: /* lp ::= LP */ yytestcase(yyruleno==289); +{sqlite3VtabArgExtend(pParse,&yymsp[0].minor.yy0);} + break; + case 290: /* with ::= WITH wqlist */ + case 291: /* with ::= WITH RECURSIVE wqlist */ yytestcase(yyruleno==291); +{ sqlite3WithPush(pParse, yymsp[0].minor.yy131, 1); } + break; + case 292: /* wqlist ::= nm eidlist_opt AS LP select RP */ +{ + yymsp[-5].minor.yy131 = sqlite3WithAdd(pParse, 0, &yymsp[-5].minor.yy0, yymsp[-4].minor.yy242, yymsp[-1].minor.yy539); /*A-overwrites-X*/ +} + break; + case 293: /* wqlist ::= wqlist COMMA nm eidlist_opt AS LP select RP */ +{ + yymsp[-7].minor.yy131 = sqlite3WithAdd(pParse, yymsp[-7].minor.yy131, &yymsp[-5].minor.yy0, yymsp[-4].minor.yy242, yymsp[-1].minor.yy539); +} + break; + case 294: /* windowdefn_list ::= windowdefn */ +{ yylhsminor.yy303 = yymsp[0].minor.yy303; } + yymsp[0].minor.yy303 = yylhsminor.yy303; + break; + case 295: /* windowdefn_list ::= windowdefn_list COMMA windowdefn */ +{ + assert( yymsp[0].minor.yy303!=0 ); + sqlite3WindowChain(pParse, yymsp[0].minor.yy303, yymsp[-2].minor.yy303); + yymsp[0].minor.yy303->pNextWin = yymsp[-2].minor.yy303; + yylhsminor.yy303 = yymsp[0].minor.yy303; +} + yymsp[-2].minor.yy303 = yylhsminor.yy303; + break; + case 296: /* windowdefn ::= nm AS LP window RP */ +{ + if( ALWAYS(yymsp[-1].minor.yy303) ){ + yymsp[-1].minor.yy303->zName = sqlite3DbStrNDup(pParse->db, yymsp[-4].minor.yy0.z, yymsp[-4].minor.yy0.n); + } + yylhsminor.yy303 = yymsp[-1].minor.yy303; +} + yymsp[-4].minor.yy303 = yylhsminor.yy303; + break; + case 297: /* window ::= PARTITION BY nexprlist orderby_opt frame_opt */ +{ + yymsp[-4].minor.yy303 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy303, yymsp[-2].minor.yy242, yymsp[-1].minor.yy242, 0); +} + break; + case 298: /* window ::= nm PARTITION BY nexprlist orderby_opt frame_opt */ +{ + yylhsminor.yy303 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy303, yymsp[-2].minor.yy242, yymsp[-1].minor.yy242, &yymsp[-5].minor.yy0); +} + yymsp[-5].minor.yy303 = yylhsminor.yy303; + break; + case 299: /* window ::= ORDER BY sortlist frame_opt */ +{ + yymsp[-3].minor.yy303 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy303, 0, yymsp[-1].minor.yy242, 0); +} + break; + case 300: /* window ::= nm ORDER BY sortlist frame_opt */ +{ + yylhsminor.yy303 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy303, 0, yymsp[-1].minor.yy242, &yymsp[-4].minor.yy0); +} + yymsp[-4].minor.yy303 = yylhsminor.yy303; + break; + case 301: /* window ::= frame_opt */ + case 320: /* filter_over ::= over_clause */ yytestcase(yyruleno==320); +{ + yylhsminor.yy303 = yymsp[0].minor.yy303; +} + yymsp[0].minor.yy303 = yylhsminor.yy303; + break; + case 302: /* window ::= nm frame_opt */ +{ + yylhsminor.yy303 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy303, 0, 0, &yymsp[-1].minor.yy0); +} + yymsp[-1].minor.yy303 = yylhsminor.yy303; + break; + case 303: /* frame_opt ::= */ +{ + yymsp[1].minor.yy303 = sqlite3WindowAlloc(pParse, 0, TK_UNBOUNDED, 0, TK_CURRENT, 0, 0); +} + break; + case 304: /* frame_opt ::= range_or_rows frame_bound_s frame_exclude_opt */ +{ + yylhsminor.yy303 = sqlite3WindowAlloc(pParse, yymsp[-2].minor.yy192, yymsp[-1].minor.yy77.eType, yymsp[-1].minor.yy77.pExpr, TK_CURRENT, 0, yymsp[0].minor.yy58); +} + yymsp[-2].minor.yy303 = yylhsminor.yy303; + break; + case 305: /* frame_opt ::= range_or_rows BETWEEN frame_bound_s AND frame_bound_e frame_exclude_opt */ +{ + yylhsminor.yy303 = sqlite3WindowAlloc(pParse, yymsp[-5].minor.yy192, yymsp[-3].minor.yy77.eType, yymsp[-3].minor.yy77.pExpr, yymsp[-1].minor.yy77.eType, yymsp[-1].minor.yy77.pExpr, yymsp[0].minor.yy58); +} + yymsp[-5].minor.yy303 = yylhsminor.yy303; + break; + case 307: /* frame_bound_s ::= frame_bound */ + case 309: /* frame_bound_e ::= frame_bound */ yytestcase(yyruleno==309); +{yylhsminor.yy77 = yymsp[0].minor.yy77;} + yymsp[0].minor.yy77 = yylhsminor.yy77; + break; + case 308: /* frame_bound_s ::= UNBOUNDED PRECEDING */ + case 310: /* frame_bound_e ::= UNBOUNDED FOLLOWING */ yytestcase(yyruleno==310); + case 312: /* frame_bound ::= CURRENT ROW */ yytestcase(yyruleno==312); +{yylhsminor.yy77.eType = yymsp[-1].major; yylhsminor.yy77.pExpr = 0;} + yymsp[-1].minor.yy77 = yylhsminor.yy77; + break; + case 311: /* frame_bound ::= expr PRECEDING|FOLLOWING */ +{yylhsminor.yy77.eType = yymsp[0].major; yylhsminor.yy77.pExpr = yymsp[-1].minor.yy202;} + yymsp[-1].minor.yy77 = yylhsminor.yy77; + break; + case 313: /* frame_exclude_opt ::= */ +{yymsp[1].minor.yy58 = 0;} + break; + case 314: /* frame_exclude_opt ::= EXCLUDE frame_exclude */ +{yymsp[-1].minor.yy58 = yymsp[0].minor.yy58;} + break; + case 315: /* frame_exclude ::= NO OTHERS */ + case 316: /* frame_exclude ::= CURRENT ROW */ yytestcase(yyruleno==316); +{yymsp[-1].minor.yy58 = yymsp[-1].major; /*A-overwrites-X*/} + break; + case 317: /* frame_exclude ::= GROUP|TIES */ +{yymsp[0].minor.yy58 = yymsp[0].major; /*A-overwrites-X*/} + break; + case 318: /* window_clause ::= WINDOW windowdefn_list */ +{ yymsp[-1].minor.yy303 = yymsp[0].minor.yy303; } + break; + case 319: /* filter_over ::= filter_clause over_clause */ +{ + yymsp[0].minor.yy303->pFilter = yymsp[-1].minor.yy202; + yylhsminor.yy303 = yymsp[0].minor.yy303; +} + yymsp[-1].minor.yy303 = yylhsminor.yy303; + break; + case 321: /* filter_over ::= filter_clause */ +{ + yylhsminor.yy303 = (Window*)sqlite3DbMallocZero(pParse->db, sizeof(Window)); + if( yylhsminor.yy303 ){ + yylhsminor.yy303->eFrmType = TK_FILTER; + yylhsminor.yy303->pFilter = yymsp[0].minor.yy202; + }else{ + sqlite3ExprDelete(pParse->db, yymsp[0].minor.yy202); + } +} + yymsp[0].minor.yy303 = yylhsminor.yy303; + break; + case 322: /* over_clause ::= OVER LP window RP */ +{ + yymsp[-3].minor.yy303 = yymsp[-1].minor.yy303; + assert( yymsp[-3].minor.yy303!=0 ); +} + break; + case 323: /* over_clause ::= OVER nm */ +{ + yymsp[-1].minor.yy303 = (Window*)sqlite3DbMallocZero(pParse->db, sizeof(Window)); + if( yymsp[-1].minor.yy303 ){ + yymsp[-1].minor.yy303->zName = sqlite3DbStrNDup(pParse->db, yymsp[0].minor.yy0.z, yymsp[0].minor.yy0.n); + } +} + break; + case 324: /* filter_clause ::= FILTER LP WHERE expr RP */ +{ yymsp[-4].minor.yy202 = yymsp[-1].minor.yy202; } + break; + default: + /* (325) input ::= cmdlist */ yytestcase(yyruleno==325); + /* (326) cmdlist ::= cmdlist ecmd */ yytestcase(yyruleno==326); + /* (327) cmdlist ::= ecmd (OPTIMIZED OUT) */ assert(yyruleno!=327); + /* (328) ecmd ::= SEMI */ yytestcase(yyruleno==328); + /* (329) ecmd ::= cmdx SEMI */ yytestcase(yyruleno==329); + /* (330) ecmd ::= explain cmdx SEMI (NEVER REDUCES) */ assert(yyruleno!=330); + /* (331) trans_opt ::= */ yytestcase(yyruleno==331); + /* (332) trans_opt ::= TRANSACTION */ yytestcase(yyruleno==332); + /* (333) trans_opt ::= TRANSACTION nm */ yytestcase(yyruleno==333); + /* (334) savepoint_opt ::= SAVEPOINT */ yytestcase(yyruleno==334); + /* (335) savepoint_opt ::= */ yytestcase(yyruleno==335); + /* (336) cmd ::= create_table create_table_args */ yytestcase(yyruleno==336); + /* (337) columnlist ::= columnlist COMMA columnname carglist */ yytestcase(yyruleno==337); + /* (338) columnlist ::= columnname carglist */ yytestcase(yyruleno==338); + /* (339) nm ::= ID|INDEXED */ yytestcase(yyruleno==339); + /* (340) nm ::= STRING */ yytestcase(yyruleno==340); + /* (341) nm ::= JOIN_KW */ yytestcase(yyruleno==341); + /* (342) typetoken ::= typename */ yytestcase(yyruleno==342); + /* (343) typename ::= ID|STRING */ yytestcase(yyruleno==343); + /* (344) signed ::= plus_num (OPTIMIZED OUT) */ assert(yyruleno!=344); + /* (345) signed ::= minus_num (OPTIMIZED OUT) */ assert(yyruleno!=345); + /* (346) carglist ::= carglist ccons */ yytestcase(yyruleno==346); + /* (347) carglist ::= */ yytestcase(yyruleno==347); + /* (348) ccons ::= NULL onconf */ yytestcase(yyruleno==348); + /* (349) ccons ::= GENERATED ALWAYS AS generated */ yytestcase(yyruleno==349); + /* (350) ccons ::= AS generated */ yytestcase(yyruleno==350); + /* (351) conslist_opt ::= COMMA conslist */ yytestcase(yyruleno==351); + /* (352) conslist ::= conslist tconscomma tcons */ yytestcase(yyruleno==352); + /* (353) conslist ::= tcons (OPTIMIZED OUT) */ assert(yyruleno!=353); + /* (354) tconscomma ::= */ yytestcase(yyruleno==354); + /* (355) defer_subclause_opt ::= defer_subclause (OPTIMIZED OUT) */ assert(yyruleno!=355); + /* (356) resolvetype ::= raisetype (OPTIMIZED OUT) */ assert(yyruleno!=356); + /* (357) selectnowith ::= oneselect (OPTIMIZED OUT) */ assert(yyruleno!=357); + /* (358) oneselect ::= values */ yytestcase(yyruleno==358); + /* (359) sclp ::= selcollist COMMA */ yytestcase(yyruleno==359); + /* (360) as ::= ID|STRING */ yytestcase(yyruleno==360); + /* (361) expr ::= term (OPTIMIZED OUT) */ assert(yyruleno!=361); + /* (362) likeop ::= LIKE_KW|MATCH */ yytestcase(yyruleno==362); + /* (363) exprlist ::= nexprlist */ yytestcase(yyruleno==363); + /* (364) nmnum ::= plus_num (OPTIMIZED OUT) */ assert(yyruleno!=364); + /* (365) nmnum ::= nm (OPTIMIZED OUT) */ assert(yyruleno!=365); + /* (366) nmnum ::= ON */ yytestcase(yyruleno==366); + /* (367) nmnum ::= DELETE */ yytestcase(yyruleno==367); + /* (368) nmnum ::= DEFAULT */ yytestcase(yyruleno==368); + /* (369) plus_num ::= INTEGER|FLOAT */ yytestcase(yyruleno==369); + /* (370) foreach_clause ::= */ yytestcase(yyruleno==370); + /* (371) foreach_clause ::= FOR EACH ROW */ yytestcase(yyruleno==371); + /* (372) trnm ::= nm */ yytestcase(yyruleno==372); + /* (373) tridxby ::= */ yytestcase(yyruleno==373); + /* (374) database_kw_opt ::= DATABASE */ yytestcase(yyruleno==374); + /* (375) database_kw_opt ::= */ yytestcase(yyruleno==375); + /* (376) kwcolumn_opt ::= */ yytestcase(yyruleno==376); + /* (377) kwcolumn_opt ::= COLUMNKW */ yytestcase(yyruleno==377); + /* (378) vtabarglist ::= vtabarg */ yytestcase(yyruleno==378); + /* (379) vtabarglist ::= vtabarglist COMMA vtabarg */ yytestcase(yyruleno==379); + /* (380) vtabarg ::= vtabarg vtabargtoken */ yytestcase(yyruleno==380); + /* (381) anylist ::= */ yytestcase(yyruleno==381); + /* (382) anylist ::= anylist LP anylist RP */ yytestcase(yyruleno==382); + /* (383) anylist ::= anylist ANY */ yytestcase(yyruleno==383); + /* (384) with ::= */ yytestcase(yyruleno==384); + break; +/********** End reduce actions ************************************************/ + }; + assert( yyrulenoYY_MAX_SHIFT && yyact<=YY_MAX_SHIFTREDUCE) ); + + /* It is not possible for a REDUCE to be followed by an error */ + assert( yyact!=YY_ERROR_ACTION ); + + yymsp += yysize+1; + yypParser->yytos = yymsp; + yymsp->stateno = (YYACTIONTYPE)yyact; + yymsp->major = (YYCODETYPE)yygoto; + yyTraceShift(yypParser, yyact, "... then shift"); + return yyact; +} + +/* +** The following code executes when the parse fails +*/ +#ifndef YYNOERRORRECOVERY +static void yy_parse_failed( + yyParser *yypParser /* The parser */ +){ + sqlite3ParserARG_FETCH + sqlite3ParserCTX_FETCH +#ifndef NDEBUG + if( yyTraceFILE ){ + fprintf(yyTraceFILE,"%sFail!\n",yyTracePrompt); + } +#endif + while( yypParser->yytos>yypParser->yystack ) yy_pop_parser_stack(yypParser); + /* Here code is inserted which will be executed whenever the + ** parser fails */ +/************ Begin %parse_failure code ***************************************/ +/************ End %parse_failure code *****************************************/ + sqlite3ParserARG_STORE /* Suppress warning about unused %extra_argument variable */ + sqlite3ParserCTX_STORE +} +#endif /* YYNOERRORRECOVERY */ + +/* +** The following code executes when a syntax error first occurs. +*/ +static void yy_syntax_error( + yyParser *yypParser, /* The parser */ + int yymajor, /* The major type of the error token */ + sqlite3ParserTOKENTYPE yyminor /* The minor type of the error token */ +){ + sqlite3ParserARG_FETCH + sqlite3ParserCTX_FETCH +#define TOKEN yyminor +/************ Begin %syntax_error code ****************************************/ + + UNUSED_PARAMETER(yymajor); /* Silence some compiler warnings */ + if( TOKEN.z[0] ){ + sqlite3ErrorMsg(pParse, "near \"%T\": syntax error", &TOKEN); + }else{ + sqlite3ErrorMsg(pParse, "incomplete input"); + } +/************ End %syntax_error code ******************************************/ + sqlite3ParserARG_STORE /* Suppress warning about unused %extra_argument variable */ + sqlite3ParserCTX_STORE +} + +/* +** The following is executed when the parser accepts +*/ +static void yy_accept( + yyParser *yypParser /* The parser */ +){ + sqlite3ParserARG_FETCH + sqlite3ParserCTX_FETCH +#ifndef NDEBUG + if( yyTraceFILE ){ + fprintf(yyTraceFILE,"%sAccept!\n",yyTracePrompt); + } +#endif +#ifndef YYNOERRORRECOVERY + yypParser->yyerrcnt = -1; +#endif + assert( yypParser->yytos==yypParser->yystack ); + /* Here code is inserted which will be executed whenever the + ** parser accepts */ +/*********** Begin %parse_accept code *****************************************/ +/*********** End %parse_accept code *******************************************/ + sqlite3ParserARG_STORE /* Suppress warning about unused %extra_argument variable */ + sqlite3ParserCTX_STORE +} + +/* The main parser program. +** The first argument is a pointer to a structure obtained from +** "sqlite3ParserAlloc" which describes the current state of the parser. +** The second argument is the major token number. The third is +** the minor token. The fourth optional argument is whatever the +** user wants (and specified in the grammar) and is available for +** use by the action routines. +** +** Inputs: +**
          +**
        • A pointer to the parser (an opaque structure.) +**
        • The major token number. +**
        • The minor token number. +**
        • An option argument of a grammar-specified type. +**
        +** +** Outputs: +** None. +*/ +SQLITE_PRIVATE void sqlite3Parser( + void *yyp, /* The parser */ + int yymajor, /* The major token code number */ + sqlite3ParserTOKENTYPE yyminor /* The value for the token */ + sqlite3ParserARG_PDECL /* Optional %extra_argument parameter */ +){ + YYMINORTYPE yyminorunion; + YYACTIONTYPE yyact; /* The parser action. */ +#if !defined(YYERRORSYMBOL) && !defined(YYNOERRORRECOVERY) + int yyendofinput; /* True if we are at the end of input */ +#endif +#ifdef YYERRORSYMBOL + int yyerrorhit = 0; /* True if yymajor has invoked an error */ +#endif + yyParser *yypParser = (yyParser*)yyp; /* The parser */ + sqlite3ParserCTX_FETCH + sqlite3ParserARG_STORE + + assert( yypParser->yytos!=0 ); +#if !defined(YYERRORSYMBOL) && !defined(YYNOERRORRECOVERY) + yyendofinput = (yymajor==0); +#endif + + yyact = yypParser->yytos->stateno; +#ifndef NDEBUG + if( yyTraceFILE ){ + if( yyact < YY_MIN_REDUCE ){ + fprintf(yyTraceFILE,"%sInput '%s' in state %d\n", + yyTracePrompt,yyTokenName[yymajor],yyact); + }else{ + fprintf(yyTraceFILE,"%sInput '%s' with pending reduce %d\n", + yyTracePrompt,yyTokenName[yymajor],yyact-YY_MIN_REDUCE); + } + } +#endif + + do{ + assert( yyact==yypParser->yytos->stateno ); + yyact = yy_find_shift_action((YYCODETYPE)yymajor,yyact); + if( yyact >= YY_MIN_REDUCE ){ + yyact = yy_reduce(yypParser,yyact-YY_MIN_REDUCE,yymajor, + yyminor sqlite3ParserCTX_PARAM); + }else if( yyact <= YY_MAX_SHIFTREDUCE ){ + yy_shift(yypParser,yyact,(YYCODETYPE)yymajor,yyminor); +#ifndef YYNOERRORRECOVERY + yypParser->yyerrcnt--; +#endif + break; + }else if( yyact==YY_ACCEPT_ACTION ){ + yypParser->yytos--; + yy_accept(yypParser); + return; + }else{ + assert( yyact == YY_ERROR_ACTION ); + yyminorunion.yy0 = yyminor; +#ifdef YYERRORSYMBOL + int yymx; +#endif +#ifndef NDEBUG + if( yyTraceFILE ){ + fprintf(yyTraceFILE,"%sSyntax Error!\n",yyTracePrompt); + } +#endif +#ifdef YYERRORSYMBOL + /* A syntax error has occurred. + ** The response to an error depends upon whether or not the + ** grammar defines an error token "ERROR". + ** + ** This is what we do if the grammar does define ERROR: + ** + ** * Call the %syntax_error function. + ** + ** * Begin popping the stack until we enter a state where + ** it is legal to shift the error symbol, then shift + ** the error symbol. + ** + ** * Set the error count to three. + ** + ** * Begin accepting and shifting new tokens. No new error + ** processing will occur until three tokens have been + ** shifted successfully. + ** + */ + if( yypParser->yyerrcnt<0 ){ + yy_syntax_error(yypParser,yymajor,yyminor); + } + yymx = yypParser->yytos->major; + if( yymx==YYERRORSYMBOL || yyerrorhit ){ +#ifndef NDEBUG + if( yyTraceFILE ){ + fprintf(yyTraceFILE,"%sDiscard input token %s\n", + yyTracePrompt,yyTokenName[yymajor]); + } +#endif + yy_destructor(yypParser, (YYCODETYPE)yymajor, &yyminorunion); + yymajor = YYNOCODE; + }else{ + while( yypParser->yytos >= yypParser->yystack + && (yyact = yy_find_reduce_action( + yypParser->yytos->stateno, + YYERRORSYMBOL)) > YY_MAX_SHIFTREDUCE + ){ + yy_pop_parser_stack(yypParser); + } + if( yypParser->yytos < yypParser->yystack || yymajor==0 ){ + yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion); + yy_parse_failed(yypParser); +#ifndef YYNOERRORRECOVERY + yypParser->yyerrcnt = -1; +#endif + yymajor = YYNOCODE; + }else if( yymx!=YYERRORSYMBOL ){ + yy_shift(yypParser,yyact,YYERRORSYMBOL,yyminor); + } + } + yypParser->yyerrcnt = 3; + yyerrorhit = 1; + if( yymajor==YYNOCODE ) break; + yyact = yypParser->yytos->stateno; +#elif defined(YYNOERRORRECOVERY) + /* If the YYNOERRORRECOVERY macro is defined, then do not attempt to + ** do any kind of error recovery. Instead, simply invoke the syntax + ** error routine and continue going as if nothing had happened. + ** + ** Applications can set this macro (for example inside %include) if + ** they intend to abandon the parse upon the first syntax error seen. + */ + yy_syntax_error(yypParser,yymajor, yyminor); + yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion); + break; +#else /* YYERRORSYMBOL is not defined */ + /* This is what we do if the grammar does not define ERROR: + ** + ** * Report an error message, and throw away the input token. + ** + ** * If the input token is $, then fail the parse. + ** + ** As before, subsequent error messages are suppressed until + ** three input tokens have been successfully shifted. + */ + if( yypParser->yyerrcnt<=0 ){ + yy_syntax_error(yypParser,yymajor, yyminor); + } + yypParser->yyerrcnt = 3; + yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion); + if( yyendofinput ){ + yy_parse_failed(yypParser); +#ifndef YYNOERRORRECOVERY + yypParser->yyerrcnt = -1; +#endif + } + break; +#endif + } + }while( yypParser->yytos>yypParser->yystack ); +#ifndef NDEBUG + if( yyTraceFILE ){ + yyStackEntry *i; + char cDiv = '['; + fprintf(yyTraceFILE,"%sReturn. Stack=",yyTracePrompt); + for(i=&yypParser->yystack[1]; i<=yypParser->yytos; i++){ + fprintf(yyTraceFILE,"%c%s", cDiv, yyTokenName[i->major]); + cDiv = ' '; + } + fprintf(yyTraceFILE,"]\n"); + } +#endif + return; +} + +/* +** Return the fallback token corresponding to canonical token iToken, or +** 0 if iToken has no fallback. +*/ +SQLITE_PRIVATE int sqlite3ParserFallback(int iToken){ +#ifdef YYFALLBACK + assert( iToken<(int)(sizeof(yyFallback)/sizeof(yyFallback[0])) ); + return yyFallback[iToken]; +#else + (void)iToken; + return 0; +#endif +} + +/************** End of parse.c ***********************************************/ +/************** Begin file tokenize.c ****************************************/ +/* +** 2001 September 15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** An tokenizer for SQL +** +** This file contains C code that splits an SQL input string up into +** individual tokens and sends those tokens one-by-one over to the +** parser for analysis. +*/ +/* #include "sqliteInt.h" */ +/* #include */ + +/* Character classes for tokenizing +** +** In the sqlite3GetToken() function, a switch() on aiClass[c] is implemented +** using a lookup table, whereas a switch() directly on c uses a binary search. +** The lookup table is much faster. To maximize speed, and to ensure that +** a lookup table is used, all of the classes need to be small integers and +** all of them need to be used within the switch. +*/ +#define CC_X 0 /* The letter 'x', or start of BLOB literal */ +#define CC_KYWD 1 /* Alphabetics or '_'. Usable in a keyword */ +#define CC_ID 2 /* unicode characters usable in IDs */ +#define CC_DIGIT 3 /* Digits */ +#define CC_DOLLAR 4 /* '$' */ +#define CC_VARALPHA 5 /* '@', '#', ':'. Alphabetic SQL variables */ +#define CC_VARNUM 6 /* '?'. Numeric SQL variables */ +#define CC_SPACE 7 /* Space characters */ +#define CC_QUOTE 8 /* '"', '\'', or '`'. String literals, quoted ids */ +#define CC_QUOTE2 9 /* '['. [...] style quoted ids */ +#define CC_PIPE 10 /* '|'. Bitwise OR or concatenate */ +#define CC_MINUS 11 /* '-'. Minus or SQL-style comment */ +#define CC_LT 12 /* '<'. Part of < or <= or <> */ +#define CC_GT 13 /* '>'. Part of > or >= */ +#define CC_EQ 14 /* '='. Part of = or == */ +#define CC_BANG 15 /* '!'. Part of != */ +#define CC_SLASH 16 /* '/'. / or c-style comment */ +#define CC_LP 17 /* '(' */ +#define CC_RP 18 /* ')' */ +#define CC_SEMI 19 /* ';' */ +#define CC_PLUS 20 /* '+' */ +#define CC_STAR 21 /* '*' */ +#define CC_PERCENT 22 /* '%' */ +#define CC_COMMA 23 /* ',' */ +#define CC_AND 24 /* '&' */ +#define CC_TILDA 25 /* '~' */ +#define CC_DOT 26 /* '.' */ +#define CC_ILLEGAL 27 /* Illegal character */ +#define CC_NUL 28 /* 0x00 */ + +static const unsigned char aiClass[] = { +#ifdef SQLITE_ASCII +/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xa xb xc xd xe xf */ +/* 0x */ 28, 27, 27, 27, 27, 27, 27, 27, 27, 7, 7, 27, 7, 7, 27, 27, +/* 1x */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, +/* 2x */ 7, 15, 8, 5, 4, 22, 24, 8, 17, 18, 21, 20, 23, 11, 26, 16, +/* 3x */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 5, 19, 12, 14, 13, 6, +/* 4x */ 5, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, +/* 5x */ 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 9, 27, 27, 27, 1, +/* 6x */ 8, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, +/* 7x */ 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 27, 10, 27, 25, 27, +/* 8x */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, +/* 9x */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, +/* Ax */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, +/* Bx */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, +/* Cx */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, +/* Dx */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, +/* Ex */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, +/* Fx */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 +#endif +#ifdef SQLITE_EBCDIC +/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xa xb xc xd xe xf */ +/* 0x */ 27, 27, 27, 27, 27, 7, 27, 27, 27, 27, 27, 27, 7, 7, 27, 27, +/* 1x */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, +/* 2x */ 27, 27, 27, 27, 27, 7, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, +/* 3x */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, +/* 4x */ 7, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 26, 12, 17, 20, 10, +/* 5x */ 24, 27, 27, 27, 27, 27, 27, 27, 27, 27, 15, 4, 21, 18, 19, 27, +/* 6x */ 11, 16, 27, 27, 27, 27, 27, 27, 27, 27, 27, 23, 22, 1, 13, 6, +/* 7x */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 8, 5, 5, 5, 8, 14, 8, +/* 8x */ 27, 1, 1, 1, 1, 1, 1, 1, 1, 1, 27, 27, 27, 27, 27, 27, +/* 9x */ 27, 1, 1, 1, 1, 1, 1, 1, 1, 1, 27, 27, 27, 27, 27, 27, +/* Ax */ 27, 25, 1, 1, 1, 1, 1, 0, 1, 1, 27, 27, 27, 27, 27, 27, +/* Bx */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 9, 27, 27, 27, 27, 27, +/* Cx */ 27, 1, 1, 1, 1, 1, 1, 1, 1, 1, 27, 27, 27, 27, 27, 27, +/* Dx */ 27, 1, 1, 1, 1, 1, 1, 1, 1, 1, 27, 27, 27, 27, 27, 27, +/* Ex */ 27, 27, 1, 1, 1, 1, 1, 0, 1, 1, 27, 27, 27, 27, 27, 27, +/* Fx */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 27, 27, 27, 27, 27, 27, +#endif +}; + +/* +** The charMap() macro maps alphabetic characters (only) into their +** lower-case ASCII equivalent. On ASCII machines, this is just +** an upper-to-lower case map. On EBCDIC machines we also need +** to adjust the encoding. The mapping is only valid for alphabetics +** which are the only characters for which this feature is used. +** +** Used by keywordhash.h +*/ +#ifdef SQLITE_ASCII +# define charMap(X) sqlite3UpperToLower[(unsigned char)X] +#endif +#ifdef SQLITE_EBCDIC +# define charMap(X) ebcdicToAscii[(unsigned char)X] +const unsigned char ebcdicToAscii[] = { +/* 0 1 2 3 4 5 6 7 8 9 A B C D E F */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 1x */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 2x */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 3x */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 4x */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 5x */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 0, 0, /* 6x */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 7x */ + 0, 97, 98, 99,100,101,102,103,104,105, 0, 0, 0, 0, 0, 0, /* 8x */ + 0,106,107,108,109,110,111,112,113,114, 0, 0, 0, 0, 0, 0, /* 9x */ + 0, 0,115,116,117,118,119,120,121,122, 0, 0, 0, 0, 0, 0, /* Ax */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* Bx */ + 0, 97, 98, 99,100,101,102,103,104,105, 0, 0, 0, 0, 0, 0, /* Cx */ + 0,106,107,108,109,110,111,112,113,114, 0, 0, 0, 0, 0, 0, /* Dx */ + 0, 0,115,116,117,118,119,120,121,122, 0, 0, 0, 0, 0, 0, /* Ex */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* Fx */ +}; +#endif + +/* +** The sqlite3KeywordCode function looks up an identifier to determine if +** it is a keyword. If it is a keyword, the token code of that keyword is +** returned. If the input is not a keyword, TK_ID is returned. +** +** The implementation of this routine was generated by a program, +** mkkeywordhash.c, located in the tool subdirectory of the distribution. +** The output of the mkkeywordhash.c program is written into a file +** named keywordhash.h and then included into this source file by +** the #include below. +*/ +/************** Include keywordhash.h in the middle of tokenize.c ************/ +/************** Begin file keywordhash.h *************************************/ +/***** This file contains automatically generated code ****** +** +** The code in this file has been automatically generated by +** +** sqlite/tool/mkkeywordhash.c +** +** The code in this file implements a function that determines whether +** or not a given identifier is really an SQL keyword. The same thing +** might be implemented more directly using a hand-written hash table. +** But by using this automatically generated code, the size of the code +** is substantially reduced. This is important for embedded applications +** on platforms with limited memory. +*/ +/* Hash score: 227 */ +/* zKWText[] encodes 984 bytes of keyword text in 648 bytes */ +/* REINDEXEDESCAPEACHECKEYBEFOREIGNOREGEXPLAINSTEADDATABASELECT */ +/* ABLEFTHENDEFERRABLELSEXCLUDELETEMPORARYISNULLSAVEPOINTERSECT */ +/* IESNOTNULLIKEXCEPTRANSACTIONATURALTERAISEXCLUSIVEXISTS */ +/* CONSTRAINTOFFSETRIGGERANGENERATEDETACHAVINGLOBEGINNEREFERENCES */ +/* UNIQUERYWITHOUTERELEASEATTACHBETWEENOTHINGROUPSCASCADEFAULT */ +/* CASECOLLATECREATECURRENT_DATEIMMEDIATEJOINSERTMATCHPLANALYZE */ +/* PRAGMABORTUPDATEVALUESVIRTUALWAYSWHENWHERECURSIVEAFTERENAMEAND */ +/* EFERREDISTINCTAUTOINCREMENTCASTCOLUMNCOMMITCONFLICTCROSS */ +/* CURRENT_TIMESTAMPARTITIONDROPRECEDINGFAILASTFILTEREPLACEFIRST */ +/* FOLLOWINGFROMFULLIMITIFORDERESTRICTOTHERSOVERIGHTROLLBACKROWS */ +/* UNBOUNDEDUNIONUSINGVACUUMVIEWINDOWBYINITIALLYPRIMARY */ +static const char zKWText[647] = { + 'R','E','I','N','D','E','X','E','D','E','S','C','A','P','E','A','C','H', + 'E','C','K','E','Y','B','E','F','O','R','E','I','G','N','O','R','E','G', + 'E','X','P','L','A','I','N','S','T','E','A','D','D','A','T','A','B','A', + 'S','E','L','E','C','T','A','B','L','E','F','T','H','E','N','D','E','F', + 'E','R','R','A','B','L','E','L','S','E','X','C','L','U','D','E','L','E', + 'T','E','M','P','O','R','A','R','Y','I','S','N','U','L','L','S','A','V', + 'E','P','O','I','N','T','E','R','S','E','C','T','I','E','S','N','O','T', + 'N','U','L','L','I','K','E','X','C','E','P','T','R','A','N','S','A','C', + 'T','I','O','N','A','T','U','R','A','L','T','E','R','A','I','S','E','X', + 'C','L','U','S','I','V','E','X','I','S','T','S','C','O','N','S','T','R', + 'A','I','N','T','O','F','F','S','E','T','R','I','G','G','E','R','A','N', + 'G','E','N','E','R','A','T','E','D','E','T','A','C','H','A','V','I','N', + 'G','L','O','B','E','G','I','N','N','E','R','E','F','E','R','E','N','C', + 'E','S','U','N','I','Q','U','E','R','Y','W','I','T','H','O','U','T','E', + 'R','E','L','E','A','S','E','A','T','T','A','C','H','B','E','T','W','E', + 'E','N','O','T','H','I','N','G','R','O','U','P','S','C','A','S','C','A', + 'D','E','F','A','U','L','T','C','A','S','E','C','O','L','L','A','T','E', + 'C','R','E','A','T','E','C','U','R','R','E','N','T','_','D','A','T','E', + 'I','M','M','E','D','I','A','T','E','J','O','I','N','S','E','R','T','M', + 'A','T','C','H','P','L','A','N','A','L','Y','Z','E','P','R','A','G','M', + 'A','B','O','R','T','U','P','D','A','T','E','V','A','L','U','E','S','V', + 'I','R','T','U','A','L','W','A','Y','S','W','H','E','N','W','H','E','R', + 'E','C','U','R','S','I','V','E','A','F','T','E','R','E','N','A','M','E', + 'A','N','D','E','F','E','R','R','E','D','I','S','T','I','N','C','T','A', + 'U','T','O','I','N','C','R','E','M','E','N','T','C','A','S','T','C','O', + 'L','U','M','N','C','O','M','M','I','T','C','O','N','F','L','I','C','T', + 'C','R','O','S','S','C','U','R','R','E','N','T','_','T','I','M','E','S', + 'T','A','M','P','A','R','T','I','T','I','O','N','D','R','O','P','R','E', + 'C','E','D','I','N','G','F','A','I','L','A','S','T','F','I','L','T','E', + 'R','E','P','L','A','C','E','F','I','R','S','T','F','O','L','L','O','W', + 'I','N','G','F','R','O','M','F','U','L','L','I','M','I','T','I','F','O', + 'R','D','E','R','E','S','T','R','I','C','T','O','T','H','E','R','S','O', + 'V','E','R','I','G','H','T','R','O','L','L','B','A','C','K','R','O','W', + 'S','U','N','B','O','U','N','D','E','D','U','N','I','O','N','U','S','I', + 'N','G','V','A','C','U','U','M','V','I','E','W','I','N','D','O','W','B', + 'Y','I','N','I','T','I','A','L','L','Y','P','R','I','M','A','R','Y', +}; +/* aKWHash[i] is the hash value for the i-th keyword */ +static const unsigned char aKWHash[127] = { + 84, 102, 132, 82, 114, 29, 0, 0, 91, 0, 85, 72, 0, + 53, 35, 86, 15, 0, 42, 94, 54, 126, 133, 19, 0, 0, + 138, 0, 40, 128, 0, 22, 104, 0, 9, 0, 0, 122, 80, + 0, 78, 6, 0, 65, 99, 145, 0, 134, 112, 0, 0, 48, + 0, 100, 24, 0, 17, 0, 27, 70, 23, 26, 5, 60, 140, + 107, 121, 0, 73, 101, 71, 143, 61, 119, 74, 0, 49, 0, + 11, 41, 0, 110, 0, 0, 0, 106, 10, 108, 113, 124, 14, + 50, 123, 0, 89, 0, 18, 120, 142, 56, 129, 137, 88, 83, + 37, 30, 125, 0, 0, 105, 51, 130, 127, 0, 34, 0, 0, + 44, 0, 95, 38, 39, 0, 20, 45, 116, 90, +}; +/* aKWNext[] forms the hash collision chain. If aKWHash[i]==0 +** then the i-th keyword has no more hash collisions. Otherwise, +** the next keyword with the same hash is aKWHash[i]-1. */ +static const unsigned char aKWNext[145] = { + 0, 0, 0, 0, 4, 0, 43, 0, 0, 103, 111, 0, 0, + 0, 2, 0, 0, 141, 0, 0, 0, 13, 0, 0, 0, 0, + 139, 0, 0, 118, 52, 0, 0, 135, 12, 0, 0, 62, 0, + 136, 0, 131, 0, 0, 36, 0, 0, 28, 77, 0, 0, 0, + 0, 59, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 69, 0, 0, 0, 0, 0, 144, 3, 0, 58, 0, 1, + 75, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 64, 66, + 63, 0, 0, 0, 0, 46, 0, 16, 0, 115, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 81, 97, 0, 8, 0, 109, + 21, 7, 67, 0, 79, 93, 117, 0, 0, 68, 0, 0, 96, + 0, 55, 0, 76, 0, 92, 32, 33, 57, 25, 0, 98, 0, + 0, 87, +}; +/* aKWLen[i] is the length (in bytes) of the i-th keyword */ +static const unsigned char aKWLen[145] = { + 7, 7, 5, 4, 6, 4, 5, 3, 6, 7, 3, 6, 6, + 7, 7, 3, 8, 2, 6, 5, 4, 4, 3, 10, 4, 7, + 6, 9, 4, 2, 6, 5, 9, 9, 4, 7, 3, 2, 4, + 4, 6, 11, 6, 2, 7, 5, 5, 9, 6, 10, 4, 6, + 2, 3, 7, 5, 9, 6, 6, 4, 5, 5, 10, 6, 5, + 7, 4, 5, 7, 6, 7, 7, 6, 5, 7, 3, 7, 4, + 7, 6, 12, 9, 4, 6, 5, 4, 7, 6, 5, 6, 6, + 7, 6, 4, 5, 9, 5, 6, 3, 8, 8, 2, 13, 2, + 2, 4, 6, 6, 8, 5, 17, 12, 7, 9, 4, 9, 4, + 4, 6, 7, 5, 9, 4, 4, 5, 2, 5, 8, 6, 4, + 5, 8, 4, 3, 9, 5, 5, 6, 4, 6, 2, 2, 9, + 3, 7, +}; +/* aKWOffset[i] is the index into zKWText[] of the start of +** the text for the i-th keyword. */ +static const unsigned short int aKWOffset[145] = { + 0, 2, 2, 8, 9, 14, 16, 20, 23, 25, 25, 29, 33, + 36, 41, 46, 48, 53, 54, 59, 62, 65, 67, 69, 78, 81, + 86, 90, 90, 94, 99, 101, 105, 111, 119, 123, 123, 123, 126, + 129, 132, 137, 142, 146, 147, 152, 156, 160, 168, 174, 181, 184, + 184, 187, 189, 195, 198, 206, 211, 216, 219, 222, 226, 236, 239, + 244, 244, 248, 252, 259, 265, 271, 277, 277, 283, 284, 288, 295, + 299, 306, 312, 324, 333, 335, 341, 346, 348, 355, 360, 365, 371, + 377, 382, 388, 392, 395, 404, 408, 414, 416, 423, 424, 431, 433, + 435, 444, 448, 454, 460, 468, 473, 473, 473, 489, 498, 501, 510, + 513, 517, 522, 529, 534, 543, 547, 550, 555, 557, 561, 569, 575, + 578, 583, 591, 591, 595, 604, 609, 614, 620, 623, 626, 629, 631, + 636, 640, +}; +/* aKWCode[i] is the parser symbol code for the i-th keyword */ +static const unsigned char aKWCode[145] = { + TK_REINDEX, TK_INDEXED, TK_INDEX, TK_DESC, TK_ESCAPE, + TK_EACH, TK_CHECK, TK_KEY, TK_BEFORE, TK_FOREIGN, + TK_FOR, TK_IGNORE, TK_LIKE_KW, TK_EXPLAIN, TK_INSTEAD, + TK_ADD, TK_DATABASE, TK_AS, TK_SELECT, TK_TABLE, + TK_JOIN_KW, TK_THEN, TK_END, TK_DEFERRABLE, TK_ELSE, + TK_EXCLUDE, TK_DELETE, TK_TEMP, TK_TEMP, TK_OR, + TK_ISNULL, TK_NULLS, TK_SAVEPOINT, TK_INTERSECT, TK_TIES, + TK_NOTNULL, TK_NOT, TK_NO, TK_NULL, TK_LIKE_KW, + TK_EXCEPT, TK_TRANSACTION,TK_ACTION, TK_ON, TK_JOIN_KW, + TK_ALTER, TK_RAISE, TK_EXCLUSIVE, TK_EXISTS, TK_CONSTRAINT, + TK_INTO, TK_OFFSET, TK_OF, TK_SET, TK_TRIGGER, + TK_RANGE, TK_GENERATED, TK_DETACH, TK_HAVING, TK_LIKE_KW, + TK_BEGIN, TK_JOIN_KW, TK_REFERENCES, TK_UNIQUE, TK_QUERY, + TK_WITHOUT, TK_WITH, TK_JOIN_KW, TK_RELEASE, TK_ATTACH, + TK_BETWEEN, TK_NOTHING, TK_GROUPS, TK_GROUP, TK_CASCADE, + TK_ASC, TK_DEFAULT, TK_CASE, TK_COLLATE, TK_CREATE, + TK_CTIME_KW, TK_IMMEDIATE, TK_JOIN, TK_INSERT, TK_MATCH, + TK_PLAN, TK_ANALYZE, TK_PRAGMA, TK_ABORT, TK_UPDATE, + TK_VALUES, TK_VIRTUAL, TK_ALWAYS, TK_WHEN, TK_WHERE, + TK_RECURSIVE, TK_AFTER, TK_RENAME, TK_AND, TK_DEFERRED, + TK_DISTINCT, TK_IS, TK_AUTOINCR, TK_TO, TK_IN, + TK_CAST, TK_COLUMNKW, TK_COMMIT, TK_CONFLICT, TK_JOIN_KW, + TK_CTIME_KW, TK_CTIME_KW, TK_CURRENT, TK_PARTITION, TK_DROP, + TK_PRECEDING, TK_FAIL, TK_LAST, TK_FILTER, TK_REPLACE, + TK_FIRST, TK_FOLLOWING, TK_FROM, TK_JOIN_KW, TK_LIMIT, + TK_IF, TK_ORDER, TK_RESTRICT, TK_OTHERS, TK_OVER, + TK_JOIN_KW, TK_ROLLBACK, TK_ROWS, TK_ROW, TK_UNBOUNDED, + TK_UNION, TK_USING, TK_VACUUM, TK_VIEW, TK_WINDOW, + TK_DO, TK_BY, TK_INITIALLY, TK_ALL, TK_PRIMARY, +}; +/* Hash table decoded: +** 0: INSERT +** 1: IS +** 2: ROLLBACK TRIGGER +** 3: IMMEDIATE +** 4: PARTITION +** 5: TEMP +** 6: +** 7: +** 8: VALUES WITHOUT +** 9: +** 10: MATCH +** 11: NOTHING +** 12: +** 13: OF +** 14: TIES IGNORE +** 15: PLAN +** 16: INSTEAD INDEXED +** 17: +** 18: TRANSACTION RIGHT +** 19: WHEN +** 20: SET HAVING +** 21: IF +** 22: ROWS +** 23: SELECT +** 24: +** 25: +** 26: VACUUM SAVEPOINT +** 27: +** 28: LIKE UNION VIRTUAL REFERENCES +** 29: RESTRICT +** 30: +** 31: THEN REGEXP +** 32: TO +** 33: +** 34: BEFORE +** 35: +** 36: +** 37: FOLLOWING COLLATE CASCADE +** 38: CREATE +** 39: +** 40: CASE REINDEX +** 41: EACH +** 42: +** 43: QUERY +** 44: AND ADD +** 45: PRIMARY ANALYZE +** 46: +** 47: ROW ASC DETACH +** 48: CURRENT_TIME CURRENT_DATE +** 49: +** 50: +** 51: EXCLUSIVE TEMPORARY +** 52: +** 53: DEFERRED +** 54: DEFERRABLE +** 55: +** 56: DATABASE +** 57: +** 58: DELETE VIEW GENERATED +** 59: ATTACH +** 60: END +** 61: EXCLUDE +** 62: ESCAPE DESC +** 63: GLOB +** 64: WINDOW ELSE +** 65: COLUMN +** 66: FIRST +** 67: +** 68: GROUPS ALL +** 69: DISTINCT DROP KEY +** 70: BETWEEN +** 71: INITIALLY +** 72: BEGIN +** 73: FILTER CHECK ACTION +** 74: GROUP INDEX +** 75: +** 76: EXISTS DEFAULT +** 77: +** 78: FOR CURRENT_TIMESTAMP +** 79: EXCEPT +** 80: +** 81: CROSS +** 82: +** 83: +** 84: +** 85: CAST +** 86: FOREIGN AUTOINCREMENT +** 87: COMMIT +** 88: CURRENT AFTER ALTER +** 89: FULL FAIL CONFLICT +** 90: EXPLAIN +** 91: CONSTRAINT +** 92: FROM ALWAYS +** 93: +** 94: ABORT +** 95: +** 96: AS DO +** 97: REPLACE WITH RELEASE +** 98: BY RENAME +** 99: RANGE RAISE +** 100: OTHERS +** 101: USING NULLS +** 102: PRAGMA +** 103: JOIN ISNULL OFFSET +** 104: NOT +** 105: OR LAST LEFT +** 106: LIMIT +** 107: +** 108: +** 109: IN +** 110: INTO +** 111: OVER RECURSIVE +** 112: ORDER OUTER +** 113: +** 114: INTERSECT UNBOUNDED +** 115: +** 116: +** 117: ON +** 118: +** 119: WHERE +** 120: NO INNER +** 121: NULL +** 122: +** 123: TABLE +** 124: NATURAL NOTNULL +** 125: PRECEDING +** 126: UPDATE UNIQUE +*/ +/* Check to see if z[0..n-1] is a keyword. If it is, write the +** parser symbol code for that keyword into *pType. Always +** return the integer n (the length of the token). */ +static int keywordCode(const char *z, int n, int *pType){ + int i, j; + const char *zKW; + if( n>=2 ){ + i = ((charMap(z[0])*4) ^ (charMap(z[n-1])*3) ^ n) % 127; + for(i=((int)aKWHash[i])-1; i>=0; i=((int)aKWNext[i])-1){ + if( aKWLen[i]!=n ) continue; + zKW = &zKWText[aKWOffset[i]]; +#ifdef SQLITE_ASCII + if( (z[0]&~0x20)!=zKW[0] ) continue; + if( (z[1]&~0x20)!=zKW[1] ) continue; + j = 2; + while( j=SQLITE_N_KEYWORD ) return SQLITE_ERROR; + *pzName = zKWText + aKWOffset[i]; + *pnName = aKWLen[i]; + return SQLITE_OK; +} +SQLITE_API int sqlite3_keyword_count(void){ return SQLITE_N_KEYWORD; } +SQLITE_API int sqlite3_keyword_check(const char *zName, int nName){ + return TK_ID!=sqlite3KeywordCode((const u8*)zName, nName); +} + +/************** End of keywordhash.h *****************************************/ +/************** Continuing where we left off in tokenize.c *******************/ + + +/* +** If X is a character that can be used in an identifier then +** IdChar(X) will be true. Otherwise it is false. +** +** For ASCII, any character with the high-order bit set is +** allowed in an identifier. For 7-bit characters, +** sqlite3IsIdChar[X] must be 1. +** +** For EBCDIC, the rules are more complex but have the same +** end result. +** +** Ticket #1066. the SQL standard does not allow '$' in the +** middle of identifiers. But many SQL implementations do. +** SQLite will allow '$' in identifiers for compatibility. +** But the feature is undocumented. +*/ +#ifdef SQLITE_ASCII +#define IdChar(C) ((sqlite3CtypeMap[(unsigned char)C]&0x46)!=0) +#endif +#ifdef SQLITE_EBCDIC +SQLITE_PRIVATE const char sqlite3IsEbcdicIdChar[] = { +/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */ + 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 4x */ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 0, 0, /* 5x */ + 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 0, 0, /* 6x */ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, /* 7x */ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 0, /* 8x */ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 1, 0, /* 9x */ + 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 0, /* Ax */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* Bx */ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, /* Cx */ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, /* Dx */ + 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, /* Ex */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0, /* Fx */ +}; +#define IdChar(C) (((c=C)>=0x42 && sqlite3IsEbcdicIdChar[c-0x40])) +#endif + +/* Make the IdChar function accessible from ctime.c and alter.c */ +SQLITE_PRIVATE int sqlite3IsIdChar(u8 c){ return IdChar(c); } + +#ifndef SQLITE_OMIT_WINDOWFUNC +/* +** Return the id of the next token in string (*pz). Before returning, set +** (*pz) to point to the byte following the parsed token. +*/ +static int getToken(const unsigned char **pz){ + const unsigned char *z = *pz; + int t; /* Token type to return */ + do { + z += sqlite3GetToken(z, &t); + }while( t==TK_SPACE ); + if( t==TK_ID + || t==TK_STRING + || t==TK_JOIN_KW + || t==TK_WINDOW + || t==TK_OVER + || sqlite3ParserFallback(t)==TK_ID + ){ + t = TK_ID; + } + *pz = z; + return t; +} + +/* +** The following three functions are called immediately after the tokenizer +** reads the keywords WINDOW, OVER and FILTER, respectively, to determine +** whether the token should be treated as a keyword or an SQL identifier. +** This cannot be handled by the usual lemon %fallback method, due to +** the ambiguity in some constructions. e.g. +** +** SELECT sum(x) OVER ... +** +** In the above, "OVER" might be a keyword, or it might be an alias for the +** sum(x) expression. If a "%fallback ID OVER" directive were added to +** grammar, then SQLite would always treat "OVER" as an alias, making it +** impossible to call a window-function without a FILTER clause. +** +** WINDOW is treated as a keyword if: +** +** * the following token is an identifier, or a keyword that can fallback +** to being an identifier, and +** * the token after than one is TK_AS. +** +** OVER is a keyword if: +** +** * the previous token was TK_RP, and +** * the next token is either TK_LP or an identifier. +** +** FILTER is a keyword if: +** +** * the previous token was TK_RP, and +** * the next token is TK_LP. +*/ +static int analyzeWindowKeyword(const unsigned char *z){ + int t; + t = getToken(&z); + if( t!=TK_ID ) return TK_ID; + t = getToken(&z); + if( t!=TK_AS ) return TK_ID; + return TK_WINDOW; +} +static int analyzeOverKeyword(const unsigned char *z, int lastToken){ + if( lastToken==TK_RP ){ + int t = getToken(&z); + if( t==TK_LP || t==TK_ID ) return TK_OVER; + } + return TK_ID; +} +static int analyzeFilterKeyword(const unsigned char *z, int lastToken){ + if( lastToken==TK_RP && getToken(&z)==TK_LP ){ + return TK_FILTER; + } + return TK_ID; +} +#endif /* SQLITE_OMIT_WINDOWFUNC */ + +/* +** Return the length (in bytes) of the token that begins at z[0]. +** Store the token type in *tokenType before returning. +*/ +SQLITE_PRIVATE int sqlite3GetToken(const unsigned char *z, int *tokenType){ + int i, c; + switch( aiClass[*z] ){ /* Switch on the character-class of the first byte + ** of the token. See the comment on the CC_ defines + ** above. */ + case CC_SPACE: { + testcase( z[0]==' ' ); + testcase( z[0]=='\t' ); + testcase( z[0]=='\n' ); + testcase( z[0]=='\f' ); + testcase( z[0]=='\r' ); + for(i=1; sqlite3Isspace(z[i]); i++){} + *tokenType = TK_SPACE; + return i; + } + case CC_MINUS: { + if( z[1]=='-' ){ + for(i=2; (c=z[i])!=0 && c!='\n'; i++){} + *tokenType = TK_SPACE; /* IMP: R-22934-25134 */ + return i; + } + *tokenType = TK_MINUS; + return 1; + } + case CC_LP: { + *tokenType = TK_LP; + return 1; + } + case CC_RP: { + *tokenType = TK_RP; + return 1; + } + case CC_SEMI: { + *tokenType = TK_SEMI; + return 1; + } + case CC_PLUS: { + *tokenType = TK_PLUS; + return 1; + } + case CC_STAR: { + *tokenType = TK_STAR; + return 1; + } + case CC_SLASH: { + if( z[1]!='*' || z[2]==0 ){ + *tokenType = TK_SLASH; + return 1; + } + for(i=3, c=z[2]; (c!='*' || z[i]!='/') && (c=z[i])!=0; i++){} + if( c ) i++; + *tokenType = TK_SPACE; /* IMP: R-22934-25134 */ + return i; } case CC_PERCENT: { *tokenType = TK_REM; return 1; } - case CC_EQ: { - *tokenType = TK_EQ; - return 1 + (z[1]=='='); + case CC_EQ: { + *tokenType = TK_EQ; + return 1 + (z[1]=='='); + } + case CC_LT: { + if( (c=z[1])=='=' ){ + *tokenType = TK_LE; + return 2; + }else if( c=='>' ){ + *tokenType = TK_NE; + return 2; + }else if( c=='<' ){ + *tokenType = TK_LSHIFT; + return 2; + }else{ + *tokenType = TK_LT; + return 1; + } + } + case CC_GT: { + if( (c=z[1])=='=' ){ + *tokenType = TK_GE; + return 2; + }else if( c=='>' ){ + *tokenType = TK_RSHIFT; + return 2; + }else{ + *tokenType = TK_GT; + return 1; + } + } + case CC_BANG: { + if( z[1]!='=' ){ + *tokenType = TK_ILLEGAL; + return 1; + }else{ + *tokenType = TK_NE; + return 2; + } + } + case CC_PIPE: { + if( z[1]!='|' ){ + *tokenType = TK_BITOR; + return 1; + }else{ + *tokenType = TK_CONCAT; + return 2; + } + } + case CC_COMMA: { + *tokenType = TK_COMMA; + return 1; + } + case CC_AND: { + *tokenType = TK_BITAND; + return 1; + } + case CC_TILDA: { + *tokenType = TK_BITNOT; + return 1; + } + case CC_QUOTE: { + int delim = z[0]; + testcase( delim=='`' ); + testcase( delim=='\'' ); + testcase( delim=='"' ); + for(i=1; (c=z[i])!=0; i++){ + if( c==delim ){ + if( z[i+1]==delim ){ + i++; + }else{ + break; + } + } + } + if( c=='\'' ){ + *tokenType = TK_STRING; + return i+1; + }else if( c!=0 ){ + *tokenType = TK_ID; + return i+1; + }else{ + *tokenType = TK_ILLEGAL; + return i; + } + } + case CC_DOT: { +#ifndef SQLITE_OMIT_FLOATING_POINT + if( !sqlite3Isdigit(z[1]) ) +#endif + { + *tokenType = TK_DOT; + return 1; + } + /* If the next character is a digit, this is a floating point + ** number that begins with ".". Fall thru into the next case */ + } + case CC_DIGIT: { + testcase( z[0]=='0' ); testcase( z[0]=='1' ); testcase( z[0]=='2' ); + testcase( z[0]=='3' ); testcase( z[0]=='4' ); testcase( z[0]=='5' ); + testcase( z[0]=='6' ); testcase( z[0]=='7' ); testcase( z[0]=='8' ); + testcase( z[0]=='9' ); + *tokenType = TK_INTEGER; +#ifndef SQLITE_OMIT_HEX_INTEGER + if( z[0]=='0' && (z[1]=='x' || z[1]=='X') && sqlite3Isxdigit(z[2]) ){ + for(i=3; sqlite3Isxdigit(z[i]); i++){} + return i; + } +#endif + for(i=0; sqlite3Isdigit(z[i]); i++){} +#ifndef SQLITE_OMIT_FLOATING_POINT + if( z[i]=='.' ){ + i++; + while( sqlite3Isdigit(z[i]) ){ i++; } + *tokenType = TK_FLOAT; + } + if( (z[i]=='e' || z[i]=='E') && + ( sqlite3Isdigit(z[i+1]) + || ((z[i+1]=='+' || z[i+1]=='-') && sqlite3Isdigit(z[i+2])) + ) + ){ + i += 2; + while( sqlite3Isdigit(z[i]) ){ i++; } + *tokenType = TK_FLOAT; + } +#endif + while( IdChar(z[i]) ){ + *tokenType = TK_ILLEGAL; + i++; + } + return i; + } + case CC_QUOTE2: { + for(i=1, c=z[0]; c!=']' && (c=z[i])!=0; i++){} + *tokenType = c==']' ? TK_ID : TK_ILLEGAL; + return i; + } + case CC_VARNUM: { + *tokenType = TK_VARIABLE; + for(i=1; sqlite3Isdigit(z[i]); i++){} + return i; + } + case CC_DOLLAR: + case CC_VARALPHA: { + int n = 0; + testcase( z[0]=='$' ); testcase( z[0]=='@' ); + testcase( z[0]==':' ); testcase( z[0]=='#' ); + *tokenType = TK_VARIABLE; + for(i=1; (c=z[i])!=0; i++){ + if( IdChar(c) ){ + n++; +#ifndef SQLITE_OMIT_TCL_VARIABLE + }else if( c=='(' && n>0 ){ + do{ + i++; + }while( (c=z[i])!=0 && !sqlite3Isspace(c) && c!=')' ); + if( c==')' ){ + i++; + }else{ + *tokenType = TK_ILLEGAL; + } + break; + }else if( c==':' && z[i+1]==':' ){ + i++; +#endif + }else{ + break; + } + } + if( n==0 ) *tokenType = TK_ILLEGAL; + return i; + } + case CC_KYWD: { + for(i=1; aiClass[z[i]]<=CC_KYWD; i++){} + if( IdChar(z[i]) ){ + /* This token started out using characters that can appear in keywords, + ** but z[i] is a character not allowed within keywords, so this must + ** be an identifier instead */ + i++; + break; + } + *tokenType = TK_ID; + return keywordCode((char*)z, i, tokenType); + } + case CC_X: { +#ifndef SQLITE_OMIT_BLOB_LITERAL + testcase( z[0]=='x' ); testcase( z[0]=='X' ); + if( z[1]=='\'' ){ + *tokenType = TK_BLOB; + for(i=2; sqlite3Isxdigit(z[i]); i++){} + if( z[i]!='\'' || i%2 ){ + *tokenType = TK_ILLEGAL; + while( z[i] && z[i]!='\'' ){ i++; } + } + if( z[i] ) i++; + return i; + } +#endif + /* If it is not a BLOB literal, then it must be an ID, since no + ** SQL keywords start with the letter 'x'. Fall through */ + } + case CC_ID: { + i = 1; + break; + } + case CC_NUL: { + *tokenType = TK_ILLEGAL; + return 0; + } + default: { + *tokenType = TK_ILLEGAL; + return 1; + } + } + while( IdChar(z[i]) ){ i++; } + *tokenType = TK_ID; + return i; +} + +/* +** Run the parser on the given SQL string. The parser structure is +** passed in. An SQLITE_ status code is returned. If an error occurs +** then an and attempt is made to write an error message into +** memory obtained from sqlite3_malloc() and to make *pzErrMsg point to that +** error message. +*/ +SQLITE_PRIVATE int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzErrMsg){ + int nErr = 0; /* Number of errors encountered */ + void *pEngine; /* The LEMON-generated LALR(1) parser */ + int n = 0; /* Length of the next token token */ + int tokenType; /* type of the next token */ + int lastTokenParsed = -1; /* type of the previous token */ + sqlite3 *db = pParse->db; /* The database connection */ + int mxSqlLen; /* Max length of an SQL string */ +#ifdef sqlite3Parser_ENGINEALWAYSONSTACK + yyParser sEngine; /* Space to hold the Lemon-generated Parser object */ +#endif + VVA_ONLY( u8 startedWithOom = db->mallocFailed ); + + assert( zSql!=0 ); + mxSqlLen = db->aLimit[SQLITE_LIMIT_SQL_LENGTH]; + if( db->nVdbeActive==0 ){ + AtomicStore(&db->u1.isInterrupted, 0); + } + pParse->rc = SQLITE_OK; + pParse->zTail = zSql; + assert( pzErrMsg!=0 ); +#ifdef SQLITE_DEBUG + if( db->flags & SQLITE_ParserTrace ){ + printf("parser: [[[%s]]]\n", zSql); + sqlite3ParserTrace(stdout, "parser: "); + }else{ + sqlite3ParserTrace(0, 0); + } +#endif +#ifdef sqlite3Parser_ENGINEALWAYSONSTACK + pEngine = &sEngine; + sqlite3ParserInit(pEngine, pParse); +#else + pEngine = sqlite3ParserAlloc(sqlite3Malloc, pParse); + if( pEngine==0 ){ + sqlite3OomFault(db); + return SQLITE_NOMEM_BKPT; + } +#endif + assert( pParse->pNewTable==0 ); + assert( pParse->pNewTrigger==0 ); + assert( pParse->nVar==0 ); + assert( pParse->pVList==0 ); + pParse->pParentParse = db->pParse; + db->pParse = pParse; + while( 1 ){ + n = sqlite3GetToken((u8*)zSql, &tokenType); + mxSqlLen -= n; + if( mxSqlLen<0 ){ + pParse->rc = SQLITE_TOOBIG; + break; + } +#ifndef SQLITE_OMIT_WINDOWFUNC + if( tokenType>=TK_WINDOW ){ + assert( tokenType==TK_SPACE || tokenType==TK_OVER || tokenType==TK_FILTER + || tokenType==TK_ILLEGAL || tokenType==TK_WINDOW + ); +#else + if( tokenType>=TK_SPACE ){ + assert( tokenType==TK_SPACE || tokenType==TK_ILLEGAL ); +#endif /* SQLITE_OMIT_WINDOWFUNC */ + if( AtomicLoad(&db->u1.isInterrupted) ){ + pParse->rc = SQLITE_INTERRUPT; + break; + } + if( tokenType==TK_SPACE ){ + zSql += n; + continue; + } + if( zSql[0]==0 ){ + /* Upon reaching the end of input, call the parser two more times + ** with tokens TK_SEMI and 0, in that order. */ + if( lastTokenParsed==TK_SEMI ){ + tokenType = 0; + }else if( lastTokenParsed==0 ){ + break; + }else{ + tokenType = TK_SEMI; + } + n = 0; +#ifndef SQLITE_OMIT_WINDOWFUNC + }else if( tokenType==TK_WINDOW ){ + assert( n==6 ); + tokenType = analyzeWindowKeyword((const u8*)&zSql[6]); + }else if( tokenType==TK_OVER ){ + assert( n==4 ); + tokenType = analyzeOverKeyword((const u8*)&zSql[4], lastTokenParsed); + }else if( tokenType==TK_FILTER ){ + assert( n==6 ); + tokenType = analyzeFilterKeyword((const u8*)&zSql[6], lastTokenParsed); +#endif /* SQLITE_OMIT_WINDOWFUNC */ + }else{ + sqlite3ErrorMsg(pParse, "unrecognized token: \"%.*s\"", n, zSql); + break; + } + } + pParse->sLastToken.z = zSql; + pParse->sLastToken.n = n; + sqlite3Parser(pEngine, tokenType, pParse->sLastToken); + lastTokenParsed = tokenType; + zSql += n; + assert( db->mallocFailed==0 || pParse->rc!=SQLITE_OK || startedWithOom ); + if( pParse->rc!=SQLITE_OK ) break; + } + assert( nErr==0 ); +#ifdef YYTRACKMAXSTACKDEPTH + sqlite3_mutex_enter(sqlite3MallocMutex()); + sqlite3StatusHighwater(SQLITE_STATUS_PARSER_STACK, + sqlite3ParserStackPeak(pEngine) + ); + sqlite3_mutex_leave(sqlite3MallocMutex()); +#endif /* YYDEBUG */ +#ifdef sqlite3Parser_ENGINEALWAYSONSTACK + sqlite3ParserFinalize(pEngine); +#else + sqlite3ParserFree(pEngine, sqlite3_free); +#endif + if( db->mallocFailed ){ + pParse->rc = SQLITE_NOMEM_BKPT; + } + if( pParse->rc!=SQLITE_OK && pParse->rc!=SQLITE_DONE && pParse->zErrMsg==0 ){ + pParse->zErrMsg = sqlite3MPrintf(db, "%s", sqlite3ErrStr(pParse->rc)); + } + assert( pzErrMsg!=0 ); + if( pParse->zErrMsg ){ + *pzErrMsg = pParse->zErrMsg; + sqlite3_log(pParse->rc, "%s in \"%s\"", + *pzErrMsg, pParse->zTail); + pParse->zErrMsg = 0; + nErr++; + } + pParse->zTail = zSql; + if( pParse->pVdbe && pParse->nErr>0 && pParse->nested==0 ){ + sqlite3VdbeDelete(pParse->pVdbe); + pParse->pVdbe = 0; + } +#ifndef SQLITE_OMIT_SHARED_CACHE + if( pParse->nested==0 ){ + sqlite3DbFree(db, pParse->aTableLock); + pParse->aTableLock = 0; + pParse->nTableLock = 0; + } +#endif +#ifndef SQLITE_OMIT_VIRTUALTABLE + sqlite3_free(pParse->apVtabLock); +#endif + + if( !IN_SPECIAL_PARSE ){ + /* If the pParse->declareVtab flag is set, do not delete any table + ** structure built up in pParse->pNewTable. The calling code (see vtab.c) + ** will take responsibility for freeing the Table structure. + */ + sqlite3DeleteTable(db, pParse->pNewTable); + } + if( !IN_RENAME_OBJECT ){ + sqlite3DeleteTrigger(db, pParse->pNewTrigger); + } + + if( pParse->pWithToFree ) sqlite3WithDelete(db, pParse->pWithToFree); + sqlite3DbFree(db, pParse->pVList); + while( pParse->pAinc ){ + AutoincInfo *p = pParse->pAinc; + pParse->pAinc = p->pNext; + sqlite3DbFreeNN(db, p); + } + while( pParse->pZombieTab ){ + Table *p = pParse->pZombieTab; + pParse->pZombieTab = p->pNextZombie; + sqlite3DeleteTable(db, p); + } + db->pParse = pParse->pParentParse; + pParse->pParentParse = 0; + assert( nErr==0 || pParse->rc!=SQLITE_OK ); + return nErr; +} + + +#ifdef SQLITE_ENABLE_NORMALIZE +/* +** Insert a single space character into pStr if the current string +** ends with an identifier +*/ +static void addSpaceSeparator(sqlite3_str *pStr){ + if( pStr->nChar && sqlite3IsIdChar(pStr->zText[pStr->nChar-1]) ){ + sqlite3_str_append(pStr, " ", 1); + } +} + +/* +** Compute a normalization of the SQL given by zSql[0..nSql-1]. Return +** the normalization in space obtained from sqlite3DbMalloc(). Or return +** NULL if anything goes wrong or if zSql is NULL. +*/ +SQLITE_PRIVATE char *sqlite3Normalize( + Vdbe *pVdbe, /* VM being reprepared */ + const char *zSql /* The original SQL string */ +){ + sqlite3 *db; /* The database connection */ + int i; /* Next unread byte of zSql[] */ + int n; /* length of current token */ + int tokenType; /* type of current token */ + int prevType = 0; /* Previous non-whitespace token */ + int nParen; /* Number of nested levels of parentheses */ + int iStartIN; /* Start of RHS of IN operator in z[] */ + int nParenAtIN; /* Value of nParent at start of RHS of IN operator */ + u32 j; /* Bytes of normalized SQL generated so far */ + sqlite3_str *pStr; /* The normalized SQL string under construction */ + + db = sqlite3VdbeDb(pVdbe); + tokenType = -1; + nParen = iStartIN = nParenAtIN = 0; + pStr = sqlite3_str_new(db); + assert( pStr!=0 ); /* sqlite3_str_new() never returns NULL */ + for(i=0; zSql[i] && pStr->accError==0; i+=n){ + if( tokenType!=TK_SPACE ){ + prevType = tokenType; + } + n = sqlite3GetToken((unsigned char*)zSql+i, &tokenType); + if( NEVER(n<=0) ) break; + switch( tokenType ){ + case TK_SPACE: { + break; + } + case TK_NULL: { + if( prevType==TK_IS || prevType==TK_NOT ){ + sqlite3_str_append(pStr, " NULL", 5); + break; + } + /* Fall through */ + } + case TK_STRING: + case TK_INTEGER: + case TK_FLOAT: + case TK_VARIABLE: + case TK_BLOB: { + sqlite3_str_append(pStr, "?", 1); + break; + } + case TK_LP: { + nParen++; + if( prevType==TK_IN ){ + iStartIN = pStr->nChar; + nParenAtIN = nParen; + } + sqlite3_str_append(pStr, "(", 1); + break; + } + case TK_RP: { + if( iStartIN>0 && nParen==nParenAtIN ){ + assert( pStr->nChar>=(u32)iStartIN ); + pStr->nChar = iStartIN+1; + sqlite3_str_append(pStr, "?,?,?", 5); + iStartIN = 0; + } + nParen--; + sqlite3_str_append(pStr, ")", 1); + break; + } + case TK_ID: { + iStartIN = 0; + j = pStr->nChar; + if( sqlite3Isquote(zSql[i]) ){ + char *zId = sqlite3DbStrNDup(db, zSql+i, n); + int nId; + int eType = 0; + if( zId==0 ) break; + sqlite3Dequote(zId); + if( zSql[i]=='"' && sqlite3VdbeUsesDoubleQuotedString(pVdbe, zId) ){ + sqlite3_str_append(pStr, "?", 1); + sqlite3DbFree(db, zId); + break; + } + nId = sqlite3Strlen30(zId); + if( sqlite3GetToken((u8*)zId, &eType)==nId && eType==TK_ID ){ + addSpaceSeparator(pStr); + sqlite3_str_append(pStr, zId, nId); + }else{ + sqlite3_str_appendf(pStr, "\"%w\"", zId); + } + sqlite3DbFree(db, zId); + }else{ + addSpaceSeparator(pStr); + sqlite3_str_append(pStr, zSql+i, n); + } + while( jnChar ){ + pStr->zText[j] = sqlite3Tolower(pStr->zText[j]); + j++; + } + break; + } + case TK_SELECT: { + iStartIN = 0; + /* fall through */ + } + default: { + if( sqlite3IsIdChar(zSql[i]) ) addSpaceSeparator(pStr); + j = pStr->nChar; + sqlite3_str_append(pStr, zSql+i, n); + while( jnChar ){ + pStr->zText[j] = sqlite3Toupper(pStr->zText[j]); + j++; + } + break; + } + } + } + if( tokenType!=TK_SEMI ) sqlite3_str_append(pStr, ";", 1); + return sqlite3_str_finish(pStr); +} +#endif /* SQLITE_ENABLE_NORMALIZE */ + +/************** End of tokenize.c ********************************************/ +/************** Begin file complete.c ****************************************/ +/* +** 2001 September 15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** An tokenizer for SQL +** +** This file contains C code that implements the sqlite3_complete() API. +** This code used to be part of the tokenizer.c source file. But by +** separating it out, the code will be automatically omitted from +** static links that do not use it. +*/ +/* #include "sqliteInt.h" */ +#ifndef SQLITE_OMIT_COMPLETE + +/* +** This is defined in tokenize.c. We just have to import the definition. +*/ +#ifndef SQLITE_AMALGAMATION +#ifdef SQLITE_ASCII +#define IdChar(C) ((sqlite3CtypeMap[(unsigned char)C]&0x46)!=0) +#endif +#ifdef SQLITE_EBCDIC +SQLITE_PRIVATE const char sqlite3IsEbcdicIdChar[]; +#define IdChar(C) (((c=C)>=0x42 && sqlite3IsEbcdicIdChar[c-0x40])) +#endif +#endif /* SQLITE_AMALGAMATION */ + + +/* +** Token types used by the sqlite3_complete() routine. See the header +** comments on that procedure for additional information. +*/ +#define tkSEMI 0 +#define tkWS 1 +#define tkOTHER 2 +#ifndef SQLITE_OMIT_TRIGGER +#define tkEXPLAIN 3 +#define tkCREATE 4 +#define tkTEMP 5 +#define tkTRIGGER 6 +#define tkEND 7 +#endif + +/* +** Return TRUE if the given SQL string ends in a semicolon. +** +** Special handling is require for CREATE TRIGGER statements. +** Whenever the CREATE TRIGGER keywords are seen, the statement +** must end with ";END;". +** +** This implementation uses a state machine with 8 states: +** +** (0) INVALID We have not yet seen a non-whitespace character. +** +** (1) START At the beginning or end of an SQL statement. This routine +** returns 1 if it ends in the START state and 0 if it ends +** in any other state. +** +** (2) NORMAL We are in the middle of statement which ends with a single +** semicolon. +** +** (3) EXPLAIN The keyword EXPLAIN has been seen at the beginning of +** a statement. +** +** (4) CREATE The keyword CREATE has been seen at the beginning of a +** statement, possibly preceded by EXPLAIN and/or followed by +** TEMP or TEMPORARY +** +** (5) TRIGGER We are in the middle of a trigger definition that must be +** ended by a semicolon, the keyword END, and another semicolon. +** +** (6) SEMI We've seen the first semicolon in the ";END;" that occurs at +** the end of a trigger definition. +** +** (7) END We've seen the ";END" of the ";END;" that occurs at the end +** of a trigger definition. +** +** Transitions between states above are determined by tokens extracted +** from the input. The following tokens are significant: +** +** (0) tkSEMI A semicolon. +** (1) tkWS Whitespace. +** (2) tkOTHER Any other SQL token. +** (3) tkEXPLAIN The "explain" keyword. +** (4) tkCREATE The "create" keyword. +** (5) tkTEMP The "temp" or "temporary" keyword. +** (6) tkTRIGGER The "trigger" keyword. +** (7) tkEND The "end" keyword. +** +** Whitespace never causes a state transition and is always ignored. +** This means that a SQL string of all whitespace is invalid. +** +** If we compile with SQLITE_OMIT_TRIGGER, all of the computation needed +** to recognize the end of a trigger can be omitted. All we have to do +** is look for a semicolon that is not part of an string or comment. +*/ +SQLITE_API int sqlite3_complete(const char *zSql){ + u8 state = 0; /* Current state, using numbers defined in header comment */ + u8 token; /* Value of the next token */ + +#ifndef SQLITE_OMIT_TRIGGER + /* A complex statement machine used to detect the end of a CREATE TRIGGER + ** statement. This is the normal case. + */ + static const u8 trans[8][8] = { + /* Token: */ + /* State: ** SEMI WS OTHER EXPLAIN CREATE TEMP TRIGGER END */ + /* 0 INVALID: */ { 1, 0, 2, 3, 4, 2, 2, 2, }, + /* 1 START: */ { 1, 1, 2, 3, 4, 2, 2, 2, }, + /* 2 NORMAL: */ { 1, 2, 2, 2, 2, 2, 2, 2, }, + /* 3 EXPLAIN: */ { 1, 3, 3, 2, 4, 2, 2, 2, }, + /* 4 CREATE: */ { 1, 4, 2, 2, 2, 4, 5, 2, }, + /* 5 TRIGGER: */ { 6, 5, 5, 5, 5, 5, 5, 5, }, + /* 6 SEMI: */ { 6, 6, 5, 5, 5, 5, 5, 7, }, + /* 7 END: */ { 1, 7, 5, 5, 5, 5, 5, 5, }, + }; +#else + /* If triggers are not supported by this compile then the statement machine + ** used to detect the end of a statement is much simpler + */ + static const u8 trans[3][3] = { + /* Token: */ + /* State: ** SEMI WS OTHER */ + /* 0 INVALID: */ { 1, 0, 2, }, + /* 1 START: */ { 1, 1, 2, }, + /* 2 NORMAL: */ { 1, 2, 2, }, + }; +#endif /* SQLITE_OMIT_TRIGGER */ + +#ifdef SQLITE_ENABLE_API_ARMOR + if( zSql==0 ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + + while( *zSql ){ + switch( *zSql ){ + case ';': { /* A semicolon */ + token = tkSEMI; + break; + } + case ' ': + case '\r': + case '\t': + case '\n': + case '\f': { /* White space is ignored */ + token = tkWS; + break; + } + case '/': { /* C-style comments */ + if( zSql[1]!='*' ){ + token = tkOTHER; + break; + } + zSql += 2; + while( zSql[0] && (zSql[0]!='*' || zSql[1]!='/') ){ zSql++; } + if( zSql[0]==0 ) return 0; + zSql++; + token = tkWS; + break; + } + case '-': { /* SQL-style comments from "--" to end of line */ + if( zSql[1]!='-' ){ + token = tkOTHER; + break; + } + while( *zSql && *zSql!='\n' ){ zSql++; } + if( *zSql==0 ) return state==1; + token = tkWS; + break; + } + case '[': { /* Microsoft-style identifiers in [...] */ + zSql++; + while( *zSql && *zSql!=']' ){ zSql++; } + if( *zSql==0 ) return 0; + token = tkOTHER; + break; + } + case '`': /* Grave-accent quoted symbols used by MySQL */ + case '"': /* single- and double-quoted strings */ + case '\'': { + int c = *zSql; + zSql++; + while( *zSql && *zSql!=c ){ zSql++; } + if( *zSql==0 ) return 0; + token = tkOTHER; + break; + } + default: { +#ifdef SQLITE_EBCDIC + unsigned char c; +#endif + if( IdChar((u8)*zSql) ){ + /* Keywords and unquoted identifiers */ + int nId; + for(nId=1; IdChar(zSql[nId]); nId++){} +#ifdef SQLITE_OMIT_TRIGGER + token = tkOTHER; +#else + switch( *zSql ){ + case 'c': case 'C': { + if( nId==6 && sqlite3StrNICmp(zSql, "create", 6)==0 ){ + token = tkCREATE; + }else{ + token = tkOTHER; + } + break; + } + case 't': case 'T': { + if( nId==7 && sqlite3StrNICmp(zSql, "trigger", 7)==0 ){ + token = tkTRIGGER; + }else if( nId==4 && sqlite3StrNICmp(zSql, "temp", 4)==0 ){ + token = tkTEMP; + }else if( nId==9 && sqlite3StrNICmp(zSql, "temporary", 9)==0 ){ + token = tkTEMP; + }else{ + token = tkOTHER; + } + break; + } + case 'e': case 'E': { + if( nId==3 && sqlite3StrNICmp(zSql, "end", 3)==0 ){ + token = tkEND; + }else +#ifndef SQLITE_OMIT_EXPLAIN + if( nId==7 && sqlite3StrNICmp(zSql, "explain", 7)==0 ){ + token = tkEXPLAIN; + }else +#endif + { + token = tkOTHER; + } + break; + } + default: { + token = tkOTHER; + break; + } + } +#endif /* SQLITE_OMIT_TRIGGER */ + zSql += nId-1; + }else{ + /* Operators and special symbols */ + token = tkOTHER; + } + break; + } + } + state = trans[state][token]; + zSql++; + } + return state==1; +} + +#ifndef SQLITE_OMIT_UTF16 +/* +** This routine is the same as the sqlite3_complete() routine described +** above, except that the parameter is required to be UTF-16 encoded, not +** UTF-8. +*/ +SQLITE_API int sqlite3_complete16(const void *zSql){ + sqlite3_value *pVal; + char const *zSql8; + int rc; + +#ifndef SQLITE_OMIT_AUTOINIT + rc = sqlite3_initialize(); + if( rc ) return rc; +#endif + pVal = sqlite3ValueNew(0); + sqlite3ValueSetStr(pVal, -1, zSql, SQLITE_UTF16NATIVE, SQLITE_STATIC); + zSql8 = sqlite3ValueText(pVal, SQLITE_UTF8); + if( zSql8 ){ + rc = sqlite3_complete(zSql8); + }else{ + rc = SQLITE_NOMEM_BKPT; + } + sqlite3ValueFree(pVal); + return rc & 0xff; +} +#endif /* SQLITE_OMIT_UTF16 */ +#endif /* SQLITE_OMIT_COMPLETE */ + +/************** End of complete.c ********************************************/ +/************** Begin file main.c ********************************************/ +/* +** 2001 September 15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** Main file for the SQLite library. The routines in this file +** implement the programmer interface to the library. Routines in +** other files are for internal use by SQLite and should not be +** accessed by users of the library. +*/ +/* #include "sqliteInt.h" */ + +#ifdef SQLITE_ENABLE_FTS3 +/************** Include fts3.h in the middle of main.c ***********************/ +/************** Begin file fts3.h ********************************************/ +/* +** 2006 Oct 10 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This header file is used by programs that want to link against the +** FTS3 library. All it does is declare the sqlite3Fts3Init() interface. +*/ +/* #include "sqlite3.h" */ + +#if 0 +extern "C" { +#endif /* __cplusplus */ + +SQLITE_PRIVATE int sqlite3Fts3Init(sqlite3 *db); + +#if 0 +} /* extern "C" */ +#endif /* __cplusplus */ + +/************** End of fts3.h ************************************************/ +/************** Continuing where we left off in main.c ***********************/ +#endif +#ifdef SQLITE_ENABLE_RTREE +/************** Include rtree.h in the middle of main.c **********************/ +/************** Begin file rtree.h *******************************************/ +/* +** 2008 May 26 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This header file is used by programs that want to link against the +** RTREE library. All it does is declare the sqlite3RtreeInit() interface. +*/ +/* #include "sqlite3.h" */ + +#ifdef SQLITE_OMIT_VIRTUALTABLE +# undef SQLITE_ENABLE_RTREE +#endif + +#if 0 +extern "C" { +#endif /* __cplusplus */ + +SQLITE_PRIVATE int sqlite3RtreeInit(sqlite3 *db); + +#if 0 +} /* extern "C" */ +#endif /* __cplusplus */ + +/************** End of rtree.h ***********************************************/ +/************** Continuing where we left off in main.c ***********************/ +#endif +#if defined(SQLITE_ENABLE_ICU) || defined(SQLITE_ENABLE_ICU_COLLATIONS) +/************** Include sqliteicu.h in the middle of main.c ******************/ +/************** Begin file sqliteicu.h ***************************************/ +/* +** 2008 May 26 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This header file is used by programs that want to link against the +** ICU extension. All it does is declare the sqlite3IcuInit() interface. +*/ +/* #include "sqlite3.h" */ + +#if 0 +extern "C" { +#endif /* __cplusplus */ + +SQLITE_PRIVATE int sqlite3IcuInit(sqlite3 *db); + +#if 0 +} /* extern "C" */ +#endif /* __cplusplus */ + + +/************** End of sqliteicu.h *******************************************/ +/************** Continuing where we left off in main.c ***********************/ +#endif + +/* +** This is an extension initializer that is a no-op and always +** succeeds, except that it fails if the fault-simulation is set +** to 500. +*/ +static int sqlite3TestExtInit(sqlite3 *db){ + (void)db; + return sqlite3FaultSim(500); +} + + +/* +** Forward declarations of external module initializer functions +** for modules that need them. +*/ +#ifdef SQLITE_ENABLE_FTS1 +SQLITE_PRIVATE int sqlite3Fts1Init(sqlite3*); +#endif +#ifdef SQLITE_ENABLE_FTS2 +SQLITE_PRIVATE int sqlite3Fts2Init(sqlite3*); +#endif +#ifdef SQLITE_ENABLE_FTS5 +SQLITE_PRIVATE int sqlite3Fts5Init(sqlite3*); +#endif +#ifdef SQLITE_ENABLE_JSON1 +SQLITE_PRIVATE int sqlite3Json1Init(sqlite3*); +#endif +#ifdef SQLITE_ENABLE_STMTVTAB +SQLITE_PRIVATE int sqlite3StmtVtabInit(sqlite3*); +#endif + +/* +** An array of pointers to extension initializer functions for +** built-in extensions. +*/ +static int (*const sqlite3BuiltinExtensions[])(sqlite3*) = { +#ifdef SQLITE_ENABLE_FTS1 + sqlite3Fts1Init, +#endif +#ifdef SQLITE_ENABLE_FTS2 + sqlite3Fts2Init, +#endif +#ifdef SQLITE_ENABLE_FTS3 + sqlite3Fts3Init, +#endif +#ifdef SQLITE_ENABLE_FTS5 + sqlite3Fts5Init, +#endif +#if defined(SQLITE_ENABLE_ICU) || defined(SQLITE_ENABLE_ICU_COLLATIONS) + sqlite3IcuInit, +#endif +#ifdef SQLITE_ENABLE_RTREE + sqlite3RtreeInit, +#endif +#ifdef SQLITE_ENABLE_DBPAGE_VTAB + sqlite3DbpageRegister, +#endif +#ifdef SQLITE_ENABLE_DBSTAT_VTAB + sqlite3DbstatRegister, +#endif + sqlite3TestExtInit, +#ifdef SQLITE_ENABLE_JSON1 + sqlite3Json1Init, +#endif +#ifdef SQLITE_ENABLE_STMTVTAB + sqlite3StmtVtabInit, +#endif +#ifdef SQLITE_ENABLE_BYTECODE_VTAB + sqlite3VdbeBytecodeVtabInit, +#endif +}; + +#ifndef SQLITE_AMALGAMATION +/* IMPLEMENTATION-OF: R-46656-45156 The sqlite3_version[] string constant +** contains the text of SQLITE_VERSION macro. +*/ +SQLITE_API const char sqlite3_version[] = SQLITE_VERSION; +#endif + +/* IMPLEMENTATION-OF: R-53536-42575 The sqlite3_libversion() function returns +** a pointer to the to the sqlite3_version[] string constant. +*/ +SQLITE_API const char *sqlite3_libversion(void){ return sqlite3_version; } + +/* IMPLEMENTATION-OF: R-25063-23286 The sqlite3_sourceid() function returns a +** pointer to a string constant whose value is the same as the +** SQLITE_SOURCE_ID C preprocessor macro. Except if SQLite is built using +** an edited copy of the amalgamation, then the last four characters of +** the hash might be different from SQLITE_SOURCE_ID. +*/ +/* SQLITE_API const char *sqlite3_sourceid(void){ return SQLITE_SOURCE_ID; } */ + +/* IMPLEMENTATION-OF: R-35210-63508 The sqlite3_libversion_number() function +** returns an integer equal to SQLITE_VERSION_NUMBER. +*/ +SQLITE_API int sqlite3_libversion_number(void){ return SQLITE_VERSION_NUMBER; } + +/* IMPLEMENTATION-OF: R-20790-14025 The sqlite3_threadsafe() function returns +** zero if and only if SQLite was compiled with mutexing code omitted due to +** the SQLITE_THREADSAFE compile-time option being set to 0. +*/ +SQLITE_API int sqlite3_threadsafe(void){ return SQLITE_THREADSAFE; } + +/* +** When compiling the test fixture or with debugging enabled (on Win32), +** this variable being set to non-zero will cause OSTRACE macros to emit +** extra diagnostic information. +*/ +#ifdef SQLITE_HAVE_OS_TRACE +# ifndef SQLITE_DEBUG_OS_TRACE +# define SQLITE_DEBUG_OS_TRACE 0 +# endif + int sqlite3OSTrace = SQLITE_DEBUG_OS_TRACE; +#endif + +#if !defined(SQLITE_OMIT_TRACE) && defined(SQLITE_ENABLE_IOTRACE) +/* +** If the following function pointer is not NULL and if +** SQLITE_ENABLE_IOTRACE is enabled, then messages describing +** I/O active are written using this function. These messages +** are intended for debugging activity only. +*/ +SQLITE_API void (SQLITE_CDECL *sqlite3IoTrace)(const char*, ...) = 0; +#endif + +/* +** If the following global variable points to a string which is the +** name of a directory, then that directory will be used to store +** temporary files. +** +** See also the "PRAGMA temp_store_directory" SQL command. +*/ +SQLITE_API char *sqlite3_temp_directory = 0; + +/* +** If the following global variable points to a string which is the +** name of a directory, then that directory will be used to store +** all database files specified with a relative pathname. +** +** See also the "PRAGMA data_store_directory" SQL command. +*/ +SQLITE_API char *sqlite3_data_directory = 0; + +/* +** Initialize SQLite. +** +** This routine must be called to initialize the memory allocation, +** VFS, and mutex subsystems prior to doing any serious work with +** SQLite. But as long as you do not compile with SQLITE_OMIT_AUTOINIT +** this routine will be called automatically by key routines such as +** sqlite3_open(). +** +** This routine is a no-op except on its very first call for the process, +** or for the first call after a call to sqlite3_shutdown. +** +** The first thread to call this routine runs the initialization to +** completion. If subsequent threads call this routine before the first +** thread has finished the initialization process, then the subsequent +** threads must block until the first thread finishes with the initialization. +** +** The first thread might call this routine recursively. Recursive +** calls to this routine should not block, of course. Otherwise the +** initialization process would never complete. +** +** Let X be the first thread to enter this routine. Let Y be some other +** thread. Then while the initial invocation of this routine by X is +** incomplete, it is required that: +** +** * Calls to this routine from Y must block until the outer-most +** call by X completes. +** +** * Recursive calls to this routine from thread X return immediately +** without blocking. +*/ +SQLITE_API int sqlite3_initialize(void){ + MUTEX_LOGIC( sqlite3_mutex *pMaster; ) /* The main static mutex */ + int rc; /* Result code */ +#ifdef SQLITE_EXTRA_INIT + int bRunExtraInit = 0; /* Extra initialization needed */ +#endif + +#ifdef SQLITE_OMIT_WSD + rc = sqlite3_wsd_init(4096, 24); + if( rc!=SQLITE_OK ){ + return rc; + } +#endif + + /* If the following assert() fails on some obscure processor/compiler + ** combination, the work-around is to set the correct pointer + ** size at compile-time using -DSQLITE_PTRSIZE=n compile-time option */ + assert( SQLITE_PTRSIZE==sizeof(char*) ); + + /* If SQLite is already completely initialized, then this call + ** to sqlite3_initialize() should be a no-op. But the initialization + ** must be complete. So isInit must not be set until the very end + ** of this routine. + */ + if( sqlite3GlobalConfig.isInit ){ + sqlite3MemoryBarrier(); + return SQLITE_OK; + } + + /* Make sure the mutex subsystem is initialized. If unable to + ** initialize the mutex subsystem, return early with the error. + ** If the system is so sick that we are unable to allocate a mutex, + ** there is not much SQLite is going to be able to do. + ** + ** The mutex subsystem must take care of serializing its own + ** initialization. + */ + rc = sqlite3MutexInit(); + if( rc ) return rc; + + /* Initialize the malloc() system and the recursive pInitMutex mutex. + ** This operation is protected by the STATIC_MASTER mutex. Note that + ** MutexAlloc() is called for a static mutex prior to initializing the + ** malloc subsystem - this implies that the allocation of a static + ** mutex must not require support from the malloc subsystem. + */ + MUTEX_LOGIC( pMaster = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); ) + sqlite3_mutex_enter(pMaster); + sqlite3GlobalConfig.isMutexInit = 1; + if( !sqlite3GlobalConfig.isMallocInit ){ + rc = sqlite3MallocInit(); + } + if( rc==SQLITE_OK ){ + sqlite3GlobalConfig.isMallocInit = 1; + if( !sqlite3GlobalConfig.pInitMutex ){ + sqlite3GlobalConfig.pInitMutex = + sqlite3MutexAlloc(SQLITE_MUTEX_RECURSIVE); + if( sqlite3GlobalConfig.bCoreMutex && !sqlite3GlobalConfig.pInitMutex ){ + rc = SQLITE_NOMEM_BKPT; + } + } + } + if( rc==SQLITE_OK ){ + sqlite3GlobalConfig.nRefInitMutex++; + } + sqlite3_mutex_leave(pMaster); + + /* If rc is not SQLITE_OK at this point, then either the malloc + ** subsystem could not be initialized or the system failed to allocate + ** the pInitMutex mutex. Return an error in either case. */ + if( rc!=SQLITE_OK ){ + return rc; + } + + /* Do the rest of the initialization under the recursive mutex so + ** that we will be able to handle recursive calls into + ** sqlite3_initialize(). The recursive calls normally come through + ** sqlite3_os_init() when it invokes sqlite3_vfs_register(), but other + ** recursive calls might also be possible. + ** + ** IMPLEMENTATION-OF: R-00140-37445 SQLite automatically serializes calls + ** to the xInit method, so the xInit method need not be threadsafe. + ** + ** The following mutex is what serializes access to the appdef pcache xInit + ** methods. The sqlite3_pcache_methods.xInit() all is embedded in the + ** call to sqlite3PcacheInitialize(). + */ + sqlite3_mutex_enter(sqlite3GlobalConfig.pInitMutex); + if( sqlite3GlobalConfig.isInit==0 && sqlite3GlobalConfig.inProgress==0 ){ + sqlite3GlobalConfig.inProgress = 1; +#ifdef SQLITE_ENABLE_SQLLOG + { + extern void sqlite3_init_sqllog(void); + sqlite3_init_sqllog(); } - case CC_LT: { - if( (c=z[1])=='=' ){ - *tokenType = TK_LE; - return 2; - }else if( c=='>' ){ - *tokenType = TK_NE; - return 2; - }else if( c=='<' ){ - *tokenType = TK_LSHIFT; - return 2; - }else{ - *tokenType = TK_LT; - return 1; - } +#endif + memset(&sqlite3BuiltinFunctions, 0, sizeof(sqlite3BuiltinFunctions)); + sqlite3RegisterBuiltinFunctions(); + if( sqlite3GlobalConfig.isPCacheInit==0 ){ + rc = sqlite3PcacheInitialize(); } - case CC_GT: { - if( (c=z[1])=='=' ){ - *tokenType = TK_GE; - return 2; - }else if( c=='>' ){ - *tokenType = TK_RSHIFT; - return 2; - }else{ - *tokenType = TK_GT; - return 1; - } + if( rc==SQLITE_OK ){ + sqlite3GlobalConfig.isPCacheInit = 1; + rc = sqlite3OsInit(); } - case CC_BANG: { - if( z[1]!='=' ){ - *tokenType = TK_ILLEGAL; - return 1; - }else{ - *tokenType = TK_NE; - return 2; - } +#ifdef SQLITE_ENABLE_DESERIALIZE + if( rc==SQLITE_OK ){ + rc = sqlite3MemdbInit(); } - case CC_PIPE: { - if( z[1]!='|' ){ - *tokenType = TK_BITOR; - return 1; - }else{ - *tokenType = TK_CONCAT; - return 2; - } +#endif + if( rc==SQLITE_OK ){ + sqlite3PCacheBufferSetup( sqlite3GlobalConfig.pPage, + sqlite3GlobalConfig.szPage, sqlite3GlobalConfig.nPage); + sqlite3MemoryBarrier(); + sqlite3GlobalConfig.isInit = 1; +#ifdef SQLITE_EXTRA_INIT + bRunExtraInit = 1; +#endif } - case CC_COMMA: { - *tokenType = TK_COMMA; - return 1; + sqlite3GlobalConfig.inProgress = 0; + } + sqlite3_mutex_leave(sqlite3GlobalConfig.pInitMutex); + + /* Go back under the static mutex and clean up the recursive + ** mutex to prevent a resource leak. + */ + sqlite3_mutex_enter(pMaster); + sqlite3GlobalConfig.nRefInitMutex--; + if( sqlite3GlobalConfig.nRefInitMutex<=0 ){ + assert( sqlite3GlobalConfig.nRefInitMutex==0 ); + sqlite3_mutex_free(sqlite3GlobalConfig.pInitMutex); + sqlite3GlobalConfig.pInitMutex = 0; + } + sqlite3_mutex_leave(pMaster); + + /* The following is just a sanity check to make sure SQLite has + ** been compiled correctly. It is important to run this code, but + ** we don't want to run it too often and soak up CPU cycles for no + ** reason. So we run it once during initialization. + */ +#ifndef NDEBUG +#ifndef SQLITE_OMIT_FLOATING_POINT + /* This section of code's only "output" is via assert() statements. */ + if( rc==SQLITE_OK ){ + u64 x = (((u64)1)<<63)-1; + double y; + assert(sizeof(x)==8); + assert(sizeof(x)==sizeof(y)); + memcpy(&y, &x, 8); + assert( sqlite3IsNaN(y) ); + } +#endif +#endif + + /* Do extra initialization steps requested by the SQLITE_EXTRA_INIT + ** compile-time option. + */ +#ifdef SQLITE_EXTRA_INIT + if( bRunExtraInit ){ + int SQLITE_EXTRA_INIT(const char*); + rc = SQLITE_EXTRA_INIT(0); + } +#endif + + return rc; +} + +/* +** Undo the effects of sqlite3_initialize(). Must not be called while +** there are outstanding database connections or memory allocations or +** while any part of SQLite is otherwise in use in any thread. This +** routine is not threadsafe. But it is safe to invoke this routine +** on when SQLite is already shut down. If SQLite is already shut down +** when this routine is invoked, then this routine is a harmless no-op. +*/ +SQLITE_API int sqlite3_shutdown(void){ +#ifdef SQLITE_OMIT_WSD + int rc = sqlite3_wsd_init(4096, 24); + if( rc!=SQLITE_OK ){ + return rc; + } +#endif + + if( sqlite3GlobalConfig.isInit ){ +#ifdef SQLITE_EXTRA_SHUTDOWN + void SQLITE_EXTRA_SHUTDOWN(void); + SQLITE_EXTRA_SHUTDOWN(); +#endif + sqlite3_os_end(); + sqlite3_reset_auto_extension(); + sqlite3GlobalConfig.isInit = 0; + } + if( sqlite3GlobalConfig.isPCacheInit ){ + sqlite3PcacheShutdown(); + sqlite3GlobalConfig.isPCacheInit = 0; + } + if( sqlite3GlobalConfig.isMallocInit ){ + sqlite3MallocEnd(); + sqlite3GlobalConfig.isMallocInit = 0; + +#ifndef SQLITE_OMIT_SHUTDOWN_DIRECTORIES + /* The heap subsystem has now been shutdown and these values are supposed + ** to be NULL or point to memory that was obtained from sqlite3_malloc(), + ** which would rely on that heap subsystem; therefore, make sure these + ** values cannot refer to heap memory that was just invalidated when the + ** heap subsystem was shutdown. This is only done if the current call to + ** this function resulted in the heap subsystem actually being shutdown. + */ + sqlite3_data_directory = 0; + sqlite3_temp_directory = 0; +#endif + } + if( sqlite3GlobalConfig.isMutexInit ){ + sqlite3MutexEnd(); + sqlite3GlobalConfig.isMutexInit = 0; + } + + return SQLITE_OK; +} + +/* +** This API allows applications to modify the global configuration of +** the SQLite library at run-time. +** +** This routine should only be called when there are no outstanding +** database connections or memory allocations. This routine is not +** threadsafe. Failure to heed these warnings can lead to unpredictable +** behavior. +*/ +SQLITE_API int sqlite3_config(int op, ...){ + va_list ap; + int rc = SQLITE_OK; + + /* sqlite3_config() shall return SQLITE_MISUSE if it is invoked while + ** the SQLite library is in use. */ + if( sqlite3GlobalConfig.isInit ) return SQLITE_MISUSE_BKPT; + + va_start(ap, op); + switch( op ){ + + /* Mutex configuration options are only available in a threadsafe + ** compile. + */ +#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE>0 /* IMP: R-54466-46756 */ + case SQLITE_CONFIG_SINGLETHREAD: { + /* EVIDENCE-OF: R-02748-19096 This option sets the threading mode to + ** Single-thread. */ + sqlite3GlobalConfig.bCoreMutex = 0; /* Disable mutex on core */ + sqlite3GlobalConfig.bFullMutex = 0; /* Disable mutex on connections */ + break; } - case CC_AND: { - *tokenType = TK_BITAND; - return 1; +#endif +#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE>0 /* IMP: R-20520-54086 */ + case SQLITE_CONFIG_MULTITHREAD: { + /* EVIDENCE-OF: R-14374-42468 This option sets the threading mode to + ** Multi-thread. */ + sqlite3GlobalConfig.bCoreMutex = 1; /* Enable mutex on core */ + sqlite3GlobalConfig.bFullMutex = 0; /* Disable mutex on connections */ + break; } - case CC_TILDA: { - *tokenType = TK_BITNOT; - return 1; +#endif +#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE>0 /* IMP: R-59593-21810 */ + case SQLITE_CONFIG_SERIALIZED: { + /* EVIDENCE-OF: R-41220-51800 This option sets the threading mode to + ** Serialized. */ + sqlite3GlobalConfig.bCoreMutex = 1; /* Enable mutex on core */ + sqlite3GlobalConfig.bFullMutex = 1; /* Enable mutex on connections */ + break; } - case CC_QUOTE: { - int delim = z[0]; - testcase( delim=='`' ); - testcase( delim=='\'' ); - testcase( delim=='"' ); - for(i=1; (c=z[i])!=0; i++){ - if( c==delim ){ - if( z[i+1]==delim ){ - i++; - }else{ - break; - } - } +#endif +#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE>0 /* IMP: R-63666-48755 */ + case SQLITE_CONFIG_MUTEX: { + /* Specify an alternative mutex implementation */ + sqlite3GlobalConfig.mutex = *va_arg(ap, sqlite3_mutex_methods*); + break; + } +#endif +#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE>0 /* IMP: R-14450-37597 */ + case SQLITE_CONFIG_GETMUTEX: { + /* Retrieve the current mutex implementation */ + *va_arg(ap, sqlite3_mutex_methods*) = sqlite3GlobalConfig.mutex; + break; + } +#endif + + case SQLITE_CONFIG_MALLOC: { + /* EVIDENCE-OF: R-55594-21030 The SQLITE_CONFIG_MALLOC option takes a + ** single argument which is a pointer to an instance of the + ** sqlite3_mem_methods structure. The argument specifies alternative + ** low-level memory allocation routines to be used in place of the memory + ** allocation routines built into SQLite. */ + sqlite3GlobalConfig.m = *va_arg(ap, sqlite3_mem_methods*); + break; + } + case SQLITE_CONFIG_GETMALLOC: { + /* EVIDENCE-OF: R-51213-46414 The SQLITE_CONFIG_GETMALLOC option takes a + ** single argument which is a pointer to an instance of the + ** sqlite3_mem_methods structure. The sqlite3_mem_methods structure is + ** filled with the currently defined memory allocation routines. */ + if( sqlite3GlobalConfig.m.xMalloc==0 ) sqlite3MemSetDefault(); + *va_arg(ap, sqlite3_mem_methods*) = sqlite3GlobalConfig.m; + break; + } + case SQLITE_CONFIG_MEMSTATUS: { + /* EVIDENCE-OF: R-61275-35157 The SQLITE_CONFIG_MEMSTATUS option takes + ** single argument of type int, interpreted as a boolean, which enables + ** or disables the collection of memory allocation statistics. */ + sqlite3GlobalConfig.bMemstat = va_arg(ap, int); + break; + } + case SQLITE_CONFIG_SMALL_MALLOC: { + sqlite3GlobalConfig.bSmallMalloc = va_arg(ap, int); + break; + } + case SQLITE_CONFIG_PAGECACHE: { + /* EVIDENCE-OF: R-18761-36601 There are three arguments to + ** SQLITE_CONFIG_PAGECACHE: A pointer to 8-byte aligned memory (pMem), + ** the size of each page cache line (sz), and the number of cache lines + ** (N). */ + sqlite3GlobalConfig.pPage = va_arg(ap, void*); + sqlite3GlobalConfig.szPage = va_arg(ap, int); + sqlite3GlobalConfig.nPage = va_arg(ap, int); + break; + } + case SQLITE_CONFIG_PCACHE_HDRSZ: { + /* EVIDENCE-OF: R-39100-27317 The SQLITE_CONFIG_PCACHE_HDRSZ option takes + ** a single parameter which is a pointer to an integer and writes into + ** that integer the number of extra bytes per page required for each page + ** in SQLITE_CONFIG_PAGECACHE. */ + *va_arg(ap, int*) = + sqlite3HeaderSizeBtree() + + sqlite3HeaderSizePcache() + + sqlite3HeaderSizePcache1(); + break; + } + + case SQLITE_CONFIG_PCACHE: { + /* no-op */ + break; + } + case SQLITE_CONFIG_GETPCACHE: { + /* now an error */ + rc = SQLITE_ERROR; + break; + } + + case SQLITE_CONFIG_PCACHE2: { + /* EVIDENCE-OF: R-63325-48378 The SQLITE_CONFIG_PCACHE2 option takes a + ** single argument which is a pointer to an sqlite3_pcache_methods2 + ** object. This object specifies the interface to a custom page cache + ** implementation. */ + sqlite3GlobalConfig.pcache2 = *va_arg(ap, sqlite3_pcache_methods2*); + break; + } + case SQLITE_CONFIG_GETPCACHE2: { + /* EVIDENCE-OF: R-22035-46182 The SQLITE_CONFIG_GETPCACHE2 option takes a + ** single argument which is a pointer to an sqlite3_pcache_methods2 + ** object. SQLite copies of the current page cache implementation into + ** that object. */ + if( sqlite3GlobalConfig.pcache2.xInit==0 ){ + sqlite3PCacheSetDefault(); } - if( c=='\'' ){ - *tokenType = TK_STRING; - return i+1; - }else if( c!=0 ){ - *tokenType = TK_ID; - return i+1; + *va_arg(ap, sqlite3_pcache_methods2*) = sqlite3GlobalConfig.pcache2; + break; + } + +/* EVIDENCE-OF: R-06626-12911 The SQLITE_CONFIG_HEAP option is only +** available if SQLite is compiled with either SQLITE_ENABLE_MEMSYS3 or +** SQLITE_ENABLE_MEMSYS5 and returns SQLITE_ERROR if invoked otherwise. */ +#if defined(SQLITE_ENABLE_MEMSYS3) || defined(SQLITE_ENABLE_MEMSYS5) + case SQLITE_CONFIG_HEAP: { + /* EVIDENCE-OF: R-19854-42126 There are three arguments to + ** SQLITE_CONFIG_HEAP: An 8-byte aligned pointer to the memory, the + ** number of bytes in the memory buffer, and the minimum allocation size. + */ + sqlite3GlobalConfig.pHeap = va_arg(ap, void*); + sqlite3GlobalConfig.nHeap = va_arg(ap, int); + sqlite3GlobalConfig.mnReq = va_arg(ap, int); + + if( sqlite3GlobalConfig.mnReq<1 ){ + sqlite3GlobalConfig.mnReq = 1; + }else if( sqlite3GlobalConfig.mnReq>(1<<12) ){ + /* cap min request size at 2^12 */ + sqlite3GlobalConfig.mnReq = (1<<12); + } + + if( sqlite3GlobalConfig.pHeap==0 ){ + /* EVIDENCE-OF: R-49920-60189 If the first pointer (the memory pointer) + ** is NULL, then SQLite reverts to using its default memory allocator + ** (the system malloc() implementation), undoing any prior invocation of + ** SQLITE_CONFIG_MALLOC. + ** + ** Setting sqlite3GlobalConfig.m to all zeros will cause malloc to + ** revert to its default implementation when sqlite3_initialize() is run + */ + memset(&sqlite3GlobalConfig.m, 0, sizeof(sqlite3GlobalConfig.m)); }else{ - *tokenType = TK_ILLEGAL; - return i; + /* EVIDENCE-OF: R-61006-08918 If the memory pointer is not NULL then the + ** alternative memory allocator is engaged to handle all of SQLites + ** memory allocation needs. */ +#ifdef SQLITE_ENABLE_MEMSYS3 + sqlite3GlobalConfig.m = *sqlite3MemGetMemsys3(); +#endif +#ifdef SQLITE_ENABLE_MEMSYS5 + sqlite3GlobalConfig.m = *sqlite3MemGetMemsys5(); +#endif } + break; } - case CC_DOT: { -#ifndef SQLITE_OMIT_FLOATING_POINT - if( !sqlite3Isdigit(z[1]) ) #endif - { - *tokenType = TK_DOT; - return 1; - } - /* If the next character is a digit, this is a floating point - ** number that begins with ".". Fall thru into the next case */ + + case SQLITE_CONFIG_LOOKASIDE: { + sqlite3GlobalConfig.szLookaside = va_arg(ap, int); + sqlite3GlobalConfig.nLookaside = va_arg(ap, int); + break; + } + + /* Record a pointer to the logger function and its first argument. + ** The default is NULL. Logging is disabled if the function pointer is + ** NULL. + */ + case SQLITE_CONFIG_LOG: { + /* MSVC is picky about pulling func ptrs from va lists. + ** http://support.microsoft.com/kb/47961 + ** sqlite3GlobalConfig.xLog = va_arg(ap, void(*)(void*,int,const char*)); + */ + typedef void(*LOGFUNC_t)(void*,int,const char*); + sqlite3GlobalConfig.xLog = va_arg(ap, LOGFUNC_t); + sqlite3GlobalConfig.pLogArg = va_arg(ap, void*); + break; + } + + /* EVIDENCE-OF: R-55548-33817 The compile-time setting for URI filenames + ** can be changed at start-time using the + ** sqlite3_config(SQLITE_CONFIG_URI,1) or + ** sqlite3_config(SQLITE_CONFIG_URI,0) configuration calls. + */ + case SQLITE_CONFIG_URI: { + /* EVIDENCE-OF: R-25451-61125 The SQLITE_CONFIG_URI option takes a single + ** argument of type int. If non-zero, then URI handling is globally + ** enabled. If the parameter is zero, then URI handling is globally + ** disabled. */ + sqlite3GlobalConfig.bOpenUri = va_arg(ap, int); + break; + } + + case SQLITE_CONFIG_COVERING_INDEX_SCAN: { + /* EVIDENCE-OF: R-36592-02772 The SQLITE_CONFIG_COVERING_INDEX_SCAN + ** option takes a single integer argument which is interpreted as a + ** boolean in order to enable or disable the use of covering indices for + ** full table scans in the query optimizer. */ + sqlite3GlobalConfig.bUseCis = va_arg(ap, int); + break; + } + +#ifdef SQLITE_ENABLE_SQLLOG + case SQLITE_CONFIG_SQLLOG: { + typedef void(*SQLLOGFUNC_t)(void*, sqlite3*, const char*, int); + sqlite3GlobalConfig.xSqllog = va_arg(ap, SQLLOGFUNC_t); + sqlite3GlobalConfig.pSqllogArg = va_arg(ap, void *); + break; } - case CC_DIGIT: { - testcase( z[0]=='0' ); testcase( z[0]=='1' ); testcase( z[0]=='2' ); - testcase( z[0]=='3' ); testcase( z[0]=='4' ); testcase( z[0]=='5' ); - testcase( z[0]=='6' ); testcase( z[0]=='7' ); testcase( z[0]=='8' ); - testcase( z[0]=='9' ); - *tokenType = TK_INTEGER; -#ifndef SQLITE_OMIT_HEX_INTEGER - if( z[0]=='0' && (z[1]=='x' || z[1]=='X') && sqlite3Isxdigit(z[2]) ){ - for(i=3; sqlite3Isxdigit(z[i]); i++){} - return i; - } #endif - for(i=0; sqlite3Isdigit(z[i]); i++){} -#ifndef SQLITE_OMIT_FLOATING_POINT - if( z[i]=='.' ){ - i++; - while( sqlite3Isdigit(z[i]) ){ i++; } - *tokenType = TK_FLOAT; + + case SQLITE_CONFIG_MMAP_SIZE: { + /* EVIDENCE-OF: R-58063-38258 SQLITE_CONFIG_MMAP_SIZE takes two 64-bit + ** integer (sqlite3_int64) values that are the default mmap size limit + ** (the default setting for PRAGMA mmap_size) and the maximum allowed + ** mmap size limit. */ + sqlite3_int64 szMmap = va_arg(ap, sqlite3_int64); + sqlite3_int64 mxMmap = va_arg(ap, sqlite3_int64); + /* EVIDENCE-OF: R-53367-43190 If either argument to this option is + ** negative, then that argument is changed to its compile-time default. + ** + ** EVIDENCE-OF: R-34993-45031 The maximum allowed mmap size will be + ** silently truncated if necessary so that it does not exceed the + ** compile-time maximum mmap size set by the SQLITE_MAX_MMAP_SIZE + ** compile-time option. + */ + if( mxMmap<0 || mxMmap>SQLITE_MAX_MMAP_SIZE ){ + mxMmap = SQLITE_MAX_MMAP_SIZE; } - if( (z[i]=='e' || z[i]=='E') && - ( sqlite3Isdigit(z[i+1]) - || ((z[i+1]=='+' || z[i+1]=='-') && sqlite3Isdigit(z[i+2])) - ) - ){ - i += 2; - while( sqlite3Isdigit(z[i]) ){ i++; } - *tokenType = TK_FLOAT; + if( szMmap<0 ) szMmap = SQLITE_DEFAULT_MMAP_SIZE; + if( szMmap>mxMmap) szMmap = mxMmap; + sqlite3GlobalConfig.mxMmap = mxMmap; + sqlite3GlobalConfig.szMmap = szMmap; + break; + } + +#if SQLITE_OS_WIN && defined(SQLITE_WIN32_MALLOC) /* IMP: R-04780-55815 */ + case SQLITE_CONFIG_WIN32_HEAPSIZE: { + /* EVIDENCE-OF: R-34926-03360 SQLITE_CONFIG_WIN32_HEAPSIZE takes a 32-bit + ** unsigned integer value that specifies the maximum size of the created + ** heap. */ + sqlite3GlobalConfig.nHeap = va_arg(ap, int); + break; + } +#endif + + case SQLITE_CONFIG_PMASZ: { + sqlite3GlobalConfig.szPma = va_arg(ap, unsigned int); + break; + } + + case SQLITE_CONFIG_STMTJRNL_SPILL: { + sqlite3GlobalConfig.nStmtSpill = va_arg(ap, int); + break; + } + +#ifdef SQLITE_ENABLE_SORTER_REFERENCES + case SQLITE_CONFIG_SORTERREF_SIZE: { + int iVal = va_arg(ap, int); + if( iVal<0 ){ + iVal = SQLITE_DEFAULT_SORTERREF_SIZE; } + sqlite3GlobalConfig.szSorterRef = (u32)iVal; + break; + } +#endif /* SQLITE_ENABLE_SORTER_REFERENCES */ + +#ifdef SQLITE_ENABLE_DESERIALIZE + case SQLITE_CONFIG_MEMDB_MAXSIZE: { + sqlite3GlobalConfig.mxMemdbSize = va_arg(ap, sqlite3_int64); + break; + } +#endif /* SQLITE_ENABLE_DESERIALIZE */ + + default: { + rc = SQLITE_ERROR; + break; + } + } + va_end(ap); + return rc; +} + +/* +** Set up the lookaside buffers for a database connection. +** Return SQLITE_OK on success. +** If lookaside is already active, return SQLITE_BUSY. +** +** The sz parameter is the number of bytes in each lookaside slot. +** The cnt parameter is the number of slots. If pStart is NULL the +** space for the lookaside memory is obtained from sqlite3_malloc(). +** If pStart is not NULL then it is sz*cnt bytes of memory to use for +** the lookaside memory. +*/ +static int setupLookaside(sqlite3 *db, void *pBuf, int sz, int cnt){ +#ifndef SQLITE_OMIT_LOOKASIDE + void *pStart; + sqlite3_int64 szAlloc = sz*(sqlite3_int64)cnt; + int nBig; /* Number of full-size slots */ + int nSm; /* Number smaller LOOKASIDE_SMALL-byte slots */ + + if( sqlite3LookasideUsed(db,0)>0 ){ + return SQLITE_BUSY; + } + /* Free any existing lookaside buffer for this handle before + ** allocating a new one so we don't have to have space for + ** both at the same time. + */ + if( db->lookaside.bMalloced ){ + sqlite3_free(db->lookaside.pStart); + } + /* The size of a lookaside slot after ROUNDDOWN8 needs to be larger + ** than a pointer to be useful. + */ + sz = ROUNDDOWN8(sz); /* IMP: R-33038-09382 */ + if( sz<=(int)sizeof(LookasideSlot*) ) sz = 0; + if( cnt<0 ) cnt = 0; + if( sz==0 || cnt==0 ){ + sz = 0; + pStart = 0; + }else if( pBuf==0 ){ + sqlite3BeginBenignMalloc(); + pStart = sqlite3Malloc( szAlloc ); /* IMP: R-61949-35727 */ + sqlite3EndBenignMalloc(); + if( pStart ) szAlloc = sqlite3MallocSize(pStart); + }else{ + pStart = pBuf; + } +#ifndef SQLITE_OMIT_TWOSIZE_LOOKASIDE + if( sz>=LOOKASIDE_SMALL*3 ){ + nBig = szAlloc/(3*LOOKASIDE_SMALL+sz); + nSm = (szAlloc - sz*nBig)/LOOKASIDE_SMALL; + }else if( sz>=LOOKASIDE_SMALL*2 ){ + nBig = szAlloc/(LOOKASIDE_SMALL+sz); + nSm = (szAlloc - sz*nBig)/LOOKASIDE_SMALL; + }else +#endif /* SQLITE_OMIT_TWOSIZE_LOOKASIDE */ + if( sz>0 ){ + nBig = szAlloc/sz; + nSm = 0; + }else{ + nBig = nSm = 0; + } + db->lookaside.pStart = pStart; + db->lookaside.pInit = 0; + db->lookaside.pFree = 0; + db->lookaside.sz = (u16)sz; + db->lookaside.szTrue = (u16)sz; + if( pStart ){ + int i; + LookasideSlot *p; + assert( sz > (int)sizeof(LookasideSlot*) ); + p = (LookasideSlot*)pStart; + for(i=0; ipNext = db->lookaside.pInit; + db->lookaside.pInit = p; + p = (LookasideSlot*)&((u8*)p)[sz]; + } +#ifndef SQLITE_OMIT_TWOSIZE_LOOKASIDE + db->lookaside.pSmallInit = 0; + db->lookaside.pSmallFree = 0; + db->lookaside.pMiddle = p; + for(i=0; ipNext = db->lookaside.pSmallInit; + db->lookaside.pSmallInit = p; + p = (LookasideSlot*)&((u8*)p)[LOOKASIDE_SMALL]; + } +#endif /* SQLITE_OMIT_TWOSIZE_LOOKASIDE */ + assert( ((uptr)p)<=szAlloc + (uptr)pStart ); + db->lookaside.pEnd = p; + db->lookaside.bDisable = 0; + db->lookaside.bMalloced = pBuf==0 ?1:0; + db->lookaside.nSlot = nBig+nSm; + }else{ + db->lookaside.pStart = db; +#ifndef SQLITE_OMIT_TWOSIZE_LOOKASIDE + db->lookaside.pSmallInit = 0; + db->lookaside.pSmallFree = 0; + db->lookaside.pMiddle = db; +#endif /* SQLITE_OMIT_TWOSIZE_LOOKASIDE */ + db->lookaside.pEnd = db; + db->lookaside.bDisable = 1; + db->lookaside.sz = 0; + db->lookaside.bMalloced = 0; + db->lookaside.nSlot = 0; + } + assert( sqlite3LookasideUsed(db,0)==0 ); +#endif /* SQLITE_OMIT_LOOKASIDE */ + return SQLITE_OK; +} + +/* +** Return the mutex associated with a database connection. +*/ +SQLITE_API sqlite3_mutex *sqlite3_db_mutex(sqlite3 *db){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } #endif - while( IdChar(z[i]) ){ - *tokenType = TK_ILLEGAL; - i++; + return db->mutex; +} + +/* +** Free up as much memory as we can from the given database +** connection. +*/ +SQLITE_API int sqlite3_db_release_memory(sqlite3 *db){ + int i; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif + sqlite3_mutex_enter(db->mutex); + sqlite3BtreeEnterAll(db); + for(i=0; inDb; i++){ + Btree *pBt = db->aDb[i].pBt; + if( pBt ){ + Pager *pPager = sqlite3BtreePager(pBt); + sqlite3PagerShrink(pPager); + } + } + sqlite3BtreeLeaveAll(db); + sqlite3_mutex_leave(db->mutex); + return SQLITE_OK; +} + +/* +** Flush any dirty pages in the pager-cache for any attached database +** to disk. +*/ +SQLITE_API int sqlite3_db_cacheflush(sqlite3 *db){ + int i; + int rc = SQLITE_OK; + int bSeenBusy = 0; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif + sqlite3_mutex_enter(db->mutex); + sqlite3BtreeEnterAll(db); + for(i=0; rc==SQLITE_OK && inDb; i++){ + Btree *pBt = db->aDb[i].pBt; + if( pBt && sqlite3BtreeIsInTrans(pBt) ){ + Pager *pPager = sqlite3BtreePager(pBt); + rc = sqlite3PagerFlush(pPager); + if( rc==SQLITE_BUSY ){ + bSeenBusy = 1; + rc = SQLITE_OK; } - return i; } - case CC_QUOTE2: { - for(i=1, c=z[0]; c!=']' && (c=z[i])!=0; i++){} - *tokenType = c==']' ? TK_ID : TK_ILLEGAL; - return i; + } + sqlite3BtreeLeaveAll(db); + sqlite3_mutex_leave(db->mutex); + return ((rc==SQLITE_OK && bSeenBusy) ? SQLITE_BUSY : rc); +} + +/* +** Configuration settings for an individual database connection +*/ +SQLITE_API int sqlite3_db_config(sqlite3 *db, int op, ...){ + va_list ap; + int rc; + va_start(ap, op); + switch( op ){ + case SQLITE_DBCONFIG_MAINDBNAME: { + /* IMP: R-06824-28531 */ + /* IMP: R-36257-52125 */ + db->aDb[0].zDbSName = va_arg(ap,char*); + rc = SQLITE_OK; + break; } - case CC_VARNUM: { - *tokenType = TK_VARIABLE; - for(i=1; sqlite3Isdigit(z[i]); i++){} - return i; + case SQLITE_DBCONFIG_LOOKASIDE: { + void *pBuf = va_arg(ap, void*); /* IMP: R-26835-10964 */ + int sz = va_arg(ap, int); /* IMP: R-47871-25994 */ + int cnt = va_arg(ap, int); /* IMP: R-04460-53386 */ + rc = setupLookaside(db, pBuf, sz, cnt); + break; } - case CC_DOLLAR: - case CC_VARALPHA: { - int n = 0; - testcase( z[0]=='$' ); testcase( z[0]=='@' ); - testcase( z[0]==':' ); testcase( z[0]=='#' ); - *tokenType = TK_VARIABLE; - for(i=1; (c=z[i])!=0; i++){ - if( IdChar(c) ){ - n++; -#ifndef SQLITE_OMIT_TCL_VARIABLE - }else if( c=='(' && n>0 ){ - do{ - i++; - }while( (c=z[i])!=0 && !sqlite3Isspace(c) && c!=')' ); - if( c==')' ){ - i++; - }else{ - *tokenType = TK_ILLEGAL; + default: { + static const struct { + int op; /* The opcode */ + u32 mask; /* Mask of the bit in sqlite3.flags to set/clear */ + } aFlagOp[] = { + { SQLITE_DBCONFIG_ENABLE_FKEY, SQLITE_ForeignKeys }, + { SQLITE_DBCONFIG_ENABLE_TRIGGER, SQLITE_EnableTrigger }, + { SQLITE_DBCONFIG_ENABLE_VIEW, SQLITE_EnableView }, + { SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER, SQLITE_Fts3Tokenizer }, + { SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION, SQLITE_LoadExtension }, + { SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE, SQLITE_NoCkptOnClose }, + { SQLITE_DBCONFIG_ENABLE_QPSG, SQLITE_EnableQPSG }, + { SQLITE_DBCONFIG_TRIGGER_EQP, SQLITE_TriggerEQP }, + { SQLITE_DBCONFIG_RESET_DATABASE, SQLITE_ResetDatabase }, + { SQLITE_DBCONFIG_DEFENSIVE, SQLITE_Defensive }, + { SQLITE_DBCONFIG_WRITABLE_SCHEMA, SQLITE_WriteSchema| + SQLITE_NoSchemaError }, + { SQLITE_DBCONFIG_LEGACY_ALTER_TABLE, SQLITE_LegacyAlter }, + { SQLITE_DBCONFIG_DQS_DDL, SQLITE_DqsDDL }, + { SQLITE_DBCONFIG_DQS_DML, SQLITE_DqsDML }, + { SQLITE_DBCONFIG_LEGACY_FILE_FORMAT, SQLITE_LegacyFileFmt }, + { SQLITE_DBCONFIG_TRUSTED_SCHEMA, SQLITE_TrustedSchema }, + }; + unsigned int i; + rc = SQLITE_ERROR; /* IMP: R-42790-23372 */ + for(i=0; iflags; + if( onoff>0 ){ + db->flags |= aFlagOp[i].mask; + }else if( onoff==0 ){ + db->flags &= ~(u64)aFlagOp[i].mask; } - break; - }else if( c==':' && z[i+1]==':' ){ - i++; -#endif - }else{ + if( oldFlags!=db->flags ){ + sqlite3ExpirePreparedStatements(db, 0); + } + if( pRes ){ + *pRes = (db->flags & aFlagOp[i].mask)!=0; + } + rc = SQLITE_OK; break; } } - if( n==0 ) *tokenType = TK_ILLEGAL; - return i; + break; } - case CC_KYWD: { - for(i=1; aiClass[z[i]]<=CC_KYWD; i++){} - if( IdChar(z[i]) ){ - /* This token started out using characters that can appear in keywords, - ** but z[i] is a character not allowed within keywords, so this must - ** be an identifier instead */ - i++; - break; - } - *tokenType = TK_ID; - return keywordCode((char*)z, i, tokenType); + } + va_end(ap); + return rc; +} + +/* +** This is the default collating function named "BINARY" which is always +** available. +*/ +static int binCollFunc( + void *NotUsed, + int nKey1, const void *pKey1, + int nKey2, const void *pKey2 +){ + int rc, n; + UNUSED_PARAMETER(NotUsed); + n = nKey1xCmp!=binCollFunc || strcmp(p->zName,"BINARY")==0 ); + return p==0 || p->xCmp==binCollFunc; +} + +/* +** Another built-in collating sequence: NOCASE. +** +** This collating sequence is intended to be used for "case independent +** comparison". SQLite's knowledge of upper and lower case equivalents +** extends only to the 26 characters used in the English language. +** +** At the moment there is only a UTF-8 implementation. +*/ +static int nocaseCollatingFunc( + void *NotUsed, + int nKey1, const void *pKey1, + int nKey2, const void *pKey2 +){ + int r = sqlite3StrNICmp( + (const char *)pKey1, (const char *)pKey2, (nKey1lastRowid; +} + +/* +** Set the value returned by the sqlite3_last_insert_rowid() API function. +*/ +SQLITE_API void sqlite3_set_last_insert_rowid(sqlite3 *db, sqlite3_int64 iRowid){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return; + } +#endif + sqlite3_mutex_enter(db->mutex); + db->lastRowid = iRowid; + sqlite3_mutex_leave(db->mutex); +} + +/* +** Return the number of changes in the most recent call to sqlite3_exec(). +*/ +SQLITE_API int sqlite3_changes(sqlite3 *db){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + return db->nChange; +} + +/* +** Return the number of changes since the database handle was opened. +*/ +SQLITE_API int sqlite3_total_changes(sqlite3 *db){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + return db->nTotalChange; +} + +/* +** Close all open savepoints. This function only manipulates fields of the +** database handle object, it does not close any savepoints that may be open +** at the b-tree/pager level. +*/ +SQLITE_PRIVATE void sqlite3CloseSavepoints(sqlite3 *db){ + while( db->pSavepoint ){ + Savepoint *pTmp = db->pSavepoint; + db->pSavepoint = pTmp->pNext; + sqlite3DbFree(db, pTmp); + } + db->nSavepoint = 0; + db->nStatement = 0; + db->isTransactionSavepoint = 0; +} + +/* +** Invoke the destructor function associated with FuncDef p, if any. Except, +** if this is not the last copy of the function, do not invoke it. Multiple +** copies of a single function are created when create_function() is called +** with SQLITE_ANY as the encoding. +*/ +static void functionDestroy(sqlite3 *db, FuncDef *p){ + FuncDestructor *pDestructor = p->u.pDestructor; + if( pDestructor ){ + pDestructor->nRef--; + if( pDestructor->nRef==0 ){ + pDestructor->xDestroy(pDestructor->pUserData); + sqlite3DbFree(db, pDestructor); } - case CC_X: { -#ifndef SQLITE_OMIT_BLOB_LITERAL - testcase( z[0]=='x' ); testcase( z[0]=='X' ); - if( z[1]=='\'' ){ - *tokenType = TK_BLOB; - for(i=2; sqlite3Isxdigit(z[i]); i++){} - if( z[i]!='\'' || i%2 ){ - *tokenType = TK_ILLEGAL; - while( z[i] && z[i]!='\'' ){ i++; } - } - if( z[i] ) i++; - return i; + } +} + +/* +** Disconnect all sqlite3_vtab objects that belong to database connection +** db. This is called when db is being closed. +*/ +static void disconnectAllVtab(sqlite3 *db){ +#ifndef SQLITE_OMIT_VIRTUALTABLE + int i; + HashElem *p; + sqlite3BtreeEnterAll(db); + for(i=0; inDb; i++){ + Schema *pSchema = db->aDb[i].pSchema; + if( pSchema ){ + for(p=sqliteHashFirst(&pSchema->tblHash); p; p=sqliteHashNext(p)){ + Table *pTab = (Table *)sqliteHashData(p); + if( IsVirtual(pTab) ) sqlite3VtabDisconnect(db, pTab); } -#endif - /* If it is not a BLOB literal, then it must be an ID, since no - ** SQL keywords start with the letter 'x'. Fall through */ - } - case CC_ID: { - i = 1; - break; } - default: { - *tokenType = TK_ILLEGAL; - return 1; + } + for(p=sqliteHashFirst(&db->aModule); p; p=sqliteHashNext(p)){ + Module *pMod = (Module *)sqliteHashData(p); + if( pMod->pEpoTab ){ + sqlite3VtabDisconnect(db, pMod->pEpoTab); } } - while( IdChar(z[i]) ){ i++; } - *tokenType = TK_ID; - return i; + sqlite3VtabUnlockList(db); + sqlite3BtreeLeaveAll(db); +#else + UNUSED_PARAMETER(db); +#endif } /* -** Run the parser on the given SQL string. The parser structure is -** passed in. An SQLITE_ status code is returned. If an error occurs -** then an and attempt is made to write an error message into -** memory obtained from sqlite3_malloc() and to make *pzErrMsg point to that -** error message. +** Return TRUE if database connection db has unfinalized prepared +** statements or unfinished sqlite3_backup objects. */ -SQLITE_PRIVATE int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzErrMsg){ - int nErr = 0; /* Number of errors encountered */ - void *pEngine; /* The LEMON-generated LALR(1) parser */ - int n = 0; /* Length of the next token token */ - int tokenType; /* type of the next token */ - int lastTokenParsed = -1; /* type of the previous token */ - sqlite3 *db = pParse->db; /* The database connection */ - int mxSqlLen; /* Max length of an SQL string */ -#ifdef sqlite3Parser_ENGINEALWAYSONSTACK - yyParser sEngine; /* Space to hold the Lemon-generated Parser object */ -#endif +static int connectionIsBusy(sqlite3 *db){ + int j; + assert( sqlite3_mutex_held(db->mutex) ); + if( db->pVdbe ) return 1; + for(j=0; jnDb; j++){ + Btree *pBt = db->aDb[j].pBt; + if( pBt && sqlite3BtreeIsInBackup(pBt) ) return 1; + } + return 0; +} - assert( zSql!=0 ); - mxSqlLen = db->aLimit[SQLITE_LIMIT_SQL_LENGTH]; - if( db->nVdbeActive==0 ){ - db->u1.isInterrupted = 0; +/* +** Close an existing SQLite database +*/ +static int sqlite3Close(sqlite3 *db, int forceZombie){ + if( !db ){ + /* EVIDENCE-OF: R-63257-11740 Calling sqlite3_close() or + ** sqlite3_close_v2() with a NULL pointer argument is a harmless no-op. */ + return SQLITE_OK; } - pParse->rc = SQLITE_OK; - pParse->zTail = zSql; - assert( pzErrMsg!=0 ); - /* sqlite3ParserTrace(stdout, "parser: "); */ -#ifdef sqlite3Parser_ENGINEALWAYSONSTACK - pEngine = &sEngine; - sqlite3ParserInit(pEngine, pParse); -#else - pEngine = sqlite3ParserAlloc(sqlite3Malloc, pParse); - if( pEngine==0 ){ - sqlite3OomFault(db); - return SQLITE_NOMEM_BKPT; + if( !sqlite3SafetyCheckSickOrOk(db) ){ + return SQLITE_MISUSE_BKPT; + } + sqlite3_mutex_enter(db->mutex); + if( db->mTrace & SQLITE_TRACE_CLOSE ){ + db->xTrace(SQLITE_TRACE_CLOSE, db->pTraceArg, db, 0); + } + + /* Force xDisconnect calls on all virtual tables */ + disconnectAllVtab(db); + + /* If a transaction is open, the disconnectAllVtab() call above + ** will not have called the xDisconnect() method on any virtual + ** tables in the db->aVTrans[] array. The following sqlite3VtabRollback() + ** call will do so. We need to do this before the check for active + ** SQL statements below, as the v-table implementation may be storing + ** some prepared statements internally. + */ + sqlite3VtabRollback(db); + + /* Legacy behavior (sqlite3_close() behavior) is to return + ** SQLITE_BUSY if the connection can not be closed immediately. + */ + if( !forceZombie && connectionIsBusy(db) ){ + sqlite3ErrorWithMsg(db, SQLITE_BUSY, "unable to close due to unfinalized " + "statements or unfinished backups"); + sqlite3_mutex_leave(db->mutex); + return SQLITE_BUSY; + } + +#ifdef SQLITE_ENABLE_SQLLOG + if( sqlite3GlobalConfig.xSqllog ){ + /* Closing the handle. Fourth parameter is passed the value 2. */ + sqlite3GlobalConfig.xSqllog(sqlite3GlobalConfig.pSqllogArg, db, 0, 2); } #endif - assert( pParse->pNewTable==0 ); - assert( pParse->pNewTrigger==0 ); - assert( pParse->nVar==0 ); - assert( pParse->pVList==0 ); - while( 1 ){ - if( zSql[0]!=0 ){ - n = sqlite3GetToken((u8*)zSql, &tokenType); - mxSqlLen -= n; - if( mxSqlLen<0 ){ - pParse->rc = SQLITE_TOOBIG; - break; - } - }else{ - /* Upon reaching the end of input, call the parser two more times - ** with tokens TK_SEMI and 0, in that order. */ - if( lastTokenParsed==TK_SEMI ){ - tokenType = 0; - }else if( lastTokenParsed==0 ){ - break; - }else{ - tokenType = TK_SEMI; + + /* Convert the connection into a zombie and then close it. + */ + db->magic = SQLITE_MAGIC_ZOMBIE; + sqlite3LeaveMutexAndCloseZombie(db); + return SQLITE_OK; +} + +/* +** Two variations on the public interface for closing a database +** connection. The sqlite3_close() version returns SQLITE_BUSY and +** leaves the connection option if there are unfinalized prepared +** statements or unfinished sqlite3_backups. The sqlite3_close_v2() +** version forces the connection to become a zombie if there are +** unclosed resources, and arranges for deallocation when the last +** prepare statement or sqlite3_backup closes. +*/ +SQLITE_API int sqlite3_close(sqlite3 *db){ return sqlite3Close(db,0); } +SQLITE_API int sqlite3_close_v2(sqlite3 *db){ return sqlite3Close(db,1); } + + +/* +** Close the mutex on database connection db. +** +** Furthermore, if database connection db is a zombie (meaning that there +** has been a prior call to sqlite3_close(db) or sqlite3_close_v2(db)) and +** every sqlite3_stmt has now been finalized and every sqlite3_backup has +** finished, then free all resources. +*/ +SQLITE_PRIVATE void sqlite3LeaveMutexAndCloseZombie(sqlite3 *db){ + HashElem *i; /* Hash table iterator */ + int j; + + /* If there are outstanding sqlite3_stmt or sqlite3_backup objects + ** or if the connection has not yet been closed by sqlite3_close_v2(), + ** then just leave the mutex and return. + */ + if( db->magic!=SQLITE_MAGIC_ZOMBIE || connectionIsBusy(db) ){ + sqlite3_mutex_leave(db->mutex); + return; + } + + /* If we reach this point, it means that the database connection has + ** closed all sqlite3_stmt and sqlite3_backup objects and has been + ** passed to sqlite3_close (meaning that it is a zombie). Therefore, + ** go ahead and free all resources. + */ + + /* If a transaction is open, roll it back. This also ensures that if + ** any database schemas have been modified by an uncommitted transaction + ** they are reset. And that the required b-tree mutex is held to make + ** the pager rollback and schema reset an atomic operation. */ + sqlite3RollbackAll(db, SQLITE_OK); + + /* Free any outstanding Savepoint structures. */ + sqlite3CloseSavepoints(db); + + /* Close all database connections */ + for(j=0; jnDb; j++){ + struct Db *pDb = &db->aDb[j]; + if( pDb->pBt ){ + sqlite3BtreeClose(pDb->pBt); + pDb->pBt = 0; + if( j!=1 ){ + pDb->pSchema = 0; } - n = 0; } - if( tokenType>=TK_SPACE ){ - assert( tokenType==TK_SPACE || tokenType==TK_ILLEGAL ); - if( db->u1.isInterrupted ){ - pParse->rc = SQLITE_INTERRUPT; - break; - } - if( tokenType==TK_ILLEGAL ){ - sqlite3ErrorMsg(pParse, "unrecognized token: \"%.*s\"", n, zSql); - break; + } + /* Clear the TEMP schema separately and last */ + if( db->aDb[1].pSchema ){ + sqlite3SchemaClear(db->aDb[1].pSchema); + } + sqlite3VtabUnlockList(db); + + /* Free up the array of auxiliary databases */ + sqlite3CollapseDatabaseArray(db); + assert( db->nDb<=2 ); + assert( db->aDb==db->aDbStatic ); + + /* Tell the code in notify.c that the connection no longer holds any + ** locks and does not require any further unlock-notify callbacks. + */ + sqlite3ConnectionClosed(db); + + for(i=sqliteHashFirst(&db->aFunc); i; i=sqliteHashNext(i)){ + FuncDef *pNext, *p; + p = sqliteHashData(i); + do{ + functionDestroy(db, p); + pNext = p->pNext; + sqlite3DbFree(db, p); + p = pNext; + }while( p ); + } + sqlite3HashClear(&db->aFunc); + for(i=sqliteHashFirst(&db->aCollSeq); i; i=sqliteHashNext(i)){ + CollSeq *pColl = (CollSeq *)sqliteHashData(i); + /* Invoke any destructors registered for collation sequence user data. */ + for(j=0; j<3; j++){ + if( pColl[j].xDel ){ + pColl[j].xDel(pColl[j].pUser); } - zSql += n; - }else{ - pParse->sLastToken.z = zSql; - pParse->sLastToken.n = n; - sqlite3Parser(pEngine, tokenType, pParse->sLastToken); - lastTokenParsed = tokenType; - zSql += n; - if( pParse->rc!=SQLITE_OK || db->mallocFailed ) break; } + sqlite3DbFree(db, pColl); } - assert( nErr==0 ); - pParse->zTail = zSql; -#ifdef YYTRACKMAXSTACKDEPTH - sqlite3_mutex_enter(sqlite3MallocMutex()); - sqlite3StatusHighwater(SQLITE_STATUS_PARSER_STACK, - sqlite3ParserStackPeak(pEngine) - ); - sqlite3_mutex_leave(sqlite3MallocMutex()); -#endif /* YYDEBUG */ -#ifdef sqlite3Parser_ENGINEALWAYSONSTACK - sqlite3ParserFinalize(pEngine); -#else - sqlite3ParserFree(pEngine, sqlite3_free); + sqlite3HashClear(&db->aCollSeq); +#ifndef SQLITE_OMIT_VIRTUALTABLE + for(i=sqliteHashFirst(&db->aModule); i; i=sqliteHashNext(i)){ + Module *pMod = (Module *)sqliteHashData(i); + sqlite3VtabEponymousTableClear(db, pMod); + sqlite3VtabModuleUnref(db, pMod); + } + sqlite3HashClear(&db->aModule); #endif - if( db->mallocFailed ){ - pParse->rc = SQLITE_NOMEM_BKPT; + + sqlite3Error(db, SQLITE_OK); /* Deallocates any cached error strings. */ + sqlite3ValueFree(db->pErr); + sqlite3CloseExtensions(db); +#if SQLITE_USER_AUTHENTICATION + sqlite3_free(db->auth.zAuthUser); + sqlite3_free(db->auth.zAuthPW); +#endif + + db->magic = SQLITE_MAGIC_ERROR; + + /* The temp-database schema is allocated differently from the other schema + ** objects (using sqliteMalloc() directly, instead of sqlite3BtreeSchema()). + ** So it needs to be freed here. Todo: Why not roll the temp schema into + ** the same sqliteMalloc() as the one that allocates the database + ** structure? + */ + sqlite3DbFree(db, db->aDb[1].pSchema); + sqlite3_mutex_leave(db->mutex); + db->magic = SQLITE_MAGIC_CLOSED; + sqlite3_mutex_free(db->mutex); + assert( sqlite3LookasideUsed(db,0)==0 ); + if( db->lookaside.bMalloced ){ + sqlite3_free(db->lookaside.pStart); } - if( pParse->rc!=SQLITE_OK && pParse->rc!=SQLITE_DONE && pParse->zErrMsg==0 ){ - pParse->zErrMsg = sqlite3MPrintf(db, "%s", sqlite3ErrStr(pParse->rc)); + sqlite3_free(db); +} + +/* +** Rollback all database files. If tripCode is not SQLITE_OK, then +** any write cursors are invalidated ("tripped" - as in "tripping a circuit +** breaker") and made to return tripCode if there are any further +** attempts to use that cursor. Read cursors remain open and valid +** but are "saved" in case the table pages are moved around. +*/ +SQLITE_PRIVATE void sqlite3RollbackAll(sqlite3 *db, int tripCode){ + int i; + int inTrans = 0; + int schemaChange; + assert( sqlite3_mutex_held(db->mutex) ); + sqlite3BeginBenignMalloc(); + + /* Obtain all b-tree mutexes before making any calls to BtreeRollback(). + ** This is important in case the transaction being rolled back has + ** modified the database schema. If the b-tree mutexes are not taken + ** here, then another shared-cache connection might sneak in between + ** the database rollback and schema reset, which can cause false + ** corruption reports in some cases. */ + sqlite3BtreeEnterAll(db); + schemaChange = (db->mDbFlags & DBFLAG_SchemaChange)!=0 && db->init.busy==0; + + for(i=0; inDb; i++){ + Btree *p = db->aDb[i].pBt; + if( p ){ + if( sqlite3BtreeIsInTrans(p) ){ + inTrans = 1; + } + sqlite3BtreeRollback(p, tripCode, !schemaChange); + } } - assert( pzErrMsg!=0 ); - if( pParse->zErrMsg ){ - *pzErrMsg = pParse->zErrMsg; - sqlite3_log(pParse->rc, "%s", *pzErrMsg); - pParse->zErrMsg = 0; - nErr++; + sqlite3VtabRollback(db); + sqlite3EndBenignMalloc(); + + if( schemaChange ){ + sqlite3ExpirePreparedStatements(db, 0); + sqlite3ResetAllSchemasOfConnection(db); } - if( pParse->pVdbe && pParse->nErr>0 && pParse->nested==0 ){ - sqlite3VdbeDelete(pParse->pVdbe); - pParse->pVdbe = 0; + sqlite3BtreeLeaveAll(db); + + /* Any deferred constraint violations have now been resolved. */ + db->nDeferredCons = 0; + db->nDeferredImmCons = 0; + db->flags &= ~(u64)SQLITE_DeferFKs; + + /* If one has been configured, invoke the rollback-hook callback */ + if( db->xRollbackCallback && (inTrans || !db->autoCommit) ){ + db->xRollbackCallback(db->pRollbackArg); } -#ifndef SQLITE_OMIT_SHARED_CACHE - if( pParse->nested==0 ){ - sqlite3DbFree(db, pParse->aTableLock); - pParse->aTableLock = 0; - pParse->nTableLock = 0; +} + +/* +** Return a static string containing the name corresponding to the error code +** specified in the argument. +*/ +#if defined(SQLITE_NEED_ERR_NAME) +SQLITE_PRIVATE const char *sqlite3ErrName(int rc){ + const char *zName = 0; + int i, origRc = rc; + for(i=0; i<2 && zName==0; i++, rc &= 0xff){ + switch( rc ){ + case SQLITE_OK: zName = "SQLITE_OK"; break; + case SQLITE_ERROR: zName = "SQLITE_ERROR"; break; + case SQLITE_ERROR_SNAPSHOT: zName = "SQLITE_ERROR_SNAPSHOT"; break; + case SQLITE_INTERNAL: zName = "SQLITE_INTERNAL"; break; + case SQLITE_PERM: zName = "SQLITE_PERM"; break; + case SQLITE_ABORT: zName = "SQLITE_ABORT"; break; + case SQLITE_ABORT_ROLLBACK: zName = "SQLITE_ABORT_ROLLBACK"; break; + case SQLITE_BUSY: zName = "SQLITE_BUSY"; break; + case SQLITE_BUSY_RECOVERY: zName = "SQLITE_BUSY_RECOVERY"; break; + case SQLITE_BUSY_SNAPSHOT: zName = "SQLITE_BUSY_SNAPSHOT"; break; + case SQLITE_LOCKED: zName = "SQLITE_LOCKED"; break; + case SQLITE_LOCKED_SHAREDCACHE: zName = "SQLITE_LOCKED_SHAREDCACHE";break; + case SQLITE_NOMEM: zName = "SQLITE_NOMEM"; break; + case SQLITE_READONLY: zName = "SQLITE_READONLY"; break; + case SQLITE_READONLY_RECOVERY: zName = "SQLITE_READONLY_RECOVERY"; break; + case SQLITE_READONLY_CANTINIT: zName = "SQLITE_READONLY_CANTINIT"; break; + case SQLITE_READONLY_ROLLBACK: zName = "SQLITE_READONLY_ROLLBACK"; break; + case SQLITE_READONLY_DBMOVED: zName = "SQLITE_READONLY_DBMOVED"; break; + case SQLITE_READONLY_DIRECTORY: zName = "SQLITE_READONLY_DIRECTORY";break; + case SQLITE_INTERRUPT: zName = "SQLITE_INTERRUPT"; break; + case SQLITE_IOERR: zName = "SQLITE_IOERR"; break; + case SQLITE_IOERR_READ: zName = "SQLITE_IOERR_READ"; break; + case SQLITE_IOERR_SHORT_READ: zName = "SQLITE_IOERR_SHORT_READ"; break; + case SQLITE_IOERR_WRITE: zName = "SQLITE_IOERR_WRITE"; break; + case SQLITE_IOERR_FSYNC: zName = "SQLITE_IOERR_FSYNC"; break; + case SQLITE_IOERR_DIR_FSYNC: zName = "SQLITE_IOERR_DIR_FSYNC"; break; + case SQLITE_IOERR_TRUNCATE: zName = "SQLITE_IOERR_TRUNCATE"; break; + case SQLITE_IOERR_FSTAT: zName = "SQLITE_IOERR_FSTAT"; break; + case SQLITE_IOERR_UNLOCK: zName = "SQLITE_IOERR_UNLOCK"; break; + case SQLITE_IOERR_RDLOCK: zName = "SQLITE_IOERR_RDLOCK"; break; + case SQLITE_IOERR_DELETE: zName = "SQLITE_IOERR_DELETE"; break; + case SQLITE_IOERR_NOMEM: zName = "SQLITE_IOERR_NOMEM"; break; + case SQLITE_IOERR_ACCESS: zName = "SQLITE_IOERR_ACCESS"; break; + case SQLITE_IOERR_CHECKRESERVEDLOCK: + zName = "SQLITE_IOERR_CHECKRESERVEDLOCK"; break; + case SQLITE_IOERR_LOCK: zName = "SQLITE_IOERR_LOCK"; break; + case SQLITE_IOERR_CLOSE: zName = "SQLITE_IOERR_CLOSE"; break; + case SQLITE_IOERR_DIR_CLOSE: zName = "SQLITE_IOERR_DIR_CLOSE"; break; + case SQLITE_IOERR_SHMOPEN: zName = "SQLITE_IOERR_SHMOPEN"; break; + case SQLITE_IOERR_SHMSIZE: zName = "SQLITE_IOERR_SHMSIZE"; break; + case SQLITE_IOERR_SHMLOCK: zName = "SQLITE_IOERR_SHMLOCK"; break; + case SQLITE_IOERR_SHMMAP: zName = "SQLITE_IOERR_SHMMAP"; break; + case SQLITE_IOERR_SEEK: zName = "SQLITE_IOERR_SEEK"; break; + case SQLITE_IOERR_DELETE_NOENT: zName = "SQLITE_IOERR_DELETE_NOENT";break; + case SQLITE_IOERR_MMAP: zName = "SQLITE_IOERR_MMAP"; break; + case SQLITE_IOERR_GETTEMPPATH: zName = "SQLITE_IOERR_GETTEMPPATH"; break; + case SQLITE_IOERR_CONVPATH: zName = "SQLITE_IOERR_CONVPATH"; break; + case SQLITE_CORRUPT: zName = "SQLITE_CORRUPT"; break; + case SQLITE_CORRUPT_VTAB: zName = "SQLITE_CORRUPT_VTAB"; break; + case SQLITE_NOTFOUND: zName = "SQLITE_NOTFOUND"; break; + case SQLITE_FULL: zName = "SQLITE_FULL"; break; + case SQLITE_CANTOPEN: zName = "SQLITE_CANTOPEN"; break; + case SQLITE_CANTOPEN_NOTEMPDIR: zName = "SQLITE_CANTOPEN_NOTEMPDIR";break; + case SQLITE_CANTOPEN_ISDIR: zName = "SQLITE_CANTOPEN_ISDIR"; break; + case SQLITE_CANTOPEN_FULLPATH: zName = "SQLITE_CANTOPEN_FULLPATH"; break; + case SQLITE_CANTOPEN_CONVPATH: zName = "SQLITE_CANTOPEN_CONVPATH"; break; + case SQLITE_CANTOPEN_SYMLINK: zName = "SQLITE_CANTOPEN_SYMLINK"; break; + case SQLITE_PROTOCOL: zName = "SQLITE_PROTOCOL"; break; + case SQLITE_EMPTY: zName = "SQLITE_EMPTY"; break; + case SQLITE_SCHEMA: zName = "SQLITE_SCHEMA"; break; + case SQLITE_TOOBIG: zName = "SQLITE_TOOBIG"; break; + case SQLITE_CONSTRAINT: zName = "SQLITE_CONSTRAINT"; break; + case SQLITE_CONSTRAINT_UNIQUE: zName = "SQLITE_CONSTRAINT_UNIQUE"; break; + case SQLITE_CONSTRAINT_TRIGGER: zName = "SQLITE_CONSTRAINT_TRIGGER";break; + case SQLITE_CONSTRAINT_FOREIGNKEY: + zName = "SQLITE_CONSTRAINT_FOREIGNKEY"; break; + case SQLITE_CONSTRAINT_CHECK: zName = "SQLITE_CONSTRAINT_CHECK"; break; + case SQLITE_CONSTRAINT_PRIMARYKEY: + zName = "SQLITE_CONSTRAINT_PRIMARYKEY"; break; + case SQLITE_CONSTRAINT_NOTNULL: zName = "SQLITE_CONSTRAINT_NOTNULL";break; + case SQLITE_CONSTRAINT_COMMITHOOK: + zName = "SQLITE_CONSTRAINT_COMMITHOOK"; break; + case SQLITE_CONSTRAINT_VTAB: zName = "SQLITE_CONSTRAINT_VTAB"; break; + case SQLITE_CONSTRAINT_FUNCTION: + zName = "SQLITE_CONSTRAINT_FUNCTION"; break; + case SQLITE_CONSTRAINT_ROWID: zName = "SQLITE_CONSTRAINT_ROWID"; break; + case SQLITE_MISMATCH: zName = "SQLITE_MISMATCH"; break; + case SQLITE_MISUSE: zName = "SQLITE_MISUSE"; break; + case SQLITE_NOLFS: zName = "SQLITE_NOLFS"; break; + case SQLITE_AUTH: zName = "SQLITE_AUTH"; break; + case SQLITE_FORMAT: zName = "SQLITE_FORMAT"; break; + case SQLITE_RANGE: zName = "SQLITE_RANGE"; break; + case SQLITE_NOTADB: zName = "SQLITE_NOTADB"; break; + case SQLITE_ROW: zName = "SQLITE_ROW"; break; + case SQLITE_NOTICE: zName = "SQLITE_NOTICE"; break; + case SQLITE_NOTICE_RECOVER_WAL: zName = "SQLITE_NOTICE_RECOVER_WAL";break; + case SQLITE_NOTICE_RECOVER_ROLLBACK: + zName = "SQLITE_NOTICE_RECOVER_ROLLBACK"; break; + case SQLITE_WARNING: zName = "SQLITE_WARNING"; break; + case SQLITE_WARNING_AUTOINDEX: zName = "SQLITE_WARNING_AUTOINDEX"; break; + case SQLITE_DONE: zName = "SQLITE_DONE"; break; + } } -#endif -#ifndef SQLITE_OMIT_VIRTUALTABLE - sqlite3_free(pParse->apVtabLock); + if( zName==0 ){ + static char zBuf[50]; + sqlite3_snprintf(sizeof(zBuf), zBuf, "SQLITE_UNKNOWN(%d)", origRc); + zName = zBuf; + } + return zName; +} #endif - if( !IN_DECLARE_VTAB ){ - /* If the pParse->declareVtab flag is set, do not delete any table - ** structure built up in pParse->pNewTable. The calling code (see vtab.c) - ** will take responsibility for freeing the Table structure. - */ - sqlite3DeleteTable(db, pParse->pNewTable); +/* +** Return a static string that describes the kind of error specified in the +** argument. +*/ +SQLITE_PRIVATE const char *sqlite3ErrStr(int rc){ + static const char* const aMsg[] = { + /* SQLITE_OK */ "not an error", + /* SQLITE_ERROR */ "SQL logic error", + /* SQLITE_INTERNAL */ 0, + /* SQLITE_PERM */ "access permission denied", + /* SQLITE_ABORT */ "query aborted", + /* SQLITE_BUSY */ "database is locked", + /* SQLITE_LOCKED */ "database table is locked", + /* SQLITE_NOMEM */ "out of memory", + /* SQLITE_READONLY */ "attempt to write a readonly database", + /* SQLITE_INTERRUPT */ "interrupted", + /* SQLITE_IOERR */ "disk I/O error", + /* SQLITE_CORRUPT */ "database disk image is malformed", + /* SQLITE_NOTFOUND */ "unknown operation", + /* SQLITE_FULL */ "database or disk is full", + /* SQLITE_CANTOPEN */ "unable to open database file", + /* SQLITE_PROTOCOL */ "locking protocol", + /* SQLITE_EMPTY */ 0, + /* SQLITE_SCHEMA */ "database schema has changed", + /* SQLITE_TOOBIG */ "string or blob too big", + /* SQLITE_CONSTRAINT */ "constraint failed", + /* SQLITE_MISMATCH */ "datatype mismatch", + /* SQLITE_MISUSE */ "bad parameter or other API misuse", +#ifdef SQLITE_DISABLE_LFS + /* SQLITE_NOLFS */ "large file support is disabled", +#else + /* SQLITE_NOLFS */ 0, +#endif + /* SQLITE_AUTH */ "authorization denied", + /* SQLITE_FORMAT */ 0, + /* SQLITE_RANGE */ "column index out of range", + /* SQLITE_NOTADB */ "file is not a database", + /* SQLITE_NOTICE */ "notification message", + /* SQLITE_WARNING */ "warning message", + }; + const char *zErr = "unknown error"; + switch( rc ){ + case SQLITE_ABORT_ROLLBACK: { + zErr = "abort due to ROLLBACK"; + break; + } + case SQLITE_ROW: { + zErr = "another row available"; + break; + } + case SQLITE_DONE: { + zErr = "no more rows available"; + break; + } + default: { + rc &= 0xff; + if( ALWAYS(rc>=0) && rcpWithToFree ) sqlite3WithDelete(db, pParse->pWithToFree); - sqlite3DeleteTrigger(db, pParse->pNewTrigger); - sqlite3DbFree(db, pParse->pVList); - while( pParse->pAinc ){ - AutoincInfo *p = pParse->pAinc; - pParse->pAinc = p->pNext; - sqlite3DbFreeNN(db, p); +/* +** This routine implements a busy callback that sleeps and tries +** again until a timeout value is reached. The timeout value is +** an integer number of milliseconds passed in as the first +** argument. +** +** Return non-zero to retry the lock. Return zero to stop trying +** and cause SQLite to return SQLITE_BUSY. +*/ +static int sqliteDefaultBusyCallback( + void *ptr, /* Database connection */ + int count /* Number of times table has been busy */ +){ +#if SQLITE_OS_WIN || HAVE_USLEEP + /* This case is for systems that have support for sleeping for fractions of + ** a second. Examples: All windows systems, unix systems with usleep() */ + static const u8 delays[] = + { 1, 2, 5, 10, 15, 20, 25, 25, 25, 50, 50, 100 }; + static const u8 totals[] = + { 0, 1, 3, 8, 18, 33, 53, 78, 103, 128, 178, 228 }; +# define NDELAY ArraySize(delays) + sqlite3 *db = (sqlite3 *)ptr; + int tmout = db->busyTimeout; + int delay, prior; + + assert( count>=0 ); + if( count < NDELAY ){ + delay = delays[count]; + prior = totals[count]; + }else{ + delay = delays[NDELAY-1]; + prior = totals[NDELAY-1] + delay*(count-(NDELAY-1)); } - while( pParse->pZombieTab ){ - Table *p = pParse->pZombieTab; - pParse->pZombieTab = p->pNextZombie; - sqlite3DeleteTable(db, p); + if( prior + delay > tmout ){ + delay = tmout - prior; + if( delay<=0 ) return 0; } - assert( nErr==0 || pParse->rc!=SQLITE_OK ); - return nErr; + sqlite3OsSleep(db->pVfs, delay*1000); + return 1; +#else + /* This case for unix systems that lack usleep() support. Sleeping + ** must be done in increments of whole seconds */ + sqlite3 *db = (sqlite3 *)ptr; + int tmout = ((sqlite3 *)ptr)->busyTimeout; + if( (count+1)*1000 > tmout ){ + return 0; + } + sqlite3OsSleep(db->pVfs, 1000000); + return 1; +#endif } -/************** End of tokenize.c ********************************************/ -/************** Begin file complete.c ****************************************/ /* -** 2001 September 15 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. +** Invoke the given busy handler. ** -************************************************************************* -** An tokenizer for SQL +** This routine is called when an operation failed to acquire a +** lock on VFS file pFile. ** -** This file contains C code that implements the sqlite3_complete() API. -** This code used to be part of the tokenizer.c source file. But by -** separating it out, the code will be automatically omitted from -** static links that do not use it. +** If this routine returns non-zero, the lock is retried. If it +** returns 0, the operation aborts with an SQLITE_BUSY error. */ -/* #include "sqliteInt.h" */ -#ifndef SQLITE_OMIT_COMPLETE +SQLITE_PRIVATE int sqlite3InvokeBusyHandler(BusyHandler *p){ + int rc; + if( p->xBusyHandler==0 || p->nBusy<0 ) return 0; + rc = p->xBusyHandler(p->pBusyArg, p->nBusy); + if( rc==0 ){ + p->nBusy = -1; + }else{ + p->nBusy++; + } + return rc; +} /* -** This is defined in tokenize.c. We just have to import the definition. +** This routine sets the busy callback for an Sqlite database to the +** given callback function with the given argument. */ -#ifndef SQLITE_AMALGAMATION -#ifdef SQLITE_ASCII -#define IdChar(C) ((sqlite3CtypeMap[(unsigned char)C]&0x46)!=0) +SQLITE_API int sqlite3_busy_handler( + sqlite3 *db, + int (*xBusy)(void*,int), + void *pArg +){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; #endif -#ifdef SQLITE_EBCDIC -SQLITE_PRIVATE const char sqlite3IsEbcdicIdChar[]; -#define IdChar(C) (((c=C)>=0x42 && sqlite3IsEbcdicIdChar[c-0x40])) + sqlite3_mutex_enter(db->mutex); + db->busyHandler.xBusyHandler = xBusy; + db->busyHandler.pBusyArg = pArg; + db->busyHandler.nBusy = 0; + db->busyTimeout = 0; + sqlite3_mutex_leave(db->mutex); + return SQLITE_OK; +} + +#ifndef SQLITE_OMIT_PROGRESS_CALLBACK +/* +** This routine sets the progress callback for an Sqlite database to the +** given callback function with the given argument. The progress callback will +** be invoked every nOps opcodes. +*/ +SQLITE_API void sqlite3_progress_handler( + sqlite3 *db, + int nOps, + int (*xProgress)(void*), + void *pArg +){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return; + } +#endif + sqlite3_mutex_enter(db->mutex); + if( nOps>0 ){ + db->xProgress = xProgress; + db->nProgressOps = (unsigned)nOps; + db->pProgressArg = pArg; + }else{ + db->xProgress = 0; + db->nProgressOps = 0; + db->pProgressArg = 0; + } + sqlite3_mutex_leave(db->mutex); +} #endif -#endif /* SQLITE_AMALGAMATION */ /* -** Token types used by the sqlite3_complete() routine. See the header -** comments on that procedure for additional information. +** This routine installs a default busy handler that waits for the +** specified number of milliseconds before returning 0. */ -#define tkSEMI 0 -#define tkWS 1 -#define tkOTHER 2 -#ifndef SQLITE_OMIT_TRIGGER -#define tkEXPLAIN 3 -#define tkCREATE 4 -#define tkTEMP 5 -#define tkTRIGGER 6 -#define tkEND 7 +SQLITE_API int sqlite3_busy_timeout(sqlite3 *db, int ms){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; #endif + if( ms>0 ){ + sqlite3_busy_handler(db, (int(*)(void*,int))sqliteDefaultBusyCallback, + (void*)db); + db->busyTimeout = ms; + }else{ + sqlite3_busy_handler(db, 0, 0); + } + return SQLITE_OK; +} /* -** Return TRUE if the given SQL string ends in a semicolon. -** -** Special handling is require for CREATE TRIGGER statements. -** Whenever the CREATE TRIGGER keywords are seen, the statement -** must end with ";END;". -** -** This implementation uses a state machine with 8 states: -** -** (0) INVALID We have not yet seen a non-whitespace character. -** -** (1) START At the beginning or end of an SQL statement. This routine -** returns 1 if it ends in the START state and 0 if it ends -** in any other state. -** -** (2) NORMAL We are in the middle of statement which ends with a single -** semicolon. -** -** (3) EXPLAIN The keyword EXPLAIN has been seen at the beginning of -** a statement. -** -** (4) CREATE The keyword CREATE has been seen at the beginning of a -** statement, possibly preceded by EXPLAIN and/or followed by -** TEMP or TEMPORARY -** -** (5) TRIGGER We are in the middle of a trigger definition that must be -** ended by a semicolon, the keyword END, and another semicolon. -** -** (6) SEMI We've seen the first semicolon in the ";END;" that occurs at -** the end of a trigger definition. -** -** (7) END We've seen the ";END" of the ";END;" that occurs at the end -** of a trigger definition. -** -** Transitions between states above are determined by tokens extracted -** from the input. The following tokens are significant: -** -** (0) tkSEMI A semicolon. -** (1) tkWS Whitespace. -** (2) tkOTHER Any other SQL token. -** (3) tkEXPLAIN The "explain" keyword. -** (4) tkCREATE The "create" keyword. -** (5) tkTEMP The "temp" or "temporary" keyword. -** (6) tkTRIGGER The "trigger" keyword. -** (7) tkEND The "end" keyword. -** -** Whitespace never causes a state transition and is always ignored. -** This means that a SQL string of all whitespace is invalid. -** -** If we compile with SQLITE_OMIT_TRIGGER, all of the computation needed -** to recognize the end of a trigger can be omitted. All we have to do -** is look for a semicolon that is not part of an string or comment. +** Cause any pending operation to stop at its earliest opportunity. */ -SQLITE_API int sqlite3_complete(const char *zSql){ - u8 state = 0; /* Current state, using numbers defined in header comment */ - u8 token; /* Value of the next token */ - -#ifndef SQLITE_OMIT_TRIGGER - /* A complex statement machine used to detect the end of a CREATE TRIGGER - ** statement. This is the normal case. - */ - static const u8 trans[8][8] = { - /* Token: */ - /* State: ** SEMI WS OTHER EXPLAIN CREATE TEMP TRIGGER END */ - /* 0 INVALID: */ { 1, 0, 2, 3, 4, 2, 2, 2, }, - /* 1 START: */ { 1, 1, 2, 3, 4, 2, 2, 2, }, - /* 2 NORMAL: */ { 1, 2, 2, 2, 2, 2, 2, 2, }, - /* 3 EXPLAIN: */ { 1, 3, 3, 2, 4, 2, 2, 2, }, - /* 4 CREATE: */ { 1, 4, 2, 2, 2, 4, 5, 2, }, - /* 5 TRIGGER: */ { 6, 5, 5, 5, 5, 5, 5, 5, }, - /* 6 SEMI: */ { 6, 6, 5, 5, 5, 5, 5, 7, }, - /* 7 END: */ { 1, 7, 5, 5, 5, 5, 5, 5, }, - }; -#else - /* If triggers are not supported by this compile then the statement machine - ** used to detect the end of a statement is much simpler - */ - static const u8 trans[3][3] = { - /* Token: */ - /* State: ** SEMI WS OTHER */ - /* 0 INVALID: */ { 1, 0, 2, }, - /* 1 START: */ { 1, 1, 2, }, - /* 2 NORMAL: */ { 1, 2, 2, }, - }; -#endif /* SQLITE_OMIT_TRIGGER */ - +SQLITE_API void sqlite3_interrupt(sqlite3 *db){ #ifdef SQLITE_ENABLE_API_ARMOR - if( zSql==0 ){ + if( !sqlite3SafetyCheckOk(db) && (db==0 || db->magic!=SQLITE_MAGIC_ZOMBIE) ){ (void)SQLITE_MISUSE_BKPT; - return 0; + return; } #endif + AtomicStore(&db->u1.isInterrupted, 1); +} - while( *zSql ){ - switch( *zSql ){ - case ';': { /* A semicolon */ - token = tkSEMI; - break; - } - case ' ': - case '\r': - case '\t': - case '\n': - case '\f': { /* White space is ignored */ - token = tkWS; - break; - } - case '/': { /* C-style comments */ - if( zSql[1]!='*' ){ - token = tkOTHER; - break; - } - zSql += 2; - while( zSql[0] && (zSql[0]!='*' || zSql[1]!='/') ){ zSql++; } - if( zSql[0]==0 ) return 0; - zSql++; - token = tkWS; - break; - } - case '-': { /* SQL-style comments from "--" to end of line */ - if( zSql[1]!='-' ){ - token = tkOTHER; - break; - } - while( *zSql && *zSql!='\n' ){ zSql++; } - if( *zSql==0 ) return state==1; - token = tkWS; - break; - } - case '[': { /* Microsoft-style identifiers in [...] */ - zSql++; - while( *zSql && *zSql!=']' ){ zSql++; } - if( *zSql==0 ) return 0; - token = tkOTHER; - break; - } - case '`': /* Grave-accent quoted symbols used by MySQL */ - case '"': /* single- and double-quoted strings */ - case '\'': { - int c = *zSql; - zSql++; - while( *zSql && *zSql!=c ){ zSql++; } - if( *zSql==0 ) return 0; - token = tkOTHER; - break; - } - default: { -#ifdef SQLITE_EBCDIC - unsigned char c; -#endif - if( IdChar((u8)*zSql) ){ - /* Keywords and unquoted identifiers */ - int nId; - for(nId=1; IdChar(zSql[nId]); nId++){} -#ifdef SQLITE_OMIT_TRIGGER - token = tkOTHER; + +/* +** This function is exactly the same as sqlite3_create_function(), except +** that it is designed to be called by internal code. The difference is +** that if a malloc() fails in sqlite3_create_function(), an error code +** is returned and the mallocFailed flag cleared. +*/ +SQLITE_PRIVATE int sqlite3CreateFunc( + sqlite3 *db, + const char *zFunctionName, + int nArg, + int enc, + void *pUserData, + void (*xSFunc)(sqlite3_context*,int,sqlite3_value **), + void (*xStep)(sqlite3_context*,int,sqlite3_value **), + void (*xFinal)(sqlite3_context*), + void (*xValue)(sqlite3_context*), + void (*xInverse)(sqlite3_context*,int,sqlite3_value **), + FuncDestructor *pDestructor +){ + FuncDef *p; + int nName; + int extraFlags; + + assert( sqlite3_mutex_held(db->mutex) ); + assert( xValue==0 || xSFunc==0 ); + if( zFunctionName==0 /* Must have a valid name */ + || (xSFunc!=0 && xFinal!=0) /* Not both xSFunc and xFinal */ + || ((xFinal==0)!=(xStep==0)) /* Both or neither of xFinal and xStep */ + || ((xValue==0)!=(xInverse==0)) /* Both or neither of xValue, xInverse */ + || (nArg<-1 || nArg>SQLITE_MAX_FUNCTION_ARG) + || (255<(nName = sqlite3Strlen30( zFunctionName))) + ){ + return SQLITE_MISUSE_BKPT; + } + + assert( SQLITE_FUNC_CONSTANT==SQLITE_DETERMINISTIC ); + assert( SQLITE_FUNC_DIRECT==SQLITE_DIRECTONLY ); + extraFlags = enc & (SQLITE_DETERMINISTIC|SQLITE_DIRECTONLY| + SQLITE_SUBTYPE|SQLITE_INNOCUOUS); + enc &= (SQLITE_FUNC_ENCMASK|SQLITE_ANY); + + /* The SQLITE_INNOCUOUS flag is the same bit as SQLITE_FUNC_UNSAFE. But + ** the meaning is inverted. So flip the bit. */ + assert( SQLITE_FUNC_UNSAFE==SQLITE_INNOCUOUS ); + extraFlags ^= SQLITE_FUNC_UNSAFE; + + +#ifndef SQLITE_OMIT_UTF16 + /* If SQLITE_UTF16 is specified as the encoding type, transform this + ** to one of SQLITE_UTF16LE or SQLITE_UTF16BE using the + ** SQLITE_UTF16NATIVE macro. SQLITE_UTF16 is not used internally. + ** + ** If SQLITE_ANY is specified, add three versions of the function + ** to the hash table. + */ + if( enc==SQLITE_UTF16 ){ + enc = SQLITE_UTF16NATIVE; + }else if( enc==SQLITE_ANY ){ + int rc; + rc = sqlite3CreateFunc(db, zFunctionName, nArg, + (SQLITE_UTF8|extraFlags)^SQLITE_FUNC_UNSAFE, + pUserData, xSFunc, xStep, xFinal, xValue, xInverse, pDestructor); + if( rc==SQLITE_OK ){ + rc = sqlite3CreateFunc(db, zFunctionName, nArg, + (SQLITE_UTF16LE|extraFlags)^SQLITE_FUNC_UNSAFE, + pUserData, xSFunc, xStep, xFinal, xValue, xInverse, pDestructor); + } + if( rc!=SQLITE_OK ){ + return rc; + } + enc = SQLITE_UTF16BE; + } #else - switch( *zSql ){ - case 'c': case 'C': { - if( nId==6 && sqlite3StrNICmp(zSql, "create", 6)==0 ){ - token = tkCREATE; - }else{ - token = tkOTHER; - } - break; - } - case 't': case 'T': { - if( nId==7 && sqlite3StrNICmp(zSql, "trigger", 7)==0 ){ - token = tkTRIGGER; - }else if( nId==4 && sqlite3StrNICmp(zSql, "temp", 4)==0 ){ - token = tkTEMP; - }else if( nId==9 && sqlite3StrNICmp(zSql, "temporary", 9)==0 ){ - token = tkTEMP; - }else{ - token = tkOTHER; - } - break; - } - case 'e': case 'E': { - if( nId==3 && sqlite3StrNICmp(zSql, "end", 3)==0 ){ - token = tkEND; - }else -#ifndef SQLITE_OMIT_EXPLAIN - if( nId==7 && sqlite3StrNICmp(zSql, "explain", 7)==0 ){ - token = tkEXPLAIN; - }else + enc = SQLITE_UTF8; #endif - { - token = tkOTHER; - } - break; - } - default: { - token = tkOTHER; - break; - } - } -#endif /* SQLITE_OMIT_TRIGGER */ - zSql += nId-1; - }else{ - /* Operators and special symbols */ - token = tkOTHER; - } - break; - } + + /* Check if an existing function is being overridden or deleted. If so, + ** and there are active VMs, then return SQLITE_BUSY. If a function + ** is being overridden/deleted but there are no active VMs, allow the + ** operation to continue but invalidate all precompiled statements. + */ + p = sqlite3FindFunction(db, zFunctionName, nArg, (u8)enc, 0); + if( p && (p->funcFlags & SQLITE_FUNC_ENCMASK)==(u32)enc && p->nArg==nArg ){ + if( db->nVdbeActive ){ + sqlite3ErrorWithMsg(db, SQLITE_BUSY, + "unable to delete/modify user-function due to active statements"); + assert( !db->mallocFailed ); + return SQLITE_BUSY; + }else{ + sqlite3ExpirePreparedStatements(db, 0); + } + } + + p = sqlite3FindFunction(db, zFunctionName, nArg, (u8)enc, 1); + assert(p || db->mallocFailed); + if( !p ){ + return SQLITE_NOMEM_BKPT; + } + + /* If an older version of the function with a configured destructor is + ** being replaced invoke the destructor function here. */ + functionDestroy(db, p); + + if( pDestructor ){ + pDestructor->nRef++; + } + p->u.pDestructor = pDestructor; + p->funcFlags = (p->funcFlags & SQLITE_FUNC_ENCMASK) | extraFlags; + testcase( p->funcFlags & SQLITE_DETERMINISTIC ); + testcase( p->funcFlags & SQLITE_DIRECTONLY ); + p->xSFunc = xSFunc ? xSFunc : xStep; + p->xFinalize = xFinal; + p->xValue = xValue; + p->xInverse = xInverse; + p->pUserData = pUserData; + p->nArg = (u16)nArg; + return SQLITE_OK; +} + +/* +** Worker function used by utf-8 APIs that create new functions: +** +** sqlite3_create_function() +** sqlite3_create_function_v2() +** sqlite3_create_window_function() +*/ +static int createFunctionApi( + sqlite3 *db, + const char *zFunc, + int nArg, + int enc, + void *p, + void (*xSFunc)(sqlite3_context*,int,sqlite3_value**), + void (*xStep)(sqlite3_context*,int,sqlite3_value**), + void (*xFinal)(sqlite3_context*), + void (*xValue)(sqlite3_context*), + void (*xInverse)(sqlite3_context*,int,sqlite3_value**), + void(*xDestroy)(void*) +){ + int rc = SQLITE_ERROR; + FuncDestructor *pArg = 0; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + return SQLITE_MISUSE_BKPT; + } +#endif + sqlite3_mutex_enter(db->mutex); + if( xDestroy ){ + pArg = (FuncDestructor *)sqlite3Malloc(sizeof(FuncDestructor)); + if( !pArg ){ + sqlite3OomFault(db); + xDestroy(p); + goto out; } - state = trans[state][token]; - zSql++; + pArg->nRef = 0; + pArg->xDestroy = xDestroy; + pArg->pUserData = p; } - return state==1; + rc = sqlite3CreateFunc(db, zFunc, nArg, enc, p, + xSFunc, xStep, xFinal, xValue, xInverse, pArg + ); + if( pArg && pArg->nRef==0 ){ + assert( rc!=SQLITE_OK ); + xDestroy(p); + sqlite3_free(pArg); + } + + out: + rc = sqlite3ApiExit(db, rc); + sqlite3_mutex_leave(db->mutex); + return rc; } -#ifndef SQLITE_OMIT_UTF16 /* -** This routine is the same as the sqlite3_complete() routine described -** above, except that the parameter is required to be UTF-16 encoded, not -** UTF-8. +** Create new user functions. */ -SQLITE_API int sqlite3_complete16(const void *zSql){ - sqlite3_value *pVal; - char const *zSql8; +SQLITE_API int sqlite3_create_function( + sqlite3 *db, + const char *zFunc, + int nArg, + int enc, + void *p, + void (*xSFunc)(sqlite3_context*,int,sqlite3_value **), + void (*xStep)(sqlite3_context*,int,sqlite3_value **), + void (*xFinal)(sqlite3_context*) +){ + return createFunctionApi(db, zFunc, nArg, enc, p, xSFunc, xStep, + xFinal, 0, 0, 0); +} +SQLITE_API int sqlite3_create_function_v2( + sqlite3 *db, + const char *zFunc, + int nArg, + int enc, + void *p, + void (*xSFunc)(sqlite3_context*,int,sqlite3_value **), + void (*xStep)(sqlite3_context*,int,sqlite3_value **), + void (*xFinal)(sqlite3_context*), + void (*xDestroy)(void *) +){ + return createFunctionApi(db, zFunc, nArg, enc, p, xSFunc, xStep, + xFinal, 0, 0, xDestroy); +} +SQLITE_API int sqlite3_create_window_function( + sqlite3 *db, + const char *zFunc, + int nArg, + int enc, + void *p, + void (*xStep)(sqlite3_context*,int,sqlite3_value **), + void (*xFinal)(sqlite3_context*), + void (*xValue)(sqlite3_context*), + void (*xInverse)(sqlite3_context*,int,sqlite3_value **), + void (*xDestroy)(void *) +){ + return createFunctionApi(db, zFunc, nArg, enc, p, 0, xStep, + xFinal, xValue, xInverse, xDestroy); +} + +#ifndef SQLITE_OMIT_UTF16 +SQLITE_API int sqlite3_create_function16( + sqlite3 *db, + const void *zFunctionName, + int nArg, + int eTextRep, + void *p, + void (*xSFunc)(sqlite3_context*,int,sqlite3_value**), + void (*xStep)(sqlite3_context*,int,sqlite3_value**), + void (*xFinal)(sqlite3_context*) +){ int rc; + char *zFunc8; -#ifndef SQLITE_OMIT_AUTOINIT - rc = sqlite3_initialize(); - if( rc ) return rc; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) || zFunctionName==0 ) return SQLITE_MISUSE_BKPT; #endif - pVal = sqlite3ValueNew(0); - sqlite3ValueSetStr(pVal, -1, zSql, SQLITE_UTF16NATIVE, SQLITE_STATIC); - zSql8 = sqlite3ValueText(pVal, SQLITE_UTF8); - if( zSql8 ){ - rc = sqlite3_complete(zSql8); - }else{ - rc = SQLITE_NOMEM_BKPT; - } - sqlite3ValueFree(pVal); - return rc & 0xff; + sqlite3_mutex_enter(db->mutex); + assert( !db->mallocFailed ); + zFunc8 = sqlite3Utf16to8(db, zFunctionName, -1, SQLITE_UTF16NATIVE); + rc = sqlite3CreateFunc(db, zFunc8, nArg, eTextRep, p, xSFunc,xStep,xFinal,0,0,0); + sqlite3DbFree(db, zFunc8); + rc = sqlite3ApiExit(db, rc); + sqlite3_mutex_leave(db->mutex); + return rc; } -#endif /* SQLITE_OMIT_UTF16 */ -#endif /* SQLITE_OMIT_COMPLETE */ +#endif + -/************** End of complete.c ********************************************/ -/************** Begin file main.c ********************************************/ /* -** 2001 September 15 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** Main file for the SQLite library. The routines in this file -** implement the programmer interface to the library. Routines in -** other files are for internal use by SQLite and should not be -** accessed by users of the library. +** The following is the implementation of an SQL function that always +** fails with an error message stating that the function is used in the +** wrong context. The sqlite3_overload_function() API might construct +** SQL function that use this routine so that the functions will exist +** for name resolution but are actually overloaded by the xFindFunction +** method of virtual tables. */ -/* #include "sqliteInt.h" */ +static void sqlite3InvalidFunction( + sqlite3_context *context, /* The function calling context */ + int NotUsed, /* Number of arguments to the function */ + sqlite3_value **NotUsed2 /* Value of each argument */ +){ + const char *zName = (const char*)sqlite3_user_data(context); + char *zErr; + UNUSED_PARAMETER2(NotUsed, NotUsed2); + zErr = sqlite3_mprintf( + "unable to use function %s in the requested context", zName); + sqlite3_result_error(context, zErr, -1); + sqlite3_free(zErr); +} -#ifdef SQLITE_ENABLE_FTS3 -/************** Include fts3.h in the middle of main.c ***********************/ -/************** Begin file fts3.h ********************************************/ /* -** 2006 Oct 10 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. +** Declare that a function has been overloaded by a virtual table. ** -****************************************************************************** +** If the function already exists as a regular global function, then +** this routine is a no-op. If the function does not exist, then create +** a new one that always throws a run-time error. ** -** This header file is used by programs that want to link against the -** FTS3 library. All it does is declare the sqlite3Fts3Init() interface. +** When virtual tables intend to provide an overloaded function, they +** should call this routine to make sure the global function exists. +** A global function must exist in order for name resolution to work +** properly. */ -/* #include "sqlite3.h" */ - -#if 0 -extern "C" { -#endif /* __cplusplus */ - -SQLITE_PRIVATE int sqlite3Fts3Init(sqlite3 *db); - -#if 0 -} /* extern "C" */ -#endif /* __cplusplus */ +SQLITE_API int sqlite3_overload_function( + sqlite3 *db, + const char *zName, + int nArg +){ + int rc; + char *zCopy; -/************** End of fts3.h ************************************************/ -/************** Continuing where we left off in main.c ***********************/ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) || zName==0 || nArg<-2 ){ + return SQLITE_MISUSE_BKPT; + } #endif -#ifdef SQLITE_ENABLE_RTREE -/************** Include rtree.h in the middle of main.c **********************/ -/************** Begin file rtree.h *******************************************/ + sqlite3_mutex_enter(db->mutex); + rc = sqlite3FindFunction(db, zName, nArg, SQLITE_UTF8, 0)!=0; + sqlite3_mutex_leave(db->mutex); + if( rc ) return SQLITE_OK; + zCopy = sqlite3_mprintf(zName); + if( zCopy==0 ) return SQLITE_NOMEM; + return sqlite3_create_function_v2(db, zName, nArg, SQLITE_UTF8, + zCopy, sqlite3InvalidFunction, 0, 0, sqlite3_free); +} + +#ifndef SQLITE_OMIT_TRACE /* -** 2008 May 26 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -****************************************************************************** +** Register a trace function. The pArg from the previously registered trace +** is returned. ** -** This header file is used by programs that want to link against the -** RTREE library. All it does is declare the sqlite3RtreeInit() interface. +** A NULL trace function means that no tracing is executes. A non-NULL +** trace is a pointer to a function that is invoked at the start of each +** SQL statement. */ -/* #include "sqlite3.h" */ +#ifndef SQLITE_OMIT_DEPRECATED +SQLITE_API void *sqlite3_trace(sqlite3 *db, void(*xTrace)(void*,const char*), void *pArg){ + void *pOld; -#ifdef SQLITE_OMIT_VIRTUALTABLE -# undef SQLITE_ENABLE_RTREE +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } #endif + sqlite3_mutex_enter(db->mutex); + pOld = db->pTraceArg; + db->mTrace = xTrace ? SQLITE_TRACE_LEGACY : 0; + db->xTrace = (int(*)(u32,void*,void*,void*))xTrace; + db->pTraceArg = pArg; + sqlite3_mutex_leave(db->mutex); + return pOld; +} +#endif /* SQLITE_OMIT_DEPRECATED */ -#if 0 -extern "C" { -#endif /* __cplusplus */ - -SQLITE_PRIVATE int sqlite3RtreeInit(sqlite3 *db); - -#if 0 -} /* extern "C" */ -#endif /* __cplusplus */ - -/************** End of rtree.h ***********************************************/ -/************** Continuing where we left off in main.c ***********************/ +/* Register a trace callback using the version-2 interface. +*/ +SQLITE_API int sqlite3_trace_v2( + sqlite3 *db, /* Trace this connection */ + unsigned mTrace, /* Mask of events to be traced */ + int(*xTrace)(unsigned,void*,void*,void*), /* Callback to invoke */ + void *pArg /* Context */ +){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + return SQLITE_MISUSE_BKPT; + } #endif -#if defined(SQLITE_ENABLE_ICU) || defined(SQLITE_ENABLE_ICU_COLLATIONS) -/************** Include sqliteicu.h in the middle of main.c ******************/ -/************** Begin file sqliteicu.h ***************************************/ + sqlite3_mutex_enter(db->mutex); + if( mTrace==0 ) xTrace = 0; + if( xTrace==0 ) mTrace = 0; + db->mTrace = mTrace; + db->xTrace = xTrace; + db->pTraceArg = pArg; + sqlite3_mutex_leave(db->mutex); + return SQLITE_OK; +} + +#ifndef SQLITE_OMIT_DEPRECATED /* -** 2008 May 26 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -****************************************************************************** +** Register a profile function. The pArg from the previously registered +** profile function is returned. ** -** This header file is used by programs that want to link against the -** ICU extension. All it does is declare the sqlite3IcuInit() interface. +** A NULL profile function means that no profiling is executes. A non-NULL +** profile is a pointer to a function that is invoked at the conclusion of +** each SQL statement that is run. */ -/* #include "sqlite3.h" */ - -#if 0 -extern "C" { -#endif /* __cplusplus */ - -SQLITE_PRIVATE int sqlite3IcuInit(sqlite3 *db); +SQLITE_API void *sqlite3_profile( + sqlite3 *db, + void (*xProfile)(void*,const char*,sqlite_uint64), + void *pArg +){ + void *pOld; -#if 0 -} /* extern "C" */ -#endif /* __cplusplus */ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + sqlite3_mutex_enter(db->mutex); + pOld = db->pProfileArg; + db->xProfile = xProfile; + db->pProfileArg = pArg; + db->mTrace &= SQLITE_TRACE_NONLEGACY_MASK; + if( db->xProfile ) db->mTrace |= SQLITE_TRACE_XPROFILE; + sqlite3_mutex_leave(db->mutex); + return pOld; +} +#endif /* SQLITE_OMIT_DEPRECATED */ +#endif /* SQLITE_OMIT_TRACE */ +/* +** Register a function to be invoked when a transaction commits. +** If the invoked function returns non-zero, then the commit becomes a +** rollback. +*/ +SQLITE_API void *sqlite3_commit_hook( + sqlite3 *db, /* Attach the hook to this database */ + int (*xCallback)(void*), /* Function to invoke on each commit */ + void *pArg /* Argument to the function */ +){ + void *pOld; -/************** End of sqliteicu.h *******************************************/ -/************** Continuing where we left off in main.c ***********************/ -#endif -#ifdef SQLITE_ENABLE_JSON1 -SQLITE_PRIVATE int sqlite3Json1Init(sqlite3*); -#endif -#ifdef SQLITE_ENABLE_STMTVTAB -SQLITE_PRIVATE int sqlite3StmtVtabInit(sqlite3*); -#endif -#ifdef SQLITE_ENABLE_FTS5 -SQLITE_PRIVATE int sqlite3Fts5Init(sqlite3*); +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } #endif + sqlite3_mutex_enter(db->mutex); + pOld = db->pCommitArg; + db->xCommitCallback = xCallback; + db->pCommitArg = pArg; + sqlite3_mutex_leave(db->mutex); + return pOld; +} -#ifndef SQLITE_AMALGAMATION -/* IMPLEMENTATION-OF: R-46656-45156 The sqlite3_version[] string constant -** contains the text of SQLITE_VERSION macro. +/* +** Register a callback to be invoked each time a row is updated, +** inserted or deleted using this database connection. */ -SQLITE_API const char sqlite3_version[] = SQLITE_VERSION; +SQLITE_API void *sqlite3_update_hook( + sqlite3 *db, /* Attach the hook to this database */ + void (*xCallback)(void*,int,char const *,char const *,sqlite_int64), + void *pArg /* Argument to the function */ +){ + void *pRet; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } #endif + sqlite3_mutex_enter(db->mutex); + pRet = db->pUpdateArg; + db->xUpdateCallback = xCallback; + db->pUpdateArg = pArg; + sqlite3_mutex_leave(db->mutex); + return pRet; +} -/* IMPLEMENTATION-OF: R-53536-42575 The sqlite3_libversion() function returns -** a pointer to the to the sqlite3_version[] string constant. +/* +** Register a callback to be invoked each time a transaction is rolled +** back by this database connection. */ -SQLITE_API const char *sqlite3_libversion(void){ return sqlite3_version; } +SQLITE_API void *sqlite3_rollback_hook( + sqlite3 *db, /* Attach the hook to this database */ + void (*xCallback)(void*), /* Callback function */ + void *pArg /* Argument to the function */ +){ + void *pRet; -/* IMPLEMENTATION-OF: R-25063-23286 The sqlite3_sourceid() function returns a -** pointer to a string constant whose value is the same as the -** SQLITE_SOURCE_ID C preprocessor macro. Except if SQLite is built using -** an edited copy of the amalgamation, then the last four characters of -** the hash might be different from SQLITE_SOURCE_ID. -*/ -/* SQLITE_API const char *sqlite3_sourceid(void){ return SQLITE_SOURCE_ID; } */ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + sqlite3_mutex_enter(db->mutex); + pRet = db->pRollbackArg; + db->xRollbackCallback = xCallback; + db->pRollbackArg = pArg; + sqlite3_mutex_leave(db->mutex); + return pRet; +} -/* IMPLEMENTATION-OF: R-35210-63508 The sqlite3_libversion_number() function -** returns an integer equal to SQLITE_VERSION_NUMBER. +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK +/* +** Register a callback to be invoked each time a row is updated, +** inserted or deleted using this database connection. */ -SQLITE_API int sqlite3_libversion_number(void){ return SQLITE_VERSION_NUMBER; } +SQLITE_API void *sqlite3_preupdate_hook( + sqlite3 *db, /* Attach the hook to this database */ + void(*xCallback)( /* Callback function */ + void*,sqlite3*,int,char const*,char const*,sqlite3_int64,sqlite3_int64), + void *pArg /* First callback argument */ +){ + void *pRet; + sqlite3_mutex_enter(db->mutex); + pRet = db->pPreUpdateArg; + db->xPreUpdateCallback = xCallback; + db->pPreUpdateArg = pArg; + sqlite3_mutex_leave(db->mutex); + return pRet; +} +#endif /* SQLITE_ENABLE_PREUPDATE_HOOK */ -/* IMPLEMENTATION-OF: R-20790-14025 The sqlite3_threadsafe() function returns -** zero if and only if SQLite was compiled with mutexing code omitted due to -** the SQLITE_THREADSAFE compile-time option being set to 0. -*/ -SQLITE_API int sqlite3_threadsafe(void){ return SQLITE_THREADSAFE; } +#ifndef SQLITE_OMIT_WAL +/* +** The sqlite3_wal_hook() callback registered by sqlite3_wal_autocheckpoint(). +** Invoke sqlite3_wal_checkpoint if the number of frames in the log file +** is greater than sqlite3.pWalArg cast to an integer (the value configured by +** wal_autocheckpoint()). +*/ +SQLITE_PRIVATE int sqlite3WalDefaultHook( + void *pClientData, /* Argument */ + sqlite3 *db, /* Connection */ + const char *zDb, /* Database */ + int nFrame /* Size of WAL */ +){ + if( nFrame>=SQLITE_PTR_TO_INT(pClientData) ){ + sqlite3BeginBenignMalloc(); + sqlite3_wal_checkpoint(db, zDb); + sqlite3EndBenignMalloc(); + } + return SQLITE_OK; +} +#endif /* SQLITE_OMIT_WAL */ /* -** When compiling the test fixture or with debugging enabled (on Win32), -** this variable being set to non-zero will cause OSTRACE macros to emit -** extra diagnostic information. +** Configure an sqlite3_wal_hook() callback to automatically checkpoint +** a database after committing a transaction if there are nFrame or +** more frames in the log file. Passing zero or a negative value as the +** nFrame parameter disables automatic checkpoints entirely. +** +** The callback registered by this function replaces any existing callback +** registered using sqlite3_wal_hook(). Likewise, registering a callback +** using sqlite3_wal_hook() disables the automatic checkpoint mechanism +** configured by this function. */ -#ifdef SQLITE_HAVE_OS_TRACE -# ifndef SQLITE_DEBUG_OS_TRACE -# define SQLITE_DEBUG_OS_TRACE 0 -# endif - int sqlite3OSTrace = SQLITE_DEBUG_OS_TRACE; +SQLITE_API int sqlite3_wal_autocheckpoint(sqlite3 *db, int nFrame){ +#ifdef SQLITE_OMIT_WAL + UNUSED_PARAMETER(db); + UNUSED_PARAMETER(nFrame); +#else +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif + if( nFrame>0 ){ + sqlite3_wal_hook(db, sqlite3WalDefaultHook, SQLITE_INT_TO_PTR(nFrame)); + }else{ + sqlite3_wal_hook(db, 0, 0); + } #endif + return SQLITE_OK; +} -#if !defined(SQLITE_OMIT_TRACE) && defined(SQLITE_ENABLE_IOTRACE) /* -** If the following function pointer is not NULL and if -** SQLITE_ENABLE_IOTRACE is enabled, then messages describing -** I/O active are written using this function. These messages -** are intended for debugging activity only. +** Register a callback to be invoked each time a transaction is written +** into the write-ahead-log by this database connection. */ -SQLITE_API void (SQLITE_CDECL *sqlite3IoTrace)(const char*, ...) = 0; +SQLITE_API void *sqlite3_wal_hook( + sqlite3 *db, /* Attach the hook to this db handle */ + int(*xCallback)(void *, sqlite3*, const char*, int), + void *pArg /* First argument passed to xCallback() */ +){ +#ifndef SQLITE_OMIT_WAL + void *pRet; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + sqlite3_mutex_enter(db->mutex); + pRet = db->pWalArg; + db->xWalCallback = xCallback; + db->pWalArg = pArg; + sqlite3_mutex_leave(db->mutex); + return pRet; +#else + return 0; #endif +} /* -** If the following global variable points to a string which is the -** name of a directory, then that directory will be used to store -** temporary files. -** -** See also the "PRAGMA temp_store_directory" SQL command. +** Checkpoint database zDb. */ -SQLITE_API char *sqlite3_temp_directory = 0; +SQLITE_API int sqlite3_wal_checkpoint_v2( + sqlite3 *db, /* Database handle */ + const char *zDb, /* Name of attached database (or NULL) */ + int eMode, /* SQLITE_CHECKPOINT_* value */ + int *pnLog, /* OUT: Size of WAL log in frames */ + int *pnCkpt /* OUT: Total number of frames checkpointed */ +){ +#ifdef SQLITE_OMIT_WAL + return SQLITE_OK; +#else + int rc; /* Return code */ + int iDb = SQLITE_MAX_ATTACHED; /* sqlite3.aDb[] index of db to checkpoint */ + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif + + /* Initialize the output variables to -1 in case an error occurs. */ + if( pnLog ) *pnLog = -1; + if( pnCkpt ) *pnCkpt = -1; + + assert( SQLITE_CHECKPOINT_PASSIVE==0 ); + assert( SQLITE_CHECKPOINT_FULL==1 ); + assert( SQLITE_CHECKPOINT_RESTART==2 ); + assert( SQLITE_CHECKPOINT_TRUNCATE==3 ); + if( eModeSQLITE_CHECKPOINT_TRUNCATE ){ + /* EVIDENCE-OF: R-03996-12088 The M parameter must be a valid checkpoint + ** mode: */ + return SQLITE_MISUSE; + } + + sqlite3_mutex_enter(db->mutex); + if( zDb && zDb[0] ){ + iDb = sqlite3FindDbName(db, zDb); + } + if( iDb<0 ){ + rc = SQLITE_ERROR; + sqlite3ErrorWithMsg(db, SQLITE_ERROR, "unknown database: %s", zDb); + }else{ + db->busyHandler.nBusy = 0; + rc = sqlite3Checkpoint(db, iDb, eMode, pnLog, pnCkpt); + sqlite3Error(db, rc); + } + rc = sqlite3ApiExit(db, rc); + + /* If there are no active statements, clear the interrupt flag at this + ** point. */ + if( db->nVdbeActive==0 ){ + AtomicStore(&db->u1.isInterrupted, 0); + } + + sqlite3_mutex_leave(db->mutex); + return rc; +#endif +} + /* -** If the following global variable points to a string which is the -** name of a directory, then that directory will be used to store -** all database files specified with a relative pathname. -** -** See also the "PRAGMA data_store_directory" SQL command. +** Checkpoint database zDb. If zDb is NULL, or if the buffer zDb points +** to contains a zero-length string, all attached databases are +** checkpointed. */ -SQLITE_API char *sqlite3_data_directory = 0; +SQLITE_API int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb){ + /* EVIDENCE-OF: R-41613-20553 The sqlite3_wal_checkpoint(D,X) is equivalent to + ** sqlite3_wal_checkpoint_v2(D,X,SQLITE_CHECKPOINT_PASSIVE,0,0). */ + return sqlite3_wal_checkpoint_v2(db,zDb,SQLITE_CHECKPOINT_PASSIVE,0,0); +} +#ifndef SQLITE_OMIT_WAL /* -** Initialize SQLite. -** -** This routine must be called to initialize the memory allocation, -** VFS, and mutex subsystems prior to doing any serious work with -** SQLite. But as long as you do not compile with SQLITE_OMIT_AUTOINIT -** this routine will be called automatically by key routines such as -** sqlite3_open(). -** -** This routine is a no-op except on its very first call for the process, -** or for the first call after a call to sqlite3_shutdown. -** -** The first thread to call this routine runs the initialization to -** completion. If subsequent threads call this routine before the first -** thread has finished the initialization process, then the subsequent -** threads must block until the first thread finishes with the initialization. +** Run a checkpoint on database iDb. This is a no-op if database iDb is +** not currently open in WAL mode. ** -** The first thread might call this routine recursively. Recursive -** calls to this routine should not block, of course. Otherwise the -** initialization process would never complete. +** If a transaction is open on the database being checkpointed, this +** function returns SQLITE_LOCKED and a checkpoint is not attempted. If +** an error occurs while running the checkpoint, an SQLite error code is +** returned (i.e. SQLITE_IOERR). Otherwise, SQLITE_OK. ** -** Let X be the first thread to enter this routine. Let Y be some other -** thread. Then while the initial invocation of this routine by X is -** incomplete, it is required that: +** The mutex on database handle db should be held by the caller. The mutex +** associated with the specific b-tree being checkpointed is taken by +** this function while the checkpoint is running. ** -** * Calls to this routine from Y must block until the outer-most -** call by X completes. +** If iDb is passed SQLITE_MAX_ATTACHED, then all attached databases are +** checkpointed. If an error is encountered it is returned immediately - +** no attempt is made to checkpoint any remaining databases. ** -** * Recursive calls to this routine from thread X return immediately -** without blocking. +** Parameter eMode is one of SQLITE_CHECKPOINT_PASSIVE, FULL, RESTART +** or TRUNCATE. */ -SQLITE_API int sqlite3_initialize(void){ - MUTEX_LOGIC( sqlite3_mutex *pMaster; ) /* The main static mutex */ - int rc; /* Result code */ -#ifdef SQLITE_EXTRA_INIT - int bRunExtraInit = 0; /* Extra initialization needed */ -#endif +SQLITE_PRIVATE int sqlite3Checkpoint(sqlite3 *db, int iDb, int eMode, int *pnLog, int *pnCkpt){ + int rc = SQLITE_OK; /* Return code */ + int i; /* Used to iterate through attached dbs */ + int bBusy = 0; /* True if SQLITE_BUSY has been encountered */ -#ifdef SQLITE_OMIT_WSD - rc = sqlite3_wsd_init(4096, 24); - if( rc!=SQLITE_OK ){ - return rc; + assert( sqlite3_mutex_held(db->mutex) ); + assert( !pnLog || *pnLog==-1 ); + assert( !pnCkpt || *pnCkpt==-1 ); + + for(i=0; inDb && rc==SQLITE_OK; i++){ + if( i==iDb || iDb==SQLITE_MAX_ATTACHED ){ + rc = sqlite3BtreeCheckpoint(db->aDb[i].pBt, eMode, pnLog, pnCkpt); + pnLog = 0; + pnCkpt = 0; + if( rc==SQLITE_BUSY ){ + bBusy = 1; + rc = SQLITE_OK; + } + } } -#endif - /* If the following assert() fails on some obscure processor/compiler - ** combination, the work-around is to set the correct pointer - ** size at compile-time using -DSQLITE_PTRSIZE=n compile-time option */ - assert( SQLITE_PTRSIZE==sizeof(char*) ); + return (rc==SQLITE_OK && bBusy) ? SQLITE_BUSY : rc; +} +#endif /* SQLITE_OMIT_WAL */ - /* If SQLite is already completely initialized, then this call - ** to sqlite3_initialize() should be a no-op. But the initialization - ** must be complete. So isInit must not be set until the very end - ** of this routine. - */ - if( sqlite3GlobalConfig.isInit ) return SQLITE_OK; +/* +** This function returns true if main-memory should be used instead of +** a temporary file for transient pager files and statement journals. +** The value returned depends on the value of db->temp_store (runtime +** parameter) and the compile time value of SQLITE_TEMP_STORE. The +** following table describes the relationship between these two values +** and this functions return value. +** +** SQLITE_TEMP_STORE db->temp_store Location of temporary database +** ----------------- -------------- ------------------------------ +** 0 any file (return 0) +** 1 1 file (return 0) +** 1 2 memory (return 1) +** 1 0 file (return 0) +** 2 1 file (return 0) +** 2 2 memory (return 1) +** 2 0 memory (return 1) +** 3 any memory (return 1) +*/ +SQLITE_PRIVATE int sqlite3TempInMemory(const sqlite3 *db){ +#if SQLITE_TEMP_STORE==1 + return ( db->temp_store==2 ); +#endif +#if SQLITE_TEMP_STORE==2 + return ( db->temp_store!=1 ); +#endif +#if SQLITE_TEMP_STORE==3 + UNUSED_PARAMETER(db); + return 1; +#endif +#if SQLITE_TEMP_STORE<1 || SQLITE_TEMP_STORE>3 + UNUSED_PARAMETER(db); + return 0; +#endif +} - /* Make sure the mutex subsystem is initialized. If unable to - ** initialize the mutex subsystem, return early with the error. - ** If the system is so sick that we are unable to allocate a mutex, - ** there is not much SQLite is going to be able to do. - ** - ** The mutex subsystem must take care of serializing its own - ** initialization. - */ - rc = sqlite3MutexInit(); - if( rc ) return rc; +/* +** Return UTF-8 encoded English language explanation of the most recent +** error. +*/ +SQLITE_API const char *sqlite3_errmsg(sqlite3 *db){ + const char *z; + if( !db ){ + return sqlite3ErrStr(SQLITE_NOMEM_BKPT); + } + if( !sqlite3SafetyCheckSickOrOk(db) ){ + return sqlite3ErrStr(SQLITE_MISUSE_BKPT); + } + sqlite3_mutex_enter(db->mutex); + if( db->mallocFailed ){ + z = sqlite3ErrStr(SQLITE_NOMEM_BKPT); + }else{ + testcase( db->pErr==0 ); + z = db->errCode ? (char*)sqlite3_value_text(db->pErr) : 0; + assert( !db->mallocFailed ); + if( z==0 ){ + z = sqlite3ErrStr(db->errCode); + } + } + sqlite3_mutex_leave(db->mutex); + return z; +} - /* Initialize the malloc() system and the recursive pInitMutex mutex. - ** This operation is protected by the STATIC_MASTER mutex. Note that - ** MutexAlloc() is called for a static mutex prior to initializing the - ** malloc subsystem - this implies that the allocation of a static - ** mutex must not require support from the malloc subsystem. - */ - MUTEX_LOGIC( pMaster = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); ) - sqlite3_mutex_enter(pMaster); - sqlite3GlobalConfig.isMutexInit = 1; - if( !sqlite3GlobalConfig.isMallocInit ){ - rc = sqlite3MallocInit(); +#ifndef SQLITE_OMIT_UTF16 +/* +** Return UTF-16 encoded English language explanation of the most recent +** error. +*/ +SQLITE_API const void *sqlite3_errmsg16(sqlite3 *db){ + static const u16 outOfMem[] = { + 'o', 'u', 't', ' ', 'o', 'f', ' ', 'm', 'e', 'm', 'o', 'r', 'y', 0 + }; + static const u16 misuse[] = { + 'b', 'a', 'd', ' ', 'p', 'a', 'r', 'a', 'm', 'e', 't', 'e', 'r', ' ', + 'o', 'r', ' ', 'o', 't', 'h', 'e', 'r', ' ', 'A', 'P', 'I', ' ', + 'm', 'i', 's', 'u', 's', 'e', 0 + }; + + const void *z; + if( !db ){ + return (void *)outOfMem; } - if( rc==SQLITE_OK ){ - sqlite3GlobalConfig.isMallocInit = 1; - if( !sqlite3GlobalConfig.pInitMutex ){ - sqlite3GlobalConfig.pInitMutex = - sqlite3MutexAlloc(SQLITE_MUTEX_RECURSIVE); - if( sqlite3GlobalConfig.bCoreMutex && !sqlite3GlobalConfig.pInitMutex ){ - rc = SQLITE_NOMEM_BKPT; - } + if( !sqlite3SafetyCheckSickOrOk(db) ){ + return (void *)misuse; + } + sqlite3_mutex_enter(db->mutex); + if( db->mallocFailed ){ + z = (void *)outOfMem; + }else{ + z = sqlite3_value_text16(db->pErr); + if( z==0 ){ + sqlite3ErrorWithMsg(db, db->errCode, sqlite3ErrStr(db->errCode)); + z = sqlite3_value_text16(db->pErr); } + /* A malloc() may have failed within the call to sqlite3_value_text16() + ** above. If this is the case, then the db->mallocFailed flag needs to + ** be cleared before returning. Do this directly, instead of via + ** sqlite3ApiExit(), to avoid setting the database handle error message. + */ + sqlite3OomClear(db); } - if( rc==SQLITE_OK ){ - sqlite3GlobalConfig.nRefInitMutex++; + sqlite3_mutex_leave(db->mutex); + return z; +} +#endif /* SQLITE_OMIT_UTF16 */ + +/* +** Return the most recent error code generated by an SQLite routine. If NULL is +** passed to this function, we assume a malloc() failed during sqlite3_open(). +*/ +SQLITE_API int sqlite3_errcode(sqlite3 *db){ + if( db && !sqlite3SafetyCheckSickOrOk(db) ){ + return SQLITE_MISUSE_BKPT; } - sqlite3_mutex_leave(pMaster); + if( !db || db->mallocFailed ){ + return SQLITE_NOMEM_BKPT; + } + return db->errCode & db->errMask; +} +SQLITE_API int sqlite3_extended_errcode(sqlite3 *db){ + if( db && !sqlite3SafetyCheckSickOrOk(db) ){ + return SQLITE_MISUSE_BKPT; + } + if( !db || db->mallocFailed ){ + return SQLITE_NOMEM_BKPT; + } + return db->errCode; +} +SQLITE_API int sqlite3_system_errno(sqlite3 *db){ + return db ? db->iSysErrno : 0; +} - /* If rc is not SQLITE_OK at this point, then either the malloc - ** subsystem could not be initialized or the system failed to allocate - ** the pInitMutex mutex. Return an error in either case. */ - if( rc!=SQLITE_OK ){ - return rc; +/* +** Return a string that describes the kind of error specified in the +** argument. For now, this simply calls the internal sqlite3ErrStr() +** function. +*/ +SQLITE_API const char *sqlite3_errstr(int rc){ + return sqlite3ErrStr(rc); +} + +/* +** Create a new collating function for database "db". The name is zName +** and the encoding is enc. +*/ +static int createCollation( + sqlite3* db, + const char *zName, + u8 enc, + void* pCtx, + int(*xCompare)(void*,int,const void*,int,const void*), + void(*xDel)(void*) +){ + CollSeq *pColl; + int enc2; + + assert( sqlite3_mutex_held(db->mutex) ); + + /* If SQLITE_UTF16 is specified as the encoding type, transform this + ** to one of SQLITE_UTF16LE or SQLITE_UTF16BE using the + ** SQLITE_UTF16NATIVE macro. SQLITE_UTF16 is not used internally. + */ + enc2 = enc; + testcase( enc2==SQLITE_UTF16 ); + testcase( enc2==SQLITE_UTF16_ALIGNED ); + if( enc2==SQLITE_UTF16 || enc2==SQLITE_UTF16_ALIGNED ){ + enc2 = SQLITE_UTF16NATIVE; + } + if( enc2SQLITE_UTF16BE ){ + return SQLITE_MISUSE_BKPT; } - /* Do the rest of the initialization under the recursive mutex so - ** that we will be able to handle recursive calls into - ** sqlite3_initialize(). The recursive calls normally come through - ** sqlite3_os_init() when it invokes sqlite3_vfs_register(), but other - ** recursive calls might also be possible. - ** - ** IMPLEMENTATION-OF: R-00140-37445 SQLite automatically serializes calls - ** to the xInit method, so the xInit method need not be threadsafe. - ** - ** The following mutex is what serializes access to the appdef pcache xInit - ** methods. The sqlite3_pcache_methods.xInit() all is embedded in the - ** call to sqlite3PcacheInitialize(). + /* Check if this call is removing or replacing an existing collation + ** sequence. If so, and there are active VMs, return busy. If there + ** are no active VMs, invalidate any pre-compiled statements. */ - sqlite3_mutex_enter(sqlite3GlobalConfig.pInitMutex); - if( sqlite3GlobalConfig.isInit==0 && sqlite3GlobalConfig.inProgress==0 ){ - sqlite3GlobalConfig.inProgress = 1; -#ifdef SQLITE_ENABLE_SQLLOG - { - extern void sqlite3_init_sqllog(void); - sqlite3_init_sqllog(); - } -#endif - memset(&sqlite3BuiltinFunctions, 0, sizeof(sqlite3BuiltinFunctions)); - sqlite3RegisterBuiltinFunctions(); - if( sqlite3GlobalConfig.isPCacheInit==0 ){ - rc = sqlite3PcacheInitialize(); - } - if( rc==SQLITE_OK ){ - sqlite3GlobalConfig.isPCacheInit = 1; - rc = sqlite3OsInit(); - } -#ifdef SQLITE_ENABLE_DESERIALIZE - if( rc==SQLITE_OK ){ - rc = sqlite3MemdbInit(); + pColl = sqlite3FindCollSeq(db, (u8)enc2, zName, 0); + if( pColl && pColl->xCmp ){ + if( db->nVdbeActive ){ + sqlite3ErrorWithMsg(db, SQLITE_BUSY, + "unable to delete/modify collation sequence due to active statements"); + return SQLITE_BUSY; } -#endif - if( rc==SQLITE_OK ){ - sqlite3PCacheBufferSetup( sqlite3GlobalConfig.pPage, - sqlite3GlobalConfig.szPage, sqlite3GlobalConfig.nPage); - sqlite3GlobalConfig.isInit = 1; -#ifdef SQLITE_EXTRA_INIT - bRunExtraInit = 1; -#endif + sqlite3ExpirePreparedStatements(db, 0); + + /* If collation sequence pColl was created directly by a call to + ** sqlite3_create_collation, and not generated by synthCollSeq(), + ** then any copies made by synthCollSeq() need to be invalidated. + ** Also, collation destructor - CollSeq.xDel() - function may need + ** to be called. + */ + if( (pColl->enc & ~SQLITE_UTF16_ALIGNED)==enc2 ){ + CollSeq *aColl = sqlite3HashFind(&db->aCollSeq, zName); + int j; + for(j=0; j<3; j++){ + CollSeq *p = &aColl[j]; + if( p->enc==pColl->enc ){ + if( p->xDel ){ + p->xDel(p->pUser); + } + p->xCmp = 0; + } + } } - sqlite3GlobalConfig.inProgress = 0; } - sqlite3_mutex_leave(sqlite3GlobalConfig.pInitMutex); - /* Go back under the static mutex and clean up the recursive - ** mutex to prevent a resource leak. - */ - sqlite3_mutex_enter(pMaster); - sqlite3GlobalConfig.nRefInitMutex--; - if( sqlite3GlobalConfig.nRefInitMutex<=0 ){ - assert( sqlite3GlobalConfig.nRefInitMutex==0 ); - sqlite3_mutex_free(sqlite3GlobalConfig.pInitMutex); - sqlite3GlobalConfig.pInitMutex = 0; - } - sqlite3_mutex_leave(pMaster); + pColl = sqlite3FindCollSeq(db, (u8)enc2, zName, 1); + if( pColl==0 ) return SQLITE_NOMEM_BKPT; + pColl->xCmp = xCompare; + pColl->pUser = pCtx; + pColl->xDel = xDel; + pColl->enc = (u8)(enc2 | (enc & SQLITE_UTF16_ALIGNED)); + sqlite3Error(db, SQLITE_OK); + return SQLITE_OK; +} - /* The following is just a sanity check to make sure SQLite has - ** been compiled correctly. It is important to run this code, but - ** we don't want to run it too often and soak up CPU cycles for no - ** reason. So we run it once during initialization. - */ -#ifndef NDEBUG -#ifndef SQLITE_OMIT_FLOATING_POINT - /* This section of code's only "output" is via assert() statements. */ - if( rc==SQLITE_OK ){ - u64 x = (((u64)1)<<63)-1; - double y; - assert(sizeof(x)==8); - assert(sizeof(x)==sizeof(y)); - memcpy(&y, &x, 8); - assert( sqlite3IsNaN(y) ); - } + +/* +** This array defines hard upper bounds on limit values. The +** initializer must be kept in sync with the SQLITE_LIMIT_* +** #defines in sqlite3.h. +*/ +static const int aHardLimit[] = { + SQLITE_MAX_LENGTH, + SQLITE_MAX_SQL_LENGTH, + SQLITE_MAX_COLUMN, + SQLITE_MAX_EXPR_DEPTH, + SQLITE_MAX_COMPOUND_SELECT, + SQLITE_MAX_VDBE_OP, + SQLITE_MAX_FUNCTION_ARG, + SQLITE_MAX_ATTACHED, + SQLITE_MAX_LIKE_PATTERN_LENGTH, + SQLITE_MAX_VARIABLE_NUMBER, /* IMP: R-38091-32352 */ + SQLITE_MAX_TRIGGER_DEPTH, + SQLITE_MAX_WORKER_THREADS, +}; + +/* +** Make sure the hard limits are set to reasonable values +*/ +#if SQLITE_MAX_LENGTH<100 +# error SQLITE_MAX_LENGTH must be at least 100 +#endif +#if SQLITE_MAX_SQL_LENGTH<100 +# error SQLITE_MAX_SQL_LENGTH must be at least 100 #endif +#if SQLITE_MAX_SQL_LENGTH>SQLITE_MAX_LENGTH +# error SQLITE_MAX_SQL_LENGTH must not be greater than SQLITE_MAX_LENGTH #endif - - /* Do extra initialization steps requested by the SQLITE_EXTRA_INIT - ** compile-time option. - */ -#ifdef SQLITE_EXTRA_INIT - if( bRunExtraInit ){ - int SQLITE_EXTRA_INIT(const char*); - rc = SQLITE_EXTRA_INIT(0); - } +#if SQLITE_MAX_COMPOUND_SELECT<2 +# error SQLITE_MAX_COMPOUND_SELECT must be at least 2 +#endif +#if SQLITE_MAX_VDBE_OP<40 +# error SQLITE_MAX_VDBE_OP must be at least 40 +#endif +#if SQLITE_MAX_FUNCTION_ARG<0 || SQLITE_MAX_FUNCTION_ARG>127 +# error SQLITE_MAX_FUNCTION_ARG must be between 0 and 127 +#endif +#if SQLITE_MAX_ATTACHED<0 || SQLITE_MAX_ATTACHED>125 +# error SQLITE_MAX_ATTACHED must be between 0 and 125 +#endif +#if SQLITE_MAX_LIKE_PATTERN_LENGTH<1 +# error SQLITE_MAX_LIKE_PATTERN_LENGTH must be at least 1 +#endif +#if SQLITE_MAX_COLUMN>32767 +# error SQLITE_MAX_COLUMN must not exceed 32767 +#endif +#if SQLITE_MAX_TRIGGER_DEPTH<1 +# error SQLITE_MAX_TRIGGER_DEPTH must be at least 1 +#endif +#if SQLITE_MAX_WORKER_THREADS<0 || SQLITE_MAX_WORKER_THREADS>50 +# error SQLITE_MAX_WORKER_THREADS must be between 0 and 50 #endif - return rc; -} /* -** Undo the effects of sqlite3_initialize(). Must not be called while -** there are outstanding database connections or memory allocations or -** while any part of SQLite is otherwise in use in any thread. This -** routine is not threadsafe. But it is safe to invoke this routine -** on when SQLite is already shut down. If SQLite is already shut down -** when this routine is invoked, then this routine is a harmless no-op. +** Change the value of a limit. Report the old value. +** If an invalid limit index is supplied, report -1. +** Make no changes but still report the old value if the +** new limit is negative. +** +** A new lower limit does not shrink existing constructs. +** It merely prevents new constructs that exceed the limit +** from forming. */ -SQLITE_API int sqlite3_shutdown(void){ -#ifdef SQLITE_OMIT_WSD - int rc = sqlite3_wsd_init(4096, 24); - if( rc!=SQLITE_OK ){ - return rc; +SQLITE_API int sqlite3_limit(sqlite3 *db, int limitId, int newLimit){ + int oldLimit; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return -1; } #endif - if( sqlite3GlobalConfig.isInit ){ -#ifdef SQLITE_EXTRA_SHUTDOWN - void SQLITE_EXTRA_SHUTDOWN(void); - SQLITE_EXTRA_SHUTDOWN(); -#endif - sqlite3_os_end(); - sqlite3_reset_auto_extension(); - sqlite3GlobalConfig.isInit = 0; - } - if( sqlite3GlobalConfig.isPCacheInit ){ - sqlite3PcacheShutdown(); - sqlite3GlobalConfig.isPCacheInit = 0; - } - if( sqlite3GlobalConfig.isMallocInit ){ - sqlite3MallocEnd(); - sqlite3GlobalConfig.isMallocInit = 0; + /* EVIDENCE-OF: R-30189-54097 For each limit category SQLITE_LIMIT_NAME + ** there is a hard upper bound set at compile-time by a C preprocessor + ** macro called SQLITE_MAX_NAME. (The "_LIMIT_" in the name is changed to + ** "_MAX_".) + */ + assert( aHardLimit[SQLITE_LIMIT_LENGTH]==SQLITE_MAX_LENGTH ); + assert( aHardLimit[SQLITE_LIMIT_SQL_LENGTH]==SQLITE_MAX_SQL_LENGTH ); + assert( aHardLimit[SQLITE_LIMIT_COLUMN]==SQLITE_MAX_COLUMN ); + assert( aHardLimit[SQLITE_LIMIT_EXPR_DEPTH]==SQLITE_MAX_EXPR_DEPTH ); + assert( aHardLimit[SQLITE_LIMIT_COMPOUND_SELECT]==SQLITE_MAX_COMPOUND_SELECT); + assert( aHardLimit[SQLITE_LIMIT_VDBE_OP]==SQLITE_MAX_VDBE_OP ); + assert( aHardLimit[SQLITE_LIMIT_FUNCTION_ARG]==SQLITE_MAX_FUNCTION_ARG ); + assert( aHardLimit[SQLITE_LIMIT_ATTACHED]==SQLITE_MAX_ATTACHED ); + assert( aHardLimit[SQLITE_LIMIT_LIKE_PATTERN_LENGTH]== + SQLITE_MAX_LIKE_PATTERN_LENGTH ); + assert( aHardLimit[SQLITE_LIMIT_VARIABLE_NUMBER]==SQLITE_MAX_VARIABLE_NUMBER); + assert( aHardLimit[SQLITE_LIMIT_TRIGGER_DEPTH]==SQLITE_MAX_TRIGGER_DEPTH ); + assert( aHardLimit[SQLITE_LIMIT_WORKER_THREADS]==SQLITE_MAX_WORKER_THREADS ); + assert( SQLITE_LIMIT_WORKER_THREADS==(SQLITE_N_LIMIT-1) ); -#ifndef SQLITE_OMIT_SHUTDOWN_DIRECTORIES - /* The heap subsystem has now been shutdown and these values are supposed - ** to be NULL or point to memory that was obtained from sqlite3_malloc(), - ** which would rely on that heap subsystem; therefore, make sure these - ** values cannot refer to heap memory that was just invalidated when the - ** heap subsystem was shutdown. This is only done if the current call to - ** this function resulted in the heap subsystem actually being shutdown. - */ - sqlite3_data_directory = 0; - sqlite3_temp_directory = 0; -#endif + + if( limitId<0 || limitId>=SQLITE_N_LIMIT ){ + return -1; } - if( sqlite3GlobalConfig.isMutexInit ){ - sqlite3MutexEnd(); - sqlite3GlobalConfig.isMutexInit = 0; + oldLimit = db->aLimit[limitId]; + if( newLimit>=0 ){ /* IMP: R-52476-28732 */ + if( newLimit>aHardLimit[limitId] ){ + newLimit = aHardLimit[limitId]; /* IMP: R-51463-25634 */ + } + db->aLimit[limitId] = newLimit; } - - return SQLITE_OK; + return oldLimit; /* IMP: R-53341-35419 */ } /* -** This API allows applications to modify the global configuration of -** the SQLite library at run-time. +** This function is used to parse both URIs and non-URI filenames passed by the +** user to API functions sqlite3_open() or sqlite3_open_v2(), and for database +** URIs specified as part of ATTACH statements. ** -** This routine should only be called when there are no outstanding -** database connections or memory allocations. This routine is not -** threadsafe. Failure to heed these warnings can lead to unpredictable -** behavior. +** The first argument to this function is the name of the VFS to use (or +** a NULL to signify the default VFS) if the URI does not contain a "vfs=xxx" +** query parameter. The second argument contains the URI (or non-URI filename) +** itself. When this function is called the *pFlags variable should contain +** the default flags to open the database handle with. The value stored in +** *pFlags may be updated before returning if the URI filename contains +** "cache=xxx" or "mode=xxx" query parameters. +** +** If successful, SQLITE_OK is returned. In this case *ppVfs is set to point to +** the VFS that should be used to open the database file. *pzFile is set to +** point to a buffer containing the name of the file to open. The value +** stored in *pzFile is a database name acceptable to sqlite3_uri_parameter() +** and is in the same format as names created using sqlite3_create_filename(). +** The caller must invoke sqlite3_free_filename() (not sqlite3_free()!) on +** the value returned in *pzFile to avoid a memory leak. +** +** If an error occurs, then an SQLite error code is returned and *pzErrMsg +** may be set to point to a buffer containing an English language error +** message. It is the responsibility of the caller to eventually release +** this buffer by calling sqlite3_free(). */ -SQLITE_API int sqlite3_config(int op, ...){ - va_list ap; +SQLITE_PRIVATE int sqlite3ParseUri( + const char *zDefaultVfs, /* VFS to use if no "vfs=xxx" query option */ + const char *zUri, /* Nul-terminated URI to parse */ + unsigned int *pFlags, /* IN/OUT: SQLITE_OPEN_XXX flags */ + sqlite3_vfs **ppVfs, /* OUT: VFS to use */ + char **pzFile, /* OUT: Filename component of URI */ + char **pzErrMsg /* OUT: Error message (if rc!=SQLITE_OK) */ +){ int rc = SQLITE_OK; + unsigned int flags = *pFlags; + const char *zVfs = zDefaultVfs; + char *zFile; + char c; + int nUri = sqlite3Strlen30(zUri); - /* sqlite3_config() shall return SQLITE_MISUSE if it is invoked while - ** the SQLite library is in use. */ - if( sqlite3GlobalConfig.isInit ) return SQLITE_MISUSE_BKPT; + assert( *pzErrMsg==0 ); - va_start(ap, op); - switch( op ){ + if( ((flags & SQLITE_OPEN_URI) /* IMP: R-48725-32206 */ + || sqlite3GlobalConfig.bOpenUri) /* IMP: R-51689-46548 */ + && nUri>=5 && memcmp(zUri, "file:", 5)==0 /* IMP: R-57884-37496 */ + ){ + char *zOpt; + int eState; /* Parser state when parsing URI */ + int iIn; /* Input character index */ + int iOut = 0; /* Output character index */ + u64 nByte = nUri+8; /* Bytes of space to allocate */ - /* Mutex configuration options are only available in a threadsafe - ** compile. - */ -#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE>0 /* IMP: R-54466-46756 */ - case SQLITE_CONFIG_SINGLETHREAD: { - /* EVIDENCE-OF: R-02748-19096 This option sets the threading mode to - ** Single-thread. */ - sqlite3GlobalConfig.bCoreMutex = 0; /* Disable mutex on core */ - sqlite3GlobalConfig.bFullMutex = 0; /* Disable mutex on connections */ - break; - } -#endif -#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE>0 /* IMP: R-20520-54086 */ - case SQLITE_CONFIG_MULTITHREAD: { - /* EVIDENCE-OF: R-14374-42468 This option sets the threading mode to - ** Multi-thread. */ - sqlite3GlobalConfig.bCoreMutex = 1; /* Enable mutex on core */ - sqlite3GlobalConfig.bFullMutex = 0; /* Disable mutex on connections */ - break; - } -#endif -#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE>0 /* IMP: R-59593-21810 */ - case SQLITE_CONFIG_SERIALIZED: { - /* EVIDENCE-OF: R-41220-51800 This option sets the threading mode to - ** Serialized. */ - sqlite3GlobalConfig.bCoreMutex = 1; /* Enable mutex on core */ - sqlite3GlobalConfig.bFullMutex = 1; /* Enable mutex on connections */ - break; - } -#endif -#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE>0 /* IMP: R-63666-48755 */ - case SQLITE_CONFIG_MUTEX: { - /* Specify an alternative mutex implementation */ - sqlite3GlobalConfig.mutex = *va_arg(ap, sqlite3_mutex_methods*); - break; - } -#endif -#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE>0 /* IMP: R-14450-37597 */ - case SQLITE_CONFIG_GETMUTEX: { - /* Retrieve the current mutex implementation */ - *va_arg(ap, sqlite3_mutex_methods*) = sqlite3GlobalConfig.mutex; - break; - } -#endif + /* Make sure the SQLITE_OPEN_URI flag is set to indicate to the VFS xOpen + ** method that there may be extra parameters following the file-name. */ + flags |= SQLITE_OPEN_URI; - case SQLITE_CONFIG_MALLOC: { - /* EVIDENCE-OF: R-55594-21030 The SQLITE_CONFIG_MALLOC option takes a - ** single argument which is a pointer to an instance of the - ** sqlite3_mem_methods structure. The argument specifies alternative - ** low-level memory allocation routines to be used in place of the memory - ** allocation routines built into SQLite. */ - sqlite3GlobalConfig.m = *va_arg(ap, sqlite3_mem_methods*); - break; - } - case SQLITE_CONFIG_GETMALLOC: { - /* EVIDENCE-OF: R-51213-46414 The SQLITE_CONFIG_GETMALLOC option takes a - ** single argument which is a pointer to an instance of the - ** sqlite3_mem_methods structure. The sqlite3_mem_methods structure is - ** filled with the currently defined memory allocation routines. */ - if( sqlite3GlobalConfig.m.xMalloc==0 ) sqlite3MemSetDefault(); - *va_arg(ap, sqlite3_mem_methods*) = sqlite3GlobalConfig.m; - break; - } - case SQLITE_CONFIG_MEMSTATUS: { - /* EVIDENCE-OF: R-61275-35157 The SQLITE_CONFIG_MEMSTATUS option takes - ** single argument of type int, interpreted as a boolean, which enables - ** or disables the collection of memory allocation statistics. */ - sqlite3GlobalConfig.bMemstat = va_arg(ap, int); - break; - } - case SQLITE_CONFIG_SMALL_MALLOC: { - sqlite3GlobalConfig.bSmallMalloc = va_arg(ap, int); - break; - } - case SQLITE_CONFIG_PAGECACHE: { - /* EVIDENCE-OF: R-18761-36601 There are three arguments to - ** SQLITE_CONFIG_PAGECACHE: A pointer to 8-byte aligned memory (pMem), - ** the size of each page cache line (sz), and the number of cache lines - ** (N). */ - sqlite3GlobalConfig.pPage = va_arg(ap, void*); - sqlite3GlobalConfig.szPage = va_arg(ap, int); - sqlite3GlobalConfig.nPage = va_arg(ap, int); - break; - } - case SQLITE_CONFIG_PCACHE_HDRSZ: { - /* EVIDENCE-OF: R-39100-27317 The SQLITE_CONFIG_PCACHE_HDRSZ option takes - ** a single parameter which is a pointer to an integer and writes into - ** that integer the number of extra bytes per page required for each page - ** in SQLITE_CONFIG_PAGECACHE. */ - *va_arg(ap, int*) = - sqlite3HeaderSizeBtree() + - sqlite3HeaderSizePcache() + - sqlite3HeaderSizePcache1(); - break; - } + for(iIn=0; iIn(1<<12) ){ - /* cap min request size at 2^12 */ - sqlite3GlobalConfig.mnReq = (1<<12); - } + /* Copy the filename and any query parameters into the zFile buffer. + ** Decode %HH escape codes along the way. + ** + ** Within this loop, variable eState may be set to 0, 1 or 2, depending + ** on the parsing context. As follows: + ** + ** 0: Parsing file-name. + ** 1: Parsing name section of a name=value query parameter. + ** 2: Parsing value section of a name=value query parameter. + */ + eState = 0; + while( (c = zUri[iIn])!=0 && c!='#' ){ + iIn++; + if( c=='%' + && sqlite3Isxdigit(zUri[iIn]) + && sqlite3Isxdigit(zUri[iIn+1]) + ){ + int octet = (sqlite3HexToInt(zUri[iIn++]) << 4); + octet += sqlite3HexToInt(zUri[iIn++]); - if( sqlite3GlobalConfig.pHeap==0 ){ - /* EVIDENCE-OF: R-49920-60189 If the first pointer (the memory pointer) - ** is NULL, then SQLite reverts to using its default memory allocator - ** (the system malloc() implementation), undoing any prior invocation of - ** SQLITE_CONFIG_MALLOC. - ** - ** Setting sqlite3GlobalConfig.m to all zeros will cause malloc to - ** revert to its default implementation when sqlite3_initialize() is run - */ - memset(&sqlite3GlobalConfig.m, 0, sizeof(sqlite3GlobalConfig.m)); - }else{ - /* EVIDENCE-OF: R-61006-08918 If the memory pointer is not NULL then the - ** alternative memory allocator is engaged to handle all of SQLites - ** memory allocation needs. */ -#ifdef SQLITE_ENABLE_MEMSYS3 - sqlite3GlobalConfig.m = *sqlite3MemGetMemsys3(); -#endif -#ifdef SQLITE_ENABLE_MEMSYS5 - sqlite3GlobalConfig.m = *sqlite3MemGetMemsys5(); + assert( octet>=0 && octet<256 ); + if( octet==0 ){ +#ifndef SQLITE_ENABLE_URI_00_ERROR + /* This branch is taken when "%00" appears within the URI. In this + ** case we ignore all text in the remainder of the path, name or + ** value currently being parsed. So ignore the current character + ** and skip to the next "?", "=" or "&", as appropriate. */ + while( (c = zUri[iIn])!=0 && c!='#' + && (eState!=0 || c!='?') + && (eState!=1 || (c!='=' && c!='&')) + && (eState!=2 || c!='&') + ){ + iIn++; + } + continue; +#else + /* If ENABLE_URI_00_ERROR is defined, "%00" in a URI is an error. */ + *pzErrMsg = sqlite3_mprintf("unexpected %%00 in uri"); + rc = SQLITE_ERROR; + goto parse_uri_out; #endif + } + c = octet; + }else if( eState==1 && (c=='&' || c=='=') ){ + if( zFile[iOut-1]==0 ){ + /* An empty option name. Ignore this option altogether. */ + while( zUri[iIn] && zUri[iIn]!='#' && zUri[iIn-1]!='&' ) iIn++; + continue; + } + if( c=='&' ){ + zFile[iOut++] = '\0'; + }else{ + eState = 2; + } + c = 0; + }else if( (eState==0 && c=='?') || (eState==2 && c=='&') ){ + c = 0; + eState = 1; } - break; - } -#endif - - case SQLITE_CONFIG_LOOKASIDE: { - sqlite3GlobalConfig.szLookaside = va_arg(ap, int); - sqlite3GlobalConfig.nLookaside = va_arg(ap, int); - break; - } - - /* Record a pointer to the logger function and its first argument. - ** The default is NULL. Logging is disabled if the function pointer is - ** NULL. - */ - case SQLITE_CONFIG_LOG: { - /* MSVC is picky about pulling func ptrs from va lists. - ** http://support.microsoft.com/kb/47961 - ** sqlite3GlobalConfig.xLog = va_arg(ap, void(*)(void*,int,const char*)); - */ - typedef void(*LOGFUNC_t)(void*,int,const char*); - sqlite3GlobalConfig.xLog = va_arg(ap, LOGFUNC_t); - sqlite3GlobalConfig.pLogArg = va_arg(ap, void*); - break; + zFile[iOut++] = c; } + if( eState==1 ) zFile[iOut++] = '\0'; + memset(zFile+iOut, 0, 4); /* end-of-options + empty journal filenames */ - /* EVIDENCE-OF: R-55548-33817 The compile-time setting for URI filenames - ** can be changed at start-time using the - ** sqlite3_config(SQLITE_CONFIG_URI,1) or - ** sqlite3_config(SQLITE_CONFIG_URI,0) configuration calls. - */ - case SQLITE_CONFIG_URI: { - /* EVIDENCE-OF: R-25451-61125 The SQLITE_CONFIG_URI option takes a single - ** argument of type int. If non-zero, then URI handling is globally - ** enabled. If the parameter is zero, then URI handling is globally - ** disabled. */ - sqlite3GlobalConfig.bOpenUri = va_arg(ap, int); - break; - } + /* Check if there were any options specified that should be interpreted + ** here. Options that are interpreted here include "vfs" and those that + ** correspond to flags that may be passed to the sqlite3_open_v2() + ** method. */ + zOpt = &zFile[sqlite3Strlen30(zFile)+1]; + while( zOpt[0] ){ + int nOpt = sqlite3Strlen30(zOpt); + char *zVal = &zOpt[nOpt+1]; + int nVal = sqlite3Strlen30(zVal); - case SQLITE_CONFIG_COVERING_INDEX_SCAN: { - /* EVIDENCE-OF: R-36592-02772 The SQLITE_CONFIG_COVERING_INDEX_SCAN - ** option takes a single integer argument which is interpreted as a - ** boolean in order to enable or disable the use of covering indices for - ** full table scans in the query optimizer. */ - sqlite3GlobalConfig.bUseCis = va_arg(ap, int); - break; - } + if( nOpt==3 && memcmp("vfs", zOpt, 3)==0 ){ + zVfs = zVal; + }else{ + struct OpenMode { + const char *z; + int mode; + } *aMode = 0; + char *zModeType = 0; + int mask = 0; + int limit = 0; -#ifdef SQLITE_ENABLE_SQLLOG - case SQLITE_CONFIG_SQLLOG: { - typedef void(*SQLLOGFUNC_t)(void*, sqlite3*, const char*, int); - sqlite3GlobalConfig.xSqllog = va_arg(ap, SQLLOGFUNC_t); - sqlite3GlobalConfig.pSqllogArg = va_arg(ap, void *); - break; - } -#endif + if( nOpt==5 && memcmp("cache", zOpt, 5)==0 ){ + static struct OpenMode aCacheMode[] = { + { "shared", SQLITE_OPEN_SHAREDCACHE }, + { "private", SQLITE_OPEN_PRIVATECACHE }, + { 0, 0 } + }; - case SQLITE_CONFIG_MMAP_SIZE: { - /* EVIDENCE-OF: R-58063-38258 SQLITE_CONFIG_MMAP_SIZE takes two 64-bit - ** integer (sqlite3_int64) values that are the default mmap size limit - ** (the default setting for PRAGMA mmap_size) and the maximum allowed - ** mmap size limit. */ - sqlite3_int64 szMmap = va_arg(ap, sqlite3_int64); - sqlite3_int64 mxMmap = va_arg(ap, sqlite3_int64); - /* EVIDENCE-OF: R-53367-43190 If either argument to this option is - ** negative, then that argument is changed to its compile-time default. - ** - ** EVIDENCE-OF: R-34993-45031 The maximum allowed mmap size will be - ** silently truncated if necessary so that it does not exceed the - ** compile-time maximum mmap size set by the SQLITE_MAX_MMAP_SIZE - ** compile-time option. - */ - if( mxMmap<0 || mxMmap>SQLITE_MAX_MMAP_SIZE ){ - mxMmap = SQLITE_MAX_MMAP_SIZE; - } - if( szMmap<0 ) szMmap = SQLITE_DEFAULT_MMAP_SIZE; - if( szMmap>mxMmap) szMmap = mxMmap; - sqlite3GlobalConfig.mxMmap = mxMmap; - sqlite3GlobalConfig.szMmap = szMmap; - break; - } + mask = SQLITE_OPEN_SHAREDCACHE|SQLITE_OPEN_PRIVATECACHE; + aMode = aCacheMode; + limit = mask; + zModeType = "cache"; + } + if( nOpt==4 && memcmp("mode", zOpt, 4)==0 ){ + static struct OpenMode aOpenMode[] = { + { "ro", SQLITE_OPEN_READONLY }, + { "rw", SQLITE_OPEN_READWRITE }, + { "rwc", SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE }, + { "memory", SQLITE_OPEN_MEMORY }, + { 0, 0 } + }; -#if SQLITE_OS_WIN && defined(SQLITE_WIN32_MALLOC) /* IMP: R-04780-55815 */ - case SQLITE_CONFIG_WIN32_HEAPSIZE: { - /* EVIDENCE-OF: R-34926-03360 SQLITE_CONFIG_WIN32_HEAPSIZE takes a 32-bit - ** unsigned integer value that specifies the maximum size of the created - ** heap. */ - sqlite3GlobalConfig.nHeap = va_arg(ap, int); - break; - } -#endif + mask = SQLITE_OPEN_READONLY | SQLITE_OPEN_READWRITE + | SQLITE_OPEN_CREATE | SQLITE_OPEN_MEMORY; + aMode = aOpenMode; + limit = mask & flags; + zModeType = "access"; + } - case SQLITE_CONFIG_PMASZ: { - sqlite3GlobalConfig.szPma = va_arg(ap, unsigned int); - break; - } + if( aMode ){ + int i; + int mode = 0; + for(i=0; aMode[i].z; i++){ + const char *z = aMode[i].z; + if( nVal==sqlite3Strlen30(z) && 0==memcmp(zVal, z, nVal) ){ + mode = aMode[i].mode; + break; + } + } + if( mode==0 ){ + *pzErrMsg = sqlite3_mprintf("no such %s mode: %s", zModeType, zVal); + rc = SQLITE_ERROR; + goto parse_uri_out; + } + if( (mode & ~SQLITE_OPEN_MEMORY)>limit ){ + *pzErrMsg = sqlite3_mprintf("%s mode not allowed: %s", + zModeType, zVal); + rc = SQLITE_PERM; + goto parse_uri_out; + } + flags = (flags & ~mask) | mode; + } + } - case SQLITE_CONFIG_STMTJRNL_SPILL: { - sqlite3GlobalConfig.nStmtSpill = va_arg(ap, int); - break; + zOpt = &zVal[nVal+1]; } -#ifdef SQLITE_ENABLE_SORTER_REFERENCES - case SQLITE_CONFIG_SORTERREF_SIZE: { - int iVal = va_arg(ap, int); - if( iVal<0 ){ - iVal = SQLITE_DEFAULT_SORTERREF_SIZE; - } - sqlite3GlobalConfig.szSorterRef = (u32)iVal; - break; + }else{ + zFile = sqlite3_malloc64(nUri+8); + if( !zFile ) return SQLITE_NOMEM_BKPT; + memset(zFile, 0, 4); + zFile += 4; + if( nUri ){ + memcpy(zFile, zUri, nUri); } -#endif /* SQLITE_ENABLE_SORTER_REFERENCES */ + memset(zFile+nUri, 0, 4); + flags &= ~SQLITE_OPEN_URI; + } - default: { - rc = SQLITE_ERROR; - break; - } + *ppVfs = sqlite3_vfs_find(zVfs); + if( *ppVfs==0 ){ + *pzErrMsg = sqlite3_mprintf("no such vfs: %s", zVfs); + rc = SQLITE_ERROR; } - va_end(ap); + parse_uri_out: + if( rc!=SQLITE_OK ){ + sqlite3_free_filename(zFile); + zFile = 0; + } + *pFlags = flags; + *pzFile = zFile; return rc; } /* -** Set up the lookaside buffers for a database connection. -** Return SQLITE_OK on success. -** If lookaside is already active, return SQLITE_BUSY. -** -** The sz parameter is the number of bytes in each lookaside slot. -** The cnt parameter is the number of slots. If pStart is NULL the -** space for the lookaside memory is obtained from sqlite3_malloc(). -** If pStart is not NULL then it is sz*cnt bytes of memory to use for -** the lookaside memory. +** This routine does the core work of extracting URI parameters from a +** database filename for the sqlite3_uri_parameter() interface. */ -static int setupLookaside(sqlite3 *db, void *pBuf, int sz, int cnt){ -#ifndef SQLITE_OMIT_LOOKASIDE - void *pStart; - - if( sqlite3LookasideUsed(db,0)>0 ){ - return SQLITE_BUSY; - } - /* Free any existing lookaside buffer for this handle before - ** allocating a new one so we don't have to have space for - ** both at the same time. - */ - if( db->lookaside.bMalloced ){ - sqlite3_free(db->lookaside.pStart); - } - /* The size of a lookaside slot after ROUNDDOWN8 needs to be larger - ** than a pointer to be useful. - */ - sz = ROUNDDOWN8(sz); /* IMP: R-33038-09382 */ - if( sz<=(int)sizeof(LookasideSlot*) ) sz = 0; - if( cnt<0 ) cnt = 0; - if( sz==0 || cnt==0 ){ - sz = 0; - pStart = 0; - }else if( pBuf==0 ){ - sqlite3BeginBenignMalloc(); - pStart = sqlite3Malloc( sz*cnt ); /* IMP: R-61949-35727 */ - sqlite3EndBenignMalloc(); - if( pStart ) cnt = sqlite3MallocSize(pStart)/sz; - }else{ - pStart = pBuf; - } - db->lookaside.pStart = pStart; - db->lookaside.pInit = 0; - db->lookaside.pFree = 0; - db->lookaside.sz = (u16)sz; - if( pStart ){ - int i; - LookasideSlot *p; - assert( sz > (int)sizeof(LookasideSlot*) ); - db->lookaside.nSlot = cnt; - p = (LookasideSlot*)pStart; - for(i=cnt-1; i>=0; i--){ - p->pNext = db->lookaside.pInit; - db->lookaside.pInit = p; - p = (LookasideSlot*)&((u8*)p)[sz]; - } - db->lookaside.pEnd = p; - db->lookaside.bDisable = 0; - db->lookaside.bMalloced = pBuf==0 ?1:0; - }else{ - db->lookaside.pStart = db; - db->lookaside.pEnd = db; - db->lookaside.bDisable = 1; - db->lookaside.bMalloced = 0; - db->lookaside.nSlot = 0; +static const char *uriParameter(const char *zFilename, const char *zParam){ + zFilename += sqlite3Strlen30(zFilename) + 1; + while( zFilename[0] ){ + int x = strcmp(zFilename, zParam); + zFilename += sqlite3Strlen30(zFilename) + 1; + if( x==0 ) return zFilename; + zFilename += sqlite3Strlen30(zFilename) + 1; } -#endif /* SQLITE_OMIT_LOOKASIDE */ - return SQLITE_OK; + return 0; } + + /* -** Return the mutex associated with a database connection. +** This routine does the work of opening a database on behalf of +** sqlite3_open() and sqlite3_open16(). The database filename "zFilename" +** is UTF-8 encoded. */ -SQLITE_API sqlite3_mutex *sqlite3_db_mutex(sqlite3 *db){ +static int openDatabase( + const char *zFilename, /* Database filename UTF-8 encoded */ + sqlite3 **ppDb, /* OUT: Returned database handle */ + unsigned int flags, /* Operational flags */ + const char *zVfs /* Name of the VFS to use */ +){ + sqlite3 *db; /* Store allocated handle here */ + int rc; /* Return code */ + int isThreadsafe; /* True for threadsafe connections */ + char *zOpen = 0; /* Filename argument to pass to BtreeOpen() */ + char *zErrMsg = 0; /* Error message from sqlite3ParseUri() */ + int i; /* Loop counter */ + #ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) ){ - (void)SQLITE_MISUSE_BKPT; - return 0; - } + if( ppDb==0 ) return SQLITE_MISUSE_BKPT; +#endif + *ppDb = 0; +#ifndef SQLITE_OMIT_AUTOINIT + rc = sqlite3_initialize(); + if( rc ) return rc; #endif - return db->mutex; -} -/* -** Free up as much memory as we can from the given database -** connection. -*/ -SQLITE_API int sqlite3_db_release_memory(sqlite3 *db){ - int i; + if( sqlite3GlobalConfig.bCoreMutex==0 ){ + isThreadsafe = 0; + }else if( flags & SQLITE_OPEN_NOMUTEX ){ + isThreadsafe = 0; + }else if( flags & SQLITE_OPEN_FULLMUTEX ){ + isThreadsafe = 1; + }else{ + isThreadsafe = sqlite3GlobalConfig.bFullMutex; + } -#ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; + if( flags & SQLITE_OPEN_PRIVATECACHE ){ + flags &= ~SQLITE_OPEN_SHAREDCACHE; + }else if( sqlite3GlobalConfig.sharedCacheEnabled ){ + flags |= SQLITE_OPEN_SHAREDCACHE; + } + + /* Remove harmful bits from the flags parameter + ** + ** The SQLITE_OPEN_NOMUTEX and SQLITE_OPEN_FULLMUTEX flags were + ** dealt with in the previous code block. Besides these, the only + ** valid input flags for sqlite3_open_v2() are SQLITE_OPEN_READONLY, + ** SQLITE_OPEN_READWRITE, SQLITE_OPEN_CREATE, SQLITE_OPEN_SHAREDCACHE, + ** SQLITE_OPEN_PRIVATECACHE, and some reserved bits. Silently mask + ** off all other flags. + */ + flags &= ~( SQLITE_OPEN_DELETEONCLOSE | + SQLITE_OPEN_EXCLUSIVE | + SQLITE_OPEN_MAIN_DB | + SQLITE_OPEN_TEMP_DB | + SQLITE_OPEN_TRANSIENT_DB | + SQLITE_OPEN_MAIN_JOURNAL | + SQLITE_OPEN_TEMP_JOURNAL | + SQLITE_OPEN_SUBJOURNAL | + SQLITE_OPEN_MASTER_JOURNAL | + SQLITE_OPEN_NOMUTEX | + SQLITE_OPEN_FULLMUTEX | + SQLITE_OPEN_WAL + ); + + /* Allocate the sqlite data structure */ + db = sqlite3MallocZero( sizeof(sqlite3) ); + if( db==0 ) goto opendb_out; + if( isThreadsafe +#ifdef SQLITE_ENABLE_MULTITHREADED_CHECKS + || sqlite3GlobalConfig.bCoreMutex #endif - sqlite3_mutex_enter(db->mutex); - sqlite3BtreeEnterAll(db); - for(i=0; inDb; i++){ - Btree *pBt = db->aDb[i].pBt; - if( pBt ){ - Pager *pPager = sqlite3BtreePager(pBt); - sqlite3PagerShrink(pPager); + ){ + db->mutex = sqlite3MutexAlloc(SQLITE_MUTEX_RECURSIVE); + if( db->mutex==0 ){ + sqlite3_free(db); + db = 0; + goto opendb_out; + } + if( isThreadsafe==0 ){ + sqlite3MutexWarnOnContention(db->mutex); } } - sqlite3BtreeLeaveAll(db); - sqlite3_mutex_leave(db->mutex); - return SQLITE_OK; -} + sqlite3_mutex_enter(db->mutex); + db->errMask = 0xff; + db->nDb = 2; + db->magic = SQLITE_MAGIC_BUSY; + db->aDb = db->aDbStatic; + db->lookaside.bDisable = 1; + db->lookaside.sz = 0; -/* -** Flush any dirty pages in the pager-cache for any attached database -** to disk. + assert( sizeof(db->aLimit)==sizeof(aHardLimit) ); + memcpy(db->aLimit, aHardLimit, sizeof(db->aLimit)); + db->aLimit[SQLITE_LIMIT_WORKER_THREADS] = SQLITE_DEFAULT_WORKER_THREADS; + db->autoCommit = 1; + db->nextAutovac = -1; + db->szMmap = sqlite3GlobalConfig.szMmap; + db->nextPagesize = 0; + db->nMaxSorterMmap = 0x7FFFFFFF; + db->flags |= SQLITE_ShortColNames + | SQLITE_EnableTrigger + | SQLITE_EnableView + | SQLITE_CacheSpill +#if !defined(SQLITE_TRUSTED_SCHEMA) || SQLITE_TRUSTED_SCHEMA+0!=0 + | SQLITE_TrustedSchema +#endif +/* The SQLITE_DQS compile-time option determines the default settings +** for SQLITE_DBCONFIG_DQS_DDL and SQLITE_DBCONFIG_DQS_DML. +** +** SQLITE_DQS SQLITE_DBCONFIG_DQS_DDL SQLITE_DBCONFIG_DQS_DML +** ---------- ----------------------- ----------------------- +** undefined on on +** 3 on on +** 2 on off +** 1 off on +** 0 off off +** +** Legacy behavior is 3 (double-quoted string literals are allowed anywhere) +** and so that is the default. But developers are encouranged to use +** -DSQLITE_DQS=0 (best) or -DSQLITE_DQS=1 (second choice) if possible. */ -SQLITE_API int sqlite3_db_cacheflush(sqlite3 *db){ - int i; - int rc = SQLITE_OK; - int bSeenBusy = 0; +#if !defined(SQLITE_DQS) +# define SQLITE_DQS 3 +#endif +#if (SQLITE_DQS&1)==1 + | SQLITE_DqsDML +#endif +#if (SQLITE_DQS&2)==2 + | SQLITE_DqsDDL +#endif -#ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#if !defined(SQLITE_DEFAULT_AUTOMATIC_INDEX) || SQLITE_DEFAULT_AUTOMATIC_INDEX + | SQLITE_AutoIndex #endif - sqlite3_mutex_enter(db->mutex); - sqlite3BtreeEnterAll(db); - for(i=0; rc==SQLITE_OK && inDb; i++){ - Btree *pBt = db->aDb[i].pBt; - if( pBt && sqlite3BtreeIsInTrans(pBt) ){ - Pager *pPager = sqlite3BtreePager(pBt); - rc = sqlite3PagerFlush(pPager); - if( rc==SQLITE_BUSY ){ - bSeenBusy = 1; - rc = SQLITE_OK; - } - } +#if SQLITE_DEFAULT_CKPTFULLFSYNC + | SQLITE_CkptFullFSync +#endif +#if SQLITE_DEFAULT_FILE_FORMAT<4 + | SQLITE_LegacyFileFmt +#endif +#ifdef SQLITE_ENABLE_LOAD_EXTENSION + | SQLITE_LoadExtension +#endif +#if SQLITE_DEFAULT_RECURSIVE_TRIGGERS + | SQLITE_RecTriggers +#endif +#if defined(SQLITE_DEFAULT_FOREIGN_KEYS) && SQLITE_DEFAULT_FOREIGN_KEYS + | SQLITE_ForeignKeys +#endif +#if defined(SQLITE_REVERSE_UNORDERED_SELECTS) + | SQLITE_ReverseOrder +#endif +#if defined(SQLITE_ENABLE_OVERSIZE_CELL_CHECK) + | SQLITE_CellSizeCk +#endif +#if defined(SQLITE_ENABLE_FTS3_TOKENIZER) + | SQLITE_Fts3Tokenizer +#endif +#if defined(SQLITE_ENABLE_QPSG) + | SQLITE_EnableQPSG +#endif +#if defined(SQLITE_DEFAULT_DEFENSIVE) + | SQLITE_Defensive +#endif +#if defined(SQLITE_DEFAULT_LEGACY_ALTER_TABLE) + | SQLITE_LegacyAlter +#endif + ; + sqlite3HashInit(&db->aCollSeq); +#ifndef SQLITE_OMIT_VIRTUALTABLE + sqlite3HashInit(&db->aModule); +#endif + + /* Add the default collation sequence BINARY. BINARY works for both UTF-8 + ** and UTF-16, so add a version for each to avoid any unnecessary + ** conversions. The only error that can occur here is a malloc() failure. + ** + ** EVIDENCE-OF: R-52786-44878 SQLite defines three built-in collating + ** functions: + */ + createCollation(db, sqlite3StrBINARY, SQLITE_UTF8, 0, binCollFunc, 0); + createCollation(db, sqlite3StrBINARY, SQLITE_UTF16BE, 0, binCollFunc, 0); + createCollation(db, sqlite3StrBINARY, SQLITE_UTF16LE, 0, binCollFunc, 0); + createCollation(db, "NOCASE", SQLITE_UTF8, 0, nocaseCollatingFunc, 0); + createCollation(db, "RTRIM", SQLITE_UTF8, 0, rtrimCollFunc, 0); + if( db->mallocFailed ){ + goto opendb_out; } - sqlite3BtreeLeaveAll(db); - sqlite3_mutex_leave(db->mutex); - return ((rc==SQLITE_OK && bSeenBusy) ? SQLITE_BUSY : rc); -} -/* -** Configuration settings for an individual database connection -*/ -SQLITE_API int sqlite3_db_config(sqlite3 *db, int op, ...){ - va_list ap; - int rc; - va_start(ap, op); - switch( op ){ - case SQLITE_DBCONFIG_MAINDBNAME: { - /* IMP: R-06824-28531 */ - /* IMP: R-36257-52125 */ - db->aDb[0].zDbSName = va_arg(ap,char*); - rc = SQLITE_OK; - break; - } - case SQLITE_DBCONFIG_LOOKASIDE: { - void *pBuf = va_arg(ap, void*); /* IMP: R-26835-10964 */ - int sz = va_arg(ap, int); /* IMP: R-47871-25994 */ - int cnt = va_arg(ap, int); /* IMP: R-04460-53386 */ - rc = setupLookaside(db, pBuf, sz, cnt); - break; + /* Parse the filename/URI argument + ** + ** Only allow sensible combinations of bits in the flags argument. + ** Throw an error if any non-sense combination is used. If we + ** do not block illegal combinations here, it could trigger + ** assert() statements in deeper layers. Sensible combinations + ** are: + ** + ** 1: SQLITE_OPEN_READONLY + ** 2: SQLITE_OPEN_READWRITE + ** 6: SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE + */ + db->openFlags = flags; + assert( SQLITE_OPEN_READONLY == 0x01 ); + assert( SQLITE_OPEN_READWRITE == 0x02 ); + assert( SQLITE_OPEN_CREATE == 0x04 ); + testcase( (1<<(flags&7))==0x02 ); /* READONLY */ + testcase( (1<<(flags&7))==0x04 ); /* READWRITE */ + testcase( (1<<(flags&7))==0x40 ); /* READWRITE | CREATE */ + if( ((1<<(flags&7)) & 0x46)==0 ){ + rc = SQLITE_MISUSE_BKPT; /* IMP: R-18321-05872 */ + }else{ + rc = sqlite3ParseUri(zVfs, zFilename, &flags, &db->pVfs, &zOpen, &zErrMsg); + } + if( rc!=SQLITE_OK ){ + if( rc==SQLITE_NOMEM ) sqlite3OomFault(db); + sqlite3ErrorWithMsg(db, rc, zErrMsg ? "%s" : 0, zErrMsg); + sqlite3_free(zErrMsg); + goto opendb_out; + } + + /* Open the backend database driver */ + rc = sqlite3BtreeOpen(db->pVfs, zOpen, db, &db->aDb[0].pBt, 0, + flags | SQLITE_OPEN_MAIN_DB); + if( rc!=SQLITE_OK ){ + if( rc==SQLITE_IOERR_NOMEM ){ + rc = SQLITE_NOMEM_BKPT; } - default: { - static const struct { - int op; /* The opcode */ - u32 mask; /* Mask of the bit in sqlite3.flags to set/clear */ - } aFlagOp[] = { - { SQLITE_DBCONFIG_ENABLE_FKEY, SQLITE_ForeignKeys }, - { SQLITE_DBCONFIG_ENABLE_TRIGGER, SQLITE_EnableTrigger }, - { SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER, SQLITE_Fts3Tokenizer }, - { SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION, SQLITE_LoadExtension }, - { SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE, SQLITE_NoCkptOnClose }, - { SQLITE_DBCONFIG_ENABLE_QPSG, SQLITE_EnableQPSG }, - { SQLITE_DBCONFIG_TRIGGER_EQP, SQLITE_TriggerEQP }, - { SQLITE_DBCONFIG_RESET_DATABASE, SQLITE_ResetDatabase }, - }; - unsigned int i; - rc = SQLITE_ERROR; /* IMP: R-42790-23372 */ - for(i=0; iflags; - if( onoff>0 ){ - db->flags |= aFlagOp[i].mask; - }else if( onoff==0 ){ - db->flags &= ~aFlagOp[i].mask; - } - if( oldFlags!=db->flags ){ - sqlite3ExpirePreparedStatements(db); - } - if( pRes ){ - *pRes = (db->flags & aFlagOp[i].mask)!=0; - } - rc = SQLITE_OK; - break; - } - } - break; + sqlite3Error(db, rc); + goto opendb_out; + } + sqlite3BtreeEnter(db->aDb[0].pBt); + db->aDb[0].pSchema = sqlite3SchemaGet(db, db->aDb[0].pBt); + if( !db->mallocFailed ){ + sqlite3SetTextEncoding(db, SCHEMA_ENC(db)); + } + sqlite3BtreeLeave(db->aDb[0].pBt); + db->aDb[1].pSchema = sqlite3SchemaGet(db, 0); + + /* The default safety_level for the main database is FULL; for the temp + ** database it is OFF. This matches the pager layer defaults. + */ + db->aDb[0].zDbSName = "main"; + db->aDb[0].safety_level = SQLITE_DEFAULT_SYNCHRONOUS+1; + db->aDb[1].zDbSName = "temp"; + db->aDb[1].safety_level = PAGER_SYNCHRONOUS_OFF; + + db->magic = SQLITE_MAGIC_OPEN; + if( db->mallocFailed ){ + goto opendb_out; + } + + /* Register all built-in functions, but do not attempt to read the + ** database schema yet. This is delayed until the first time the database + ** is accessed. + */ + sqlite3Error(db, SQLITE_OK); + sqlite3RegisterPerConnectionBuiltinFunctions(db); + rc = sqlite3_errcode(db); + + + /* Load compiled-in extensions */ + for(i=0; rc==SQLITE_OK && imDbFlags |= DBFLAG_InternalFunc; +#endif + + /* -DSQLITE_DEFAULT_LOCKING_MODE=1 makes EXCLUSIVE the default locking + ** mode. -DSQLITE_DEFAULT_LOCKING_MODE=0 make NORMAL the default locking + ** mode. Doing nothing at all also makes NORMAL the default. + */ +#ifdef SQLITE_DEFAULT_LOCKING_MODE + db->dfltLockMode = SQLITE_DEFAULT_LOCKING_MODE; + sqlite3PagerLockingMode(sqlite3BtreePager(db->aDb[0].pBt), + SQLITE_DEFAULT_LOCKING_MODE); +#endif + + if( rc ) sqlite3Error(db, rc); + + /* Enable the lookaside-malloc subsystem */ + setupLookaside(db, 0, sqlite3GlobalConfig.szLookaside, + sqlite3GlobalConfig.nLookaside); + + sqlite3_wal_autocheckpoint(db, SQLITE_DEFAULT_WAL_AUTOCHECKPOINT); + +opendb_out: + if( db ){ + assert( db->mutex!=0 || isThreadsafe==0 + || sqlite3GlobalConfig.bFullMutex==0 ); + sqlite3_mutex_leave(db->mutex); + } + rc = sqlite3_errcode(db); + assert( db!=0 || rc==SQLITE_NOMEM ); + if( rc==SQLITE_NOMEM ){ + sqlite3_close(db); + db = 0; + }else if( rc!=SQLITE_OK ){ + db->magic = SQLITE_MAGIC_SICK; + } + *ppDb = db; +#ifdef SQLITE_ENABLE_SQLLOG + if( sqlite3GlobalConfig.xSqllog ){ + /* Opening a db handle. Fourth parameter is passed 0. */ + void *pArg = sqlite3GlobalConfig.pSqllogArg; + sqlite3GlobalConfig.xSqllog(pArg, db, zFilename, 0); + } +#endif + sqlite3_free_filename(zOpen); + return rc & 0xff; } /* -** Return true if the buffer z[0..n-1] contains all spaces. +** Open a new database handle. */ -static int allSpaces(const char *z, int n){ - while( n>0 && z[n-1]==' ' ){ n--; } - return n==0; +SQLITE_API int sqlite3_open( + const char *zFilename, + sqlite3 **ppDb +){ + return openDatabase(zFilename, ppDb, + SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, 0); +} +SQLITE_API int sqlite3_open_v2( + const char *filename, /* Database filename (UTF-8) */ + sqlite3 **ppDb, /* OUT: SQLite db handle */ + int flags, /* Flags */ + const char *zVfs /* Name of VFS module to use */ +){ + return openDatabase(filename, ppDb, (unsigned int)flags, zVfs); } +#ifndef SQLITE_OMIT_UTF16 /* -** This is the default collating function named "BINARY" which is always -** available. -** -** If the padFlag argument is not NULL then space padding at the end -** of strings is ignored. This implements the RTRIM collation. +** Open a new database handle. */ -static int binCollFunc( - void *padFlag, - int nKey1, const void *pKey1, - int nKey2, const void *pKey2 +SQLITE_API int sqlite3_open16( + const void *zFilename, + sqlite3 **ppDb ){ - int rc, n; - n = nKey1lastRowid; + sqlite3_mutex_enter(db->mutex); + assert( !db->mallocFailed ); + rc = createCollation(db, zName, (u8)enc, pCtx, xCompare, xDel); + rc = sqlite3ApiExit(db, rc); + sqlite3_mutex_leave(db->mutex); + return rc; } +#ifndef SQLITE_OMIT_UTF16 /* -** Set the value returned by the sqlite3_last_insert_rowid() API function. +** Register a new collation sequence with the database handle db. */ -SQLITE_API void sqlite3_set_last_insert_rowid(sqlite3 *db, sqlite3_int64 iRowid){ +SQLITE_API int sqlite3_create_collation16( + sqlite3* db, + const void *zName, + int enc, + void* pCtx, + int(*xCompare)(void*,int,const void*,int,const void*) +){ + int rc = SQLITE_OK; + char *zName8; + #ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) ){ - (void)SQLITE_MISUSE_BKPT; - return; - } + if( !sqlite3SafetyCheckOk(db) || zName==0 ) return SQLITE_MISUSE_BKPT; #endif sqlite3_mutex_enter(db->mutex); - db->lastRowid = iRowid; + assert( !db->mallocFailed ); + zName8 = sqlite3Utf16to8(db, zName, -1, SQLITE_UTF16NATIVE); + if( zName8 ){ + rc = createCollation(db, zName8, (u8)enc, pCtx, xCompare, 0); + sqlite3DbFree(db, zName8); + } + rc = sqlite3ApiExit(db, rc); sqlite3_mutex_leave(db->mutex); + return rc; } +#endif /* SQLITE_OMIT_UTF16 */ /* -** Return the number of changes in the most recent call to sqlite3_exec(). +** Register a collation sequence factory callback with the database handle +** db. Replace any previously installed collation sequence factory. */ -SQLITE_API int sqlite3_changes(sqlite3 *db){ +SQLITE_API int sqlite3_collation_needed( + sqlite3 *db, + void *pCollNeededArg, + void(*xCollNeeded)(void*,sqlite3*,int eTextRep,const char*) +){ #ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) ){ - (void)SQLITE_MISUSE_BKPT; - return 0; - } + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; #endif - return db->nChange; + sqlite3_mutex_enter(db->mutex); + db->xCollNeeded = xCollNeeded; + db->xCollNeeded16 = 0; + db->pCollNeededArg = pCollNeededArg; + sqlite3_mutex_leave(db->mutex); + return SQLITE_OK; } +#ifndef SQLITE_OMIT_UTF16 /* -** Return the number of changes since the database handle was opened. +** Register a collation sequence factory callback with the database handle +** db. Replace any previously installed collation sequence factory. */ -SQLITE_API int sqlite3_total_changes(sqlite3 *db){ +SQLITE_API int sqlite3_collation_needed16( + sqlite3 *db, + void *pCollNeededArg, + void(*xCollNeeded16)(void*,sqlite3*,int eTextRep,const void*) +){ #ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) ){ - (void)SQLITE_MISUSE_BKPT; - return 0; - } + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; #endif - return db->nTotalChange; + sqlite3_mutex_enter(db->mutex); + db->xCollNeeded = 0; + db->xCollNeeded16 = xCollNeeded16; + db->pCollNeededArg = pCollNeededArg; + sqlite3_mutex_leave(db->mutex); + return SQLITE_OK; } +#endif /* SQLITE_OMIT_UTF16 */ +#ifndef SQLITE_OMIT_DEPRECATED /* -** Close all open savepoints. This function only manipulates fields of the -** database handle object, it does not close any savepoints that may be open -** at the b-tree/pager level. +** This function is now an anachronism. It used to be used to recover from a +** malloc() failure, but SQLite now does this automatically. */ -SQLITE_PRIVATE void sqlite3CloseSavepoints(sqlite3 *db){ - while( db->pSavepoint ){ - Savepoint *pTmp = db->pSavepoint; - db->pSavepoint = pTmp->pNext; - sqlite3DbFree(db, pTmp); - } - db->nSavepoint = 0; - db->nStatement = 0; - db->isTransactionSavepoint = 0; +SQLITE_API int sqlite3_global_recover(void){ + return SQLITE_OK; } +#endif /* -** Invoke the destructor function associated with FuncDef p, if any. Except, -** if this is not the last copy of the function, do not invoke it. Multiple -** copies of a single function are created when create_function() is called -** with SQLITE_ANY as the encoding. +** Test to see whether or not the database connection is in autocommit +** mode. Return TRUE if it is and FALSE if not. Autocommit mode is on +** by default. Autocommit is disabled by a BEGIN statement and reenabled +** by the next COMMIT or ROLLBACK. */ -static void functionDestroy(sqlite3 *db, FuncDef *p){ - FuncDestructor *pDestructor = p->u.pDestructor; - if( pDestructor ){ - pDestructor->nRef--; - if( pDestructor->nRef==0 ){ - pDestructor->xDestroy(pDestructor->pUserData); - sqlite3DbFree(db, pDestructor); - } +SQLITE_API int sqlite3_get_autocommit(sqlite3 *db){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; } +#endif + return db->autoCommit; } /* -** Disconnect all sqlite3_vtab objects that belong to database connection -** db. This is called when db is being closed. +** The following routines are substitutes for constants SQLITE_CORRUPT, +** SQLITE_MISUSE, SQLITE_CANTOPEN, SQLITE_NOMEM and possibly other error +** constants. They serve two purposes: +** +** 1. Serve as a convenient place to set a breakpoint in a debugger +** to detect when version error conditions occurs. +** +** 2. Invoke sqlite3_log() to provide the source code location where +** a low-level error is first detected. */ -static void disconnectAllVtab(sqlite3 *db){ -#ifndef SQLITE_OMIT_VIRTUALTABLE - int i; - HashElem *p; - sqlite3BtreeEnterAll(db); - for(i=0; inDb; i++){ - Schema *pSchema = db->aDb[i].pSchema; - if( db->aDb[i].pSchema ){ - for(p=sqliteHashFirst(&pSchema->tblHash); p; p=sqliteHashNext(p)){ - Table *pTab = (Table *)sqliteHashData(p); - if( IsVirtual(pTab) ) sqlite3VtabDisconnect(db, pTab); - } - } - } - for(p=sqliteHashFirst(&db->aModule); p; p=sqliteHashNext(p)){ - Module *pMod = (Module *)sqliteHashData(p); - if( pMod->pEpoTab ){ - sqlite3VtabDisconnect(db, pMod->pEpoTab); - } - } - sqlite3VtabUnlockList(db); - sqlite3BtreeLeaveAll(db); -#else - UNUSED_PARAMETER(db); +SQLITE_PRIVATE int sqlite3ReportError(int iErr, int lineno, const char *zType){ + sqlite3_log(iErr, "%s at line %d of [%.10s]", + zType, lineno, 20+sqlite3_sourceid()); + return iErr; +} +SQLITE_PRIVATE int sqlite3CorruptError(int lineno){ + testcase( sqlite3GlobalConfig.xLog!=0 ); + return sqlite3ReportError(SQLITE_CORRUPT, lineno, "database corruption"); +} +SQLITE_PRIVATE int sqlite3MisuseError(int lineno){ + testcase( sqlite3GlobalConfig.xLog!=0 ); + return sqlite3ReportError(SQLITE_MISUSE, lineno, "misuse"); +} +SQLITE_PRIVATE int sqlite3CantopenError(int lineno){ + testcase( sqlite3GlobalConfig.xLog!=0 ); + return sqlite3ReportError(SQLITE_CANTOPEN, lineno, "cannot open file"); +} +#if defined(SQLITE_DEBUG) || defined(SQLITE_ENABLE_CORRUPT_PGNO) +SQLITE_PRIVATE int sqlite3CorruptPgnoError(int lineno, Pgno pgno){ + char zMsg[100]; + sqlite3_snprintf(sizeof(zMsg), zMsg, "database corruption page %d", pgno); + testcase( sqlite3GlobalConfig.xLog!=0 ); + return sqlite3ReportError(SQLITE_CORRUPT, lineno, zMsg); +} #endif +#ifdef SQLITE_DEBUG +SQLITE_PRIVATE int sqlite3NomemError(int lineno){ + testcase( sqlite3GlobalConfig.xLog!=0 ); + return sqlite3ReportError(SQLITE_NOMEM, lineno, "OOM"); +} +SQLITE_PRIVATE int sqlite3IoerrnomemError(int lineno){ + testcase( sqlite3GlobalConfig.xLog!=0 ); + return sqlite3ReportError(SQLITE_IOERR_NOMEM, lineno, "I/O OOM error"); } +#endif +#ifndef SQLITE_OMIT_DEPRECATED /* -** Return TRUE if database connection db has unfinalized prepared -** statements or unfinished sqlite3_backup objects. +** This is a convenience routine that makes sure that all thread-specific +** data for this thread has been deallocated. +** +** SQLite no longer uses thread-specific data so this routine is now a +** no-op. It is retained for historical compatibility. */ -static int connectionIsBusy(sqlite3 *db){ - int j; - assert( sqlite3_mutex_held(db->mutex) ); - if( db->pVdbe ) return 1; - for(j=0; jnDb; j++){ - Btree *pBt = db->aDb[j].pBt; - if( pBt && sqlite3BtreeIsInBackup(pBt) ) return 1; - } - return 0; +SQLITE_API void sqlite3_thread_cleanup(void){ } +#endif /* -** Close an existing SQLite database +** Return meta information about a specific column of a database table. +** See comment in sqlite3.h (sqlite.h.in) for details. */ -static int sqlite3Close(sqlite3 *db, int forceZombie){ - if( !db ){ - /* EVIDENCE-OF: R-63257-11740 Calling sqlite3_close() or - ** sqlite3_close_v2() with a NULL pointer argument is a harmless no-op. */ - return SQLITE_OK; - } - if( !sqlite3SafetyCheckSickOrOk(db) ){ +SQLITE_API int sqlite3_table_column_metadata( + sqlite3 *db, /* Connection handle */ + const char *zDbName, /* Database name or NULL */ + const char *zTableName, /* Table name */ + const char *zColumnName, /* Column name */ + char const **pzDataType, /* OUTPUT: Declared data type */ + char const **pzCollSeq, /* OUTPUT: Collation sequence name */ + int *pNotNull, /* OUTPUT: True if NOT NULL constraint exists */ + int *pPrimaryKey, /* OUTPUT: True if column part of PK */ + int *pAutoinc /* OUTPUT: True if column is auto-increment */ +){ + int rc; + char *zErrMsg = 0; + Table *pTab = 0; + Column *pCol = 0; + int iCol = 0; + char const *zDataType = 0; + char const *zCollSeq = 0; + int notnull = 0; + int primarykey = 0; + int autoinc = 0; + + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) || zTableName==0 ){ return SQLITE_MISUSE_BKPT; } +#endif + + /* Ensure the database schema has been loaded */ sqlite3_mutex_enter(db->mutex); - if( db->mTrace & SQLITE_TRACE_CLOSE ){ - db->xTrace(SQLITE_TRACE_CLOSE, db->pTraceArg, db, 0); + sqlite3BtreeEnterAll(db); + rc = sqlite3Init(db, &zErrMsg); + if( SQLITE_OK!=rc ){ + goto error_out; } - /* Force xDisconnect calls on all virtual tables */ - disconnectAllVtab(db); - - /* If a transaction is open, the disconnectAllVtab() call above - ** will not have called the xDisconnect() method on any virtual - ** tables in the db->aVTrans[] array. The following sqlite3VtabRollback() - ** call will do so. We need to do this before the check for active - ** SQL statements below, as the v-table implementation may be storing - ** some prepared statements internally. - */ - sqlite3VtabRollback(db); + /* Locate the table in question */ + pTab = sqlite3FindTable(db, zTableName, zDbName); + if( !pTab || pTab->pSelect ){ + pTab = 0; + goto error_out; + } - /* Legacy behavior (sqlite3_close() behavior) is to return - ** SQLITE_BUSY if the connection can not be closed immediately. - */ - if( !forceZombie && connectionIsBusy(db) ){ - sqlite3ErrorWithMsg(db, SQLITE_BUSY, "unable to close due to unfinalized " - "statements or unfinished backups"); - sqlite3_mutex_leave(db->mutex); - return SQLITE_BUSY; + /* Find the column for which info is requested */ + if( zColumnName==0 ){ + /* Query for existance of table only */ + }else{ + for(iCol=0; iColnCol; iCol++){ + pCol = &pTab->aCol[iCol]; + if( 0==sqlite3StrICmp(pCol->zName, zColumnName) ){ + break; + } + } + if( iCol==pTab->nCol ){ + if( HasRowid(pTab) && sqlite3IsRowid(zColumnName) ){ + iCol = pTab->iPKey; + pCol = iCol>=0 ? &pTab->aCol[iCol] : 0; + }else{ + pTab = 0; + goto error_out; + } + } } -#ifdef SQLITE_ENABLE_SQLLOG - if( sqlite3GlobalConfig.xSqllog ){ - /* Closing the handle. Fourth parameter is passed the value 2. */ - sqlite3GlobalConfig.xSqllog(sqlite3GlobalConfig.pSqllogArg, db, 0, 2); + /* The following block stores the meta information that will be returned + ** to the caller in local variables zDataType, zCollSeq, notnull, primarykey + ** and autoinc. At this point there are two possibilities: + ** + ** 1. The specified column name was rowid", "oid" or "_rowid_" + ** and there is no explicitly declared IPK column. + ** + ** 2. The table is not a view and the column name identified an + ** explicitly declared column. Copy meta information from *pCol. + */ + if( pCol ){ + zDataType = sqlite3ColumnType(pCol,0); + zCollSeq = pCol->zColl; + notnull = pCol->notNull!=0; + primarykey = (pCol->colFlags & COLFLAG_PRIMKEY)!=0; + autoinc = pTab->iPKey==iCol && (pTab->tabFlags & TF_Autoincrement)!=0; + }else{ + zDataType = "INTEGER"; + primarykey = 1; + } + if( !zCollSeq ){ + zCollSeq = sqlite3StrBINARY; } -#endif - /* Convert the connection into a zombie and then close it. +error_out: + sqlite3BtreeLeaveAll(db); + + /* Whether the function call succeeded or failed, set the output parameters + ** to whatever their local counterparts contain. If an error did occur, + ** this has the effect of zeroing all output parameters. */ - db->magic = SQLITE_MAGIC_ZOMBIE; - sqlite3LeaveMutexAndCloseZombie(db); - return SQLITE_OK; + if( pzDataType ) *pzDataType = zDataType; + if( pzCollSeq ) *pzCollSeq = zCollSeq; + if( pNotNull ) *pNotNull = notnull; + if( pPrimaryKey ) *pPrimaryKey = primarykey; + if( pAutoinc ) *pAutoinc = autoinc; + + if( SQLITE_OK==rc && !pTab ){ + sqlite3DbFree(db, zErrMsg); + zErrMsg = sqlite3MPrintf(db, "no such table column: %s.%s", zTableName, + zColumnName); + rc = SQLITE_ERROR; + } + sqlite3ErrorWithMsg(db, rc, (zErrMsg?"%s":0), zErrMsg); + sqlite3DbFree(db, zErrMsg); + rc = sqlite3ApiExit(db, rc); + sqlite3_mutex_leave(db->mutex); + return rc; } /* -** Two variations on the public interface for closing a database -** connection. The sqlite3_close() version returns SQLITE_BUSY and -** leaves the connection option if there are unfinalized prepared -** statements or unfinished sqlite3_backups. The sqlite3_close_v2() -** version forces the connection to become a zombie if there are -** unclosed resources, and arranges for deallocation when the last -** prepare statement or sqlite3_backup closes. +** Sleep for a little while. Return the amount of time slept. */ -SQLITE_API int sqlite3_close(sqlite3 *db){ return sqlite3Close(db,0); } -SQLITE_API int sqlite3_close_v2(sqlite3 *db){ return sqlite3Close(db,1); } +SQLITE_API int sqlite3_sleep(int ms){ + sqlite3_vfs *pVfs; + int rc; + pVfs = sqlite3_vfs_find(0); + if( pVfs==0 ) return 0; + + /* This function works in milliseconds, but the underlying OsSleep() + ** API uses microseconds. Hence the 1000's. + */ + rc = (sqlite3OsSleep(pVfs, 1000*ms)/1000); + return rc; +} +/* +** Enable or disable the extended result codes. +*/ +SQLITE_API int sqlite3_extended_result_codes(sqlite3 *db, int onoff){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif + sqlite3_mutex_enter(db->mutex); + db->errMask = onoff ? 0xffffffff : 0xff; + sqlite3_mutex_leave(db->mutex); + return SQLITE_OK; +} /* -** Close the mutex on database connection db. -** -** Furthermore, if database connection db is a zombie (meaning that there -** has been a prior call to sqlite3_close(db) or sqlite3_close_v2(db)) and -** every sqlite3_stmt has now been finalized and every sqlite3_backup has -** finished, then free all resources. +** Invoke the xFileControl method on a particular database. */ -SQLITE_PRIVATE void sqlite3LeaveMutexAndCloseZombie(sqlite3 *db){ - HashElem *i; /* Hash table iterator */ - int j; +SQLITE_API int sqlite3_file_control(sqlite3 *db, const char *zDbName, int op, void *pArg){ + int rc = SQLITE_ERROR; + Btree *pBtree; - /* If there are outstanding sqlite3_stmt or sqlite3_backup objects - ** or if the connection has not yet been closed by sqlite3_close_v2(), - ** then just leave the mutex and return. - */ - if( db->magic!=SQLITE_MAGIC_ZOMBIE || connectionIsBusy(db) ){ - sqlite3_mutex_leave(db->mutex); - return; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif + sqlite3_mutex_enter(db->mutex); + pBtree = sqlite3DbNameToBtree(db, zDbName); + if( pBtree ){ + Pager *pPager; + sqlite3_file *fd; + sqlite3BtreeEnter(pBtree); + pPager = sqlite3BtreePager(pBtree); + assert( pPager!=0 ); + fd = sqlite3PagerFile(pPager); + assert( fd!=0 ); + if( op==SQLITE_FCNTL_FILE_POINTER ){ + *(sqlite3_file**)pArg = fd; + rc = SQLITE_OK; + }else if( op==SQLITE_FCNTL_VFS_POINTER ){ + *(sqlite3_vfs**)pArg = sqlite3PagerVfs(pPager); + rc = SQLITE_OK; + }else if( op==SQLITE_FCNTL_JOURNAL_POINTER ){ + *(sqlite3_file**)pArg = sqlite3PagerJrnlFile(pPager); + rc = SQLITE_OK; + }else if( op==SQLITE_FCNTL_DATA_VERSION ){ + *(unsigned int*)pArg = sqlite3PagerDataVersion(pPager); + rc = SQLITE_OK; + }else if( op==SQLITE_FCNTL_RESERVE_BYTES ){ + int iNew = *(int*)pArg; + *(int*)pArg = sqlite3BtreeGetRequestedReserve(pBtree); + if( iNew>=0 && iNew<=255 ){ + sqlite3BtreeSetPageSize(pBtree, 0, iNew, 0); + } + rc = SQLITE_OK; + }else{ + rc = sqlite3OsFileControl(fd, op, pArg); + } + sqlite3BtreeLeave(pBtree); } + sqlite3_mutex_leave(db->mutex); + return rc; +} - /* If we reach this point, it means that the database connection has - ** closed all sqlite3_stmt and sqlite3_backup objects and has been - ** passed to sqlite3_close (meaning that it is a zombie). Therefore, - ** go ahead and free all resources. - */ - - /* If a transaction is open, roll it back. This also ensures that if - ** any database schemas have been modified by an uncommitted transaction - ** they are reset. And that the required b-tree mutex is held to make - ** the pager rollback and schema reset an atomic operation. */ - sqlite3RollbackAll(db, SQLITE_OK); +/* +** Interface to the testing logic. +*/ +SQLITE_API int sqlite3_test_control(int op, ...){ + int rc = 0; +#ifdef SQLITE_UNTESTABLE + UNUSED_PARAMETER(op); +#else + va_list ap; + va_start(ap, op); + switch( op ){ - /* Free any outstanding Savepoint structures. */ - sqlite3CloseSavepoints(db); + /* + ** Save the current state of the PRNG. + */ + case SQLITE_TESTCTRL_PRNG_SAVE: { + sqlite3PrngSaveState(); + break; + } - /* Close all database connections */ - for(j=0; jnDb; j++){ - struct Db *pDb = &db->aDb[j]; - if( pDb->pBt ){ - sqlite3BtreeClose(pDb->pBt); - pDb->pBt = 0; - if( j!=1 ){ - pDb->pSchema = 0; - } + /* + ** Restore the state of the PRNG to the last state saved using + ** PRNG_SAVE. If PRNG_SAVE has never before been called, then + ** this verb acts like PRNG_RESET. + */ + case SQLITE_TESTCTRL_PRNG_RESTORE: { + sqlite3PrngRestoreState(); + break; } - } - /* Clear the TEMP schema separately and last */ - if( db->aDb[1].pSchema ){ - sqlite3SchemaClear(db->aDb[1].pSchema); - } - sqlite3VtabUnlockList(db); - /* Free up the array of auxiliary databases */ - sqlite3CollapseDatabaseArray(db); - assert( db->nDb<=2 ); - assert( db->aDb==db->aDbStatic ); + /* sqlite3_test_control(SQLITE_TESTCTRL_PRNG_SEED, int x, sqlite3 *db); + ** + ** Control the seed for the pseudo-random number generator (PRNG) that + ** is built into SQLite. Cases: + ** + ** x!=0 && db!=0 Seed the PRNG to the current value of the + ** schema cookie in the main database for db, or + ** x if the schema cookie is zero. This case + ** is convenient to use with database fuzzers + ** as it allows the fuzzer some control over the + ** the PRNG seed. + ** + ** x!=0 && db==0 Seed the PRNG to the value of x. + ** + ** x==0 && db==0 Revert to default behavior of using the + ** xRandomness method on the primary VFS. + ** + ** This test-control also resets the PRNG so that the new seed will + ** be used for the next call to sqlite3_randomness(). + */ +#ifndef SQLITE_OMIT_WSD + case SQLITE_TESTCTRL_PRNG_SEED: { + int x = va_arg(ap, int); + int y; + sqlite3 *db = va_arg(ap, sqlite3*); + assert( db==0 || db->aDb[0].pSchema!=0 ); + if( db && (y = db->aDb[0].pSchema->schema_cookie)!=0 ){ x = y; } + sqlite3Config.iPrngSeed = x; + sqlite3_randomness(0,0); + break; + } +#endif - /* Tell the code in notify.c that the connection no longer holds any - ** locks and does not require any further unlock-notify callbacks. - */ - sqlite3ConnectionClosed(db); + /* + ** sqlite3_test_control(BITVEC_TEST, size, program) + ** + ** Run a test against a Bitvec object of size. The program argument + ** is an array of integers that defines the test. Return -1 on a + ** memory allocation error, 0 on success, or non-zero for an error. + ** See the sqlite3BitvecBuiltinTest() for additional information. + */ + case SQLITE_TESTCTRL_BITVEC_TEST: { + int sz = va_arg(ap, int); + int *aProg = va_arg(ap, int*); + rc = sqlite3BitvecBuiltinTest(sz, aProg); + break; + } - for(i=sqliteHashFirst(&db->aFunc); i; i=sqliteHashNext(i)){ - FuncDef *pNext, *p; - p = sqliteHashData(i); - do{ - functionDestroy(db, p); - pNext = p->pNext; - sqlite3DbFree(db, p); - p = pNext; - }while( p ); - } - sqlite3HashClear(&db->aFunc); - for(i=sqliteHashFirst(&db->aCollSeq); i; i=sqliteHashNext(i)){ - CollSeq *pColl = (CollSeq *)sqliteHashData(i); - /* Invoke any destructors registered for collation sequence user data. */ - for(j=0; j<3; j++){ - if( pColl[j].xDel ){ - pColl[j].xDel(pColl[j].pUser); - } + /* + ** sqlite3_test_control(FAULT_INSTALL, xCallback) + ** + ** Arrange to invoke xCallback() whenever sqlite3FaultSim() is called, + ** if xCallback is not NULL. + ** + ** As a test of the fault simulator mechanism itself, sqlite3FaultSim(0) + ** is called immediately after installing the new callback and the return + ** value from sqlite3FaultSim(0) becomes the return from + ** sqlite3_test_control(). + */ + case SQLITE_TESTCTRL_FAULT_INSTALL: { + /* MSVC is picky about pulling func ptrs from va lists. + ** http://support.microsoft.com/kb/47961 + ** sqlite3GlobalConfig.xTestCallback = va_arg(ap, int(*)(int)); + */ + typedef int(*TESTCALLBACKFUNC_t)(int); + sqlite3GlobalConfig.xTestCallback = va_arg(ap, TESTCALLBACKFUNC_t); + rc = sqlite3FaultSim(0); + break; } - sqlite3DbFree(db, pColl); - } - sqlite3HashClear(&db->aCollSeq); -#ifndef SQLITE_OMIT_VIRTUALTABLE - for(i=sqliteHashFirst(&db->aModule); i; i=sqliteHashNext(i)){ - Module *pMod = (Module *)sqliteHashData(i); - if( pMod->xDestroy ){ - pMod->xDestroy(pMod->pAux); + + /* + ** sqlite3_test_control(BENIGN_MALLOC_HOOKS, xBegin, xEnd) + ** + ** Register hooks to call to indicate which malloc() failures + ** are benign. + */ + case SQLITE_TESTCTRL_BENIGN_MALLOC_HOOKS: { + typedef void (*void_function)(void); + void_function xBenignBegin; + void_function xBenignEnd; + xBenignBegin = va_arg(ap, void_function); + xBenignEnd = va_arg(ap, void_function); + sqlite3BenignMallocHooks(xBenignBegin, xBenignEnd); + break; } - sqlite3VtabEponymousTableClear(db, pMod); - sqlite3DbFree(db, pMod); - } - sqlite3HashClear(&db->aModule); -#endif - sqlite3Error(db, SQLITE_OK); /* Deallocates any cached error strings. */ - sqlite3ValueFree(db->pErr); - sqlite3CloseExtensions(db); -#if SQLITE_USER_AUTHENTICATION - sqlite3_free(db->auth.zAuthUser); - sqlite3_free(db->auth.zAuthPW); + /* + ** sqlite3_test_control(SQLITE_TESTCTRL_PENDING_BYTE, unsigned int X) + ** + ** Set the PENDING byte to the value in the argument, if X>0. + ** Make no changes if X==0. Return the value of the pending byte + ** as it existing before this routine was called. + ** + ** IMPORTANT: Changing the PENDING byte from 0x40000000 results in + ** an incompatible database file format. Changing the PENDING byte + ** while any database connection is open results in undefined and + ** deleterious behavior. + */ + case SQLITE_TESTCTRL_PENDING_BYTE: { + rc = PENDING_BYTE; +#ifndef SQLITE_OMIT_WSD + { + unsigned int newVal = va_arg(ap, unsigned int); + if( newVal ) sqlite3PendingByte = newVal; + } #endif + break; + } - db->magic = SQLITE_MAGIC_ERROR; + /* + ** sqlite3_test_control(SQLITE_TESTCTRL_ASSERT, int X) + ** + ** This action provides a run-time test to see whether or not + ** assert() was enabled at compile-time. If X is true and assert() + ** is enabled, then the return value is true. If X is true and + ** assert() is disabled, then the return value is zero. If X is + ** false and assert() is enabled, then the assertion fires and the + ** process aborts. If X is false and assert() is disabled, then the + ** return value is zero. + */ + case SQLITE_TESTCTRL_ASSERT: { + volatile int x = 0; + assert( /*side-effects-ok*/ (x = va_arg(ap,int))!=0 ); + rc = x; + break; + } - /* The temp-database schema is allocated differently from the other schema - ** objects (using sqliteMalloc() directly, instead of sqlite3BtreeSchema()). - ** So it needs to be freed here. Todo: Why not roll the temp schema into - ** the same sqliteMalloc() as the one that allocates the database - ** structure? - */ - sqlite3DbFree(db, db->aDb[1].pSchema); - sqlite3_mutex_leave(db->mutex); - db->magic = SQLITE_MAGIC_CLOSED; - sqlite3_mutex_free(db->mutex); - assert( sqlite3LookasideUsed(db,0)==0 ); - if( db->lookaside.bMalloced ){ - sqlite3_free(db->lookaside.pStart); - } - sqlite3_free(db); -} -/* -** Rollback all database files. If tripCode is not SQLITE_OK, then -** any write cursors are invalidated ("tripped" - as in "tripping a circuit -** breaker") and made to return tripCode if there are any further -** attempts to use that cursor. Read cursors remain open and valid -** but are "saved" in case the table pages are moved around. -*/ -SQLITE_PRIVATE void sqlite3RollbackAll(sqlite3 *db, int tripCode){ - int i; - int inTrans = 0; - int schemaChange; - assert( sqlite3_mutex_held(db->mutex) ); - sqlite3BeginBenignMalloc(); + /* + ** sqlite3_test_control(SQLITE_TESTCTRL_ALWAYS, int X) + ** + ** This action provides a run-time test to see how the ALWAYS and + ** NEVER macros were defined at compile-time. + ** + ** The return value is ALWAYS(X) if X is true, or 0 if X is false. + ** + ** The recommended test is X==2. If the return value is 2, that means + ** ALWAYS() and NEVER() are both no-op pass-through macros, which is the + ** default setting. If the return value is 1, then ALWAYS() is either + ** hard-coded to true or else it asserts if its argument is false. + ** The first behavior (hard-coded to true) is the case if + ** SQLITE_TESTCTRL_ASSERT shows that assert() is disabled and the second + ** behavior (assert if the argument to ALWAYS() is false) is the case if + ** SQLITE_TESTCTRL_ASSERT shows that assert() is enabled. + ** + ** The run-time test procedure might look something like this: + ** + ** if( sqlite3_test_control(SQLITE_TESTCTRL_ALWAYS, 2)==2 ){ + ** // ALWAYS() and NEVER() are no-op pass-through macros + ** }else if( sqlite3_test_control(SQLITE_TESTCTRL_ASSERT, 1) ){ + ** // ALWAYS(x) asserts that x is true. NEVER(x) asserts x is false. + ** }else{ + ** // ALWAYS(x) is a constant 1. NEVER(x) is a constant 0. + ** } + */ + case SQLITE_TESTCTRL_ALWAYS: { + int x = va_arg(ap,int); + rc = x ? ALWAYS(x) : 0; + break; + } - /* Obtain all b-tree mutexes before making any calls to BtreeRollback(). - ** This is important in case the transaction being rolled back has - ** modified the database schema. If the b-tree mutexes are not taken - ** here, then another shared-cache connection might sneak in between - ** the database rollback and schema reset, which can cause false - ** corruption reports in some cases. */ - sqlite3BtreeEnterAll(db); - schemaChange = (db->mDbFlags & DBFLAG_SchemaChange)!=0 && db->init.busy==0; + /* + ** sqlite3_test_control(SQLITE_TESTCTRL_BYTEORDER); + ** + ** The integer returned reveals the byte-order of the computer on which + ** SQLite is running: + ** + ** 1 big-endian, determined at run-time + ** 10 little-endian, determined at run-time + ** 432101 big-endian, determined at compile-time + ** 123410 little-endian, determined at compile-time + */ + case SQLITE_TESTCTRL_BYTEORDER: { + rc = SQLITE_BYTEORDER*100 + SQLITE_LITTLEENDIAN*10 + SQLITE_BIGENDIAN; + break; + } - for(i=0; inDb; i++){ - Btree *p = db->aDb[i].pBt; - if( p ){ - if( sqlite3BtreeIsInTrans(p) ){ - inTrans = 1; - } - sqlite3BtreeRollback(p, tripCode, !schemaChange); + /* sqlite3_test_control(SQLITE_TESTCTRL_OPTIMIZATIONS, sqlite3 *db, int N) + ** + ** Enable or disable various optimizations for testing purposes. The + ** argument N is a bitmask of optimizations to be disabled. For normal + ** operation N should be 0. The idea is that a test program (like the + ** SQL Logic Test or SLT test module) can run the same SQL multiple times + ** with various optimizations disabled to verify that the same answer + ** is obtained in every case. + */ + case SQLITE_TESTCTRL_OPTIMIZATIONS: { + sqlite3 *db = va_arg(ap, sqlite3*); + db->dbOptFlags = (u16)(va_arg(ap, int) & 0xffff); + break; } - } - sqlite3VtabRollback(db); - sqlite3EndBenignMalloc(); - if( (db->mDbFlags&DBFLAG_SchemaChange)!=0 && db->init.busy==0 ){ - sqlite3ExpirePreparedStatements(db); - sqlite3ResetAllSchemasOfConnection(db); - } - sqlite3BtreeLeaveAll(db); + /* sqlite3_test_control(SQLITE_TESTCTRL_LOCALTIME_FAULT, int onoff); + ** + ** If parameter onoff is non-zero, subsequent calls to localtime() + ** and its variants fail. If onoff is zero, undo this setting. + */ + case SQLITE_TESTCTRL_LOCALTIME_FAULT: { + sqlite3GlobalConfig.bLocaltimeFault = va_arg(ap, int); + break; + } - /* Any deferred constraint violations have now been resolved. */ - db->nDeferredCons = 0; - db->nDeferredImmCons = 0; - db->flags &= ~SQLITE_DeferFKs; + /* sqlite3_test_control(SQLITE_TESTCTRL_INTERNAL_FUNCTIONS, sqlite3*); + ** + ** Toggle the ability to use internal functions on or off for + ** the database connection given in the argument. + */ + case SQLITE_TESTCTRL_INTERNAL_FUNCTIONS: { + sqlite3 *db = va_arg(ap, sqlite3*); + db->mDbFlags ^= DBFLAG_InternalFunc; + break; + } - /* If one has been configured, invoke the rollback-hook callback */ - if( db->xRollbackCallback && (inTrans || !db->autoCommit) ){ - db->xRollbackCallback(db->pRollbackArg); - } -} + /* sqlite3_test_control(SQLITE_TESTCTRL_NEVER_CORRUPT, int); + ** + ** Set or clear a flag that indicates that the database file is always well- + ** formed and never corrupt. This flag is clear by default, indicating that + ** database files might have arbitrary corruption. Setting the flag during + ** testing causes certain assert() statements in the code to be activated + ** that demonstrat invariants on well-formed database files. + */ + case SQLITE_TESTCTRL_NEVER_CORRUPT: { + sqlite3GlobalConfig.neverCorrupt = va_arg(ap, int); + break; + } -/* -** Return a static string containing the name corresponding to the error code -** specified in the argument. -*/ -#if defined(SQLITE_NEED_ERR_NAME) -SQLITE_PRIVATE const char *sqlite3ErrName(int rc){ - const char *zName = 0; - int i, origRc = rc; - for(i=0; i<2 && zName==0; i++, rc &= 0xff){ - switch( rc ){ - case SQLITE_OK: zName = "SQLITE_OK"; break; - case SQLITE_ERROR: zName = "SQLITE_ERROR"; break; - case SQLITE_INTERNAL: zName = "SQLITE_INTERNAL"; break; - case SQLITE_PERM: zName = "SQLITE_PERM"; break; - case SQLITE_ABORT: zName = "SQLITE_ABORT"; break; - case SQLITE_ABORT_ROLLBACK: zName = "SQLITE_ABORT_ROLLBACK"; break; - case SQLITE_BUSY: zName = "SQLITE_BUSY"; break; - case SQLITE_BUSY_RECOVERY: zName = "SQLITE_BUSY_RECOVERY"; break; - case SQLITE_BUSY_SNAPSHOT: zName = "SQLITE_BUSY_SNAPSHOT"; break; - case SQLITE_LOCKED: zName = "SQLITE_LOCKED"; break; - case SQLITE_LOCKED_SHAREDCACHE: zName = "SQLITE_LOCKED_SHAREDCACHE";break; - case SQLITE_NOMEM: zName = "SQLITE_NOMEM"; break; - case SQLITE_READONLY: zName = "SQLITE_READONLY"; break; - case SQLITE_READONLY_RECOVERY: zName = "SQLITE_READONLY_RECOVERY"; break; - case SQLITE_READONLY_CANTINIT: zName = "SQLITE_READONLY_CANTINIT"; break; - case SQLITE_READONLY_ROLLBACK: zName = "SQLITE_READONLY_ROLLBACK"; break; - case SQLITE_READONLY_DBMOVED: zName = "SQLITE_READONLY_DBMOVED"; break; - case SQLITE_READONLY_DIRECTORY: zName = "SQLITE_READONLY_DIRECTORY";break; - case SQLITE_INTERRUPT: zName = "SQLITE_INTERRUPT"; break; - case SQLITE_IOERR: zName = "SQLITE_IOERR"; break; - case SQLITE_IOERR_READ: zName = "SQLITE_IOERR_READ"; break; - case SQLITE_IOERR_SHORT_READ: zName = "SQLITE_IOERR_SHORT_READ"; break; - case SQLITE_IOERR_WRITE: zName = "SQLITE_IOERR_WRITE"; break; - case SQLITE_IOERR_FSYNC: zName = "SQLITE_IOERR_FSYNC"; break; - case SQLITE_IOERR_DIR_FSYNC: zName = "SQLITE_IOERR_DIR_FSYNC"; break; - case SQLITE_IOERR_TRUNCATE: zName = "SQLITE_IOERR_TRUNCATE"; break; - case SQLITE_IOERR_FSTAT: zName = "SQLITE_IOERR_FSTAT"; break; - case SQLITE_IOERR_UNLOCK: zName = "SQLITE_IOERR_UNLOCK"; break; - case SQLITE_IOERR_RDLOCK: zName = "SQLITE_IOERR_RDLOCK"; break; - case SQLITE_IOERR_DELETE: zName = "SQLITE_IOERR_DELETE"; break; - case SQLITE_IOERR_NOMEM: zName = "SQLITE_IOERR_NOMEM"; break; - case SQLITE_IOERR_ACCESS: zName = "SQLITE_IOERR_ACCESS"; break; - case SQLITE_IOERR_CHECKRESERVEDLOCK: - zName = "SQLITE_IOERR_CHECKRESERVEDLOCK"; break; - case SQLITE_IOERR_LOCK: zName = "SQLITE_IOERR_LOCK"; break; - case SQLITE_IOERR_CLOSE: zName = "SQLITE_IOERR_CLOSE"; break; - case SQLITE_IOERR_DIR_CLOSE: zName = "SQLITE_IOERR_DIR_CLOSE"; break; - case SQLITE_IOERR_SHMOPEN: zName = "SQLITE_IOERR_SHMOPEN"; break; - case SQLITE_IOERR_SHMSIZE: zName = "SQLITE_IOERR_SHMSIZE"; break; - case SQLITE_IOERR_SHMLOCK: zName = "SQLITE_IOERR_SHMLOCK"; break; - case SQLITE_IOERR_SHMMAP: zName = "SQLITE_IOERR_SHMMAP"; break; - case SQLITE_IOERR_SEEK: zName = "SQLITE_IOERR_SEEK"; break; - case SQLITE_IOERR_DELETE_NOENT: zName = "SQLITE_IOERR_DELETE_NOENT";break; - case SQLITE_IOERR_MMAP: zName = "SQLITE_IOERR_MMAP"; break; - case SQLITE_IOERR_GETTEMPPATH: zName = "SQLITE_IOERR_GETTEMPPATH"; break; - case SQLITE_IOERR_CONVPATH: zName = "SQLITE_IOERR_CONVPATH"; break; - case SQLITE_CORRUPT: zName = "SQLITE_CORRUPT"; break; - case SQLITE_CORRUPT_VTAB: zName = "SQLITE_CORRUPT_VTAB"; break; - case SQLITE_NOTFOUND: zName = "SQLITE_NOTFOUND"; break; - case SQLITE_FULL: zName = "SQLITE_FULL"; break; - case SQLITE_CANTOPEN: zName = "SQLITE_CANTOPEN"; break; - case SQLITE_CANTOPEN_NOTEMPDIR: zName = "SQLITE_CANTOPEN_NOTEMPDIR";break; - case SQLITE_CANTOPEN_ISDIR: zName = "SQLITE_CANTOPEN_ISDIR"; break; - case SQLITE_CANTOPEN_FULLPATH: zName = "SQLITE_CANTOPEN_FULLPATH"; break; - case SQLITE_CANTOPEN_CONVPATH: zName = "SQLITE_CANTOPEN_CONVPATH"; break; - case SQLITE_PROTOCOL: zName = "SQLITE_PROTOCOL"; break; - case SQLITE_EMPTY: zName = "SQLITE_EMPTY"; break; - case SQLITE_SCHEMA: zName = "SQLITE_SCHEMA"; break; - case SQLITE_TOOBIG: zName = "SQLITE_TOOBIG"; break; - case SQLITE_CONSTRAINT: zName = "SQLITE_CONSTRAINT"; break; - case SQLITE_CONSTRAINT_UNIQUE: zName = "SQLITE_CONSTRAINT_UNIQUE"; break; - case SQLITE_CONSTRAINT_TRIGGER: zName = "SQLITE_CONSTRAINT_TRIGGER";break; - case SQLITE_CONSTRAINT_FOREIGNKEY: - zName = "SQLITE_CONSTRAINT_FOREIGNKEY"; break; - case SQLITE_CONSTRAINT_CHECK: zName = "SQLITE_CONSTRAINT_CHECK"; break; - case SQLITE_CONSTRAINT_PRIMARYKEY: - zName = "SQLITE_CONSTRAINT_PRIMARYKEY"; break; - case SQLITE_CONSTRAINT_NOTNULL: zName = "SQLITE_CONSTRAINT_NOTNULL";break; - case SQLITE_CONSTRAINT_COMMITHOOK: - zName = "SQLITE_CONSTRAINT_COMMITHOOK"; break; - case SQLITE_CONSTRAINT_VTAB: zName = "SQLITE_CONSTRAINT_VTAB"; break; - case SQLITE_CONSTRAINT_FUNCTION: - zName = "SQLITE_CONSTRAINT_FUNCTION"; break; - case SQLITE_CONSTRAINT_ROWID: zName = "SQLITE_CONSTRAINT_ROWID"; break; - case SQLITE_MISMATCH: zName = "SQLITE_MISMATCH"; break; - case SQLITE_MISUSE: zName = "SQLITE_MISUSE"; break; - case SQLITE_NOLFS: zName = "SQLITE_NOLFS"; break; - case SQLITE_AUTH: zName = "SQLITE_AUTH"; break; - case SQLITE_FORMAT: zName = "SQLITE_FORMAT"; break; - case SQLITE_RANGE: zName = "SQLITE_RANGE"; break; - case SQLITE_NOTADB: zName = "SQLITE_NOTADB"; break; - case SQLITE_ROW: zName = "SQLITE_ROW"; break; - case SQLITE_NOTICE: zName = "SQLITE_NOTICE"; break; - case SQLITE_NOTICE_RECOVER_WAL: zName = "SQLITE_NOTICE_RECOVER_WAL";break; - case SQLITE_NOTICE_RECOVER_ROLLBACK: - zName = "SQLITE_NOTICE_RECOVER_ROLLBACK"; break; - case SQLITE_WARNING: zName = "SQLITE_WARNING"; break; - case SQLITE_WARNING_AUTOINDEX: zName = "SQLITE_WARNING_AUTOINDEX"; break; - case SQLITE_DONE: zName = "SQLITE_DONE"; break; + /* sqlite3_test_control(SQLITE_TESTCTRL_EXTRA_SCHEMA_CHECKS, int); + ** + ** Set or clear a flag that causes SQLite to verify that type, name, + ** and tbl_name fields of the sqlite_master table. This is normally + ** on, but it is sometimes useful to turn it off for testing. + */ + case SQLITE_TESTCTRL_EXTRA_SCHEMA_CHECKS: { + sqlite3GlobalConfig.bExtraSchemaChecks = va_arg(ap, int); + break; + } + + /* Set the threshold at which OP_Once counters reset back to zero. + ** By default this is 0x7ffffffe (over 2 billion), but that value is + ** too big to test in a reasonable amount of time, so this control is + ** provided to set a small and easily reachable reset value. + */ + case SQLITE_TESTCTRL_ONCE_RESET_THRESHOLD: { + sqlite3GlobalConfig.iOnceResetThreshold = va_arg(ap, int); + break; } - } - if( zName==0 ){ - static char zBuf[50]; - sqlite3_snprintf(sizeof(zBuf), zBuf, "SQLITE_UNKNOWN(%d)", origRc); - zName = zBuf; - } - return zName; -} -#endif -/* -** Return a static string that describes the kind of error specified in the -** argument. -*/ -SQLITE_PRIVATE const char *sqlite3ErrStr(int rc){ - static const char* const aMsg[] = { - /* SQLITE_OK */ "not an error", - /* SQLITE_ERROR */ "SQL logic error", - /* SQLITE_INTERNAL */ 0, - /* SQLITE_PERM */ "access permission denied", - /* SQLITE_ABORT */ "query aborted", - /* SQLITE_BUSY */ "database is locked", - /* SQLITE_LOCKED */ "database table is locked", - /* SQLITE_NOMEM */ "out of memory", - /* SQLITE_READONLY */ "attempt to write a readonly database", - /* SQLITE_INTERRUPT */ "interrupted", - /* SQLITE_IOERR */ "disk I/O error", - /* SQLITE_CORRUPT */ "database disk image is malformed", - /* SQLITE_NOTFOUND */ "unknown operation", - /* SQLITE_FULL */ "database or disk is full", - /* SQLITE_CANTOPEN */ "unable to open database file", - /* SQLITE_PROTOCOL */ "locking protocol", - /* SQLITE_EMPTY */ 0, - /* SQLITE_SCHEMA */ "database schema has changed", - /* SQLITE_TOOBIG */ "string or blob too big", - /* SQLITE_CONSTRAINT */ "constraint failed", - /* SQLITE_MISMATCH */ "datatype mismatch", - /* SQLITE_MISUSE */ "bad parameter or other API misuse", -#ifdef SQLITE_DISABLE_LFS - /* SQLITE_NOLFS */ "large file support is disabled", -#else - /* SQLITE_NOLFS */ 0, + /* sqlite3_test_control(SQLITE_TESTCTRL_VDBE_COVERAGE, xCallback, ptr); + ** + ** Set the VDBE coverage callback function to xCallback with context + ** pointer ptr. + */ + case SQLITE_TESTCTRL_VDBE_COVERAGE: { +#ifdef SQLITE_VDBE_COVERAGE + typedef void (*branch_callback)(void*,unsigned int, + unsigned char,unsigned char); + sqlite3GlobalConfig.xVdbeBranch = va_arg(ap,branch_callback); + sqlite3GlobalConfig.pVdbeBranchArg = va_arg(ap,void*); #endif - /* SQLITE_AUTH */ "authorization denied", - /* SQLITE_FORMAT */ 0, - /* SQLITE_RANGE */ "column index out of range", - /* SQLITE_NOTADB */ "file is not a database", - /* SQLITE_NOTICE */ "notification message", - /* SQLITE_WARNING */ "warning message", - }; - const char *zErr = "unknown error"; - switch( rc ){ - case SQLITE_ABORT_ROLLBACK: { - zErr = "abort due to ROLLBACK"; break; } - case SQLITE_ROW: { - zErr = "another row available"; + + /* sqlite3_test_control(SQLITE_TESTCTRL_SORTER_MMAP, db, nMax); */ + case SQLITE_TESTCTRL_SORTER_MMAP: { + sqlite3 *db = va_arg(ap, sqlite3*); + db->nMaxSorterMmap = va_arg(ap, int); break; } - case SQLITE_DONE: { - zErr = "no more rows available"; + + /* sqlite3_test_control(SQLITE_TESTCTRL_ISINIT); + ** + ** Return SQLITE_OK if SQLite has been initialized and SQLITE_ERROR if + ** not. + */ + case SQLITE_TESTCTRL_ISINIT: { + if( sqlite3GlobalConfig.isInit==0 ) rc = SQLITE_ERROR; break; } - default: { - rc &= 0xff; - if( ALWAYS(rc>=0) && rc0 then reset the schema for all databases, causing + ** the schema to be reparsed the next time it is needed. This has the + ** effect of erasing all imposter tables. + */ + case SQLITE_TESTCTRL_IMPOSTER: { + sqlite3 *db = va_arg(ap, sqlite3*); + sqlite3_mutex_enter(db->mutex); + db->init.iDb = sqlite3FindDbName(db, va_arg(ap,const char*)); + db->init.busy = db->init.imposterTable = va_arg(ap,int); + db->init.newTnum = va_arg(ap,int); + if( db->init.busy==0 && db->init.newTnum>0 ){ + sqlite3ResetAllSchemasOfConnection(db); } + sqlite3_mutex_leave(db->mutex); break; } - } - return zErr; -} -/* -** This routine implements a busy callback that sleeps and tries -** again until a timeout value is reached. The timeout value is -** an integer number of milliseconds passed in as the first -** argument. -** -** Return non-zero to retry the lock. Return zero to stop trying -** and cause SQLite to return SQLITE_BUSY. -*/ -static int sqliteDefaultBusyCallback( - void *ptr, /* Database connection */ - int count, /* Number of times table has been busy */ - sqlite3_file *pFile /* The file on which the lock occurred */ -){ -#if SQLITE_OS_WIN || HAVE_USLEEP - /* This case is for systems that have support for sleeping for fractions of - ** a second. Examples: All windows systems, unix systems with usleep() */ - static const u8 delays[] = - { 1, 2, 5, 10, 15, 20, 25, 25, 25, 50, 50, 100 }; - static const u8 totals[] = - { 0, 1, 3, 8, 18, 33, 53, 78, 103, 128, 178, 228 }; -# define NDELAY ArraySize(delays) - sqlite3 *db = (sqlite3 *)ptr; - int tmout = db->busyTimeout; - int delay, prior; +#if defined(YYCOVERAGE) + /* sqlite3_test_control(SQLITE_TESTCTRL_PARSER_COVERAGE, FILE *out) + ** + ** This test control (only available when SQLite is compiled with + ** -DYYCOVERAGE) writes a report onto "out" that shows all + ** state/lookahead combinations in the parser state machine + ** which are never exercised. If any state is missed, make the + ** return code SQLITE_ERROR. + */ + case SQLITE_TESTCTRL_PARSER_COVERAGE: { + FILE *out = va_arg(ap, FILE*); + if( sqlite3ParserCoverage(out) ) rc = SQLITE_ERROR; + break; + } +#endif /* defined(YYCOVERAGE) */ -#ifdef SQLITE_ENABLE_SETLK_TIMEOUT - if( sqlite3OsFileControl(pFile,SQLITE_FCNTL_LOCK_TIMEOUT,&tmout)==SQLITE_OK ){ - if( count ){ - tmout = 0; - sqlite3OsFileControl(pFile, SQLITE_FCNTL_LOCK_TIMEOUT, &tmout); - return 0; - }else{ - return 1; + /* sqlite3_test_control(SQLITE_TESTCTRL_RESULT_INTREAL, sqlite3_context*); + ** + ** This test-control causes the most recent sqlite3_result_int64() value + ** to be interpreted as a MEM_IntReal instead of as an MEM_Int. Normally, + ** MEM_IntReal values only arise during an INSERT operation of integer + ** values into a REAL column, so they can be challenging to test. This + ** test-control enables us to write an intreal() SQL function that can + ** inject an intreal() value at arbitrary places in an SQL statement, + ** for testing purposes. + */ + case SQLITE_TESTCTRL_RESULT_INTREAL: { + sqlite3_context *pCtx = va_arg(ap, sqlite3_context*); + sqlite3ResultIntReal(pCtx); + break; } } -#else - UNUSED_PARAMETER(pFile); -#endif - assert( count>=0 ); - if( count < NDELAY ){ - delay = delays[count]; - prior = totals[count]; - }else{ - delay = delays[NDELAY-1]; - prior = totals[NDELAY-1] + delay*(count-(NDELAY-1)); - } - if( prior + delay > tmout ){ - delay = tmout - prior; - if( delay<=0 ) return 0; - } - sqlite3OsSleep(db->pVfs, delay*1000); - return 1; -#else - /* This case for unix systems that lack usleep() support. Sleeping - ** must be done in increments of whole seconds */ - sqlite3 *db = (sqlite3 *)ptr; - int tmout = ((sqlite3 *)ptr)->busyTimeout; - UNUSED_PARAMETER(pFile); - if( (count+1)*1000 > tmout ){ - return 0; - } - sqlite3OsSleep(db->pVfs, 1000000); - return 1; -#endif + va_end(ap); +#endif /* SQLITE_UNTESTABLE */ + return rc; } /* -** Invoke the given busy handler. -** -** This routine is called when an operation failed to acquire a -** lock on VFS file pFile. +** The Pager stores the Database filename, Journal filename, and WAL filename +** consecutively in memory, in that order. The database filename is prefixed +** by four zero bytes. Locate the start of the database filename by searching +** backwards for the first byte following four consecutive zero bytes. ** -** If this routine returns non-zero, the lock is retried. If it -** returns 0, the operation aborts with an SQLITE_BUSY error. +** This only works if the filename passed in was obtained from the Pager. */ -SQLITE_PRIVATE int sqlite3InvokeBusyHandler(BusyHandler *p, sqlite3_file *pFile){ - int rc; - if( p->xBusyHandler==0 || p->nBusy<0 ) return 0; - if( p->bExtraFileArg ){ - /* Add an extra parameter with the pFile pointer to the end of the - ** callback argument list */ - int (*xTra)(void*,int,sqlite3_file*); - xTra = (int(*)(void*,int,sqlite3_file*))p->xBusyHandler; - rc = xTra(p->pBusyArg, p->nBusy, pFile); - }else{ - /* Legacy style busy handler callback */ - rc = p->xBusyHandler(p->pBusyArg, p->nBusy); - } - if( rc==0 ){ - p->nBusy = -1; - }else{ - p->nBusy++; +static const char *databaseName(const char *zName){ + while( zName[-1]!=0 || zName[-2]!=0 || zName[-3]!=0 || zName[-4]!=0 ){ + zName--; } - return rc; + return zName; } /* -** This routine sets the busy callback for an Sqlite database to the -** given callback function with the given argument. +** Append text z[] to the end of p[]. Return a pointer to the first +** character after then zero terminator on the new text in p[]. */ -SQLITE_API int sqlite3_busy_handler( - sqlite3 *db, - int (*xBusy)(void*,int), - void *pArg -){ -#ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; -#endif - sqlite3_mutex_enter(db->mutex); - db->busyHandler.xBusyHandler = xBusy; - db->busyHandler.pBusyArg = pArg; - db->busyHandler.nBusy = 0; - db->busyHandler.bExtraFileArg = 0; - db->busyTimeout = 0; - sqlite3_mutex_leave(db->mutex); - return SQLITE_OK; +static char *appendText(char *p, const char *z){ + size_t n = strlen(z); + memcpy(p, z, n+1); + return p+n+1; } -#ifndef SQLITE_OMIT_PROGRESS_CALLBACK /* -** This routine sets the progress callback for an Sqlite database to the -** given callback function with the given argument. The progress callback will -** be invoked every nOps opcodes. +** Allocate memory to hold names for a database, journal file, WAL file, +** and query parameters. The pointer returned is valid for use by +** sqlite3_filename_database() and sqlite3_uri_parameter() and related +** functions. +** +** Memory layout must be compatible with that generated by the pager +** and expected by sqlite3_uri_parameter() and databaseName(). */ -SQLITE_API void sqlite3_progress_handler( - sqlite3 *db, - int nOps, - int (*xProgress)(void*), - void *pArg +SQLITE_API char *sqlite3_create_filename( + const char *zDatabase, + const char *zJournal, + const char *zWal, + int nParam, + const char **azParam ){ -#ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) ){ - (void)SQLITE_MISUSE_BKPT; - return; + sqlite3_int64 nByte; + int i; + char *pResult, *p; + nByte = strlen(zDatabase) + strlen(zJournal) + strlen(zWal) + 10; + for(i=0; imutex); - if( nOps>0 ){ - db->xProgress = xProgress; - db->nProgressOps = (unsigned)nOps; - db->pProgressArg = pArg; - }else{ - db->xProgress = 0; - db->nProgressOps = 0; - db->pProgressArg = 0; + pResult = p = sqlite3_malloc64( nByte ); + if( p==0 ) return 0; + memset(p, 0, 4); + p += 4; + p = appendText(p, zDatabase); + for(i=0; imutex); + *(p++) = 0; + p = appendText(p, zJournal); + p = appendText(p, zWal); + *(p++) = 0; + *(p++) = 0; + assert( (sqlite3_int64)(p - pResult)==nByte ); + return pResult + 4; } -#endif - /* -** This routine installs a default busy handler that waits for the -** specified number of milliseconds before returning 0. +** Free memory obtained from sqlite3_create_filename(). It is a severe +** error to call this routine with any parameter other than a pointer +** previously obtained from sqlite3_create_filename() or a NULL pointer. */ -SQLITE_API int sqlite3_busy_timeout(sqlite3 *db, int ms){ -#ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; -#endif - if( ms>0 ){ - sqlite3_busy_handler(db, (int(*)(void*,int))sqliteDefaultBusyCallback, - (void*)db); - db->busyTimeout = ms; - db->busyHandler.bExtraFileArg = 1; - }else{ - sqlite3_busy_handler(db, 0, 0); - } - return SQLITE_OK; +SQLITE_API void sqlite3_free_filename(char *p){ + if( p==0 ) return; + p = (char*)databaseName(p); + sqlite3_free(p - 4); } + /* -** Cause any pending operation to stop at its earliest opportunity. +** This is a utility routine, useful to VFS implementations, that checks +** to see if a database file was a URI that contained a specific query +** parameter, and if so obtains the value of the query parameter. +** +** The zFilename argument is the filename pointer passed into the xOpen() +** method of a VFS implementation. The zParam argument is the name of the +** query parameter we seek. This routine returns the value of the zParam +** parameter if it exists. If the parameter does not exist, this routine +** returns a NULL pointer. */ -SQLITE_API void sqlite3_interrupt(sqlite3 *db){ -#ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) && (db==0 || db->magic!=SQLITE_MAGIC_ZOMBIE) ){ - (void)SQLITE_MISUSE_BKPT; - return; - } -#endif - db->u1.isInterrupted = 1; +SQLITE_API const char *sqlite3_uri_parameter(const char *zFilename, const char *zParam){ + if( zFilename==0 || zParam==0 ) return 0; + zFilename = databaseName(zFilename); + return uriParameter(zFilename, zParam); } - /* -** This function is exactly the same as sqlite3_create_function(), except -** that it is designed to be called by internal code. The difference is -** that if a malloc() fails in sqlite3_create_function(), an error code -** is returned and the mallocFailed flag cleared. +** Return a pointer to the name of Nth query parameter of the filename. */ -SQLITE_PRIVATE int sqlite3CreateFunc( - sqlite3 *db, - const char *zFunctionName, - int nArg, - int enc, - void *pUserData, - void (*xSFunc)(sqlite3_context*,int,sqlite3_value **), - void (*xStep)(sqlite3_context*,int,sqlite3_value **), - void (*xFinal)(sqlite3_context*), - FuncDestructor *pDestructor -){ - FuncDef *p; - int nName; - int extraFlags; - - assert( sqlite3_mutex_held(db->mutex) ); - if( zFunctionName==0 || - (xSFunc && (xFinal || xStep)) || - (!xSFunc && (xFinal && !xStep)) || - (!xSFunc && (!xFinal && xStep)) || - (nArg<-1 || nArg>SQLITE_MAX_FUNCTION_ARG) || - (255<(nName = sqlite3Strlen30( zFunctionName))) ){ - return SQLITE_MISUSE_BKPT; - } - - assert( SQLITE_FUNC_CONSTANT==SQLITE_DETERMINISTIC ); - extraFlags = enc & SQLITE_DETERMINISTIC; - enc &= (SQLITE_FUNC_ENCMASK|SQLITE_ANY); - -#ifndef SQLITE_OMIT_UTF16 - /* If SQLITE_UTF16 is specified as the encoding type, transform this - ** to one of SQLITE_UTF16LE or SQLITE_UTF16BE using the - ** SQLITE_UTF16NATIVE macro. SQLITE_UTF16 is not used internally. - ** - ** If SQLITE_ANY is specified, add three versions of the function - ** to the hash table. - */ - if( enc==SQLITE_UTF16 ){ - enc = SQLITE_UTF16NATIVE; - }else if( enc==SQLITE_ANY ){ - int rc; - rc = sqlite3CreateFunc(db, zFunctionName, nArg, SQLITE_UTF8|extraFlags, - pUserData, xSFunc, xStep, xFinal, pDestructor); - if( rc==SQLITE_OK ){ - rc = sqlite3CreateFunc(db, zFunctionName, nArg, SQLITE_UTF16LE|extraFlags, - pUserData, xSFunc, xStep, xFinal, pDestructor); - } - if( rc!=SQLITE_OK ){ - return rc; - } - enc = SQLITE_UTF16BE; - } -#else - enc = SQLITE_UTF8; -#endif - - /* Check if an existing function is being overridden or deleted. If so, - ** and there are active VMs, then return SQLITE_BUSY. If a function - ** is being overridden/deleted but there are no active VMs, allow the - ** operation to continue but invalidate all precompiled statements. - */ - p = sqlite3FindFunction(db, zFunctionName, nArg, (u8)enc, 0); - if( p && (p->funcFlags & SQLITE_FUNC_ENCMASK)==enc && p->nArg==nArg ){ - if( db->nVdbeActive ){ - sqlite3ErrorWithMsg(db, SQLITE_BUSY, - "unable to delete/modify user-function due to active statements"); - assert( !db->mallocFailed ); - return SQLITE_BUSY; - }else{ - sqlite3ExpirePreparedStatements(db); - } - } - - p = sqlite3FindFunction(db, zFunctionName, nArg, (u8)enc, 1); - assert(p || db->mallocFailed); - if( !p ){ - return SQLITE_NOMEM_BKPT; - } - - /* If an older version of the function with a configured destructor is - ** being replaced invoke the destructor function here. */ - functionDestroy(db, p); - - if( pDestructor ){ - pDestructor->nRef++; +SQLITE_API const char *sqlite3_uri_key(const char *zFilename, int N){ + if( zFilename==0 || N<0 ) return 0; + zFilename = databaseName(zFilename); + zFilename += sqlite3Strlen30(zFilename) + 1; + while( zFilename[0] && (N--)>0 ){ + zFilename += sqlite3Strlen30(zFilename) + 1; + zFilename += sqlite3Strlen30(zFilename) + 1; } - p->u.pDestructor = pDestructor; - p->funcFlags = (p->funcFlags & SQLITE_FUNC_ENCMASK) | extraFlags; - testcase( p->funcFlags & SQLITE_DETERMINISTIC ); - p->xSFunc = xSFunc ? xSFunc : xStep; - p->xFinalize = xFinal; - p->pUserData = pUserData; - p->nArg = (u16)nArg; - return SQLITE_OK; + return zFilename[0] ? zFilename : 0; } /* -** Create new user functions. +** Return a boolean value for a query parameter. */ -SQLITE_API int sqlite3_create_function( - sqlite3 *db, - const char *zFunc, - int nArg, - int enc, - void *p, - void (*xSFunc)(sqlite3_context*,int,sqlite3_value **), - void (*xStep)(sqlite3_context*,int,sqlite3_value **), - void (*xFinal)(sqlite3_context*) -){ - return sqlite3_create_function_v2(db, zFunc, nArg, enc, p, xSFunc, xStep, - xFinal, 0); -} - -SQLITE_API int sqlite3_create_function_v2( - sqlite3 *db, - const char *zFunc, - int nArg, - int enc, - void *p, - void (*xSFunc)(sqlite3_context*,int,sqlite3_value **), - void (*xStep)(sqlite3_context*,int,sqlite3_value **), - void (*xFinal)(sqlite3_context*), - void (*xDestroy)(void *) -){ - int rc = SQLITE_ERROR; - FuncDestructor *pArg = 0; - -#ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) ){ - return SQLITE_MISUSE_BKPT; - } -#endif - sqlite3_mutex_enter(db->mutex); - if( xDestroy ){ - pArg = (FuncDestructor *)sqlite3Malloc(sizeof(FuncDestructor)); - if( !pArg ){ - sqlite3OomFault(db); - xDestroy(p); - goto out; - } - pArg->nRef = 0; - pArg->xDestroy = xDestroy; - pArg->pUserData = p; - } - rc = sqlite3CreateFunc(db, zFunc, nArg, enc, p, xSFunc, xStep, xFinal, pArg); - if( pArg && pArg->nRef==0 ){ - assert( rc!=SQLITE_OK ); - xDestroy(p); - sqlite3_free(pArg); - } - - out: - rc = sqlite3ApiExit(db, rc); - sqlite3_mutex_leave(db->mutex); - return rc; -} - -#ifndef SQLITE_OMIT_UTF16 -SQLITE_API int sqlite3_create_function16( - sqlite3 *db, - const void *zFunctionName, - int nArg, - int eTextRep, - void *p, - void (*xSFunc)(sqlite3_context*,int,sqlite3_value**), - void (*xStep)(sqlite3_context*,int,sqlite3_value**), - void (*xFinal)(sqlite3_context*) -){ - int rc; - char *zFunc8; - -#ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) || zFunctionName==0 ) return SQLITE_MISUSE_BKPT; -#endif - sqlite3_mutex_enter(db->mutex); - assert( !db->mallocFailed ); - zFunc8 = sqlite3Utf16to8(db, zFunctionName, -1, SQLITE_UTF16NATIVE); - rc = sqlite3CreateFunc(db, zFunc8, nArg, eTextRep, p, xSFunc,xStep,xFinal,0); - sqlite3DbFree(db, zFunc8); - rc = sqlite3ApiExit(db, rc); - sqlite3_mutex_leave(db->mutex); - return rc; +SQLITE_API int sqlite3_uri_boolean(const char *zFilename, const char *zParam, int bDflt){ + const char *z = sqlite3_uri_parameter(zFilename, zParam); + bDflt = bDflt!=0; + return z ? sqlite3GetBoolean(z, bDflt) : bDflt; } -#endif - /* -** The following is the implementation of an SQL function that always -** fails with an error message stating that the function is used in the -** wrong context. The sqlite3_overload_function() API might construct -** SQL function that use this routine so that the functions will exist -** for name resolution but are actually overloaded by the xFindFunction -** method of virtual tables. +** Return a 64-bit integer value for a query parameter. */ -static void sqlite3InvalidFunction( - sqlite3_context *context, /* The function calling context */ - int NotUsed, /* Number of arguments to the function */ - sqlite3_value **NotUsed2 /* Value of each argument */ +SQLITE_API sqlite3_int64 sqlite3_uri_int64( + const char *zFilename, /* Filename as passed to xOpen */ + const char *zParam, /* URI parameter sought */ + sqlite3_int64 bDflt /* return if parameter is missing */ ){ - const char *zName = (const char*)sqlite3_user_data(context); - char *zErr; - UNUSED_PARAMETER2(NotUsed, NotUsed2); - zErr = sqlite3_mprintf( - "unable to use function %s in the requested context", zName); - sqlite3_result_error(context, zErr, -1); - sqlite3_free(zErr); + const char *z = sqlite3_uri_parameter(zFilename, zParam); + sqlite3_int64 v; + if( z && sqlite3DecOrHexToI64(z, &v)==0 ){ + bDflt = v; + } + return bDflt; } /* -** Declare that a function has been overloaded by a virtual table. -** -** If the function already exists as a regular global function, then -** this routine is a no-op. If the function does not exist, then create -** a new one that always throws a run-time error. +** Translate a filename that was handed to a VFS routine into the corresponding +** database, journal, or WAL file. ** -** When virtual tables intend to provide an overloaded function, they -** should call this routine to make sure the global function exists. -** A global function must exist in order for name resolution to work -** properly. +** It is an error to pass this routine a filename string that was not +** passed into the VFS from the SQLite core. Doing so is similar to +** passing free() a pointer that was not obtained from malloc() - it is +** an error that we cannot easily detect but that will likely cause memory +** corruption. */ -SQLITE_API int sqlite3_overload_function( - sqlite3 *db, - const char *zName, - int nArg -){ - int rc; - char *zCopy; - -#ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) || zName==0 || nArg<-2 ){ - return SQLITE_MISUSE_BKPT; +SQLITE_API const char *sqlite3_filename_database(const char *zFilename){ + return databaseName(zFilename); +} +SQLITE_API const char *sqlite3_filename_journal(const char *zFilename){ + zFilename = databaseName(zFilename); + zFilename += sqlite3Strlen30(zFilename) + 1; + while( zFilename[0] ){ + zFilename += sqlite3Strlen30(zFilename) + 1; + zFilename += sqlite3Strlen30(zFilename) + 1; } + return zFilename + 1; +} +SQLITE_API const char *sqlite3_filename_wal(const char *zFilename){ +#ifdef SQLITE_OMIT_WAL + return 0; +#else + zFilename = sqlite3_filename_journal(zFilename); + zFilename += sqlite3Strlen30(zFilename) + 1; + return zFilename; #endif - sqlite3_mutex_enter(db->mutex); - rc = sqlite3FindFunction(db, zName, nArg, SQLITE_UTF8, 0)!=0; - sqlite3_mutex_leave(db->mutex); - if( rc ) return SQLITE_OK; - zCopy = sqlite3_mprintf(zName); - if( zCopy==0 ) return SQLITE_NOMEM; - return sqlite3_create_function_v2(db, zName, nArg, SQLITE_UTF8, - zCopy, sqlite3InvalidFunction, 0, 0, sqlite3_free); } -#ifndef SQLITE_OMIT_TRACE /* -** Register a trace function. The pArg from the previously registered trace -** is returned. -** -** A NULL trace function means that no tracing is executes. A non-NULL -** trace is a pointer to a function that is invoked at the start of each -** SQL statement. +** Return the Btree pointer identified by zDbName. Return NULL if not found. */ -#ifndef SQLITE_OMIT_DEPRECATED -SQLITE_API void *sqlite3_trace(sqlite3 *db, void(*xTrace)(void*,const char*), void *pArg){ - void *pOld; - -#ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) ){ - (void)SQLITE_MISUSE_BKPT; - return 0; - } -#endif - sqlite3_mutex_enter(db->mutex); - pOld = db->pTraceArg; - db->mTrace = xTrace ? SQLITE_TRACE_LEGACY : 0; - db->xTrace = (int(*)(u32,void*,void*,void*))xTrace; - db->pTraceArg = pArg; - sqlite3_mutex_leave(db->mutex); - return pOld; +SQLITE_PRIVATE Btree *sqlite3DbNameToBtree(sqlite3 *db, const char *zDbName){ + int iDb = zDbName ? sqlite3FindDbName(db, zDbName) : 0; + return iDb<0 ? 0 : db->aDb[iDb].pBt; } -#endif /* SQLITE_OMIT_DEPRECATED */ -/* Register a trace callback using the version-2 interface. +/* +** Return the filename of the database associated with a database +** connection. */ -SQLITE_API int sqlite3_trace_v2( - sqlite3 *db, /* Trace this connection */ - unsigned mTrace, /* Mask of events to be traced */ - int(*xTrace)(unsigned,void*,void*,void*), /* Callback to invoke */ - void *pArg /* Context */ -){ +SQLITE_API const char *sqlite3_db_filename(sqlite3 *db, const char *zDbName){ + Btree *pBt; #ifdef SQLITE_ENABLE_API_ARMOR if( !sqlite3SafetyCheckOk(db) ){ - return SQLITE_MISUSE_BKPT; + (void)SQLITE_MISUSE_BKPT; + return 0; } #endif - sqlite3_mutex_enter(db->mutex); - if( mTrace==0 ) xTrace = 0; - if( xTrace==0 ) mTrace = 0; - db->mTrace = mTrace; - db->xTrace = xTrace; - db->pTraceArg = pArg; - sqlite3_mutex_leave(db->mutex); - return SQLITE_OK; + pBt = sqlite3DbNameToBtree(db, zDbName); + return pBt ? sqlite3BtreeGetFilename(pBt) : 0; } -#ifndef SQLITE_OMIT_DEPRECATED /* -** Register a profile function. The pArg from the previously registered -** profile function is returned. -** -** A NULL profile function means that no profiling is executes. A non-NULL -** profile is a pointer to a function that is invoked at the conclusion of -** each SQL statement that is run. +** Return 1 if database is read-only or 0 if read/write. Return -1 if +** no such database exists. */ -SQLITE_API void *sqlite3_profile( - sqlite3 *db, - void (*xProfile)(void*,const char*,sqlite_uint64), - void *pArg -){ - void *pOld; - +SQLITE_API int sqlite3_db_readonly(sqlite3 *db, const char *zDbName){ + Btree *pBt; #ifdef SQLITE_ENABLE_API_ARMOR if( !sqlite3SafetyCheckOk(db) ){ (void)SQLITE_MISUSE_BKPT; - return 0; + return -1; } #endif - sqlite3_mutex_enter(db->mutex); - pOld = db->pProfileArg; - db->xProfile = xProfile; - db->pProfileArg = pArg; - sqlite3_mutex_leave(db->mutex); - return pOld; + pBt = sqlite3DbNameToBtree(db, zDbName); + return pBt ? sqlite3BtreeIsReadonly(pBt) : -1; } -#endif /* SQLITE_OMIT_DEPRECATED */ -#endif /* SQLITE_OMIT_TRACE */ +#ifdef SQLITE_ENABLE_SNAPSHOT /* -** Register a function to be invoked when a transaction commits. -** If the invoked function returns non-zero, then the commit becomes a -** rollback. +** Obtain a snapshot handle for the snapshot of database zDb currently +** being read by handle db. */ -SQLITE_API void *sqlite3_commit_hook( - sqlite3 *db, /* Attach the hook to this database */ - int (*xCallback)(void*), /* Function to invoke on each commit */ - void *pArg /* Argument to the function */ +SQLITE_API int sqlite3_snapshot_get( + sqlite3 *db, + const char *zDb, + sqlite3_snapshot **ppSnapshot ){ - void *pOld; + int rc = SQLITE_ERROR; +#ifndef SQLITE_OMIT_WAL #ifdef SQLITE_ENABLE_API_ARMOR if( !sqlite3SafetyCheckOk(db) ){ - (void)SQLITE_MISUSE_BKPT; - return 0; + return SQLITE_MISUSE_BKPT; } #endif sqlite3_mutex_enter(db->mutex); - pOld = db->pCommitArg; - db->xCommitCallback = xCallback; - db->pCommitArg = pArg; + + if( db->autoCommit==0 ){ + int iDb = sqlite3FindDbName(db, zDb); + if( iDb==0 || iDb>1 ){ + Btree *pBt = db->aDb[iDb].pBt; + if( 0==sqlite3BtreeIsInTrans(pBt) ){ + rc = sqlite3BtreeBeginTrans(pBt, 0, 0); + if( rc==SQLITE_OK ){ + rc = sqlite3PagerSnapshotGet(sqlite3BtreePager(pBt), ppSnapshot); + } + } + } + } + sqlite3_mutex_leave(db->mutex); - return pOld; +#endif /* SQLITE_OMIT_WAL */ + return rc; } /* -** Register a callback to be invoked each time a row is updated, -** inserted or deleted using this database connection. +** Open a read-transaction on the snapshot idendified by pSnapshot. */ -SQLITE_API void *sqlite3_update_hook( - sqlite3 *db, /* Attach the hook to this database */ - void (*xCallback)(void*,int,char const *,char const *,sqlite_int64), - void *pArg /* Argument to the function */ +SQLITE_API int sqlite3_snapshot_open( + sqlite3 *db, + const char *zDb, + sqlite3_snapshot *pSnapshot ){ - void *pRet; + int rc = SQLITE_ERROR; +#ifndef SQLITE_OMIT_WAL #ifdef SQLITE_ENABLE_API_ARMOR if( !sqlite3SafetyCheckOk(db) ){ - (void)SQLITE_MISUSE_BKPT; - return 0; + return SQLITE_MISUSE_BKPT; } #endif sqlite3_mutex_enter(db->mutex); - pRet = db->pUpdateArg; - db->xUpdateCallback = xCallback; - db->pUpdateArg = pArg; + if( db->autoCommit==0 ){ + int iDb; + iDb = sqlite3FindDbName(db, zDb); + if( iDb==0 || iDb>1 ){ + Btree *pBt = db->aDb[iDb].pBt; + if( sqlite3BtreeIsInTrans(pBt)==0 ){ + Pager *pPager = sqlite3BtreePager(pBt); + int bUnlock = 0; + if( sqlite3BtreeIsInReadTrans(pBt) ){ + if( db->nVdbeActive==0 ){ + rc = sqlite3PagerSnapshotCheck(pPager, pSnapshot); + if( rc==SQLITE_OK ){ + bUnlock = 1; + rc = sqlite3BtreeCommit(pBt); + } + } + }else{ + rc = SQLITE_OK; + } + if( rc==SQLITE_OK ){ + rc = sqlite3PagerSnapshotOpen(pPager, pSnapshot); + } + if( rc==SQLITE_OK ){ + rc = sqlite3BtreeBeginTrans(pBt, 0, 0); + sqlite3PagerSnapshotOpen(pPager, 0); + } + if( bUnlock ){ + sqlite3PagerSnapshotUnlock(pPager); + } + } + } + } + sqlite3_mutex_leave(db->mutex); - return pRet; +#endif /* SQLITE_OMIT_WAL */ + return rc; } /* -** Register a callback to be invoked each time a transaction is rolled -** back by this database connection. +** Recover as many snapshots as possible from the wal file associated with +** schema zDb of database db. */ -SQLITE_API void *sqlite3_rollback_hook( - sqlite3 *db, /* Attach the hook to this database */ - void (*xCallback)(void*), /* Callback function */ - void *pArg /* Argument to the function */ -){ - void *pRet; +SQLITE_API int sqlite3_snapshot_recover(sqlite3 *db, const char *zDb){ + int rc = SQLITE_ERROR; + int iDb; +#ifndef SQLITE_OMIT_WAL #ifdef SQLITE_ENABLE_API_ARMOR if( !sqlite3SafetyCheckOk(db) ){ - (void)SQLITE_MISUSE_BKPT; - return 0; + return SQLITE_MISUSE_BKPT; } #endif - sqlite3_mutex_enter(db->mutex); - pRet = db->pRollbackArg; - db->xRollbackCallback = xCallback; - db->pRollbackArg = pArg; - sqlite3_mutex_leave(db->mutex); - return pRet; -} -#ifdef SQLITE_ENABLE_PREUPDATE_HOOK -/* -** Register a callback to be invoked each time a row is updated, -** inserted or deleted using this database connection. -*/ -SQLITE_API void *sqlite3_preupdate_hook( - sqlite3 *db, /* Attach the hook to this database */ - void(*xCallback)( /* Callback function */ - void*,sqlite3*,int,char const*,char const*,sqlite3_int64,sqlite3_int64), - void *pArg /* First callback argument */ -){ - void *pRet; sqlite3_mutex_enter(db->mutex); - pRet = db->pPreUpdateArg; - db->xPreUpdateCallback = xCallback; - db->pPreUpdateArg = pArg; + iDb = sqlite3FindDbName(db, zDb); + if( iDb==0 || iDb>1 ){ + Btree *pBt = db->aDb[iDb].pBt; + if( 0==sqlite3BtreeIsInReadTrans(pBt) ){ + rc = sqlite3BtreeBeginTrans(pBt, 0, 0); + if( rc==SQLITE_OK ){ + rc = sqlite3PagerSnapshotRecover(sqlite3BtreePager(pBt)); + sqlite3BtreeCommit(pBt); + } + } + } sqlite3_mutex_leave(db->mutex); - return pRet; +#endif /* SQLITE_OMIT_WAL */ + return rc; } -#endif /* SQLITE_ENABLE_PREUPDATE_HOOK */ -#ifndef SQLITE_OMIT_WAL /* -** The sqlite3_wal_hook() callback registered by sqlite3_wal_autocheckpoint(). -** Invoke sqlite3_wal_checkpoint if the number of frames in the log file -** is greater than sqlite3.pWalArg cast to an integer (the value configured by -** wal_autocheckpoint()). -*/ -SQLITE_PRIVATE int sqlite3WalDefaultHook( - void *pClientData, /* Argument */ - sqlite3 *db, /* Connection */ - const char *zDb, /* Database */ - int nFrame /* Size of WAL */ -){ - if( nFrame>=SQLITE_PTR_TO_INT(pClientData) ){ - sqlite3BeginBenignMalloc(); - sqlite3_wal_checkpoint(db, zDb); - sqlite3EndBenignMalloc(); - } - return SQLITE_OK; +** Free a snapshot handle obtained from sqlite3_snapshot_get(). +*/ +SQLITE_API void sqlite3_snapshot_free(sqlite3_snapshot *pSnapshot){ + sqlite3_free(pSnapshot); } -#endif /* SQLITE_OMIT_WAL */ +#endif /* SQLITE_ENABLE_SNAPSHOT */ +#ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS /* -** Configure an sqlite3_wal_hook() callback to automatically checkpoint -** a database after committing a transaction if there are nFrame or -** more frames in the log file. Passing zero or a negative value as the -** nFrame parameter disables automatic checkpoints entirely. +** Given the name of a compile-time option, return true if that option +** was used and false if not. ** -** The callback registered by this function replaces any existing callback -** registered using sqlite3_wal_hook(). Likewise, registering a callback -** using sqlite3_wal_hook() disables the automatic checkpoint mechanism -** configured by this function. +** The name can optionally begin with "SQLITE_" but the "SQLITE_" prefix +** is not required for a match. */ -SQLITE_API int sqlite3_wal_autocheckpoint(sqlite3 *db, int nFrame){ -#ifdef SQLITE_OMIT_WAL - UNUSED_PARAMETER(db); - UNUSED_PARAMETER(nFrame); -#else -#ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; -#endif - if( nFrame>0 ){ - sqlite3_wal_hook(db, sqlite3WalDefaultHook, SQLITE_INT_TO_PTR(nFrame)); - }else{ - sqlite3_wal_hook(db, 0, 0); +SQLITE_API int sqlite3_compileoption_used(const char *zOptName){ + int i, n; + int nOpt; + const char **azCompileOpt; + +#if SQLITE_ENABLE_API_ARMOR + if( zOptName==0 ){ + (void)SQLITE_MISUSE_BKPT; + return 0; } #endif - return SQLITE_OK; + + azCompileOpt = sqlite3CompileOptions(&nOpt); + + if( sqlite3StrNICmp(zOptName, "SQLITE_", 7)==0 ) zOptName += 7; + n = sqlite3Strlen30(zOptName); + + /* Since nOpt is normally in single digits, a linear search is + ** adequate. No need for a binary search. */ + for(i=0; i=0 && Nmutex); - pRet = db->pWalArg; - db->xWalCallback = xCallback; - db->pWalArg = pArg; - sqlite3_mutex_leave(db->mutex); - return pRet; -#else return 0; -#endif } +#endif /* SQLITE_OMIT_COMPILEOPTION_DIAGS */ +/************** End of main.c ************************************************/ +/************** Begin file notify.c ******************************************/ /* -** Checkpoint database zDb. +** 2009 March 3 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains the implementation of the sqlite3_unlock_notify() +** API method and its associated functionality. */ -SQLITE_API int sqlite3_wal_checkpoint_v2( - sqlite3 *db, /* Database handle */ - const char *zDb, /* Name of attached database (or NULL) */ - int eMode, /* SQLITE_CHECKPOINT_* value */ - int *pnLog, /* OUT: Size of WAL log in frames */ - int *pnCkpt /* OUT: Total number of frames checkpointed */ -){ -#ifdef SQLITE_OMIT_WAL - return SQLITE_OK; -#else - int rc; /* Return code */ - int iDb = SQLITE_MAX_ATTACHED; /* sqlite3.aDb[] index of db to checkpoint */ - -#ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; -#endif - - /* Initialize the output variables to -1 in case an error occurs. */ - if( pnLog ) *pnLog = -1; - if( pnCkpt ) *pnCkpt = -1; - - assert( SQLITE_CHECKPOINT_PASSIVE==0 ); - assert( SQLITE_CHECKPOINT_FULL==1 ); - assert( SQLITE_CHECKPOINT_RESTART==2 ); - assert( SQLITE_CHECKPOINT_TRUNCATE==3 ); - if( eModeSQLITE_CHECKPOINT_TRUNCATE ){ - /* EVIDENCE-OF: R-03996-12088 The M parameter must be a valid checkpoint - ** mode: */ - return SQLITE_MISUSE; - } - - sqlite3_mutex_enter(db->mutex); - if( zDb && zDb[0] ){ - iDb = sqlite3FindDbName(db, zDb); - } - if( iDb<0 ){ - rc = SQLITE_ERROR; - sqlite3ErrorWithMsg(db, SQLITE_ERROR, "unknown database: %s", zDb); - }else{ - db->busyHandler.nBusy = 0; - rc = sqlite3Checkpoint(db, iDb, eMode, pnLog, pnCkpt); - sqlite3Error(db, rc); - } - rc = sqlite3ApiExit(db, rc); +/* #include "sqliteInt.h" */ +/* #include "btreeInt.h" */ - /* If there are no active statements, clear the interrupt flag at this - ** point. */ - if( db->nVdbeActive==0 ){ - db->u1.isInterrupted = 0; - } +/* Omit this entire file if SQLITE_ENABLE_UNLOCK_NOTIFY is not defined. */ +#ifdef SQLITE_ENABLE_UNLOCK_NOTIFY - sqlite3_mutex_leave(db->mutex); - return rc; -#endif -} +/* +** Public interfaces: +** +** sqlite3ConnectionBlocked() +** sqlite3ConnectionUnlocked() +** sqlite3ConnectionClosed() +** sqlite3_unlock_notify() +*/ +#define assertMutexHeld() \ + assert( sqlite3_mutex_held(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)) ) /* -** Checkpoint database zDb. If zDb is NULL, or if the buffer zDb points -** to contains a zero-length string, all attached databases are -** checkpointed. +** Head of a linked list of all sqlite3 objects created by this process +** for which either sqlite3.pBlockingConnection or sqlite3.pUnlockConnection +** is not NULL. This variable may only accessed while the STATIC_MASTER +** mutex is held. */ -SQLITE_API int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb){ - /* EVIDENCE-OF: R-41613-20553 The sqlite3_wal_checkpoint(D,X) is equivalent to - ** sqlite3_wal_checkpoint_v2(D,X,SQLITE_CHECKPOINT_PASSIVE,0,0). */ - return sqlite3_wal_checkpoint_v2(db,zDb,SQLITE_CHECKPOINT_PASSIVE,0,0); -} +static sqlite3 *SQLITE_WSD sqlite3BlockedList = 0; -#ifndef SQLITE_OMIT_WAL +#ifndef NDEBUG /* -** Run a checkpoint on database iDb. This is a no-op if database iDb is -** not currently open in WAL mode. -** -** If a transaction is open on the database being checkpointed, this -** function returns SQLITE_LOCKED and a checkpoint is not attempted. If -** an error occurs while running the checkpoint, an SQLite error code is -** returned (i.e. SQLITE_IOERR). Otherwise, SQLITE_OK. +** This function is a complex assert() that verifies the following +** properties of the blocked connections list: ** -** The mutex on database handle db should be held by the caller. The mutex -** associated with the specific b-tree being checkpointed is taken by -** this function while the checkpoint is running. +** 1) Each entry in the list has a non-NULL value for either +** pUnlockConnection or pBlockingConnection, or both. ** -** If iDb is passed SQLITE_MAX_ATTACHED, then all attached databases are -** checkpointed. If an error is encountered it is returned immediately - -** no attempt is made to checkpoint any remaining databases. +** 2) All entries in the list that share a common value for +** xUnlockNotify are grouped together. ** -** Parameter eMode is one of SQLITE_CHECKPOINT_PASSIVE, FULL, RESTART -** or TRUNCATE. +** 3) If the argument db is not NULL, then none of the entries in the +** blocked connections list have pUnlockConnection or pBlockingConnection +** set to db. This is used when closing connection db. */ -SQLITE_PRIVATE int sqlite3Checkpoint(sqlite3 *db, int iDb, int eMode, int *pnLog, int *pnCkpt){ - int rc = SQLITE_OK; /* Return code */ - int i; /* Used to iterate through attached dbs */ - int bBusy = 0; /* True if SQLITE_BUSY has been encountered */ +static void checkListProperties(sqlite3 *db){ + sqlite3 *p; + for(p=sqlite3BlockedList; p; p=p->pNextBlocked){ + int seen = 0; + sqlite3 *p2; - assert( sqlite3_mutex_held(db->mutex) ); - assert( !pnLog || *pnLog==-1 ); - assert( !pnCkpt || *pnCkpt==-1 ); + /* Verify property (1) */ + assert( p->pUnlockConnection || p->pBlockingConnection ); - for(i=0; inDb && rc==SQLITE_OK; i++){ - if( i==iDb || iDb==SQLITE_MAX_ATTACHED ){ - rc = sqlite3BtreeCheckpoint(db->aDb[i].pBt, eMode, pnLog, pnCkpt); - pnLog = 0; - pnCkpt = 0; - if( rc==SQLITE_BUSY ){ - bBusy = 1; - rc = SQLITE_OK; - } + /* Verify property (2) */ + for(p2=sqlite3BlockedList; p2!=p; p2=p2->pNextBlocked){ + if( p2->xUnlockNotify==p->xUnlockNotify ) seen = 1; + assert( p2->xUnlockNotify==p->xUnlockNotify || !seen ); + assert( db==0 || p->pUnlockConnection!=db ); + assert( db==0 || p->pBlockingConnection!=db ); + } + } +} +#else +# define checkListProperties(x) +#endif + +/* +** Remove connection db from the blocked connections list. If connection +** db is not currently a part of the list, this function is a no-op. +*/ +static void removeFromBlockedList(sqlite3 *db){ + sqlite3 **pp; + assertMutexHeld(); + for(pp=&sqlite3BlockedList; *pp; pp = &(*pp)->pNextBlocked){ + if( *pp==db ){ + *pp = (*pp)->pNextBlocked; + break; } } +} - return (rc==SQLITE_OK && bBusy) ? SQLITE_BUSY : rc; +/* +** Add connection db to the blocked connections list. It is assumed +** that it is not already a part of the list. +*/ +static void addToBlockedList(sqlite3 *db){ + sqlite3 **pp; + assertMutexHeld(); + for( + pp=&sqlite3BlockedList; + *pp && (*pp)->xUnlockNotify!=db->xUnlockNotify; + pp=&(*pp)->pNextBlocked + ); + db->pNextBlocked = *pp; + *pp = db; } -#endif /* SQLITE_OMIT_WAL */ /* -** This function returns true if main-memory should be used instead of -** a temporary file for transient pager files and statement journals. -** The value returned depends on the value of db->temp_store (runtime -** parameter) and the compile time value of SQLITE_TEMP_STORE. The -** following table describes the relationship between these two values -** and this functions return value. -** -** SQLITE_TEMP_STORE db->temp_store Location of temporary database -** ----------------- -------------- ------------------------------ -** 0 any file (return 0) -** 1 1 file (return 0) -** 1 2 memory (return 1) -** 1 0 file (return 0) -** 2 1 file (return 0) -** 2 2 memory (return 1) -** 2 0 memory (return 1) -** 3 any memory (return 1) +** Obtain the STATIC_MASTER mutex. */ -SQLITE_PRIVATE int sqlite3TempInMemory(const sqlite3 *db){ -#if SQLITE_TEMP_STORE==1 - return ( db->temp_store==2 ); -#endif -#if SQLITE_TEMP_STORE==2 - return ( db->temp_store!=1 ); -#endif -#if SQLITE_TEMP_STORE==3 - UNUSED_PARAMETER(db); - return 1; -#endif -#if SQLITE_TEMP_STORE<1 || SQLITE_TEMP_STORE>3 - UNUSED_PARAMETER(db); - return 0; -#endif +static void enterMutex(void){ + sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); + checkListProperties(0); } /* -** Return UTF-8 encoded English language explanation of the most recent -** error. +** Release the STATIC_MASTER mutex. */ -SQLITE_API const char *sqlite3_errmsg(sqlite3 *db){ - const char *z; - if( !db ){ - return sqlite3ErrStr(SQLITE_NOMEM_BKPT); - } - if( !sqlite3SafetyCheckSickOrOk(db) ){ - return sqlite3ErrStr(SQLITE_MISUSE_BKPT); - } - sqlite3_mutex_enter(db->mutex); - if( db->mallocFailed ){ - z = sqlite3ErrStr(SQLITE_NOMEM_BKPT); - }else{ - testcase( db->pErr==0 ); - z = (char*)sqlite3_value_text(db->pErr); - assert( !db->mallocFailed ); - if( z==0 ){ - z = sqlite3ErrStr(db->errCode); - } - } - sqlite3_mutex_leave(db->mutex); - return z; +static void leaveMutex(void){ + assertMutexHeld(); + checkListProperties(0); + sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); } -#ifndef SQLITE_OMIT_UTF16 /* -** Return UTF-16 encoded English language explanation of the most recent -** error. +** Register an unlock-notify callback. +** +** This is called after connection "db" has attempted some operation +** but has received an SQLITE_LOCKED error because another connection +** (call it pOther) in the same process was busy using the same shared +** cache. pOther is found by looking at db->pBlockingConnection. +** +** If there is no blocking connection, the callback is invoked immediately, +** before this routine returns. +** +** If pOther is already blocked on db, then report SQLITE_LOCKED, to indicate +** a deadlock. +** +** Otherwise, make arrangements to invoke xNotify when pOther drops +** its locks. +** +** Each call to this routine overrides any prior callbacks registered +** on the same "db". If xNotify==0 then any prior callbacks are immediately +** cancelled. */ -SQLITE_API const void *sqlite3_errmsg16(sqlite3 *db){ - static const u16 outOfMem[] = { - 'o', 'u', 't', ' ', 'o', 'f', ' ', 'm', 'e', 'm', 'o', 'r', 'y', 0 - }; - static const u16 misuse[] = { - 'b', 'a', 'd', ' ', 'p', 'a', 'r', 'a', 'm', 'e', 't', 'e', 'r', ' ', - 'o', 'r', ' ', 'o', 't', 'h', 'e', 'r', ' ', 'A', 'P', 'I', ' ', - 'm', 'i', 's', 'u', 's', 'e', 0 - }; +SQLITE_API int sqlite3_unlock_notify( + sqlite3 *db, + void (*xNotify)(void **, int), + void *pArg +){ + int rc = SQLITE_OK; - const void *z; - if( !db ){ - return (void *)outOfMem; - } - if( !sqlite3SafetyCheckSickOrOk(db) ){ - return (void *)misuse; - } sqlite3_mutex_enter(db->mutex); - if( db->mallocFailed ){ - z = (void *)outOfMem; + enterMutex(); + + if( xNotify==0 ){ + removeFromBlockedList(db); + db->pBlockingConnection = 0; + db->pUnlockConnection = 0; + db->xUnlockNotify = 0; + db->pUnlockArg = 0; + }else if( 0==db->pBlockingConnection ){ + /* The blocking transaction has been concluded. Or there never was a + ** blocking transaction. In either case, invoke the notify callback + ** immediately. + */ + xNotify(&pArg, 1); }else{ - z = sqlite3_value_text16(db->pErr); - if( z==0 ){ - sqlite3ErrorWithMsg(db, db->errCode, sqlite3ErrStr(db->errCode)); - z = sqlite3_value_text16(db->pErr); + sqlite3 *p; + + for(p=db->pBlockingConnection; p && p!=db; p=p->pUnlockConnection){} + if( p ){ + rc = SQLITE_LOCKED; /* Deadlock detected. */ + }else{ + db->pUnlockConnection = db->pBlockingConnection; + db->xUnlockNotify = xNotify; + db->pUnlockArg = pArg; + removeFromBlockedList(db); + addToBlockedList(db); } - /* A malloc() may have failed within the call to sqlite3_value_text16() - ** above. If this is the case, then the db->mallocFailed flag needs to - ** be cleared before returning. Do this directly, instead of via - ** sqlite3ApiExit(), to avoid setting the database handle error message. - */ - sqlite3OomClear(db); } + + leaveMutex(); + assert( !db->mallocFailed ); + sqlite3ErrorWithMsg(db, rc, (rc?"database is deadlocked":0)); sqlite3_mutex_leave(db->mutex); - return z; + return rc; } -#endif /* SQLITE_OMIT_UTF16 */ /* -** Return the most recent error code generated by an SQLite routine. If NULL is -** passed to this function, we assume a malloc() failed during sqlite3_open(). +** This function is called while stepping or preparing a statement +** associated with connection db. The operation will return SQLITE_LOCKED +** to the user because it requires a lock that will not be available +** until connection pBlocker concludes its current transaction. */ -SQLITE_API int sqlite3_errcode(sqlite3 *db){ - if( db && !sqlite3SafetyCheckSickOrOk(db) ){ - return SQLITE_MISUSE_BKPT; - } - if( !db || db->mallocFailed ){ - return SQLITE_NOMEM_BKPT; - } - return db->errCode & db->errMask; -} -SQLITE_API int sqlite3_extended_errcode(sqlite3 *db){ - if( db && !sqlite3SafetyCheckSickOrOk(db) ){ - return SQLITE_MISUSE_BKPT; - } - if( !db || db->mallocFailed ){ - return SQLITE_NOMEM_BKPT; +SQLITE_PRIVATE void sqlite3ConnectionBlocked(sqlite3 *db, sqlite3 *pBlocker){ + enterMutex(); + if( db->pBlockingConnection==0 && db->pUnlockConnection==0 ){ + addToBlockedList(db); } - return db->errCode; + db->pBlockingConnection = pBlocker; + leaveMutex(); } -SQLITE_API int sqlite3_system_errno(sqlite3 *db){ - return db ? db->iSysErrno : 0; -} /* -** Return a string that describes the kind of error specified in the -** argument. For now, this simply calls the internal sqlite3ErrStr() -** function. +** This function is called when +** the transaction opened by database db has just finished. Locks held +** by database connection db have been released. +** +** This function loops through each entry in the blocked connections +** list and does the following: +** +** 1) If the sqlite3.pBlockingConnection member of a list entry is +** set to db, then set pBlockingConnection=0. +** +** 2) If the sqlite3.pUnlockConnection member of a list entry is +** set to db, then invoke the configured unlock-notify callback and +** set pUnlockConnection=0. +** +** 3) If the two steps above mean that pBlockingConnection==0 and +** pUnlockConnection==0, remove the entry from the blocked connections +** list. */ -SQLITE_API const char *sqlite3_errstr(int rc){ - return sqlite3ErrStr(rc); -} +SQLITE_PRIVATE void sqlite3ConnectionUnlocked(sqlite3 *db){ + void (*xUnlockNotify)(void **, int) = 0; /* Unlock-notify cb to invoke */ + int nArg = 0; /* Number of entries in aArg[] */ + sqlite3 **pp; /* Iterator variable */ + void **aArg; /* Arguments to the unlock callback */ + void **aDyn = 0; /* Dynamically allocated space for aArg[] */ + void *aStatic[16]; /* Starter space for aArg[]. No malloc required */ -/* -** Create a new collating function for database "db". The name is zName -** and the encoding is enc. -*/ -static int createCollation( - sqlite3* db, - const char *zName, - u8 enc, - void* pCtx, - int(*xCompare)(void*,int,const void*,int,const void*), - void(*xDel)(void*) -){ - CollSeq *pColl; - int enc2; - - assert( sqlite3_mutex_held(db->mutex) ); + aArg = aStatic; + enterMutex(); /* Enter STATIC_MASTER mutex */ - /* If SQLITE_UTF16 is specified as the encoding type, transform this - ** to one of SQLITE_UTF16LE or SQLITE_UTF16BE using the - ** SQLITE_UTF16NATIVE macro. SQLITE_UTF16 is not used internally. - */ - enc2 = enc; - testcase( enc2==SQLITE_UTF16 ); - testcase( enc2==SQLITE_UTF16_ALIGNED ); - if( enc2==SQLITE_UTF16 || enc2==SQLITE_UTF16_ALIGNED ){ - enc2 = SQLITE_UTF16NATIVE; - } - if( enc2SQLITE_UTF16BE ){ - return SQLITE_MISUSE_BKPT; - } + /* This loop runs once for each entry in the blocked-connections list. */ + for(pp=&sqlite3BlockedList; *pp; /* no-op */ ){ + sqlite3 *p = *pp; - /* Check if this call is removing or replacing an existing collation - ** sequence. If so, and there are active VMs, return busy. If there - ** are no active VMs, invalidate any pre-compiled statements. - */ - pColl = sqlite3FindCollSeq(db, (u8)enc2, zName, 0); - if( pColl && pColl->xCmp ){ - if( db->nVdbeActive ){ - sqlite3ErrorWithMsg(db, SQLITE_BUSY, - "unable to delete/modify collation sequence due to active statements"); - return SQLITE_BUSY; + /* Step 1. */ + if( p->pBlockingConnection==db ){ + p->pBlockingConnection = 0; } - sqlite3ExpirePreparedStatements(db); - /* If collation sequence pColl was created directly by a call to - ** sqlite3_create_collation, and not generated by synthCollSeq(), - ** then any copies made by synthCollSeq() need to be invalidated. - ** Also, collation destructor - CollSeq.xDel() - function may need - ** to be called. - */ - if( (pColl->enc & ~SQLITE_UTF16_ALIGNED)==enc2 ){ - CollSeq *aColl = sqlite3HashFind(&db->aCollSeq, zName); - int j; - for(j=0; j<3; j++){ - CollSeq *p = &aColl[j]; - if( p->enc==pColl->enc ){ - if( p->xDel ){ - p->xDel(p->pUser); - } - p->xCmp = 0; + /* Step 2. */ + if( p->pUnlockConnection==db ){ + assert( p->xUnlockNotify ); + if( p->xUnlockNotify!=xUnlockNotify && nArg!=0 ){ + xUnlockNotify(aArg, nArg); + nArg = 0; + } + + sqlite3BeginBenignMalloc(); + assert( aArg==aDyn || (aDyn==0 && aArg==aStatic) ); + assert( nArg<=(int)ArraySize(aStatic) || aArg==aDyn ); + if( (!aDyn && nArg==(int)ArraySize(aStatic)) + || (aDyn && nArg==(int)(sqlite3MallocSize(aDyn)/sizeof(void*))) + ){ + /* The aArg[] array needs to grow. */ + void **pNew = (void **)sqlite3Malloc(nArg*sizeof(void *)*2); + if( pNew ){ + memcpy(pNew, aArg, nArg*sizeof(void *)); + sqlite3_free(aDyn); + aDyn = aArg = pNew; + }else{ + /* This occurs when the array of context pointers that need to + ** be passed to the unlock-notify callback is larger than the + ** aStatic[] array allocated on the stack and the attempt to + ** allocate a larger array from the heap has failed. + ** + ** This is a difficult situation to handle. Returning an error + ** code to the caller is insufficient, as even if an error code + ** is returned the transaction on connection db will still be + ** closed and the unlock-notify callbacks on blocked connections + ** will go unissued. This might cause the application to wait + ** indefinitely for an unlock-notify callback that will never + ** arrive. + ** + ** Instead, invoke the unlock-notify callback with the context + ** array already accumulated. We can then clear the array and + ** begin accumulating any further context pointers without + ** requiring any dynamic allocation. This is sub-optimal because + ** it means that instead of one callback with a large array of + ** context pointers the application will receive two or more + ** callbacks with smaller arrays of context pointers, which will + ** reduce the applications ability to prioritize multiple + ** connections. But it is the best that can be done under the + ** circumstances. + */ + xUnlockNotify(aArg, nArg); + nArg = 0; } } + sqlite3EndBenignMalloc(); + + aArg[nArg++] = p->pUnlockArg; + xUnlockNotify = p->xUnlockNotify; + p->pUnlockConnection = 0; + p->xUnlockNotify = 0; + p->pUnlockArg = 0; + } + + /* Step 3. */ + if( p->pBlockingConnection==0 && p->pUnlockConnection==0 ){ + /* Remove connection p from the blocked connections list. */ + *pp = p->pNextBlocked; + p->pNextBlocked = 0; + }else{ + pp = &p->pNextBlocked; } } - pColl = sqlite3FindCollSeq(db, (u8)enc2, zName, 1); - if( pColl==0 ) return SQLITE_NOMEM_BKPT; - pColl->xCmp = xCompare; - pColl->pUser = pCtx; - pColl->xDel = xDel; - pColl->enc = (u8)(enc2 | (enc & SQLITE_UTF16_ALIGNED)); - sqlite3Error(db, SQLITE_OK); - return SQLITE_OK; + if( nArg!=0 ){ + xUnlockNotify(aArg, nArg); + } + sqlite3_free(aDyn); + leaveMutex(); /* Leave STATIC_MASTER mutex */ } - /* -** This array defines hard upper bounds on limit values. The -** initializer must be kept in sync with the SQLITE_LIMIT_* -** #defines in sqlite3.h. +** This is called when the database connection passed as an argument is +** being closed. The connection is removed from the blocked list. */ -static const int aHardLimit[] = { - SQLITE_MAX_LENGTH, - SQLITE_MAX_SQL_LENGTH, - SQLITE_MAX_COLUMN, - SQLITE_MAX_EXPR_DEPTH, - SQLITE_MAX_COMPOUND_SELECT, - SQLITE_MAX_VDBE_OP, - SQLITE_MAX_FUNCTION_ARG, - SQLITE_MAX_ATTACHED, - SQLITE_MAX_LIKE_PATTERN_LENGTH, - SQLITE_MAX_VARIABLE_NUMBER, /* IMP: R-38091-32352 */ - SQLITE_MAX_TRIGGER_DEPTH, - SQLITE_MAX_WORKER_THREADS, -}; +SQLITE_PRIVATE void sqlite3ConnectionClosed(sqlite3 *db){ + sqlite3ConnectionUnlocked(db); + enterMutex(); + removeFromBlockedList(db); + checkListProperties(db); + leaveMutex(); +} +#endif +/************** End of notify.c **********************************************/ +/************** Begin file fts3.c ********************************************/ /* -** Make sure the hard limits are set to reasonable values +** 2006 Oct 10 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This is an SQLite module implementing full-text search. */ -#if SQLITE_MAX_LENGTH<100 -# error SQLITE_MAX_LENGTH must be at least 100 -#endif -#if SQLITE_MAX_SQL_LENGTH<100 -# error SQLITE_MAX_SQL_LENGTH must be at least 100 -#endif -#if SQLITE_MAX_SQL_LENGTH>SQLITE_MAX_LENGTH -# error SQLITE_MAX_SQL_LENGTH must not be greater than SQLITE_MAX_LENGTH -#endif -#if SQLITE_MAX_COMPOUND_SELECT<2 -# error SQLITE_MAX_COMPOUND_SELECT must be at least 2 -#endif -#if SQLITE_MAX_VDBE_OP<40 -# error SQLITE_MAX_VDBE_OP must be at least 40 -#endif -#if SQLITE_MAX_FUNCTION_ARG<0 || SQLITE_MAX_FUNCTION_ARG>127 -# error SQLITE_MAX_FUNCTION_ARG must be between 0 and 127 -#endif -#if SQLITE_MAX_ATTACHED<0 || SQLITE_MAX_ATTACHED>125 -# error SQLITE_MAX_ATTACHED must be between 0 and 125 -#endif -#if SQLITE_MAX_LIKE_PATTERN_LENGTH<1 -# error SQLITE_MAX_LIKE_PATTERN_LENGTH must be at least 1 -#endif -#if SQLITE_MAX_COLUMN>32767 -# error SQLITE_MAX_COLUMN must not exceed 32767 -#endif -#if SQLITE_MAX_TRIGGER_DEPTH<1 -# error SQLITE_MAX_TRIGGER_DEPTH must be at least 1 -#endif -#if SQLITE_MAX_WORKER_THREADS<0 || SQLITE_MAX_WORKER_THREADS>50 -# error SQLITE_MAX_WORKER_THREADS must be between 0 and 50 -#endif - /* -** Change the value of a limit. Report the old value. -** If an invalid limit index is supplied, report -1. -** Make no changes but still report the old value if the -** new limit is negative. +** The code in this file is only compiled if: ** -** A new lower limit does not shrink existing constructs. -** It merely prevents new constructs that exceed the limit -** from forming. +** * The FTS3 module is being built as an extension +** (in which case SQLITE_CORE is not defined), or +** +** * The FTS3 module is being built into the core of +** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). */ -SQLITE_API int sqlite3_limit(sqlite3 *db, int limitId, int newLimit){ - int oldLimit; - -#ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) ){ - (void)SQLITE_MISUSE_BKPT; - return -1; - } -#endif - - /* EVIDENCE-OF: R-30189-54097 For each limit category SQLITE_LIMIT_NAME - ** there is a hard upper bound set at compile-time by a C preprocessor - ** macro called SQLITE_MAX_NAME. (The "_LIMIT_" in the name is changed to - ** "_MAX_".) - */ - assert( aHardLimit[SQLITE_LIMIT_LENGTH]==SQLITE_MAX_LENGTH ); - assert( aHardLimit[SQLITE_LIMIT_SQL_LENGTH]==SQLITE_MAX_SQL_LENGTH ); - assert( aHardLimit[SQLITE_LIMIT_COLUMN]==SQLITE_MAX_COLUMN ); - assert( aHardLimit[SQLITE_LIMIT_EXPR_DEPTH]==SQLITE_MAX_EXPR_DEPTH ); - assert( aHardLimit[SQLITE_LIMIT_COMPOUND_SELECT]==SQLITE_MAX_COMPOUND_SELECT); - assert( aHardLimit[SQLITE_LIMIT_VDBE_OP]==SQLITE_MAX_VDBE_OP ); - assert( aHardLimit[SQLITE_LIMIT_FUNCTION_ARG]==SQLITE_MAX_FUNCTION_ARG ); - assert( aHardLimit[SQLITE_LIMIT_ATTACHED]==SQLITE_MAX_ATTACHED ); - assert( aHardLimit[SQLITE_LIMIT_LIKE_PATTERN_LENGTH]== - SQLITE_MAX_LIKE_PATTERN_LENGTH ); - assert( aHardLimit[SQLITE_LIMIT_VARIABLE_NUMBER]==SQLITE_MAX_VARIABLE_NUMBER); - assert( aHardLimit[SQLITE_LIMIT_TRIGGER_DEPTH]==SQLITE_MAX_TRIGGER_DEPTH ); - assert( aHardLimit[SQLITE_LIMIT_WORKER_THREADS]==SQLITE_MAX_WORKER_THREADS ); - assert( SQLITE_LIMIT_WORKER_THREADS==(SQLITE_N_LIMIT-1) ); - - if( limitId<0 || limitId>=SQLITE_N_LIMIT ){ - return -1; - } - oldLimit = db->aLimit[limitId]; - if( newLimit>=0 ){ /* IMP: R-52476-28732 */ - if( newLimit>aHardLimit[limitId] ){ - newLimit = aHardLimit[limitId]; /* IMP: R-51463-25634 */ - } - db->aLimit[limitId] = newLimit; - } - return oldLimit; /* IMP: R-53341-35419 */ -} +/* The full-text index is stored in a series of b+tree (-like) +** structures called segments which map terms to doclists. The +** structures are like b+trees in layout, but are constructed from the +** bottom up in optimal fashion and are not updatable. Since trees +** are built from the bottom up, things will be described from the +** bottom up. +** +** +**** Varints **** +** The basic unit of encoding is a variable-length integer called a +** varint. We encode variable-length integers in little-endian order +** using seven bits * per byte as follows: +** +** KEY: +** A = 0xxxxxxx 7 bits of data and one flag bit +** B = 1xxxxxxx 7 bits of data and one flag bit +** +** 7 bits - A +** 14 bits - BA +** 21 bits - BBA +** and so on. +** +** This is similar in concept to how sqlite encodes "varints" but +** the encoding is not the same. SQLite varints are big-endian +** are are limited to 9 bytes in length whereas FTS3 varints are +** little-endian and can be up to 10 bytes in length (in theory). +** +** Example encodings: +** +** 1: 0x01 +** 127: 0x7f +** 128: 0x81 0x00 +** +** +**** Document lists **** +** A doclist (document list) holds a docid-sorted list of hits for a +** given term. Doclists hold docids and associated token positions. +** A docid is the unique integer identifier for a single document. +** A position is the index of a word within the document. The first +** word of the document has a position of 0. +** +** FTS3 used to optionally store character offsets using a compile-time +** option. But that functionality is no longer supported. +** +** A doclist is stored like this: +** +** array { +** varint docid; (delta from previous doclist) +** array { (position list for column 0) +** varint position; (2 more than the delta from previous position) +** } +** array { +** varint POS_COLUMN; (marks start of position list for new column) +** varint column; (index of new column) +** array { +** varint position; (2 more than the delta from previous position) +** } +** } +** varint POS_END; (marks end of positions for this document. +** } +** +** Here, array { X } means zero or more occurrences of X, adjacent in +** memory. A "position" is an index of a token in the token stream +** generated by the tokenizer. Note that POS_END and POS_COLUMN occur +** in the same logical place as the position element, and act as sentinals +** ending a position list array. POS_END is 0. POS_COLUMN is 1. +** The positions numbers are not stored literally but rather as two more +** than the difference from the prior position, or the just the position plus +** 2 for the first position. Example: +** +** label: A B C D E F G H I J K +** value: 123 5 9 1 1 14 35 0 234 72 0 +** +** The 123 value is the first docid. For column zero in this document +** there are two matches at positions 3 and 10 (5-2 and 9-2+3). The 1 +** at D signals the start of a new column; the 1 at E indicates that the +** new column is column number 1. There are two positions at 12 and 45 +** (14-2 and 35-2+12). The 0 at H indicate the end-of-document. The +** 234 at I is the delta to next docid (357). It has one position 70 +** (72-2) and then terminates with the 0 at K. +** +** A "position-list" is the list of positions for multiple columns for +** a single docid. A "column-list" is the set of positions for a single +** column. Hence, a position-list consists of one or more column-lists, +** a document record consists of a docid followed by a position-list and +** a doclist consists of one or more document records. +** +** A bare doclist omits the position information, becoming an +** array of varint-encoded docids. +** +**** Segment leaf nodes **** +** Segment leaf nodes store terms and doclists, ordered by term. Leaf +** nodes are written using LeafWriter, and read using LeafReader (to +** iterate through a single leaf node's data) and LeavesReader (to +** iterate through a segment's entire leaf layer). Leaf nodes have +** the format: +** +** varint iHeight; (height from leaf level, always 0) +** varint nTerm; (length of first term) +** char pTerm[nTerm]; (content of first term) +** varint nDoclist; (length of term's associated doclist) +** char pDoclist[nDoclist]; (content of doclist) +** array { +** (further terms are delta-encoded) +** varint nPrefix; (length of prefix shared with previous term) +** varint nSuffix; (length of unshared suffix) +** char pTermSuffix[nSuffix];(unshared suffix of next term) +** varint nDoclist; (length of term's associated doclist) +** char pDoclist[nDoclist]; (content of doclist) +** } +** +** Here, array { X } means zero or more occurrences of X, adjacent in +** memory. +** +** Leaf nodes are broken into blocks which are stored contiguously in +** the %_segments table in sorted order. This means that when the end +** of a node is reached, the next term is in the node with the next +** greater node id. +** +** New data is spilled to a new leaf node when the current node +** exceeds LEAF_MAX bytes (default 2048). New data which itself is +** larger than STANDALONE_MIN (default 1024) is placed in a standalone +** node (a leaf node with a single term and doclist). The goal of +** these settings is to pack together groups of small doclists while +** making it efficient to directly access large doclists. The +** assumption is that large doclists represent terms which are more +** likely to be query targets. +** +** TODO(shess) It may be useful for blocking decisions to be more +** dynamic. For instance, it may make more sense to have a 2.5k leaf +** node rather than splitting into 2k and .5k nodes. My intuition is +** that this might extend through 2x or 4x the pagesize. +** +** +**** Segment interior nodes **** +** Segment interior nodes store blockids for subtree nodes and terms +** to describe what data is stored by the each subtree. Interior +** nodes are written using InteriorWriter, and read using +** InteriorReader. InteriorWriters are created as needed when +** SegmentWriter creates new leaf nodes, or when an interior node +** itself grows too big and must be split. The format of interior +** nodes: +** +** varint iHeight; (height from leaf level, always >0) +** varint iBlockid; (block id of node's leftmost subtree) +** optional { +** varint nTerm; (length of first term) +** char pTerm[nTerm]; (content of first term) +** array { +** (further terms are delta-encoded) +** varint nPrefix; (length of shared prefix with previous term) +** varint nSuffix; (length of unshared suffix) +** char pTermSuffix[nSuffix]; (unshared suffix of next term) +** } +** } +** +** Here, optional { X } means an optional element, while array { X } +** means zero or more occurrences of X, adjacent in memory. +** +** An interior node encodes n terms separating n+1 subtrees. The +** subtree blocks are contiguous, so only the first subtree's blockid +** is encoded. The subtree at iBlockid will contain all terms less +** than the first term encoded (or all terms if no term is encoded). +** Otherwise, for terms greater than or equal to pTerm[i] but less +** than pTerm[i+1], the subtree for that term will be rooted at +** iBlockid+i. Interior nodes only store enough term data to +** distinguish adjacent children (if the rightmost term of the left +** child is "something", and the leftmost term of the right child is +** "wicked", only "w" is stored). +** +** New data is spilled to a new interior node at the same height when +** the current node exceeds INTERIOR_MAX bytes (default 2048). +** INTERIOR_MIN_TERMS (default 7) keeps large terms from monopolizing +** interior nodes and making the tree too skinny. The interior nodes +** at a given height are naturally tracked by interior nodes at +** height+1, and so on. +** +** +**** Segment directory **** +** The segment directory in table %_segdir stores meta-information for +** merging and deleting segments, and also the root node of the +** segment's tree. +** +** The root node is the top node of the segment's tree after encoding +** the entire segment, restricted to ROOT_MAX bytes (default 1024). +** This could be either a leaf node or an interior node. If the top +** node requires more than ROOT_MAX bytes, it is flushed to %_segments +** and a new root interior node is generated (which should always fit +** within ROOT_MAX because it only needs space for 2 varints, the +** height and the blockid of the previous root). +** +** The meta-information in the segment directory is: +** level - segment level (see below) +** idx - index within level +** - (level,idx uniquely identify a segment) +** start_block - first leaf node +** leaves_end_block - last leaf node +** end_block - last block (including interior nodes) +** root - contents of root node +** +** If the root node is a leaf node, then start_block, +** leaves_end_block, and end_block are all 0. +** +** +**** Segment merging **** +** To amortize update costs, segments are grouped into levels and +** merged in batches. Each increase in level represents exponentially +** more documents. +** +** New documents (actually, document updates) are tokenized and +** written individually (using LeafWriter) to a level 0 segment, with +** incrementing idx. When idx reaches MERGE_COUNT (default 16), all +** level 0 segments are merged into a single level 1 segment. Level 1 +** is populated like level 0, and eventually MERGE_COUNT level 1 +** segments are merged to a single level 2 segment (representing +** MERGE_COUNT^2 updates), and so on. +** +** A segment merge traverses all segments at a given level in +** parallel, performing a straightforward sorted merge. Since segment +** leaf nodes are written in to the %_segments table in order, this +** merge traverses the underlying sqlite disk structures efficiently. +** After the merge, all segment blocks from the merged level are +** deleted. +** +** MERGE_COUNT controls how often we merge segments. 16 seems to be +** somewhat of a sweet spot for insertion performance. 32 and 64 show +** very similar performance numbers to 16 on insertion, though they're +** a tiny bit slower (perhaps due to more overhead in merge-time +** sorting). 8 is about 20% slower than 16, 4 about 50% slower than +** 16, 2 about 66% slower than 16. +** +** At query time, high MERGE_COUNT increases the number of segments +** which need to be scanned and merged. For instance, with 100k docs +** inserted: +** +** MERGE_COUNT segments +** 16 25 +** 8 12 +** 4 10 +** 2 6 +** +** This appears to have only a moderate impact on queries for very +** frequent terms (which are somewhat dominated by segment merge +** costs), and infrequent and non-existent terms still seem to be fast +** even with many segments. +** +** TODO(shess) That said, it would be nice to have a better query-side +** argument for MERGE_COUNT of 16. Also, it is possible/likely that +** optimizations to things like doclist merging will swing the sweet +** spot around. +** +** +** +**** Handling of deletions and updates **** +** Since we're using a segmented structure, with no docid-oriented +** index into the term index, we clearly cannot simply update the term +** index when a document is deleted or updated. For deletions, we +** write an empty doclist (varint(docid) varint(POS_END)), for updates +** we simply write the new doclist. Segment merges overwrite older +** data for a particular docid with newer data, so deletes or updates +** will eventually overtake the earlier data and knock it out. The +** query logic likewise merges doclists so that newer data knocks out +** older data. +*/ +/************** Include fts3Int.h in the middle of fts3.c ********************/ +/************** Begin file fts3Int.h *****************************************/ /* -** This function is used to parse both URIs and non-URI filenames passed by the -** user to API functions sqlite3_open() or sqlite3_open_v2(), and for database -** URIs specified as part of ATTACH statements. +** 2009 Nov 12 ** -** The first argument to this function is the name of the VFS to use (or -** a NULL to signify the default VFS) if the URI does not contain a "vfs=xxx" -** query parameter. The second argument contains the URI (or non-URI filename) -** itself. When this function is called the *pFlags variable should contain -** the default flags to open the database handle with. The value stored in -** *pFlags may be updated before returning if the URI filename contains -** "cache=xxx" or "mode=xxx" query parameters. +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: ** -** If successful, SQLITE_OK is returned. In this case *ppVfs is set to point to -** the VFS that should be used to open the database file. *pzFile is set to -** point to a buffer containing the name of the file to open. It is the -** responsibility of the caller to eventually call sqlite3_free() to release -** this buffer. +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** ** -** If an error occurs, then an SQLite error code is returned and *pzErrMsg -** may be set to point to a buffer containing an English language error -** message. It is the responsibility of the caller to eventually release -** this buffer by calling sqlite3_free(). */ -SQLITE_PRIVATE int sqlite3ParseUri( - const char *zDefaultVfs, /* VFS to use if no "vfs=xxx" query option */ - const char *zUri, /* Nul-terminated URI to parse */ - unsigned int *pFlags, /* IN/OUT: SQLITE_OPEN_XXX flags */ - sqlite3_vfs **ppVfs, /* OUT: VFS to use */ - char **pzFile, /* OUT: Filename component of URI */ - char **pzErrMsg /* OUT: Error message (if rc!=SQLITE_OK) */ -){ - int rc = SQLITE_OK; - unsigned int flags = *pFlags; - const char *zVfs = zDefaultVfs; - char *zFile; - char c; - int nUri = sqlite3Strlen30(zUri); - - assert( *pzErrMsg==0 ); - - if( ((flags & SQLITE_OPEN_URI) /* IMP: R-48725-32206 */ - || sqlite3GlobalConfig.bOpenUri) /* IMP: R-51689-46548 */ - && nUri>=5 && memcmp(zUri, "file:", 5)==0 /* IMP: R-57884-37496 */ - ){ - char *zOpt; - int eState; /* Parser state when parsing URI */ - int iIn; /* Input character index */ - int iOut = 0; /* Output character index */ - u64 nByte = nUri+2; /* Bytes of space to allocate */ - - /* Make sure the SQLITE_OPEN_URI flag is set to indicate to the VFS xOpen - ** method that there may be extra parameters following the file-name. */ - flags |= SQLITE_OPEN_URI; - - for(iIn=0; iIn=0 && octet<256 ); - if( octet==0 ){ -#ifndef SQLITE_ENABLE_URI_00_ERROR - /* This branch is taken when "%00" appears within the URI. In this - ** case we ignore all text in the remainder of the path, name or - ** value currently being parsed. So ignore the current character - ** and skip to the next "?", "=" or "&", as appropriate. */ - while( (c = zUri[iIn])!=0 && c!='#' - && (eState!=0 || c!='?') - && (eState!=1 || (c!='=' && c!='&')) - && (eState!=2 || c!='&') - ){ - iIn++; - } - continue; -#else - /* If ENABLE_URI_00_ERROR is defined, "%00" in a URI is an error. */ - *pzErrMsg = sqlite3_mprintf("unexpected %%00 in uri"); - rc = SQLITE_ERROR; - goto parse_uri_out; +/* FTS3/FTS4 require virtual tables */ +#ifdef SQLITE_OMIT_VIRTUALTABLE +# undef SQLITE_ENABLE_FTS3 +# undef SQLITE_ENABLE_FTS4 #endif - } - c = octet; - }else if( eState==1 && (c=='&' || c=='=') ){ - if( zFile[iOut-1]==0 ){ - /* An empty option name. Ignore this option altogether. */ - while( zUri[iIn] && zUri[iIn]!='#' && zUri[iIn-1]!='&' ) iIn++; - continue; - } - if( c=='&' ){ - zFile[iOut++] = '\0'; - }else{ - eState = 2; - } - c = 0; - }else if( (eState==0 && c=='?') || (eState==2 && c=='&') ){ - c = 0; - eState = 1; - } - zFile[iOut++] = c; - } - if( eState==1 ) zFile[iOut++] = '\0'; - zFile[iOut++] = '\0'; - zFile[iOut++] = '\0'; - - /* Check if there were any options specified that should be interpreted - ** here. Options that are interpreted here include "vfs" and those that - ** correspond to flags that may be passed to the sqlite3_open_v2() - ** method. */ - zOpt = &zFile[sqlite3Strlen30(zFile)+1]; - while( zOpt[0] ){ - int nOpt = sqlite3Strlen30(zOpt); - char *zVal = &zOpt[nOpt+1]; - int nVal = sqlite3Strlen30(zVal); - - if( nOpt==3 && memcmp("vfs", zOpt, 3)==0 ){ - zVfs = zVal; - }else{ - struct OpenMode { - const char *z; - int mode; - } *aMode = 0; - char *zModeType = 0; - int mask = 0; - int limit = 0; - - if( nOpt==5 && memcmp("cache", zOpt, 5)==0 ){ - static struct OpenMode aCacheMode[] = { - { "shared", SQLITE_OPEN_SHAREDCACHE }, - { "private", SQLITE_OPEN_PRIVATECACHE }, - { 0, 0 } - }; - - mask = SQLITE_OPEN_SHAREDCACHE|SQLITE_OPEN_PRIVATECACHE; - aMode = aCacheMode; - limit = mask; - zModeType = "cache"; - } - if( nOpt==4 && memcmp("mode", zOpt, 4)==0 ){ - static struct OpenMode aOpenMode[] = { - { "ro", SQLITE_OPEN_READONLY }, - { "rw", SQLITE_OPEN_READWRITE }, - { "rwc", SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE }, - { "memory", SQLITE_OPEN_MEMORY }, - { 0, 0 } - }; - - mask = SQLITE_OPEN_READONLY | SQLITE_OPEN_READWRITE - | SQLITE_OPEN_CREATE | SQLITE_OPEN_MEMORY; - aMode = aOpenMode; - limit = mask & flags; - zModeType = "access"; - } - - if( aMode ){ - int i; - int mode = 0; - for(i=0; aMode[i].z; i++){ - const char *z = aMode[i].z; - if( nVal==sqlite3Strlen30(z) && 0==memcmp(zVal, z, nVal) ){ - mode = aMode[i].mode; - break; - } - } - if( mode==0 ){ - *pzErrMsg = sqlite3_mprintf("no such %s mode: %s", zModeType, zVal); - rc = SQLITE_ERROR; - goto parse_uri_out; - } - if( (mode & ~SQLITE_OPEN_MEMORY)>limit ){ - *pzErrMsg = sqlite3_mprintf("%s mode not allowed: %s", - zModeType, zVal); - rc = SQLITE_PERM; - goto parse_uri_out; - } - flags = (flags & ~mask) | mode; - } - } - - zOpt = &zVal[nVal+1]; - } - }else{ - zFile = sqlite3_malloc64(nUri+2); - if( !zFile ) return SQLITE_NOMEM_BKPT; - if( nUri ){ - memcpy(zFile, zUri, nUri); - } - zFile[nUri] = '\0'; - zFile[nUri+1] = '\0'; - flags &= ~SQLITE_OPEN_URI; - } +/* +** FTS4 is really an extension for FTS3. It is enabled using the +** SQLITE_ENABLE_FTS3 macro. But to avoid confusion we also all +** the SQLITE_ENABLE_FTS4 macro to serve as an alisse for SQLITE_ENABLE_FTS3. +*/ +#if defined(SQLITE_ENABLE_FTS4) && !defined(SQLITE_ENABLE_FTS3) +# define SQLITE_ENABLE_FTS3 +#endif - *ppVfs = sqlite3_vfs_find(zVfs); - if( *ppVfs==0 ){ - *pzErrMsg = sqlite3_mprintf("no such vfs: %s", zVfs); - rc = SQLITE_ERROR; - } - parse_uri_out: - if( rc!=SQLITE_OK ){ - sqlite3_free(zFile); - zFile = 0; - } - *pFlags = flags; - *pzFile = zFile; - return rc; -} +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) +/* If not building as part of the core, include sqlite3ext.h. */ +#ifndef SQLITE_CORE +/* # include "sqlite3ext.h" */ +SQLITE_EXTENSION_INIT3 +#endif +/* #include "sqlite3.h" */ +/************** Include fts3_tokenizer.h in the middle of fts3Int.h **********/ +/************** Begin file fts3_tokenizer.h **********************************/ /* -** This routine does the work of opening a database on behalf of -** sqlite3_open() and sqlite3_open16(). The database filename "zFilename" -** is UTF-8 encoded. +** 2006 July 10 +** +** The author disclaims copyright to this source code. +** +************************************************************************* +** Defines the interface to tokenizers used by fulltext-search. There +** are three basic components: +** +** sqlite3_tokenizer_module is a singleton defining the tokenizer +** interface functions. This is essentially the class structure for +** tokenizers. +** +** sqlite3_tokenizer is used to define a particular tokenizer, perhaps +** including customization information defined at creation time. +** +** sqlite3_tokenizer_cursor is generated by a tokenizer to generate +** tokens from a particular input. */ -static int openDatabase( - const char *zFilename, /* Database filename UTF-8 encoded */ - sqlite3 **ppDb, /* OUT: Returned database handle */ - unsigned int flags, /* Operational flags */ - const char *zVfs /* Name of the VFS to use */ -){ - sqlite3 *db; /* Store allocated handle here */ - int rc; /* Return code */ - int isThreadsafe; /* True for threadsafe connections */ - char *zOpen = 0; /* Filename argument to pass to BtreeOpen() */ - char *zErrMsg = 0; /* Error message from sqlite3ParseUri() */ +#ifndef _FTS3_TOKENIZER_H_ +#define _FTS3_TOKENIZER_H_ -#ifdef SQLITE_ENABLE_API_ARMOR - if( ppDb==0 ) return SQLITE_MISUSE_BKPT; -#endif - *ppDb = 0; -#ifndef SQLITE_OMIT_AUTOINIT - rc = sqlite3_initialize(); - if( rc ) return rc; -#endif +/* TODO(shess) Only used for SQLITE_OK and SQLITE_DONE at this time. +** If tokenizers are to be allowed to call sqlite3_*() functions, then +** we will need a way to register the API consistently. +*/ +/* #include "sqlite3.h" */ - if( sqlite3GlobalConfig.bCoreMutex==0 ){ - isThreadsafe = 0; - }else if( flags & SQLITE_OPEN_NOMUTEX ){ - isThreadsafe = 0; - }else if( flags & SQLITE_OPEN_FULLMUTEX ){ - isThreadsafe = 1; - }else{ - isThreadsafe = sqlite3GlobalConfig.bFullMutex; - } +/* +** Structures used by the tokenizer interface. When a new tokenizer +** implementation is registered, the caller provides a pointer to +** an sqlite3_tokenizer_module containing pointers to the callback +** functions that make up an implementation. +** +** When an fts3 table is created, it passes any arguments passed to +** the tokenizer clause of the CREATE VIRTUAL TABLE statement to the +** sqlite3_tokenizer_module.xCreate() function of the requested tokenizer +** implementation. The xCreate() function in turn returns an +** sqlite3_tokenizer structure representing the specific tokenizer to +** be used for the fts3 table (customized by the tokenizer clause arguments). +** +** To tokenize an input buffer, the sqlite3_tokenizer_module.xOpen() +** method is called. It returns an sqlite3_tokenizer_cursor object +** that may be used to tokenize a specific input buffer based on +** the tokenization rules supplied by a specific sqlite3_tokenizer +** object. +*/ +typedef struct sqlite3_tokenizer_module sqlite3_tokenizer_module; +typedef struct sqlite3_tokenizer sqlite3_tokenizer; +typedef struct sqlite3_tokenizer_cursor sqlite3_tokenizer_cursor; - if( flags & SQLITE_OPEN_PRIVATECACHE ){ - flags &= ~SQLITE_OPEN_SHAREDCACHE; - }else if( sqlite3GlobalConfig.sharedCacheEnabled ){ - flags |= SQLITE_OPEN_SHAREDCACHE; - } +struct sqlite3_tokenizer_module { - /* Remove harmful bits from the flags parameter - ** - ** The SQLITE_OPEN_NOMUTEX and SQLITE_OPEN_FULLMUTEX flags were - ** dealt with in the previous code block. Besides these, the only - ** valid input flags for sqlite3_open_v2() are SQLITE_OPEN_READONLY, - ** SQLITE_OPEN_READWRITE, SQLITE_OPEN_CREATE, SQLITE_OPEN_SHAREDCACHE, - ** SQLITE_OPEN_PRIVATECACHE, and some reserved bits. Silently mask - ** off all other flags. + /* + ** Structure version. Should always be set to 0 or 1. */ - flags &= ~( SQLITE_OPEN_DELETEONCLOSE | - SQLITE_OPEN_EXCLUSIVE | - SQLITE_OPEN_MAIN_DB | - SQLITE_OPEN_TEMP_DB | - SQLITE_OPEN_TRANSIENT_DB | - SQLITE_OPEN_MAIN_JOURNAL | - SQLITE_OPEN_TEMP_JOURNAL | - SQLITE_OPEN_SUBJOURNAL | - SQLITE_OPEN_MASTER_JOURNAL | - SQLITE_OPEN_NOMUTEX | - SQLITE_OPEN_FULLMUTEX | - SQLITE_OPEN_WAL - ); + int iVersion; - /* Allocate the sqlite data structure */ - db = sqlite3MallocZero( sizeof(sqlite3) ); - if( db==0 ) goto opendb_out; - if( isThreadsafe -#ifdef SQLITE_ENABLE_MULTITHREADED_CHECKS - || sqlite3GlobalConfig.bCoreMutex -#endif - ){ - db->mutex = sqlite3MutexAlloc(SQLITE_MUTEX_RECURSIVE); - if( db->mutex==0 ){ - sqlite3_free(db); - db = 0; - goto opendb_out; - } - if( isThreadsafe==0 ){ - sqlite3MutexWarnOnContention(db->mutex); - } - } - sqlite3_mutex_enter(db->mutex); - db->errMask = 0xff; - db->nDb = 2; - db->magic = SQLITE_MAGIC_BUSY; - db->aDb = db->aDbStatic; + /* + ** Create a new tokenizer. The values in the argv[] array are the + ** arguments passed to the "tokenizer" clause of the CREATE VIRTUAL + ** TABLE statement that created the fts3 table. For example, if + ** the following SQL is executed: + ** + ** CREATE .. USING fts3( ... , tokenizer arg1 arg2) + ** + ** then argc is set to 2, and the argv[] array contains pointers + ** to the strings "arg1" and "arg2". + ** + ** This method should return either SQLITE_OK (0), or an SQLite error + ** code. If SQLITE_OK is returned, then *ppTokenizer should be set + ** to point at the newly created tokenizer structure. The generic + ** sqlite3_tokenizer.pModule variable should not be initialized by + ** this callback. The caller will do so. + */ + int (*xCreate)( + int argc, /* Size of argv array */ + const char *const*argv, /* Tokenizer argument strings */ + sqlite3_tokenizer **ppTokenizer /* OUT: Created tokenizer */ + ); - assert( sizeof(db->aLimit)==sizeof(aHardLimit) ); - memcpy(db->aLimit, aHardLimit, sizeof(db->aLimit)); - db->aLimit[SQLITE_LIMIT_WORKER_THREADS] = SQLITE_DEFAULT_WORKER_THREADS; - db->autoCommit = 1; - db->nextAutovac = -1; - db->szMmap = sqlite3GlobalConfig.szMmap; - db->nextPagesize = 0; - db->nMaxSorterMmap = 0x7FFFFFFF; - db->flags |= SQLITE_ShortColNames | SQLITE_EnableTrigger | SQLITE_CacheSpill -#if !defined(SQLITE_DEFAULT_AUTOMATIC_INDEX) || SQLITE_DEFAULT_AUTOMATIC_INDEX - | SQLITE_AutoIndex -#endif -#if SQLITE_DEFAULT_CKPTFULLFSYNC - | SQLITE_CkptFullFSync -#endif -#if SQLITE_DEFAULT_FILE_FORMAT<4 - | SQLITE_LegacyFileFmt -#endif -#ifdef SQLITE_ENABLE_LOAD_EXTENSION - | SQLITE_LoadExtension -#endif -#if SQLITE_DEFAULT_RECURSIVE_TRIGGERS - | SQLITE_RecTriggers -#endif -#if defined(SQLITE_DEFAULT_FOREIGN_KEYS) && SQLITE_DEFAULT_FOREIGN_KEYS - | SQLITE_ForeignKeys -#endif -#if defined(SQLITE_REVERSE_UNORDERED_SELECTS) - | SQLITE_ReverseOrder -#endif -#if defined(SQLITE_ENABLE_OVERSIZE_CELL_CHECK) - | SQLITE_CellSizeCk -#endif -#if defined(SQLITE_ENABLE_FTS3_TOKENIZER) - | SQLITE_Fts3Tokenizer -#endif -#if defined(SQLITE_ENABLE_QPSG) - | SQLITE_EnableQPSG -#endif - ; - sqlite3HashInit(&db->aCollSeq); -#ifndef SQLITE_OMIT_VIRTUALTABLE - sqlite3HashInit(&db->aModule); -#endif + /* + ** Destroy an existing tokenizer. The fts3 module calls this method + ** exactly once for each successful call to xCreate(). + */ + int (*xDestroy)(sqlite3_tokenizer *pTokenizer); - /* Add the default collation sequence BINARY. BINARY works for both UTF-8 - ** and UTF-16, so add a version for each to avoid any unnecessary - ** conversions. The only error that can occur here is a malloc() failure. - ** - ** EVIDENCE-OF: R-52786-44878 SQLite defines three built-in collating - ** functions: + /* + ** Create a tokenizer cursor to tokenize an input buffer. The caller + ** is responsible for ensuring that the input buffer remains valid + ** until the cursor is closed (using the xClose() method). */ - createCollation(db, sqlite3StrBINARY, SQLITE_UTF8, 0, binCollFunc, 0); - createCollation(db, sqlite3StrBINARY, SQLITE_UTF16BE, 0, binCollFunc, 0); - createCollation(db, sqlite3StrBINARY, SQLITE_UTF16LE, 0, binCollFunc, 0); - createCollation(db, "NOCASE", SQLITE_UTF8, 0, nocaseCollatingFunc, 0); - createCollation(db, "RTRIM", SQLITE_UTF8, (void*)1, binCollFunc, 0); - if( db->mallocFailed ){ - goto opendb_out; - } - /* EVIDENCE-OF: R-08308-17224 The default collating function for all - ** strings is BINARY. + int (*xOpen)( + sqlite3_tokenizer *pTokenizer, /* Tokenizer object */ + const char *pInput, int nBytes, /* Input buffer */ + sqlite3_tokenizer_cursor **ppCursor /* OUT: Created tokenizer cursor */ + ); + + /* + ** Destroy an existing tokenizer cursor. The fts3 module calls this + ** method exactly once for each successful call to xOpen(). */ - db->pDfltColl = sqlite3FindCollSeq(db, SQLITE_UTF8, sqlite3StrBINARY, 0); - assert( db->pDfltColl!=0 ); + int (*xClose)(sqlite3_tokenizer_cursor *pCursor); - /* Parse the filename/URI argument + /* + ** Retrieve the next token from the tokenizer cursor pCursor. This + ** method should either return SQLITE_OK and set the values of the + ** "OUT" variables identified below, or SQLITE_DONE to indicate that + ** the end of the buffer has been reached, or an SQLite error code. ** - ** Only allow sensible combinations of bits in the flags argument. - ** Throw an error if any non-sense combination is used. If we - ** do not block illegal combinations here, it could trigger - ** assert() statements in deeper layers. Sensible combinations - ** are: + ** *ppToken should be set to point at a buffer containing the + ** normalized version of the token (i.e. after any case-folding and/or + ** stemming has been performed). *pnBytes should be set to the length + ** of this buffer in bytes. The input text that generated the token is + ** identified by the byte offsets returned in *piStartOffset and + ** *piEndOffset. *piStartOffset should be set to the index of the first + ** byte of the token in the input buffer. *piEndOffset should be set + ** to the index of the first byte just past the end of the token in + ** the input buffer. ** - ** 1: SQLITE_OPEN_READONLY - ** 2: SQLITE_OPEN_READWRITE - ** 6: SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE + ** The buffer *ppToken is set to point at is managed by the tokenizer + ** implementation. It is only required to be valid until the next call + ** to xNext() or xClose(). */ - db->openFlags = flags; - assert( SQLITE_OPEN_READONLY == 0x01 ); - assert( SQLITE_OPEN_READWRITE == 0x02 ); - assert( SQLITE_OPEN_CREATE == 0x04 ); - testcase( (1<<(flags&7))==0x02 ); /* READONLY */ - testcase( (1<<(flags&7))==0x04 ); /* READWRITE */ - testcase( (1<<(flags&7))==0x40 ); /* READWRITE | CREATE */ - if( ((1<<(flags&7)) & 0x46)==0 ){ - rc = SQLITE_MISUSE_BKPT; /* IMP: R-65497-44594 */ - }else{ - rc = sqlite3ParseUri(zVfs, zFilename, &flags, &db->pVfs, &zOpen, &zErrMsg); - } - if( rc!=SQLITE_OK ){ - if( rc==SQLITE_NOMEM ) sqlite3OomFault(db); - sqlite3ErrorWithMsg(db, rc, zErrMsg ? "%s" : 0, zErrMsg); - sqlite3_free(zErrMsg); - goto opendb_out; - } - - /* Open the backend database driver */ - rc = sqlite3BtreeOpen(db->pVfs, zOpen, db, &db->aDb[0].pBt, 0, - flags | SQLITE_OPEN_MAIN_DB); - if( rc!=SQLITE_OK ){ - if( rc==SQLITE_IOERR_NOMEM ){ - rc = SQLITE_NOMEM_BKPT; - } - sqlite3Error(db, rc); - goto opendb_out; - } - sqlite3BtreeEnter(db->aDb[0].pBt); - db->aDb[0].pSchema = sqlite3SchemaGet(db, db->aDb[0].pBt); - if( !db->mallocFailed ) ENC(db) = SCHEMA_ENC(db); - sqlite3BtreeLeave(db->aDb[0].pBt); - db->aDb[1].pSchema = sqlite3SchemaGet(db, 0); - - /* The default safety_level for the main database is FULL; for the temp - ** database it is OFF. This matches the pager layer defaults. + /* TODO(shess) current implementation requires pInput to be + ** nul-terminated. This should either be fixed, or pInput/nBytes + ** should be converted to zInput. */ - db->aDb[0].zDbSName = "main"; - db->aDb[0].safety_level = SQLITE_DEFAULT_SYNCHRONOUS+1; - db->aDb[1].zDbSName = "temp"; - db->aDb[1].safety_level = PAGER_SYNCHRONOUS_OFF; + int (*xNext)( + sqlite3_tokenizer_cursor *pCursor, /* Tokenizer cursor */ + const char **ppToken, int *pnBytes, /* OUT: Normalized text for token */ + int *piStartOffset, /* OUT: Byte offset of token in input buffer */ + int *piEndOffset, /* OUT: Byte offset of end of token in input buffer */ + int *piPosition /* OUT: Number of tokens returned before this one */ + ); - db->magic = SQLITE_MAGIC_OPEN; - if( db->mallocFailed ){ - goto opendb_out; - } + /*********************************************************************** + ** Methods below this point are only available if iVersion>=1. + */ - /* Register all built-in functions, but do not attempt to read the - ** database schema yet. This is delayed until the first time the database - ** is accessed. + /* + ** Configure the language id of a tokenizer cursor. */ - sqlite3Error(db, SQLITE_OK); - sqlite3RegisterPerConnectionBuiltinFunctions(db); - rc = sqlite3_errcode(db); + int (*xLanguageid)(sqlite3_tokenizer_cursor *pCsr, int iLangid); +}; -#ifdef SQLITE_ENABLE_FTS5 - /* Register any built-in FTS5 module before loading the automatic - ** extensions. This allows automatic extensions to register FTS5 - ** tokenizers and auxiliary functions. */ - if( !db->mallocFailed && rc==SQLITE_OK ){ - rc = sqlite3Fts5Init(db); - } -#endif +struct sqlite3_tokenizer { + const sqlite3_tokenizer_module *pModule; /* The module for this tokenizer */ + /* Tokenizer implementations will typically add additional fields */ +}; - /* Load automatic extensions - extensions that have been registered - ** using the sqlite3_automatic_extension() API. - */ - if( rc==SQLITE_OK ){ - sqlite3AutoLoadExtensions(db); - rc = sqlite3_errcode(db); - if( rc!=SQLITE_OK ){ - goto opendb_out; - } - } +struct sqlite3_tokenizer_cursor { + sqlite3_tokenizer *pTokenizer; /* Tokenizer for this cursor. */ + /* Tokenizer implementations will typically add additional fields */ +}; -#ifdef SQLITE_ENABLE_FTS1 - if( !db->mallocFailed ){ - extern int sqlite3Fts1Init(sqlite3*); - rc = sqlite3Fts1Init(db); - } -#endif +int fts3_global_term_cnt(int iTerm, int iCol); +int fts3_term_cnt(int iTerm, int iCol); -#ifdef SQLITE_ENABLE_FTS2 - if( !db->mallocFailed && rc==SQLITE_OK ){ - extern int sqlite3Fts2Init(sqlite3*); - rc = sqlite3Fts2Init(db); - } -#endif -#ifdef SQLITE_ENABLE_FTS3 /* automatically defined by SQLITE_ENABLE_FTS4 */ - if( !db->mallocFailed && rc==SQLITE_OK ){ - rc = sqlite3Fts3Init(db); - } -#endif +#endif /* _FTS3_TOKENIZER_H_ */ -#if defined(SQLITE_ENABLE_ICU) || defined(SQLITE_ENABLE_ICU_COLLATIONS) - if( !db->mallocFailed && rc==SQLITE_OK ){ - rc = sqlite3IcuInit(db); - } -#endif +/************** End of fts3_tokenizer.h **************************************/ +/************** Continuing where we left off in fts3Int.h ********************/ +/************** Include fts3_hash.h in the middle of fts3Int.h ***************/ +/************** Begin file fts3_hash.h ***************************************/ +/* +** 2001 September 22 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This is the header file for the generic hash-table implementation +** used in SQLite. We've modified it slightly to serve as a standalone +** hash table implementation for the full-text indexing module. +** +*/ +#ifndef _FTS3_HASH_H_ +#define _FTS3_HASH_H_ -#ifdef SQLITE_ENABLE_RTREE - if( !db->mallocFailed && rc==SQLITE_OK){ - rc = sqlite3RtreeInit(db); - } -#endif +/* Forward declarations of structures. */ +typedef struct Fts3Hash Fts3Hash; +typedef struct Fts3HashElem Fts3HashElem; -#ifdef SQLITE_ENABLE_DBPAGE_VTAB - if( !db->mallocFailed && rc==SQLITE_OK){ - rc = sqlite3DbpageRegister(db); - } -#endif +/* A complete hash table is an instance of the following structure. +** The internals of this structure are intended to be opaque -- client +** code should not attempt to access or modify the fields of this structure +** directly. Change this structure only by using the routines below. +** However, many of the "procedures" and "functions" for modifying and +** accessing this structure are really macros, so we can't really make +** this structure opaque. +*/ +struct Fts3Hash { + char keyClass; /* HASH_INT, _POINTER, _STRING, _BINARY */ + char copyKey; /* True if copy of key made on insert */ + int count; /* Number of entries in this table */ + Fts3HashElem *first; /* The first element of the array */ + int htsize; /* Number of buckets in the hash table */ + struct _fts3ht { /* the hash table */ + int count; /* Number of entries with this hash */ + Fts3HashElem *chain; /* Pointer to first entry with this hash */ + } *ht; +}; -#ifdef SQLITE_ENABLE_DBSTAT_VTAB - if( !db->mallocFailed && rc==SQLITE_OK){ - rc = sqlite3DbstatRegister(db); - } -#endif +/* Each element in the hash table is an instance of the following +** structure. All elements are stored on a single doubly-linked list. +** +** Again, this structure is intended to be opaque, but it can't really +** be opaque because it is used by macros. +*/ +struct Fts3HashElem { + Fts3HashElem *next, *prev; /* Next and previous elements in the table */ + void *data; /* Data associated with this element */ + void *pKey; int nKey; /* Key associated with this element */ +}; -#ifdef SQLITE_ENABLE_JSON1 - if( !db->mallocFailed && rc==SQLITE_OK){ - rc = sqlite3Json1Init(db); - } -#endif +/* +** There are 2 different modes of operation for a hash table: +** +** FTS3_HASH_STRING pKey points to a string that is nKey bytes long +** (including the null-terminator, if any). Case +** is respected in comparisons. +** +** FTS3_HASH_BINARY pKey points to binary data nKey bytes long. +** memcmp() is used to compare keys. +** +** A copy of the key is made if the copyKey parameter to fts3HashInit is 1. +*/ +#define FTS3_HASH_STRING 1 +#define FTS3_HASH_BINARY 2 -#ifdef SQLITE_ENABLE_STMTVTAB - if( !db->mallocFailed && rc==SQLITE_OK){ - rc = sqlite3StmtVtabInit(db); - } -#endif +/* +** Access routines. To delete, insert a NULL pointer. +*/ +SQLITE_PRIVATE void sqlite3Fts3HashInit(Fts3Hash *pNew, char keyClass, char copyKey); +SQLITE_PRIVATE void *sqlite3Fts3HashInsert(Fts3Hash*, const void *pKey, int nKey, void *pData); +SQLITE_PRIVATE void *sqlite3Fts3HashFind(const Fts3Hash*, const void *pKey, int nKey); +SQLITE_PRIVATE void sqlite3Fts3HashClear(Fts3Hash*); +SQLITE_PRIVATE Fts3HashElem *sqlite3Fts3HashFindElem(const Fts3Hash *, const void *, int); - /* -DSQLITE_DEFAULT_LOCKING_MODE=1 makes EXCLUSIVE the default locking - ** mode. -DSQLITE_DEFAULT_LOCKING_MODE=0 make NORMAL the default locking - ** mode. Doing nothing at all also makes NORMAL the default. - */ -#ifdef SQLITE_DEFAULT_LOCKING_MODE - db->dfltLockMode = SQLITE_DEFAULT_LOCKING_MODE; - sqlite3PagerLockingMode(sqlite3BtreePager(db->aDb[0].pBt), - SQLITE_DEFAULT_LOCKING_MODE); -#endif +/* +** Shorthand for the functions above +*/ +#define fts3HashInit sqlite3Fts3HashInit +#define fts3HashInsert sqlite3Fts3HashInsert +#define fts3HashFind sqlite3Fts3HashFind +#define fts3HashClear sqlite3Fts3HashClear +#define fts3HashFindElem sqlite3Fts3HashFindElem - if( rc ) sqlite3Error(db, rc); +/* +** Macros for looping over all elements of a hash table. The idiom is +** like this: +** +** Fts3Hash h; +** Fts3HashElem *p; +** ... +** for(p=fts3HashFirst(&h); p; p=fts3HashNext(p)){ +** SomeStructure *pData = fts3HashData(p); +** // do something with pData +** } +*/ +#define fts3HashFirst(H) ((H)->first) +#define fts3HashNext(E) ((E)->next) +#define fts3HashData(E) ((E)->data) +#define fts3HashKey(E) ((E)->pKey) +#define fts3HashKeysize(E) ((E)->nKey) - /* Enable the lookaside-malloc subsystem */ - setupLookaside(db, 0, sqlite3GlobalConfig.szLookaside, - sqlite3GlobalConfig.nLookaside); +/* +** Number of entries in a hash table +*/ +#define fts3HashCount(H) ((H)->count) - sqlite3_wal_autocheckpoint(db, SQLITE_DEFAULT_WAL_AUTOCHECKPOINT); +#endif /* _FTS3_HASH_H_ */ -opendb_out: - if( db ){ - assert( db->mutex!=0 || isThreadsafe==0 - || sqlite3GlobalConfig.bFullMutex==0 ); - sqlite3_mutex_leave(db->mutex); - } - rc = sqlite3_errcode(db); - assert( db!=0 || rc==SQLITE_NOMEM ); - if( rc==SQLITE_NOMEM ){ - sqlite3_close(db); - db = 0; - }else if( rc!=SQLITE_OK ){ - db->magic = SQLITE_MAGIC_SICK; - } - *ppDb = db; -#ifdef SQLITE_ENABLE_SQLLOG - if( sqlite3GlobalConfig.xSqllog ){ - /* Opening a db handle. Fourth parameter is passed 0. */ - void *pArg = sqlite3GlobalConfig.pSqllogArg; - sqlite3GlobalConfig.xSqllog(pArg, db, zFilename, 0); - } -#endif -#if defined(SQLITE_HAS_CODEC) - if( rc==SQLITE_OK ){ - const char *zKey; - if( (zKey = sqlite3_uri_parameter(zOpen, "hexkey"))!=0 && zKey[0] ){ - u8 iByte; - int i; - char zDecoded[40]; - for(i=0, iByte=0; i(y)?(x):(y)) #endif - if( zFilename==0 ) zFilename = "\000\000"; - pVal = sqlite3ValueNew(0); - sqlite3ValueSetStr(pVal, -1, zFilename, SQLITE_UTF16NATIVE, SQLITE_STATIC); - zFilename8 = sqlite3ValueText(pVal, SQLITE_UTF8); - if( zFilename8 ){ - rc = openDatabase(zFilename8, ppDb, - SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, 0); - assert( *ppDb || rc==SQLITE_NOMEM ); - if( rc==SQLITE_OK && !DbHasProperty(*ppDb, 0, DB_SchemaLoaded) ){ - SCHEMA_ENC(*ppDb) = ENC(*ppDb) = SQLITE_UTF16NATIVE; - } - }else{ - rc = SQLITE_NOMEM_BKPT; - } - sqlite3ValueFree(pVal); - - return rc & 0xff; -} -#endif /* SQLITE_OMIT_UTF16 */ /* -** Register a new collation sequence with the database handle db. +** Maximum length of a varint encoded integer. The varint format is different +** from that used by SQLite, so the maximum length is 10, not 9. */ -SQLITE_API int sqlite3_create_collation( - sqlite3* db, - const char *zName, - int enc, - void* pCtx, - int(*xCompare)(void*,int,const void*,int,const void*) -){ - return sqlite3_create_collation_v2(db, zName, enc, pCtx, xCompare, 0); -} +#define FTS3_VARINT_MAX 10 + +#define FTS3_BUFFER_PADDING 8 /* -** Register a new collation sequence with the database handle db. +** FTS4 virtual tables may maintain multiple indexes - one index of all terms +** in the document set and zero or more prefix indexes. All indexes are stored +** as one or more b+-trees in the %_segments and %_segdir tables. +** +** It is possible to determine which index a b+-tree belongs to based on the +** value stored in the "%_segdir.level" column. Given this value L, the index +** that the b+-tree belongs to is (L<<10). In other words, all b+-trees with +** level values between 0 and 1023 (inclusive) belong to index 0, all levels +** between 1024 and 2047 to index 1, and so on. +** +** It is considered impossible for an index to use more than 1024 levels. In +** theory though this may happen, but only after at least +** (FTS3_MERGE_COUNT^1024) separate flushes of the pending-terms tables. */ -SQLITE_API int sqlite3_create_collation_v2( - sqlite3* db, - const char *zName, - int enc, - void* pCtx, - int(*xCompare)(void*,int,const void*,int,const void*), - void(*xDel)(void*) -){ - int rc; +#define FTS3_SEGDIR_MAXLEVEL 1024 +#define FTS3_SEGDIR_MAXLEVEL_STR "1024" -#ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) || zName==0 ) return SQLITE_MISUSE_BKPT; +/* +** The testcase() macro is only used by the amalgamation. If undefined, +** make it a no-op. +*/ +#ifndef testcase +# define testcase(X) #endif - sqlite3_mutex_enter(db->mutex); - assert( !db->mallocFailed ); - rc = createCollation(db, zName, (u8)enc, pCtx, xCompare, xDel); - rc = sqlite3ApiExit(db, rc); - sqlite3_mutex_leave(db->mutex); - return rc; -} -#ifndef SQLITE_OMIT_UTF16 /* -** Register a new collation sequence with the database handle db. +** Terminator values for position-lists and column-lists. */ -SQLITE_API int sqlite3_create_collation16( - sqlite3* db, - const void *zName, - int enc, - void* pCtx, - int(*xCompare)(void*,int,const void*,int,const void*) -){ - int rc = SQLITE_OK; - char *zName8; +#define POS_COLUMN (1) /* Column-list terminator */ +#define POS_END (0) /* Position-list terminator */ -#ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) || zName==0 ) return SQLITE_MISUSE_BKPT; +/* +** The assert_fts3_nc() macro is similar to the assert() macro, except that it +** is used for assert() conditions that are true only if it can be +** guranteed that the database is not corrupt. +*/ +#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) +SQLITE_API extern int sqlite3_fts3_may_be_corrupt; +# define assert_fts3_nc(x) assert(sqlite3_fts3_may_be_corrupt || (x)) +#else +# define assert_fts3_nc(x) assert(x) #endif - sqlite3_mutex_enter(db->mutex); - assert( !db->mallocFailed ); - zName8 = sqlite3Utf16to8(db, zName, -1, SQLITE_UTF16NATIVE); - if( zName8 ){ - rc = createCollation(db, zName8, (u8)enc, pCtx, xCompare, 0); - sqlite3DbFree(db, zName8); - } - rc = sqlite3ApiExit(db, rc); - sqlite3_mutex_leave(db->mutex); - return rc; -} -#endif /* SQLITE_OMIT_UTF16 */ /* -** Register a collation sequence factory callback with the database handle -** db. Replace any previously installed collation sequence factory. +** This section provides definitions to allow the +** FTS3 extension to be compiled outside of the +** amalgamation. */ -SQLITE_API int sqlite3_collation_needed( - sqlite3 *db, - void *pCollNeededArg, - void(*xCollNeeded)(void*,sqlite3*,int eTextRep,const char*) -){ -#ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#ifndef SQLITE_AMALGAMATION +/* +** Macros indicating that conditional expressions are always true or +** false. +*/ +#ifdef SQLITE_COVERAGE_TEST +# define ALWAYS(x) (1) +# define NEVER(X) (0) +#elif defined(SQLITE_DEBUG) +# define ALWAYS(x) sqlite3Fts3Always((x)!=0) +# define NEVER(x) sqlite3Fts3Never((x)!=0) +SQLITE_PRIVATE int sqlite3Fts3Always(int b); +SQLITE_PRIVATE int sqlite3Fts3Never(int b); +#else +# define ALWAYS(x) (x) +# define NEVER(x) (x) #endif - sqlite3_mutex_enter(db->mutex); - db->xCollNeeded = xCollNeeded; - db->xCollNeeded16 = 0; - db->pCollNeededArg = pCollNeededArg; - sqlite3_mutex_leave(db->mutex); - return SQLITE_OK; -} -#ifndef SQLITE_OMIT_UTF16 /* -** Register a collation sequence factory callback with the database handle -** db. Replace any previously installed collation sequence factory. +** Internal types used by SQLite. */ -SQLITE_API int sqlite3_collation_needed16( - sqlite3 *db, - void *pCollNeededArg, - void(*xCollNeeded16)(void*,sqlite3*,int eTextRep,const void*) -){ -#ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +typedef unsigned char u8; /* 1-byte (or larger) unsigned integer */ +typedef short int i16; /* 2-byte (or larger) signed integer */ +typedef unsigned int u32; /* 4-byte unsigned integer */ +typedef sqlite3_uint64 u64; /* 8-byte unsigned integer */ +typedef sqlite3_int64 i64; /* 8-byte signed integer */ + +/* +** Macro used to suppress compiler warnings for unused parameters. +*/ +#define UNUSED_PARAMETER(x) (void)(x) + +/* +** Activate assert() only if SQLITE_TEST is enabled. +*/ +#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) +# define NDEBUG 1 #endif - sqlite3_mutex_enter(db->mutex); - db->xCollNeeded = 0; - db->xCollNeeded16 = xCollNeeded16; - db->pCollNeededArg = pCollNeededArg; - sqlite3_mutex_leave(db->mutex); - return SQLITE_OK; -} -#endif /* SQLITE_OMIT_UTF16 */ -#ifndef SQLITE_OMIT_DEPRECATED /* -** This function is now an anachronism. It used to be used to recover from a -** malloc() failure, but SQLite now does this automatically. +** The TESTONLY macro is used to enclose variable declarations or +** other bits of code that are needed to support the arguments +** within testcase() and assert() macros. */ -SQLITE_API int sqlite3_global_recover(void){ - return SQLITE_OK; -} +#if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST) +# define TESTONLY(X) X +#else +# define TESTONLY(X) +#endif + +#define LARGEST_INT64 (0xffffffff|(((i64)0x7fffffff)<<32)) +#define SMALLEST_INT64 (((i64)-1) - LARGEST_INT64) + +#endif /* SQLITE_AMALGAMATION */ + +#ifdef SQLITE_DEBUG +SQLITE_PRIVATE int sqlite3Fts3Corrupt(void); +# define FTS_CORRUPT_VTAB sqlite3Fts3Corrupt() +#else +# define FTS_CORRUPT_VTAB SQLITE_CORRUPT_VTAB #endif +typedef struct Fts3Table Fts3Table; +typedef struct Fts3Cursor Fts3Cursor; +typedef struct Fts3Expr Fts3Expr; +typedef struct Fts3Phrase Fts3Phrase; +typedef struct Fts3PhraseToken Fts3PhraseToken; + +typedef struct Fts3Doclist Fts3Doclist; +typedef struct Fts3SegFilter Fts3SegFilter; +typedef struct Fts3DeferredToken Fts3DeferredToken; +typedef struct Fts3SegReader Fts3SegReader; +typedef struct Fts3MultiSegReader Fts3MultiSegReader; + +typedef struct MatchinfoBuffer MatchinfoBuffer; + /* -** Test to see whether or not the database connection is in autocommit -** mode. Return TRUE if it is and FALSE if not. Autocommit mode is on -** by default. Autocommit is disabled by a BEGIN statement and reenabled -** by the next COMMIT or ROLLBACK. +** A connection to a fulltext index is an instance of the following +** structure. The xCreate and xConnect methods create an instance +** of this structure and xDestroy and xDisconnect free that instance. +** All other methods receive a pointer to the structure as one of their +** arguments. */ -SQLITE_API int sqlite3_get_autocommit(sqlite3 *db){ -#ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) ){ - (void)SQLITE_MISUSE_BKPT; - return 0; - } +struct Fts3Table { + sqlite3_vtab base; /* Base class used by SQLite core */ + sqlite3 *db; /* The database connection */ + const char *zDb; /* logical database name */ + const char *zName; /* virtual table name */ + int nColumn; /* number of named columns in virtual table */ + char **azColumn; /* column names. malloced */ + u8 *abNotindexed; /* True for 'notindexed' columns */ + sqlite3_tokenizer *pTokenizer; /* tokenizer for inserts and queries */ + char *zContentTbl; /* content=xxx option, or NULL */ + char *zLanguageid; /* languageid=xxx option, or NULL */ + int nAutoincrmerge; /* Value configured by 'automerge' */ + u32 nLeafAdd; /* Number of leaf blocks added this trans */ + int bLock; /* Used to prevent recursive content= tbls */ + + /* Precompiled statements used by the implementation. Each of these + ** statements is run and reset within a single virtual table API call. + */ + sqlite3_stmt *aStmt[40]; + sqlite3_stmt *pSeekStmt; /* Cache for fts3CursorSeekStmt() */ + + char *zReadExprlist; + char *zWriteExprlist; + + int nNodeSize; /* Soft limit for node size */ + u8 bFts4; /* True for FTS4, false for FTS3 */ + u8 bHasStat; /* True if %_stat table exists (2==unknown) */ + u8 bHasDocsize; /* True if %_docsize table exists */ + u8 bDescIdx; /* True if doclists are in reverse order */ + u8 bIgnoreSavepoint; /* True to ignore xSavepoint invocations */ + int nPgsz; /* Page size for host database */ + char *zSegmentsTbl; /* Name of %_segments table */ + sqlite3_blob *pSegments; /* Blob handle open on %_segments table */ + + /* + ** The following array of hash tables is used to buffer pending index + ** updates during transactions. All pending updates buffered at any one + ** time must share a common language-id (see the FTS4 langid= feature). + ** The current language id is stored in variable iPrevLangid. + ** + ** A single FTS4 table may have multiple full-text indexes. For each index + ** there is an entry in the aIndex[] array. Index 0 is an index of all the + ** terms that appear in the document set. Each subsequent index in aIndex[] + ** is an index of prefixes of a specific length. + ** + ** Variable nPendingData contains an estimate the memory consumed by the + ** pending data structures, including hash table overhead, but not including + ** malloc overhead. When nPendingData exceeds nMaxPendingData, all hash + ** tables are flushed to disk. Variable iPrevDocid is the docid of the most + ** recently inserted record. + */ + int nIndex; /* Size of aIndex[] */ + struct Fts3Index { + int nPrefix; /* Prefix length (0 for main terms index) */ + Fts3Hash hPending; /* Pending terms table for this index */ + } *aIndex; + int nMaxPendingData; /* Max pending data before flush to disk */ + int nPendingData; /* Current bytes of pending data */ + sqlite_int64 iPrevDocid; /* Docid of most recently inserted document */ + int iPrevLangid; /* Langid of recently inserted document */ + int bPrevDelete; /* True if last operation was a delete */ + +#if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST) + /* State variables used for validating that the transaction control + ** methods of the virtual table are called at appropriate times. These + ** values do not contribute to FTS functionality; they are used for + ** verifying the operation of the SQLite core. + */ + int inTransaction; /* True after xBegin but before xCommit/xRollback */ + int mxSavepoint; /* Largest valid xSavepoint integer */ +#endif + +#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) + /* True to disable the incremental doclist optimization. This is controled + ** by special insert command 'test-no-incr-doclist'. */ + int bNoIncrDoclist; + + /* Number of segments in a level */ + int nMergeCount; +#endif +}; + +/* Macro to find the number of segments to merge */ +#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) +# define MergeCount(P) ((P)->nMergeCount) +#else +# define MergeCount(P) FTS3_MERGE_COUNT #endif - return db->autoCommit; -} /* -** The following routines are substitutes for constants SQLITE_CORRUPT, -** SQLITE_MISUSE, SQLITE_CANTOPEN, SQLITE_NOMEM and possibly other error -** constants. They serve two purposes: -** -** 1. Serve as a convenient place to set a breakpoint in a debugger -** to detect when version error conditions occurs. +** When the core wants to read from the virtual table, it creates a +** virtual table cursor (an instance of the following structure) using +** the xOpen method. Cursors are destroyed using the xClose method. +*/ +struct Fts3Cursor { + sqlite3_vtab_cursor base; /* Base class used by SQLite core */ + i16 eSearch; /* Search strategy (see below) */ + u8 isEof; /* True if at End Of Results */ + u8 isRequireSeek; /* True if must seek pStmt to %_content row */ + u8 bSeekStmt; /* True if pStmt is a seek */ + sqlite3_stmt *pStmt; /* Prepared statement in use by the cursor */ + Fts3Expr *pExpr; /* Parsed MATCH query string */ + int iLangid; /* Language being queried for */ + int nPhrase; /* Number of matchable phrases in query */ + Fts3DeferredToken *pDeferred; /* Deferred search tokens, if any */ + sqlite3_int64 iPrevId; /* Previous id read from aDoclist */ + char *pNextId; /* Pointer into the body of aDoclist */ + char *aDoclist; /* List of docids for full-text queries */ + int nDoclist; /* Size of buffer at aDoclist */ + u8 bDesc; /* True to sort in descending order */ + int eEvalmode; /* An FTS3_EVAL_XX constant */ + int nRowAvg; /* Average size of database rows, in pages */ + sqlite3_int64 nDoc; /* Documents in table */ + i64 iMinDocid; /* Minimum docid to return */ + i64 iMaxDocid; /* Maximum docid to return */ + int isMatchinfoNeeded; /* True when aMatchinfo[] needs filling in */ + MatchinfoBuffer *pMIBuffer; /* Buffer for matchinfo data */ +}; + +#define FTS3_EVAL_FILTER 0 +#define FTS3_EVAL_NEXT 1 +#define FTS3_EVAL_MATCHINFO 2 + +/* +** The Fts3Cursor.eSearch member is always set to one of the following. +** Actualy, Fts3Cursor.eSearch can be greater than or equal to +** FTS3_FULLTEXT_SEARCH. If so, then Fts3Cursor.eSearch - 2 is the index +** of the column to be searched. For example, in ** -** 2. Invoke sqlite3_log() to provide the source code location where -** a low-level error is first detected. +** CREATE VIRTUAL TABLE ex1 USING fts3(a,b,c,d); +** SELECT docid FROM ex1 WHERE b MATCH 'one two three'; +** +** Because the LHS of the MATCH operator is 2nd column "b", +** Fts3Cursor.eSearch will be set to FTS3_FULLTEXT_SEARCH+1. (+0 for a, +** +1 for b, +2 for c, +3 for d.) If the LHS of MATCH were "ex1" +** indicating that all columns should be searched, +** then eSearch would be set to FTS3_FULLTEXT_SEARCH+4. */ -SQLITE_PRIVATE int sqlite3ReportError(int iErr, int lineno, const char *zType){ - sqlite3_log(iErr, "%s at line %d of [%.10s]", - zType, lineno, 20+sqlite3_sourceid()); - return iErr; -} -SQLITE_PRIVATE int sqlite3CorruptError(int lineno){ - testcase( sqlite3GlobalConfig.xLog!=0 ); - return sqlite3ReportError(SQLITE_CORRUPT, lineno, "database corruption"); -} -SQLITE_PRIVATE int sqlite3MisuseError(int lineno){ - testcase( sqlite3GlobalConfig.xLog!=0 ); - return sqlite3ReportError(SQLITE_MISUSE, lineno, "misuse"); -} -SQLITE_PRIVATE int sqlite3CantopenError(int lineno){ - testcase( sqlite3GlobalConfig.xLog!=0 ); - return sqlite3ReportError(SQLITE_CANTOPEN, lineno, "cannot open file"); -} -#ifdef SQLITE_DEBUG -SQLITE_PRIVATE int sqlite3CorruptPgnoError(int lineno, Pgno pgno){ - char zMsg[100]; - sqlite3_snprintf(sizeof(zMsg), zMsg, "database corruption page %d", pgno); - testcase( sqlite3GlobalConfig.xLog!=0 ); - return sqlite3ReportError(SQLITE_CORRUPT, lineno, zMsg); -} -SQLITE_PRIVATE int sqlite3NomemError(int lineno){ - testcase( sqlite3GlobalConfig.xLog!=0 ); - return sqlite3ReportError(SQLITE_NOMEM, lineno, "OOM"); -} -SQLITE_PRIVATE int sqlite3IoerrnomemError(int lineno){ - testcase( sqlite3GlobalConfig.xLog!=0 ); - return sqlite3ReportError(SQLITE_IOERR_NOMEM, lineno, "I/O OOM error"); -} -#endif +#define FTS3_FULLSCAN_SEARCH 0 /* Linear scan of %_content table */ +#define FTS3_DOCID_SEARCH 1 /* Lookup by rowid on %_content table */ +#define FTS3_FULLTEXT_SEARCH 2 /* Full-text index search */ + +/* +** The lower 16-bits of the sqlite3_index_info.idxNum value set by +** the xBestIndex() method contains the Fts3Cursor.eSearch value described +** above. The upper 16-bits contain a combination of the following +** bits, used to describe extra constraints on full-text searches. +*/ +#define FTS3_HAVE_LANGID 0x00010000 /* languageid=? */ +#define FTS3_HAVE_DOCID_GE 0x00020000 /* docid>=? */ +#define FTS3_HAVE_DOCID_LE 0x00040000 /* docid<=? */ + +struct Fts3Doclist { + char *aAll; /* Array containing doclist (or NULL) */ + int nAll; /* Size of a[] in bytes */ + char *pNextDocid; /* Pointer to next docid */ + + sqlite3_int64 iDocid; /* Current docid (if pList!=0) */ + int bFreeList; /* True if pList should be sqlite3_free()d */ + char *pList; /* Pointer to position list following iDocid */ + int nList; /* Length of position list */ +}; -#ifndef SQLITE_OMIT_DEPRECATED /* -** This is a convenience routine that makes sure that all thread-specific -** data for this thread has been deallocated. -** -** SQLite no longer uses thread-specific data so this routine is now a -** no-op. It is retained for historical compatibility. +** A "phrase" is a sequence of one or more tokens that must match in +** sequence. A single token is the base case and the most common case. +** For a sequence of tokens contained in double-quotes (i.e. "one two three") +** nToken will be the number of tokens in the string. */ -SQLITE_API void sqlite3_thread_cleanup(void){ -} -#endif +struct Fts3PhraseToken { + char *z; /* Text of the token */ + int n; /* Number of bytes in buffer z */ + int isPrefix; /* True if token ends with a "*" character */ + int bFirst; /* True if token must appear at position 0 */ + + /* Variables above this point are populated when the expression is + ** parsed (by code in fts3_expr.c). Below this point the variables are + ** used when evaluating the expression. */ + Fts3DeferredToken *pDeferred; /* Deferred token object for this token */ + Fts3MultiSegReader *pSegcsr; /* Segment-reader for this token */ +}; + +struct Fts3Phrase { + /* Cache of doclist for this phrase. */ + Fts3Doclist doclist; + int bIncr; /* True if doclist is loaded incrementally */ + int iDoclistToken; + + /* Used by sqlite3Fts3EvalPhrasePoslist() if this is a descendent of an + ** OR condition. */ + char *pOrPoslist; + i64 iOrDocid; + + /* Variables below this point are populated by fts3_expr.c when parsing + ** a MATCH expression. Everything above is part of the evaluation phase. + */ + int nToken; /* Number of tokens in the phrase */ + int iColumn; /* Index of column this phrase must match */ + Fts3PhraseToken aToken[1]; /* One entry for each token in the phrase */ +}; /* -** Return meta information about a specific column of a database table. -** See comment in sqlite3.h (sqlite.h.in) for details. +** A tree of these objects forms the RHS of a MATCH operator. +** +** If Fts3Expr.eType is FTSQUERY_PHRASE and isLoaded is true, then aDoclist +** points to a malloced buffer, size nDoclist bytes, containing the results +** of this phrase query in FTS3 doclist format. As usual, the initial +** "Length" field found in doclists stored on disk is omitted from this +** buffer. +** +** Variable aMI is used only for FTSQUERY_NEAR nodes to store the global +** matchinfo data. If it is not NULL, it points to an array of size nCol*3, +** where nCol is the number of columns in the queried FTS table. The array +** is populated as follows: +** +** aMI[iCol*3 + 0] = Undefined +** aMI[iCol*3 + 1] = Number of occurrences +** aMI[iCol*3 + 2] = Number of rows containing at least one instance +** +** The aMI array is allocated using sqlite3_malloc(). It should be freed +** when the expression node is. */ -SQLITE_API int sqlite3_table_column_metadata( - sqlite3 *db, /* Connection handle */ - const char *zDbName, /* Database name or NULL */ - const char *zTableName, /* Table name */ - const char *zColumnName, /* Column name */ - char const **pzDataType, /* OUTPUT: Declared data type */ - char const **pzCollSeq, /* OUTPUT: Collation sequence name */ - int *pNotNull, /* OUTPUT: True if NOT NULL constraint exists */ - int *pPrimaryKey, /* OUTPUT: True if column part of PK */ - int *pAutoinc /* OUTPUT: True if column is auto-increment */ -){ - int rc; - char *zErrMsg = 0; - Table *pTab = 0; - Column *pCol = 0; - int iCol = 0; - char const *zDataType = 0; - char const *zCollSeq = 0; - int notnull = 0; - int primarykey = 0; - int autoinc = 0; +struct Fts3Expr { + int eType; /* One of the FTSQUERY_XXX values defined below */ + int nNear; /* Valid if eType==FTSQUERY_NEAR */ + Fts3Expr *pParent; /* pParent->pLeft==this or pParent->pRight==this */ + Fts3Expr *pLeft; /* Left operand */ + Fts3Expr *pRight; /* Right operand */ + Fts3Phrase *pPhrase; /* Valid if eType==FTSQUERY_PHRASE */ + /* The following are used by the fts3_eval.c module. */ + sqlite3_int64 iDocid; /* Current docid */ + u8 bEof; /* True this expression is at EOF already */ + u8 bStart; /* True if iDocid is valid */ + u8 bDeferred; /* True if this expression is entirely deferred */ -#ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) || zTableName==0 ){ - return SQLITE_MISUSE_BKPT; - } -#endif + /* The following are used by the fts3_snippet.c module. */ + int iPhrase; /* Index of this phrase in matchinfo() results */ + u32 *aMI; /* See above */ +}; - /* Ensure the database schema has been loaded */ - sqlite3_mutex_enter(db->mutex); - sqlite3BtreeEnterAll(db); - rc = sqlite3Init(db, &zErrMsg); - if( SQLITE_OK!=rc ){ - goto error_out; - } +/* +** Candidate values for Fts3Query.eType. Note that the order of the first +** four values is in order of precedence when parsing expressions. For +** example, the following: +** +** "a OR b AND c NOT d NEAR e" +** +** is equivalent to: +** +** "a OR (b AND (c NOT (d NEAR e)))" +*/ +#define FTSQUERY_NEAR 1 +#define FTSQUERY_NOT 2 +#define FTSQUERY_AND 3 +#define FTSQUERY_OR 4 +#define FTSQUERY_PHRASE 5 - /* Locate the table in question */ - pTab = sqlite3FindTable(db, zTableName, zDbName); - if( !pTab || pTab->pSelect ){ - pTab = 0; - goto error_out; - } - /* Find the column for which info is requested */ - if( zColumnName==0 ){ - /* Query for existance of table only */ - }else{ - for(iCol=0; iColnCol; iCol++){ - pCol = &pTab->aCol[iCol]; - if( 0==sqlite3StrICmp(pCol->zName, zColumnName) ){ - break; - } - } - if( iCol==pTab->nCol ){ - if( HasRowid(pTab) && sqlite3IsRowid(zColumnName) ){ - iCol = pTab->iPKey; - pCol = iCol>=0 ? &pTab->aCol[iCol] : 0; - }else{ - pTab = 0; - goto error_out; - } - } - } +/* fts3_write.c */ +SQLITE_PRIVATE int sqlite3Fts3UpdateMethod(sqlite3_vtab*,int,sqlite3_value**,sqlite3_int64*); +SQLITE_PRIVATE int sqlite3Fts3PendingTermsFlush(Fts3Table *); +SQLITE_PRIVATE void sqlite3Fts3PendingTermsClear(Fts3Table *); +SQLITE_PRIVATE int sqlite3Fts3Optimize(Fts3Table *); +SQLITE_PRIVATE int sqlite3Fts3SegReaderNew(int, int, sqlite3_int64, + sqlite3_int64, sqlite3_int64, const char *, int, Fts3SegReader**); +SQLITE_PRIVATE int sqlite3Fts3SegReaderPending( + Fts3Table*,int,const char*,int,int,Fts3SegReader**); +SQLITE_PRIVATE void sqlite3Fts3SegReaderFree(Fts3SegReader *); +SQLITE_PRIVATE int sqlite3Fts3AllSegdirs(Fts3Table*, int, int, int, sqlite3_stmt **); +SQLITE_PRIVATE int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char **, int*, int*); - /* The following block stores the meta information that will be returned - ** to the caller in local variables zDataType, zCollSeq, notnull, primarykey - ** and autoinc. At this point there are two possibilities: - ** - ** 1. The specified column name was rowid", "oid" or "_rowid_" - ** and there is no explicitly declared IPK column. - ** - ** 2. The table is not a view and the column name identified an - ** explicitly declared column. Copy meta information from *pCol. - */ - if( pCol ){ - zDataType = sqlite3ColumnType(pCol,0); - zCollSeq = pCol->zColl; - notnull = pCol->notNull!=0; - primarykey = (pCol->colFlags & COLFLAG_PRIMKEY)!=0; - autoinc = pTab->iPKey==iCol && (pTab->tabFlags & TF_Autoincrement)!=0; - }else{ - zDataType = "INTEGER"; - primarykey = 1; - } - if( !zCollSeq ){ - zCollSeq = sqlite3StrBINARY; - } +SQLITE_PRIVATE int sqlite3Fts3SelectDoctotal(Fts3Table *, sqlite3_stmt **); +SQLITE_PRIVATE int sqlite3Fts3SelectDocsize(Fts3Table *, sqlite3_int64, sqlite3_stmt **); -error_out: - sqlite3BtreeLeaveAll(db); +#ifndef SQLITE_DISABLE_FTS4_DEFERRED +SQLITE_PRIVATE void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *); +SQLITE_PRIVATE int sqlite3Fts3DeferToken(Fts3Cursor *, Fts3PhraseToken *, int); +SQLITE_PRIVATE int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *); +SQLITE_PRIVATE void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *); +SQLITE_PRIVATE int sqlite3Fts3DeferredTokenList(Fts3DeferredToken *, char **, int *); +#else +# define sqlite3Fts3FreeDeferredTokens(x) +# define sqlite3Fts3DeferToken(x,y,z) SQLITE_OK +# define sqlite3Fts3CacheDeferredDoclists(x) SQLITE_OK +# define sqlite3Fts3FreeDeferredDoclists(x) +# define sqlite3Fts3DeferredTokenList(x,y,z) SQLITE_OK +#endif - /* Whether the function call succeeded or failed, set the output parameters - ** to whatever their local counterparts contain. If an error did occur, - ** this has the effect of zeroing all output parameters. - */ - if( pzDataType ) *pzDataType = zDataType; - if( pzCollSeq ) *pzCollSeq = zCollSeq; - if( pNotNull ) *pNotNull = notnull; - if( pPrimaryKey ) *pPrimaryKey = primarykey; - if( pAutoinc ) *pAutoinc = autoinc; +SQLITE_PRIVATE void sqlite3Fts3SegmentsClose(Fts3Table *); +SQLITE_PRIVATE int sqlite3Fts3MaxLevel(Fts3Table *, int *); - if( SQLITE_OK==rc && !pTab ){ - sqlite3DbFree(db, zErrMsg); - zErrMsg = sqlite3MPrintf(db, "no such table column: %s.%s", zTableName, - zColumnName); - rc = SQLITE_ERROR; - } - sqlite3ErrorWithMsg(db, rc, (zErrMsg?"%s":0), zErrMsg); - sqlite3DbFree(db, zErrMsg); - rc = sqlite3ApiExit(db, rc); - sqlite3_mutex_leave(db->mutex); - return rc; -} +/* Special values interpreted by sqlite3SegReaderCursor() */ +#define FTS3_SEGCURSOR_PENDING -1 +#define FTS3_SEGCURSOR_ALL -2 -/* -** Sleep for a little while. Return the amount of time slept. -*/ -SQLITE_API int sqlite3_sleep(int ms){ - sqlite3_vfs *pVfs; - int rc; - pVfs = sqlite3_vfs_find(0); - if( pVfs==0 ) return 0; +SQLITE_PRIVATE int sqlite3Fts3SegReaderStart(Fts3Table*, Fts3MultiSegReader*, Fts3SegFilter*); +SQLITE_PRIVATE int sqlite3Fts3SegReaderStep(Fts3Table *, Fts3MultiSegReader *); +SQLITE_PRIVATE void sqlite3Fts3SegReaderFinish(Fts3MultiSegReader *); - /* This function works in milliseconds, but the underlying OsSleep() - ** API uses microseconds. Hence the 1000's. - */ - rc = (sqlite3OsSleep(pVfs, 1000*ms)/1000); - return rc; -} +SQLITE_PRIVATE int sqlite3Fts3SegReaderCursor(Fts3Table *, + int, int, int, const char *, int, int, int, Fts3MultiSegReader *); -/* -** Enable or disable the extended result codes. -*/ -SQLITE_API int sqlite3_extended_result_codes(sqlite3 *db, int onoff){ -#ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; -#endif - sqlite3_mutex_enter(db->mutex); - db->errMask = onoff ? 0xffffffff : 0xff; - sqlite3_mutex_leave(db->mutex); - return SQLITE_OK; -} +/* Flags allowed as part of the 4th argument to SegmentReaderIterate() */ +#define FTS3_SEGMENT_REQUIRE_POS 0x00000001 +#define FTS3_SEGMENT_IGNORE_EMPTY 0x00000002 +#define FTS3_SEGMENT_COLUMN_FILTER 0x00000004 +#define FTS3_SEGMENT_PREFIX 0x00000008 +#define FTS3_SEGMENT_SCAN 0x00000010 +#define FTS3_SEGMENT_FIRST 0x00000020 -/* -** Invoke the xFileControl method on a particular database. -*/ -SQLITE_API int sqlite3_file_control(sqlite3 *db, const char *zDbName, int op, void *pArg){ - int rc = SQLITE_ERROR; - Btree *pBtree; +/* Type passed as 4th argument to SegmentReaderIterate() */ +struct Fts3SegFilter { + const char *zTerm; + int nTerm; + int iCol; + int flags; +}; -#ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; -#endif - sqlite3_mutex_enter(db->mutex); - pBtree = sqlite3DbNameToBtree(db, zDbName); - if( pBtree ){ - Pager *pPager; - sqlite3_file *fd; - sqlite3BtreeEnter(pBtree); - pPager = sqlite3BtreePager(pBtree); - assert( pPager!=0 ); - fd = sqlite3PagerFile(pPager); - assert( fd!=0 ); - if( op==SQLITE_FCNTL_FILE_POINTER ){ - *(sqlite3_file**)pArg = fd; - rc = SQLITE_OK; - }else if( op==SQLITE_FCNTL_VFS_POINTER ){ - *(sqlite3_vfs**)pArg = sqlite3PagerVfs(pPager); - rc = SQLITE_OK; - }else if( op==SQLITE_FCNTL_JOURNAL_POINTER ){ - *(sqlite3_file**)pArg = sqlite3PagerJrnlFile(pPager); - rc = SQLITE_OK; - }else{ - rc = sqlite3OsFileControl(fd, op, pArg); - } - sqlite3BtreeLeave(pBtree); - } - sqlite3_mutex_leave(db->mutex); - return rc; -} +struct Fts3MultiSegReader { + /* Used internally by sqlite3Fts3SegReaderXXX() calls */ + Fts3SegReader **apSegment; /* Array of Fts3SegReader objects */ + int nSegment; /* Size of apSegment array */ + int nAdvance; /* How many seg-readers to advance */ + Fts3SegFilter *pFilter; /* Pointer to filter object */ + char *aBuffer; /* Buffer to merge doclists in */ + int nBuffer; /* Allocated size of aBuffer[] in bytes */ -/* -** Interface to the testing logic. -*/ -SQLITE_API int sqlite3_test_control(int op, ...){ - int rc = 0; -#ifdef SQLITE_UNTESTABLE - UNUSED_PARAMETER(op); -#else - va_list ap; - va_start(ap, op); - switch( op ){ + int iColFilter; /* If >=0, filter for this column */ + int bRestart; - /* - ** Save the current state of the PRNG. - */ - case SQLITE_TESTCTRL_PRNG_SAVE: { - sqlite3PrngSaveState(); - break; - } + /* Used by fts3.c only. */ + int nCost; /* Cost of running iterator */ + int bLookup; /* True if a lookup of a single entry. */ - /* - ** Restore the state of the PRNG to the last state saved using - ** PRNG_SAVE. If PRNG_SAVE has never before been called, then - ** this verb acts like PRNG_RESET. - */ - case SQLITE_TESTCTRL_PRNG_RESTORE: { - sqlite3PrngRestoreState(); - break; - } + /* Output values. Valid only after Fts3SegReaderStep() returns SQLITE_ROW. */ + char *zTerm; /* Pointer to term buffer */ + int nTerm; /* Size of zTerm in bytes */ + char *aDoclist; /* Pointer to doclist buffer */ + int nDoclist; /* Size of aDoclist[] in bytes */ +}; - /* - ** Reset the PRNG back to its uninitialized state. The next call - ** to sqlite3_randomness() will reseed the PRNG using a single call - ** to the xRandomness method of the default VFS. - */ - case SQLITE_TESTCTRL_PRNG_RESET: { - sqlite3_randomness(0,0); - break; - } +SQLITE_PRIVATE int sqlite3Fts3Incrmerge(Fts3Table*,int,int); - /* - ** sqlite3_test_control(BITVEC_TEST, size, program) - ** - ** Run a test against a Bitvec object of size. The program argument - ** is an array of integers that defines the test. Return -1 on a - ** memory allocation error, 0 on success, or non-zero for an error. - ** See the sqlite3BitvecBuiltinTest() for additional information. - */ - case SQLITE_TESTCTRL_BITVEC_TEST: { - int sz = va_arg(ap, int); - int *aProg = va_arg(ap, int*); - rc = sqlite3BitvecBuiltinTest(sz, aProg); - break; - } +#define fts3GetVarint32(p, piVal) ( \ + (*(u8*)(p)&0x80) ? sqlite3Fts3GetVarint32(p, piVal) : (*piVal=*(u8*)(p), 1) \ +) - /* - ** sqlite3_test_control(FAULT_INSTALL, xCallback) - ** - ** Arrange to invoke xCallback() whenever sqlite3FaultSim() is called, - ** if xCallback is not NULL. - ** - ** As a test of the fault simulator mechanism itself, sqlite3FaultSim(0) - ** is called immediately after installing the new callback and the return - ** value from sqlite3FaultSim(0) becomes the return from - ** sqlite3_test_control(). - */ - case SQLITE_TESTCTRL_FAULT_INSTALL: { - /* MSVC is picky about pulling func ptrs from va lists. - ** http://support.microsoft.com/kb/47961 - ** sqlite3GlobalConfig.xTestCallback = va_arg(ap, int(*)(int)); - */ - typedef int(*TESTCALLBACKFUNC_t)(int); - sqlite3GlobalConfig.xTestCallback = va_arg(ap, TESTCALLBACKFUNC_t); - rc = sqlite3FaultSim(0); - break; - } +/* fts3.c */ +SQLITE_PRIVATE void sqlite3Fts3ErrMsg(char**,const char*,...); +SQLITE_PRIVATE int sqlite3Fts3PutVarint(char *, sqlite3_int64); +SQLITE_PRIVATE int sqlite3Fts3GetVarint(const char *, sqlite_int64 *); +SQLITE_PRIVATE int sqlite3Fts3GetVarintU(const char *, sqlite_uint64 *); +SQLITE_PRIVATE int sqlite3Fts3GetVarintBounded(const char*,const char*,sqlite3_int64*); +SQLITE_PRIVATE int sqlite3Fts3GetVarint32(const char *, int *); +SQLITE_PRIVATE int sqlite3Fts3VarintLen(sqlite3_uint64); +SQLITE_PRIVATE void sqlite3Fts3Dequote(char *); +SQLITE_PRIVATE void sqlite3Fts3DoclistPrev(int,char*,int,char**,sqlite3_int64*,int*,u8*); +SQLITE_PRIVATE int sqlite3Fts3EvalPhraseStats(Fts3Cursor *, Fts3Expr *, u32 *); +SQLITE_PRIVATE int sqlite3Fts3FirstFilter(sqlite3_int64, char *, int, char *); +SQLITE_PRIVATE void sqlite3Fts3CreateStatTable(int*, Fts3Table*); +SQLITE_PRIVATE int sqlite3Fts3EvalTestDeferred(Fts3Cursor *pCsr, int *pRc); +SQLITE_PRIVATE int sqlite3Fts3ReadInt(const char *z, int *pnOut); - /* - ** sqlite3_test_control(BENIGN_MALLOC_HOOKS, xBegin, xEnd) - ** - ** Register hooks to call to indicate which malloc() failures - ** are benign. - */ - case SQLITE_TESTCTRL_BENIGN_MALLOC_HOOKS: { - typedef void (*void_function)(void); - void_function xBenignBegin; - void_function xBenignEnd; - xBenignBegin = va_arg(ap, void_function); - xBenignEnd = va_arg(ap, void_function); - sqlite3BenignMallocHooks(xBenignBegin, xBenignEnd); - break; - } +/* fts3_tokenizer.c */ +SQLITE_PRIVATE const char *sqlite3Fts3NextToken(const char *, int *); +SQLITE_PRIVATE int sqlite3Fts3InitHashTable(sqlite3 *, Fts3Hash *, const char *); +SQLITE_PRIVATE int sqlite3Fts3InitTokenizer(Fts3Hash *pHash, const char *, + sqlite3_tokenizer **, char ** +); +SQLITE_PRIVATE int sqlite3Fts3IsIdChar(char); - /* - ** sqlite3_test_control(SQLITE_TESTCTRL_PENDING_BYTE, unsigned int X) - ** - ** Set the PENDING byte to the value in the argument, if X>0. - ** Make no changes if X==0. Return the value of the pending byte - ** as it existing before this routine was called. - ** - ** IMPORTANT: Changing the PENDING byte from 0x40000000 results in - ** an incompatible database file format. Changing the PENDING byte - ** while any database connection is open results in undefined and - ** deleterious behavior. - */ - case SQLITE_TESTCTRL_PENDING_BYTE: { - rc = PENDING_BYTE; -#ifndef SQLITE_OMIT_WSD - { - unsigned int newVal = va_arg(ap, unsigned int); - if( newVal ) sqlite3PendingByte = newVal; - } -#endif - break; - } +/* fts3_snippet.c */ +SQLITE_PRIVATE void sqlite3Fts3Offsets(sqlite3_context*, Fts3Cursor*); +SQLITE_PRIVATE void sqlite3Fts3Snippet(sqlite3_context *, Fts3Cursor *, const char *, + const char *, const char *, int, int +); +SQLITE_PRIVATE void sqlite3Fts3Matchinfo(sqlite3_context *, Fts3Cursor *, const char *); +SQLITE_PRIVATE void sqlite3Fts3MIBufferFree(MatchinfoBuffer *p); - /* - ** sqlite3_test_control(SQLITE_TESTCTRL_ASSERT, int X) - ** - ** This action provides a run-time test to see whether or not - ** assert() was enabled at compile-time. If X is true and assert() - ** is enabled, then the return value is true. If X is true and - ** assert() is disabled, then the return value is zero. If X is - ** false and assert() is enabled, then the assertion fires and the - ** process aborts. If X is false and assert() is disabled, then the - ** return value is zero. - */ - case SQLITE_TESTCTRL_ASSERT: { - volatile int x = 0; - assert( /*side-effects-ok*/ (x = va_arg(ap,int))!=0 ); - rc = x; - break; - } +/* fts3_expr.c */ +SQLITE_PRIVATE int sqlite3Fts3ExprParse(sqlite3_tokenizer *, int, + char **, int, int, int, const char *, int, Fts3Expr **, char ** +); +SQLITE_PRIVATE void sqlite3Fts3ExprFree(Fts3Expr *); +#ifdef SQLITE_TEST +SQLITE_PRIVATE int sqlite3Fts3ExprInitTestInterface(sqlite3 *db, Fts3Hash*); +SQLITE_PRIVATE int sqlite3Fts3InitTerm(sqlite3 *db); +#endif +SQLITE_PRIVATE int sqlite3Fts3OpenTokenizer(sqlite3_tokenizer *, int, const char *, int, + sqlite3_tokenizer_cursor ** +); - /* - ** sqlite3_test_control(SQLITE_TESTCTRL_ALWAYS, int X) - ** - ** This action provides a run-time test to see how the ALWAYS and - ** NEVER macros were defined at compile-time. - ** - ** The return value is ALWAYS(X) if X is true, or 0 if X is false. - ** - ** The recommended test is X==2. If the return value is 2, that means - ** ALWAYS() and NEVER() are both no-op pass-through macros, which is the - ** default setting. If the return value is 1, then ALWAYS() is either - ** hard-coded to true or else it asserts if its argument is false. - ** The first behavior (hard-coded to true) is the case if - ** SQLITE_TESTCTRL_ASSERT shows that assert() is disabled and the second - ** behavior (assert if the argument to ALWAYS() is false) is the case if - ** SQLITE_TESTCTRL_ASSERT shows that assert() is enabled. - ** - ** The run-time test procedure might look something like this: - ** - ** if( sqlite3_test_control(SQLITE_TESTCTRL_ALWAYS, 2)==2 ){ - ** // ALWAYS() and NEVER() are no-op pass-through macros - ** }else if( sqlite3_test_control(SQLITE_TESTCTRL_ASSERT, 1) ){ - ** // ALWAYS(x) asserts that x is true. NEVER(x) asserts x is false. - ** }else{ - ** // ALWAYS(x) is a constant 1. NEVER(x) is a constant 0. - ** } - */ - case SQLITE_TESTCTRL_ALWAYS: { - int x = va_arg(ap,int); - rc = x ? ALWAYS(x) : 0; - break; - } +/* fts3_aux.c */ +SQLITE_PRIVATE int sqlite3Fts3InitAux(sqlite3 *db); - /* - ** sqlite3_test_control(SQLITE_TESTCTRL_BYTEORDER); - ** - ** The integer returned reveals the byte-order of the computer on which - ** SQLite is running: - ** - ** 1 big-endian, determined at run-time - ** 10 little-endian, determined at run-time - ** 432101 big-endian, determined at compile-time - ** 123410 little-endian, determined at compile-time - */ - case SQLITE_TESTCTRL_BYTEORDER: { - rc = SQLITE_BYTEORDER*100 + SQLITE_LITTLEENDIAN*10 + SQLITE_BIGENDIAN; - break; - } +SQLITE_PRIVATE void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *); - /* sqlite3_test_control(SQLITE_TESTCTRL_RESERVE, sqlite3 *db, int N) - ** - ** Set the nReserve size to N for the main database on the database - ** connection db. - */ - case SQLITE_TESTCTRL_RESERVE: { - sqlite3 *db = va_arg(ap, sqlite3*); - int x = va_arg(ap,int); - sqlite3_mutex_enter(db->mutex); - sqlite3BtreeSetPageSize(db->aDb[0].pBt, 0, x, 0); - sqlite3_mutex_leave(db->mutex); - break; - } +SQLITE_PRIVATE int sqlite3Fts3MsrIncrStart( + Fts3Table*, Fts3MultiSegReader*, int, const char*, int); +SQLITE_PRIVATE int sqlite3Fts3MsrIncrNext( + Fts3Table *, Fts3MultiSegReader *, sqlite3_int64 *, char **, int *); +SQLITE_PRIVATE int sqlite3Fts3EvalPhrasePoslist(Fts3Cursor *, Fts3Expr *, int iCol, char **); +SQLITE_PRIVATE int sqlite3Fts3MsrOvfl(Fts3Cursor *, Fts3MultiSegReader *, int *); +SQLITE_PRIVATE int sqlite3Fts3MsrIncrRestart(Fts3MultiSegReader *pCsr); - /* sqlite3_test_control(SQLITE_TESTCTRL_OPTIMIZATIONS, sqlite3 *db, int N) - ** - ** Enable or disable various optimizations for testing purposes. The - ** argument N is a bitmask of optimizations to be disabled. For normal - ** operation N should be 0. The idea is that a test program (like the - ** SQL Logic Test or SLT test module) can run the same SQL multiple times - ** with various optimizations disabled to verify that the same answer - ** is obtained in every case. - */ - case SQLITE_TESTCTRL_OPTIMIZATIONS: { - sqlite3 *db = va_arg(ap, sqlite3*); - db->dbOptFlags = (u16)(va_arg(ap, int) & 0xffff); - break; - } +/* fts3_tokenize_vtab.c */ +SQLITE_PRIVATE int sqlite3Fts3InitTok(sqlite3*, Fts3Hash *); - /* sqlite3_test_control(SQLITE_TESTCTRL_LOCALTIME_FAULT, int onoff); - ** - ** If parameter onoff is non-zero, configure the wrappers so that all - ** subsequent calls to localtime() and variants fail. If onoff is zero, - ** undo this setting. - */ - case SQLITE_TESTCTRL_LOCALTIME_FAULT: { - sqlite3GlobalConfig.bLocaltimeFault = va_arg(ap, int); - break; - } +/* fts3_unicode2.c (functions generated by parsing unicode text files) */ +#ifndef SQLITE_DISABLE_FTS3_UNICODE +SQLITE_PRIVATE int sqlite3FtsUnicodeFold(int, int); +SQLITE_PRIVATE int sqlite3FtsUnicodeIsalnum(int); +SQLITE_PRIVATE int sqlite3FtsUnicodeIsdiacritic(int); +#endif - /* sqlite3_test_control(SQLITE_TESTCTRL_NEVER_CORRUPT, int); - ** - ** Set or clear a flag that indicates that the database file is always well- - ** formed and never corrupt. This flag is clear by default, indicating that - ** database files might have arbitrary corruption. Setting the flag during - ** testing causes certain assert() statements in the code to be activated - ** that demonstrat invariants on well-formed database files. - */ - case SQLITE_TESTCTRL_NEVER_CORRUPT: { - sqlite3GlobalConfig.neverCorrupt = va_arg(ap, int); - break; - } +#endif /* !SQLITE_CORE || SQLITE_ENABLE_FTS3 */ +#endif /* _FTSINT_H */ - /* Set the threshold at which OP_Once counters reset back to zero. - ** By default this is 0x7ffffffe (over 2 billion), but that value is - ** too big to test in a reasonable amount of time, so this control is - ** provided to set a small and easily reachable reset value. - */ - case SQLITE_TESTCTRL_ONCE_RESET_THRESHOLD: { - sqlite3GlobalConfig.iOnceResetThreshold = va_arg(ap, int); - break; - } +/************** End of fts3Int.h *********************************************/ +/************** Continuing where we left off in fts3.c ***********************/ +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) - /* sqlite3_test_control(SQLITE_TESTCTRL_VDBE_COVERAGE, xCallback, ptr); - ** - ** Set the VDBE coverage callback function to xCallback with context - ** pointer ptr. - */ - case SQLITE_TESTCTRL_VDBE_COVERAGE: { -#ifdef SQLITE_VDBE_COVERAGE - typedef void (*branch_callback)(void*,int,u8,u8); - sqlite3GlobalConfig.xVdbeBranch = va_arg(ap,branch_callback); - sqlite3GlobalConfig.pVdbeBranchArg = va_arg(ap,void*); +#if defined(SQLITE_ENABLE_FTS3) && !defined(SQLITE_CORE) +# define SQLITE_CORE 1 #endif - break; - } - /* sqlite3_test_control(SQLITE_TESTCTRL_SORTER_MMAP, db, nMax); */ - case SQLITE_TESTCTRL_SORTER_MMAP: { - sqlite3 *db = va_arg(ap, sqlite3*); - db->nMaxSorterMmap = va_arg(ap, int); - break; - } +/* #include */ +/* #include */ +/* #include */ +/* #include */ +/* #include */ +/* #include */ - /* sqlite3_test_control(SQLITE_TESTCTRL_ISINIT); - ** - ** Return SQLITE_OK if SQLite has been initialized and SQLITE_ERROR if - ** not. - */ - case SQLITE_TESTCTRL_ISINIT: { - if( sqlite3GlobalConfig.isInit==0 ) rc = SQLITE_ERROR; - break; - } +/* #include "fts3.h" */ +#ifndef SQLITE_CORE +/* # include "sqlite3ext.h" */ + SQLITE_EXTENSION_INIT1 +#endif - /* sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, db, dbName, onOff, tnum); - ** - ** This test control is used to create imposter tables. "db" is a pointer - ** to the database connection. dbName is the database name (ex: "main" or - ** "temp") which will receive the imposter. "onOff" turns imposter mode on - ** or off. "tnum" is the root page of the b-tree to which the imposter - ** table should connect. - ** - ** Enable imposter mode only when the schema has already been parsed. Then - ** run a single CREATE TABLE statement to construct the imposter table in - ** the parsed schema. Then turn imposter mode back off again. - ** - ** If onOff==0 and tnum>0 then reset the schema for all databases, causing - ** the schema to be reparsed the next time it is needed. This has the - ** effect of erasing all imposter tables. - */ - case SQLITE_TESTCTRL_IMPOSTER: { - sqlite3 *db = va_arg(ap, sqlite3*); - sqlite3_mutex_enter(db->mutex); - db->init.iDb = sqlite3FindDbName(db, va_arg(ap,const char*)); - db->init.busy = db->init.imposterTable = va_arg(ap,int); - db->init.newTnum = va_arg(ap,int); - if( db->init.busy==0 && db->init.newTnum>0 ){ - sqlite3ResetAllSchemasOfConnection(db); - } - sqlite3_mutex_leave(db->mutex); - break; - } +static int fts3EvalNext(Fts3Cursor *pCsr); +static int fts3EvalStart(Fts3Cursor *pCsr); +static int fts3TermSegReaderCursor( + Fts3Cursor *, const char *, int, int, Fts3MultiSegReader **); -#if defined(YYCOVERAGE) - /* sqlite3_test_control(SQLITE_TESTCTRL_PARSER_COVERAGE, FILE *out) - ** - ** This test control (only available when SQLite is compiled with - ** -DYYCOVERAGE) writes a report onto "out" that shows all - ** state/lookahead combinations in the parser state machine - ** which are never exercised. If any state is missed, make the - ** return code SQLITE_ERROR. - */ - case SQLITE_TESTCTRL_PARSER_COVERAGE: { - FILE *out = va_arg(ap, FILE*); - if( sqlite3ParserCoverage(out) ) rc = SQLITE_ERROR; - break; - } -#endif /* defined(YYCOVERAGE) */ - } - va_end(ap); -#endif /* SQLITE_UNTESTABLE */ - return rc; -} +#ifndef SQLITE_AMALGAMATION +# if defined(SQLITE_DEBUG) +SQLITE_PRIVATE int sqlite3Fts3Always(int b) { assert( b ); return b; } +SQLITE_PRIVATE int sqlite3Fts3Never(int b) { assert( !b ); return b; } +# endif +#endif /* -** This is a utility routine, useful to VFS implementations, that checks -** to see if a database file was a URI that contained a specific query -** parameter, and if so obtains the value of the query parameter. -** -** The zFilename argument is the filename pointer passed into the xOpen() -** method of a VFS implementation. The zParam argument is the name of the -** query parameter we seek. This routine returns the value of the zParam -** parameter if it exists. If the parameter does not exist, this routine -** returns a NULL pointer. +** This variable is set to false when running tests for which the on disk +** structures should not be corrupt. Otherwise, true. If it is false, extra +** assert() conditions in the fts3 code are activated - conditions that are +** only true if it is guaranteed that the fts3 database is not corrupt. */ -SQLITE_API const char *sqlite3_uri_parameter(const char *zFilename, const char *zParam){ - if( zFilename==0 || zParam==0 ) return 0; - zFilename += sqlite3Strlen30(zFilename) + 1; - while( zFilename[0] ){ - int x = strcmp(zFilename, zParam); - zFilename += sqlite3Strlen30(zFilename) + 1; - if( x==0 ) return zFilename; - zFilename += sqlite3Strlen30(zFilename) + 1; - } - return 0; -} +SQLITE_API int sqlite3_fts3_may_be_corrupt = 1; -/* -** Return a boolean value for a query parameter. +/* +** Write a 64-bit variable-length integer to memory starting at p[0]. +** The length of data written will be between 1 and FTS3_VARINT_MAX bytes. +** The number of bytes written is returned. */ -SQLITE_API int sqlite3_uri_boolean(const char *zFilename, const char *zParam, int bDflt){ - const char *z = sqlite3_uri_parameter(zFilename, zParam); - bDflt = bDflt!=0; - return z ? sqlite3GetBoolean(z, bDflt) : bDflt; +SQLITE_PRIVATE int sqlite3Fts3PutVarint(char *p, sqlite_int64 v){ + unsigned char *q = (unsigned char *) p; + sqlite_uint64 vu = v; + do{ + *q++ = (unsigned char) ((vu & 0x7f) | 0x80); + vu >>= 7; + }while( vu!=0 ); + q[-1] &= 0x7f; /* turn off high bit in final byte */ + assert( q - (unsigned char *)p <= FTS3_VARINT_MAX ); + return (int) (q - (unsigned char *)p); } -/* -** Return a 64-bit integer value for a query parameter. -*/ -SQLITE_API sqlite3_int64 sqlite3_uri_int64( - const char *zFilename, /* Filename as passed to xOpen */ - const char *zParam, /* URI parameter sought */ - sqlite3_int64 bDflt /* return if parameter is missing */ -){ - const char *z = sqlite3_uri_parameter(zFilename, zParam); - sqlite3_int64 v; - if( z && sqlite3DecOrHexToI64(z, &v)==0 ){ - bDflt = v; - } - return bDflt; -} +#define GETVARINT_STEP(v, ptr, shift, mask1, mask2, var, ret) \ + v = (v & mask1) | ( (*(const unsigned char*)(ptr++)) << shift ); \ + if( (v & mask2)==0 ){ var = v; return ret; } +#define GETVARINT_INIT(v, ptr, shift, mask1, mask2, var, ret) \ + v = (*ptr++); \ + if( (v & mask2)==0 ){ var = v; return ret; } -/* -** Return the Btree pointer identified by zDbName. Return NULL if not found. -*/ -SQLITE_PRIVATE Btree *sqlite3DbNameToBtree(sqlite3 *db, const char *zDbName){ - int iDb = zDbName ? sqlite3FindDbName(db, zDbName) : 0; - return iDb<0 ? 0 : db->aDb[iDb].pBt; +SQLITE_PRIVATE int sqlite3Fts3GetVarintU(const char *pBuf, sqlite_uint64 *v){ + const unsigned char *p = (const unsigned char*)pBuf; + const unsigned char *pStart = p; + u32 a; + u64 b; + int shift; + + GETVARINT_INIT(a, p, 0, 0x00, 0x80, *v, 1); + GETVARINT_STEP(a, p, 7, 0x7F, 0x4000, *v, 2); + GETVARINT_STEP(a, p, 14, 0x3FFF, 0x200000, *v, 3); + GETVARINT_STEP(a, p, 21, 0x1FFFFF, 0x10000000, *v, 4); + b = (a & 0x0FFFFFFF ); + + for(shift=28; shift<=63; shift+=7){ + u64 c = *p++; + b += (c&0x7F) << shift; + if( (c & 0x80)==0 ) break; + } + *v = b; + return (int)(p - pStart); } -/* -** Return the filename of the database associated with a database -** connection. +/* +** Read a 64-bit variable-length integer from memory starting at p[0]. +** Return the number of bytes read, or 0 on error. +** The value is stored in *v. */ -SQLITE_API const char *sqlite3_db_filename(sqlite3 *db, const char *zDbName){ - Btree *pBt; -#ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) ){ - (void)SQLITE_MISUSE_BKPT; - return 0; - } -#endif - pBt = sqlite3DbNameToBtree(db, zDbName); - return pBt ? sqlite3BtreeGetFilename(pBt) : 0; +SQLITE_PRIVATE int sqlite3Fts3GetVarint(const char *pBuf, sqlite_int64 *v){ + return sqlite3Fts3GetVarintU(pBuf, (sqlite3_uint64*)v); } -/* -** Return 1 if database is read-only or 0 if read/write. Return -1 if -** no such database exists. +/* +** Read a 64-bit variable-length integer from memory starting at p[0] and +** not extending past pEnd[-1]. +** Return the number of bytes read, or 0 on error. +** The value is stored in *v. */ -SQLITE_API int sqlite3_db_readonly(sqlite3 *db, const char *zDbName){ - Btree *pBt; -#ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) ){ - (void)SQLITE_MISUSE_BKPT; - return -1; +SQLITE_PRIVATE int sqlite3Fts3GetVarintBounded( + const char *pBuf, + const char *pEnd, + sqlite_int64 *v +){ + const unsigned char *p = (const unsigned char*)pBuf; + const unsigned char *pStart = p; + const unsigned char *pX = (const unsigned char*)pEnd; + u64 b = 0; + int shift; + for(shift=0; shift<=63; shift+=7){ + u64 c = pmutex); - - if( db->autoCommit==0 ){ - int iDb = sqlite3FindDbName(db, zDb); - if( iDb==0 || iDb>1 ){ - Btree *pBt = db->aDb[iDb].pBt; - if( 0==sqlite3BtreeIsInTrans(pBt) ){ - rc = sqlite3BtreeBeginTrans(pBt, 0); - if( rc==SQLITE_OK ){ - rc = sqlite3PagerSnapshotGet(sqlite3BtreePager(pBt), ppSnapshot); - } - } - } - } - sqlite3_mutex_leave(db->mutex); -#endif /* SQLITE_OMIT_WAL */ - return rc; + GETVARINT_STEP(a, ptr, 7, 0x7F, 0x4000, *pi, 2); + GETVARINT_STEP(a, ptr, 14, 0x3FFF, 0x200000, *pi, 3); + GETVARINT_STEP(a, ptr, 21, 0x1FFFFF, 0x10000000, *pi, 4); + a = (a & 0x0FFFFFFF ); + *pi = (int)(a | ((u32)(*ptr & 0x07) << 28)); + assert( 0==(a & 0x80000000) ); + assert( *pi>=0 ); + return 5; } /* -** Open a read-transaction on the snapshot idendified by pSnapshot. +** Return the number of bytes required to encode v as a varint */ -SQLITE_API int sqlite3_snapshot_open( - sqlite3 *db, - const char *zDb, - sqlite3_snapshot *pSnapshot -){ - int rc = SQLITE_ERROR; -#ifndef SQLITE_OMIT_WAL - -#ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) ){ - return SQLITE_MISUSE_BKPT; - } -#endif - sqlite3_mutex_enter(db->mutex); - if( db->autoCommit==0 ){ - int iDb; - iDb = sqlite3FindDbName(db, zDb); - if( iDb==0 || iDb>1 ){ - Btree *pBt = db->aDb[iDb].pBt; - if( 0==sqlite3BtreeIsInReadTrans(pBt) ){ - rc = sqlite3PagerSnapshotOpen(sqlite3BtreePager(pBt), pSnapshot); - if( rc==SQLITE_OK ){ - rc = sqlite3BtreeBeginTrans(pBt, 0); - sqlite3PagerSnapshotOpen(sqlite3BtreePager(pBt), 0); - } - } - } - } - - sqlite3_mutex_leave(db->mutex); -#endif /* SQLITE_OMIT_WAL */ - return rc; +SQLITE_PRIVATE int sqlite3Fts3VarintLen(sqlite3_uint64 v){ + int i = 0; + do{ + i++; + v >>= 7; + }while( v!=0 ); + return i; } /* -** Recover as many snapshots as possible from the wal file associated with -** schema zDb of database db. +** Convert an SQL-style quoted string into a normal string by removing +** the quote characters. The conversion is done in-place. If the +** input does not begin with a quote character, then this routine +** is a no-op. +** +** Examples: +** +** "abc" becomes abc +** 'xyz' becomes xyz +** [pqr] becomes pqr +** `mno` becomes mno +** */ -SQLITE_API int sqlite3_snapshot_recover(sqlite3 *db, const char *zDb){ - int rc = SQLITE_ERROR; - int iDb; -#ifndef SQLITE_OMIT_WAL +SQLITE_PRIVATE void sqlite3Fts3Dequote(char *z){ + char quote; /* Quote character (if any ) */ -#ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) ){ - return SQLITE_MISUSE_BKPT; - } -#endif + quote = z[0]; + if( quote=='[' || quote=='\'' || quote=='"' || quote=='`' ){ + int iIn = 1; /* Index of next byte to read from input */ + int iOut = 0; /* Index of next byte to write to output */ - sqlite3_mutex_enter(db->mutex); - iDb = sqlite3FindDbName(db, zDb); - if( iDb==0 || iDb>1 ){ - Btree *pBt = db->aDb[iDb].pBt; - if( 0==sqlite3BtreeIsInReadTrans(pBt) ){ - rc = sqlite3BtreeBeginTrans(pBt, 0); - if( rc==SQLITE_OK ){ - rc = sqlite3PagerSnapshotRecover(sqlite3BtreePager(pBt)); - sqlite3BtreeCommit(pBt); + /* If the first byte was a '[', then the close-quote character is a ']' */ + if( quote=='[' ) quote = ']'; + + while( z[iIn] ){ + if( z[iIn]==quote ){ + if( z[iIn+1]!=quote ) break; + z[iOut++] = quote; + iIn += 2; + }else{ + z[iOut++] = z[iIn++]; } } + z[iOut] = '\0'; } - sqlite3_mutex_leave(db->mutex); -#endif /* SQLITE_OMIT_WAL */ - return rc; } /* -** Free a snapshot handle obtained from sqlite3_snapshot_get(). +** Read a single varint from the doclist at *pp and advance *pp to point +** to the first byte past the end of the varint. Add the value of the varint +** to *pVal. */ -SQLITE_API void sqlite3_snapshot_free(sqlite3_snapshot *pSnapshot){ - sqlite3_free(pSnapshot); +static void fts3GetDeltaVarint(char **pp, sqlite3_int64 *pVal){ + sqlite3_int64 iVal; + *pp += sqlite3Fts3GetVarint(*pp, &iVal); + *pVal += iVal; } -#endif /* SQLITE_ENABLE_SNAPSHOT */ -#ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS /* -** Given the name of a compile-time option, return true if that option -** was used and false if not. +** When this function is called, *pp points to the first byte following a +** varint that is part of a doclist (or position-list, or any other list +** of varints). This function moves *pp to point to the start of that varint, +** and sets *pVal by the varint value. ** -** The name can optionally begin with "SQLITE_" but the "SQLITE_" prefix -** is not required for a match. +** Argument pStart points to the first byte of the doclist that the +** varint is part of. */ -SQLITE_API int sqlite3_compileoption_used(const char *zOptName){ - int i, n; - int nOpt; - const char **azCompileOpt; - -#if SQLITE_ENABLE_API_ARMOR - if( zOptName==0 ){ - (void)SQLITE_MISUSE_BKPT; - return 0; - } -#endif +static void fts3GetReverseVarint( + char **pp, + char *pStart, + sqlite3_int64 *pVal +){ + sqlite3_int64 iVal; + char *p; - azCompileOpt = sqlite3CompileOptions(&nOpt); + /* Pointer p now points at the first byte past the varint we are + ** interested in. So, unless the doclist is corrupt, the 0x80 bit is + ** clear on character p[-1]. */ + for(p = (*pp)-2; p>=pStart && *p&0x80; p--); + p++; + *pp = p; + + sqlite3Fts3GetVarint(p, &iVal); + *pVal = iVal; +} + +/* +** The xDisconnect() virtual table method. +*/ +static int fts3DisconnectMethod(sqlite3_vtab *pVtab){ + Fts3Table *p = (Fts3Table *)pVtab; + int i; - if( sqlite3StrNICmp(zOptName, "SQLITE_", 7)==0 ) zOptName += 7; - n = sqlite3Strlen30(zOptName); + assert( p->nPendingData==0 ); + assert( p->pSegments==0 ); - /* Since nOpt is normally in single digits, a linear search is - ** adequate. No need for a binary search. */ - for(i=0; ipSeekStmt); + for(i=0; iaStmt); i++){ + sqlite3_finalize(p->aStmt[i]); } - return 0; + sqlite3_free(p->zSegmentsTbl); + sqlite3_free(p->zReadExprlist); + sqlite3_free(p->zWriteExprlist); + sqlite3_free(p->zContentTbl); + sqlite3_free(p->zLanguageid); + + /* Invoke the tokenizer destructor to free the tokenizer. */ + p->pTokenizer->pModule->xDestroy(p->pTokenizer); + + sqlite3_free(p); + return SQLITE_OK; } /* -** Return the N-th compile-time option string. If N is out of range, -** return a NULL pointer. +** Write an error message into *pzErr */ -SQLITE_API const char *sqlite3_compileoption_get(int N){ - int nOpt; - const char **azCompileOpt; - azCompileOpt = sqlite3CompileOptions(&nOpt); - if( N>=0 && NzDb; /* Name of database (e.g. "main", "temp") */ + sqlite3 *db = p->db; /* Database handle */ -#define assertMutexHeld() \ - assert( sqlite3_mutex_held(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)) ) + /* Drop the shadow tables */ + fts3DbExec(&rc, db, + "DROP TABLE IF EXISTS %Q.'%q_segments';" + "DROP TABLE IF EXISTS %Q.'%q_segdir';" + "DROP TABLE IF EXISTS %Q.'%q_docsize';" + "DROP TABLE IF EXISTS %Q.'%q_stat';" + "%s DROP TABLE IF EXISTS %Q.'%q_content';", + zDb, p->zName, + zDb, p->zName, + zDb, p->zName, + zDb, p->zName, + (p->zContentTbl ? "--" : ""), zDb,p->zName + ); + + /* If everything has worked, invoke fts3DisconnectMethod() to free the + ** memory associated with the Fts3Table structure and return SQLITE_OK. + ** Otherwise, return an SQLite error code. + */ + return (rc==SQLITE_OK ? fts3DisconnectMethod(pVtab) : rc); +} -/* -** Head of a linked list of all sqlite3 objects created by this process -** for which either sqlite3.pBlockingConnection or sqlite3.pUnlockConnection -** is not NULL. This variable may only accessed while the STATIC_MASTER -** mutex is held. -*/ -static sqlite3 *SQLITE_WSD sqlite3BlockedList = 0; -#ifndef NDEBUG /* -** This function is a complex assert() that verifies the following -** properties of the blocked connections list: -** -** 1) Each entry in the list has a non-NULL value for either -** pUnlockConnection or pBlockingConnection, or both. -** -** 2) All entries in the list that share a common value for -** xUnlockNotify are grouped together. +** Invoke sqlite3_declare_vtab() to declare the schema for the FTS3 table +** passed as the first argument. This is done as part of the xConnect() +** and xCreate() methods. ** -** 3) If the argument db is not NULL, then none of the entries in the -** blocked connections list have pUnlockConnection or pBlockingConnection -** set to db. This is used when closing connection db. +** If *pRc is non-zero when this function is called, it is a no-op. +** Otherwise, if an error occurs, an SQLite error code is stored in *pRc +** before returning. */ -static void checkListProperties(sqlite3 *db){ - sqlite3 *p; - for(p=sqlite3BlockedList; p; p=p->pNextBlocked){ - int seen = 0; - sqlite3 *p2; +static void fts3DeclareVtab(int *pRc, Fts3Table *p){ + if( *pRc==SQLITE_OK ){ + int i; /* Iterator variable */ + int rc; /* Return code */ + char *zSql; /* SQL statement passed to declare_vtab() */ + char *zCols; /* List of user defined columns */ + const char *zLanguageid; - /* Verify property (1) */ - assert( p->pUnlockConnection || p->pBlockingConnection ); + zLanguageid = (p->zLanguageid ? p->zLanguageid : "__langid"); + sqlite3_vtab_config(p->db, SQLITE_VTAB_CONSTRAINT_SUPPORT, 1); - /* Verify property (2) */ - for(p2=sqlite3BlockedList; p2!=p; p2=p2->pNextBlocked){ - if( p2->xUnlockNotify==p->xUnlockNotify ) seen = 1; - assert( p2->xUnlockNotify==p->xUnlockNotify || !seen ); - assert( db==0 || p->pUnlockConnection!=db ); - assert( db==0 || p->pBlockingConnection!=db ); + /* Create a list of user columns for the virtual table */ + zCols = sqlite3_mprintf("%Q, ", p->azColumn[0]); + for(i=1; zCols && inColumn; i++){ + zCols = sqlite3_mprintf("%z%Q, ", zCols, p->azColumn[i]); } - } -} -#else -# define checkListProperties(x) -#endif -/* -** Remove connection db from the blocked connections list. If connection -** db is not currently a part of the list, this function is a no-op. -*/ -static void removeFromBlockedList(sqlite3 *db){ - sqlite3 **pp; - assertMutexHeld(); - for(pp=&sqlite3BlockedList; *pp; pp = &(*pp)->pNextBlocked){ - if( *pp==db ){ - *pp = (*pp)->pNextBlocked; - break; + /* Create the whole "CREATE TABLE" statement to pass to SQLite */ + zSql = sqlite3_mprintf( + "CREATE TABLE x(%s %Q HIDDEN, docid HIDDEN, %Q HIDDEN)", + zCols, p->zName, zLanguageid + ); + if( !zCols || !zSql ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_declare_vtab(p->db, zSql); } + + sqlite3_free(zSql); + sqlite3_free(zCols); + *pRc = rc; } } /* -** Add connection db to the blocked connections list. It is assumed -** that it is not already a part of the list. +** Create the %_stat table if it does not already exist. */ -static void addToBlockedList(sqlite3 *db){ - sqlite3 **pp; - assertMutexHeld(); - for( - pp=&sqlite3BlockedList; - *pp && (*pp)->xUnlockNotify!=db->xUnlockNotify; - pp=&(*pp)->pNextBlocked +SQLITE_PRIVATE void sqlite3Fts3CreateStatTable(int *pRc, Fts3Table *p){ + fts3DbExec(pRc, p->db, + "CREATE TABLE IF NOT EXISTS %Q.'%q_stat'" + "(id INTEGER PRIMARY KEY, value BLOB);", + p->zDb, p->zName ); - db->pNextBlocked = *pp; - *pp = db; + if( (*pRc)==SQLITE_OK ) p->bHasStat = 1; } /* -** Obtain the STATIC_MASTER mutex. +** Create the backing store tables (%_content, %_segments and %_segdir) +** required by the FTS3 table passed as the only argument. This is done +** as part of the vtab xCreate() method. +** +** If the p->bHasDocsize boolean is true (indicating that this is an +** FTS4 table, not an FTS3 table) then also create the %_docsize and +** %_stat tables required by FTS4. */ -static void enterMutex(void){ - sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); - checkListProperties(0); +static int fts3CreateTables(Fts3Table *p){ + int rc = SQLITE_OK; /* Return code */ + int i; /* Iterator variable */ + sqlite3 *db = p->db; /* The database connection */ + + if( p->zContentTbl==0 ){ + const char *zLanguageid = p->zLanguageid; + char *zContentCols; /* Columns of %_content table */ + + /* Create a list of user columns for the content table */ + zContentCols = sqlite3_mprintf("docid INTEGER PRIMARY KEY"); + for(i=0; zContentCols && inColumn; i++){ + char *z = p->azColumn[i]; + zContentCols = sqlite3_mprintf("%z, 'c%d%q'", zContentCols, i, z); + } + if( zLanguageid && zContentCols ){ + zContentCols = sqlite3_mprintf("%z, langid", zContentCols, zLanguageid); + } + if( zContentCols==0 ) rc = SQLITE_NOMEM; + + /* Create the content table */ + fts3DbExec(&rc, db, + "CREATE TABLE %Q.'%q_content'(%s)", + p->zDb, p->zName, zContentCols + ); + sqlite3_free(zContentCols); + } + + /* Create other tables */ + fts3DbExec(&rc, db, + "CREATE TABLE %Q.'%q_segments'(blockid INTEGER PRIMARY KEY, block BLOB);", + p->zDb, p->zName + ); + fts3DbExec(&rc, db, + "CREATE TABLE %Q.'%q_segdir'(" + "level INTEGER," + "idx INTEGER," + "start_block INTEGER," + "leaves_end_block INTEGER," + "end_block INTEGER," + "root BLOB," + "PRIMARY KEY(level, idx)" + ");", + p->zDb, p->zName + ); + if( p->bHasDocsize ){ + fts3DbExec(&rc, db, + "CREATE TABLE %Q.'%q_docsize'(docid INTEGER PRIMARY KEY, size BLOB);", + p->zDb, p->zName + ); + } + assert( p->bHasStat==p->bFts4 ); + if( p->bHasStat ){ + sqlite3Fts3CreateStatTable(&rc, p); + } + return rc; } /* -** Release the STATIC_MASTER mutex. +** Store the current database page-size in bytes in p->nPgsz. +** +** If *pRc is non-zero when this function is called, it is a no-op. +** Otherwise, if an error occurs, an SQLite error code is stored in *pRc +** before returning. */ -static void leaveMutex(void){ - assertMutexHeld(); - checkListProperties(0); - sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); +static void fts3DatabasePageSize(int *pRc, Fts3Table *p){ + if( *pRc==SQLITE_OK ){ + int rc; /* Return code */ + char *zSql; /* SQL text "PRAGMA %Q.page_size" */ + sqlite3_stmt *pStmt; /* Compiled "PRAGMA %Q.page_size" statement */ + + zSql = sqlite3_mprintf("PRAGMA %Q.page_size", p->zDb); + if( !zSql ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare(p->db, zSql, -1, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_step(pStmt); + p->nPgsz = sqlite3_column_int(pStmt, 0); + rc = sqlite3_finalize(pStmt); + }else if( rc==SQLITE_AUTH ){ + p->nPgsz = 1024; + rc = SQLITE_OK; + } + } + assert( p->nPgsz>0 || rc!=SQLITE_OK ); + sqlite3_free(zSql); + *pRc = rc; + } } /* -** Register an unlock-notify callback. -** -** This is called after connection "db" has attempted some operation -** but has received an SQLITE_LOCKED error because another connection -** (call it pOther) in the same process was busy using the same shared -** cache. pOther is found by looking at db->pBlockingConnection. -** -** If there is no blocking connection, the callback is invoked immediately, -** before this routine returns. -** -** If pOther is already blocked on db, then report SQLITE_LOCKED, to indicate -** a deadlock. +** "Special" FTS4 arguments are column specifications of the following form: ** -** Otherwise, make arrangements to invoke xNotify when pOther drops -** its locks. +** = ** -** Each call to this routine overrides any prior callbacks registered -** on the same "db". If xNotify==0 then any prior callbacks are immediately -** cancelled. +** There may not be whitespace surrounding the "=" character. The +** term may be quoted, but the may not. */ -SQLITE_API int sqlite3_unlock_notify( - sqlite3 *db, - void (*xNotify)(void **, int), - void *pArg +static int fts3IsSpecialColumn( + const char *z, + int *pnKey, + char **pzValue ){ - int rc = SQLITE_OK; + char *zValue; + const char *zCsr = z; - sqlite3_mutex_enter(db->mutex); - enterMutex(); + while( *zCsr!='=' ){ + if( *zCsr=='\0' ) return 0; + zCsr++; + } - if( xNotify==0 ){ - removeFromBlockedList(db); - db->pBlockingConnection = 0; - db->pUnlockConnection = 0; - db->xUnlockNotify = 0; - db->pUnlockArg = 0; - }else if( 0==db->pBlockingConnection ){ - /* The blocking transaction has been concluded. Or there never was a - ** blocking transaction. In either case, invoke the notify callback - ** immediately. - */ - xNotify(&pArg, 1); - }else{ - sqlite3 *p; + *pnKey = (int)(zCsr-z); + zValue = sqlite3_mprintf("%s", &zCsr[1]); + if( zValue ){ + sqlite3Fts3Dequote(zValue); + } + *pzValue = zValue; + return 1; +} - for(p=db->pBlockingConnection; p && p!=db; p=p->pUnlockConnection){} - if( p ){ - rc = SQLITE_LOCKED; /* Deadlock detected. */ - }else{ - db->pUnlockConnection = db->pBlockingConnection; - db->xUnlockNotify = xNotify; - db->pUnlockArg = pArg; - removeFromBlockedList(db); - addToBlockedList(db); +/* +** Append the output of a printf() style formatting to an existing string. +*/ +static void fts3Appendf( + int *pRc, /* IN/OUT: Error code */ + char **pz, /* IN/OUT: Pointer to string buffer */ + const char *zFormat, /* Printf format string to append */ + ... /* Arguments for printf format string */ +){ + if( *pRc==SQLITE_OK ){ + va_list ap; + char *z; + va_start(ap, zFormat); + z = sqlite3_vmprintf(zFormat, ap); + va_end(ap); + if( z && *pz ){ + char *z2 = sqlite3_mprintf("%s%s", *pz, z); + sqlite3_free(z); + z = z2; } + if( z==0 ) *pRc = SQLITE_NOMEM; + sqlite3_free(*pz); + *pz = z; } - - leaveMutex(); - assert( !db->mallocFailed ); - sqlite3ErrorWithMsg(db, rc, (rc?"database is deadlocked":0)); - sqlite3_mutex_leave(db->mutex); - return rc; } /* -** This function is called while stepping or preparing a statement -** associated with connection db. The operation will return SQLITE_LOCKED -** to the user because it requires a lock that will not be available -** until connection pBlocker concludes its current transaction. +** Return a copy of input string zInput enclosed in double-quotes (") and +** with all double quote characters escaped. For example: +** +** fts3QuoteId("un \"zip\"") -> "un \"\"zip\"\"" +** +** The pointer returned points to memory obtained from sqlite3_malloc(). It +** is the callers responsibility to call sqlite3_free() to release this +** memory. */ -SQLITE_PRIVATE void sqlite3ConnectionBlocked(sqlite3 *db, sqlite3 *pBlocker){ - enterMutex(); - if( db->pBlockingConnection==0 && db->pUnlockConnection==0 ){ - addToBlockedList(db); +static char *fts3QuoteId(char const *zInput){ + sqlite3_int64 nRet; + char *zRet; + nRet = 2 + (int)strlen(zInput)*2 + 1; + zRet = sqlite3_malloc64(nRet); + if( zRet ){ + int i; + char *z = zRet; + *(z++) = '"'; + for(i=0; zInput[i]; i++){ + if( zInput[i]=='"' ) *(z++) = '"'; + *(z++) = zInput[i]; + } + *(z++) = '"'; + *(z++) = '\0'; } - db->pBlockingConnection = pBlocker; - leaveMutex(); + return zRet; } /* -** This function is called when -** the transaction opened by database db has just finished. Locks held -** by database connection db have been released. +** Return a list of comma separated SQL expressions and a FROM clause that +** could be used in a SELECT statement such as the following: ** -** This function loops through each entry in the blocked connections -** list and does the following: +** SELECT FROM %_content AS x ... ** -** 1) If the sqlite3.pBlockingConnection member of a list entry is -** set to db, then set pBlockingConnection=0. +** to return the docid, followed by each column of text data in order +** from left to write. If parameter zFunc is not NULL, then instead of +** being returned directly each column of text data is passed to an SQL +** function named zFunc first. For example, if zFunc is "unzip" and the +** table has the three user-defined columns "a", "b", and "c", the following +** string is returned: ** -** 2) If the sqlite3.pUnlockConnection member of a list entry is -** set to db, then invoke the configured unlock-notify callback and -** set pUnlockConnection=0. +** "docid, unzip(x.'a'), unzip(x.'b'), unzip(x.'c') FROM %_content AS x" ** -** 3) If the two steps above mean that pBlockingConnection==0 and -** pUnlockConnection==0, remove the entry from the blocked connections -** list. +** The pointer returned points to a buffer allocated by sqlite3_malloc(). It +** is the responsibility of the caller to eventually free it. +** +** If *pRc is not SQLITE_OK when this function is called, it is a no-op (and +** a NULL pointer is returned). Otherwise, if an OOM error is encountered +** by this function, NULL is returned and *pRc is set to SQLITE_NOMEM. If +** no error occurs, *pRc is left unmodified. */ -SQLITE_PRIVATE void sqlite3ConnectionUnlocked(sqlite3 *db){ - void (*xUnlockNotify)(void **, int) = 0; /* Unlock-notify cb to invoke */ - int nArg = 0; /* Number of entries in aArg[] */ - sqlite3 **pp; /* Iterator variable */ - void **aArg; /* Arguments to the unlock callback */ - void **aDyn = 0; /* Dynamically allocated space for aArg[] */ - void *aStatic[16]; /* Starter space for aArg[]. No malloc required */ - - aArg = aStatic; - enterMutex(); /* Enter STATIC_MASTER mutex */ - - /* This loop runs once for each entry in the blocked-connections list. */ - for(pp=&sqlite3BlockedList; *pp; /* no-op */ ){ - sqlite3 *p = *pp; +static char *fts3ReadExprList(Fts3Table *p, const char *zFunc, int *pRc){ + char *zRet = 0; + char *zFree = 0; + char *zFunction; + int i; - /* Step 1. */ - if( p->pBlockingConnection==db ){ - p->pBlockingConnection = 0; + if( p->zContentTbl==0 ){ + if( !zFunc ){ + zFunction = ""; + }else{ + zFree = zFunction = fts3QuoteId(zFunc); } - - /* Step 2. */ - if( p->pUnlockConnection==db ){ - assert( p->xUnlockNotify ); - if( p->xUnlockNotify!=xUnlockNotify && nArg!=0 ){ - xUnlockNotify(aArg, nArg); - nArg = 0; - } - - sqlite3BeginBenignMalloc(); - assert( aArg==aDyn || (aDyn==0 && aArg==aStatic) ); - assert( nArg<=(int)ArraySize(aStatic) || aArg==aDyn ); - if( (!aDyn && nArg==(int)ArraySize(aStatic)) - || (aDyn && nArg==(int)(sqlite3MallocSize(aDyn)/sizeof(void*))) - ){ - /* The aArg[] array needs to grow. */ - void **pNew = (void **)sqlite3Malloc(nArg*sizeof(void *)*2); - if( pNew ){ - memcpy(pNew, aArg, nArg*sizeof(void *)); - sqlite3_free(aDyn); - aDyn = aArg = pNew; - }else{ - /* This occurs when the array of context pointers that need to - ** be passed to the unlock-notify callback is larger than the - ** aStatic[] array allocated on the stack and the attempt to - ** allocate a larger array from the heap has failed. - ** - ** This is a difficult situation to handle. Returning an error - ** code to the caller is insufficient, as even if an error code - ** is returned the transaction on connection db will still be - ** closed and the unlock-notify callbacks on blocked connections - ** will go unissued. This might cause the application to wait - ** indefinitely for an unlock-notify callback that will never - ** arrive. - ** - ** Instead, invoke the unlock-notify callback with the context - ** array already accumulated. We can then clear the array and - ** begin accumulating any further context pointers without - ** requiring any dynamic allocation. This is sub-optimal because - ** it means that instead of one callback with a large array of - ** context pointers the application will receive two or more - ** callbacks with smaller arrays of context pointers, which will - ** reduce the applications ability to prioritize multiple - ** connections. But it is the best that can be done under the - ** circumstances. - */ - xUnlockNotify(aArg, nArg); - nArg = 0; - } - } - sqlite3EndBenignMalloc(); - - aArg[nArg++] = p->pUnlockArg; - xUnlockNotify = p->xUnlockNotify; - p->pUnlockConnection = 0; - p->xUnlockNotify = 0; - p->pUnlockArg = 0; + fts3Appendf(pRc, &zRet, "docid"); + for(i=0; inColumn; i++){ + fts3Appendf(pRc, &zRet, ",%s(x.'c%d%q')", zFunction, i, p->azColumn[i]); } - - /* Step 3. */ - if( p->pBlockingConnection==0 && p->pUnlockConnection==0 ){ - /* Remove connection p from the blocked connections list. */ - *pp = p->pNextBlocked; - p->pNextBlocked = 0; - }else{ - pp = &p->pNextBlocked; + if( p->zLanguageid ){ + fts3Appendf(pRc, &zRet, ", x.%Q", "langid"); + } + sqlite3_free(zFree); + }else{ + fts3Appendf(pRc, &zRet, "rowid"); + for(i=0; inColumn; i++){ + fts3Appendf(pRc, &zRet, ", x.'%q'", p->azColumn[i]); + } + if( p->zLanguageid ){ + fts3Appendf(pRc, &zRet, ", x.%Q", p->zLanguageid); } } - - if( nArg!=0 ){ - xUnlockNotify(aArg, nArg); - } - sqlite3_free(aDyn); - leaveMutex(); /* Leave STATIC_MASTER mutex */ -} - -/* -** This is called when the database connection passed as an argument is -** being closed. The connection is removed from the blocked list. -*/ -SQLITE_PRIVATE void sqlite3ConnectionClosed(sqlite3 *db){ - sqlite3ConnectionUnlocked(db); - enterMutex(); - removeFromBlockedList(db); - checkListProperties(db); - leaveMutex(); + fts3Appendf(pRc, &zRet, " FROM '%q'.'%q%s' AS x", + p->zDb, + (p->zContentTbl ? p->zContentTbl : p->zName), + (p->zContentTbl ? "" : "_content") + ); + return zRet; } -#endif -/************** End of notify.c **********************************************/ -/************** Begin file fts3.c ********************************************/ /* -** 2006 Oct 10 +** Return a list of N comma separated question marks, where N is the number +** of columns in the %_content table (one for the docid plus one for each +** user-defined text column). ** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: +** If argument zFunc is not NULL, then all but the first question mark +** is preceded by zFunc and an open bracket, and followed by a closed +** bracket. For example, if zFunc is "zip" and the FTS3 table has three +** user-defined text columns, the following string is returned: ** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. +** "?, zip(?), zip(?), zip(?)" ** -****************************************************************************** +** The pointer returned points to a buffer allocated by sqlite3_malloc(). It +** is the responsibility of the caller to eventually free it. ** -** This is an SQLite module implementing full-text search. +** If *pRc is not SQLITE_OK when this function is called, it is a no-op (and +** a NULL pointer is returned). Otherwise, if an OOM error is encountered +** by this function, NULL is returned and *pRc is set to SQLITE_NOMEM. If +** no error occurs, *pRc is left unmodified. */ +static char *fts3WriteExprList(Fts3Table *p, const char *zFunc, int *pRc){ + char *zRet = 0; + char *zFree = 0; + char *zFunction; + int i; + + if( !zFunc ){ + zFunction = ""; + }else{ + zFree = zFunction = fts3QuoteId(zFunc); + } + fts3Appendf(pRc, &zRet, "?"); + for(i=0; inColumn; i++){ + fts3Appendf(pRc, &zRet, ",%s(?)", zFunction); + } + if( p->zLanguageid ){ + fts3Appendf(pRc, &zRet, ", ?"); + } + sqlite3_free(zFree); + return zRet; +} /* -** The code in this file is only compiled if: -** -** * The FTS3 module is being built as an extension -** (in which case SQLITE_CORE is not defined), or -** -** * The FTS3 module is being built into the core of -** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). +** Buffer z contains a positive integer value encoded as utf-8 text. +** Decode this value and store it in *pnOut, returning the number of bytes +** consumed. If an overflow error occurs return a negative value. */ +SQLITE_PRIVATE int sqlite3Fts3ReadInt(const char *z, int *pnOut){ + u64 iVal = 0; + int i; + for(i=0; z[i]>='0' && z[i]<='9'; i++){ + iVal = iVal*10 + (z[i] - '0'); + if( iVal>0x7FFFFFFF ) return -1; + } + *pnOut = (int)iVal; + return i; +} -/* The full-text index is stored in a series of b+tree (-like) -** structures called segments which map terms to doclists. The -** structures are like b+trees in layout, but are constructed from the -** bottom up in optimal fashion and are not updatable. Since trees -** are built from the bottom up, things will be described from the -** bottom up. -** -** -**** Varints **** -** The basic unit of encoding is a variable-length integer called a -** varint. We encode variable-length integers in little-endian order -** using seven bits * per byte as follows: -** -** KEY: -** A = 0xxxxxxx 7 bits of data and one flag bit -** B = 1xxxxxxx 7 bits of data and one flag bit -** -** 7 bits - A -** 14 bits - BA -** 21 bits - BBA -** and so on. -** -** This is similar in concept to how sqlite encodes "varints" but -** the encoding is not the same. SQLite varints are big-endian -** are are limited to 9 bytes in length whereas FTS3 varints are -** little-endian and can be up to 10 bytes in length (in theory). -** -** Example encodings: -** -** 1: 0x01 -** 127: 0x7f -** 128: 0x81 0x00 -** -** -**** Document lists **** -** A doclist (document list) holds a docid-sorted list of hits for a -** given term. Doclists hold docids and associated token positions. -** A docid is the unique integer identifier for a single document. -** A position is the index of a word within the document. The first -** word of the document has a position of 0. -** -** FTS3 used to optionally store character offsets using a compile-time -** option. But that functionality is no longer supported. -** -** A doclist is stored like this: -** -** array { -** varint docid; (delta from previous doclist) -** array { (position list for column 0) -** varint position; (2 more than the delta from previous position) -** } -** array { -** varint POS_COLUMN; (marks start of position list for new column) -** varint column; (index of new column) -** array { -** varint position; (2 more than the delta from previous position) -** } -** } -** varint POS_END; (marks end of positions for this document. -** } -** -** Here, array { X } means zero or more occurrences of X, adjacent in -** memory. A "position" is an index of a token in the token stream -** generated by the tokenizer. Note that POS_END and POS_COLUMN occur -** in the same logical place as the position element, and act as sentinals -** ending a position list array. POS_END is 0. POS_COLUMN is 1. -** The positions numbers are not stored literally but rather as two more -** than the difference from the prior position, or the just the position plus -** 2 for the first position. Example: -** -** label: A B C D E F G H I J K -** value: 123 5 9 1 1 14 35 0 234 72 0 -** -** The 123 value is the first docid. For column zero in this document -** there are two matches at positions 3 and 10 (5-2 and 9-2+3). The 1 -** at D signals the start of a new column; the 1 at E indicates that the -** new column is column number 1. There are two positions at 12 and 45 -** (14-2 and 35-2+12). The 0 at H indicate the end-of-document. The -** 234 at I is the delta to next docid (357). It has one position 70 -** (72-2) and then terminates with the 0 at K. -** -** A "position-list" is the list of positions for multiple columns for -** a single docid. A "column-list" is the set of positions for a single -** column. Hence, a position-list consists of one or more column-lists, -** a document record consists of a docid followed by a position-list and -** a doclist consists of one or more document records. -** -** A bare doclist omits the position information, becoming an -** array of varint-encoded docids. -** -**** Segment leaf nodes **** -** Segment leaf nodes store terms and doclists, ordered by term. Leaf -** nodes are written using LeafWriter, and read using LeafReader (to -** iterate through a single leaf node's data) and LeavesReader (to -** iterate through a segment's entire leaf layer). Leaf nodes have -** the format: -** -** varint iHeight; (height from leaf level, always 0) -** varint nTerm; (length of first term) -** char pTerm[nTerm]; (content of first term) -** varint nDoclist; (length of term's associated doclist) -** char pDoclist[nDoclist]; (content of doclist) -** array { -** (further terms are delta-encoded) -** varint nPrefix; (length of prefix shared with previous term) -** varint nSuffix; (length of unshared suffix) -** char pTermSuffix[nSuffix];(unshared suffix of next term) -** varint nDoclist; (length of term's associated doclist) -** char pDoclist[nDoclist]; (content of doclist) -** } -** -** Here, array { X } means zero or more occurrences of X, adjacent in -** memory. -** -** Leaf nodes are broken into blocks which are stored contiguously in -** the %_segments table in sorted order. This means that when the end -** of a node is reached, the next term is in the node with the next -** greater node id. -** -** New data is spilled to a new leaf node when the current node -** exceeds LEAF_MAX bytes (default 2048). New data which itself is -** larger than STANDALONE_MIN (default 1024) is placed in a standalone -** node (a leaf node with a single term and doclist). The goal of -** these settings is to pack together groups of small doclists while -** making it efficient to directly access large doclists. The -** assumption is that large doclists represent terms which are more -** likely to be query targets. -** -** TODO(shess) It may be useful for blocking decisions to be more -** dynamic. For instance, it may make more sense to have a 2.5k leaf -** node rather than splitting into 2k and .5k nodes. My intuition is -** that this might extend through 2x or 4x the pagesize. -** -** -**** Segment interior nodes **** -** Segment interior nodes store blockids for subtree nodes and terms -** to describe what data is stored by the each subtree. Interior -** nodes are written using InteriorWriter, and read using -** InteriorReader. InteriorWriters are created as needed when -** SegmentWriter creates new leaf nodes, or when an interior node -** itself grows too big and must be split. The format of interior -** nodes: -** -** varint iHeight; (height from leaf level, always >0) -** varint iBlockid; (block id of node's leftmost subtree) -** optional { -** varint nTerm; (length of first term) -** char pTerm[nTerm]; (content of first term) -** array { -** (further terms are delta-encoded) -** varint nPrefix; (length of shared prefix with previous term) -** varint nSuffix; (length of unshared suffix) -** char pTermSuffix[nSuffix]; (unshared suffix of next term) -** } -** } -** -** Here, optional { X } means an optional element, while array { X } -** means zero or more occurrences of X, adjacent in memory. -** -** An interior node encodes n terms separating n+1 subtrees. The -** subtree blocks are contiguous, so only the first subtree's blockid -** is encoded. The subtree at iBlockid will contain all terms less -** than the first term encoded (or all terms if no term is encoded). -** Otherwise, for terms greater than or equal to pTerm[i] but less -** than pTerm[i+1], the subtree for that term will be rooted at -** iBlockid+i. Interior nodes only store enough term data to -** distinguish adjacent children (if the rightmost term of the left -** child is "something", and the leftmost term of the right child is -** "wicked", only "w" is stored). -** -** New data is spilled to a new interior node at the same height when -** the current node exceeds INTERIOR_MAX bytes (default 2048). -** INTERIOR_MIN_TERMS (default 7) keeps large terms from monopolizing -** interior nodes and making the tree too skinny. The interior nodes -** at a given height are naturally tracked by interior nodes at -** height+1, and so on. -** -** -**** Segment directory **** -** The segment directory in table %_segdir stores meta-information for -** merging and deleting segments, and also the root node of the -** segment's tree. -** -** The root node is the top node of the segment's tree after encoding -** the entire segment, restricted to ROOT_MAX bytes (default 1024). -** This could be either a leaf node or an interior node. If the top -** node requires more than ROOT_MAX bytes, it is flushed to %_segments -** and a new root interior node is generated (which should always fit -** within ROOT_MAX because it only needs space for 2 varints, the -** height and the blockid of the previous root). -** -** The meta-information in the segment directory is: -** level - segment level (see below) -** idx - index within level -** - (level,idx uniquely identify a segment) -** start_block - first leaf node -** leaves_end_block - last leaf node -** end_block - last block (including interior nodes) -** root - contents of root node -** -** If the root node is a leaf node, then start_block, -** leaves_end_block, and end_block are all 0. -** -** -**** Segment merging **** -** To amortize update costs, segments are grouped into levels and -** merged in batches. Each increase in level represents exponentially -** more documents. -** -** New documents (actually, document updates) are tokenized and -** written individually (using LeafWriter) to a level 0 segment, with -** incrementing idx. When idx reaches MERGE_COUNT (default 16), all -** level 0 segments are merged into a single level 1 segment. Level 1 -** is populated like level 0, and eventually MERGE_COUNT level 1 -** segments are merged to a single level 2 segment (representing -** MERGE_COUNT^2 updates), and so on. -** -** A segment merge traverses all segments at a given level in -** parallel, performing a straightforward sorted merge. Since segment -** leaf nodes are written in to the %_segments table in order, this -** merge traverses the underlying sqlite disk structures efficiently. -** After the merge, all segment blocks from the merged level are -** deleted. -** -** MERGE_COUNT controls how often we merge segments. 16 seems to be -** somewhat of a sweet spot for insertion performance. 32 and 64 show -** very similar performance numbers to 16 on insertion, though they're -** a tiny bit slower (perhaps due to more overhead in merge-time -** sorting). 8 is about 20% slower than 16, 4 about 50% slower than -** 16, 2 about 66% slower than 16. -** -** At query time, high MERGE_COUNT increases the number of segments -** which need to be scanned and merged. For instance, with 100k docs -** inserted: -** -** MERGE_COUNT segments -** 16 25 -** 8 12 -** 4 10 -** 2 6 -** -** This appears to have only a moderate impact on queries for very -** frequent terms (which are somewhat dominated by segment merge -** costs), and infrequent and non-existent terms still seem to be fast -** even with many segments. -** -** TODO(shess) That said, it would be nice to have a better query-side -** argument for MERGE_COUNT of 16. Also, it is possible/likely that -** optimizations to things like doclist merging will swing the sweet -** spot around. +/* +** This function interprets the string at (*pp) as a non-negative integer +** value. It reads the integer and sets *pnOut to the value read, then +** sets *pp to point to the byte immediately following the last byte of +** the integer value. ** +** Only decimal digits ('0'..'9') may be part of an integer value. ** +** If *pp does not being with a decimal digit SQLITE_ERROR is returned and +** the output value undefined. Otherwise SQLITE_OK is returned. ** -**** Handling of deletions and updates **** -** Since we're using a segmented structure, with no docid-oriented -** index into the term index, we clearly cannot simply update the term -** index when a document is deleted or updated. For deletions, we -** write an empty doclist (varint(docid) varint(POS_END)), for updates -** we simply write the new doclist. Segment merges overwrite older -** data for a particular docid with newer data, so deletes or updates -** will eventually overtake the earlier data and knock it out. The -** query logic likewise merges doclists so that newer data knocks out -** older data. +** This function is used when parsing the "prefix=" FTS4 parameter. */ +static int fts3GobbleInt(const char **pp, int *pnOut){ + const int MAX_NPREFIX = 10000000; + int nInt = 0; /* Output value */ + int nByte; + nByte = sqlite3Fts3ReadInt(*pp, &nInt); + if( nInt>MAX_NPREFIX ){ + nInt = 0; + } + if( nByte==0 ){ + return SQLITE_ERROR; + } + *pnOut = nInt; + *pp += nByte; + return SQLITE_OK; +} -/************** Include fts3Int.h in the middle of fts3.c ********************/ -/************** Begin file fts3Int.h *****************************************/ /* -** 2009 Nov 12 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: +** This function is called to allocate an array of Fts3Index structures +** representing the indexes maintained by the current FTS table. FTS tables +** always maintain the main "terms" index, but may also maintain one or +** more "prefix" indexes, depending on the value of the "prefix=" parameter +** (if any) specified as part of the CREATE VIRTUAL TABLE statement. ** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. +** Argument zParam is passed the value of the "prefix=" option if one was +** specified, or NULL otherwise. ** -****************************************************************************** +** If no error occurs, SQLITE_OK is returned and *apIndex set to point to +** the allocated array. *pnIndex is set to the number of elements in the +** array. If an error does occur, an SQLite error code is returned. ** +** Regardless of whether or not an error is returned, it is the responsibility +** of the caller to call sqlite3_free() on the output array to free it. */ -#ifndef _FTSINT_H -#define _FTSINT_H - -#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) -# define NDEBUG 1 -#endif - -/* FTS3/FTS4 require virtual tables */ -#ifdef SQLITE_OMIT_VIRTUALTABLE -# undef SQLITE_ENABLE_FTS3 -# undef SQLITE_ENABLE_FTS4 -#endif +static int fts3PrefixParameter( + const char *zParam, /* ABC in prefix=ABC parameter to parse */ + int *pnIndex, /* OUT: size of *apIndex[] array */ + struct Fts3Index **apIndex /* OUT: Array of indexes for this table */ +){ + struct Fts3Index *aIndex; /* Allocated array */ + int nIndex = 1; /* Number of entries in array */ -/* -** FTS4 is really an extension for FTS3. It is enabled using the -** SQLITE_ENABLE_FTS3 macro. But to avoid confusion we also all -** the SQLITE_ENABLE_FTS4 macro to serve as an alisse for SQLITE_ENABLE_FTS3. -*/ -#if defined(SQLITE_ENABLE_FTS4) && !defined(SQLITE_ENABLE_FTS3) -# define SQLITE_ENABLE_FTS3 -#endif + if( zParam && zParam[0] ){ + const char *p; + nIndex++; + for(p=zParam; *p; p++){ + if( *p==',' ) nIndex++; + } + } -#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + aIndex = sqlite3_malloc64(sizeof(struct Fts3Index) * nIndex); + *apIndex = aIndex; + if( !aIndex ){ + return SQLITE_NOMEM; + } -/* If not building as part of the core, include sqlite3ext.h. */ -#ifndef SQLITE_CORE -/* # include "sqlite3ext.h" */ -SQLITE_EXTENSION_INIT3 -#endif + memset(aIndex, 0, sizeof(struct Fts3Index) * nIndex); + if( zParam ){ + const char *p = zParam; + int i; + for(i=1; i=0 ); + if( nPrefix==0 ){ + nIndex--; + i--; + }else{ + aIndex[i].nPrefix = nPrefix; + } + p++; + } + } + + *pnIndex = nIndex; + return SQLITE_OK; +} -/* #include "sqlite3.h" */ -/************** Include fts3_tokenizer.h in the middle of fts3Int.h **********/ -/************** Begin file fts3_tokenizer.h **********************************/ /* -** 2006 July 10 +** This function is called when initializing an FTS4 table that uses the +** content=xxx option. It determines the number of and names of the columns +** of the new FTS4 table. ** -** The author disclaims copyright to this source code. +** The third argument passed to this function is the value passed to the +** config=xxx option (i.e. "xxx"). This function queries the database for +** a table of that name. If found, the output variables are populated +** as follows: ** -************************************************************************* -** Defines the interface to tokenizers used by fulltext-search. There -** are three basic components: +** *pnCol: Set to the number of columns table xxx has, ** -** sqlite3_tokenizer_module is a singleton defining the tokenizer -** interface functions. This is essentially the class structure for -** tokenizers. +** *pnStr: Set to the total amount of space required to store a copy +** of each columns name, including the nul-terminator. ** -** sqlite3_tokenizer is used to define a particular tokenizer, perhaps -** including customization information defined at creation time. +** *pazCol: Set to point to an array of *pnCol strings. Each string is +** the name of the corresponding column in table xxx. The array +** and its contents are allocated using a single allocation. It +** is the responsibility of the caller to free this allocation +** by eventually passing the *pazCol value to sqlite3_free(). ** -** sqlite3_tokenizer_cursor is generated by a tokenizer to generate -** tokens from a particular input. +** If the table cannot be found, an error code is returned and the output +** variables are undefined. Or, if an OOM is encountered, SQLITE_NOMEM is +** returned (and the output variables are undefined). */ -#ifndef _FTS3_TOKENIZER_H_ -#define _FTS3_TOKENIZER_H_ +static int fts3ContentColumns( + sqlite3 *db, /* Database handle */ + const char *zDb, /* Name of db (i.e. "main", "temp" etc.) */ + const char *zTbl, /* Name of content table */ + const char ***pazCol, /* OUT: Malloc'd array of column names */ + int *pnCol, /* OUT: Size of array *pazCol */ + int *pnStr, /* OUT: Bytes of string content */ + char **pzErr /* OUT: error message */ +){ + int rc = SQLITE_OK; /* Return code */ + char *zSql; /* "SELECT *" statement on zTbl */ + sqlite3_stmt *pStmt = 0; /* Compiled version of zSql */ -/* TODO(shess) Only used for SQLITE_OK and SQLITE_DONE at this time. -** If tokenizers are to be allowed to call sqlite3_*() functions, then -** we will need a way to register the API consistently. -*/ -/* #include "sqlite3.h" */ + zSql = sqlite3_mprintf("SELECT * FROM %Q.%Q", zDb, zTbl); + if( !zSql ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare(db, zSql, -1, &pStmt, 0); + if( rc!=SQLITE_OK ){ + sqlite3Fts3ErrMsg(pzErr, "%s", sqlite3_errmsg(db)); + } + } + sqlite3_free(zSql); + + if( rc==SQLITE_OK ){ + const char **azCol; /* Output array */ + sqlite3_int64 nStr = 0; /* Size of all column names (incl. 0x00) */ + int nCol; /* Number of table columns */ + int i; /* Used to iterate through columns */ + + /* Loop through the returned columns. Set nStr to the number of bytes of + ** space required to store a copy of each column name, including the + ** nul-terminator byte. */ + nCol = sqlite3_column_count(pStmt); + for(i=0; i module name ("fts3" or "fts4") +** argv[1] -> database name +** argv[2] -> table name +** argv[...] -> "column name" and other module argument fields. */ -typedef struct sqlite3_tokenizer_module sqlite3_tokenizer_module; -typedef struct sqlite3_tokenizer sqlite3_tokenizer; -typedef struct sqlite3_tokenizer_cursor sqlite3_tokenizer_cursor; +static int fts3InitVtab( + int isCreate, /* True for xCreate, false for xConnect */ + sqlite3 *db, /* The SQLite database connection */ + void *pAux, /* Hash table containing tokenizers */ + int argc, /* Number of elements in argv array */ + const char * const *argv, /* xCreate/xConnect argument array */ + sqlite3_vtab **ppVTab, /* Write the resulting vtab structure here */ + char **pzErr /* Write any error message here */ +){ + Fts3Hash *pHash = (Fts3Hash *)pAux; + Fts3Table *p = 0; /* Pointer to allocated vtab */ + int rc = SQLITE_OK; /* Return code */ + int i; /* Iterator variable */ + sqlite3_int64 nByte; /* Size of allocation used for *p */ + int iCol; /* Column index */ + int nString = 0; /* Bytes required to hold all column names */ + int nCol = 0; /* Number of columns in the FTS table */ + char *zCsr; /* Space for holding column names */ + int nDb; /* Bytes required to hold database name */ + int nName; /* Bytes required to hold table name */ + int isFts4 = (argv[0][3]=='4'); /* True for FTS4, false for FTS3 */ + const char **aCol; /* Array of column names */ + sqlite3_tokenizer *pTokenizer = 0; /* Tokenizer for this table */ -struct sqlite3_tokenizer_module { + int nIndex = 0; /* Size of aIndex[] array */ + struct Fts3Index *aIndex = 0; /* Array of indexes for this table */ - /* - ** Structure version. Should always be set to 0 or 1. - */ - int iVersion; + /* The results of parsing supported FTS4 key=value options: */ + int bNoDocsize = 0; /* True to omit %_docsize table */ + int bDescIdx = 0; /* True to store descending indexes */ + char *zPrefix = 0; /* Prefix parameter value (or NULL) */ + char *zCompress = 0; /* compress=? parameter (or NULL) */ + char *zUncompress = 0; /* uncompress=? parameter (or NULL) */ + char *zContent = 0; /* content=? parameter (or NULL) */ + char *zLanguageid = 0; /* languageid=? parameter (or NULL) */ + char **azNotindexed = 0; /* The set of notindexed= columns */ + int nNotindexed = 0; /* Size of azNotindexed[] array */ - /* - ** Create a new tokenizer. The values in the argv[] array are the - ** arguments passed to the "tokenizer" clause of the CREATE VIRTUAL - ** TABLE statement that created the fts3 table. For example, if - ** the following SQL is executed: - ** - ** CREATE .. USING fts3( ... , tokenizer arg1 arg2) + assert( strlen(argv[0])==4 ); + assert( (sqlite3_strnicmp(argv[0], "fts4", 4)==0 && isFts4) + || (sqlite3_strnicmp(argv[0], "fts3", 4)==0 && !isFts4) + ); + + nDb = (int)strlen(argv[1]) + 1; + nName = (int)strlen(argv[2]) + 1; + + nByte = sizeof(const char *) * (argc-2); + aCol = (const char **)sqlite3_malloc64(nByte); + if( aCol ){ + memset((void*)aCol, 0, nByte); + azNotindexed = (char **)sqlite3_malloc64(nByte); + } + if( azNotindexed ){ + memset(azNotindexed, 0, nByte); + } + if( !aCol || !azNotindexed ){ + rc = SQLITE_NOMEM; + goto fts3_init_out; + } + + /* Loop through all of the arguments passed by the user to the FTS3/4 + ** module (i.e. all the column names and special arguments). This loop + ** does the following: ** - ** then argc is set to 2, and the argv[] array contains pointers - ** to the strings "arg1" and "arg2". + ** + Figures out the number of columns the FTSX table will have, and + ** the number of bytes of space that must be allocated to store copies + ** of the column names. ** - ** This method should return either SQLITE_OK (0), or an SQLite error - ** code. If SQLITE_OK is returned, then *ppTokenizer should be set - ** to point at the newly created tokenizer structure. The generic - ** sqlite3_tokenizer.pModule variable should not be initialized by - ** this callback. The caller will do so. + ** + If there is a tokenizer specification included in the arguments, + ** initializes the tokenizer pTokenizer. */ - int (*xCreate)( - int argc, /* Size of argv array */ - const char *const*argv, /* Tokenizer argument strings */ - sqlite3_tokenizer **ppTokenizer /* OUT: Created tokenizer */ - ); + for(i=3; rc==SQLITE_OK && i8 + && 0==sqlite3_strnicmp(z, "tokenize", 8) + && 0==sqlite3Fts3IsIdChar(z[8]) + ){ + rc = sqlite3Fts3InitTokenizer(pHash, &z[9], &pTokenizer, pzErr); + } - /* - ** Create a tokenizer cursor to tokenize an input buffer. The caller - ** is responsible for ensuring that the input buffer remains valid - ** until the cursor is closed (using the xClose() method). - */ - int (*xOpen)( - sqlite3_tokenizer *pTokenizer, /* Tokenizer object */ - const char *pInput, int nBytes, /* Input buffer */ - sqlite3_tokenizer_cursor **ppCursor /* OUT: Created tokenizer cursor */ - ); + /* Check if it is an FTS4 special argument. */ + else if( isFts4 && fts3IsSpecialColumn(z, &nKey, &zVal) ){ + struct Fts4Option { + const char *zOpt; + int nOpt; + } aFts4Opt[] = { + { "matchinfo", 9 }, /* 0 -> MATCHINFO */ + { "prefix", 6 }, /* 1 -> PREFIX */ + { "compress", 8 }, /* 2 -> COMPRESS */ + { "uncompress", 10 }, /* 3 -> UNCOMPRESS */ + { "order", 5 }, /* 4 -> ORDER */ + { "content", 7 }, /* 5 -> CONTENT */ + { "languageid", 10 }, /* 6 -> LANGUAGEID */ + { "notindexed", 10 } /* 7 -> NOTINDEXED */ + }; - /* - ** Destroy an existing tokenizer cursor. The fts3 module calls this - ** method exactly once for each successful call to xOpen(). - */ - int (*xClose)(sqlite3_tokenizer_cursor *pCursor); + int iOpt; + if( !zVal ){ + rc = SQLITE_NOMEM; + }else{ + for(iOpt=0; iOptnOpt && !sqlite3_strnicmp(z, pOp->zOpt, pOp->nOpt) ){ + break; + } + } + switch( iOpt ){ + case 0: /* MATCHINFO */ + if( strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "fts3", 4) ){ + sqlite3Fts3ErrMsg(pzErr, "unrecognized matchinfo: %s", zVal); + rc = SQLITE_ERROR; + } + bNoDocsize = 1; + break; - /* - ** Retrieve the next token from the tokenizer cursor pCursor. This - ** method should either return SQLITE_OK and set the values of the - ** "OUT" variables identified below, or SQLITE_DONE to indicate that - ** the end of the buffer has been reached, or an SQLite error code. + case 1: /* PREFIX */ + sqlite3_free(zPrefix); + zPrefix = zVal; + zVal = 0; + break; + + case 2: /* COMPRESS */ + sqlite3_free(zCompress); + zCompress = zVal; + zVal = 0; + break; + + case 3: /* UNCOMPRESS */ + sqlite3_free(zUncompress); + zUncompress = zVal; + zVal = 0; + break; + + case 4: /* ORDER */ + if( (strlen(zVal)!=3 || sqlite3_strnicmp(zVal, "asc", 3)) + && (strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "desc", 4)) + ){ + sqlite3Fts3ErrMsg(pzErr, "unrecognized order: %s", zVal); + rc = SQLITE_ERROR; + } + bDescIdx = (zVal[0]=='d' || zVal[0]=='D'); + break; + + case 5: /* CONTENT */ + sqlite3_free(zContent); + zContent = zVal; + zVal = 0; + break; + + case 6: /* LANGUAGEID */ + assert( iOpt==6 ); + sqlite3_free(zLanguageid); + zLanguageid = zVal; + zVal = 0; + break; + + case 7: /* NOTINDEXED */ + azNotindexed[nNotindexed++] = zVal; + zVal = 0; + break; + + default: + assert( iOpt==SizeofArray(aFts4Opt) ); + sqlite3Fts3ErrMsg(pzErr, "unrecognized parameter: %s", z); + rc = SQLITE_ERROR; + break; + } + sqlite3_free(zVal); + } + } + + /* Otherwise, the argument is a column name. */ + else { + nString += (int)(strlen(z) + 1); + aCol[nCol++] = z; + } + } + + /* If a content=xxx option was specified, the following: ** - ** *ppToken should be set to point at a buffer containing the - ** normalized version of the token (i.e. after any case-folding and/or - ** stemming has been performed). *pnBytes should be set to the length - ** of this buffer in bytes. The input text that generated the token is - ** identified by the byte offsets returned in *piStartOffset and - ** *piEndOffset. *piStartOffset should be set to the index of the first - ** byte of the token in the input buffer. *piEndOffset should be set - ** to the index of the first byte just past the end of the token in - ** the input buffer. + ** 1. Ignore any compress= and uncompress= options. ** - ** The buffer *ppToken is set to point at is managed by the tokenizer - ** implementation. It is only required to be valid until the next call - ** to xNext() or xClose(). - */ - /* TODO(shess) current implementation requires pInput to be - ** nul-terminated. This should either be fixed, or pInput/nBytes - ** should be converted to zInput. + ** 2. If no column names were specified as part of the CREATE VIRTUAL + ** TABLE statement, use all columns from the content table. */ - int (*xNext)( - sqlite3_tokenizer_cursor *pCursor, /* Tokenizer cursor */ - const char **ppToken, int *pnBytes, /* OUT: Normalized text for token */ - int *piStartOffset, /* OUT: Byte offset of token in input buffer */ - int *piEndOffset, /* OUT: Byte offset of end of token in input buffer */ - int *piPosition /* OUT: Number of tokens returned before this one */ - ); + if( rc==SQLITE_OK && zContent ){ + sqlite3_free(zCompress); + sqlite3_free(zUncompress); + zCompress = 0; + zUncompress = 0; + if( nCol==0 ){ + sqlite3_free((void*)aCol); + aCol = 0; + rc = fts3ContentColumns(db, argv[1], zContent,&aCol,&nCol,&nString,pzErr); - /*********************************************************************** - ** Methods below this point are only available if iVersion>=1. - */ + /* If a languageid= option was specified, remove the language id + ** column from the aCol[] array. */ + if( rc==SQLITE_OK && zLanguageid ){ + int j; + for(j=0; jdb = db; + p->nColumn = nCol; + p->nPendingData = 0; + p->azColumn = (char **)&p[1]; + p->pTokenizer = pTokenizer; + p->nMaxPendingData = FTS3_MAX_PENDING_DATA; + p->bHasDocsize = (isFts4 && bNoDocsize==0); + p->bHasStat = (u8)isFts4; + p->bFts4 = (u8)isFts4; + p->bDescIdx = (u8)bDescIdx; + p->nAutoincrmerge = 0xff; /* 0xff means setting unknown */ + p->zContentTbl = zContent; + p->zLanguageid = zLanguageid; + zContent = 0; + zLanguageid = 0; + TESTONLY( p->inTransaction = -1 ); + TESTONLY( p->mxSavepoint = -1 ); + + p->aIndex = (struct Fts3Index *)&p->azColumn[nCol]; + memcpy(p->aIndex, aIndex, sizeof(struct Fts3Index) * nIndex); + p->nIndex = nIndex; + for(i=0; iaIndex[i].hPending, FTS3_HASH_STRING, 1); + } + p->abNotindexed = (u8 *)&p->aIndex[nIndex]; + + /* Fill in the zName and zDb fields of the vtab structure. */ + zCsr = (char *)&p->abNotindexed[nCol]; + p->zName = zCsr; + memcpy(zCsr, argv[2], nName); + zCsr += nName; + p->zDb = zCsr; + memcpy(zCsr, argv[1], nDb); + zCsr += nDb; + + /* Fill in the azColumn array */ + for(iCol=0; iCol0 ){ + memcpy(zCsr, z, n); + } + zCsr[n] = '\0'; + sqlite3Fts3Dequote(zCsr); + p->azColumn[iCol] = zCsr; + zCsr += n+1; + assert( zCsr <= &((char *)p)[nByte] ); + } + + /* Fill in the abNotindexed array */ + for(iCol=0; iColazColumn[iCol]); + for(i=0; iazColumn[iCol], zNot, n) + ){ + p->abNotindexed[iCol] = 1; + sqlite3_free(zNot); + azNotindexed[i] = 0; + } + } + } + for(i=0; izReadExprlist = fts3ReadExprList(p, zUncompress, &rc); + p->zWriteExprlist = fts3WriteExprList(p, zCompress, &rc); + if( rc!=SQLITE_OK ) goto fts3_init_out; + + /* If this is an xCreate call, create the underlying tables in the + ** database. TODO: For xConnect(), it could verify that said tables exist. */ - int (*xLanguageid)(sqlite3_tokenizer_cursor *pCsr, int iLangid); -}; + if( isCreate ){ + rc = fts3CreateTables(p); + } -struct sqlite3_tokenizer { - const sqlite3_tokenizer_module *pModule; /* The module for this tokenizer */ - /* Tokenizer implementations will typically add additional fields */ -}; + /* Check to see if a legacy fts3 table has been "upgraded" by the + ** addition of a %_stat table so that it can use incremental merge. + */ + if( !isFts4 && !isCreate ){ + p->bHasStat = 2; + } -struct sqlite3_tokenizer_cursor { - sqlite3_tokenizer *pTokenizer; /* Tokenizer for this cursor. */ - /* Tokenizer implementations will typically add additional fields */ -}; + /* Figure out the page-size for the database. This is required in order to + ** estimate the cost of loading large doclists from the database. */ + fts3DatabasePageSize(&rc, p); + p->nNodeSize = p->nPgsz-35; -int fts3_global_term_cnt(int iTerm, int iCol); -int fts3_term_cnt(int iTerm, int iCol); +#if defined(SQLITE_DEBUG)||defined(SQLITE_TEST) + p->nMergeCount = FTS3_MERGE_COUNT; +#endif + /* Declare the table schema to SQLite. */ + fts3DeclareVtab(&rc, p); -#endif /* _FTS3_TOKENIZER_H_ */ +fts3_init_out: + sqlite3_free(zPrefix); + sqlite3_free(aIndex); + sqlite3_free(zCompress); + sqlite3_free(zUncompress); + sqlite3_free(zContent); + sqlite3_free(zLanguageid); + for(i=0; ipModule->xDestroy(pTokenizer); + } + }else{ + assert( p->pSegments==0 ); + *ppVTab = &p->base; + } + return rc; +} -/************** End of fts3_tokenizer.h **************************************/ -/************** Continuing where we left off in fts3Int.h ********************/ -/************** Include fts3_hash.h in the middle of fts3Int.h ***************/ -/************** Begin file fts3_hash.h ***************************************/ /* -** 2001 September 22 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** This is the header file for the generic hash-table implementation -** used in SQLite. We've modified it slightly to serve as a standalone -** hash table implementation for the full-text indexing module. -** +** The xConnect() and xCreate() methods for the virtual table. All the +** work is done in function fts3InitVtab(). */ -#ifndef _FTS3_HASH_H_ -#define _FTS3_HASH_H_ +static int fts3ConnectMethod( + sqlite3 *db, /* Database connection */ + void *pAux, /* Pointer to tokenizer hash table */ + int argc, /* Number of elements in argv array */ + const char * const *argv, /* xCreate/xConnect argument array */ + sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */ + char **pzErr /* OUT: sqlite3_malloc'd error message */ +){ + return fts3InitVtab(0, db, pAux, argc, argv, ppVtab, pzErr); +} +static int fts3CreateMethod( + sqlite3 *db, /* Database connection */ + void *pAux, /* Pointer to tokenizer hash table */ + int argc, /* Number of elements in argv array */ + const char * const *argv, /* xCreate/xConnect argument array */ + sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */ + char **pzErr /* OUT: sqlite3_malloc'd error message */ +){ + return fts3InitVtab(1, db, pAux, argc, argv, ppVtab, pzErr); +} -/* Forward declarations of structures. */ -typedef struct Fts3Hash Fts3Hash; -typedef struct Fts3HashElem Fts3HashElem; +/* +** Set the pIdxInfo->estimatedRows variable to nRow. Unless this +** extension is currently being used by a version of SQLite too old to +** support estimatedRows. In that case this function is a no-op. +*/ +static void fts3SetEstimatedRows(sqlite3_index_info *pIdxInfo, i64 nRow){ +#if SQLITE_VERSION_NUMBER>=3008002 + if( sqlite3_libversion_number()>=3008002 ){ + pIdxInfo->estimatedRows = nRow; + } +#endif +} -/* A complete hash table is an instance of the following structure. -** The internals of this structure are intended to be opaque -- client -** code should not attempt to access or modify the fields of this structure -** directly. Change this structure only by using the routines below. -** However, many of the "procedures" and "functions" for modifying and -** accessing this structure are really macros, so we can't really make -** this structure opaque. +/* +** Set the SQLITE_INDEX_SCAN_UNIQUE flag in pIdxInfo->flags. Unless this +** extension is currently being used by a version of SQLite too old to +** support index-info flags. In that case this function is a no-op. */ -struct Fts3Hash { - char keyClass; /* HASH_INT, _POINTER, _STRING, _BINARY */ - char copyKey; /* True if copy of key made on insert */ - int count; /* Number of entries in this table */ - Fts3HashElem *first; /* The first element of the array */ - int htsize; /* Number of buckets in the hash table */ - struct _fts3ht { /* the hash table */ - int count; /* Number of entries with this hash */ - Fts3HashElem *chain; /* Pointer to first entry with this hash */ - } *ht; -}; +static void fts3SetUniqueFlag(sqlite3_index_info *pIdxInfo){ +#if SQLITE_VERSION_NUMBER>=3008012 + if( sqlite3_libversion_number()>=3008012 ){ + pIdxInfo->idxFlags |= SQLITE_INDEX_SCAN_UNIQUE; + } +#endif +} -/* Each element in the hash table is an instance of the following -** structure. All elements are stored on a single doubly-linked list. +/* +** Implementation of the xBestIndex method for FTS3 tables. There +** are three possible strategies, in order of preference: ** -** Again, this structure is intended to be opaque, but it can't really -** be opaque because it is used by macros. +** 1. Direct lookup by rowid or docid. +** 2. Full-text search using a MATCH operator on a non-docid column. +** 3. Linear scan of %_content table. */ -struct Fts3HashElem { - Fts3HashElem *next, *prev; /* Next and previous elements in the table */ - void *data; /* Data associated with this element */ - void *pKey; int nKey; /* Key associated with this element */ -}; +static int fts3BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ + Fts3Table *p = (Fts3Table *)pVTab; + int i; /* Iterator variable */ + int iCons = -1; /* Index of constraint to use */ + + int iLangidCons = -1; /* Index of langid=x constraint, if present */ + int iDocidGe = -1; /* Index of docid>=x constraint, if present */ + int iDocidLe = -1; /* Index of docid<=x constraint, if present */ + int iIdx; + + if( p->bLock ){ + return SQLITE_ERROR; + } + + /* By default use a full table scan. This is an expensive option, + ** so search through the constraints to see if a more efficient + ** strategy is possible. + */ + pInfo->idxNum = FTS3_FULLSCAN_SEARCH; + pInfo->estimatedCost = 5000000; + for(i=0; inConstraint; i++){ + int bDocid; /* True if this constraint is on docid */ + struct sqlite3_index_constraint *pCons = &pInfo->aConstraint[i]; + if( pCons->usable==0 ){ + if( pCons->op==SQLITE_INDEX_CONSTRAINT_MATCH ){ + /* There exists an unusable MATCH constraint. This means that if + ** the planner does elect to use the results of this call as part + ** of the overall query plan the user will see an "unable to use + ** function MATCH in the requested context" error. To discourage + ** this, return a very high cost here. */ + pInfo->idxNum = FTS3_FULLSCAN_SEARCH; + pInfo->estimatedCost = 1e50; + fts3SetEstimatedRows(pInfo, ((sqlite3_int64)1) << 50); + return SQLITE_OK; + } + continue; + } + + bDocid = (pCons->iColumn<0 || pCons->iColumn==p->nColumn+1); + + /* A direct lookup on the rowid or docid column. Assign a cost of 1.0. */ + if( iCons<0 && pCons->op==SQLITE_INDEX_CONSTRAINT_EQ && bDocid ){ + pInfo->idxNum = FTS3_DOCID_SEARCH; + pInfo->estimatedCost = 1.0; + iCons = i; + } + + /* A MATCH constraint. Use a full-text search. + ** + ** If there is more than one MATCH constraint available, use the first + ** one encountered. If there is both a MATCH constraint and a direct + ** rowid/docid lookup, prefer the MATCH strategy. This is done even + ** though the rowid/docid lookup is faster than a MATCH query, selecting + ** it would lead to an "unable to use function MATCH in the requested + ** context" error. + */ + if( pCons->op==SQLITE_INDEX_CONSTRAINT_MATCH + && pCons->iColumn>=0 && pCons->iColumn<=p->nColumn + ){ + pInfo->idxNum = FTS3_FULLTEXT_SEARCH + pCons->iColumn; + pInfo->estimatedCost = 2.0; + iCons = i; + } + + /* Equality constraint on the langid column */ + if( pCons->op==SQLITE_INDEX_CONSTRAINT_EQ + && pCons->iColumn==p->nColumn + 2 + ){ + iLangidCons = i; + } + + if( bDocid ){ + switch( pCons->op ){ + case SQLITE_INDEX_CONSTRAINT_GE: + case SQLITE_INDEX_CONSTRAINT_GT: + iDocidGe = i; + break; + + case SQLITE_INDEX_CONSTRAINT_LE: + case SQLITE_INDEX_CONSTRAINT_LT: + iDocidLe = i; + break; + } + } + } + + /* If using a docid=? or rowid=? strategy, set the UNIQUE flag. */ + if( pInfo->idxNum==FTS3_DOCID_SEARCH ) fts3SetUniqueFlag(pInfo); + + iIdx = 1; + if( iCons>=0 ){ + pInfo->aConstraintUsage[iCons].argvIndex = iIdx++; + pInfo->aConstraintUsage[iCons].omit = 1; + } + if( iLangidCons>=0 ){ + pInfo->idxNum |= FTS3_HAVE_LANGID; + pInfo->aConstraintUsage[iLangidCons].argvIndex = iIdx++; + } + if( iDocidGe>=0 ){ + pInfo->idxNum |= FTS3_HAVE_DOCID_GE; + pInfo->aConstraintUsage[iDocidGe].argvIndex = iIdx++; + } + if( iDocidLe>=0 ){ + pInfo->idxNum |= FTS3_HAVE_DOCID_LE; + pInfo->aConstraintUsage[iDocidLe].argvIndex = iIdx++; + } + + /* Regardless of the strategy selected, FTS can deliver rows in rowid (or + ** docid) order. Both ascending and descending are possible. + */ + if( pInfo->nOrderBy==1 ){ + struct sqlite3_index_orderby *pOrder = &pInfo->aOrderBy[0]; + if( pOrder->iColumn<0 || pOrder->iColumn==p->nColumn+1 ){ + if( pOrder->desc ){ + pInfo->idxStr = "DESC"; + }else{ + pInfo->idxStr = "ASC"; + } + pInfo->orderByConsumed = 1; + } + } + + assert( p->pSegments==0 ); + return SQLITE_OK; +} /* -** There are 2 different modes of operation for a hash table: -** -** FTS3_HASH_STRING pKey points to a string that is nKey bytes long -** (including the null-terminator, if any). Case -** is respected in comparisons. -** -** FTS3_HASH_BINARY pKey points to binary data nKey bytes long. -** memcmp() is used to compare keys. +** Implementation of xOpen method. +*/ +static int fts3OpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){ + sqlite3_vtab_cursor *pCsr; /* Allocated cursor */ + + UNUSED_PARAMETER(pVTab); + + /* Allocate a buffer large enough for an Fts3Cursor structure. If the + ** allocation succeeds, zero it and return SQLITE_OK. Otherwise, + ** if the allocation fails, return SQLITE_NOMEM. + */ + *ppCsr = pCsr = (sqlite3_vtab_cursor *)sqlite3_malloc(sizeof(Fts3Cursor)); + if( !pCsr ){ + return SQLITE_NOMEM; + } + memset(pCsr, 0, sizeof(Fts3Cursor)); + return SQLITE_OK; +} + +/* +** Finalize the statement handle at pCsr->pStmt. ** -** A copy of the key is made if the copyKey parameter to fts3HashInit is 1. +** Or, if that statement handle is one created by fts3CursorSeekStmt(), +** and the Fts3Table.pSeekStmt slot is currently NULL, save the statement +** pointer there instead of finalizing it. */ -#define FTS3_HASH_STRING 1 -#define FTS3_HASH_BINARY 2 +static void fts3CursorFinalizeStmt(Fts3Cursor *pCsr){ + if( pCsr->bSeekStmt ){ + Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; + if( p->pSeekStmt==0 ){ + p->pSeekStmt = pCsr->pStmt; + sqlite3_reset(pCsr->pStmt); + pCsr->pStmt = 0; + } + pCsr->bSeekStmt = 0; + } + sqlite3_finalize(pCsr->pStmt); +} /* -** Access routines. To delete, insert a NULL pointer. +** Free all resources currently held by the cursor passed as the only +** argument. */ -SQLITE_PRIVATE void sqlite3Fts3HashInit(Fts3Hash *pNew, char keyClass, char copyKey); -SQLITE_PRIVATE void *sqlite3Fts3HashInsert(Fts3Hash*, const void *pKey, int nKey, void *pData); -SQLITE_PRIVATE void *sqlite3Fts3HashFind(const Fts3Hash*, const void *pKey, int nKey); -SQLITE_PRIVATE void sqlite3Fts3HashClear(Fts3Hash*); -SQLITE_PRIVATE Fts3HashElem *sqlite3Fts3HashFindElem(const Fts3Hash *, const void *, int); +static void fts3ClearCursor(Fts3Cursor *pCsr){ + fts3CursorFinalizeStmt(pCsr); + sqlite3Fts3FreeDeferredTokens(pCsr); + sqlite3_free(pCsr->aDoclist); + sqlite3Fts3MIBufferFree(pCsr->pMIBuffer); + sqlite3Fts3ExprFree(pCsr->pExpr); + memset(&(&pCsr->base)[1], 0, sizeof(Fts3Cursor)-sizeof(sqlite3_vtab_cursor)); +} /* -** Shorthand for the functions above +** Close the cursor. For additional information see the documentation +** on the xClose method of the virtual table interface. */ -#define fts3HashInit sqlite3Fts3HashInit -#define fts3HashInsert sqlite3Fts3HashInsert -#define fts3HashFind sqlite3Fts3HashFind -#define fts3HashClear sqlite3Fts3HashClear -#define fts3HashFindElem sqlite3Fts3HashFindElem +static int fts3CloseMethod(sqlite3_vtab_cursor *pCursor){ + Fts3Cursor *pCsr = (Fts3Cursor *)pCursor; + assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); + fts3ClearCursor(pCsr); + assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); + sqlite3_free(pCsr); + return SQLITE_OK; +} /* -** Macros for looping over all elements of a hash table. The idiom is -** like this: +** If pCsr->pStmt has not been prepared (i.e. if pCsr->pStmt==0), then +** compose and prepare an SQL statement of the form: ** -** Fts3Hash h; -** Fts3HashElem *p; -** ... -** for(p=fts3HashFirst(&h); p; p=fts3HashNext(p)){ -** SomeStructure *pData = fts3HashData(p); -** // do something with pData -** } +** "SELECT FROM %_content WHERE rowid = ?" +** +** (or the equivalent for a content=xxx table) and set pCsr->pStmt to +** it. If an error occurs, return an SQLite error code. */ -#define fts3HashFirst(H) ((H)->first) -#define fts3HashNext(E) ((E)->next) -#define fts3HashData(E) ((E)->data) -#define fts3HashKey(E) ((E)->pKey) -#define fts3HashKeysize(E) ((E)->nKey) +static int fts3CursorSeekStmt(Fts3Cursor *pCsr){ + int rc = SQLITE_OK; + if( pCsr->pStmt==0 ){ + Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; + char *zSql; + if( p->pSeekStmt ){ + pCsr->pStmt = p->pSeekStmt; + p->pSeekStmt = 0; + }else{ + zSql = sqlite3_mprintf("SELECT %s WHERE rowid = ?", p->zReadExprlist); + if( !zSql ) return SQLITE_NOMEM; + p->bLock++; + rc = sqlite3_prepare_v3( + p->db, zSql,-1,SQLITE_PREPARE_PERSISTENT,&pCsr->pStmt,0 + ); + p->bLock--; + sqlite3_free(zSql); + } + if( rc==SQLITE_OK ) pCsr->bSeekStmt = 1; + } + return rc; +} /* -** Number of entries in a hash table +** Position the pCsr->pStmt statement so that it is on the row +** of the %_content table that contains the last match. Return +** SQLITE_OK on success. */ -#define fts3HashCount(H) ((H)->count) - -#endif /* _FTS3_HASH_H_ */ +static int fts3CursorSeek(sqlite3_context *pContext, Fts3Cursor *pCsr){ + int rc = SQLITE_OK; + if( pCsr->isRequireSeek ){ + rc = fts3CursorSeekStmt(pCsr); + if( rc==SQLITE_OK ){ + Fts3Table *pTab = (Fts3Table*)pCsr->base.pVtab; + pTab->bLock++; + sqlite3_bind_int64(pCsr->pStmt, 1, pCsr->iPrevId); + pCsr->isRequireSeek = 0; + if( SQLITE_ROW==sqlite3_step(pCsr->pStmt) ){ + pTab->bLock--; + return SQLITE_OK; + }else{ + pTab->bLock--; + rc = sqlite3_reset(pCsr->pStmt); + if( rc==SQLITE_OK && ((Fts3Table *)pCsr->base.pVtab)->zContentTbl==0 ){ + /* If no row was found and no error has occurred, then the %_content + ** table is missing a row that is present in the full-text index. + ** The data structures are corrupt. */ + rc = FTS_CORRUPT_VTAB; + pCsr->isEof = 1; + } + } + } + } -/************** End of fts3_hash.h *******************************************/ -/************** Continuing where we left off in fts3Int.h ********************/ + if( rc!=SQLITE_OK && pContext ){ + sqlite3_result_error_code(pContext, rc); + } + return rc; +} /* -** This constant determines the maximum depth of an FTS expression tree -** that the library will create and use. FTS uses recursion to perform -** various operations on the query tree, so the disadvantage of a large -** limit is that it may allow very large queries to use large amounts -** of stack space (perhaps causing a stack overflow). +** This function is used to process a single interior node when searching +** a b-tree for a term or term prefix. The node data is passed to this +** function via the zNode/nNode parameters. The term to search for is +** passed in zTerm/nTerm. +** +** If piFirst is not NULL, then this function sets *piFirst to the blockid +** of the child node that heads the sub-tree that may contain the term. +** +** If piLast is not NULL, then *piLast is set to the right-most child node +** that heads a sub-tree that may contain a term for which zTerm/nTerm is +** a prefix. +** +** If an OOM error occurs, SQLITE_NOMEM is returned. Otherwise, SQLITE_OK. */ -#ifndef SQLITE_FTS3_MAX_EXPR_DEPTH -# define SQLITE_FTS3_MAX_EXPR_DEPTH 12 -#endif +static int fts3ScanInteriorNode( + const char *zTerm, /* Term to select leaves for */ + int nTerm, /* Size of term zTerm in bytes */ + const char *zNode, /* Buffer containing segment interior node */ + int nNode, /* Size of buffer at zNode */ + sqlite3_int64 *piFirst, /* OUT: Selected child node */ + sqlite3_int64 *piLast /* OUT: Selected child node */ +){ + int rc = SQLITE_OK; /* Return code */ + const char *zCsr = zNode; /* Cursor to iterate through node */ + const char *zEnd = &zCsr[nNode];/* End of interior node buffer */ + char *zBuffer = 0; /* Buffer to load terms into */ + i64 nAlloc = 0; /* Size of allocated buffer */ + int isFirstTerm = 1; /* True when processing first term on page */ + sqlite3_int64 iChild; /* Block id of child node to descend to */ + int nBuffer = 0; /* Total term size */ + /* Skip over the 'height' varint that occurs at the start of every + ** interior node. Then load the blockid of the left-child of the b-tree + ** node into variable iChild. + ** + ** Even if the data structure on disk is corrupted, this (reading two + ** varints from the buffer) does not risk an overread. If zNode is a + ** root node, then the buffer comes from a SELECT statement. SQLite does + ** not make this guarantee explicitly, but in practice there are always + ** either more than 20 bytes of allocated space following the nNode bytes of + ** contents, or two zero bytes. Or, if the node is read from the %_segments + ** table, then there are always 20 bytes of zeroed padding following the + ** nNode bytes of content (see sqlite3Fts3ReadBlock() for details). + */ + zCsr += sqlite3Fts3GetVarint(zCsr, &iChild); + zCsr += sqlite3Fts3GetVarint(zCsr, &iChild); + if( zCsr>zEnd ){ + return FTS_CORRUPT_VTAB; + } + + while( zCsrnBuffer ){ + rc = FTS_CORRUPT_VTAB; + goto finish_scan; + } + } + isFirstTerm = 0; + zCsr += fts3GetVarint32(zCsr, &nSuffix); + + assert( nPrefix>=0 && nSuffix>=0 ); + if( nPrefix>zCsr-zNode || nSuffix>zEnd-zCsr || nSuffix==0 ){ + rc = FTS_CORRUPT_VTAB; + goto finish_scan; + } + if( (i64)nPrefix+nSuffix>nAlloc ){ + char *zNew; + nAlloc = ((i64)nPrefix+nSuffix) * 2; + zNew = (char *)sqlite3_realloc64(zBuffer, nAlloc); + if( !zNew ){ + rc = SQLITE_NOMEM; + goto finish_scan; + } + zBuffer = zNew; + } + assert( zBuffer ); + memcpy(&zBuffer[nPrefix], zCsr, nSuffix); + nBuffer = nPrefix + nSuffix; + zCsr += nSuffix; -/* -** This constant controls how often segments are merged. Once there are -** FTS3_MERGE_COUNT segments of level N, they are merged into a single -** segment of level N+1. -*/ -#define FTS3_MERGE_COUNT 16 + /* Compare the term we are searching for with the term just loaded from + ** the interior node. If the specified term is greater than or equal + ** to the term from the interior node, then all terms on the sub-tree + ** headed by node iChild are smaller than zTerm. No need to search + ** iChild. + ** + ** If the interior node term is larger than the specified term, then + ** the tree headed by iChild may contain the specified term. + */ + cmp = memcmp(zTerm, zBuffer, (nBuffer>nTerm ? nTerm : nBuffer)); + if( piFirst && (cmp<0 || (cmp==0 && nBuffer>nTerm)) ){ + *piFirst = iChild; + piFirst = 0; + } -/* -** This is the maximum amount of data (in bytes) to store in the -** Fts3Table.pendingTerms hash table. Normally, the hash table is -** populated as documents are inserted/updated/deleted in a transaction -** and used to create a new segment when the transaction is committed. -** However if this limit is reached midway through a transaction, a new -** segment is created and the hash table cleared immediately. -*/ -#define FTS3_MAX_PENDING_DATA (1*1024*1024) + if( piLast && cmp<0 ){ + *piLast = iChild; + piLast = 0; + } -/* -** Macro to return the number of elements in an array. SQLite has a -** similar macro called ArraySize(). Use a different name to avoid -** a collision when building an amalgamation with built-in FTS3. -*/ -#define SizeofArray(X) ((int)(sizeof(X)/sizeof(X[0]))) + iChild++; + }; + if( piFirst ) *piFirst = iChild; + if( piLast ) *piLast = iChild; -#ifndef MIN -# define MIN(x,y) ((x)<(y)?(x):(y)) -#endif -#ifndef MAX -# define MAX(x,y) ((x)>(y)?(x):(y)) -#endif + finish_scan: + sqlite3_free(zBuffer); + return rc; +} -/* -** Maximum length of a varint encoded integer. The varint format is different -** from that used by SQLite, so the maximum length is 10, not 9. -*/ -#define FTS3_VARINT_MAX 10 /* -** FTS4 virtual tables may maintain multiple indexes - one index of all terms -** in the document set and zero or more prefix indexes. All indexes are stored -** as one or more b+-trees in the %_segments and %_segdir tables. +** The buffer pointed to by argument zNode (size nNode bytes) contains an +** interior node of a b-tree segment. The zTerm buffer (size nTerm bytes) +** contains a term. This function searches the sub-tree headed by the zNode +** node for the range of leaf nodes that may contain the specified term +** or terms for which the specified term is a prefix. ** -** It is possible to determine which index a b+-tree belongs to based on the -** value stored in the "%_segdir.level" column. Given this value L, the index -** that the b+-tree belongs to is (L<<10). In other words, all b+-trees with -** level values between 0 and 1023 (inclusive) belong to index 0, all levels -** between 1024 and 2047 to index 1, and so on. +** If piLeaf is not NULL, then *piLeaf is set to the blockid of the +** left-most leaf node in the tree that may contain the specified term. +** If piLeaf2 is not NULL, then *piLeaf2 is set to the blockid of the +** right-most leaf node that may contain a term for which the specified +** term is a prefix. ** -** It is considered impossible for an index to use more than 1024 levels. In -** theory though this may happen, but only after at least -** (FTS3_MERGE_COUNT^1024) separate flushes of the pending-terms tables. -*/ -#define FTS3_SEGDIR_MAXLEVEL 1024 -#define FTS3_SEGDIR_MAXLEVEL_STR "1024" +** It is possible that the range of returned leaf nodes does not contain +** the specified term or any terms for which it is a prefix. However, if the +** segment does contain any such terms, they are stored within the identified +** range. Because this function only inspects interior segment nodes (and +** never loads leaf nodes into memory), it is not possible to be sure. +** +** If an error occurs, an error code other than SQLITE_OK is returned. +*/ +static int fts3SelectLeaf( + Fts3Table *p, /* Virtual table handle */ + const char *zTerm, /* Term to select leaves for */ + int nTerm, /* Size of term zTerm in bytes */ + const char *zNode, /* Buffer containing segment interior node */ + int nNode, /* Size of buffer at zNode */ + sqlite3_int64 *piLeaf, /* Selected leaf node */ + sqlite3_int64 *piLeaf2 /* Selected leaf node */ +){ + int rc = SQLITE_OK; /* Return code */ + int iHeight; /* Height of this node in tree */ -/* -** The testcase() macro is only used by the amalgamation. If undefined, -** make it a no-op. -*/ -#ifndef testcase -# define testcase(X) -#endif + assert( piLeaf || piLeaf2 ); -/* -** Terminator values for position-lists and column-lists. -*/ -#define POS_COLUMN (1) /* Column-list terminator */ -#define POS_END (0) /* Position-list terminator */ + fts3GetVarint32(zNode, &iHeight); + rc = fts3ScanInteriorNode(zTerm, nTerm, zNode, nNode, piLeaf, piLeaf2); + assert_fts3_nc( !piLeaf2 || !piLeaf || rc!=SQLITE_OK || (*piLeaf<=*piLeaf2) ); -/* -** This section provides definitions to allow the -** FTS3 extension to be compiled outside of the -** amalgamation. -*/ -#ifndef SQLITE_AMALGAMATION -/* -** Macros indicating that conditional expressions are always true or -** false. -*/ -#ifdef SQLITE_COVERAGE_TEST -# define ALWAYS(x) (1) -# define NEVER(X) (0) -#elif defined(SQLITE_DEBUG) -# define ALWAYS(x) sqlite3Fts3Always((x)!=0) -# define NEVER(x) sqlite3Fts3Never((x)!=0) -SQLITE_PRIVATE int sqlite3Fts3Always(int b); -SQLITE_PRIVATE int sqlite3Fts3Never(int b); -#else -# define ALWAYS(x) (x) -# define NEVER(x) (x) -#endif + if( rc==SQLITE_OK && iHeight>1 ){ + char *zBlob = 0; /* Blob read from %_segments table */ + int nBlob = 0; /* Size of zBlob in bytes */ -/* -** Internal types used by SQLite. -*/ -typedef unsigned char u8; /* 1-byte (or larger) unsigned integer */ -typedef short int i16; /* 2-byte (or larger) signed integer */ -typedef unsigned int u32; /* 4-byte unsigned integer */ -typedef sqlite3_uint64 u64; /* 8-byte unsigned integer */ -typedef sqlite3_int64 i64; /* 8-byte signed integer */ + if( piLeaf && piLeaf2 && (*piLeaf!=*piLeaf2) ){ + rc = sqlite3Fts3ReadBlock(p, *piLeaf, &zBlob, &nBlob, 0); + if( rc==SQLITE_OK ){ + rc = fts3SelectLeaf(p, zTerm, nTerm, zBlob, nBlob, piLeaf, 0); + } + sqlite3_free(zBlob); + piLeaf = 0; + zBlob = 0; + } -/* -** Macro used to suppress compiler warnings for unused parameters. -*/ -#define UNUSED_PARAMETER(x) (void)(x) + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3ReadBlock(p, piLeaf?*piLeaf:*piLeaf2, &zBlob, &nBlob, 0); + } + if( rc==SQLITE_OK ){ + int iNewHeight = 0; + fts3GetVarint32(zBlob, &iNewHeight); + if( iNewHeight>=iHeight ){ + rc = FTS_CORRUPT_VTAB; + }else{ + rc = fts3SelectLeaf(p, zTerm, nTerm, zBlob, nBlob, piLeaf, piLeaf2); + } + } + sqlite3_free(zBlob); + } -/* -** Activate assert() only if SQLITE_TEST is enabled. -*/ -#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) -# define NDEBUG 1 -#endif + return rc; +} /* -** The TESTONLY macro is used to enclose variable declarations or -** other bits of code that are needed to support the arguments -** within testcase() and assert() macros. +** This function is used to create delta-encoded serialized lists of FTS3 +** varints. Each call to this function appends a single varint to a list. */ -#if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST) -# define TESTONLY(X) X -#else -# define TESTONLY(X) -#endif - -#endif /* SQLITE_AMALGAMATION */ - -#ifdef SQLITE_DEBUG -SQLITE_PRIVATE int sqlite3Fts3Corrupt(void); -# define FTS_CORRUPT_VTAB sqlite3Fts3Corrupt() -#else -# define FTS_CORRUPT_VTAB SQLITE_CORRUPT_VTAB -#endif - -typedef struct Fts3Table Fts3Table; -typedef struct Fts3Cursor Fts3Cursor; -typedef struct Fts3Expr Fts3Expr; -typedef struct Fts3Phrase Fts3Phrase; -typedef struct Fts3PhraseToken Fts3PhraseToken; - -typedef struct Fts3Doclist Fts3Doclist; -typedef struct Fts3SegFilter Fts3SegFilter; -typedef struct Fts3DeferredToken Fts3DeferredToken; -typedef struct Fts3SegReader Fts3SegReader; -typedef struct Fts3MultiSegReader Fts3MultiSegReader; - -typedef struct MatchinfoBuffer MatchinfoBuffer; +static void fts3PutDeltaVarint( + char **pp, /* IN/OUT: Output pointer */ + sqlite3_int64 *piPrev, /* IN/OUT: Previous value written to list */ + sqlite3_int64 iVal /* Write this value to the list */ +){ + assert( iVal-*piPrev > 0 || (*piPrev==0 && iVal==0) ); + *pp += sqlite3Fts3PutVarint(*pp, iVal-*piPrev); + *piPrev = iVal; +} /* -** A connection to a fulltext index is an instance of the following -** structure. The xCreate and xConnect methods create an instance -** of this structure and xDestroy and xDisconnect free that instance. -** All other methods receive a pointer to the structure as one of their -** arguments. +** When this function is called, *ppPoslist is assumed to point to the +** start of a position-list. After it returns, *ppPoslist points to the +** first byte after the position-list. +** +** A position list is list of positions (delta encoded) and columns for +** a single document record of a doclist. So, in other words, this +** routine advances *ppPoslist so that it points to the next docid in +** the doclist, or to the first byte past the end of the doclist. +** +** If pp is not NULL, then the contents of the position list are copied +** to *pp. *pp is set to point to the first byte past the last byte copied +** before this function returns. */ -struct Fts3Table { - sqlite3_vtab base; /* Base class used by SQLite core */ - sqlite3 *db; /* The database connection */ - const char *zDb; /* logical database name */ - const char *zName; /* virtual table name */ - int nColumn; /* number of named columns in virtual table */ - char **azColumn; /* column names. malloced */ - u8 *abNotindexed; /* True for 'notindexed' columns */ - sqlite3_tokenizer *pTokenizer; /* tokenizer for inserts and queries */ - char *zContentTbl; /* content=xxx option, or NULL */ - char *zLanguageid; /* languageid=xxx option, or NULL */ - int nAutoincrmerge; /* Value configured by 'automerge' */ - u32 nLeafAdd; /* Number of leaf blocks added this trans */ +static void fts3PoslistCopy(char **pp, char **ppPoslist){ + char *pEnd = *ppPoslist; + char c = 0; - /* Precompiled statements used by the implementation. Each of these - ** statements is run and reset within a single virtual table API call. + /* The end of a position list is marked by a zero encoded as an FTS3 + ** varint. A single POS_END (0) byte. Except, if the 0 byte is preceded by + ** a byte with the 0x80 bit set, then it is not a varint 0, but the tail + ** of some other, multi-byte, value. + ** + ** The following while-loop moves pEnd to point to the first byte that is not + ** immediately preceded by a byte with the 0x80 bit set. Then increments + ** pEnd once more so that it points to the byte immediately following the + ** last byte in the position-list. */ - sqlite3_stmt *aStmt[40]; - sqlite3_stmt *pSeekStmt; /* Cache for fts3CursorSeekStmt() */ - - char *zReadExprlist; - char *zWriteExprlist; + while( *pEnd | c ){ + c = *pEnd++ & 0x80; + testcase( c!=0 && (*pEnd)==0 ); + } + pEnd++; /* Advance past the POS_END terminator byte */ - int nNodeSize; /* Soft limit for node size */ - u8 bFts4; /* True for FTS4, false for FTS3 */ - u8 bHasStat; /* True if %_stat table exists (2==unknown) */ - u8 bHasDocsize; /* True if %_docsize table exists */ - u8 bDescIdx; /* True if doclists are in reverse order */ - u8 bIgnoreSavepoint; /* True to ignore xSavepoint invocations */ - int nPgsz; /* Page size for host database */ - char *zSegmentsTbl; /* Name of %_segments table */ - sqlite3_blob *pSegments; /* Blob handle open on %_segments table */ + if( pp ){ + int n = (int)(pEnd - *ppPoslist); + char *p = *pp; + memcpy(p, *ppPoslist, n); + p += n; + *pp = p; + } + *ppPoslist = pEnd; +} - /* - ** The following array of hash tables is used to buffer pending index - ** updates during transactions. All pending updates buffered at any one - ** time must share a common language-id (see the FTS4 langid= feature). - ** The current language id is stored in variable iPrevLangid. - ** - ** A single FTS4 table may have multiple full-text indexes. For each index - ** there is an entry in the aIndex[] array. Index 0 is an index of all the - ** terms that appear in the document set. Each subsequent index in aIndex[] - ** is an index of prefixes of a specific length. - ** - ** Variable nPendingData contains an estimate the memory consumed by the - ** pending data structures, including hash table overhead, but not including - ** malloc overhead. When nPendingData exceeds nMaxPendingData, all hash - ** tables are flushed to disk. Variable iPrevDocid is the docid of the most - ** recently inserted record. - */ - int nIndex; /* Size of aIndex[] */ - struct Fts3Index { - int nPrefix; /* Prefix length (0 for main terms index) */ - Fts3Hash hPending; /* Pending terms table for this index */ - } *aIndex; - int nMaxPendingData; /* Max pending data before flush to disk */ - int nPendingData; /* Current bytes of pending data */ - sqlite_int64 iPrevDocid; /* Docid of most recently inserted document */ - int iPrevLangid; /* Langid of recently inserted document */ - int bPrevDelete; /* True if last operation was a delete */ +/* +** When this function is called, *ppPoslist is assumed to point to the +** start of a column-list. After it returns, *ppPoslist points to the +** to the terminator (POS_COLUMN or POS_END) byte of the column-list. +** +** A column-list is list of delta-encoded positions for a single column +** within a single document within a doclist. +** +** The column-list is terminated either by a POS_COLUMN varint (1) or +** a POS_END varint (0). This routine leaves *ppPoslist pointing to +** the POS_COLUMN or POS_END that terminates the column-list. +** +** If pp is not NULL, then the contents of the column-list are copied +** to *pp. *pp is set to point to the first byte past the last byte copied +** before this function returns. The POS_COLUMN or POS_END terminator +** is not copied into *pp. +*/ +static void fts3ColumnlistCopy(char **pp, char **ppPoslist){ + char *pEnd = *ppPoslist; + char c = 0; -#if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST) - /* State variables used for validating that the transaction control - ** methods of the virtual table are called at appropriate times. These - ** values do not contribute to FTS functionality; they are used for - ** verifying the operation of the SQLite core. + /* A column-list is terminated by either a 0x01 or 0x00 byte that is + ** not part of a multi-byte varint. */ - int inTransaction; /* True after xBegin but before xCommit/xRollback */ - int mxSavepoint; /* Largest valid xSavepoint integer */ -#endif - -#ifdef SQLITE_TEST - /* True to disable the incremental doclist optimization. This is controled - ** by special insert command 'test-no-incr-doclist'. */ - int bNoIncrDoclist; -#endif -}; + while( 0xFE & (*pEnd | c) ){ + c = *pEnd++ & 0x80; + testcase( c!=0 && ((*pEnd)&0xfe)==0 ); + } + if( pp ){ + int n = (int)(pEnd - *ppPoslist); + char *p = *pp; + memcpy(p, *ppPoslist, n); + p += n; + *pp = p; + } + *ppPoslist = pEnd; +} /* -** When the core wants to read from the virtual table, it creates a -** virtual table cursor (an instance of the following structure) using -** the xOpen method. Cursors are destroyed using the xClose method. +** Value used to signify the end of an position-list. This must be +** as large or larger than any value that might appear on the +** position-list, even a position list that has been corrupted. */ -struct Fts3Cursor { - sqlite3_vtab_cursor base; /* Base class used by SQLite core */ - i16 eSearch; /* Search strategy (see below) */ - u8 isEof; /* True if at End Of Results */ - u8 isRequireSeek; /* True if must seek pStmt to %_content row */ - u8 bSeekStmt; /* True if pStmt is a seek */ - sqlite3_stmt *pStmt; /* Prepared statement in use by the cursor */ - Fts3Expr *pExpr; /* Parsed MATCH query string */ - int iLangid; /* Language being queried for */ - int nPhrase; /* Number of matchable phrases in query */ - Fts3DeferredToken *pDeferred; /* Deferred search tokens, if any */ - sqlite3_int64 iPrevId; /* Previous id read from aDoclist */ - char *pNextId; /* Pointer into the body of aDoclist */ - char *aDoclist; /* List of docids for full-text queries */ - int nDoclist; /* Size of buffer at aDoclist */ - u8 bDesc; /* True to sort in descending order */ - int eEvalmode; /* An FTS3_EVAL_XX constant */ - int nRowAvg; /* Average size of database rows, in pages */ - sqlite3_int64 nDoc; /* Documents in table */ - i64 iMinDocid; /* Minimum docid to return */ - i64 iMaxDocid; /* Maximum docid to return */ - int isMatchinfoNeeded; /* True when aMatchinfo[] needs filling in */ - MatchinfoBuffer *pMIBuffer; /* Buffer for matchinfo data */ -}; - -#define FTS3_EVAL_FILTER 0 -#define FTS3_EVAL_NEXT 1 -#define FTS3_EVAL_MATCHINFO 2 +#define POSITION_LIST_END LARGEST_INT64 /* -** The Fts3Cursor.eSearch member is always set to one of the following. -** Actualy, Fts3Cursor.eSearch can be greater than or equal to -** FTS3_FULLTEXT_SEARCH. If so, then Fts3Cursor.eSearch - 2 is the index -** of the column to be searched. For example, in +** This function is used to help parse position-lists. When this function is +** called, *pp may point to the start of the next varint in the position-list +** being parsed, or it may point to 1 byte past the end of the position-list +** (in which case **pp will be a terminator bytes POS_END (0) or +** (1)). ** -** CREATE VIRTUAL TABLE ex1 USING fts3(a,b,c,d); -** SELECT docid FROM ex1 WHERE b MATCH 'one two three'; -** -** Because the LHS of the MATCH operator is 2nd column "b", -** Fts3Cursor.eSearch will be set to FTS3_FULLTEXT_SEARCH+1. (+0 for a, -** +1 for b, +2 for c, +3 for d.) If the LHS of MATCH were "ex1" -** indicating that all columns should be searched, -** then eSearch would be set to FTS3_FULLTEXT_SEARCH+4. +** If *pp points past the end of the current position-list, set *pi to +** POSITION_LIST_END and return. Otherwise, read the next varint from *pp, +** increment the current value of *pi by the value read, and set *pp to +** point to the next value before returning. +** +** Before calling this routine *pi must be initialized to the value of +** the previous position, or zero if we are reading the first position +** in the position-list. Because positions are delta-encoded, the value +** of the previous position is needed in order to compute the value of +** the next position. */ -#define FTS3_FULLSCAN_SEARCH 0 /* Linear scan of %_content table */ -#define FTS3_DOCID_SEARCH 1 /* Lookup by rowid on %_content table */ -#define FTS3_FULLTEXT_SEARCH 2 /* Full-text index search */ +static void fts3ReadNextPos( + char **pp, /* IN/OUT: Pointer into position-list buffer */ + sqlite3_int64 *pi /* IN/OUT: Value read from position-list */ +){ + if( (**pp)&0xFE ){ + int iVal; + *pp += fts3GetVarint32((*pp), &iVal); + *pi += iVal; + *pi -= 2; + }else{ + *pi = POSITION_LIST_END; + } +} /* -** The lower 16-bits of the sqlite3_index_info.idxNum value set by -** the xBestIndex() method contains the Fts3Cursor.eSearch value described -** above. The upper 16-bits contain a combination of the following -** bits, used to describe extra constraints on full-text searches. +** If parameter iCol is not 0, write an POS_COLUMN (1) byte followed by +** the value of iCol encoded as a varint to *pp. This will start a new +** column list. +** +** Set *pp to point to the byte just after the last byte written before +** returning (do not modify it if iCol==0). Return the total number of bytes +** written (0 if iCol==0). */ -#define FTS3_HAVE_LANGID 0x00010000 /* languageid=? */ -#define FTS3_HAVE_DOCID_GE 0x00020000 /* docid>=? */ -#define FTS3_HAVE_DOCID_LE 0x00040000 /* docid<=? */ - -struct Fts3Doclist { - char *aAll; /* Array containing doclist (or NULL) */ - int nAll; /* Size of a[] in bytes */ - char *pNextDocid; /* Pointer to next docid */ - - sqlite3_int64 iDocid; /* Current docid (if pList!=0) */ - int bFreeList; /* True if pList should be sqlite3_free()d */ - char *pList; /* Pointer to position list following iDocid */ - int nList; /* Length of position list */ -}; +static int fts3PutColNumber(char **pp, int iCol){ + int n = 0; /* Number of bytes written */ + if( iCol ){ + char *p = *pp; /* Output pointer */ + n = 1 + sqlite3Fts3PutVarint(&p[1], iCol); + *p = 0x01; + *pp = &p[n]; + } + return n; +} /* -** A "phrase" is a sequence of one or more tokens that must match in -** sequence. A single token is the base case and the most common case. -** For a sequence of tokens contained in double-quotes (i.e. "one two three") -** nToken will be the number of tokens in the string. +** Compute the union of two position lists. The output written +** into *pp contains all positions of both *pp1 and *pp2 in sorted +** order and with any duplicates removed. All pointers are +** updated appropriately. The caller is responsible for insuring +** that there is enough space in *pp to hold the complete output. */ -struct Fts3PhraseToken { - char *z; /* Text of the token */ - int n; /* Number of bytes in buffer z */ - int isPrefix; /* True if token ends with a "*" character */ - int bFirst; /* True if token must appear at position 0 */ +static int fts3PoslistMerge( + char **pp, /* Output buffer */ + char **pp1, /* Left input list */ + char **pp2 /* Right input list */ +){ + char *p = *pp; + char *p1 = *pp1; + char *p2 = *pp2; - /* Variables above this point are populated when the expression is - ** parsed (by code in fts3_expr.c). Below this point the variables are - ** used when evaluating the expression. */ - Fts3DeferredToken *pDeferred; /* Deferred token object for this token */ - Fts3MultiSegReader *pSegcsr; /* Segment-reader for this token */ -}; + while( *p1 || *p2 ){ + int iCol1; /* The current column index in pp1 */ + int iCol2; /* The current column index in pp2 */ -struct Fts3Phrase { - /* Cache of doclist for this phrase. */ - Fts3Doclist doclist; - int bIncr; /* True if doclist is loaded incrementally */ - int iDoclistToken; + if( *p1==POS_COLUMN ){ + fts3GetVarint32(&p1[1], &iCol1); + if( iCol1==0 ) return FTS_CORRUPT_VTAB; + } + else if( *p1==POS_END ) iCol1 = 0x7fffffff; + else iCol1 = 0; - /* Used by sqlite3Fts3EvalPhrasePoslist() if this is a descendent of an - ** OR condition. */ - char *pOrPoslist; - i64 iOrDocid; + if( *p2==POS_COLUMN ){ + fts3GetVarint32(&p2[1], &iCol2); + if( iCol2==0 ) return FTS_CORRUPT_VTAB; + } + else if( *p2==POS_END ) iCol2 = 0x7fffffff; + else iCol2 = 0; - /* Variables below this point are populated by fts3_expr.c when parsing - ** a MATCH expression. Everything above is part of the evaluation phase. - */ - int nToken; /* Number of tokens in the phrase */ - int iColumn; /* Index of column this phrase must match */ - Fts3PhraseToken aToken[1]; /* One entry for each token in the phrase */ -}; + if( iCol1==iCol2 ){ + sqlite3_int64 i1 = 0; /* Last position from pp1 */ + sqlite3_int64 i2 = 0; /* Last position from pp2 */ + sqlite3_int64 iPrev = 0; + int n = fts3PutColNumber(&p, iCol1); + p1 += n; + p2 += n; + + /* At this point, both p1 and p2 point to the start of column-lists + ** for the same column (the column with index iCol1 and iCol2). + ** A column-list is a list of non-negative delta-encoded varints, each + ** incremented by 2 before being stored. Each list is terminated by a + ** POS_END (0) or POS_COLUMN (1). The following block merges the two lists + ** and writes the results to buffer p. p is left pointing to the byte + ** after the list written. No terminator (POS_END or POS_COLUMN) is + ** written to the output. + */ + fts3GetDeltaVarint(&p1, &i1); + fts3GetDeltaVarint(&p2, &i2); + do { + fts3PutDeltaVarint(&p, &iPrev, (i1pLeft==this or pParent->pRight==this */ - Fts3Expr *pLeft; /* Left operand */ - Fts3Expr *pRight; /* Right operand */ - Fts3Phrase *pPhrase; /* Valid if eType==FTSQUERY_PHRASE */ - - /* The following are used by the fts3_eval.c module. */ - sqlite3_int64 iDocid; /* Current docid */ - u8 bEof; /* True this expression is at EOF already */ - u8 bStart; /* True if iDocid is valid */ - u8 bDeferred; /* True if this expression is entirely deferred */ - - /* The following are used by the fts3_snippet.c module. */ - int iPhrase; /* Index of this phrase in matchinfo() results */ - u32 *aMI; /* See above */ -}; - -/* -** Candidate values for Fts3Query.eType. Note that the order of the first -** four values is in order of precedence when parsing expressions. For -** example, the following: +** 0x02 0x01 0x02 0x03 0x03 0x00 ** -** "a OR b AND c NOT d NEAR e" +** When this function returns, both *pp1 and *pp2 are left pointing to the +** byte following the 0x00 terminator of their respective position lists. ** -** is equivalent to: +** If isSaveLeft is 0, an entry is added to the output position list for +** each position in *pp2 for which there exists one or more positions in +** *pp1 so that (pos(*pp2)>pos(*pp1) && pos(*pp2)-pos(*pp1)<=nToken). i.e. +** when the *pp1 token appears before the *pp2 token, but not more than nToken +** slots before it. ** -** "a OR (b AND (c NOT (d NEAR e)))" +** e.g. nToken==1 searches for adjacent positions. */ -#define FTSQUERY_NEAR 1 -#define FTSQUERY_NOT 2 -#define FTSQUERY_AND 3 -#define FTSQUERY_OR 4 -#define FTSQUERY_PHRASE 5 - - -/* fts3_write.c */ -SQLITE_PRIVATE int sqlite3Fts3UpdateMethod(sqlite3_vtab*,int,sqlite3_value**,sqlite3_int64*); -SQLITE_PRIVATE int sqlite3Fts3PendingTermsFlush(Fts3Table *); -SQLITE_PRIVATE void sqlite3Fts3PendingTermsClear(Fts3Table *); -SQLITE_PRIVATE int sqlite3Fts3Optimize(Fts3Table *); -SQLITE_PRIVATE int sqlite3Fts3SegReaderNew(int, int, sqlite3_int64, - sqlite3_int64, sqlite3_int64, const char *, int, Fts3SegReader**); -SQLITE_PRIVATE int sqlite3Fts3SegReaderPending( - Fts3Table*,int,const char*,int,int,Fts3SegReader**); -SQLITE_PRIVATE void sqlite3Fts3SegReaderFree(Fts3SegReader *); -SQLITE_PRIVATE int sqlite3Fts3AllSegdirs(Fts3Table*, int, int, int, sqlite3_stmt **); -SQLITE_PRIVATE int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char **, int*, int*); - -SQLITE_PRIVATE int sqlite3Fts3SelectDoctotal(Fts3Table *, sqlite3_stmt **); -SQLITE_PRIVATE int sqlite3Fts3SelectDocsize(Fts3Table *, sqlite3_int64, sqlite3_stmt **); - -#ifndef SQLITE_DISABLE_FTS4_DEFERRED -SQLITE_PRIVATE void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *); -SQLITE_PRIVATE int sqlite3Fts3DeferToken(Fts3Cursor *, Fts3PhraseToken *, int); -SQLITE_PRIVATE int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *); -SQLITE_PRIVATE void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *); -SQLITE_PRIVATE int sqlite3Fts3DeferredTokenList(Fts3DeferredToken *, char **, int *); -#else -# define sqlite3Fts3FreeDeferredTokens(x) -# define sqlite3Fts3DeferToken(x,y,z) SQLITE_OK -# define sqlite3Fts3CacheDeferredDoclists(x) SQLITE_OK -# define sqlite3Fts3FreeDeferredDoclists(x) -# define sqlite3Fts3DeferredTokenList(x,y,z) SQLITE_OK -#endif - -SQLITE_PRIVATE void sqlite3Fts3SegmentsClose(Fts3Table *); -SQLITE_PRIVATE int sqlite3Fts3MaxLevel(Fts3Table *, int *); - -/* Special values interpreted by sqlite3SegReaderCursor() */ -#define FTS3_SEGCURSOR_PENDING -1 -#define FTS3_SEGCURSOR_ALL -2 - -SQLITE_PRIVATE int sqlite3Fts3SegReaderStart(Fts3Table*, Fts3MultiSegReader*, Fts3SegFilter*); -SQLITE_PRIVATE int sqlite3Fts3SegReaderStep(Fts3Table *, Fts3MultiSegReader *); -SQLITE_PRIVATE void sqlite3Fts3SegReaderFinish(Fts3MultiSegReader *); - -SQLITE_PRIVATE int sqlite3Fts3SegReaderCursor(Fts3Table *, - int, int, int, const char *, int, int, int, Fts3MultiSegReader *); - -/* Flags allowed as part of the 4th argument to SegmentReaderIterate() */ -#define FTS3_SEGMENT_REQUIRE_POS 0x00000001 -#define FTS3_SEGMENT_IGNORE_EMPTY 0x00000002 -#define FTS3_SEGMENT_COLUMN_FILTER 0x00000004 -#define FTS3_SEGMENT_PREFIX 0x00000008 -#define FTS3_SEGMENT_SCAN 0x00000010 -#define FTS3_SEGMENT_FIRST 0x00000020 - -/* Type passed as 4th argument to SegmentReaderIterate() */ -struct Fts3SegFilter { - const char *zTerm; - int nTerm; - int iCol; - int flags; -}; - -struct Fts3MultiSegReader { - /* Used internally by sqlite3Fts3SegReaderXXX() calls */ - Fts3SegReader **apSegment; /* Array of Fts3SegReader objects */ - int nSegment; /* Size of apSegment array */ - int nAdvance; /* How many seg-readers to advance */ - Fts3SegFilter *pFilter; /* Pointer to filter object */ - char *aBuffer; /* Buffer to merge doclists in */ - int nBuffer; /* Allocated size of aBuffer[] in bytes */ - - int iColFilter; /* If >=0, filter for this column */ - int bRestart; - - /* Used by fts3.c only. */ - int nCost; /* Cost of running iterator */ - int bLookup; /* True if a lookup of a single entry. */ - - /* Output values. Valid only after Fts3SegReaderStep() returns SQLITE_ROW. */ - char *zTerm; /* Pointer to term buffer */ - int nTerm; /* Size of zTerm in bytes */ - char *aDoclist; /* Pointer to doclist buffer */ - int nDoclist; /* Size of aDoclist[] in bytes */ -}; - -SQLITE_PRIVATE int sqlite3Fts3Incrmerge(Fts3Table*,int,int); - -#define fts3GetVarint32(p, piVal) ( \ - (*(u8*)(p)&0x80) ? sqlite3Fts3GetVarint32(p, piVal) : (*piVal=*(u8*)(p), 1) \ -) - -/* fts3.c */ -SQLITE_PRIVATE void sqlite3Fts3ErrMsg(char**,const char*,...); -SQLITE_PRIVATE int sqlite3Fts3PutVarint(char *, sqlite3_int64); -SQLITE_PRIVATE int sqlite3Fts3GetVarint(const char *, sqlite_int64 *); -SQLITE_PRIVATE int sqlite3Fts3GetVarint32(const char *, int *); -SQLITE_PRIVATE int sqlite3Fts3VarintLen(sqlite3_uint64); -SQLITE_PRIVATE void sqlite3Fts3Dequote(char *); -SQLITE_PRIVATE void sqlite3Fts3DoclistPrev(int,char*,int,char**,sqlite3_int64*,int*,u8*); -SQLITE_PRIVATE int sqlite3Fts3EvalPhraseStats(Fts3Cursor *, Fts3Expr *, u32 *); -SQLITE_PRIVATE int sqlite3Fts3FirstFilter(sqlite3_int64, char *, int, char *); -SQLITE_PRIVATE void sqlite3Fts3CreateStatTable(int*, Fts3Table*); -SQLITE_PRIVATE int sqlite3Fts3EvalTestDeferred(Fts3Cursor *pCsr, int *pRc); - -/* fts3_tokenizer.c */ -SQLITE_PRIVATE const char *sqlite3Fts3NextToken(const char *, int *); -SQLITE_PRIVATE int sqlite3Fts3InitHashTable(sqlite3 *, Fts3Hash *, const char *); -SQLITE_PRIVATE int sqlite3Fts3InitTokenizer(Fts3Hash *pHash, const char *, - sqlite3_tokenizer **, char ** -); -SQLITE_PRIVATE int sqlite3Fts3IsIdChar(char); - -/* fts3_snippet.c */ -SQLITE_PRIVATE void sqlite3Fts3Offsets(sqlite3_context*, Fts3Cursor*); -SQLITE_PRIVATE void sqlite3Fts3Snippet(sqlite3_context *, Fts3Cursor *, const char *, - const char *, const char *, int, int -); -SQLITE_PRIVATE void sqlite3Fts3Matchinfo(sqlite3_context *, Fts3Cursor *, const char *); -SQLITE_PRIVATE void sqlite3Fts3MIBufferFree(MatchinfoBuffer *p); +static int fts3PoslistPhraseMerge( + char **pp, /* IN/OUT: Preallocated output buffer */ + int nToken, /* Maximum difference in token positions */ + int isSaveLeft, /* Save the left position */ + int isExact, /* If *pp1 is exactly nTokens before *pp2 */ + char **pp1, /* IN/OUT: Left input list */ + char **pp2 /* IN/OUT: Right input list */ +){ + char *p = *pp; + char *p1 = *pp1; + char *p2 = *pp2; + int iCol1 = 0; + int iCol2 = 0; -/* fts3_expr.c */ -SQLITE_PRIVATE int sqlite3Fts3ExprParse(sqlite3_tokenizer *, int, - char **, int, int, int, const char *, int, Fts3Expr **, char ** -); -SQLITE_PRIVATE void sqlite3Fts3ExprFree(Fts3Expr *); -#ifdef SQLITE_TEST -SQLITE_PRIVATE int sqlite3Fts3ExprInitTestInterface(sqlite3 *db, Fts3Hash*); -SQLITE_PRIVATE int sqlite3Fts3InitTerm(sqlite3 *db); -#endif + /* Never set both isSaveLeft and isExact for the same invocation. */ + assert( isSaveLeft==0 || isExact==0 ); -SQLITE_PRIVATE int sqlite3Fts3OpenTokenizer(sqlite3_tokenizer *, int, const char *, int, - sqlite3_tokenizer_cursor ** -); + assert( p!=0 && *p1!=0 && *p2!=0 ); + if( *p1==POS_COLUMN ){ + p1++; + p1 += fts3GetVarint32(p1, &iCol1); + } + if( *p2==POS_COLUMN ){ + p2++; + p2 += fts3GetVarint32(p2, &iCol2); + } -/* fts3_aux.c */ -SQLITE_PRIVATE int sqlite3Fts3InitAux(sqlite3 *db); + while( 1 ){ + if( iCol1==iCol2 ){ + char *pSave = p; + sqlite3_int64 iPrev = 0; + sqlite3_int64 iPos1 = 0; + sqlite3_int64 iPos2 = 0; -SQLITE_PRIVATE void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *); + if( iCol1 ){ + *p++ = POS_COLUMN; + p += sqlite3Fts3PutVarint(p, iCol1); + } -SQLITE_PRIVATE int sqlite3Fts3MsrIncrStart( - Fts3Table*, Fts3MultiSegReader*, int, const char*, int); -SQLITE_PRIVATE int sqlite3Fts3MsrIncrNext( - Fts3Table *, Fts3MultiSegReader *, sqlite3_int64 *, char **, int *); -SQLITE_PRIVATE int sqlite3Fts3EvalPhrasePoslist(Fts3Cursor *, Fts3Expr *, int iCol, char **); -SQLITE_PRIVATE int sqlite3Fts3MsrOvfl(Fts3Cursor *, Fts3MultiSegReader *, int *); -SQLITE_PRIVATE int sqlite3Fts3MsrIncrRestart(Fts3MultiSegReader *pCsr); + fts3GetDeltaVarint(&p1, &iPos1); iPos1 -= 2; + fts3GetDeltaVarint(&p2, &iPos2); iPos2 -= 2; + if( iPos1<0 || iPos2<0 ) break; -/* fts3_tokenize_vtab.c */ -SQLITE_PRIVATE int sqlite3Fts3InitTok(sqlite3*, Fts3Hash *); + while( 1 ){ + if( iPos2==iPos1+nToken + || (isExact==0 && iPos2>iPos1 && iPos2<=iPos1+nToken) + ){ + sqlite3_int64 iSave; + iSave = isSaveLeft ? iPos1 : iPos2; + fts3PutDeltaVarint(&p, &iPrev, iSave+2); iPrev -= 2; + pSave = 0; + assert( p ); + } + if( (!isSaveLeft && iPos2<=(iPos1+nToken)) || iPos2<=iPos1 ){ + if( (*p2&0xFE)==0 ) break; + fts3GetDeltaVarint(&p2, &iPos2); iPos2 -= 2; + }else{ + if( (*p1&0xFE)==0 ) break; + fts3GetDeltaVarint(&p1, &iPos1); iPos1 -= 2; + } + } -/* fts3_unicode2.c (functions generated by parsing unicode text files) */ -#ifndef SQLITE_DISABLE_FTS3_UNICODE -SQLITE_PRIVATE int sqlite3FtsUnicodeFold(int, int); -SQLITE_PRIVATE int sqlite3FtsUnicodeIsalnum(int); -SQLITE_PRIVATE int sqlite3FtsUnicodeIsdiacritic(int); -#endif + if( pSave ){ + assert( pp && p ); + p = pSave; + } -#endif /* !SQLITE_CORE || SQLITE_ENABLE_FTS3 */ -#endif /* _FTSINT_H */ + fts3ColumnlistCopy(0, &p1); + fts3ColumnlistCopy(0, &p2); + assert( (*p1&0xFE)==0 && (*p2&0xFE)==0 ); + if( 0==*p1 || 0==*p2 ) break; -/************** End of fts3Int.h *********************************************/ -/************** Continuing where we left off in fts3.c ***********************/ -#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + p1++; + p1 += fts3GetVarint32(p1, &iCol1); + p2++; + p2 += fts3GetVarint32(p2, &iCol2); + } -#if defined(SQLITE_ENABLE_FTS3) && !defined(SQLITE_CORE) -# define SQLITE_CORE 1 -#endif + /* Advance pointer p1 or p2 (whichever corresponds to the smaller of + ** iCol1 and iCol2) so that it points to either the 0x00 that marks the + ** end of the position list, or the 0x01 that precedes the next + ** column-number in the position list. + */ + else if( iCol1 */ -/* #include */ -/* #include */ -/* #include */ -/* #include */ -/* #include */ + fts3PoslistCopy(0, &p2); + fts3PoslistCopy(0, &p1); + *pp1 = p1; + *pp2 = p2; + if( *pp==p ){ + return 0; + } + *p++ = 0x00; + *pp = p; + return 1; +} -/* #include "fts3.h" */ -#ifndef SQLITE_CORE -/* # include "sqlite3ext.h" */ - SQLITE_EXTENSION_INIT1 -#endif +/* +** Merge two position-lists as required by the NEAR operator. The argument +** position lists correspond to the left and right phrases of an expression +** like: +** +** "phrase 1" NEAR "phrase number 2" +** +** Position list *pp1 corresponds to the left-hand side of the NEAR +** expression and *pp2 to the right. As usual, the indexes in the position +** lists are the offsets of the last token in each phrase (tokens "1" and "2" +** in the example above). +** +** The output position list - written to *pp - is a copy of *pp2 with those +** entries that are not sufficiently NEAR entries in *pp1 removed. +*/ +static int fts3PoslistNearMerge( + char **pp, /* Output buffer */ + char *aTmp, /* Temporary buffer space */ + int nRight, /* Maximum difference in token positions */ + int nLeft, /* Maximum difference in token positions */ + char **pp1, /* IN/OUT: Left input list */ + char **pp2 /* IN/OUT: Right input list */ +){ + char *p1 = *pp1; + char *p2 = *pp2; -static int fts3EvalNext(Fts3Cursor *pCsr); -static int fts3EvalStart(Fts3Cursor *pCsr); -static int fts3TermSegReaderCursor( - Fts3Cursor *, const char *, int, int, Fts3MultiSegReader **); + char *pTmp1 = aTmp; + char *pTmp2; + char *aTmp2; + int res = 1; -#ifndef SQLITE_AMALGAMATION -# if defined(SQLITE_DEBUG) -SQLITE_PRIVATE int sqlite3Fts3Always(int b) { assert( b ); return b; } -SQLITE_PRIVATE int sqlite3Fts3Never(int b) { assert( !b ); return b; } -# endif -#endif + fts3PoslistPhraseMerge(&pTmp1, nRight, 0, 0, pp1, pp2); + aTmp2 = pTmp2 = pTmp1; + *pp1 = p1; + *pp2 = p2; + fts3PoslistPhraseMerge(&pTmp2, nLeft, 1, 0, pp2, pp1); + if( pTmp1!=aTmp && pTmp2!=aTmp2 ){ + fts3PoslistMerge(pp, &aTmp, &aTmp2); + }else if( pTmp1!=aTmp ){ + fts3PoslistCopy(pp, &aTmp); + }else if( pTmp2!=aTmp2 ){ + fts3PoslistCopy(pp, &aTmp2); + }else{ + res = 0; + } -/* -** Write a 64-bit variable-length integer to memory starting at p[0]. -** The length of data written will be between 1 and FTS3_VARINT_MAX bytes. -** The number of bytes written is returned. -*/ -SQLITE_PRIVATE int sqlite3Fts3PutVarint(char *p, sqlite_int64 v){ - unsigned char *q = (unsigned char *) p; - sqlite_uint64 vu = v; - do{ - *q++ = (unsigned char) ((vu & 0x7f) | 0x80); - vu >>= 7; - }while( vu!=0 ); - q[-1] &= 0x7f; /* turn off high bit in final byte */ - assert( q - (unsigned char *)p <= FTS3_VARINT_MAX ); - return (int) (q - (unsigned char *)p); + return res; } -#define GETVARINT_STEP(v, ptr, shift, mask1, mask2, var, ret) \ - v = (v & mask1) | ( (*ptr++) << shift ); \ - if( (v & mask2)==0 ){ var = v; return ret; } -#define GETVARINT_INIT(v, ptr, shift, mask1, mask2, var, ret) \ - v = (*ptr++); \ - if( (v & mask2)==0 ){ var = v; return ret; } - /* -** Read a 64-bit variable-length integer from memory starting at p[0]. -** Return the number of bytes read, or 0 on error. -** The value is stored in *v. +** An instance of this function is used to merge together the (potentially +** large number of) doclists for each term that matches a prefix query. +** See function fts3TermSelectMerge() for details. */ -SQLITE_PRIVATE int sqlite3Fts3GetVarint(const char *pBuf, sqlite_int64 *v){ - const unsigned char *p = (const unsigned char*)pBuf; - const unsigned char *pStart = p; - u32 a; - u64 b; - int shift; - - GETVARINT_INIT(a, p, 0, 0x00, 0x80, *v, 1); - GETVARINT_STEP(a, p, 7, 0x7F, 0x4000, *v, 2); - GETVARINT_STEP(a, p, 14, 0x3FFF, 0x200000, *v, 3); - GETVARINT_STEP(a, p, 21, 0x1FFFFF, 0x10000000, *v, 4); - b = (a & 0x0FFFFFFF ); +typedef struct TermSelect TermSelect; +struct TermSelect { + char *aaOutput[16]; /* Malloc'd output buffers */ + int anOutput[16]; /* Size each output buffer in bytes */ +}; - for(shift=28; shift<=63; shift+=7){ - u64 c = *p++; - b += (c&0x7F) << shift; - if( (c & 0x80)==0 ) break; +/* +** This function is used to read a single varint from a buffer. Parameter +** pEnd points 1 byte past the end of the buffer. When this function is +** called, if *pp points to pEnd or greater, then the end of the buffer +** has been reached. In this case *pp is set to 0 and the function returns. +** +** If *pp does not point to or past pEnd, then a single varint is read +** from *pp. *pp is then set to point 1 byte past the end of the read varint. +** +** If bDescIdx is false, the value read is added to *pVal before returning. +** If it is true, the value read is subtracted from *pVal before this +** function returns. +*/ +static void fts3GetDeltaVarint3( + char **pp, /* IN/OUT: Point to read varint from */ + char *pEnd, /* End of buffer */ + int bDescIdx, /* True if docids are descending */ + sqlite3_int64 *pVal /* IN/OUT: Integer value */ +){ + if( *pp>=pEnd ){ + *pp = 0; + }else{ + u64 iVal; + *pp += sqlite3Fts3GetVarintU(*pp, &iVal); + if( bDescIdx ){ + *pVal = (i64)((u64)*pVal - iVal); + }else{ + *pVal = (i64)((u64)*pVal + iVal); + } } - *v = b; - return (int)(p - pStart); } /* -** Similar to sqlite3Fts3GetVarint(), except that the output is truncated to -** a non-negative 32-bit integer before it is returned. +** This function is used to write a single varint to a buffer. The varint +** is written to *pp. Before returning, *pp is set to point 1 byte past the +** end of the value written. +** +** If *pbFirst is zero when this function is called, the value written to +** the buffer is that of parameter iVal. +** +** If *pbFirst is non-zero when this function is called, then the value +** written is either (iVal-*piPrev) (if bDescIdx is zero) or (*piPrev-iVal) +** (if bDescIdx is non-zero). +** +** Before returning, this function always sets *pbFirst to 1 and *piPrev +** to the value of parameter iVal. */ -SQLITE_PRIVATE int sqlite3Fts3GetVarint32(const char *p, int *pi){ - u32 a; - -#ifndef fts3GetVarint32 - GETVARINT_INIT(a, p, 0, 0x00, 0x80, *pi, 1); -#else - a = (*p++); - assert( a & 0x80 ); -#endif - - GETVARINT_STEP(a, p, 7, 0x7F, 0x4000, *pi, 2); - GETVARINT_STEP(a, p, 14, 0x3FFF, 0x200000, *pi, 3); - GETVARINT_STEP(a, p, 21, 0x1FFFFF, 0x10000000, *pi, 4); - a = (a & 0x0FFFFFFF ); - *pi = (int)(a | ((u32)(*p & 0x07) << 28)); - assert( 0==(a & 0x80000000) ); - assert( *pi>=0 ); - return 5; +static void fts3PutDeltaVarint3( + char **pp, /* IN/OUT: Output pointer */ + int bDescIdx, /* True for descending docids */ + sqlite3_int64 *piPrev, /* IN/OUT: Previous value written to list */ + int *pbFirst, /* IN/OUT: True after first int written */ + sqlite3_int64 iVal /* Write this value to the list */ +){ + sqlite3_uint64 iWrite; + if( bDescIdx==0 || *pbFirst==0 ){ + assert_fts3_nc( *pbFirst==0 || iVal>=*piPrev ); + iWrite = (u64)iVal - (u64)*piPrev; + }else{ + assert_fts3_nc( *piPrev>=iVal ); + iWrite = (u64)*piPrev - (u64)iVal; + } + assert( *pbFirst || *piPrev==0 ); + assert_fts3_nc( *pbFirst==0 || iWrite>0 ); + *pp += sqlite3Fts3PutVarint(*pp, iWrite); + *piPrev = iVal; + *pbFirst = 1; } + /* -** Return the number of bytes required to encode v as a varint +** This macro is used by various functions that merge doclists. The two +** arguments are 64-bit docid values. If the value of the stack variable +** bDescDoclist is 0 when this macro is invoked, then it returns (i1-i2). +** Otherwise, (i2-i1). +** +** Using this makes it easier to write code that can merge doclists that are +** sorted in either ascending or descending order. */ -SQLITE_PRIVATE int sqlite3Fts3VarintLen(sqlite3_uint64 v){ - int i = 0; - do{ - i++; - v >>= 7; - }while( v!=0 ); - return i; -} +/* #define DOCID_CMP(i1, i2) ((bDescDoclist?-1:1) * (i64)((u64)i1-i2)) */ +#define DOCID_CMP(i1, i2) ((bDescDoclist?-1:1) * (i1>i2?1:((i1==i2)?0:-1))) /* -** Convert an SQL-style quoted string into a normal string by removing -** the quote characters. The conversion is done in-place. If the -** input does not begin with a quote character, then this routine -** is a no-op. -** -** Examples: +** This function does an "OR" merge of two doclists (output contains all +** positions contained in either argument doclist). If the docids in the +** input doclists are sorted in ascending order, parameter bDescDoclist +** should be false. If they are sorted in ascending order, it should be +** passed a non-zero value. ** -** "abc" becomes abc -** 'xyz' becomes xyz -** [pqr] becomes pqr -** `mno` becomes mno +** If no error occurs, *paOut is set to point at an sqlite3_malloc'd buffer +** containing the output doclist and SQLITE_OK is returned. In this case +** *pnOut is set to the number of bytes in the output doclist. ** +** If an error occurs, an SQLite error code is returned. The output values +** are undefined in this case. */ -SQLITE_PRIVATE void sqlite3Fts3Dequote(char *z){ - char quote; /* Quote character (if any ) */ +static int fts3DoclistOrMerge( + int bDescDoclist, /* True if arguments are desc */ + char *a1, int n1, /* First doclist */ + char *a2, int n2, /* Second doclist */ + char **paOut, int *pnOut /* OUT: Malloc'd doclist */ +){ + int rc = SQLITE_OK; + sqlite3_int64 i1 = 0; + sqlite3_int64 i2 = 0; + sqlite3_int64 iPrev = 0; + char *pEnd1 = &a1[n1]; + char *pEnd2 = &a2[n2]; + char *p1 = a1; + char *p2 = a2; + char *p; + char *aOut; + int bFirstOut = 0; - quote = z[0]; - if( quote=='[' || quote=='\'' || quote=='"' || quote=='`' ){ - int iIn = 1; /* Index of next byte to read from input */ - int iOut = 0; /* Index of next byte to write to output */ + *paOut = 0; + *pnOut = 0; - /* If the first byte was a '[', then the close-quote character is a ']' */ - if( quote=='[' ) quote = ']'; + /* Allocate space for the output. Both the input and output doclists + ** are delta encoded. If they are in ascending order (bDescDoclist==0), + ** then the first docid in each list is simply encoded as a varint. For + ** each subsequent docid, the varint stored is the difference between the + ** current and previous docid (a positive number - since the list is in + ** ascending order). + ** + ** The first docid written to the output is therefore encoded using the + ** same number of bytes as it is in whichever of the input lists it is + ** read from. And each subsequent docid read from the same input list + ** consumes either the same or less bytes as it did in the input (since + ** the difference between it and the previous value in the output must + ** be a positive value less than or equal to the delta value read from + ** the input list). The same argument applies to all but the first docid + ** read from the 'other' list. And to the contents of all position lists + ** that will be copied and merged from the input to the output. + ** + ** However, if the first docid copied to the output is a negative number, + ** then the encoding of the first docid from the 'other' input list may + ** be larger in the output than it was in the input (since the delta value + ** may be a larger positive integer than the actual docid). + ** + ** The space required to store the output is therefore the sum of the + ** sizes of the two inputs, plus enough space for exactly one of the input + ** docids to grow. + ** + ** A symetric argument may be made if the doclists are in descending + ** order. + */ + aOut = sqlite3_malloc64((i64)n1+n2+FTS3_VARINT_MAX-1+FTS3_BUFFER_PADDING); + if( !aOut ) return SQLITE_NOMEM; - while( z[iIn] ){ - if( z[iIn]==quote ){ - if( z[iIn+1]!=quote ) break; - z[iOut++] = quote; - iIn += 2; - }else{ - z[iOut++] = z[iIn++]; - } + p = aOut; + fts3GetDeltaVarint3(&p1, pEnd1, 0, &i1); + fts3GetDeltaVarint3(&p2, pEnd2, 0, &i2); + while( p1 || p2 ){ + sqlite3_int64 iDiff = DOCID_CMP(i1, i2); + + if( p2 && p1 && iDiff==0 ){ + fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i1); + rc = fts3PoslistMerge(&p, &p1, &p2); + if( rc ) break; + fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1); + fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2); + }else if( !p2 || (p1 && iDiff<0) ){ + fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i1); + fts3PoslistCopy(&p, &p1); + fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1); + }else{ + fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i2); + fts3PoslistCopy(&p, &p2); + fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2); } - z[iOut] = '\0'; + + assert( (p-aOut)<=((p1?(p1-a1):n1)+(p2?(p2-a2):n2)+FTS3_VARINT_MAX-1) ); } -} -/* -** Read a single varint from the doclist at *pp and advance *pp to point -** to the first byte past the end of the varint. Add the value of the varint -** to *pVal. -*/ -static void fts3GetDeltaVarint(char **pp, sqlite3_int64 *pVal){ - sqlite3_int64 iVal; - *pp += sqlite3Fts3GetVarint(*pp, &iVal); - *pVal += iVal; + if( rc!=SQLITE_OK ){ + sqlite3_free(aOut); + p = aOut = 0; + }else{ + assert( (p-aOut)<=n1+n2+FTS3_VARINT_MAX-1 ); + memset(&aOut[(p-aOut)], 0, FTS3_BUFFER_PADDING); + } + *paOut = aOut; + *pnOut = (int)(p-aOut); + return rc; } /* -** When this function is called, *pp points to the first byte following a -** varint that is part of a doclist (or position-list, or any other list -** of varints). This function moves *pp to point to the start of that varint, -** and sets *pVal by the varint value. +** This function does a "phrase" merge of two doclists. In a phrase merge, +** the output contains a copy of each position from the right-hand input +** doclist for which there is a position in the left-hand input doclist +** exactly nDist tokens before it. ** -** Argument pStart points to the first byte of the doclist that the -** varint is part of. +** If the docids in the input doclists are sorted in ascending order, +** parameter bDescDoclist should be false. If they are sorted in ascending +** order, it should be passed a non-zero value. +** +** The right-hand input doclist is overwritten by this function. */ -static void fts3GetReverseVarint( - char **pp, - char *pStart, - sqlite3_int64 *pVal +static int fts3DoclistPhraseMerge( + int bDescDoclist, /* True if arguments are desc */ + int nDist, /* Distance from left to right (1=adjacent) */ + char *aLeft, int nLeft, /* Left doclist */ + char **paRight, int *pnRight /* IN/OUT: Right/output doclist */ ){ - sqlite3_int64 iVal; + sqlite3_int64 i1 = 0; + sqlite3_int64 i2 = 0; + sqlite3_int64 iPrev = 0; + char *aRight = *paRight; + char *pEnd1 = &aLeft[nLeft]; + char *pEnd2 = &aRight[*pnRight]; + char *p1 = aLeft; + char *p2 = aRight; char *p; + int bFirstOut = 0; + char *aOut; - /* Pointer p now points at the first byte past the varint we are - ** interested in. So, unless the doclist is corrupt, the 0x80 bit is - ** clear on character p[-1]. */ - for(p = (*pp)-2; p>=pStart && *p&0x80; p--); - p++; - *pp = p; - - sqlite3Fts3GetVarint(p, &iVal); - *pVal = iVal; -} + assert( nDist>0 ); + if( bDescDoclist ){ + aOut = sqlite3_malloc64((sqlite3_int64)*pnRight + FTS3_VARINT_MAX); + if( aOut==0 ) return SQLITE_NOMEM; + }else{ + aOut = aRight; + } + p = aOut; -/* -** The xDisconnect() virtual table method. -*/ -static int fts3DisconnectMethod(sqlite3_vtab *pVtab){ - Fts3Table *p = (Fts3Table *)pVtab; - int i; + fts3GetDeltaVarint3(&p1, pEnd1, 0, &i1); + fts3GetDeltaVarint3(&p2, pEnd2, 0, &i2); - assert( p->nPendingData==0 ); - assert( p->pSegments==0 ); + while( p1 && p2 ){ + sqlite3_int64 iDiff = DOCID_CMP(i1, i2); + if( iDiff==0 ){ + char *pSave = p; + sqlite3_int64 iPrevSave = iPrev; + int bFirstOutSave = bFirstOut; - /* Free any prepared statements held */ - sqlite3_finalize(p->pSeekStmt); - for(i=0; iaStmt); i++){ - sqlite3_finalize(p->aStmt[i]); + fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i1); + if( 0==fts3PoslistPhraseMerge(&p, nDist, 0, 1, &p1, &p2) ){ + p = pSave; + iPrev = iPrevSave; + bFirstOut = bFirstOutSave; + } + fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1); + fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2); + }else if( iDiff<0 ){ + fts3PoslistCopy(0, &p1); + fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1); + }else{ + fts3PoslistCopy(0, &p2); + fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2); + } } - sqlite3_free(p->zSegmentsTbl); - sqlite3_free(p->zReadExprlist); - sqlite3_free(p->zWriteExprlist); - sqlite3_free(p->zContentTbl); - sqlite3_free(p->zLanguageid); - /* Invoke the tokenizer destructor to free the tokenizer. */ - p->pTokenizer->pModule->xDestroy(p->pTokenizer); + *pnRight = (int)(p - aOut); + if( bDescDoclist ){ + sqlite3_free(aRight); + *paRight = aOut; + } - sqlite3_free(p); return SQLITE_OK; } /* -** Write an error message into *pzErr -*/ -SQLITE_PRIVATE void sqlite3Fts3ErrMsg(char **pzErr, const char *zFormat, ...){ - va_list ap; - sqlite3_free(*pzErr); - va_start(ap, zFormat); - *pzErr = sqlite3_vmprintf(zFormat, ap); - va_end(ap); -} - -/* -** Construct one or more SQL statements from the format string given -** and then evaluate those statements. The success code is written -** into *pRc. -** -** If *pRc is initially non-zero then this routine is a no-op. +** Argument pList points to a position list nList bytes in size. This +** function checks to see if the position list contains any entries for +** a token in position 0 (of any column). If so, it writes argument iDelta +** to the output buffer pOut, followed by a position list consisting only +** of the entries from pList at position 0, and terminated by an 0x00 byte. +** The value returned is the number of bytes written to pOut (if any). */ -static void fts3DbExec( - int *pRc, /* Success code */ - sqlite3 *db, /* Database in which to run SQL */ - const char *zFormat, /* Format string for SQL */ - ... /* Arguments to the format string */ +SQLITE_PRIVATE int sqlite3Fts3FirstFilter( + sqlite3_int64 iDelta, /* Varint that may be written to pOut */ + char *pList, /* Position list (no 0x00 term) */ + int nList, /* Size of pList in bytes */ + char *pOut /* Write output here */ ){ - va_list ap; - char *zSql; - if( *pRc ) return; - va_start(ap, zFormat); - zSql = sqlite3_vmprintf(zFormat, ap); - va_end(ap); - if( zSql==0 ){ - *pRc = SQLITE_NOMEM; - }else{ - *pRc = sqlite3_exec(db, zSql, 0, 0, 0); - sqlite3_free(zSql); - } -} + int nOut = 0; + int bWritten = 0; /* True once iDelta has been written */ + char *p = pList; + char *pEnd = &pList[nList]; -/* -** The xDestroy() virtual table method. -*/ -static int fts3DestroyMethod(sqlite3_vtab *pVtab){ - Fts3Table *p = (Fts3Table *)pVtab; - int rc = SQLITE_OK; /* Return code */ - const char *zDb = p->zDb; /* Name of database (e.g. "main", "temp") */ - sqlite3 *db = p->db; /* Database handle */ + if( *p!=0x01 ){ + if( *p==0x02 ){ + nOut += sqlite3Fts3PutVarint(&pOut[nOut], iDelta); + pOut[nOut++] = 0x02; + bWritten = 1; + } + fts3ColumnlistCopy(0, &p); + } - /* Drop the shadow tables */ - if( p->zContentTbl==0 ){ - fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_content'", zDb, p->zName); + while( pzName); - fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_segdir'", zDb, p->zName); - fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_docsize'", zDb, p->zName); - fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_stat'", zDb, p->zName); - /* If everything has worked, invoke fts3DisconnectMethod() to free the - ** memory associated with the Fts3Table structure and return SQLITE_OK. - ** Otherwise, return an SQLite error code. - */ - return (rc==SQLITE_OK ? fts3DisconnectMethod(pVtab) : rc); + return nOut; } /* -** Invoke sqlite3_declare_vtab() to declare the schema for the FTS3 table -** passed as the first argument. This is done as part of the xConnect() -** and xCreate() methods. +** Merge all doclists in the TermSelect.aaOutput[] array into a single +** doclist stored in TermSelect.aaOutput[0]. If successful, delete all +** other doclists (except the aaOutput[0] one) and return SQLITE_OK. ** -** If *pRc is non-zero when this function is called, it is a no-op. -** Otherwise, if an error occurs, an SQLite error code is stored in *pRc -** before returning. +** If an OOM error occurs, return SQLITE_NOMEM. In this case it is +** the responsibility of the caller to free any doclists left in the +** TermSelect.aaOutput[] array. */ -static void fts3DeclareVtab(int *pRc, Fts3Table *p){ - if( *pRc==SQLITE_OK ){ - int i; /* Iterator variable */ - int rc; /* Return code */ - char *zSql; /* SQL statement passed to declare_vtab() */ - char *zCols; /* List of user defined columns */ - const char *zLanguageid; +static int fts3TermSelectFinishMerge(Fts3Table *p, TermSelect *pTS){ + char *aOut = 0; + int nOut = 0; + int i; - zLanguageid = (p->zLanguageid ? p->zLanguageid : "__langid"); - sqlite3_vtab_config(p->db, SQLITE_VTAB_CONSTRAINT_SUPPORT, 1); + /* Loop through the doclists in the aaOutput[] array. Merge them all + ** into a single doclist. + */ + for(i=0; iaaOutput); i++){ + if( pTS->aaOutput[i] ){ + if( !aOut ){ + aOut = pTS->aaOutput[i]; + nOut = pTS->anOutput[i]; + pTS->aaOutput[i] = 0; + }else{ + int nNew; + char *aNew; - /* Create a list of user columns for the virtual table */ - zCols = sqlite3_mprintf("%Q, ", p->azColumn[0]); - for(i=1; zCols && inColumn; i++){ - zCols = sqlite3_mprintf("%z%Q, ", zCols, p->azColumn[i]); - } + int rc = fts3DoclistOrMerge(p->bDescIdx, + pTS->aaOutput[i], pTS->anOutput[i], aOut, nOut, &aNew, &nNew + ); + if( rc!=SQLITE_OK ){ + sqlite3_free(aOut); + return rc; + } - /* Create the whole "CREATE TABLE" statement to pass to SQLite */ - zSql = sqlite3_mprintf( - "CREATE TABLE x(%s %Q HIDDEN, docid HIDDEN, %Q HIDDEN)", - zCols, p->zName, zLanguageid - ); - if( !zCols || !zSql ){ - rc = SQLITE_NOMEM; - }else{ - rc = sqlite3_declare_vtab(p->db, zSql); + sqlite3_free(pTS->aaOutput[i]); + sqlite3_free(aOut); + pTS->aaOutput[i] = 0; + aOut = aNew; + nOut = nNew; + } } - - sqlite3_free(zSql); - sqlite3_free(zCols); - *pRc = rc; } -} -/* -** Create the %_stat table if it does not already exist. -*/ -SQLITE_PRIVATE void sqlite3Fts3CreateStatTable(int *pRc, Fts3Table *p){ - fts3DbExec(pRc, p->db, - "CREATE TABLE IF NOT EXISTS %Q.'%q_stat'" - "(id INTEGER PRIMARY KEY, value BLOB);", - p->zDb, p->zName - ); - if( (*pRc)==SQLITE_OK ) p->bHasStat = 1; + pTS->aaOutput[0] = aOut; + pTS->anOutput[0] = nOut; + return SQLITE_OK; } /* -** Create the backing store tables (%_content, %_segments and %_segdir) -** required by the FTS3 table passed as the only argument. This is done -** as part of the vtab xCreate() method. +** Merge the doclist aDoclist/nDoclist into the TermSelect object passed +** as the first argument. The merge is an "OR" merge (see function +** fts3DoclistOrMerge() for details). ** -** If the p->bHasDocsize boolean is true (indicating that this is an -** FTS4 table, not an FTS3 table) then also create the %_docsize and -** %_stat tables required by FTS4. +** This function is called with the doclist for each term that matches +** a queried prefix. It merges all these doclists into one, the doclist +** for the specified prefix. Since there can be a very large number of +** doclists to merge, the merging is done pair-wise using the TermSelect +** object. +** +** This function returns SQLITE_OK if the merge is successful, or an +** SQLite error code (SQLITE_NOMEM) if an error occurs. */ -static int fts3CreateTables(Fts3Table *p){ - int rc = SQLITE_OK; /* Return code */ - int i; /* Iterator variable */ - sqlite3 *db = p->db; /* The database connection */ +static int fts3TermSelectMerge( + Fts3Table *p, /* FTS table handle */ + TermSelect *pTS, /* TermSelect object to merge into */ + char *aDoclist, /* Pointer to doclist */ + int nDoclist /* Size of aDoclist in bytes */ +){ + if( pTS->aaOutput[0]==0 ){ + /* If this is the first term selected, copy the doclist to the output + ** buffer using memcpy(). + ** + ** Add FTS3_VARINT_MAX bytes of unused space to the end of the + ** allocation. This is so as to ensure that the buffer is big enough + ** to hold the current doclist AND'd with any other doclist. If the + ** doclists are stored in order=ASC order, this padding would not be + ** required (since the size of [doclistA AND doclistB] is always less + ** than or equal to the size of [doclistA] in that case). But this is + ** not true for order=DESC. For example, a doclist containing (1, -1) + ** may be smaller than (-1), as in the first example the -1 may be stored + ** as a single-byte delta, whereas in the second it must be stored as a + ** FTS3_VARINT_MAX byte varint. + ** + ** Similar padding is added in the fts3DoclistOrMerge() function. + */ + pTS->aaOutput[0] = sqlite3_malloc(nDoclist + FTS3_VARINT_MAX + 1); + pTS->anOutput[0] = nDoclist; + if( pTS->aaOutput[0] ){ + memcpy(pTS->aaOutput[0], aDoclist, nDoclist); + memset(&pTS->aaOutput[0][nDoclist], 0, FTS3_VARINT_MAX); + }else{ + return SQLITE_NOMEM; + } + }else{ + char *aMerge = aDoclist; + int nMerge = nDoclist; + int iOut; - if( p->zContentTbl==0 ){ - const char *zLanguageid = p->zLanguageid; - char *zContentCols; /* Columns of %_content table */ + for(iOut=0; iOutaaOutput); iOut++){ + if( pTS->aaOutput[iOut]==0 ){ + assert( iOut>0 ); + pTS->aaOutput[iOut] = aMerge; + pTS->anOutput[iOut] = nMerge; + break; + }else{ + char *aNew; + int nNew; - /* Create a list of user columns for the content table */ - zContentCols = sqlite3_mprintf("docid INTEGER PRIMARY KEY"); - for(i=0; zContentCols && inColumn; i++){ - char *z = p->azColumn[i]; - zContentCols = sqlite3_mprintf("%z, 'c%d%q'", zContentCols, i, z); - } - if( zLanguageid && zContentCols ){ - zContentCols = sqlite3_mprintf("%z, langid", zContentCols, zLanguageid); - } - if( zContentCols==0 ) rc = SQLITE_NOMEM; - - /* Create the content table */ - fts3DbExec(&rc, db, - "CREATE TABLE %Q.'%q_content'(%s)", - p->zDb, p->zName, zContentCols - ); - sqlite3_free(zContentCols); - } + int rc = fts3DoclistOrMerge(p->bDescIdx, aMerge, nMerge, + pTS->aaOutput[iOut], pTS->anOutput[iOut], &aNew, &nNew + ); + if( rc!=SQLITE_OK ){ + if( aMerge!=aDoclist ) sqlite3_free(aMerge); + return rc; + } - /* Create other tables */ - fts3DbExec(&rc, db, - "CREATE TABLE %Q.'%q_segments'(blockid INTEGER PRIMARY KEY, block BLOB);", - p->zDb, p->zName - ); - fts3DbExec(&rc, db, - "CREATE TABLE %Q.'%q_segdir'(" - "level INTEGER," - "idx INTEGER," - "start_block INTEGER," - "leaves_end_block INTEGER," - "end_block INTEGER," - "root BLOB," - "PRIMARY KEY(level, idx)" - ");", - p->zDb, p->zName - ); - if( p->bHasDocsize ){ - fts3DbExec(&rc, db, - "CREATE TABLE %Q.'%q_docsize'(docid INTEGER PRIMARY KEY, size BLOB);", - p->zDb, p->zName - ); - } - assert( p->bHasStat==p->bFts4 ); - if( p->bHasStat ){ - sqlite3Fts3CreateStatTable(&rc, p); + if( aMerge!=aDoclist ) sqlite3_free(aMerge); + sqlite3_free(pTS->aaOutput[iOut]); + pTS->aaOutput[iOut] = 0; + + aMerge = aNew; + nMerge = nNew; + if( (iOut+1)==SizeofArray(pTS->aaOutput) ){ + pTS->aaOutput[iOut] = aMerge; + pTS->anOutput[iOut] = nMerge; + } + } + } } - return rc; + return SQLITE_OK; } /* -** Store the current database page-size in bytes in p->nPgsz. -** -** If *pRc is non-zero when this function is called, it is a no-op. -** Otherwise, if an error occurs, an SQLite error code is stored in *pRc -** before returning. +** Append SegReader object pNew to the end of the pCsr->apSegment[] array. */ -static void fts3DatabasePageSize(int *pRc, Fts3Table *p){ - if( *pRc==SQLITE_OK ){ - int rc; /* Return code */ - char *zSql; /* SQL text "PRAGMA %Q.page_size" */ - sqlite3_stmt *pStmt; /* Compiled "PRAGMA %Q.page_size" statement */ - - zSql = sqlite3_mprintf("PRAGMA %Q.page_size", p->zDb); - if( !zSql ){ - rc = SQLITE_NOMEM; - }else{ - rc = sqlite3_prepare(p->db, zSql, -1, &pStmt, 0); - if( rc==SQLITE_OK ){ - sqlite3_step(pStmt); - p->nPgsz = sqlite3_column_int(pStmt, 0); - rc = sqlite3_finalize(pStmt); - }else if( rc==SQLITE_AUTH ){ - p->nPgsz = 1024; - rc = SQLITE_OK; - } +static int fts3SegReaderCursorAppend( + Fts3MultiSegReader *pCsr, + Fts3SegReader *pNew +){ + if( (pCsr->nSegment%16)==0 ){ + Fts3SegReader **apNew; + sqlite3_int64 nByte = (pCsr->nSegment + 16)*sizeof(Fts3SegReader*); + apNew = (Fts3SegReader **)sqlite3_realloc64(pCsr->apSegment, nByte); + if( !apNew ){ + sqlite3Fts3SegReaderFree(pNew); + return SQLITE_NOMEM; } - assert( p->nPgsz>0 || rc!=SQLITE_OK ); - sqlite3_free(zSql); - *pRc = rc; + pCsr->apSegment = apNew; } + pCsr->apSegment[pCsr->nSegment++] = pNew; + return SQLITE_OK; } /* -** "Special" FTS4 arguments are column specifications of the following form: -** -** = +** Add seg-reader objects to the Fts3MultiSegReader object passed as the +** 8th argument. ** -** There may not be whitespace surrounding the "=" character. The -** term may be quoted, but the may not. +** This function returns SQLITE_OK if successful, or an SQLite error code +** otherwise. */ -static int fts3IsSpecialColumn( - const char *z, - int *pnKey, - char **pzValue +static int fts3SegReaderCursor( + Fts3Table *p, /* FTS3 table handle */ + int iLangid, /* Language id */ + int iIndex, /* Index to search (from 0 to p->nIndex-1) */ + int iLevel, /* Level of segments to scan */ + const char *zTerm, /* Term to query for */ + int nTerm, /* Size of zTerm in bytes */ + int isPrefix, /* True for a prefix search */ + int isScan, /* True to scan from zTerm to EOF */ + Fts3MultiSegReader *pCsr /* Cursor object to populate */ ){ - char *zValue; - const char *zCsr = z; + int rc = SQLITE_OK; /* Error code */ + sqlite3_stmt *pStmt = 0; /* Statement to iterate through segments */ + int rc2; /* Result of sqlite3_reset() */ - while( *zCsr!='=' ){ - if( *zCsr=='\0' ) return 0; - zCsr++; + /* If iLevel is less than 0 and this is not a scan, include a seg-reader + ** for the pending-terms. If this is a scan, then this call must be being + ** made by an fts4aux module, not an FTS table. In this case calling + ** Fts3SegReaderPending might segfault, as the data structures used by + ** fts4aux are not completely populated. So it's easiest to filter these + ** calls out here. */ + if( iLevel<0 && p->aIndex && p->iPrevLangid==iLangid ){ + Fts3SegReader *pSeg = 0; + rc = sqlite3Fts3SegReaderPending(p, iIndex, zTerm, nTerm, isPrefix||isScan, &pSeg); + if( rc==SQLITE_OK && pSeg ){ + rc = fts3SegReaderCursorAppend(pCsr, pSeg); + } } - *pnKey = (int)(zCsr-z); - zValue = sqlite3_mprintf("%s", &zCsr[1]); - if( zValue ){ - sqlite3Fts3Dequote(zValue); + if( iLevel!=FTS3_SEGCURSOR_PENDING ){ + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3AllSegdirs(p, iLangid, iIndex, iLevel, &pStmt); + } + + while( rc==SQLITE_OK && SQLITE_ROW==(rc = sqlite3_step(pStmt)) ){ + Fts3SegReader *pSeg = 0; + + /* Read the values returned by the SELECT into local variables. */ + sqlite3_int64 iStartBlock = sqlite3_column_int64(pStmt, 1); + sqlite3_int64 iLeavesEndBlock = sqlite3_column_int64(pStmt, 2); + sqlite3_int64 iEndBlock = sqlite3_column_int64(pStmt, 3); + int nRoot = sqlite3_column_bytes(pStmt, 4); + char const *zRoot = sqlite3_column_blob(pStmt, 4); + + /* If zTerm is not NULL, and this segment is not stored entirely on its + ** root node, the range of leaves scanned can be reduced. Do this. */ + if( iStartBlock && zTerm && zRoot ){ + sqlite3_int64 *pi = (isPrefix ? &iLeavesEndBlock : 0); + rc = fts3SelectLeaf(p, zTerm, nTerm, zRoot, nRoot, &iStartBlock, pi); + if( rc!=SQLITE_OK ) goto finished; + if( isPrefix==0 && isScan==0 ) iLeavesEndBlock = iStartBlock; + } + + rc = sqlite3Fts3SegReaderNew(pCsr->nSegment+1, + (isPrefix==0 && isScan==0), + iStartBlock, iLeavesEndBlock, + iEndBlock, zRoot, nRoot, &pSeg + ); + if( rc!=SQLITE_OK ) goto finished; + rc = fts3SegReaderCursorAppend(pCsr, pSeg); + } } - *pzValue = zValue; - return 1; + + finished: + rc2 = sqlite3_reset(pStmt); + if( rc==SQLITE_DONE ) rc = rc2; + + return rc; } /* -** Append the output of a printf() style formatting to an existing string. +** Set up a cursor object for iterating through a full-text index or a +** single level therein. */ -static void fts3Appendf( - int *pRc, /* IN/OUT: Error code */ - char **pz, /* IN/OUT: Pointer to string buffer */ - const char *zFormat, /* Printf format string to append */ - ... /* Arguments for printf format string */ +SQLITE_PRIVATE int sqlite3Fts3SegReaderCursor( + Fts3Table *p, /* FTS3 table handle */ + int iLangid, /* Language-id to search */ + int iIndex, /* Index to search (from 0 to p->nIndex-1) */ + int iLevel, /* Level of segments to scan */ + const char *zTerm, /* Term to query for */ + int nTerm, /* Size of zTerm in bytes */ + int isPrefix, /* True for a prefix search */ + int isScan, /* True to scan from zTerm to EOF */ + Fts3MultiSegReader *pCsr /* Cursor object to populate */ ){ - if( *pRc==SQLITE_OK ){ - va_list ap; - char *z; - va_start(ap, zFormat); - z = sqlite3_vmprintf(zFormat, ap); - va_end(ap); - if( z && *pz ){ - char *z2 = sqlite3_mprintf("%s%s", *pz, z); - sqlite3_free(z); - z = z2; - } - if( z==0 ) *pRc = SQLITE_NOMEM; - sqlite3_free(*pz); - *pz = z; - } + assert( iIndex>=0 && iIndexnIndex ); + assert( iLevel==FTS3_SEGCURSOR_ALL + || iLevel==FTS3_SEGCURSOR_PENDING + || iLevel>=0 + ); + assert( iLevel "un \"\"zip\"\"" +** In addition to its current configuration, have the Fts3MultiSegReader +** passed as the 4th argument also scan the doclist for term zTerm/nTerm. ** -** The pointer returned points to memory obtained from sqlite3_malloc(). It -** is the callers responsibility to call sqlite3_free() to release this -** memory. +** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code. */ -static char *fts3QuoteId(char const *zInput){ - int nRet; - char *zRet; - nRet = 2 + (int)strlen(zInput)*2 + 1; - zRet = sqlite3_malloc(nRet); - if( zRet ){ - int i; - char *z = zRet; - *(z++) = '"'; - for(i=0; zInput[i]; i++){ - if( zInput[i]=='"' ) *(z++) = '"'; - *(z++) = zInput[i]; - } - *(z++) = '"'; - *(z++) = '\0'; - } - return zRet; +static int fts3SegReaderCursorAddZero( + Fts3Table *p, /* FTS virtual table handle */ + int iLangid, + const char *zTerm, /* Term to scan doclist of */ + int nTerm, /* Number of bytes in zTerm */ + Fts3MultiSegReader *pCsr /* Fts3MultiSegReader to modify */ +){ + return fts3SegReaderCursor(p, + iLangid, 0, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 0, 0,pCsr + ); } /* -** Return a list of comma separated SQL expressions and a FROM clause that -** could be used in a SELECT statement such as the following: -** -** SELECT FROM %_content AS x ... -** -** to return the docid, followed by each column of text data in order -** from left to write. If parameter zFunc is not NULL, then instead of -** being returned directly each column of text data is passed to an SQL -** function named zFunc first. For example, if zFunc is "unzip" and the -** table has the three user-defined columns "a", "b", and "c", the following -** string is returned: -** -** "docid, unzip(x.'a'), unzip(x.'b'), unzip(x.'c') FROM %_content AS x" +** Open an Fts3MultiSegReader to scan the doclist for term zTerm/nTerm. Or, +** if isPrefix is true, to scan the doclist for all terms for which +** zTerm/nTerm is a prefix. If successful, return SQLITE_OK and write +** a pointer to the new Fts3MultiSegReader to *ppSegcsr. Otherwise, return +** an SQLite error code. ** -** The pointer returned points to a buffer allocated by sqlite3_malloc(). It -** is the responsibility of the caller to eventually free it. +** It is the responsibility of the caller to free this object by eventually +** passing it to fts3SegReaderCursorFree() ** -** If *pRc is not SQLITE_OK when this function is called, it is a no-op (and -** a NULL pointer is returned). Otherwise, if an OOM error is encountered -** by this function, NULL is returned and *pRc is set to SQLITE_NOMEM. If -** no error occurs, *pRc is left unmodified. +** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code. +** Output parameter *ppSegcsr is set to 0 if an error occurs. */ -static char *fts3ReadExprList(Fts3Table *p, const char *zFunc, int *pRc){ - char *zRet = 0; - char *zFree = 0; - char *zFunction; - int i; +static int fts3TermSegReaderCursor( + Fts3Cursor *pCsr, /* Virtual table cursor handle */ + const char *zTerm, /* Term to query for */ + int nTerm, /* Size of zTerm in bytes */ + int isPrefix, /* True for a prefix search */ + Fts3MultiSegReader **ppSegcsr /* OUT: Allocated seg-reader cursor */ +){ + Fts3MultiSegReader *pSegcsr; /* Object to allocate and return */ + int rc = SQLITE_NOMEM; /* Return code */ - if( p->zContentTbl==0 ){ - if( !zFunc ){ - zFunction = ""; - }else{ - zFree = zFunction = fts3QuoteId(zFunc); - } - fts3Appendf(pRc, &zRet, "docid"); - for(i=0; inColumn; i++){ - fts3Appendf(pRc, &zRet, ",%s(x.'c%d%q')", zFunction, i, p->azColumn[i]); - } - if( p->zLanguageid ){ - fts3Appendf(pRc, &zRet, ", x.%Q", "langid"); - } - sqlite3_free(zFree); - }else{ - fts3Appendf(pRc, &zRet, "rowid"); - for(i=0; inColumn; i++){ - fts3Appendf(pRc, &zRet, ", x.'%q'", p->azColumn[i]); + pSegcsr = sqlite3_malloc(sizeof(Fts3MultiSegReader)); + if( pSegcsr ){ + int i; + int bFound = 0; /* True once an index has been found */ + Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; + + if( isPrefix ){ + for(i=1; bFound==0 && inIndex; i++){ + if( p->aIndex[i].nPrefix==nTerm ){ + bFound = 1; + rc = sqlite3Fts3SegReaderCursor(p, pCsr->iLangid, + i, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 0, 0, pSegcsr + ); + pSegcsr->bLookup = 1; + } + } + + for(i=1; bFound==0 && inIndex; i++){ + if( p->aIndex[i].nPrefix==nTerm+1 ){ + bFound = 1; + rc = sqlite3Fts3SegReaderCursor(p, pCsr->iLangid, + i, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 1, 0, pSegcsr + ); + if( rc==SQLITE_OK ){ + rc = fts3SegReaderCursorAddZero( + p, pCsr->iLangid, zTerm, nTerm, pSegcsr + ); + } + } + } } - if( p->zLanguageid ){ - fts3Appendf(pRc, &zRet, ", x.%Q", p->zLanguageid); + + if( bFound==0 ){ + rc = sqlite3Fts3SegReaderCursor(p, pCsr->iLangid, + 0, FTS3_SEGCURSOR_ALL, zTerm, nTerm, isPrefix, 0, pSegcsr + ); + pSegcsr->bLookup = !isPrefix; } } - fts3Appendf(pRc, &zRet, " FROM '%q'.'%q%s' AS x", - p->zDb, - (p->zContentTbl ? p->zContentTbl : p->zName), - (p->zContentTbl ? "" : "_content") - ); - return zRet; -} - -/* -** Return a list of N comma separated question marks, where N is the number -** of columns in the %_content table (one for the docid plus one for each -** user-defined text column). -** -** If argument zFunc is not NULL, then all but the first question mark -** is preceded by zFunc and an open bracket, and followed by a closed -** bracket. For example, if zFunc is "zip" and the FTS3 table has three -** user-defined text columns, the following string is returned: -** -** "?, zip(?), zip(?), zip(?)" -** -** The pointer returned points to a buffer allocated by sqlite3_malloc(). It -** is the responsibility of the caller to eventually free it. -** -** If *pRc is not SQLITE_OK when this function is called, it is a no-op (and -** a NULL pointer is returned). Otherwise, if an OOM error is encountered -** by this function, NULL is returned and *pRc is set to SQLITE_NOMEM. If -** no error occurs, *pRc is left unmodified. -*/ -static char *fts3WriteExprList(Fts3Table *p, const char *zFunc, int *pRc){ - char *zRet = 0; - char *zFree = 0; - char *zFunction; - int i; - if( !zFunc ){ - zFunction = ""; - }else{ - zFree = zFunction = fts3QuoteId(zFunc); - } - fts3Appendf(pRc, &zRet, "?"); - for(i=0; inColumn; i++){ - fts3Appendf(pRc, &zRet, ",%s(?)", zFunction); - } - if( p->zLanguageid ){ - fts3Appendf(pRc, &zRet, ", ?"); - } - sqlite3_free(zFree); - return zRet; + *ppSegcsr = pSegcsr; + return rc; } /* -** This function interprets the string at (*pp) as a non-negative integer -** value. It reads the integer and sets *pnOut to the value read, then -** sets *pp to point to the byte immediately following the last byte of -** the integer value. -** -** Only decimal digits ('0'..'9') may be part of an integer value. -** -** If *pp does not being with a decimal digit SQLITE_ERROR is returned and -** the output value undefined. Otherwise SQLITE_OK is returned. -** -** This function is used when parsing the "prefix=" FTS4 parameter. +** Free an Fts3MultiSegReader allocated by fts3TermSegReaderCursor(). */ -static int fts3GobbleInt(const char **pp, int *pnOut){ - const int MAX_NPREFIX = 10000000; - const char *p; /* Iterator pointer */ - int nInt = 0; /* Output value */ - - for(p=*pp; p[0]>='0' && p[0]<='9'; p++){ - nInt = nInt * 10 + (p[0] - '0'); - if( nInt>MAX_NPREFIX ){ - nInt = 0; - break; - } - } - if( p==*pp ) return SQLITE_ERROR; - *pnOut = nInt; - *pp = p; - return SQLITE_OK; +static void fts3SegReaderCursorFree(Fts3MultiSegReader *pSegcsr){ + sqlite3Fts3SegReaderFinish(pSegcsr); + sqlite3_free(pSegcsr); } /* -** This function is called to allocate an array of Fts3Index structures -** representing the indexes maintained by the current FTS table. FTS tables -** always maintain the main "terms" index, but may also maintain one or -** more "prefix" indexes, depending on the value of the "prefix=" parameter -** (if any) specified as part of the CREATE VIRTUAL TABLE statement. -** -** Argument zParam is passed the value of the "prefix=" option if one was -** specified, or NULL otherwise. -** -** If no error occurs, SQLITE_OK is returned and *apIndex set to point to -** the allocated array. *pnIndex is set to the number of elements in the -** array. If an error does occur, an SQLite error code is returned. -** -** Regardless of whether or not an error is returned, it is the responsibility -** of the caller to call sqlite3_free() on the output array to free it. +** This function retrieves the doclist for the specified term (or term +** prefix) from the database. */ -static int fts3PrefixParameter( - const char *zParam, /* ABC in prefix=ABC parameter to parse */ - int *pnIndex, /* OUT: size of *apIndex[] array */ - struct Fts3Index **apIndex /* OUT: Array of indexes for this table */ +static int fts3TermSelect( + Fts3Table *p, /* Virtual table handle */ + Fts3PhraseToken *pTok, /* Token to query for */ + int iColumn, /* Column to query (or -ve for all columns) */ + int *pnOut, /* OUT: Size of buffer at *ppOut */ + char **ppOut /* OUT: Malloced result buffer */ ){ - struct Fts3Index *aIndex; /* Allocated array */ - int nIndex = 1; /* Number of entries in array */ + int rc; /* Return code */ + Fts3MultiSegReader *pSegcsr; /* Seg-reader cursor for this term */ + TermSelect tsc; /* Object for pair-wise doclist merging */ + Fts3SegFilter filter; /* Segment term filter configuration */ - if( zParam && zParam[0] ){ - const char *p; - nIndex++; - for(p=zParam; *p; p++){ - if( *p==',' ) nIndex++; - } - } + pSegcsr = pTok->pSegcsr; + memset(&tsc, 0, sizeof(TermSelect)); + + filter.flags = FTS3_SEGMENT_IGNORE_EMPTY | FTS3_SEGMENT_REQUIRE_POS + | (pTok->isPrefix ? FTS3_SEGMENT_PREFIX : 0) + | (pTok->bFirst ? FTS3_SEGMENT_FIRST : 0) + | (iColumnnColumn ? FTS3_SEGMENT_COLUMN_FILTER : 0); + filter.iCol = iColumn; + filter.zTerm = pTok->z; + filter.nTerm = pTok->n; - aIndex = sqlite3_malloc(sizeof(struct Fts3Index) * nIndex); - *apIndex = aIndex; - if( !aIndex ){ - return SQLITE_NOMEM; + rc = sqlite3Fts3SegReaderStart(p, pSegcsr, &filter); + while( SQLITE_OK==rc + && SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, pSegcsr)) + ){ + rc = fts3TermSelectMerge(p, &tsc, pSegcsr->aDoclist, pSegcsr->nDoclist); } - memset(aIndex, 0, sizeof(struct Fts3Index) * nIndex); - if( zParam ){ - const char *p = zParam; + if( rc==SQLITE_OK ){ + rc = fts3TermSelectFinishMerge(p, &tsc); + } + if( rc==SQLITE_OK ){ + *ppOut = tsc.aaOutput[0]; + *pnOut = tsc.anOutput[0]; + }else{ int i; - for(i=1; i=0 ); - if( nPrefix==0 ){ - nIndex--; - i--; - }else{ - aIndex[i].nPrefix = nPrefix; - } - p++; + for(i=0; ipSegcsr = 0; + return rc; } /* -** This function is called when initializing an FTS4 table that uses the -** content=xxx option. It determines the number of and names of the columns -** of the new FTS4 table. -** -** The third argument passed to this function is the value passed to the -** config=xxx option (i.e. "xxx"). This function queries the database for -** a table of that name. If found, the output variables are populated -** as follows: -** -** *pnCol: Set to the number of columns table xxx has, -** -** *pnStr: Set to the total amount of space required to store a copy -** of each columns name, including the nul-terminator. -** -** *pazCol: Set to point to an array of *pnCol strings. Each string is -** the name of the corresponding column in table xxx. The array -** and its contents are allocated using a single allocation. It -** is the responsibility of the caller to free this allocation -** by eventually passing the *pazCol value to sqlite3_free(). +** This function counts the total number of docids in the doclist stored +** in buffer aList[], size nList bytes. ** -** If the table cannot be found, an error code is returned and the output -** variables are undefined. Or, if an OOM is encountered, SQLITE_NOMEM is -** returned (and the output variables are undefined). +** If the isPoslist argument is true, then it is assumed that the doclist +** contains a position-list following each docid. Otherwise, it is assumed +** that the doclist is simply a list of docids stored as delta encoded +** varints. */ -static int fts3ContentColumns( - sqlite3 *db, /* Database handle */ - const char *zDb, /* Name of db (i.e. "main", "temp" etc.) */ - const char *zTbl, /* Name of content table */ - const char ***pazCol, /* OUT: Malloc'd array of column names */ - int *pnCol, /* OUT: Size of array *pazCol */ - int *pnStr, /* OUT: Bytes of string content */ - char **pzErr /* OUT: error message */ -){ - int rc = SQLITE_OK; /* Return code */ - char *zSql; /* "SELECT *" statement on zTbl */ - sqlite3_stmt *pStmt = 0; /* Compiled version of zSql */ - - zSql = sqlite3_mprintf("SELECT * FROM %Q.%Q", zDb, zTbl); - if( !zSql ){ - rc = SQLITE_NOMEM; - }else{ - rc = sqlite3_prepare(db, zSql, -1, &pStmt, 0); - if( rc!=SQLITE_OK ){ - sqlite3Fts3ErrMsg(pzErr, "%s", sqlite3_errmsg(db)); +static int fts3DoclistCountDocids(char *aList, int nList){ + int nDoc = 0; /* Return value */ + if( aList ){ + char *aEnd = &aList[nList]; /* Pointer to one byte after EOF */ + char *p = aList; /* Cursor */ + while( peSearch==FTS3_DOCID_SEARCH || pCsr->eSearch==FTS3_FULLSCAN_SEARCH ){ + Fts3Table *pTab = (Fts3Table*)pCursor->pVtab; + pTab->bLock++; + if( SQLITE_ROW!=sqlite3_step(pCsr->pStmt) ){ + pCsr->isEof = 1; + rc = sqlite3_reset(pCsr->pStmt); }else{ - char *p = (char *)&azCol[nCol]; - for(i=0; iiPrevId = sqlite3_column_int64(pCsr->pStmt, 0); + rc = SQLITE_OK; } - sqlite3_finalize(pStmt); - - /* Set the output variables. */ - *pnCol = nCol; - *pnStr = nStr; - *pazCol = azCol; + pTab->bLock--; + }else{ + rc = fts3EvalNext((Fts3Cursor *)pCursor); } - + assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); return rc; } /* -** This function is the implementation of both the xConnect and xCreate -** methods of the FTS3 virtual table. +** If the numeric type of argument pVal is "integer", then return it +** converted to a 64-bit signed integer. Otherwise, return a copy of +** the second parameter, iDefault. +*/ +static sqlite3_int64 fts3DocidRange(sqlite3_value *pVal, i64 iDefault){ + if( pVal ){ + int eType = sqlite3_value_numeric_type(pVal); + if( eType==SQLITE_INTEGER ){ + return sqlite3_value_int64(pVal); + } + } + return iDefault; +} + +/* +** This is the xFilter interface for the virtual table. See +** the virtual table xFilter method documentation for additional +** information. ** -** The argv[] array contains the following: +** If idxNum==FTS3_FULLSCAN_SEARCH then do a full table scan against +** the %_content table. ** -** argv[0] -> module name ("fts3" or "fts4") -** argv[1] -> database name -** argv[2] -> table name -** argv[...] -> "column name" and other module argument fields. +** If idxNum==FTS3_DOCID_SEARCH then do a docid lookup for a single entry +** in the %_content table. +** +** If idxNum>=FTS3_FULLTEXT_SEARCH then use the full text index. The +** column on the left-hand side of the MATCH operator is column +** number idxNum-FTS3_FULLTEXT_SEARCH, 0 indexed. argv[0] is the right-hand +** side of the MATCH operator. */ -static int fts3InitVtab( - int isCreate, /* True for xCreate, false for xConnect */ - sqlite3 *db, /* The SQLite database connection */ - void *pAux, /* Hash table containing tokenizers */ - int argc, /* Number of elements in argv array */ - const char * const *argv, /* xCreate/xConnect argument array */ - sqlite3_vtab **ppVTab, /* Write the resulting vtab structure here */ - char **pzErr /* Write any error message here */ +static int fts3FilterMethod( + sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */ + int idxNum, /* Strategy index */ + const char *idxStr, /* Unused */ + int nVal, /* Number of elements in apVal */ + sqlite3_value **apVal /* Arguments for the indexing scheme */ ){ - Fts3Hash *pHash = (Fts3Hash *)pAux; - Fts3Table *p = 0; /* Pointer to allocated vtab */ - int rc = SQLITE_OK; /* Return code */ - int i; /* Iterator variable */ - int nByte; /* Size of allocation used for *p */ - int iCol; /* Column index */ - int nString = 0; /* Bytes required to hold all column names */ - int nCol = 0; /* Number of columns in the FTS table */ - char *zCsr; /* Space for holding column names */ - int nDb; /* Bytes required to hold database name */ - int nName; /* Bytes required to hold table name */ - int isFts4 = (argv[0][3]=='4'); /* True for FTS4, false for FTS3 */ - const char **aCol; /* Array of column names */ - sqlite3_tokenizer *pTokenizer = 0; /* Tokenizer for this table */ - - int nIndex = 0; /* Size of aIndex[] array */ - struct Fts3Index *aIndex = 0; /* Array of indexes for this table */ - - /* The results of parsing supported FTS4 key=value options: */ - int bNoDocsize = 0; /* True to omit %_docsize table */ - int bDescIdx = 0; /* True to store descending indexes */ - char *zPrefix = 0; /* Prefix parameter value (or NULL) */ - char *zCompress = 0; /* compress=? parameter (or NULL) */ - char *zUncompress = 0; /* uncompress=? parameter (or NULL) */ - char *zContent = 0; /* content=? parameter (or NULL) */ - char *zLanguageid = 0; /* languageid=? parameter (or NULL) */ - char **azNotindexed = 0; /* The set of notindexed= columns */ - int nNotindexed = 0; /* Size of azNotindexed[] array */ + int rc = SQLITE_OK; + char *zSql; /* SQL statement used to access %_content */ + int eSearch; + Fts3Table *p = (Fts3Table *)pCursor->pVtab; + Fts3Cursor *pCsr = (Fts3Cursor *)pCursor; - assert( strlen(argv[0])==4 ); - assert( (sqlite3_strnicmp(argv[0], "fts4", 4)==0 && isFts4) - || (sqlite3_strnicmp(argv[0], "fts3", 4)==0 && !isFts4) - ); + sqlite3_value *pCons = 0; /* The MATCH or rowid constraint, if any */ + sqlite3_value *pLangid = 0; /* The "langid = ?" constraint, if any */ + sqlite3_value *pDocidGe = 0; /* The "docid >= ?" constraint, if any */ + sqlite3_value *pDocidLe = 0; /* The "docid <= ?" constraint, if any */ + int iIdx; - nDb = (int)strlen(argv[1]) + 1; - nName = (int)strlen(argv[2]) + 1; + UNUSED_PARAMETER(idxStr); + UNUSED_PARAMETER(nVal); - nByte = sizeof(const char *) * (argc-2); - aCol = (const char **)sqlite3_malloc(nByte); - if( aCol ){ - memset((void*)aCol, 0, nByte); - azNotindexed = (char **)sqlite3_malloc(nByte); - } - if( azNotindexed ){ - memset(azNotindexed, 0, nByte); - } - if( !aCol || !azNotindexed ){ - rc = SQLITE_NOMEM; - goto fts3_init_out; + if( p->bLock ){ + return SQLITE_ERROR; } - /* Loop through all of the arguments passed by the user to the FTS3/4 - ** module (i.e. all the column names and special arguments). This loop - ** does the following: - ** - ** + Figures out the number of columns the FTSX table will have, and - ** the number of bytes of space that must be allocated to store copies - ** of the column names. - ** - ** + If there is a tokenizer specification included in the arguments, - ** initializes the tokenizer pTokenizer. - */ - for(i=3; rc==SQLITE_OK && i8 - && 0==sqlite3_strnicmp(z, "tokenize", 8) - && 0==sqlite3Fts3IsIdChar(z[8]) - ){ - rc = sqlite3Fts3InitTokenizer(pHash, &z[9], &pTokenizer, pzErr); - } - - /* Check if it is an FTS4 special argument. */ - else if( isFts4 && fts3IsSpecialColumn(z, &nKey, &zVal) ){ - struct Fts4Option { - const char *zOpt; - int nOpt; - } aFts4Opt[] = { - { "matchinfo", 9 }, /* 0 -> MATCHINFO */ - { "prefix", 6 }, /* 1 -> PREFIX */ - { "compress", 8 }, /* 2 -> COMPRESS */ - { "uncompress", 10 }, /* 3 -> UNCOMPRESS */ - { "order", 5 }, /* 4 -> ORDER */ - { "content", 7 }, /* 5 -> CONTENT */ - { "languageid", 10 }, /* 6 -> LANGUAGEID */ - { "notindexed", 10 } /* 7 -> NOTINDEXED */ - }; - - int iOpt; - if( !zVal ){ - rc = SQLITE_NOMEM; - }else{ - for(iOpt=0; iOptnOpt && !sqlite3_strnicmp(z, pOp->zOpt, pOp->nOpt) ){ - break; - } - } - switch( iOpt ){ - case 0: /* MATCHINFO */ - if( strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "fts3", 4) ){ - sqlite3Fts3ErrMsg(pzErr, "unrecognized matchinfo: %s", zVal); - rc = SQLITE_ERROR; - } - bNoDocsize = 1; - break; + eSearch = (idxNum & 0x0000FFFF); + assert( eSearch>=0 && eSearch<=(FTS3_FULLTEXT_SEARCH+p->nColumn) ); + assert( p->pSegments==0 ); - case 1: /* PREFIX */ - sqlite3_free(zPrefix); - zPrefix = zVal; - zVal = 0; - break; + /* Collect arguments into local variables */ + iIdx = 0; + if( eSearch!=FTS3_FULLSCAN_SEARCH ) pCons = apVal[iIdx++]; + if( idxNum & FTS3_HAVE_LANGID ) pLangid = apVal[iIdx++]; + if( idxNum & FTS3_HAVE_DOCID_GE ) pDocidGe = apVal[iIdx++]; + if( idxNum & FTS3_HAVE_DOCID_LE ) pDocidLe = apVal[iIdx++]; + assert( iIdx==nVal ); - case 2: /* COMPRESS */ - sqlite3_free(zCompress); - zCompress = zVal; - zVal = 0; - break; + /* In case the cursor has been used before, clear it now. */ + fts3ClearCursor(pCsr); - case 3: /* UNCOMPRESS */ - sqlite3_free(zUncompress); - zUncompress = zVal; - zVal = 0; - break; + /* Set the lower and upper bounds on docids to return */ + pCsr->iMinDocid = fts3DocidRange(pDocidGe, SMALLEST_INT64); + pCsr->iMaxDocid = fts3DocidRange(pDocidLe, LARGEST_INT64); - case 4: /* ORDER */ - if( (strlen(zVal)!=3 || sqlite3_strnicmp(zVal, "asc", 3)) - && (strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "desc", 4)) - ){ - sqlite3Fts3ErrMsg(pzErr, "unrecognized order: %s", zVal); - rc = SQLITE_ERROR; - } - bDescIdx = (zVal[0]=='d' || zVal[0]=='D'); - break; + if( idxStr ){ + pCsr->bDesc = (idxStr[0]=='D'); + }else{ + pCsr->bDesc = p->bDescIdx; + } + pCsr->eSearch = (i16)eSearch; - case 5: /* CONTENT */ - sqlite3_free(zContent); - zContent = zVal; - zVal = 0; - break; + if( eSearch!=FTS3_DOCID_SEARCH && eSearch!=FTS3_FULLSCAN_SEARCH ){ + int iCol = eSearch-FTS3_FULLTEXT_SEARCH; + const char *zQuery = (const char *)sqlite3_value_text(pCons); - case 6: /* LANGUAGEID */ - assert( iOpt==6 ); - sqlite3_free(zLanguageid); - zLanguageid = zVal; - zVal = 0; - break; + if( zQuery==0 && sqlite3_value_type(pCons)!=SQLITE_NULL ){ + return SQLITE_NOMEM; + } - case 7: /* NOTINDEXED */ - azNotindexed[nNotindexed++] = zVal; - zVal = 0; - break; + pCsr->iLangid = 0; + if( pLangid ) pCsr->iLangid = sqlite3_value_int(pLangid); - default: - assert( iOpt==SizeofArray(aFts4Opt) ); - sqlite3Fts3ErrMsg(pzErr, "unrecognized parameter: %s", z); - rc = SQLITE_ERROR; - break; - } - sqlite3_free(zVal); - } + assert( p->base.zErrMsg==0 ); + rc = sqlite3Fts3ExprParse(p->pTokenizer, pCsr->iLangid, + p->azColumn, p->bFts4, p->nColumn, iCol, zQuery, -1, &pCsr->pExpr, + &p->base.zErrMsg + ); + if( rc!=SQLITE_OK ){ + return rc; } - /* Otherwise, the argument is a column name. */ - else { - nString += (int)(strlen(z) + 1); - aCol[nCol++] = z; - } + rc = fts3EvalStart(pCsr); + sqlite3Fts3SegmentsClose(p); + if( rc!=SQLITE_OK ) return rc; + pCsr->pNextId = pCsr->aDoclist; + pCsr->iPrevId = 0; } - /* If a content=xxx option was specified, the following: - ** - ** 1. Ignore any compress= and uncompress= options. - ** - ** 2. If no column names were specified as part of the CREATE VIRTUAL - ** TABLE statement, use all columns from the content table. + /* Compile a SELECT statement for this cursor. For a full-table-scan, the + ** statement loops through all rows of the %_content table. For a + ** full-text query or docid lookup, the statement retrieves a single + ** row by docid. */ - if( rc==SQLITE_OK && zContent ){ - sqlite3_free(zCompress); - sqlite3_free(zUncompress); - zCompress = 0; - zUncompress = 0; - if( nCol==0 ){ - sqlite3_free((void*)aCol); - aCol = 0; - rc = fts3ContentColumns(db, argv[1], zContent,&aCol,&nCol,&nString,pzErr); - - /* If a languageid= option was specified, remove the language id - ** column from the aCol[] array. */ - if( rc==SQLITE_OK && zLanguageid ){ - int j; - for(j=0; jzReadExprlist, pCsr->iMinDocid, pCsr->iMaxDocid, + (pCsr->bDesc ? "DESC" : "ASC") + ); + }else{ + zSql = sqlite3_mprintf("SELECT %s ORDER BY rowid %s", + p->zReadExprlist, (pCsr->bDesc ? "DESC" : "ASC") + ); + } + if( zSql ){ + p->bLock++; + rc = sqlite3_prepare_v3( + p->db,zSql,-1,SQLITE_PREPARE_PERSISTENT,&pCsr->pStmt,0 + ); + p->bLock--; + sqlite3_free(zSql); + }else{ + rc = SQLITE_NOMEM; + } + }else if( eSearch==FTS3_DOCID_SEARCH ){ + rc = fts3CursorSeekStmt(pCsr); + if( rc==SQLITE_OK ){ + rc = sqlite3_bind_value(pCsr->pStmt, 1, pCons); } } - if( rc!=SQLITE_OK ) goto fts3_init_out; + if( rc!=SQLITE_OK ) return rc; - if( nCol==0 ){ - assert( nString==0 ); - aCol[0] = "content"; - nString = 8; - nCol = 1; - } + return fts3NextMethod(pCursor); +} - if( pTokenizer==0 ){ - rc = sqlite3Fts3InitTokenizer(pHash, "simple", &pTokenizer, pzErr); - if( rc!=SQLITE_OK ) goto fts3_init_out; +/* +** This is the xEof method of the virtual table. SQLite calls this +** routine to find out if it has reached the end of a result set. +*/ +static int fts3EofMethod(sqlite3_vtab_cursor *pCursor){ + Fts3Cursor *pCsr = (Fts3Cursor*)pCursor; + if( pCsr->isEof ){ + fts3ClearCursor(pCsr); + pCsr->isEof = 1; } - assert( pTokenizer ); + return pCsr->isEof; +} - rc = fts3PrefixParameter(zPrefix, &nIndex, &aIndex); - if( rc==SQLITE_ERROR ){ - assert( zPrefix ); - sqlite3Fts3ErrMsg(pzErr, "error parsing prefix parameter: %s", zPrefix); - } - if( rc!=SQLITE_OK ) goto fts3_init_out; +/* +** This is the xRowid method. The SQLite core calls this routine to +** retrieve the rowid for the current row of the result set. fts3 +** exposes %_content.docid as the rowid for the virtual table. The +** rowid should be written to *pRowid. +*/ +static int fts3RowidMethod(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ + Fts3Cursor *pCsr = (Fts3Cursor *) pCursor; + *pRowid = pCsr->iPrevId; + return SQLITE_OK; +} - /* Allocate and populate the Fts3Table structure. */ - nByte = sizeof(Fts3Table) + /* Fts3Table */ - nCol * sizeof(char *) + /* azColumn */ - nIndex * sizeof(struct Fts3Index) + /* aIndex */ - nCol * sizeof(u8) + /* abNotindexed */ - nName + /* zName */ - nDb + /* zDb */ - nString; /* Space for azColumn strings */ - p = (Fts3Table*)sqlite3_malloc(nByte); - if( p==0 ){ - rc = SQLITE_NOMEM; - goto fts3_init_out; - } - memset(p, 0, nByte); - p->db = db; - p->nColumn = nCol; - p->nPendingData = 0; - p->azColumn = (char **)&p[1]; - p->pTokenizer = pTokenizer; - p->nMaxPendingData = FTS3_MAX_PENDING_DATA; - p->bHasDocsize = (isFts4 && bNoDocsize==0); - p->bHasStat = (u8)isFts4; - p->bFts4 = (u8)isFts4; - p->bDescIdx = (u8)bDescIdx; - p->nAutoincrmerge = 0xff; /* 0xff means setting unknown */ - p->zContentTbl = zContent; - p->zLanguageid = zLanguageid; - zContent = 0; - zLanguageid = 0; - TESTONLY( p->inTransaction = -1 ); - TESTONLY( p->mxSavepoint = -1 ); +/* +** This is the xColumn method, called by SQLite to request a value from +** the row that the supplied cursor currently points to. +** +** If: +** +** (iCol < p->nColumn) -> The value of the iCol'th user column. +** (iCol == p->nColumn) -> Magic column with the same name as the table. +** (iCol == p->nColumn+1) -> Docid column +** (iCol == p->nColumn+2) -> Langid column +*/ +static int fts3ColumnMethod( + sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ + sqlite3_context *pCtx, /* Context for sqlite3_result_xxx() calls */ + int iCol /* Index of column to read value from */ +){ + int rc = SQLITE_OK; /* Return Code */ + Fts3Cursor *pCsr = (Fts3Cursor *) pCursor; + Fts3Table *p = (Fts3Table *)pCursor->pVtab; - p->aIndex = (struct Fts3Index *)&p->azColumn[nCol]; - memcpy(p->aIndex, aIndex, sizeof(struct Fts3Index) * nIndex); - p->nIndex = nIndex; - for(i=0; iaIndex[i].hPending, FTS3_HASH_STRING, 1); - } - p->abNotindexed = (u8 *)&p->aIndex[nIndex]; + /* The column value supplied by SQLite must be in range. */ + assert( iCol>=0 && iCol<=p->nColumn+2 ); - /* Fill in the zName and zDb fields of the vtab structure. */ - zCsr = (char *)&p->abNotindexed[nCol]; - p->zName = zCsr; - memcpy(zCsr, argv[2], nName); - zCsr += nName; - p->zDb = zCsr; - memcpy(zCsr, argv[1], nDb); - zCsr += nDb; + switch( iCol-p->nColumn ){ + case 0: + /* The special 'table-name' column */ + sqlite3_result_pointer(pCtx, pCsr, "fts3cursor", 0); + break; - /* Fill in the azColumn array */ - for(iCol=0; iCol0 ){ - memcpy(zCsr, z, n); - } - zCsr[n] = '\0'; - sqlite3Fts3Dequote(zCsr); - p->azColumn[iCol] = zCsr; - zCsr += n+1; - assert( zCsr <= &((char *)p)[nByte] ); - } + case 1: + /* The docid column */ + sqlite3_result_int64(pCtx, pCsr->iPrevId); + break; - /* Fill in the abNotindexed array */ - for(iCol=0; iColazColumn[iCol]); - for(i=0; iazColumn[iCol], zNot, n) - ){ - p->abNotindexed[iCol] = 1; - sqlite3_free(zNot); - azNotindexed[i] = 0; + case 2: + if( pCsr->pExpr ){ + sqlite3_result_int64(pCtx, pCsr->iLangid); + break; + }else if( p->zLanguageid==0 ){ + sqlite3_result_int(pCtx, 0); + break; + }else{ + iCol = p->nColumn; + /* fall-through */ } - } - } - for(i=0; ipStmt)-1>iCol ){ + sqlite3_result_value(pCtx, sqlite3_column_value(pCsr->pStmt, iCol+1)); + } + break; } - p->zReadExprlist = fts3ReadExprList(p, zUncompress, &rc); - p->zWriteExprlist = fts3WriteExprList(p, zCompress, &rc); - if( rc!=SQLITE_OK ) goto fts3_init_out; - /* If this is an xCreate call, create the underlying tables in the - ** database. TODO: For xConnect(), it could verify that said tables exist. - */ - if( isCreate ){ - rc = fts3CreateTables(p); - } + assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); + return rc; +} - /* Check to see if a legacy fts3 table has been "upgraded" by the - ** addition of a %_stat table so that it can use incremental merge. +/* +** This function is the implementation of the xUpdate callback used by +** FTS3 virtual tables. It is invoked by SQLite each time a row is to be +** inserted, updated or deleted. +*/ +static int fts3UpdateMethod( + sqlite3_vtab *pVtab, /* Virtual table handle */ + int nArg, /* Size of argument array */ + sqlite3_value **apVal, /* Array of arguments */ + sqlite_int64 *pRowid /* OUT: The affected (or effected) rowid */ +){ + return sqlite3Fts3UpdateMethod(pVtab, nArg, apVal, pRowid); +} + +/* +** Implementation of xSync() method. Flush the contents of the pending-terms +** hash-table to the database. +*/ +static int fts3SyncMethod(sqlite3_vtab *pVtab){ + + /* Following an incremental-merge operation, assuming that the input + ** segments are not completely consumed (the usual case), they are updated + ** in place to remove the entries that have already been merged. This + ** involves updating the leaf block that contains the smallest unmerged + ** entry and each block (if any) between the leaf and the root node. So + ** if the height of the input segment b-trees is N, and input segments + ** are merged eight at a time, updating the input segments at the end + ** of an incremental-merge requires writing (8*(1+N)) blocks. N is usually + ** small - often between 0 and 2. So the overhead of the incremental + ** merge is somewhere between 8 and 24 blocks. To avoid this overhead + ** dwarfing the actual productive work accomplished, the incremental merge + ** is only attempted if it will write at least 64 leaf blocks. Hence + ** nMinMerge. + ** + ** Of course, updating the input segments also involves deleting a bunch + ** of blocks from the segments table. But this is not considered overhead + ** as it would also be required by a crisis-merge that used the same input + ** segments. */ - if( !isFts4 && !isCreate ){ - p->bHasStat = 2; - } + const u32 nMinMerge = 64; /* Minimum amount of incr-merge work to do */ - /* Figure out the page-size for the database. This is required in order to - ** estimate the cost of loading large doclists from the database. */ - fts3DatabasePageSize(&rc, p); - p->nNodeSize = p->nPgsz-35; + Fts3Table *p = (Fts3Table*)pVtab; + int rc; + i64 iLastRowid = sqlite3_last_insert_rowid(p->db); - /* Declare the table schema to SQLite. */ - fts3DeclareVtab(&rc, p); + rc = sqlite3Fts3PendingTermsFlush(p); + if( rc==SQLITE_OK + && p->nLeafAdd>(nMinMerge/16) + && p->nAutoincrmerge && p->nAutoincrmerge!=0xff + ){ + int mxLevel = 0; /* Maximum relative level value in db */ + int A; /* Incr-merge parameter A */ -fts3_init_out: - sqlite3_free(zPrefix); - sqlite3_free(aIndex); - sqlite3_free(zCompress); - sqlite3_free(zUncompress); - sqlite3_free(zContent); - sqlite3_free(zLanguageid); - for(i=0; ipModule->xDestroy(pTokenizer); + rc = sqlite3Fts3MaxLevel(p, &mxLevel); + assert( rc==SQLITE_OK || mxLevel==0 ); + A = p->nLeafAdd * mxLevel; + A += (A/2); + if( A>(int)nMinMerge ) rc = sqlite3Fts3Incrmerge(p, A, p->nAutoincrmerge); + } + sqlite3Fts3SegmentsClose(p); + sqlite3_set_last_insert_rowid(p->db, iLastRowid); + return rc; +} + +/* +** If it is currently unknown whether or not the FTS table has an %_stat +** table (if p->bHasStat==2), attempt to determine this (set p->bHasStat +** to 0 or 1). Return SQLITE_OK if successful, or an SQLite error code +** if an error occurs. +*/ +static int fts3SetHasStat(Fts3Table *p){ + int rc = SQLITE_OK; + if( p->bHasStat==2 ){ + char *zTbl = sqlite3_mprintf("%s_stat", p->zName); + if( zTbl ){ + int res = sqlite3_table_column_metadata(p->db, p->zDb, zTbl, 0,0,0,0,0,0); + sqlite3_free(zTbl); + p->bHasStat = (res==SQLITE_OK); + }else{ + rc = SQLITE_NOMEM; } - }else{ - assert( p->pSegments==0 ); - *ppVTab = &p->base; } return rc; } /* -** The xConnect() and xCreate() methods for the virtual table. All the -** work is done in function fts3InitVtab(). +** Implementation of xBegin() method. */ -static int fts3ConnectMethod( - sqlite3 *db, /* Database connection */ - void *pAux, /* Pointer to tokenizer hash table */ - int argc, /* Number of elements in argv array */ - const char * const *argv, /* xCreate/xConnect argument array */ - sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */ - char **pzErr /* OUT: sqlite3_malloc'd error message */ -){ - return fts3InitVtab(0, db, pAux, argc, argv, ppVtab, pzErr); +static int fts3BeginMethod(sqlite3_vtab *pVtab){ + Fts3Table *p = (Fts3Table*)pVtab; + UNUSED_PARAMETER(pVtab); + assert( p->pSegments==0 ); + assert( p->nPendingData==0 ); + assert( p->inTransaction!=1 ); + TESTONLY( p->inTransaction = 1 ); + TESTONLY( p->mxSavepoint = -1; ); + p->nLeafAdd = 0; + return fts3SetHasStat(p); } -static int fts3CreateMethod( - sqlite3 *db, /* Database connection */ - void *pAux, /* Pointer to tokenizer hash table */ - int argc, /* Number of elements in argv array */ - const char * const *argv, /* xCreate/xConnect argument array */ - sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */ - char **pzErr /* OUT: sqlite3_malloc'd error message */ -){ - return fts3InitVtab(1, db, pAux, argc, argv, ppVtab, pzErr); + +/* +** Implementation of xCommit() method. This is a no-op. The contents of +** the pending-terms hash-table have already been flushed into the database +** by fts3SyncMethod(). +*/ +static int fts3CommitMethod(sqlite3_vtab *pVtab){ + TESTONLY( Fts3Table *p = (Fts3Table*)pVtab ); + UNUSED_PARAMETER(pVtab); + assert( p->nPendingData==0 ); + assert( p->inTransaction!=0 ); + assert( p->pSegments==0 ); + TESTONLY( p->inTransaction = 0 ); + TESTONLY( p->mxSavepoint = -1; ); + return SQLITE_OK; } /* -** Set the pIdxInfo->estimatedRows variable to nRow. Unless this -** extension is currently being used by a version of SQLite too old to -** support estimatedRows. In that case this function is a no-op. +** Implementation of xRollback(). Discard the contents of the pending-terms +** hash-table. Any changes made to the database are reverted by SQLite. */ -static void fts3SetEstimatedRows(sqlite3_index_info *pIdxInfo, i64 nRow){ -#if SQLITE_VERSION_NUMBER>=3008002 - if( sqlite3_libversion_number()>=3008002 ){ - pIdxInfo->estimatedRows = nRow; - } -#endif +static int fts3RollbackMethod(sqlite3_vtab *pVtab){ + Fts3Table *p = (Fts3Table*)pVtab; + sqlite3Fts3PendingTermsClear(p); + assert( p->inTransaction!=0 ); + TESTONLY( p->inTransaction = 0 ); + TESTONLY( p->mxSavepoint = -1; ); + return SQLITE_OK; } /* -** Set the SQLITE_INDEX_SCAN_UNIQUE flag in pIdxInfo->flags. Unless this -** extension is currently being used by a version of SQLite too old to -** support index-info flags. In that case this function is a no-op. +** When called, *ppPoslist must point to the byte immediately following the +** end of a position-list. i.e. ( (*ppPoslist)[-1]==POS_END ). This function +** moves *ppPoslist so that it instead points to the first byte of the +** same position list. */ -static void fts3SetUniqueFlag(sqlite3_index_info *pIdxInfo){ -#if SQLITE_VERSION_NUMBER>=3008012 - if( sqlite3_libversion_number()>=3008012 ){ - pIdxInfo->idxFlags |= SQLITE_INDEX_SCAN_UNIQUE; +static void fts3ReversePoslist(char *pStart, char **ppPoslist){ + char *p = &(*ppPoslist)[-2]; + char c = 0; + + /* Skip backwards passed any trailing 0x00 bytes added by NearTrim() */ + while( p>pStart && (c=*p--)==0 ); + + /* Search backwards for a varint with value zero (the end of the previous + ** poslist). This is an 0x00 byte preceded by some byte that does not + ** have the 0x80 bit set. */ + while( p>pStart && (*p & 0x80) | c ){ + c = *p--; } -#endif + assert( p==pStart || c==0 ); + + /* At this point p points to that preceding byte without the 0x80 bit + ** set. So to find the start of the poslist, skip forward 2 bytes then + ** over a varint. + ** + ** Normally. The other case is that p==pStart and the poslist to return + ** is the first in the doclist. In this case do not skip forward 2 bytes. + ** The second part of the if condition (c==0 && *ppPoslist>&p[2]) + ** is required for cases where the first byte of a doclist and the + ** doclist is empty. For example, if the first docid is 10, a doclist + ** that begins with: + ** + ** 0x0A 0x00 + */ + if( p>pStart || (c==0 && *ppPoslist>&p[2]) ){ p = &p[2]; } + while( *p++&0x80 ); + *ppPoslist = p; } -/* -** Implementation of the xBestIndex method for FTS3 tables. There -** are three possible strategies, in order of preference: +/* +** Helper function used by the implementation of the overloaded snippet(), +** offsets() and optimize() SQL functions. ** -** 1. Direct lookup by rowid or docid. -** 2. Full-text search using a MATCH operator on a non-docid column. -** 3. Linear scan of %_content table. +** If the value passed as the third argument is a blob of size +** sizeof(Fts3Cursor*), then the blob contents are copied to the +** output variable *ppCsr and SQLITE_OK is returned. Otherwise, an error +** message is written to context pContext and SQLITE_ERROR returned. The +** string passed via zFunc is used as part of the error message. */ -static int fts3BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ - Fts3Table *p = (Fts3Table *)pVTab; - int i; /* Iterator variable */ - int iCons = -1; /* Index of constraint to use */ +static int fts3FunctionArg( + sqlite3_context *pContext, /* SQL function call context */ + const char *zFunc, /* Function name */ + sqlite3_value *pVal, /* argv[0] passed to function */ + Fts3Cursor **ppCsr /* OUT: Store cursor handle here */ +){ + int rc; + *ppCsr = (Fts3Cursor*)sqlite3_value_pointer(pVal, "fts3cursor"); + if( (*ppCsr)!=0 ){ + rc = SQLITE_OK; + }else{ + char *zErr = sqlite3_mprintf("illegal first argument to %s", zFunc); + sqlite3_result_error(pContext, zErr, -1); + sqlite3_free(zErr); + rc = SQLITE_ERROR; + } + return rc; +} - int iLangidCons = -1; /* Index of langid=x constraint, if present */ - int iDocidGe = -1; /* Index of docid>=x constraint, if present */ - int iDocidLe = -1; /* Index of docid<=x constraint, if present */ - int iIdx; +/* +** Implementation of the snippet() function for FTS3 +*/ +static void fts3SnippetFunc( + sqlite3_context *pContext, /* SQLite function call context */ + int nVal, /* Size of apVal[] array */ + sqlite3_value **apVal /* Array of arguments */ +){ + Fts3Cursor *pCsr; /* Cursor handle passed through apVal[0] */ + const char *zStart = ""; + const char *zEnd = ""; + const char *zEllipsis = "..."; + int iCol = -1; + int nToken = 15; /* Default number of tokens in snippet */ - /* By default use a full table scan. This is an expensive option, - ** so search through the constraints to see if a more efficient - ** strategy is possible. + /* There must be at least one argument passed to this function (otherwise + ** the non-overloaded version would have been called instead of this one). */ - pInfo->idxNum = FTS3_FULLSCAN_SEARCH; - pInfo->estimatedCost = 5000000; - for(i=0; inConstraint; i++){ - int bDocid; /* True if this constraint is on docid */ - struct sqlite3_index_constraint *pCons = &pInfo->aConstraint[i]; - if( pCons->usable==0 ){ - if( pCons->op==SQLITE_INDEX_CONSTRAINT_MATCH ){ - /* There exists an unusable MATCH constraint. This means that if - ** the planner does elect to use the results of this call as part - ** of the overall query plan the user will see an "unable to use - ** function MATCH in the requested context" error. To discourage - ** this, return a very high cost here. */ - pInfo->idxNum = FTS3_FULLSCAN_SEARCH; - pInfo->estimatedCost = 1e50; - fts3SetEstimatedRows(pInfo, ((sqlite3_int64)1) << 50); - return SQLITE_OK; - } - continue; - } + assert( nVal>=1 ); - bDocid = (pCons->iColumn<0 || pCons->iColumn==p->nColumn+1); + if( nVal>6 ){ + sqlite3_result_error(pContext, + "wrong number of arguments to function snippet()", -1); + return; + } + if( fts3FunctionArg(pContext, "snippet", apVal[0], &pCsr) ) return; - /* A direct lookup on the rowid or docid column. Assign a cost of 1.0. */ - if( iCons<0 && pCons->op==SQLITE_INDEX_CONSTRAINT_EQ && bDocid ){ - pInfo->idxNum = FTS3_DOCID_SEARCH; - pInfo->estimatedCost = 1.0; - iCons = i; - } + switch( nVal ){ + case 6: nToken = sqlite3_value_int(apVal[5]); + case 5: iCol = sqlite3_value_int(apVal[4]); + case 4: zEllipsis = (const char*)sqlite3_value_text(apVal[3]); + case 3: zEnd = (const char*)sqlite3_value_text(apVal[2]); + case 2: zStart = (const char*)sqlite3_value_text(apVal[1]); + } + if( !zEllipsis || !zEnd || !zStart ){ + sqlite3_result_error_nomem(pContext); + }else if( nToken==0 ){ + sqlite3_result_text(pContext, "", -1, SQLITE_STATIC); + }else if( SQLITE_OK==fts3CursorSeek(pContext, pCsr) ){ + sqlite3Fts3Snippet(pContext, pCsr, zStart, zEnd, zEllipsis, iCol, nToken); + } +} - /* A MATCH constraint. Use a full-text search. - ** - ** If there is more than one MATCH constraint available, use the first - ** one encountered. If there is both a MATCH constraint and a direct - ** rowid/docid lookup, prefer the MATCH strategy. This is done even - ** though the rowid/docid lookup is faster than a MATCH query, selecting - ** it would lead to an "unable to use function MATCH in the requested - ** context" error. - */ - if( pCons->op==SQLITE_INDEX_CONSTRAINT_MATCH - && pCons->iColumn>=0 && pCons->iColumn<=p->nColumn - ){ - pInfo->idxNum = FTS3_FULLTEXT_SEARCH + pCons->iColumn; - pInfo->estimatedCost = 2.0; - iCons = i; - } +/* +** Implementation of the offsets() function for FTS3 +*/ +static void fts3OffsetsFunc( + sqlite3_context *pContext, /* SQLite function call context */ + int nVal, /* Size of argument array */ + sqlite3_value **apVal /* Array of arguments */ +){ + Fts3Cursor *pCsr; /* Cursor handle passed through apVal[0] */ + + UNUSED_PARAMETER(nVal); + + assert( nVal==1 ); + if( fts3FunctionArg(pContext, "offsets", apVal[0], &pCsr) ) return; + assert( pCsr ); + if( SQLITE_OK==fts3CursorSeek(pContext, pCsr) ){ + sqlite3Fts3Offsets(pContext, pCsr); + } +} + +/* +** Implementation of the special optimize() function for FTS3. This +** function merges all segments in the database to a single segment. +** Example usage is: +** +** SELECT optimize(t) FROM t LIMIT 1; +** +** where 't' is the name of an FTS3 table. +*/ +static void fts3OptimizeFunc( + sqlite3_context *pContext, /* SQLite function call context */ + int nVal, /* Size of argument array */ + sqlite3_value **apVal /* Array of arguments */ +){ + int rc; /* Return code */ + Fts3Table *p; /* Virtual table handle */ + Fts3Cursor *pCursor; /* Cursor handle passed through apVal[0] */ + + UNUSED_PARAMETER(nVal); + + assert( nVal==1 ); + if( fts3FunctionArg(pContext, "optimize", apVal[0], &pCursor) ) return; + p = (Fts3Table *)pCursor->base.pVtab; + assert( p ); - /* Equality constraint on the langid column */ - if( pCons->op==SQLITE_INDEX_CONSTRAINT_EQ - && pCons->iColumn==p->nColumn + 2 - ){ - iLangidCons = i; - } + rc = sqlite3Fts3Optimize(p); - if( bDocid ){ - switch( pCons->op ){ - case SQLITE_INDEX_CONSTRAINT_GE: - case SQLITE_INDEX_CONSTRAINT_GT: - iDocidGe = i; - break; + switch( rc ){ + case SQLITE_OK: + sqlite3_result_text(pContext, "Index optimized", -1, SQLITE_STATIC); + break; + case SQLITE_DONE: + sqlite3_result_text(pContext, "Index already optimal", -1, SQLITE_STATIC); + break; + default: + sqlite3_result_error_code(pContext, rc); + break; + } +} - case SQLITE_INDEX_CONSTRAINT_LE: - case SQLITE_INDEX_CONSTRAINT_LT: - iDocidLe = i; - break; - } +/* +** Implementation of the matchinfo() function for FTS3 +*/ +static void fts3MatchinfoFunc( + sqlite3_context *pContext, /* SQLite function call context */ + int nVal, /* Size of argument array */ + sqlite3_value **apVal /* Array of arguments */ +){ + Fts3Cursor *pCsr; /* Cursor handle passed through apVal[0] */ + assert( nVal==1 || nVal==2 ); + if( SQLITE_OK==fts3FunctionArg(pContext, "matchinfo", apVal[0], &pCsr) ){ + const char *zArg = 0; + if( nVal>1 ){ + zArg = (const char *)sqlite3_value_text(apVal[1]); } + sqlite3Fts3Matchinfo(pContext, pCsr, zArg); } +} - /* If using a docid=? or rowid=? strategy, set the UNIQUE flag. */ - if( pInfo->idxNum==FTS3_DOCID_SEARCH ) fts3SetUniqueFlag(pInfo); +/* +** This routine implements the xFindFunction method for the FTS3 +** virtual table. +*/ +static int fts3FindFunctionMethod( + sqlite3_vtab *pVtab, /* Virtual table handle */ + int nArg, /* Number of SQL function arguments */ + const char *zName, /* Name of SQL function */ + void (**pxFunc)(sqlite3_context*,int,sqlite3_value**), /* OUT: Result */ + void **ppArg /* Unused */ +){ + struct Overloaded { + const char *zName; + void (*xFunc)(sqlite3_context*,int,sqlite3_value**); + } aOverload[] = { + { "snippet", fts3SnippetFunc }, + { "offsets", fts3OffsetsFunc }, + { "optimize", fts3OptimizeFunc }, + { "matchinfo", fts3MatchinfoFunc }, + }; + int i; /* Iterator variable */ - iIdx = 1; - if( iCons>=0 ){ - pInfo->aConstraintUsage[iCons].argvIndex = iIdx++; - pInfo->aConstraintUsage[iCons].omit = 1; - } - if( iLangidCons>=0 ){ - pInfo->idxNum |= FTS3_HAVE_LANGID; - pInfo->aConstraintUsage[iLangidCons].argvIndex = iIdx++; - } - if( iDocidGe>=0 ){ - pInfo->idxNum |= FTS3_HAVE_DOCID_GE; - pInfo->aConstraintUsage[iDocidGe].argvIndex = iIdx++; - } - if( iDocidLe>=0 ){ - pInfo->idxNum |= FTS3_HAVE_DOCID_LE; - pInfo->aConstraintUsage[iDocidLe].argvIndex = iIdx++; - } + UNUSED_PARAMETER(pVtab); + UNUSED_PARAMETER(nArg); + UNUSED_PARAMETER(ppArg); - /* Regardless of the strategy selected, FTS can deliver rows in rowid (or - ** docid) order. Both ascending and descending are possible. - */ - if( pInfo->nOrderBy==1 ){ - struct sqlite3_index_orderby *pOrder = &pInfo->aOrderBy[0]; - if( pOrder->iColumn<0 || pOrder->iColumn==p->nColumn+1 ){ - if( pOrder->desc ){ - pInfo->idxStr = "DESC"; - }else{ - pInfo->idxStr = "ASC"; - } - pInfo->orderByConsumed = 1; + for(i=0; ipSegments==0 ); - return SQLITE_OK; + /* No function of the specified name was found. Return 0. */ + return 0; } /* -** Implementation of xOpen method. +** Implementation of FTS3 xRename method. Rename an fts3 table. */ -static int fts3OpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){ - sqlite3_vtab_cursor *pCsr; /* Allocated cursor */ - - UNUSED_PARAMETER(pVTab); +static int fts3RenameMethod( + sqlite3_vtab *pVtab, /* Virtual table handle */ + const char *zName /* New name of table */ +){ + Fts3Table *p = (Fts3Table *)pVtab; + sqlite3 *db = p->db; /* Database connection */ + int rc; /* Return Code */ - /* Allocate a buffer large enough for an Fts3Cursor structure. If the - ** allocation succeeds, zero it and return SQLITE_OK. Otherwise, - ** if the allocation fails, return SQLITE_NOMEM. + /* At this point it must be known if the %_stat table exists or not. + ** So bHasStat may not be 2. */ + rc = fts3SetHasStat(p); + + /* As it happens, the pending terms table is always empty here. This is + ** because an "ALTER TABLE RENAME TABLE" statement inside a transaction + ** always opens a savepoint transaction. And the xSavepoint() method + ** flushes the pending terms table. But leave the (no-op) call to + ** PendingTermsFlush() in in case that changes. */ - *ppCsr = pCsr = (sqlite3_vtab_cursor *)sqlite3_malloc(sizeof(Fts3Cursor)); - if( !pCsr ){ - return SQLITE_NOMEM; + assert( p->nPendingData==0 ); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3PendingTermsFlush(p); } - memset(pCsr, 0, sizeof(Fts3Cursor)); - return SQLITE_OK; + + if( p->zContentTbl==0 ){ + fts3DbExec(&rc, db, + "ALTER TABLE %Q.'%q_content' RENAME TO '%q_content';", + p->zDb, p->zName, zName + ); + } + + if( p->bHasDocsize ){ + fts3DbExec(&rc, db, + "ALTER TABLE %Q.'%q_docsize' RENAME TO '%q_docsize';", + p->zDb, p->zName, zName + ); + } + if( p->bHasStat ){ + fts3DbExec(&rc, db, + "ALTER TABLE %Q.'%q_stat' RENAME TO '%q_stat';", + p->zDb, p->zName, zName + ); + } + fts3DbExec(&rc, db, + "ALTER TABLE %Q.'%q_segments' RENAME TO '%q_segments';", + p->zDb, p->zName, zName + ); + fts3DbExec(&rc, db, + "ALTER TABLE %Q.'%q_segdir' RENAME TO '%q_segdir';", + p->zDb, p->zName, zName + ); + return rc; } /* -** Finalize the statement handle at pCsr->pStmt. +** The xSavepoint() method. ** -** Or, if that statement handle is one created by fts3CursorSeekStmt(), -** and the Fts3Table.pSeekStmt slot is currently NULL, save the statement -** pointer there instead of finalizing it. +** Flush the contents of the pending-terms table to disk. */ -static void fts3CursorFinalizeStmt(Fts3Cursor *pCsr){ - if( pCsr->bSeekStmt ){ - Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; - if( p->pSeekStmt==0 ){ - p->pSeekStmt = pCsr->pStmt; - sqlite3_reset(pCsr->pStmt); - pCsr->pStmt = 0; - } - pCsr->bSeekStmt = 0; +static int fts3SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){ + int rc = SQLITE_OK; + UNUSED_PARAMETER(iSavepoint); + assert( ((Fts3Table *)pVtab)->inTransaction ); + assert( ((Fts3Table *)pVtab)->mxSavepoint <= iSavepoint ); + TESTONLY( ((Fts3Table *)pVtab)->mxSavepoint = iSavepoint ); + if( ((Fts3Table *)pVtab)->bIgnoreSavepoint==0 ){ + rc = fts3SyncMethod(pVtab); } - sqlite3_finalize(pCsr->pStmt); + return rc; } /* -** Free all resources currently held by the cursor passed as the only -** argument. +** The xRelease() method. +** +** This is a no-op. */ -static void fts3ClearCursor(Fts3Cursor *pCsr){ - fts3CursorFinalizeStmt(pCsr); - sqlite3Fts3FreeDeferredTokens(pCsr); - sqlite3_free(pCsr->aDoclist); - sqlite3Fts3MIBufferFree(pCsr->pMIBuffer); - sqlite3Fts3ExprFree(pCsr->pExpr); - memset(&(&pCsr->base)[1], 0, sizeof(Fts3Cursor)-sizeof(sqlite3_vtab_cursor)); +static int fts3ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){ + TESTONLY( Fts3Table *p = (Fts3Table*)pVtab ); + UNUSED_PARAMETER(iSavepoint); + UNUSED_PARAMETER(pVtab); + assert( p->inTransaction ); + assert( p->mxSavepoint >= iSavepoint ); + TESTONLY( p->mxSavepoint = iSavepoint-1 ); + return SQLITE_OK; } /* -** Close the cursor. For additional information see the documentation -** on the xClose method of the virtual table interface. +** The xRollbackTo() method. +** +** Discard the contents of the pending terms table. */ -static int fts3CloseMethod(sqlite3_vtab_cursor *pCursor){ - Fts3Cursor *pCsr = (Fts3Cursor *)pCursor; - assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); - fts3ClearCursor(pCsr); - assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); - sqlite3_free(pCsr); +static int fts3RollbackToMethod(sqlite3_vtab *pVtab, int iSavepoint){ + Fts3Table *p = (Fts3Table*)pVtab; + UNUSED_PARAMETER(iSavepoint); + assert( p->inTransaction ); + TESTONLY( p->mxSavepoint = iSavepoint ); + sqlite3Fts3PendingTermsClear(p); return SQLITE_OK; } /* -** If pCsr->pStmt has not been prepared (i.e. if pCsr->pStmt==0), then -** compose and prepare an SQL statement of the form: -** -** "SELECT FROM %_content WHERE rowid = ?" -** -** (or the equivalent for a content=xxx table) and set pCsr->pStmt to -** it. If an error occurs, return an SQLite error code. +** Return true if zName is the extension on one of the shadow tables used +** by this module. */ -static int fts3CursorSeekStmt(Fts3Cursor *pCsr){ - int rc = SQLITE_OK; - if( pCsr->pStmt==0 ){ - Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; - char *zSql; - if( p->pSeekStmt ){ - pCsr->pStmt = p->pSeekStmt; - p->pSeekStmt = 0; - }else{ - zSql = sqlite3_mprintf("SELECT %s WHERE rowid = ?", p->zReadExprlist); - if( !zSql ) return SQLITE_NOMEM; - rc = sqlite3_prepare_v3(p->db, zSql,-1,SQLITE_PREPARE_PERSISTENT,&pCsr->pStmt,0); - sqlite3_free(zSql); - } - if( rc==SQLITE_OK ) pCsr->bSeekStmt = 1; +static int fts3ShadowName(const char *zName){ + static const char *azName[] = { + "content", "docsize", "segdir", "segments", "stat", + }; + unsigned int i; + for(i=0; ipStmt statement so that it is on the row -** of the %_content table that contains the last match. Return -** SQLITE_OK on success. +** This function is registered as the module destructor (called when an +** FTS3 enabled database connection is closed). It frees the memory +** allocated for the tokenizer hash table. */ -static int fts3CursorSeek(sqlite3_context *pContext, Fts3Cursor *pCsr){ - int rc = SQLITE_OK; - if( pCsr->isRequireSeek ){ - rc = fts3CursorSeekStmt(pCsr); - if( rc==SQLITE_OK ){ - sqlite3_bind_int64(pCsr->pStmt, 1, pCsr->iPrevId); - pCsr->isRequireSeek = 0; - if( SQLITE_ROW==sqlite3_step(pCsr->pStmt) ){ - return SQLITE_OK; - }else{ - rc = sqlite3_reset(pCsr->pStmt); - if( rc==SQLITE_OK && ((Fts3Table *)pCsr->base.pVtab)->zContentTbl==0 ){ - /* If no row was found and no error has occurred, then the %_content - ** table is missing a row that is present in the full-text index. - ** The data structures are corrupt. */ - rc = FTS_CORRUPT_VTAB; - pCsr->isEof = 1; - } - } - } - } - - if( rc!=SQLITE_OK && pContext ){ - sqlite3_result_error_code(pContext, rc); - } - return rc; +static void hashDestroy(void *p){ + Fts3Hash *pHash = (Fts3Hash *)p; + sqlite3Fts3HashClear(pHash); + sqlite3_free(pHash); } /* -** This function is used to process a single interior node when searching -** a b-tree for a term or term prefix. The node data is passed to this -** function via the zNode/nNode parameters. The term to search for is -** passed in zTerm/nTerm. -** -** If piFirst is not NULL, then this function sets *piFirst to the blockid -** of the child node that heads the sub-tree that may contain the term. -** -** If piLast is not NULL, then *piLast is set to the right-most child node -** that heads a sub-tree that may contain a term for which zTerm/nTerm is -** a prefix. +** The fts3 built-in tokenizers - "simple", "porter" and "icu"- are +** implemented in files fts3_tokenizer1.c, fts3_porter.c and fts3_icu.c +** respectively. The following three forward declarations are for functions +** declared in these files used to retrieve the respective implementations. ** -** If an OOM error occurs, SQLITE_NOMEM is returned. Otherwise, SQLITE_OK. +** Calling sqlite3Fts3SimpleTokenizerModule() sets the value pointed +** to by the argument to point to the "simple" tokenizer implementation. +** And so on. */ -static int fts3ScanInteriorNode( - const char *zTerm, /* Term to select leaves for */ - int nTerm, /* Size of term zTerm in bytes */ - const char *zNode, /* Buffer containing segment interior node */ - int nNode, /* Size of buffer at zNode */ - sqlite3_int64 *piFirst, /* OUT: Selected child node */ - sqlite3_int64 *piLast /* OUT: Selected child node */ -){ - int rc = SQLITE_OK; /* Return code */ - const char *zCsr = zNode; /* Cursor to iterate through node */ - const char *zEnd = &zCsr[nNode];/* End of interior node buffer */ - char *zBuffer = 0; /* Buffer to load terms into */ - int nAlloc = 0; /* Size of allocated buffer */ - int isFirstTerm = 1; /* True when processing first term on page */ - sqlite3_int64 iChild; /* Block id of child node to descend to */ - - /* Skip over the 'height' varint that occurs at the start of every - ** interior node. Then load the blockid of the left-child of the b-tree - ** node into variable iChild. - ** - ** Even if the data structure on disk is corrupted, this (reading two - ** varints from the buffer) does not risk an overread. If zNode is a - ** root node, then the buffer comes from a SELECT statement. SQLite does - ** not make this guarantee explicitly, but in practice there are always - ** either more than 20 bytes of allocated space following the nNode bytes of - ** contents, or two zero bytes. Or, if the node is read from the %_segments - ** table, then there are always 20 bytes of zeroed padding following the - ** nNode bytes of content (see sqlite3Fts3ReadBlock() for details). - */ - zCsr += sqlite3Fts3GetVarint(zCsr, &iChild); - zCsr += sqlite3Fts3GetVarint(zCsr, &iChild); - if( zCsr>zEnd ){ - return FTS_CORRUPT_VTAB; - } - - while( zCsr=0 && nSuffix>=0 ); - if( &zCsr[nSuffix]>zEnd ){ - rc = FTS_CORRUPT_VTAB; - goto finish_scan; - } - if( nPrefix+nSuffix>nAlloc ){ - char *zNew; - nAlloc = (nPrefix+nSuffix) * 2; - zNew = (char *)sqlite3_realloc(zBuffer, nAlloc); - if( !zNew ){ - rc = SQLITE_NOMEM; - goto finish_scan; - } - zBuffer = zNew; - } - assert( zBuffer ); - memcpy(&zBuffer[nPrefix], zCsr, nSuffix); - nBuffer = nPrefix + nSuffix; - zCsr += nSuffix; - - /* Compare the term we are searching for with the term just loaded from - ** the interior node. If the specified term is greater than or equal - ** to the term from the interior node, then all terms on the sub-tree - ** headed by node iChild are smaller than zTerm. No need to search - ** iChild. - ** - ** If the interior node term is larger than the specified term, then - ** the tree headed by iChild may contain the specified term. - */ - cmp = memcmp(zTerm, zBuffer, (nBuffer>nTerm ? nTerm : nBuffer)); - if( piFirst && (cmp<0 || (cmp==0 && nBuffer>nTerm)) ){ - *piFirst = iChild; - piFirst = 0; - } - - if( piLast && cmp<0 ){ - *piLast = iChild; - piLast = 0; - } +SQLITE_PRIVATE void sqlite3Fts3SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule); +SQLITE_PRIVATE void sqlite3Fts3PorterTokenizerModule(sqlite3_tokenizer_module const**ppModule); +#ifndef SQLITE_DISABLE_FTS3_UNICODE +SQLITE_PRIVATE void sqlite3Fts3UnicodeTokenizer(sqlite3_tokenizer_module const**ppModule); +#endif +#ifdef SQLITE_ENABLE_ICU +SQLITE_PRIVATE void sqlite3Fts3IcuTokenizerModule(sqlite3_tokenizer_module const**ppModule); +#endif - iChild++; - }; +/* +** Initialize the fts3 extension. If this extension is built as part +** of the sqlite library, then this function is called directly by +** SQLite. If fts3 is built as a dynamically loadable extension, this +** function is called by the sqlite3_extension_init() entry point. +*/ +SQLITE_PRIVATE int sqlite3Fts3Init(sqlite3 *db){ + int rc = SQLITE_OK; + Fts3Hash *pHash = 0; + const sqlite3_tokenizer_module *pSimple = 0; + const sqlite3_tokenizer_module *pPorter = 0; +#ifndef SQLITE_DISABLE_FTS3_UNICODE + const sqlite3_tokenizer_module *pUnicode = 0; +#endif - if( piFirst ) *piFirst = iChild; - if( piLast ) *piLast = iChild; +#ifdef SQLITE_ENABLE_ICU + const sqlite3_tokenizer_module *pIcu = 0; + sqlite3Fts3IcuTokenizerModule(&pIcu); +#endif - finish_scan: - sqlite3_free(zBuffer); - return rc; -} +#ifndef SQLITE_DISABLE_FTS3_UNICODE + sqlite3Fts3UnicodeTokenizer(&pUnicode); +#endif +#ifdef SQLITE_TEST + rc = sqlite3Fts3InitTerm(db); + if( rc!=SQLITE_OK ) return rc; +#endif -/* -** The buffer pointed to by argument zNode (size nNode bytes) contains an -** interior node of a b-tree segment. The zTerm buffer (size nTerm bytes) -** contains a term. This function searches the sub-tree headed by the zNode -** node for the range of leaf nodes that may contain the specified term -** or terms for which the specified term is a prefix. -** -** If piLeaf is not NULL, then *piLeaf is set to the blockid of the -** left-most leaf node in the tree that may contain the specified term. -** If piLeaf2 is not NULL, then *piLeaf2 is set to the blockid of the -** right-most leaf node that may contain a term for which the specified -** term is a prefix. -** -** It is possible that the range of returned leaf nodes does not contain -** the specified term or any terms for which it is a prefix. However, if the -** segment does contain any such terms, they are stored within the identified -** range. Because this function only inspects interior segment nodes (and -** never loads leaf nodes into memory), it is not possible to be sure. -** -** If an error occurs, an error code other than SQLITE_OK is returned. -*/ -static int fts3SelectLeaf( - Fts3Table *p, /* Virtual table handle */ - const char *zTerm, /* Term to select leaves for */ - int nTerm, /* Size of term zTerm in bytes */ - const char *zNode, /* Buffer containing segment interior node */ - int nNode, /* Size of buffer at zNode */ - sqlite3_int64 *piLeaf, /* Selected leaf node */ - sqlite3_int64 *piLeaf2 /* Selected leaf node */ -){ - int rc = SQLITE_OK; /* Return code */ - int iHeight; /* Height of this node in tree */ + rc = sqlite3Fts3InitAux(db); + if( rc!=SQLITE_OK ) return rc; - assert( piLeaf || piLeaf2 ); + sqlite3Fts3SimpleTokenizerModule(&pSimple); + sqlite3Fts3PorterTokenizerModule(&pPorter); - fts3GetVarint32(zNode, &iHeight); - rc = fts3ScanInteriorNode(zTerm, nTerm, zNode, nNode, piLeaf, piLeaf2); - assert( !piLeaf2 || !piLeaf || rc!=SQLITE_OK || (*piLeaf<=*piLeaf2) ); + /* Allocate and initialize the hash-table used to store tokenizers. */ + pHash = sqlite3_malloc(sizeof(Fts3Hash)); + if( !pHash ){ + rc = SQLITE_NOMEM; + }else{ + sqlite3Fts3HashInit(pHash, FTS3_HASH_STRING, 1); + } - if( rc==SQLITE_OK && iHeight>1 ){ - char *zBlob = 0; /* Blob read from %_segments table */ - int nBlob = 0; /* Size of zBlob in bytes */ + /* Load the built-in tokenizers into the hash table */ + if( rc==SQLITE_OK ){ + if( sqlite3Fts3HashInsert(pHash, "simple", 7, (void *)pSimple) + || sqlite3Fts3HashInsert(pHash, "porter", 7, (void *)pPorter) - if( piLeaf && piLeaf2 && (*piLeaf!=*piLeaf2) ){ - rc = sqlite3Fts3ReadBlock(p, *piLeaf, &zBlob, &nBlob, 0); - if( rc==SQLITE_OK ){ - rc = fts3SelectLeaf(p, zTerm, nTerm, zBlob, nBlob, piLeaf, 0); - } - sqlite3_free(zBlob); - piLeaf = 0; - zBlob = 0; +#ifndef SQLITE_DISABLE_FTS3_UNICODE + || sqlite3Fts3HashInsert(pHash, "unicode61", 10, (void *)pUnicode) +#endif +#ifdef SQLITE_ENABLE_ICU + || (pIcu && sqlite3Fts3HashInsert(pHash, "icu", 4, (void *)pIcu)) +#endif + ){ + rc = SQLITE_NOMEM; } + } + +#ifdef SQLITE_TEST + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3ExprInitTestInterface(db, pHash); + } +#endif + /* Create the virtual table wrapper around the hash-table and overload + ** the four scalar functions. If this is successful, register the + ** module with sqlite. + */ + if( SQLITE_OK==rc + && SQLITE_OK==(rc = sqlite3Fts3InitHashTable(db, pHash, "fts3_tokenizer")) + && SQLITE_OK==(rc = sqlite3_overload_function(db, "snippet", -1)) + && SQLITE_OK==(rc = sqlite3_overload_function(db, "offsets", 1)) + && SQLITE_OK==(rc = sqlite3_overload_function(db, "matchinfo", 1)) + && SQLITE_OK==(rc = sqlite3_overload_function(db, "matchinfo", 2)) + && SQLITE_OK==(rc = sqlite3_overload_function(db, "optimize", 1)) + ){ + rc = sqlite3_create_module_v2( + db, "fts3", &fts3Module, (void *)pHash, hashDestroy + ); if( rc==SQLITE_OK ){ - rc = sqlite3Fts3ReadBlock(p, piLeaf?*piLeaf:*piLeaf2, &zBlob, &nBlob, 0); + rc = sqlite3_create_module_v2( + db, "fts4", &fts3Module, (void *)pHash, 0 + ); } if( rc==SQLITE_OK ){ - rc = fts3SelectLeaf(p, zTerm, nTerm, zBlob, nBlob, piLeaf, piLeaf2); + rc = sqlite3Fts3InitTok(db, (void *)pHash); } - sqlite3_free(zBlob); + return rc; } + + /* An error has occurred. Delete the hash table and return the error code. */ + assert( rc!=SQLITE_OK ); + if( pHash ){ + sqlite3Fts3HashClear(pHash); + sqlite3_free(pHash); + } return rc; } /* -** This function is used to create delta-encoded serialized lists of FTS3 -** varints. Each call to this function appends a single varint to a list. +** Allocate an Fts3MultiSegReader for each token in the expression headed +** by pExpr. +** +** An Fts3SegReader object is a cursor that can seek or scan a range of +** entries within a single segment b-tree. An Fts3MultiSegReader uses multiple +** Fts3SegReader objects internally to provide an interface to seek or scan +** within the union of all segments of a b-tree. Hence the name. +** +** If the allocated Fts3MultiSegReader just seeks to a single entry in a +** segment b-tree (if the term is not a prefix or it is a prefix for which +** there exists prefix b-tree of the right length) then it may be traversed +** and merged incrementally. Otherwise, it has to be merged into an in-memory +** doclist and then traversed. */ -static void fts3PutDeltaVarint( - char **pp, /* IN/OUT: Output pointer */ - sqlite3_int64 *piPrev, /* IN/OUT: Previous value written to list */ - sqlite3_int64 iVal /* Write this value to the list */ +static void fts3EvalAllocateReaders( + Fts3Cursor *pCsr, /* FTS cursor handle */ + Fts3Expr *pExpr, /* Allocate readers for this expression */ + int *pnToken, /* OUT: Total number of tokens in phrase. */ + int *pnOr, /* OUT: Total number of OR nodes in expr. */ + int *pRc /* IN/OUT: Error code */ ){ - assert( iVal-*piPrev > 0 || (*piPrev==0 && iVal==0) ); - *pp += sqlite3Fts3PutVarint(*pp, iVal-*piPrev); - *piPrev = iVal; + if( pExpr && SQLITE_OK==*pRc ){ + if( pExpr->eType==FTSQUERY_PHRASE ){ + int i; + int nToken = pExpr->pPhrase->nToken; + *pnToken += nToken; + for(i=0; ipPhrase->aToken[i]; + int rc = fts3TermSegReaderCursor(pCsr, + pToken->z, pToken->n, pToken->isPrefix, &pToken->pSegcsr + ); + if( rc!=SQLITE_OK ){ + *pRc = rc; + return; + } + } + assert( pExpr->pPhrase->iDoclistToken==0 ); + pExpr->pPhrase->iDoclistToken = -1; + }else{ + *pnOr += (pExpr->eType==FTSQUERY_OR); + fts3EvalAllocateReaders(pCsr, pExpr->pLeft, pnToken, pnOr, pRc); + fts3EvalAllocateReaders(pCsr, pExpr->pRight, pnToken, pnOr, pRc); + } + } } /* -** When this function is called, *ppPoslist is assumed to point to the -** start of a position-list. After it returns, *ppPoslist points to the -** first byte after the position-list. +** Arguments pList/nList contain the doclist for token iToken of phrase p. +** It is merged into the main doclist stored in p->doclist.aAll/nAll. ** -** A position list is list of positions (delta encoded) and columns for -** a single document record of a doclist. So, in other words, this -** routine advances *ppPoslist so that it points to the next docid in -** the doclist, or to the first byte past the end of the doclist. +** This function assumes that pList points to a buffer allocated using +** sqlite3_malloc(). This function takes responsibility for eventually +** freeing the buffer. ** -** If pp is not NULL, then the contents of the position list are copied -** to *pp. *pp is set to point to the first byte past the last byte copied -** before this function returns. +** SQLITE_OK is returned if successful, or SQLITE_NOMEM if an error occurs. */ -static void fts3PoslistCopy(char **pp, char **ppPoslist){ - char *pEnd = *ppPoslist; - char c = 0; +static int fts3EvalPhraseMergeToken( + Fts3Table *pTab, /* FTS Table pointer */ + Fts3Phrase *p, /* Phrase to merge pList/nList into */ + int iToken, /* Token pList/nList corresponds to */ + char *pList, /* Pointer to doclist */ + int nList /* Number of bytes in pList */ +){ + int rc = SQLITE_OK; + assert( iToken!=p->iDoclistToken ); - /* The end of a position list is marked by a zero encoded as an FTS3 - ** varint. A single POS_END (0) byte. Except, if the 0 byte is preceded by - ** a byte with the 0x80 bit set, then it is not a varint 0, but the tail - ** of some other, multi-byte, value. - ** - ** The following while-loop moves pEnd to point to the first byte that is not - ** immediately preceded by a byte with the 0x80 bit set. Then increments - ** pEnd once more so that it points to the byte immediately following the - ** last byte in the position-list. - */ - while( *pEnd | c ){ - c = *pEnd++ & 0x80; - testcase( c!=0 && (*pEnd)==0 ); + if( pList==0 ){ + sqlite3_free(p->doclist.aAll); + p->doclist.aAll = 0; + p->doclist.nAll = 0; } - pEnd++; /* Advance past the POS_END terminator byte */ - if( pp ){ - int n = (int)(pEnd - *ppPoslist); - char *p = *pp; - memcpy(p, *ppPoslist, n); - p += n; - *pp = p; + else if( p->iDoclistToken<0 ){ + p->doclist.aAll = pList; + p->doclist.nAll = nList; } - *ppPoslist = pEnd; -} - -/* -** When this function is called, *ppPoslist is assumed to point to the -** start of a column-list. After it returns, *ppPoslist points to the -** to the terminator (POS_COLUMN or POS_END) byte of the column-list. -** -** A column-list is list of delta-encoded positions for a single column -** within a single document within a doclist. -** -** The column-list is terminated either by a POS_COLUMN varint (1) or -** a POS_END varint (0). This routine leaves *ppPoslist pointing to -** the POS_COLUMN or POS_END that terminates the column-list. -** -** If pp is not NULL, then the contents of the column-list are copied -** to *pp. *pp is set to point to the first byte past the last byte copied -** before this function returns. The POS_COLUMN or POS_END terminator -** is not copied into *pp. -*/ -static void fts3ColumnlistCopy(char **pp, char **ppPoslist){ - char *pEnd = *ppPoslist; - char c = 0; - /* A column-list is terminated by either a 0x01 or 0x00 byte that is - ** not part of a multi-byte varint. - */ - while( 0xFE & (*pEnd | c) ){ - c = *pEnd++ & 0x80; - testcase( c!=0 && ((*pEnd)&0xfe)==0 ); - } - if( pp ){ - int n = (int)(pEnd - *ppPoslist); - char *p = *pp; - memcpy(p, *ppPoslist, n); - p += n; - *pp = p; + else if( p->doclist.aAll==0 ){ + sqlite3_free(pList); } - *ppPoslist = pEnd; -} -/* -** Value used to signify the end of an position-list. This is safe because -** it is not possible to have a document with 2^31 terms. -*/ -#define POSITION_LIST_END 0x7fffffff + else { + char *pLeft; + char *pRight; + int nLeft; + int nRight; + int nDiff; -/* -** This function is used to help parse position-lists. When this function is -** called, *pp may point to the start of the next varint in the position-list -** being parsed, or it may point to 1 byte past the end of the position-list -** (in which case **pp will be a terminator bytes POS_END (0) or -** (1)). -** -** If *pp points past the end of the current position-list, set *pi to -** POSITION_LIST_END and return. Otherwise, read the next varint from *pp, -** increment the current value of *pi by the value read, and set *pp to -** point to the next value before returning. -** -** Before calling this routine *pi must be initialized to the value of -** the previous position, or zero if we are reading the first position -** in the position-list. Because positions are delta-encoded, the value -** of the previous position is needed in order to compute the value of -** the next position. -*/ -static void fts3ReadNextPos( - char **pp, /* IN/OUT: Pointer into position-list buffer */ - sqlite3_int64 *pi /* IN/OUT: Value read from position-list */ -){ - if( (**pp)&0xFE ){ - fts3GetDeltaVarint(pp, pi); - *pi -= 2; - }else{ - *pi = POSITION_LIST_END; - } -} + if( p->iDoclistTokendoclist.aAll; + nLeft = p->doclist.nAll; + pRight = pList; + nRight = nList; + nDiff = iToken - p->iDoclistToken; + }else{ + pRight = p->doclist.aAll; + nRight = p->doclist.nAll; + pLeft = pList; + nLeft = nList; + nDiff = p->iDoclistToken - iToken; + } -/* -** If parameter iCol is not 0, write an POS_COLUMN (1) byte followed by -** the value of iCol encoded as a varint to *pp. This will start a new -** column list. -** -** Set *pp to point to the byte just after the last byte written before -** returning (do not modify it if iCol==0). Return the total number of bytes -** written (0 if iCol==0). -*/ -static int fts3PutColNumber(char **pp, int iCol){ - int n = 0; /* Number of bytes written */ - if( iCol ){ - char *p = *pp; /* Output pointer */ - n = 1 + sqlite3Fts3PutVarint(&p[1], iCol); - *p = 0x01; - *pp = &p[n]; + rc = fts3DoclistPhraseMerge( + pTab->bDescIdx, nDiff, pLeft, nLeft, &pRight, &nRight + ); + sqlite3_free(pLeft); + p->doclist.aAll = pRight; + p->doclist.nAll = nRight; } - return n; + + if( iToken>p->iDoclistToken ) p->iDoclistToken = iToken; + return rc; } /* -** Compute the union of two position lists. The output written -** into *pp contains all positions of both *pp1 and *pp2 in sorted -** order and with any duplicates removed. All pointers are -** updated appropriately. The caller is responsible for insuring -** that there is enough space in *pp to hold the complete output. +** Load the doclist for phrase p into p->doclist.aAll/nAll. The loaded doclist +** does not take deferred tokens into account. +** +** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code. */ -static void fts3PoslistMerge( - char **pp, /* Output buffer */ - char **pp1, /* Left input list */ - char **pp2 /* Right input list */ +static int fts3EvalPhraseLoad( + Fts3Cursor *pCsr, /* FTS Cursor handle */ + Fts3Phrase *p /* Phrase object */ ){ - char *p = *pp; - char *p1 = *pp1; - char *p2 = *pp2; - - while( *p1 || *p2 ){ - int iCol1; /* The current column index in pp1 */ - int iCol2; /* The current column index in pp2 */ - - if( *p1==POS_COLUMN ) fts3GetVarint32(&p1[1], &iCol1); - else if( *p1==POS_END ) iCol1 = POSITION_LIST_END; - else iCol1 = 0; - - if( *p2==POS_COLUMN ) fts3GetVarint32(&p2[1], &iCol2); - else if( *p2==POS_END ) iCol2 = POSITION_LIST_END; - else iCol2 = 0; + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + int iToken; + int rc = SQLITE_OK; - if( iCol1==iCol2 ){ - sqlite3_int64 i1 = 0; /* Last position from pp1 */ - sqlite3_int64 i2 = 0; /* Last position from pp2 */ - sqlite3_int64 iPrev = 0; - int n = fts3PutColNumber(&p, iCol1); - p1 += n; - p2 += n; + for(iToken=0; rc==SQLITE_OK && iTokennToken; iToken++){ + Fts3PhraseToken *pToken = &p->aToken[iToken]; + assert( pToken->pDeferred==0 || pToken->pSegcsr==0 ); - /* At this point, both p1 and p2 point to the start of column-lists - ** for the same column (the column with index iCol1 and iCol2). - ** A column-list is a list of non-negative delta-encoded varints, each - ** incremented by 2 before being stored. Each list is terminated by a - ** POS_END (0) or POS_COLUMN (1). The following block merges the two lists - ** and writes the results to buffer p. p is left pointing to the byte - ** after the list written. No terminator (POS_END or POS_COLUMN) is - ** written to the output. - */ - fts3GetDeltaVarint(&p1, &i1); - fts3GetDeltaVarint(&p2, &i2); - do { - fts3PutDeltaVarint(&p, &iPrev, (i1pSegcsr ){ + int nThis = 0; + char *pThis = 0; + rc = fts3TermSelect(pTab, pToken, p->iColumn, &nThis, &pThis); + if( rc==SQLITE_OK ){ + rc = fts3EvalPhraseMergeToken(pTab, p, iToken, pThis, nThis); + } } + assert( pToken->pSegcsr==0 ); } - *p++ = POS_END; - *pp = p; - *pp1 = p1 + 1; - *pp2 = p2 + 1; + return rc; } +#ifndef SQLITE_DISABLE_FTS4_DEFERRED /* -** This function is used to merge two position lists into one. When it is -** called, *pp1 and *pp2 must both point to position lists. A position-list is -** the part of a doclist that follows each document id. For example, if a row -** contains: -** -** 'a b c'|'x y z'|'a b b a' -** -** Then the position list for this row for token 'b' would consist of: -** -** 0x02 0x01 0x02 0x03 0x03 0x00 -** -** When this function returns, both *pp1 and *pp2 are left pointing to the -** byte following the 0x00 terminator of their respective position lists. -** -** If isSaveLeft is 0, an entry is added to the output position list for -** each position in *pp2 for which there exists one or more positions in -** *pp1 so that (pos(*pp2)>pos(*pp1) && pos(*pp2)-pos(*pp1)<=nToken). i.e. -** when the *pp1 token appears before the *pp2 token, but not more than nToken -** slots before it. +** This function is called on each phrase after the position lists for +** any deferred tokens have been loaded into memory. It updates the phrases +** current position list to include only those positions that are really +** instances of the phrase (after considering deferred tokens). If this +** means that the phrase does not appear in the current row, doclist.pList +** and doclist.nList are both zeroed. ** -** e.g. nToken==1 searches for adjacent positions. +** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code. */ -static int fts3PoslistPhraseMerge( - char **pp, /* IN/OUT: Preallocated output buffer */ - int nToken, /* Maximum difference in token positions */ - int isSaveLeft, /* Save the left position */ - int isExact, /* If *pp1 is exactly nTokens before *pp2 */ - char **pp1, /* IN/OUT: Left input list */ - char **pp2 /* IN/OUT: Right input list */ -){ - char *p = *pp; - char *p1 = *pp1; - char *p2 = *pp2; - int iCol1 = 0; - int iCol2 = 0; +static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){ + int iToken; /* Used to iterate through phrase tokens */ + char *aPoslist = 0; /* Position list for deferred tokens */ + int nPoslist = 0; /* Number of bytes in aPoslist */ + int iPrev = -1; /* Token number of previous deferred token */ - /* Never set both isSaveLeft and isExact for the same invocation. */ - assert( isSaveLeft==0 || isExact==0 ); + assert( pPhrase->doclist.bFreeList==0 ); - assert( p!=0 && *p1!=0 && *p2!=0 ); - if( *p1==POS_COLUMN ){ - p1++; - p1 += fts3GetVarint32(p1, &iCol1); - } - if( *p2==POS_COLUMN ){ - p2++; - p2 += fts3GetVarint32(p2, &iCol2); - } + for(iToken=0; iTokennToken; iToken++){ + Fts3PhraseToken *pToken = &pPhrase->aToken[iToken]; + Fts3DeferredToken *pDeferred = pToken->pDeferred; - while( 1 ){ - if( iCol1==iCol2 ){ - char *pSave = p; - sqlite3_int64 iPrev = 0; - sqlite3_int64 iPos1 = 0; - sqlite3_int64 iPos2 = 0; + if( pDeferred ){ + char *pList; + int nList; + int rc = sqlite3Fts3DeferredTokenList(pDeferred, &pList, &nList); + if( rc!=SQLITE_OK ) return rc; - if( iCol1 ){ - *p++ = POS_COLUMN; - p += sqlite3Fts3PutVarint(p, iCol1); - } + if( pList==0 ){ + sqlite3_free(aPoslist); + pPhrase->doclist.pList = 0; + pPhrase->doclist.nList = 0; + return SQLITE_OK; - assert( *p1!=POS_END && *p1!=POS_COLUMN ); - assert( *p2!=POS_END && *p2!=POS_COLUMN ); - fts3GetDeltaVarint(&p1, &iPos1); iPos1 -= 2; - fts3GetDeltaVarint(&p2, &iPos2); iPos2 -= 2; + }else if( aPoslist==0 ){ + aPoslist = pList; + nPoslist = nList; - while( 1 ){ - if( iPos2==iPos1+nToken - || (isExact==0 && iPos2>iPos1 && iPos2<=iPos1+nToken) - ){ - sqlite3_int64 iSave; - iSave = isSaveLeft ? iPos1 : iPos2; - fts3PutDeltaVarint(&p, &iPrev, iSave+2); iPrev -= 2; - pSave = 0; - assert( p ); - } - if( (!isSaveLeft && iPos2<=(iPos1+nToken)) || iPos2<=iPos1 ){ - if( (*p2&0xFE)==0 ) break; - fts3GetDeltaVarint(&p2, &iPos2); iPos2 -= 2; - }else{ - if( (*p1&0xFE)==0 ) break; - fts3GetDeltaVarint(&p1, &iPos1); iPos1 -= 2; + }else{ + char *aOut = pList; + char *p1 = aPoslist; + char *p2 = aOut; + + assert( iPrev>=0 ); + fts3PoslistPhraseMerge(&aOut, iToken-iPrev, 0, 1, &p1, &p2); + sqlite3_free(aPoslist); + aPoslist = pList; + nPoslist = (int)(aOut - aPoslist); + if( nPoslist==0 ){ + sqlite3_free(aPoslist); + pPhrase->doclist.pList = 0; + pPhrase->doclist.nList = 0; + return SQLITE_OK; } } + iPrev = iToken; + } + } + + if( iPrev>=0 ){ + int nMaxUndeferred = pPhrase->iDoclistToken; + if( nMaxUndeferred<0 ){ + pPhrase->doclist.pList = aPoslist; + pPhrase->doclist.nList = nPoslist; + pPhrase->doclist.iDocid = pCsr->iPrevId; + pPhrase->doclist.bFreeList = 1; + }else{ + int nDistance; + char *p1; + char *p2; + char *aOut; + + if( nMaxUndeferred>iPrev ){ + p1 = aPoslist; + p2 = pPhrase->doclist.pList; + nDistance = nMaxUndeferred - iPrev; + }else{ + p1 = pPhrase->doclist.pList; + p2 = aPoslist; + nDistance = iPrev - nMaxUndeferred; + } + + aOut = (char *)sqlite3_malloc(nPoslist+8); + if( !aOut ){ + sqlite3_free(aPoslist); + return SQLITE_NOMEM; + } + + pPhrase->doclist.pList = aOut; + if( fts3PoslistPhraseMerge(&aOut, nDistance, 0, 1, &p1, &p2) ){ + pPhrase->doclist.bFreeList = 1; + pPhrase->doclist.nList = (int)(aOut - pPhrase->doclist.pList); + }else{ + sqlite3_free(aOut); + pPhrase->doclist.pList = 0; + pPhrase->doclist.nList = 0; + } + sqlite3_free(aPoslist); + } + } - if( pSave ){ - assert( pp && p ); - p = pSave; - } + return SQLITE_OK; +} +#endif /* SQLITE_DISABLE_FTS4_DEFERRED */ - fts3ColumnlistCopy(0, &p1); - fts3ColumnlistCopy(0, &p2); - assert( (*p1&0xFE)==0 && (*p2&0xFE)==0 ); - if( 0==*p1 || 0==*p2 ) break; +/* +** Maximum number of tokens a phrase may have to be considered for the +** incremental doclists strategy. +*/ +#define MAX_INCR_PHRASE_TOKENS 4 - p1++; - p1 += fts3GetVarint32(p1, &iCol1); - p2++; - p2 += fts3GetVarint32(p2, &iCol2); - } +/* +** This function is called for each Fts3Phrase in a full-text query +** expression to initialize the mechanism for returning rows. Once this +** function has been called successfully on an Fts3Phrase, it may be +** used with fts3EvalPhraseNext() to iterate through the matching docids. +** +** If parameter bOptOk is true, then the phrase may (or may not) use the +** incremental loading strategy. Otherwise, the entire doclist is loaded into +** memory within this call. +** +** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code. +*/ +static int fts3EvalPhraseStart(Fts3Cursor *pCsr, int bOptOk, Fts3Phrase *p){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + int rc = SQLITE_OK; /* Error code */ + int i; - /* Advance pointer p1 or p2 (whichever corresponds to the smaller of - ** iCol1 and iCol2) so that it points to either the 0x00 that marks the - ** end of the position list, or the 0x01 that precedes the next - ** column-number in the position list. - */ - else if( iCol1bDesc==pTab->bDescIdx + && p->nToken<=MAX_INCR_PHRASE_TOKENS && p->nToken>0 +#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) + && pTab->bNoIncrDoclist==0 +#endif + ); + for(i=0; bIncrOk==1 && inToken; i++){ + Fts3PhraseToken *pToken = &p->aToken[i]; + if( pToken->bFirst || (pToken->pSegcsr!=0 && !pToken->pSegcsr->bLookup) ){ + bIncrOk = 0; } + if( pToken->pSegcsr ) bHaveIncr = 1; } - fts3PoslistCopy(0, &p2); - fts3PoslistCopy(0, &p1); - *pp1 = p1; - *pp2 = p2; - if( *pp==p ){ - return 0; + if( bIncrOk && bHaveIncr ){ + /* Use the incremental approach. */ + int iCol = (p->iColumn >= pTab->nColumn ? -1 : p->iColumn); + for(i=0; rc==SQLITE_OK && inToken; i++){ + Fts3PhraseToken *pToken = &p->aToken[i]; + Fts3MultiSegReader *pSegcsr = pToken->pSegcsr; + if( pSegcsr ){ + rc = sqlite3Fts3MsrIncrStart(pTab, pSegcsr, iCol, pToken->z, pToken->n); + } + } + p->bIncr = 1; + }else{ + /* Load the full doclist for the phrase into memory. */ + rc = fts3EvalPhraseLoad(pCsr, p); + p->bIncr = 0; } - *p++ = 0x00; - *pp = p; - return 1; + + assert( rc!=SQLITE_OK || p->nToken<1 || p->aToken[0].pSegcsr==0 || p->bIncr ); + return rc; } /* -** Merge two position-lists as required by the NEAR operator. The argument -** position lists correspond to the left and right phrases of an expression -** like: -** -** "phrase 1" NEAR "phrase number 2" -** -** Position list *pp1 corresponds to the left-hand side of the NEAR -** expression and *pp2 to the right. As usual, the indexes in the position -** lists are the offsets of the last token in each phrase (tokens "1" and "2" -** in the example above). +** This function is used to iterate backwards (from the end to start) +** through doclists. It is used by this module to iterate through phrase +** doclists in reverse and by the fts3_write.c module to iterate through +** pending-terms lists when writing to databases with "order=desc". ** -** The output position list - written to *pp - is a copy of *pp2 with those -** entries that are not sufficiently NEAR entries in *pp1 removed. +** The doclist may be sorted in ascending (parameter bDescIdx==0) or +** descending (parameter bDescIdx==1) order of docid. Regardless, this +** function iterates from the end of the doclist to the beginning. */ -static int fts3PoslistNearMerge( - char **pp, /* Output buffer */ - char *aTmp, /* Temporary buffer space */ - int nRight, /* Maximum difference in token positions */ - int nLeft, /* Maximum difference in token positions */ - char **pp1, /* IN/OUT: Left input list */ - char **pp2 /* IN/OUT: Right input list */ +SQLITE_PRIVATE void sqlite3Fts3DoclistPrev( + int bDescIdx, /* True if the doclist is desc */ + char *aDoclist, /* Pointer to entire doclist */ + int nDoclist, /* Length of aDoclist in bytes */ + char **ppIter, /* IN/OUT: Iterator pointer */ + sqlite3_int64 *piDocid, /* IN/OUT: Docid pointer */ + int *pnList, /* OUT: List length pointer */ + u8 *pbEof /* OUT: End-of-file flag */ ){ - char *p1 = *pp1; - char *p2 = *pp2; + char *p = *ppIter; - char *pTmp1 = aTmp; - char *pTmp2; - char *aTmp2; - int res = 1; + assert( nDoclist>0 ); + assert( *pbEof==0 ); + assert( p || *piDocid==0 ); + assert( !p || (p>aDoclist && p<&aDoclist[nDoclist]) ); - fts3PoslistPhraseMerge(&pTmp1, nRight, 0, 0, pp1, pp2); - aTmp2 = pTmp2 = pTmp1; - *pp1 = p1; - *pp2 = p2; - fts3PoslistPhraseMerge(&pTmp2, nLeft, 1, 0, pp2, pp1); - if( pTmp1!=aTmp && pTmp2!=aTmp2 ){ - fts3PoslistMerge(pp, &aTmp, &aTmp2); - }else if( pTmp1!=aTmp ){ - fts3PoslistCopy(pp, &aTmp); - }else if( pTmp2!=aTmp2 ){ - fts3PoslistCopy(pp, &aTmp2); + if( p==0 ){ + sqlite3_int64 iDocid = 0; + char *pNext = 0; + char *pDocid = aDoclist; + char *pEnd = &aDoclist[nDoclist]; + int iMul = 1; + + while( pDocid=pEnd ){ - *pp = 0; + char *p = *ppIter; + + assert( nDoclist>0 ); + assert( *pbEof==0 ); + assert( p || *piDocid==0 ); + assert( !p || (p>=aDoclist && p<=&aDoclist[nDoclist]) ); + + if( p==0 ){ + p = aDoclist; + p += sqlite3Fts3GetVarint(p, piDocid); }else{ - sqlite3_int64 iVal; - *pp += sqlite3Fts3GetVarint(*pp, &iVal); - if( bDescIdx ){ - *pVal -= iVal; + fts3PoslistCopy(0, &p); + while( p<&aDoclist[nDoclist] && *p==0 ) p++; + if( p>=&aDoclist[nDoclist] ){ + *pbEof = 1; }else{ - *pVal += iVal; + sqlite3_int64 iVar; + p += sqlite3Fts3GetVarint(p, &iVar); + *piDocid += ((bDescIdx ? -1 : 1) * iVar); } } + + *ppIter = p; } /* -** This function is used to write a single varint to a buffer. The varint -** is written to *pp. Before returning, *pp is set to point 1 byte past the -** end of the value written. -** -** If *pbFirst is zero when this function is called, the value written to -** the buffer is that of parameter iVal. -** -** If *pbFirst is non-zero when this function is called, then the value -** written is either (iVal-*piPrev) (if bDescIdx is zero) or (*piPrev-iVal) -** (if bDescIdx is non-zero). -** -** Before returning, this function always sets *pbFirst to 1 and *piPrev -** to the value of parameter iVal. +** Advance the iterator pDL to the next entry in pDL->aAll/nAll. Set *pbEof +** to true if EOF is reached. */ -static void fts3PutDeltaVarint3( - char **pp, /* IN/OUT: Output pointer */ - int bDescIdx, /* True for descending docids */ - sqlite3_int64 *piPrev, /* IN/OUT: Previous value written to list */ - int *pbFirst, /* IN/OUT: True after first int written */ - sqlite3_int64 iVal /* Write this value to the list */ +static void fts3EvalDlPhraseNext( + Fts3Table *pTab, + Fts3Doclist *pDL, + u8 *pbEof ){ - sqlite3_int64 iWrite; - if( bDescIdx==0 || *pbFirst==0 ){ - iWrite = iVal - *piPrev; + char *pIter; /* Used to iterate through aAll */ + char *pEnd; /* 1 byte past end of aAll */ + + if( pDL->pNextDocid ){ + pIter = pDL->pNextDocid; + assert( pDL->aAll!=0 || pIter==0 ); }else{ - iWrite = *piPrev - iVal; + pIter = pDL->aAll; } - assert( *pbFirst || *piPrev==0 ); - assert( *pbFirst==0 || iWrite>0 ); - *pp += sqlite3Fts3PutVarint(*pp, iWrite); - *piPrev = iVal; - *pbFirst = 1; -} + if( pIter==0 || pIter>=(pEnd = pDL->aAll + pDL->nAll) ){ + /* We have already reached the end of this doclist. EOF. */ + *pbEof = 1; + }else{ + sqlite3_int64 iDelta; + pIter += sqlite3Fts3GetVarint(pIter, &iDelta); + if( pTab->bDescIdx==0 || pDL->pNextDocid==0 ){ + pDL->iDocid += iDelta; + }else{ + pDL->iDocid -= iDelta; + } + pDL->pList = pIter; + fts3PoslistCopy(0, &pIter); + pDL->nList = (int)(pIter - pDL->pList); + + /* pIter now points just past the 0x00 that terminates the position- + ** list for document pDL->iDocid. However, if this position-list was + ** edited in place by fts3EvalNearTrim(), then pIter may not actually + ** point to the start of the next docid value. The following line deals + ** with this case by advancing pIter past the zero-padding added by + ** fts3EvalNearTrim(). */ + while( pIterpNextDocid = pIter; + assert( pIter>=&pDL->aAll[pDL->nAll] || *pIter ); + *pbEof = 0; + } +} /* -** This macro is used by various functions that merge doclists. The two -** arguments are 64-bit docid values. If the value of the stack variable -** bDescDoclist is 0 when this macro is invoked, then it returns (i1-i2). -** Otherwise, (i2-i1). -** -** Using this makes it easier to write code that can merge doclists that are -** sorted in either ascending or descending order. +** Helper type used by fts3EvalIncrPhraseNext() and incrPhraseTokenNext(). */ -#define DOCID_CMP(i1, i2) ((bDescDoclist?-1:1) * (i1-i2)) +typedef struct TokenDoclist TokenDoclist; +struct TokenDoclist { + int bIgnore; + sqlite3_int64 iDocid; + char *pList; + int nList; +}; /* -** This function does an "OR" merge of two doclists (output contains all -** positions contained in either argument doclist). If the docids in the -** input doclists are sorted in ascending order, parameter bDescDoclist -** should be false. If they are sorted in ascending order, it should be -** passed a non-zero value. -** -** If no error occurs, *paOut is set to point at an sqlite3_malloc'd buffer -** containing the output doclist and SQLITE_OK is returned. In this case -** *pnOut is set to the number of bytes in the output doclist. +** Token pToken is an incrementally loaded token that is part of a +** multi-token phrase. Advance it to the next matching document in the +** database and populate output variable *p with the details of the new +** entry. Or, if the iterator has reached EOF, set *pbEof to true. ** -** If an error occurs, an SQLite error code is returned. The output values -** are undefined in this case. +** If an error occurs, return an SQLite error code. Otherwise, return +** SQLITE_OK. */ -static int fts3DoclistOrMerge( - int bDescDoclist, /* True if arguments are desc */ - char *a1, int n1, /* First doclist */ - char *a2, int n2, /* Second doclist */ - char **paOut, int *pnOut /* OUT: Malloc'd doclist */ +static int incrPhraseTokenNext( + Fts3Table *pTab, /* Virtual table handle */ + Fts3Phrase *pPhrase, /* Phrase to advance token of */ + int iToken, /* Specific token to advance */ + TokenDoclist *p, /* OUT: Docid and doclist for new entry */ + u8 *pbEof /* OUT: True if iterator is at EOF */ ){ - sqlite3_int64 i1 = 0; - sqlite3_int64 i2 = 0; - sqlite3_int64 iPrev = 0; - char *pEnd1 = &a1[n1]; - char *pEnd2 = &a2[n2]; - char *p1 = a1; - char *p2 = a2; - char *p; - char *aOut; - int bFirstOut = 0; - - *paOut = 0; - *pnOut = 0; - - /* Allocate space for the output. Both the input and output doclists - ** are delta encoded. If they are in ascending order (bDescDoclist==0), - ** then the first docid in each list is simply encoded as a varint. For - ** each subsequent docid, the varint stored is the difference between the - ** current and previous docid (a positive number - since the list is in - ** ascending order). - ** - ** The first docid written to the output is therefore encoded using the - ** same number of bytes as it is in whichever of the input lists it is - ** read from. And each subsequent docid read from the same input list - ** consumes either the same or less bytes as it did in the input (since - ** the difference between it and the previous value in the output must - ** be a positive value less than or equal to the delta value read from - ** the input list). The same argument applies to all but the first docid - ** read from the 'other' list. And to the contents of all position lists - ** that will be copied and merged from the input to the output. - ** - ** However, if the first docid copied to the output is a negative number, - ** then the encoding of the first docid from the 'other' input list may - ** be larger in the output than it was in the input (since the delta value - ** may be a larger positive integer than the actual docid). - ** - ** The space required to store the output is therefore the sum of the - ** sizes of the two inputs, plus enough space for exactly one of the input - ** docids to grow. - ** - ** A symetric argument may be made if the doclists are in descending - ** order. - */ - aOut = sqlite3_malloc(n1+n2+FTS3_VARINT_MAX-1); - if( !aOut ) return SQLITE_NOMEM; - - p = aOut; - fts3GetDeltaVarint3(&p1, pEnd1, 0, &i1); - fts3GetDeltaVarint3(&p2, pEnd2, 0, &i2); - while( p1 || p2 ){ - sqlite3_int64 iDiff = DOCID_CMP(i1, i2); + int rc = SQLITE_OK; - if( p2 && p1 && iDiff==0 ){ - fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i1); - fts3PoslistMerge(&p, &p1, &p2); - fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1); - fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2); - }else if( !p2 || (p1 && iDiff<0) ){ - fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i1); - fts3PoslistCopy(&p, &p1); - fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1); + if( pPhrase->iDoclistToken==iToken ){ + assert( p->bIgnore==0 ); + assert( pPhrase->aToken[iToken].pSegcsr==0 ); + fts3EvalDlPhraseNext(pTab, &pPhrase->doclist, pbEof); + p->pList = pPhrase->doclist.pList; + p->nList = pPhrase->doclist.nList; + p->iDocid = pPhrase->doclist.iDocid; + }else{ + Fts3PhraseToken *pToken = &pPhrase->aToken[iToken]; + assert( pToken->pDeferred==0 ); + assert( pToken->pSegcsr || pPhrase->iDoclistToken>=0 ); + if( pToken->pSegcsr ){ + assert( p->bIgnore==0 ); + rc = sqlite3Fts3MsrIncrNext( + pTab, pToken->pSegcsr, &p->iDocid, &p->pList, &p->nList + ); + if( p->pList==0 ) *pbEof = 1; }else{ - fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i2); - fts3PoslistCopy(&p, &p2); - fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2); + p->bIgnore = 1; } } - *paOut = aOut; - *pnOut = (int)(p-aOut); - assert( *pnOut<=n1+n2+FTS3_VARINT_MAX-1 ); - return SQLITE_OK; + return rc; } + /* -** This function does a "phrase" merge of two doclists. In a phrase merge, -** the output contains a copy of each position from the right-hand input -** doclist for which there is a position in the left-hand input doclist -** exactly nDist tokens before it. +** The phrase iterator passed as the second argument: ** -** If the docids in the input doclists are sorted in ascending order, -** parameter bDescDoclist should be false. If they are sorted in ascending -** order, it should be passed a non-zero value. +** * features at least one token that uses an incremental doclist, and ** -** The right-hand input doclist is overwritten by this function. +** * does not contain any deferred tokens. +** +** Advance it to the next matching documnent in the database and populate +** the Fts3Doclist.pList and nList fields. +** +** If there is no "next" entry and no error occurs, then *pbEof is set to +** 1 before returning. Otherwise, if no error occurs and the iterator is +** successfully advanced, *pbEof is set to 0. +** +** If an error occurs, return an SQLite error code. Otherwise, return +** SQLITE_OK. */ -static int fts3DoclistPhraseMerge( - int bDescDoclist, /* True if arguments are desc */ - int nDist, /* Distance from left to right (1=adjacent) */ - char *aLeft, int nLeft, /* Left doclist */ - char **paRight, int *pnRight /* IN/OUT: Right/output doclist */ +static int fts3EvalIncrPhraseNext( + Fts3Cursor *pCsr, /* FTS Cursor handle */ + Fts3Phrase *p, /* Phrase object to advance to next docid */ + u8 *pbEof /* OUT: Set to 1 if EOF */ ){ - sqlite3_int64 i1 = 0; - sqlite3_int64 i2 = 0; - sqlite3_int64 iPrev = 0; - char *aRight = *paRight; - char *pEnd1 = &aLeft[nLeft]; - char *pEnd2 = &aRight[*pnRight]; - char *p1 = aLeft; - char *p2 = aRight; - char *p; - int bFirstOut = 0; - char *aOut; + int rc = SQLITE_OK; + Fts3Doclist *pDL = &p->doclist; + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + u8 bEof = 0; - assert( nDist>0 ); - if( bDescDoclist ){ - aOut = sqlite3_malloc(*pnRight + FTS3_VARINT_MAX); - if( aOut==0 ) return SQLITE_NOMEM; + /* This is only called if it is guaranteed that the phrase has at least + ** one incremental token. In which case the bIncr flag is set. */ + assert( p->bIncr==1 ); + + if( p->nToken==1 ){ + rc = sqlite3Fts3MsrIncrNext(pTab, p->aToken[0].pSegcsr, + &pDL->iDocid, &pDL->pList, &pDL->nList + ); + if( pDL->pList==0 ) bEof = 1; }else{ - aOut = aRight; - } - p = aOut; + int bDescDoclist = pCsr->bDesc; + struct TokenDoclist a[MAX_INCR_PHRASE_TOKENS]; - fts3GetDeltaVarint3(&p1, pEnd1, 0, &i1); - fts3GetDeltaVarint3(&p2, pEnd2, 0, &i2); + memset(a, 0, sizeof(a)); + assert( p->nToken<=MAX_INCR_PHRASE_TOKENS ); + assert( p->iDoclistTokennToken && bEof==0; i++){ + rc = incrPhraseTokenNext(pTab, p, i, &a[i], &bEof); + if( a[i].bIgnore==0 && (bMaxSet==0 || DOCID_CMP(iMax, a[i].iDocid)<0) ){ + iMax = a[i].iDocid; + bMaxSet = 1; + } } - fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1); - fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2); - }else if( iDiff<0 ){ - fts3PoslistCopy(0, &p1); - fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1); - }else{ - fts3PoslistCopy(0, &p2); - fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2); - } - } - - *pnRight = (int)(p - aOut); - if( bDescDoclist ){ - sqlite3_free(aRight); - *paRight = aOut; - } - - return SQLITE_OK; -} + assert( rc!=SQLITE_OK || (p->nToken>=1 && a[p->nToken-1].bIgnore==0) ); + assert( rc!=SQLITE_OK || bMaxSet ); -/* -** Argument pList points to a position list nList bytes in size. This -** function checks to see if the position list contains any entries for -** a token in position 0 (of any column). If so, it writes argument iDelta -** to the output buffer pOut, followed by a position list consisting only -** of the entries from pList at position 0, and terminated by an 0x00 byte. -** The value returned is the number of bytes written to pOut (if any). -*/ -SQLITE_PRIVATE int sqlite3Fts3FirstFilter( - sqlite3_int64 iDelta, /* Varint that may be written to pOut */ - char *pList, /* Position list (no 0x00 term) */ - int nList, /* Size of pList in bytes */ - char *pOut /* Write output here */ -){ - int nOut = 0; - int bWritten = 0; /* True once iDelta has been written */ - char *p = pList; - char *pEnd = &pList[nList]; + /* Keep advancing iterators until they all point to the same document */ + for(i=0; inToken; i++){ + while( rc==SQLITE_OK && bEof==0 + && a[i].bIgnore==0 && DOCID_CMP(a[i].iDocid, iMax)<0 + ){ + rc = incrPhraseTokenNext(pTab, p, i, &a[i], &bEof); + if( DOCID_CMP(a[i].iDocid, iMax)>0 ){ + iMax = a[i].iDocid; + i = 0; + } + } + } - if( *p!=0x01 ){ - if( *p==0x02 ){ - nOut += sqlite3Fts3PutVarint(&pOut[nOut], iDelta); - pOut[nOut++] = 0x02; - bWritten = 1; - } - fts3ColumnlistCopy(0, &p); - } + /* Check if the current entries really are a phrase match */ + if( bEof==0 ){ + int nList = 0; + int nByte = a[p->nToken-1].nList; + char *aDoclist = sqlite3_malloc(nByte+FTS3_BUFFER_PADDING); + if( !aDoclist ) return SQLITE_NOMEM; + memcpy(aDoclist, a[p->nToken-1].pList, nByte+1); + memset(&aDoclist[nByte], 0, FTS3_BUFFER_PADDING); - while( pnToken-1); i++){ + if( a[i].bIgnore==0 ){ + char *pL = a[i].pList; + char *pR = aDoclist; + char *pOut = aDoclist; + int nDist = p->nToken-1-i; + int res = fts3PoslistPhraseMerge(&pOut, nDist, 0, 1, &pL, &pR); + if( res==0 ) break; + nList = (int)(pOut - aDoclist); + } + } + if( i==(p->nToken-1) ){ + pDL->iDocid = iMax; + pDL->pList = aDoclist; + pDL->nList = nList; + pDL->bFreeList = 1; + break; + } + sqlite3_free(aDoclist); } - pOut[nOut++] = 0x01; - nOut += sqlite3Fts3PutVarint(&pOut[nOut], iCol); - pOut[nOut++] = 0x02; } - fts3ColumnlistCopy(0, &p); - } - if( bWritten ){ - pOut[nOut++] = 0x00; } - return nOut; + *pbEof = bEof; + return rc; } - /* -** Merge all doclists in the TermSelect.aaOutput[] array into a single -** doclist stored in TermSelect.aaOutput[0]. If successful, delete all -** other doclists (except the aaOutput[0] one) and return SQLITE_OK. +** Attempt to move the phrase iterator to point to the next matching docid. +** If an error occurs, return an SQLite error code. Otherwise, return +** SQLITE_OK. ** -** If an OOM error occurs, return SQLITE_NOMEM. In this case it is -** the responsibility of the caller to free any doclists left in the -** TermSelect.aaOutput[] array. +** If there is no "next" entry and no error occurs, then *pbEof is set to +** 1 before returning. Otherwise, if no error occurs and the iterator is +** successfully advanced, *pbEof is set to 0. */ -static int fts3TermSelectFinishMerge(Fts3Table *p, TermSelect *pTS){ - char *aOut = 0; - int nOut = 0; - int i; - - /* Loop through the doclists in the aaOutput[] array. Merge them all - ** into a single doclist. - */ - for(i=0; iaaOutput); i++){ - if( pTS->aaOutput[i] ){ - if( !aOut ){ - aOut = pTS->aaOutput[i]; - nOut = pTS->anOutput[i]; - pTS->aaOutput[i] = 0; - }else{ - int nNew; - char *aNew; - - int rc = fts3DoclistOrMerge(p->bDescIdx, - pTS->aaOutput[i], pTS->anOutput[i], aOut, nOut, &aNew, &nNew - ); - if( rc!=SQLITE_OK ){ - sqlite3_free(aOut); - return rc; - } +static int fts3EvalPhraseNext( + Fts3Cursor *pCsr, /* FTS Cursor handle */ + Fts3Phrase *p, /* Phrase object to advance to next docid */ + u8 *pbEof /* OUT: Set to 1 if EOF */ +){ + int rc = SQLITE_OK; + Fts3Doclist *pDL = &p->doclist; + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; - sqlite3_free(pTS->aaOutput[i]); - sqlite3_free(aOut); - pTS->aaOutput[i] = 0; - aOut = aNew; - nOut = nNew; - } - } + if( p->bIncr ){ + rc = fts3EvalIncrPhraseNext(pCsr, p, pbEof); + }else if( pCsr->bDesc!=pTab->bDescIdx && pDL->nAll ){ + sqlite3Fts3DoclistPrev(pTab->bDescIdx, pDL->aAll, pDL->nAll, + &pDL->pNextDocid, &pDL->iDocid, &pDL->nList, pbEof + ); + pDL->pList = pDL->pNextDocid; + }else{ + fts3EvalDlPhraseNext(pTab, pDL, pbEof); } - pTS->aaOutput[0] = aOut; - pTS->anOutput[0] = nOut; - return SQLITE_OK; + return rc; } /* -** Merge the doclist aDoclist/nDoclist into the TermSelect object passed -** as the first argument. The merge is an "OR" merge (see function -** fts3DoclistOrMerge() for details). ** -** This function is called with the doclist for each term that matches -** a queried prefix. It merges all these doclists into one, the doclist -** for the specified prefix. Since there can be a very large number of -** doclists to merge, the merging is done pair-wise using the TermSelect -** object. +** If *pRc is not SQLITE_OK when this function is called, it is a no-op. +** Otherwise, fts3EvalPhraseStart() is called on all phrases within the +** expression. Also the Fts3Expr.bDeferred variable is set to true for any +** expressions for which all descendent tokens are deferred. ** -** This function returns SQLITE_OK if the merge is successful, or an -** SQLite error code (SQLITE_NOMEM) if an error occurs. +** If parameter bOptOk is zero, then it is guaranteed that the +** Fts3Phrase.doclist.aAll/nAll variables contain the entire doclist for +** each phrase in the expression (subject to deferred token processing). +** Or, if bOptOk is non-zero, then one or more tokens within the expression +** may be loaded incrementally, meaning doclist.aAll/nAll is not available. +** +** If an error occurs within this function, *pRc is set to an SQLite error +** code before returning. */ -static int fts3TermSelectMerge( - Fts3Table *p, /* FTS table handle */ - TermSelect *pTS, /* TermSelect object to merge into */ - char *aDoclist, /* Pointer to doclist */ - int nDoclist /* Size of aDoclist in bytes */ +static void fts3EvalStartReaders( + Fts3Cursor *pCsr, /* FTS Cursor handle */ + Fts3Expr *pExpr, /* Expression to initialize phrases in */ + int *pRc /* IN/OUT: Error code */ ){ - if( pTS->aaOutput[0]==0 ){ - /* If this is the first term selected, copy the doclist to the output - ** buffer using memcpy(). - ** - ** Add FTS3_VARINT_MAX bytes of unused space to the end of the - ** allocation. This is so as to ensure that the buffer is big enough - ** to hold the current doclist AND'd with any other doclist. If the - ** doclists are stored in order=ASC order, this padding would not be - ** required (since the size of [doclistA AND doclistB] is always less - ** than or equal to the size of [doclistA] in that case). But this is - ** not true for order=DESC. For example, a doclist containing (1, -1) - ** may be smaller than (-1), as in the first example the -1 may be stored - ** as a single-byte delta, whereas in the second it must be stored as a - ** FTS3_VARINT_MAX byte varint. - ** - ** Similar padding is added in the fts3DoclistOrMerge() function. - */ - pTS->aaOutput[0] = sqlite3_malloc(nDoclist + FTS3_VARINT_MAX + 1); - pTS->anOutput[0] = nDoclist; - if( pTS->aaOutput[0] ){ - memcpy(pTS->aaOutput[0], aDoclist, nDoclist); - }else{ - return SQLITE_NOMEM; - } - }else{ - char *aMerge = aDoclist; - int nMerge = nDoclist; - int iOut; - - for(iOut=0; iOutaaOutput); iOut++){ - if( pTS->aaOutput[iOut]==0 ){ - assert( iOut>0 ); - pTS->aaOutput[iOut] = aMerge; - pTS->anOutput[iOut] = nMerge; - break; - }else{ - char *aNew; - int nNew; - - int rc = fts3DoclistOrMerge(p->bDescIdx, aMerge, nMerge, - pTS->aaOutput[iOut], pTS->anOutput[iOut], &aNew, &nNew - ); - if( rc!=SQLITE_OK ){ - if( aMerge!=aDoclist ) sqlite3_free(aMerge); - return rc; - } - - if( aMerge!=aDoclist ) sqlite3_free(aMerge); - sqlite3_free(pTS->aaOutput[iOut]); - pTS->aaOutput[iOut] = 0; - - aMerge = aNew; - nMerge = nNew; - if( (iOut+1)==SizeofArray(pTS->aaOutput) ){ - pTS->aaOutput[iOut] = aMerge; - pTS->anOutput[iOut] = nMerge; + if( pExpr && SQLITE_OK==*pRc ){ + if( pExpr->eType==FTSQUERY_PHRASE ){ + int nToken = pExpr->pPhrase->nToken; + if( nToken ){ + int i; + for(i=0; ipPhrase->aToken[i].pDeferred==0 ) break; } + pExpr->bDeferred = (i==nToken); } + *pRc = fts3EvalPhraseStart(pCsr, 1, pExpr->pPhrase); + }else{ + fts3EvalStartReaders(pCsr, pExpr->pLeft, pRc); + fts3EvalStartReaders(pCsr, pExpr->pRight, pRc); + pExpr->bDeferred = (pExpr->pLeft->bDeferred && pExpr->pRight->bDeferred); } } - return SQLITE_OK; } /* -** Append SegReader object pNew to the end of the pCsr->apSegment[] array. +** An array of the following structures is assembled as part of the process +** of selecting tokens to defer before the query starts executing (as part +** of the xFilter() method). There is one element in the array for each +** token in the FTS expression. +** +** Tokens are divided into AND/NEAR clusters. All tokens in a cluster belong +** to phrases that are connected only by AND and NEAR operators (not OR or +** NOT). When determining tokens to defer, each AND/NEAR cluster is considered +** separately. The root of a tokens AND/NEAR cluster is stored in +** Fts3TokenAndCost.pRoot. */ -static int fts3SegReaderCursorAppend( - Fts3MultiSegReader *pCsr, - Fts3SegReader *pNew -){ - if( (pCsr->nSegment%16)==0 ){ - Fts3SegReader **apNew; - int nByte = (pCsr->nSegment + 16)*sizeof(Fts3SegReader*); - apNew = (Fts3SegReader **)sqlite3_realloc(pCsr->apSegment, nByte); - if( !apNew ){ - sqlite3Fts3SegReaderFree(pNew); - return SQLITE_NOMEM; - } - pCsr->apSegment = apNew; - } - pCsr->apSegment[pCsr->nSegment++] = pNew; - return SQLITE_OK; -} +typedef struct Fts3TokenAndCost Fts3TokenAndCost; +struct Fts3TokenAndCost { + Fts3Phrase *pPhrase; /* The phrase the token belongs to */ + int iToken; /* Position of token in phrase */ + Fts3PhraseToken *pToken; /* The token itself */ + Fts3Expr *pRoot; /* Root of NEAR/AND cluster */ + int nOvfl; /* Number of overflow pages to load doclist */ + int iCol; /* The column the token must match */ +}; /* -** Add seg-reader objects to the Fts3MultiSegReader object passed as the -** 8th argument. +** This function is used to populate an allocated Fts3TokenAndCost array. ** -** This function returns SQLITE_OK if successful, or an SQLite error code -** otherwise. +** If *pRc is not SQLITE_OK when this function is called, it is a no-op. +** Otherwise, if an error occurs during execution, *pRc is set to an +** SQLite error code. */ -static int fts3SegReaderCursor( - Fts3Table *p, /* FTS3 table handle */ - int iLangid, /* Language id */ - int iIndex, /* Index to search (from 0 to p->nIndex-1) */ - int iLevel, /* Level of segments to scan */ - const char *zTerm, /* Term to query for */ - int nTerm, /* Size of zTerm in bytes */ - int isPrefix, /* True for a prefix search */ - int isScan, /* True to scan from zTerm to EOF */ - Fts3MultiSegReader *pCsr /* Cursor object to populate */ +static void fts3EvalTokenCosts( + Fts3Cursor *pCsr, /* FTS Cursor handle */ + Fts3Expr *pRoot, /* Root of current AND/NEAR cluster */ + Fts3Expr *pExpr, /* Expression to consider */ + Fts3TokenAndCost **ppTC, /* Write new entries to *(*ppTC)++ */ + Fts3Expr ***ppOr, /* Write new OR root to *(*ppOr)++ */ + int *pRc /* IN/OUT: Error code */ ){ - int rc = SQLITE_OK; /* Error code */ - sqlite3_stmt *pStmt = 0; /* Statement to iterate through segments */ - int rc2; /* Result of sqlite3_reset() */ - - /* If iLevel is less than 0 and this is not a scan, include a seg-reader - ** for the pending-terms. If this is a scan, then this call must be being - ** made by an fts4aux module, not an FTS table. In this case calling - ** Fts3SegReaderPending might segfault, as the data structures used by - ** fts4aux are not completely populated. So it's easiest to filter these - ** calls out here. */ - if( iLevel<0 && p->aIndex ){ - Fts3SegReader *pSeg = 0; - rc = sqlite3Fts3SegReaderPending(p, iIndex, zTerm, nTerm, isPrefix||isScan, &pSeg); - if( rc==SQLITE_OK && pSeg ){ - rc = fts3SegReaderCursorAppend(pCsr, pSeg); - } - } - - if( iLevel!=FTS3_SEGCURSOR_PENDING ){ - if( rc==SQLITE_OK ){ - rc = sqlite3Fts3AllSegdirs(p, iLangid, iIndex, iLevel, &pStmt); - } - - while( rc==SQLITE_OK && SQLITE_ROW==(rc = sqlite3_step(pStmt)) ){ - Fts3SegReader *pSeg = 0; - - /* Read the values returned by the SELECT into local variables. */ - sqlite3_int64 iStartBlock = sqlite3_column_int64(pStmt, 1); - sqlite3_int64 iLeavesEndBlock = sqlite3_column_int64(pStmt, 2); - sqlite3_int64 iEndBlock = sqlite3_column_int64(pStmt, 3); - int nRoot = sqlite3_column_bytes(pStmt, 4); - char const *zRoot = sqlite3_column_blob(pStmt, 4); - - /* If zTerm is not NULL, and this segment is not stored entirely on its - ** root node, the range of leaves scanned can be reduced. Do this. */ - if( iStartBlock && zTerm ){ - sqlite3_int64 *pi = (isPrefix ? &iLeavesEndBlock : 0); - rc = fts3SelectLeaf(p, zTerm, nTerm, zRoot, nRoot, &iStartBlock, pi); - if( rc!=SQLITE_OK ) goto finished; - if( isPrefix==0 && isScan==0 ) iLeavesEndBlock = iStartBlock; + if( *pRc==SQLITE_OK ){ + if( pExpr->eType==FTSQUERY_PHRASE ){ + Fts3Phrase *pPhrase = pExpr->pPhrase; + int i; + for(i=0; *pRc==SQLITE_OK && inToken; i++){ + Fts3TokenAndCost *pTC = (*ppTC)++; + pTC->pPhrase = pPhrase; + pTC->iToken = i; + pTC->pRoot = pRoot; + pTC->pToken = &pPhrase->aToken[i]; + pTC->iCol = pPhrase->iColumn; + *pRc = sqlite3Fts3MsrOvfl(pCsr, pTC->pToken->pSegcsr, &pTC->nOvfl); } - - rc = sqlite3Fts3SegReaderNew(pCsr->nSegment+1, - (isPrefix==0 && isScan==0), - iStartBlock, iLeavesEndBlock, - iEndBlock, zRoot, nRoot, &pSeg + }else if( pExpr->eType!=FTSQUERY_NOT ){ + assert( pExpr->eType==FTSQUERY_OR + || pExpr->eType==FTSQUERY_AND + || pExpr->eType==FTSQUERY_NEAR ); - if( rc!=SQLITE_OK ) goto finished; - rc = fts3SegReaderCursorAppend(pCsr, pSeg); + assert( pExpr->pLeft && pExpr->pRight ); + if( pExpr->eType==FTSQUERY_OR ){ + pRoot = pExpr->pLeft; + **ppOr = pRoot; + (*ppOr)++; + } + fts3EvalTokenCosts(pCsr, pRoot, pExpr->pLeft, ppTC, ppOr, pRc); + if( pExpr->eType==FTSQUERY_OR ){ + pRoot = pExpr->pRight; + **ppOr = pRoot; + (*ppOr)++; + } + fts3EvalTokenCosts(pCsr, pRoot, pExpr->pRight, ppTC, ppOr, pRc); } } - - finished: - rc2 = sqlite3_reset(pStmt); - if( rc==SQLITE_DONE ) rc = rc2; - - return rc; } /* -** Set up a cursor object for iterating through a full-text index or a -** single level therein. +** Determine the average document (row) size in pages. If successful, +** write this value to *pnPage and return SQLITE_OK. Otherwise, return +** an SQLite error code. +** +** The average document size in pages is calculated by first calculating +** determining the average size in bytes, B. If B is less than the amount +** of data that will fit on a single leaf page of an intkey table in +** this database, then the average docsize is 1. Otherwise, it is 1 plus +** the number of overflow pages consumed by a record B bytes in size. */ -SQLITE_PRIVATE int sqlite3Fts3SegReaderCursor( - Fts3Table *p, /* FTS3 table handle */ - int iLangid, /* Language-id to search */ - int iIndex, /* Index to search (from 0 to p->nIndex-1) */ - int iLevel, /* Level of segments to scan */ - const char *zTerm, /* Term to query for */ - int nTerm, /* Size of zTerm in bytes */ - int isPrefix, /* True for a prefix search */ - int isScan, /* True to scan from zTerm to EOF */ - Fts3MultiSegReader *pCsr /* Cursor object to populate */ -){ - assert( iIndex>=0 && iIndexnIndex ); - assert( iLevel==FTS3_SEGCURSOR_ALL - || iLevel==FTS3_SEGCURSOR_PENDING - || iLevel>=0 - ); - assert( iLevelnRowAvg==0 ){ + /* The average document size, which is required to calculate the cost + ** of each doclist, has not yet been determined. Read the required + ** data from the %_stat table to calculate it. + ** + ** Entry 0 of the %_stat table is a blob containing (nCol+1) FTS3 + ** varints, where nCol is the number of columns in the FTS3 table. + ** The first varint is the number of documents currently stored in + ** the table. The following nCol varints contain the total amount of + ** data stored in all rows of each column of the table, from left + ** to right. + */ + Fts3Table *p = (Fts3Table*)pCsr->base.pVtab; + sqlite3_stmt *pStmt; + sqlite3_int64 nDoc = 0; + sqlite3_int64 nByte = 0; + const char *pEnd; + const char *a; - memset(pCsr, 0, sizeof(Fts3MultiSegReader)); - return fts3SegReaderCursor( - p, iLangid, iIndex, iLevel, zTerm, nTerm, isPrefix, isScan, pCsr - ); -} + rc = sqlite3Fts3SelectDoctotal(p, &pStmt); + if( rc!=SQLITE_OK ) return rc; + a = sqlite3_column_blob(pStmt, 0); + testcase( a==0 ); /* If %_stat.value set to X'' */ + if( a ){ + pEnd = &a[sqlite3_column_bytes(pStmt, 0)]; + a += sqlite3Fts3GetVarintBounded(a, pEnd, &nDoc); + while( anDoc = nDoc; + pCsr->nRowAvg = (int)(((nByte / nDoc) + p->nPgsz) / p->nPgsz); + assert( pCsr->nRowAvg>0 ); + rc = sqlite3_reset(pStmt); + } + + *pnPage = pCsr->nRowAvg; + return rc; } /* -** Open an Fts3MultiSegReader to scan the doclist for term zTerm/nTerm. Or, -** if isPrefix is true, to scan the doclist for all terms for which -** zTerm/nTerm is a prefix. If successful, return SQLITE_OK and write -** a pointer to the new Fts3MultiSegReader to *ppSegcsr. Otherwise, return -** an SQLite error code. +** This function is called to select the tokens (if any) that will be +** deferred. The array aTC[] has already been populated when this is +** called. ** -** It is the responsibility of the caller to free this object by eventually -** passing it to fts3SegReaderCursorFree() +** This function is called once for each AND/NEAR cluster in the +** expression. Each invocation determines which tokens to defer within +** the cluster with root node pRoot. See comments above the definition +** of struct Fts3TokenAndCost for more details. ** -** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code. -** Output parameter *ppSegcsr is set to 0 if an error occurs. +** If no error occurs, SQLITE_OK is returned and sqlite3Fts3DeferToken() +** called on each token to defer. Otherwise, an SQLite error code is +** returned. */ -static int fts3TermSegReaderCursor( - Fts3Cursor *pCsr, /* Virtual table cursor handle */ - const char *zTerm, /* Term to query for */ - int nTerm, /* Size of zTerm in bytes */ - int isPrefix, /* True for a prefix search */ - Fts3MultiSegReader **ppSegcsr /* OUT: Allocated seg-reader cursor */ +static int fts3EvalSelectDeferred( + Fts3Cursor *pCsr, /* FTS Cursor handle */ + Fts3Expr *pRoot, /* Consider tokens with this root node */ + Fts3TokenAndCost *aTC, /* Array of expression tokens and costs */ + int nTC /* Number of entries in aTC[] */ ){ - Fts3MultiSegReader *pSegcsr; /* Object to allocate and return */ - int rc = SQLITE_NOMEM; /* Return code */ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + int nDocSize = 0; /* Number of pages per doc loaded */ + int rc = SQLITE_OK; /* Return code */ + int ii; /* Iterator variable for various purposes */ + int nOvfl = 0; /* Total overflow pages used by doclists */ + int nToken = 0; /* Total number of tokens in cluster */ - pSegcsr = sqlite3_malloc(sizeof(Fts3MultiSegReader)); - if( pSegcsr ){ - int i; - int bFound = 0; /* True once an index has been found */ - Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; + int nMinEst = 0; /* The minimum count for any phrase so far. */ + int nLoad4 = 1; /* (Phrases that will be loaded)^4. */ - if( isPrefix ){ - for(i=1; bFound==0 && inIndex; i++){ - if( p->aIndex[i].nPrefix==nTerm ){ - bFound = 1; - rc = sqlite3Fts3SegReaderCursor(p, pCsr->iLangid, - i, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 0, 0, pSegcsr - ); - pSegcsr->bLookup = 1; - } + /* Tokens are never deferred for FTS tables created using the content=xxx + ** option. The reason being that it is not guaranteed that the content + ** table actually contains the same data as the index. To prevent this from + ** causing any problems, the deferred token optimization is completely + ** disabled for content=xxx tables. */ + if( pTab->zContentTbl ){ + return SQLITE_OK; + } + + /* Count the tokens in this AND/NEAR cluster. If none of the doclists + ** associated with the tokens spill onto overflow pages, or if there is + ** only 1 token, exit early. No tokens to defer in this case. */ + for(ii=0; ii0 ); + + + /* Iterate through all tokens in this AND/NEAR cluster, in ascending order + ** of the number of overflow pages that will be loaded by the pager layer + ** to retrieve the entire doclist for the token from the full-text index. + ** Load the doclists for tokens that are either: + ** + ** a. The cheapest token in the entire query (i.e. the one visited by the + ** first iteration of this loop), or + ** + ** b. Part of a multi-token phrase. + ** + ** After each token doclist is loaded, merge it with the others from the + ** same phrase and count the number of documents that the merged doclist + ** contains. Set variable "nMinEst" to the smallest number of documents in + ** any phrase doclist for which 1 or more token doclists have been loaded. + ** Let nOther be the number of other phrases for which it is certain that + ** one or more tokens will not be deferred. + ** + ** Then, for each token, defer it if loading the doclist would result in + ** loading N or more overflow pages into memory, where N is computed as: + ** + ** (nMinEst + 4^nOther - 1) / (4^nOther) + */ + for(ii=0; iinOvfl) + ){ + pTC = &aTC[iTC]; } + } + assert( pTC ); - for(i=1; bFound==0 && inIndex; i++){ - if( p->aIndex[i].nPrefix==nTerm+1 ){ - bFound = 1; - rc = sqlite3Fts3SegReaderCursor(p, pCsr->iLangid, - i, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 1, 0, pSegcsr + if( ii && pTC->nOvfl>=((nMinEst+(nLoad4/4)-1)/(nLoad4/4))*nDocSize ){ + /* The number of overflow pages to load for this (and therefore all + ** subsequent) tokens is greater than the estimated number of pages + ** that will be loaded if all subsequent tokens are deferred. + */ + Fts3PhraseToken *pToken = pTC->pToken; + rc = sqlite3Fts3DeferToken(pCsr, pToken, pTC->iCol); + fts3SegReaderCursorFree(pToken->pSegcsr); + pToken->pSegcsr = 0; + }else{ + /* Set nLoad4 to the value of (4^nOther) for the next iteration of the + ** for-loop. Except, limit the value to 2^24 to prevent it from + ** overflowing the 32-bit integer it is stored in. */ + if( ii<12 ) nLoad4 = nLoad4*4; + + if( ii==0 || (pTC->pPhrase->nToken>1 && ii!=nToken-1) ){ + /* Either this is the cheapest token in the entire query, or it is + ** part of a multi-token phrase. Either way, the entire doclist will + ** (eventually) be loaded into memory. It may as well be now. */ + Fts3PhraseToken *pToken = pTC->pToken; + int nList = 0; + char *pList = 0; + rc = fts3TermSelect(pTab, pToken, pTC->iCol, &nList, &pList); + assert( rc==SQLITE_OK || pList==0 ); + if( rc==SQLITE_OK ){ + rc = fts3EvalPhraseMergeToken( + pTab, pTC->pPhrase, pTC->iToken,pList,nList ); - if( rc==SQLITE_OK ){ - rc = fts3SegReaderCursorAddZero( - p, pCsr->iLangid, zTerm, nTerm, pSegcsr - ); - } + } + if( rc==SQLITE_OK ){ + int nCount; + nCount = fts3DoclistCountDocids( + pTC->pPhrase->doclist.aAll, pTC->pPhrase->doclist.nAll + ); + if( ii==0 || nCountiLangid, - 0, FTS3_SEGCURSOR_ALL, zTerm, nTerm, isPrefix, 0, pSegcsr - ); - pSegcsr->bLookup = !isPrefix; - } + pTC->pToken = 0; } - *ppSegcsr = pSegcsr; return rc; } /* -** Free an Fts3MultiSegReader allocated by fts3TermSegReaderCursor(). +** This function is called from within the xFilter method. It initializes +** the full-text query currently stored in pCsr->pExpr. To iterate through +** the results of a query, the caller does: +** +** fts3EvalStart(pCsr); +** while( 1 ){ +** fts3EvalNext(pCsr); +** if( pCsr->bEof ) break; +** ... return row pCsr->iPrevId to the caller ... +** } */ -static void fts3SegReaderCursorFree(Fts3MultiSegReader *pSegcsr){ - sqlite3Fts3SegReaderFinish(pSegcsr); - sqlite3_free(pSegcsr); -} +static int fts3EvalStart(Fts3Cursor *pCsr){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + int rc = SQLITE_OK; + int nToken = 0; + int nOr = 0; -/* -** This function retrieves the doclist for the specified term (or term -** prefix) from the database. -*/ -static int fts3TermSelect( - Fts3Table *p, /* Virtual table handle */ - Fts3PhraseToken *pTok, /* Token to query for */ - int iColumn, /* Column to query (or -ve for all columns) */ - int *pnOut, /* OUT: Size of buffer at *ppOut */ - char **ppOut /* OUT: Malloced result buffer */ -){ - int rc; /* Return code */ - Fts3MultiSegReader *pSegcsr; /* Seg-reader cursor for this term */ - TermSelect tsc; /* Object for pair-wise doclist merging */ - Fts3SegFilter filter; /* Segment term filter configuration */ + /* Allocate a MultiSegReader for each token in the expression. */ + fts3EvalAllocateReaders(pCsr, pCsr->pExpr, &nToken, &nOr, &rc); - pSegcsr = pTok->pSegcsr; - memset(&tsc, 0, sizeof(TermSelect)); + /* Determine which, if any, tokens in the expression should be deferred. */ +#ifndef SQLITE_DISABLE_FTS4_DEFERRED + if( rc==SQLITE_OK && nToken>1 && pTab->bFts4 ){ + Fts3TokenAndCost *aTC; + Fts3Expr **apOr; + aTC = (Fts3TokenAndCost *)sqlite3_malloc64( + sizeof(Fts3TokenAndCost) * nToken + + sizeof(Fts3Expr *) * nOr * 2 + ); + apOr = (Fts3Expr **)&aTC[nToken]; - filter.flags = FTS3_SEGMENT_IGNORE_EMPTY | FTS3_SEGMENT_REQUIRE_POS - | (pTok->isPrefix ? FTS3_SEGMENT_PREFIX : 0) - | (pTok->bFirst ? FTS3_SEGMENT_FIRST : 0) - | (iColumnnColumn ? FTS3_SEGMENT_COLUMN_FILTER : 0); - filter.iCol = iColumn; - filter.zTerm = pTok->z; - filter.nTerm = pTok->n; + if( !aTC ){ + rc = SQLITE_NOMEM; + }else{ + int ii; + Fts3TokenAndCost *pTC = aTC; + Fts3Expr **ppOr = apOr; - rc = sqlite3Fts3SegReaderStart(p, pSegcsr, &filter); - while( SQLITE_OK==rc - && SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, pSegcsr)) - ){ - rc = fts3TermSelectMerge(p, &tsc, pSegcsr->aDoclist, pSegcsr->nDoclist); - } + fts3EvalTokenCosts(pCsr, 0, pCsr->pExpr, &pTC, &ppOr, &rc); + nToken = (int)(pTC-aTC); + nOr = (int)(ppOr-apOr); - if( rc==SQLITE_OK ){ - rc = fts3TermSelectFinishMerge(p, &tsc); - } - if( rc==SQLITE_OK ){ - *ppOut = tsc.aaOutput[0]; - *pnOut = tsc.anOutput[0]; - }else{ - int i; - for(i=0; ipSegcsr = 0; + fts3EvalStartReaders(pCsr, pCsr->pExpr, &rc); return rc; } /* -** This function counts the total number of docids in the doclist stored -** in buffer aList[], size nList bytes. -** -** If the isPoslist argument is true, then it is assumed that the doclist -** contains a position-list following each docid. Otherwise, it is assumed -** that the doclist is simply a list of docids stored as delta encoded -** varints. +** Invalidate the current position list for phrase pPhrase. */ -static int fts3DoclistCountDocids(char *aList, int nList){ - int nDoc = 0; /* Return value */ - if( aList ){ - char *aEnd = &aList[nList]; /* Pointer to one byte after EOF */ - char *p = aList; /* Cursor */ - while( pdoclist.bFreeList ){ + sqlite3_free(pPhrase->doclist.pList); } - - return nDoc; + pPhrase->doclist.pList = 0; + pPhrase->doclist.nList = 0; + pPhrase->doclist.bFreeList = 0; } /* -** Advance the cursor to the next row in the %_content table that -** matches the search criteria. For a MATCH search, this will be -** the next row that matches. For a full-table scan, this will be -** simply the next row in the %_content table. For a docid lookup, -** this routine simply sets the EOF flag. +** This function is called to edit the position list associated with +** the phrase object passed as the fifth argument according to a NEAR +** condition. For example: ** -** Return SQLITE_OK if nothing goes wrong. SQLITE_OK is returned -** even if we reach end-of-file. The fts3EofMethod() will be called -** subsequently to determine whether or not an EOF was hit. -*/ -static int fts3NextMethod(sqlite3_vtab_cursor *pCursor){ - int rc; - Fts3Cursor *pCsr = (Fts3Cursor *)pCursor; - if( pCsr->eSearch==FTS3_DOCID_SEARCH || pCsr->eSearch==FTS3_FULLSCAN_SEARCH ){ - if( SQLITE_ROW!=sqlite3_step(pCsr->pStmt) ){ - pCsr->isEof = 1; - rc = sqlite3_reset(pCsr->pStmt); - }else{ - pCsr->iPrevId = sqlite3_column_int64(pCsr->pStmt, 0); - rc = SQLITE_OK; - } - }else{ - rc = fts3EvalNext((Fts3Cursor *)pCursor); - } - assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); - return rc; -} - -/* -** The following are copied from sqliteInt.h. +** abc NEAR/5 "def ghi" ** -** Constants for the largest and smallest possible 64-bit signed integers. -** These macros are designed to work correctly on both 32-bit and 64-bit -** compilers. +** Parameter nNear is passed the NEAR distance of the expression (5 in +** the example above). When this function is called, *paPoslist points to +** the position list, and *pnToken is the number of phrase tokens in, the +** phrase on the other side of the NEAR operator to pPhrase. For example, +** if pPhrase refers to the "def ghi" phrase, then *paPoslist points to +** the position list associated with phrase "abc". +** +** All positions in the pPhrase position list that are not sufficiently +** close to a position in the *paPoslist position list are removed. If this +** leaves 0 positions, zero is returned. Otherwise, non-zero. +** +** Before returning, *paPoslist is set to point to the position lsit +** associated with pPhrase. And *pnToken is set to the number of tokens in +** pPhrase. */ -#ifndef SQLITE_AMALGAMATION -# define LARGEST_INT64 (0xffffffff|(((sqlite3_int64)0x7fffffff)<<32)) -# define SMALLEST_INT64 (((sqlite3_int64)-1) - LARGEST_INT64) -#endif +static int fts3EvalNearTrim( + int nNear, /* NEAR distance. As in "NEAR/nNear". */ + char *aTmp, /* Temporary space to use */ + char **paPoslist, /* IN/OUT: Position list */ + int *pnToken, /* IN/OUT: Tokens in phrase of *paPoslist */ + Fts3Phrase *pPhrase /* The phrase object to trim the doclist of */ +){ + int nParam1 = nNear + pPhrase->nToken; + int nParam2 = nNear + *pnToken; + int nNew; + char *p2; + char *pOut; + int res; -/* -** If the numeric type of argument pVal is "integer", then return it -** converted to a 64-bit signed integer. Otherwise, return a copy of -** the second parameter, iDefault. -*/ -static sqlite3_int64 fts3DocidRange(sqlite3_value *pVal, i64 iDefault){ - if( pVal ){ - int eType = sqlite3_value_numeric_type(pVal); - if( eType==SQLITE_INTEGER ){ - return sqlite3_value_int64(pVal); - } + assert( pPhrase->doclist.pList ); + + p2 = pOut = pPhrase->doclist.pList; + res = fts3PoslistNearMerge( + &pOut, aTmp, nParam1, nParam2, paPoslist, &p2 + ); + if( res ){ + nNew = (int)(pOut - pPhrase->doclist.pList) - 1; + assert( pPhrase->doclist.pList[nNew]=='\0' ); + assert( nNew<=pPhrase->doclist.nList && nNew>0 ); + memset(&pPhrase->doclist.pList[nNew], 0, pPhrase->doclist.nList - nNew); + pPhrase->doclist.nList = nNew; + *paPoslist = pPhrase->doclist.pList; + *pnToken = pPhrase->nToken; } - return iDefault; + + return res; } /* -** This is the xFilter interface for the virtual table. See -** the virtual table xFilter method documentation for additional -** information. +** This function is a no-op if *pRc is other than SQLITE_OK when it is called. +** Otherwise, it advances the expression passed as the second argument to +** point to the next matching row in the database. Expressions iterate through +** matching rows in docid order. Ascending order if Fts3Cursor.bDesc is zero, +** or descending if it is non-zero. ** -** If idxNum==FTS3_FULLSCAN_SEARCH then do a full table scan against -** the %_content table. +** If an error occurs, *pRc is set to an SQLite error code. Otherwise, if +** successful, the following variables in pExpr are set: ** -** If idxNum==FTS3_DOCID_SEARCH then do a docid lookup for a single entry -** in the %_content table. +** Fts3Expr.bEof (non-zero if EOF - there is no next row) +** Fts3Expr.iDocid (valid if bEof==0. The docid of the next row) ** -** If idxNum>=FTS3_FULLTEXT_SEARCH then use the full text index. The -** column on the left-hand side of the MATCH operator is column -** number idxNum-FTS3_FULLTEXT_SEARCH, 0 indexed. argv[0] is the right-hand -** side of the MATCH operator. +** If the expression is of type FTSQUERY_PHRASE, and the expression is not +** at EOF, then the following variables are populated with the position list +** for the phrase for the visited row: +** +** FTs3Expr.pPhrase->doclist.nList (length of pList in bytes) +** FTs3Expr.pPhrase->doclist.pList (pointer to position list) +** +** It says above that this function advances the expression to the next +** matching row. This is usually true, but there are the following exceptions: +** +** 1. Deferred tokens are not taken into account. If a phrase consists +** entirely of deferred tokens, it is assumed to match every row in +** the db. In this case the position-list is not populated at all. +** +** Or, if a phrase contains one or more deferred tokens and one or +** more non-deferred tokens, then the expression is advanced to the +** next possible match, considering only non-deferred tokens. In other +** words, if the phrase is "A B C", and "B" is deferred, the expression +** is advanced to the next row that contains an instance of "A * C", +** where "*" may match any single token. The position list in this case +** is populated as for "A * C" before returning. +** +** 2. NEAR is treated as AND. If the expression is "x NEAR y", it is +** advanced to point to the next row that matches "x AND y". +** +** See sqlite3Fts3EvalTestDeferred() for details on testing if a row is +** really a match, taking into account deferred tokens and NEAR operators. */ -static int fts3FilterMethod( - sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */ - int idxNum, /* Strategy index */ - const char *idxStr, /* Unused */ - int nVal, /* Number of elements in apVal */ - sqlite3_value **apVal /* Arguments for the indexing scheme */ +static void fts3EvalNextRow( + Fts3Cursor *pCsr, /* FTS Cursor handle */ + Fts3Expr *pExpr, /* Expr. to advance to next matching row */ + int *pRc /* IN/OUT: Error code */ ){ - int rc = SQLITE_OK; - char *zSql; /* SQL statement used to access %_content */ - int eSearch; - Fts3Table *p = (Fts3Table *)pCursor->pVtab; - Fts3Cursor *pCsr = (Fts3Cursor *)pCursor; - - sqlite3_value *pCons = 0; /* The MATCH or rowid constraint, if any */ - sqlite3_value *pLangid = 0; /* The "langid = ?" constraint, if any */ - sqlite3_value *pDocidGe = 0; /* The "docid >= ?" constraint, if any */ - sqlite3_value *pDocidLe = 0; /* The "docid <= ?" constraint, if any */ - int iIdx; - - UNUSED_PARAMETER(idxStr); - UNUSED_PARAMETER(nVal); - - eSearch = (idxNum & 0x0000FFFF); - assert( eSearch>=0 && eSearch<=(FTS3_FULLTEXT_SEARCH+p->nColumn) ); - assert( p->pSegments==0 ); + if( *pRc==SQLITE_OK ){ + int bDescDoclist = pCsr->bDesc; /* Used by DOCID_CMP() macro */ + assert( pExpr->bEof==0 ); + pExpr->bStart = 1; - /* Collect arguments into local variables */ - iIdx = 0; - if( eSearch!=FTS3_FULLSCAN_SEARCH ) pCons = apVal[iIdx++]; - if( idxNum & FTS3_HAVE_LANGID ) pLangid = apVal[iIdx++]; - if( idxNum & FTS3_HAVE_DOCID_GE ) pDocidGe = apVal[iIdx++]; - if( idxNum & FTS3_HAVE_DOCID_LE ) pDocidLe = apVal[iIdx++]; - assert( iIdx==nVal ); + switch( pExpr->eType ){ + case FTSQUERY_NEAR: + case FTSQUERY_AND: { + Fts3Expr *pLeft = pExpr->pLeft; + Fts3Expr *pRight = pExpr->pRight; + assert( !pLeft->bDeferred || !pRight->bDeferred ); - /* In case the cursor has been used before, clear it now. */ - fts3ClearCursor(pCsr); + if( pLeft->bDeferred ){ + /* LHS is entirely deferred. So we assume it matches every row. + ** Advance the RHS iterator to find the next row visited. */ + fts3EvalNextRow(pCsr, pRight, pRc); + pExpr->iDocid = pRight->iDocid; + pExpr->bEof = pRight->bEof; + }else if( pRight->bDeferred ){ + /* RHS is entirely deferred. So we assume it matches every row. + ** Advance the LHS iterator to find the next row visited. */ + fts3EvalNextRow(pCsr, pLeft, pRc); + pExpr->iDocid = pLeft->iDocid; + pExpr->bEof = pLeft->bEof; + }else{ + /* Neither the RHS or LHS are deferred. */ + fts3EvalNextRow(pCsr, pLeft, pRc); + fts3EvalNextRow(pCsr, pRight, pRc); + while( !pLeft->bEof && !pRight->bEof && *pRc==SQLITE_OK ){ + sqlite3_int64 iDiff = DOCID_CMP(pLeft->iDocid, pRight->iDocid); + if( iDiff==0 ) break; + if( iDiff<0 ){ + fts3EvalNextRow(pCsr, pLeft, pRc); + }else{ + fts3EvalNextRow(pCsr, pRight, pRc); + } + } + pExpr->iDocid = pLeft->iDocid; + pExpr->bEof = (pLeft->bEof || pRight->bEof); + if( pExpr->eType==FTSQUERY_NEAR && pExpr->bEof ){ + assert( pRight->eType==FTSQUERY_PHRASE ); + if( pRight->pPhrase->doclist.aAll ){ + Fts3Doclist *pDl = &pRight->pPhrase->doclist; + while( *pRc==SQLITE_OK && pRight->bEof==0 ){ + memset(pDl->pList, 0, pDl->nList); + fts3EvalNextRow(pCsr, pRight, pRc); + } + } + if( pLeft->pPhrase && pLeft->pPhrase->doclist.aAll ){ + Fts3Doclist *pDl = &pLeft->pPhrase->doclist; + while( *pRc==SQLITE_OK && pLeft->bEof==0 ){ + memset(pDl->pList, 0, pDl->nList); + fts3EvalNextRow(pCsr, pLeft, pRc); + } + } + pRight->bEof = pLeft->bEof = 1; + } + } + break; + } + + case FTSQUERY_OR: { + Fts3Expr *pLeft = pExpr->pLeft; + Fts3Expr *pRight = pExpr->pRight; + sqlite3_int64 iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid); - /* Set the lower and upper bounds on docids to return */ - pCsr->iMinDocid = fts3DocidRange(pDocidGe, SMALLEST_INT64); - pCsr->iMaxDocid = fts3DocidRange(pDocidLe, LARGEST_INT64); + assert( pLeft->bStart || pLeft->iDocid==pRight->iDocid ); + assert( pRight->bStart || pLeft->iDocid==pRight->iDocid ); - if( idxStr ){ - pCsr->bDesc = (idxStr[0]=='D'); - }else{ - pCsr->bDesc = p->bDescIdx; - } - pCsr->eSearch = (i16)eSearch; + if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){ + fts3EvalNextRow(pCsr, pLeft, pRc); + }else if( pLeft->bEof || iCmp>0 ){ + fts3EvalNextRow(pCsr, pRight, pRc); + }else{ + fts3EvalNextRow(pCsr, pLeft, pRc); + fts3EvalNextRow(pCsr, pRight, pRc); + } - if( eSearch!=FTS3_DOCID_SEARCH && eSearch!=FTS3_FULLSCAN_SEARCH ){ - int iCol = eSearch-FTS3_FULLTEXT_SEARCH; - const char *zQuery = (const char *)sqlite3_value_text(pCons); + pExpr->bEof = (pLeft->bEof && pRight->bEof); + iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid); + if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){ + pExpr->iDocid = pLeft->iDocid; + }else{ + pExpr->iDocid = pRight->iDocid; + } - if( zQuery==0 && sqlite3_value_type(pCons)!=SQLITE_NULL ){ - return SQLITE_NOMEM; - } + break; + } - pCsr->iLangid = 0; - if( pLangid ) pCsr->iLangid = sqlite3_value_int(pLangid); + case FTSQUERY_NOT: { + Fts3Expr *pLeft = pExpr->pLeft; + Fts3Expr *pRight = pExpr->pRight; - assert( p->base.zErrMsg==0 ); - rc = sqlite3Fts3ExprParse(p->pTokenizer, pCsr->iLangid, - p->azColumn, p->bFts4, p->nColumn, iCol, zQuery, -1, &pCsr->pExpr, - &p->base.zErrMsg - ); - if( rc!=SQLITE_OK ){ - return rc; - } + if( pRight->bStart==0 ){ + fts3EvalNextRow(pCsr, pRight, pRc); + assert( *pRc!=SQLITE_OK || pRight->bStart ); + } - rc = fts3EvalStart(pCsr); - sqlite3Fts3SegmentsClose(p); - if( rc!=SQLITE_OK ) return rc; - pCsr->pNextId = pCsr->aDoclist; - pCsr->iPrevId = 0; - } + fts3EvalNextRow(pCsr, pLeft, pRc); + if( pLeft->bEof==0 ){ + while( !*pRc + && !pRight->bEof + && DOCID_CMP(pLeft->iDocid, pRight->iDocid)>0 + ){ + fts3EvalNextRow(pCsr, pRight, pRc); + } + } + pExpr->iDocid = pLeft->iDocid; + pExpr->bEof = pLeft->bEof; + break; + } - /* Compile a SELECT statement for this cursor. For a full-table-scan, the - ** statement loops through all rows of the %_content table. For a - ** full-text query or docid lookup, the statement retrieves a single - ** row by docid. - */ - if( eSearch==FTS3_FULLSCAN_SEARCH ){ - if( pDocidGe || pDocidLe ){ - zSql = sqlite3_mprintf( - "SELECT %s WHERE rowid BETWEEN %lld AND %lld ORDER BY rowid %s", - p->zReadExprlist, pCsr->iMinDocid, pCsr->iMaxDocid, - (pCsr->bDesc ? "DESC" : "ASC") - ); - }else{ - zSql = sqlite3_mprintf("SELECT %s ORDER BY rowid %s", - p->zReadExprlist, (pCsr->bDesc ? "DESC" : "ASC") - ); - } - if( zSql ){ - rc = sqlite3_prepare_v3(p->db,zSql,-1,SQLITE_PREPARE_PERSISTENT,&pCsr->pStmt,0); - sqlite3_free(zSql); - }else{ - rc = SQLITE_NOMEM; - } - }else if( eSearch==FTS3_DOCID_SEARCH ){ - rc = fts3CursorSeekStmt(pCsr); - if( rc==SQLITE_OK ){ - rc = sqlite3_bind_value(pCsr->pStmt, 1, pCons); + default: { + Fts3Phrase *pPhrase = pExpr->pPhrase; + fts3EvalInvalidatePoslist(pPhrase); + *pRc = fts3EvalPhraseNext(pCsr, pPhrase, &pExpr->bEof); + pExpr->iDocid = pPhrase->doclist.iDocid; + break; + } } } - if( rc!=SQLITE_OK ) return rc; - - return fts3NextMethod(pCursor); -} - -/* -** This is the xEof method of the virtual table. SQLite calls this -** routine to find out if it has reached the end of a result set. -*/ -static int fts3EofMethod(sqlite3_vtab_cursor *pCursor){ - Fts3Cursor *pCsr = (Fts3Cursor*)pCursor; - if( pCsr->isEof ){ - fts3ClearCursor(pCsr); - pCsr->isEof = 1; - } - return pCsr->isEof; -} - -/* -** This is the xRowid method. The SQLite core calls this routine to -** retrieve the rowid for the current row of the result set. fts3 -** exposes %_content.docid as the rowid for the virtual table. The -** rowid should be written to *pRowid. -*/ -static int fts3RowidMethod(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ - Fts3Cursor *pCsr = (Fts3Cursor *) pCursor; - *pRowid = pCsr->iPrevId; - return SQLITE_OK; } -/* -** This is the xColumn method, called by SQLite to request a value from -** the row that the supplied cursor currently points to. +/* +** If *pRc is not SQLITE_OK, or if pExpr is not the root node of a NEAR +** cluster, then this function returns 1 immediately. ** -** If: +** Otherwise, it checks if the current row really does match the NEAR +** expression, using the data currently stored in the position lists +** (Fts3Expr->pPhrase.doclist.pList/nList) for each phrase in the expression. ** -** (iCol < p->nColumn) -> The value of the iCol'th user column. -** (iCol == p->nColumn) -> Magic column with the same name as the table. -** (iCol == p->nColumn+1) -> Docid column -** (iCol == p->nColumn+2) -> Langid column +** If the current row is a match, the position list associated with each +** phrase in the NEAR expression is edited in place to contain only those +** phrase instances sufficiently close to their peers to satisfy all NEAR +** constraints. In this case it returns 1. If the NEAR expression does not +** match the current row, 0 is returned. The position lists may or may not +** be edited if 0 is returned. */ -static int fts3ColumnMethod( - sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ - sqlite3_context *pCtx, /* Context for sqlite3_result_xxx() calls */ - int iCol /* Index of column to read value from */ -){ - int rc = SQLITE_OK; /* Return Code */ - Fts3Cursor *pCsr = (Fts3Cursor *) pCursor; - Fts3Table *p = (Fts3Table *)pCursor->pVtab; - - /* The column value supplied by SQLite must be in range. */ - assert( iCol>=0 && iCol<=p->nColumn+2 ); +static int fts3EvalNearTest(Fts3Expr *pExpr, int *pRc){ + int res = 1; - switch( iCol-p->nColumn ){ - case 0: - /* The special 'table-name' column */ - sqlite3_result_pointer(pCtx, pCsr, "fts3cursor", 0); - break; + /* The following block runs if pExpr is the root of a NEAR query. + ** For example, the query: + ** + ** "w" NEAR "x" NEAR "y" NEAR "z" + ** + ** which is represented in tree form as: + ** + ** | + ** +--NEAR--+ <-- root of NEAR query + ** | | + ** +--NEAR--+ "z" + ** | | + ** +--NEAR--+ "y" + ** | | + ** "w" "x" + ** + ** The right-hand child of a NEAR node is always a phrase. The + ** left-hand child may be either a phrase or a NEAR node. There are + ** no exceptions to this - it's the way the parser in fts3_expr.c works. + */ + if( *pRc==SQLITE_OK + && pExpr->eType==FTSQUERY_NEAR + && (pExpr->pParent==0 || pExpr->pParent->eType!=FTSQUERY_NEAR) + ){ + Fts3Expr *p; + sqlite3_int64 nTmp = 0; /* Bytes of temp space */ + char *aTmp; /* Temp space for PoslistNearMerge() */ - case 1: - /* The docid column */ - sqlite3_result_int64(pCtx, pCsr->iPrevId); - break; + /* Allocate temporary working space. */ + for(p=pExpr; p->pLeft; p=p->pLeft){ + assert( p->pRight->pPhrase->doclist.nList>0 ); + nTmp += p->pRight->pPhrase->doclist.nList; + } + nTmp += p->pPhrase->doclist.nList; + aTmp = sqlite3_malloc64(nTmp*2); + if( !aTmp ){ + *pRc = SQLITE_NOMEM; + res = 0; + }else{ + char *aPoslist = p->pPhrase->doclist.pList; + int nToken = p->pPhrase->nToken; - case 2: - if( pCsr->pExpr ){ - sqlite3_result_int64(pCtx, pCsr->iLangid); - break; - }else if( p->zLanguageid==0 ){ - sqlite3_result_int(pCtx, 0); - break; - }else{ - iCol = p->nColumn; - /* fall-through */ + for(p=p->pParent;res && p && p->eType==FTSQUERY_NEAR; p=p->pParent){ + Fts3Phrase *pPhrase = p->pRight->pPhrase; + int nNear = p->nNear; + res = fts3EvalNearTrim(nNear, aTmp, &aPoslist, &nToken, pPhrase); } - default: - /* A user column. Or, if this is a full-table scan, possibly the - ** language-id column. Seek the cursor. */ - rc = fts3CursorSeek(0, pCsr); - if( rc==SQLITE_OK && sqlite3_data_count(pCsr->pStmt)-1>iCol ){ - sqlite3_result_value(pCtx, sqlite3_column_value(pCsr->pStmt, iCol+1)); + aPoslist = pExpr->pRight->pPhrase->doclist.pList; + nToken = pExpr->pRight->pPhrase->nToken; + for(p=pExpr->pLeft; p && res; p=p->pLeft){ + int nNear; + Fts3Phrase *pPhrase; + assert( p->pParent && p->pParent->pLeft==p ); + nNear = p->pParent->nNear; + pPhrase = ( + p->eType==FTSQUERY_NEAR ? p->pRight->pPhrase : p->pPhrase + ); + res = fts3EvalNearTrim(nNear, aTmp, &aPoslist, &nToken, pPhrase); } - break; - } + } - assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); - return rc; -} + sqlite3_free(aTmp); + } -/* -** This function is the implementation of the xUpdate callback used by -** FTS3 virtual tables. It is invoked by SQLite each time a row is to be -** inserted, updated or deleted. -*/ -static int fts3UpdateMethod( - sqlite3_vtab *pVtab, /* Virtual table handle */ - int nArg, /* Size of argument array */ - sqlite3_value **apVal, /* Array of arguments */ - sqlite_int64 *pRowid /* OUT: The affected (or effected) rowid */ -){ - return sqlite3Fts3UpdateMethod(pVtab, nArg, apVal, pRowid); + return res; } /* -** Implementation of xSync() method. Flush the contents of the pending-terms -** hash-table to the database. +** This function is a helper function for sqlite3Fts3EvalTestDeferred(). +** Assuming no error occurs or has occurred, It returns non-zero if the +** expression passed as the second argument matches the row that pCsr +** currently points to, or zero if it does not. +** +** If *pRc is not SQLITE_OK when this function is called, it is a no-op. +** If an error occurs during execution of this function, *pRc is set to +** the appropriate SQLite error code. In this case the returned value is +** undefined. */ -static int fts3SyncMethod(sqlite3_vtab *pVtab){ +static int fts3EvalTestExpr( + Fts3Cursor *pCsr, /* FTS cursor handle */ + Fts3Expr *pExpr, /* Expr to test. May or may not be root. */ + int *pRc /* IN/OUT: Error code */ +){ + int bHit = 1; /* Return value */ + if( *pRc==SQLITE_OK ){ + switch( pExpr->eType ){ + case FTSQUERY_NEAR: + case FTSQUERY_AND: + bHit = ( + fts3EvalTestExpr(pCsr, pExpr->pLeft, pRc) + && fts3EvalTestExpr(pCsr, pExpr->pRight, pRc) + && fts3EvalNearTest(pExpr, pRc) + ); - /* Following an incremental-merge operation, assuming that the input - ** segments are not completely consumed (the usual case), they are updated - ** in place to remove the entries that have already been merged. This - ** involves updating the leaf block that contains the smallest unmerged - ** entry and each block (if any) between the leaf and the root node. So - ** if the height of the input segment b-trees is N, and input segments - ** are merged eight at a time, updating the input segments at the end - ** of an incremental-merge requires writing (8*(1+N)) blocks. N is usually - ** small - often between 0 and 2. So the overhead of the incremental - ** merge is somewhere between 8 and 24 blocks. To avoid this overhead - ** dwarfing the actual productive work accomplished, the incremental merge - ** is only attempted if it will write at least 64 leaf blocks. Hence - ** nMinMerge. - ** - ** Of course, updating the input segments also involves deleting a bunch - ** of blocks from the segments table. But this is not considered overhead - ** as it would also be required by a crisis-merge that used the same input - ** segments. - */ - const u32 nMinMerge = 64; /* Minimum amount of incr-merge work to do */ + /* If the NEAR expression does not match any rows, zero the doclist for + ** all phrases involved in the NEAR. This is because the snippet(), + ** offsets() and matchinfo() functions are not supposed to recognize + ** any instances of phrases that are part of unmatched NEAR queries. + ** For example if this expression: + ** + ** ... MATCH 'a OR (b NEAR c)' + ** + ** is matched against a row containing: + ** + ** 'a b d e' + ** + ** then any snippet() should ony highlight the "a" term, not the "b" + ** (as "b" is part of a non-matching NEAR clause). + */ + if( bHit==0 + && pExpr->eType==FTSQUERY_NEAR + && (pExpr->pParent==0 || pExpr->pParent->eType!=FTSQUERY_NEAR) + ){ + Fts3Expr *p; + for(p=pExpr; p->pPhrase==0; p=p->pLeft){ + if( p->pRight->iDocid==pCsr->iPrevId ){ + fts3EvalInvalidatePoslist(p->pRight->pPhrase); + } + } + if( p->iDocid==pCsr->iPrevId ){ + fts3EvalInvalidatePoslist(p->pPhrase); + } + } - Fts3Table *p = (Fts3Table*)pVtab; - int rc; - i64 iLastRowid = sqlite3_last_insert_rowid(p->db); + break; - rc = sqlite3Fts3PendingTermsFlush(p); - if( rc==SQLITE_OK - && p->nLeafAdd>(nMinMerge/16) - && p->nAutoincrmerge && p->nAutoincrmerge!=0xff - ){ - int mxLevel = 0; /* Maximum relative level value in db */ - int A; /* Incr-merge parameter A */ + case FTSQUERY_OR: { + int bHit1 = fts3EvalTestExpr(pCsr, pExpr->pLeft, pRc); + int bHit2 = fts3EvalTestExpr(pCsr, pExpr->pRight, pRc); + bHit = bHit1 || bHit2; + break; + } - rc = sqlite3Fts3MaxLevel(p, &mxLevel); - assert( rc==SQLITE_OK || mxLevel==0 ); - A = p->nLeafAdd * mxLevel; - A += (A/2); - if( A>(int)nMinMerge ) rc = sqlite3Fts3Incrmerge(p, A, p->nAutoincrmerge); - } - sqlite3Fts3SegmentsClose(p); - sqlite3_set_last_insert_rowid(p->db, iLastRowid); - return rc; -} + case FTSQUERY_NOT: + bHit = ( + fts3EvalTestExpr(pCsr, pExpr->pLeft, pRc) + && !fts3EvalTestExpr(pCsr, pExpr->pRight, pRc) + ); + break; -/* -** If it is currently unknown whether or not the FTS table has an %_stat -** table (if p->bHasStat==2), attempt to determine this (set p->bHasStat -** to 0 or 1). Return SQLITE_OK if successful, or an SQLite error code -** if an error occurs. -*/ -static int fts3SetHasStat(Fts3Table *p){ - int rc = SQLITE_OK; - if( p->bHasStat==2 ){ - char *zTbl = sqlite3_mprintf("%s_stat", p->zName); - if( zTbl ){ - int res = sqlite3_table_column_metadata(p->db, p->zDb, zTbl, 0,0,0,0,0,0); - sqlite3_free(zTbl); - p->bHasStat = (res==SQLITE_OK); - }else{ - rc = SQLITE_NOMEM; + default: { +#ifndef SQLITE_DISABLE_FTS4_DEFERRED + if( pCsr->pDeferred + && (pExpr->iDocid==pCsr->iPrevId || pExpr->bDeferred) + ){ + Fts3Phrase *pPhrase = pExpr->pPhrase; + assert( pExpr->bDeferred || pPhrase->doclist.bFreeList==0 ); + if( pExpr->bDeferred ){ + fts3EvalInvalidatePoslist(pPhrase); + } + *pRc = fts3EvalDeferredPhrase(pCsr, pPhrase); + bHit = (pPhrase->doclist.pList!=0); + pExpr->iDocid = pCsr->iPrevId; + }else +#endif + { + bHit = (pExpr->bEof==0 && pExpr->iDocid==pCsr->iPrevId); + } + break; + } } } - return rc; -} - -/* -** Implementation of xBegin() method. -*/ -static int fts3BeginMethod(sqlite3_vtab *pVtab){ - Fts3Table *p = (Fts3Table*)pVtab; - UNUSED_PARAMETER(pVtab); - assert( p->pSegments==0 ); - assert( p->nPendingData==0 ); - assert( p->inTransaction!=1 ); - TESTONLY( p->inTransaction = 1 ); - TESTONLY( p->mxSavepoint = -1; ); - p->nLeafAdd = 0; - return fts3SetHasStat(p); -} - -/* -** Implementation of xCommit() method. This is a no-op. The contents of -** the pending-terms hash-table have already been flushed into the database -** by fts3SyncMethod(). -*/ -static int fts3CommitMethod(sqlite3_vtab *pVtab){ - TESTONLY( Fts3Table *p = (Fts3Table*)pVtab ); - UNUSED_PARAMETER(pVtab); - assert( p->nPendingData==0 ); - assert( p->inTransaction!=0 ); - assert( p->pSegments==0 ); - TESTONLY( p->inTransaction = 0 ); - TESTONLY( p->mxSavepoint = -1; ); - return SQLITE_OK; -} - -/* -** Implementation of xRollback(). Discard the contents of the pending-terms -** hash-table. Any changes made to the database are reverted by SQLite. -*/ -static int fts3RollbackMethod(sqlite3_vtab *pVtab){ - Fts3Table *p = (Fts3Table*)pVtab; - sqlite3Fts3PendingTermsClear(p); - assert( p->inTransaction!=0 ); - TESTONLY( p->inTransaction = 0 ); - TESTONLY( p->mxSavepoint = -1; ); - return SQLITE_OK; + return bHit; } /* -** When called, *ppPoslist must point to the byte immediately following the -** end of a position-list. i.e. ( (*ppPoslist)[-1]==POS_END ). This function -** moves *ppPoslist so that it instead points to the first byte of the -** same position list. +** This function is called as the second part of each xNext operation when +** iterating through the results of a full-text query. At this point the +** cursor points to a row that matches the query expression, with the +** following caveats: +** +** * Up until this point, "NEAR" operators in the expression have been +** treated as "AND". +** +** * Deferred tokens have not yet been considered. +** +** If *pRc is not SQLITE_OK when this function is called, it immediately +** returns 0. Otherwise, it tests whether or not after considering NEAR +** operators and deferred tokens the current row is still a match for the +** expression. It returns 1 if both of the following are true: +** +** 1. *pRc is SQLITE_OK when this function returns, and +** +** 2. After scanning the current FTS table row for the deferred tokens, +** it is determined that the row does *not* match the query. +** +** Or, if no error occurs and it seems the current row does match the FTS +** query, return 0. */ -static void fts3ReversePoslist(char *pStart, char **ppPoslist){ - char *p = &(*ppPoslist)[-2]; - char c = 0; +SQLITE_PRIVATE int sqlite3Fts3EvalTestDeferred(Fts3Cursor *pCsr, int *pRc){ + int rc = *pRc; + int bMiss = 0; + if( rc==SQLITE_OK ){ - /* Skip backwards passed any trailing 0x00 bytes added by NearTrim() */ - while( p>pStart && (c=*p--)==0 ); + /* If there are one or more deferred tokens, load the current row into + ** memory and scan it to determine the position list for each deferred + ** token. Then, see if this row is really a match, considering deferred + ** tokens and NEAR operators (neither of which were taken into account + ** earlier, by fts3EvalNextRow()). + */ + if( pCsr->pDeferred ){ + rc = fts3CursorSeek(0, pCsr); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3CacheDeferredDoclists(pCsr); + } + } + bMiss = (0==fts3EvalTestExpr(pCsr, pCsr->pExpr, &rc)); - /* Search backwards for a varint with value zero (the end of the previous - ** poslist). This is an 0x00 byte preceded by some byte that does not - ** have the 0x80 bit set. */ - while( p>pStart && (*p & 0x80) | c ){ - c = *p--; + /* Free the position-lists accumulated for each deferred token above. */ + sqlite3Fts3FreeDeferredDoclists(pCsr); + *pRc = rc; } - assert( p==pStart || c==0 ); - - /* At this point p points to that preceding byte without the 0x80 bit - ** set. So to find the start of the poslist, skip forward 2 bytes then - ** over a varint. - ** - ** Normally. The other case is that p==pStart and the poslist to return - ** is the first in the doclist. In this case do not skip forward 2 bytes. - ** The second part of the if condition (c==0 && *ppPoslist>&p[2]) - ** is required for cases where the first byte of a doclist and the - ** doclist is empty. For example, if the first docid is 10, a doclist - ** that begins with: - ** - ** 0x0A 0x00 - */ - if( p>pStart || (c==0 && *ppPoslist>&p[2]) ){ p = &p[2]; } - while( *p++&0x80 ); - *ppPoslist = p; + return (rc==SQLITE_OK && bMiss); } /* -** Helper function used by the implementation of the overloaded snippet(), -** offsets() and optimize() SQL functions. -** -** If the value passed as the third argument is a blob of size -** sizeof(Fts3Cursor*), then the blob contents are copied to the -** output variable *ppCsr and SQLITE_OK is returned. Otherwise, an error -** message is written to context pContext and SQLITE_ERROR returned. The -** string passed via zFunc is used as part of the error message. +** Advance to the next document that matches the FTS expression in +** Fts3Cursor.pExpr. */ -static int fts3FunctionArg( - sqlite3_context *pContext, /* SQL function call context */ - const char *zFunc, /* Function name */ - sqlite3_value *pVal, /* argv[0] passed to function */ - Fts3Cursor **ppCsr /* OUT: Store cursor handle here */ -){ - int rc; - *ppCsr = (Fts3Cursor*)sqlite3_value_pointer(pVal, "fts3cursor"); - if( (*ppCsr)!=0 ){ - rc = SQLITE_OK; +static int fts3EvalNext(Fts3Cursor *pCsr){ + int rc = SQLITE_OK; /* Return Code */ + Fts3Expr *pExpr = pCsr->pExpr; + assert( pCsr->isEof==0 ); + if( pExpr==0 ){ + pCsr->isEof = 1; }else{ - char *zErr = sqlite3_mprintf("illegal first argument to %s", zFunc); - sqlite3_result_error(pContext, zErr, -1); - sqlite3_free(zErr); - rc = SQLITE_ERROR; + do { + if( pCsr->isRequireSeek==0 ){ + sqlite3_reset(pCsr->pStmt); + } + assert( sqlite3_data_count(pCsr->pStmt)==0 ); + fts3EvalNextRow(pCsr, pExpr, &rc); + pCsr->isEof = pExpr->bEof; + pCsr->isRequireSeek = 1; + pCsr->isMatchinfoNeeded = 1; + pCsr->iPrevId = pExpr->iDocid; + }while( pCsr->isEof==0 && sqlite3Fts3EvalTestDeferred(pCsr, &rc) ); + } + + /* Check if the cursor is past the end of the docid range specified + ** by Fts3Cursor.iMinDocid/iMaxDocid. If so, set the EOF flag. */ + if( rc==SQLITE_OK && ( + (pCsr->bDesc==0 && pCsr->iPrevId>pCsr->iMaxDocid) + || (pCsr->bDesc!=0 && pCsr->iPrevIdiMinDocid) + )){ + pCsr->isEof = 1; } + return rc; } /* -** Implementation of the snippet() function for FTS3 +** Restart interation for expression pExpr so that the next call to +** fts3EvalNext() visits the first row. Do not allow incremental +** loading or merging of phrase doclists for this iteration. +** +** If *pRc is other than SQLITE_OK when this function is called, it is +** a no-op. If an error occurs within this function, *pRc is set to an +** SQLite error code before returning. */ -static void fts3SnippetFunc( - sqlite3_context *pContext, /* SQLite function call context */ - int nVal, /* Size of apVal[] array */ - sqlite3_value **apVal /* Array of arguments */ +static void fts3EvalRestart( + Fts3Cursor *pCsr, + Fts3Expr *pExpr, + int *pRc ){ - Fts3Cursor *pCsr; /* Cursor handle passed through apVal[0] */ - const char *zStart = ""; - const char *zEnd = ""; - const char *zEllipsis = "..."; - int iCol = -1; - int nToken = 15; /* Default number of tokens in snippet */ + if( pExpr && *pRc==SQLITE_OK ){ + Fts3Phrase *pPhrase = pExpr->pPhrase; - /* There must be at least one argument passed to this function (otherwise - ** the non-overloaded version would have been called instead of this one). - */ - assert( nVal>=1 ); + if( pPhrase ){ + fts3EvalInvalidatePoslist(pPhrase); + if( pPhrase->bIncr ){ + int i; + for(i=0; inToken; i++){ + Fts3PhraseToken *pToken = &pPhrase->aToken[i]; + assert( pToken->pDeferred==0 ); + if( pToken->pSegcsr ){ + sqlite3Fts3MsrIncrRestart(pToken->pSegcsr); + } + } + *pRc = fts3EvalPhraseStart(pCsr, 0, pPhrase); + } + pPhrase->doclist.pNextDocid = 0; + pPhrase->doclist.iDocid = 0; + pPhrase->pOrPoslist = 0; + } - if( nVal>6 ){ - sqlite3_result_error(pContext, - "wrong number of arguments to function snippet()", -1); - return; - } - if( fts3FunctionArg(pContext, "snippet", apVal[0], &pCsr) ) return; + pExpr->iDocid = 0; + pExpr->bEof = 0; + pExpr->bStart = 0; - switch( nVal ){ - case 6: nToken = sqlite3_value_int(apVal[5]); - case 5: iCol = sqlite3_value_int(apVal[4]); - case 4: zEllipsis = (const char*)sqlite3_value_text(apVal[3]); - case 3: zEnd = (const char*)sqlite3_value_text(apVal[2]); - case 2: zStart = (const char*)sqlite3_value_text(apVal[1]); - } - if( !zEllipsis || !zEnd || !zStart ){ - sqlite3_result_error_nomem(pContext); - }else if( nToken==0 ){ - sqlite3_result_text(pContext, "", -1, SQLITE_STATIC); - }else if( SQLITE_OK==fts3CursorSeek(pContext, pCsr) ){ - sqlite3Fts3Snippet(pContext, pCsr, zStart, zEnd, zEllipsis, iCol, nToken); + fts3EvalRestart(pCsr, pExpr->pLeft, pRc); + fts3EvalRestart(pCsr, pExpr->pRight, pRc); } } /* -** Implementation of the offsets() function for FTS3 +** After allocating the Fts3Expr.aMI[] array for each phrase in the +** expression rooted at pExpr, the cursor iterates through all rows matched +** by pExpr, calling this function for each row. This function increments +** the values in Fts3Expr.aMI[] according to the position-list currently +** found in Fts3Expr.pPhrase->doclist.pList for each of the phrase +** expression nodes. */ -static void fts3OffsetsFunc( - sqlite3_context *pContext, /* SQLite function call context */ - int nVal, /* Size of argument array */ - sqlite3_value **apVal /* Array of arguments */ -){ - Fts3Cursor *pCsr; /* Cursor handle passed through apVal[0] */ +static void fts3EvalUpdateCounts(Fts3Expr *pExpr, int nCol){ + if( pExpr ){ + Fts3Phrase *pPhrase = pExpr->pPhrase; + if( pPhrase && pPhrase->doclist.pList ){ + int iCol = 0; + char *p = pPhrase->doclist.pList; - UNUSED_PARAMETER(nVal); + do{ + u8 c = 0; + int iCnt = 0; + while( 0xFE & (*p | c) ){ + if( (c&0x80)==0 ) iCnt++; + c = *p++ & 0x80; + } - assert( nVal==1 ); - if( fts3FunctionArg(pContext, "offsets", apVal[0], &pCsr) ) return; - assert( pCsr ); - if( SQLITE_OK==fts3CursorSeek(pContext, pCsr) ){ - sqlite3Fts3Offsets(pContext, pCsr); + /* aMI[iCol*3 + 1] = Number of occurrences + ** aMI[iCol*3 + 2] = Number of rows containing at least one instance + */ + pExpr->aMI[iCol*3 + 1] += iCnt; + pExpr->aMI[iCol*3 + 2] += (iCnt>0); + if( *p==0x00 ) break; + p++; + p += fts3GetVarint32(p, &iCol); + }while( iColpLeft, nCol); + fts3EvalUpdateCounts(pExpr->pRight, nCol); } } -/* -** Implementation of the special optimize() function for FTS3. This -** function merges all segments in the database to a single segment. -** Example usage is: +/* +** Expression pExpr must be of type FTSQUERY_PHRASE. ** -** SELECT optimize(t) FROM t LIMIT 1; +** If it is not already allocated and populated, this function allocates and +** populates the Fts3Expr.aMI[] array for expression pExpr. If pExpr is part +** of a NEAR expression, then it also allocates and populates the same array +** for all other phrases that are part of the NEAR expression. ** -** where 't' is the name of an FTS3 table. +** SQLITE_OK is returned if the aMI[] array is successfully allocated and +** populated. Otherwise, if an error occurs, an SQLite error code is returned. */ -static void fts3OptimizeFunc( - sqlite3_context *pContext, /* SQLite function call context */ - int nVal, /* Size of argument array */ - sqlite3_value **apVal /* Array of arguments */ +static int fts3EvalGatherStats( + Fts3Cursor *pCsr, /* Cursor object */ + Fts3Expr *pExpr /* FTSQUERY_PHRASE expression */ ){ - int rc; /* Return code */ - Fts3Table *p; /* Virtual table handle */ - Fts3Cursor *pCursor; /* Cursor handle passed through apVal[0] */ + int rc = SQLITE_OK; /* Return code */ - UNUSED_PARAMETER(nVal); + assert( pExpr->eType==FTSQUERY_PHRASE ); + if( pExpr->aMI==0 ){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + Fts3Expr *pRoot; /* Root of NEAR expression */ + Fts3Expr *p; /* Iterator used for several purposes */ - assert( nVal==1 ); - if( fts3FunctionArg(pContext, "optimize", apVal[0], &pCursor) ) return; - p = (Fts3Table *)pCursor->base.pVtab; - assert( p ); + sqlite3_int64 iPrevId = pCsr->iPrevId; + sqlite3_int64 iDocid; + u8 bEof; - rc = sqlite3Fts3Optimize(p); + /* Find the root of the NEAR expression */ + pRoot = pExpr; + while( pRoot->pParent && pRoot->pParent->eType==FTSQUERY_NEAR ){ + pRoot = pRoot->pParent; + } + iDocid = pRoot->iDocid; + bEof = pRoot->bEof; + assert( pRoot->bStart ); - switch( rc ){ - case SQLITE_OK: - sqlite3_result_text(pContext, "Index optimized", -1, SQLITE_STATIC); - break; - case SQLITE_DONE: - sqlite3_result_text(pContext, "Index already optimal", -1, SQLITE_STATIC); - break; - default: - sqlite3_result_error_code(pContext, rc); - break; - } -} + /* Allocate space for the aMSI[] array of each FTSQUERY_PHRASE node */ + for(p=pRoot; p; p=p->pLeft){ + Fts3Expr *pE = (p->eType==FTSQUERY_PHRASE?p:p->pRight); + assert( pE->aMI==0 ); + pE->aMI = (u32 *)sqlite3_malloc64(pTab->nColumn * 3 * sizeof(u32)); + if( !pE->aMI ) return SQLITE_NOMEM; + memset(pE->aMI, 0, pTab->nColumn * 3 * sizeof(u32)); + } -/* -** Implementation of the matchinfo() function for FTS3 -*/ -static void fts3MatchinfoFunc( - sqlite3_context *pContext, /* SQLite function call context */ - int nVal, /* Size of argument array */ - sqlite3_value **apVal /* Array of arguments */ -){ - Fts3Cursor *pCsr; /* Cursor handle passed through apVal[0] */ - assert( nVal==1 || nVal==2 ); - if( SQLITE_OK==fts3FunctionArg(pContext, "matchinfo", apVal[0], &pCsr) ){ - const char *zArg = 0; - if( nVal>1 ){ - zArg = (const char *)sqlite3_value_text(apVal[1]); + fts3EvalRestart(pCsr, pRoot, &rc); + + while( pCsr->isEof==0 && rc==SQLITE_OK ){ + + do { + /* Ensure the %_content statement is reset. */ + if( pCsr->isRequireSeek==0 ) sqlite3_reset(pCsr->pStmt); + assert( sqlite3_data_count(pCsr->pStmt)==0 ); + + /* Advance to the next document */ + fts3EvalNextRow(pCsr, pRoot, &rc); + pCsr->isEof = pRoot->bEof; + pCsr->isRequireSeek = 1; + pCsr->isMatchinfoNeeded = 1; + pCsr->iPrevId = pRoot->iDocid; + }while( pCsr->isEof==0 + && pRoot->eType==FTSQUERY_NEAR + && sqlite3Fts3EvalTestDeferred(pCsr, &rc) + ); + + if( rc==SQLITE_OK && pCsr->isEof==0 ){ + fts3EvalUpdateCounts(pRoot, pTab->nColumn); + } + } + + pCsr->isEof = 0; + pCsr->iPrevId = iPrevId; + + if( bEof ){ + pRoot->bEof = bEof; + }else{ + /* Caution: pRoot may iterate through docids in ascending or descending + ** order. For this reason, even though it seems more defensive, the + ** do loop can not be written: + ** + ** do {...} while( pRoot->iDocidbEof==0 ); + }while( pRoot->iDocid!=iDocid && rc==SQLITE_OK ); } - sqlite3Fts3Matchinfo(pContext, pCsr, zArg); } + return rc; } /* -** This routine implements the xFindFunction method for the FTS3 -** virtual table. +** This function is used by the matchinfo() module to query a phrase +** expression node for the following information: +** +** 1. The total number of occurrences of the phrase in each column of +** the FTS table (considering all rows), and +** +** 2. For each column, the number of rows in the table for which the +** column contains at least one instance of the phrase. +** +** If no error occurs, SQLITE_OK is returned and the values for each column +** written into the array aiOut as follows: +** +** aiOut[iCol*3 + 1] = Number of occurrences +** aiOut[iCol*3 + 2] = Number of rows containing at least one instance +** +** Caveats: +** +** * If a phrase consists entirely of deferred tokens, then all output +** values are set to the number of documents in the table. In other +** words we assume that very common tokens occur exactly once in each +** column of each row of the table. +** +** * If a phrase contains some deferred tokens (and some non-deferred +** tokens), count the potential occurrence identified by considering +** the non-deferred tokens instead of actual phrase occurrences. +** +** * If the phrase is part of a NEAR expression, then only phrase instances +** that meet the NEAR constraint are included in the counts. */ -static int fts3FindFunctionMethod( - sqlite3_vtab *pVtab, /* Virtual table handle */ - int nArg, /* Number of SQL function arguments */ - const char *zName, /* Name of SQL function */ - void (**pxFunc)(sqlite3_context*,int,sqlite3_value**), /* OUT: Result */ - void **ppArg /* Unused */ +SQLITE_PRIVATE int sqlite3Fts3EvalPhraseStats( + Fts3Cursor *pCsr, /* FTS cursor handle */ + Fts3Expr *pExpr, /* Phrase expression */ + u32 *aiOut /* Array to write results into (see above) */ ){ - struct Overloaded { - const char *zName; - void (*xFunc)(sqlite3_context*,int,sqlite3_value**); - } aOverload[] = { - { "snippet", fts3SnippetFunc }, - { "offsets", fts3OffsetsFunc }, - { "optimize", fts3OptimizeFunc }, - { "matchinfo", fts3MatchinfoFunc }, - }; - int i; /* Iterator variable */ - - UNUSED_PARAMETER(pVtab); - UNUSED_PARAMETER(nArg); - UNUSED_PARAMETER(ppArg); + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + int rc = SQLITE_OK; + int iCol; - for(i=0; ibDeferred && pExpr->pParent->eType!=FTSQUERY_NEAR ){ + assert( pCsr->nDoc>0 ); + for(iCol=0; iColnColumn; iCol++){ + aiOut[iCol*3 + 1] = (u32)pCsr->nDoc; + aiOut[iCol*3 + 2] = (u32)pCsr->nDoc; + } + }else{ + rc = fts3EvalGatherStats(pCsr, pExpr); + if( rc==SQLITE_OK ){ + assert( pExpr->aMI ); + for(iCol=0; iColnColumn; iCol++){ + aiOut[iCol*3 + 1] = pExpr->aMI[iCol*3 + 1]; + aiOut[iCol*3 + 2] = pExpr->aMI[iCol*3 + 2]; + } } } - /* No function of the specified name was found. Return 0. */ - return 0; + return rc; } /* -** Implementation of FTS3 xRename method. Rename an fts3 table. +** The expression pExpr passed as the second argument to this function +** must be of type FTSQUERY_PHRASE. +** +** The returned value is either NULL or a pointer to a buffer containing +** a position-list indicating the occurrences of the phrase in column iCol +** of the current row. +** +** More specifically, the returned buffer contains 1 varint for each +** occurrence of the phrase in the column, stored using the normal (delta+2) +** compression and is terminated by either an 0x01 or 0x00 byte. For example, +** if the requested column contains "a b X c d X X" and the position-list +** for 'X' is requested, the buffer returned may contain: +** +** 0x04 0x05 0x03 0x01 or 0x04 0x05 0x03 0x00 +** +** This function works regardless of whether or not the phrase is deferred, +** incremental, or neither. */ -static int fts3RenameMethod( - sqlite3_vtab *pVtab, /* Virtual table handle */ - const char *zName /* New name of table */ +SQLITE_PRIVATE int sqlite3Fts3EvalPhrasePoslist( + Fts3Cursor *pCsr, /* FTS3 cursor object */ + Fts3Expr *pExpr, /* Phrase to return doclist for */ + int iCol, /* Column to return position list for */ + char **ppOut /* OUT: Pointer to position list */ ){ - Fts3Table *p = (Fts3Table *)pVtab; - sqlite3 *db = p->db; /* Database connection */ - int rc; /* Return Code */ + Fts3Phrase *pPhrase = pExpr->pPhrase; + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + char *pIter; + int iThis; + sqlite3_int64 iDocid; - /* At this point it must be known if the %_stat table exists or not. - ** So bHasStat may not be 2. */ - rc = fts3SetHasStat(p); - - /* As it happens, the pending terms table is always empty here. This is - ** because an "ALTER TABLE RENAME TABLE" statement inside a transaction - ** always opens a savepoint transaction. And the xSavepoint() method - ** flushes the pending terms table. But leave the (no-op) call to - ** PendingTermsFlush() in in case that changes. - */ - assert( p->nPendingData==0 ); - if( rc==SQLITE_OK ){ - rc = sqlite3Fts3PendingTermsFlush(p); + /* If this phrase is applies specifically to some column other than + ** column iCol, return a NULL pointer. */ + *ppOut = 0; + assert( iCol>=0 && iColnColumn ); + if( (pPhrase->iColumnnColumn && pPhrase->iColumn!=iCol) ){ + return SQLITE_OK; } - if( p->zContentTbl==0 ){ - fts3DbExec(&rc, db, - "ALTER TABLE %Q.'%q_content' RENAME TO '%q_content';", - p->zDb, p->zName, zName - ); + iDocid = pExpr->iDocid; + pIter = pPhrase->doclist.pList; + if( iDocid!=pCsr->iPrevId || pExpr->bEof ){ + int rc = SQLITE_OK; + int bDescDoclist = pTab->bDescIdx; /* For DOCID_CMP macro */ + int bOr = 0; + u8 bTreeEof = 0; + Fts3Expr *p; /* Used to iterate from pExpr to root */ + Fts3Expr *pNear; /* Most senior NEAR ancestor (or pExpr) */ + int bMatch; + + /* Check if this phrase descends from an OR expression node. If not, + ** return NULL. Otherwise, the entry that corresponds to docid + ** pCsr->iPrevId may lie earlier in the doclist buffer. Or, if the + ** tree that the node is part of has been marked as EOF, but the node + ** itself is not EOF, then it may point to an earlier entry. */ + pNear = pExpr; + for(p=pExpr->pParent; p; p=p->pParent){ + if( p->eType==FTSQUERY_OR ) bOr = 1; + if( p->eType==FTSQUERY_NEAR ) pNear = p; + if( p->bEof ) bTreeEof = 1; + } + if( bOr==0 ) return SQLITE_OK; + + /* This is the descendent of an OR node. In this case we cannot use + ** an incremental phrase. Load the entire doclist for the phrase + ** into memory in this case. */ + if( pPhrase->bIncr ){ + int bEofSave = pNear->bEof; + fts3EvalRestart(pCsr, pNear, &rc); + while( rc==SQLITE_OK && !pNear->bEof ){ + fts3EvalNextRow(pCsr, pNear, &rc); + if( bEofSave==0 && pNear->iDocid==iDocid ) break; + } + assert( rc!=SQLITE_OK || pPhrase->bIncr==0 ); + } + if( bTreeEof ){ + while( rc==SQLITE_OK && !pNear->bEof ){ + fts3EvalNextRow(pCsr, pNear, &rc); + } + } + if( rc!=SQLITE_OK ) return rc; + + bMatch = 1; + for(p=pNear; p; p=p->pLeft){ + u8 bEof = 0; + Fts3Expr *pTest = p; + Fts3Phrase *pPh; + assert( pTest->eType==FTSQUERY_NEAR || pTest->eType==FTSQUERY_PHRASE ); + if( pTest->eType==FTSQUERY_NEAR ) pTest = pTest->pRight; + assert( pTest->eType==FTSQUERY_PHRASE ); + pPh = pTest->pPhrase; + + pIter = pPh->pOrPoslist; + iDocid = pPh->iOrDocid; + if( pCsr->bDesc==bDescDoclist ){ + bEof = !pPh->doclist.nAll || + (pIter >= (pPh->doclist.aAll + pPh->doclist.nAll)); + while( (pIter==0 || DOCID_CMP(iDocid, pCsr->iPrevId)<0 ) && bEof==0 ){ + sqlite3Fts3DoclistNext( + bDescDoclist, pPh->doclist.aAll, pPh->doclist.nAll, + &pIter, &iDocid, &bEof + ); + } + }else{ + bEof = !pPh->doclist.nAll || (pIter && pIter<=pPh->doclist.aAll); + while( (pIter==0 || DOCID_CMP(iDocid, pCsr->iPrevId)>0 ) && bEof==0 ){ + int dummy; + sqlite3Fts3DoclistPrev( + bDescDoclist, pPh->doclist.aAll, pPh->doclist.nAll, + &pIter, &iDocid, &dummy, &bEof + ); + } + } + pPh->pOrPoslist = pIter; + pPh->iOrDocid = iDocid; + if( bEof || iDocid!=pCsr->iPrevId ) bMatch = 0; + } + + if( bMatch ){ + pIter = pPhrase->pOrPoslist; + }else{ + pIter = 0; + } } + if( pIter==0 ) return SQLITE_OK; - if( p->bHasDocsize ){ - fts3DbExec(&rc, db, - "ALTER TABLE %Q.'%q_docsize' RENAME TO '%q_docsize';", - p->zDb, p->zName, zName - ); + if( *pIter==0x01 ){ + pIter++; + pIter += fts3GetVarint32(pIter, &iThis); + }else{ + iThis = 0; } - if( p->bHasStat ){ - fts3DbExec(&rc, db, - "ALTER TABLE %Q.'%q_stat' RENAME TO '%q_stat';", - p->zDb, p->zName, zName - ); + while( iThiszDb, p->zName, zName - ); - fts3DbExec(&rc, db, - "ALTER TABLE %Q.'%q_segdir' RENAME TO '%q_segdir';", - p->zDb, p->zName, zName - ); - return rc; -} - -/* -** The xSavepoint() method. -** -** Flush the contents of the pending-terms table to disk. -*/ -static int fts3SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){ - int rc = SQLITE_OK; - UNUSED_PARAMETER(iSavepoint); - assert( ((Fts3Table *)pVtab)->inTransaction ); - assert( ((Fts3Table *)pVtab)->mxSavepoint < iSavepoint ); - TESTONLY( ((Fts3Table *)pVtab)->mxSavepoint = iSavepoint ); - if( ((Fts3Table *)pVtab)->bIgnoreSavepoint==0 ){ - rc = fts3SyncMethod(pVtab); + if( *pIter==0x00 ){ + pIter = 0; } - return rc; -} -/* -** The xRelease() method. -** -** This is a no-op. -*/ -static int fts3ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){ - TESTONLY( Fts3Table *p = (Fts3Table*)pVtab ); - UNUSED_PARAMETER(iSavepoint); - UNUSED_PARAMETER(pVtab); - assert( p->inTransaction ); - assert( p->mxSavepoint >= iSavepoint ); - TESTONLY( p->mxSavepoint = iSavepoint-1 ); + *ppOut = ((iCol==iThis)?pIter:0); return SQLITE_OK; } /* -** The xRollbackTo() method. +** Free all components of the Fts3Phrase structure that were allocated by +** the eval module. Specifically, this means to free: ** -** Discard the contents of the pending terms table. +** * the contents of pPhrase->doclist, and +** * any Fts3MultiSegReader objects held by phrase tokens. */ -static int fts3RollbackToMethod(sqlite3_vtab *pVtab, int iSavepoint){ - Fts3Table *p = (Fts3Table*)pVtab; - UNUSED_PARAMETER(iSavepoint); - assert( p->inTransaction ); - assert( p->mxSavepoint >= iSavepoint ); - TESTONLY( p->mxSavepoint = iSavepoint ); - sqlite3Fts3PendingTermsClear(p); - return SQLITE_OK; +SQLITE_PRIVATE void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *pPhrase){ + if( pPhrase ){ + int i; + sqlite3_free(pPhrase->doclist.aAll); + fts3EvalInvalidatePoslist(pPhrase); + memset(&pPhrase->doclist, 0, sizeof(Fts3Doclist)); + for(i=0; inToken; i++){ + fts3SegReaderCursorFree(pPhrase->aToken[i].pSegcsr); + pPhrase->aToken[i].pSegcsr = 0; + } + } } -static const sqlite3_module fts3Module = { - /* iVersion */ 2, - /* xCreate */ fts3CreateMethod, - /* xConnect */ fts3ConnectMethod, - /* xBestIndex */ fts3BestIndexMethod, - /* xDisconnect */ fts3DisconnectMethod, - /* xDestroy */ fts3DestroyMethod, - /* xOpen */ fts3OpenMethod, - /* xClose */ fts3CloseMethod, - /* xFilter */ fts3FilterMethod, - /* xNext */ fts3NextMethod, - /* xEof */ fts3EofMethod, - /* xColumn */ fts3ColumnMethod, - /* xRowid */ fts3RowidMethod, - /* xUpdate */ fts3UpdateMethod, - /* xBegin */ fts3BeginMethod, - /* xSync */ fts3SyncMethod, - /* xCommit */ fts3CommitMethod, - /* xRollback */ fts3RollbackMethod, - /* xFindFunction */ fts3FindFunctionMethod, - /* xRename */ fts3RenameMethod, - /* xSavepoint */ fts3SavepointMethod, - /* xRelease */ fts3ReleaseMethod, - /* xRollbackTo */ fts3RollbackToMethod, -}; /* -** This function is registered as the module destructor (called when an -** FTS3 enabled database connection is closed). It frees the memory -** allocated for the tokenizer hash table. +** Return SQLITE_CORRUPT_VTAB. */ -static void hashDestroy(void *p){ - Fts3Hash *pHash = (Fts3Hash *)p; - sqlite3Fts3HashClear(pHash); - sqlite3_free(pHash); +#ifdef SQLITE_DEBUG +SQLITE_PRIVATE int sqlite3Fts3Corrupt(){ + return SQLITE_CORRUPT_VTAB; } +#endif +#if !SQLITE_CORE /* -** The fts3 built-in tokenizers - "simple", "porter" and "icu"- are -** implemented in files fts3_tokenizer1.c, fts3_porter.c and fts3_icu.c -** respectively. The following three forward declarations are for functions -** declared in these files used to retrieve the respective implementations. -** -** Calling sqlite3Fts3SimpleTokenizerModule() sets the value pointed -** to by the argument to point to the "simple" tokenizer implementation. -** And so on. +** Initialize API pointer table, if required. */ -SQLITE_PRIVATE void sqlite3Fts3SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule); -SQLITE_PRIVATE void sqlite3Fts3PorterTokenizerModule(sqlite3_tokenizer_module const**ppModule); -#ifndef SQLITE_DISABLE_FTS3_UNICODE -SQLITE_PRIVATE void sqlite3Fts3UnicodeTokenizer(sqlite3_tokenizer_module const**ppModule); +#ifdef _WIN32 +__declspec(dllexport) #endif -#ifdef SQLITE_ENABLE_ICU -SQLITE_PRIVATE void sqlite3Fts3IcuTokenizerModule(sqlite3_tokenizer_module const**ppModule); +SQLITE_API int sqlite3_fts3_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + SQLITE_EXTENSION_INIT2(pApi) + return sqlite3Fts3Init(db); +} #endif -/* -** Initialize the fts3 extension. If this extension is built as part -** of the sqlite library, then this function is called directly by -** SQLite. If fts3 is built as a dynamically loadable extension, this -** function is called by the sqlite3_extension_init() entry point. -*/ -SQLITE_PRIVATE int sqlite3Fts3Init(sqlite3 *db){ - int rc = SQLITE_OK; - Fts3Hash *pHash = 0; - const sqlite3_tokenizer_module *pSimple = 0; - const sqlite3_tokenizer_module *pPorter = 0; -#ifndef SQLITE_DISABLE_FTS3_UNICODE - const sqlite3_tokenizer_module *pUnicode = 0; #endif -#ifdef SQLITE_ENABLE_ICU - const sqlite3_tokenizer_module *pIcu = 0; - sqlite3Fts3IcuTokenizerModule(&pIcu); -#endif +/************** End of fts3.c ************************************************/ +/************** Begin file fts3_aux.c ****************************************/ +/* +** 2011 Jan 27 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +*/ +/* #include "fts3Int.h" */ +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) -#ifndef SQLITE_DISABLE_FTS3_UNICODE - sqlite3Fts3UnicodeTokenizer(&pUnicode); -#endif +/* #include */ +/* #include */ -#ifdef SQLITE_TEST - rc = sqlite3Fts3InitTerm(db); - if( rc!=SQLITE_OK ) return rc; -#endif +typedef struct Fts3auxTable Fts3auxTable; +typedef struct Fts3auxCursor Fts3auxCursor; - rc = sqlite3Fts3InitAux(db); - if( rc!=SQLITE_OK ) return rc; +struct Fts3auxTable { + sqlite3_vtab base; /* Base class used by SQLite core */ + Fts3Table *pFts3Tab; +}; - sqlite3Fts3SimpleTokenizerModule(&pSimple); - sqlite3Fts3PorterTokenizerModule(&pPorter); +struct Fts3auxCursor { + sqlite3_vtab_cursor base; /* Base class used by SQLite core */ + Fts3MultiSegReader csr; /* Must be right after "base" */ + Fts3SegFilter filter; + char *zStop; + int nStop; /* Byte-length of string zStop */ + int iLangid; /* Language id to query */ + int isEof; /* True if cursor is at EOF */ + sqlite3_int64 iRowid; /* Current rowid */ - /* Allocate and initialize the hash-table used to store tokenizers. */ - pHash = sqlite3_malloc(sizeof(Fts3Hash)); - if( !pHash ){ - rc = SQLITE_NOMEM; - }else{ - sqlite3Fts3HashInit(pHash, FTS3_HASH_STRING, 1); - } + int iCol; /* Current value of 'col' column */ + int nStat; /* Size of aStat[] array */ + struct Fts3auxColstats { + sqlite3_int64 nDoc; /* 'documents' values for current csr row */ + sqlite3_int64 nOcc; /* 'occurrences' values for current csr row */ + } *aStat; +}; - /* Load the built-in tokenizers into the hash table */ - if( rc==SQLITE_OK ){ - if( sqlite3Fts3HashInsert(pHash, "simple", 7, (void *)pSimple) - || sqlite3Fts3HashInsert(pHash, "porter", 7, (void *)pPorter) +/* +** Schema of the terms table. +*/ +#define FTS3_AUX_SCHEMA \ + "CREATE TABLE x(term, col, documents, occurrences, languageid HIDDEN)" -#ifndef SQLITE_DISABLE_FTS3_UNICODE - || sqlite3Fts3HashInsert(pHash, "unicode61", 10, (void *)pUnicode) -#endif -#ifdef SQLITE_ENABLE_ICU - || (pIcu && sqlite3Fts3HashInsert(pHash, "icu", 4, (void *)pIcu)) -#endif - ){ - rc = SQLITE_NOMEM; - } - } +/* +** This function does all the work for both the xConnect and xCreate methods. +** These tables have no persistent representation of their own, so xConnect +** and xCreate are identical operations. +*/ +static int fts3auxConnectMethod( + sqlite3 *db, /* Database connection */ + void *pUnused, /* Unused */ + int argc, /* Number of elements in argv array */ + const char * const *argv, /* xCreate/xConnect argument array */ + sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */ + char **pzErr /* OUT: sqlite3_malloc'd error message */ +){ + char const *zDb; /* Name of database (e.g. "main") */ + char const *zFts3; /* Name of fts3 table */ + int nDb; /* Result of strlen(zDb) */ + int nFts3; /* Result of strlen(zFts3) */ + sqlite3_int64 nByte; /* Bytes of space to allocate here */ + int rc; /* value returned by declare_vtab() */ + Fts3auxTable *p; /* Virtual table object to return */ -#ifdef SQLITE_TEST - if( rc==SQLITE_OK ){ - rc = sqlite3Fts3ExprInitTestInterface(db, pHash); - } -#endif + UNUSED_PARAMETER(pUnused); - /* Create the virtual table wrapper around the hash-table and overload - ** the four scalar functions. If this is successful, register the - ** module with sqlite. + /* The user should invoke this in one of two forms: + ** + ** CREATE VIRTUAL TABLE xxx USING fts4aux(fts4-table); + ** CREATE VIRTUAL TABLE xxx USING fts4aux(fts4-table-db, fts4-table); */ - if( SQLITE_OK==rc - && SQLITE_OK==(rc = sqlite3Fts3InitHashTable(db, pHash, "fts3_tokenizer")) - && SQLITE_OK==(rc = sqlite3_overload_function(db, "snippet", -1)) - && SQLITE_OK==(rc = sqlite3_overload_function(db, "offsets", 1)) - && SQLITE_OK==(rc = sqlite3_overload_function(db, "matchinfo", 1)) - && SQLITE_OK==(rc = sqlite3_overload_function(db, "matchinfo", 2)) - && SQLITE_OK==(rc = sqlite3_overload_function(db, "optimize", 1)) - ){ - rc = sqlite3_create_module_v2( - db, "fts3", &fts3Module, (void *)pHash, hashDestroy - ); - if( rc==SQLITE_OK ){ - rc = sqlite3_create_module_v2( - db, "fts4", &fts3Module, (void *)pHash, 0 - ); - } - if( rc==SQLITE_OK ){ - rc = sqlite3Fts3InitTok(db, (void *)pHash); + if( argc!=4 && argc!=5 ) goto bad_args; + + zDb = argv[1]; + nDb = (int)strlen(zDb); + if( argc==5 ){ + if( nDb==4 && 0==sqlite3_strnicmp("temp", zDb, 4) ){ + zDb = argv[3]; + nDb = (int)strlen(zDb); + zFts3 = argv[4]; + }else{ + goto bad_args; } - return rc; + }else{ + zFts3 = argv[3]; } + nFts3 = (int)strlen(zFts3); + rc = sqlite3_declare_vtab(db, FTS3_AUX_SCHEMA); + if( rc!=SQLITE_OK ) return rc; - /* An error has occurred. Delete the hash table and return the error code. */ - assert( rc!=SQLITE_OK ); - if( pHash ){ - sqlite3Fts3HashClear(pHash); - sqlite3_free(pHash); - } - return rc; + nByte = sizeof(Fts3auxTable) + sizeof(Fts3Table) + nDb + nFts3 + 2; + p = (Fts3auxTable *)sqlite3_malloc64(nByte); + if( !p ) return SQLITE_NOMEM; + memset(p, 0, nByte); + + p->pFts3Tab = (Fts3Table *)&p[1]; + p->pFts3Tab->zDb = (char *)&p->pFts3Tab[1]; + p->pFts3Tab->zName = &p->pFts3Tab->zDb[nDb+1]; + p->pFts3Tab->db = db; + p->pFts3Tab->nIndex = 1; + + memcpy((char *)p->pFts3Tab->zDb, zDb, nDb); + memcpy((char *)p->pFts3Tab->zName, zFts3, nFts3); + sqlite3Fts3Dequote((char *)p->pFts3Tab->zName); + + *ppVtab = (sqlite3_vtab *)p; + return SQLITE_OK; + + bad_args: + sqlite3Fts3ErrMsg(pzErr, "invalid arguments to fts4aux constructor"); + return SQLITE_ERROR; } /* -** Allocate an Fts3MultiSegReader for each token in the expression headed -** by pExpr. -** -** An Fts3SegReader object is a cursor that can seek or scan a range of -** entries within a single segment b-tree. An Fts3MultiSegReader uses multiple -** Fts3SegReader objects internally to provide an interface to seek or scan -** within the union of all segments of a b-tree. Hence the name. -** -** If the allocated Fts3MultiSegReader just seeks to a single entry in a -** segment b-tree (if the term is not a prefix or it is a prefix for which -** there exists prefix b-tree of the right length) then it may be traversed -** and merged incrementally. Otherwise, it has to be merged into an in-memory -** doclist and then traversed. +** This function does the work for both the xDisconnect and xDestroy methods. +** These tables have no persistent representation of their own, so xDisconnect +** and xDestroy are identical operations. */ -static void fts3EvalAllocateReaders( - Fts3Cursor *pCsr, /* FTS cursor handle */ - Fts3Expr *pExpr, /* Allocate readers for this expression */ - int *pnToken, /* OUT: Total number of tokens in phrase. */ - int *pnOr, /* OUT: Total number of OR nodes in expr. */ - int *pRc /* IN/OUT: Error code */ -){ - if( pExpr && SQLITE_OK==*pRc ){ - if( pExpr->eType==FTSQUERY_PHRASE ){ - int i; - int nToken = pExpr->pPhrase->nToken; - *pnToken += nToken; - for(i=0; ipPhrase->aToken[i]; - int rc = fts3TermSegReaderCursor(pCsr, - pToken->z, pToken->n, pToken->isPrefix, &pToken->pSegcsr - ); - if( rc!=SQLITE_OK ){ - *pRc = rc; - return; - } - } - assert( pExpr->pPhrase->iDoclistToken==0 ); - pExpr->pPhrase->iDoclistToken = -1; - }else{ - *pnOr += (pExpr->eType==FTSQUERY_OR); - fts3EvalAllocateReaders(pCsr, pExpr->pLeft, pnToken, pnOr, pRc); - fts3EvalAllocateReaders(pCsr, pExpr->pRight, pnToken, pnOr, pRc); - } +static int fts3auxDisconnectMethod(sqlite3_vtab *pVtab){ + Fts3auxTable *p = (Fts3auxTable *)pVtab; + Fts3Table *pFts3 = p->pFts3Tab; + int i; + + /* Free any prepared statements held */ + for(i=0; iaStmt); i++){ + sqlite3_finalize(pFts3->aStmt[i]); } + sqlite3_free(pFts3->zSegmentsTbl); + sqlite3_free(p); + return SQLITE_OK; } +#define FTS4AUX_EQ_CONSTRAINT 1 +#define FTS4AUX_GE_CONSTRAINT 2 +#define FTS4AUX_LE_CONSTRAINT 4 + /* -** Arguments pList/nList contain the doclist for token iToken of phrase p. -** It is merged into the main doclist stored in p->doclist.aAll/nAll. -** -** This function assumes that pList points to a buffer allocated using -** sqlite3_malloc(). This function takes responsibility for eventually -** freeing the buffer. -** -** SQLITE_OK is returned if successful, or SQLITE_NOMEM if an error occurs. +** xBestIndex - Analyze a WHERE and ORDER BY clause. */ -static int fts3EvalPhraseMergeToken( - Fts3Table *pTab, /* FTS Table pointer */ - Fts3Phrase *p, /* Phrase to merge pList/nList into */ - int iToken, /* Token pList/nList corresponds to */ - char *pList, /* Pointer to doclist */ - int nList /* Number of bytes in pList */ +static int fts3auxBestIndexMethod( + sqlite3_vtab *pVTab, + sqlite3_index_info *pInfo ){ - int rc = SQLITE_OK; - assert( iToken!=p->iDoclistToken ); + int i; + int iEq = -1; + int iGe = -1; + int iLe = -1; + int iLangid = -1; + int iNext = 1; /* Next free argvIndex value */ - if( pList==0 ){ - sqlite3_free(p->doclist.aAll); - p->doclist.aAll = 0; - p->doclist.nAll = 0; - } + UNUSED_PARAMETER(pVTab); - else if( p->iDoclistToken<0 ){ - p->doclist.aAll = pList; - p->doclist.nAll = nList; + /* This vtab delivers always results in "ORDER BY term ASC" order. */ + if( pInfo->nOrderBy==1 + && pInfo->aOrderBy[0].iColumn==0 + && pInfo->aOrderBy[0].desc==0 + ){ + pInfo->orderByConsumed = 1; } - else if( p->doclist.aAll==0 ){ - sqlite3_free(pList); - } + /* Search for equality and range constraints on the "term" column. + ** And equality constraints on the hidden "languageid" column. */ + for(i=0; inConstraint; i++){ + if( pInfo->aConstraint[i].usable ){ + int op = pInfo->aConstraint[i].op; + int iCol = pInfo->aConstraint[i].iColumn; - else { - char *pLeft; - char *pRight; - int nLeft; - int nRight; - int nDiff; + if( iCol==0 ){ + if( op==SQLITE_INDEX_CONSTRAINT_EQ ) iEq = i; + if( op==SQLITE_INDEX_CONSTRAINT_LT ) iLe = i; + if( op==SQLITE_INDEX_CONSTRAINT_LE ) iLe = i; + if( op==SQLITE_INDEX_CONSTRAINT_GT ) iGe = i; + if( op==SQLITE_INDEX_CONSTRAINT_GE ) iGe = i; + } + if( iCol==4 ){ + if( op==SQLITE_INDEX_CONSTRAINT_EQ ) iLangid = i; + } + } + } - if( p->iDoclistTokendoclist.aAll; - nLeft = p->doclist.nAll; - pRight = pList; - nRight = nList; - nDiff = iToken - p->iDoclistToken; - }else{ - pRight = p->doclist.aAll; - nRight = p->doclist.nAll; - pLeft = pList; - nLeft = nList; - nDiff = p->iDoclistToken - iToken; + if( iEq>=0 ){ + pInfo->idxNum = FTS4AUX_EQ_CONSTRAINT; + pInfo->aConstraintUsage[iEq].argvIndex = iNext++; + pInfo->estimatedCost = 5; + }else{ + pInfo->idxNum = 0; + pInfo->estimatedCost = 20000; + if( iGe>=0 ){ + pInfo->idxNum += FTS4AUX_GE_CONSTRAINT; + pInfo->aConstraintUsage[iGe].argvIndex = iNext++; + pInfo->estimatedCost /= 2; + } + if( iLe>=0 ){ + pInfo->idxNum += FTS4AUX_LE_CONSTRAINT; + pInfo->aConstraintUsage[iLe].argvIndex = iNext++; + pInfo->estimatedCost /= 2; } - - rc = fts3DoclistPhraseMerge( - pTab->bDescIdx, nDiff, pLeft, nLeft, &pRight, &nRight - ); - sqlite3_free(pLeft); - p->doclist.aAll = pRight; - p->doclist.nAll = nRight; + } + if( iLangid>=0 ){ + pInfo->aConstraintUsage[iLangid].argvIndex = iNext++; + pInfo->estimatedCost--; } - if( iToken>p->iDoclistToken ) p->iDoclistToken = iToken; - return rc; + return SQLITE_OK; } /* -** Load the doclist for phrase p into p->doclist.aAll/nAll. The loaded doclist -** does not take deferred tokens into account. -** -** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code. +** xOpen - Open a cursor. */ -static int fts3EvalPhraseLoad( - Fts3Cursor *pCsr, /* FTS Cursor handle */ - Fts3Phrase *p /* Phrase object */ -){ - Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; - int iToken; - int rc = SQLITE_OK; +static int fts3auxOpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){ + Fts3auxCursor *pCsr; /* Pointer to cursor object to return */ - for(iToken=0; rc==SQLITE_OK && iTokennToken; iToken++){ - Fts3PhraseToken *pToken = &p->aToken[iToken]; - assert( pToken->pDeferred==0 || pToken->pSegcsr==0 ); + UNUSED_PARAMETER(pVTab); - if( pToken->pSegcsr ){ - int nThis = 0; - char *pThis = 0; - rc = fts3TermSelect(pTab, pToken, p->iColumn, &nThis, &pThis); - if( rc==SQLITE_OK ){ - rc = fts3EvalPhraseMergeToken(pTab, p, iToken, pThis, nThis); - } - } - assert( pToken->pSegcsr==0 ); - } + pCsr = (Fts3auxCursor *)sqlite3_malloc(sizeof(Fts3auxCursor)); + if( !pCsr ) return SQLITE_NOMEM; + memset(pCsr, 0, sizeof(Fts3auxCursor)); - return rc; + *ppCsr = (sqlite3_vtab_cursor *)pCsr; + return SQLITE_OK; } /* -** This function is called on each phrase after the position lists for -** any deferred tokens have been loaded into memory. It updates the phrases -** current position list to include only those positions that are really -** instances of the phrase (after considering deferred tokens). If this -** means that the phrase does not appear in the current row, doclist.pList -** and doclist.nList are both zeroed. -** -** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code. +** xClose - Close a cursor. */ -static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){ - int iToken; /* Used to iterate through phrase tokens */ - char *aPoslist = 0; /* Position list for deferred tokens */ - int nPoslist = 0; /* Number of bytes in aPoslist */ - int iPrev = -1; /* Token number of previous deferred token */ - - assert( pPhrase->doclist.bFreeList==0 ); - - for(iToken=0; iTokennToken; iToken++){ - Fts3PhraseToken *pToken = &pPhrase->aToken[iToken]; - Fts3DeferredToken *pDeferred = pToken->pDeferred; +static int fts3auxCloseMethod(sqlite3_vtab_cursor *pCursor){ + Fts3Table *pFts3 = ((Fts3auxTable *)pCursor->pVtab)->pFts3Tab; + Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor; - if( pDeferred ){ - char *pList; - int nList; - int rc = sqlite3Fts3DeferredTokenList(pDeferred, &pList, &nList); - if( rc!=SQLITE_OK ) return rc; + sqlite3Fts3SegmentsClose(pFts3); + sqlite3Fts3SegReaderFinish(&pCsr->csr); + sqlite3_free((void *)pCsr->filter.zTerm); + sqlite3_free(pCsr->zStop); + sqlite3_free(pCsr->aStat); + sqlite3_free(pCsr); + return SQLITE_OK; +} - if( pList==0 ){ - sqlite3_free(aPoslist); - pPhrase->doclist.pList = 0; - pPhrase->doclist.nList = 0; - return SQLITE_OK; +static int fts3auxGrowStatArray(Fts3auxCursor *pCsr, int nSize){ + if( nSize>pCsr->nStat ){ + struct Fts3auxColstats *aNew; + aNew = (struct Fts3auxColstats *)sqlite3_realloc64(pCsr->aStat, + sizeof(struct Fts3auxColstats) * nSize + ); + if( aNew==0 ) return SQLITE_NOMEM; + memset(&aNew[pCsr->nStat], 0, + sizeof(struct Fts3auxColstats) * (nSize - pCsr->nStat) + ); + pCsr->aStat = aNew; + pCsr->nStat = nSize; + } + return SQLITE_OK; +} - }else if( aPoslist==0 ){ - aPoslist = pList; - nPoslist = nList; +/* +** xNext - Advance the cursor to the next row, if any. +*/ +static int fts3auxNextMethod(sqlite3_vtab_cursor *pCursor){ + Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor; + Fts3Table *pFts3 = ((Fts3auxTable *)pCursor->pVtab)->pFts3Tab; + int rc; - }else{ - char *aOut = pList; - char *p1 = aPoslist; - char *p2 = aOut; + /* Increment our pretend rowid value. */ + pCsr->iRowid++; - assert( iPrev>=0 ); - fts3PoslistPhraseMerge(&aOut, iToken-iPrev, 0, 1, &p1, &p2); - sqlite3_free(aPoslist); - aPoslist = pList; - nPoslist = (int)(aOut - aPoslist); - if( nPoslist==0 ){ - sqlite3_free(aPoslist); - pPhrase->doclist.pList = 0; - pPhrase->doclist.nList = 0; - return SQLITE_OK; - } - } - iPrev = iToken; - } + for(pCsr->iCol++; pCsr->iColnStat; pCsr->iCol++){ + if( pCsr->aStat[pCsr->iCol].nDoc>0 ) return SQLITE_OK; } - if( iPrev>=0 ){ - int nMaxUndeferred = pPhrase->iDoclistToken; - if( nMaxUndeferred<0 ){ - pPhrase->doclist.pList = aPoslist; - pPhrase->doclist.nList = nPoslist; - pPhrase->doclist.iDocid = pCsr->iPrevId; - pPhrase->doclist.bFreeList = 1; - }else{ - int nDistance; - char *p1; - char *p2; - char *aOut; + rc = sqlite3Fts3SegReaderStep(pFts3, &pCsr->csr); + if( rc==SQLITE_ROW ){ + int i = 0; + int nDoclist = pCsr->csr.nDoclist; + char *aDoclist = pCsr->csr.aDoclist; + int iCol; - if( nMaxUndeferred>iPrev ){ - p1 = aPoslist; - p2 = pPhrase->doclist.pList; - nDistance = nMaxUndeferred - iPrev; - }else{ - p1 = pPhrase->doclist.pList; - p2 = aPoslist; - nDistance = iPrev - nMaxUndeferred; - } + int eState = 0; - aOut = (char *)sqlite3_malloc(nPoslist+8); - if( !aOut ){ - sqlite3_free(aPoslist); - return SQLITE_NOMEM; - } - - pPhrase->doclist.pList = aOut; - if( fts3PoslistPhraseMerge(&aOut, nDistance, 0, 1, &p1, &p2) ){ - pPhrase->doclist.bFreeList = 1; - pPhrase->doclist.nList = (int)(aOut - pPhrase->doclist.pList); - }else{ - sqlite3_free(aOut); - pPhrase->doclist.pList = 0; - pPhrase->doclist.nList = 0; + if( pCsr->zStop ){ + int n = (pCsr->nStopcsr.nTerm) ? pCsr->nStop : pCsr->csr.nTerm; + int mc = memcmp(pCsr->zStop, pCsr->csr.zTerm, n); + if( mc<0 || (mc==0 && pCsr->csr.nTerm>pCsr->nStop) ){ + pCsr->isEof = 1; + return SQLITE_OK; } - sqlite3_free(aPoslist); } - } - return SQLITE_OK; -} + if( fts3auxGrowStatArray(pCsr, 2) ) return SQLITE_NOMEM; + memset(pCsr->aStat, 0, sizeof(struct Fts3auxColstats) * pCsr->nStat); + iCol = 0; -/* -** Maximum number of tokens a phrase may have to be considered for the -** incremental doclists strategy. -*/ -#define MAX_INCR_PHRASE_TOKENS 4 + while( ibase.pVtab; - int rc = SQLITE_OK; /* Error code */ - int i; + i += sqlite3Fts3GetVarint(&aDoclist[i], &v); + switch( eState ){ + /* State 0. In this state the integer just read was a docid. */ + case 0: + pCsr->aStat[0].nDoc++; + eState = 1; + iCol = 0; + break; - /* Determine if doclists may be loaded from disk incrementally. This is - ** possible if the bOptOk argument is true, the FTS doclists will be - ** scanned in forward order, and the phrase consists of - ** MAX_INCR_PHRASE_TOKENS or fewer tokens, none of which are are "^first" - ** tokens or prefix tokens that cannot use a prefix-index. */ - int bHaveIncr = 0; - int bIncrOk = (bOptOk - && pCsr->bDesc==pTab->bDescIdx - && p->nToken<=MAX_INCR_PHRASE_TOKENS && p->nToken>0 -#ifdef SQLITE_TEST - && pTab->bNoIncrDoclist==0 -#endif - ); - for(i=0; bIncrOk==1 && inToken; i++){ - Fts3PhraseToken *pToken = &p->aToken[i]; - if( pToken->bFirst || (pToken->pSegcsr!=0 && !pToken->pSegcsr->bLookup) ){ - bIncrOk = 0; - } - if( pToken->pSegcsr ) bHaveIncr = 1; - } + /* State 1. In this state we are expecting either a 1, indicating + ** that the following integer will be a column number, or the + ** start of a position list for column 0. + ** + ** The only difference between state 1 and state 2 is that if the + ** integer encountered in state 1 is not 0 or 1, then we need to + ** increment the column 0 "nDoc" count for this term. + */ + case 1: + assert( iCol==0 ); + if( v>1 ){ + pCsr->aStat[1].nDoc++; + } + eState = 2; + /* fall through */ - if( bIncrOk && bHaveIncr ){ - /* Use the incremental approach. */ - int iCol = (p->iColumn >= pTab->nColumn ? -1 : p->iColumn); - for(i=0; rc==SQLITE_OK && inToken; i++){ - Fts3PhraseToken *pToken = &p->aToken[i]; - Fts3MultiSegReader *pSegcsr = pToken->pSegcsr; - if( pSegcsr ){ - rc = sqlite3Fts3MsrIncrStart(pTab, pSegcsr, iCol, pToken->z, pToken->n); + case 2: + if( v==0 ){ /* 0x00. Next integer will be a docid. */ + eState = 0; + }else if( v==1 ){ /* 0x01. Next integer will be a column number. */ + eState = 3; + }else{ /* 2 or greater. A position. */ + pCsr->aStat[iCol+1].nOcc++; + pCsr->aStat[0].nOcc++; + } + break; + + /* State 3. The integer just read is a column number. */ + default: assert( eState==3 ); + iCol = (int)v; + if( fts3auxGrowStatArray(pCsr, iCol+2) ) return SQLITE_NOMEM; + pCsr->aStat[iCol+1].nDoc++; + eState = 2; + break; } } - p->bIncr = 1; + + pCsr->iCol = 0; + rc = SQLITE_OK; }else{ - /* Load the full doclist for the phrase into memory. */ - rc = fts3EvalPhraseLoad(pCsr, p); - p->bIncr = 0; + pCsr->isEof = 1; } - - assert( rc!=SQLITE_OK || p->nToken<1 || p->aToken[0].pSegcsr==0 || p->bIncr ); return rc; } /* -** This function is used to iterate backwards (from the end to start) -** through doclists. It is used by this module to iterate through phrase -** doclists in reverse and by the fts3_write.c module to iterate through -** pending-terms lists when writing to databases with "order=desc". -** -** The doclist may be sorted in ascending (parameter bDescIdx==0) or -** descending (parameter bDescIdx==1) order of docid. Regardless, this -** function iterates from the end of the doclist to the beginning. +** xFilter - Initialize a cursor to point at the start of its data. */ -SQLITE_PRIVATE void sqlite3Fts3DoclistPrev( - int bDescIdx, /* True if the doclist is desc */ - char *aDoclist, /* Pointer to entire doclist */ - int nDoclist, /* Length of aDoclist in bytes */ - char **ppIter, /* IN/OUT: Iterator pointer */ - sqlite3_int64 *piDocid, /* IN/OUT: Docid pointer */ - int *pnList, /* OUT: List length pointer */ - u8 *pbEof /* OUT: End-of-file flag */ +static int fts3auxFilterMethod( + sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */ + int idxNum, /* Strategy index */ + const char *idxStr, /* Unused */ + int nVal, /* Number of elements in apVal */ + sqlite3_value **apVal /* Arguments for the indexing scheme */ ){ - char *p = *ppIter; + Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor; + Fts3Table *pFts3 = ((Fts3auxTable *)pCursor->pVtab)->pFts3Tab; + int rc; + int isScan = 0; + int iLangVal = 0; /* Language id to query */ - assert( nDoclist>0 ); - assert( *pbEof==0 ); - assert( p || *piDocid==0 ); - assert( !p || (p>aDoclist && p<&aDoclist[nDoclist]) ); + int iEq = -1; /* Index of term=? value in apVal */ + int iGe = -1; /* Index of term>=? value in apVal */ + int iLe = -1; /* Index of term<=? value in apVal */ + int iLangid = -1; /* Index of languageid=? value in apVal */ + int iNext = 0; - if( p==0 ){ - sqlite3_int64 iDocid = 0; - char *pNext = 0; - char *pDocid = aDoclist; - char *pEnd = &aDoclist[nDoclist]; - int iMul = 1; + UNUSED_PARAMETER(nVal); + UNUSED_PARAMETER(idxStr); - while( pDocidfilter.zTerm); + sqlite3Fts3SegReaderFinish(&pCsr->csr); + sqlite3_free((void *)pCsr->filter.zTerm); + sqlite3_free(pCsr->aStat); + memset(&pCsr->csr, 0, ((u8*)&pCsr[1]) - (u8*)&pCsr->csr); - assert( nDoclist>0 ); - assert( *pbEof==0 ); - assert( p || *piDocid==0 ); - assert( !p || (p>=aDoclist && p<=&aDoclist[nDoclist]) ); + pCsr->filter.flags = FTS3_SEGMENT_REQUIRE_POS|FTS3_SEGMENT_IGNORE_EMPTY; + if( isScan ) pCsr->filter.flags |= FTS3_SEGMENT_SCAN; - if( p==0 ){ - p = aDoclist; - p += sqlite3Fts3GetVarint(p, piDocid); - }else{ - fts3PoslistCopy(0, &p); - while( p<&aDoclist[nDoclist] && *p==0 ) p++; - if( p>=&aDoclist[nDoclist] ){ - *pbEof = 1; - }else{ - sqlite3_int64 iVar; - p += sqlite3Fts3GetVarint(p, &iVar); - *piDocid += ((bDescIdx ? -1 : 1) * iVar); + if( iEq>=0 || iGe>=0 ){ + const unsigned char *zStr = sqlite3_value_text(apVal[0]); + assert( (iEq==0 && iGe==-1) || (iEq==-1 && iGe==0) ); + if( zStr ){ + pCsr->filter.zTerm = sqlite3_mprintf("%s", zStr); + if( pCsr->filter.zTerm==0 ) return SQLITE_NOMEM; + pCsr->filter.nTerm = (int)strlen(pCsr->filter.zTerm); } } - *ppIter = p; + if( iLe>=0 ){ + pCsr->zStop = sqlite3_mprintf("%s", sqlite3_value_text(apVal[iLe])); + if( pCsr->zStop==0 ) return SQLITE_NOMEM; + pCsr->nStop = (int)strlen(pCsr->zStop); + } + + if( iLangid>=0 ){ + iLangVal = sqlite3_value_int(apVal[iLangid]); + + /* If the user specified a negative value for the languageid, use zero + ** instead. This works, as the "languageid=?" constraint will also + ** be tested by the VDBE layer. The test will always be false (since + ** this module will not return a row with a negative languageid), and + ** so the overall query will return zero rows. */ + if( iLangVal<0 ) iLangVal = 0; + } + pCsr->iLangid = iLangVal; + + rc = sqlite3Fts3SegReaderCursor(pFts3, iLangVal, 0, FTS3_SEGCURSOR_ALL, + pCsr->filter.zTerm, pCsr->filter.nTerm, 0, isScan, &pCsr->csr + ); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3SegReaderStart(pFts3, &pCsr->csr, &pCsr->filter); + } + + if( rc==SQLITE_OK ) rc = fts3auxNextMethod(pCursor); + return rc; } /* -** Advance the iterator pDL to the next entry in pDL->aAll/nAll. Set *pbEof -** to true if EOF is reached. +** xEof - Return true if the cursor is at EOF, or false otherwise. */ -static void fts3EvalDlPhraseNext( - Fts3Table *pTab, - Fts3Doclist *pDL, - u8 *pbEof +static int fts3auxEofMethod(sqlite3_vtab_cursor *pCursor){ + Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor; + return pCsr->isEof; +} + +/* +** xColumn - Return a column value. +*/ +static int fts3auxColumnMethod( + sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ + sqlite3_context *pCtx, /* Context for sqlite3_result_xxx() calls */ + int iCol /* Index of column to read value from */ ){ - char *pIter; /* Used to iterate through aAll */ - char *pEnd = &pDL->aAll[pDL->nAll]; /* 1 byte past end of aAll */ - - if( pDL->pNextDocid ){ - pIter = pDL->pNextDocid; - }else{ - pIter = pDL->aAll; - } + Fts3auxCursor *p = (Fts3auxCursor *)pCursor; - if( pIter>=pEnd ){ - /* We have already reached the end of this doclist. EOF. */ - *pbEof = 1; - }else{ - sqlite3_int64 iDelta; - pIter += sqlite3Fts3GetVarint(pIter, &iDelta); - if( pTab->bDescIdx==0 || pDL->pNextDocid==0 ){ - pDL->iDocid += iDelta; - }else{ - pDL->iDocid -= iDelta; - } - pDL->pList = pIter; - fts3PoslistCopy(0, &pIter); - pDL->nList = (int)(pIter - pDL->pList); + assert( p->isEof==0 ); + switch( iCol ){ + case 0: /* term */ + sqlite3_result_text(pCtx, p->csr.zTerm, p->csr.nTerm, SQLITE_TRANSIENT); + break; - /* pIter now points just past the 0x00 that terminates the position- - ** list for document pDL->iDocid. However, if this position-list was - ** edited in place by fts3EvalNearTrim(), then pIter may not actually - ** point to the start of the next docid value. The following line deals - ** with this case by advancing pIter past the zero-padding added by - ** fts3EvalNearTrim(). */ - while( pIteriCol ){ + sqlite3_result_int(pCtx, p->iCol-1); + }else{ + sqlite3_result_text(pCtx, "*", -1, SQLITE_STATIC); + } + break; - pDL->pNextDocid = pIter; - assert( pIter>=&pDL->aAll[pDL->nAll] || *pIter ); - *pbEof = 0; + case 2: /* documents */ + sqlite3_result_int64(pCtx, p->aStat[p->iCol].nDoc); + break; + + case 3: /* occurrences */ + sqlite3_result_int64(pCtx, p->aStat[p->iCol].nOcc); + break; + + default: /* languageid */ + assert( iCol==4 ); + sqlite3_result_int(pCtx, p->iLangid); + break; } + + return SQLITE_OK; } /* -** Helper type used by fts3EvalIncrPhraseNext() and incrPhraseTokenNext(). +** xRowid - Return the current rowid for the cursor. */ -typedef struct TokenDoclist TokenDoclist; -struct TokenDoclist { - int bIgnore; - sqlite3_int64 iDocid; - char *pList; - int nList; -}; +static int fts3auxRowidMethod( + sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ + sqlite_int64 *pRowid /* OUT: Rowid value */ +){ + Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor; + *pRowid = pCsr->iRowid; + return SQLITE_OK; +} /* -** Token pToken is an incrementally loaded token that is part of a -** multi-token phrase. Advance it to the next matching document in the -** database and populate output variable *p with the details of the new -** entry. Or, if the iterator has reached EOF, set *pbEof to true. -** -** If an error occurs, return an SQLite error code. Otherwise, return -** SQLITE_OK. +** Register the fts3aux module with database connection db. Return SQLITE_OK +** if successful or an error code if sqlite3_create_module() fails. */ -static int incrPhraseTokenNext( - Fts3Table *pTab, /* Virtual table handle */ - Fts3Phrase *pPhrase, /* Phrase to advance token of */ - int iToken, /* Specific token to advance */ - TokenDoclist *p, /* OUT: Docid and doclist for new entry */ - u8 *pbEof /* OUT: True if iterator is at EOF */ -){ - int rc = SQLITE_OK; - - if( pPhrase->iDoclistToken==iToken ){ - assert( p->bIgnore==0 ); - assert( pPhrase->aToken[iToken].pSegcsr==0 ); - fts3EvalDlPhraseNext(pTab, &pPhrase->doclist, pbEof); - p->pList = pPhrase->doclist.pList; - p->nList = pPhrase->doclist.nList; - p->iDocid = pPhrase->doclist.iDocid; - }else{ - Fts3PhraseToken *pToken = &pPhrase->aToken[iToken]; - assert( pToken->pDeferred==0 ); - assert( pToken->pSegcsr || pPhrase->iDoclistToken>=0 ); - if( pToken->pSegcsr ){ - assert( p->bIgnore==0 ); - rc = sqlite3Fts3MsrIncrNext( - pTab, pToken->pSegcsr, &p->iDocid, &p->pList, &p->nList - ); - if( p->pList==0 ) *pbEof = 1; - }else{ - p->bIgnore = 1; - } - } +SQLITE_PRIVATE int sqlite3Fts3InitAux(sqlite3 *db){ + static const sqlite3_module fts3aux_module = { + 0, /* iVersion */ + fts3auxConnectMethod, /* xCreate */ + fts3auxConnectMethod, /* xConnect */ + fts3auxBestIndexMethod, /* xBestIndex */ + fts3auxDisconnectMethod, /* xDisconnect */ + fts3auxDisconnectMethod, /* xDestroy */ + fts3auxOpenMethod, /* xOpen */ + fts3auxCloseMethod, /* xClose */ + fts3auxFilterMethod, /* xFilter */ + fts3auxNextMethod, /* xNext */ + fts3auxEofMethod, /* xEof */ + fts3auxColumnMethod, /* xColumn */ + fts3auxRowidMethod, /* xRowid */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindFunction */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0, /* xRollbackTo */ + 0 /* xShadowName */ + }; + int rc; /* Return code */ + rc = sqlite3_create_module(db, "fts4aux", &fts3aux_module, 0); return rc; } +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ +/************** End of fts3_aux.c ********************************************/ +/************** Begin file fts3_expr.c ***************************************/ /* -** The phrase iterator passed as the second argument: -** -** * features at least one token that uses an incremental doclist, and +** 2008 Nov 28 ** -** * does not contain any deferred tokens. +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: ** -** Advance it to the next matching documnent in the database and populate -** the Fts3Doclist.pList and nList fields. +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. ** -** If there is no "next" entry and no error occurs, then *pbEof is set to -** 1 before returning. Otherwise, if no error occurs and the iterator is -** successfully advanced, *pbEof is set to 0. +****************************************************************************** ** -** If an error occurs, return an SQLite error code. Otherwise, return -** SQLITE_OK. +** This module contains code that implements a parser for fts3 query strings +** (the right-hand argument to the MATCH operator). Because the supported +** syntax is relatively simple, the whole tokenizer/parser system is +** hand-coded. */ -static int fts3EvalIncrPhraseNext( - Fts3Cursor *pCsr, /* FTS Cursor handle */ - Fts3Phrase *p, /* Phrase object to advance to next docid */ - u8 *pbEof /* OUT: Set to 1 if EOF */ -){ - int rc = SQLITE_OK; - Fts3Doclist *pDL = &p->doclist; - Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; - u8 bEof = 0; - - /* This is only called if it is guaranteed that the phrase has at least - ** one incremental token. In which case the bIncr flag is set. */ - assert( p->bIncr==1 ); - - if( p->nToken==1 ){ - rc = sqlite3Fts3MsrIncrNext(pTab, p->aToken[0].pSegcsr, - &pDL->iDocid, &pDL->pList, &pDL->nList - ); - if( pDL->pList==0 ) bEof = 1; - }else{ - int bDescDoclist = pCsr->bDesc; - struct TokenDoclist a[MAX_INCR_PHRASE_TOKENS]; - - memset(a, 0, sizeof(a)); - assert( p->nToken<=MAX_INCR_PHRASE_TOKENS ); - assert( p->iDoclistTokennToken && bEof==0; i++){ - rc = incrPhraseTokenNext(pTab, p, i, &a[i], &bEof); - if( a[i].bIgnore==0 && (bMaxSet==0 || DOCID_CMP(iMax, a[i].iDocid)<0) ){ - iMax = a[i].iDocid; - bMaxSet = 1; - } - } - assert( rc!=SQLITE_OK || (p->nToken>=1 && a[p->nToken-1].bIgnore==0) ); - assert( rc!=SQLITE_OK || bMaxSet ); +/* +** By default, this module parses the legacy syntax that has been +** traditionally used by fts3. Or, if SQLITE_ENABLE_FTS3_PARENTHESIS +** is defined, then it uses the new syntax. The differences between +** the new and the old syntaxes are: +** +** a) The new syntax supports parenthesis. The old does not. +** +** b) The new syntax supports the AND and NOT operators. The old does not. +** +** c) The old syntax supports the "-" token qualifier. This is not +** supported by the new syntax (it is replaced by the NOT operator). +** +** d) When using the old syntax, the OR operator has a greater precedence +** than an implicit AND. When using the new, both implicity and explicit +** AND operators have a higher precedence than OR. +** +** If compiled with SQLITE_TEST defined, then this module exports the +** symbol "int sqlite3_fts3_enable_parentheses". Setting this variable +** to zero causes the module to use the old syntax. If it is set to +** non-zero the new syntax is activated. This is so both syntaxes can +** be tested using a single build of testfixture. +** +** The following describes the syntax supported by the fts3 MATCH +** operator in a similar format to that used by the lemon parser +** generator. This module does not use actually lemon, it uses a +** custom parser. +** +** query ::= andexpr (OR andexpr)*. +** +** andexpr ::= notexpr (AND? notexpr)*. +** +** notexpr ::= nearexpr (NOT nearexpr|-TOKEN)*. +** notexpr ::= LP query RP. +** +** nearexpr ::= phrase (NEAR distance_opt nearexpr)*. +** +** distance_opt ::= . +** distance_opt ::= / INTEGER. +** +** phrase ::= TOKEN. +** phrase ::= COLUMN:TOKEN. +** phrase ::= "TOKEN TOKEN TOKEN...". +*/ - /* Keep advancing iterators until they all point to the same document */ - for(i=0; inToken; i++){ - while( rc==SQLITE_OK && bEof==0 - && a[i].bIgnore==0 && DOCID_CMP(a[i].iDocid, iMax)<0 - ){ - rc = incrPhraseTokenNext(pTab, p, i, &a[i], &bEof); - if( DOCID_CMP(a[i].iDocid, iMax)>0 ){ - iMax = a[i].iDocid; - i = 0; - } - } - } +#ifdef SQLITE_TEST +SQLITE_API int sqlite3_fts3_enable_parentheses = 0; +#else +# ifdef SQLITE_ENABLE_FTS3_PARENTHESIS +# define sqlite3_fts3_enable_parentheses 1 +# else +# define sqlite3_fts3_enable_parentheses 0 +# endif +#endif - /* Check if the current entries really are a phrase match */ - if( bEof==0 ){ - int nList = 0; - int nByte = a[p->nToken-1].nList; - char *aDoclist = sqlite3_malloc(nByte+1); - if( !aDoclist ) return SQLITE_NOMEM; - memcpy(aDoclist, a[p->nToken-1].pList, nByte+1); +/* +** Default span for NEAR operators. +*/ +#define SQLITE_FTS3_DEFAULT_NEAR_PARAM 10 - for(i=0; i<(p->nToken-1); i++){ - if( a[i].bIgnore==0 ){ - char *pL = a[i].pList; - char *pR = aDoclist; - char *pOut = aDoclist; - int nDist = p->nToken-1-i; - int res = fts3PoslistPhraseMerge(&pOut, nDist, 0, 1, &pL, &pR); - if( res==0 ) break; - nList = (int)(pOut - aDoclist); - } - } - if( i==(p->nToken-1) ){ - pDL->iDocid = iMax; - pDL->pList = aDoclist; - pDL->nList = nList; - pDL->bFreeList = 1; - break; - } - sqlite3_free(aDoclist); - } - } - } +/* #include */ +/* #include */ - *pbEof = bEof; - return rc; -} +/* +** isNot: +** This variable is used by function getNextNode(). When getNextNode() is +** called, it sets ParseContext.isNot to true if the 'next node' is a +** FTSQUERY_PHRASE with a unary "-" attached to it. i.e. "mysql" in the +** FTS3 query "sqlite -mysql". Otherwise, ParseContext.isNot is set to +** zero. +*/ +typedef struct ParseContext ParseContext; +struct ParseContext { + sqlite3_tokenizer *pTokenizer; /* Tokenizer module */ + int iLangid; /* Language id used with tokenizer */ + const char **azCol; /* Array of column names for fts3 table */ + int bFts4; /* True to allow FTS4-only syntax */ + int nCol; /* Number of entries in azCol[] */ + int iDefaultCol; /* Default column to query */ + int isNot; /* True if getNextNode() sees a unary - */ + sqlite3_context *pCtx; /* Write error message here */ + int nNest; /* Number of nested brackets */ +}; /* -** Attempt to move the phrase iterator to point to the next matching docid. -** If an error occurs, return an SQLite error code. Otherwise, return -** SQLITE_OK. +** This function is equivalent to the standard isspace() function. ** -** If there is no "next" entry and no error occurs, then *pbEof is set to -** 1 before returning. Otherwise, if no error occurs and the iterator is -** successfully advanced, *pbEof is set to 0. +** The standard isspace() can be awkward to use safely, because although it +** is defined to accept an argument of type int, its behavior when passed +** an integer that falls outside of the range of the unsigned char type +** is undefined (and sometimes, "undefined" means segfault). This wrapper +** is defined to accept an argument of type char, and always returns 0 for +** any values that fall outside of the range of the unsigned char type (i.e. +** negative values). */ -static int fts3EvalPhraseNext( - Fts3Cursor *pCsr, /* FTS Cursor handle */ - Fts3Phrase *p, /* Phrase object to advance to next docid */ - u8 *pbEof /* OUT: Set to 1 if EOF */ -){ - int rc = SQLITE_OK; - Fts3Doclist *pDL = &p->doclist; - Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; - - if( p->bIncr ){ - rc = fts3EvalIncrPhraseNext(pCsr, p, pbEof); - }else if( pCsr->bDesc!=pTab->bDescIdx && pDL->nAll ){ - sqlite3Fts3DoclistPrev(pTab->bDescIdx, pDL->aAll, pDL->nAll, - &pDL->pNextDocid, &pDL->iDocid, &pDL->nList, pbEof - ); - pDL->pList = pDL->pNextDocid; - }else{ - fts3EvalDlPhraseNext(pTab, pDL, pbEof); - } - - return rc; +static int fts3isspace(char c){ + return c==' ' || c=='\t' || c=='\n' || c=='\r' || c=='\v' || c=='\f'; } /* -** -** If *pRc is not SQLITE_OK when this function is called, it is a no-op. -** Otherwise, fts3EvalPhraseStart() is called on all phrases within the -** expression. Also the Fts3Expr.bDeferred variable is set to true for any -** expressions for which all descendent tokens are deferred. -** -** If parameter bOptOk is zero, then it is guaranteed that the -** Fts3Phrase.doclist.aAll/nAll variables contain the entire doclist for -** each phrase in the expression (subject to deferred token processing). -** Or, if bOptOk is non-zero, then one or more tokens within the expression -** may be loaded incrementally, meaning doclist.aAll/nAll is not available. -** -** If an error occurs within this function, *pRc is set to an SQLite error -** code before returning. +** Allocate nByte bytes of memory using sqlite3_malloc(). If successful, +** zero the memory before returning a pointer to it. If unsuccessful, +** return NULL. */ -static void fts3EvalStartReaders( - Fts3Cursor *pCsr, /* FTS Cursor handle */ - Fts3Expr *pExpr, /* Expression to initialize phrases in */ - int *pRc /* IN/OUT: Error code */ +static void *fts3MallocZero(sqlite3_int64 nByte){ + void *pRet = sqlite3_malloc64(nByte); + if( pRet ) memset(pRet, 0, nByte); + return pRet; +} + +SQLITE_PRIVATE int sqlite3Fts3OpenTokenizer( + sqlite3_tokenizer *pTokenizer, + int iLangid, + const char *z, + int n, + sqlite3_tokenizer_cursor **ppCsr ){ - if( pExpr && SQLITE_OK==*pRc ){ - if( pExpr->eType==FTSQUERY_PHRASE ){ - int nToken = pExpr->pPhrase->nToken; - if( nToken ){ - int i; - for(i=0; ipPhrase->aToken[i].pDeferred==0 ) break; - } - pExpr->bDeferred = (i==nToken); + sqlite3_tokenizer_module const *pModule = pTokenizer->pModule; + sqlite3_tokenizer_cursor *pCsr = 0; + int rc; + + rc = pModule->xOpen(pTokenizer, z, n, &pCsr); + assert( rc==SQLITE_OK || pCsr==0 ); + if( rc==SQLITE_OK ){ + pCsr->pTokenizer = pTokenizer; + if( pModule->iVersion>=1 ){ + rc = pModule->xLanguageid(pCsr, iLangid); + if( rc!=SQLITE_OK ){ + pModule->xClose(pCsr); + pCsr = 0; } - *pRc = fts3EvalPhraseStart(pCsr, 1, pExpr->pPhrase); - }else{ - fts3EvalStartReaders(pCsr, pExpr->pLeft, pRc); - fts3EvalStartReaders(pCsr, pExpr->pRight, pRc); - pExpr->bDeferred = (pExpr->pLeft->bDeferred && pExpr->pRight->bDeferred); } } + *ppCsr = pCsr; + return rc; } /* -** An array of the following structures is assembled as part of the process -** of selecting tokens to defer before the query starts executing (as part -** of the xFilter() method). There is one element in the array for each -** token in the FTS expression. -** -** Tokens are divided into AND/NEAR clusters. All tokens in a cluster belong -** to phrases that are connected only by AND and NEAR operators (not OR or -** NOT). When determining tokens to defer, each AND/NEAR cluster is considered -** separately. The root of a tokens AND/NEAR cluster is stored in -** Fts3TokenAndCost.pRoot. +** Function getNextNode(), which is called by fts3ExprParse(), may itself +** call fts3ExprParse(). So this forward declaration is required. */ -typedef struct Fts3TokenAndCost Fts3TokenAndCost; -struct Fts3TokenAndCost { - Fts3Phrase *pPhrase; /* The phrase the token belongs to */ - int iToken; /* Position of token in phrase */ - Fts3PhraseToken *pToken; /* The token itself */ - Fts3Expr *pRoot; /* Root of NEAR/AND cluster */ - int nOvfl; /* Number of overflow pages to load doclist */ - int iCol; /* The column the token must match */ -}; +static int fts3ExprParse(ParseContext *, const char *, int, Fts3Expr **, int *); /* -** This function is used to populate an allocated Fts3TokenAndCost array. +** Extract the next token from buffer z (length n) using the tokenizer +** and other information (column names etc.) in pParse. Create an Fts3Expr +** structure of type FTSQUERY_PHRASE containing a phrase consisting of this +** single token and set *ppExpr to point to it. If the end of the buffer is +** reached before a token is found, set *ppExpr to zero. It is the +** responsibility of the caller to eventually deallocate the allocated +** Fts3Expr structure (if any) by passing it to sqlite3_free(). ** -** If *pRc is not SQLITE_OK when this function is called, it is a no-op. -** Otherwise, if an error occurs during execution, *pRc is set to an -** SQLite error code. +** Return SQLITE_OK if successful, or SQLITE_NOMEM if a memory allocation +** fails. */ -static void fts3EvalTokenCosts( - Fts3Cursor *pCsr, /* FTS Cursor handle */ - Fts3Expr *pRoot, /* Root of current AND/NEAR cluster */ - Fts3Expr *pExpr, /* Expression to consider */ - Fts3TokenAndCost **ppTC, /* Write new entries to *(*ppTC)++ */ - Fts3Expr ***ppOr, /* Write new OR root to *(*ppOr)++ */ - int *pRc /* IN/OUT: Error code */ +static int getNextToken( + ParseContext *pParse, /* fts3 query parse context */ + int iCol, /* Value for Fts3Phrase.iColumn */ + const char *z, int n, /* Input string */ + Fts3Expr **ppExpr, /* OUT: expression */ + int *pnConsumed /* OUT: Number of bytes consumed */ ){ - if( *pRc==SQLITE_OK ){ - if( pExpr->eType==FTSQUERY_PHRASE ){ - Fts3Phrase *pPhrase = pExpr->pPhrase; - int i; - for(i=0; *pRc==SQLITE_OK && inToken; i++){ - Fts3TokenAndCost *pTC = (*ppTC)++; - pTC->pPhrase = pPhrase; - pTC->iToken = i; - pTC->pRoot = pRoot; - pTC->pToken = &pPhrase->aToken[i]; - pTC->iCol = pPhrase->iColumn; - *pRc = sqlite3Fts3MsrOvfl(pCsr, pTC->pToken->pSegcsr, &pTC->nOvfl); - } - }else if( pExpr->eType!=FTSQUERY_NOT ){ - assert( pExpr->eType==FTSQUERY_OR - || pExpr->eType==FTSQUERY_AND - || pExpr->eType==FTSQUERY_NEAR - ); - assert( pExpr->pLeft && pExpr->pRight ); - if( pExpr->eType==FTSQUERY_OR ){ - pRoot = pExpr->pLeft; - **ppOr = pRoot; - (*ppOr)++; - } - fts3EvalTokenCosts(pCsr, pRoot, pExpr->pLeft, ppTC, ppOr, pRc); - if( pExpr->eType==FTSQUERY_OR ){ - pRoot = pExpr->pRight; - **ppOr = pRoot; - (*ppOr)++; + sqlite3_tokenizer *pTokenizer = pParse->pTokenizer; + sqlite3_tokenizer_module const *pModule = pTokenizer->pModule; + int rc; + sqlite3_tokenizer_cursor *pCursor; + Fts3Expr *pRet = 0; + int i = 0; + + /* Set variable i to the maximum number of bytes of input to tokenize. */ + for(i=0; iiLangid, z, i, &pCursor); + if( rc==SQLITE_OK ){ + const char *zToken; + int nToken = 0, iStart = 0, iEnd = 0, iPosition = 0; + sqlite3_int64 nByte; /* total space to allocate */ + + rc = pModule->xNext(pCursor, &zToken, &nToken, &iStart, &iEnd, &iPosition); + if( rc==SQLITE_OK ){ + nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase) + nToken; + pRet = (Fts3Expr *)fts3MallocZero(nByte); + if( !pRet ){ + rc = SQLITE_NOMEM; + }else{ + pRet->eType = FTSQUERY_PHRASE; + pRet->pPhrase = (Fts3Phrase *)&pRet[1]; + pRet->pPhrase->nToken = 1; + pRet->pPhrase->iColumn = iCol; + pRet->pPhrase->aToken[0].n = nToken; + pRet->pPhrase->aToken[0].z = (char *)&pRet->pPhrase[1]; + memcpy(pRet->pPhrase->aToken[0].z, zToken, nToken); + + if( iEndpPhrase->aToken[0].isPrefix = 1; + iEnd++; + } + + while( 1 ){ + if( !sqlite3_fts3_enable_parentheses + && iStart>0 && z[iStart-1]=='-' + ){ + pParse->isNot = 1; + iStart--; + }else if( pParse->bFts4 && iStart>0 && z[iStart-1]=='^' ){ + pRet->pPhrase->aToken[0].bFirst = 1; + iStart--; + }else{ + break; + } + } + } - fts3EvalTokenCosts(pCsr, pRoot, pExpr->pRight, ppTC, ppOr, pRc); + *pnConsumed = iEnd; + }else if( i && rc==SQLITE_DONE ){ + rc = SQLITE_OK; } + + pModule->xClose(pCursor); } + + *ppExpr = pRet; + return rc; } + /* -** Determine the average document (row) size in pages. If successful, -** write this value to *pnPage and return SQLITE_OK. Otherwise, return -** an SQLite error code. -** -** The average document size in pages is calculated by first calculating -** determining the average size in bytes, B. If B is less than the amount -** of data that will fit on a single leaf page of an intkey table in -** this database, then the average docsize is 1. Otherwise, it is 1 plus -** the number of overflow pages consumed by a record B bytes in size. +** Enlarge a memory allocation. If an out-of-memory allocation occurs, +** then free the old allocation. */ -static int fts3EvalAverageDocsize(Fts3Cursor *pCsr, int *pnPage){ - int rc = SQLITE_OK; - if( pCsr->nRowAvg==0 ){ - /* The average document size, which is required to calculate the cost - ** of each doclist, has not yet been determined. Read the required - ** data from the %_stat table to calculate it. - ** - ** Entry 0 of the %_stat table is a blob containing (nCol+1) FTS3 - ** varints, where nCol is the number of columns in the FTS3 table. - ** The first varint is the number of documents currently stored in - ** the table. The following nCol varints contain the total amount of - ** data stored in all rows of each column of the table, from left - ** to right. - */ - Fts3Table *p = (Fts3Table*)pCsr->base.pVtab; - sqlite3_stmt *pStmt; - sqlite3_int64 nDoc = 0; - sqlite3_int64 nByte = 0; - const char *pEnd; - const char *a; - - rc = sqlite3Fts3SelectDoctotal(p, &pStmt); - if( rc!=SQLITE_OK ) return rc; - a = sqlite3_column_blob(pStmt, 0); - assert( a ); - - pEnd = &a[sqlite3_column_bytes(pStmt, 0)]; - a += sqlite3Fts3GetVarint(a, &nDoc); - while( anDoc = nDoc; - pCsr->nRowAvg = (int)(((nByte / nDoc) + p->nPgsz) / p->nPgsz); - assert( pCsr->nRowAvg>0 ); - rc = sqlite3_reset(pStmt); +static void *fts3ReallocOrFree(void *pOrig, sqlite3_int64 nNew){ + void *pRet = sqlite3_realloc64(pOrig, nNew); + if( !pRet ){ + sqlite3_free(pOrig); } - - *pnPage = pCsr->nRowAvg; - return rc; + return pRet; } /* -** This function is called to select the tokens (if any) that will be -** deferred. The array aTC[] has already been populated when this is -** called. -** -** This function is called once for each AND/NEAR cluster in the -** expression. Each invocation determines which tokens to defer within -** the cluster with root node pRoot. See comments above the definition -** of struct Fts3TokenAndCost for more details. +** Buffer zInput, length nInput, contains the contents of a quoted string +** that appeared as part of an fts3 query expression. Neither quote character +** is included in the buffer. This function attempts to tokenize the entire +** input buffer and create an Fts3Expr structure of type FTSQUERY_PHRASE +** containing the results. ** -** If no error occurs, SQLITE_OK is returned and sqlite3Fts3DeferToken() -** called on each token to defer. Otherwise, an SQLite error code is -** returned. +** If successful, SQLITE_OK is returned and *ppExpr set to point at the +** allocated Fts3Expr structure. Otherwise, either SQLITE_NOMEM (out of memory +** error) or SQLITE_ERROR (tokenization error) is returned and *ppExpr set +** to 0. */ -static int fts3EvalSelectDeferred( - Fts3Cursor *pCsr, /* FTS Cursor handle */ - Fts3Expr *pRoot, /* Consider tokens with this root node */ - Fts3TokenAndCost *aTC, /* Array of expression tokens and costs */ - int nTC /* Number of entries in aTC[] */ +static int getNextString( + ParseContext *pParse, /* fts3 query parse context */ + const char *zInput, int nInput, /* Input string */ + Fts3Expr **ppExpr /* OUT: expression */ ){ - Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; - int nDocSize = 0; /* Number of pages per doc loaded */ - int rc = SQLITE_OK; /* Return code */ - int ii; /* Iterator variable for various purposes */ - int nOvfl = 0; /* Total overflow pages used by doclists */ - int nToken = 0; /* Total number of tokens in cluster */ - - int nMinEst = 0; /* The minimum count for any phrase so far. */ - int nLoad4 = 1; /* (Phrases that will be loaded)^4. */ - - /* Tokens are never deferred for FTS tables created using the content=xxx - ** option. The reason being that it is not guaranteed that the content - ** table actually contains the same data as the index. To prevent this from - ** causing any problems, the deferred token optimization is completely - ** disabled for content=xxx tables. */ - if( pTab->zContentTbl ){ - return SQLITE_OK; - } - - /* Count the tokens in this AND/NEAR cluster. If none of the doclists - ** associated with the tokens spill onto overflow pages, or if there is - ** only 1 token, exit early. No tokens to defer in this case. */ - for(ii=0; ii0 ); + sqlite3_tokenizer *pTokenizer = pParse->pTokenizer; + sqlite3_tokenizer_module const *pModule = pTokenizer->pModule; + int rc; + Fts3Expr *p = 0; + sqlite3_tokenizer_cursor *pCursor = 0; + char *zTemp = 0; + int nTemp = 0; + const int nSpace = sizeof(Fts3Expr) + sizeof(Fts3Phrase); + int nToken = 0; - /* Iterate through all tokens in this AND/NEAR cluster, in ascending order - ** of the number of overflow pages that will be loaded by the pager layer - ** to retrieve the entire doclist for the token from the full-text index. - ** Load the doclists for tokens that are either: - ** - ** a. The cheapest token in the entire query (i.e. the one visited by the - ** first iteration of this loop), or + /* The final Fts3Expr data structure, including the Fts3Phrase, + ** Fts3PhraseToken structures token buffers are all stored as a single + ** allocation so that the expression can be freed with a single call to + ** sqlite3_free(). Setting this up requires a two pass approach. ** - ** b. Part of a multi-token phrase. + ** The first pass, in the block below, uses a tokenizer cursor to iterate + ** through the tokens in the expression. This pass uses fts3ReallocOrFree() + ** to assemble data in two dynamic buffers: ** - ** After each token doclist is loaded, merge it with the others from the - ** same phrase and count the number of documents that the merged doclist - ** contains. Set variable "nMinEst" to the smallest number of documents in - ** any phrase doclist for which 1 or more token doclists have been loaded. - ** Let nOther be the number of other phrases for which it is certain that - ** one or more tokens will not be deferred. + ** Buffer p: Points to the Fts3Expr structure, followed by the Fts3Phrase + ** structure, followed by the array of Fts3PhraseToken + ** structures. This pass only populates the Fts3PhraseToken array. ** - ** Then, for each token, defer it if loading the doclist would result in - ** loading N or more overflow pages into memory, where N is computed as: + ** Buffer zTemp: Contains copies of all tokens. ** - ** (nMinEst + 4^nOther - 1) / (4^nOther) + ** The second pass, in the block that begins "if( rc==SQLITE_DONE )" below, + ** appends buffer zTemp to buffer p, and fills in the Fts3Expr and Fts3Phrase + ** structures. */ - for(ii=0; iiiLangid, zInput, nInput, &pCursor); + if( rc==SQLITE_OK ){ + int ii; + for(ii=0; rc==SQLITE_OK; ii++){ + const char *zByte; + int nByte = 0, iBegin = 0, iEnd = 0, iPos = 0; + rc = pModule->xNext(pCursor, &zByte, &nByte, &iBegin, &iEnd, &iPos); + if( rc==SQLITE_OK ){ + Fts3PhraseToken *pToken; - /* Set pTC to point to the cheapest remaining token. */ - for(iTC=0; iTCnOvfl) - ){ - pTC = &aTC[iTC]; + p = fts3ReallocOrFree(p, nSpace + ii*sizeof(Fts3PhraseToken)); + if( !p ) goto no_mem; + + zTemp = fts3ReallocOrFree(zTemp, nTemp + nByte); + if( !zTemp ) goto no_mem; + + assert( nToken==ii ); + pToken = &((Fts3Phrase *)(&p[1]))->aToken[ii]; + memset(pToken, 0, sizeof(Fts3PhraseToken)); + + memcpy(&zTemp[nTemp], zByte, nByte); + nTemp += nByte; + + pToken->n = nByte; + pToken->isPrefix = (iEndbFirst = (iBegin>0 && zInput[iBegin-1]=='^'); + nToken = ii+1; } } - assert( pTC ); - if( ii && pTC->nOvfl>=((nMinEst+(nLoad4/4)-1)/(nLoad4/4))*nDocSize ){ - /* The number of overflow pages to load for this (and therefore all - ** subsequent) tokens is greater than the estimated number of pages - ** that will be loaded if all subsequent tokens are deferred. - */ - Fts3PhraseToken *pToken = pTC->pToken; - rc = sqlite3Fts3DeferToken(pCsr, pToken, pTC->iCol); - fts3SegReaderCursorFree(pToken->pSegcsr); - pToken->pSegcsr = 0; + pModule->xClose(pCursor); + pCursor = 0; + } + + if( rc==SQLITE_DONE ){ + int jj; + char *zBuf = 0; + + p = fts3ReallocOrFree(p, nSpace + nToken*sizeof(Fts3PhraseToken) + nTemp); + if( !p ) goto no_mem; + memset(p, 0, (char *)&(((Fts3Phrase *)&p[1])->aToken[0])-(char *)p); + p->eType = FTSQUERY_PHRASE; + p->pPhrase = (Fts3Phrase *)&p[1]; + p->pPhrase->iColumn = pParse->iDefaultCol; + p->pPhrase->nToken = nToken; + + zBuf = (char *)&p->pPhrase->aToken[nToken]; + if( zTemp ){ + memcpy(zBuf, zTemp, nTemp); + sqlite3_free(zTemp); }else{ - /* Set nLoad4 to the value of (4^nOther) for the next iteration of the - ** for-loop. Except, limit the value to 2^24 to prevent it from - ** overflowing the 32-bit integer it is stored in. */ - if( ii<12 ) nLoad4 = nLoad4*4; + assert( nTemp==0 ); + } - if( ii==0 || (pTC->pPhrase->nToken>1 && ii!=nToken-1) ){ - /* Either this is the cheapest token in the entire query, or it is - ** part of a multi-token phrase. Either way, the entire doclist will - ** (eventually) be loaded into memory. It may as well be now. */ - Fts3PhraseToken *pToken = pTC->pToken; - int nList = 0; - char *pList = 0; - rc = fts3TermSelect(pTab, pToken, pTC->iCol, &nList, &pList); - assert( rc==SQLITE_OK || pList==0 ); - if( rc==SQLITE_OK ){ - rc = fts3EvalPhraseMergeToken( - pTab, pTC->pPhrase, pTC->iToken,pList,nList - ); - } - if( rc==SQLITE_OK ){ - int nCount; - nCount = fts3DoclistCountDocids( - pTC->pPhrase->doclist.aAll, pTC->pPhrase->doclist.nAll - ); - if( ii==0 || nCountpPhrase->nToken; jj++){ + p->pPhrase->aToken[jj].z = zBuf; + zBuf += p->pPhrase->aToken[jj].n; } - pTC->pToken = 0; + rc = SQLITE_OK; } + *ppExpr = p; return rc; +no_mem: + + if( pCursor ){ + pModule->xClose(pCursor); + } + sqlite3_free(zTemp); + sqlite3_free(p); + *ppExpr = 0; + return SQLITE_NOMEM; } /* -** This function is called from within the xFilter method. It initializes -** the full-text query currently stored in pCsr->pExpr. To iterate through -** the results of a query, the caller does: +** The output variable *ppExpr is populated with an allocated Fts3Expr +** structure, or set to 0 if the end of the input buffer is reached. ** -** fts3EvalStart(pCsr); -** while( 1 ){ -** fts3EvalNext(pCsr); -** if( pCsr->bEof ) break; -** ... return row pCsr->iPrevId to the caller ... -** } +** Returns an SQLite error code. SQLITE_OK if everything works, SQLITE_NOMEM +** if a malloc failure occurs, or SQLITE_ERROR if a parse error is encountered. +** If SQLITE_ERROR is returned, pContext is populated with an error message. */ -static int fts3EvalStart(Fts3Cursor *pCsr){ - Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; - int rc = SQLITE_OK; - int nToken = 0; - int nOr = 0; +static int getNextNode( + ParseContext *pParse, /* fts3 query parse context */ + const char *z, int n, /* Input string */ + Fts3Expr **ppExpr, /* OUT: expression */ + int *pnConsumed /* OUT: Number of bytes consumed */ +){ + static const struct Fts3Keyword { + char *z; /* Keyword text */ + unsigned char n; /* Length of the keyword */ + unsigned char parenOnly; /* Only valid in paren mode */ + unsigned char eType; /* Keyword code */ + } aKeyword[] = { + { "OR" , 2, 0, FTSQUERY_OR }, + { "AND", 3, 1, FTSQUERY_AND }, + { "NOT", 3, 1, FTSQUERY_NOT }, + { "NEAR", 4, 0, FTSQUERY_NEAR } + }; + int ii; + int iCol; + int iColLen; + int rc; + Fts3Expr *pRet = 0; - /* Allocate a MultiSegReader for each token in the expression. */ - fts3EvalAllocateReaders(pCsr, pCsr->pExpr, &nToken, &nOr, &rc); + const char *zInput = z; + int nInput = n; - /* Determine which, if any, tokens in the expression should be deferred. */ -#ifndef SQLITE_DISABLE_FTS4_DEFERRED - if( rc==SQLITE_OK && nToken>1 && pTab->bFts4 ){ - Fts3TokenAndCost *aTC; - Fts3Expr **apOr; - aTC = (Fts3TokenAndCost *)sqlite3_malloc( - sizeof(Fts3TokenAndCost) * nToken - + sizeof(Fts3Expr *) * nOr * 2 - ); - apOr = (Fts3Expr **)&aTC[nToken]; + pParse->isNot = 0; - if( !aTC ){ - rc = SQLITE_NOMEM; - }else{ - int ii; - Fts3TokenAndCost *pTC = aTC; - Fts3Expr **ppOr = apOr; + /* Skip over any whitespace before checking for a keyword, an open or + ** close bracket, or a quoted string. + */ + while( nInput>0 && fts3isspace(*zInput) ){ + nInput--; + zInput++; + } + if( nInput==0 ){ + return SQLITE_DONE; + } - fts3EvalTokenCosts(pCsr, 0, pCsr->pExpr, &pTC, &ppOr, &rc); - nToken = (int)(pTC-aTC); - nOr = (int)(ppOr-apOr); + /* See if we are dealing with a keyword. */ + for(ii=0; ii<(int)(sizeof(aKeyword)/sizeof(struct Fts3Keyword)); ii++){ + const struct Fts3Keyword *pKey = &aKeyword[ii]; - if( rc==SQLITE_OK ){ - rc = fts3EvalSelectDeferred(pCsr, 0, aTC, nToken); - for(ii=0; rc==SQLITE_OK && iiparenOnly & ~sqlite3_fts3_enable_parentheses)!=0 ){ + continue; + } + + if( nInput>=pKey->n && 0==memcmp(zInput, pKey->z, pKey->n) ){ + int nNear = SQLITE_FTS3_DEFAULT_NEAR_PARAM; + int nKey = pKey->n; + char cNext; + + /* If this is a "NEAR" keyword, check for an explicit nearness. */ + if( pKey->eType==FTSQUERY_NEAR ){ + assert( nKey==4 ); + if( zInput[4]=='/' && zInput[5]>='0' && zInput[5]<='9' ){ + nKey += 1+sqlite3Fts3ReadInt(&zInput[nKey+1], &nNear); } } - sqlite3_free(aTC); + /* At this point this is probably a keyword. But for that to be true, + ** the next byte must contain either whitespace, an open or close + ** parenthesis, a quote character, or EOF. + */ + cNext = zInput[nKey]; + if( fts3isspace(cNext) + || cNext=='"' || cNext=='(' || cNext==')' || cNext==0 + ){ + pRet = (Fts3Expr *)fts3MallocZero(sizeof(Fts3Expr)); + if( !pRet ){ + return SQLITE_NOMEM; + } + pRet->eType = pKey->eType; + pRet->nNear = nNear; + *ppExpr = pRet; + *pnConsumed = (int)((zInput - z) + nKey); + return SQLITE_OK; + } + + /* Turns out that wasn't a keyword after all. This happens if the + ** user has supplied a token such as "ORacle". Continue. + */ } } -#endif - fts3EvalStartReaders(pCsr, pCsr->pExpr, &rc); + /* See if we are dealing with a quoted phrase. If this is the case, then + ** search for the closing quote and pass the whole string to getNextString() + ** for processing. This is easy to do, as fts3 has no syntax for escaping + ** a quote character embedded in a string. + */ + if( *zInput=='"' ){ + for(ii=1; iinNest++; + rc = fts3ExprParse(pParse, zInput+1, nInput-1, ppExpr, &nConsumed); + *pnConsumed = (int)(zInput - z) + 1 + nConsumed; + return rc; + }else if( *zInput==')' ){ + pParse->nNest--; + *pnConsumed = (int)((zInput - z) + 1); + *ppExpr = 0; + return SQLITE_DONE; + } + } + + /* If control flows to this point, this must be a regular token, or + ** the end of the input. Read a regular token using the sqlite3_tokenizer + ** interface. Before doing so, figure out if there is an explicit + ** column specifier for the token. + ** + ** TODO: Strangely, it is not possible to associate a column specifier + ** with a quoted phrase, only with a single token. Not sure if this was + ** an implementation artifact or an intentional decision when fts3 was + ** first implemented. Whichever it was, this module duplicates the + ** limitation. + */ + iCol = pParse->iDefaultCol; + iColLen = 0; + for(ii=0; iinCol; ii++){ + const char *zStr = pParse->azCol[ii]; + int nStr = (int)strlen(zStr); + if( nInput>nStr && zInput[nStr]==':' + && sqlite3_strnicmp(zStr, zInput, nStr)==0 + ){ + iCol = ii; + iColLen = (int)((zInput - z) + nStr + 1); + break; + } + } + rc = getNextToken(pParse, iCol, &z[iColLen], n-iColLen, ppExpr, pnConsumed); + *pnConsumed += iColLen; return rc; } /* -** Invalidate the current position list for phrase pPhrase. +** The argument is an Fts3Expr structure for a binary operator (any type +** except an FTSQUERY_PHRASE). Return an integer value representing the +** precedence of the operator. Lower values have a higher precedence (i.e. +** group more tightly). For example, in the C language, the == operator +** groups more tightly than ||, and would therefore have a higher precedence. +** +** When using the new fts3 query syntax (when SQLITE_ENABLE_FTS3_PARENTHESIS +** is defined), the order of the operators in precedence from highest to +** lowest is: +** +** NEAR +** NOT +** AND (including implicit ANDs) +** OR +** +** Note that when using the old query syntax, the OR operator has a higher +** precedence than the AND operator. */ -static void fts3EvalInvalidatePoslist(Fts3Phrase *pPhrase){ - if( pPhrase->doclist.bFreeList ){ - sqlite3_free(pPhrase->doclist.pList); +static int opPrecedence(Fts3Expr *p){ + assert( p->eType!=FTSQUERY_PHRASE ); + if( sqlite3_fts3_enable_parentheses ){ + return p->eType; + }else if( p->eType==FTSQUERY_NEAR ){ + return 1; + }else if( p->eType==FTSQUERY_OR ){ + return 2; } - pPhrase->doclist.pList = 0; - pPhrase->doclist.nList = 0; - pPhrase->doclist.bFreeList = 0; + assert( p->eType==FTSQUERY_AND ); + return 3; } /* -** This function is called to edit the position list associated with -** the phrase object passed as the fifth argument according to a NEAR -** condition. For example: -** -** abc NEAR/5 "def ghi" -** -** Parameter nNear is passed the NEAR distance of the expression (5 in -** the example above). When this function is called, *paPoslist points to -** the position list, and *pnToken is the number of phrase tokens in, the -** phrase on the other side of the NEAR operator to pPhrase. For example, -** if pPhrase refers to the "def ghi" phrase, then *paPoslist points to -** the position list associated with phrase "abc". -** -** All positions in the pPhrase position list that are not sufficiently -** close to a position in the *paPoslist position list are removed. If this -** leaves 0 positions, zero is returned. Otherwise, non-zero. -** -** Before returning, *paPoslist is set to point to the position lsit -** associated with pPhrase. And *pnToken is set to the number of tokens in -** pPhrase. +** Argument ppHead contains a pointer to the current head of a query +** expression tree being parsed. pPrev is the expression node most recently +** inserted into the tree. This function adds pNew, which is always a binary +** operator node, into the expression tree based on the relative precedence +** of pNew and the existing nodes of the tree. This may result in the head +** of the tree changing, in which case *ppHead is set to the new root node. */ -static int fts3EvalNearTrim( - int nNear, /* NEAR distance. As in "NEAR/nNear". */ - char *aTmp, /* Temporary space to use */ - char **paPoslist, /* IN/OUT: Position list */ - int *pnToken, /* IN/OUT: Tokens in phrase of *paPoslist */ - Fts3Phrase *pPhrase /* The phrase object to trim the doclist of */ +static void insertBinaryOperator( + Fts3Expr **ppHead, /* Pointer to the root node of a tree */ + Fts3Expr *pPrev, /* Node most recently inserted into the tree */ + Fts3Expr *pNew /* New binary node to insert into expression tree */ ){ - int nParam1 = nNear + pPhrase->nToken; - int nParam2 = nNear + *pnToken; - int nNew; - char *p2; - char *pOut; - int res; - - assert( pPhrase->doclist.pList ); - - p2 = pOut = pPhrase->doclist.pList; - res = fts3PoslistNearMerge( - &pOut, aTmp, nParam1, nParam2, paPoslist, &p2 - ); - if( res ){ - nNew = (int)(pOut - pPhrase->doclist.pList) - 1; - assert( pPhrase->doclist.pList[nNew]=='\0' ); - assert( nNew<=pPhrase->doclist.nList && nNew>0 ); - memset(&pPhrase->doclist.pList[nNew], 0, pPhrase->doclist.nList - nNew); - pPhrase->doclist.nList = nNew; - *paPoslist = pPhrase->doclist.pList; - *pnToken = pPhrase->nToken; + Fts3Expr *pSplit = pPrev; + while( pSplit->pParent && opPrecedence(pSplit->pParent)<=opPrecedence(pNew) ){ + pSplit = pSplit->pParent; } - return res; + if( pSplit->pParent ){ + assert( pSplit->pParent->pRight==pSplit ); + pSplit->pParent->pRight = pNew; + pNew->pParent = pSplit->pParent; + }else{ + *ppHead = pNew; + } + pNew->pLeft = pSplit; + pSplit->pParent = pNew; } /* -** This function is a no-op if *pRc is other than SQLITE_OK when it is called. -** Otherwise, it advances the expression passed as the second argument to -** point to the next matching row in the database. Expressions iterate through -** matching rows in docid order. Ascending order if Fts3Cursor.bDesc is zero, -** or descending if it is non-zero. -** -** If an error occurs, *pRc is set to an SQLite error code. Otherwise, if -** successful, the following variables in pExpr are set: -** -** Fts3Expr.bEof (non-zero if EOF - there is no next row) -** Fts3Expr.iDocid (valid if bEof==0. The docid of the next row) -** -** If the expression is of type FTSQUERY_PHRASE, and the expression is not -** at EOF, then the following variables are populated with the position list -** for the phrase for the visited row: -** -** FTs3Expr.pPhrase->doclist.nList (length of pList in bytes) -** FTs3Expr.pPhrase->doclist.pList (pointer to position list) -** -** It says above that this function advances the expression to the next -** matching row. This is usually true, but there are the following exceptions: -** -** 1. Deferred tokens are not taken into account. If a phrase consists -** entirely of deferred tokens, it is assumed to match every row in -** the db. In this case the position-list is not populated at all. -** -** Or, if a phrase contains one or more deferred tokens and one or -** more non-deferred tokens, then the expression is advanced to the -** next possible match, considering only non-deferred tokens. In other -** words, if the phrase is "A B C", and "B" is deferred, the expression -** is advanced to the next row that contains an instance of "A * C", -** where "*" may match any single token. The position list in this case -** is populated as for "A * C" before returning. +** Parse the fts3 query expression found in buffer z, length n. This function +** returns either when the end of the buffer is reached or an unmatched +** closing bracket - ')' - is encountered. ** -** 2. NEAR is treated as AND. If the expression is "x NEAR y", it is -** advanced to point to the next row that matches "x AND y". -** -** See sqlite3Fts3EvalTestDeferred() for details on testing if a row is -** really a match, taking into account deferred tokens and NEAR operators. +** If successful, SQLITE_OK is returned, *ppExpr is set to point to the +** parsed form of the expression and *pnConsumed is set to the number of +** bytes read from buffer z. Otherwise, *ppExpr is set to 0 and SQLITE_NOMEM +** (out of memory error) or SQLITE_ERROR (parse error) is returned. */ -static void fts3EvalNextRow( - Fts3Cursor *pCsr, /* FTS Cursor handle */ - Fts3Expr *pExpr, /* Expr. to advance to next matching row */ - int *pRc /* IN/OUT: Error code */ +static int fts3ExprParse( + ParseContext *pParse, /* fts3 query parse context */ + const char *z, int n, /* Text of MATCH query */ + Fts3Expr **ppExpr, /* OUT: Parsed query structure */ + int *pnConsumed /* OUT: Number of bytes consumed */ ){ - if( *pRc==SQLITE_OK ){ - int bDescDoclist = pCsr->bDesc; /* Used by DOCID_CMP() macro */ - assert( pExpr->bEof==0 ); - pExpr->bStart = 1; + Fts3Expr *pRet = 0; + Fts3Expr *pPrev = 0; + Fts3Expr *pNotBranch = 0; /* Only used in legacy parse mode */ + int nIn = n; + const char *zIn = z; + int rc = SQLITE_OK; + int isRequirePhrase = 1; - switch( pExpr->eType ){ - case FTSQUERY_NEAR: - case FTSQUERY_AND: { - Fts3Expr *pLeft = pExpr->pLeft; - Fts3Expr *pRight = pExpr->pRight; - assert( !pLeft->bDeferred || !pRight->bDeferred ); + while( rc==SQLITE_OK ){ + Fts3Expr *p = 0; + int nByte = 0; - if( pLeft->bDeferred ){ - /* LHS is entirely deferred. So we assume it matches every row. - ** Advance the RHS iterator to find the next row visited. */ - fts3EvalNextRow(pCsr, pRight, pRc); - pExpr->iDocid = pRight->iDocid; - pExpr->bEof = pRight->bEof; - }else if( pRight->bDeferred ){ - /* RHS is entirely deferred. So we assume it matches every row. - ** Advance the LHS iterator to find the next row visited. */ - fts3EvalNextRow(pCsr, pLeft, pRc); - pExpr->iDocid = pLeft->iDocid; - pExpr->bEof = pLeft->bEof; + rc = getNextNode(pParse, zIn, nIn, &p, &nByte); + assert( nByte>0 || (rc!=SQLITE_OK && p==0) ); + if( rc==SQLITE_OK ){ + if( p ){ + int isPhrase; + + if( !sqlite3_fts3_enable_parentheses + && p->eType==FTSQUERY_PHRASE && pParse->isNot + ){ + /* Create an implicit NOT operator. */ + Fts3Expr *pNot = fts3MallocZero(sizeof(Fts3Expr)); + if( !pNot ){ + sqlite3Fts3ExprFree(p); + rc = SQLITE_NOMEM; + goto exprparse_out; + } + pNot->eType = FTSQUERY_NOT; + pNot->pRight = p; + p->pParent = pNot; + if( pNotBranch ){ + pNot->pLeft = pNotBranch; + pNotBranch->pParent = pNot; + } + pNotBranch = pNot; + p = pPrev; }else{ - /* Neither the RHS or LHS are deferred. */ - fts3EvalNextRow(pCsr, pLeft, pRc); - fts3EvalNextRow(pCsr, pRight, pRc); - while( !pLeft->bEof && !pRight->bEof && *pRc==SQLITE_OK ){ - sqlite3_int64 iDiff = DOCID_CMP(pLeft->iDocid, pRight->iDocid); - if( iDiff==0 ) break; - if( iDiff<0 ){ - fts3EvalNextRow(pCsr, pLeft, pRc); - }else{ - fts3EvalNextRow(pCsr, pRight, pRc); - } + int eType = p->eType; + isPhrase = (eType==FTSQUERY_PHRASE || p->pLeft); + + /* The isRequirePhrase variable is set to true if a phrase or + ** an expression contained in parenthesis is required. If a + ** binary operator (AND, OR, NOT or NEAR) is encounted when + ** isRequirePhrase is set, this is a syntax error. + */ + if( !isPhrase && isRequirePhrase ){ + sqlite3Fts3ExprFree(p); + rc = SQLITE_ERROR; + goto exprparse_out; } - pExpr->iDocid = pLeft->iDocid; - pExpr->bEof = (pLeft->bEof || pRight->bEof); - if( pExpr->eType==FTSQUERY_NEAR && pExpr->bEof ){ - assert( pRight->eType==FTSQUERY_PHRASE ); - if( pRight->pPhrase->doclist.aAll ){ - Fts3Doclist *pDl = &pRight->pPhrase->doclist; - while( *pRc==SQLITE_OK && pRight->bEof==0 ){ - memset(pDl->pList, 0, pDl->nList); - fts3EvalNextRow(pCsr, pRight, pRc); - } - } - if( pLeft->pPhrase && pLeft->pPhrase->doclist.aAll ){ - Fts3Doclist *pDl = &pLeft->pPhrase->doclist; - while( *pRc==SQLITE_OK && pLeft->bEof==0 ){ - memset(pDl->pList, 0, pDl->nList); - fts3EvalNextRow(pCsr, pLeft, pRc); - } + + if( isPhrase && !isRequirePhrase ){ + /* Insert an implicit AND operator. */ + Fts3Expr *pAnd; + assert( pRet && pPrev ); + pAnd = fts3MallocZero(sizeof(Fts3Expr)); + if( !pAnd ){ + sqlite3Fts3ExprFree(p); + rc = SQLITE_NOMEM; + goto exprparse_out; } + pAnd->eType = FTSQUERY_AND; + insertBinaryOperator(&pRet, pPrev, pAnd); + pPrev = pAnd; } - } - break; - } - - case FTSQUERY_OR: { - Fts3Expr *pLeft = pExpr->pLeft; - Fts3Expr *pRight = pExpr->pRight; - sqlite3_int64 iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid); - - assert( pLeft->bStart || pLeft->iDocid==pRight->iDocid ); - assert( pRight->bStart || pLeft->iDocid==pRight->iDocid ); - if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){ - fts3EvalNextRow(pCsr, pLeft, pRc); - }else if( pLeft->bEof || iCmp>0 ){ - fts3EvalNextRow(pCsr, pRight, pRc); - }else{ - fts3EvalNextRow(pCsr, pLeft, pRc); - fts3EvalNextRow(pCsr, pRight, pRc); - } + /* This test catches attempts to make either operand of a NEAR + ** operator something other than a phrase. For example, either of + ** the following: + ** + ** (bracketed expression) NEAR phrase + ** phrase NEAR (bracketed expression) + ** + ** Return an error in either case. + */ + if( pPrev && ( + (eType==FTSQUERY_NEAR && !isPhrase && pPrev->eType!=FTSQUERY_PHRASE) + || (eType!=FTSQUERY_PHRASE && isPhrase && pPrev->eType==FTSQUERY_NEAR) + )){ + sqlite3Fts3ExprFree(p); + rc = SQLITE_ERROR; + goto exprparse_out; + } - pExpr->bEof = (pLeft->bEof && pRight->bEof); - iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid); - if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){ - pExpr->iDocid = pLeft->iDocid; - }else{ - pExpr->iDocid = pRight->iDocid; + if( isPhrase ){ + if( pRet ){ + assert( pPrev && pPrev->pLeft && pPrev->pRight==0 ); + pPrev->pRight = p; + p->pParent = pPrev; + }else{ + pRet = p; + } + }else{ + insertBinaryOperator(&pRet, pPrev, p); + } + isRequirePhrase = !isPhrase; } - - break; + pPrev = p; } + assert( nByte>0 ); + } + assert( rc!=SQLITE_OK || (nByte>0 && nByte<=nIn) ); + nIn -= nByte; + zIn += nByte; + } - case FTSQUERY_NOT: { - Fts3Expr *pLeft = pExpr->pLeft; - Fts3Expr *pRight = pExpr->pRight; - - if( pRight->bStart==0 ){ - fts3EvalNextRow(pCsr, pRight, pRc); - assert( *pRc!=SQLITE_OK || pRight->bStart ); - } + if( rc==SQLITE_DONE && pRet && isRequirePhrase ){ + rc = SQLITE_ERROR; + } - fts3EvalNextRow(pCsr, pLeft, pRc); - if( pLeft->bEof==0 ){ - while( !*pRc - && !pRight->bEof - && DOCID_CMP(pLeft->iDocid, pRight->iDocid)>0 - ){ - fts3EvalNextRow(pCsr, pRight, pRc); - } + if( rc==SQLITE_DONE ){ + rc = SQLITE_OK; + if( !sqlite3_fts3_enable_parentheses && pNotBranch ){ + if( !pRet ){ + rc = SQLITE_ERROR; + }else{ + Fts3Expr *pIter = pNotBranch; + while( pIter->pLeft ){ + pIter = pIter->pLeft; } - pExpr->iDocid = pLeft->iDocid; - pExpr->bEof = pLeft->bEof; - break; + pIter->pLeft = pRet; + pRet->pParent = pIter; + pRet = pNotBranch; } + } + } + *pnConsumed = n - nIn; - default: { - Fts3Phrase *pPhrase = pExpr->pPhrase; - fts3EvalInvalidatePoslist(pPhrase); - *pRc = fts3EvalPhraseNext(pCsr, pPhrase, &pExpr->bEof); - pExpr->iDocid = pPhrase->doclist.iDocid; - break; +exprparse_out: + if( rc!=SQLITE_OK ){ + sqlite3Fts3ExprFree(pRet); + sqlite3Fts3ExprFree(pNotBranch); + pRet = 0; + } + *ppExpr = pRet; + return rc; +} + +/* +** Return SQLITE_ERROR if the maximum depth of the expression tree passed +** as the only argument is more than nMaxDepth. +*/ +static int fts3ExprCheckDepth(Fts3Expr *p, int nMaxDepth){ + int rc = SQLITE_OK; + if( p ){ + if( nMaxDepth<0 ){ + rc = SQLITE_TOOBIG; + }else{ + rc = fts3ExprCheckDepth(p->pLeft, nMaxDepth-1); + if( rc==SQLITE_OK ){ + rc = fts3ExprCheckDepth(p->pRight, nMaxDepth-1); } } } + return rc; } /* -** If *pRc is not SQLITE_OK, or if pExpr is not the root node of a NEAR -** cluster, then this function returns 1 immediately. +** This function attempts to transform the expression tree at (*pp) to +** an equivalent but more balanced form. The tree is modified in place. +** If successful, SQLITE_OK is returned and (*pp) set to point to the +** new root expression node. ** -** Otherwise, it checks if the current row really does match the NEAR -** expression, using the data currently stored in the position lists -** (Fts3Expr->pPhrase.doclist.pList/nList) for each phrase in the expression. +** nMaxDepth is the maximum allowable depth of the balanced sub-tree. ** -** If the current row is a match, the position list associated with each -** phrase in the NEAR expression is edited in place to contain only those -** phrase instances sufficiently close to their peers to satisfy all NEAR -** constraints. In this case it returns 1. If the NEAR expression does not -** match the current row, 0 is returned. The position lists may or may not -** be edited if 0 is returned. +** Otherwise, if an error occurs, an SQLite error code is returned and +** expression (*pp) freed. */ -static int fts3EvalNearTest(Fts3Expr *pExpr, int *pRc){ - int res = 1; - - /* The following block runs if pExpr is the root of a NEAR query. - ** For example, the query: - ** - ** "w" NEAR "x" NEAR "y" NEAR "z" - ** - ** which is represented in tree form as: - ** - ** | - ** +--NEAR--+ <-- root of NEAR query - ** | | - ** +--NEAR--+ "z" - ** | | - ** +--NEAR--+ "y" - ** | | - ** "w" "x" - ** - ** The right-hand child of a NEAR node is always a phrase. The - ** left-hand child may be either a phrase or a NEAR node. There are - ** no exceptions to this - it's the way the parser in fts3_expr.c works. - */ - if( *pRc==SQLITE_OK - && pExpr->eType==FTSQUERY_NEAR - && (pExpr->pParent==0 || pExpr->pParent->eType!=FTSQUERY_NEAR) - ){ - Fts3Expr *p; - int nTmp = 0; /* Bytes of temp space */ - char *aTmp; /* Temp space for PoslistNearMerge() */ +static int fts3ExprBalance(Fts3Expr **pp, int nMaxDepth){ + int rc = SQLITE_OK; /* Return code */ + Fts3Expr *pRoot = *pp; /* Initial root node */ + Fts3Expr *pFree = 0; /* List of free nodes. Linked by pParent. */ + int eType = pRoot->eType; /* Type of node in this tree */ - /* Allocate temporary working space. */ - for(p=pExpr; p->pLeft; p=p->pLeft){ - assert( p->pRight->pPhrase->doclist.nList>0 ); - nTmp += p->pRight->pPhrase->doclist.nList; - } - nTmp += p->pPhrase->doclist.nList; - aTmp = sqlite3_malloc(nTmp*2); - if( !aTmp ){ - *pRc = SQLITE_NOMEM; - res = 0; - }else{ - char *aPoslist = p->pPhrase->doclist.pList; - int nToken = p->pPhrase->nToken; + if( nMaxDepth==0 ){ + rc = SQLITE_ERROR; + } - for(p=p->pParent;res && p && p->eType==FTSQUERY_NEAR; p=p->pParent){ - Fts3Phrase *pPhrase = p->pRight->pPhrase; - int nNear = p->nNear; - res = fts3EvalNearTrim(nNear, aTmp, &aPoslist, &nToken, pPhrase); + if( rc==SQLITE_OK ){ + if( (eType==FTSQUERY_AND || eType==FTSQUERY_OR) ){ + Fts3Expr **apLeaf; + apLeaf = (Fts3Expr **)sqlite3_malloc64(sizeof(Fts3Expr *) * nMaxDepth); + if( 0==apLeaf ){ + rc = SQLITE_NOMEM; + }else{ + memset(apLeaf, 0, sizeof(Fts3Expr *) * nMaxDepth); } - aPoslist = pExpr->pRight->pPhrase->doclist.pList; - nToken = pExpr->pRight->pPhrase->nToken; - for(p=pExpr->pLeft; p && res; p=p->pLeft){ - int nNear; - Fts3Phrase *pPhrase; - assert( p->pParent && p->pParent->pLeft==p ); - nNear = p->pParent->nNear; - pPhrase = ( - p->eType==FTSQUERY_NEAR ? p->pRight->pPhrase : p->pPhrase - ); - res = fts3EvalNearTrim(nNear, aTmp, &aPoslist, &nToken, pPhrase); - } - } + if( rc==SQLITE_OK ){ + int i; + Fts3Expr *p; - sqlite3_free(aTmp); - } + /* Set $p to point to the left-most leaf in the tree of eType nodes. */ + for(p=pRoot; p->eType==eType; p=p->pLeft){ + assert( p->pParent==0 || p->pParent->pLeft==p ); + assert( p->pLeft && p->pRight ); + } - return res; -} + /* This loop runs once for each leaf in the tree of eType nodes. */ + while( 1 ){ + int iLvl; + Fts3Expr *pParent = p->pParent; /* Current parent of p */ -/* -** This function is a helper function for sqlite3Fts3EvalTestDeferred(). -** Assuming no error occurs or has occurred, It returns non-zero if the -** expression passed as the second argument matches the row that pCsr -** currently points to, or zero if it does not. -** -** If *pRc is not SQLITE_OK when this function is called, it is a no-op. -** If an error occurs during execution of this function, *pRc is set to -** the appropriate SQLite error code. In this case the returned value is -** undefined. -*/ -static int fts3EvalTestExpr( - Fts3Cursor *pCsr, /* FTS cursor handle */ - Fts3Expr *pExpr, /* Expr to test. May or may not be root. */ - int *pRc /* IN/OUT: Error code */ -){ - int bHit = 1; /* Return value */ - if( *pRc==SQLITE_OK ){ - switch( pExpr->eType ){ - case FTSQUERY_NEAR: - case FTSQUERY_AND: - bHit = ( - fts3EvalTestExpr(pCsr, pExpr->pLeft, pRc) - && fts3EvalTestExpr(pCsr, pExpr->pRight, pRc) - && fts3EvalNearTest(pExpr, pRc) - ); + assert( pParent==0 || pParent->pLeft==p ); + p->pParent = 0; + if( pParent ){ + pParent->pLeft = 0; + }else{ + pRoot = 0; + } + rc = fts3ExprBalance(&p, nMaxDepth-1); + if( rc!=SQLITE_OK ) break; - /* If the NEAR expression does not match any rows, zero the doclist for - ** all phrases involved in the NEAR. This is because the snippet(), - ** offsets() and matchinfo() functions are not supposed to recognize - ** any instances of phrases that are part of unmatched NEAR queries. - ** For example if this expression: - ** - ** ... MATCH 'a OR (b NEAR c)' - ** - ** is matched against a row containing: - ** - ** 'a b d e' - ** - ** then any snippet() should ony highlight the "a" term, not the "b" - ** (as "b" is part of a non-matching NEAR clause). - */ - if( bHit==0 - && pExpr->eType==FTSQUERY_NEAR - && (pExpr->pParent==0 || pExpr->pParent->eType!=FTSQUERY_NEAR) - ){ - Fts3Expr *p; - for(p=pExpr; p->pPhrase==0; p=p->pLeft){ - if( p->pRight->iDocid==pCsr->iPrevId ){ - fts3EvalInvalidatePoslist(p->pRight->pPhrase); + for(iLvl=0; p && iLvlpLeft = apLeaf[iLvl]; + pFree->pRight = p; + pFree->pLeft->pParent = pFree; + pFree->pRight->pParent = pFree; + + p = pFree; + pFree = pFree->pParent; + p->pParent = 0; + apLeaf[iLvl] = 0; } } - if( p->iDocid==pCsr->iPrevId ){ - fts3EvalInvalidatePoslist(p->pPhrase); + if( p ){ + sqlite3Fts3ExprFree(p); + rc = SQLITE_TOOBIG; + break; } - } - break; + /* If that was the last leaf node, break out of the loop */ + if( pParent==0 ) break; - case FTSQUERY_OR: { - int bHit1 = fts3EvalTestExpr(pCsr, pExpr->pLeft, pRc); - int bHit2 = fts3EvalTestExpr(pCsr, pExpr->pRight, pRc); - bHit = bHit1 || bHit2; - break; - } + /* Set $p to point to the next leaf in the tree of eType nodes */ + for(p=pParent->pRight; p->eType==eType; p=p->pLeft); - case FTSQUERY_NOT: - bHit = ( - fts3EvalTestExpr(pCsr, pExpr->pLeft, pRc) - && !fts3EvalTestExpr(pCsr, pExpr->pRight, pRc) - ); - break; + /* Remove pParent from the original tree. */ + assert( pParent->pParent==0 || pParent->pParent->pLeft==pParent ); + pParent->pRight->pParent = pParent->pParent; + if( pParent->pParent ){ + pParent->pParent->pLeft = pParent->pRight; + }else{ + assert( pParent==pRoot ); + pRoot = pParent->pRight; + } - default: { -#ifndef SQLITE_DISABLE_FTS4_DEFERRED - if( pCsr->pDeferred - && (pExpr->iDocid==pCsr->iPrevId || pExpr->bDeferred) - ){ - Fts3Phrase *pPhrase = pExpr->pPhrase; - assert( pExpr->bDeferred || pPhrase->doclist.bFreeList==0 ); - if( pExpr->bDeferred ){ - fts3EvalInvalidatePoslist(pPhrase); + /* Link pParent into the free node list. It will be used as an + ** internal node of the new tree. */ + pParent->pParent = pFree; + pFree = pParent; + } + + if( rc==SQLITE_OK ){ + p = 0; + for(i=0; ipParent = 0; + }else{ + assert( pFree!=0 ); + pFree->pRight = p; + pFree->pLeft = apLeaf[i]; + pFree->pLeft->pParent = pFree; + pFree->pRight->pParent = pFree; + + p = pFree; + pFree = pFree->pParent; + p->pParent = 0; + } + } + } + pRoot = p; + }else{ + /* An error occurred. Delete the contents of the apLeaf[] array + ** and pFree list. Everything else is cleaned up by the call to + ** sqlite3Fts3ExprFree(pRoot) below. */ + Fts3Expr *pDel; + for(i=0; ipParent; + sqlite3_free(pDel); } - *pRc = fts3EvalDeferredPhrase(pCsr, pPhrase); - bHit = (pPhrase->doclist.pList!=0); - pExpr->iDocid = pCsr->iPrevId; - }else -#endif - { - bHit = (pExpr->bEof==0 && pExpr->iDocid==pCsr->iPrevId); } - break; + + assert( pFree==0 ); + sqlite3_free( apLeaf ); + } + }else if( eType==FTSQUERY_NOT ){ + Fts3Expr *pLeft = pRoot->pLeft; + Fts3Expr *pRight = pRoot->pRight; + + pRoot->pLeft = 0; + pRoot->pRight = 0; + pLeft->pParent = 0; + pRight->pParent = 0; + + rc = fts3ExprBalance(&pLeft, nMaxDepth-1); + if( rc==SQLITE_OK ){ + rc = fts3ExprBalance(&pRight, nMaxDepth-1); + } + + if( rc!=SQLITE_OK ){ + sqlite3Fts3ExprFree(pRight); + sqlite3Fts3ExprFree(pLeft); + }else{ + assert( pLeft && pRight ); + pRoot->pLeft = pLeft; + pLeft->pParent = pRoot; + pRoot->pRight = pRight; + pRight->pParent = pRoot; } } } - return bHit; + + if( rc!=SQLITE_OK ){ + sqlite3Fts3ExprFree(pRoot); + pRoot = 0; + } + *pp = pRoot; + return rc; } /* -** This function is called as the second part of each xNext operation when -** iterating through the results of a full-text query. At this point the -** cursor points to a row that matches the query expression, with the -** following caveats: -** -** * Up until this point, "NEAR" operators in the expression have been -** treated as "AND". -** -** * Deferred tokens have not yet been considered. -** -** If *pRc is not SQLITE_OK when this function is called, it immediately -** returns 0. Otherwise, it tests whether or not after considering NEAR -** operators and deferred tokens the current row is still a match for the -** expression. It returns 1 if both of the following are true: -** -** 1. *pRc is SQLITE_OK when this function returns, and -** -** 2. After scanning the current FTS table row for the deferred tokens, -** it is determined that the row does *not* match the query. +** This function is similar to sqlite3Fts3ExprParse(), with the following +** differences: ** -** Or, if no error occurs and it seems the current row does match the FTS -** query, return 0. +** 1. It does not do expression rebalancing. +** 2. It does not check that the expression does not exceed the +** maximum allowable depth. +** 3. Even if it fails, *ppExpr may still be set to point to an +** expression tree. It should be deleted using sqlite3Fts3ExprFree() +** in this case. */ -SQLITE_PRIVATE int sqlite3Fts3EvalTestDeferred(Fts3Cursor *pCsr, int *pRc){ - int rc = *pRc; - int bMiss = 0; - if( rc==SQLITE_OK ){ +static int fts3ExprParseUnbalanced( + sqlite3_tokenizer *pTokenizer, /* Tokenizer module */ + int iLangid, /* Language id for tokenizer */ + char **azCol, /* Array of column names for fts3 table */ + int bFts4, /* True to allow FTS4-only syntax */ + int nCol, /* Number of entries in azCol[] */ + int iDefaultCol, /* Default column to query */ + const char *z, int n, /* Text of MATCH query */ + Fts3Expr **ppExpr /* OUT: Parsed query structure */ +){ + int nParsed; + int rc; + ParseContext sParse; - /* If there are one or more deferred tokens, load the current row into - ** memory and scan it to determine the position list for each deferred - ** token. Then, see if this row is really a match, considering deferred - ** tokens and NEAR operators (neither of which were taken into account - ** earlier, by fts3EvalNextRow()). - */ - if( pCsr->pDeferred ){ - rc = fts3CursorSeek(0, pCsr); - if( rc==SQLITE_OK ){ - rc = sqlite3Fts3CacheDeferredDoclists(pCsr); - } - } - bMiss = (0==fts3EvalTestExpr(pCsr, pCsr->pExpr, &rc)); + memset(&sParse, 0, sizeof(ParseContext)); + sParse.pTokenizer = pTokenizer; + sParse.iLangid = iLangid; + sParse.azCol = (const char **)azCol; + sParse.nCol = nCol; + sParse.iDefaultCol = iDefaultCol; + sParse.bFts4 = bFts4; + if( z==0 ){ + *ppExpr = 0; + return SQLITE_OK; + } + if( n<0 ){ + n = (int)strlen(z); + } + rc = fts3ExprParse(&sParse, z, n, ppExpr, &nParsed); + assert( rc==SQLITE_OK || *ppExpr==0 ); - /* Free the position-lists accumulated for each deferred token above. */ - sqlite3Fts3FreeDeferredDoclists(pCsr); - *pRc = rc; + /* Check for mismatched parenthesis */ + if( rc==SQLITE_OK && sParse.nNest ){ + rc = SQLITE_ERROR; } - return (rc==SQLITE_OK && bMiss); + + return rc; } /* -** Advance to the next document that matches the FTS expression in -** Fts3Cursor.pExpr. +** Parameters z and n contain a pointer to and length of a buffer containing +** an fts3 query expression, respectively. This function attempts to parse the +** query expression and create a tree of Fts3Expr structures representing the +** parsed expression. If successful, *ppExpr is set to point to the head +** of the parsed expression tree and SQLITE_OK is returned. If an error +** occurs, either SQLITE_NOMEM (out-of-memory error) or SQLITE_ERROR (parse +** error) is returned and *ppExpr is set to 0. +** +** If parameter n is a negative number, then z is assumed to point to a +** nul-terminated string and the length is determined using strlen(). +** +** The first parameter, pTokenizer, is passed the fts3 tokenizer module to +** use to normalize query tokens while parsing the expression. The azCol[] +** array, which is assumed to contain nCol entries, should contain the names +** of each column in the target fts3 table, in order from left to right. +** Column names must be nul-terminated strings. +** +** The iDefaultCol parameter should be passed the index of the table column +** that appears on the left-hand-side of the MATCH operator (the default +** column to match against for tokens for which a column name is not explicitly +** specified as part of the query string), or -1 if tokens may by default +** match any table column. */ -static int fts3EvalNext(Fts3Cursor *pCsr){ - int rc = SQLITE_OK; /* Return Code */ - Fts3Expr *pExpr = pCsr->pExpr; - assert( pCsr->isEof==0 ); - if( pExpr==0 ){ - pCsr->isEof = 1; - }else{ - do { - if( pCsr->isRequireSeek==0 ){ - sqlite3_reset(pCsr->pStmt); - } - assert( sqlite3_data_count(pCsr->pStmt)==0 ); - fts3EvalNextRow(pCsr, pExpr, &rc); - pCsr->isEof = pExpr->bEof; - pCsr->isRequireSeek = 1; - pCsr->isMatchinfoNeeded = 1; - pCsr->iPrevId = pExpr->iDocid; - }while( pCsr->isEof==0 && sqlite3Fts3EvalTestDeferred(pCsr, &rc) ); +SQLITE_PRIVATE int sqlite3Fts3ExprParse( + sqlite3_tokenizer *pTokenizer, /* Tokenizer module */ + int iLangid, /* Language id for tokenizer */ + char **azCol, /* Array of column names for fts3 table */ + int bFts4, /* True to allow FTS4-only syntax */ + int nCol, /* Number of entries in azCol[] */ + int iDefaultCol, /* Default column to query */ + const char *z, int n, /* Text of MATCH query */ + Fts3Expr **ppExpr, /* OUT: Parsed query structure */ + char **pzErr /* OUT: Error message (sqlite3_malloc) */ +){ + int rc = fts3ExprParseUnbalanced( + pTokenizer, iLangid, azCol, bFts4, nCol, iDefaultCol, z, n, ppExpr + ); + + /* Rebalance the expression. And check that its depth does not exceed + ** SQLITE_FTS3_MAX_EXPR_DEPTH. */ + if( rc==SQLITE_OK && *ppExpr ){ + rc = fts3ExprBalance(ppExpr, SQLITE_FTS3_MAX_EXPR_DEPTH); + if( rc==SQLITE_OK ){ + rc = fts3ExprCheckDepth(*ppExpr, SQLITE_FTS3_MAX_EXPR_DEPTH); + } } - /* Check if the cursor is past the end of the docid range specified - ** by Fts3Cursor.iMinDocid/iMaxDocid. If so, set the EOF flag. */ - if( rc==SQLITE_OK && ( - (pCsr->bDesc==0 && pCsr->iPrevId>pCsr->iMaxDocid) - || (pCsr->bDesc!=0 && pCsr->iPrevIdiMinDocid) - )){ - pCsr->isEof = 1; + if( rc!=SQLITE_OK ){ + sqlite3Fts3ExprFree(*ppExpr); + *ppExpr = 0; + if( rc==SQLITE_TOOBIG ){ + sqlite3Fts3ErrMsg(pzErr, + "FTS expression tree is too large (maximum depth %d)", + SQLITE_FTS3_MAX_EXPR_DEPTH + ); + rc = SQLITE_ERROR; + }else if( rc==SQLITE_ERROR ){ + sqlite3Fts3ErrMsg(pzErr, "malformed MATCH expression: [%s]", z); + } } return rc; } /* -** Restart interation for expression pExpr so that the next call to -** fts3EvalNext() visits the first row. Do not allow incremental -** loading or merging of phrase doclists for this iteration. -** -** If *pRc is other than SQLITE_OK when this function is called, it is -** a no-op. If an error occurs within this function, *pRc is set to an -** SQLite error code before returning. +** Free a single node of an expression tree. */ -static void fts3EvalRestart( - Fts3Cursor *pCsr, - Fts3Expr *pExpr, - int *pRc -){ - if( pExpr && *pRc==SQLITE_OK ){ - Fts3Phrase *pPhrase = pExpr->pPhrase; +static void fts3FreeExprNode(Fts3Expr *p){ + assert( p->eType==FTSQUERY_PHRASE || p->pPhrase==0 ); + sqlite3Fts3EvalPhraseCleanup(p->pPhrase); + sqlite3_free(p->aMI); + sqlite3_free(p); +} - if( pPhrase ){ - fts3EvalInvalidatePoslist(pPhrase); - if( pPhrase->bIncr ){ - int i; - for(i=0; inToken; i++){ - Fts3PhraseToken *pToken = &pPhrase->aToken[i]; - assert( pToken->pDeferred==0 ); - if( pToken->pSegcsr ){ - sqlite3Fts3MsrIncrRestart(pToken->pSegcsr); - } - } - *pRc = fts3EvalPhraseStart(pCsr, 0, pPhrase); +/* +** Free a parsed fts3 query expression allocated by sqlite3Fts3ExprParse(). +** +** This function would be simpler if it recursively called itself. But +** that would mean passing a sufficiently large expression to ExprParse() +** could cause a stack overflow. +*/ +SQLITE_PRIVATE void sqlite3Fts3ExprFree(Fts3Expr *pDel){ + Fts3Expr *p; + assert( pDel==0 || pDel->pParent==0 ); + for(p=pDel; p && (p->pLeft||p->pRight); p=(p->pLeft ? p->pLeft : p->pRight)){ + assert( p->pParent==0 || p==p->pParent->pRight || p==p->pParent->pLeft ); + } + while( p ){ + Fts3Expr *pParent = p->pParent; + fts3FreeExprNode(p); + if( pParent && p==pParent->pLeft && pParent->pRight ){ + p = pParent->pRight; + while( p && (p->pLeft || p->pRight) ){ + assert( p==p->pParent->pRight || p==p->pParent->pLeft ); + p = (p->pLeft ? p->pLeft : p->pRight); } - pPhrase->doclist.pNextDocid = 0; - pPhrase->doclist.iDocid = 0; - pPhrase->pOrPoslist = 0; + }else{ + p = pParent; } - - pExpr->iDocid = 0; - pExpr->bEof = 0; - pExpr->bStart = 0; - - fts3EvalRestart(pCsr, pExpr->pLeft, pRc); - fts3EvalRestart(pCsr, pExpr->pRight, pRc); } } -/* -** After allocating the Fts3Expr.aMI[] array for each phrase in the -** expression rooted at pExpr, the cursor iterates through all rows matched -** by pExpr, calling this function for each row. This function increments -** the values in Fts3Expr.aMI[] according to the position-list currently -** found in Fts3Expr.pPhrase->doclist.pList for each of the phrase -** expression nodes. +/**************************************************************************** +***************************************************************************** +** Everything after this point is just test code. */ -static void fts3EvalUpdateCounts(Fts3Expr *pExpr){ - if( pExpr ){ - Fts3Phrase *pPhrase = pExpr->pPhrase; - if( pPhrase && pPhrase->doclist.pList ){ - int iCol = 0; - char *p = pPhrase->doclist.pList; - assert( *p ); - while( 1 ){ - u8 c = 0; - int iCnt = 0; - while( 0xFE & (*p | c) ){ - if( (c&0x80)==0 ) iCnt++; - c = *p++ & 0x80; - } +#ifdef SQLITE_TEST - /* aMI[iCol*3 + 1] = Number of occurrences - ** aMI[iCol*3 + 2] = Number of rows containing at least one instance - */ - pExpr->aMI[iCol*3 + 1] += iCnt; - pExpr->aMI[iCol*3 + 2] += (iCnt>0); - if( *p==0x00 ) break; - p++; - p += fts3GetVarint32(p, &iCol); +/* #include */ + +/* +** Return a pointer to a buffer containing a text representation of the +** expression passed as the first argument. The buffer is obtained from +** sqlite3_malloc(). It is the responsibility of the caller to use +** sqlite3_free() to release the memory. If an OOM condition is encountered, +** NULL is returned. +** +** If the second argument is not NULL, then its contents are prepended to +** the returned expression text and then freed using sqlite3_free(). +*/ +static char *exprToString(Fts3Expr *pExpr, char *zBuf){ + if( pExpr==0 ){ + return sqlite3_mprintf(""); + } + switch( pExpr->eType ){ + case FTSQUERY_PHRASE: { + Fts3Phrase *pPhrase = pExpr->pPhrase; + int i; + zBuf = sqlite3_mprintf( + "%zPHRASE %d 0", zBuf, pPhrase->iColumn); + for(i=0; zBuf && inToken; i++){ + zBuf = sqlite3_mprintf("%z %.*s%s", zBuf, + pPhrase->aToken[i].n, pPhrase->aToken[i].z, + (pPhrase->aToken[i].isPrefix?"+":"") + ); } + return zBuf; } - fts3EvalUpdateCounts(pExpr->pLeft); - fts3EvalUpdateCounts(pExpr->pRight); + case FTSQUERY_NEAR: + zBuf = sqlite3_mprintf("%zNEAR/%d ", zBuf, pExpr->nNear); + break; + case FTSQUERY_NOT: + zBuf = sqlite3_mprintf("%zNOT ", zBuf); + break; + case FTSQUERY_AND: + zBuf = sqlite3_mprintf("%zAND ", zBuf); + break; + case FTSQUERY_OR: + zBuf = sqlite3_mprintf("%zOR ", zBuf); + break; } + + if( zBuf ) zBuf = sqlite3_mprintf("%z{", zBuf); + if( zBuf ) zBuf = exprToString(pExpr->pLeft, zBuf); + if( zBuf ) zBuf = sqlite3_mprintf("%z} {", zBuf); + + if( zBuf ) zBuf = exprToString(pExpr->pRight, zBuf); + if( zBuf ) zBuf = sqlite3_mprintf("%z}", zBuf); + + return zBuf; } /* -** Expression pExpr must be of type FTSQUERY_PHRASE. +** This is the implementation of a scalar SQL function used to test the +** expression parser. It should be called as follows: ** -** If it is not already allocated and populated, this function allocates and -** populates the Fts3Expr.aMI[] array for expression pExpr. If pExpr is part -** of a NEAR expression, then it also allocates and populates the same array -** for all other phrases that are part of the NEAR expression. +** fts3_exprtest(, , , ...); ** -** SQLITE_OK is returned if the aMI[] array is successfully allocated and -** populated. Otherwise, if an error occurs, an SQLite error code is returned. +** The first argument, , is the name of the fts3 tokenizer used +** to parse the query expression (see README.tokenizers). The second argument +** is the query expression to parse. Each subsequent argument is the name +** of a column of the fts3 table that the query expression may refer to. +** For example: +** +** SELECT fts3_exprtest('simple', 'Bill col2:Bloggs', 'col1', 'col2'); */ -static int fts3EvalGatherStats( - Fts3Cursor *pCsr, /* Cursor object */ - Fts3Expr *pExpr /* FTSQUERY_PHRASE expression */ +static void fts3ExprTestCommon( + int bRebalance, + sqlite3_context *context, + int argc, + sqlite3_value **argv ){ - int rc = SQLITE_OK; /* Return code */ - - assert( pExpr->eType==FTSQUERY_PHRASE ); - if( pExpr->aMI==0 ){ - Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; - Fts3Expr *pRoot; /* Root of NEAR expression */ - Fts3Expr *p; /* Iterator used for several purposes */ - - sqlite3_int64 iPrevId = pCsr->iPrevId; - sqlite3_int64 iDocid; - u8 bEof; + sqlite3_tokenizer *pTokenizer = 0; + int rc; + char **azCol = 0; + const char *zExpr; + int nExpr; + int nCol; + int ii; + Fts3Expr *pExpr; + char *zBuf = 0; + Fts3Hash *pHash = (Fts3Hash*)sqlite3_user_data(context); + const char *zTokenizer = 0; + char *zErr = 0; - /* Find the root of the NEAR expression */ - pRoot = pExpr; - while( pRoot->pParent && pRoot->pParent->eType==FTSQUERY_NEAR ){ - pRoot = pRoot->pParent; - } - iDocid = pRoot->iDocid; - bEof = pRoot->bEof; - assert( pRoot->bStart ); + if( argc<3 ){ + sqlite3_result_error(context, + "Usage: fts3_exprtest(tokenizer, expr, col1, ...", -1 + ); + return; + } - /* Allocate space for the aMSI[] array of each FTSQUERY_PHRASE node */ - for(p=pRoot; p; p=p->pLeft){ - Fts3Expr *pE = (p->eType==FTSQUERY_PHRASE?p:p->pRight); - assert( pE->aMI==0 ); - pE->aMI = (u32 *)sqlite3_malloc(pTab->nColumn * 3 * sizeof(u32)); - if( !pE->aMI ) return SQLITE_NOMEM; - memset(pE->aMI, 0, pTab->nColumn * 3 * sizeof(u32)); + zTokenizer = (const char*)sqlite3_value_text(argv[0]); + rc = sqlite3Fts3InitTokenizer(pHash, zTokenizer, &pTokenizer, &zErr); + if( rc!=SQLITE_OK ){ + if( rc==SQLITE_NOMEM ){ + sqlite3_result_error_nomem(context); + }else{ + sqlite3_result_error(context, zErr, -1); } + sqlite3_free(zErr); + return; + } - fts3EvalRestart(pCsr, pRoot, &rc); + zExpr = (const char *)sqlite3_value_text(argv[1]); + nExpr = sqlite3_value_bytes(argv[1]); + nCol = argc-2; + azCol = (char **)sqlite3_malloc64(nCol*sizeof(char *)); + if( !azCol ){ + sqlite3_result_error_nomem(context); + goto exprtest_out; + } + for(ii=0; iiisEof==0 && rc==SQLITE_OK ){ + if( bRebalance ){ + char *zDummy = 0; + rc = sqlite3Fts3ExprParse( + pTokenizer, 0, azCol, 0, nCol, nCol, zExpr, nExpr, &pExpr, &zDummy + ); + assert( rc==SQLITE_OK || pExpr==0 ); + sqlite3_free(zDummy); + }else{ + rc = fts3ExprParseUnbalanced( + pTokenizer, 0, azCol, 0, nCol, nCol, zExpr, nExpr, &pExpr + ); + } - do { - /* Ensure the %_content statement is reset. */ - if( pCsr->isRequireSeek==0 ) sqlite3_reset(pCsr->pStmt); - assert( sqlite3_data_count(pCsr->pStmt)==0 ); + if( rc!=SQLITE_OK && rc!=SQLITE_NOMEM ){ + sqlite3Fts3ExprFree(pExpr); + sqlite3_result_error(context, "Error parsing expression", -1); + }else if( rc==SQLITE_NOMEM || !(zBuf = exprToString(pExpr, 0)) ){ + sqlite3_result_error_nomem(context); + }else{ + sqlite3_result_text(context, zBuf, -1, SQLITE_TRANSIENT); + sqlite3_free(zBuf); + } - /* Advance to the next document */ - fts3EvalNextRow(pCsr, pRoot, &rc); - pCsr->isEof = pRoot->bEof; - pCsr->isRequireSeek = 1; - pCsr->isMatchinfoNeeded = 1; - pCsr->iPrevId = pRoot->iDocid; - }while( pCsr->isEof==0 - && pRoot->eType==FTSQUERY_NEAR - && sqlite3Fts3EvalTestDeferred(pCsr, &rc) - ); + sqlite3Fts3ExprFree(pExpr); - if( rc==SQLITE_OK && pCsr->isEof==0 ){ - fts3EvalUpdateCounts(pRoot); - } - } +exprtest_out: + if( pTokenizer ){ + rc = pTokenizer->pModule->xDestroy(pTokenizer); + } + sqlite3_free(azCol); +} - pCsr->isEof = 0; - pCsr->iPrevId = iPrevId; +static void fts3ExprTest( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + fts3ExprTestCommon(0, context, argc, argv); +} +static void fts3ExprTestRebalance( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + fts3ExprTestCommon(1, context, argc, argv); +} - if( bEof ){ - pRoot->bEof = bEof; - }else{ - /* Caution: pRoot may iterate through docids in ascending or descending - ** order. For this reason, even though it seems more defensive, the - ** do loop can not be written: - ** - ** do {...} while( pRoot->iDocidbEof==0 ); - }while( pRoot->iDocid!=iDocid && rc==SQLITE_OK ); - } +/* +** Register the query expression parser test function fts3_exprtest() +** with database connection db. +*/ +SQLITE_PRIVATE int sqlite3Fts3ExprInitTestInterface(sqlite3 *db, Fts3Hash *pHash){ + int rc = sqlite3_create_function( + db, "fts3_exprtest", -1, SQLITE_UTF8, (void*)pHash, fts3ExprTest, 0, 0 + ); + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "fts3_exprtest_rebalance", + -1, SQLITE_UTF8, (void*)pHash, fts3ExprTestRebalance, 0, 0 + ); } return rc; } +#endif +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ + +/************** End of fts3_expr.c *******************************************/ +/************** Begin file fts3_hash.c ***************************************/ /* -** This function is used by the matchinfo() module to query a phrase -** expression node for the following information: -** -** 1. The total number of occurrences of the phrase in each column of -** the FTS table (considering all rows), and -** -** 2. For each column, the number of rows in the table for which the -** column contains at least one instance of the phrase. +** 2001 September 22 ** -** If no error occurs, SQLITE_OK is returned and the values for each column -** written into the array aiOut as follows: +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: ** -** aiOut[iCol*3 + 1] = Number of occurrences -** aiOut[iCol*3 + 2] = Number of rows containing at least one instance +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. ** -** Caveats: +************************************************************************* +** This is the implementation of generic hash-tables used in SQLite. +** We've modified it slightly to serve as a standalone hash table +** implementation for the full-text indexing module. +*/ + +/* +** The code in this file is only compiled if: ** -** * If a phrase consists entirely of deferred tokens, then all output -** values are set to the number of documents in the table. In other -** words we assume that very common tokens occur exactly once in each -** column of each row of the table. +** * The FTS3 module is being built as an extension +** (in which case SQLITE_CORE is not defined), or ** -** * If a phrase contains some deferred tokens (and some non-deferred -** tokens), count the potential occurrence identified by considering -** the non-deferred tokens instead of actual phrase occurrences. +** * The FTS3 module is being built into the core of +** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). +*/ +/* #include "fts3Int.h" */ +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + +/* #include */ +/* #include */ +/* #include */ + +/* #include "fts3_hash.h" */ + +/* +** Malloc and Free functions +*/ +static void *fts3HashMalloc(sqlite3_int64 n){ + void *p = sqlite3_malloc64(n); + if( p ){ + memset(p, 0, n); + } + return p; +} +static void fts3HashFree(void *p){ + sqlite3_free(p); +} + +/* Turn bulk memory into a hash table object by initializing the +** fields of the Hash structure. ** -** * If the phrase is part of a NEAR expression, then only phrase instances -** that meet the NEAR constraint are included in the counts. +** "pNew" is a pointer to the hash table that is to be initialized. +** keyClass is one of the constants +** FTS3_HASH_BINARY or FTS3_HASH_STRING. The value of keyClass +** determines what kind of key the hash table will use. "copyKey" is +** true if the hash table should make its own private copy of keys and +** false if it should just use the supplied pointer. */ -SQLITE_PRIVATE int sqlite3Fts3EvalPhraseStats( - Fts3Cursor *pCsr, /* FTS cursor handle */ - Fts3Expr *pExpr, /* Phrase expression */ - u32 *aiOut /* Array to write results into (see above) */ -){ - Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; - int rc = SQLITE_OK; - int iCol; +SQLITE_PRIVATE void sqlite3Fts3HashInit(Fts3Hash *pNew, char keyClass, char copyKey){ + assert( pNew!=0 ); + assert( keyClass>=FTS3_HASH_STRING && keyClass<=FTS3_HASH_BINARY ); + pNew->keyClass = keyClass; + pNew->copyKey = copyKey; + pNew->first = 0; + pNew->count = 0; + pNew->htsize = 0; + pNew->ht = 0; +} - if( pExpr->bDeferred && pExpr->pParent->eType!=FTSQUERY_NEAR ){ - assert( pCsr->nDoc>0 ); - for(iCol=0; iColnColumn; iCol++){ - aiOut[iCol*3 + 1] = (u32)pCsr->nDoc; - aiOut[iCol*3 + 2] = (u32)pCsr->nDoc; - } - }else{ - rc = fts3EvalGatherStats(pCsr, pExpr); - if( rc==SQLITE_OK ){ - assert( pExpr->aMI ); - for(iCol=0; iColnColumn; iCol++){ - aiOut[iCol*3 + 1] = pExpr->aMI[iCol*3 + 1]; - aiOut[iCol*3 + 2] = pExpr->aMI[iCol*3 + 2]; - } +/* Remove all entries from a hash table. Reclaim all memory. +** Call this routine to delete a hash table or to reset a hash table +** to the empty state. +*/ +SQLITE_PRIVATE void sqlite3Fts3HashClear(Fts3Hash *pH){ + Fts3HashElem *elem; /* For looping over all elements of the table */ + + assert( pH!=0 ); + elem = pH->first; + pH->first = 0; + fts3HashFree(pH->ht); + pH->ht = 0; + pH->htsize = 0; + while( elem ){ + Fts3HashElem *next_elem = elem->next; + if( pH->copyKey && elem->pKey ){ + fts3HashFree(elem->pKey); } + fts3HashFree(elem); + elem = next_elem; } + pH->count = 0; +} - return rc; +/* +** Hash and comparison functions when the mode is FTS3_HASH_STRING +*/ +static int fts3StrHash(const void *pKey, int nKey){ + const char *z = (const char *)pKey; + unsigned h = 0; + if( nKey<=0 ) nKey = (int) strlen(z); + while( nKey > 0 ){ + h = (h<<3) ^ h ^ *z++; + nKey--; + } + return (int)(h & 0x7fffffff); +} +static int fts3StrCompare(const void *pKey1, int n1, const void *pKey2, int n2){ + if( n1!=n2 ) return 1; + return strncmp((const char*)pKey1,(const char*)pKey2,n1); } /* -** The expression pExpr passed as the second argument to this function -** must be of type FTSQUERY_PHRASE. -** -** The returned value is either NULL or a pointer to a buffer containing -** a position-list indicating the occurrences of the phrase in column iCol -** of the current row. +** Hash and comparison functions when the mode is FTS3_HASH_BINARY +*/ +static int fts3BinHash(const void *pKey, int nKey){ + int h = 0; + const char *z = (const char *)pKey; + while( nKey-- > 0 ){ + h = (h<<3) ^ h ^ *(z++); + } + return h & 0x7fffffff; +} +static int fts3BinCompare(const void *pKey1, int n1, const void *pKey2, int n2){ + if( n1!=n2 ) return 1; + return memcmp(pKey1,pKey2,n1); +} + +/* +** Return a pointer to the appropriate hash function given the key class. ** -** More specifically, the returned buffer contains 1 varint for each -** occurrence of the phrase in the column, stored using the normal (delta+2) -** compression and is terminated by either an 0x01 or 0x00 byte. For example, -** if the requested column contains "a b X c d X X" and the position-list -** for 'X' is requested, the buffer returned may contain: +** The C syntax in this function definition may be unfamilar to some +** programmers, so we provide the following additional explanation: ** -** 0x04 0x05 0x03 0x01 or 0x04 0x05 0x03 0x00 +** The name of the function is "ftsHashFunction". The function takes a +** single parameter "keyClass". The return value of ftsHashFunction() +** is a pointer to another function. Specifically, the return value +** of ftsHashFunction() is a pointer to a function that takes two parameters +** with types "const void*" and "int" and returns an "int". +*/ +static int (*ftsHashFunction(int keyClass))(const void*,int){ + if( keyClass==FTS3_HASH_STRING ){ + return &fts3StrHash; + }else{ + assert( keyClass==FTS3_HASH_BINARY ); + return &fts3BinHash; + } +} + +/* +** Return a pointer to the appropriate hash function given the key class. ** -** This function works regardless of whether or not the phrase is deferred, -** incremental, or neither. +** For help in interpreted the obscure C code in the function definition, +** see the header comment on the previous function. */ -SQLITE_PRIVATE int sqlite3Fts3EvalPhrasePoslist( - Fts3Cursor *pCsr, /* FTS3 cursor object */ - Fts3Expr *pExpr, /* Phrase to return doclist for */ - int iCol, /* Column to return position list for */ - char **ppOut /* OUT: Pointer to position list */ -){ - Fts3Phrase *pPhrase = pExpr->pPhrase; - Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; - char *pIter; - int iThis; - sqlite3_int64 iDocid; +static int (*ftsCompareFunction(int keyClass))(const void*,int,const void*,int){ + if( keyClass==FTS3_HASH_STRING ){ + return &fts3StrCompare; + }else{ + assert( keyClass==FTS3_HASH_BINARY ); + return &fts3BinCompare; + } +} - /* If this phrase is applies specifically to some column other than - ** column iCol, return a NULL pointer. */ - *ppOut = 0; - assert( iCol>=0 && iColnColumn ); - if( (pPhrase->iColumnnColumn && pPhrase->iColumn!=iCol) ){ - return SQLITE_OK; +/* Link an element into the hash table +*/ +static void fts3HashInsertElement( + Fts3Hash *pH, /* The complete hash table */ + struct _fts3ht *pEntry, /* The entry into which pNew is inserted */ + Fts3HashElem *pNew /* The element to be inserted */ +){ + Fts3HashElem *pHead; /* First element already in pEntry */ + pHead = pEntry->chain; + if( pHead ){ + pNew->next = pHead; + pNew->prev = pHead->prev; + if( pHead->prev ){ pHead->prev->next = pNew; } + else { pH->first = pNew; } + pHead->prev = pNew; + }else{ + pNew->next = pH->first; + if( pH->first ){ pH->first->prev = pNew; } + pNew->prev = 0; + pH->first = pNew; } + pEntry->count++; + pEntry->chain = pNew; +} - iDocid = pExpr->iDocid; - pIter = pPhrase->doclist.pList; - if( iDocid!=pCsr->iPrevId || pExpr->bEof ){ - int rc = SQLITE_OK; - int bDescDoclist = pTab->bDescIdx; /* For DOCID_CMP macro */ - int bOr = 0; - u8 bTreeEof = 0; - Fts3Expr *p; /* Used to iterate from pExpr to root */ - Fts3Expr *pNear; /* Most senior NEAR ancestor (or pExpr) */ - int bMatch; - /* Check if this phrase descends from an OR expression node. If not, - ** return NULL. Otherwise, the entry that corresponds to docid - ** pCsr->iPrevId may lie earlier in the doclist buffer. Or, if the - ** tree that the node is part of has been marked as EOF, but the node - ** itself is not EOF, then it may point to an earlier entry. */ - pNear = pExpr; - for(p=pExpr->pParent; p; p=p->pParent){ - if( p->eType==FTSQUERY_OR ) bOr = 1; - if( p->eType==FTSQUERY_NEAR ) pNear = p; - if( p->bEof ) bTreeEof = 1; - } - if( bOr==0 ) return SQLITE_OK; +/* Resize the hash table so that it cantains "new_size" buckets. +** "new_size" must be a power of 2. The hash table might fail +** to resize if sqliteMalloc() fails. +** +** Return non-zero if a memory allocation error occurs. +*/ +static int fts3Rehash(Fts3Hash *pH, int new_size){ + struct _fts3ht *new_ht; /* The new hash table */ + Fts3HashElem *elem, *next_elem; /* For looping over existing elements */ + int (*xHash)(const void*,int); /* The hash function */ - /* This is the descendent of an OR node. In this case we cannot use - ** an incremental phrase. Load the entire doclist for the phrase - ** into memory in this case. */ - if( pPhrase->bIncr ){ - int bEofSave = pNear->bEof; - fts3EvalRestart(pCsr, pNear, &rc); - while( rc==SQLITE_OK && !pNear->bEof ){ - fts3EvalNextRow(pCsr, pNear, &rc); - if( bEofSave==0 && pNear->iDocid==iDocid ) break; - } - assert( rc!=SQLITE_OK || pPhrase->bIncr==0 ); - } - if( bTreeEof ){ - while( rc==SQLITE_OK && !pNear->bEof ){ - fts3EvalNextRow(pCsr, pNear, &rc); - } - } - if( rc!=SQLITE_OK ) return rc; + assert( (new_size & (new_size-1))==0 ); + new_ht = (struct _fts3ht *)fts3HashMalloc( new_size*sizeof(struct _fts3ht) ); + if( new_ht==0 ) return 1; + fts3HashFree(pH->ht); + pH->ht = new_ht; + pH->htsize = new_size; + xHash = ftsHashFunction(pH->keyClass); + for(elem=pH->first, pH->first=0; elem; elem = next_elem){ + int h = (*xHash)(elem->pKey, elem->nKey) & (new_size-1); + next_elem = elem->next; + fts3HashInsertElement(pH, &new_ht[h], elem); + } + return 0; +} - bMatch = 1; - for(p=pNear; p; p=p->pLeft){ - u8 bEof = 0; - Fts3Expr *pTest = p; - Fts3Phrase *pPh; - assert( pTest->eType==FTSQUERY_NEAR || pTest->eType==FTSQUERY_PHRASE ); - if( pTest->eType==FTSQUERY_NEAR ) pTest = pTest->pRight; - assert( pTest->eType==FTSQUERY_PHRASE ); - pPh = pTest->pPhrase; +/* This function (for internal use only) locates an element in an +** hash table that matches the given key. The hash for this key has +** already been computed and is passed as the 4th parameter. +*/ +static Fts3HashElem *fts3FindElementByHash( + const Fts3Hash *pH, /* The pH to be searched */ + const void *pKey, /* The key we are searching for */ + int nKey, + int h /* The hash for this key. */ +){ + Fts3HashElem *elem; /* Used to loop thru the element list */ + int count; /* Number of elements left to test */ + int (*xCompare)(const void*,int,const void*,int); /* comparison function */ - pIter = pPh->pOrPoslist; - iDocid = pPh->iOrDocid; - if( pCsr->bDesc==bDescDoclist ){ - bEof = !pPh->doclist.nAll || - (pIter >= (pPh->doclist.aAll + pPh->doclist.nAll)); - while( (pIter==0 || DOCID_CMP(iDocid, pCsr->iPrevId)<0 ) && bEof==0 ){ - sqlite3Fts3DoclistNext( - bDescDoclist, pPh->doclist.aAll, pPh->doclist.nAll, - &pIter, &iDocid, &bEof - ); - } - }else{ - bEof = !pPh->doclist.nAll || (pIter && pIter<=pPh->doclist.aAll); - while( (pIter==0 || DOCID_CMP(iDocid, pCsr->iPrevId)>0 ) && bEof==0 ){ - int dummy; - sqlite3Fts3DoclistPrev( - bDescDoclist, pPh->doclist.aAll, pPh->doclist.nAll, - &pIter, &iDocid, &dummy, &bEof - ); - } + if( pH->ht ){ + struct _fts3ht *pEntry = &pH->ht[h]; + elem = pEntry->chain; + count = pEntry->count; + xCompare = ftsCompareFunction(pH->keyClass); + while( count-- && elem ){ + if( (*xCompare)(elem->pKey,elem->nKey,pKey,nKey)==0 ){ + return elem; } - pPh->pOrPoslist = pIter; - pPh->iOrDocid = iDocid; - if( bEof || iDocid!=pCsr->iPrevId ) bMatch = 0; - } - - if( bMatch ){ - pIter = pPhrase->pOrPoslist; - }else{ - pIter = 0; + elem = elem->next; } } - if( pIter==0 ) return SQLITE_OK; + return 0; +} - if( *pIter==0x01 ){ - pIter++; - pIter += fts3GetVarint32(pIter, &iThis); +/* Remove a single entry from the hash table given a pointer to that +** element and a hash on the element's key. +*/ +static void fts3RemoveElementByHash( + Fts3Hash *pH, /* The pH containing "elem" */ + Fts3HashElem* elem, /* The element to be removed from the pH */ + int h /* Hash value for the element */ +){ + struct _fts3ht *pEntry; + if( elem->prev ){ + elem->prev->next = elem->next; }else{ - iThis = 0; + pH->first = elem->next; + } + if( elem->next ){ + elem->next->prev = elem->prev; + } + pEntry = &pH->ht[h]; + if( pEntry->chain==elem ){ + pEntry->chain = elem->next; + } + pEntry->count--; + if( pEntry->count<=0 ){ + pEntry->chain = 0; } - while( iThiscopyKey && elem->pKey ){ + fts3HashFree(elem->pKey); } - if( *pIter==0x00 ){ - pIter = 0; + fts3HashFree( elem ); + pH->count--; + if( pH->count<=0 ){ + assert( pH->first==0 ); + assert( pH->count==0 ); + fts3HashClear(pH); } - - *ppOut = ((iCol==iThis)?pIter:0); - return SQLITE_OK; } -/* -** Free all components of the Fts3Phrase structure that were allocated by -** the eval module. Specifically, this means to free: -** -** * the contents of pPhrase->doclist, and -** * any Fts3MultiSegReader objects held by phrase tokens. -*/ -SQLITE_PRIVATE void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *pPhrase){ - if( pPhrase ){ - int i; - sqlite3_free(pPhrase->doclist.aAll); - fts3EvalInvalidatePoslist(pPhrase); - memset(&pPhrase->doclist, 0, sizeof(Fts3Doclist)); - for(i=0; inToken; i++){ - fts3SegReaderCursorFree(pPhrase->aToken[i].pSegcsr); - pPhrase->aToken[i].pSegcsr = 0; - } - } -} +SQLITE_PRIVATE Fts3HashElem *sqlite3Fts3HashFindElem( + const Fts3Hash *pH, + const void *pKey, + int nKey +){ + int h; /* A hash on key */ + int (*xHash)(const void*,int); /* The hash function */ + if( pH==0 || pH->ht==0 ) return 0; + xHash = ftsHashFunction(pH->keyClass); + assert( xHash!=0 ); + h = (*xHash)(pKey,nKey); + assert( (pH->htsize & (pH->htsize-1))==0 ); + return fts3FindElementByHash(pH,pKey,nKey, h & (pH->htsize-1)); +} -/* -** Return SQLITE_CORRUPT_VTAB. +/* +** Attempt to locate an element of the hash table pH with a key +** that matches pKey,nKey. Return the data for this element if it is +** found, or NULL if there is no match. */ -#ifdef SQLITE_DEBUG -SQLITE_PRIVATE int sqlite3Fts3Corrupt(){ - return SQLITE_CORRUPT_VTAB; +SQLITE_PRIVATE void *sqlite3Fts3HashFind(const Fts3Hash *pH, const void *pKey, int nKey){ + Fts3HashElem *pElem; /* The element that matches key (if any) */ + + pElem = sqlite3Fts3HashFindElem(pH, pKey, nKey); + return pElem ? pElem->data : 0; } -#endif -#if !SQLITE_CORE -/* -** Initialize API pointer table, if required. +/* Insert an element into the hash table pH. The key is pKey,nKey +** and the data is "data". +** +** If no element exists with a matching key, then a new +** element is created. A copy of the key is made if the copyKey +** flag is set. NULL is returned. +** +** If another element already exists with the same key, then the +** new data replaces the old data and the old data is returned. +** The key is not copied in this instance. If a malloc fails, then +** the new data is returned and the hash table is unchanged. +** +** If the "data" parameter to this function is NULL, then the +** element corresponding to "key" is removed from the hash table. */ -#ifdef _WIN32 -__declspec(dllexport) -#endif -SQLITE_API int sqlite3_fts3_init( - sqlite3 *db, - char **pzErrMsg, - const sqlite3_api_routines *pApi +SQLITE_PRIVATE void *sqlite3Fts3HashInsert( + Fts3Hash *pH, /* The hash table to insert into */ + const void *pKey, /* The key */ + int nKey, /* Number of bytes in the key */ + void *data /* The data */ ){ - SQLITE_EXTENSION_INIT2(pApi) - return sqlite3Fts3Init(db); + int hraw; /* Raw hash value of the key */ + int h; /* the hash of the key modulo hash table size */ + Fts3HashElem *elem; /* Used to loop thru the element list */ + Fts3HashElem *new_elem; /* New element added to the pH */ + int (*xHash)(const void*,int); /* The hash function */ + + assert( pH!=0 ); + xHash = ftsHashFunction(pH->keyClass); + assert( xHash!=0 ); + hraw = (*xHash)(pKey, nKey); + assert( (pH->htsize & (pH->htsize-1))==0 ); + h = hraw & (pH->htsize-1); + elem = fts3FindElementByHash(pH,pKey,nKey,h); + if( elem ){ + void *old_data = elem->data; + if( data==0 ){ + fts3RemoveElementByHash(pH,elem,h); + }else{ + elem->data = data; + } + return old_data; + } + if( data==0 ) return 0; + if( (pH->htsize==0 && fts3Rehash(pH,8)) + || (pH->count>=pH->htsize && fts3Rehash(pH, pH->htsize*2)) + ){ + pH->count = 0; + return data; + } + assert( pH->htsize>0 ); + new_elem = (Fts3HashElem*)fts3HashMalloc( sizeof(Fts3HashElem) ); + if( new_elem==0 ) return data; + if( pH->copyKey && pKey!=0 ){ + new_elem->pKey = fts3HashMalloc( nKey ); + if( new_elem->pKey==0 ){ + fts3HashFree(new_elem); + return data; + } + memcpy((void*)new_elem->pKey, pKey, nKey); + }else{ + new_elem->pKey = (void*)pKey; + } + new_elem->nKey = nKey; + pH->count++; + assert( pH->htsize>0 ); + assert( (pH->htsize & (pH->htsize-1))==0 ); + h = hraw & (pH->htsize-1); + fts3HashInsertElement(pH, &pH->ht[h], new_elem); + new_elem->data = data; + return 0; } -#endif -#endif +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ -/************** End of fts3.c ************************************************/ -/************** Begin file fts3_aux.c ****************************************/ +/************** End of fts3_hash.c *******************************************/ +/************** Begin file fts3_porter.c *************************************/ /* -** 2011 Jan 27 +** 2006 September 30 ** ** The author disclaims copyright to this source code. In place of ** a legal notice, here is a blessing: @@ -158257,551 +174363,663 @@ SQLITE_API int sqlite3_fts3_init( ** May you find forgiveness for yourself and forgive others. ** May you share freely, never taking more than you give. ** -****************************************************************************** +************************************************************************* +** Implementation of the full-text-search tokenizer that implements +** a Porter stemmer. +*/ + +/* +** The code in this file is only compiled if: ** +** * The FTS3 module is being built as an extension +** (in which case SQLITE_CORE is not defined), or +** +** * The FTS3 module is being built into the core of +** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). */ /* #include "fts3Int.h" */ #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) -/* #include */ /* #include */ +/* #include */ +/* #include */ +/* #include */ -typedef struct Fts3auxTable Fts3auxTable; -typedef struct Fts3auxCursor Fts3auxCursor; - -struct Fts3auxTable { - sqlite3_vtab base; /* Base class used by SQLite core */ - Fts3Table *pFts3Tab; -}; - -struct Fts3auxCursor { - sqlite3_vtab_cursor base; /* Base class used by SQLite core */ - Fts3MultiSegReader csr; /* Must be right after "base" */ - Fts3SegFilter filter; - char *zStop; - int nStop; /* Byte-length of string zStop */ - int iLangid; /* Language id to query */ - int isEof; /* True if cursor is at EOF */ - sqlite3_int64 iRowid; /* Current rowid */ - - int iCol; /* Current value of 'col' column */ - int nStat; /* Size of aStat[] array */ - struct Fts3auxColstats { - sqlite3_int64 nDoc; /* 'documents' values for current csr row */ - sqlite3_int64 nOcc; /* 'occurrences' values for current csr row */ - } *aStat; -}; +/* #include "fts3_tokenizer.h" */ /* -** Schema of the terms table. +** Class derived from sqlite3_tokenizer */ -#define FTS3_AUX_SCHEMA \ - "CREATE TABLE x(term, col, documents, occurrences, languageid HIDDEN)" +typedef struct porter_tokenizer { + sqlite3_tokenizer base; /* Base class */ +} porter_tokenizer; /* -** This function does all the work for both the xConnect and xCreate methods. -** These tables have no persistent representation of their own, so xConnect -** and xCreate are identical operations. +** Class derived from sqlite3_tokenizer_cursor */ -static int fts3auxConnectMethod( - sqlite3 *db, /* Database connection */ - void *pUnused, /* Unused */ - int argc, /* Number of elements in argv array */ - const char * const *argv, /* xCreate/xConnect argument array */ - sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */ - char **pzErr /* OUT: sqlite3_malloc'd error message */ -){ - char const *zDb; /* Name of database (e.g. "main") */ - char const *zFts3; /* Name of fts3 table */ - int nDb; /* Result of strlen(zDb) */ - int nFts3; /* Result of strlen(zFts3) */ - int nByte; /* Bytes of space to allocate here */ - int rc; /* value returned by declare_vtab() */ - Fts3auxTable *p; /* Virtual table object to return */ - - UNUSED_PARAMETER(pUnused); - - /* The user should invoke this in one of two forms: - ** - ** CREATE VIRTUAL TABLE xxx USING fts4aux(fts4-table); - ** CREATE VIRTUAL TABLE xxx USING fts4aux(fts4-table-db, fts4-table); - */ - if( argc!=4 && argc!=5 ) goto bad_args; - - zDb = argv[1]; - nDb = (int)strlen(zDb); - if( argc==5 ){ - if( nDb==4 && 0==sqlite3_strnicmp("temp", zDb, 4) ){ - zDb = argv[3]; - nDb = (int)strlen(zDb); - zFts3 = argv[4]; - }else{ - goto bad_args; - } - }else{ - zFts3 = argv[3]; - } - nFts3 = (int)strlen(zFts3); - - rc = sqlite3_declare_vtab(db, FTS3_AUX_SCHEMA); - if( rc!=SQLITE_OK ) return rc; +typedef struct porter_tokenizer_cursor { + sqlite3_tokenizer_cursor base; + const char *zInput; /* input we are tokenizing */ + int nInput; /* size of the input */ + int iOffset; /* current position in zInput */ + int iToken; /* index of next token to be returned */ + char *zToken; /* storage for current token */ + int nAllocated; /* space allocated to zToken buffer */ +} porter_tokenizer_cursor; - nByte = sizeof(Fts3auxTable) + sizeof(Fts3Table) + nDb + nFts3 + 2; - p = (Fts3auxTable *)sqlite3_malloc(nByte); - if( !p ) return SQLITE_NOMEM; - memset(p, 0, nByte); - p->pFts3Tab = (Fts3Table *)&p[1]; - p->pFts3Tab->zDb = (char *)&p->pFts3Tab[1]; - p->pFts3Tab->zName = &p->pFts3Tab->zDb[nDb+1]; - p->pFts3Tab->db = db; - p->pFts3Tab->nIndex = 1; +/* +** Create a new tokenizer instance. +*/ +static int porterCreate( + int argc, const char * const *argv, + sqlite3_tokenizer **ppTokenizer +){ + porter_tokenizer *t; - memcpy((char *)p->pFts3Tab->zDb, zDb, nDb); - memcpy((char *)p->pFts3Tab->zName, zFts3, nFts3); - sqlite3Fts3Dequote((char *)p->pFts3Tab->zName); + UNUSED_PARAMETER(argc); + UNUSED_PARAMETER(argv); - *ppVtab = (sqlite3_vtab *)p; + t = (porter_tokenizer *) sqlite3_malloc(sizeof(*t)); + if( t==NULL ) return SQLITE_NOMEM; + memset(t, 0, sizeof(*t)); + *ppTokenizer = &t->base; return SQLITE_OK; - - bad_args: - sqlite3Fts3ErrMsg(pzErr, "invalid arguments to fts4aux constructor"); - return SQLITE_ERROR; } /* -** This function does the work for both the xDisconnect and xDestroy methods. -** These tables have no persistent representation of their own, so xDisconnect -** and xDestroy are identical operations. +** Destroy a tokenizer */ -static int fts3auxDisconnectMethod(sqlite3_vtab *pVtab){ - Fts3auxTable *p = (Fts3auxTable *)pVtab; - Fts3Table *pFts3 = p->pFts3Tab; - int i; - - /* Free any prepared statements held */ - for(i=0; iaStmt); i++){ - sqlite3_finalize(pFts3->aStmt[i]); - } - sqlite3_free(pFts3->zSegmentsTbl); - sqlite3_free(p); +static int porterDestroy(sqlite3_tokenizer *pTokenizer){ + sqlite3_free(pTokenizer); return SQLITE_OK; } -#define FTS4AUX_EQ_CONSTRAINT 1 -#define FTS4AUX_GE_CONSTRAINT 2 -#define FTS4AUX_LE_CONSTRAINT 4 - /* -** xBestIndex - Analyze a WHERE and ORDER BY clause. +** Prepare to begin tokenizing a particular string. The input +** string to be tokenized is zInput[0..nInput-1]. A cursor +** used to incrementally tokenize this string is returned in +** *ppCursor. */ -static int fts3auxBestIndexMethod( - sqlite3_vtab *pVTab, - sqlite3_index_info *pInfo +static int porterOpen( + sqlite3_tokenizer *pTokenizer, /* The tokenizer */ + const char *zInput, int nInput, /* String to be tokenized */ + sqlite3_tokenizer_cursor **ppCursor /* OUT: Tokenization cursor */ ){ - int i; - int iEq = -1; - int iGe = -1; - int iLe = -1; - int iLangid = -1; - int iNext = 1; /* Next free argvIndex value */ - - UNUSED_PARAMETER(pVTab); - - /* This vtab delivers always results in "ORDER BY term ASC" order. */ - if( pInfo->nOrderBy==1 - && pInfo->aOrderBy[0].iColumn==0 - && pInfo->aOrderBy[0].desc==0 - ){ - pInfo->orderByConsumed = 1; - } + porter_tokenizer_cursor *c; - /* Search for equality and range constraints on the "term" column. - ** And equality constraints on the hidden "languageid" column. */ - for(i=0; inConstraint; i++){ - if( pInfo->aConstraint[i].usable ){ - int op = pInfo->aConstraint[i].op; - int iCol = pInfo->aConstraint[i].iColumn; + UNUSED_PARAMETER(pTokenizer); - if( iCol==0 ){ - if( op==SQLITE_INDEX_CONSTRAINT_EQ ) iEq = i; - if( op==SQLITE_INDEX_CONSTRAINT_LT ) iLe = i; - if( op==SQLITE_INDEX_CONSTRAINT_LE ) iLe = i; - if( op==SQLITE_INDEX_CONSTRAINT_GT ) iGe = i; - if( op==SQLITE_INDEX_CONSTRAINT_GE ) iGe = i; - } - if( iCol==4 ){ - if( op==SQLITE_INDEX_CONSTRAINT_EQ ) iLangid = i; - } - } - } + c = (porter_tokenizer_cursor *) sqlite3_malloc(sizeof(*c)); + if( c==NULL ) return SQLITE_NOMEM; - if( iEq>=0 ){ - pInfo->idxNum = FTS4AUX_EQ_CONSTRAINT; - pInfo->aConstraintUsage[iEq].argvIndex = iNext++; - pInfo->estimatedCost = 5; + c->zInput = zInput; + if( zInput==0 ){ + c->nInput = 0; + }else if( nInput<0 ){ + c->nInput = (int)strlen(zInput); }else{ - pInfo->idxNum = 0; - pInfo->estimatedCost = 20000; - if( iGe>=0 ){ - pInfo->idxNum += FTS4AUX_GE_CONSTRAINT; - pInfo->aConstraintUsage[iGe].argvIndex = iNext++; - pInfo->estimatedCost /= 2; - } - if( iLe>=0 ){ - pInfo->idxNum += FTS4AUX_LE_CONSTRAINT; - pInfo->aConstraintUsage[iLe].argvIndex = iNext++; - pInfo->estimatedCost /= 2; - } - } - if( iLangid>=0 ){ - pInfo->aConstraintUsage[iLangid].argvIndex = iNext++; - pInfo->estimatedCost--; + c->nInput = nInput; } + c->iOffset = 0; /* start tokenizing at the beginning */ + c->iToken = 0; + c->zToken = NULL; /* no space allocated, yet. */ + c->nAllocated = 0; + *ppCursor = &c->base; return SQLITE_OK; } /* -** xOpen - Open a cursor. +** Close a tokenization cursor previously opened by a call to +** porterOpen() above. */ -static int fts3auxOpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){ - Fts3auxCursor *pCsr; /* Pointer to cursor object to return */ - - UNUSED_PARAMETER(pVTab); - - pCsr = (Fts3auxCursor *)sqlite3_malloc(sizeof(Fts3auxCursor)); - if( !pCsr ) return SQLITE_NOMEM; - memset(pCsr, 0, sizeof(Fts3auxCursor)); - - *ppCsr = (sqlite3_vtab_cursor *)pCsr; +static int porterClose(sqlite3_tokenizer_cursor *pCursor){ + porter_tokenizer_cursor *c = (porter_tokenizer_cursor *) pCursor; + sqlite3_free(c->zToken); + sqlite3_free(c); return SQLITE_OK; } - /* -** xClose - Close a cursor. +** Vowel or consonant */ -static int fts3auxCloseMethod(sqlite3_vtab_cursor *pCursor){ - Fts3Table *pFts3 = ((Fts3auxTable *)pCursor->pVtab)->pFts3Tab; - Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor; +static const char cType[] = { + 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, + 1, 1, 1, 2, 1 +}; - sqlite3Fts3SegmentsClose(pFts3); - sqlite3Fts3SegReaderFinish(&pCsr->csr); - sqlite3_free((void *)pCsr->filter.zTerm); - sqlite3_free(pCsr->zStop); - sqlite3_free(pCsr->aStat); - sqlite3_free(pCsr); - return SQLITE_OK; +/* +** isConsonant() and isVowel() determine if their first character in +** the string they point to is a consonant or a vowel, according +** to Porter ruls. +** +** A consonate is any letter other than 'a', 'e', 'i', 'o', or 'u'. +** 'Y' is a consonant unless it follows another consonant, +** in which case it is a vowel. +** +** In these routine, the letters are in reverse order. So the 'y' rule +** is that 'y' is a consonant unless it is followed by another +** consonent. +*/ +static int isVowel(const char*); +static int isConsonant(const char *z){ + int j; + char x = *z; + if( x==0 ) return 0; + assert( x>='a' && x<='z' ); + j = cType[x-'a']; + if( j<2 ) return j; + return z[1]==0 || isVowel(z + 1); } - -static int fts3auxGrowStatArray(Fts3auxCursor *pCsr, int nSize){ - if( nSize>pCsr->nStat ){ - struct Fts3auxColstats *aNew; - aNew = (struct Fts3auxColstats *)sqlite3_realloc(pCsr->aStat, - sizeof(struct Fts3auxColstats) * nSize - ); - if( aNew==0 ) return SQLITE_NOMEM; - memset(&aNew[pCsr->nStat], 0, - sizeof(struct Fts3auxColstats) * (nSize - pCsr->nStat) - ); - pCsr->aStat = aNew; - pCsr->nStat = nSize; - } - return SQLITE_OK; +static int isVowel(const char *z){ + int j; + char x = *z; + if( x==0 ) return 0; + assert( x>='a' && x<='z' ); + j = cType[x-'a']; + if( j<2 ) return 1-j; + return isConsonant(z + 1); } /* -** xNext - Advance the cursor to the next row, if any. +** Let any sequence of one or more vowels be represented by V and let +** C be sequence of one or more consonants. Then every word can be +** represented as: +** +** [C] (VC){m} [V] +** +** In prose: A word is an optional consonant followed by zero or +** vowel-consonant pairs followed by an optional vowel. "m" is the +** number of vowel consonant pairs. This routine computes the value +** of m for the first i bytes of a word. +** +** Return true if the m-value for z is 1 or more. In other words, +** return true if z contains at least one vowel that is followed +** by a consonant. +** +** In this routine z[] is in reverse order. So we are really looking +** for an instance of a consonant followed by a vowel. */ -static int fts3auxNextMethod(sqlite3_vtab_cursor *pCursor){ - Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor; - Fts3Table *pFts3 = ((Fts3auxTable *)pCursor->pVtab)->pFts3Tab; - int rc; - - /* Increment our pretend rowid value. */ - pCsr->iRowid++; - - for(pCsr->iCol++; pCsr->iColnStat; pCsr->iCol++){ - if( pCsr->aStat[pCsr->iCol].nDoc>0 ) return SQLITE_OK; - } - - rc = sqlite3Fts3SegReaderStep(pFts3, &pCsr->csr); - if( rc==SQLITE_ROW ){ - int i = 0; - int nDoclist = pCsr->csr.nDoclist; - char *aDoclist = pCsr->csr.aDoclist; - int iCol; - - int eState = 0; - - if( pCsr->zStop ){ - int n = (pCsr->nStopcsr.nTerm) ? pCsr->nStop : pCsr->csr.nTerm; - int mc = memcmp(pCsr->zStop, pCsr->csr.zTerm, n); - if( mc<0 || (mc==0 && pCsr->csr.nTerm>pCsr->nStop) ){ - pCsr->isEof = 1; - return SQLITE_OK; - } - } - - if( fts3auxGrowStatArray(pCsr, 2) ) return SQLITE_NOMEM; - memset(pCsr->aStat, 0, sizeof(struct Fts3auxColstats) * pCsr->nStat); - iCol = 0; +static int m_gt_0(const char *z){ + while( isVowel(z) ){ z++; } + if( *z==0 ) return 0; + while( isConsonant(z) ){ z++; } + return *z!=0; +} - while( iaStat[0].nDoc++; - eState = 1; - iCol = 0; - break; +/* Like mgt0 above except we are looking for a value of m>1 instead +** or m>0 +*/ +static int m_gt_1(const char *z){ + while( isVowel(z) ){ z++; } + if( *z==0 ) return 0; + while( isConsonant(z) ){ z++; } + if( *z==0 ) return 0; + while( isVowel(z) ){ z++; } + if( *z==0 ) return 0; + while( isConsonant(z) ){ z++; } + return *z!=0; +} - /* State 1. In this state we are expecting either a 1, indicating - ** that the following integer will be a column number, or the - ** start of a position list for column 0. - ** - ** The only difference between state 1 and state 2 is that if the - ** integer encountered in state 1 is not 0 or 1, then we need to - ** increment the column 0 "nDoc" count for this term. - */ - case 1: - assert( iCol==0 ); - if( v>1 ){ - pCsr->aStat[1].nDoc++; - } - eState = 2; - /* fall through */ +/* +** Return TRUE if there is a vowel anywhere within z[0..n-1] +*/ +static int hasVowel(const char *z){ + while( isConsonant(z) ){ z++; } + return *z!=0; +} - case 2: - if( v==0 ){ /* 0x00. Next integer will be a docid. */ - eState = 0; - }else if( v==1 ){ /* 0x01. Next integer will be a column number. */ - eState = 3; - }else{ /* 2 or greater. A position. */ - pCsr->aStat[iCol+1].nOcc++; - pCsr->aStat[0].nOcc++; - } - break; +/* +** Return TRUE if the word ends in a double consonant. +** +** The text is reversed here. So we are really looking at +** the first two characters of z[]. +*/ +static int doubleConsonant(const char *z){ + return isConsonant(z) && z[0]==z[1]; +} - /* State 3. The integer just read is a column number. */ - default: assert( eState==3 ); - iCol = (int)v; - if( fts3auxGrowStatArray(pCsr, iCol+2) ) return SQLITE_NOMEM; - pCsr->aStat[iCol+1].nDoc++; - eState = 2; - break; - } - } +/* +** Return TRUE if the word ends with three letters which +** are consonant-vowel-consonent and where the final consonant +** is not 'w', 'x', or 'y'. +** +** The word is reversed here. So we are really checking the +** first three letters and the first one cannot be in [wxy]. +*/ +static int star_oh(const char *z){ + return + isConsonant(z) && + z[0]!='w' && z[0]!='x' && z[0]!='y' && + isVowel(z+1) && + isConsonant(z+2); +} - pCsr->iCol = 0; - rc = SQLITE_OK; - }else{ - pCsr->isEof = 1; +/* +** If the word ends with zFrom and xCond() is true for the stem +** of the word that preceeds the zFrom ending, then change the +** ending to zTo. +** +** The input word *pz and zFrom are both in reverse order. zTo +** is in normal order. +** +** Return TRUE if zFrom matches. Return FALSE if zFrom does not +** match. Not that TRUE is returned even if xCond() fails and +** no substitution occurs. +*/ +static int stem( + char **pz, /* The word being stemmed (Reversed) */ + const char *zFrom, /* If the ending matches this... (Reversed) */ + const char *zTo, /* ... change the ending to this (not reversed) */ + int (*xCond)(const char*) /* Condition that must be true */ +){ + char *z = *pz; + while( *zFrom && *zFrom==*z ){ z++; zFrom++; } + if( *zFrom!=0 ) return 0; + if( xCond && !xCond(z) ) return 1; + while( *zTo ){ + *(--z) = *(zTo++); } - return rc; + *pz = z; + return 1; } /* -** xFilter - Initialize a cursor to point at the start of its data. +** This is the fallback stemmer used when the porter stemmer is +** inappropriate. The input word is copied into the output with +** US-ASCII case folding. If the input word is too long (more +** than 20 bytes if it contains no digits or more than 6 bytes if +** it contains digits) then word is truncated to 20 or 6 bytes +** by taking 10 or 3 bytes from the beginning and end. */ -static int fts3auxFilterMethod( - sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */ - int idxNum, /* Strategy index */ - const char *idxStr, /* Unused */ - int nVal, /* Number of elements in apVal */ - sqlite3_value **apVal /* Arguments for the indexing scheme */ -){ - Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor; - Fts3Table *pFts3 = ((Fts3auxTable *)pCursor->pVtab)->pFts3Tab; - int rc; - int isScan = 0; - int iLangVal = 0; /* Language id to query */ +static void copy_stemmer(const char *zIn, int nIn, char *zOut, int *pnOut){ + int i, mx, j; + int hasDigit = 0; + for(i=0; i='A' && c<='Z' ){ + zOut[i] = c - 'A' + 'a'; + }else{ + if( c>='0' && c<='9' ) hasDigit = 1; + zOut[i] = c; + } + } + mx = hasDigit ? 3 : 10; + if( nIn>mx*2 ){ + for(j=mx, i=nIn-mx; i=? value in apVal */ - int iLe = -1; /* Index of term<=? value in apVal */ - int iLangid = -1; /* Index of languageid=? value in apVal */ - int iNext = 0; - UNUSED_PARAMETER(nVal); - UNUSED_PARAMETER(idxStr); +/* +** Stem the input word zIn[0..nIn-1]. Store the output in zOut. +** zOut is at least big enough to hold nIn bytes. Write the actual +** size of the output word (exclusive of the '\0' terminator) into *pnOut. +** +** Any upper-case characters in the US-ASCII character set ([A-Z]) +** are converted to lower case. Upper-case UTF characters are +** unchanged. +** +** Words that are longer than about 20 bytes are stemmed by retaining +** a few bytes from the beginning and the end of the word. If the +** word contains digits, 3 bytes are taken from the beginning and +** 3 bytes from the end. For long words without digits, 10 bytes +** are taken from each end. US-ASCII case folding still applies. +** +** If the input word contains not digits but does characters not +** in [a-zA-Z] then no stemming is attempted and this routine just +** copies the input into the input into the output with US-ASCII +** case folding. +** +** Stemming never increases the length of the word. So there is +** no chance of overflowing the zOut buffer. +*/ +static void porter_stemmer(const char *zIn, int nIn, char *zOut, int *pnOut){ + int i, j; + char zReverse[28]; + char *z, *z2; + if( nIn<3 || nIn>=(int)sizeof(zReverse)-7 ){ + /* The word is too big or too small for the porter stemmer. + ** Fallback to the copy stemmer */ + copy_stemmer(zIn, nIn, zOut, pnOut); + return; + } + for(i=0, j=sizeof(zReverse)-6; i='A' && c<='Z' ){ + zReverse[j] = c + 'a' - 'A'; + }else if( c>='a' && c<='z' ){ + zReverse[j] = c; + }else{ + /* The use of a character not in [a-zA-Z] means that we fallback + ** to the copy stemmer */ + copy_stemmer(zIn, nIn, zOut, pnOut); + return; + } + } + memset(&zReverse[sizeof(zReverse)-5], 0, 5); + z = &zReverse[j+1]; - assert( idxStr==0 ); - assert( idxNum==FTS4AUX_EQ_CONSTRAINT || idxNum==0 - || idxNum==FTS4AUX_LE_CONSTRAINT || idxNum==FTS4AUX_GE_CONSTRAINT - || idxNum==(FTS4AUX_LE_CONSTRAINT|FTS4AUX_GE_CONSTRAINT) - ); - if( idxNum==FTS4AUX_EQ_CONSTRAINT ){ - iEq = iNext++; - }else{ - isScan = 1; - if( idxNum & FTS4AUX_GE_CONSTRAINT ){ - iGe = iNext++; - } - if( idxNum & FTS4AUX_LE_CONSTRAINT ){ - iLe = iNext++; + /* Step 1a */ + if( z[0]=='s' ){ + if( + !stem(&z, "sess", "ss", 0) && + !stem(&z, "sei", "i", 0) && + !stem(&z, "ss", "ss", 0) + ){ + z++; } } - if( iNextfilter.zTerm); - sqlite3Fts3SegReaderFinish(&pCsr->csr); - sqlite3_free((void *)pCsr->filter.zTerm); - sqlite3_free(pCsr->aStat); - memset(&pCsr->csr, 0, ((u8*)&pCsr[1]) - (u8*)&pCsr->csr); + /* Step 1b */ + z2 = z; + if( stem(&z, "dee", "ee", m_gt_0) ){ + /* Do nothing. The work was all in the test */ + }else if( + (stem(&z, "gni", "", hasVowel) || stem(&z, "de", "", hasVowel)) + && z!=z2 + ){ + if( stem(&z, "ta", "ate", 0) || + stem(&z, "lb", "ble", 0) || + stem(&z, "zi", "ize", 0) ){ + /* Do nothing. The work was all in the test */ + }else if( doubleConsonant(z) && (*z!='l' && *z!='s' && *z!='z') ){ + z++; + }else if( m_eq_1(z) && star_oh(z) ){ + *(--z) = 'e'; + } + } - pCsr->filter.flags = FTS3_SEGMENT_REQUIRE_POS|FTS3_SEGMENT_IGNORE_EMPTY; - if( isScan ) pCsr->filter.flags |= FTS3_SEGMENT_SCAN; + /* Step 1c */ + if( z[0]=='y' && hasVowel(z+1) ){ + z[0] = 'i'; + } - if( iEq>=0 || iGe>=0 ){ - const unsigned char *zStr = sqlite3_value_text(apVal[0]); - assert( (iEq==0 && iGe==-1) || (iEq==-1 && iGe==0) ); - if( zStr ){ - pCsr->filter.zTerm = sqlite3_mprintf("%s", zStr); - pCsr->filter.nTerm = sqlite3_value_bytes(apVal[0]); - if( pCsr->filter.zTerm==0 ) return SQLITE_NOMEM; - } + /* Step 2 */ + switch( z[1] ){ + case 'a': + if( !stem(&z, "lanoita", "ate", m_gt_0) ){ + stem(&z, "lanoit", "tion", m_gt_0); + } + break; + case 'c': + if( !stem(&z, "icne", "ence", m_gt_0) ){ + stem(&z, "icna", "ance", m_gt_0); + } + break; + case 'e': + stem(&z, "rezi", "ize", m_gt_0); + break; + case 'g': + stem(&z, "igol", "log", m_gt_0); + break; + case 'l': + if( !stem(&z, "ilb", "ble", m_gt_0) + && !stem(&z, "illa", "al", m_gt_0) + && !stem(&z, "iltne", "ent", m_gt_0) + && !stem(&z, "ile", "e", m_gt_0) + ){ + stem(&z, "ilsuo", "ous", m_gt_0); + } + break; + case 'o': + if( !stem(&z, "noitazi", "ize", m_gt_0) + && !stem(&z, "noita", "ate", m_gt_0) + ){ + stem(&z, "rota", "ate", m_gt_0); + } + break; + case 's': + if( !stem(&z, "msila", "al", m_gt_0) + && !stem(&z, "ssenevi", "ive", m_gt_0) + && !stem(&z, "ssenluf", "ful", m_gt_0) + ){ + stem(&z, "ssensuo", "ous", m_gt_0); + } + break; + case 't': + if( !stem(&z, "itila", "al", m_gt_0) + && !stem(&z, "itivi", "ive", m_gt_0) + ){ + stem(&z, "itilib", "ble", m_gt_0); + } + break; } - if( iLe>=0 ){ - pCsr->zStop = sqlite3_mprintf("%s", sqlite3_value_text(apVal[iLe])); - pCsr->nStop = sqlite3_value_bytes(apVal[iLe]); - if( pCsr->zStop==0 ) return SQLITE_NOMEM; + /* Step 3 */ + switch( z[0] ){ + case 'e': + if( !stem(&z, "etaci", "ic", m_gt_0) + && !stem(&z, "evita", "", m_gt_0) + ){ + stem(&z, "ezila", "al", m_gt_0); + } + break; + case 'i': + stem(&z, "itici", "ic", m_gt_0); + break; + case 'l': + if( !stem(&z, "laci", "ic", m_gt_0) ){ + stem(&z, "luf", "", m_gt_0); + } + break; + case 's': + stem(&z, "ssen", "", m_gt_0); + break; } - - if( iLangid>=0 ){ - iLangVal = sqlite3_value_int(apVal[iLangid]); - /* If the user specified a negative value for the languageid, use zero - ** instead. This works, as the "languageid=?" constraint will also - ** be tested by the VDBE layer. The test will always be false (since - ** this module will not return a row with a negative languageid), and - ** so the overall query will return zero rows. */ - if( iLangVal<0 ) iLangVal = 0; + /* Step 4 */ + switch( z[1] ){ + case 'a': + if( z[0]=='l' && m_gt_1(z+2) ){ + z += 2; + } + break; + case 'c': + if( z[0]=='e' && z[2]=='n' && (z[3]=='a' || z[3]=='e') && m_gt_1(z+4) ){ + z += 4; + } + break; + case 'e': + if( z[0]=='r' && m_gt_1(z+2) ){ + z += 2; + } + break; + case 'i': + if( z[0]=='c' && m_gt_1(z+2) ){ + z += 2; + } + break; + case 'l': + if( z[0]=='e' && z[2]=='b' && (z[3]=='a' || z[3]=='i') && m_gt_1(z+4) ){ + z += 4; + } + break; + case 'n': + if( z[0]=='t' ){ + if( z[2]=='a' ){ + if( m_gt_1(z+3) ){ + z += 3; + } + }else if( z[2]=='e' ){ + if( !stem(&z, "tneme", "", m_gt_1) + && !stem(&z, "tnem", "", m_gt_1) + ){ + stem(&z, "tne", "", m_gt_1); + } + } + } + break; + case 'o': + if( z[0]=='u' ){ + if( m_gt_1(z+2) ){ + z += 2; + } + }else if( z[3]=='s' || z[3]=='t' ){ + stem(&z, "noi", "", m_gt_1); + } + break; + case 's': + if( z[0]=='m' && z[2]=='i' && m_gt_1(z+3) ){ + z += 3; + } + break; + case 't': + if( !stem(&z, "eta", "", m_gt_1) ){ + stem(&z, "iti", "", m_gt_1); + } + break; + case 'u': + if( z[0]=='s' && z[2]=='o' && m_gt_1(z+3) ){ + z += 3; + } + break; + case 'v': + case 'z': + if( z[0]=='e' && z[2]=='i' && m_gt_1(z+3) ){ + z += 3; + } + break; } - pCsr->iLangid = iLangVal; - rc = sqlite3Fts3SegReaderCursor(pFts3, iLangVal, 0, FTS3_SEGCURSOR_ALL, - pCsr->filter.zTerm, pCsr->filter.nTerm, 0, isScan, &pCsr->csr - ); - if( rc==SQLITE_OK ){ - rc = sqlite3Fts3SegReaderStart(pFts3, &pCsr->csr, &pCsr->filter); + /* Step 5a */ + if( z[0]=='e' ){ + if( m_gt_1(z+1) ){ + z++; + }else if( m_eq_1(z+1) && !star_oh(z+1) ){ + z++; + } } - if( rc==SQLITE_OK ) rc = fts3auxNextMethod(pCursor); - return rc; -} + /* Step 5b */ + if( m_gt_1(z) && z[0]=='l' && z[1]=='l' ){ + z++; + } -/* -** xEof - Return true if the cursor is at EOF, or false otherwise. -*/ -static int fts3auxEofMethod(sqlite3_vtab_cursor *pCursor){ - Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor; - return pCsr->isEof; + /* z[] is now the stemmed word in reverse order. Flip it back + ** around into forward order and return. + */ + *pnOut = i = (int)strlen(z); + zOut[i] = 0; + while( *z ){ + zOut[--i] = *(z++); + } } /* -** xColumn - Return a column value. +** Characters that can be part of a token. We assume any character +** whose value is greater than 0x80 (any UTF character) can be +** part of a token. In other words, delimiters all must have +** values of 0x7f or lower. */ -static int fts3auxColumnMethod( - sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ - sqlite3_context *pCtx, /* Context for sqlite3_result_xxx() calls */ - int iCol /* Index of column to read value from */ -){ - Fts3auxCursor *p = (Fts3auxCursor *)pCursor; +static const char porterIdChar[] = { +/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 3x */ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4x */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 5x */ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 6x */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */ +}; +#define isDelim(C) (((ch=C)&0x80)==0 && (ch<0x30 || !porterIdChar[ch-0x30])) - assert( p->isEof==0 ); - switch( iCol ){ - case 0: /* term */ - sqlite3_result_text(pCtx, p->csr.zTerm, p->csr.nTerm, SQLITE_TRANSIENT); - break; +/* +** Extract the next token from a tokenization cursor. The cursor must +** have been opened by a prior call to porterOpen(). +*/ +static int porterNext( + sqlite3_tokenizer_cursor *pCursor, /* Cursor returned by porterOpen */ + const char **pzToken, /* OUT: *pzToken is the token text */ + int *pnBytes, /* OUT: Number of bytes in token */ + int *piStartOffset, /* OUT: Starting offset of token */ + int *piEndOffset, /* OUT: Ending offset of token */ + int *piPosition /* OUT: Position integer of token */ +){ + porter_tokenizer_cursor *c = (porter_tokenizer_cursor *) pCursor; + const char *z = c->zInput; - case 1: /* col */ - if( p->iCol ){ - sqlite3_result_int(pCtx, p->iCol-1); - }else{ - sqlite3_result_text(pCtx, "*", -1, SQLITE_STATIC); - } - break; + while( c->iOffsetnInput ){ + int iStartOffset, ch; - case 2: /* documents */ - sqlite3_result_int64(pCtx, p->aStat[p->iCol].nDoc); - break; + /* Scan past delimiter characters */ + while( c->iOffsetnInput && isDelim(z[c->iOffset]) ){ + c->iOffset++; + } - case 3: /* occurrences */ - sqlite3_result_int64(pCtx, p->aStat[p->iCol].nOcc); - break; + /* Count non-delimiter characters. */ + iStartOffset = c->iOffset; + while( c->iOffsetnInput && !isDelim(z[c->iOffset]) ){ + c->iOffset++; + } - default: /* languageid */ - assert( iCol==4 ); - sqlite3_result_int(pCtx, p->iLangid); - break; + if( c->iOffset>iStartOffset ){ + int n = c->iOffset-iStartOffset; + if( n>c->nAllocated ){ + char *pNew; + c->nAllocated = n+20; + pNew = sqlite3_realloc(c->zToken, c->nAllocated); + if( !pNew ) return SQLITE_NOMEM; + c->zToken = pNew; + } + porter_stemmer(&z[iStartOffset], n, c->zToken, pnBytes); + *pzToken = c->zToken; + *piStartOffset = iStartOffset; + *piEndOffset = c->iOffset; + *piPosition = c->iToken++; + return SQLITE_OK; + } } - - return SQLITE_OK; + return SQLITE_DONE; } /* -** xRowid - Return the current rowid for the cursor. +** The set of routines that implement the porter-stemmer tokenizer */ -static int fts3auxRowidMethod( - sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ - sqlite_int64 *pRowid /* OUT: Rowid value */ -){ - Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor; - *pRowid = pCsr->iRowid; - return SQLITE_OK; -} +static const sqlite3_tokenizer_module porterTokenizerModule = { + 0, + porterCreate, + porterDestroy, + porterOpen, + porterClose, + porterNext, + 0 +}; /* -** Register the fts3aux module with database connection db. Return SQLITE_OK -** if successful or an error code if sqlite3_create_module() fails. +** Allocate a new porter tokenizer. Return a pointer to the new +** tokenizer in *ppModule */ -SQLITE_PRIVATE int sqlite3Fts3InitAux(sqlite3 *db){ - static const sqlite3_module fts3aux_module = { - 0, /* iVersion */ - fts3auxConnectMethod, /* xCreate */ - fts3auxConnectMethod, /* xConnect */ - fts3auxBestIndexMethod, /* xBestIndex */ - fts3auxDisconnectMethod, /* xDisconnect */ - fts3auxDisconnectMethod, /* xDestroy */ - fts3auxOpenMethod, /* xOpen */ - fts3auxCloseMethod, /* xClose */ - fts3auxFilterMethod, /* xFilter */ - fts3auxNextMethod, /* xNext */ - fts3auxEofMethod, /* xEof */ - fts3auxColumnMethod, /* xColumn */ - fts3auxRowidMethod, /* xRowid */ - 0, /* xUpdate */ - 0, /* xBegin */ - 0, /* xSync */ - 0, /* xCommit */ - 0, /* xRollback */ - 0, /* xFindFunction */ - 0, /* xRename */ - 0, /* xSavepoint */ - 0, /* xRelease */ - 0 /* xRollbackTo */ - }; - int rc; /* Return code */ - - rc = sqlite3_create_module(db, "fts4aux", &fts3aux_module, 0); - return rc; +SQLITE_PRIVATE void sqlite3Fts3PorterTokenizerModule( + sqlite3_tokenizer_module const**ppModule +){ + *ppModule = &porterTokenizerModule; } #endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ -/************** End of fts3_aux.c ********************************************/ -/************** Begin file fts3_expr.c ***************************************/ +/************** End of fts3_porter.c *****************************************/ +/************** Begin file fts3_tokenizer.c **********************************/ /* -** 2008 Nov 28 +** 2007 June 22 ** ** The author disclaims copyright to this source code. In place of ** a legal notice, here is a blessing: @@ -158812,1291 +175030,1214 @@ SQLITE_PRIVATE int sqlite3Fts3InitAux(sqlite3 *db){ ** ****************************************************************************** ** -** This module contains code that implements a parser for fts3 query strings -** (the right-hand argument to the MATCH operator). Because the supported -** syntax is relatively simple, the whole tokenizer/parser system is -** hand-coded. +** This is part of an SQLite module implementing full-text search. +** This particular file implements the generic tokenizer interface. */ -/* #include "fts3Int.h" */ -#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) /* -** By default, this module parses the legacy syntax that has been -** traditionally used by fts3. Or, if SQLITE_ENABLE_FTS3_PARENTHESIS -** is defined, then it uses the new syntax. The differences between -** the new and the old syntaxes are: -** -** a) The new syntax supports parenthesis. The old does not. -** -** b) The new syntax supports the AND and NOT operators. The old does not. -** -** c) The old syntax supports the "-" token qualifier. This is not -** supported by the new syntax (it is replaced by the NOT operator). -** -** d) When using the old syntax, the OR operator has a greater precedence -** than an implicit AND. When using the new, both implicity and explicit -** AND operators have a higher precedence than OR. -** -** If compiled with SQLITE_TEST defined, then this module exports the -** symbol "int sqlite3_fts3_enable_parentheses". Setting this variable -** to zero causes the module to use the old syntax. If it is set to -** non-zero the new syntax is activated. This is so both syntaxes can -** be tested using a single build of testfixture. -** -** The following describes the syntax supported by the fts3 MATCH -** operator in a similar format to that used by the lemon parser -** generator. This module does not use actually lemon, it uses a -** custom parser. -** -** query ::= andexpr (OR andexpr)*. -** -** andexpr ::= notexpr (AND? notexpr)*. -** -** notexpr ::= nearexpr (NOT nearexpr|-TOKEN)*. -** notexpr ::= LP query RP. -** -** nearexpr ::= phrase (NEAR distance_opt nearexpr)*. +** The code in this file is only compiled if: ** -** distance_opt ::= . -** distance_opt ::= / INTEGER. +** * The FTS3 module is being built as an extension +** (in which case SQLITE_CORE is not defined), or ** -** phrase ::= TOKEN. -** phrase ::= COLUMN:TOKEN. -** phrase ::= "TOKEN TOKEN TOKEN...". -*/ - -#ifdef SQLITE_TEST -SQLITE_API int sqlite3_fts3_enable_parentheses = 0; -#else -# ifdef SQLITE_ENABLE_FTS3_PARENTHESIS -# define sqlite3_fts3_enable_parentheses 1 -# else -# define sqlite3_fts3_enable_parentheses 0 -# endif -#endif - -/* -** Default span for NEAR operators. +** * The FTS3 module is being built into the core of +** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). */ -#define SQLITE_FTS3_DEFAULT_NEAR_PARAM 10 +/* #include "fts3Int.h" */ +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) -/* #include */ /* #include */ +/* #include */ /* -** isNot: -** This variable is used by function getNextNode(). When getNextNode() is -** called, it sets ParseContext.isNot to true if the 'next node' is a -** FTSQUERY_PHRASE with a unary "-" attached to it. i.e. "mysql" in the -** FTS3 query "sqlite -mysql". Otherwise, ParseContext.isNot is set to -** zero. +** Return true if the two-argument version of fts3_tokenizer() +** has been activated via a prior call to sqlite3_db_config(db, +** SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER, 1, 0); */ -typedef struct ParseContext ParseContext; -struct ParseContext { - sqlite3_tokenizer *pTokenizer; /* Tokenizer module */ - int iLangid; /* Language id used with tokenizer */ - const char **azCol; /* Array of column names for fts3 table */ - int bFts4; /* True to allow FTS4-only syntax */ - int nCol; /* Number of entries in azCol[] */ - int iDefaultCol; /* Default column to query */ - int isNot; /* True if getNextNode() sees a unary - */ - sqlite3_context *pCtx; /* Write error message here */ - int nNest; /* Number of nested brackets */ -}; +static int fts3TokenizerEnabled(sqlite3_context *context){ + sqlite3 *db = sqlite3_context_db_handle(context); + int isEnabled = 0; + sqlite3_db_config(db,SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER,-1,&isEnabled); + return isEnabled; +} /* -** This function is equivalent to the standard isspace() function. +** Implementation of the SQL scalar function for accessing the underlying +** hash table. This function may be called as follows: ** -** The standard isspace() can be awkward to use safely, because although it -** is defined to accept an argument of type int, its behavior when passed -** an integer that falls outside of the range of the unsigned char type -** is undefined (and sometimes, "undefined" means segfault). This wrapper -** is defined to accept an argument of type char, and always returns 0 for -** any values that fall outside of the range of the unsigned char type (i.e. -** negative values). +** SELECT (); +** SELECT (, ); +** +** where is the name passed as the second argument +** to the sqlite3Fts3InitHashTable() function (e.g. 'fts3_tokenizer'). +** +** If the argument is specified, it must be a blob value +** containing a pointer to be stored as the hash data corresponding +** to the string . If is not specified, then +** the string must already exist in the has table. Otherwise, +** an error is returned. +** +** Whether or not the argument is specified, the value returned +** is a blob containing the pointer stored as the hash data corresponding +** to string (after the hash-table is updated, if applicable). */ -static int fts3isspace(char c){ - return c==' ' || c=='\t' || c=='\n' || c=='\r' || c=='\v' || c=='\f'; +static void fts3TokenizerFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + Fts3Hash *pHash; + void *pPtr = 0; + const unsigned char *zName; + int nName; + + assert( argc==1 || argc==2 ); + + pHash = (Fts3Hash *)sqlite3_user_data(context); + + zName = sqlite3_value_text(argv[0]); + nName = sqlite3_value_bytes(argv[0])+1; + + if( argc==2 ){ + if( fts3TokenizerEnabled(context) || sqlite3_value_frombind(argv[1]) ){ + void *pOld; + int n = sqlite3_value_bytes(argv[1]); + if( zName==0 || n!=sizeof(pPtr) ){ + sqlite3_result_error(context, "argument type mismatch", -1); + return; + } + pPtr = *(void **)sqlite3_value_blob(argv[1]); + pOld = sqlite3Fts3HashInsert(pHash, (void *)zName, nName, pPtr); + if( pOld==pPtr ){ + sqlite3_result_error(context, "out of memory", -1); + } + }else{ + sqlite3_result_error(context, "fts3tokenize disabled", -1); + return; + } + }else{ + if( zName ){ + pPtr = sqlite3Fts3HashFind(pHash, zName, nName); + } + if( !pPtr ){ + char *zErr = sqlite3_mprintf("unknown tokenizer: %s", zName); + sqlite3_result_error(context, zErr, -1); + sqlite3_free(zErr); + return; + } + } + if( fts3TokenizerEnabled(context) || sqlite3_value_frombind(argv[0]) ){ + sqlite3_result_blob(context, (void *)&pPtr, sizeof(pPtr), SQLITE_TRANSIENT); + } } -/* -** Allocate nByte bytes of memory using sqlite3_malloc(). If successful, -** zero the memory before returning a pointer to it. If unsuccessful, -** return NULL. -*/ -static void *fts3MallocZero(int nByte){ - void *pRet = sqlite3_malloc(nByte); - if( pRet ) memset(pRet, 0, nByte); - return pRet; +SQLITE_PRIVATE int sqlite3Fts3IsIdChar(char c){ + static const char isFtsIdChar[] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 1x */ + 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 2x */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 3x */ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4x */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 5x */ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 6x */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */ + }; + return (c&0x80 || isFtsIdChar[(int)(c)]); } -SQLITE_PRIVATE int sqlite3Fts3OpenTokenizer( - sqlite3_tokenizer *pTokenizer, - int iLangid, - const char *z, - int n, - sqlite3_tokenizer_cursor **ppCsr +SQLITE_PRIVATE const char *sqlite3Fts3NextToken(const char *zStr, int *pn){ + const char *z1; + const char *z2 = 0; + + /* Find the start of the next token. */ + z1 = zStr; + while( z2==0 ){ + char c = *z1; + switch( c ){ + case '\0': return 0; /* No more tokens here */ + case '\'': + case '"': + case '`': { + z2 = z1; + while( *++z2 && (*z2!=c || *++z2==c) ); + break; + } + case '[': + z2 = &z1[1]; + while( *z2 && z2[0]!=']' ) z2++; + if( *z2 ) z2++; + break; + + default: + if( sqlite3Fts3IsIdChar(*z1) ){ + z2 = &z1[1]; + while( sqlite3Fts3IsIdChar(*z2) ) z2++; + }else{ + z1++; + } + } + } + + *pn = (int)(z2-z1); + return z1; +} + +SQLITE_PRIVATE int sqlite3Fts3InitTokenizer( + Fts3Hash *pHash, /* Tokenizer hash table */ + const char *zArg, /* Tokenizer name */ + sqlite3_tokenizer **ppTok, /* OUT: Tokenizer (if applicable) */ + char **pzErr /* OUT: Set to malloced error message */ ){ - sqlite3_tokenizer_module const *pModule = pTokenizer->pModule; - sqlite3_tokenizer_cursor *pCsr = 0; int rc; + char *z = (char *)zArg; + int n = 0; + char *zCopy; + char *zEnd; /* Pointer to nul-term of zCopy */ + sqlite3_tokenizer_module *m; - rc = pModule->xOpen(pTokenizer, z, n, &pCsr); - assert( rc==SQLITE_OK || pCsr==0 ); - if( rc==SQLITE_OK ){ - pCsr->pTokenizer = pTokenizer; - if( pModule->iVersion>=1 ){ - rc = pModule->xLanguageid(pCsr, iLangid); - if( rc!=SQLITE_OK ){ - pModule->xClose(pCsr); - pCsr = 0; + zCopy = sqlite3_mprintf("%s", zArg); + if( !zCopy ) return SQLITE_NOMEM; + zEnd = &zCopy[strlen(zCopy)]; + + z = (char *)sqlite3Fts3NextToken(zCopy, &n); + if( z==0 ){ + assert( n==0 ); + z = zCopy; + } + z[n] = '\0'; + sqlite3Fts3Dequote(z); + + m = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash,z,(int)strlen(z)+1); + if( !m ){ + sqlite3Fts3ErrMsg(pzErr, "unknown tokenizer: %s", z); + rc = SQLITE_ERROR; + }else{ + char const **aArg = 0; + int iArg = 0; + z = &z[n+1]; + while( zxCreate(iArg, aArg, ppTok); + assert( rc!=SQLITE_OK || *ppTok ); + if( rc!=SQLITE_OK ){ + sqlite3Fts3ErrMsg(pzErr, "unknown tokenizer"); + }else{ + (*ppTok)->pModule = m; } + sqlite3_free((void *)aArg); } - *ppCsr = pCsr; + + sqlite3_free(zCopy); return rc; } -/* -** Function getNextNode(), which is called by fts3ExprParse(), may itself -** call fts3ExprParse(). So this forward declaration is required. -*/ -static int fts3ExprParse(ParseContext *, const char *, int, Fts3Expr **, int *); + +#ifdef SQLITE_TEST + +#if defined(INCLUDE_SQLITE_TCL_H) +# include "sqlite_tcl.h" +#else +# include "tcl.h" +#endif +/* #include */ /* -** Extract the next token from buffer z (length n) using the tokenizer -** and other information (column names etc.) in pParse. Create an Fts3Expr -** structure of type FTSQUERY_PHRASE containing a phrase consisting of this -** single token and set *ppExpr to point to it. If the end of the buffer is -** reached before a token is found, set *ppExpr to zero. It is the -** responsibility of the caller to eventually deallocate the allocated -** Fts3Expr structure (if any) by passing it to sqlite3_free(). +** Implementation of a special SQL scalar function for testing tokenizers +** designed to be used in concert with the Tcl testing framework. This +** function must be called with two or more arguments: ** -** Return SQLITE_OK if successful, or SQLITE_NOMEM if a memory allocation -** fails. +** SELECT (, ..., ); +** +** where is the name passed as the second argument +** to the sqlite3Fts3InitHashTable() function (e.g. 'fts3_tokenizer') +** concatenated with the string '_test' (e.g. 'fts3_tokenizer_test'). +** +** The return value is a string that may be interpreted as a Tcl +** list. For each token in the , three elements are +** added to the returned list. The first is the token position, the +** second is the token text (folded, stemmed, etc.) and the third is the +** substring of associated with the token. For example, +** using the built-in "simple" tokenizer: +** +** SELECT fts_tokenizer_test('simple', 'I don't see how'); +** +** will return the string: +** +** "{0 i I 1 dont don't 2 see see 3 how how}" +** */ -static int getNextToken( - ParseContext *pParse, /* fts3 query parse context */ - int iCol, /* Value for Fts3Phrase.iColumn */ - const char *z, int n, /* Input string */ - Fts3Expr **ppExpr, /* OUT: expression */ - int *pnConsumed /* OUT: Number of bytes consumed */ +static void testFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv ){ - sqlite3_tokenizer *pTokenizer = pParse->pTokenizer; - sqlite3_tokenizer_module const *pModule = pTokenizer->pModule; - int rc; - sqlite3_tokenizer_cursor *pCursor; - Fts3Expr *pRet = 0; - int i = 0; + Fts3Hash *pHash; + sqlite3_tokenizer_module *p; + sqlite3_tokenizer *pTokenizer = 0; + sqlite3_tokenizer_cursor *pCsr = 0; - /* Set variable i to the maximum number of bytes of input to tokenize. */ - for(i=0; iiLangid, z, i, &pCursor); - if( rc==SQLITE_OK ){ - const char *zToken; - int nToken = 0, iStart = 0, iEnd = 0, iPosition = 0; - int nByte; /* total space to allocate */ + nName = sqlite3_value_bytes(argv[0]); + zName = (const char *)sqlite3_value_text(argv[0]); + nInput = sqlite3_value_bytes(argv[argc-1]); + zInput = (const char *)sqlite3_value_text(argv[argc-1]); - rc = pModule->xNext(pCursor, &zToken, &nToken, &iStart, &iEnd, &iPosition); - if( rc==SQLITE_OK ){ - nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase) + nToken; - pRet = (Fts3Expr *)fts3MallocZero(nByte); - if( !pRet ){ - rc = SQLITE_NOMEM; - }else{ - pRet->eType = FTSQUERY_PHRASE; - pRet->pPhrase = (Fts3Phrase *)&pRet[1]; - pRet->pPhrase->nToken = 1; - pRet->pPhrase->iColumn = iCol; - pRet->pPhrase->aToken[0].n = nToken; - pRet->pPhrase->aToken[0].z = (char *)&pRet->pPhrase[1]; - memcpy(pRet->pPhrase->aToken[0].z, zToken, nToken); + pHash = (Fts3Hash *)sqlite3_user_data(context); + p = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash, zName, nName+1); - if( iEndpPhrase->aToken[0].isPrefix = 1; - iEnd++; - } + if( !p ){ + char *zErr2 = sqlite3_mprintf("unknown tokenizer: %s", zName); + sqlite3_result_error(context, zErr2, -1); + sqlite3_free(zErr2); + return; + } - while( 1 ){ - if( !sqlite3_fts3_enable_parentheses - && iStart>0 && z[iStart-1]=='-' - ){ - pParse->isNot = 1; - iStart--; - }else if( pParse->bFts4 && iStart>0 && z[iStart-1]=='^' ){ - pRet->pPhrase->aToken[0].bFirst = 1; - iStart--; - }else{ - break; - } - } + pRet = Tcl_NewObj(); + Tcl_IncrRefCount(pRet); - } - *pnConsumed = iEnd; - }else if( i && rc==SQLITE_DONE ){ - rc = SQLITE_OK; - } + for(i=1; ixClose(pCursor); + if( SQLITE_OK!=p->xCreate(argc-2, azArg, &pTokenizer) ){ + zErr = "error in xCreate()"; + goto finish; + } + pTokenizer->pModule = p; + if( sqlite3Fts3OpenTokenizer(pTokenizer, 0, zInput, nInput, &pCsr) ){ + zErr = "error in xOpen()"; + goto finish; } - - *ppExpr = pRet; - return rc; -} + while( SQLITE_OK==p->xNext(pCsr, &zToken, &nToken, &iStart, &iEnd, &iPos) ){ + Tcl_ListObjAppendElement(0, pRet, Tcl_NewIntObj(iPos)); + Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj(zToken, nToken)); + zToken = &zInput[iStart]; + nToken = iEnd-iStart; + Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj(zToken, nToken)); + } -/* -** Enlarge a memory allocation. If an out-of-memory allocation occurs, -** then free the old allocation. -*/ -static void *fts3ReallocOrFree(void *pOrig, int nNew){ - void *pRet = sqlite3_realloc(pOrig, nNew); - if( !pRet ){ - sqlite3_free(pOrig); + if( SQLITE_OK!=p->xClose(pCsr) ){ + zErr = "error in xClose()"; + goto finish; } - return pRet; + if( SQLITE_OK!=p->xDestroy(pTokenizer) ){ + zErr = "error in xDestroy()"; + goto finish; + } + +finish: + if( zErr ){ + sqlite3_result_error(context, zErr, -1); + }else{ + sqlite3_result_text(context, Tcl_GetString(pRet), -1, SQLITE_TRANSIENT); + } + Tcl_DecrRefCount(pRet); } -/* -** Buffer zInput, length nInput, contains the contents of a quoted string -** that appeared as part of an fts3 query expression. Neither quote character -** is included in the buffer. This function attempts to tokenize the entire -** input buffer and create an Fts3Expr structure of type FTSQUERY_PHRASE -** containing the results. -** -** If successful, SQLITE_OK is returned and *ppExpr set to point at the -** allocated Fts3Expr structure. Otherwise, either SQLITE_NOMEM (out of memory -** error) or SQLITE_ERROR (tokenization error) is returned and *ppExpr set -** to 0. -*/ -static int getNextString( - ParseContext *pParse, /* fts3 query parse context */ - const char *zInput, int nInput, /* Input string */ - Fts3Expr **ppExpr /* OUT: expression */ +static +int registerTokenizer( + sqlite3 *db, + char *zName, + const sqlite3_tokenizer_module *p ){ - sqlite3_tokenizer *pTokenizer = pParse->pTokenizer; - sqlite3_tokenizer_module const *pModule = pTokenizer->pModule; int rc; - Fts3Expr *p = 0; - sqlite3_tokenizer_cursor *pCursor = 0; - char *zTemp = 0; - int nTemp = 0; + sqlite3_stmt *pStmt; + const char zSql[] = "SELECT fts3_tokenizer(?, ?)"; - const int nSpace = sizeof(Fts3Expr) + sizeof(Fts3Phrase); - int nToken = 0; + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); + if( rc!=SQLITE_OK ){ + return rc; + } - /* The final Fts3Expr data structure, including the Fts3Phrase, - ** Fts3PhraseToken structures token buffers are all stored as a single - ** allocation so that the expression can be freed with a single call to - ** sqlite3_free(). Setting this up requires a two pass approach. - ** - ** The first pass, in the block below, uses a tokenizer cursor to iterate - ** through the tokens in the expression. This pass uses fts3ReallocOrFree() - ** to assemble data in two dynamic buffers: - ** - ** Buffer p: Points to the Fts3Expr structure, followed by the Fts3Phrase - ** structure, followed by the array of Fts3PhraseToken - ** structures. This pass only populates the Fts3PhraseToken array. - ** - ** Buffer zTemp: Contains copies of all tokens. - ** - ** The second pass, in the block that begins "if( rc==SQLITE_DONE )" below, - ** appends buffer zTemp to buffer p, and fills in the Fts3Expr and Fts3Phrase - ** structures. - */ - rc = sqlite3Fts3OpenTokenizer( - pTokenizer, pParse->iLangid, zInput, nInput, &pCursor); - if( rc==SQLITE_OK ){ - int ii; - for(ii=0; rc==SQLITE_OK; ii++){ - const char *zByte; - int nByte = 0, iBegin = 0, iEnd = 0, iPos = 0; - rc = pModule->xNext(pCursor, &zByte, &nByte, &iBegin, &iEnd, &iPos); - if( rc==SQLITE_OK ){ - Fts3PhraseToken *pToken; + sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC); + sqlite3_bind_blob(pStmt, 2, &p, sizeof(p), SQLITE_STATIC); + sqlite3_step(pStmt); - p = fts3ReallocOrFree(p, nSpace + ii*sizeof(Fts3PhraseToken)); - if( !p ) goto no_mem; + return sqlite3_finalize(pStmt); +} - zTemp = fts3ReallocOrFree(zTemp, nTemp + nByte); - if( !zTemp ) goto no_mem; - assert( nToken==ii ); - pToken = &((Fts3Phrase *)(&p[1]))->aToken[ii]; - memset(pToken, 0, sizeof(Fts3PhraseToken)); +static +int queryTokenizer( + sqlite3 *db, + char *zName, + const sqlite3_tokenizer_module **pp +){ + int rc; + sqlite3_stmt *pStmt; + const char zSql[] = "SELECT fts3_tokenizer(?)"; - memcpy(&zTemp[nTemp], zByte, nByte); - nTemp += nByte; + *pp = 0; + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); + if( rc!=SQLITE_OK ){ + return rc; + } - pToken->n = nByte; - pToken->isPrefix = (iEndbFirst = (iBegin>0 && zInput[iBegin-1]=='^'); - nToken = ii+1; - } + sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC); + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + if( sqlite3_column_type(pStmt, 0)==SQLITE_BLOB + && sqlite3_column_bytes(pStmt, 0)==sizeof(*pp) + ){ + memcpy((void *)pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp)); } - - pModule->xClose(pCursor); - pCursor = 0; } - if( rc==SQLITE_DONE ){ - int jj; - char *zBuf = 0; + return sqlite3_finalize(pStmt); +} - p = fts3ReallocOrFree(p, nSpace + nToken*sizeof(Fts3PhraseToken) + nTemp); - if( !p ) goto no_mem; - memset(p, 0, (char *)&(((Fts3Phrase *)&p[1])->aToken[0])-(char *)p); - p->eType = FTSQUERY_PHRASE; - p->pPhrase = (Fts3Phrase *)&p[1]; - p->pPhrase->iColumn = pParse->iDefaultCol; - p->pPhrase->nToken = nToken; +SQLITE_PRIVATE void sqlite3Fts3SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule); - zBuf = (char *)&p->pPhrase->aToken[nToken]; - if( zTemp ){ - memcpy(zBuf, zTemp, nTemp); - sqlite3_free(zTemp); - }else{ - assert( nTemp==0 ); - } +/* +** Implementation of the scalar function fts3_tokenizer_internal_test(). +** This function is used for testing only, it is not included in the +** build unless SQLITE_TEST is defined. +** +** The purpose of this is to test that the fts3_tokenizer() function +** can be used as designed by the C-code in the queryTokenizer and +** registerTokenizer() functions above. These two functions are repeated +** in the README.tokenizer file as an example, so it is important to +** test them. +** +** To run the tests, evaluate the fts3_tokenizer_internal_test() scalar +** function with no arguments. An assert() will fail if a problem is +** detected. i.e.: +** +** SELECT fts3_tokenizer_internal_test(); +** +*/ +static void intTestFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + int rc; + const sqlite3_tokenizer_module *p1; + const sqlite3_tokenizer_module *p2; + sqlite3 *db = (sqlite3 *)sqlite3_user_data(context); - for(jj=0; jjpPhrase->nToken; jj++){ - p->pPhrase->aToken[jj].z = zBuf; - zBuf += p->pPhrase->aToken[jj].n; - } - rc = SQLITE_OK; - } + UNUSED_PARAMETER(argc); + UNUSED_PARAMETER(argv); - *ppExpr = p; - return rc; -no_mem: + /* Test the query function */ + sqlite3Fts3SimpleTokenizerModule(&p1); + rc = queryTokenizer(db, "simple", &p2); + assert( rc==SQLITE_OK ); + assert( p1==p2 ); + rc = queryTokenizer(db, "nosuchtokenizer", &p2); + assert( rc==SQLITE_ERROR ); + assert( p2==0 ); + assert( 0==strcmp(sqlite3_errmsg(db), "unknown tokenizer: nosuchtokenizer") ); - if( pCursor ){ - pModule->xClose(pCursor); + /* Test the storage function */ + if( fts3TokenizerEnabled(context) ){ + rc = registerTokenizer(db, "nosuchtokenizer", p1); + assert( rc==SQLITE_OK ); + rc = queryTokenizer(db, "nosuchtokenizer", &p2); + assert( rc==SQLITE_OK ); + assert( p2==p1 ); } - sqlite3_free(zTemp); - sqlite3_free(p); - *ppExpr = 0; - return SQLITE_NOMEM; + + sqlite3_result_text(context, "ok", -1, SQLITE_STATIC); } +#endif + /* -** The output variable *ppExpr is populated with an allocated Fts3Expr -** structure, or set to 0 if the end of the input buffer is reached. +** Set up SQL objects in database db used to access the contents of +** the hash table pointed to by argument pHash. The hash table must +** been initialized to use string keys, and to take a private copy +** of the key when a value is inserted. i.e. by a call similar to: ** -** Returns an SQLite error code. SQLITE_OK if everything works, SQLITE_NOMEM -** if a malloc failure occurs, or SQLITE_ERROR if a parse error is encountered. -** If SQLITE_ERROR is returned, pContext is populated with an error message. +** sqlite3Fts3HashInit(pHash, FTS3_HASH_STRING, 1); +** +** This function adds a scalar function (see header comment above +** fts3TokenizerFunc() in this file for details) and, if ENABLE_TABLE is +** defined at compilation time, a temporary virtual table (see header +** comment above struct HashTableVtab) to the database schema. Both +** provide read/write access to the contents of *pHash. +** +** The third argument to this function, zName, is used as the name +** of both the scalar and, if created, the virtual table. */ -static int getNextNode( - ParseContext *pParse, /* fts3 query parse context */ - const char *z, int n, /* Input string */ - Fts3Expr **ppExpr, /* OUT: expression */ - int *pnConsumed /* OUT: Number of bytes consumed */ +SQLITE_PRIVATE int sqlite3Fts3InitHashTable( + sqlite3 *db, + Fts3Hash *pHash, + const char *zName ){ - static const struct Fts3Keyword { - char *z; /* Keyword text */ - unsigned char n; /* Length of the keyword */ - unsigned char parenOnly; /* Only valid in paren mode */ - unsigned char eType; /* Keyword code */ - } aKeyword[] = { - { "OR" , 2, 0, FTSQUERY_OR }, - { "AND", 3, 1, FTSQUERY_AND }, - { "NOT", 3, 1, FTSQUERY_NOT }, - { "NEAR", 4, 0, FTSQUERY_NEAR } - }; - int ii; - int iCol; - int iColLen; - int rc; - Fts3Expr *pRet = 0; - - const char *zInput = z; - int nInput = n; + int rc = SQLITE_OK; + void *p = (void *)pHash; + const int any = SQLITE_UTF8|SQLITE_DIRECTONLY; - pParse->isNot = 0; +#ifdef SQLITE_TEST + char *zTest = 0; + char *zTest2 = 0; + void *pdb = (void *)db; + zTest = sqlite3_mprintf("%s_test", zName); + zTest2 = sqlite3_mprintf("%s_internal_test", zName); + if( !zTest || !zTest2 ){ + rc = SQLITE_NOMEM; + } +#endif - /* Skip over any whitespace before checking for a keyword, an open or - ** close bracket, or a quoted string. - */ - while( nInput>0 && fts3isspace(*zInput) ){ - nInput--; - zInput++; + if( SQLITE_OK==rc ){ + rc = sqlite3_create_function(db, zName, 1, any, p, fts3TokenizerFunc, 0, 0); } - if( nInput==0 ){ - return SQLITE_DONE; + if( SQLITE_OK==rc ){ + rc = sqlite3_create_function(db, zName, 2, any, p, fts3TokenizerFunc, 0, 0); + } +#ifdef SQLITE_TEST + if( SQLITE_OK==rc ){ + rc = sqlite3_create_function(db, zTest, -1, any, p, testFunc, 0, 0); + } + if( SQLITE_OK==rc ){ + rc = sqlite3_create_function(db, zTest2, 0, any, pdb, intTestFunc, 0, 0); } +#endif - /* See if we are dealing with a keyword. */ - for(ii=0; ii<(int)(sizeof(aKeyword)/sizeof(struct Fts3Keyword)); ii++){ - const struct Fts3Keyword *pKey = &aKeyword[ii]; +#ifdef SQLITE_TEST + sqlite3_free(zTest); + sqlite3_free(zTest2); +#endif - if( (pKey->parenOnly & ~sqlite3_fts3_enable_parentheses)!=0 ){ - continue; - } + return rc; +} - if( nInput>=pKey->n && 0==memcmp(zInput, pKey->z, pKey->n) ){ - int nNear = SQLITE_FTS3_DEFAULT_NEAR_PARAM; - int nKey = pKey->n; - char cNext; +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ - /* If this is a "NEAR" keyword, check for an explicit nearness. */ - if( pKey->eType==FTSQUERY_NEAR ){ - assert( nKey==4 ); - if( zInput[4]=='/' && zInput[5]>='0' && zInput[5]<='9' ){ - nNear = 0; - for(nKey=5; zInput[nKey]>='0' && zInput[nKey]<='9'; nKey++){ - nNear = nNear * 10 + (zInput[nKey] - '0'); - } - } - } +/************** End of fts3_tokenizer.c **************************************/ +/************** Begin file fts3_tokenizer1.c *********************************/ +/* +** 2006 Oct 10 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** Implementation of the "simple" full-text-search tokenizer. +*/ - /* At this point this is probably a keyword. But for that to be true, - ** the next byte must contain either whitespace, an open or close - ** parenthesis, a quote character, or EOF. - */ - cNext = zInput[nKey]; - if( fts3isspace(cNext) - || cNext=='"' || cNext=='(' || cNext==')' || cNext==0 - ){ - pRet = (Fts3Expr *)fts3MallocZero(sizeof(Fts3Expr)); - if( !pRet ){ - return SQLITE_NOMEM; - } - pRet->eType = pKey->eType; - pRet->nNear = nNear; - *ppExpr = pRet; - *pnConsumed = (int)((zInput - z) + nKey); - return SQLITE_OK; - } +/* +** The code in this file is only compiled if: +** +** * The FTS3 module is being built as an extension +** (in which case SQLITE_CORE is not defined), or +** +** * The FTS3 module is being built into the core of +** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). +*/ +/* #include "fts3Int.h" */ +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) - /* Turns out that wasn't a keyword after all. This happens if the - ** user has supplied a token such as "ORacle". Continue. - */ - } - } +/* #include */ +/* #include */ +/* #include */ +/* #include */ - /* See if we are dealing with a quoted phrase. If this is the case, then - ** search for the closing quote and pass the whole string to getNextString() - ** for processing. This is easy to do, as fts3 has no syntax for escaping - ** a quote character embedded in a string. - */ - if( *zInput=='"' ){ - for(ii=1; iinNest++; - rc = fts3ExprParse(pParse, zInput+1, nInput-1, ppExpr, &nConsumed); - if( rc==SQLITE_OK && !*ppExpr ){ rc = SQLITE_DONE; } - *pnConsumed = (int)(zInput - z) + 1 + nConsumed; - return rc; - }else if( *zInput==')' ){ - pParse->nNest--; - *pnConsumed = (int)((zInput - z) + 1); - *ppExpr = 0; - return SQLITE_DONE; - } - } +typedef struct simple_tokenizer { + sqlite3_tokenizer base; + char delim[128]; /* flag ASCII delimiters */ +} simple_tokenizer; - /* If control flows to this point, this must be a regular token, or - ** the end of the input. Read a regular token using the sqlite3_tokenizer - ** interface. Before doing so, figure out if there is an explicit - ** column specifier for the token. - ** - ** TODO: Strangely, it is not possible to associate a column specifier - ** with a quoted phrase, only with a single token. Not sure if this was - ** an implementation artifact or an intentional decision when fts3 was - ** first implemented. Whichever it was, this module duplicates the - ** limitation. +typedef struct simple_tokenizer_cursor { + sqlite3_tokenizer_cursor base; + const char *pInput; /* input we are tokenizing */ + int nBytes; /* size of the input */ + int iOffset; /* current position in pInput */ + int iToken; /* index of next token to be returned */ + char *pToken; /* storage for current token */ + int nTokenAllocated; /* space allocated to zToken buffer */ +} simple_tokenizer_cursor; + + +static int simpleDelim(simple_tokenizer *t, unsigned char c){ + return c<0x80 && t->delim[c]; +} +static int fts3_isalnum(int x){ + return (x>='0' && x<='9') || (x>='A' && x<='Z') || (x>='a' && x<='z'); +} + +/* +** Create a new tokenizer instance. +*/ +static int simpleCreate( + int argc, const char * const *argv, + sqlite3_tokenizer **ppTokenizer +){ + simple_tokenizer *t; + + t = (simple_tokenizer *) sqlite3_malloc(sizeof(*t)); + if( t==NULL ) return SQLITE_NOMEM; + memset(t, 0, sizeof(*t)); + + /* TODO(shess) Delimiters need to remain the same from run to run, + ** else we need to reindex. One solution would be a meta-table to + ** track such information in the database, then we'd only want this + ** information on the initial create. */ - iCol = pParse->iDefaultCol; - iColLen = 0; - for(ii=0; iinCol; ii++){ - const char *zStr = pParse->azCol[ii]; - int nStr = (int)strlen(zStr); - if( nInput>nStr && zInput[nStr]==':' - && sqlite3_strnicmp(zStr, zInput, nStr)==0 - ){ - iCol = ii; - iColLen = (int)((zInput - z) + nStr + 1); - break; + if( argc>1 ){ + int i, n = (int)strlen(argv[1]); + for(i=0; i=0x80 ){ + sqlite3_free(t); + return SQLITE_ERROR; + } + t->delim[ch] = 1; + } + } else { + /* Mark non-alphanumeric ASCII characters as delimiters */ + int i; + for(i=1; i<0x80; i++){ + t->delim[i] = !fts3_isalnum(i) ? -1 : 0; } } - rc = getNextToken(pParse, iCol, &z[iColLen], n-iColLen, ppExpr, pnConsumed); - *pnConsumed += iColLen; - return rc; + + *ppTokenizer = &t->base; + return SQLITE_OK; } /* -** The argument is an Fts3Expr structure for a binary operator (any type -** except an FTSQUERY_PHRASE). Return an integer value representing the -** precedence of the operator. Lower values have a higher precedence (i.e. -** group more tightly). For example, in the C language, the == operator -** groups more tightly than ||, and would therefore have a higher precedence. -** -** When using the new fts3 query syntax (when SQLITE_ENABLE_FTS3_PARENTHESIS -** is defined), the order of the operators in precedence from highest to -** lowest is: -** -** NEAR -** NOT -** AND (including implicit ANDs) -** OR -** -** Note that when using the old query syntax, the OR operator has a higher -** precedence than the AND operator. +** Destroy a tokenizer */ -static int opPrecedence(Fts3Expr *p){ - assert( p->eType!=FTSQUERY_PHRASE ); - if( sqlite3_fts3_enable_parentheses ){ - return p->eType; - }else if( p->eType==FTSQUERY_NEAR ){ - return 1; - }else if( p->eType==FTSQUERY_OR ){ - return 2; - } - assert( p->eType==FTSQUERY_AND ); - return 3; +static int simpleDestroy(sqlite3_tokenizer *pTokenizer){ + sqlite3_free(pTokenizer); + return SQLITE_OK; } /* -** Argument ppHead contains a pointer to the current head of a query -** expression tree being parsed. pPrev is the expression node most recently -** inserted into the tree. This function adds pNew, which is always a binary -** operator node, into the expression tree based on the relative precedence -** of pNew and the existing nodes of the tree. This may result in the head -** of the tree changing, in which case *ppHead is set to the new root node. +** Prepare to begin tokenizing a particular string. The input +** string to be tokenized is pInput[0..nBytes-1]. A cursor +** used to incrementally tokenize this string is returned in +** *ppCursor. */ -static void insertBinaryOperator( - Fts3Expr **ppHead, /* Pointer to the root node of a tree */ - Fts3Expr *pPrev, /* Node most recently inserted into the tree */ - Fts3Expr *pNew /* New binary node to insert into expression tree */ +static int simpleOpen( + sqlite3_tokenizer *pTokenizer, /* The tokenizer */ + const char *pInput, int nBytes, /* String to be tokenized */ + sqlite3_tokenizer_cursor **ppCursor /* OUT: Tokenization cursor */ ){ - Fts3Expr *pSplit = pPrev; - while( pSplit->pParent && opPrecedence(pSplit->pParent)<=opPrecedence(pNew) ){ - pSplit = pSplit->pParent; - } + simple_tokenizer_cursor *c; - if( pSplit->pParent ){ - assert( pSplit->pParent->pRight==pSplit ); - pSplit->pParent->pRight = pNew; - pNew->pParent = pSplit->pParent; + UNUSED_PARAMETER(pTokenizer); + + c = (simple_tokenizer_cursor *) sqlite3_malloc(sizeof(*c)); + if( c==NULL ) return SQLITE_NOMEM; + + c->pInput = pInput; + if( pInput==0 ){ + c->nBytes = 0; + }else if( nBytes<0 ){ + c->nBytes = (int)strlen(pInput); }else{ - *ppHead = pNew; + c->nBytes = nBytes; } - pNew->pLeft = pSplit; - pSplit->pParent = pNew; + c->iOffset = 0; /* start tokenizing at the beginning */ + c->iToken = 0; + c->pToken = NULL; /* no space allocated, yet. */ + c->nTokenAllocated = 0; + + *ppCursor = &c->base; + return SQLITE_OK; } /* -** Parse the fts3 query expression found in buffer z, length n. This function -** returns either when the end of the buffer is reached or an unmatched -** closing bracket - ')' - is encountered. -** -** If successful, SQLITE_OK is returned, *ppExpr is set to point to the -** parsed form of the expression and *pnConsumed is set to the number of -** bytes read from buffer z. Otherwise, *ppExpr is set to 0 and SQLITE_NOMEM -** (out of memory error) or SQLITE_ERROR (parse error) is returned. +** Close a tokenization cursor previously opened by a call to +** simpleOpen() above. */ -static int fts3ExprParse( - ParseContext *pParse, /* fts3 query parse context */ - const char *z, int n, /* Text of MATCH query */ - Fts3Expr **ppExpr, /* OUT: Parsed query structure */ - int *pnConsumed /* OUT: Number of bytes consumed */ -){ - Fts3Expr *pRet = 0; - Fts3Expr *pPrev = 0; - Fts3Expr *pNotBranch = 0; /* Only used in legacy parse mode */ - int nIn = n; - const char *zIn = z; - int rc = SQLITE_OK; - int isRequirePhrase = 1; - - while( rc==SQLITE_OK ){ - Fts3Expr *p = 0; - int nByte = 0; - - rc = getNextNode(pParse, zIn, nIn, &p, &nByte); - assert( nByte>0 || (rc!=SQLITE_OK && p==0) ); - if( rc==SQLITE_OK ){ - if( p ){ - int isPhrase; - - if( !sqlite3_fts3_enable_parentheses - && p->eType==FTSQUERY_PHRASE && pParse->isNot - ){ - /* Create an implicit NOT operator. */ - Fts3Expr *pNot = fts3MallocZero(sizeof(Fts3Expr)); - if( !pNot ){ - sqlite3Fts3ExprFree(p); - rc = SQLITE_NOMEM; - goto exprparse_out; - } - pNot->eType = FTSQUERY_NOT; - pNot->pRight = p; - p->pParent = pNot; - if( pNotBranch ){ - pNot->pLeft = pNotBranch; - pNotBranch->pParent = pNot; - } - pNotBranch = pNot; - p = pPrev; - }else{ - int eType = p->eType; - isPhrase = (eType==FTSQUERY_PHRASE || p->pLeft); - - /* The isRequirePhrase variable is set to true if a phrase or - ** an expression contained in parenthesis is required. If a - ** binary operator (AND, OR, NOT or NEAR) is encounted when - ** isRequirePhrase is set, this is a syntax error. - */ - if( !isPhrase && isRequirePhrase ){ - sqlite3Fts3ExprFree(p); - rc = SQLITE_ERROR; - goto exprparse_out; - } +static int simpleClose(sqlite3_tokenizer_cursor *pCursor){ + simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor; + sqlite3_free(c->pToken); + sqlite3_free(c); + return SQLITE_OK; +} - if( isPhrase && !isRequirePhrase ){ - /* Insert an implicit AND operator. */ - Fts3Expr *pAnd; - assert( pRet && pPrev ); - pAnd = fts3MallocZero(sizeof(Fts3Expr)); - if( !pAnd ){ - sqlite3Fts3ExprFree(p); - rc = SQLITE_NOMEM; - goto exprparse_out; - } - pAnd->eType = FTSQUERY_AND; - insertBinaryOperator(&pRet, pPrev, pAnd); - pPrev = pAnd; - } +/* +** Extract the next token from a tokenization cursor. The cursor must +** have been opened by a prior call to simpleOpen(). +*/ +static int simpleNext( + sqlite3_tokenizer_cursor *pCursor, /* Cursor returned by simpleOpen */ + const char **ppToken, /* OUT: *ppToken is the token text */ + int *pnBytes, /* OUT: Number of bytes in token */ + int *piStartOffset, /* OUT: Starting offset of token */ + int *piEndOffset, /* OUT: Ending offset of token */ + int *piPosition /* OUT: Position integer of token */ +){ + simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor; + simple_tokenizer *t = (simple_tokenizer *) pCursor->pTokenizer; + unsigned char *p = (unsigned char *)c->pInput; - /* This test catches attempts to make either operand of a NEAR - ** operator something other than a phrase. For example, either of - ** the following: - ** - ** (bracketed expression) NEAR phrase - ** phrase NEAR (bracketed expression) - ** - ** Return an error in either case. - */ - if( pPrev && ( - (eType==FTSQUERY_NEAR && !isPhrase && pPrev->eType!=FTSQUERY_PHRASE) - || (eType!=FTSQUERY_PHRASE && isPhrase && pPrev->eType==FTSQUERY_NEAR) - )){ - sqlite3Fts3ExprFree(p); - rc = SQLITE_ERROR; - goto exprparse_out; - } + while( c->iOffsetnBytes ){ + int iStartOffset; - if( isPhrase ){ - if( pRet ){ - assert( pPrev && pPrev->pLeft && pPrev->pRight==0 ); - pPrev->pRight = p; - p->pParent = pPrev; - }else{ - pRet = p; - } - }else{ - insertBinaryOperator(&pRet, pPrev, p); - } - isRequirePhrase = !isPhrase; - } - pPrev = p; - } - assert( nByte>0 ); + /* Scan past delimiter characters */ + while( c->iOffsetnBytes && simpleDelim(t, p[c->iOffset]) ){ + c->iOffset++; } - assert( rc!=SQLITE_OK || (nByte>0 && nByte<=nIn) ); - nIn -= nByte; - zIn += nByte; - } - if( rc==SQLITE_DONE && pRet && isRequirePhrase ){ - rc = SQLITE_ERROR; - } + /* Count non-delimiter characters. */ + iStartOffset = c->iOffset; + while( c->iOffsetnBytes && !simpleDelim(t, p[c->iOffset]) ){ + c->iOffset++; + } - if( rc==SQLITE_DONE ){ - rc = SQLITE_OK; - if( !sqlite3_fts3_enable_parentheses && pNotBranch ){ - if( !pRet ){ - rc = SQLITE_ERROR; - }else{ - Fts3Expr *pIter = pNotBranch; - while( pIter->pLeft ){ - pIter = pIter->pLeft; - } - pIter->pLeft = pRet; - pRet->pParent = pIter; - pRet = pNotBranch; + if( c->iOffset>iStartOffset ){ + int i, n = c->iOffset-iStartOffset; + if( n>c->nTokenAllocated ){ + char *pNew; + c->nTokenAllocated = n+20; + pNew = sqlite3_realloc(c->pToken, c->nTokenAllocated); + if( !pNew ) return SQLITE_NOMEM; + c->pToken = pNew; } - } - } - *pnConsumed = n - nIn; + for(i=0; ipToken[i] = (char)((ch>='A' && ch<='Z') ? ch-'A'+'a' : ch); + } + *ppToken = c->pToken; + *pnBytes = n; + *piStartOffset = iStartOffset; + *piEndOffset = c->iOffset; + *piPosition = c->iToken++; -exprparse_out: - if( rc!=SQLITE_OK ){ - sqlite3Fts3ExprFree(pRet); - sqlite3Fts3ExprFree(pNotBranch); - pRet = 0; + return SQLITE_OK; + } } - *ppExpr = pRet; - return rc; + return SQLITE_DONE; } /* -** Return SQLITE_ERROR if the maximum depth of the expression tree passed -** as the only argument is more than nMaxDepth. +** The set of routines that implement the simple tokenizer */ -static int fts3ExprCheckDepth(Fts3Expr *p, int nMaxDepth){ - int rc = SQLITE_OK; - if( p ){ - if( nMaxDepth<0 ){ - rc = SQLITE_TOOBIG; - }else{ - rc = fts3ExprCheckDepth(p->pLeft, nMaxDepth-1); - if( rc==SQLITE_OK ){ - rc = fts3ExprCheckDepth(p->pRight, nMaxDepth-1); - } - } - } - return rc; +static const sqlite3_tokenizer_module simpleTokenizerModule = { + 0, + simpleCreate, + simpleDestroy, + simpleOpen, + simpleClose, + simpleNext, + 0, +}; + +/* +** Allocate a new simple tokenizer. Return a pointer to the new +** tokenizer in *ppModule +*/ +SQLITE_PRIVATE void sqlite3Fts3SimpleTokenizerModule( + sqlite3_tokenizer_module const**ppModule +){ + *ppModule = &simpleTokenizerModule; } +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ + +/************** End of fts3_tokenizer1.c *************************************/ +/************** Begin file fts3_tokenize_vtab.c ******************************/ /* -** This function attempts to transform the expression tree at (*pp) to -** an equivalent but more balanced form. The tree is modified in place. -** If successful, SQLITE_OK is returned and (*pp) set to point to the -** new root expression node. +** 2013 Apr 22 ** -** nMaxDepth is the maximum allowable depth of the balanced sub-tree. +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains code for the "fts3tokenize" virtual table module. +** An fts3tokenize virtual table is created as follows: +** +** CREATE VIRTUAL TABLE USING fts3tokenize( +** , , ... +** ); +** +** The table created has the following schema: +** +** CREATE TABLE (input, token, start, end, position) +** +** When queried, the query must include a WHERE clause of type: +** +** input = +** +** The virtual table module tokenizes this , using the FTS3 +** tokenizer specified by the arguments to the CREATE VIRTUAL TABLE +** statement and returns one row for each token in the result. With +** fields set as follows: +** +** input: Always set to a copy of +** token: A token from the input. +** start: Byte offset of the token within the input . +** end: Byte offset of the byte immediately following the end of the +** token within the input string. +** pos: Token offset of token within input. ** -** Otherwise, if an error occurs, an SQLite error code is returned and -** expression (*pp) freed. */ -static int fts3ExprBalance(Fts3Expr **pp, int nMaxDepth){ - int rc = SQLITE_OK; /* Return code */ - Fts3Expr *pRoot = *pp; /* Initial root node */ - Fts3Expr *pFree = 0; /* List of free nodes. Linked by pParent. */ - int eType = pRoot->eType; /* Type of node in this tree */ - - if( nMaxDepth==0 ){ - rc = SQLITE_ERROR; - } - - if( rc==SQLITE_OK ){ - if( (eType==FTSQUERY_AND || eType==FTSQUERY_OR) ){ - Fts3Expr **apLeaf; - apLeaf = (Fts3Expr **)sqlite3_malloc(sizeof(Fts3Expr *) * nMaxDepth); - if( 0==apLeaf ){ - rc = SQLITE_NOMEM; - }else{ - memset(apLeaf, 0, sizeof(Fts3Expr *) * nMaxDepth); - } - - if( rc==SQLITE_OK ){ - int i; - Fts3Expr *p; - - /* Set $p to point to the left-most leaf in the tree of eType nodes. */ - for(p=pRoot; p->eType==eType; p=p->pLeft){ - assert( p->pParent==0 || p->pParent->pLeft==p ); - assert( p->pLeft && p->pRight ); - } - - /* This loop runs once for each leaf in the tree of eType nodes. */ - while( 1 ){ - int iLvl; - Fts3Expr *pParent = p->pParent; /* Current parent of p */ - - assert( pParent==0 || pParent->pLeft==p ); - p->pParent = 0; - if( pParent ){ - pParent->pLeft = 0; - }else{ - pRoot = 0; - } - rc = fts3ExprBalance(&p, nMaxDepth-1); - if( rc!=SQLITE_OK ) break; - - for(iLvl=0; p && iLvlpLeft = apLeaf[iLvl]; - pFree->pRight = p; - pFree->pLeft->pParent = pFree; - pFree->pRight->pParent = pFree; - - p = pFree; - pFree = pFree->pParent; - p->pParent = 0; - apLeaf[iLvl] = 0; - } - } - if( p ){ - sqlite3Fts3ExprFree(p); - rc = SQLITE_TOOBIG; - break; - } +/* #include "fts3Int.h" */ +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) - /* If that was the last leaf node, break out of the loop */ - if( pParent==0 ) break; +/* #include */ +/* #include */ - /* Set $p to point to the next leaf in the tree of eType nodes */ - for(p=pParent->pRight; p->eType==eType; p=p->pLeft); +typedef struct Fts3tokTable Fts3tokTable; +typedef struct Fts3tokCursor Fts3tokCursor; - /* Remove pParent from the original tree. */ - assert( pParent->pParent==0 || pParent->pParent->pLeft==pParent ); - pParent->pRight->pParent = pParent->pParent; - if( pParent->pParent ){ - pParent->pParent->pLeft = pParent->pRight; - }else{ - assert( pParent==pRoot ); - pRoot = pParent->pRight; - } +/* +** Virtual table structure. +*/ +struct Fts3tokTable { + sqlite3_vtab base; /* Base class used by SQLite core */ + const sqlite3_tokenizer_module *pMod; + sqlite3_tokenizer *pTok; +}; - /* Link pParent into the free node list. It will be used as an - ** internal node of the new tree. */ - pParent->pParent = pFree; - pFree = pParent; - } +/* +** Virtual table cursor structure. +*/ +struct Fts3tokCursor { + sqlite3_vtab_cursor base; /* Base class used by SQLite core */ + char *zInput; /* Input string */ + sqlite3_tokenizer_cursor *pCsr; /* Cursor to iterate through zInput */ + int iRowid; /* Current 'rowid' value */ + const char *zToken; /* Current 'token' value */ + int nToken; /* Size of zToken in bytes */ + int iStart; /* Current 'start' value */ + int iEnd; /* Current 'end' value */ + int iPos; /* Current 'pos' value */ +}; - if( rc==SQLITE_OK ){ - p = 0; - for(i=0; ipParent = 0; - }else{ - assert( pFree!=0 ); - pFree->pRight = p; - pFree->pLeft = apLeaf[i]; - pFree->pLeft->pParent = pFree; - pFree->pRight->pParent = pFree; +/* +** Query FTS for the tokenizer implementation named zName. +*/ +static int fts3tokQueryTokenizer( + Fts3Hash *pHash, + const char *zName, + const sqlite3_tokenizer_module **pp, + char **pzErr +){ + sqlite3_tokenizer_module *p; + int nName = (int)strlen(zName); - p = pFree; - pFree = pFree->pParent; - p->pParent = 0; - } - } - } - pRoot = p; - }else{ - /* An error occurred. Delete the contents of the apLeaf[] array - ** and pFree list. Everything else is cleaned up by the call to - ** sqlite3Fts3ExprFree(pRoot) below. */ - Fts3Expr *pDel; - for(i=0; ipParent; - sqlite3_free(pDel); - } - } + p = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash, zName, nName+1); + if( !p ){ + sqlite3Fts3ErrMsg(pzErr, "unknown tokenizer: %s", zName); + return SQLITE_ERROR; + } - assert( pFree==0 ); - sqlite3_free( apLeaf ); - } - }else if( eType==FTSQUERY_NOT ){ - Fts3Expr *pLeft = pRoot->pLeft; - Fts3Expr *pRight = pRoot->pRight; + *pp = p; + return SQLITE_OK; +} - pRoot->pLeft = 0; - pRoot->pRight = 0; - pLeft->pParent = 0; - pRight->pParent = 0; +/* +** The second argument, argv[], is an array of pointers to nul-terminated +** strings. This function makes a copy of the array and strings into a +** single block of memory. It then dequotes any of the strings that appear +** to be quoted. +** +** If successful, output parameter *pazDequote is set to point at the +** array of dequoted strings and SQLITE_OK is returned. The caller is +** responsible for eventually calling sqlite3_free() to free the array +** in this case. Or, if an error occurs, an SQLite error code is returned. +** The final value of *pazDequote is undefined in this case. +*/ +static int fts3tokDequoteArray( + int argc, /* Number of elements in argv[] */ + const char * const *argv, /* Input array */ + char ***pazDequote /* Output array */ +){ + int rc = SQLITE_OK; /* Return code */ + if( argc==0 ){ + *pazDequote = 0; + }else{ + int i; + int nByte = 0; + char **azDequote; - rc = fts3ExprBalance(&pLeft, nMaxDepth-1); - if( rc==SQLITE_OK ){ - rc = fts3ExprBalance(&pRight, nMaxDepth-1); - } + for(i=0; ipLeft = pLeft; - pLeft->pParent = pRoot; - pRoot->pRight = pRight; - pRight->pParent = pRoot; + *pazDequote = azDequote = sqlite3_malloc64(sizeof(char *)*argc + nByte); + if( azDequote==0 ){ + rc = SQLITE_NOMEM; + }else{ + char *pSpace = (char *)&azDequote[argc]; + for(i=0; ixCreate((nDequote>1 ? nDequote-1 : 0), azArg, &pTok); } - - return rc; -} -/* -** Parameters z and n contain a pointer to and length of a buffer containing -** an fts3 query expression, respectively. This function attempts to parse the -** query expression and create a tree of Fts3Expr structures representing the -** parsed expression. If successful, *ppExpr is set to point to the head -** of the parsed expression tree and SQLITE_OK is returned. If an error -** occurs, either SQLITE_NOMEM (out-of-memory error) or SQLITE_ERROR (parse -** error) is returned and *ppExpr is set to 0. -** -** If parameter n is a negative number, then z is assumed to point to a -** nul-terminated string and the length is determined using strlen(). -** -** The first parameter, pTokenizer, is passed the fts3 tokenizer module to -** use to normalize query tokens while parsing the expression. The azCol[] -** array, which is assumed to contain nCol entries, should contain the names -** of each column in the target fts3 table, in order from left to right. -** Column names must be nul-terminated strings. -** -** The iDefaultCol parameter should be passed the index of the table column -** that appears on the left-hand-side of the MATCH operator (the default -** column to match against for tokens for which a column name is not explicitly -** specified as part of the query string), or -1 if tokens may by default -** match any table column. -*/ -SQLITE_PRIVATE int sqlite3Fts3ExprParse( - sqlite3_tokenizer *pTokenizer, /* Tokenizer module */ - int iLangid, /* Language id for tokenizer */ - char **azCol, /* Array of column names for fts3 table */ - int bFts4, /* True to allow FTS4-only syntax */ - int nCol, /* Number of entries in azCol[] */ - int iDefaultCol, /* Default column to query */ - const char *z, int n, /* Text of MATCH query */ - Fts3Expr **ppExpr, /* OUT: Parsed query structure */ - char **pzErr /* OUT: Error message (sqlite3_malloc) */ -){ - int rc = fts3ExprParseUnbalanced( - pTokenizer, iLangid, azCol, bFts4, nCol, iDefaultCol, z, n, ppExpr - ); - - /* Rebalance the expression. And check that its depth does not exceed - ** SQLITE_FTS3_MAX_EXPR_DEPTH. */ - if( rc==SQLITE_OK && *ppExpr ){ - rc = fts3ExprBalance(ppExpr, SQLITE_FTS3_MAX_EXPR_DEPTH); - if( rc==SQLITE_OK ){ - rc = fts3ExprCheckDepth(*ppExpr, SQLITE_FTS3_MAX_EXPR_DEPTH); + if( rc==SQLITE_OK ){ + pTab = (Fts3tokTable *)sqlite3_malloc(sizeof(Fts3tokTable)); + if( pTab==0 ){ + rc = SQLITE_NOMEM; } } - if( rc!=SQLITE_OK ){ - sqlite3Fts3ExprFree(*ppExpr); - *ppExpr = 0; - if( rc==SQLITE_TOOBIG ){ - sqlite3Fts3ErrMsg(pzErr, - "FTS expression tree is too large (maximum depth %d)", - SQLITE_FTS3_MAX_EXPR_DEPTH - ); - rc = SQLITE_ERROR; - }else if( rc==SQLITE_ERROR ){ - sqlite3Fts3ErrMsg(pzErr, "malformed MATCH expression: [%s]", z); + if( rc==SQLITE_OK ){ + memset(pTab, 0, sizeof(Fts3tokTable)); + pTab->pMod = pMod; + pTab->pTok = pTok; + *ppVtab = &pTab->base; + }else{ + if( pTok ){ + pMod->xDestroy(pTok); } } + sqlite3_free(azDequote); return rc; } /* -** Free a single node of an expression tree. +** This function does the work for both the xDisconnect and xDestroy methods. +** These tables have no persistent representation of their own, so xDisconnect +** and xDestroy are identical operations. */ -static void fts3FreeExprNode(Fts3Expr *p){ - assert( p->eType==FTSQUERY_PHRASE || p->pPhrase==0 ); - sqlite3Fts3EvalPhraseCleanup(p->pPhrase); - sqlite3_free(p->aMI); - sqlite3_free(p); +static int fts3tokDisconnectMethod(sqlite3_vtab *pVtab){ + Fts3tokTable *pTab = (Fts3tokTable *)pVtab; + + pTab->pMod->xDestroy(pTab->pTok); + sqlite3_free(pTab); + return SQLITE_OK; } /* -** Free a parsed fts3 query expression allocated by sqlite3Fts3ExprParse(). -** -** This function would be simpler if it recursively called itself. But -** that would mean passing a sufficiently large expression to ExprParse() -** could cause a stack overflow. +** xBestIndex - Analyze a WHERE and ORDER BY clause. */ -SQLITE_PRIVATE void sqlite3Fts3ExprFree(Fts3Expr *pDel){ - Fts3Expr *p; - assert( pDel==0 || pDel->pParent==0 ); - for(p=pDel; p && (p->pLeft||p->pRight); p=(p->pLeft ? p->pLeft : p->pRight)){ - assert( p->pParent==0 || p==p->pParent->pRight || p==p->pParent->pLeft ); - } - while( p ){ - Fts3Expr *pParent = p->pParent; - fts3FreeExprNode(p); - if( pParent && p==pParent->pLeft && pParent->pRight ){ - p = pParent->pRight; - while( p && (p->pLeft || p->pRight) ){ - assert( p==p->pParent->pRight || p==p->pParent->pLeft ); - p = (p->pLeft ? p->pLeft : p->pRight); - } - }else{ - p = pParent; +static int fts3tokBestIndexMethod( + sqlite3_vtab *pVTab, + sqlite3_index_info *pInfo +){ + int i; + UNUSED_PARAMETER(pVTab); + + for(i=0; inConstraint; i++){ + if( pInfo->aConstraint[i].usable + && pInfo->aConstraint[i].iColumn==0 + && pInfo->aConstraint[i].op==SQLITE_INDEX_CONSTRAINT_EQ + ){ + pInfo->idxNum = 1; + pInfo->aConstraintUsage[i].argvIndex = 1; + pInfo->aConstraintUsage[i].omit = 1; + pInfo->estimatedCost = 1; + return SQLITE_OK; } } + + pInfo->idxNum = 0; + assert( pInfo->estimatedCost>1000000.0 ); + + return SQLITE_OK; } -/**************************************************************************** -***************************************************************************** -** Everything after this point is just test code. +/* +** xOpen - Open a cursor. */ +static int fts3tokOpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){ + Fts3tokCursor *pCsr; + UNUSED_PARAMETER(pVTab); -#ifdef SQLITE_TEST + pCsr = (Fts3tokCursor *)sqlite3_malloc(sizeof(Fts3tokCursor)); + if( pCsr==0 ){ + return SQLITE_NOMEM; + } + memset(pCsr, 0, sizeof(Fts3tokCursor)); -/* #include */ + *ppCsr = (sqlite3_vtab_cursor *)pCsr; + return SQLITE_OK; +} /* -** Return a pointer to a buffer containing a text representation of the -** expression passed as the first argument. The buffer is obtained from -** sqlite3_malloc(). It is the responsibility of the caller to use -** sqlite3_free() to release the memory. If an OOM condition is encountered, -** NULL is returned. -** -** If the second argument is not NULL, then its contents are prepended to -** the returned expression text and then freed using sqlite3_free(). +** Reset the tokenizer cursor passed as the only argument. As if it had +** just been returned by fts3tokOpenMethod(). */ -static char *exprToString(Fts3Expr *pExpr, char *zBuf){ - if( pExpr==0 ){ - return sqlite3_mprintf(""); - } - switch( pExpr->eType ){ - case FTSQUERY_PHRASE: { - Fts3Phrase *pPhrase = pExpr->pPhrase; - int i; - zBuf = sqlite3_mprintf( - "%zPHRASE %d 0", zBuf, pPhrase->iColumn); - for(i=0; zBuf && inToken; i++){ - zBuf = sqlite3_mprintf("%z %.*s%s", zBuf, - pPhrase->aToken[i].n, pPhrase->aToken[i].z, - (pPhrase->aToken[i].isPrefix?"+":"") - ); - } - return zBuf; - } - - case FTSQUERY_NEAR: - zBuf = sqlite3_mprintf("%zNEAR/%d ", zBuf, pExpr->nNear); - break; - case FTSQUERY_NOT: - zBuf = sqlite3_mprintf("%zNOT ", zBuf); - break; - case FTSQUERY_AND: - zBuf = sqlite3_mprintf("%zAND ", zBuf); - break; - case FTSQUERY_OR: - zBuf = sqlite3_mprintf("%zOR ", zBuf); - break; +static void fts3tokResetCursor(Fts3tokCursor *pCsr){ + if( pCsr->pCsr ){ + Fts3tokTable *pTab = (Fts3tokTable *)(pCsr->base.pVtab); + pTab->pMod->xClose(pCsr->pCsr); + pCsr->pCsr = 0; } + sqlite3_free(pCsr->zInput); + pCsr->zInput = 0; + pCsr->zToken = 0; + pCsr->nToken = 0; + pCsr->iStart = 0; + pCsr->iEnd = 0; + pCsr->iPos = 0; + pCsr->iRowid = 0; +} - if( zBuf ) zBuf = sqlite3_mprintf("%z{", zBuf); - if( zBuf ) zBuf = exprToString(pExpr->pLeft, zBuf); - if( zBuf ) zBuf = sqlite3_mprintf("%z} {", zBuf); - - if( zBuf ) zBuf = exprToString(pExpr->pRight, zBuf); - if( zBuf ) zBuf = sqlite3_mprintf("%z}", zBuf); +/* +** xClose - Close a cursor. +*/ +static int fts3tokCloseMethod(sqlite3_vtab_cursor *pCursor){ + Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor; - return zBuf; + fts3tokResetCursor(pCsr); + sqlite3_free(pCsr); + return SQLITE_OK; } /* -** This is the implementation of a scalar SQL function used to test the -** expression parser. It should be called as follows: -** -** fts3_exprtest(, , , ...); -** -** The first argument, , is the name of the fts3 tokenizer used -** to parse the query expression (see README.tokenizers). The second argument -** is the query expression to parse. Each subsequent argument is the name -** of a column of the fts3 table that the query expression may refer to. -** For example: -** -** SELECT fts3_exprtest('simple', 'Bill col2:Bloggs', 'col1', 'col2'); +** xNext - Advance the cursor to the next row, if any. */ -static void fts3ExprTestCommon( - int bRebalance, - sqlite3_context *context, - int argc, - sqlite3_value **argv -){ - sqlite3_tokenizer *pTokenizer = 0; - int rc; - char **azCol = 0; - const char *zExpr; - int nExpr; - int nCol; - int ii; - Fts3Expr *pExpr; - char *zBuf = 0; - Fts3Hash *pHash = (Fts3Hash*)sqlite3_user_data(context); - const char *zTokenizer = 0; - char *zErr = 0; +static int fts3tokNextMethod(sqlite3_vtab_cursor *pCursor){ + Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor; + Fts3tokTable *pTab = (Fts3tokTable *)(pCursor->pVtab); + int rc; /* Return code */ - if( argc<3 ){ - sqlite3_result_error(context, - "Usage: fts3_exprtest(tokenizer, expr, col1, ...", -1 - ); - return; - } + pCsr->iRowid++; + rc = pTab->pMod->xNext(pCsr->pCsr, + &pCsr->zToken, &pCsr->nToken, + &pCsr->iStart, &pCsr->iEnd, &pCsr->iPos + ); - zTokenizer = (const char*)sqlite3_value_text(argv[0]); - rc = sqlite3Fts3InitTokenizer(pHash, zTokenizer, &pTokenizer, &zErr); if( rc!=SQLITE_OK ){ - if( rc==SQLITE_NOMEM ){ - sqlite3_result_error_nomem(context); - }else{ - sqlite3_result_error(context, zErr, -1); - } - sqlite3_free(zErr); - return; - } - - zExpr = (const char *)sqlite3_value_text(argv[1]); - nExpr = sqlite3_value_bytes(argv[1]); - nCol = argc-2; - azCol = (char **)sqlite3_malloc(nCol*sizeof(char *)); - if( !azCol ){ - sqlite3_result_error_nomem(context); - goto exprtest_out; - } - for(ii=0; iipModule->xDestroy(pTokenizer); +/* +** xFilter - Initialize a cursor to point at the start of its data. +*/ +static int fts3tokFilterMethod( + sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */ + int idxNum, /* Strategy index */ + const char *idxStr, /* Unused */ + int nVal, /* Number of elements in apVal */ + sqlite3_value **apVal /* Arguments for the indexing scheme */ +){ + int rc = SQLITE_ERROR; + Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor; + Fts3tokTable *pTab = (Fts3tokTable *)(pCursor->pVtab); + UNUSED_PARAMETER(idxStr); + UNUSED_PARAMETER(nVal); + + fts3tokResetCursor(pCsr); + if( idxNum==1 ){ + const char *zByte = (const char *)sqlite3_value_text(apVal[0]); + int nByte = sqlite3_value_bytes(apVal[0]); + pCsr->zInput = sqlite3_malloc64(nByte+1); + if( pCsr->zInput==0 ){ + rc = SQLITE_NOMEM; + }else{ + if( nByte>0 ) memcpy(pCsr->zInput, zByte, nByte); + pCsr->zInput[nByte] = 0; + rc = pTab->pMod->xOpen(pTab->pTok, pCsr->zInput, nByte, &pCsr->pCsr); + if( rc==SQLITE_OK ){ + pCsr->pCsr->pTokenizer = pTab->pTok; + } + } } - sqlite3_free(azCol); + + if( rc!=SQLITE_OK ) return rc; + return fts3tokNextMethod(pCursor); } -static void fts3ExprTest( - sqlite3_context *context, - int argc, - sqlite3_value **argv +/* +** xEof - Return true if the cursor is at EOF, or false otherwise. +*/ +static int fts3tokEofMethod(sqlite3_vtab_cursor *pCursor){ + Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor; + return (pCsr->zToken==0); +} + +/* +** xColumn - Return a column value. +*/ +static int fts3tokColumnMethod( + sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ + sqlite3_context *pCtx, /* Context for sqlite3_result_xxx() calls */ + int iCol /* Index of column to read value from */ ){ - fts3ExprTestCommon(0, context, argc, argv); + Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor; + + /* CREATE TABLE x(input, token, start, end, position) */ + switch( iCol ){ + case 0: + sqlite3_result_text(pCtx, pCsr->zInput, -1, SQLITE_TRANSIENT); + break; + case 1: + sqlite3_result_text(pCtx, pCsr->zToken, pCsr->nToken, SQLITE_TRANSIENT); + break; + case 2: + sqlite3_result_int(pCtx, pCsr->iStart); + break; + case 3: + sqlite3_result_int(pCtx, pCsr->iEnd); + break; + default: + assert( iCol==4 ); + sqlite3_result_int(pCtx, pCsr->iPos); + break; + } + return SQLITE_OK; } -static void fts3ExprTestRebalance( - sqlite3_context *context, - int argc, - sqlite3_value **argv + +/* +** xRowid - Return the current rowid for the cursor. +*/ +static int fts3tokRowidMethod( + sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ + sqlite_int64 *pRowid /* OUT: Rowid value */ ){ - fts3ExprTestCommon(1, context, argc, argv); + Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor; + *pRowid = (sqlite3_int64)pCsr->iRowid; + return SQLITE_OK; } /* -** Register the query expression parser test function fts3_exprtest() -** with database connection db. +** Register the fts3tok module with database connection db. Return SQLITE_OK +** if successful or an error code if sqlite3_create_module() fails. */ -SQLITE_PRIVATE int sqlite3Fts3ExprInitTestInterface(sqlite3 *db, Fts3Hash *pHash){ - int rc = sqlite3_create_function( - db, "fts3_exprtest", -1, SQLITE_UTF8, (void*)pHash, fts3ExprTest, 0, 0 - ); - if( rc==SQLITE_OK ){ - rc = sqlite3_create_function(db, "fts3_exprtest_rebalance", - -1, SQLITE_UTF8, (void*)pHash, fts3ExprTestRebalance, 0, 0 - ); - } +SQLITE_PRIVATE int sqlite3Fts3InitTok(sqlite3 *db, Fts3Hash *pHash){ + static const sqlite3_module fts3tok_module = { + 0, /* iVersion */ + fts3tokConnectMethod, /* xCreate */ + fts3tokConnectMethod, /* xConnect */ + fts3tokBestIndexMethod, /* xBestIndex */ + fts3tokDisconnectMethod, /* xDisconnect */ + fts3tokDisconnectMethod, /* xDestroy */ + fts3tokOpenMethod, /* xOpen */ + fts3tokCloseMethod, /* xClose */ + fts3tokFilterMethod, /* xFilter */ + fts3tokNextMethod, /* xNext */ + fts3tokEofMethod, /* xEof */ + fts3tokColumnMethod, /* xColumn */ + fts3tokRowidMethod, /* xRowid */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindFunction */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0, /* xRollbackTo */ + 0 /* xShadowName */ + }; + int rc; /* Return code */ + + rc = sqlite3_create_module(db, "fts3tokenize", &fts3tok_module, (void*)pHash); return rc; } -#endif #endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ -/************** End of fts3_expr.c *******************************************/ -/************** Begin file fts3_hash.c ***************************************/ +/************** End of fts3_tokenize_vtab.c **********************************/ +/************** Begin file fts3_write.c **************************************/ /* -** 2001 September 22 +** 2009 Oct 23 ** ** The author disclaims copyright to this source code. In place of ** a legal notice, here is a blessing: @@ -160105,7950 +176246,8319 @@ SQLITE_PRIVATE int sqlite3Fts3ExprInitTestInterface(sqlite3 *db, Fts3Hash *pHash ** May you find forgiveness for yourself and forgive others. ** May you share freely, never taking more than you give. ** -************************************************************************* -** This is the implementation of generic hash-tables used in SQLite. -** We've modified it slightly to serve as a standalone hash table -** implementation for the full-text indexing module. -*/ - -/* -** The code in this file is only compiled if: -** -** * The FTS3 module is being built as an extension -** (in which case SQLITE_CORE is not defined), or +****************************************************************************** ** -** * The FTS3 module is being built into the core of -** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). +** This file is part of the SQLite FTS3 extension module. Specifically, +** this file contains code to insert, update and delete rows from FTS3 +** tables. It also contains code to merge FTS3 b-tree segments. Some +** of the sub-routines used to merge segments are also used by the query +** code in fts3.c. */ + /* #include "fts3Int.h" */ #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) +/* #include */ /* #include */ /* #include */ -/* #include */ +/* #include */ -/* #include "fts3_hash.h" */ +#define FTS_MAX_APPENDABLE_HEIGHT 16 /* -** Malloc and Free functions +** When full-text index nodes are loaded from disk, the buffer that they +** are loaded into has the following number of bytes of padding at the end +** of it. i.e. if a full-text index node is 900 bytes in size, then a buffer +** of 920 bytes is allocated for it. +** +** This means that if we have a pointer into a buffer containing node data, +** it is always safe to read up to two varints from it without risking an +** overread, even if the node data is corrupted. */ -static void *fts3HashMalloc(int n){ - void *p = sqlite3_malloc(n); - if( p ){ - memset(p, 0, n); - } - return p; -} -static void fts3HashFree(void *p){ - sqlite3_free(p); -} +#define FTS3_NODE_PADDING (FTS3_VARINT_MAX*2) -/* Turn bulk memory into a hash table object by initializing the -** fields of the Hash structure. +/* +** Under certain circumstances, b-tree nodes (doclists) can be loaded into +** memory incrementally instead of all at once. This can be a big performance +** win (reduced IO and CPU) if SQLite stops calling the virtual table xNext() +** method before retrieving all query results (as may happen, for example, +** if a query has a LIMIT clause). ** -** "pNew" is a pointer to the hash table that is to be initialized. -** keyClass is one of the constants -** FTS3_HASH_BINARY or FTS3_HASH_STRING. The value of keyClass -** determines what kind of key the hash table will use. "copyKey" is -** true if the hash table should make its own private copy of keys and -** false if it should just use the supplied pointer. +** Incremental loading is used for b-tree nodes FTS3_NODE_CHUNK_THRESHOLD +** bytes and larger. Nodes are loaded in chunks of FTS3_NODE_CHUNKSIZE bytes. +** The code is written so that the hard lower-limit for each of these values +** is 1. Clearly such small values would be inefficient, but can be useful +** for testing purposes. +** +** If this module is built with SQLITE_TEST defined, these constants may +** be overridden at runtime for testing purposes. File fts3_test.c contains +** a Tcl interface to read and write the values. */ -SQLITE_PRIVATE void sqlite3Fts3HashInit(Fts3Hash *pNew, char keyClass, char copyKey){ - assert( pNew!=0 ); - assert( keyClass>=FTS3_HASH_STRING && keyClass<=FTS3_HASH_BINARY ); - pNew->keyClass = keyClass; - pNew->copyKey = copyKey; - pNew->first = 0; - pNew->count = 0; - pNew->htsize = 0; - pNew->ht = 0; -} +#ifdef SQLITE_TEST +int test_fts3_node_chunksize = (4*1024); +int test_fts3_node_chunk_threshold = (4*1024)*4; +# define FTS3_NODE_CHUNKSIZE test_fts3_node_chunksize +# define FTS3_NODE_CHUNK_THRESHOLD test_fts3_node_chunk_threshold +#else +# define FTS3_NODE_CHUNKSIZE (4*1024) +# define FTS3_NODE_CHUNK_THRESHOLD (FTS3_NODE_CHUNKSIZE*4) +#endif -/* Remove all entries from a hash table. Reclaim all memory. -** Call this routine to delete a hash table or to reset a hash table -** to the empty state. +/* +** The values that may be meaningfully bound to the :1 parameter in +** statements SQL_REPLACE_STAT and SQL_SELECT_STAT. */ -SQLITE_PRIVATE void sqlite3Fts3HashClear(Fts3Hash *pH){ - Fts3HashElem *elem; /* For looping over all elements of the table */ +#define FTS_STAT_DOCTOTAL 0 +#define FTS_STAT_INCRMERGEHINT 1 +#define FTS_STAT_AUTOINCRMERGE 2 - assert( pH!=0 ); - elem = pH->first; - pH->first = 0; - fts3HashFree(pH->ht); - pH->ht = 0; - pH->htsize = 0; - while( elem ){ - Fts3HashElem *next_elem = elem->next; - if( pH->copyKey && elem->pKey ){ - fts3HashFree(elem->pKey); - } - fts3HashFree(elem); - elem = next_elem; - } - pH->count = 0; +/* +** If FTS_LOG_MERGES is defined, call sqlite3_log() to report each automatic +** and incremental merge operation that takes place. This is used for +** debugging FTS only, it should not usually be turned on in production +** systems. +*/ +#ifdef FTS3_LOG_MERGES +static void fts3LogMerge(int nMerge, sqlite3_int64 iAbsLevel){ + sqlite3_log(SQLITE_OK, "%d-way merge from level %d", nMerge, (int)iAbsLevel); } +#else +#define fts3LogMerge(x, y) +#endif + + +typedef struct PendingList PendingList; +typedef struct SegmentNode SegmentNode; +typedef struct SegmentWriter SegmentWriter; /* -** Hash and comparison functions when the mode is FTS3_HASH_STRING +** An instance of the following data structure is used to build doclists +** incrementally. See function fts3PendingListAppend() for details. */ -static int fts3StrHash(const void *pKey, int nKey){ - const char *z = (const char *)pKey; - unsigned h = 0; - if( nKey<=0 ) nKey = (int) strlen(z); - while( nKey > 0 ){ - h = (h<<3) ^ h ^ *z++; - nKey--; - } - return (int)(h & 0x7fffffff); -} -static int fts3StrCompare(const void *pKey1, int n1, const void *pKey2, int n2){ - if( n1!=n2 ) return 1; - return strncmp((const char*)pKey1,(const char*)pKey2,n1); -} +struct PendingList { + int nData; + char *aData; + int nSpace; + sqlite3_int64 iLastDocid; + sqlite3_int64 iLastCol; + sqlite3_int64 iLastPos; +}; + /* -** Hash and comparison functions when the mode is FTS3_HASH_BINARY +** Each cursor has a (possibly empty) linked list of the following objects. */ -static int fts3BinHash(const void *pKey, int nKey){ - int h = 0; - const char *z = (const char *)pKey; - while( nKey-- > 0 ){ - h = (h<<3) ^ h ^ *(z++); - } - return h & 0x7fffffff; -} -static int fts3BinCompare(const void *pKey1, int n1, const void *pKey2, int n2){ - if( n1!=n2 ) return 1; - return memcmp(pKey1,pKey2,n1); -} +struct Fts3DeferredToken { + Fts3PhraseToken *pToken; /* Pointer to corresponding expr token */ + int iCol; /* Column token must occur in */ + Fts3DeferredToken *pNext; /* Next in list of deferred tokens */ + PendingList *pList; /* Doclist is assembled here */ +}; /* -** Return a pointer to the appropriate hash function given the key class. +** An instance of this structure is used to iterate through the terms on +** a contiguous set of segment b-tree leaf nodes. Although the details of +** this structure are only manipulated by code in this file, opaque handles +** of type Fts3SegReader* are also used by code in fts3.c to iterate through +** terms when querying the full-text index. See functions: ** -** The C syntax in this function definition may be unfamilar to some -** programmers, so we provide the following additional explanation: +** sqlite3Fts3SegReaderNew() +** sqlite3Fts3SegReaderFree() +** sqlite3Fts3SegReaderIterate() ** -** The name of the function is "ftsHashFunction". The function takes a -** single parameter "keyClass". The return value of ftsHashFunction() -** is a pointer to another function. Specifically, the return value -** of ftsHashFunction() is a pointer to a function that takes two parameters -** with types "const void*" and "int" and returns an "int". +** Methods used to manipulate Fts3SegReader structures: +** +** fts3SegReaderNext() +** fts3SegReaderFirstDocid() +** fts3SegReaderNextDocid() */ -static int (*ftsHashFunction(int keyClass))(const void*,int){ - if( keyClass==FTS3_HASH_STRING ){ - return &fts3StrHash; - }else{ - assert( keyClass==FTS3_HASH_BINARY ); - return &fts3BinHash; - } -} +struct Fts3SegReader { + int iIdx; /* Index within level, or 0x7FFFFFFF for PT */ + u8 bLookup; /* True for a lookup only */ + u8 rootOnly; /* True for a root-only reader */ + + sqlite3_int64 iStartBlock; /* Rowid of first leaf block to traverse */ + sqlite3_int64 iLeafEndBlock; /* Rowid of final leaf block to traverse */ + sqlite3_int64 iEndBlock; /* Rowid of final block in segment (or 0) */ + sqlite3_int64 iCurrentBlock; /* Current leaf block (or 0) */ + + char *aNode; /* Pointer to node data (or NULL) */ + int nNode; /* Size of buffer at aNode (or 0) */ + int nPopulate; /* If >0, bytes of buffer aNode[] loaded */ + sqlite3_blob *pBlob; /* If not NULL, blob handle to read node */ + + Fts3HashElem **ppNextElem; + + /* Variables set by fts3SegReaderNext(). These may be read directly + ** by the caller. They are valid from the time SegmentReaderNew() returns + ** until SegmentReaderNext() returns something other than SQLITE_OK + ** (i.e. SQLITE_DONE). + */ + int nTerm; /* Number of bytes in current term */ + char *zTerm; /* Pointer to current term */ + int nTermAlloc; /* Allocated size of zTerm buffer */ + char *aDoclist; /* Pointer to doclist of current entry */ + int nDoclist; /* Size of doclist in current entry */ + + /* The following variables are used by fts3SegReaderNextDocid() to iterate + ** through the current doclist (aDoclist/nDoclist). + */ + char *pOffsetList; + int nOffsetList; /* For descending pending seg-readers only */ + sqlite3_int64 iDocid; +}; + +#define fts3SegReaderIsPending(p) ((p)->ppNextElem!=0) +#define fts3SegReaderIsRootOnly(p) ((p)->rootOnly!=0) /* -** Return a pointer to the appropriate hash function given the key class. +** An instance of this structure is used to create a segment b-tree in the +** database. The internal details of this type are only accessed by the +** following functions: ** -** For help in interpreted the obscure C code in the function definition, -** see the header comment on the previous function. +** fts3SegWriterAdd() +** fts3SegWriterFlush() +** fts3SegWriterFree() */ -static int (*ftsCompareFunction(int keyClass))(const void*,int,const void*,int){ - if( keyClass==FTS3_HASH_STRING ){ - return &fts3StrCompare; - }else{ - assert( keyClass==FTS3_HASH_BINARY ); - return &fts3BinCompare; - } -} +struct SegmentWriter { + SegmentNode *pTree; /* Pointer to interior tree structure */ + sqlite3_int64 iFirst; /* First slot in %_segments written */ + sqlite3_int64 iFree; /* Next free slot in %_segments */ + char *zTerm; /* Pointer to previous term buffer */ + int nTerm; /* Number of bytes in zTerm */ + int nMalloc; /* Size of malloc'd buffer at zMalloc */ + char *zMalloc; /* Malloc'd space (possibly) used for zTerm */ + int nSize; /* Size of allocation at aData */ + int nData; /* Bytes of data in aData */ + char *aData; /* Pointer to block from malloc() */ + i64 nLeafData; /* Number of bytes of leaf data written */ +}; -/* Link an element into the hash table +/* +** Type SegmentNode is used by the following three functions to create +** the interior part of the segment b+-tree structures (everything except +** the leaf nodes). These functions and type are only ever used by code +** within the fts3SegWriterXXX() family of functions described above. +** +** fts3NodeAddTerm() +** fts3NodeWrite() +** fts3NodeFree() +** +** When a b+tree is written to the database (either as a result of a merge +** or the pending-terms table being flushed), leaves are written into the +** database file as soon as they are completely populated. The interior of +** the tree is assembled in memory and written out only once all leaves have +** been populated and stored. This is Ok, as the b+-tree fanout is usually +** very large, meaning that the interior of the tree consumes relatively +** little memory. */ -static void fts3HashInsertElement( - Fts3Hash *pH, /* The complete hash table */ - struct _fts3ht *pEntry, /* The entry into which pNew is inserted */ - Fts3HashElem *pNew /* The element to be inserted */ -){ - Fts3HashElem *pHead; /* First element already in pEntry */ - pHead = pEntry->chain; - if( pHead ){ - pNew->next = pHead; - pNew->prev = pHead->prev; - if( pHead->prev ){ pHead->prev->next = pNew; } - else { pH->first = pNew; } - pHead->prev = pNew; - }else{ - pNew->next = pH->first; - if( pH->first ){ pH->first->prev = pNew; } - pNew->prev = 0; - pH->first = pNew; - } - pEntry->count++; - pEntry->chain = pNew; -} +struct SegmentNode { + SegmentNode *pParent; /* Parent node (or NULL for root node) */ + SegmentNode *pRight; /* Pointer to right-sibling */ + SegmentNode *pLeftmost; /* Pointer to left-most node of this depth */ + int nEntry; /* Number of terms written to node so far */ + char *zTerm; /* Pointer to previous term buffer */ + int nTerm; /* Number of bytes in zTerm */ + int nMalloc; /* Size of malloc'd buffer at zMalloc */ + char *zMalloc; /* Malloc'd space (possibly) used for zTerm */ + int nData; /* Bytes of valid data so far */ + char *aData; /* Node data */ +}; + +/* +** Valid values for the second argument to fts3SqlStmt(). +*/ +#define SQL_DELETE_CONTENT 0 +#define SQL_IS_EMPTY 1 +#define SQL_DELETE_ALL_CONTENT 2 +#define SQL_DELETE_ALL_SEGMENTS 3 +#define SQL_DELETE_ALL_SEGDIR 4 +#define SQL_DELETE_ALL_DOCSIZE 5 +#define SQL_DELETE_ALL_STAT 6 +#define SQL_SELECT_CONTENT_BY_ROWID 7 +#define SQL_NEXT_SEGMENT_INDEX 8 +#define SQL_INSERT_SEGMENTS 9 +#define SQL_NEXT_SEGMENTS_ID 10 +#define SQL_INSERT_SEGDIR 11 +#define SQL_SELECT_LEVEL 12 +#define SQL_SELECT_LEVEL_RANGE 13 +#define SQL_SELECT_LEVEL_COUNT 14 +#define SQL_SELECT_SEGDIR_MAX_LEVEL 15 +#define SQL_DELETE_SEGDIR_LEVEL 16 +#define SQL_DELETE_SEGMENTS_RANGE 17 +#define SQL_CONTENT_INSERT 18 +#define SQL_DELETE_DOCSIZE 19 +#define SQL_REPLACE_DOCSIZE 20 +#define SQL_SELECT_DOCSIZE 21 +#define SQL_SELECT_STAT 22 +#define SQL_REPLACE_STAT 23 +#define SQL_SELECT_ALL_PREFIX_LEVEL 24 +#define SQL_DELETE_ALL_TERMS_SEGDIR 25 +#define SQL_DELETE_SEGDIR_RANGE 26 +#define SQL_SELECT_ALL_LANGID 27 +#define SQL_FIND_MERGE_LEVEL 28 +#define SQL_MAX_LEAF_NODE_ESTIMATE 29 +#define SQL_DELETE_SEGDIR_ENTRY 30 +#define SQL_SHIFT_SEGDIR_ENTRY 31 +#define SQL_SELECT_SEGDIR 32 +#define SQL_CHOMP_SEGDIR 33 +#define SQL_SEGMENT_IS_APPENDABLE 34 +#define SQL_SELECT_INDEXES 35 +#define SQL_SELECT_MXLEVEL 36 -/* Resize the hash table so that it cantains "new_size" buckets. -** "new_size" must be a power of 2. The hash table might fail -** to resize if sqliteMalloc() fails. +#define SQL_SELECT_LEVEL_RANGE2 37 +#define SQL_UPDATE_LEVEL_IDX 38 +#define SQL_UPDATE_LEVEL 39 + +/* +** This function is used to obtain an SQLite prepared statement handle +** for the statement identified by the second argument. If successful, +** *pp is set to the requested statement handle and SQLITE_OK returned. +** Otherwise, an SQLite error code is returned and *pp is set to 0. ** -** Return non-zero if a memory allocation error occurs. +** If argument apVal is not NULL, then it must point to an array with +** at least as many entries as the requested statement has bound +** parameters. The values are bound to the statements parameters before +** returning. */ -static int fts3Rehash(Fts3Hash *pH, int new_size){ - struct _fts3ht *new_ht; /* The new hash table */ - Fts3HashElem *elem, *next_elem; /* For looping over existing elements */ - int (*xHash)(const void*,int); /* The hash function */ +static int fts3SqlStmt( + Fts3Table *p, /* Virtual table handle */ + int eStmt, /* One of the SQL_XXX constants above */ + sqlite3_stmt **pp, /* OUT: Statement handle */ + sqlite3_value **apVal /* Values to bind to statement */ +){ + const char *azSql[] = { +/* 0 */ "DELETE FROM %Q.'%q_content' WHERE rowid = ?", +/* 1 */ "SELECT NOT EXISTS(SELECT docid FROM %Q.'%q_content' WHERE rowid!=?)", +/* 2 */ "DELETE FROM %Q.'%q_content'", +/* 3 */ "DELETE FROM %Q.'%q_segments'", +/* 4 */ "DELETE FROM %Q.'%q_segdir'", +/* 5 */ "DELETE FROM %Q.'%q_docsize'", +/* 6 */ "DELETE FROM %Q.'%q_stat'", +/* 7 */ "SELECT %s WHERE rowid=?", +/* 8 */ "SELECT (SELECT max(idx) FROM %Q.'%q_segdir' WHERE level = ?) + 1", +/* 9 */ "REPLACE INTO %Q.'%q_segments'(blockid, block) VALUES(?, ?)", +/* 10 */ "SELECT coalesce((SELECT max(blockid) FROM %Q.'%q_segments') + 1, 1)", +/* 11 */ "REPLACE INTO %Q.'%q_segdir' VALUES(?,?,?,?,?,?)", - assert( (new_size & (new_size-1))==0 ); - new_ht = (struct _fts3ht *)fts3HashMalloc( new_size*sizeof(struct _fts3ht) ); - if( new_ht==0 ) return 1; - fts3HashFree(pH->ht); - pH->ht = new_ht; - pH->htsize = new_size; - xHash = ftsHashFunction(pH->keyClass); - for(elem=pH->first, pH->first=0; elem; elem = next_elem){ - int h = (*xHash)(elem->pKey, elem->nKey) & (new_size-1); - next_elem = elem->next; - fts3HashInsertElement(pH, &new_ht[h], elem); + /* Return segments in order from oldest to newest.*/ +/* 12 */ "SELECT idx, start_block, leaves_end_block, end_block, root " + "FROM %Q.'%q_segdir' WHERE level = ? ORDER BY idx ASC", +/* 13 */ "SELECT idx, start_block, leaves_end_block, end_block, root " + "FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?" + "ORDER BY level DESC, idx ASC", + +/* 14 */ "SELECT count(*) FROM %Q.'%q_segdir' WHERE level = ?", +/* 15 */ "SELECT max(level) FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?", + +/* 16 */ "DELETE FROM %Q.'%q_segdir' WHERE level = ?", +/* 17 */ "DELETE FROM %Q.'%q_segments' WHERE blockid BETWEEN ? AND ?", +/* 18 */ "INSERT INTO %Q.'%q_content' VALUES(%s)", +/* 19 */ "DELETE FROM %Q.'%q_docsize' WHERE docid = ?", +/* 20 */ "REPLACE INTO %Q.'%q_docsize' VALUES(?,?)", +/* 21 */ "SELECT size FROM %Q.'%q_docsize' WHERE docid=?", +/* 22 */ "SELECT value FROM %Q.'%q_stat' WHERE id=?", +/* 23 */ "REPLACE INTO %Q.'%q_stat' VALUES(?,?)", +/* 24 */ "", +/* 25 */ "", + +/* 26 */ "DELETE FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?", +/* 27 */ "SELECT ? UNION SELECT level / (1024 * ?) FROM %Q.'%q_segdir'", + +/* This statement is used to determine which level to read the input from +** when performing an incremental merge. It returns the absolute level number +** of the oldest level in the db that contains at least ? segments. Or, +** if no level in the FTS index contains more than ? segments, the statement +** returns zero rows. */ +/* 28 */ "SELECT level, count(*) AS cnt FROM %Q.'%q_segdir' " + " GROUP BY level HAVING cnt>=?" + " ORDER BY (level %% 1024) ASC, 2 DESC LIMIT 1", + +/* Estimate the upper limit on the number of leaf nodes in a new segment +** created by merging the oldest :2 segments from absolute level :1. See +** function sqlite3Fts3Incrmerge() for details. */ +/* 29 */ "SELECT 2 * total(1 + leaves_end_block - start_block) " + " FROM %Q.'%q_segdir' WHERE level = ? AND idx < ?", + +/* SQL_DELETE_SEGDIR_ENTRY +** Delete the %_segdir entry on absolute level :1 with index :2. */ +/* 30 */ "DELETE FROM %Q.'%q_segdir' WHERE level = ? AND idx = ?", + +/* SQL_SHIFT_SEGDIR_ENTRY +** Modify the idx value for the segment with idx=:3 on absolute level :2 +** to :1. */ +/* 31 */ "UPDATE %Q.'%q_segdir' SET idx = ? WHERE level=? AND idx=?", + +/* SQL_SELECT_SEGDIR +** Read a single entry from the %_segdir table. The entry from absolute +** level :1 with index value :2. */ +/* 32 */ "SELECT idx, start_block, leaves_end_block, end_block, root " + "FROM %Q.'%q_segdir' WHERE level = ? AND idx = ?", + +/* SQL_CHOMP_SEGDIR +** Update the start_block (:1) and root (:2) fields of the %_segdir +** entry located on absolute level :3 with index :4. */ +/* 33 */ "UPDATE %Q.'%q_segdir' SET start_block = ?, root = ?" + "WHERE level = ? AND idx = ?", + +/* SQL_SEGMENT_IS_APPENDABLE +** Return a single row if the segment with end_block=? is appendable. Or +** no rows otherwise. */ +/* 34 */ "SELECT 1 FROM %Q.'%q_segments' WHERE blockid=? AND block IS NULL", + +/* SQL_SELECT_INDEXES +** Return the list of valid segment indexes for absolute level ? */ +/* 35 */ "SELECT idx FROM %Q.'%q_segdir' WHERE level=? ORDER BY 1 ASC", + +/* SQL_SELECT_MXLEVEL +** Return the largest relative level in the FTS index or indexes. */ +/* 36 */ "SELECT max( level %% 1024 ) FROM %Q.'%q_segdir'", + + /* Return segments in order from oldest to newest.*/ +/* 37 */ "SELECT level, idx, end_block " + "FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ? " + "ORDER BY level DESC, idx ASC", + + /* Update statements used while promoting segments */ +/* 38 */ "UPDATE OR FAIL %Q.'%q_segdir' SET level=-1,idx=? " + "WHERE level=? AND idx=?", +/* 39 */ "UPDATE OR FAIL %Q.'%q_segdir' SET level=? WHERE level=-1" + + }; + int rc = SQLITE_OK; + sqlite3_stmt *pStmt; + + assert( SizeofArray(azSql)==SizeofArray(p->aStmt) ); + assert( eStmt=0 ); + + pStmt = p->aStmt[eStmt]; + if( !pStmt ){ + int f = SQLITE_PREPARE_PERSISTENT|SQLITE_PREPARE_NO_VTAB; + char *zSql; + if( eStmt==SQL_CONTENT_INSERT ){ + zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName, p->zWriteExprlist); + }else if( eStmt==SQL_SELECT_CONTENT_BY_ROWID ){ + f &= ~SQLITE_PREPARE_NO_VTAB; + zSql = sqlite3_mprintf(azSql[eStmt], p->zReadExprlist); + }else{ + zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName); + } + if( !zSql ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare_v3(p->db, zSql, -1, f, &pStmt, NULL); + sqlite3_free(zSql); + assert( rc==SQLITE_OK || pStmt==0 ); + p->aStmt[eStmt] = pStmt; + } } - return 0; + if( apVal ){ + int i; + int nParam = sqlite3_bind_parameter_count(pStmt); + for(i=0; rc==SQLITE_OK && iht ){ - struct _fts3ht *pEntry = &pH->ht[h]; - elem = pEntry->chain; - count = pEntry->count; - xCompare = ftsCompareFunction(pH->keyClass); - while( count-- && elem ){ - if( (*xCompare)(elem->pKey,elem->nKey,pKey,nKey)==0 ){ - return elem; - } - elem = elem->next; + rc = fts3SqlStmt(pTab, SQL_SELECT_DOCSIZE, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pStmt, 1, iDocid); + rc = sqlite3_step(pStmt); + if( rc!=SQLITE_ROW || sqlite3_column_type(pStmt, 0)!=SQLITE_BLOB ){ + rc = sqlite3_reset(pStmt); + if( rc==SQLITE_OK ) rc = FTS_CORRUPT_VTAB; + pStmt = 0; + }else{ + rc = SQLITE_OK; } } - return 0; + + *ppStmt = pStmt; + return rc; } -/* Remove a single entry from the hash table given a pointer to that -** element and a hash on the element's key. -*/ -static void fts3RemoveElementByHash( - Fts3Hash *pH, /* The pH containing "elem" */ - Fts3HashElem* elem, /* The element to be removed from the pH */ - int h /* Hash value for the element */ +SQLITE_PRIVATE int sqlite3Fts3SelectDoctotal( + Fts3Table *pTab, /* Fts3 table handle */ + sqlite3_stmt **ppStmt /* OUT: Statement handle */ ){ - struct _fts3ht *pEntry; - if( elem->prev ){ - elem->prev->next = elem->next; - }else{ - pH->first = elem->next; - } - if( elem->next ){ - elem->next->prev = elem->prev; - } - pEntry = &pH->ht[h]; - if( pEntry->chain==elem ){ - pEntry->chain = elem->next; - } - pEntry->count--; - if( pEntry->count<=0 ){ - pEntry->chain = 0; + sqlite3_stmt *pStmt = 0; + int rc; + rc = fts3SqlStmt(pTab, SQL_SELECT_STAT, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int(pStmt, 1, FTS_STAT_DOCTOTAL); + if( sqlite3_step(pStmt)!=SQLITE_ROW + || sqlite3_column_type(pStmt, 0)!=SQLITE_BLOB + ){ + rc = sqlite3_reset(pStmt); + if( rc==SQLITE_OK ) rc = FTS_CORRUPT_VTAB; + pStmt = 0; + } } - if( pH->copyKey && elem->pKey ){ - fts3HashFree(elem->pKey); + *ppStmt = pStmt; + return rc; +} + +SQLITE_PRIVATE int sqlite3Fts3SelectDocsize( + Fts3Table *pTab, /* Fts3 table handle */ + sqlite3_int64 iDocid, /* Docid to read size data for */ + sqlite3_stmt **ppStmt /* OUT: Statement handle */ +){ + return fts3SelectDocsize(pTab, iDocid, ppStmt); +} + +/* +** Similar to fts3SqlStmt(). Except, after binding the parameters in +** array apVal[] to the SQL statement identified by eStmt, the statement +** is executed. +** +** Returns SQLITE_OK if the statement is successfully executed, or an +** SQLite error code otherwise. +*/ +static void fts3SqlExec( + int *pRC, /* Result code */ + Fts3Table *p, /* The FTS3 table */ + int eStmt, /* Index of statement to evaluate */ + sqlite3_value **apVal /* Parameters to bind */ +){ + sqlite3_stmt *pStmt; + int rc; + if( *pRC ) return; + rc = fts3SqlStmt(p, eStmt, &pStmt, apVal); + if( rc==SQLITE_OK ){ + sqlite3_step(pStmt); + rc = sqlite3_reset(pStmt); } - fts3HashFree( elem ); - pH->count--; - if( pH->count<=0 ){ - assert( pH->first==0 ); - assert( pH->count==0 ); - fts3HashClear(pH); + *pRC = rc; +} + + +/* +** This function ensures that the caller has obtained an exclusive +** shared-cache table-lock on the %_segdir table. This is required before +** writing data to the fts3 table. If this lock is not acquired first, then +** the caller may end up attempting to take this lock as part of committing +** a transaction, causing SQLite to return SQLITE_LOCKED or +** LOCKED_SHAREDCACHEto a COMMIT command. +** +** It is best to avoid this because if FTS3 returns any error when +** committing a transaction, the whole transaction will be rolled back. +** And this is not what users expect when they get SQLITE_LOCKED_SHAREDCACHE. +** It can still happen if the user locks the underlying tables directly +** instead of accessing them via FTS. +*/ +static int fts3Writelock(Fts3Table *p){ + int rc = SQLITE_OK; + + if( p->nPendingData==0 ){ + sqlite3_stmt *pStmt; + rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_LEVEL, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_null(pStmt, 1); + sqlite3_step(pStmt); + rc = sqlite3_reset(pStmt); + } } + + return rc; } -SQLITE_PRIVATE Fts3HashElem *sqlite3Fts3HashFindElem( - const Fts3Hash *pH, - const void *pKey, - int nKey +/* +** FTS maintains a separate indexes for each language-id (a 32-bit integer). +** Within each language id, a separate index is maintained to store the +** document terms, and each configured prefix size (configured the FTS +** "prefix=" option). And each index consists of multiple levels ("relative +** levels"). +** +** All three of these values (the language id, the specific index and the +** level within the index) are encoded in 64-bit integer values stored +** in the %_segdir table on disk. This function is used to convert three +** separate component values into the single 64-bit integer value that +** can be used to query the %_segdir table. +** +** Specifically, each language-id/index combination is allocated 1024 +** 64-bit integer level values ("absolute levels"). The main terms index +** for language-id 0 is allocate values 0-1023. The first prefix index +** (if any) for language-id 0 is allocated values 1024-2047. And so on. +** Language 1 indexes are allocated immediately following language 0. +** +** So, for a system with nPrefix prefix indexes configured, the block of +** absolute levels that corresponds to language-id iLangid and index +** iIndex starts at absolute level ((iLangid * (nPrefix+1) + iIndex) * 1024). +*/ +static sqlite3_int64 getAbsoluteLevel( + Fts3Table *p, /* FTS3 table handle */ + int iLangid, /* Language id */ + int iIndex, /* Index in p->aIndex[] */ + int iLevel /* Level of segments */ ){ - int h; /* A hash on key */ - int (*xHash)(const void*,int); /* The hash function */ + sqlite3_int64 iBase; /* First absolute level for iLangid/iIndex */ + assert_fts3_nc( iLangid>=0 ); + assert( p->nIndex>0 ); + assert( iIndex>=0 && iIndexnIndex ); - if( pH==0 || pH->ht==0 ) return 0; - xHash = ftsHashFunction(pH->keyClass); - assert( xHash!=0 ); - h = (*xHash)(pKey,nKey); - assert( (pH->htsize & (pH->htsize-1))==0 ); - return fts3FindElementByHash(pH,pKey,nKey, h & (pH->htsize-1)); + iBase = ((sqlite3_int64)iLangid * p->nIndex + iIndex) * FTS3_SEGDIR_MAXLEVEL; + return iBase + iLevel; } -/* -** Attempt to locate an element of the hash table pH with a key -** that matches pKey,nKey. Return the data for this element if it is -** found, or NULL if there is no match. +/* +** Set *ppStmt to a statement handle that may be used to iterate through +** all rows in the %_segdir table, from oldest to newest. If successful, +** return SQLITE_OK. If an error occurs while preparing the statement, +** return an SQLite error code. +** +** There is only ever one instance of this SQL statement compiled for +** each FTS3 table. +** +** The statement returns the following columns from the %_segdir table: +** +** 0: idx +** 1: start_block +** 2: leaves_end_block +** 3: end_block +** 4: root */ -SQLITE_PRIVATE void *sqlite3Fts3HashFind(const Fts3Hash *pH, const void *pKey, int nKey){ - Fts3HashElem *pElem; /* The element that matches key (if any) */ +SQLITE_PRIVATE int sqlite3Fts3AllSegdirs( + Fts3Table *p, /* FTS3 table */ + int iLangid, /* Language being queried */ + int iIndex, /* Index for p->aIndex[] */ + int iLevel, /* Level to select (relative level) */ + sqlite3_stmt **ppStmt /* OUT: Compiled statement */ +){ + int rc; + sqlite3_stmt *pStmt = 0; - pElem = sqlite3Fts3HashFindElem(pH, pKey, nKey); - return pElem ? pElem->data : 0; + assert( iLevel==FTS3_SEGCURSOR_ALL || iLevel>=0 ); + assert( iLevel=0 && iIndexnIndex ); + + if( iLevel<0 ){ + /* "SELECT * FROM %_segdir WHERE level BETWEEN ? AND ? ORDER BY ..." */ + rc = fts3SqlStmt(p, SQL_SELECT_LEVEL_RANGE, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pStmt, 1, getAbsoluteLevel(p, iLangid, iIndex, 0)); + sqlite3_bind_int64(pStmt, 2, + getAbsoluteLevel(p, iLangid, iIndex, FTS3_SEGDIR_MAXLEVEL-1) + ); + } + }else{ + /* "SELECT * FROM %_segdir WHERE level = ? ORDER BY ..." */ + rc = fts3SqlStmt(p, SQL_SELECT_LEVEL, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pStmt, 1, getAbsoluteLevel(p, iLangid, iIndex,iLevel)); + } + } + *ppStmt = pStmt; + return rc; } -/* Insert an element into the hash table pH. The key is pKey,nKey -** and the data is "data". + +/* +** Append a single varint to a PendingList buffer. SQLITE_OK is returned +** if successful, or an SQLite error code otherwise. ** -** If no element exists with a matching key, then a new -** element is created. A copy of the key is made if the copyKey -** flag is set. NULL is returned. +** This function also serves to allocate the PendingList structure itself. +** For example, to create a new PendingList structure containing two +** varints: ** -** If another element already exists with the same key, then the -** new data replaces the old data and the old data is returned. -** The key is not copied in this instance. If a malloc fails, then -** the new data is returned and the hash table is unchanged. +** PendingList *p = 0; +** fts3PendingListAppendVarint(&p, 1); +** fts3PendingListAppendVarint(&p, 2); +*/ +static int fts3PendingListAppendVarint( + PendingList **pp, /* IN/OUT: Pointer to PendingList struct */ + sqlite3_int64 i /* Value to append to data */ +){ + PendingList *p = *pp; + + /* Allocate or grow the PendingList as required. */ + if( !p ){ + p = sqlite3_malloc(sizeof(*p) + 100); + if( !p ){ + return SQLITE_NOMEM; + } + p->nSpace = 100; + p->aData = (char *)&p[1]; + p->nData = 0; + } + else if( p->nData+FTS3_VARINT_MAX+1>p->nSpace ){ + int nNew = p->nSpace * 2; + p = sqlite3_realloc(p, sizeof(*p) + nNew); + if( !p ){ + sqlite3_free(*pp); + *pp = 0; + return SQLITE_NOMEM; + } + p->nSpace = nNew; + p->aData = (char *)&p[1]; + } + + /* Append the new serialized varint to the end of the list. */ + p->nData += sqlite3Fts3PutVarint(&p->aData[p->nData], i); + p->aData[p->nData] = '\0'; + *pp = p; + return SQLITE_OK; +} + +/* +** Add a docid/column/position entry to a PendingList structure. Non-zero +** is returned if the structure is sqlite3_realloced as part of adding +** the entry. Otherwise, zero. ** -** If the "data" parameter to this function is NULL, then the -** element corresponding to "key" is removed from the hash table. +** If an OOM error occurs, *pRc is set to SQLITE_NOMEM before returning. +** Zero is always returned in this case. Otherwise, if no OOM error occurs, +** it is set to SQLITE_OK. */ -SQLITE_PRIVATE void *sqlite3Fts3HashInsert( - Fts3Hash *pH, /* The hash table to insert into */ - const void *pKey, /* The key */ - int nKey, /* Number of bytes in the key */ - void *data /* The data */ +static int fts3PendingListAppend( + PendingList **pp, /* IN/OUT: PendingList structure */ + sqlite3_int64 iDocid, /* Docid for entry to add */ + sqlite3_int64 iCol, /* Column for entry to add */ + sqlite3_int64 iPos, /* Position of term for entry to add */ + int *pRc /* OUT: Return code */ ){ - int hraw; /* Raw hash value of the key */ - int h; /* the hash of the key modulo hash table size */ - Fts3HashElem *elem; /* Used to loop thru the element list */ - Fts3HashElem *new_elem; /* New element added to the pH */ - int (*xHash)(const void*,int); /* The hash function */ + PendingList *p = *pp; + int rc = SQLITE_OK; - assert( pH!=0 ); - xHash = ftsHashFunction(pH->keyClass); - assert( xHash!=0 ); - hraw = (*xHash)(pKey, nKey); - assert( (pH->htsize & (pH->htsize-1))==0 ); - h = hraw & (pH->htsize-1); - elem = fts3FindElementByHash(pH,pKey,nKey,h); - if( elem ){ - void *old_data = elem->data; - if( data==0 ){ - fts3RemoveElementByHash(pH,elem,h); - }else{ - elem->data = data; + assert( !p || p->iLastDocid<=iDocid ); + + if( !p || p->iLastDocid!=iDocid ){ + u64 iDelta = (u64)iDocid - (u64)(p ? p->iLastDocid : 0); + if( p ){ + assert( p->nDatanSpace ); + assert( p->aData[p->nData]==0 ); + p->nData++; } - return old_data; + if( SQLITE_OK!=(rc = fts3PendingListAppendVarint(&p, iDelta)) ){ + goto pendinglistappend_out; + } + p->iLastCol = -1; + p->iLastPos = 0; + p->iLastDocid = iDocid; } - if( data==0 ) return 0; - if( (pH->htsize==0 && fts3Rehash(pH,8)) - || (pH->count>=pH->htsize && fts3Rehash(pH, pH->htsize*2)) - ){ - pH->count = 0; - return data; + if( iCol>0 && p->iLastCol!=iCol ){ + if( SQLITE_OK!=(rc = fts3PendingListAppendVarint(&p, 1)) + || SQLITE_OK!=(rc = fts3PendingListAppendVarint(&p, iCol)) + ){ + goto pendinglistappend_out; + } + p->iLastCol = iCol; + p->iLastPos = 0; } - assert( pH->htsize>0 ); - new_elem = (Fts3HashElem*)fts3HashMalloc( sizeof(Fts3HashElem) ); - if( new_elem==0 ) return data; - if( pH->copyKey && pKey!=0 ){ - new_elem->pKey = fts3HashMalloc( nKey ); - if( new_elem->pKey==0 ){ - fts3HashFree(new_elem); - return data; + if( iCol>=0 ){ + assert( iPos>p->iLastPos || (iPos==0 && p->iLastPos==0) ); + rc = fts3PendingListAppendVarint(&p, 2+iPos-p->iLastPos); + if( rc==SQLITE_OK ){ + p->iLastPos = iPos; } - memcpy((void*)new_elem->pKey, pKey, nKey); - }else{ - new_elem->pKey = (void*)pKey; } - new_elem->nKey = nKey; - pH->count++; - assert( pH->htsize>0 ); - assert( (pH->htsize & (pH->htsize-1))==0 ); - h = hraw & (pH->htsize-1); - fts3HashInsertElement(pH, &pH->ht[h], new_elem); - new_elem->data = data; + + pendinglistappend_out: + *pRc = rc; + if( p!=*pp ){ + *pp = p; + return 1; + } return 0; } -#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ +/* +** Free a PendingList object allocated by fts3PendingListAppend(). +*/ +static void fts3PendingListDelete(PendingList *pList){ + sqlite3_free(pList); +} -/************** End of fts3_hash.c *******************************************/ -/************** Begin file fts3_porter.c *************************************/ /* -** 2006 September 30 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** Implementation of the full-text-search tokenizer that implements -** a Porter stemmer. +** Add an entry to one of the pending-terms hash tables. */ +static int fts3PendingTermsAddOne( + Fts3Table *p, + int iCol, + int iPos, + Fts3Hash *pHash, /* Pending terms hash table to add entry to */ + const char *zToken, + int nToken +){ + PendingList *pList; + int rc = SQLITE_OK; + + pList = (PendingList *)fts3HashFind(pHash, zToken, nToken); + if( pList ){ + p->nPendingData -= (pList->nData + nToken + sizeof(Fts3HashElem)); + } + if( fts3PendingListAppend(&pList, p->iPrevDocid, iCol, iPos, &rc) ){ + if( pList==fts3HashInsert(pHash, zToken, nToken, pList) ){ + /* Malloc failed while inserting the new entry. This can only + ** happen if there was no previous entry for this token. + */ + assert( 0==fts3HashFind(pHash, zToken, nToken) ); + sqlite3_free(pList); + rc = SQLITE_NOMEM; + } + } + if( rc==SQLITE_OK ){ + p->nPendingData += (pList->nData + nToken + sizeof(Fts3HashElem)); + } + return rc; +} /* -** The code in this file is only compiled if: -** -** * The FTS3 module is being built as an extension -** (in which case SQLITE_CORE is not defined), or +** Tokenize the nul-terminated string zText and add all tokens to the +** pending-terms hash-table. The docid used is that currently stored in +** p->iPrevDocid, and the column is specified by argument iCol. ** -** * The FTS3 module is being built into the core of -** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). +** If successful, SQLITE_OK is returned. Otherwise, an SQLite error code. */ -/* #include "fts3Int.h" */ -#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) +static int fts3PendingTermsAdd( + Fts3Table *p, /* Table into which text will be inserted */ + int iLangid, /* Language id to use */ + const char *zText, /* Text of document to be inserted */ + int iCol, /* Column into which text is being inserted */ + u32 *pnWord /* IN/OUT: Incr. by number tokens inserted */ +){ + int rc; + int iStart = 0; + int iEnd = 0; + int iPos = 0; + int nWord = 0; -/* #include */ -/* #include */ -/* #include */ -/* #include */ + char const *zToken; + int nToken = 0; -/* #include "fts3_tokenizer.h" */ + sqlite3_tokenizer *pTokenizer = p->pTokenizer; + sqlite3_tokenizer_module const *pModule = pTokenizer->pModule; + sqlite3_tokenizer_cursor *pCsr; + int (*xNext)(sqlite3_tokenizer_cursor *pCursor, + const char**,int*,int*,int*,int*); -/* -** Class derived from sqlite3_tokenizer -*/ -typedef struct porter_tokenizer { - sqlite3_tokenizer base; /* Base class */ -} porter_tokenizer; + assert( pTokenizer && pModule ); -/* -** Class derived from sqlite3_tokenizer_cursor -*/ -typedef struct porter_tokenizer_cursor { - sqlite3_tokenizer_cursor base; - const char *zInput; /* input we are tokenizing */ - int nInput; /* size of the input */ - int iOffset; /* current position in zInput */ - int iToken; /* index of next token to be returned */ - char *zToken; /* storage for current token */ - int nAllocated; /* space allocated to zToken buffer */ -} porter_tokenizer_cursor; + /* If the user has inserted a NULL value, this function may be called with + ** zText==0. In this case, add zero token entries to the hash table and + ** return early. */ + if( zText==0 ){ + *pnWord = 0; + return SQLITE_OK; + } + rc = sqlite3Fts3OpenTokenizer(pTokenizer, iLangid, zText, -1, &pCsr); + if( rc!=SQLITE_OK ){ + return rc; + } -/* -** Create a new tokenizer instance. -*/ -static int porterCreate( - int argc, const char * const *argv, - sqlite3_tokenizer **ppTokenizer -){ - porter_tokenizer *t; + xNext = pModule->xNext; + while( SQLITE_OK==rc + && SQLITE_OK==(rc = xNext(pCsr, &zToken, &nToken, &iStart, &iEnd, &iPos)) + ){ + int i; + if( iPos>=nWord ) nWord = iPos+1; - UNUSED_PARAMETER(argc); - UNUSED_PARAMETER(argv); + /* Positions cannot be negative; we use -1 as a terminator internally. + ** Tokens must have a non-zero length. + */ + if( iPos<0 || !zToken || nToken<=0 ){ + rc = SQLITE_ERROR; + break; + } - t = (porter_tokenizer *) sqlite3_malloc(sizeof(*t)); - if( t==NULL ) return SQLITE_NOMEM; - memset(t, 0, sizeof(*t)); - *ppTokenizer = &t->base; - return SQLITE_OK; -} + /* Add the term to the terms index */ + rc = fts3PendingTermsAddOne( + p, iCol, iPos, &p->aIndex[0].hPending, zToken, nToken + ); + + /* Add the term to each of the prefix indexes that it is not too + ** short for. */ + for(i=1; rc==SQLITE_OK && inIndex; i++){ + struct Fts3Index *pIndex = &p->aIndex[i]; + if( nTokennPrefix ) continue; + rc = fts3PendingTermsAddOne( + p, iCol, iPos, &pIndex->hPending, zToken, pIndex->nPrefix + ); + } + } -/* -** Destroy a tokenizer -*/ -static int porterDestroy(sqlite3_tokenizer *pTokenizer){ - sqlite3_free(pTokenizer); - return SQLITE_OK; + pModule->xClose(pCsr); + *pnWord += nWord; + return (rc==SQLITE_DONE ? SQLITE_OK : rc); } -/* -** Prepare to begin tokenizing a particular string. The input -** string to be tokenized is zInput[0..nInput-1]. A cursor -** used to incrementally tokenize this string is returned in -** *ppCursor. +/* +** Calling this function indicates that subsequent calls to +** fts3PendingTermsAdd() are to add term/position-list pairs for the +** contents of the document with docid iDocid. */ -static int porterOpen( - sqlite3_tokenizer *pTokenizer, /* The tokenizer */ - const char *zInput, int nInput, /* String to be tokenized */ - sqlite3_tokenizer_cursor **ppCursor /* OUT: Tokenization cursor */ +static int fts3PendingTermsDocid( + Fts3Table *p, /* Full-text table handle */ + int bDelete, /* True if this op is a delete */ + int iLangid, /* Language id of row being written */ + sqlite_int64 iDocid /* Docid of row being written */ ){ - porter_tokenizer_cursor *c; - - UNUSED_PARAMETER(pTokenizer); - - c = (porter_tokenizer_cursor *) sqlite3_malloc(sizeof(*c)); - if( c==NULL ) return SQLITE_NOMEM; + assert( iLangid>=0 ); + assert( bDelete==1 || bDelete==0 ); - c->zInput = zInput; - if( zInput==0 ){ - c->nInput = 0; - }else if( nInput<0 ){ - c->nInput = (int)strlen(zInput); - }else{ - c->nInput = nInput; + /* TODO(shess) Explore whether partially flushing the buffer on + ** forced-flush would provide better performance. I suspect that if + ** we ordered the doclists by size and flushed the largest until the + ** buffer was half empty, that would let the less frequent terms + ** generate longer doclists. + */ + if( iDocidiPrevDocid + || (iDocid==p->iPrevDocid && p->bPrevDelete==0) + || p->iPrevLangid!=iLangid + || p->nPendingData>p->nMaxPendingData + ){ + int rc = sqlite3Fts3PendingTermsFlush(p); + if( rc!=SQLITE_OK ) return rc; } - c->iOffset = 0; /* start tokenizing at the beginning */ - c->iToken = 0; - c->zToken = NULL; /* no space allocated, yet. */ - c->nAllocated = 0; - - *ppCursor = &c->base; + p->iPrevDocid = iDocid; + p->iPrevLangid = iLangid; + p->bPrevDelete = bDelete; return SQLITE_OK; } /* -** Close a tokenization cursor previously opened by a call to -** porterOpen() above. +** Discard the contents of the pending-terms hash tables. */ -static int porterClose(sqlite3_tokenizer_cursor *pCursor){ - porter_tokenizer_cursor *c = (porter_tokenizer_cursor *) pCursor; - sqlite3_free(c->zToken); - sqlite3_free(c); - return SQLITE_OK; +SQLITE_PRIVATE void sqlite3Fts3PendingTermsClear(Fts3Table *p){ + int i; + for(i=0; inIndex; i++){ + Fts3HashElem *pElem; + Fts3Hash *pHash = &p->aIndex[i].hPending; + for(pElem=fts3HashFirst(pHash); pElem; pElem=fts3HashNext(pElem)){ + PendingList *pList = (PendingList *)fts3HashData(pElem); + fts3PendingListDelete(pList); + } + fts3HashClear(pHash); + } + p->nPendingData = 0; } -/* -** Vowel or consonant -*/ -static const char cType[] = { - 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, - 1, 1, 1, 2, 1 -}; /* -** isConsonant() and isVowel() determine if their first character in -** the string they point to is a consonant or a vowel, according -** to Porter ruls. -** -** A consonate is any letter other than 'a', 'e', 'i', 'o', or 'u'. -** 'Y' is a consonant unless it follows another consonant, -** in which case it is a vowel. +** This function is called by the xUpdate() method as part of an INSERT +** operation. It adds entries for each term in the new record to the +** pendingTerms hash table. ** -** In these routine, the letters are in reverse order. So the 'y' rule -** is that 'y' is a consonant unless it is followed by another -** consonent. +** Argument apVal is the same as the similarly named argument passed to +** fts3InsertData(). Parameter iDocid is the docid of the new row. */ -static int isVowel(const char*); -static int isConsonant(const char *z){ - int j; - char x = *z; - if( x==0 ) return 0; - assert( x>='a' && x<='z' ); - j = cType[x-'a']; - if( j<2 ) return j; - return z[1]==0 || isVowel(z + 1); -} -static int isVowel(const char *z){ - int j; - char x = *z; - if( x==0 ) return 0; - assert( x>='a' && x<='z' ); - j = cType[x-'a']; - if( j<2 ) return 1-j; - return isConsonant(z + 1); +static int fts3InsertTerms( + Fts3Table *p, + int iLangid, + sqlite3_value **apVal, + u32 *aSz +){ + int i; /* Iterator variable */ + for(i=2; inColumn+2; i++){ + int iCol = i-2; + if( p->abNotindexed[iCol]==0 ){ + const char *zText = (const char *)sqlite3_value_text(apVal[i]); + int rc = fts3PendingTermsAdd(p, iLangid, zText, iCol, &aSz[iCol]); + if( rc!=SQLITE_OK ){ + return rc; + } + aSz[p->nColumn] += sqlite3_value_bytes(apVal[i]); + } + } + return SQLITE_OK; } /* -** Let any sequence of one or more vowels be represented by V and let -** C be sequence of one or more consonants. Then every word can be -** represented as: -** -** [C] (VC){m} [V] -** -** In prose: A word is an optional consonant followed by zero or -** vowel-consonant pairs followed by an optional vowel. "m" is the -** number of vowel consonant pairs. This routine computes the value -** of m for the first i bytes of a word. -** -** Return true if the m-value for z is 1 or more. In other words, -** return true if z contains at least one vowel that is followed -** by a consonant. +** This function is called by the xUpdate() method for an INSERT operation. +** The apVal parameter is passed a copy of the apVal argument passed by +** SQLite to the xUpdate() method. i.e: ** -** In this routine z[] is in reverse order. So we are really looking -** for an instance of a consonant followed by a vowel. +** apVal[0] Not used for INSERT. +** apVal[1] rowid +** apVal[2] Left-most user-defined column +** ... +** apVal[p->nColumn+1] Right-most user-defined column +** apVal[p->nColumn+2] Hidden column with same name as table +** apVal[p->nColumn+3] Hidden "docid" column (alias for rowid) +** apVal[p->nColumn+4] Hidden languageid column */ -static int m_gt_0(const char *z){ - while( isVowel(z) ){ z++; } - if( *z==0 ) return 0; - while( isConsonant(z) ){ z++; } - return *z!=0; -} +static int fts3InsertData( + Fts3Table *p, /* Full-text table */ + sqlite3_value **apVal, /* Array of values to insert */ + sqlite3_int64 *piDocid /* OUT: Docid for row just inserted */ +){ + int rc; /* Return code */ + sqlite3_stmt *pContentInsert; /* INSERT INTO %_content VALUES(...) */ -/* Like mgt0 above except we are looking for a value of m which is -** exactly 1 -*/ -static int m_eq_1(const char *z){ - while( isVowel(z) ){ z++; } - if( *z==0 ) return 0; - while( isConsonant(z) ){ z++; } - if( *z==0 ) return 0; - while( isVowel(z) ){ z++; } - if( *z==0 ) return 1; - while( isConsonant(z) ){ z++; } - return *z==0; -} + if( p->zContentTbl ){ + sqlite3_value *pRowid = apVal[p->nColumn+3]; + if( sqlite3_value_type(pRowid)==SQLITE_NULL ){ + pRowid = apVal[1]; + } + if( sqlite3_value_type(pRowid)!=SQLITE_INTEGER ){ + return SQLITE_CONSTRAINT; + } + *piDocid = sqlite3_value_int64(pRowid); + return SQLITE_OK; + } -/* Like mgt0 above except we are looking for a value of m>1 instead -** or m>0 -*/ -static int m_gt_1(const char *z){ - while( isVowel(z) ){ z++; } - if( *z==0 ) return 0; - while( isConsonant(z) ){ z++; } - if( *z==0 ) return 0; - while( isVowel(z) ){ z++; } - if( *z==0 ) return 0; - while( isConsonant(z) ){ z++; } - return *z!=0; + /* Locate the statement handle used to insert data into the %_content + ** table. The SQL for this statement is: + ** + ** INSERT INTO %_content VALUES(?, ?, ?, ...) + ** + ** The statement features N '?' variables, where N is the number of user + ** defined columns in the FTS3 table, plus one for the docid field. + */ + rc = fts3SqlStmt(p, SQL_CONTENT_INSERT, &pContentInsert, &apVal[1]); + if( rc==SQLITE_OK && p->zLanguageid ){ + rc = sqlite3_bind_int( + pContentInsert, p->nColumn+2, + sqlite3_value_int(apVal[p->nColumn+4]) + ); + } + if( rc!=SQLITE_OK ) return rc; + + /* There is a quirk here. The users INSERT statement may have specified + ** a value for the "rowid" field, for the "docid" field, or for both. + ** Which is a problem, since "rowid" and "docid" are aliases for the + ** same value. For example: + ** + ** INSERT INTO fts3tbl(rowid, docid) VALUES(1, 2); + ** + ** In FTS3, this is an error. It is an error to specify non-NULL values + ** for both docid and some other rowid alias. + */ + if( SQLITE_NULL!=sqlite3_value_type(apVal[3+p->nColumn]) ){ + if( SQLITE_NULL==sqlite3_value_type(apVal[0]) + && SQLITE_NULL!=sqlite3_value_type(apVal[1]) + ){ + /* A rowid/docid conflict. */ + return SQLITE_ERROR; + } + rc = sqlite3_bind_value(pContentInsert, 1, apVal[3+p->nColumn]); + if( rc!=SQLITE_OK ) return rc; + } + + /* Execute the statement to insert the record. Set *piDocid to the + ** new docid value. + */ + sqlite3_step(pContentInsert); + rc = sqlite3_reset(pContentInsert); + + *piDocid = sqlite3_last_insert_rowid(p->db); + return rc; } + + /* -** Return TRUE if there is a vowel anywhere within z[0..n-1] +** Remove all data from the FTS3 table. Clear the hash table containing +** pending terms. */ -static int hasVowel(const char *z){ - while( isConsonant(z) ){ z++; } - return *z!=0; +static int fts3DeleteAll(Fts3Table *p, int bContent){ + int rc = SQLITE_OK; /* Return code */ + + /* Discard the contents of the pending-terms hash table. */ + sqlite3Fts3PendingTermsClear(p); + + /* Delete everything from the shadow tables. Except, leave %_content as + ** is if bContent is false. */ + assert( p->zContentTbl==0 || bContent==0 ); + if( bContent ) fts3SqlExec(&rc, p, SQL_DELETE_ALL_CONTENT, 0); + fts3SqlExec(&rc, p, SQL_DELETE_ALL_SEGMENTS, 0); + fts3SqlExec(&rc, p, SQL_DELETE_ALL_SEGDIR, 0); + if( p->bHasDocsize ){ + fts3SqlExec(&rc, p, SQL_DELETE_ALL_DOCSIZE, 0); + } + if( p->bHasStat ){ + fts3SqlExec(&rc, p, SQL_DELETE_ALL_STAT, 0); + } + return rc; } /* -** Return TRUE if the word ends in a double consonant. ** -** The text is reversed here. So we are really looking at -** the first two characters of z[]. */ -static int doubleConsonant(const char *z){ - return isConsonant(z) && z[0]==z[1]; +static int langidFromSelect(Fts3Table *p, sqlite3_stmt *pSelect){ + int iLangid = 0; + if( p->zLanguageid ) iLangid = sqlite3_column_int(pSelect, p->nColumn+1); + return iLangid; } /* -** Return TRUE if the word ends with three letters which -** are consonant-vowel-consonent and where the final consonant -** is not 'w', 'x', or 'y'. -** -** The word is reversed here. So we are really checking the -** first three letters and the first one cannot be in [wxy]. +** The first element in the apVal[] array is assumed to contain the docid +** (an integer) of a row about to be deleted. Remove all terms from the +** full-text index. */ -static int star_oh(const char *z){ - return - isConsonant(z) && - z[0]!='w' && z[0]!='x' && z[0]!='y' && - isVowel(z+1) && - isConsonant(z+2); +static void fts3DeleteTerms( + int *pRC, /* Result code */ + Fts3Table *p, /* The FTS table to delete from */ + sqlite3_value *pRowid, /* The docid to be deleted */ + u32 *aSz, /* Sizes of deleted document written here */ + int *pbFound /* OUT: Set to true if row really does exist */ +){ + int rc; + sqlite3_stmt *pSelect; + + assert( *pbFound==0 ); + if( *pRC ) return; + rc = fts3SqlStmt(p, SQL_SELECT_CONTENT_BY_ROWID, &pSelect, &pRowid); + if( rc==SQLITE_OK ){ + if( SQLITE_ROW==sqlite3_step(pSelect) ){ + int i; + int iLangid = langidFromSelect(p, pSelect); + i64 iDocid = sqlite3_column_int64(pSelect, 0); + rc = fts3PendingTermsDocid(p, 1, iLangid, iDocid); + for(i=1; rc==SQLITE_OK && i<=p->nColumn; i++){ + int iCol = i-1; + if( p->abNotindexed[iCol]==0 ){ + const char *zText = (const char *)sqlite3_column_text(pSelect, i); + rc = fts3PendingTermsAdd(p, iLangid, zText, -1, &aSz[iCol]); + aSz[p->nColumn] += sqlite3_column_bytes(pSelect, i); + } + } + if( rc!=SQLITE_OK ){ + sqlite3_reset(pSelect); + *pRC = rc; + return; + } + *pbFound = 1; + } + rc = sqlite3_reset(pSelect); + }else{ + sqlite3_reset(pSelect); + } + *pRC = rc; } /* -** If the word ends with zFrom and xCond() is true for the stem -** of the word that preceeds the zFrom ending, then change the -** ending to zTo. +** Forward declaration to account for the circular dependency between +** functions fts3SegmentMerge() and fts3AllocateSegdirIdx(). +*/ +static int fts3SegmentMerge(Fts3Table *, int, int, int); + +/* +** This function allocates a new level iLevel index in the segdir table. +** Usually, indexes are allocated within a level sequentially starting +** with 0, so the allocated index is one greater than the value returned +** by: ** -** The input word *pz and zFrom are both in reverse order. zTo -** is in normal order. +** SELECT max(idx) FROM %_segdir WHERE level = :iLevel ** -** Return TRUE if zFrom matches. Return FALSE if zFrom does not -** match. Not that TRUE is returned even if xCond() fails and -** no substitution occurs. +** However, if there are already FTS3_MERGE_COUNT indexes at the requested +** level, they are merged into a single level (iLevel+1) segment and the +** allocated index is 0. +** +** If successful, *piIdx is set to the allocated index slot and SQLITE_OK +** returned. Otherwise, an SQLite error code is returned. */ -static int stem( - char **pz, /* The word being stemmed (Reversed) */ - const char *zFrom, /* If the ending matches this... (Reversed) */ - const char *zTo, /* ... change the ending to this (not reversed) */ - int (*xCond)(const char*) /* Condition that must be true */ +static int fts3AllocateSegdirIdx( + Fts3Table *p, + int iLangid, /* Language id */ + int iIndex, /* Index for p->aIndex */ + int iLevel, + int *piIdx ){ - char *z = *pz; - while( *zFrom && *zFrom==*z ){ z++; zFrom++; } - if( *zFrom!=0 ) return 0; - if( xCond && !xCond(z) ) return 1; - while( *zTo ){ - *(--z) = *(zTo++); - } - *pz = z; - return 1; -} + int rc; /* Return Code */ + sqlite3_stmt *pNextIdx; /* Query for next idx at level iLevel */ + int iNext = 0; /* Result of query pNextIdx */ -/* -** This is the fallback stemmer used when the porter stemmer is -** inappropriate. The input word is copied into the output with -** US-ASCII case folding. If the input word is too long (more -** than 20 bytes if it contains no digits or more than 6 bytes if -** it contains digits) then word is truncated to 20 or 6 bytes -** by taking 10 or 3 bytes from the beginning and end. -*/ -static void copy_stemmer(const char *zIn, int nIn, char *zOut, int *pnOut){ - int i, mx, j; - int hasDigit = 0; - for(i=0; i='A' && c<='Z' ){ - zOut[i] = c - 'A' + 'a'; - }else{ - if( c>='0' && c<='9' ) hasDigit = 1; - zOut[i] = c; + assert( iLangid>=0 ); + assert( p->nIndex>=1 ); + + /* Set variable iNext to the next available segdir index at level iLevel. */ + rc = fts3SqlStmt(p, SQL_NEXT_SEGMENT_INDEX, &pNextIdx, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64( + pNextIdx, 1, getAbsoluteLevel(p, iLangid, iIndex, iLevel) + ); + if( SQLITE_ROW==sqlite3_step(pNextIdx) ){ + iNext = sqlite3_column_int(pNextIdx, 0); } + rc = sqlite3_reset(pNextIdx); } - mx = hasDigit ? 3 : 10; - if( nIn>mx*2 ){ - for(j=mx, i=nIn-mx; i=MergeCount(p) ){ + fts3LogMerge(16, getAbsoluteLevel(p, iLangid, iIndex, iLevel)); + rc = fts3SegmentMerge(p, iLangid, iIndex, iLevel); + *piIdx = 0; + }else{ + *piIdx = iNext; } - i = j; } - zOut[i] = 0; - *pnOut = i; -} + return rc; +} /* -** Stem the input word zIn[0..nIn-1]. Store the output in zOut. -** zOut is at least big enough to hold nIn bytes. Write the actual -** size of the output word (exclusive of the '\0' terminator) into *pnOut. +** The %_segments table is declared as follows: ** -** Any upper-case characters in the US-ASCII character set ([A-Z]) -** are converted to lower case. Upper-case UTF characters are -** unchanged. +** CREATE TABLE %_segments(blockid INTEGER PRIMARY KEY, block BLOB) ** -** Words that are longer than about 20 bytes are stemmed by retaining -** a few bytes from the beginning and the end of the word. If the -** word contains digits, 3 bytes are taken from the beginning and -** 3 bytes from the end. For long words without digits, 10 bytes -** are taken from each end. US-ASCII case folding still applies. -** -** If the input word contains not digits but does characters not -** in [a-zA-Z] then no stemming is attempted and this routine just -** copies the input into the input into the output with US-ASCII -** case folding. +** This function reads data from a single row of the %_segments table. The +** specific row is identified by the iBlockid parameter. If paBlob is not +** NULL, then a buffer is allocated using sqlite3_malloc() and populated +** with the contents of the blob stored in the "block" column of the +** identified table row is. Whether or not paBlob is NULL, *pnBlob is set +** to the size of the blob in bytes before returning. ** -** Stemming never increases the length of the word. So there is -** no chance of overflowing the zOut buffer. +** If an error occurs, or the table does not contain the specified row, +** an SQLite error code is returned. Otherwise, SQLITE_OK is returned. If +** paBlob is non-NULL, then it is the responsibility of the caller to +** eventually free the returned buffer. +** +** This function may leave an open sqlite3_blob* handle in the +** Fts3Table.pSegments variable. This handle is reused by subsequent calls +** to this function. The handle may be closed by calling the +** sqlite3Fts3SegmentsClose() function. Reusing a blob handle is a handy +** performance improvement, but the blob handle should always be closed +** before control is returned to the user (to prevent a lock being held +** on the database file for longer than necessary). Thus, any virtual table +** method (xFilter etc.) that may directly or indirectly call this function +** must call sqlite3Fts3SegmentsClose() before returning. */ -static void porter_stemmer(const char *zIn, int nIn, char *zOut, int *pnOut){ - int i, j; - char zReverse[28]; - char *z, *z2; - if( nIn<3 || nIn>=(int)sizeof(zReverse)-7 ){ - /* The word is too big or too small for the porter stemmer. - ** Fallback to the copy stemmer */ - copy_stemmer(zIn, nIn, zOut, pnOut); - return; +SQLITE_PRIVATE int sqlite3Fts3ReadBlock( + Fts3Table *p, /* FTS3 table handle */ + sqlite3_int64 iBlockid, /* Access the row with blockid=$iBlockid */ + char **paBlob, /* OUT: Blob data in malloc'd buffer */ + int *pnBlob, /* OUT: Size of blob data */ + int *pnLoad /* OUT: Bytes actually loaded */ +){ + int rc; /* Return code */ + + /* pnBlob must be non-NULL. paBlob may be NULL or non-NULL. */ + assert( pnBlob ); + + if( p->pSegments ){ + rc = sqlite3_blob_reopen(p->pSegments, iBlockid); + }else{ + if( 0==p->zSegmentsTbl ){ + p->zSegmentsTbl = sqlite3_mprintf("%s_segments", p->zName); + if( 0==p->zSegmentsTbl ) return SQLITE_NOMEM; + } + rc = sqlite3_blob_open( + p->db, p->zDb, p->zSegmentsTbl, "block", iBlockid, 0, &p->pSegments + ); } - for(i=0, j=sizeof(zReverse)-6; i='A' && c<='Z' ){ - zReverse[j] = c + 'a' - 'A'; - }else if( c>='a' && c<='z' ){ - zReverse[j] = c; - }else{ - /* The use of a character not in [a-zA-Z] means that we fallback - ** to the copy stemmer */ - copy_stemmer(zIn, nIn, zOut, pnOut); - return; + + if( rc==SQLITE_OK ){ + int nByte = sqlite3_blob_bytes(p->pSegments); + *pnBlob = nByte; + if( paBlob ){ + char *aByte = sqlite3_malloc(nByte + FTS3_NODE_PADDING); + if( !aByte ){ + rc = SQLITE_NOMEM; + }else{ + if( pnLoad && nByte>(FTS3_NODE_CHUNK_THRESHOLD) ){ + nByte = FTS3_NODE_CHUNKSIZE; + *pnLoad = nByte; + } + rc = sqlite3_blob_read(p->pSegments, aByte, nByte, 0); + memset(&aByte[nByte], 0, FTS3_NODE_PADDING); + if( rc!=SQLITE_OK ){ + sqlite3_free(aByte); + aByte = 0; + } + } + *paBlob = aByte; } + }else if( rc==SQLITE_ERROR ){ + rc = FTS_CORRUPT_VTAB; } - memset(&zReverse[sizeof(zReverse)-5], 0, 5); - z = &zReverse[j+1]; + return rc; +} - /* Step 1a */ - if( z[0]=='s' ){ - if( - !stem(&z, "sess", "ss", 0) && - !stem(&z, "sei", "i", 0) && - !stem(&z, "ss", "ss", 0) - ){ - z++; +/* +** Close the blob handle at p->pSegments, if it is open. See comments above +** the sqlite3Fts3ReadBlock() function for details. +*/ +SQLITE_PRIVATE void sqlite3Fts3SegmentsClose(Fts3Table *p){ + sqlite3_blob_close(p->pSegments); + p->pSegments = 0; +} + +static int fts3SegReaderIncrRead(Fts3SegReader *pReader){ + int nRead; /* Number of bytes to read */ + int rc; /* Return code */ + + nRead = MIN(pReader->nNode - pReader->nPopulate, FTS3_NODE_CHUNKSIZE); + rc = sqlite3_blob_read( + pReader->pBlob, + &pReader->aNode[pReader->nPopulate], + nRead, + pReader->nPopulate + ); + + if( rc==SQLITE_OK ){ + pReader->nPopulate += nRead; + memset(&pReader->aNode[pReader->nPopulate], 0, FTS3_NODE_PADDING); + if( pReader->nPopulate==pReader->nNode ){ + sqlite3_blob_close(pReader->pBlob); + pReader->pBlob = 0; + pReader->nPopulate = 0; } } + return rc; +} - /* Step 1b */ - z2 = z; - if( stem(&z, "dee", "ee", m_gt_0) ){ - /* Do nothing. The work was all in the test */ - }else if( - (stem(&z, "gni", "", hasVowel) || stem(&z, "de", "", hasVowel)) - && z!=z2 +static int fts3SegReaderRequire(Fts3SegReader *pReader, char *pFrom, int nByte){ + int rc = SQLITE_OK; + assert( !pReader->pBlob + || (pFrom>=pReader->aNode && pFrom<&pReader->aNode[pReader->nNode]) + ); + while( pReader->pBlob && rc==SQLITE_OK + && (pFrom - pReader->aNode + nByte)>pReader->nPopulate ){ - if( stem(&z, "ta", "ate", 0) || - stem(&z, "lb", "ble", 0) || - stem(&z, "zi", "ize", 0) ){ - /* Do nothing. The work was all in the test */ - }else if( doubleConsonant(z) && (*z!='l' && *z!='s' && *z!='z') ){ - z++; - }else if( m_eq_1(z) && star_oh(z) ){ - *(--z) = 'e'; - } + rc = fts3SegReaderIncrRead(pReader); } + return rc; +} - /* Step 1c */ - if( z[0]=='y' && hasVowel(z+1) ){ - z[0] = 'i'; +/* +** Set an Fts3SegReader cursor to point at EOF. +*/ +static void fts3SegReaderSetEof(Fts3SegReader *pSeg){ + if( !fts3SegReaderIsRootOnly(pSeg) ){ + sqlite3_free(pSeg->aNode); + sqlite3_blob_close(pSeg->pBlob); + pSeg->pBlob = 0; } + pSeg->aNode = 0; +} - /* Step 2 */ - switch( z[1] ){ - case 'a': - if( !stem(&z, "lanoita", "ate", m_gt_0) ){ - stem(&z, "lanoit", "tion", m_gt_0); - } - break; - case 'c': - if( !stem(&z, "icne", "ence", m_gt_0) ){ - stem(&z, "icna", "ance", m_gt_0); - } - break; - case 'e': - stem(&z, "rezi", "ize", m_gt_0); - break; - case 'g': - stem(&z, "igol", "log", m_gt_0); - break; - case 'l': - if( !stem(&z, "ilb", "ble", m_gt_0) - && !stem(&z, "illa", "al", m_gt_0) - && !stem(&z, "iltne", "ent", m_gt_0) - && !stem(&z, "ile", "e", m_gt_0) - ){ - stem(&z, "ilsuo", "ous", m_gt_0); - } - break; - case 'o': - if( !stem(&z, "noitazi", "ize", m_gt_0) - && !stem(&z, "noita", "ate", m_gt_0) - ){ - stem(&z, "rota", "ate", m_gt_0); - } - break; - case 's': - if( !stem(&z, "msila", "al", m_gt_0) - && !stem(&z, "ssenevi", "ive", m_gt_0) - && !stem(&z, "ssenluf", "ful", m_gt_0) - ){ - stem(&z, "ssensuo", "ous", m_gt_0); - } - break; - case 't': - if( !stem(&z, "itila", "al", m_gt_0) - && !stem(&z, "itivi", "ive", m_gt_0) - ){ - stem(&z, "itilib", "ble", m_gt_0); - } - break; - } +/* +** Move the iterator passed as the first argument to the next term in the +** segment. If successful, SQLITE_OK is returned. If there is no next term, +** SQLITE_DONE. Otherwise, an SQLite error code. +*/ +static int fts3SegReaderNext( + Fts3Table *p, + Fts3SegReader *pReader, + int bIncr +){ + int rc; /* Return code of various sub-routines */ + char *pNext; /* Cursor variable */ + int nPrefix; /* Number of bytes in term prefix */ + int nSuffix; /* Number of bytes in term suffix */ - /* Step 3 */ - switch( z[0] ){ - case 'e': - if( !stem(&z, "etaci", "ic", m_gt_0) - && !stem(&z, "evita", "", m_gt_0) - ){ - stem(&z, "ezila", "al", m_gt_0); - } - break; - case 'i': - stem(&z, "itici", "ic", m_gt_0); - break; - case 'l': - if( !stem(&z, "laci", "ic", m_gt_0) ){ - stem(&z, "luf", "", m_gt_0); - } - break; - case 's': - stem(&z, "ssen", "", m_gt_0); - break; + if( !pReader->aDoclist ){ + pNext = pReader->aNode; + }else{ + pNext = &pReader->aDoclist[pReader->nDoclist]; } - /* Step 4 */ - switch( z[1] ){ - case 'a': - if( z[0]=='l' && m_gt_1(z+2) ){ - z += 2; - } - break; - case 'c': - if( z[0]=='e' && z[2]=='n' && (z[3]=='a' || z[3]=='e') && m_gt_1(z+4) ){ - z += 4; - } - break; - case 'e': - if( z[0]=='r' && m_gt_1(z+2) ){ - z += 2; - } - break; - case 'i': - if( z[0]=='c' && m_gt_1(z+2) ){ - z += 2; - } - break; - case 'l': - if( z[0]=='e' && z[2]=='b' && (z[3]=='a' || z[3]=='i') && m_gt_1(z+4) ){ - z += 4; - } - break; - case 'n': - if( z[0]=='t' ){ - if( z[2]=='a' ){ - if( m_gt_1(z+3) ){ - z += 3; - } - }else if( z[2]=='e' ){ - if( !stem(&z, "tneme", "", m_gt_1) - && !stem(&z, "tnem", "", m_gt_1) - ){ - stem(&z, "tne", "", m_gt_1); - } - } - } - break; - case 'o': - if( z[0]=='u' ){ - if( m_gt_1(z+2) ){ - z += 2; - } - }else if( z[3]=='s' || z[3]=='t' ){ - stem(&z, "noi", "", m_gt_1); - } - break; - case 's': - if( z[0]=='m' && z[2]=='i' && m_gt_1(z+3) ){ - z += 3; - } - break; - case 't': - if( !stem(&z, "eta", "", m_gt_1) ){ - stem(&z, "iti", "", m_gt_1); - } - break; - case 'u': - if( z[0]=='s' && z[2]=='o' && m_gt_1(z+3) ){ - z += 3; - } - break; - case 'v': - case 'z': - if( z[0]=='e' && z[2]=='i' && m_gt_1(z+3) ){ - z += 3; - } - break; - } + if( !pNext || pNext>=&pReader->aNode[pReader->nNode] ){ + + if( fts3SegReaderIsPending(pReader) ){ + Fts3HashElem *pElem = *(pReader->ppNextElem); + sqlite3_free(pReader->aNode); + pReader->aNode = 0; + if( pElem ){ + char *aCopy; + PendingList *pList = (PendingList *)fts3HashData(pElem); + int nCopy = pList->nData+1; + pReader->zTerm = (char *)fts3HashKey(pElem); + pReader->nTerm = fts3HashKeysize(pElem); + aCopy = (char*)sqlite3_malloc(nCopy); + if( !aCopy ) return SQLITE_NOMEM; + memcpy(aCopy, pList->aData, nCopy); + pReader->nNode = pReader->nDoclist = nCopy; + pReader->aNode = pReader->aDoclist = aCopy; + pReader->ppNextElem++; + assert( pReader->aNode ); + } + return SQLITE_OK; + } - /* Step 5a */ - if( z[0]=='e' ){ - if( m_gt_1(z+1) ){ - z++; - }else if( m_eq_1(z+1) && !star_oh(z+1) ){ - z++; + fts3SegReaderSetEof(pReader); + + /* If iCurrentBlock>=iLeafEndBlock, this is an EOF condition. All leaf + ** blocks have already been traversed. */ +#ifdef CORRUPT_DB + assert( pReader->iCurrentBlock<=pReader->iLeafEndBlock || CORRUPT_DB ); +#endif + if( pReader->iCurrentBlock>=pReader->iLeafEndBlock ){ + return SQLITE_OK; + } + + rc = sqlite3Fts3ReadBlock( + p, ++pReader->iCurrentBlock, &pReader->aNode, &pReader->nNode, + (bIncr ? &pReader->nPopulate : 0) + ); + if( rc!=SQLITE_OK ) return rc; + assert( pReader->pBlob==0 ); + if( bIncr && pReader->nPopulatenNode ){ + pReader->pBlob = p->pSegments; + p->pSegments = 0; } + pNext = pReader->aNode; } - /* Step 5b */ - if( m_gt_1(z) && z[0]=='l' && z[1]=='l' ){ - z++; + assert( !fts3SegReaderIsPending(pReader) ); + + rc = fts3SegReaderRequire(pReader, pNext, FTS3_VARINT_MAX*2); + if( rc!=SQLITE_OK ) return rc; + + /* Because of the FTS3_NODE_PADDING bytes of padding, the following is + ** safe (no risk of overread) even if the node data is corrupted. */ + pNext += fts3GetVarint32(pNext, &nPrefix); + pNext += fts3GetVarint32(pNext, &nSuffix); + if( nSuffix<=0 + || (&pReader->aNode[pReader->nNode] - pNext)pReader->nTerm + ){ + return FTS_CORRUPT_VTAB; } - /* z[] is now the stemmed word in reverse order. Flip it back - ** around into forward order and return. + /* Both nPrefix and nSuffix were read by fts3GetVarint32() and so are + ** between 0 and 0x7FFFFFFF. But the sum of the two may cause integer + ** overflow - hence the (i64) casts. */ + if( (i64)nPrefix+nSuffix>(i64)pReader->nTermAlloc ){ + i64 nNew = ((i64)nPrefix+nSuffix)*2; + char *zNew = sqlite3_realloc64(pReader->zTerm, nNew); + if( !zNew ){ + return SQLITE_NOMEM; + } + pReader->zTerm = zNew; + pReader->nTermAlloc = nNew; + } + + rc = fts3SegReaderRequire(pReader, pNext, nSuffix+FTS3_VARINT_MAX); + if( rc!=SQLITE_OK ) return rc; + + memcpy(&pReader->zTerm[nPrefix], pNext, nSuffix); + pReader->nTerm = nPrefix+nSuffix; + pNext += nSuffix; + pNext += fts3GetVarint32(pNext, &pReader->nDoclist); + pReader->aDoclist = pNext; + pReader->pOffsetList = 0; + + /* Check that the doclist does not appear to extend past the end of the + ** b-tree node. And that the final byte of the doclist is 0x00. If either + ** of these statements is untrue, then the data structure is corrupt. */ - *pnOut = i = (int)strlen(z); - zOut[i] = 0; - while( *z ){ - zOut[--i] = *(z++); + if( pReader->nDoclist > pReader->nNode-(pReader->aDoclist-pReader->aNode) + || (pReader->nPopulate==0 && pReader->aDoclist[pReader->nDoclist-1]) + || pReader->nDoclist==0 + ){ + return FTS_CORRUPT_VTAB; } + return SQLITE_OK; } /* -** Characters that can be part of a token. We assume any character -** whose value is greater than 0x80 (any UTF character) can be -** part of a token. In other words, delimiters all must have -** values of 0x7f or lower. +** Set the SegReader to point to the first docid in the doclist associated +** with the current term. */ -static const char porterIdChar[] = { -/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */ - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 3x */ - 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4x */ - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 5x */ - 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 6x */ - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */ -}; -#define isDelim(C) (((ch=C)&0x80)==0 && (ch<0x30 || !porterIdChar[ch-0x30])) +static int fts3SegReaderFirstDocid(Fts3Table *pTab, Fts3SegReader *pReader){ + int rc = SQLITE_OK; + assert( pReader->aDoclist ); + assert( !pReader->pOffsetList ); + if( pTab->bDescIdx && fts3SegReaderIsPending(pReader) ){ + u8 bEof = 0; + pReader->iDocid = 0; + pReader->nOffsetList = 0; + sqlite3Fts3DoclistPrev(0, + pReader->aDoclist, pReader->nDoclist, &pReader->pOffsetList, + &pReader->iDocid, &pReader->nOffsetList, &bEof + ); + }else{ + rc = fts3SegReaderRequire(pReader, pReader->aDoclist, FTS3_VARINT_MAX); + if( rc==SQLITE_OK ){ + int n = sqlite3Fts3GetVarint(pReader->aDoclist, &pReader->iDocid); + pReader->pOffsetList = &pReader->aDoclist[n]; + } + } + return rc; +} /* -** Extract the next token from a tokenization cursor. The cursor must -** have been opened by a prior call to porterOpen(). +** Advance the SegReader to point to the next docid in the doclist +** associated with the current term. +** +** If arguments ppOffsetList and pnOffsetList are not NULL, then +** *ppOffsetList is set to point to the first column-offset list +** in the doclist entry (i.e. immediately past the docid varint). +** *pnOffsetList is set to the length of the set of column-offset +** lists, not including the nul-terminator byte. For example: */ -static int porterNext( - sqlite3_tokenizer_cursor *pCursor, /* Cursor returned by porterOpen */ - const char **pzToken, /* OUT: *pzToken is the token text */ - int *pnBytes, /* OUT: Number of bytes in token */ - int *piStartOffset, /* OUT: Starting offset of token */ - int *piEndOffset, /* OUT: Ending offset of token */ - int *piPosition /* OUT: Position integer of token */ +static int fts3SegReaderNextDocid( + Fts3Table *pTab, + Fts3SegReader *pReader, /* Reader to advance to next docid */ + char **ppOffsetList, /* OUT: Pointer to current position-list */ + int *pnOffsetList /* OUT: Length of *ppOffsetList in bytes */ ){ - porter_tokenizer_cursor *c = (porter_tokenizer_cursor *) pCursor; - const char *z = c->zInput; + int rc = SQLITE_OK; + char *p = pReader->pOffsetList; + char c = 0; - while( c->iOffsetnInput ){ - int iStartOffset, ch; + assert( p ); - /* Scan past delimiter characters */ - while( c->iOffsetnInput && isDelim(z[c->iOffset]) ){ - c->iOffset++; + if( pTab->bDescIdx && fts3SegReaderIsPending(pReader) ){ + /* A pending-terms seg-reader for an FTS4 table that uses order=desc. + ** Pending-terms doclists are always built up in ascending order, so + ** we have to iterate through them backwards here. */ + u8 bEof = 0; + if( ppOffsetList ){ + *ppOffsetList = pReader->pOffsetList; + *pnOffsetList = pReader->nOffsetList - 1; + } + sqlite3Fts3DoclistPrev(0, + pReader->aDoclist, pReader->nDoclist, &p, &pReader->iDocid, + &pReader->nOffsetList, &bEof + ); + if( bEof ){ + pReader->pOffsetList = 0; + }else{ + pReader->pOffsetList = p; } + }else{ + char *pEnd = &pReader->aDoclist[pReader->nDoclist]; - /* Count non-delimiter characters. */ - iStartOffset = c->iOffset; - while( c->iOffsetnInput && !isDelim(z[c->iOffset]) ){ - c->iOffset++; + /* Pointer p currently points at the first byte of an offset list. The + ** following block advances it to point one byte past the end of + ** the same offset list. */ + while( 1 ){ + + /* The following line of code (and the "p++" below the while() loop) is + ** normally all that is required to move pointer p to the desired + ** position. The exception is if this node is being loaded from disk + ** incrementally and pointer "p" now points to the first byte past + ** the populated part of pReader->aNode[]. + */ + while( *p | c ) c = *p++ & 0x80; + assert( *p==0 ); + + if( pReader->pBlob==0 || p<&pReader->aNode[pReader->nPopulate] ) break; + rc = fts3SegReaderIncrRead(pReader); + if( rc!=SQLITE_OK ) return rc; + } + p++; + + /* If required, populate the output variables with a pointer to and the + ** size of the previous offset-list. + */ + if( ppOffsetList ){ + *ppOffsetList = pReader->pOffsetList; + *pnOffsetList = (int)(p - pReader->pOffsetList - 1); } - if( c->iOffset>iStartOffset ){ - int n = c->iOffset-iStartOffset; - if( n>c->nAllocated ){ - char *pNew; - c->nAllocated = n+20; - pNew = sqlite3_realloc(c->zToken, c->nAllocated); - if( !pNew ) return SQLITE_NOMEM; - c->zToken = pNew; + /* List may have been edited in place by fts3EvalNearTrim() */ + while( p=pEnd ){ + pReader->pOffsetList = 0; + }else{ + rc = fts3SegReaderRequire(pReader, p, FTS3_VARINT_MAX); + if( rc==SQLITE_OK ){ + u64 iDelta; + pReader->pOffsetList = p + sqlite3Fts3GetVarintU(p, &iDelta); + if( pTab->bDescIdx ){ + pReader->iDocid = (i64)((u64)pReader->iDocid - iDelta); + }else{ + pReader->iDocid = (i64)((u64)pReader->iDocid + iDelta); + } } - porter_stemmer(&z[iStartOffset], n, c->zToken, pnBytes); - *pzToken = c->zToken; - *piStartOffset = iStartOffset; - *piEndOffset = c->iOffset; - *piPosition = c->iToken++; - return SQLITE_OK; } } - return SQLITE_DONE; + + return rc; } -/* -** The set of routines that implement the porter-stemmer tokenizer -*/ -static const sqlite3_tokenizer_module porterTokenizerModule = { - 0, - porterCreate, - porterDestroy, - porterOpen, - porterClose, - porterNext, - 0 -}; -/* -** Allocate a new porter tokenizer. Return a pointer to the new -** tokenizer in *ppModule -*/ -SQLITE_PRIVATE void sqlite3Fts3PorterTokenizerModule( - sqlite3_tokenizer_module const**ppModule +SQLITE_PRIVATE int sqlite3Fts3MsrOvfl( + Fts3Cursor *pCsr, + Fts3MultiSegReader *pMsr, + int *pnOvfl ){ - *ppModule = &porterTokenizerModule; -} + Fts3Table *p = (Fts3Table*)pCsr->base.pVtab; + int nOvfl = 0; + int ii; + int rc = SQLITE_OK; + int pgsz = p->nPgsz; -#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ + assert( p->bFts4 ); + assert( pgsz>0 ); + + for(ii=0; rc==SQLITE_OK && iinSegment; ii++){ + Fts3SegReader *pReader = pMsr->apSegment[ii]; + if( !fts3SegReaderIsPending(pReader) + && !fts3SegReaderIsRootOnly(pReader) + ){ + sqlite3_int64 jj; + for(jj=pReader->iStartBlock; jj<=pReader->iLeafEndBlock; jj++){ + int nBlob; + rc = sqlite3Fts3ReadBlock(p, jj, 0, &nBlob, 0); + if( rc!=SQLITE_OK ) break; + if( (nBlob+35)>pgsz ){ + nOvfl += (nBlob + 34)/pgsz; + } + } + } + } + *pnOvfl = nOvfl; + return rc; +} -/************** End of fts3_porter.c *****************************************/ -/************** Begin file fts3_tokenizer.c **********************************/ /* -** 2007 June 22 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -****************************************************************************** -** -** This is part of an SQLite module implementing full-text search. -** This particular file implements the generic tokenizer interface. +** Free all allocations associated with the iterator passed as the +** second argument. */ +SQLITE_PRIVATE void sqlite3Fts3SegReaderFree(Fts3SegReader *pReader){ + if( pReader ){ + if( !fts3SegReaderIsPending(pReader) ){ + sqlite3_free(pReader->zTerm); + } + if( !fts3SegReaderIsRootOnly(pReader) ){ + sqlite3_free(pReader->aNode); + } + sqlite3_blob_close(pReader->pBlob); + } + sqlite3_free(pReader); +} /* -** The code in this file is only compiled if: -** -** * The FTS3 module is being built as an extension -** (in which case SQLITE_CORE is not defined), or -** -** * The FTS3 module is being built into the core of -** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). +** Allocate a new SegReader object. */ -/* #include "fts3Int.h" */ -#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) +SQLITE_PRIVATE int sqlite3Fts3SegReaderNew( + int iAge, /* Segment "age". */ + int bLookup, /* True for a lookup only */ + sqlite3_int64 iStartLeaf, /* First leaf to traverse */ + sqlite3_int64 iEndLeaf, /* Final leaf to traverse */ + sqlite3_int64 iEndBlock, /* Final block of segment */ + const char *zRoot, /* Buffer containing root node */ + int nRoot, /* Size of buffer containing root node */ + Fts3SegReader **ppReader /* OUT: Allocated Fts3SegReader */ +){ + Fts3SegReader *pReader; /* Newly allocated SegReader object */ + int nExtra = 0; /* Bytes to allocate segment root node */ -/* #include */ -/* #include */ + assert( zRoot!=0 || nRoot==0 ); +#ifdef CORRUPT_DB + assert( zRoot!=0 || CORRUPT_DB ); +#endif + + if( iStartLeaf==0 ){ + if( iEndLeaf!=0 ) return FTS_CORRUPT_VTAB; + nExtra = nRoot + FTS3_NODE_PADDING; + } + + pReader = (Fts3SegReader *)sqlite3_malloc(sizeof(Fts3SegReader) + nExtra); + if( !pReader ){ + return SQLITE_NOMEM; + } + memset(pReader, 0, sizeof(Fts3SegReader)); + pReader->iIdx = iAge; + pReader->bLookup = bLookup!=0; + pReader->iStartBlock = iStartLeaf; + pReader->iLeafEndBlock = iEndLeaf; + pReader->iEndBlock = iEndBlock; + + if( nExtra ){ + /* The entire segment is stored in the root node. */ + pReader->aNode = (char *)&pReader[1]; + pReader->rootOnly = 1; + pReader->nNode = nRoot; + if( nRoot ) memcpy(pReader->aNode, zRoot, nRoot); + memset(&pReader->aNode[nRoot], 0, FTS3_NODE_PADDING); + }else{ + pReader->iCurrentBlock = iStartLeaf-1; + } + *ppReader = pReader; + return SQLITE_OK; +} /* -** Return true if the two-argument version of fts3_tokenizer() -** has been activated via a prior call to sqlite3_db_config(db, -** SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER, 1, 0); +** This is a comparison function used as a qsort() callback when sorting +** an array of pending terms by term. This occurs as part of flushing +** the contents of the pending-terms hash table to the database. */ -static int fts3TokenizerEnabled(sqlite3_context *context){ - sqlite3 *db = sqlite3_context_db_handle(context); - int isEnabled = 0; - sqlite3_db_config(db,SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER,-1,&isEnabled); - return isEnabled; +static int SQLITE_CDECL fts3CompareElemByTerm( + const void *lhs, + const void *rhs +){ + char *z1 = fts3HashKey(*(Fts3HashElem **)lhs); + char *z2 = fts3HashKey(*(Fts3HashElem **)rhs); + int n1 = fts3HashKeysize(*(Fts3HashElem **)lhs); + int n2 = fts3HashKeysize(*(Fts3HashElem **)rhs); + + int n = (n1(); -** SELECT (, ); +** If the isPrefixIter parameter is zero, then the returned SegReader iterates +** through each term in the pending-terms table. Or, if isPrefixIter is +** non-zero, it iterates through each term and its prefixes. For example, if +** the pending terms hash table contains the terms "sqlite", "mysql" and +** "firebird", then the iterator visits the following 'terms' (in the order +** shown): ** -** where is the name passed as the second argument -** to the sqlite3Fts3InitHashTable() function (e.g. 'fts3_tokenizer'). +** f fi fir fire fireb firebi firebir firebird +** m my mys mysq mysql +** s sq sql sqli sqlit sqlite ** -** If the argument is specified, it must be a blob value -** containing a pointer to be stored as the hash data corresponding -** to the string . If is not specified, then -** the string must already exist in the has table. Otherwise, -** an error is returned. +** Whereas if isPrefixIter is zero, the terms visited are: ** -** Whether or not the argument is specified, the value returned -** is a blob containing the pointer stored as the hash data corresponding -** to string (after the hash-table is updated, if applicable). +** firebird mysql sqlite */ -static void fts3TokenizerFunc( - sqlite3_context *context, - int argc, - sqlite3_value **argv +SQLITE_PRIVATE int sqlite3Fts3SegReaderPending( + Fts3Table *p, /* Virtual table handle */ + int iIndex, /* Index for p->aIndex */ + const char *zTerm, /* Term to search for */ + int nTerm, /* Size of buffer zTerm */ + int bPrefix, /* True for a prefix iterator */ + Fts3SegReader **ppReader /* OUT: SegReader for pending-terms */ ){ + Fts3SegReader *pReader = 0; /* Fts3SegReader object to return */ + Fts3HashElem *pE; /* Iterator variable */ + Fts3HashElem **aElem = 0; /* Array of term hash entries to scan */ + int nElem = 0; /* Size of array at aElem */ + int rc = SQLITE_OK; /* Return Code */ Fts3Hash *pHash; - void *pPtr = 0; - const unsigned char *zName; - int nName; - assert( argc==1 || argc==2 ); - - pHash = (Fts3Hash *)sqlite3_user_data(context); + pHash = &p->aIndex[iIndex].hPending; + if( bPrefix ){ + int nAlloc = 0; /* Size of allocated array at aElem */ - zName = sqlite3_value_text(argv[0]); - nName = sqlite3_value_bytes(argv[0])+1; + for(pE=fts3HashFirst(pHash); pE; pE=fts3HashNext(pE)){ + char *zKey = (char *)fts3HashKey(pE); + int nKey = fts3HashKeysize(pE); + if( nTerm==0 || (nKey>=nTerm && 0==memcmp(zKey, zTerm, nTerm)) ){ + if( nElem==nAlloc ){ + Fts3HashElem **aElem2; + nAlloc += 16; + aElem2 = (Fts3HashElem **)sqlite3_realloc( + aElem, nAlloc*sizeof(Fts3HashElem *) + ); + if( !aElem2 ){ + rc = SQLITE_NOMEM; + nElem = 0; + break; + } + aElem = aElem2; + } - if( argc==2 ){ - if( fts3TokenizerEnabled(context) ){ - void *pOld; - int n = sqlite3_value_bytes(argv[1]); - if( zName==0 || n!=sizeof(pPtr) ){ - sqlite3_result_error(context, "argument type mismatch", -1); - return; - } - pPtr = *(void **)sqlite3_value_blob(argv[1]); - pOld = sqlite3Fts3HashInsert(pHash, (void *)zName, nName, pPtr); - if( pOld==pPtr ){ - sqlite3_result_error(context, "out of memory", -1); + aElem[nElem++] = pE; } - }else{ - sqlite3_result_error(context, "fts3tokenize disabled", -1); - return; } - }else{ - if( zName ){ - pPtr = sqlite3Fts3HashFind(pHash, zName, nName); + + /* If more than one term matches the prefix, sort the Fts3HashElem + ** objects in term order using qsort(). This uses the same comparison + ** callback as is used when flushing terms to disk. + */ + if( nElem>1 ){ + qsort(aElem, nElem, sizeof(Fts3HashElem *), fts3CompareElemByTerm); } - if( !pPtr ){ - char *zErr = sqlite3_mprintf("unknown tokenizer: %s", zName); - sqlite3_result_error(context, zErr, -1); - sqlite3_free(zErr); - return; + + }else{ + /* The query is a simple term lookup that matches at most one term in + ** the index. All that is required is a straight hash-lookup. + ** + ** Because the stack address of pE may be accessed via the aElem pointer + ** below, the "Fts3HashElem *pE" must be declared so that it is valid + ** within this entire function, not just this "else{...}" block. + */ + pE = fts3HashFindElem(pHash, zTerm, nTerm); + if( pE ){ + aElem = &pE; + nElem = 1; } } - sqlite3_result_blob(context, (void *)&pPtr, sizeof(pPtr), SQLITE_TRANSIENT); -} - -SQLITE_PRIVATE int sqlite3Fts3IsIdChar(char c){ - static const char isFtsIdChar[] = { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x */ - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 1x */ - 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 2x */ - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 3x */ - 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4x */ - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 5x */ - 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 6x */ - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */ - }; - return (c&0x80 || isFtsIdChar[(int)(c)]); -} - -SQLITE_PRIVATE const char *sqlite3Fts3NextToken(const char *zStr, int *pn){ - const char *z1; - const char *z2 = 0; - - /* Find the start of the next token. */ - z1 = zStr; - while( z2==0 ){ - char c = *z1; - switch( c ){ - case '\0': return 0; /* No more tokens here */ - case '\'': - case '"': - case '`': { - z2 = z1; - while( *++z2 && (*z2!=c || *++z2==c) ); - break; - } - case '[': - z2 = &z1[1]; - while( *z2 && z2[0]!=']' ) z2++; - if( *z2 ) z2++; - break; - default: - if( sqlite3Fts3IsIdChar(*z1) ){ - z2 = &z1[1]; - while( sqlite3Fts3IsIdChar(*z2) ) z2++; - }else{ - z1++; - } + if( nElem>0 ){ + sqlite3_int64 nByte; + nByte = sizeof(Fts3SegReader) + (nElem+1)*sizeof(Fts3HashElem *); + pReader = (Fts3SegReader *)sqlite3_malloc64(nByte); + if( !pReader ){ + rc = SQLITE_NOMEM; + }else{ + memset(pReader, 0, nByte); + pReader->iIdx = 0x7FFFFFFF; + pReader->ppNextElem = (Fts3HashElem **)&pReader[1]; + memcpy(pReader->ppNextElem, aElem, nElem*sizeof(Fts3HashElem *)); } } - *pn = (int)(z2-z1); - return z1; + if( bPrefix ){ + sqlite3_free(aElem); + } + *ppReader = pReader; + return rc; } -SQLITE_PRIVATE int sqlite3Fts3InitTokenizer( - Fts3Hash *pHash, /* Tokenizer hash table */ - const char *zArg, /* Tokenizer name */ - sqlite3_tokenizer **ppTok, /* OUT: Tokenizer (if applicable) */ - char **pzErr /* OUT: Set to malloced error message */ -){ +/* +** Compare the entries pointed to by two Fts3SegReader structures. +** Comparison is as follows: +** +** 1) EOF is greater than not EOF. +** +** 2) The current terms (if any) are compared using memcmp(). If one +** term is a prefix of another, the longer term is considered the +** larger. +** +** 3) By segment age. An older segment is considered larger. +*/ +static int fts3SegReaderCmp(Fts3SegReader *pLhs, Fts3SegReader *pRhs){ int rc; - char *z = (char *)zArg; - int n = 0; - char *zCopy; - char *zEnd; /* Pointer to nul-term of zCopy */ - sqlite3_tokenizer_module *m; - - zCopy = sqlite3_mprintf("%s", zArg); - if( !zCopy ) return SQLITE_NOMEM; - zEnd = &zCopy[strlen(zCopy)]; - - z = (char *)sqlite3Fts3NextToken(zCopy, &n); - if( z==0 ){ - assert( n==0 ); - z = zCopy; - } - z[n] = '\0'; - sqlite3Fts3Dequote(z); - - m = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash,z,(int)strlen(z)+1); - if( !m ){ - sqlite3Fts3ErrMsg(pzErr, "unknown tokenizer: %s", z); - rc = SQLITE_ERROR; - }else{ - char const **aArg = 0; - int iArg = 0; - z = &z[n+1]; - while( zxCreate(iArg, aArg, ppTok); - assert( rc!=SQLITE_OK || *ppTok ); - if( rc!=SQLITE_OK ){ - sqlite3Fts3ErrMsg(pzErr, "unknown tokenizer"); + if( pLhs->aNode && pRhs->aNode ){ + int rc2 = pLhs->nTerm - pRhs->nTerm; + if( rc2<0 ){ + rc = memcmp(pLhs->zTerm, pRhs->zTerm, pLhs->nTerm); }else{ - (*ppTok)->pModule = m; + rc = memcmp(pLhs->zTerm, pRhs->zTerm, pRhs->nTerm); } - sqlite3_free((void *)aArg); + if( rc==0 ){ + rc = rc2; + } + }else{ + rc = (pLhs->aNode==0) - (pRhs->aNode==0); } - - sqlite3_free(zCopy); + if( rc==0 ){ + rc = pRhs->iIdx - pLhs->iIdx; + } + assert( rc!=0 ); return rc; } - -#ifdef SQLITE_TEST - -#if defined(INCLUDE_SQLITE_TCL_H) -# include "sqlite_tcl.h" -#else -# include "tcl.h" -#endif -/* #include */ - /* -** Implementation of a special SQL scalar function for testing tokenizers -** designed to be used in concert with the Tcl testing framework. This -** function must be called with two or more arguments: -** -** SELECT (, ..., ); -** -** where is the name passed as the second argument -** to the sqlite3Fts3InitHashTable() function (e.g. 'fts3_tokenizer') -** concatenated with the string '_test' (e.g. 'fts3_tokenizer_test'). +** A different comparison function for SegReader structures. In this +** version, it is assumed that each SegReader points to an entry in +** a doclist for identical terms. Comparison is made as follows: ** -** The return value is a string that may be interpreted as a Tcl -** list. For each token in the , three elements are -** added to the returned list. The first is the token position, the -** second is the token text (folded, stemmed, etc.) and the third is the -** substring of associated with the token. For example, -** using the built-in "simple" tokenizer: +** 1) EOF (end of doclist in this case) is greater than not EOF. ** -** SELECT fts_tokenizer_test('simple', 'I don't see how'); +** 2) By current docid. ** -** will return the string: +** 3) By segment age. An older segment is considered larger. +*/ +static int fts3SegReaderDoclistCmp(Fts3SegReader *pLhs, Fts3SegReader *pRhs){ + int rc = (pLhs->pOffsetList==0)-(pRhs->pOffsetList==0); + if( rc==0 ){ + if( pLhs->iDocid==pRhs->iDocid ){ + rc = pRhs->iIdx - pLhs->iIdx; + }else{ + rc = (pLhs->iDocid > pRhs->iDocid) ? 1 : -1; + } + } + assert( pLhs->aNode && pRhs->aNode ); + return rc; +} +static int fts3SegReaderDoclistCmpRev(Fts3SegReader *pLhs, Fts3SegReader *pRhs){ + int rc = (pLhs->pOffsetList==0)-(pRhs->pOffsetList==0); + if( rc==0 ){ + if( pLhs->iDocid==pRhs->iDocid ){ + rc = pRhs->iIdx - pLhs->iIdx; + }else{ + rc = (pLhs->iDocid < pRhs->iDocid) ? 1 : -1; + } + } + assert( pLhs->aNode && pRhs->aNode ); + return rc; +} + +/* +** Compare the term that the Fts3SegReader object passed as the first argument +** points to with the term specified by arguments zTerm and nTerm. ** -** "{0 i I 1 dont don't 2 see see 3 how how}" -** +** If the pSeg iterator is already at EOF, return 0. Otherwise, return +** -ve if the pSeg term is less than zTerm/nTerm, 0 if the two terms are +** equal, or +ve if the pSeg term is greater than zTerm/nTerm. */ -static void testFunc( - sqlite3_context *context, - int argc, - sqlite3_value **argv +static int fts3SegReaderTermCmp( + Fts3SegReader *pSeg, /* Segment reader object */ + const char *zTerm, /* Term to compare to */ + int nTerm /* Size of term zTerm in bytes */ ){ - Fts3Hash *pHash; - sqlite3_tokenizer_module *p; - sqlite3_tokenizer *pTokenizer = 0; - sqlite3_tokenizer_cursor *pCsr = 0; - - const char *zErr = 0; - - const char *zName; - int nName; - const char *zInput; - int nInput; - - const char *azArg[64]; - - const char *zToken; - int nToken = 0; - int iStart = 0; - int iEnd = 0; - int iPos = 0; - int i; - - Tcl_Obj *pRet; - - if( argc<2 ){ - sqlite3_result_error(context, "insufficient arguments", -1); - return; + int res = 0; + if( pSeg->aNode ){ + if( pSeg->nTerm>nTerm ){ + res = memcmp(pSeg->zTerm, zTerm, nTerm); + }else{ + res = memcmp(pSeg->zTerm, zTerm, pSeg->nTerm); + } + if( res==0 ){ + res = pSeg->nTerm-nTerm; + } } + return res; +} - nName = sqlite3_value_bytes(argv[0]); - zName = (const char *)sqlite3_value_text(argv[0]); - nInput = sqlite3_value_bytes(argv[argc-1]); - zInput = (const char *)sqlite3_value_text(argv[argc-1]); +/* +** Argument apSegment is an array of nSegment elements. It is known that +** the final (nSegment-nSuspect) members are already in sorted order +** (according to the comparison function provided). This function shuffles +** the array around until all entries are in sorted order. +*/ +static void fts3SegReaderSort( + Fts3SegReader **apSegment, /* Array to sort entries of */ + int nSegment, /* Size of apSegment array */ + int nSuspect, /* Unsorted entry count */ + int (*xCmp)(Fts3SegReader *, Fts3SegReader *) /* Comparison function */ +){ + int i; /* Iterator variable */ - pHash = (Fts3Hash *)sqlite3_user_data(context); - p = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash, zName, nName+1); + assert( nSuspect<=nSegment ); - if( !p ){ - char *zErr2 = sqlite3_mprintf("unknown tokenizer: %s", zName); - sqlite3_result_error(context, zErr2, -1); - sqlite3_free(zErr2); - return; + if( nSuspect==nSegment ) nSuspect--; + for(i=nSuspect-1; i>=0; i--){ + int j; + for(j=i; j<(nSegment-1); j++){ + Fts3SegReader *pTmp; + if( xCmp(apSegment[j], apSegment[j+1])<0 ) break; + pTmp = apSegment[j+1]; + apSegment[j+1] = apSegment[j]; + apSegment[j] = pTmp; + } } - pRet = Tcl_NewObj(); - Tcl_IncrRefCount(pRet); - - for(i=1; ixCreate(argc-2, azArg, &pTokenizer) ){ - zErr = "error in xCreate()"; - goto finish; - } - pTokenizer->pModule = p; - if( sqlite3Fts3OpenTokenizer(pTokenizer, 0, zInput, nInput, &pCsr) ){ - zErr = "error in xOpen()"; - goto finish; +/* +** Insert a record into the %_segments table. +*/ +static int fts3WriteSegment( + Fts3Table *p, /* Virtual table handle */ + sqlite3_int64 iBlock, /* Block id for new block */ + char *z, /* Pointer to buffer containing block data */ + int n /* Size of buffer z in bytes */ +){ + sqlite3_stmt *pStmt; + int rc = fts3SqlStmt(p, SQL_INSERT_SEGMENTS, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pStmt, 1, iBlock); + sqlite3_bind_blob(pStmt, 2, z, n, SQLITE_STATIC); + sqlite3_step(pStmt); + rc = sqlite3_reset(pStmt); + sqlite3_bind_null(pStmt, 2); } + return rc; +} - while( SQLITE_OK==p->xNext(pCsr, &zToken, &nToken, &iStart, &iEnd, &iPos) ){ - Tcl_ListObjAppendElement(0, pRet, Tcl_NewIntObj(iPos)); - Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj(zToken, nToken)); - zToken = &zInput[iStart]; - nToken = iEnd-iStart; - Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj(zToken, nToken)); - } +/* +** Find the largest relative level number in the table. If successful, set +** *pnMax to this value and return SQLITE_OK. Otherwise, if an error occurs, +** set *pnMax to zero and return an SQLite error code. +*/ +SQLITE_PRIVATE int sqlite3Fts3MaxLevel(Fts3Table *p, int *pnMax){ + int rc; + int mxLevel = 0; + sqlite3_stmt *pStmt = 0; - if( SQLITE_OK!=p->xClose(pCsr) ){ - zErr = "error in xClose()"; - goto finish; - } - if( SQLITE_OK!=p->xDestroy(pTokenizer) ){ - zErr = "error in xDestroy()"; - goto finish; + rc = fts3SqlStmt(p, SQL_SELECT_MXLEVEL, &pStmt, 0); + if( rc==SQLITE_OK ){ + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + mxLevel = sqlite3_column_int(pStmt, 0); + } + rc = sqlite3_reset(pStmt); } + *pnMax = mxLevel; + return rc; +} -finish: - if( zErr ){ - sqlite3_result_error(context, zErr, -1); - }else{ - sqlite3_result_text(context, Tcl_GetString(pRet), -1, SQLITE_TRANSIENT); +/* +** Insert a record into the %_segdir table. +*/ +static int fts3WriteSegdir( + Fts3Table *p, /* Virtual table handle */ + sqlite3_int64 iLevel, /* Value for "level" field (absolute level) */ + int iIdx, /* Value for "idx" field */ + sqlite3_int64 iStartBlock, /* Value for "start_block" field */ + sqlite3_int64 iLeafEndBlock, /* Value for "leaves_end_block" field */ + sqlite3_int64 iEndBlock, /* Value for "end_block" field */ + sqlite3_int64 nLeafData, /* Bytes of leaf data in segment */ + char *zRoot, /* Blob value for "root" field */ + int nRoot /* Number of bytes in buffer zRoot */ +){ + sqlite3_stmt *pStmt; + int rc = fts3SqlStmt(p, SQL_INSERT_SEGDIR, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pStmt, 1, iLevel); + sqlite3_bind_int(pStmt, 2, iIdx); + sqlite3_bind_int64(pStmt, 3, iStartBlock); + sqlite3_bind_int64(pStmt, 4, iLeafEndBlock); + if( nLeafData==0 ){ + sqlite3_bind_int64(pStmt, 5, iEndBlock); + }else{ + char *zEnd = sqlite3_mprintf("%lld %lld", iEndBlock, nLeafData); + if( !zEnd ) return SQLITE_NOMEM; + sqlite3_bind_text(pStmt, 5, zEnd, -1, sqlite3_free); + } + sqlite3_bind_blob(pStmt, 6, zRoot, nRoot, SQLITE_STATIC); + sqlite3_step(pStmt); + rc = sqlite3_reset(pStmt); + sqlite3_bind_null(pStmt, 6); } - Tcl_DecrRefCount(pRet); + return rc; } -static -int registerTokenizer( - sqlite3 *db, - char *zName, - const sqlite3_tokenizer_module *p +/* +** Return the size of the common prefix (if any) shared by zPrev and +** zNext, in bytes. For example, +** +** fts3PrefixCompress("abc", 3, "abcdef", 6) // returns 3 +** fts3PrefixCompress("abX", 3, "abcdef", 6) // returns 2 +** fts3PrefixCompress("abX", 3, "Xbcdef", 6) // returns 0 +*/ +static int fts3PrefixCompress( + const char *zPrev, /* Buffer containing previous term */ + int nPrev, /* Size of buffer zPrev in bytes */ + const char *zNext, /* Buffer containing next term */ + int nNext /* Size of buffer zNext in bytes */ +){ + int n; + UNUSED_PARAMETER(nNext); + for(n=0; nnData; /* Current size of node in bytes */ + int nReq = nData; /* Required space after adding zTerm */ + int nPrefix; /* Number of bytes of prefix compression */ + int nSuffix; /* Suffix length */ - sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC); - sqlite3_bind_blob(pStmt, 2, &p, sizeof(p), SQLITE_STATIC); - sqlite3_step(pStmt); + nPrefix = fts3PrefixCompress(pTree->zTerm, pTree->nTerm, zTerm, nTerm); + nSuffix = nTerm-nPrefix; - return sqlite3_finalize(pStmt); -} + /* If nSuffix is zero or less, then zTerm/nTerm must be a prefix of + ** pWriter->zTerm/pWriter->nTerm. i.e. must be equal to or less than when + ** compared with BINARY collation. This indicates corruption. */ + if( nSuffix<=0 ) return FTS_CORRUPT_VTAB; + nReq += sqlite3Fts3VarintLen(nPrefix)+sqlite3Fts3VarintLen(nSuffix)+nSuffix; + if( nReq<=p->nNodeSize || !pTree->zTerm ){ -static -int queryTokenizer( - sqlite3 *db, - char *zName, - const sqlite3_tokenizer_module **pp -){ - int rc; - sqlite3_stmt *pStmt; - const char zSql[] = "SELECT fts3_tokenizer(?)"; + if( nReq>p->nNodeSize ){ + /* An unusual case: this is the first term to be added to the node + ** and the static node buffer (p->nNodeSize bytes) is not large + ** enough. Use a separately malloced buffer instead This wastes + ** p->nNodeSize bytes, but since this scenario only comes about when + ** the database contain two terms that share a prefix of almost 2KB, + ** this is not expected to be a serious problem. + */ + assert( pTree->aData==(char *)&pTree[1] ); + pTree->aData = (char *)sqlite3_malloc(nReq); + if( !pTree->aData ){ + return SQLITE_NOMEM; + } + } - *pp = 0; - rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); - if( rc!=SQLITE_OK ){ - return rc; + if( pTree->zTerm ){ + /* There is no prefix-length field for first term in a node */ + nData += sqlite3Fts3PutVarint(&pTree->aData[nData], nPrefix); + } + + nData += sqlite3Fts3PutVarint(&pTree->aData[nData], nSuffix); + memcpy(&pTree->aData[nData], &zTerm[nPrefix], nSuffix); + pTree->nData = nData + nSuffix; + pTree->nEntry++; + + if( isCopyTerm ){ + if( pTree->nMalloczMalloc, nTerm*2); + if( !zNew ){ + return SQLITE_NOMEM; + } + pTree->nMalloc = nTerm*2; + pTree->zMalloc = zNew; + } + pTree->zTerm = pTree->zMalloc; + memcpy(pTree->zTerm, zTerm, nTerm); + pTree->nTerm = nTerm; + }else{ + pTree->zTerm = (char *)zTerm; + pTree->nTerm = nTerm; + } + return SQLITE_OK; + } + } + + /* If control flows to here, it was not possible to append zTerm to the + ** current node. Create a new node (a right-sibling of the current node). + ** If this is the first node in the tree, the term is added to it. + ** + ** Otherwise, the term is not added to the new node, it is left empty for + ** now. Instead, the term is inserted into the parent of pTree. If pTree + ** has no parent, one is created here. + */ + pNew = (SegmentNode *)sqlite3_malloc(sizeof(SegmentNode) + p->nNodeSize); + if( !pNew ){ + return SQLITE_NOMEM; } + memset(pNew, 0, sizeof(SegmentNode)); + pNew->nData = 1 + FTS3_VARINT_MAX; + pNew->aData = (char *)&pNew[1]; - sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC); - if( SQLITE_ROW==sqlite3_step(pStmt) ){ - if( sqlite3_column_type(pStmt, 0)==SQLITE_BLOB ){ - memcpy((void *)pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp)); + if( pTree ){ + SegmentNode *pParent = pTree->pParent; + rc = fts3NodeAddTerm(p, &pParent, isCopyTerm, zTerm, nTerm); + if( pTree->pParent==0 ){ + pTree->pParent = pParent; } + pTree->pRight = pNew; + pNew->pLeftmost = pTree->pLeftmost; + pNew->pParent = pParent; + pNew->zMalloc = pTree->zMalloc; + pNew->nMalloc = pTree->nMalloc; + pTree->zMalloc = 0; + }else{ + pNew->pLeftmost = pNew; + rc = fts3NodeAddTerm(p, &pNew, isCopyTerm, zTerm, nTerm); } - return sqlite3_finalize(pStmt); + *ppTree = pNew; + return rc; } -SQLITE_PRIVATE void sqlite3Fts3SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule); - /* -** Implementation of the scalar function fts3_tokenizer_internal_test(). -** This function is used for testing only, it is not included in the -** build unless SQLITE_TEST is defined. -** -** The purpose of this is to test that the fts3_tokenizer() function -** can be used as designed by the C-code in the queryTokenizer and -** registerTokenizer() functions above. These two functions are repeated -** in the README.tokenizer file as an example, so it is important to -** test them. -** -** To run the tests, evaluate the fts3_tokenizer_internal_test() scalar -** function with no arguments. An assert() will fail if a problem is -** detected. i.e.: -** -** SELECT fts3_tokenizer_internal_test(); -** +** Helper function for fts3NodeWrite(). */ -static void intTestFunc( - sqlite3_context *context, - int argc, - sqlite3_value **argv +static int fts3TreeFinishNode( + SegmentNode *pTree, + int iHeight, + sqlite3_int64 iLeftChild ){ - int rc; - const sqlite3_tokenizer_module *p1; - const sqlite3_tokenizer_module *p2; - sqlite3 *db = (sqlite3 *)sqlite3_user_data(context); - - UNUSED_PARAMETER(argc); - UNUSED_PARAMETER(argv); - - /* Test the query function */ - sqlite3Fts3SimpleTokenizerModule(&p1); - rc = queryTokenizer(db, "simple", &p2); - assert( rc==SQLITE_OK ); - assert( p1==p2 ); - rc = queryTokenizer(db, "nosuchtokenizer", &p2); - assert( rc==SQLITE_ERROR ); - assert( p2==0 ); - assert( 0==strcmp(sqlite3_errmsg(db), "unknown tokenizer: nosuchtokenizer") ); - - /* Test the storage function */ - if( fts3TokenizerEnabled(context) ){ - rc = registerTokenizer(db, "nosuchtokenizer", p1); - assert( rc==SQLITE_OK ); - rc = queryTokenizer(db, "nosuchtokenizer", &p2); - assert( rc==SQLITE_OK ); - assert( p2==p1 ); - } - - sqlite3_result_text(context, "ok", -1, SQLITE_STATIC); + int nStart; + assert( iHeight>=1 && iHeight<128 ); + nStart = FTS3_VARINT_MAX - sqlite3Fts3VarintLen(iLeftChild); + pTree->aData[nStart] = (char)iHeight; + sqlite3Fts3PutVarint(&pTree->aData[nStart+1], iLeftChild); + return nStart; } -#endif - /* -** Set up SQL objects in database db used to access the contents of -** the hash table pointed to by argument pHash. The hash table must -** been initialized to use string keys, and to take a private copy -** of the key when a value is inserted. i.e. by a call similar to: -** -** sqlite3Fts3HashInit(pHash, FTS3_HASH_STRING, 1); +** Write the buffer for the segment node pTree and all of its peers to the +** database. Then call this function recursively to write the parent of +** pTree and its peers to the database. ** -** This function adds a scalar function (see header comment above -** fts3TokenizerFunc() in this file for details) and, if ENABLE_TABLE is -** defined at compilation time, a temporary virtual table (see header -** comment above struct HashTableVtab) to the database schema. Both -** provide read/write access to the contents of *pHash. +** Except, if pTree is a root node, do not write it to the database. Instead, +** set output variables *paRoot and *pnRoot to contain the root node. ** -** The third argument to this function, zName, is used as the name -** of both the scalar and, if created, the virtual table. +** If successful, SQLITE_OK is returned and output variable *piLast is +** set to the largest blockid written to the database (or zero if no +** blocks were written to the db). Otherwise, an SQLite error code is +** returned. */ -SQLITE_PRIVATE int sqlite3Fts3InitHashTable( - sqlite3 *db, - Fts3Hash *pHash, - const char *zName +static int fts3NodeWrite( + Fts3Table *p, /* Virtual table handle */ + SegmentNode *pTree, /* SegmentNode handle */ + int iHeight, /* Height of this node in tree */ + sqlite3_int64 iLeaf, /* Block id of first leaf node */ + sqlite3_int64 iFree, /* Block id of next free slot in %_segments */ + sqlite3_int64 *piLast, /* OUT: Block id of last entry written */ + char **paRoot, /* OUT: Data for root node */ + int *pnRoot /* OUT: Size of root node in bytes */ ){ int rc = SQLITE_OK; - void *p = (void *)pHash; - const int any = SQLITE_ANY; - -#ifdef SQLITE_TEST - char *zTest = 0; - char *zTest2 = 0; - void *pdb = (void *)db; - zTest = sqlite3_mprintf("%s_test", zName); - zTest2 = sqlite3_mprintf("%s_internal_test", zName); - if( !zTest || !zTest2 ){ - rc = SQLITE_NOMEM; - } -#endif - if( SQLITE_OK==rc ){ - rc = sqlite3_create_function(db, zName, 1, any, p, fts3TokenizerFunc, 0, 0); - } - if( SQLITE_OK==rc ){ - rc = sqlite3_create_function(db, zName, 2, any, p, fts3TokenizerFunc, 0, 0); - } -#ifdef SQLITE_TEST - if( SQLITE_OK==rc ){ - rc = sqlite3_create_function(db, zTest, -1, any, p, testFunc, 0, 0); - } - if( SQLITE_OK==rc ){ - rc = sqlite3_create_function(db, zTest2, 0, any, pdb, intTestFunc, 0, 0); + if( !pTree->pParent ){ + /* Root node of the tree. */ + int nStart = fts3TreeFinishNode(pTree, iHeight, iLeaf); + *piLast = iFree-1; + *pnRoot = pTree->nData - nStart; + *paRoot = &pTree->aData[nStart]; + }else{ + SegmentNode *pIter; + sqlite3_int64 iNextFree = iFree; + sqlite3_int64 iNextLeaf = iLeaf; + for(pIter=pTree->pLeftmost; pIter && rc==SQLITE_OK; pIter=pIter->pRight){ + int nStart = fts3TreeFinishNode(pIter, iHeight, iNextLeaf); + int nWrite = pIter->nData - nStart; + + rc = fts3WriteSegment(p, iNextFree, &pIter->aData[nStart], nWrite); + iNextFree++; + iNextLeaf += (pIter->nEntry+1); + } + if( rc==SQLITE_OK ){ + assert( iNextLeaf==iFree ); + rc = fts3NodeWrite( + p, pTree->pParent, iHeight+1, iFree, iNextFree, piLast, paRoot, pnRoot + ); + } } -#endif - -#ifdef SQLITE_TEST - sqlite3_free(zTest); - sqlite3_free(zTest2); -#endif return rc; } -#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ - -/************** End of fts3_tokenizer.c **************************************/ -/************** Begin file fts3_tokenizer1.c *********************************/ /* -** 2006 Oct 10 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -****************************************************************************** -** -** Implementation of the "simple" full-text-search tokenizer. +** Free all memory allocations associated with the tree pTree. */ +static void fts3NodeFree(SegmentNode *pTree){ + if( pTree ){ + SegmentNode *p = pTree->pLeftmost; + fts3NodeFree(p->pParent); + while( p ){ + SegmentNode *pRight = p->pRight; + if( p->aData!=(char *)&p[1] ){ + sqlite3_free(p->aData); + } + assert( pRight==0 || p->zMalloc==0 ); + sqlite3_free(p->zMalloc); + sqlite3_free(p); + p = pRight; + } + } +} /* -** The code in this file is only compiled if: -** -** * The FTS3 module is being built as an extension -** (in which case SQLITE_CORE is not defined), or +** Add a term to the segment being constructed by the SegmentWriter object +** *ppWriter. When adding the first term to a segment, *ppWriter should +** be passed NULL. This function will allocate a new SegmentWriter object +** and return it via the input/output variable *ppWriter in this case. ** -** * The FTS3 module is being built into the core of -** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). +** If successful, SQLITE_OK is returned. Otherwise, an SQLite error code. */ -/* #include "fts3Int.h" */ -#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) +static int fts3SegWriterAdd( + Fts3Table *p, /* Virtual table handle */ + SegmentWriter **ppWriter, /* IN/OUT: SegmentWriter handle */ + int isCopyTerm, /* True if buffer zTerm must be copied */ + const char *zTerm, /* Pointer to buffer containing term */ + int nTerm, /* Size of term in bytes */ + const char *aDoclist, /* Pointer to buffer containing doclist */ + int nDoclist /* Size of doclist in bytes */ +){ + int nPrefix; /* Size of term prefix in bytes */ + int nSuffix; /* Size of term suffix in bytes */ + int nReq; /* Number of bytes required on leaf page */ + int nData; + SegmentWriter *pWriter = *ppWriter; -/* #include */ -/* #include */ -/* #include */ -/* #include */ + if( !pWriter ){ + int rc; + sqlite3_stmt *pStmt; -/* #include "fts3_tokenizer.h" */ + /* Allocate the SegmentWriter structure */ + pWriter = (SegmentWriter *)sqlite3_malloc(sizeof(SegmentWriter)); + if( !pWriter ) return SQLITE_NOMEM; + memset(pWriter, 0, sizeof(SegmentWriter)); + *ppWriter = pWriter; -typedef struct simple_tokenizer { - sqlite3_tokenizer base; - char delim[128]; /* flag ASCII delimiters */ -} simple_tokenizer; + /* Allocate a buffer in which to accumulate data */ + pWriter->aData = (char *)sqlite3_malloc(p->nNodeSize); + if( !pWriter->aData ) return SQLITE_NOMEM; + pWriter->nSize = p->nNodeSize; -typedef struct simple_tokenizer_cursor { - sqlite3_tokenizer_cursor base; - const char *pInput; /* input we are tokenizing */ - int nBytes; /* size of the input */ - int iOffset; /* current position in pInput */ - int iToken; /* index of next token to be returned */ - char *pToken; /* storage for current token */ - int nTokenAllocated; /* space allocated to zToken buffer */ -} simple_tokenizer_cursor; + /* Find the next free blockid in the %_segments table */ + rc = fts3SqlStmt(p, SQL_NEXT_SEGMENTS_ID, &pStmt, 0); + if( rc!=SQLITE_OK ) return rc; + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + pWriter->iFree = sqlite3_column_int64(pStmt, 0); + pWriter->iFirst = pWriter->iFree; + } + rc = sqlite3_reset(pStmt); + if( rc!=SQLITE_OK ) return rc; + } + nData = pWriter->nData; + nPrefix = fts3PrefixCompress(pWriter->zTerm, pWriter->nTerm, zTerm, nTerm); + nSuffix = nTerm-nPrefix; -static int simpleDelim(simple_tokenizer *t, unsigned char c){ - return c<0x80 && t->delim[c]; -} -static int fts3_isalnum(int x){ - return (x>='0' && x<='9') || (x>='A' && x<='Z') || (x>='a' && x<='z'); -} + /* If nSuffix is zero or less, then zTerm/nTerm must be a prefix of + ** pWriter->zTerm/pWriter->nTerm. i.e. must be equal to or less than when + ** compared with BINARY collation. This indicates corruption. */ + if( nSuffix<=0 ) return FTS_CORRUPT_VTAB; -/* -** Create a new tokenizer instance. -*/ -static int simpleCreate( - int argc, const char * const *argv, - sqlite3_tokenizer **ppTokenizer -){ - simple_tokenizer *t; + /* Figure out how many bytes are required by this new entry */ + nReq = sqlite3Fts3VarintLen(nPrefix) + /* varint containing prefix size */ + sqlite3Fts3VarintLen(nSuffix) + /* varint containing suffix size */ + nSuffix + /* Term suffix */ + sqlite3Fts3VarintLen(nDoclist) + /* Size of doclist */ + nDoclist; /* Doclist data */ - t = (simple_tokenizer *) sqlite3_malloc(sizeof(*t)); - if( t==NULL ) return SQLITE_NOMEM; - memset(t, 0, sizeof(*t)); + if( nData>0 && nData+nReq>p->nNodeSize ){ + int rc; - /* TODO(shess) Delimiters need to remain the same from run to run, - ** else we need to reindex. One solution would be a meta-table to - ** track such information in the database, then we'd only want this - ** information on the initial create. - */ - if( argc>1 ){ - int i, n = (int)strlen(argv[1]); - for(i=0; i=0x80 ){ - sqlite3_free(t); - return SQLITE_ERROR; - } - t->delim[ch] = 1; - } - } else { - /* Mark non-alphanumeric ASCII characters as delimiters */ - int i; - for(i=1; i<0x80; i++){ - t->delim[i] = !fts3_isalnum(i) ? -1 : 0; - } - } + /* The current leaf node is full. Write it out to the database. */ + if( pWriter->iFree==LARGEST_INT64 ) return FTS_CORRUPT_VTAB; + rc = fts3WriteSegment(p, pWriter->iFree++, pWriter->aData, nData); + if( rc!=SQLITE_OK ) return rc; + p->nLeafAdd++; - *ppTokenizer = &t->base; - return SQLITE_OK; -} + /* Add the current term to the interior node tree. The term added to + ** the interior tree must: + ** + ** a) be greater than the largest term on the leaf node just written + ** to the database (still available in pWriter->zTerm), and + ** + ** b) be less than or equal to the term about to be added to the new + ** leaf node (zTerm/nTerm). + ** + ** In other words, it must be the prefix of zTerm 1 byte longer than + ** the common prefix (if any) of zTerm and pWriter->zTerm. + */ + assert( nPrefixpTree, isCopyTerm, zTerm, nPrefix+1); + if( rc!=SQLITE_OK ) return rc; -/* -** Destroy a tokenizer -*/ -static int simpleDestroy(sqlite3_tokenizer *pTokenizer){ - sqlite3_free(pTokenizer); - return SQLITE_OK; -} + nData = 0; + pWriter->nTerm = 0; -/* -** Prepare to begin tokenizing a particular string. The input -** string to be tokenized is pInput[0..nBytes-1]. A cursor -** used to incrementally tokenize this string is returned in -** *ppCursor. -*/ -static int simpleOpen( - sqlite3_tokenizer *pTokenizer, /* The tokenizer */ - const char *pInput, int nBytes, /* String to be tokenized */ - sqlite3_tokenizer_cursor **ppCursor /* OUT: Tokenization cursor */ -){ - simple_tokenizer_cursor *c; + nPrefix = 0; + nSuffix = nTerm; + nReq = 1 + /* varint containing prefix size */ + sqlite3Fts3VarintLen(nTerm) + /* varint containing suffix size */ + nTerm + /* Term suffix */ + sqlite3Fts3VarintLen(nDoclist) + /* Size of doclist */ + nDoclist; /* Doclist data */ + } - UNUSED_PARAMETER(pTokenizer); + /* Increase the total number of bytes written to account for the new entry. */ + pWriter->nLeafData += nReq; - c = (simple_tokenizer_cursor *) sqlite3_malloc(sizeof(*c)); - if( c==NULL ) return SQLITE_NOMEM; + /* If the buffer currently allocated is too small for this entry, realloc + ** the buffer to make it large enough. + */ + if( nReq>pWriter->nSize ){ + char *aNew = sqlite3_realloc(pWriter->aData, nReq); + if( !aNew ) return SQLITE_NOMEM; + pWriter->aData = aNew; + pWriter->nSize = nReq; + } + assert( nData+nReq<=pWriter->nSize ); - c->pInput = pInput; - if( pInput==0 ){ - c->nBytes = 0; - }else if( nBytes<0 ){ - c->nBytes = (int)strlen(pInput); + /* Append the prefix-compressed term and doclist to the buffer. */ + nData += sqlite3Fts3PutVarint(&pWriter->aData[nData], nPrefix); + nData += sqlite3Fts3PutVarint(&pWriter->aData[nData], nSuffix); + assert( nSuffix>0 ); + memcpy(&pWriter->aData[nData], &zTerm[nPrefix], nSuffix); + nData += nSuffix; + nData += sqlite3Fts3PutVarint(&pWriter->aData[nData], nDoclist); + assert( nDoclist>0 ); + memcpy(&pWriter->aData[nData], aDoclist, nDoclist); + pWriter->nData = nData + nDoclist; + + /* Save the current term so that it can be used to prefix-compress the next. + ** If the isCopyTerm parameter is true, then the buffer pointed to by + ** zTerm is transient, so take a copy of the term data. Otherwise, just + ** store a copy of the pointer. + */ + if( isCopyTerm ){ + if( nTerm>pWriter->nMalloc ){ + char *zNew = sqlite3_realloc(pWriter->zMalloc, nTerm*2); + if( !zNew ){ + return SQLITE_NOMEM; + } + pWriter->nMalloc = nTerm*2; + pWriter->zMalloc = zNew; + pWriter->zTerm = zNew; + } + assert( pWriter->zTerm==pWriter->zMalloc ); + assert( nTerm>0 ); + memcpy(pWriter->zTerm, zTerm, nTerm); }else{ - c->nBytes = nBytes; + pWriter->zTerm = (char *)zTerm; } - c->iOffset = 0; /* start tokenizing at the beginning */ - c->iToken = 0; - c->pToken = NULL; /* no space allocated, yet. */ - c->nTokenAllocated = 0; - - *ppCursor = &c->base; - return SQLITE_OK; -} + pWriter->nTerm = nTerm; -/* -** Close a tokenization cursor previously opened by a call to -** simpleOpen() above. -*/ -static int simpleClose(sqlite3_tokenizer_cursor *pCursor){ - simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor; - sqlite3_free(c->pToken); - sqlite3_free(c); return SQLITE_OK; } /* -** Extract the next token from a tokenization cursor. The cursor must -** have been opened by a prior call to simpleOpen(). +** Flush all data associated with the SegmentWriter object pWriter to the +** database. This function must be called after all terms have been added +** to the segment using fts3SegWriterAdd(). If successful, SQLITE_OK is +** returned. Otherwise, an SQLite error code. */ -static int simpleNext( - sqlite3_tokenizer_cursor *pCursor, /* Cursor returned by simpleOpen */ - const char **ppToken, /* OUT: *ppToken is the token text */ - int *pnBytes, /* OUT: Number of bytes in token */ - int *piStartOffset, /* OUT: Starting offset of token */ - int *piEndOffset, /* OUT: Ending offset of token */ - int *piPosition /* OUT: Position integer of token */ +static int fts3SegWriterFlush( + Fts3Table *p, /* Virtual table handle */ + SegmentWriter *pWriter, /* SegmentWriter to flush to the db */ + sqlite3_int64 iLevel, /* Value for 'level' column of %_segdir */ + int iIdx /* Value for 'idx' column of %_segdir */ ){ - simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor; - simple_tokenizer *t = (simple_tokenizer *) pCursor->pTokenizer; - unsigned char *p = (unsigned char *)c->pInput; - - while( c->iOffsetnBytes ){ - int iStartOffset; - - /* Scan past delimiter characters */ - while( c->iOffsetnBytes && simpleDelim(t, p[c->iOffset]) ){ - c->iOffset++; - } + int rc; /* Return code */ + if( pWriter->pTree ){ + sqlite3_int64 iLast = 0; /* Largest block id written to database */ + sqlite3_int64 iLastLeaf; /* Largest leaf block id written to db */ + char *zRoot = NULL; /* Pointer to buffer containing root node */ + int nRoot = 0; /* Size of buffer zRoot */ - /* Count non-delimiter characters. */ - iStartOffset = c->iOffset; - while( c->iOffsetnBytes && !simpleDelim(t, p[c->iOffset]) ){ - c->iOffset++; + iLastLeaf = pWriter->iFree; + rc = fts3WriteSegment(p, pWriter->iFree++, pWriter->aData, pWriter->nData); + if( rc==SQLITE_OK ){ + rc = fts3NodeWrite(p, pWriter->pTree, 1, + pWriter->iFirst, pWriter->iFree, &iLast, &zRoot, &nRoot); } - - if( c->iOffset>iStartOffset ){ - int i, n = c->iOffset-iStartOffset; - if( n>c->nTokenAllocated ){ - char *pNew; - c->nTokenAllocated = n+20; - pNew = sqlite3_realloc(c->pToken, c->nTokenAllocated); - if( !pNew ) return SQLITE_NOMEM; - c->pToken = pNew; - } - for(i=0; ipToken[i] = (char)((ch>='A' && ch<='Z') ? ch-'A'+'a' : ch); - } - *ppToken = c->pToken; - *pnBytes = n; - *piStartOffset = iStartOffset; - *piEndOffset = c->iOffset; - *piPosition = c->iToken++; - - return SQLITE_OK; + if( rc==SQLITE_OK ){ + rc = fts3WriteSegdir(p, iLevel, iIdx, + pWriter->iFirst, iLastLeaf, iLast, pWriter->nLeafData, zRoot, nRoot); } + }else{ + /* The entire tree fits on the root node. Write it to the segdir table. */ + rc = fts3WriteSegdir(p, iLevel, iIdx, + 0, 0, 0, pWriter->nLeafData, pWriter->aData, pWriter->nData); } - return SQLITE_DONE; + p->nLeafAdd++; + return rc; } /* -** The set of routines that implement the simple tokenizer +** Release all memory held by the SegmentWriter object passed as the +** first argument. */ -static const sqlite3_tokenizer_module simpleTokenizerModule = { - 0, - simpleCreate, - simpleDestroy, - simpleOpen, - simpleClose, - simpleNext, - 0, -}; +static void fts3SegWriterFree(SegmentWriter *pWriter){ + if( pWriter ){ + sqlite3_free(pWriter->aData); + sqlite3_free(pWriter->zMalloc); + fts3NodeFree(pWriter->pTree); + sqlite3_free(pWriter); + } +} /* -** Allocate a new simple tokenizer. Return a pointer to the new -** tokenizer in *ppModule +** The first value in the apVal[] array is assumed to contain an integer. +** This function tests if there exist any documents with docid values that +** are different from that integer. i.e. if deleting the document with docid +** pRowid would mean the FTS3 table were empty. +** +** If successful, *pisEmpty is set to true if the table is empty except for +** document pRowid, or false otherwise, and SQLITE_OK is returned. If an +** error occurs, an SQLite error code is returned. */ -SQLITE_PRIVATE void sqlite3Fts3SimpleTokenizerModule( - sqlite3_tokenizer_module const**ppModule -){ - *ppModule = &simpleTokenizerModule; +static int fts3IsEmpty(Fts3Table *p, sqlite3_value *pRowid, int *pisEmpty){ + sqlite3_stmt *pStmt; + int rc; + if( p->zContentTbl ){ + /* If using the content=xxx option, assume the table is never empty */ + *pisEmpty = 0; + rc = SQLITE_OK; + }else{ + rc = fts3SqlStmt(p, SQL_IS_EMPTY, &pStmt, &pRowid); + if( rc==SQLITE_OK ){ + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + *pisEmpty = sqlite3_column_int(pStmt, 0); + } + rc = sqlite3_reset(pStmt); + } + } + return rc; } -#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ - -/************** End of fts3_tokenizer1.c *************************************/ -/************** Begin file fts3_tokenize_vtab.c ******************************/ /* -** 2013 Apr 22 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -****************************************************************************** -** -** This file contains code for the "fts3tokenize" virtual table module. -** An fts3tokenize virtual table is created as follows: -** -** CREATE VIRTUAL TABLE USING fts3tokenize( -** , , ... -** ); -** -** The table created has the following schema: -** -** CREATE TABLE (input, token, start, end, position) -** -** When queried, the query must include a WHERE clause of type: -** -** input = -** -** The virtual table module tokenizes this , using the FTS3 -** tokenizer specified by the arguments to the CREATE VIRTUAL TABLE -** statement and returns one row for each token in the result. With -** fields set as follows: +** Set *pnMax to the largest segment level in the database for the index +** iIndex. ** -** input: Always set to a copy of -** token: A token from the input. -** start: Byte offset of the token within the input . -** end: Byte offset of the byte immediately following the end of the -** token within the input string. -** pos: Token offset of token within input. +** Segment levels are stored in the 'level' column of the %_segdir table. ** +** Return SQLITE_OK if successful, or an SQLite error code if not. */ -/* #include "fts3Int.h" */ -#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) - -/* #include */ -/* #include */ - -typedef struct Fts3tokTable Fts3tokTable; -typedef struct Fts3tokCursor Fts3tokCursor; +static int fts3SegmentMaxLevel( + Fts3Table *p, + int iLangid, + int iIndex, + sqlite3_int64 *pnMax +){ + sqlite3_stmt *pStmt; + int rc; + assert( iIndex>=0 && iIndexnIndex ); -/* -** Virtual table structure. -*/ -struct Fts3tokTable { - sqlite3_vtab base; /* Base class used by SQLite core */ - const sqlite3_tokenizer_module *pMod; - sqlite3_tokenizer *pTok; -}; + /* Set pStmt to the compiled version of: + ** + ** SELECT max(level) FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ? + ** + ** (1024 is actually the value of macro FTS3_SEGDIR_PREFIXLEVEL_STR). + */ + rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR_MAX_LEVEL, &pStmt, 0); + if( rc!=SQLITE_OK ) return rc; + sqlite3_bind_int64(pStmt, 1, getAbsoluteLevel(p, iLangid, iIndex, 0)); + sqlite3_bind_int64(pStmt, 2, + getAbsoluteLevel(p, iLangid, iIndex, FTS3_SEGDIR_MAXLEVEL-1) + ); + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + *pnMax = sqlite3_column_int64(pStmt, 0); + } + return sqlite3_reset(pStmt); +} /* -** Virtual table cursor structure. +** iAbsLevel is an absolute level that may be assumed to exist within +** the database. This function checks if it is the largest level number +** within its index. Assuming no error occurs, *pbMax is set to 1 if +** iAbsLevel is indeed the largest level, or 0 otherwise, and SQLITE_OK +** is returned. If an error occurs, an error code is returned and the +** final value of *pbMax is undefined. */ -struct Fts3tokCursor { - sqlite3_vtab_cursor base; /* Base class used by SQLite core */ - char *zInput; /* Input string */ - sqlite3_tokenizer_cursor *pCsr; /* Cursor to iterate through zInput */ - int iRowid; /* Current 'rowid' value */ - const char *zToken; /* Current 'token' value */ - int nToken; /* Size of zToken in bytes */ - int iStart; /* Current 'start' value */ - int iEnd; /* Current 'end' value */ - int iPos; /* Current 'pos' value */ -}; +static int fts3SegmentIsMaxLevel(Fts3Table *p, i64 iAbsLevel, int *pbMax){ -/* -** Query FTS for the tokenizer implementation named zName. -*/ -static int fts3tokQueryTokenizer( - Fts3Hash *pHash, - const char *zName, - const sqlite3_tokenizer_module **pp, - char **pzErr -){ - sqlite3_tokenizer_module *p; - int nName = (int)strlen(zName); + /* Set pStmt to the compiled version of: + ** + ** SELECT max(level) FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ? + ** + ** (1024 is actually the value of macro FTS3_SEGDIR_PREFIXLEVEL_STR). + */ + sqlite3_stmt *pStmt; + int rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR_MAX_LEVEL, &pStmt, 0); + if( rc!=SQLITE_OK ) return rc; + sqlite3_bind_int64(pStmt, 1, iAbsLevel+1); + sqlite3_bind_int64(pStmt, 2, + (((u64)iAbsLevel/FTS3_SEGDIR_MAXLEVEL)+1) * FTS3_SEGDIR_MAXLEVEL + ); - p = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash, zName, nName+1); - if( !p ){ - sqlite3Fts3ErrMsg(pzErr, "unknown tokenizer: %s", zName); - return SQLITE_ERROR; + *pbMax = 0; + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + *pbMax = sqlite3_column_type(pStmt, 0)==SQLITE_NULL; } - - *pp = p; - return SQLITE_OK; + return sqlite3_reset(pStmt); } /* -** The second argument, argv[], is an array of pointers to nul-terminated -** strings. This function makes a copy of the array and strings into a -** single block of memory. It then dequotes any of the strings that appear -** to be quoted. -** -** If successful, output parameter *pazDequote is set to point at the -** array of dequoted strings and SQLITE_OK is returned. The caller is -** responsible for eventually calling sqlite3_free() to free the array -** in this case. Or, if an error occurs, an SQLite error code is returned. -** The final value of *pazDequote is undefined in this case. +** Delete all entries in the %_segments table associated with the segment +** opened with seg-reader pSeg. This function does not affect the contents +** of the %_segdir table. */ -static int fts3tokDequoteArray( - int argc, /* Number of elements in argv[] */ - const char * const *argv, /* Input array */ - char ***pazDequote /* Output array */ +static int fts3DeleteSegment( + Fts3Table *p, /* FTS table handle */ + Fts3SegReader *pSeg /* Segment to delete */ ){ int rc = SQLITE_OK; /* Return code */ - if( argc==0 ){ - *pazDequote = 0; - }else{ - int i; - int nByte = 0; - char **azDequote; - - for(i=0; iiStartBlock ){ + sqlite3_stmt *pDelete; /* SQL statement to delete rows */ + rc = fts3SqlStmt(p, SQL_DELETE_SEGMENTS_RANGE, &pDelete, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pDelete, 1, pSeg->iStartBlock); + sqlite3_bind_int64(pDelete, 2, pSeg->iEndBlock); + sqlite3_step(pDelete); + rc = sqlite3_reset(pDelete); } } - return rc; } /* -** Schema of the tokenizer table. -*/ -#define FTS3_TOK_SCHEMA "CREATE TABLE x(input, token, start, end, position)" - -/* -** This function does all the work for both the xConnect and xCreate methods. -** These tables have no persistent representation of their own, so xConnect -** and xCreate are identical operations. +** This function is used after merging multiple segments into a single large +** segment to delete the old, now redundant, segment b-trees. Specifically, +** it: +** +** 1) Deletes all %_segments entries for the segments associated with +** each of the SegReader objects in the array passed as the third +** argument, and ** -** argv[0]: module name -** argv[1]: database name -** argv[2]: table name -** argv[3]: first argument (tokenizer name) +** 2) deletes all %_segdir entries with level iLevel, or all %_segdir +** entries regardless of level if (iLevel<0). +** +** SQLITE_OK is returned if successful, otherwise an SQLite error code. */ -static int fts3tokConnectMethod( - sqlite3 *db, /* Database connection */ - void *pHash, /* Hash table of tokenizers */ - int argc, /* Number of elements in argv array */ - const char * const *argv, /* xCreate/xConnect argument array */ - sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */ - char **pzErr /* OUT: sqlite3_malloc'd error message */ +static int fts3DeleteSegdir( + Fts3Table *p, /* Virtual table handle */ + int iLangid, /* Language id */ + int iIndex, /* Index for p->aIndex */ + int iLevel, /* Level of %_segdir entries to delete */ + Fts3SegReader **apSegment, /* Array of SegReader objects */ + int nReader /* Size of array apSegment */ ){ - Fts3tokTable *pTab = 0; - const sqlite3_tokenizer_module *pMod = 0; - sqlite3_tokenizer *pTok = 0; - int rc; - char **azDequote = 0; - int nDequote; - - rc = sqlite3_declare_vtab(db, FTS3_TOK_SCHEMA); - if( rc!=SQLITE_OK ) return rc; - - nDequote = argc-3; - rc = fts3tokDequoteArray(nDequote, &argv[3], &azDequote); + int rc = SQLITE_OK; /* Return Code */ + int i; /* Iterator variable */ + sqlite3_stmt *pDelete = 0; /* SQL statement to delete rows */ - if( rc==SQLITE_OK ){ - const char *zModule; - if( nDequote<1 ){ - zModule = "simple"; - }else{ - zModule = azDequote[0]; - } - rc = fts3tokQueryTokenizer((Fts3Hash*)pHash, zModule, &pMod, pzErr); + for(i=0; rc==SQLITE_OK && ixCreate((nDequote>1 ? nDequote-1 : 0), azArg, &pTok); + if( rc!=SQLITE_OK ){ + return rc; } - if( rc==SQLITE_OK ){ - pTab = (Fts3tokTable *)sqlite3_malloc(sizeof(Fts3tokTable)); - if( pTab==0 ){ - rc = SQLITE_NOMEM; + assert( iLevel>=0 || iLevel==FTS3_SEGCURSOR_ALL ); + if( iLevel==FTS3_SEGCURSOR_ALL ){ + rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_RANGE, &pDelete, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pDelete, 1, getAbsoluteLevel(p, iLangid, iIndex, 0)); + sqlite3_bind_int64(pDelete, 2, + getAbsoluteLevel(p, iLangid, iIndex, FTS3_SEGDIR_MAXLEVEL-1) + ); + } + }else{ + rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_LEVEL, &pDelete, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64( + pDelete, 1, getAbsoluteLevel(p, iLangid, iIndex, iLevel) + ); } } if( rc==SQLITE_OK ){ - memset(pTab, 0, sizeof(Fts3tokTable)); - pTab->pMod = pMod; - pTab->pTok = pTok; - *ppVtab = &pTab->base; - }else{ - if( pTok ){ - pMod->xDestroy(pTok); - } + sqlite3_step(pDelete); + rc = sqlite3_reset(pDelete); } - sqlite3_free(azDequote); return rc; } /* -** This function does the work for both the xDisconnect and xDestroy methods. -** These tables have no persistent representation of their own, so xDisconnect -** and xDestroy are identical operations. -*/ -static int fts3tokDisconnectMethod(sqlite3_vtab *pVtab){ - Fts3tokTable *pTab = (Fts3tokTable *)pVtab; - - pTab->pMod->xDestroy(pTab->pTok); - sqlite3_free(pTab); - return SQLITE_OK; -} - -/* -** xBestIndex - Analyze a WHERE and ORDER BY clause. +** When this function is called, buffer *ppList (size *pnList bytes) contains +** a position list that may (or may not) feature multiple columns. This +** function adjusts the pointer *ppList and the length *pnList so that they +** identify the subset of the position list that corresponds to column iCol. +** +** If there are no entries in the input position list for column iCol, then +** *pnList is set to zero before returning. +** +** If parameter bZero is non-zero, then any part of the input list following +** the end of the output list is zeroed before returning. */ -static int fts3tokBestIndexMethod( - sqlite3_vtab *pVTab, - sqlite3_index_info *pInfo +static void fts3ColumnFilter( + int iCol, /* Column to filter on */ + int bZero, /* Zero out anything following *ppList */ + char **ppList, /* IN/OUT: Pointer to position list */ + int *pnList /* IN/OUT: Size of buffer *ppList in bytes */ ){ - int i; - UNUSED_PARAMETER(pVTab); + char *pList = *ppList; + int nList = *pnList; + char *pEnd = &pList[nList]; + int iCurrent = 0; + char *p = pList; - for(i=0; inConstraint; i++){ - if( pInfo->aConstraint[i].usable - && pInfo->aConstraint[i].iColumn==0 - && pInfo->aConstraint[i].op==SQLITE_INDEX_CONSTRAINT_EQ - ){ - pInfo->idxNum = 1; - pInfo->aConstraintUsage[i].argvIndex = 1; - pInfo->aConstraintUsage[i].omit = 1; - pInfo->estimatedCost = 1; - return SQLITE_OK; + assert( iCol>=0 ); + while( 1 ){ + char c = 0; + while( pidxNum = 0; - assert( pInfo->estimatedCost>1000000.0 ); + nList -= (int)(p - pList); + pList = p; + if( nList<=0 ){ + break; + } + p = &pList[1]; + p += fts3GetVarint32(p, &iCurrent); + } - return SQLITE_OK; + if( bZero && (pEnd - &pList[nList])>0){ + memset(&pList[nList], 0, pEnd - &pList[nList]); + } + *ppList = pList; + *pnList = nList; } /* -** xOpen - Open a cursor. +** Cache data in the Fts3MultiSegReader.aBuffer[] buffer (overwriting any +** existing data). Grow the buffer if required. +** +** If successful, return SQLITE_OK. Otherwise, if an OOM error is encountered +** trying to resize the buffer, return SQLITE_NOMEM. */ -static int fts3tokOpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){ - Fts3tokCursor *pCsr; - UNUSED_PARAMETER(pVTab); - - pCsr = (Fts3tokCursor *)sqlite3_malloc(sizeof(Fts3tokCursor)); - if( pCsr==0 ){ - return SQLITE_NOMEM; +static int fts3MsrBufferData( + Fts3MultiSegReader *pMsr, /* Multi-segment-reader handle */ + char *pList, + int nList +){ + if( nList>pMsr->nBuffer ){ + char *pNew; + pMsr->nBuffer = nList*2; + pNew = (char *)sqlite3_realloc(pMsr->aBuffer, pMsr->nBuffer); + if( !pNew ) return SQLITE_NOMEM; + pMsr->aBuffer = pNew; } - memset(pCsr, 0, sizeof(Fts3tokCursor)); - *ppCsr = (sqlite3_vtab_cursor *)pCsr; + assert( nList>0 ); + memcpy(pMsr->aBuffer, pList, nList); return SQLITE_OK; } -/* -** Reset the tokenizer cursor passed as the only argument. As if it had -** just been returned by fts3tokOpenMethod(). -*/ -static void fts3tokResetCursor(Fts3tokCursor *pCsr){ - if( pCsr->pCsr ){ - Fts3tokTable *pTab = (Fts3tokTable *)(pCsr->base.pVtab); - pTab->pMod->xClose(pCsr->pCsr); - pCsr->pCsr = 0; +SQLITE_PRIVATE int sqlite3Fts3MsrIncrNext( + Fts3Table *p, /* Virtual table handle */ + Fts3MultiSegReader *pMsr, /* Multi-segment-reader handle */ + sqlite3_int64 *piDocid, /* OUT: Docid value */ + char **paPoslist, /* OUT: Pointer to position list */ + int *pnPoslist /* OUT: Size of position list in bytes */ +){ + int nMerge = pMsr->nAdvance; + Fts3SegReader **apSegment = pMsr->apSegment; + int (*xCmp)(Fts3SegReader *, Fts3SegReader *) = ( + p->bDescIdx ? fts3SegReaderDoclistCmpRev : fts3SegReaderDoclistCmp + ); + + if( nMerge==0 ){ + *paPoslist = 0; + return SQLITE_OK; } - sqlite3_free(pCsr->zInput); - pCsr->zInput = 0; - pCsr->zToken = 0; - pCsr->nToken = 0; - pCsr->iStart = 0; - pCsr->iEnd = 0; - pCsr->iPos = 0; - pCsr->iRowid = 0; -} -/* -** xClose - Close a cursor. -*/ -static int fts3tokCloseMethod(sqlite3_vtab_cursor *pCursor){ - Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor; + while( 1 ){ + Fts3SegReader *pSeg; + pSeg = pMsr->apSegment[0]; + + if( pSeg->pOffsetList==0 ){ + *paPoslist = 0; + break; + }else{ + int rc; + char *pList; + int nList; + int j; + sqlite3_int64 iDocid = apSegment[0]->iDocid; - fts3tokResetCursor(pCsr); - sqlite3_free(pCsr); - return SQLITE_OK; -} + rc = fts3SegReaderNextDocid(p, apSegment[0], &pList, &nList); + j = 1; + while( rc==SQLITE_OK + && jpOffsetList + && apSegment[j]->iDocid==iDocid + ){ + rc = fts3SegReaderNextDocid(p, apSegment[j], 0, 0); + j++; + } + if( rc!=SQLITE_OK ) return rc; + fts3SegReaderSort(pMsr->apSegment, nMerge, j, xCmp); -/* -** xNext - Advance the cursor to the next row, if any. -*/ -static int fts3tokNextMethod(sqlite3_vtab_cursor *pCursor){ - Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor; - Fts3tokTable *pTab = (Fts3tokTable *)(pCursor->pVtab); - int rc; /* Return code */ + if( nList>0 && fts3SegReaderIsPending(apSegment[0]) ){ + rc = fts3MsrBufferData(pMsr, pList, nList+1); + if( rc!=SQLITE_OK ) return rc; + assert( (pMsr->aBuffer[nList] & 0xFE)==0x00 ); + pList = pMsr->aBuffer; + } - pCsr->iRowid++; - rc = pTab->pMod->xNext(pCsr->pCsr, - &pCsr->zToken, &pCsr->nToken, - &pCsr->iStart, &pCsr->iEnd, &pCsr->iPos - ); + if( pMsr->iColFilter>=0 ){ + fts3ColumnFilter(pMsr->iColFilter, 1, &pList, &nList); + } - if( rc!=SQLITE_OK ){ - fts3tokResetCursor(pCsr); - if( rc==SQLITE_DONE ) rc = SQLITE_OK; + if( nList>0 ){ + *paPoslist = pList; + *piDocid = iDocid; + *pnPoslist = nList; + break; + } + } } - return rc; + return SQLITE_OK; } -/* -** xFilter - Initialize a cursor to point at the start of its data. -*/ -static int fts3tokFilterMethod( - sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */ - int idxNum, /* Strategy index */ - const char *idxStr, /* Unused */ - int nVal, /* Number of elements in apVal */ - sqlite3_value **apVal /* Arguments for the indexing scheme */ +static int fts3SegReaderStart( + Fts3Table *p, /* Virtual table handle */ + Fts3MultiSegReader *pCsr, /* Cursor object */ + const char *zTerm, /* Term searched for (or NULL) */ + int nTerm /* Length of zTerm in bytes */ ){ - int rc = SQLITE_ERROR; - Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor; - Fts3tokTable *pTab = (Fts3tokTable *)(pCursor->pVtab); - UNUSED_PARAMETER(idxStr); - UNUSED_PARAMETER(nVal); + int i; + int nSeg = pCsr->nSegment; - fts3tokResetCursor(pCsr); - if( idxNum==1 ){ - const char *zByte = (const char *)sqlite3_value_text(apVal[0]); - int nByte = sqlite3_value_bytes(apVal[0]); - pCsr->zInput = sqlite3_malloc(nByte+1); - if( pCsr->zInput==0 ){ - rc = SQLITE_NOMEM; - }else{ - memcpy(pCsr->zInput, zByte, nByte); - pCsr->zInput[nByte] = 0; - rc = pTab->pMod->xOpen(pTab->pTok, pCsr->zInput, nByte, &pCsr->pCsr); - if( rc==SQLITE_OK ){ - pCsr->pCsr->pTokenizer = pTab->pTok; - } + /* If the Fts3SegFilter defines a specific term (or term prefix) to search + ** for, then advance each segment iterator until it points to a term of + ** equal or greater value than the specified term. This prevents many + ** unnecessary merge/sort operations for the case where single segment + ** b-tree leaf nodes contain more than one term. + */ + for(i=0; pCsr->bRestart==0 && inSegment; i++){ + int res = 0; + Fts3SegReader *pSeg = pCsr->apSegment[i]; + do { + int rc = fts3SegReaderNext(p, pSeg, 0); + if( rc!=SQLITE_OK ) return rc; + }while( zTerm && (res = fts3SegReaderTermCmp(pSeg, zTerm, nTerm))<0 ); + + if( pSeg->bLookup && res!=0 ){ + fts3SegReaderSetEof(pSeg); } } + fts3SegReaderSort(pCsr->apSegment, nSeg, nSeg, fts3SegReaderCmp); - if( rc!=SQLITE_OK ) return rc; - return fts3tokNextMethod(pCursor); + return SQLITE_OK; } -/* -** xEof - Return true if the cursor is at EOF, or false otherwise. -*/ -static int fts3tokEofMethod(sqlite3_vtab_cursor *pCursor){ - Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor; - return (pCsr->zToken==0); +SQLITE_PRIVATE int sqlite3Fts3SegReaderStart( + Fts3Table *p, /* Virtual table handle */ + Fts3MultiSegReader *pCsr, /* Cursor object */ + Fts3SegFilter *pFilter /* Restrictions on range of iteration */ +){ + pCsr->pFilter = pFilter; + return fts3SegReaderStart(p, pCsr, pFilter->zTerm, pFilter->nTerm); } -/* -** xColumn - Return a column value. -*/ -static int fts3tokColumnMethod( - sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ - sqlite3_context *pCtx, /* Context for sqlite3_result_xxx() calls */ - int iCol /* Index of column to read value from */ +SQLITE_PRIVATE int sqlite3Fts3MsrIncrStart( + Fts3Table *p, /* Virtual table handle */ + Fts3MultiSegReader *pCsr, /* Cursor object */ + int iCol, /* Column to match on. */ + const char *zTerm, /* Term to iterate through a doclist for */ + int nTerm /* Number of bytes in zTerm */ ){ - Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor; + int i; + int rc; + int nSegment = pCsr->nSegment; + int (*xCmp)(Fts3SegReader *, Fts3SegReader *) = ( + p->bDescIdx ? fts3SegReaderDoclistCmpRev : fts3SegReaderDoclistCmp + ); - /* CREATE TABLE x(input, token, start, end, position) */ - switch( iCol ){ - case 0: - sqlite3_result_text(pCtx, pCsr->zInput, -1, SQLITE_TRANSIENT); - break; - case 1: - sqlite3_result_text(pCtx, pCsr->zToken, pCsr->nToken, SQLITE_TRANSIENT); - break; - case 2: - sqlite3_result_int(pCtx, pCsr->iStart); - break; - case 3: - sqlite3_result_int(pCtx, pCsr->iEnd); - break; - default: - assert( iCol==4 ); - sqlite3_result_int(pCtx, pCsr->iPos); + assert( pCsr->pFilter==0 ); + assert( zTerm && nTerm>0 ); + + /* Advance each segment iterator until it points to the term zTerm/nTerm. */ + rc = fts3SegReaderStart(p, pCsr, zTerm, nTerm); + if( rc!=SQLITE_OK ) return rc; + + /* Determine how many of the segments actually point to zTerm/nTerm. */ + for(i=0; iapSegment[i]; + if( !pSeg->aNode || fts3SegReaderTermCmp(pSeg, zTerm, nTerm) ){ break; + } } - return SQLITE_OK; -} + pCsr->nAdvance = i; -/* -** xRowid - Return the current rowid for the cursor. -*/ -static int fts3tokRowidMethod( - sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ - sqlite_int64 *pRowid /* OUT: Rowid value */ -){ - Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor; - *pRowid = (sqlite3_int64)pCsr->iRowid; - return SQLITE_OK; -} + /* Advance each of the segments to point to the first docid. */ + for(i=0; inAdvance; i++){ + rc = fts3SegReaderFirstDocid(p, pCsr->apSegment[i]); + if( rc!=SQLITE_OK ) return rc; + } + fts3SegReaderSort(pCsr->apSegment, i, i, xCmp); -/* -** Register the fts3tok module with database connection db. Return SQLITE_OK -** if successful or an error code if sqlite3_create_module() fails. -*/ -SQLITE_PRIVATE int sqlite3Fts3InitTok(sqlite3 *db, Fts3Hash *pHash){ - static const sqlite3_module fts3tok_module = { - 0, /* iVersion */ - fts3tokConnectMethod, /* xCreate */ - fts3tokConnectMethod, /* xConnect */ - fts3tokBestIndexMethod, /* xBestIndex */ - fts3tokDisconnectMethod, /* xDisconnect */ - fts3tokDisconnectMethod, /* xDestroy */ - fts3tokOpenMethod, /* xOpen */ - fts3tokCloseMethod, /* xClose */ - fts3tokFilterMethod, /* xFilter */ - fts3tokNextMethod, /* xNext */ - fts3tokEofMethod, /* xEof */ - fts3tokColumnMethod, /* xColumn */ - fts3tokRowidMethod, /* xRowid */ - 0, /* xUpdate */ - 0, /* xBegin */ - 0, /* xSync */ - 0, /* xCommit */ - 0, /* xRollback */ - 0, /* xFindFunction */ - 0, /* xRename */ - 0, /* xSavepoint */ - 0, /* xRelease */ - 0 /* xRollbackTo */ - }; - int rc; /* Return code */ + assert( iCol<0 || iColnColumn ); + pCsr->iColFilter = iCol; - rc = sqlite3_create_module(db, "fts3tokenize", &fts3tok_module, (void*)pHash); - return rc; + return SQLITE_OK; } -#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ - -/************** End of fts3_tokenize_vtab.c **********************************/ -/************** Begin file fts3_write.c **************************************/ /* -** 2009 Oct 23 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. +** This function is called on a MultiSegReader that has been started using +** sqlite3Fts3MsrIncrStart(). One or more calls to MsrIncrNext() may also +** have been made. Calling this function puts the MultiSegReader in such +** a state that if the next two calls are: ** -****************************************************************************** +** sqlite3Fts3SegReaderStart() +** sqlite3Fts3SegReaderStep() ** -** This file is part of the SQLite FTS3 extension module. Specifically, -** this file contains code to insert, update and delete rows from FTS3 -** tables. It also contains code to merge FTS3 b-tree segments. Some -** of the sub-routines used to merge segments are also used by the query -** code in fts3.c. +** then the entire doclist for the term is available in +** MultiSegReader.aDoclist/nDoclist. */ +SQLITE_PRIVATE int sqlite3Fts3MsrIncrRestart(Fts3MultiSegReader *pCsr){ + int i; /* Used to iterate through segment-readers */ -/* #include "fts3Int.h" */ -#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + assert( pCsr->zTerm==0 ); + assert( pCsr->nTerm==0 ); + assert( pCsr->aDoclist==0 ); + assert( pCsr->nDoclist==0 ); -/* #include */ -/* #include */ -/* #include */ + pCsr->nAdvance = 0; + pCsr->bRestart = 1; + for(i=0; inSegment; i++){ + pCsr->apSegment[i]->pOffsetList = 0; + pCsr->apSegment[i]->nOffsetList = 0; + pCsr->apSegment[i]->iDocid = 0; + } + return SQLITE_OK; +} -#define FTS_MAX_APPENDABLE_HEIGHT 16 -/* -** When full-text index nodes are loaded from disk, the buffer that they -** are loaded into has the following number of bytes of padding at the end -** of it. i.e. if a full-text index node is 900 bytes in size, then a buffer -** of 920 bytes is allocated for it. -** -** This means that if we have a pointer into a buffer containing node data, -** it is always safe to read up to two varints from it without risking an -** overread, even if the node data is corrupted. -*/ -#define FTS3_NODE_PADDING (FTS3_VARINT_MAX*2) +SQLITE_PRIVATE int sqlite3Fts3SegReaderStep( + Fts3Table *p, /* Virtual table handle */ + Fts3MultiSegReader *pCsr /* Cursor object */ +){ + int rc = SQLITE_OK; -/* -** Under certain circumstances, b-tree nodes (doclists) can be loaded into -** memory incrementally instead of all at once. This can be a big performance -** win (reduced IO and CPU) if SQLite stops calling the virtual table xNext() -** method before retrieving all query results (as may happen, for example, -** if a query has a LIMIT clause). -** -** Incremental loading is used for b-tree nodes FTS3_NODE_CHUNK_THRESHOLD -** bytes and larger. Nodes are loaded in chunks of FTS3_NODE_CHUNKSIZE bytes. -** The code is written so that the hard lower-limit for each of these values -** is 1. Clearly such small values would be inefficient, but can be useful -** for testing purposes. -** -** If this module is built with SQLITE_TEST defined, these constants may -** be overridden at runtime for testing purposes. File fts3_test.c contains -** a Tcl interface to read and write the values. -*/ -#ifdef SQLITE_TEST -int test_fts3_node_chunksize = (4*1024); -int test_fts3_node_chunk_threshold = (4*1024)*4; -# define FTS3_NODE_CHUNKSIZE test_fts3_node_chunksize -# define FTS3_NODE_CHUNK_THRESHOLD test_fts3_node_chunk_threshold -#else -# define FTS3_NODE_CHUNKSIZE (4*1024) -# define FTS3_NODE_CHUNK_THRESHOLD (FTS3_NODE_CHUNKSIZE*4) -#endif + int isIgnoreEmpty = (pCsr->pFilter->flags & FTS3_SEGMENT_IGNORE_EMPTY); + int isRequirePos = (pCsr->pFilter->flags & FTS3_SEGMENT_REQUIRE_POS); + int isColFilter = (pCsr->pFilter->flags & FTS3_SEGMENT_COLUMN_FILTER); + int isPrefix = (pCsr->pFilter->flags & FTS3_SEGMENT_PREFIX); + int isScan = (pCsr->pFilter->flags & FTS3_SEGMENT_SCAN); + int isFirst = (pCsr->pFilter->flags & FTS3_SEGMENT_FIRST); -/* -** The two values that may be meaningfully bound to the :1 parameter in -** statements SQL_REPLACE_STAT and SQL_SELECT_STAT. -*/ -#define FTS_STAT_DOCTOTAL 0 -#define FTS_STAT_INCRMERGEHINT 1 -#define FTS_STAT_AUTOINCRMERGE 2 + Fts3SegReader **apSegment = pCsr->apSegment; + int nSegment = pCsr->nSegment; + Fts3SegFilter *pFilter = pCsr->pFilter; + int (*xCmp)(Fts3SegReader *, Fts3SegReader *) = ( + p->bDescIdx ? fts3SegReaderDoclistCmpRev : fts3SegReaderDoclistCmp + ); -/* -** If FTS_LOG_MERGES is defined, call sqlite3_log() to report each automatic -** and incremental merge operation that takes place. This is used for -** debugging FTS only, it should not usually be turned on in production -** systems. -*/ -#ifdef FTS3_LOG_MERGES -static void fts3LogMerge(int nMerge, sqlite3_int64 iAbsLevel){ - sqlite3_log(SQLITE_OK, "%d-way merge from level %d", nMerge, (int)iAbsLevel); -} -#else -#define fts3LogMerge(x, y) -#endif + if( pCsr->nSegment==0 ) return SQLITE_OK; + do { + int nMerge; + int i; + + /* Advance the first pCsr->nAdvance entries in the apSegment[] array + ** forward. Then sort the list in order of current term again. + */ + for(i=0; inAdvance; i++){ + Fts3SegReader *pSeg = apSegment[i]; + if( pSeg->bLookup ){ + fts3SegReaderSetEof(pSeg); + }else{ + rc = fts3SegReaderNext(p, pSeg, 0); + } + if( rc!=SQLITE_OK ) return rc; + } + fts3SegReaderSort(apSegment, nSegment, pCsr->nAdvance, fts3SegReaderCmp); + pCsr->nAdvance = 0; -typedef struct PendingList PendingList; -typedef struct SegmentNode SegmentNode; -typedef struct SegmentWriter SegmentWriter; + /* If all the seg-readers are at EOF, we're finished. return SQLITE_OK. */ + assert( rc==SQLITE_OK ); + if( apSegment[0]->aNode==0 ) break; -/* -** An instance of the following data structure is used to build doclists -** incrementally. See function fts3PendingListAppend() for details. -*/ -struct PendingList { - int nData; - char *aData; - int nSpace; - sqlite3_int64 iLastDocid; - sqlite3_int64 iLastCol; - sqlite3_int64 iLastPos; -}; + pCsr->nTerm = apSegment[0]->nTerm; + pCsr->zTerm = apSegment[0]->zTerm; + /* If this is a prefix-search, and if the term that apSegment[0] points + ** to does not share a suffix with pFilter->zTerm/nTerm, then all + ** required callbacks have been made. In this case exit early. + ** + ** Similarly, if this is a search for an exact match, and the first term + ** of segment apSegment[0] is not a match, exit early. + */ + if( pFilter->zTerm && !isScan ){ + if( pCsr->nTermnTerm + || (!isPrefix && pCsr->nTerm>pFilter->nTerm) + || memcmp(pCsr->zTerm, pFilter->zTerm, pFilter->nTerm) + ){ + break; + } + } -/* -** Each cursor has a (possibly empty) linked list of the following objects. -*/ -struct Fts3DeferredToken { - Fts3PhraseToken *pToken; /* Pointer to corresponding expr token */ - int iCol; /* Column token must occur in */ - Fts3DeferredToken *pNext; /* Next in list of deferred tokens */ - PendingList *pList; /* Doclist is assembled here */ -}; + nMerge = 1; + while( nMergeaNode + && apSegment[nMerge]->nTerm==pCsr->nTerm + && 0==memcmp(pCsr->zTerm, apSegment[nMerge]->zTerm, pCsr->nTerm) + ){ + nMerge++; + } -/* -** An instance of this structure is used to iterate through the terms on -** a contiguous set of segment b-tree leaf nodes. Although the details of -** this structure are only manipulated by code in this file, opaque handles -** of type Fts3SegReader* are also used by code in fts3.c to iterate through -** terms when querying the full-text index. See functions: -** -** sqlite3Fts3SegReaderNew() -** sqlite3Fts3SegReaderFree() -** sqlite3Fts3SegReaderIterate() -** -** Methods used to manipulate Fts3SegReader structures: -** -** fts3SegReaderNext() -** fts3SegReaderFirstDocid() -** fts3SegReaderNextDocid() -*/ -struct Fts3SegReader { - int iIdx; /* Index within level, or 0x7FFFFFFF for PT */ - u8 bLookup; /* True for a lookup only */ - u8 rootOnly; /* True for a root-only reader */ + assert( isIgnoreEmpty || (isRequirePos && !isColFilter) ); + if( nMerge==1 + && !isIgnoreEmpty + && !isFirst + && (p->bDescIdx==0 || fts3SegReaderIsPending(apSegment[0])==0) + ){ + pCsr->nDoclist = apSegment[0]->nDoclist; + if( fts3SegReaderIsPending(apSegment[0]) ){ + rc = fts3MsrBufferData(pCsr, apSegment[0]->aDoclist, pCsr->nDoclist); + pCsr->aDoclist = pCsr->aBuffer; + }else{ + pCsr->aDoclist = apSegment[0]->aDoclist; + } + if( rc==SQLITE_OK ) rc = SQLITE_ROW; + }else{ + int nDoclist = 0; /* Size of doclist */ + sqlite3_int64 iPrev = 0; /* Previous docid stored in doclist */ - sqlite3_int64 iStartBlock; /* Rowid of first leaf block to traverse */ - sqlite3_int64 iLeafEndBlock; /* Rowid of final leaf block to traverse */ - sqlite3_int64 iEndBlock; /* Rowid of final block in segment (or 0) */ - sqlite3_int64 iCurrentBlock; /* Current leaf block (or 0) */ + /* The current term of the first nMerge entries in the array + ** of Fts3SegReader objects is the same. The doclists must be merged + ** and a single term returned with the merged doclist. + */ + for(i=0; ipOffsetList ){ + int j; /* Number of segments that share a docid */ + char *pList = 0; + int nList = 0; + int nByte; + sqlite3_int64 iDocid = apSegment[0]->iDocid; + fts3SegReaderNextDocid(p, apSegment[0], &pList, &nList); + j = 1; + while( jpOffsetList + && apSegment[j]->iDocid==iDocid + ){ + fts3SegReaderNextDocid(p, apSegment[j], 0, 0); + j++; + } - char *aNode; /* Pointer to node data (or NULL) */ - int nNode; /* Size of buffer at aNode (or 0) */ - int nPopulate; /* If >0, bytes of buffer aNode[] loaded */ - sqlite3_blob *pBlob; /* If not NULL, blob handle to read node */ + if( isColFilter ){ + fts3ColumnFilter(pFilter->iCol, 0, &pList, &nList); + } - Fts3HashElem **ppNextElem; + if( !isIgnoreEmpty || nList>0 ){ - /* Variables set by fts3SegReaderNext(). These may be read directly - ** by the caller. They are valid from the time SegmentReaderNew() returns - ** until SegmentReaderNext() returns something other than SQLITE_OK - ** (i.e. SQLITE_DONE). - */ - int nTerm; /* Number of bytes in current term */ - char *zTerm; /* Pointer to current term */ - int nTermAlloc; /* Allocated size of zTerm buffer */ - char *aDoclist; /* Pointer to doclist of current entry */ - int nDoclist; /* Size of doclist in current entry */ + /* Calculate the 'docid' delta value to write into the merged + ** doclist. */ + sqlite3_int64 iDelta; + if( p->bDescIdx && nDoclist>0 ){ + if( iPrev<=iDocid ) return FTS_CORRUPT_VTAB; + iDelta = (i64)((u64)iPrev - (u64)iDocid); + }else{ + if( nDoclist>0 && iPrev>=iDocid ) return FTS_CORRUPT_VTAB; + iDelta = (i64)((u64)iDocid - (u64)iPrev); + } - /* The following variables are used by fts3SegReaderNextDocid() to iterate - ** through the current doclist (aDoclist/nDoclist). - */ - char *pOffsetList; - int nOffsetList; /* For descending pending seg-readers only */ - sqlite3_int64 iDocid; -}; + nByte = sqlite3Fts3VarintLen(iDelta) + (isRequirePos?nList+1:0); + if( nDoclist+nByte>pCsr->nBuffer ){ + char *aNew; + pCsr->nBuffer = (nDoclist+nByte)*2; + aNew = sqlite3_realloc(pCsr->aBuffer, pCsr->nBuffer); + if( !aNew ){ + return SQLITE_NOMEM; + } + pCsr->aBuffer = aNew; + } -#define fts3SegReaderIsPending(p) ((p)->ppNextElem!=0) -#define fts3SegReaderIsRootOnly(p) ((p)->rootOnly!=0) + if( isFirst ){ + char *a = &pCsr->aBuffer[nDoclist]; + int nWrite; + + nWrite = sqlite3Fts3FirstFilter(iDelta, pList, nList, a); + if( nWrite ){ + iPrev = iDocid; + nDoclist += nWrite; + } + }else{ + nDoclist += sqlite3Fts3PutVarint(&pCsr->aBuffer[nDoclist], iDelta); + iPrev = iDocid; + if( isRequirePos ){ + memcpy(&pCsr->aBuffer[nDoclist], pList, nList); + nDoclist += nList; + pCsr->aBuffer[nDoclist++] = '\0'; + } + } + } -/* -** An instance of this structure is used to create a segment b-tree in the -** database. The internal details of this type are only accessed by the -** following functions: -** -** fts3SegWriterAdd() -** fts3SegWriterFlush() -** fts3SegWriterFree() -*/ -struct SegmentWriter { - SegmentNode *pTree; /* Pointer to interior tree structure */ - sqlite3_int64 iFirst; /* First slot in %_segments written */ - sqlite3_int64 iFree; /* Next free slot in %_segments */ - char *zTerm; /* Pointer to previous term buffer */ - int nTerm; /* Number of bytes in zTerm */ - int nMalloc; /* Size of malloc'd buffer at zMalloc */ - char *zMalloc; /* Malloc'd space (possibly) used for zTerm */ - int nSize; /* Size of allocation at aData */ - int nData; /* Bytes of data in aData */ - char *aData; /* Pointer to block from malloc() */ - i64 nLeafData; /* Number of bytes of leaf data written */ -}; + fts3SegReaderSort(apSegment, nMerge, j, xCmp); + } + if( nDoclist>0 ){ + pCsr->aDoclist = pCsr->aBuffer; + pCsr->nDoclist = nDoclist; + rc = SQLITE_ROW; + } + } + pCsr->nAdvance = nMerge; + }while( rc==SQLITE_OK ); -/* -** Type SegmentNode is used by the following three functions to create -** the interior part of the segment b+-tree structures (everything except -** the leaf nodes). These functions and type are only ever used by code -** within the fts3SegWriterXXX() family of functions described above. -** -** fts3NodeAddTerm() -** fts3NodeWrite() -** fts3NodeFree() -** -** When a b+tree is written to the database (either as a result of a merge -** or the pending-terms table being flushed), leaves are written into the -** database file as soon as they are completely populated. The interior of -** the tree is assembled in memory and written out only once all leaves have -** been populated and stored. This is Ok, as the b+-tree fanout is usually -** very large, meaning that the interior of the tree consumes relatively -** little memory. -*/ -struct SegmentNode { - SegmentNode *pParent; /* Parent node (or NULL for root node) */ - SegmentNode *pRight; /* Pointer to right-sibling */ - SegmentNode *pLeftmost; /* Pointer to left-most node of this depth */ - int nEntry; /* Number of terms written to node so far */ - char *zTerm; /* Pointer to previous term buffer */ - int nTerm; /* Number of bytes in zTerm */ - int nMalloc; /* Size of malloc'd buffer at zMalloc */ - char *zMalloc; /* Malloc'd space (possibly) used for zTerm */ - int nData; /* Bytes of valid data so far */ - char *aData; /* Node data */ -}; + return rc; +} -/* -** Valid values for the second argument to fts3SqlStmt(). -*/ -#define SQL_DELETE_CONTENT 0 -#define SQL_IS_EMPTY 1 -#define SQL_DELETE_ALL_CONTENT 2 -#define SQL_DELETE_ALL_SEGMENTS 3 -#define SQL_DELETE_ALL_SEGDIR 4 -#define SQL_DELETE_ALL_DOCSIZE 5 -#define SQL_DELETE_ALL_STAT 6 -#define SQL_SELECT_CONTENT_BY_ROWID 7 -#define SQL_NEXT_SEGMENT_INDEX 8 -#define SQL_INSERT_SEGMENTS 9 -#define SQL_NEXT_SEGMENTS_ID 10 -#define SQL_INSERT_SEGDIR 11 -#define SQL_SELECT_LEVEL 12 -#define SQL_SELECT_LEVEL_RANGE 13 -#define SQL_SELECT_LEVEL_COUNT 14 -#define SQL_SELECT_SEGDIR_MAX_LEVEL 15 -#define SQL_DELETE_SEGDIR_LEVEL 16 -#define SQL_DELETE_SEGMENTS_RANGE 17 -#define SQL_CONTENT_INSERT 18 -#define SQL_DELETE_DOCSIZE 19 -#define SQL_REPLACE_DOCSIZE 20 -#define SQL_SELECT_DOCSIZE 21 -#define SQL_SELECT_STAT 22 -#define SQL_REPLACE_STAT 23 -#define SQL_SELECT_ALL_PREFIX_LEVEL 24 -#define SQL_DELETE_ALL_TERMS_SEGDIR 25 -#define SQL_DELETE_SEGDIR_RANGE 26 -#define SQL_SELECT_ALL_LANGID 27 -#define SQL_FIND_MERGE_LEVEL 28 -#define SQL_MAX_LEAF_NODE_ESTIMATE 29 -#define SQL_DELETE_SEGDIR_ENTRY 30 -#define SQL_SHIFT_SEGDIR_ENTRY 31 -#define SQL_SELECT_SEGDIR 32 -#define SQL_CHOMP_SEGDIR 33 -#define SQL_SEGMENT_IS_APPENDABLE 34 -#define SQL_SELECT_INDEXES 35 -#define SQL_SELECT_MXLEVEL 36 +SQLITE_PRIVATE void sqlite3Fts3SegReaderFinish( + Fts3MultiSegReader *pCsr /* Cursor object */ +){ + if( pCsr ){ + int i; + for(i=0; inSegment; i++){ + sqlite3Fts3SegReaderFree(pCsr->apSegment[i]); + } + sqlite3_free(pCsr->apSegment); + sqlite3_free(pCsr->aBuffer); -#define SQL_SELECT_LEVEL_RANGE2 37 -#define SQL_UPDATE_LEVEL_IDX 38 -#define SQL_UPDATE_LEVEL 39 + pCsr->nSegment = 0; + pCsr->apSegment = 0; + pCsr->aBuffer = 0; + } +} /* -** This function is used to obtain an SQLite prepared statement handle -** for the statement identified by the second argument. If successful, -** *pp is set to the requested statement handle and SQLITE_OK returned. -** Otherwise, an SQLite error code is returned and *pp is set to 0. +** Decode the "end_block" field, selected by column iCol of the SELECT +** statement passed as the first argument. ** -** If argument apVal is not NULL, then it must point to an array with -** at least as many entries as the requested statement has bound -** parameters. The values are bound to the statements parameters before -** returning. +** The "end_block" field may contain either an integer, or a text field +** containing the text representation of two non-negative integers separated +** by one or more space (0x20) characters. In the first case, set *piEndBlock +** to the integer value and *pnByte to zero before returning. In the second, +** set *piEndBlock to the first value and *pnByte to the second. */ -static int fts3SqlStmt( - Fts3Table *p, /* Virtual table handle */ - int eStmt, /* One of the SQL_XXX constants above */ - sqlite3_stmt **pp, /* OUT: Statement handle */ - sqlite3_value **apVal /* Values to bind to statement */ +static void fts3ReadEndBlockField( + sqlite3_stmt *pStmt, + int iCol, + i64 *piEndBlock, + i64 *pnByte ){ - const char *azSql[] = { -/* 0 */ "DELETE FROM %Q.'%q_content' WHERE rowid = ?", -/* 1 */ "SELECT NOT EXISTS(SELECT docid FROM %Q.'%q_content' WHERE rowid!=?)", -/* 2 */ "DELETE FROM %Q.'%q_content'", -/* 3 */ "DELETE FROM %Q.'%q_segments'", -/* 4 */ "DELETE FROM %Q.'%q_segdir'", -/* 5 */ "DELETE FROM %Q.'%q_docsize'", -/* 6 */ "DELETE FROM %Q.'%q_stat'", -/* 7 */ "SELECT %s WHERE rowid=?", -/* 8 */ "SELECT (SELECT max(idx) FROM %Q.'%q_segdir' WHERE level = ?) + 1", -/* 9 */ "REPLACE INTO %Q.'%q_segments'(blockid, block) VALUES(?, ?)", -/* 10 */ "SELECT coalesce((SELECT max(blockid) FROM %Q.'%q_segments') + 1, 1)", -/* 11 */ "REPLACE INTO %Q.'%q_segdir' VALUES(?,?,?,?,?,?)", - - /* Return segments in order from oldest to newest.*/ -/* 12 */ "SELECT idx, start_block, leaves_end_block, end_block, root " - "FROM %Q.'%q_segdir' WHERE level = ? ORDER BY idx ASC", -/* 13 */ "SELECT idx, start_block, leaves_end_block, end_block, root " - "FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?" - "ORDER BY level DESC, idx ASC", + const unsigned char *zText = sqlite3_column_text(pStmt, iCol); + if( zText ){ + int i; + int iMul = 1; + u64 iVal = 0; + for(i=0; zText[i]>='0' && zText[i]<='9'; i++){ + iVal = iVal*10 + (zText[i] - '0'); + } + *piEndBlock = (i64)iVal; + while( zText[i]==' ' ) i++; + iVal = 0; + if( zText[i]=='-' ){ + i++; + iMul = -1; + } + for(/* no-op */; zText[i]>='0' && zText[i]<='9'; i++){ + iVal = iVal*10 + (zText[i] - '0'); + } + *pnByte = ((i64)iVal * (i64)iMul); + } +} -/* 14 */ "SELECT count(*) FROM %Q.'%q_segdir' WHERE level = ?", -/* 15 */ "SELECT max(level) FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?", -/* 16 */ "DELETE FROM %Q.'%q_segdir' WHERE level = ?", -/* 17 */ "DELETE FROM %Q.'%q_segments' WHERE blockid BETWEEN ? AND ?", -/* 18 */ "INSERT INTO %Q.'%q_content' VALUES(%s)", -/* 19 */ "DELETE FROM %Q.'%q_docsize' WHERE docid = ?", -/* 20 */ "REPLACE INTO %Q.'%q_docsize' VALUES(?,?)", -/* 21 */ "SELECT size FROM %Q.'%q_docsize' WHERE docid=?", -/* 22 */ "SELECT value FROM %Q.'%q_stat' WHERE id=?", -/* 23 */ "REPLACE INTO %Q.'%q_stat' VALUES(?,?)", -/* 24 */ "", -/* 25 */ "", +/* +** A segment of size nByte bytes has just been written to absolute level +** iAbsLevel. Promote any segments that should be promoted as a result. +*/ +static int fts3PromoteSegments( + Fts3Table *p, /* FTS table handle */ + sqlite3_int64 iAbsLevel, /* Absolute level just updated */ + sqlite3_int64 nByte /* Size of new segment at iAbsLevel */ +){ + int rc = SQLITE_OK; + sqlite3_stmt *pRange; -/* 26 */ "DELETE FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?", -/* 27 */ "SELECT ? UNION SELECT level / (1024 * ?) FROM %Q.'%q_segdir'", + rc = fts3SqlStmt(p, SQL_SELECT_LEVEL_RANGE2, &pRange, 0); -/* This statement is used to determine which level to read the input from -** when performing an incremental merge. It returns the absolute level number -** of the oldest level in the db that contains at least ? segments. Or, -** if no level in the FTS index contains more than ? segments, the statement -** returns zero rows. */ -/* 28 */ "SELECT level, count(*) AS cnt FROM %Q.'%q_segdir' " - " GROUP BY level HAVING cnt>=?" - " ORDER BY (level %% 1024) ASC LIMIT 1", + if( rc==SQLITE_OK ){ + int bOk = 0; + i64 iLast = (iAbsLevel/FTS3_SEGDIR_MAXLEVEL + 1) * FTS3_SEGDIR_MAXLEVEL - 1; + i64 nLimit = (nByte*3)/2; -/* Estimate the upper limit on the number of leaf nodes in a new segment -** created by merging the oldest :2 segments from absolute level :1. See -** function sqlite3Fts3Incrmerge() for details. */ -/* 29 */ "SELECT 2 * total(1 + leaves_end_block - start_block) " - " FROM %Q.'%q_segdir' WHERE level = ? AND idx < ?", + /* Loop through all entries in the %_segdir table corresponding to + ** segments in this index on levels greater than iAbsLevel. If there is + ** at least one such segment, and it is possible to determine that all + ** such segments are smaller than nLimit bytes in size, they will be + ** promoted to level iAbsLevel. */ + sqlite3_bind_int64(pRange, 1, iAbsLevel+1); + sqlite3_bind_int64(pRange, 2, iLast); + while( SQLITE_ROW==sqlite3_step(pRange) ){ + i64 nSize = 0, dummy; + fts3ReadEndBlockField(pRange, 2, &dummy, &nSize); + if( nSize<=0 || nSize>nLimit ){ + /* If nSize==0, then the %_segdir.end_block field does not not + ** contain a size value. This happens if it was written by an + ** old version of FTS. In this case it is not possible to determine + ** the size of the segment, and so segment promotion does not + ** take place. */ + bOk = 0; + break; + } + bOk = 1; + } + rc = sqlite3_reset(pRange); -/* SQL_DELETE_SEGDIR_ENTRY -** Delete the %_segdir entry on absolute level :1 with index :2. */ -/* 30 */ "DELETE FROM %Q.'%q_segdir' WHERE level = ? AND idx = ?", + if( bOk ){ + int iIdx = 0; + sqlite3_stmt *pUpdate1 = 0; + sqlite3_stmt *pUpdate2 = 0; -/* SQL_SHIFT_SEGDIR_ENTRY -** Modify the idx value for the segment with idx=:3 on absolute level :2 -** to :1. */ -/* 31 */ "UPDATE %Q.'%q_segdir' SET idx = ? WHERE level=? AND idx=?", + if( rc==SQLITE_OK ){ + rc = fts3SqlStmt(p, SQL_UPDATE_LEVEL_IDX, &pUpdate1, 0); + } + if( rc==SQLITE_OK ){ + rc = fts3SqlStmt(p, SQL_UPDATE_LEVEL, &pUpdate2, 0); + } -/* SQL_SELECT_SEGDIR -** Read a single entry from the %_segdir table. The entry from absolute -** level :1 with index value :2. */ -/* 32 */ "SELECT idx, start_block, leaves_end_block, end_block, root " - "FROM %Q.'%q_segdir' WHERE level = ? AND idx = ?", + if( rc==SQLITE_OK ){ -/* SQL_CHOMP_SEGDIR -** Update the start_block (:1) and root (:2) fields of the %_segdir -** entry located on absolute level :3 with index :4. */ -/* 33 */ "UPDATE %Q.'%q_segdir' SET start_block = ?, root = ?" - "WHERE level = ? AND idx = ?", + /* Loop through all %_segdir entries for segments in this index with + ** levels equal to or greater than iAbsLevel. As each entry is visited, + ** updated it to set (level = -1) and (idx = N), where N is 0 for the + ** oldest segment in the range, 1 for the next oldest, and so on. + ** + ** In other words, move all segments being promoted to level -1, + ** setting the "idx" fields as appropriate to keep them in the same + ** order. The contents of level -1 (which is never used, except + ** transiently here), will be moved back to level iAbsLevel below. */ + sqlite3_bind_int64(pRange, 1, iAbsLevel); + while( SQLITE_ROW==sqlite3_step(pRange) ){ + sqlite3_bind_int(pUpdate1, 1, iIdx++); + sqlite3_bind_int(pUpdate1, 2, sqlite3_column_int(pRange, 0)); + sqlite3_bind_int(pUpdate1, 3, sqlite3_column_int(pRange, 1)); + sqlite3_step(pUpdate1); + rc = sqlite3_reset(pUpdate1); + if( rc!=SQLITE_OK ){ + sqlite3_reset(pRange); + break; + } + } + } + if( rc==SQLITE_OK ){ + rc = sqlite3_reset(pRange); + } -/* SQL_SEGMENT_IS_APPENDABLE -** Return a single row if the segment with end_block=? is appendable. Or -** no rows otherwise. */ -/* 34 */ "SELECT 1 FROM %Q.'%q_segments' WHERE blockid=? AND block IS NULL", + /* Move level -1 to level iAbsLevel */ + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pUpdate2, 1, iAbsLevel); + sqlite3_step(pUpdate2); + rc = sqlite3_reset(pUpdate2); + } + } + } -/* SQL_SELECT_INDEXES -** Return the list of valid segment indexes for absolute level ? */ -/* 35 */ "SELECT idx FROM %Q.'%q_segdir' WHERE level=? ORDER BY 1 ASC", -/* SQL_SELECT_MXLEVEL -** Return the largest relative level in the FTS index or indexes. */ -/* 36 */ "SELECT max( level %% 1024 ) FROM %Q.'%q_segdir'", + return rc; +} - /* Return segments in order from oldest to newest.*/ -/* 37 */ "SELECT level, idx, end_block " - "FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ? " - "ORDER BY level DESC, idx ASC", +/* +** Merge all level iLevel segments in the database into a single +** iLevel+1 segment. Or, if iLevel<0, merge all segments into a +** single segment with a level equal to the numerically largest level +** currently present in the database. +** +** If this function is called with iLevel<0, but there is only one +** segment in the database, SQLITE_DONE is returned immediately. +** Otherwise, if successful, SQLITE_OK is returned. If an error occurs, +** an SQLite error code is returned. +*/ +static int fts3SegmentMerge( + Fts3Table *p, + int iLangid, /* Language id to merge */ + int iIndex, /* Index in p->aIndex[] to merge */ + int iLevel /* Level to merge */ +){ + int rc; /* Return code */ + int iIdx = 0; /* Index of new segment */ + sqlite3_int64 iNewLevel = 0; /* Level/index to create new segment at */ + SegmentWriter *pWriter = 0; /* Used to write the new, merged, segment */ + Fts3SegFilter filter; /* Segment term filter condition */ + Fts3MultiSegReader csr; /* Cursor to iterate through level(s) */ + int bIgnoreEmpty = 0; /* True to ignore empty segments */ + i64 iMaxLevel = 0; /* Max level number for this index/langid */ - /* Update statements used while promoting segments */ -/* 38 */ "UPDATE OR FAIL %Q.'%q_segdir' SET level=-1,idx=? " - "WHERE level=? AND idx=?", -/* 39 */ "UPDATE OR FAIL %Q.'%q_segdir' SET level=? WHERE level=-1" + assert( iLevel==FTS3_SEGCURSOR_ALL + || iLevel==FTS3_SEGCURSOR_PENDING + || iLevel>=0 + ); + assert( iLevel=0 && iIndexnIndex ); - }; - int rc = SQLITE_OK; - sqlite3_stmt *pStmt; + rc = sqlite3Fts3SegReaderCursor(p, iLangid, iIndex, iLevel, 0, 0, 1, 0, &csr); + if( rc!=SQLITE_OK || csr.nSegment==0 ) goto finished; - assert( SizeofArray(azSql)==SizeofArray(p->aStmt) ); - assert( eStmt=0 ); - - pStmt = p->aStmt[eStmt]; - if( !pStmt ){ - char *zSql; - if( eStmt==SQL_CONTENT_INSERT ){ - zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName, p->zWriteExprlist); - }else if( eStmt==SQL_SELECT_CONTENT_BY_ROWID ){ - zSql = sqlite3_mprintf(azSql[eStmt], p->zReadExprlist); - }else{ - zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName); - } - if( !zSql ){ - rc = SQLITE_NOMEM; - }else{ - rc = sqlite3_prepare_v3(p->db, zSql, -1, SQLITE_PREPARE_PERSISTENT, - &pStmt, NULL); - sqlite3_free(zSql); - assert( rc==SQLITE_OK || pStmt==0 ); - p->aStmt[eStmt] = pStmt; - } + if( iLevel!=FTS3_SEGCURSOR_PENDING ){ + rc = fts3SegmentMaxLevel(p, iLangid, iIndex, &iMaxLevel); + if( rc!=SQLITE_OK ) goto finished; } - if( apVal ){ - int i; - int nParam = sqlite3_bind_parameter_count(pStmt); - for(i=0; rc==SQLITE_OK && iiMaxLevel); } - *pp = pStmt; - return rc; -} + if( rc!=SQLITE_OK ) goto finished; + assert( csr.nSegment>0 ); + assert_fts3_nc( iNewLevel>=getAbsoluteLevel(p, iLangid, iIndex, 0) ); + assert_fts3_nc( + iNewLevelnLeafData); + } } } - *ppStmt = pStmt; + finished: + fts3SegWriterFree(pWriter); + sqlite3Fts3SegReaderFinish(&csr); return rc; } -SQLITE_PRIVATE int sqlite3Fts3SelectDoctotal( - Fts3Table *pTab, /* Fts3 table handle */ - sqlite3_stmt **ppStmt /* OUT: Statement handle */ -){ - sqlite3_stmt *pStmt = 0; - int rc; - rc = fts3SqlStmt(pTab, SQL_SELECT_STAT, &pStmt, 0); - if( rc==SQLITE_OK ){ - sqlite3_bind_int(pStmt, 1, FTS_STAT_DOCTOTAL); - if( sqlite3_step(pStmt)!=SQLITE_ROW - || sqlite3_column_type(pStmt, 0)!=SQLITE_BLOB - ){ + +/* +** Flush the contents of pendingTerms to level 0 segments. +*/ +SQLITE_PRIVATE int sqlite3Fts3PendingTermsFlush(Fts3Table *p){ + int rc = SQLITE_OK; + int i; + + for(i=0; rc==SQLITE_OK && inIndex; i++){ + rc = fts3SegmentMerge(p, p->iPrevLangid, i, FTS3_SEGCURSOR_PENDING); + if( rc==SQLITE_DONE ) rc = SQLITE_OK; + } + sqlite3Fts3PendingTermsClear(p); + + /* Determine the auto-incr-merge setting if unknown. If enabled, + ** estimate the number of leaf blocks of content to be written + */ + if( rc==SQLITE_OK && p->bHasStat + && p->nAutoincrmerge==0xff && p->nLeafAdd>0 + ){ + sqlite3_stmt *pStmt = 0; + rc = fts3SqlStmt(p, SQL_SELECT_STAT, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int(pStmt, 1, FTS_STAT_AUTOINCRMERGE); + rc = sqlite3_step(pStmt); + if( rc==SQLITE_ROW ){ + p->nAutoincrmerge = sqlite3_column_int(pStmt, 0); + if( p->nAutoincrmerge==1 ) p->nAutoincrmerge = 8; + }else if( rc==SQLITE_DONE ){ + p->nAutoincrmerge = 0; + } rc = sqlite3_reset(pStmt); - if( rc==SQLITE_OK ) rc = FTS_CORRUPT_VTAB; - pStmt = 0; } } - *ppStmt = pStmt; return rc; } -SQLITE_PRIVATE int sqlite3Fts3SelectDocsize( - Fts3Table *pTab, /* Fts3 table handle */ - sqlite3_int64 iDocid, /* Docid to read size data for */ - sqlite3_stmt **ppStmt /* OUT: Statement handle */ -){ - return fts3SelectDocsize(pTab, iDocid, ppStmt); -} - /* -** Similar to fts3SqlStmt(). Except, after binding the parameters in -** array apVal[] to the SQL statement identified by eStmt, the statement -** is executed. -** -** Returns SQLITE_OK if the statement is successfully executed, or an -** SQLite error code otherwise. +** Encode N integers as varints into a blob. */ -static void fts3SqlExec( - int *pRC, /* Result code */ - Fts3Table *p, /* The FTS3 table */ - int eStmt, /* Index of statement to evaluate */ - sqlite3_value **apVal /* Parameters to bind */ +static void fts3EncodeIntArray( + int N, /* The number of integers to encode */ + u32 *a, /* The integer values */ + char *zBuf, /* Write the BLOB here */ + int *pNBuf /* Write number of bytes if zBuf[] used here */ ){ - sqlite3_stmt *pStmt; - int rc; - if( *pRC ) return; - rc = fts3SqlStmt(p, eStmt, &pStmt, apVal); - if( rc==SQLITE_OK ){ - sqlite3_step(pStmt); - rc = sqlite3_reset(pStmt); + int i, j; + for(i=j=0; inPendingData==0 ){ - sqlite3_stmt *pStmt; - rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_LEVEL, &pStmt, 0); - if( rc==SQLITE_OK ){ - sqlite3_bind_null(pStmt, 1); - sqlite3_step(pStmt); - rc = sqlite3_reset(pStmt); +static void fts3DecodeIntArray( + int N, /* The number of integers to decode */ + u32 *a, /* Write the integer values */ + const char *zBuf, /* The BLOB containing the varints */ + int nBuf /* size of the BLOB */ +){ + int i = 0; + if( nBuf && (zBuf[nBuf-1]&0x80)==0 ){ + int j; + for(i=j=0; iiPrevDocid. The sizes are encoded as +** a blob of varints. */ -static sqlite3_int64 getAbsoluteLevel( - Fts3Table *p, /* FTS3 table handle */ - int iLangid, /* Language id */ - int iIndex, /* Index in p->aIndex[] */ - int iLevel /* Level of segments */ +static void fts3InsertDocsize( + int *pRC, /* Result code */ + Fts3Table *p, /* Table into which to insert */ + u32 *aSz /* Sizes of each column, in tokens */ ){ - sqlite3_int64 iBase; /* First absolute level for iLangid/iIndex */ - assert( iLangid>=0 ); - assert( p->nIndex>0 ); - assert( iIndex>=0 && iIndexnIndex ); + char *pBlob; /* The BLOB encoding of the document size */ + int nBlob; /* Number of bytes in the BLOB */ + sqlite3_stmt *pStmt; /* Statement used to insert the encoding */ + int rc; /* Result code from subfunctions */ - iBase = ((sqlite3_int64)iLangid * p->nIndex + iIndex) * FTS3_SEGDIR_MAXLEVEL; - return iBase + iLevel; + if( *pRC ) return; + pBlob = sqlite3_malloc64( 10*(sqlite3_int64)p->nColumn ); + if( pBlob==0 ){ + *pRC = SQLITE_NOMEM; + return; + } + fts3EncodeIntArray(p->nColumn, aSz, pBlob, &nBlob); + rc = fts3SqlStmt(p, SQL_REPLACE_DOCSIZE, &pStmt, 0); + if( rc ){ + sqlite3_free(pBlob); + *pRC = rc; + return; + } + sqlite3_bind_int64(pStmt, 1, p->iPrevDocid); + sqlite3_bind_blob(pStmt, 2, pBlob, nBlob, sqlite3_free); + sqlite3_step(pStmt); + *pRC = sqlite3_reset(pStmt); } /* -** Set *ppStmt to a statement handle that may be used to iterate through -** all rows in the %_segdir table, from oldest to newest. If successful, -** return SQLITE_OK. If an error occurs while preparing the statement, -** return an SQLite error code. +** Record 0 of the %_stat table contains a blob consisting of N varints, +** where N is the number of user defined columns in the fts3 table plus +** two. If nCol is the number of user defined columns, then values of the +** varints are set as follows: ** -** There is only ever one instance of this SQL statement compiled for -** each FTS3 table. +** Varint 0: Total number of rows in the table. ** -** The statement returns the following columns from the %_segdir table: +** Varint 1..nCol: For each column, the total number of tokens stored in +** the column for all rows of the table. +** +** Varint 1+nCol: The total size, in bytes, of all text values in all +** columns of all rows of the table. ** -** 0: idx -** 1: start_block -** 2: leaves_end_block -** 3: end_block -** 4: root */ -SQLITE_PRIVATE int sqlite3Fts3AllSegdirs( - Fts3Table *p, /* FTS3 table */ - int iLangid, /* Language being queried */ - int iIndex, /* Index for p->aIndex[] */ - int iLevel, /* Level to select (relative level) */ - sqlite3_stmt **ppStmt /* OUT: Compiled statement */ +static void fts3UpdateDocTotals( + int *pRC, /* The result code */ + Fts3Table *p, /* Table being updated */ + u32 *aSzIns, /* Size increases */ + u32 *aSzDel, /* Size decreases */ + int nChng /* Change in the number of documents */ ){ - int rc; - sqlite3_stmt *pStmt = 0; + char *pBlob; /* Storage for BLOB written into %_stat */ + int nBlob; /* Size of BLOB written into %_stat */ + u32 *a; /* Array of integers that becomes the BLOB */ + sqlite3_stmt *pStmt; /* Statement for reading and writing */ + int i; /* Loop counter */ + int rc; /* Result code from subfunctions */ - assert( iLevel==FTS3_SEGCURSOR_ALL || iLevel>=0 ); - assert( iLevel=0 && iIndexnIndex ); + const int nStat = p->nColumn+2; - if( iLevel<0 ){ - /* "SELECT * FROM %_segdir WHERE level BETWEEN ? AND ? ORDER BY ..." */ - rc = fts3SqlStmt(p, SQL_SELECT_LEVEL_RANGE, &pStmt, 0); - if( rc==SQLITE_OK ){ - sqlite3_bind_int64(pStmt, 1, getAbsoluteLevel(p, iLangid, iIndex, 0)); - sqlite3_bind_int64(pStmt, 2, - getAbsoluteLevel(p, iLangid, iIndex, FTS3_SEGDIR_MAXLEVEL-1) - ); - } + if( *pRC ) return; + a = sqlite3_malloc64( (sizeof(u32)+10)*(sqlite3_int64)nStat ); + if( a==0 ){ + *pRC = SQLITE_NOMEM; + return; + } + pBlob = (char*)&a[nStat]; + rc = fts3SqlStmt(p, SQL_SELECT_STAT, &pStmt, 0); + if( rc ){ + sqlite3_free(a); + *pRC = rc; + return; + } + sqlite3_bind_int(pStmt, 1, FTS_STAT_DOCTOTAL); + if( sqlite3_step(pStmt)==SQLITE_ROW ){ + fts3DecodeIntArray(nStat, a, + sqlite3_column_blob(pStmt, 0), + sqlite3_column_bytes(pStmt, 0)); }else{ - /* "SELECT * FROM %_segdir WHERE level = ? ORDER BY ..." */ - rc = fts3SqlStmt(p, SQL_SELECT_LEVEL, &pStmt, 0); - if( rc==SQLITE_OK ){ - sqlite3_bind_int64(pStmt, 1, getAbsoluteLevel(p, iLangid, iIndex,iLevel)); + memset(a, 0, sizeof(u32)*(nStat) ); + } + rc = sqlite3_reset(pStmt); + if( rc!=SQLITE_OK ){ + sqlite3_free(a); + *pRC = rc; + return; + } + if( nChng<0 && a[0]<(u32)(-nChng) ){ + a[0] = 0; + }else{ + a[0] += nChng; + } + for(i=0; inColumn+1; i++){ + u32 x = a[i+1]; + if( x+aSzIns[i] < aSzDel[i] ){ + x = 0; + }else{ + x = x + aSzIns[i] - aSzDel[i]; } + a[i+1] = x; } - *ppStmt = pStmt; - return rc; + fts3EncodeIntArray(nStat, a, pBlob, &nBlob); + rc = fts3SqlStmt(p, SQL_REPLACE_STAT, &pStmt, 0); + if( rc ){ + sqlite3_free(a); + *pRC = rc; + return; + } + sqlite3_bind_int(pStmt, 1, FTS_STAT_DOCTOTAL); + sqlite3_bind_blob(pStmt, 2, pBlob, nBlob, SQLITE_STATIC); + sqlite3_step(pStmt); + *pRC = sqlite3_reset(pStmt); + sqlite3_bind_null(pStmt, 2); + sqlite3_free(a); } - /* -** Append a single varint to a PendingList buffer. SQLITE_OK is returned -** if successful, or an SQLite error code otherwise. -** -** This function also serves to allocate the PendingList structure itself. -** For example, to create a new PendingList structure containing two -** varints: -** -** PendingList *p = 0; -** fts3PendingListAppendVarint(&p, 1); -** fts3PendingListAppendVarint(&p, 2); +** Merge the entire database so that there is one segment for each +** iIndex/iLangid combination. */ -static int fts3PendingListAppendVarint( - PendingList **pp, /* IN/OUT: Pointer to PendingList struct */ - sqlite3_int64 i /* Value to append to data */ -){ - PendingList *p = *pp; +static int fts3DoOptimize(Fts3Table *p, int bReturnDone){ + int bSeenDone = 0; + int rc; + sqlite3_stmt *pAllLangid = 0; - /* Allocate or grow the PendingList as required. */ - if( !p ){ - p = sqlite3_malloc(sizeof(*p) + 100); - if( !p ){ - return SQLITE_NOMEM; - } - p->nSpace = 100; - p->aData = (char *)&p[1]; - p->nData = 0; + rc = sqlite3Fts3PendingTermsFlush(p); + if( rc==SQLITE_OK ){ + rc = fts3SqlStmt(p, SQL_SELECT_ALL_LANGID, &pAllLangid, 0); } - else if( p->nData+FTS3_VARINT_MAX+1>p->nSpace ){ - int nNew = p->nSpace * 2; - p = sqlite3_realloc(p, sizeof(*p) + nNew); - if( !p ){ - sqlite3_free(*pp); - *pp = 0; - return SQLITE_NOMEM; + if( rc==SQLITE_OK ){ + int rc2; + sqlite3_bind_int(pAllLangid, 1, p->iPrevLangid); + sqlite3_bind_int(pAllLangid, 2, p->nIndex); + while( sqlite3_step(pAllLangid)==SQLITE_ROW ){ + int i; + int iLangid = sqlite3_column_int(pAllLangid, 0); + for(i=0; rc==SQLITE_OK && inIndex; i++){ + rc = fts3SegmentMerge(p, iLangid, i, FTS3_SEGCURSOR_ALL); + if( rc==SQLITE_DONE ){ + bSeenDone = 1; + rc = SQLITE_OK; + } + } } - p->nSpace = nNew; - p->aData = (char *)&p[1]; + rc2 = sqlite3_reset(pAllLangid); + if( rc==SQLITE_OK ) rc = rc2; } - /* Append the new serialized varint to the end of the list. */ - p->nData += sqlite3Fts3PutVarint(&p->aData[p->nData], i); - p->aData[p->nData] = '\0'; - *pp = p; - return SQLITE_OK; + sqlite3Fts3SegmentsClose(p); + + return (rc==SQLITE_OK && bReturnDone && bSeenDone) ? SQLITE_DONE : rc; } /* -** Add a docid/column/position entry to a PendingList structure. Non-zero -** is returned if the structure is sqlite3_realloced as part of adding -** the entry. Otherwise, zero. +** This function is called when the user executes the following statement: ** -** If an OOM error occurs, *pRc is set to SQLITE_NOMEM before returning. -** Zero is always returned in this case. Otherwise, if no OOM error occurs, -** it is set to SQLITE_OK. +** INSERT INTO () VALUES('rebuild'); +** +** The entire FTS index is discarded and rebuilt. If the table is one +** created using the content=xxx option, then the new index is based on +** the current contents of the xxx table. Otherwise, it is rebuilt based +** on the contents of the %_content table. */ -static int fts3PendingListAppend( - PendingList **pp, /* IN/OUT: PendingList structure */ - sqlite3_int64 iDocid, /* Docid for entry to add */ - sqlite3_int64 iCol, /* Column for entry to add */ - sqlite3_int64 iPos, /* Position of term for entry to add */ - int *pRc /* OUT: Return code */ -){ - PendingList *p = *pp; - int rc = SQLITE_OK; +static int fts3DoRebuild(Fts3Table *p){ + int rc; /* Return Code */ - assert( !p || p->iLastDocid<=iDocid ); + rc = fts3DeleteAll(p, 0); + if( rc==SQLITE_OK ){ + u32 *aSz = 0; + u32 *aSzIns = 0; + u32 *aSzDel = 0; + sqlite3_stmt *pStmt = 0; + int nEntry = 0; - if( !p || p->iLastDocid!=iDocid ){ - sqlite3_int64 iDelta = iDocid - (p ? p->iLastDocid : 0); - if( p ){ - assert( p->nDatanSpace ); - assert( p->aData[p->nData]==0 ); - p->nData++; - } - if( SQLITE_OK!=(rc = fts3PendingListAppendVarint(&p, iDelta)) ){ - goto pendinglistappend_out; - } - p->iLastCol = -1; - p->iLastPos = 0; - p->iLastDocid = iDocid; - } - if( iCol>0 && p->iLastCol!=iCol ){ - if( SQLITE_OK!=(rc = fts3PendingListAppendVarint(&p, 1)) - || SQLITE_OK!=(rc = fts3PendingListAppendVarint(&p, iCol)) - ){ - goto pendinglistappend_out; + /* Compose and prepare an SQL statement to loop through the content table */ + char *zSql = sqlite3_mprintf("SELECT %s" , p->zReadExprlist); + if( !zSql ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); } - p->iLastCol = iCol; - p->iLastPos = 0; - } - if( iCol>=0 ){ - assert( iPos>p->iLastPos || (iPos==0 && p->iLastPos==0) ); - rc = fts3PendingListAppendVarint(&p, 2+iPos-p->iLastPos); + if( rc==SQLITE_OK ){ - p->iLastPos = iPos; + sqlite3_int64 nByte = sizeof(u32) * ((sqlite3_int64)p->nColumn+1)*3; + aSz = (u32 *)sqlite3_malloc64(nByte); + if( aSz==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(aSz, 0, nByte); + aSzIns = &aSz[p->nColumn+1]; + aSzDel = &aSzIns[p->nColumn+1]; + } } - } - - pendinglistappend_out: - *pRc = rc; - if( p!=*pp ){ - *pp = p; - return 1; - } - return 0; -} - -/* -** Free a PendingList object allocated by fts3PendingListAppend(). -*/ -static void fts3PendingListDelete(PendingList *pList){ - sqlite3_free(pList); -} -/* -** Add an entry to one of the pending-terms hash tables. -*/ -static int fts3PendingTermsAddOne( - Fts3Table *p, - int iCol, - int iPos, - Fts3Hash *pHash, /* Pending terms hash table to add entry to */ - const char *zToken, - int nToken -){ - PendingList *pList; - int rc = SQLITE_OK; + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + int iCol; + int iLangid = langidFromSelect(p, pStmt); + rc = fts3PendingTermsDocid(p, 0, iLangid, sqlite3_column_int64(pStmt, 0)); + memset(aSz, 0, sizeof(aSz[0]) * (p->nColumn+1)); + for(iCol=0; rc==SQLITE_OK && iColnColumn; iCol++){ + if( p->abNotindexed[iCol]==0 ){ + const char *z = (const char *) sqlite3_column_text(pStmt, iCol+1); + rc = fts3PendingTermsAdd(p, iLangid, z, iCol, &aSz[iCol]); + aSz[p->nColumn] += sqlite3_column_bytes(pStmt, iCol+1); + } + } + if( p->bHasDocsize ){ + fts3InsertDocsize(&rc, p, aSz); + } + if( rc!=SQLITE_OK ){ + sqlite3_finalize(pStmt); + pStmt = 0; + }else{ + nEntry++; + for(iCol=0; iCol<=p->nColumn; iCol++){ + aSzIns[iCol] += aSz[iCol]; + } + } + } + if( p->bFts4 ){ + fts3UpdateDocTotals(&rc, p, aSzIns, aSzDel, nEntry); + } + sqlite3_free(aSz); - pList = (PendingList *)fts3HashFind(pHash, zToken, nToken); - if( pList ){ - p->nPendingData -= (pList->nData + nToken + sizeof(Fts3HashElem)); - } - if( fts3PendingListAppend(&pList, p->iPrevDocid, iCol, iPos, &rc) ){ - if( pList==fts3HashInsert(pHash, zToken, nToken, pList) ){ - /* Malloc failed while inserting the new entry. This can only - ** happen if there was no previous entry for this token. - */ - assert( 0==fts3HashFind(pHash, zToken, nToken) ); - sqlite3_free(pList); - rc = SQLITE_NOMEM; + if( pStmt ){ + int rc2 = sqlite3_finalize(pStmt); + if( rc==SQLITE_OK ){ + rc = rc2; + } } } - if( rc==SQLITE_OK ){ - p->nPendingData += (pList->nData + nToken + sizeof(Fts3HashElem)); - } + return rc; } + /* -** Tokenize the nul-terminated string zText and add all tokens to the -** pending-terms hash-table. The docid used is that currently stored in -** p->iPrevDocid, and the column is specified by argument iCol. -** -** If successful, SQLITE_OK is returned. Otherwise, an SQLite error code. +** This function opens a cursor used to read the input data for an +** incremental merge operation. Specifically, it opens a cursor to scan +** the oldest nSeg segments (idx=0 through idx=(nSeg-1)) in absolute +** level iAbsLevel. */ -static int fts3PendingTermsAdd( - Fts3Table *p, /* Table into which text will be inserted */ - int iLangid, /* Language id to use */ - const char *zText, /* Text of document to be inserted */ - int iCol, /* Column into which text is being inserted */ - u32 *pnWord /* IN/OUT: Incr. by number tokens inserted */ +static int fts3IncrmergeCsr( + Fts3Table *p, /* FTS3 table handle */ + sqlite3_int64 iAbsLevel, /* Absolute level to open */ + int nSeg, /* Number of segments to merge */ + Fts3MultiSegReader *pCsr /* Cursor object to populate */ ){ - int rc; - int iStart = 0; - int iEnd = 0; - int iPos = 0; - int nWord = 0; - - char const *zToken; - int nToken = 0; - - sqlite3_tokenizer *pTokenizer = p->pTokenizer; - sqlite3_tokenizer_module const *pModule = pTokenizer->pModule; - sqlite3_tokenizer_cursor *pCsr; - int (*xNext)(sqlite3_tokenizer_cursor *pCursor, - const char**,int*,int*,int*,int*); - - assert( pTokenizer && pModule ); + int rc; /* Return Code */ + sqlite3_stmt *pStmt = 0; /* Statement used to read %_segdir entry */ + sqlite3_int64 nByte; /* Bytes allocated at pCsr->apSegment[] */ - /* If the user has inserted a NULL value, this function may be called with - ** zText==0. In this case, add zero token entries to the hash table and - ** return early. */ - if( zText==0 ){ - *pnWord = 0; - return SQLITE_OK; - } + /* Allocate space for the Fts3MultiSegReader.aCsr[] array */ + memset(pCsr, 0, sizeof(*pCsr)); + nByte = sizeof(Fts3SegReader *) * nSeg; + pCsr->apSegment = (Fts3SegReader **)sqlite3_malloc64(nByte); - rc = sqlite3Fts3OpenTokenizer(pTokenizer, iLangid, zText, -1, &pCsr); - if( rc!=SQLITE_OK ){ - return rc; + if( pCsr->apSegment==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(pCsr->apSegment, 0, nByte); + rc = fts3SqlStmt(p, SQL_SELECT_LEVEL, &pStmt, 0); } - - xNext = pModule->xNext; - while( SQLITE_OK==rc - && SQLITE_OK==(rc = xNext(pCsr, &zToken, &nToken, &iStart, &iEnd, &iPos)) - ){ + if( rc==SQLITE_OK ){ int i; - if( iPos>=nWord ) nWord = iPos+1; - - /* Positions cannot be negative; we use -1 as a terminator internally. - ** Tokens must have a non-zero length. - */ - if( iPos<0 || !zToken || nToken<=0 ){ - rc = SQLITE_ERROR; - break; - } - - /* Add the term to the terms index */ - rc = fts3PendingTermsAddOne( - p, iCol, iPos, &p->aIndex[0].hPending, zToken, nToken - ); - - /* Add the term to each of the prefix indexes that it is not too - ** short for. */ - for(i=1; rc==SQLITE_OK && inIndex; i++){ - struct Fts3Index *pIndex = &p->aIndex[i]; - if( nTokennPrefix ) continue; - rc = fts3PendingTermsAddOne( - p, iCol, iPos, &pIndex->hPending, zToken, pIndex->nPrefix + int rc2; + sqlite3_bind_int64(pStmt, 1, iAbsLevel); + assert( pCsr->nSegment==0 ); + for(i=0; rc==SQLITE_OK && sqlite3_step(pStmt)==SQLITE_ROW && iapSegment[i] ); + pCsr->nSegment++; } + rc2 = sqlite3_reset(pStmt); + if( rc==SQLITE_OK ) rc = rc2; } - pModule->xClose(pCsr); - *pnWord += nWord; - return (rc==SQLITE_DONE ? SQLITE_OK : rc); + return rc; } -/* -** Calling this function indicates that subsequent calls to -** fts3PendingTermsAdd() are to add term/position-list pairs for the -** contents of the document with docid iDocid. -*/ -static int fts3PendingTermsDocid( - Fts3Table *p, /* Full-text table handle */ - int bDelete, /* True if this op is a delete */ - int iLangid, /* Language id of row being written */ - sqlite_int64 iDocid /* Docid of row being written */ -){ - assert( iLangid>=0 ); - assert( bDelete==1 || bDelete==0 ); +typedef struct IncrmergeWriter IncrmergeWriter; +typedef struct NodeWriter NodeWriter; +typedef struct Blob Blob; +typedef struct NodeReader NodeReader; - /* TODO(shess) Explore whether partially flushing the buffer on - ** forced-flush would provide better performance. I suspect that if - ** we ordered the doclists by size and flushed the largest until the - ** buffer was half empty, that would let the less frequent terms - ** generate longer doclists. - */ - if( iDocidiPrevDocid - || (iDocid==p->iPrevDocid && p->bPrevDelete==0) - || p->iPrevLangid!=iLangid - || p->nPendingData>p->nMaxPendingData - ){ - int rc = sqlite3Fts3PendingTermsFlush(p); - if( rc!=SQLITE_OK ) return rc; - } - p->iPrevDocid = iDocid; - p->iPrevLangid = iLangid; - p->bPrevDelete = bDelete; - return SQLITE_OK; -} +/* +** An instance of the following structure is used as a dynamic buffer +** to build up nodes or other blobs of data in. +** +** The function blobGrowBuffer() is used to extend the allocation. +*/ +struct Blob { + char *a; /* Pointer to allocation */ + int n; /* Number of valid bytes of data in a[] */ + int nAlloc; /* Allocated size of a[] (nAlloc>=n) */ +}; /* -** Discard the contents of the pending-terms hash tables. +** This structure is used to build up buffers containing segment b-tree +** nodes (blocks). */ -SQLITE_PRIVATE void sqlite3Fts3PendingTermsClear(Fts3Table *p){ - int i; - for(i=0; inIndex; i++){ - Fts3HashElem *pElem; - Fts3Hash *pHash = &p->aIndex[i].hPending; - for(pElem=fts3HashFirst(pHash); pElem; pElem=fts3HashNext(pElem)){ - PendingList *pList = (PendingList *)fts3HashData(pElem); - fts3PendingListDelete(pList); - } - fts3HashClear(pHash); - } - p->nPendingData = 0; -} +struct NodeWriter { + sqlite3_int64 iBlock; /* Current block id */ + Blob key; /* Last key written to the current block */ + Blob block; /* Current block image */ +}; /* -** This function is called by the xUpdate() method as part of an INSERT -** operation. It adds entries for each term in the new record to the -** pendingTerms hash table. -** -** Argument apVal is the same as the similarly named argument passed to -** fts3InsertData(). Parameter iDocid is the docid of the new row. +** An object of this type contains the state required to create or append +** to an appendable b-tree segment. */ -static int fts3InsertTerms( - Fts3Table *p, - int iLangid, - sqlite3_value **apVal, - u32 *aSz -){ - int i; /* Iterator variable */ - for(i=2; inColumn+2; i++){ - int iCol = i-2; - if( p->abNotindexed[iCol]==0 ){ - const char *zText = (const char *)sqlite3_value_text(apVal[i]); - int rc = fts3PendingTermsAdd(p, iLangid, zText, iCol, &aSz[iCol]); - if( rc!=SQLITE_OK ){ - return rc; - } - aSz[p->nColumn] += sqlite3_value_bytes(apVal[i]); - } - } - return SQLITE_OK; -} +struct IncrmergeWriter { + int nLeafEst; /* Space allocated for leaf blocks */ + int nWork; /* Number of leaf pages flushed */ + sqlite3_int64 iAbsLevel; /* Absolute level of input segments */ + int iIdx; /* Index of *output* segment in iAbsLevel+1 */ + sqlite3_int64 iStart; /* Block number of first allocated block */ + sqlite3_int64 iEnd; /* Block number of last allocated block */ + sqlite3_int64 nLeafData; /* Bytes of leaf page data so far */ + u8 bNoLeafData; /* If true, store 0 for segment size */ + NodeWriter aNodeWriter[FTS_MAX_APPENDABLE_HEIGHT]; +}; /* -** This function is called by the xUpdate() method for an INSERT operation. -** The apVal parameter is passed a copy of the apVal argument passed by -** SQLite to the xUpdate() method. i.e: +** An object of the following type is used to read data from a single +** FTS segment node. See the following functions: ** -** apVal[0] Not used for INSERT. -** apVal[1] rowid -** apVal[2] Left-most user-defined column -** ... -** apVal[p->nColumn+1] Right-most user-defined column -** apVal[p->nColumn+2] Hidden column with same name as table -** apVal[p->nColumn+3] Hidden "docid" column (alias for rowid) -** apVal[p->nColumn+4] Hidden languageid column +** nodeReaderInit() +** nodeReaderNext() +** nodeReaderRelease() */ -static int fts3InsertData( - Fts3Table *p, /* Full-text table */ - sqlite3_value **apVal, /* Array of values to insert */ - sqlite3_int64 *piDocid /* OUT: Docid for row just inserted */ -){ - int rc; /* Return code */ - sqlite3_stmt *pContentInsert; /* INSERT INTO %_content VALUES(...) */ - - if( p->zContentTbl ){ - sqlite3_value *pRowid = apVal[p->nColumn+3]; - if( sqlite3_value_type(pRowid)==SQLITE_NULL ){ - pRowid = apVal[1]; - } - if( sqlite3_value_type(pRowid)!=SQLITE_INTEGER ){ - return SQLITE_CONSTRAINT; - } - *piDocid = sqlite3_value_int64(pRowid); - return SQLITE_OK; - } +struct NodeReader { + const char *aNode; + int nNode; + int iOff; /* Current offset within aNode[] */ - /* Locate the statement handle used to insert data into the %_content - ** table. The SQL for this statement is: - ** - ** INSERT INTO %_content VALUES(?, ?, ?, ...) - ** - ** The statement features N '?' variables, where N is the number of user - ** defined columns in the FTS3 table, plus one for the docid field. - */ - rc = fts3SqlStmt(p, SQL_CONTENT_INSERT, &pContentInsert, &apVal[1]); - if( rc==SQLITE_OK && p->zLanguageid ){ - rc = sqlite3_bind_int( - pContentInsert, p->nColumn+2, - sqlite3_value_int(apVal[p->nColumn+4]) - ); - } - if( rc!=SQLITE_OK ) return rc; + /* Output variables. Containing the current node entry. */ + sqlite3_int64 iChild; /* Pointer to child node */ + Blob term; /* Current term */ + const char *aDoclist; /* Pointer to doclist */ + int nDoclist; /* Size of doclist in bytes */ +}; - /* There is a quirk here. The users INSERT statement may have specified - ** a value for the "rowid" field, for the "docid" field, or for both. - ** Which is a problem, since "rowid" and "docid" are aliases for the - ** same value. For example: - ** - ** INSERT INTO fts3tbl(rowid, docid) VALUES(1, 2); - ** - ** In FTS3, this is an error. It is an error to specify non-NULL values - ** for both docid and some other rowid alias. - */ - if( SQLITE_NULL!=sqlite3_value_type(apVal[3+p->nColumn]) ){ - if( SQLITE_NULL==sqlite3_value_type(apVal[0]) - && SQLITE_NULL!=sqlite3_value_type(apVal[1]) - ){ - /* A rowid/docid conflict. */ - return SQLITE_ERROR; +/* +** If *pRc is not SQLITE_OK when this function is called, it is a no-op. +** Otherwise, if the allocation at pBlob->a is not already at least nMin +** bytes in size, extend (realloc) it to be so. +** +** If an OOM error occurs, set *pRc to SQLITE_NOMEM and leave pBlob->a +** unmodified. Otherwise, if the allocation succeeds, update pBlob->nAlloc +** to reflect the new size of the pBlob->a[] buffer. +*/ +static void blobGrowBuffer(Blob *pBlob, int nMin, int *pRc){ + if( *pRc==SQLITE_OK && nMin>pBlob->nAlloc ){ + int nAlloc = nMin; + char *a = (char *)sqlite3_realloc(pBlob->a, nAlloc); + if( a ){ + pBlob->nAlloc = nAlloc; + pBlob->a = a; + }else{ + *pRc = SQLITE_NOMEM; } - rc = sqlite3_bind_value(pContentInsert, 1, apVal[3+p->nColumn]); - if( rc!=SQLITE_OK ) return rc; } - - /* Execute the statement to insert the record. Set *piDocid to the - ** new docid value. - */ - sqlite3_step(pContentInsert); - rc = sqlite3_reset(pContentInsert); - - *piDocid = sqlite3_last_insert_rowid(p->db); - return rc; } - - /* -** Remove all data from the FTS3 table. Clear the hash table containing -** pending terms. +** Attempt to advance the node-reader object passed as the first argument to +** the next entry on the node. +** +** Return an error code if an error occurs (SQLITE_NOMEM is possible). +** Otherwise return SQLITE_OK. If there is no next entry on the node +** (e.g. because the current entry is the last) set NodeReader->aNode to +** NULL to indicate EOF. Otherwise, populate the NodeReader structure output +** variables for the new entry. */ -static int fts3DeleteAll(Fts3Table *p, int bContent){ +static int nodeReaderNext(NodeReader *p){ + int bFirst = (p->term.n==0); /* True for first term on the node */ + int nPrefix = 0; /* Bytes to copy from previous term */ + int nSuffix = 0; /* Bytes to append to the prefix */ int rc = SQLITE_OK; /* Return code */ - /* Discard the contents of the pending-terms hash table. */ - sqlite3Fts3PendingTermsClear(p); + assert( p->aNode ); + if( p->iChild && bFirst==0 ) p->iChild++; + if( p->iOff>=p->nNode ){ + /* EOF */ + p->aNode = 0; + }else{ + if( bFirst==0 ){ + p->iOff += fts3GetVarint32(&p->aNode[p->iOff], &nPrefix); + } + p->iOff += fts3GetVarint32(&p->aNode[p->iOff], &nSuffix); - /* Delete everything from the shadow tables. Except, leave %_content as - ** is if bContent is false. */ - assert( p->zContentTbl==0 || bContent==0 ); - if( bContent ) fts3SqlExec(&rc, p, SQL_DELETE_ALL_CONTENT, 0); - fts3SqlExec(&rc, p, SQL_DELETE_ALL_SEGMENTS, 0); - fts3SqlExec(&rc, p, SQL_DELETE_ALL_SEGDIR, 0); - if( p->bHasDocsize ){ - fts3SqlExec(&rc, p, SQL_DELETE_ALL_DOCSIZE, 0); - } - if( p->bHasStat ){ - fts3SqlExec(&rc, p, SQL_DELETE_ALL_STAT, 0); + if( nPrefix>p->term.n || nSuffix>p->nNode-p->iOff || nSuffix==0 ){ + return FTS_CORRUPT_VTAB; + } + blobGrowBuffer(&p->term, nPrefix+nSuffix, &rc); + if( rc==SQLITE_OK ){ + memcpy(&p->term.a[nPrefix], &p->aNode[p->iOff], nSuffix); + p->term.n = nPrefix+nSuffix; + p->iOff += nSuffix; + if( p->iChild==0 ){ + p->iOff += fts3GetVarint32(&p->aNode[p->iOff], &p->nDoclist); + if( (p->nNode-p->iOff)nDoclist ){ + return FTS_CORRUPT_VTAB; + } + p->aDoclist = &p->aNode[p->iOff]; + p->iOff += p->nDoclist; + } + } } + + assert_fts3_nc( p->iOff<=p->nNode ); return rc; } /* -** +** Release all dynamic resources held by node-reader object *p. */ -static int langidFromSelect(Fts3Table *p, sqlite3_stmt *pSelect){ - int iLangid = 0; - if( p->zLanguageid ) iLangid = sqlite3_column_int(pSelect, p->nColumn+1); - return iLangid; +static void nodeReaderRelease(NodeReader *p){ + sqlite3_free(p->term.a); } /* -** The first element in the apVal[] array is assumed to contain the docid -** (an integer) of a row about to be deleted. Remove all terms from the -** full-text index. +** Initialize a node-reader object to read the node in buffer aNode/nNode. +** +** If successful, SQLITE_OK is returned and the NodeReader object set to +** point to the first entry on the node (if any). Otherwise, an SQLite +** error code is returned. */ -static void fts3DeleteTerms( - int *pRC, /* Result code */ - Fts3Table *p, /* The FTS table to delete from */ - sqlite3_value *pRowid, /* The docid to be deleted */ - u32 *aSz, /* Sizes of deleted document written here */ - int *pbFound /* OUT: Set to true if row really does exist */ -){ - int rc; - sqlite3_stmt *pSelect; +static int nodeReaderInit(NodeReader *p, const char *aNode, int nNode){ + memset(p, 0, sizeof(NodeReader)); + p->aNode = aNode; + p->nNode = nNode; - assert( *pbFound==0 ); - if( *pRC ) return; - rc = fts3SqlStmt(p, SQL_SELECT_CONTENT_BY_ROWID, &pSelect, &pRowid); - if( rc==SQLITE_OK ){ - if( SQLITE_ROW==sqlite3_step(pSelect) ){ - int i; - int iLangid = langidFromSelect(p, pSelect); - i64 iDocid = sqlite3_column_int64(pSelect, 0); - rc = fts3PendingTermsDocid(p, 1, iLangid, iDocid); - for(i=1; rc==SQLITE_OK && i<=p->nColumn; i++){ - int iCol = i-1; - if( p->abNotindexed[iCol]==0 ){ - const char *zText = (const char *)sqlite3_column_text(pSelect, i); - rc = fts3PendingTermsAdd(p, iLangid, zText, -1, &aSz[iCol]); - aSz[p->nColumn] += sqlite3_column_bytes(pSelect, i); - } - } - if( rc!=SQLITE_OK ){ - sqlite3_reset(pSelect); - *pRC = rc; - return; - } - *pbFound = 1; - } - rc = sqlite3_reset(pSelect); + /* Figure out if this is a leaf or an internal node. */ + if( aNode && aNode[0] ){ + /* An internal node. */ + p->iOff = 1 + sqlite3Fts3GetVarint(&p->aNode[1], &p->iChild); }else{ - sqlite3_reset(pSelect); + p->iOff = 1; } - *pRC = rc; + + return aNode ? nodeReaderNext(p) : SQLITE_OK; } /* -** Forward declaration to account for the circular dependency between -** functions fts3SegmentMerge() and fts3AllocateSegdirIdx(). -*/ -static int fts3SegmentMerge(Fts3Table *, int, int, int); - -/* -** This function allocates a new level iLevel index in the segdir table. -** Usually, indexes are allocated within a level sequentially starting -** with 0, so the allocated index is one greater than the value returned -** by: -** -** SELECT max(idx) FROM %_segdir WHERE level = :iLevel -** -** However, if there are already FTS3_MERGE_COUNT indexes at the requested -** level, they are merged into a single level (iLevel+1) segment and the -** allocated index is 0. +** This function is called while writing an FTS segment each time a leaf o +** node is finished and written to disk. The key (zTerm/nTerm) is guaranteed +** to be greater than the largest key on the node just written, but smaller +** than or equal to the first key that will be written to the next leaf +** node. ** -** If successful, *piIdx is set to the allocated index slot and SQLITE_OK -** returned. Otherwise, an SQLite error code is returned. +** The block id of the leaf node just written to disk may be found in +** (pWriter->aNodeWriter[0].iBlock) when this function is called. */ -static int fts3AllocateSegdirIdx( - Fts3Table *p, - int iLangid, /* Language id */ - int iIndex, /* Index for p->aIndex */ - int iLevel, - int *piIdx +static int fts3IncrmergePush( + Fts3Table *p, /* Fts3 table handle */ + IncrmergeWriter *pWriter, /* Writer object */ + const char *zTerm, /* Term to write to internal node */ + int nTerm /* Bytes at zTerm */ ){ - int rc; /* Return Code */ - sqlite3_stmt *pNextIdx; /* Query for next idx at level iLevel */ - int iNext = 0; /* Result of query pNextIdx */ + sqlite3_int64 iPtr = pWriter->aNodeWriter[0].iBlock; + int iLayer; - assert( iLangid>=0 ); - assert( p->nIndex>=1 ); + assert( nTerm>0 ); + for(iLayer=1; ALWAYS(iLayeraNodeWriter[iLayer]; + int rc = SQLITE_OK; + int nPrefix; + int nSuffix; + int nSpace; - /* Set variable iNext to the next available segdir index at level iLevel. */ - rc = fts3SqlStmt(p, SQL_NEXT_SEGMENT_INDEX, &pNextIdx, 0); - if( rc==SQLITE_OK ){ - sqlite3_bind_int64( - pNextIdx, 1, getAbsoluteLevel(p, iLangid, iIndex, iLevel) - ); - if( SQLITE_ROW==sqlite3_step(pNextIdx) ){ - iNext = sqlite3_column_int(pNextIdx, 0); - } - rc = sqlite3_reset(pNextIdx); - } + /* Figure out how much space the key will consume if it is written to + ** the current node of layer iLayer. Due to the prefix compression, + ** the space required changes depending on which node the key is to + ** be added to. */ + nPrefix = fts3PrefixCompress(pNode->key.a, pNode->key.n, zTerm, nTerm); + nSuffix = nTerm - nPrefix; + if(nSuffix<=0 ) return FTS_CORRUPT_VTAB; + nSpace = sqlite3Fts3VarintLen(nPrefix); + nSpace += sqlite3Fts3VarintLen(nSuffix) + nSuffix; - if( rc==SQLITE_OK ){ - /* If iNext is FTS3_MERGE_COUNT, indicating that level iLevel is already - ** full, merge all segments in level iLevel into a single iLevel+1 - ** segment and allocate (newly freed) index 0 at level iLevel. Otherwise, - ** if iNext is less than FTS3_MERGE_COUNT, allocate index iNext. - */ - if( iNext>=FTS3_MERGE_COUNT ){ - fts3LogMerge(16, getAbsoluteLevel(p, iLangid, iIndex, iLevel)); - rc = fts3SegmentMerge(p, iLangid, iIndex, iLevel); - *piIdx = 0; + if( pNode->key.n==0 || (pNode->block.n + nSpace)<=p->nNodeSize ){ + /* If the current node of layer iLayer contains zero keys, or if adding + ** the key to it will not cause it to grow to larger than nNodeSize + ** bytes in size, write the key here. */ + + Blob *pBlk = &pNode->block; + if( pBlk->n==0 ){ + blobGrowBuffer(pBlk, p->nNodeSize, &rc); + if( rc==SQLITE_OK ){ + pBlk->a[0] = (char)iLayer; + pBlk->n = 1 + sqlite3Fts3PutVarint(&pBlk->a[1], iPtr); + } + } + blobGrowBuffer(pBlk, pBlk->n + nSpace, &rc); + blobGrowBuffer(&pNode->key, nTerm, &rc); + + if( rc==SQLITE_OK ){ + if( pNode->key.n ){ + pBlk->n += sqlite3Fts3PutVarint(&pBlk->a[pBlk->n], nPrefix); + } + pBlk->n += sqlite3Fts3PutVarint(&pBlk->a[pBlk->n], nSuffix); + memcpy(&pBlk->a[pBlk->n], &zTerm[nPrefix], nSuffix); + pBlk->n += nSuffix; + + memcpy(pNode->key.a, zTerm, nTerm); + pNode->key.n = nTerm; + } }else{ - *piIdx = iNext; + /* Otherwise, flush the current node of layer iLayer to disk. + ** Then allocate a new, empty sibling node. The key will be written + ** into the parent of this node. */ + rc = fts3WriteSegment(p, pNode->iBlock, pNode->block.a, pNode->block.n); + + assert( pNode->block.nAlloc>=p->nNodeSize ); + pNode->block.a[0] = (char)iLayer; + pNode->block.n = 1 + sqlite3Fts3PutVarint(&pNode->block.a[1], iPtr+1); + + iNextPtr = pNode->iBlock; + pNode->iBlock++; + pNode->key.n = 0; } + + if( rc!=SQLITE_OK || iNextPtr==0 ) return rc; + iPtr = iNextPtr; } - return rc; + assert( 0 ); + return 0; } /* -** The %_segments table is declared as follows: +** Append a term and (optionally) doclist to the FTS segment node currently +** stored in blob *pNode. The node need not contain any terms, but the +** header must be written before this function is called. ** -** CREATE TABLE %_segments(blockid INTEGER PRIMARY KEY, block BLOB) +** A node header is a single 0x00 byte for a leaf node, or a height varint +** followed by the left-hand-child varint for an internal node. ** -** This function reads data from a single row of the %_segments table. The -** specific row is identified by the iBlockid parameter. If paBlob is not -** NULL, then a buffer is allocated using sqlite3_malloc() and populated -** with the contents of the blob stored in the "block" column of the -** identified table row is. Whether or not paBlob is NULL, *pnBlob is set -** to the size of the blob in bytes before returning. +** The term to be appended is passed via arguments zTerm/nTerm. For a +** leaf node, the doclist is passed as aDoclist/nDoclist. For an internal +** node, both aDoclist and nDoclist must be passed 0. ** -** If an error occurs, or the table does not contain the specified row, -** an SQLite error code is returned. Otherwise, SQLITE_OK is returned. If -** paBlob is non-NULL, then it is the responsibility of the caller to -** eventually free the returned buffer. +** If the size of the value in blob pPrev is zero, then this is the first +** term written to the node. Otherwise, pPrev contains a copy of the +** previous term. Before this function returns, it is updated to contain a +** copy of zTerm/nTerm. ** -** This function may leave an open sqlite3_blob* handle in the -** Fts3Table.pSegments variable. This handle is reused by subsequent calls -** to this function. The handle may be closed by calling the -** sqlite3Fts3SegmentsClose() function. Reusing a blob handle is a handy -** performance improvement, but the blob handle should always be closed -** before control is returned to the user (to prevent a lock being held -** on the database file for longer than necessary). Thus, any virtual table -** method (xFilter etc.) that may directly or indirectly call this function -** must call sqlite3Fts3SegmentsClose() before returning. +** It is assumed that the buffer associated with pNode is already large +** enough to accommodate the new entry. The buffer associated with pPrev +** is extended by this function if requrired. +** +** If an error (i.e. OOM condition) occurs, an SQLite error code is +** returned. Otherwise, SQLITE_OK. */ -SQLITE_PRIVATE int sqlite3Fts3ReadBlock( - Fts3Table *p, /* FTS3 table handle */ - sqlite3_int64 iBlockid, /* Access the row with blockid=$iBlockid */ - char **paBlob, /* OUT: Blob data in malloc'd buffer */ - int *pnBlob, /* OUT: Size of blob data */ - int *pnLoad /* OUT: Bytes actually loaded */ +static int fts3AppendToNode( + Blob *pNode, /* Current node image to append to */ + Blob *pPrev, /* Buffer containing previous term written */ + const char *zTerm, /* New term to write */ + int nTerm, /* Size of zTerm in bytes */ + const char *aDoclist, /* Doclist (or NULL) to write */ + int nDoclist /* Size of aDoclist in bytes */ ){ - int rc; /* Return code */ + int rc = SQLITE_OK; /* Return code */ + int bFirst = (pPrev->n==0); /* True if this is the first term written */ + int nPrefix; /* Size of term prefix in bytes */ + int nSuffix; /* Size of term suffix in bytes */ - /* pnBlob must be non-NULL. paBlob may be NULL or non-NULL. */ - assert( pnBlob ); + /* Node must have already been started. There must be a doclist for a + ** leaf node, and there must not be a doclist for an internal node. */ + assert( pNode->n>0 ); + assert_fts3_nc( (pNode->a[0]=='\0')==(aDoclist!=0) ); - if( p->pSegments ){ - rc = sqlite3_blob_reopen(p->pSegments, iBlockid); - }else{ - if( 0==p->zSegmentsTbl ){ - p->zSegmentsTbl = sqlite3_mprintf("%s_segments", p->zName); - if( 0==p->zSegmentsTbl ) return SQLITE_NOMEM; - } - rc = sqlite3_blob_open( - p->db, p->zDb, p->zSegmentsTbl, "block", iBlockid, 0, &p->pSegments - ); + blobGrowBuffer(pPrev, nTerm, &rc); + if( rc!=SQLITE_OK ) return rc; + + nPrefix = fts3PrefixCompress(pPrev->a, pPrev->n, zTerm, nTerm); + nSuffix = nTerm - nPrefix; + if( nSuffix<=0 ) return FTS_CORRUPT_VTAB; + memcpy(pPrev->a, zTerm, nTerm); + pPrev->n = nTerm; + + if( bFirst==0 ){ + pNode->n += sqlite3Fts3PutVarint(&pNode->a[pNode->n], nPrefix); } + pNode->n += sqlite3Fts3PutVarint(&pNode->a[pNode->n], nSuffix); + memcpy(&pNode->a[pNode->n], &zTerm[nPrefix], nSuffix); + pNode->n += nSuffix; - if( rc==SQLITE_OK ){ - int nByte = sqlite3_blob_bytes(p->pSegments); - *pnBlob = nByte; - if( paBlob ){ - char *aByte = sqlite3_malloc(nByte + FTS3_NODE_PADDING); - if( !aByte ){ - rc = SQLITE_NOMEM; - }else{ - if( pnLoad && nByte>(FTS3_NODE_CHUNK_THRESHOLD) ){ - nByte = FTS3_NODE_CHUNKSIZE; - *pnLoad = nByte; - } - rc = sqlite3_blob_read(p->pSegments, aByte, nByte, 0); - memset(&aByte[nByte], 0, FTS3_NODE_PADDING); - if( rc!=SQLITE_OK ){ - sqlite3_free(aByte); - aByte = 0; - } - } - *paBlob = aByte; - } + if( aDoclist ){ + pNode->n += sqlite3Fts3PutVarint(&pNode->a[pNode->n], nDoclist); + memcpy(&pNode->a[pNode->n], aDoclist, nDoclist); + pNode->n += nDoclist; } - return rc; + assert( pNode->n<=pNode->nAlloc ); + + return SQLITE_OK; } /* -** Close the blob handle at p->pSegments, if it is open. See comments above -** the sqlite3Fts3ReadBlock() function for details. +** Append the current term and doclist pointed to by cursor pCsr to the +** appendable b-tree segment opened for writing by pWriter. +** +** Return SQLITE_OK if successful, or an SQLite error code otherwise. */ -SQLITE_PRIVATE void sqlite3Fts3SegmentsClose(Fts3Table *p){ - sqlite3_blob_close(p->pSegments); - p->pSegments = 0; -} - -static int fts3SegReaderIncrRead(Fts3SegReader *pReader){ - int nRead; /* Number of bytes to read */ - int rc; /* Return code */ +static int fts3IncrmergeAppend( + Fts3Table *p, /* Fts3 table handle */ + IncrmergeWriter *pWriter, /* Writer object */ + Fts3MultiSegReader *pCsr /* Cursor containing term and doclist */ +){ + const char *zTerm = pCsr->zTerm; + int nTerm = pCsr->nTerm; + const char *aDoclist = pCsr->aDoclist; + int nDoclist = pCsr->nDoclist; + int rc = SQLITE_OK; /* Return code */ + int nSpace; /* Total space in bytes required on leaf */ + int nPrefix; /* Size of prefix shared with previous term */ + int nSuffix; /* Size of suffix (nTerm - nPrefix) */ + NodeWriter *pLeaf; /* Object used to write leaf nodes */ - nRead = MIN(pReader->nNode - pReader->nPopulate, FTS3_NODE_CHUNKSIZE); - rc = sqlite3_blob_read( - pReader->pBlob, - &pReader->aNode[pReader->nPopulate], - nRead, - pReader->nPopulate - ); + pLeaf = &pWriter->aNodeWriter[0]; + nPrefix = fts3PrefixCompress(pLeaf->key.a, pLeaf->key.n, zTerm, nTerm); + nSuffix = nTerm - nPrefix; - if( rc==SQLITE_OK ){ - pReader->nPopulate += nRead; - memset(&pReader->aNode[pReader->nPopulate], 0, FTS3_NODE_PADDING); - if( pReader->nPopulate==pReader->nNode ){ - sqlite3_blob_close(pReader->pBlob); - pReader->pBlob = 0; - pReader->nPopulate = 0; + nSpace = sqlite3Fts3VarintLen(nPrefix); + nSpace += sqlite3Fts3VarintLen(nSuffix) + nSuffix; + nSpace += sqlite3Fts3VarintLen(nDoclist) + nDoclist; + + /* If the current block is not empty, and if adding this term/doclist + ** to the current block would make it larger than Fts3Table.nNodeSize + ** bytes, write this block out to the database. */ + if( pLeaf->block.n>0 && (pLeaf->block.n + nSpace)>p->nNodeSize ){ + rc = fts3WriteSegment(p, pLeaf->iBlock, pLeaf->block.a, pLeaf->block.n); + pWriter->nWork++; + + /* Add the current term to the parent node. The term added to the + ** parent must: + ** + ** a) be greater than the largest term on the leaf node just written + ** to the database (still available in pLeaf->key), and + ** + ** b) be less than or equal to the term about to be added to the new + ** leaf node (zTerm/nTerm). + ** + ** In other words, it must be the prefix of zTerm 1 byte longer than + ** the common prefix (if any) of zTerm and pWriter->zTerm. + */ + if( rc==SQLITE_OK ){ + rc = fts3IncrmergePush(p, pWriter, zTerm, nPrefix+1); } - } - return rc; -} -static int fts3SegReaderRequire(Fts3SegReader *pReader, char *pFrom, int nByte){ - int rc = SQLITE_OK; - assert( !pReader->pBlob - || (pFrom>=pReader->aNode && pFrom<&pReader->aNode[pReader->nNode]) - ); - while( pReader->pBlob && rc==SQLITE_OK - && (pFrom - pReader->aNode + nByte)>pReader->nPopulate - ){ - rc = fts3SegReaderIncrRead(pReader); + /* Advance to the next output block */ + pLeaf->iBlock++; + pLeaf->key.n = 0; + pLeaf->block.n = 0; + + nSuffix = nTerm; + nSpace = 1; + nSpace += sqlite3Fts3VarintLen(nSuffix) + nSuffix; + nSpace += sqlite3Fts3VarintLen(nDoclist) + nDoclist; } - return rc; -} -/* -** Set an Fts3SegReader cursor to point at EOF. -*/ -static void fts3SegReaderSetEof(Fts3SegReader *pSeg){ - if( !fts3SegReaderIsRootOnly(pSeg) ){ - sqlite3_free(pSeg->aNode); - sqlite3_blob_close(pSeg->pBlob); - pSeg->pBlob = 0; + pWriter->nLeafData += nSpace; + blobGrowBuffer(&pLeaf->block, pLeaf->block.n + nSpace, &rc); + if( rc==SQLITE_OK ){ + if( pLeaf->block.n==0 ){ + pLeaf->block.n = 1; + pLeaf->block.a[0] = '\0'; + } + rc = fts3AppendToNode( + &pLeaf->block, &pLeaf->key, zTerm, nTerm, aDoclist, nDoclist + ); } - pSeg->aNode = 0; + + return rc; } /* -** Move the iterator passed as the first argument to the next term in the -** segment. If successful, SQLITE_OK is returned. If there is no next term, -** SQLITE_DONE. Otherwise, an SQLite error code. +** This function is called to release all dynamic resources held by the +** merge-writer object pWriter, and if no error has occurred, to flush +** all outstanding node buffers held by pWriter to disk. +** +** If *pRc is not SQLITE_OK when this function is called, then no attempt +** is made to write any data to disk. Instead, this function serves only +** to release outstanding resources. +** +** Otherwise, if *pRc is initially SQLITE_OK and an error occurs while +** flushing buffers to disk, *pRc is set to an SQLite error code before +** returning. */ -static int fts3SegReaderNext( - Fts3Table *p, - Fts3SegReader *pReader, - int bIncr +static void fts3IncrmergeRelease( + Fts3Table *p, /* FTS3 table handle */ + IncrmergeWriter *pWriter, /* Merge-writer object */ + int *pRc /* IN/OUT: Error code */ ){ - int rc; /* Return code of various sub-routines */ - char *pNext; /* Cursor variable */ - int nPrefix; /* Number of bytes in term prefix */ - int nSuffix; /* Number of bytes in term suffix */ + int i; /* Used to iterate through non-root layers */ + int iRoot; /* Index of root in pWriter->aNodeWriter */ + NodeWriter *pRoot; /* NodeWriter for root node */ + int rc = *pRc; /* Error code */ - if( !pReader->aDoclist ){ - pNext = pReader->aNode; - }else{ - pNext = &pReader->aDoclist[pReader->nDoclist]; + /* Set iRoot to the index in pWriter->aNodeWriter[] of the output segment + ** root node. If the segment fits entirely on a single leaf node, iRoot + ** will be set to 0. If the root node is the parent of the leaves, iRoot + ** will be 1. And so on. */ + for(iRoot=FTS_MAX_APPENDABLE_HEIGHT-1; iRoot>=0; iRoot--){ + NodeWriter *pNode = &pWriter->aNodeWriter[iRoot]; + if( pNode->block.n>0 ) break; + assert( *pRc || pNode->block.nAlloc==0 ); + assert( *pRc || pNode->key.nAlloc==0 ); + sqlite3_free(pNode->block.a); + sqlite3_free(pNode->key.a); } - if( !pNext || pNext>=&pReader->aNode[pReader->nNode] ){ - - if( fts3SegReaderIsPending(pReader) ){ - Fts3HashElem *pElem = *(pReader->ppNextElem); - sqlite3_free(pReader->aNode); - pReader->aNode = 0; - if( pElem ){ - char *aCopy; - PendingList *pList = (PendingList *)fts3HashData(pElem); - int nCopy = pList->nData+1; - pReader->zTerm = (char *)fts3HashKey(pElem); - pReader->nTerm = fts3HashKeysize(pElem); - aCopy = (char*)sqlite3_malloc(nCopy); - if( !aCopy ) return SQLITE_NOMEM; - memcpy(aCopy, pList->aData, nCopy); - pReader->nNode = pReader->nDoclist = nCopy; - pReader->aNode = pReader->aDoclist = aCopy; - pReader->ppNextElem++; - assert( pReader->aNode ); - } - return SQLITE_OK; - } - - fts3SegReaderSetEof(pReader); + /* Empty output segment. This is a no-op. */ + if( iRoot<0 ) return; - /* If iCurrentBlock>=iLeafEndBlock, this is an EOF condition. All leaf - ** blocks have already been traversed. */ - assert( pReader->iCurrentBlock<=pReader->iLeafEndBlock ); - if( pReader->iCurrentBlock>=pReader->iLeafEndBlock ){ - return SQLITE_OK; + /* The entire output segment fits on a single node. Normally, this means + ** the node would be stored as a blob in the "root" column of the %_segdir + ** table. However, this is not permitted in this case. The problem is that + ** space has already been reserved in the %_segments table, and so the + ** start_block and end_block fields of the %_segdir table must be populated. + ** And, by design or by accident, released versions of FTS cannot handle + ** segments that fit entirely on the root node with start_block!=0. + ** + ** Instead, create a synthetic root node that contains nothing but a + ** pointer to the single content node. So that the segment consists of a + ** single leaf and a single interior (root) node. + ** + ** Todo: Better might be to defer allocating space in the %_segments + ** table until we are sure it is needed. + */ + if( iRoot==0 ){ + Blob *pBlock = &pWriter->aNodeWriter[1].block; + blobGrowBuffer(pBlock, 1 + FTS3_VARINT_MAX, &rc); + if( rc==SQLITE_OK ){ + pBlock->a[0] = 0x01; + pBlock->n = 1 + sqlite3Fts3PutVarint( + &pBlock->a[1], pWriter->aNodeWriter[0].iBlock + ); } + iRoot = 1; + } + pRoot = &pWriter->aNodeWriter[iRoot]; - rc = sqlite3Fts3ReadBlock( - p, ++pReader->iCurrentBlock, &pReader->aNode, &pReader->nNode, - (bIncr ? &pReader->nPopulate : 0) - ); - if( rc!=SQLITE_OK ) return rc; - assert( pReader->pBlob==0 ); - if( bIncr && pReader->nPopulatenNode ){ - pReader->pBlob = p->pSegments; - p->pSegments = 0; + /* Flush all currently outstanding nodes to disk. */ + for(i=0; iaNodeWriter[i]; + if( pNode->block.n>0 && rc==SQLITE_OK ){ + rc = fts3WriteSegment(p, pNode->iBlock, pNode->block.a, pNode->block.n); } - pNext = pReader->aNode; + sqlite3_free(pNode->block.a); + sqlite3_free(pNode->key.a); } - assert( !fts3SegReaderIsPending(pReader) ); - - rc = fts3SegReaderRequire(pReader, pNext, FTS3_VARINT_MAX*2); - if( rc!=SQLITE_OK ) return rc; - - /* Because of the FTS3_NODE_PADDING bytes of padding, the following is - ** safe (no risk of overread) even if the node data is corrupted. */ - pNext += fts3GetVarint32(pNext, &nPrefix); - pNext += fts3GetVarint32(pNext, &nSuffix); - if( nPrefix<0 || nSuffix<=0 - || &pNext[nSuffix]>&pReader->aNode[pReader->nNode] - ){ - return FTS_CORRUPT_VTAB; + /* Write the %_segdir record. */ + if( rc==SQLITE_OK ){ + rc = fts3WriteSegdir(p, + pWriter->iAbsLevel+1, /* level */ + pWriter->iIdx, /* idx */ + pWriter->iStart, /* start_block */ + pWriter->aNodeWriter[0].iBlock, /* leaves_end_block */ + pWriter->iEnd, /* end_block */ + (pWriter->bNoLeafData==0 ? pWriter->nLeafData : 0), /* end_block */ + pRoot->block.a, pRoot->block.n /* root */ + ); } + sqlite3_free(pRoot->block.a); + sqlite3_free(pRoot->key.a); - if( nPrefix+nSuffix>pReader->nTermAlloc ){ - int nNew = (nPrefix+nSuffix)*2; - char *zNew = sqlite3_realloc(pReader->zTerm, nNew); - if( !zNew ){ - return SQLITE_NOMEM; - } - pReader->zTerm = zNew; - pReader->nTermAlloc = nNew; - } + *pRc = rc; +} - rc = fts3SegReaderRequire(pReader, pNext, nSuffix+FTS3_VARINT_MAX); - if( rc!=SQLITE_OK ) return rc; +/* +** Compare the term in buffer zLhs (size in bytes nLhs) with that in +** zRhs (size in bytes nRhs) using memcmp. If one term is a prefix of +** the other, it is considered to be smaller than the other. +** +** Return -ve if zLhs is smaller than zRhs, 0 if it is equal, or +ve +** if it is greater. +*/ +static int fts3TermCmp( + const char *zLhs, int nLhs, /* LHS of comparison */ + const char *zRhs, int nRhs /* RHS of comparison */ +){ + int nCmp = MIN(nLhs, nRhs); + int res; - memcpy(&pReader->zTerm[nPrefix], pNext, nSuffix); - pReader->nTerm = nPrefix+nSuffix; - pNext += nSuffix; - pNext += fts3GetVarint32(pNext, &pReader->nDoclist); - pReader->aDoclist = pNext; - pReader->pOffsetList = 0; + res = (nCmp ? memcmp(zLhs, zRhs, nCmp) : 0); + if( res==0 ) res = nLhs - nRhs; - /* Check that the doclist does not appear to extend past the end of the - ** b-tree node. And that the final byte of the doclist is 0x00. If either - ** of these statements is untrue, then the data structure is corrupt. - */ - if( &pReader->aDoclist[pReader->nDoclist]>&pReader->aNode[pReader->nNode] - || (pReader->nPopulate==0 && pReader->aDoclist[pReader->nDoclist-1]) - ){ - return FTS_CORRUPT_VTAB; - } - return SQLITE_OK; + return res; } + /* -** Set the SegReader to point to the first docid in the doclist associated -** with the current term. +** Query to see if the entry in the %_segments table with blockid iEnd is +** NULL. If no error occurs and the entry is NULL, set *pbRes 1 before +** returning. Otherwise, set *pbRes to 0. +** +** Or, if an error occurs while querying the database, return an SQLite +** error code. The final value of *pbRes is undefined in this case. +** +** This is used to test if a segment is an "appendable" segment. If it +** is, then a NULL entry has been inserted into the %_segments table +** with blockid %_segdir.end_block. */ -static int fts3SegReaderFirstDocid(Fts3Table *pTab, Fts3SegReader *pReader){ - int rc = SQLITE_OK; - assert( pReader->aDoclist ); - assert( !pReader->pOffsetList ); - if( pTab->bDescIdx && fts3SegReaderIsPending(pReader) ){ - u8 bEof = 0; - pReader->iDocid = 0; - pReader->nOffsetList = 0; - sqlite3Fts3DoclistPrev(0, - pReader->aDoclist, pReader->nDoclist, &pReader->pOffsetList, - &pReader->iDocid, &pReader->nOffsetList, &bEof - ); - }else{ - rc = fts3SegReaderRequire(pReader, pReader->aDoclist, FTS3_VARINT_MAX); - if( rc==SQLITE_OK ){ - int n = sqlite3Fts3GetVarint(pReader->aDoclist, &pReader->iDocid); - pReader->pOffsetList = &pReader->aDoclist[n]; - } +static int fts3IsAppendable(Fts3Table *p, sqlite3_int64 iEnd, int *pbRes){ + int bRes = 0; /* Result to set *pbRes to */ + sqlite3_stmt *pCheck = 0; /* Statement to query database with */ + int rc; /* Return code */ + + rc = fts3SqlStmt(p, SQL_SEGMENT_IS_APPENDABLE, &pCheck, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pCheck, 1, iEnd); + if( SQLITE_ROW==sqlite3_step(pCheck) ) bRes = 1; + rc = sqlite3_reset(pCheck); } + + *pbRes = bRes; return rc; } /* -** Advance the SegReader to point to the next docid in the doclist -** associated with the current term. -** -** If arguments ppOffsetList and pnOffsetList are not NULL, then -** *ppOffsetList is set to point to the first column-offset list -** in the doclist entry (i.e. immediately past the docid varint). -** *pnOffsetList is set to the length of the set of column-offset -** lists, not including the nul-terminator byte. For example: +** This function is called when initializing an incremental-merge operation. +** It checks if the existing segment with index value iIdx at absolute level +** (iAbsLevel+1) can be appended to by the incremental merge. If it can, the +** merge-writer object *pWriter is initialized to write to it. +** +** An existing segment can be appended to by an incremental merge if: +** +** * It was initially created as an appendable segment (with all required +** space pre-allocated), and +** +** * The first key read from the input (arguments zKey and nKey) is +** greater than the largest key currently stored in the potential +** output segment. */ -static int fts3SegReaderNextDocid( - Fts3Table *pTab, - Fts3SegReader *pReader, /* Reader to advance to next docid */ - char **ppOffsetList, /* OUT: Pointer to current position-list */ - int *pnOffsetList /* OUT: Length of *ppOffsetList in bytes */ +static int fts3IncrmergeLoad( + Fts3Table *p, /* Fts3 table handle */ + sqlite3_int64 iAbsLevel, /* Absolute level of input segments */ + int iIdx, /* Index of candidate output segment */ + const char *zKey, /* First key to write */ + int nKey, /* Number of bytes in nKey */ + IncrmergeWriter *pWriter /* Populate this object */ ){ - int rc = SQLITE_OK; - char *p = pReader->pOffsetList; - char c = 0; + int rc; /* Return code */ + sqlite3_stmt *pSelect = 0; /* SELECT to read %_segdir entry */ - assert( p ); + rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR, &pSelect, 0); + if( rc==SQLITE_OK ){ + sqlite3_int64 iStart = 0; /* Value of %_segdir.start_block */ + sqlite3_int64 iLeafEnd = 0; /* Value of %_segdir.leaves_end_block */ + sqlite3_int64 iEnd = 0; /* Value of %_segdir.end_block */ + const char *aRoot = 0; /* Pointer to %_segdir.root buffer */ + int nRoot = 0; /* Size of aRoot[] in bytes */ + int rc2; /* Return code from sqlite3_reset() */ + int bAppendable = 0; /* Set to true if segment is appendable */ - if( pTab->bDescIdx && fts3SegReaderIsPending(pReader) ){ - /* A pending-terms seg-reader for an FTS4 table that uses order=desc. - ** Pending-terms doclists are always built up in ascending order, so - ** we have to iterate through them backwards here. */ - u8 bEof = 0; - if( ppOffsetList ){ - *ppOffsetList = pReader->pOffsetList; - *pnOffsetList = pReader->nOffsetList - 1; - } - sqlite3Fts3DoclistPrev(0, - pReader->aDoclist, pReader->nDoclist, &p, &pReader->iDocid, - &pReader->nOffsetList, &bEof - ); - if( bEof ){ - pReader->pOffsetList = 0; + /* Read the %_segdir entry for index iIdx absolute level (iAbsLevel+1) */ + sqlite3_bind_int64(pSelect, 1, iAbsLevel+1); + sqlite3_bind_int(pSelect, 2, iIdx); + if( sqlite3_step(pSelect)==SQLITE_ROW ){ + iStart = sqlite3_column_int64(pSelect, 1); + iLeafEnd = sqlite3_column_int64(pSelect, 2); + fts3ReadEndBlockField(pSelect, 3, &iEnd, &pWriter->nLeafData); + if( pWriter->nLeafData<0 ){ + pWriter->nLeafData = pWriter->nLeafData * -1; + } + pWriter->bNoLeafData = (pWriter->nLeafData==0); + nRoot = sqlite3_column_bytes(pSelect, 4); + aRoot = sqlite3_column_blob(pSelect, 4); + if( aRoot==0 ){ + sqlite3_reset(pSelect); + return nRoot ? SQLITE_NOMEM : FTS_CORRUPT_VTAB; + } }else{ - pReader->pOffsetList = p; + return sqlite3_reset(pSelect); } - }else{ - char *pEnd = &pReader->aDoclist[pReader->nDoclist]; - /* Pointer p currently points at the first byte of an offset list. The - ** following block advances it to point one byte past the end of - ** the same offset list. */ - while( 1 ){ - - /* The following line of code (and the "p++" below the while() loop) is - ** normally all that is required to move pointer p to the desired - ** position. The exception is if this node is being loaded from disk - ** incrementally and pointer "p" now points to the first byte past - ** the populated part of pReader->aNode[]. - */ - while( *p | c ) c = *p++ & 0x80; - assert( *p==0 ); - - if( pReader->pBlob==0 || p<&pReader->aNode[pReader->nPopulate] ) break; - rc = fts3SegReaderIncrRead(pReader); - if( rc!=SQLITE_OK ) return rc; - } - p++; - - /* If required, populate the output variables with a pointer to and the - ** size of the previous offset-list. - */ - if( ppOffsetList ){ - *ppOffsetList = pReader->pOffsetList; - *pnOffsetList = (int)(p - pReader->pOffsetList - 1); - } + /* Check for the zero-length marker in the %_segments table */ + rc = fts3IsAppendable(p, iEnd, &bAppendable); - /* List may have been edited in place by fts3EvalNearTrim() */ - while( p=pEnd ){ - pReader->pOffsetList = 0; - }else{ - rc = fts3SegReaderRequire(pReader, p, FTS3_VARINT_MAX); + /* Check that zKey/nKey is larger than the largest key the candidate */ + if( rc==SQLITE_OK && bAppendable ){ + char *aLeaf = 0; + int nLeaf = 0; + + rc = sqlite3Fts3ReadBlock(p, iLeafEnd, &aLeaf, &nLeaf, 0); if( rc==SQLITE_OK ){ - sqlite3_int64 iDelta; - pReader->pOffsetList = p + sqlite3Fts3GetVarint(p, &iDelta); - if( pTab->bDescIdx ){ - pReader->iDocid -= iDelta; - }else{ - pReader->iDocid += iDelta; + NodeReader reader; + for(rc = nodeReaderInit(&reader, aLeaf, nLeaf); + rc==SQLITE_OK && reader.aNode; + rc = nodeReaderNext(&reader) + ){ + assert( reader.aNode ); + } + if( fts3TermCmp(zKey, nKey, reader.term.a, reader.term.n)<=0 ){ + bAppendable = 0; } + nodeReaderRelease(&reader); } + sqlite3_free(aLeaf); } - } - return SQLITE_OK; -} + if( rc==SQLITE_OK && bAppendable ){ + /* It is possible to append to this segment. Set up the IncrmergeWriter + ** object to do so. */ + int i; + int nHeight = (int)aRoot[0]; + NodeWriter *pNode; + if( nHeight<1 || nHeight>FTS_MAX_APPENDABLE_HEIGHT ){ + sqlite3_reset(pSelect); + return FTS_CORRUPT_VTAB; + } + pWriter->nLeafEst = (int)((iEnd - iStart) + 1)/FTS_MAX_APPENDABLE_HEIGHT; + pWriter->iStart = iStart; + pWriter->iEnd = iEnd; + pWriter->iAbsLevel = iAbsLevel; + pWriter->iIdx = iIdx; -SQLITE_PRIVATE int sqlite3Fts3MsrOvfl( - Fts3Cursor *pCsr, - Fts3MultiSegReader *pMsr, - int *pnOvfl -){ - Fts3Table *p = (Fts3Table*)pCsr->base.pVtab; - int nOvfl = 0; - int ii; - int rc = SQLITE_OK; - int pgsz = p->nPgsz; + for(i=nHeight+1; iaNodeWriter[i].iBlock = pWriter->iStart + i*pWriter->nLeafEst; + } - assert( p->bFts4 ); - assert( pgsz>0 ); + pNode = &pWriter->aNodeWriter[nHeight]; + pNode->iBlock = pWriter->iStart + pWriter->nLeafEst*nHeight; + blobGrowBuffer(&pNode->block, + MAX(nRoot, p->nNodeSize)+FTS3_NODE_PADDING, &rc + ); + if( rc==SQLITE_OK ){ + memcpy(pNode->block.a, aRoot, nRoot); + pNode->block.n = nRoot; + memset(&pNode->block.a[nRoot], 0, FTS3_NODE_PADDING); + } - for(ii=0; rc==SQLITE_OK && iinSegment; ii++){ - Fts3SegReader *pReader = pMsr->apSegment[ii]; - if( !fts3SegReaderIsPending(pReader) - && !fts3SegReaderIsRootOnly(pReader) - ){ - sqlite3_int64 jj; - for(jj=pReader->iStartBlock; jj<=pReader->iLeafEndBlock; jj++){ - int nBlob; - rc = sqlite3Fts3ReadBlock(p, jj, 0, &nBlob, 0); - if( rc!=SQLITE_OK ) break; - if( (nBlob+35)>pgsz ){ - nOvfl += (nBlob + 34)/pgsz; + for(i=nHeight; i>=0 && rc==SQLITE_OK; i--){ + NodeReader reader; + pNode = &pWriter->aNodeWriter[i]; + + if( pNode->block.a){ + rc = nodeReaderInit(&reader, pNode->block.a, pNode->block.n); + while( reader.aNode && rc==SQLITE_OK ) rc = nodeReaderNext(&reader); + blobGrowBuffer(&pNode->key, reader.term.n, &rc); + if( rc==SQLITE_OK ){ + memcpy(pNode->key.a, reader.term.a, reader.term.n); + pNode->key.n = reader.term.n; + if( i>0 ){ + char *aBlock = 0; + int nBlock = 0; + pNode = &pWriter->aNodeWriter[i-1]; + pNode->iBlock = reader.iChild; + rc = sqlite3Fts3ReadBlock(p, reader.iChild, &aBlock, &nBlock, 0); + blobGrowBuffer(&pNode->block, + MAX(nBlock, p->nNodeSize)+FTS3_NODE_PADDING, &rc + ); + if( rc==SQLITE_OK ){ + memcpy(pNode->block.a, aBlock, nBlock); + pNode->block.n = nBlock; + memset(&pNode->block.a[nBlock], 0, FTS3_NODE_PADDING); + } + sqlite3_free(aBlock); + } + } } + nodeReaderRelease(&reader); } } + + rc2 = sqlite3_reset(pSelect); + if( rc==SQLITE_OK ) rc = rc2; } - *pnOvfl = nOvfl; + return rc; } /* -** Free all allocations associated with the iterator passed as the -** second argument. +** Determine the largest segment index value that exists within absolute +** level iAbsLevel+1. If no error occurs, set *piIdx to this value plus +** one before returning SQLITE_OK. Or, if there are no segments at all +** within level iAbsLevel, set *piIdx to zero. +** +** If an error occurs, return an SQLite error code. The final value of +** *piIdx is undefined in this case. */ -SQLITE_PRIVATE void sqlite3Fts3SegReaderFree(Fts3SegReader *pReader){ - if( pReader ){ - if( !fts3SegReaderIsPending(pReader) ){ - sqlite3_free(pReader->zTerm); - } - if( !fts3SegReaderIsRootOnly(pReader) ){ - sqlite3_free(pReader->aNode); - } - sqlite3_blob_close(pReader->pBlob); +static int fts3IncrmergeOutputIdx( + Fts3Table *p, /* FTS Table handle */ + sqlite3_int64 iAbsLevel, /* Absolute index of input segments */ + int *piIdx /* OUT: Next free index at iAbsLevel+1 */ +){ + int rc; + sqlite3_stmt *pOutputIdx = 0; /* SQL used to find output index */ + + rc = fts3SqlStmt(p, SQL_NEXT_SEGMENT_INDEX, &pOutputIdx, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pOutputIdx, 1, iAbsLevel+1); + sqlite3_step(pOutputIdx); + *piIdx = sqlite3_column_int(pOutputIdx, 0); + rc = sqlite3_reset(pOutputIdx); } - sqlite3_free(pReader); + + return rc; } -/* -** Allocate a new SegReader object. +/* +** Allocate an appendable output segment on absolute level iAbsLevel+1 +** with idx value iIdx. +** +** In the %_segdir table, a segment is defined by the values in three +** columns: +** +** start_block +** leaves_end_block +** end_block +** +** When an appendable segment is allocated, it is estimated that the +** maximum number of leaf blocks that may be required is the sum of the +** number of leaf blocks consumed by the input segments, plus the number +** of input segments, multiplied by two. This value is stored in stack +** variable nLeafEst. +** +** A total of 16*nLeafEst blocks are allocated when an appendable segment +** is created ((1 + end_block - start_block)==16*nLeafEst). The contiguous +** array of leaf nodes starts at the first block allocated. The array +** of interior nodes that are parents of the leaf nodes start at block +** (start_block + (1 + end_block - start_block) / 16). And so on. +** +** In the actual code below, the value "16" is replaced with the +** pre-processor macro FTS_MAX_APPENDABLE_HEIGHT. */ -SQLITE_PRIVATE int sqlite3Fts3SegReaderNew( - int iAge, /* Segment "age". */ - int bLookup, /* True for a lookup only */ - sqlite3_int64 iStartLeaf, /* First leaf to traverse */ - sqlite3_int64 iEndLeaf, /* Final leaf to traverse */ - sqlite3_int64 iEndBlock, /* Final block of segment */ - const char *zRoot, /* Buffer containing root node */ - int nRoot, /* Size of buffer containing root node */ - Fts3SegReader **ppReader /* OUT: Allocated Fts3SegReader */ +static int fts3IncrmergeWriter( + Fts3Table *p, /* Fts3 table handle */ + sqlite3_int64 iAbsLevel, /* Absolute level of input segments */ + int iIdx, /* Index of new output segment */ + Fts3MultiSegReader *pCsr, /* Cursor that data will be read from */ + IncrmergeWriter *pWriter /* Populate this object */ ){ - Fts3SegReader *pReader; /* Newly allocated SegReader object */ - int nExtra = 0; /* Bytes to allocate segment root node */ + int rc; /* Return Code */ + int i; /* Iterator variable */ + int nLeafEst = 0; /* Blocks allocated for leaf nodes */ + sqlite3_stmt *pLeafEst = 0; /* SQL used to determine nLeafEst */ + sqlite3_stmt *pFirstBlock = 0; /* SQL used to determine first block */ - assert( iStartLeaf<=iEndLeaf ); - if( iStartLeaf==0 ){ - nExtra = nRoot + FTS3_NODE_PADDING; + /* Calculate nLeafEst. */ + rc = fts3SqlStmt(p, SQL_MAX_LEAF_NODE_ESTIMATE, &pLeafEst, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pLeafEst, 1, iAbsLevel); + sqlite3_bind_int64(pLeafEst, 2, pCsr->nSegment); + if( SQLITE_ROW==sqlite3_step(pLeafEst) ){ + nLeafEst = sqlite3_column_int(pLeafEst, 0); + } + rc = sqlite3_reset(pLeafEst); } + if( rc!=SQLITE_OK ) return rc; - pReader = (Fts3SegReader *)sqlite3_malloc(sizeof(Fts3SegReader) + nExtra); - if( !pReader ){ - return SQLITE_NOMEM; + /* Calculate the first block to use in the output segment */ + rc = fts3SqlStmt(p, SQL_NEXT_SEGMENTS_ID, &pFirstBlock, 0); + if( rc==SQLITE_OK ){ + if( SQLITE_ROW==sqlite3_step(pFirstBlock) ){ + pWriter->iStart = sqlite3_column_int64(pFirstBlock, 0); + pWriter->iEnd = pWriter->iStart - 1; + pWriter->iEnd += nLeafEst * FTS_MAX_APPENDABLE_HEIGHT; + } + rc = sqlite3_reset(pFirstBlock); } - memset(pReader, 0, sizeof(Fts3SegReader)); - pReader->iIdx = iAge; - pReader->bLookup = bLookup!=0; - pReader->iStartBlock = iStartLeaf; - pReader->iLeafEndBlock = iEndLeaf; - pReader->iEndBlock = iEndBlock; + if( rc!=SQLITE_OK ) return rc; - if( nExtra ){ - /* The entire segment is stored in the root node. */ - pReader->aNode = (char *)&pReader[1]; - pReader->rootOnly = 1; - pReader->nNode = nRoot; - memcpy(pReader->aNode, zRoot, nRoot); - memset(&pReader->aNode[nRoot], 0, FTS3_NODE_PADDING); - }else{ - pReader->iCurrentBlock = iStartLeaf-1; + /* Insert the marker in the %_segments table to make sure nobody tries + ** to steal the space just allocated. This is also used to identify + ** appendable segments. */ + rc = fts3WriteSegment(p, pWriter->iEnd, 0, 0); + if( rc!=SQLITE_OK ) return rc; + + pWriter->iAbsLevel = iAbsLevel; + pWriter->nLeafEst = nLeafEst; + pWriter->iIdx = iIdx; + + /* Set up the array of NodeWriter objects */ + for(i=0; iaNodeWriter[i].iBlock = pWriter->iStart + i*pWriter->nLeafEst; } - *ppReader = pReader; return SQLITE_OK; } /* -** This is a comparison function used as a qsort() callback when sorting -** an array of pending terms by term. This occurs as part of flushing -** the contents of the pending-terms hash table to the database. +** Remove an entry from the %_segdir table. This involves running the +** following two statements: +** +** DELETE FROM %_segdir WHERE level = :iAbsLevel AND idx = :iIdx +** UPDATE %_segdir SET idx = idx - 1 WHERE level = :iAbsLevel AND idx > :iIdx +** +** The DELETE statement removes the specific %_segdir level. The UPDATE +** statement ensures that the remaining segments have contiguously allocated +** idx values. */ -static int SQLITE_CDECL fts3CompareElemByTerm( - const void *lhs, - const void *rhs +static int fts3RemoveSegdirEntry( + Fts3Table *p, /* FTS3 table handle */ + sqlite3_int64 iAbsLevel, /* Absolute level to delete from */ + int iIdx /* Index of %_segdir entry to delete */ ){ - char *z1 = fts3HashKey(*(Fts3HashElem **)lhs); - char *z2 = fts3HashKey(*(Fts3HashElem **)rhs); - int n1 = fts3HashKeysize(*(Fts3HashElem **)lhs); - int n2 = fts3HashKeysize(*(Fts3HashElem **)rhs); + int rc; /* Return code */ + sqlite3_stmt *pDelete = 0; /* DELETE statement */ - int n = (n1aIndex */ - const char *zTerm, /* Term to search for */ - int nTerm, /* Size of buffer zTerm */ - int bPrefix, /* True for a prefix iterator */ - Fts3SegReader **ppReader /* OUT: SegReader for pending-terms */ +static int fts3RepackSegdirLevel( + Fts3Table *p, /* FTS3 table handle */ + sqlite3_int64 iAbsLevel /* Absolute level to repack */ ){ - Fts3SegReader *pReader = 0; /* Fts3SegReader object to return */ - Fts3HashElem *pE; /* Iterator variable */ - Fts3HashElem **aElem = 0; /* Array of term hash entries to scan */ - int nElem = 0; /* Size of array at aElem */ - int rc = SQLITE_OK; /* Return Code */ - Fts3Hash *pHash; - - pHash = &p->aIndex[iIndex].hPending; - if( bPrefix ){ - int nAlloc = 0; /* Size of allocated array at aElem */ + int rc; /* Return code */ + int *aIdx = 0; /* Array of remaining idx values */ + int nIdx = 0; /* Valid entries in aIdx[] */ + int nAlloc = 0; /* Allocated size of aIdx[] */ + int i; /* Iterator variable */ + sqlite3_stmt *pSelect = 0; /* Select statement to read idx values */ + sqlite3_stmt *pUpdate = 0; /* Update statement to modify idx values */ - for(pE=fts3HashFirst(pHash); pE; pE=fts3HashNext(pE)){ - char *zKey = (char *)fts3HashKey(pE); - int nKey = fts3HashKeysize(pE); - if( nTerm==0 || (nKey>=nTerm && 0==memcmp(zKey, zTerm, nTerm)) ){ - if( nElem==nAlloc ){ - Fts3HashElem **aElem2; - nAlloc += 16; - aElem2 = (Fts3HashElem **)sqlite3_realloc( - aElem, nAlloc*sizeof(Fts3HashElem *) - ); - if( !aElem2 ){ - rc = SQLITE_NOMEM; - nElem = 0; - break; - } - aElem = aElem2; + rc = fts3SqlStmt(p, SQL_SELECT_INDEXES, &pSelect, 0); + if( rc==SQLITE_OK ){ + int rc2; + sqlite3_bind_int64(pSelect, 1, iAbsLevel); + while( SQLITE_ROW==sqlite3_step(pSelect) ){ + if( nIdx>=nAlloc ){ + int *aNew; + nAlloc += 16; + aNew = sqlite3_realloc(aIdx, nAlloc*sizeof(int)); + if( !aNew ){ + rc = SQLITE_NOMEM; + break; } - - aElem[nElem++] = pE; + aIdx = aNew; } + aIdx[nIdx++] = sqlite3_column_int(pSelect, 0); } + rc2 = sqlite3_reset(pSelect); + if( rc==SQLITE_OK ) rc = rc2; + } - /* If more than one term matches the prefix, sort the Fts3HashElem - ** objects in term order using qsort(). This uses the same comparison - ** callback as is used when flushing terms to disk. - */ - if( nElem>1 ){ - qsort(aElem, nElem, sizeof(Fts3HashElem *), fts3CompareElemByTerm); + if( rc==SQLITE_OK ){ + rc = fts3SqlStmt(p, SQL_SHIFT_SEGDIR_ENTRY, &pUpdate, 0); + } + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pUpdate, 2, iAbsLevel); + } + + assert( p->bIgnoreSavepoint==0 ); + p->bIgnoreSavepoint = 1; + for(i=0; rc==SQLITE_OK && ibIgnoreSavepoint = 0; + sqlite3_free(aIdx); + return rc; +} + +static void fts3StartNode(Blob *pNode, int iHeight, sqlite3_int64 iChild){ + pNode->a[0] = (char)iHeight; + if( iChild ){ + assert( pNode->nAlloc>=1+sqlite3Fts3VarintLen(iChild) ); + pNode->n = 1 + sqlite3Fts3PutVarint(&pNode->a[1], iChild); }else{ - /* The query is a simple term lookup that matches at most one term in - ** the index. All that is required is a straight hash-lookup. - ** - ** Because the stack address of pE may be accessed via the aElem pointer - ** below, the "Fts3HashElem *pE" must be declared so that it is valid - ** within this entire function, not just this "else{...}" block. - */ - pE = fts3HashFindElem(pHash, zTerm, nTerm); - if( pE ){ - aElem = &pE; - nElem = 1; - } + assert( pNode->nAlloc>=1 ); + pNode->n = 1; } +} - if( nElem>0 ){ - int nByte = sizeof(Fts3SegReader) + (nElem+1)*sizeof(Fts3HashElem *); - pReader = (Fts3SegReader *)sqlite3_malloc(nByte); - if( !pReader ){ - rc = SQLITE_NOMEM; - }else{ - memset(pReader, 0, nByte); - pReader->iIdx = 0x7FFFFFFF; - pReader->ppNextElem = (Fts3HashElem **)&pReader[1]; - memcpy(pReader->ppNextElem, aElem, nElem*sizeof(Fts3HashElem *)); +/* +** The first two arguments are a pointer to and the size of a segment b-tree +** node. The node may be a leaf or an internal node. +** +** This function creates a new node image in blob object *pNew by copying +** all terms that are greater than or equal to zTerm/nTerm (for leaf nodes) +** or greater than zTerm/nTerm (for internal nodes) from aNode/nNode. +*/ +static int fts3TruncateNode( + const char *aNode, /* Current node image */ + int nNode, /* Size of aNode in bytes */ + Blob *pNew, /* OUT: Write new node image here */ + const char *zTerm, /* Omit all terms smaller than this */ + int nTerm, /* Size of zTerm in bytes */ + sqlite3_int64 *piBlock /* OUT: Block number in next layer down */ +){ + NodeReader reader; /* Reader object */ + Blob prev = {0, 0, 0}; /* Previous term written to new node */ + int rc = SQLITE_OK; /* Return code */ + int bLeaf; /* True for a leaf node */ + + if( nNode<1 ) return FTS_CORRUPT_VTAB; + bLeaf = aNode[0]=='\0'; + + /* Allocate required output space */ + blobGrowBuffer(pNew, nNode, &rc); + if( rc!=SQLITE_OK ) return rc; + pNew->n = 0; + + /* Populate new node buffer */ + for(rc = nodeReaderInit(&reader, aNode, nNode); + rc==SQLITE_OK && reader.aNode; + rc = nodeReaderNext(&reader) + ){ + if( pNew->n==0 ){ + int res = fts3TermCmp(reader.term.a, reader.term.n, zTerm, nTerm); + if( res<0 || (bLeaf==0 && res==0) ) continue; + fts3StartNode(pNew, (int)aNode[0], reader.iChild); + *piBlock = reader.iChild; } + rc = fts3AppendToNode( + pNew, &prev, reader.term.a, reader.term.n, + reader.aDoclist, reader.nDoclist + ); + if( rc!=SQLITE_OK ) break; } - - if( bPrefix ){ - sqlite3_free(aElem); + if( pNew->n==0 ){ + fts3StartNode(pNew, (int)aNode[0], reader.iChild); + *piBlock = reader.iChild; } - *ppReader = pReader; + assert( pNew->n<=pNew->nAlloc ); + + nodeReaderRelease(&reader); + sqlite3_free(prev.a); return rc; } /* -** Compare the entries pointed to by two Fts3SegReader structures. -** Comparison is as follows: -** -** 1) EOF is greater than not EOF. -** -** 2) The current terms (if any) are compared using memcmp(). If one -** term is a prefix of another, the longer term is considered the -** larger. +** Remove all terms smaller than zTerm/nTerm from segment iIdx in absolute +** level iAbsLevel. This may involve deleting entries from the %_segments +** table, and modifying existing entries in both the %_segments and %_segdir +** tables. ** -** 3) By segment age. An older segment is considered larger. +** SQLITE_OK is returned if the segment is updated successfully. Or an +** SQLite error code otherwise. */ -static int fts3SegReaderCmp(Fts3SegReader *pLhs, Fts3SegReader *pRhs){ - int rc; - if( pLhs->aNode && pRhs->aNode ){ - int rc2 = pLhs->nTerm - pRhs->nTerm; - if( rc2<0 ){ - rc = memcmp(pLhs->zTerm, pRhs->zTerm, pLhs->nTerm); - }else{ - rc = memcmp(pLhs->zTerm, pRhs->zTerm, pRhs->nTerm); - } - if( rc==0 ){ - rc = rc2; +static int fts3TruncateSegment( + Fts3Table *p, /* FTS3 table handle */ + sqlite3_int64 iAbsLevel, /* Absolute level of segment to modify */ + int iIdx, /* Index within level of segment to modify */ + const char *zTerm, /* Remove terms smaller than this */ + int nTerm /* Number of bytes in buffer zTerm */ +){ + int rc = SQLITE_OK; /* Return code */ + Blob root = {0,0,0}; /* New root page image */ + Blob block = {0,0,0}; /* Buffer used for any other block */ + sqlite3_int64 iBlock = 0; /* Block id */ + sqlite3_int64 iNewStart = 0; /* New value for iStartBlock */ + sqlite3_int64 iOldStart = 0; /* Old value for iStartBlock */ + sqlite3_stmt *pFetch = 0; /* Statement used to fetch segdir */ + + rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR, &pFetch, 0); + if( rc==SQLITE_OK ){ + int rc2; /* sqlite3_reset() return code */ + sqlite3_bind_int64(pFetch, 1, iAbsLevel); + sqlite3_bind_int(pFetch, 2, iIdx); + if( SQLITE_ROW==sqlite3_step(pFetch) ){ + const char *aRoot = sqlite3_column_blob(pFetch, 4); + int nRoot = sqlite3_column_bytes(pFetch, 4); + iOldStart = sqlite3_column_int64(pFetch, 1); + rc = fts3TruncateNode(aRoot, nRoot, &root, zTerm, nTerm, &iBlock); } - }else{ - rc = (pLhs->aNode==0) - (pRhs->aNode==0); + rc2 = sqlite3_reset(pFetch); + if( rc==SQLITE_OK ) rc = rc2; } - if( rc==0 ){ - rc = pRhs->iIdx - pLhs->iIdx; + + while( rc==SQLITE_OK && iBlock ){ + char *aBlock = 0; + int nBlock = 0; + iNewStart = iBlock; + + rc = sqlite3Fts3ReadBlock(p, iBlock, &aBlock, &nBlock, 0); + if( rc==SQLITE_OK ){ + rc = fts3TruncateNode(aBlock, nBlock, &block, zTerm, nTerm, &iBlock); + } + if( rc==SQLITE_OK ){ + rc = fts3WriteSegment(p, iNewStart, block.a, block.n); + } + sqlite3_free(aBlock); } - assert( rc!=0 ); - return rc; -} -/* -** A different comparison function for SegReader structures. In this -** version, it is assumed that each SegReader points to an entry in -** a doclist for identical terms. Comparison is made as follows: -** -** 1) EOF (end of doclist in this case) is greater than not EOF. -** -** 2) By current docid. -** -** 3) By segment age. An older segment is considered larger. -*/ -static int fts3SegReaderDoclistCmp(Fts3SegReader *pLhs, Fts3SegReader *pRhs){ - int rc = (pLhs->pOffsetList==0)-(pRhs->pOffsetList==0); - if( rc==0 ){ - if( pLhs->iDocid==pRhs->iDocid ){ - rc = pRhs->iIdx - pLhs->iIdx; - }else{ - rc = (pLhs->iDocid > pRhs->iDocid) ? 1 : -1; + /* Variable iNewStart now contains the first valid leaf node. */ + if( rc==SQLITE_OK && iNewStart ){ + sqlite3_stmt *pDel = 0; + rc = fts3SqlStmt(p, SQL_DELETE_SEGMENTS_RANGE, &pDel, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pDel, 1, iOldStart); + sqlite3_bind_int64(pDel, 2, iNewStart-1); + sqlite3_step(pDel); + rc = sqlite3_reset(pDel); } } - assert( pLhs->aNode && pRhs->aNode ); - return rc; -} -static int fts3SegReaderDoclistCmpRev(Fts3SegReader *pLhs, Fts3SegReader *pRhs){ - int rc = (pLhs->pOffsetList==0)-(pRhs->pOffsetList==0); - if( rc==0 ){ - if( pLhs->iDocid==pRhs->iDocid ){ - rc = pRhs->iIdx - pLhs->iIdx; - }else{ - rc = (pLhs->iDocid < pRhs->iDocid) ? 1 : -1; + + if( rc==SQLITE_OK ){ + sqlite3_stmt *pChomp = 0; + rc = fts3SqlStmt(p, SQL_CHOMP_SEGDIR, &pChomp, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pChomp, 1, iNewStart); + sqlite3_bind_blob(pChomp, 2, root.a, root.n, SQLITE_STATIC); + sqlite3_bind_int64(pChomp, 3, iAbsLevel); + sqlite3_bind_int(pChomp, 4, iIdx); + sqlite3_step(pChomp); + rc = sqlite3_reset(pChomp); + sqlite3_bind_null(pChomp, 2); } } - assert( pLhs->aNode && pRhs->aNode ); + + sqlite3_free(root.a); + sqlite3_free(block.a); return rc; } /* -** Compare the term that the Fts3SegReader object passed as the first argument -** points to with the term specified by arguments zTerm and nTerm. +** This function is called after an incrmental-merge operation has run to +** merge (or partially merge) two or more segments from absolute level +** iAbsLevel. ** -** If the pSeg iterator is already at EOF, return 0. Otherwise, return -** -ve if the pSeg term is less than zTerm/nTerm, 0 if the two terms are -** equal, or +ve if the pSeg term is greater than zTerm/nTerm. +** Each input segment is either removed from the db completely (if all of +** its data was copied to the output segment by the incrmerge operation) +** or modified in place so that it no longer contains those entries that +** have been duplicated in the output segment. */ -static int fts3SegReaderTermCmp( - Fts3SegReader *pSeg, /* Segment reader object */ - const char *zTerm, /* Term to compare to */ - int nTerm /* Size of term zTerm in bytes */ +static int fts3IncrmergeChomp( + Fts3Table *p, /* FTS table handle */ + sqlite3_int64 iAbsLevel, /* Absolute level containing segments */ + Fts3MultiSegReader *pCsr, /* Chomp all segments opened by this cursor */ + int *pnRem /* Number of segments not deleted */ ){ - int res = 0; - if( pSeg->aNode ){ - if( pSeg->nTerm>nTerm ){ - res = memcmp(pSeg->zTerm, zTerm, nTerm); - }else{ - res = memcmp(pSeg->zTerm, zTerm, pSeg->nTerm); - } - if( res==0 ){ - res = pSeg->nTerm-nTerm; - } - } - return res; -} + int i; + int nRem = 0; + int rc = SQLITE_OK; -/* -** Argument apSegment is an array of nSegment elements. It is known that -** the final (nSegment-nSuspect) members are already in sorted order -** (according to the comparison function provided). This function shuffles -** the array around until all entries are in sorted order. -*/ -static void fts3SegReaderSort( - Fts3SegReader **apSegment, /* Array to sort entries of */ - int nSegment, /* Size of apSegment array */ - int nSuspect, /* Unsorted entry count */ - int (*xCmp)(Fts3SegReader *, Fts3SegReader *) /* Comparison function */ -){ - int i; /* Iterator variable */ + for(i=pCsr->nSegment-1; i>=0 && rc==SQLITE_OK; i--){ + Fts3SegReader *pSeg = 0; + int j; - assert( nSuspect<=nSegment ); + /* Find the Fts3SegReader object with Fts3SegReader.iIdx==i. It is hiding + ** somewhere in the pCsr->apSegment[] array. */ + for(j=0; ALWAYS(jnSegment); j++){ + pSeg = pCsr->apSegment[j]; + if( pSeg->iIdx==i ) break; + } + assert( jnSegment && pSeg->iIdx==i ); - if( nSuspect==nSegment ) nSuspect--; - for(i=nSuspect-1; i>=0; i--){ - int j; - for(j=i; j<(nSegment-1); j++){ - Fts3SegReader *pTmp; - if( xCmp(apSegment[j], apSegment[j+1])<0 ) break; - pTmp = apSegment[j+1]; - apSegment[j+1] = apSegment[j]; - apSegment[j] = pTmp; + if( pSeg->aNode==0 ){ + /* Seg-reader is at EOF. Remove the entire input segment. */ + rc = fts3DeleteSegment(p, pSeg); + if( rc==SQLITE_OK ){ + rc = fts3RemoveSegdirEntry(p, iAbsLevel, pSeg->iIdx); + } + *pnRem = 0; + }else{ + /* The incremental merge did not copy all the data from this + ** segment to the upper level. The segment is modified in place + ** so that it contains no keys smaller than zTerm/nTerm. */ + const char *zTerm = pSeg->zTerm; + int nTerm = pSeg->nTerm; + rc = fts3TruncateSegment(p, iAbsLevel, pSeg->iIdx, zTerm, nTerm); + nRem++; } } -#ifndef NDEBUG - /* Check that the list really is sorted now. */ - for(i=0; i<(nSuspect-1); i++){ - assert( xCmp(apSegment[i], apSegment[i+1])<0 ); + if( rc==SQLITE_OK && nRem!=pCsr->nSegment ){ + rc = fts3RepackSegdirLevel(p, iAbsLevel); } -#endif + + *pnRem = nRem; + return rc; } -/* -** Insert a record into the %_segments table. +/* +** Store an incr-merge hint in the database. */ -static int fts3WriteSegment( - Fts3Table *p, /* Virtual table handle */ - sqlite3_int64 iBlock, /* Block id for new block */ - char *z, /* Pointer to buffer containing block data */ - int n /* Size of buffer z in bytes */ -){ - sqlite3_stmt *pStmt; - int rc = fts3SqlStmt(p, SQL_INSERT_SEGMENTS, &pStmt, 0); +static int fts3IncrmergeHintStore(Fts3Table *p, Blob *pHint){ + sqlite3_stmt *pReplace = 0; + int rc; /* Return code */ + + rc = fts3SqlStmt(p, SQL_REPLACE_STAT, &pReplace, 0); if( rc==SQLITE_OK ){ - sqlite3_bind_int64(pStmt, 1, iBlock); - sqlite3_bind_blob(pStmt, 2, z, n, SQLITE_STATIC); - sqlite3_step(pStmt); - rc = sqlite3_reset(pStmt); - sqlite3_bind_null(pStmt, 2); + sqlite3_bind_int(pReplace, 1, FTS_STAT_INCRMERGEHINT); + sqlite3_bind_blob(pReplace, 2, pHint->a, pHint->n, SQLITE_STATIC); + sqlite3_step(pReplace); + rc = sqlite3_reset(pReplace); + sqlite3_bind_null(pReplace, 2); } + return rc; } /* -** Find the largest relative level number in the table. If successful, set -** *pnMax to this value and return SQLITE_OK. Otherwise, if an error occurs, -** set *pnMax to zero and return an SQLite error code. +** Load an incr-merge hint from the database. The incr-merge hint, if one +** exists, is stored in the rowid==1 row of the %_stat table. +** +** If successful, populate blob *pHint with the value read from the %_stat +** table and return SQLITE_OK. Otherwise, if an error occurs, return an +** SQLite error code. */ -SQLITE_PRIVATE int sqlite3Fts3MaxLevel(Fts3Table *p, int *pnMax){ +static int fts3IncrmergeHintLoad(Fts3Table *p, Blob *pHint){ + sqlite3_stmt *pSelect = 0; int rc; - int mxLevel = 0; - sqlite3_stmt *pStmt = 0; - rc = fts3SqlStmt(p, SQL_SELECT_MXLEVEL, &pStmt, 0); + pHint->n = 0; + rc = fts3SqlStmt(p, SQL_SELECT_STAT, &pSelect, 0); if( rc==SQLITE_OK ){ - if( SQLITE_ROW==sqlite3_step(pStmt) ){ - mxLevel = sqlite3_column_int(pStmt, 0); + int rc2; + sqlite3_bind_int(pSelect, 1, FTS_STAT_INCRMERGEHINT); + if( SQLITE_ROW==sqlite3_step(pSelect) ){ + const char *aHint = sqlite3_column_blob(pSelect, 0); + int nHint = sqlite3_column_bytes(pSelect, 0); + if( aHint ){ + blobGrowBuffer(pHint, nHint, &rc); + if( rc==SQLITE_OK ){ + memcpy(pHint->a, aHint, nHint); + pHint->n = nHint; + } + } } - rc = sqlite3_reset(pStmt); + rc2 = sqlite3_reset(pSelect); + if( rc==SQLITE_OK ) rc = rc2; } - *pnMax = mxLevel; + return rc; } -/* -** Insert a record into the %_segdir table. +/* +** If *pRc is not SQLITE_OK when this function is called, it is a no-op. +** Otherwise, append an entry to the hint stored in blob *pHint. Each entry +** consists of two varints, the absolute level number of the input segments +** and the number of input segments. +** +** If successful, leave *pRc set to SQLITE_OK and return. If an error occurs, +** set *pRc to an SQLite error code before returning. */ -static int fts3WriteSegdir( - Fts3Table *p, /* Virtual table handle */ - sqlite3_int64 iLevel, /* Value for "level" field (absolute level) */ - int iIdx, /* Value for "idx" field */ - sqlite3_int64 iStartBlock, /* Value for "start_block" field */ - sqlite3_int64 iLeafEndBlock, /* Value for "leaves_end_block" field */ - sqlite3_int64 iEndBlock, /* Value for "end_block" field */ - sqlite3_int64 nLeafData, /* Bytes of leaf data in segment */ - char *zRoot, /* Blob value for "root" field */ - int nRoot /* Number of bytes in buffer zRoot */ +static void fts3IncrmergeHintPush( + Blob *pHint, /* Hint blob to append to */ + i64 iAbsLevel, /* First varint to store in hint */ + int nInput, /* Second varint to store in hint */ + int *pRc /* IN/OUT: Error code */ ){ - sqlite3_stmt *pStmt; - int rc = fts3SqlStmt(p, SQL_INSERT_SEGDIR, &pStmt, 0); - if( rc==SQLITE_OK ){ - sqlite3_bind_int64(pStmt, 1, iLevel); - sqlite3_bind_int(pStmt, 2, iIdx); - sqlite3_bind_int64(pStmt, 3, iStartBlock); - sqlite3_bind_int64(pStmt, 4, iLeafEndBlock); - if( nLeafData==0 ){ - sqlite3_bind_int64(pStmt, 5, iEndBlock); - }else{ - char *zEnd = sqlite3_mprintf("%lld %lld", iEndBlock, nLeafData); - if( !zEnd ) return SQLITE_NOMEM; - sqlite3_bind_text(pStmt, 5, zEnd, -1, sqlite3_free); - } - sqlite3_bind_blob(pStmt, 6, zRoot, nRoot, SQLITE_STATIC); - sqlite3_step(pStmt); - rc = sqlite3_reset(pStmt); - sqlite3_bind_null(pStmt, 6); + blobGrowBuffer(pHint, pHint->n + 2*FTS3_VARINT_MAX, pRc); + if( *pRc==SQLITE_OK ){ + pHint->n += sqlite3Fts3PutVarint(&pHint->a[pHint->n], iAbsLevel); + pHint->n += sqlite3Fts3PutVarint(&pHint->a[pHint->n], (i64)nInput); } - return rc; } /* -** Return the size of the common prefix (if any) shared by zPrev and -** zNext, in bytes. For example, +** Read the last entry (most recently pushed) from the hint blob *pHint +** and then remove the entry. Write the two values read to *piAbsLevel and +** *pnInput before returning. ** -** fts3PrefixCompress("abc", 3, "abcdef", 6) // returns 3 -** fts3PrefixCompress("abX", 3, "abcdef", 6) // returns 2 -** fts3PrefixCompress("abX", 3, "Xbcdef", 6) // returns 0 +** If no error occurs, return SQLITE_OK. If the hint blob in *pHint does +** not contain at least two valid varints, return SQLITE_CORRUPT_VTAB. */ -static int fts3PrefixCompress( - const char *zPrev, /* Buffer containing previous term */ - int nPrev, /* Size of buffer zPrev in bytes */ - const char *zNext, /* Buffer containing next term */ - int nNext /* Size of buffer zNext in bytes */ -){ - int n; - UNUSED_PARAMETER(nNext); - for(n=0; nn; + int i; + + i = pHint->n-1; + if( (pHint->a[i] & 0x80) ) return FTS_CORRUPT_VTAB; + while( i>0 && (pHint->a[i-1] & 0x80) ) i--; + if( i==0 ) return FTS_CORRUPT_VTAB; + i--; + while( i>0 && (pHint->a[i-1] & 0x80) ) i--; + + pHint->n = i; + i += sqlite3Fts3GetVarint(&pHint->a[i], piAbsLevel); + i += fts3GetVarint32(&pHint->a[i], pnInput); + assert( i<=nHint ); + if( i!=nHint ) return FTS_CORRUPT_VTAB; + + return SQLITE_OK; } + /* -** Add term zTerm to the SegmentNode. It is guaranteed that zTerm is larger -** (according to memcmp) than the previous term. +** Attempt an incremental merge that writes nMerge leaf blocks. +** +** Incremental merges happen nMin segments at a time. The segments +** to be merged are the nMin oldest segments (the ones with the smallest +** values for the _segdir.idx field) in the highest level that contains +** at least nMin segments. Multiple merges might occur in an attempt to +** write the quota of nMerge leaf blocks. */ -static int fts3NodeAddTerm( - Fts3Table *p, /* Virtual table handle */ - SegmentNode **ppTree, /* IN/OUT: SegmentNode handle */ - int isCopyTerm, /* True if zTerm/nTerm is transient */ - const char *zTerm, /* Pointer to buffer containing term */ - int nTerm /* Size of term in bytes */ -){ - SegmentNode *pTree = *ppTree; - int rc; - SegmentNode *pNew; +SQLITE_PRIVATE int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ + int rc; /* Return code */ + int nRem = nMerge; /* Number of leaf pages yet to be written */ + Fts3MultiSegReader *pCsr; /* Cursor used to read input data */ + Fts3SegFilter *pFilter; /* Filter used with cursor pCsr */ + IncrmergeWriter *pWriter; /* Writer object */ + int nSeg = 0; /* Number of input segments */ + sqlite3_int64 iAbsLevel = 0; /* Absolute level number to work on */ + Blob hint = {0, 0, 0}; /* Hint read from %_stat table */ + int bDirtyHint = 0; /* True if blob 'hint' has been modified */ - /* First try to append the term to the current node. Return early if - ** this is possible. - */ - if( pTree ){ - int nData = pTree->nData; /* Current size of node in bytes */ - int nReq = nData; /* Required space after adding zTerm */ - int nPrefix; /* Number of bytes of prefix compression */ - int nSuffix; /* Suffix length */ + /* Allocate space for the cursor, filter and writer objects */ + const int nAlloc = sizeof(*pCsr) + sizeof(*pFilter) + sizeof(*pWriter); + pWriter = (IncrmergeWriter *)sqlite3_malloc(nAlloc); + if( !pWriter ) return SQLITE_NOMEM; + pFilter = (Fts3SegFilter *)&pWriter[1]; + pCsr = (Fts3MultiSegReader *)&pFilter[1]; - nPrefix = fts3PrefixCompress(pTree->zTerm, pTree->nTerm, zTerm, nTerm); - nSuffix = nTerm-nPrefix; + rc = fts3IncrmergeHintLoad(p, &hint); + while( rc==SQLITE_OK && nRem>0 ){ + const i64 nMod = FTS3_SEGDIR_MAXLEVEL * p->nIndex; + sqlite3_stmt *pFindLevel = 0; /* SQL used to determine iAbsLevel */ + int bUseHint = 0; /* True if attempting to append */ + int iIdx = 0; /* Largest idx in level (iAbsLevel+1) */ - nReq += sqlite3Fts3VarintLen(nPrefix)+sqlite3Fts3VarintLen(nSuffix)+nSuffix; - if( nReq<=p->nNodeSize || !pTree->zTerm ){ + /* Search the %_segdir table for the absolute level with the smallest + ** relative level number that contains at least nMin segments, if any. + ** If one is found, set iAbsLevel to the absolute level number and + ** nSeg to nMin. If no level with at least nMin segments can be found, + ** set nSeg to -1. + */ + rc = fts3SqlStmt(p, SQL_FIND_MERGE_LEVEL, &pFindLevel, 0); + sqlite3_bind_int(pFindLevel, 1, MAX(2, nMin)); + if( sqlite3_step(pFindLevel)==SQLITE_ROW ){ + iAbsLevel = sqlite3_column_int64(pFindLevel, 0); + nSeg = sqlite3_column_int(pFindLevel, 1); + assert( nSeg>=2 ); + }else{ + nSeg = -1; + } + rc = sqlite3_reset(pFindLevel); - if( nReq>p->nNodeSize ){ - /* An unusual case: this is the first term to be added to the node - ** and the static node buffer (p->nNodeSize bytes) is not large - ** enough. Use a separately malloced buffer instead This wastes - ** p->nNodeSize bytes, but since this scenario only comes about when - ** the database contain two terms that share a prefix of almost 2KB, - ** this is not expected to be a serious problem. - */ - assert( pTree->aData==(char *)&pTree[1] ); - pTree->aData = (char *)sqlite3_malloc(nReq); - if( !pTree->aData ){ - return SQLITE_NOMEM; + /* If the hint read from the %_stat table is not empty, check if the + ** last entry in it specifies a relative level smaller than or equal + ** to the level identified by the block above (if any). If so, this + ** iteration of the loop will work on merging at the hinted level. + */ + if( rc==SQLITE_OK && hint.n ){ + int nHint = hint.n; + sqlite3_int64 iHintAbsLevel = 0; /* Hint level */ + int nHintSeg = 0; /* Hint number of segments */ + + rc = fts3IncrmergeHintPop(&hint, &iHintAbsLevel, &nHintSeg); + if( nSeg<0 || (iAbsLevel % nMod) >= (iHintAbsLevel % nMod) ){ + /* Based on the scan in the block above, it is known that there + ** are no levels with a relative level smaller than that of + ** iAbsLevel with more than nSeg segments, or if nSeg is -1, + ** no levels with more than nMin segments. Use this to limit the + ** value of nHintSeg to avoid a large memory allocation in case the + ** merge-hint is corrupt*/ + iAbsLevel = iHintAbsLevel; + nSeg = MIN(MAX(nMin,nSeg), nHintSeg); + bUseHint = 1; + bDirtyHint = 1; + }else{ + /* This undoes the effect of the HintPop() above - so that no entry + ** is removed from the hint blob. */ + hint.n = nHint; + } + } + + /* If nSeg is less that zero, then there is no level with at least + ** nMin segments and no hint in the %_stat table. No work to do. + ** Exit early in this case. */ + if( nSeg<=0 ) break; + + assert( nMod<=0x7FFFFFFF ); + if( iAbsLevel<0 || iAbsLevel>(nMod<<32) ){ + rc = FTS_CORRUPT_VTAB; + break; + } + + /* Open a cursor to iterate through the contents of the oldest nSeg + ** indexes of absolute level iAbsLevel. If this cursor is opened using + ** the 'hint' parameters, it is possible that there are less than nSeg + ** segments available in level iAbsLevel. In this case, no work is + ** done on iAbsLevel - fall through to the next iteration of the loop + ** to start work on some other level. */ + memset(pWriter, 0, nAlloc); + pFilter->flags = FTS3_SEGMENT_REQUIRE_POS; + + if( rc==SQLITE_OK ){ + rc = fts3IncrmergeOutputIdx(p, iAbsLevel, &iIdx); + assert( bUseHint==1 || bUseHint==0 ); + if( iIdx==0 || (bUseHint && iIdx==1) ){ + int bIgnore = 0; + rc = fts3SegmentIsMaxLevel(p, iAbsLevel+1, &bIgnore); + if( bIgnore ){ + pFilter->flags |= FTS3_SEGMENT_IGNORE_EMPTY; } } + } - if( pTree->zTerm ){ - /* There is no prefix-length field for first term in a node */ - nData += sqlite3Fts3PutVarint(&pTree->aData[nData], nPrefix); + if( rc==SQLITE_OK ){ + rc = fts3IncrmergeCsr(p, iAbsLevel, nSeg, pCsr); + } + if( SQLITE_OK==rc && pCsr->nSegment==nSeg + && SQLITE_OK==(rc = sqlite3Fts3SegReaderStart(p, pCsr, pFilter)) + ){ + int bEmpty = 0; + rc = sqlite3Fts3SegReaderStep(p, pCsr); + if( rc==SQLITE_OK ){ + bEmpty = 1; + }else if( rc!=SQLITE_ROW ){ + sqlite3Fts3SegReaderFinish(pCsr); + break; + } + if( bUseHint && iIdx>0 ){ + const char *zKey = pCsr->zTerm; + int nKey = pCsr->nTerm; + rc = fts3IncrmergeLoad(p, iAbsLevel, iIdx-1, zKey, nKey, pWriter); + }else{ + rc = fts3IncrmergeWriter(p, iAbsLevel, iIdx, pCsr, pWriter); } - nData += sqlite3Fts3PutVarint(&pTree->aData[nData], nSuffix); - memcpy(&pTree->aData[nData], &zTerm[nPrefix], nSuffix); - pTree->nData = nData + nSuffix; - pTree->nEntry++; + if( rc==SQLITE_OK && pWriter->nLeafEst ){ + fts3LogMerge(nSeg, iAbsLevel); + if( bEmpty==0 ){ + do { + rc = fts3IncrmergeAppend(p, pWriter, pCsr); + if( rc==SQLITE_OK ) rc = sqlite3Fts3SegReaderStep(p, pCsr); + if( pWriter->nWork>=nRem && rc==SQLITE_ROW ) rc = SQLITE_OK; + }while( rc==SQLITE_ROW ); + } - if( isCopyTerm ){ - if( pTree->nMalloczMalloc, nTerm*2); - if( !zNew ){ - return SQLITE_NOMEM; + /* Update or delete the input segments */ + if( rc==SQLITE_OK ){ + nRem -= (1 + pWriter->nWork); + rc = fts3IncrmergeChomp(p, iAbsLevel, pCsr, &nSeg); + if( nSeg!=0 ){ + bDirtyHint = 1; + fts3IncrmergeHintPush(&hint, iAbsLevel, nSeg, &rc); } - pTree->nMalloc = nTerm*2; - pTree->zMalloc = zNew; } - pTree->zTerm = pTree->zMalloc; - memcpy(pTree->zTerm, zTerm, nTerm); - pTree->nTerm = nTerm; - }else{ - pTree->zTerm = (char *)zTerm; - pTree->nTerm = nTerm; } - return SQLITE_OK; + + if( nSeg!=0 ){ + pWriter->nLeafData = pWriter->nLeafData * -1; + } + fts3IncrmergeRelease(p, pWriter, &rc); + if( nSeg==0 && pWriter->bNoLeafData==0 ){ + fts3PromoteSegments(p, iAbsLevel+1, pWriter->nLeafData); + } } - } - /* If control flows to here, it was not possible to append zTerm to the - ** current node. Create a new node (a right-sibling of the current node). - ** If this is the first node in the tree, the term is added to it. - ** - ** Otherwise, the term is not added to the new node, it is left empty for - ** now. Instead, the term is inserted into the parent of pTree. If pTree - ** has no parent, one is created here. - */ - pNew = (SegmentNode *)sqlite3_malloc(sizeof(SegmentNode) + p->nNodeSize); - if( !pNew ){ - return SQLITE_NOMEM; + sqlite3Fts3SegReaderFinish(pCsr); } - memset(pNew, 0, sizeof(SegmentNode)); - pNew->nData = 1 + FTS3_VARINT_MAX; - pNew->aData = (char *)&pNew[1]; - if( pTree ){ - SegmentNode *pParent = pTree->pParent; - rc = fts3NodeAddTerm(p, &pParent, isCopyTerm, zTerm, nTerm); - if( pTree->pParent==0 ){ - pTree->pParent = pParent; - } - pTree->pRight = pNew; - pNew->pLeftmost = pTree->pLeftmost; - pNew->pParent = pParent; - pNew->zMalloc = pTree->zMalloc; - pNew->nMalloc = pTree->nMalloc; - pTree->zMalloc = 0; - }else{ - pNew->pLeftmost = pNew; - rc = fts3NodeAddTerm(p, &pNew, isCopyTerm, zTerm, nTerm); + /* Write the hint values into the %_stat table for the next incr-merger */ + if( bDirtyHint && rc==SQLITE_OK ){ + rc = fts3IncrmergeHintStore(p, &hint); } - *ppTree = pNew; + sqlite3_free(pWriter); + sqlite3_free(hint.a); return rc; } /* -** Helper function for fts3NodeWrite(). +** Convert the text beginning at *pz into an integer and return +** its value. Advance *pz to point to the first character past +** the integer. +** +** This function used for parameters to merge= and incrmerge= +** commands. */ -static int fts3TreeFinishNode( - SegmentNode *pTree, - int iHeight, - sqlite3_int64 iLeftChild -){ - int nStart; - assert( iHeight>=1 && iHeight<128 ); - nStart = FTS3_VARINT_MAX - sqlite3Fts3VarintLen(iLeftChild); - pTree->aData[nStart] = (char)iHeight; - sqlite3Fts3PutVarint(&pTree->aData[nStart+1], iLeftChild); - return nStart; +static int fts3Getint(const char **pz){ + const char *z = *pz; + int i = 0; + while( (*z)>='0' && (*z)<='9' && i<214748363 ) i = 10*i + *(z++) - '0'; + *pz = z; + return i; } /* -** Write the buffer for the segment node pTree and all of its peers to the -** database. Then call this function recursively to write the parent of -** pTree and its peers to the database. +** Process statements of the form: ** -** Except, if pTree is a root node, do not write it to the database. Instead, -** set output variables *paRoot and *pnRoot to contain the root node. +** INSERT INTO table(table) VALUES('merge=A,B'); ** -** If successful, SQLITE_OK is returned and output variable *piLast is -** set to the largest blockid written to the database (or zero if no -** blocks were written to the db). Otherwise, an SQLite error code is -** returned. +** A and B are integers that decode to be the number of leaf pages +** written for the merge, and the minimum number of segments on a level +** before it will be selected for a merge, respectively. */ -static int fts3NodeWrite( - Fts3Table *p, /* Virtual table handle */ - SegmentNode *pTree, /* SegmentNode handle */ - int iHeight, /* Height of this node in tree */ - sqlite3_int64 iLeaf, /* Block id of first leaf node */ - sqlite3_int64 iFree, /* Block id of next free slot in %_segments */ - sqlite3_int64 *piLast, /* OUT: Block id of last entry written */ - char **paRoot, /* OUT: Data for root node */ - int *pnRoot /* OUT: Size of root node in bytes */ +static int fts3DoIncrmerge( + Fts3Table *p, /* FTS3 table handle */ + const char *zParam /* Nul-terminated string containing "A,B" */ ){ - int rc = SQLITE_OK; + int rc; + int nMin = (MergeCount(p) / 2); + int nMerge = 0; + const char *z = zParam; - if( !pTree->pParent ){ - /* Root node of the tree. */ - int nStart = fts3TreeFinishNode(pTree, iHeight, iLeaf); - *piLast = iFree-1; - *pnRoot = pTree->nData - nStart; - *paRoot = &pTree->aData[nStart]; + /* Read the first integer value */ + nMerge = fts3Getint(&z); + + /* If the first integer value is followed by a ',', read the second + ** integer value. */ + if( z[0]==',' && z[1]!='\0' ){ + z++; + nMin = fts3Getint(&z); + } + + if( z[0]!='\0' || nMin<2 ){ + rc = SQLITE_ERROR; }else{ - SegmentNode *pIter; - sqlite3_int64 iNextFree = iFree; - sqlite3_int64 iNextLeaf = iLeaf; - for(pIter=pTree->pLeftmost; pIter && rc==SQLITE_OK; pIter=pIter->pRight){ - int nStart = fts3TreeFinishNode(pIter, iHeight, iNextLeaf); - int nWrite = pIter->nData - nStart; - - rc = fts3WriteSegment(p, iNextFree, &pIter->aData[nStart], nWrite); - iNextFree++; - iNextLeaf += (pIter->nEntry+1); + rc = SQLITE_OK; + if( !p->bHasStat ){ + assert( p->bFts4==0 ); + sqlite3Fts3CreateStatTable(&rc, p); } if( rc==SQLITE_OK ){ - assert( iNextLeaf==iFree ); - rc = fts3NodeWrite( - p, pTree->pParent, iHeight+1, iFree, iNextFree, piLast, paRoot, pnRoot - ); + rc = sqlite3Fts3Incrmerge(p, nMerge, nMin); } + sqlite3Fts3SegmentsClose(p); } - return rc; } /* -** Free all memory allocations associated with the tree pTree. +** Process statements of the form: +** +** INSERT INTO table(table) VALUES('automerge=X'); +** +** where X is an integer. X==0 means to turn automerge off. X!=0 means +** turn it on. The setting is persistent. */ -static void fts3NodeFree(SegmentNode *pTree){ - if( pTree ){ - SegmentNode *p = pTree->pLeftmost; - fts3NodeFree(p->pParent); - while( p ){ - SegmentNode *pRight = p->pRight; - if( p->aData!=(char *)&p[1] ){ - sqlite3_free(p->aData); - } - assert( pRight==0 || p->zMalloc==0 ); - sqlite3_free(p->zMalloc); - sqlite3_free(p); - p = pRight; - } +static int fts3DoAutoincrmerge( + Fts3Table *p, /* FTS3 table handle */ + const char *zParam /* Nul-terminated string containing boolean */ +){ + int rc = SQLITE_OK; + sqlite3_stmt *pStmt = 0; + p->nAutoincrmerge = fts3Getint(&zParam); + if( p->nAutoincrmerge==1 || p->nAutoincrmerge>MergeCount(p) ){ + p->nAutoincrmerge = 8; + } + if( !p->bHasStat ){ + assert( p->bFts4==0 ); + sqlite3Fts3CreateStatTable(&rc, p); + if( rc ) return rc; } + rc = fts3SqlStmt(p, SQL_REPLACE_STAT, &pStmt, 0); + if( rc ) return rc; + sqlite3_bind_int(pStmt, 1, FTS_STAT_AUTOINCRMERGE); + sqlite3_bind_int(pStmt, 2, p->nAutoincrmerge); + sqlite3_step(pStmt); + rc = sqlite3_reset(pStmt); + return rc; } /* -** Add a term to the segment being constructed by the SegmentWriter object -** *ppWriter. When adding the first term to a segment, *ppWriter should -** be passed NULL. This function will allocate a new SegmentWriter object -** and return it via the input/output variable *ppWriter in this case. -** -** If successful, SQLITE_OK is returned. Otherwise, an SQLite error code. +** Return a 64-bit checksum for the FTS index entry specified by the +** arguments to this function. */ -static int fts3SegWriterAdd( - Fts3Table *p, /* Virtual table handle */ - SegmentWriter **ppWriter, /* IN/OUT: SegmentWriter handle */ - int isCopyTerm, /* True if buffer zTerm must be copied */ +static u64 fts3ChecksumEntry( const char *zTerm, /* Pointer to buffer containing term */ - int nTerm, /* Size of term in bytes */ - const char *aDoclist, /* Pointer to buffer containing doclist */ - int nDoclist /* Size of doclist in bytes */ + int nTerm, /* Size of zTerm in bytes */ + int iLangid, /* Language id for current row */ + int iIndex, /* Index (0..Fts3Table.nIndex-1) */ + i64 iDocid, /* Docid for current row. */ + int iCol, /* Column number */ + int iPos /* Position */ ){ - int nPrefix; /* Size of term prefix in bytes */ - int nSuffix; /* Size of term suffix in bytes */ - int nReq; /* Number of bytes required on leaf page */ - int nData; - SegmentWriter *pWriter = *ppWriter; - - if( !pWriter ){ - int rc; - sqlite3_stmt *pStmt; - - /* Allocate the SegmentWriter structure */ - pWriter = (SegmentWriter *)sqlite3_malloc(sizeof(SegmentWriter)); - if( !pWriter ) return SQLITE_NOMEM; - memset(pWriter, 0, sizeof(SegmentWriter)); - *ppWriter = pWriter; - - /* Allocate a buffer in which to accumulate data */ - pWriter->aData = (char *)sqlite3_malloc(p->nNodeSize); - if( !pWriter->aData ) return SQLITE_NOMEM; - pWriter->nSize = p->nNodeSize; - - /* Find the next free blockid in the %_segments table */ - rc = fts3SqlStmt(p, SQL_NEXT_SEGMENTS_ID, &pStmt, 0); - if( rc!=SQLITE_OK ) return rc; - if( SQLITE_ROW==sqlite3_step(pStmt) ){ - pWriter->iFree = sqlite3_column_int64(pStmt, 0); - pWriter->iFirst = pWriter->iFree; - } - rc = sqlite3_reset(pStmt); - if( rc!=SQLITE_OK ) return rc; - } - nData = pWriter->nData; - - nPrefix = fts3PrefixCompress(pWriter->zTerm, pWriter->nTerm, zTerm, nTerm); - nSuffix = nTerm-nPrefix; + int i; + u64 ret = (u64)iDocid; - /* Figure out how many bytes are required by this new entry */ - nReq = sqlite3Fts3VarintLen(nPrefix) + /* varint containing prefix size */ - sqlite3Fts3VarintLen(nSuffix) + /* varint containing suffix size */ - nSuffix + /* Term suffix */ - sqlite3Fts3VarintLen(nDoclist) + /* Size of doclist */ - nDoclist; /* Doclist data */ + ret += (ret<<3) + iLangid; + ret += (ret<<3) + iIndex; + ret += (ret<<3) + iCol; + ret += (ret<<3) + iPos; + for(i=0; i0 && nData+nReq>p->nNodeSize ){ - int rc; + return ret; +} - /* The current leaf node is full. Write it out to the database. */ - rc = fts3WriteSegment(p, pWriter->iFree++, pWriter->aData, nData); - if( rc!=SQLITE_OK ) return rc; - p->nLeafAdd++; +/* +** Return a checksum of all entries in the FTS index that correspond to +** language id iLangid. The checksum is calculated by XORing the checksums +** of each individual entry (see fts3ChecksumEntry()) together. +** +** If successful, the checksum value is returned and *pRc set to SQLITE_OK. +** Otherwise, if an error occurs, *pRc is set to an SQLite error code. The +** return value is undefined in this case. +*/ +static u64 fts3ChecksumIndex( + Fts3Table *p, /* FTS3 table handle */ + int iLangid, /* Language id to return cksum for */ + int iIndex, /* Index to cksum (0..p->nIndex-1) */ + int *pRc /* OUT: Return code */ +){ + Fts3SegFilter filter; + Fts3MultiSegReader csr; + int rc; + u64 cksum = 0; - /* Add the current term to the interior node tree. The term added to - ** the interior tree must: - ** - ** a) be greater than the largest term on the leaf node just written - ** to the database (still available in pWriter->zTerm), and - ** - ** b) be less than or equal to the term about to be added to the new - ** leaf node (zTerm/nTerm). - ** - ** In other words, it must be the prefix of zTerm 1 byte longer than - ** the common prefix (if any) of zTerm and pWriter->zTerm. - */ - assert( nPrefixpTree, isCopyTerm, zTerm, nPrefix+1); - if( rc!=SQLITE_OK ) return rc; + assert( *pRc==SQLITE_OK ); - nData = 0; - pWriter->nTerm = 0; + memset(&filter, 0, sizeof(filter)); + memset(&csr, 0, sizeof(csr)); + filter.flags = FTS3_SEGMENT_REQUIRE_POS|FTS3_SEGMENT_IGNORE_EMPTY; + filter.flags |= FTS3_SEGMENT_SCAN; - nPrefix = 0; - nSuffix = nTerm; - nReq = 1 + /* varint containing prefix size */ - sqlite3Fts3VarintLen(nTerm) + /* varint containing suffix size */ - nTerm + /* Term suffix */ - sqlite3Fts3VarintLen(nDoclist) + /* Size of doclist */ - nDoclist; /* Doclist data */ + rc = sqlite3Fts3SegReaderCursor( + p, iLangid, iIndex, FTS3_SEGCURSOR_ALL, 0, 0, 0, 1,&csr + ); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3SegReaderStart(p, &csr, &filter); } - /* Increase the total number of bytes written to account for the new entry. */ - pWriter->nLeafData += nReq; - - /* If the buffer currently allocated is too small for this entry, realloc - ** the buffer to make it large enough. - */ - if( nReq>pWriter->nSize ){ - char *aNew = sqlite3_realloc(pWriter->aData, nReq); - if( !aNew ) return SQLITE_NOMEM; - pWriter->aData = aNew; - pWriter->nSize = nReq; - } - assert( nData+nReq<=pWriter->nSize ); + if( rc==SQLITE_OK ){ + while( SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, &csr)) ){ + char *pCsr = csr.aDoclist; + char *pEnd = &pCsr[csr.nDoclist]; - /* Append the prefix-compressed term and doclist to the buffer. */ - nData += sqlite3Fts3PutVarint(&pWriter->aData[nData], nPrefix); - nData += sqlite3Fts3PutVarint(&pWriter->aData[nData], nSuffix); - memcpy(&pWriter->aData[nData], &zTerm[nPrefix], nSuffix); - nData += nSuffix; - nData += sqlite3Fts3PutVarint(&pWriter->aData[nData], nDoclist); - memcpy(&pWriter->aData[nData], aDoclist, nDoclist); - pWriter->nData = nData + nDoclist; + i64 iDocid = 0; + i64 iCol = 0; + u64 iPos = 0; - /* Save the current term so that it can be used to prefix-compress the next. - ** If the isCopyTerm parameter is true, then the buffer pointed to by - ** zTerm is transient, so take a copy of the term data. Otherwise, just - ** store a copy of the pointer. - */ - if( isCopyTerm ){ - if( nTerm>pWriter->nMalloc ){ - char *zNew = sqlite3_realloc(pWriter->zMalloc, nTerm*2); - if( !zNew ){ - return SQLITE_NOMEM; + pCsr += sqlite3Fts3GetVarint(pCsr, &iDocid); + while( pCsrbDescIdx ){ + iDocid = (i64)((u64)iDocid - iVal); + }else{ + iDocid = (i64)((u64)iDocid + iVal); + } + } + }else{ + iPos += (iVal - 2); + cksum = cksum ^ fts3ChecksumEntry( + csr.zTerm, csr.nTerm, iLangid, iIndex, iDocid, + (int)iCol, (int)iPos + ); + } + } } - pWriter->nMalloc = nTerm*2; - pWriter->zMalloc = zNew; - pWriter->zTerm = zNew; } - assert( pWriter->zTerm==pWriter->zMalloc ); - memcpy(pWriter->zTerm, zTerm, nTerm); - }else{ - pWriter->zTerm = (char *)zTerm; } - pWriter->nTerm = nTerm; + sqlite3Fts3SegReaderFinish(&csr); - return SQLITE_OK; + *pRc = rc; + return cksum; } /* -** Flush all data associated with the SegmentWriter object pWriter to the -** database. This function must be called after all terms have been added -** to the segment using fts3SegWriterAdd(). If successful, SQLITE_OK is -** returned. Otherwise, an SQLite error code. +** Check if the contents of the FTS index match the current contents of the +** content table. If no error occurs and the contents do match, set *pbOk +** to true and return SQLITE_OK. Or if the contents do not match, set *pbOk +** to false before returning. +** +** If an error occurs (e.g. an OOM or IO error), return an SQLite error +** code. The final value of *pbOk is undefined in this case. */ -static int fts3SegWriterFlush( - Fts3Table *p, /* Virtual table handle */ - SegmentWriter *pWriter, /* SegmentWriter to flush to the db */ - sqlite3_int64 iLevel, /* Value for 'level' column of %_segdir */ - int iIdx /* Value for 'idx' column of %_segdir */ -){ - int rc; /* Return code */ - if( pWriter->pTree ){ - sqlite3_int64 iLast = 0; /* Largest block id written to database */ - sqlite3_int64 iLastLeaf; /* Largest leaf block id written to db */ - char *zRoot = NULL; /* Pointer to buffer containing root node */ - int nRoot = 0; /* Size of buffer zRoot */ +static int fts3IntegrityCheck(Fts3Table *p, int *pbOk){ + int rc = SQLITE_OK; /* Return code */ + u64 cksum1 = 0; /* Checksum based on FTS index contents */ + u64 cksum2 = 0; /* Checksum based on %_content contents */ + sqlite3_stmt *pAllLangid = 0; /* Statement to return all language-ids */ - iLastLeaf = pWriter->iFree; - rc = fts3WriteSegment(p, pWriter->iFree++, pWriter->aData, pWriter->nData); - if( rc==SQLITE_OK ){ - rc = fts3NodeWrite(p, pWriter->pTree, 1, - pWriter->iFirst, pWriter->iFree, &iLast, &zRoot, &nRoot); + /* This block calculates the checksum according to the FTS index. */ + rc = fts3SqlStmt(p, SQL_SELECT_ALL_LANGID, &pAllLangid, 0); + if( rc==SQLITE_OK ){ + int rc2; + sqlite3_bind_int(pAllLangid, 1, p->iPrevLangid); + sqlite3_bind_int(pAllLangid, 2, p->nIndex); + while( rc==SQLITE_OK && sqlite3_step(pAllLangid)==SQLITE_ROW ){ + int iLangid = sqlite3_column_int(pAllLangid, 0); + int i; + for(i=0; inIndex; i++){ + cksum1 = cksum1 ^ fts3ChecksumIndex(p, iLangid, i, &rc); + } } - if( rc==SQLITE_OK ){ - rc = fts3WriteSegdir(p, iLevel, iIdx, - pWriter->iFirst, iLastLeaf, iLast, pWriter->nLeafData, zRoot, nRoot); + rc2 = sqlite3_reset(pAllLangid); + if( rc==SQLITE_OK ) rc = rc2; + } + + /* This block calculates the checksum according to the %_content table */ + if( rc==SQLITE_OK ){ + sqlite3_tokenizer_module const *pModule = p->pTokenizer->pModule; + sqlite3_stmt *pStmt = 0; + char *zSql; + + zSql = sqlite3_mprintf("SELECT %s" , p->zReadExprlist); + if( !zSql ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + } + + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + i64 iDocid = sqlite3_column_int64(pStmt, 0); + int iLang = langidFromSelect(p, pStmt); + int iCol; + + for(iCol=0; rc==SQLITE_OK && iColnColumn; iCol++){ + if( p->abNotindexed[iCol]==0 ){ + const char *zText = (const char *)sqlite3_column_text(pStmt, iCol+1); + sqlite3_tokenizer_cursor *pT = 0; + + rc = sqlite3Fts3OpenTokenizer(p->pTokenizer, iLang, zText, -1, &pT); + while( rc==SQLITE_OK ){ + char const *zToken; /* Buffer containing token */ + int nToken = 0; /* Number of bytes in token */ + int iDum1 = 0, iDum2 = 0; /* Dummy variables */ + int iPos = 0; /* Position of token in zText */ + + rc = pModule->xNext(pT, &zToken, &nToken, &iDum1, &iDum2, &iPos); + if( rc==SQLITE_OK ){ + int i; + cksum2 = cksum2 ^ fts3ChecksumEntry( + zToken, nToken, iLang, 0, iDocid, iCol, iPos + ); + for(i=1; inIndex; i++){ + if( p->aIndex[i].nPrefix<=nToken ){ + cksum2 = cksum2 ^ fts3ChecksumEntry( + zToken, p->aIndex[i].nPrefix, iLang, i, iDocid, iCol, iPos + ); + } + } + } + } + if( pT ) pModule->xClose(pT); + if( rc==SQLITE_DONE ) rc = SQLITE_OK; + } + } } - }else{ - /* The entire tree fits on the root node. Write it to the segdir table. */ - rc = fts3WriteSegdir(p, iLevel, iIdx, - 0, 0, 0, pWriter->nLeafData, pWriter->aData, pWriter->nData); - } - p->nLeafAdd++; - return rc; -} -/* -** Release all memory held by the SegmentWriter object passed as the -** first argument. -*/ -static void fts3SegWriterFree(SegmentWriter *pWriter){ - if( pWriter ){ - sqlite3_free(pWriter->aData); - sqlite3_free(pWriter->zMalloc); - fts3NodeFree(pWriter->pTree); - sqlite3_free(pWriter); + sqlite3_finalize(pStmt); } + + *pbOk = (cksum1==cksum2); + return rc; } /* -** The first value in the apVal[] array is assumed to contain an integer. -** This function tests if there exist any documents with docid values that -** are different from that integer. i.e. if deleting the document with docid -** pRowid would mean the FTS3 table were empty. +** Run the integrity-check. If no error occurs and the current contents of +** the FTS index are correct, return SQLITE_OK. Or, if the contents of the +** FTS index are incorrect, return SQLITE_CORRUPT_VTAB. ** -** If successful, *pisEmpty is set to true if the table is empty except for -** document pRowid, or false otherwise, and SQLITE_OK is returned. If an -** error occurs, an SQLite error code is returned. +** Or, if an error (e.g. an OOM or IO error) occurs, return an SQLite +** error code. +** +** The integrity-check works as follows. For each token and indexed token +** prefix in the document set, a 64-bit checksum is calculated (by code +** in fts3ChecksumEntry()) based on the following: +** +** + The index number (0 for the main index, 1 for the first prefix +** index etc.), +** + The token (or token prefix) text itself, +** + The language-id of the row it appears in, +** + The docid of the row it appears in, +** + The column it appears in, and +** + The tokens position within that column. +** +** The checksums for all entries in the index are XORed together to create +** a single checksum for the entire index. +** +** The integrity-check code calculates the same checksum in two ways: +** +** 1. By scanning the contents of the FTS index, and +** 2. By scanning and tokenizing the content table. +** +** If the two checksums are identical, the integrity-check is deemed to have +** passed. */ -static int fts3IsEmpty(Fts3Table *p, sqlite3_value *pRowid, int *pisEmpty){ - sqlite3_stmt *pStmt; +static int fts3DoIntegrityCheck( + Fts3Table *p /* FTS3 table handle */ +){ int rc; - if( p->zContentTbl ){ - /* If using the content=xxx option, assume the table is never empty */ - *pisEmpty = 0; - rc = SQLITE_OK; - }else{ - rc = fts3SqlStmt(p, SQL_IS_EMPTY, &pStmt, &pRowid); - if( rc==SQLITE_OK ){ - if( SQLITE_ROW==sqlite3_step(pStmt) ){ - *pisEmpty = sqlite3_column_int(pStmt, 0); - } - rc = sqlite3_reset(pStmt); - } - } + int bOk = 0; + rc = fts3IntegrityCheck(p, &bOk); + if( rc==SQLITE_OK && bOk==0 ) rc = FTS_CORRUPT_VTAB; return rc; } /* -** Set *pnMax to the largest segment level in the database for the index -** iIndex. +** Handle a 'special' INSERT of the form: ** -** Segment levels are stored in the 'level' column of the %_segdir table. +** "INSERT INTO tbl(tbl) VALUES()" ** -** Return SQLITE_OK if successful, or an SQLite error code if not. +** Argument pVal contains the result of . Currently the only +** meaningful value to insert is the text 'optimize'. */ -static int fts3SegmentMaxLevel( - Fts3Table *p, - int iLangid, - int iIndex, - sqlite3_int64 *pnMax -){ - sqlite3_stmt *pStmt; - int rc; - assert( iIndex>=0 && iIndexnIndex ); +static int fts3SpecialInsert(Fts3Table *p, sqlite3_value *pVal){ + int rc = SQLITE_ERROR; /* Return Code */ + const char *zVal = (const char *)sqlite3_value_text(pVal); + int nVal = sqlite3_value_bytes(pVal); - /* Set pStmt to the compiled version of: - ** - ** SELECT max(level) FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ? - ** - ** (1024 is actually the value of macro FTS3_SEGDIR_PREFIXLEVEL_STR). - */ - rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR_MAX_LEVEL, &pStmt, 0); - if( rc!=SQLITE_OK ) return rc; - sqlite3_bind_int64(pStmt, 1, getAbsoluteLevel(p, iLangid, iIndex, 0)); - sqlite3_bind_int64(pStmt, 2, - getAbsoluteLevel(p, iLangid, iIndex, FTS3_SEGDIR_MAXLEVEL-1) - ); - if( SQLITE_ROW==sqlite3_step(pStmt) ){ - *pnMax = sqlite3_column_int64(pStmt, 0); + if( !zVal ){ + return SQLITE_NOMEM; + }else if( nVal==8 && 0==sqlite3_strnicmp(zVal, "optimize", 8) ){ + rc = fts3DoOptimize(p, 0); + }else if( nVal==7 && 0==sqlite3_strnicmp(zVal, "rebuild", 7) ){ + rc = fts3DoRebuild(p); + }else if( nVal==15 && 0==sqlite3_strnicmp(zVal, "integrity-check", 15) ){ + rc = fts3DoIntegrityCheck(p); + }else if( nVal>6 && 0==sqlite3_strnicmp(zVal, "merge=", 6) ){ + rc = fts3DoIncrmerge(p, &zVal[6]); + }else if( nVal>10 && 0==sqlite3_strnicmp(zVal, "automerge=", 10) ){ + rc = fts3DoAutoincrmerge(p, &zVal[10]); +#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) + }else{ + int v; + if( nVal>9 && 0==sqlite3_strnicmp(zVal, "nodesize=", 9) ){ + v = atoi(&zVal[9]); + if( v>=24 && v<=p->nPgsz-35 ) p->nNodeSize = v; + rc = SQLITE_OK; + }else if( nVal>11 && 0==sqlite3_strnicmp(zVal, "maxpending=", 9) ){ + v = atoi(&zVal[11]); + if( v>=64 && v<=FTS3_MAX_PENDING_DATA ) p->nMaxPendingData = v; + rc = SQLITE_OK; + }else if( nVal>21 && 0==sqlite3_strnicmp(zVal,"test-no-incr-doclist=",21) ){ + p->bNoIncrDoclist = atoi(&zVal[21]); + rc = SQLITE_OK; + }else if( nVal>11 && 0==sqlite3_strnicmp(zVal,"mergecount=",11) ){ + v = atoi(&zVal[11]); + if( v>=4 && v<=FTS3_MERGE_COUNT && (v&1)==0 ) p->nMergeCount = v; + rc = SQLITE_OK; + } +#endif } - return sqlite3_reset(pStmt); + return rc; } +#ifndef SQLITE_DISABLE_FTS4_DEFERRED /* -** iAbsLevel is an absolute level that may be assumed to exist within -** the database. This function checks if it is the largest level number -** within its index. Assuming no error occurs, *pbMax is set to 1 if -** iAbsLevel is indeed the largest level, or 0 otherwise, and SQLITE_OK -** is returned. If an error occurs, an error code is returned and the -** final value of *pbMax is undefined. +** Delete all cached deferred doclists. Deferred doclists are cached +** (allocated) by the sqlite3Fts3CacheDeferredDoclists() function. */ -static int fts3SegmentIsMaxLevel(Fts3Table *p, i64 iAbsLevel, int *pbMax){ - - /* Set pStmt to the compiled version of: - ** - ** SELECT max(level) FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ? - ** - ** (1024 is actually the value of macro FTS3_SEGDIR_PREFIXLEVEL_STR). - */ - sqlite3_stmt *pStmt; - int rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR_MAX_LEVEL, &pStmt, 0); - if( rc!=SQLITE_OK ) return rc; - sqlite3_bind_int64(pStmt, 1, iAbsLevel+1); - sqlite3_bind_int64(pStmt, 2, - ((iAbsLevel/FTS3_SEGDIR_MAXLEVEL)+1) * FTS3_SEGDIR_MAXLEVEL - ); - - *pbMax = 0; - if( SQLITE_ROW==sqlite3_step(pStmt) ){ - *pbMax = sqlite3_column_type(pStmt, 0)==SQLITE_NULL; +SQLITE_PRIVATE void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *pCsr){ + Fts3DeferredToken *pDef; + for(pDef=pCsr->pDeferred; pDef; pDef=pDef->pNext){ + fts3PendingListDelete(pDef->pList); + pDef->pList = 0; } - return sqlite3_reset(pStmt); } /* -** Delete all entries in the %_segments table associated with the segment -** opened with seg-reader pSeg. This function does not affect the contents -** of the %_segdir table. +** Free all entries in the pCsr->pDeffered list. Entries are added to +** this list using sqlite3Fts3DeferToken(). */ -static int fts3DeleteSegment( - Fts3Table *p, /* FTS table handle */ - Fts3SegReader *pSeg /* Segment to delete */ -){ - int rc = SQLITE_OK; /* Return code */ - if( pSeg->iStartBlock ){ - sqlite3_stmt *pDelete; /* SQL statement to delete rows */ - rc = fts3SqlStmt(p, SQL_DELETE_SEGMENTS_RANGE, &pDelete, 0); - if( rc==SQLITE_OK ){ - sqlite3_bind_int64(pDelete, 1, pSeg->iStartBlock); - sqlite3_bind_int64(pDelete, 2, pSeg->iEndBlock); - sqlite3_step(pDelete); - rc = sqlite3_reset(pDelete); - } +SQLITE_PRIVATE void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *pCsr){ + Fts3DeferredToken *pDef; + Fts3DeferredToken *pNext; + for(pDef=pCsr->pDeferred; pDef; pDef=pNext){ + pNext = pDef->pNext; + fts3PendingListDelete(pDef->pList); + sqlite3_free(pDef); } - return rc; + pCsr->pDeferred = 0; } /* -** This function is used after merging multiple segments into a single large -** segment to delete the old, now redundant, segment b-trees. Specifically, -** it: -** -** 1) Deletes all %_segments entries for the segments associated with -** each of the SegReader objects in the array passed as the third -** argument, and -** -** 2) deletes all %_segdir entries with level iLevel, or all %_segdir -** entries regardless of level if (iLevel<0). +** Generate deferred-doclists for all tokens in the pCsr->pDeferred list +** based on the row that pCsr currently points to. ** -** SQLITE_OK is returned if successful, otherwise an SQLite error code. +** A deferred-doclist is like any other doclist with position information +** included, except that it only contains entries for a single row of the +** table, not for all rows. */ -static int fts3DeleteSegdir( - Fts3Table *p, /* Virtual table handle */ - int iLangid, /* Language id */ - int iIndex, /* Index for p->aIndex */ - int iLevel, /* Level of %_segdir entries to delete */ - Fts3SegReader **apSegment, /* Array of SegReader objects */ - int nReader /* Size of array apSegment */ -){ - int rc = SQLITE_OK; /* Return Code */ - int i; /* Iterator variable */ - sqlite3_stmt *pDelete = 0; /* SQL statement to delete rows */ +SQLITE_PRIVATE int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *pCsr){ + int rc = SQLITE_OK; /* Return code */ + if( pCsr->pDeferred ){ + int i; /* Used to iterate through table columns */ + sqlite3_int64 iDocid; /* Docid of the row pCsr points to */ + Fts3DeferredToken *pDef; /* Used to iterate through deferred tokens */ + + Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; + sqlite3_tokenizer *pT = p->pTokenizer; + sqlite3_tokenizer_module const *pModule = pT->pModule; + + assert( pCsr->isRequireSeek==0 ); + iDocid = sqlite3_column_int64(pCsr->pStmt, 0); + + for(i=0; inColumn && rc==SQLITE_OK; i++){ + if( p->abNotindexed[i]==0 ){ + const char *zText = (const char *)sqlite3_column_text(pCsr->pStmt, i+1); + sqlite3_tokenizer_cursor *pTC = 0; - for(i=0; rc==SQLITE_OK && iiLangid, zText, -1, &pTC); + while( rc==SQLITE_OK ){ + char const *zToken; /* Buffer containing token */ + int nToken = 0; /* Number of bytes in token */ + int iDum1 = 0, iDum2 = 0; /* Dummy variables */ + int iPos = 0; /* Position of token in zText */ - assert( iLevel>=0 || iLevel==FTS3_SEGCURSOR_ALL ); - if( iLevel==FTS3_SEGCURSOR_ALL ){ - rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_RANGE, &pDelete, 0); - if( rc==SQLITE_OK ){ - sqlite3_bind_int64(pDelete, 1, getAbsoluteLevel(p, iLangid, iIndex, 0)); - sqlite3_bind_int64(pDelete, 2, - getAbsoluteLevel(p, iLangid, iIndex, FTS3_SEGDIR_MAXLEVEL-1) - ); - } - }else{ - rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_LEVEL, &pDelete, 0); - if( rc==SQLITE_OK ){ - sqlite3_bind_int64( - pDelete, 1, getAbsoluteLevel(p, iLangid, iIndex, iLevel) - ); + rc = pModule->xNext(pTC, &zToken, &nToken, &iDum1, &iDum2, &iPos); + for(pDef=pCsr->pDeferred; pDef && rc==SQLITE_OK; pDef=pDef->pNext){ + Fts3PhraseToken *pPT = pDef->pToken; + if( (pDef->iCol>=p->nColumn || pDef->iCol==i) + && (pPT->bFirst==0 || iPos==0) + && (pPT->n==nToken || (pPT->isPrefix && pPT->nz, pPT->n)) + ){ + fts3PendingListAppend(&pDef->pList, iDocid, i, iPos, &rc); + } + } + } + if( pTC ) pModule->xClose(pTC); + if( rc==SQLITE_DONE ) rc = SQLITE_OK; + } } - } - if( rc==SQLITE_OK ){ - sqlite3_step(pDelete); - rc = sqlite3_reset(pDelete); + for(pDef=pCsr->pDeferred; pDef && rc==SQLITE_OK; pDef=pDef->pNext){ + if( pDef->pList ){ + rc = fts3PendingListAppendVarint(&pDef->pList, 0); + } + } } return rc; } -/* -** When this function is called, buffer *ppList (size *pnList bytes) contains -** a position list that may (or may not) feature multiple columns. This -** function adjusts the pointer *ppList and the length *pnList so that they -** identify the subset of the position list that corresponds to column iCol. -** -** If there are no entries in the input position list for column iCol, then -** *pnList is set to zero before returning. -** -** If parameter bZero is non-zero, then any part of the input list following -** the end of the output list is zeroed before returning. -*/ -static void fts3ColumnFilter( - int iCol, /* Column to filter on */ - int bZero, /* Zero out anything following *ppList */ - char **ppList, /* IN/OUT: Pointer to position list */ - int *pnList /* IN/OUT: Size of buffer *ppList in bytes */ +SQLITE_PRIVATE int sqlite3Fts3DeferredTokenList( + Fts3DeferredToken *p, + char **ppData, + int *pnData ){ - char *pList = *ppList; - int nList = *pnList; - char *pEnd = &pList[nList]; - int iCurrent = 0; - char *p = pList; + char *pRet; + int nSkip; + sqlite3_int64 dummy; - assert( iCol>=0 ); - while( 1 ){ - char c = 0; - while( ppList==0 ){ + return SQLITE_OK; } - if( bZero && &pList[nList]!=pEnd ){ - memset(&pList[nList], 0, pEnd - &pList[nList]); - } - *ppList = pList; - *pnList = nList; + pRet = (char *)sqlite3_malloc(p->pList->nData); + if( !pRet ) return SQLITE_NOMEM; + + nSkip = sqlite3Fts3GetVarint(p->pList->aData, &dummy); + *pnData = p->pList->nData - nSkip; + *ppData = pRet; + + memcpy(pRet, &p->pList->aData[nSkip], *pnData); + return SQLITE_OK; } /* -** Cache data in the Fts3MultiSegReader.aBuffer[] buffer (overwriting any -** existing data). Grow the buffer if required. -** -** If successful, return SQLITE_OK. Otherwise, if an OOM error is encountered -** trying to resize the buffer, return SQLITE_NOMEM. +** Add an entry for token pToken to the pCsr->pDeferred list. */ -static int fts3MsrBufferData( - Fts3MultiSegReader *pMsr, /* Multi-segment-reader handle */ - char *pList, - int nList +SQLITE_PRIVATE int sqlite3Fts3DeferToken( + Fts3Cursor *pCsr, /* Fts3 table cursor */ + Fts3PhraseToken *pToken, /* Token to defer */ + int iCol /* Column that token must appear in (or -1) */ ){ - if( nList>pMsr->nBuffer ){ - char *pNew; - pMsr->nBuffer = nList*2; - pNew = (char *)sqlite3_realloc(pMsr->aBuffer, pMsr->nBuffer); - if( !pNew ) return SQLITE_NOMEM; - pMsr->aBuffer = pNew; + Fts3DeferredToken *pDeferred; + pDeferred = sqlite3_malloc(sizeof(*pDeferred)); + if( !pDeferred ){ + return SQLITE_NOMEM; } + memset(pDeferred, 0, sizeof(*pDeferred)); + pDeferred->pToken = pToken; + pDeferred->pNext = pCsr->pDeferred; + pDeferred->iCol = iCol; + pCsr->pDeferred = pDeferred; + + assert( pToken->pDeferred==0 ); + pToken->pDeferred = pDeferred; - memcpy(pMsr->aBuffer, pList, nList); return SQLITE_OK; } +#endif -SQLITE_PRIVATE int sqlite3Fts3MsrIncrNext( - Fts3Table *p, /* Virtual table handle */ - Fts3MultiSegReader *pMsr, /* Multi-segment-reader handle */ - sqlite3_int64 *piDocid, /* OUT: Docid value */ - char **paPoslist, /* OUT: Pointer to position list */ - int *pnPoslist /* OUT: Size of position list in bytes */ +/* +** SQLite value pRowid contains the rowid of a row that may or may not be +** present in the FTS3 table. If it is, delete it and adjust the contents +** of subsiduary data structures accordingly. +*/ +static int fts3DeleteByRowid( + Fts3Table *p, + sqlite3_value *pRowid, + int *pnChng, /* IN/OUT: Decrement if row is deleted */ + u32 *aSzDel ){ - int nMerge = pMsr->nAdvance; - Fts3SegReader **apSegment = pMsr->apSegment; - int (*xCmp)(Fts3SegReader *, Fts3SegReader *) = ( - p->bDescIdx ? fts3SegReaderDoclistCmpRev : fts3SegReaderDoclistCmp - ); - - if( nMerge==0 ){ - *paPoslist = 0; - return SQLITE_OK; - } - - while( 1 ){ - Fts3SegReader *pSeg; - pSeg = pMsr->apSegment[0]; - - if( pSeg->pOffsetList==0 ){ - *paPoslist = 0; - break; - }else{ - int rc; - char *pList; - int nList; - int j; - sqlite3_int64 iDocid = apSegment[0]->iDocid; - - rc = fts3SegReaderNextDocid(p, apSegment[0], &pList, &nList); - j = 1; - while( rc==SQLITE_OK - && jpOffsetList - && apSegment[j]->iDocid==iDocid - ){ - rc = fts3SegReaderNextDocid(p, apSegment[j], 0, 0); - j++; - } - if( rc!=SQLITE_OK ) return rc; - fts3SegReaderSort(pMsr->apSegment, nMerge, j, xCmp); - - if( nList>0 && fts3SegReaderIsPending(apSegment[0]) ){ - rc = fts3MsrBufferData(pMsr, pList, nList+1); - if( rc!=SQLITE_OK ) return rc; - assert( (pMsr->aBuffer[nList] & 0xFE)==0x00 ); - pList = pMsr->aBuffer; - } - - if( pMsr->iColFilter>=0 ){ - fts3ColumnFilter(pMsr->iColFilter, 1, &pList, &nList); - } + int rc = SQLITE_OK; /* Return code */ + int bFound = 0; /* True if *pRowid really is in the table */ - if( nList>0 ){ - *paPoslist = pList; - *piDocid = iDocid; - *pnPoslist = nList; - break; + fts3DeleteTerms(&rc, p, pRowid, aSzDel, &bFound); + if( bFound && rc==SQLITE_OK ){ + int isEmpty = 0; /* Deleting *pRowid leaves the table empty */ + rc = fts3IsEmpty(p, pRowid, &isEmpty); + if( rc==SQLITE_OK ){ + if( isEmpty ){ + /* Deleting this row means the whole table is empty. In this case + ** delete the contents of all three tables and throw away any + ** data in the pendingTerms hash table. */ + rc = fts3DeleteAll(p, 1); + *pnChng = 0; + memset(aSzDel, 0, sizeof(u32) * (p->nColumn+1) * 2); + }else{ + *pnChng = *pnChng - 1; + if( p->zContentTbl==0 ){ + fts3SqlExec(&rc, p, SQL_DELETE_CONTENT, &pRowid); + } + if( p->bHasDocsize ){ + fts3SqlExec(&rc, p, SQL_DELETE_DOCSIZE, &pRowid); + } } } } - return SQLITE_OK; + return rc; } -static int fts3SegReaderStart( - Fts3Table *p, /* Virtual table handle */ - Fts3MultiSegReader *pCsr, /* Cursor object */ - const char *zTerm, /* Term searched for (or NULL) */ - int nTerm /* Length of zTerm in bytes */ +/* +** This function does the work for the xUpdate method of FTS3 virtual +** tables. The schema of the virtual table being: +** +** CREATE TABLE ( +** , +**
        HIDDEN, +** docid HIDDEN, +** HIDDEN +** ); +** +** +*/ +SQLITE_PRIVATE int sqlite3Fts3UpdateMethod( + sqlite3_vtab *pVtab, /* FTS3 vtab object */ + int nArg, /* Size of argument array */ + sqlite3_value **apVal, /* Array of arguments */ + sqlite_int64 *pRowid /* OUT: The affected (or effected) rowid */ ){ - int i; - int nSeg = pCsr->nSegment; + Fts3Table *p = (Fts3Table *)pVtab; + int rc = SQLITE_OK; /* Return Code */ + u32 *aSzIns = 0; /* Sizes of inserted documents */ + u32 *aSzDel = 0; /* Sizes of deleted documents */ + int nChng = 0; /* Net change in number of documents */ + int bInsertDone = 0; - /* If the Fts3SegFilter defines a specific term (or term prefix) to search - ** for, then advance each segment iterator until it points to a term of - ** equal or greater value than the specified term. This prevents many - ** unnecessary merge/sort operations for the case where single segment - ** b-tree leaf nodes contain more than one term. - */ - for(i=0; pCsr->bRestart==0 && inSegment; i++){ - int res = 0; - Fts3SegReader *pSeg = pCsr->apSegment[i]; - do { - int rc = fts3SegReaderNext(p, pSeg, 0); - if( rc!=SQLITE_OK ) return rc; - }while( zTerm && (res = fts3SegReaderTermCmp(pSeg, zTerm, nTerm))<0 ); + /* At this point it must be known if the %_stat table exists or not. + ** So bHasStat may not be 2. */ + assert( p->bHasStat==0 || p->bHasStat==1 ); - if( pSeg->bLookup && res!=0 ){ - fts3SegReaderSetEof(pSeg); - } + assert( p->pSegments==0 ); + assert( + nArg==1 /* DELETE operations */ + || nArg==(2 + p->nColumn + 3) /* INSERT or UPDATE operations */ + ); + + /* Check for a "special" INSERT operation. One of the form: + ** + ** INSERT INTO xyz(xyz) VALUES('command'); + */ + if( nArg>1 + && sqlite3_value_type(apVal[0])==SQLITE_NULL + && sqlite3_value_type(apVal[p->nColumn+2])!=SQLITE_NULL + ){ + rc = fts3SpecialInsert(p, apVal[p->nColumn+2]); + goto update_out; } - fts3SegReaderSort(pCsr->apSegment, nSeg, nSeg, fts3SegReaderCmp); - return SQLITE_OK; -} + if( nArg>1 && sqlite3_value_int(apVal[2 + p->nColumn + 2])<0 ){ + rc = SQLITE_CONSTRAINT; + goto update_out; + } -SQLITE_PRIVATE int sqlite3Fts3SegReaderStart( - Fts3Table *p, /* Virtual table handle */ - Fts3MultiSegReader *pCsr, /* Cursor object */ - Fts3SegFilter *pFilter /* Restrictions on range of iteration */ -){ - pCsr->pFilter = pFilter; - return fts3SegReaderStart(p, pCsr, pFilter->zTerm, pFilter->nTerm); -} + /* Allocate space to hold the change in document sizes */ + aSzDel = sqlite3_malloc64(sizeof(aSzDel[0])*((sqlite3_int64)p->nColumn+1)*2); + if( aSzDel==0 ){ + rc = SQLITE_NOMEM; + goto update_out; + } + aSzIns = &aSzDel[p->nColumn+1]; + memset(aSzDel, 0, sizeof(aSzDel[0])*(p->nColumn+1)*2); -SQLITE_PRIVATE int sqlite3Fts3MsrIncrStart( - Fts3Table *p, /* Virtual table handle */ - Fts3MultiSegReader *pCsr, /* Cursor object */ - int iCol, /* Column to match on. */ - const char *zTerm, /* Term to iterate through a doclist for */ - int nTerm /* Number of bytes in zTerm */ -){ - int i; - int rc; - int nSegment = pCsr->nSegment; - int (*xCmp)(Fts3SegReader *, Fts3SegReader *) = ( - p->bDescIdx ? fts3SegReaderDoclistCmpRev : fts3SegReaderDoclistCmp - ); + rc = fts3Writelock(p); + if( rc!=SQLITE_OK ) goto update_out; - assert( pCsr->pFilter==0 ); - assert( zTerm && nTerm>0 ); + /* If this is an INSERT operation, or an UPDATE that modifies the rowid + ** value, then this operation requires constraint handling. + ** + ** If the on-conflict mode is REPLACE, this means that the existing row + ** should be deleted from the database before inserting the new row. Or, + ** if the on-conflict mode is other than REPLACE, then this method must + ** detect the conflict and return SQLITE_CONSTRAINT before beginning to + ** modify the database file. + */ + if( nArg>1 && p->zContentTbl==0 ){ + /* Find the value object that holds the new rowid value. */ + sqlite3_value *pNewRowid = apVal[3+p->nColumn]; + if( sqlite3_value_type(pNewRowid)==SQLITE_NULL ){ + pNewRowid = apVal[1]; + } - /* Advance each segment iterator until it points to the term zTerm/nTerm. */ - rc = fts3SegReaderStart(p, pCsr, zTerm, nTerm); - if( rc!=SQLITE_OK ) return rc; + if( sqlite3_value_type(pNewRowid)!=SQLITE_NULL && ( + sqlite3_value_type(apVal[0])==SQLITE_NULL + || sqlite3_value_int64(apVal[0])!=sqlite3_value_int64(pNewRowid) + )){ + /* The new rowid is not NULL (in this case the rowid will be + ** automatically assigned and there is no chance of a conflict), and + ** the statement is either an INSERT or an UPDATE that modifies the + ** rowid column. So if the conflict mode is REPLACE, then delete any + ** existing row with rowid=pNewRowid. + ** + ** Or, if the conflict mode is not REPLACE, insert the new record into + ** the %_content table. If we hit the duplicate rowid constraint (or any + ** other error) while doing so, return immediately. + ** + ** This branch may also run if pNewRowid contains a value that cannot + ** be losslessly converted to an integer. In this case, the eventual + ** call to fts3InsertData() (either just below or further on in this + ** function) will return SQLITE_MISMATCH. If fts3DeleteByRowid is + ** invoked, it will delete zero rows (since no row will have + ** docid=$pNewRowid if $pNewRowid is not an integer value). + */ + if( sqlite3_vtab_on_conflict(p->db)==SQLITE_REPLACE ){ + rc = fts3DeleteByRowid(p, pNewRowid, &nChng, aSzDel); + }else{ + rc = fts3InsertData(p, apVal, pRowid); + bInsertDone = 1; + } + } + } + if( rc!=SQLITE_OK ){ + goto update_out; + } - /* Determine how many of the segments actually point to zTerm/nTerm. */ - for(i=0; iapSegment[i]; - if( !pSeg->aNode || fts3SegReaderTermCmp(pSeg, zTerm, nTerm) ){ - break; + /* If this is a DELETE or UPDATE operation, remove the old record. */ + if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){ + assert( sqlite3_value_type(apVal[0])==SQLITE_INTEGER ); + rc = fts3DeleteByRowid(p, apVal[0], &nChng, aSzDel); + } + + /* If this is an INSERT or UPDATE operation, insert the new record. */ + if( nArg>1 && rc==SQLITE_OK ){ + int iLangid = sqlite3_value_int(apVal[2 + p->nColumn + 2]); + if( bInsertDone==0 ){ + rc = fts3InsertData(p, apVal, pRowid); + if( rc==SQLITE_CONSTRAINT && p->zContentTbl==0 ){ + rc = FTS_CORRUPT_VTAB; + } + } + if( rc==SQLITE_OK ){ + rc = fts3PendingTermsDocid(p, 0, iLangid, *pRowid); + } + if( rc==SQLITE_OK ){ + assert( p->iPrevDocid==*pRowid ); + rc = fts3InsertTerms(p, iLangid, apVal, aSzIns); } + if( p->bHasDocsize ){ + fts3InsertDocsize(&rc, p, aSzIns); + } + nChng++; } - pCsr->nAdvance = i; - /* Advance each of the segments to point to the first docid. */ - for(i=0; inAdvance; i++){ - rc = fts3SegReaderFirstDocid(p, pCsr->apSegment[i]); - if( rc!=SQLITE_OK ) return rc; + if( p->bFts4 ){ + fts3UpdateDocTotals(&rc, p, aSzIns, aSzDel, nChng); } - fts3SegReaderSort(pCsr->apSegment, i, i, xCmp); - assert( iCol<0 || iColnColumn ); - pCsr->iColFilter = iCol; + update_out: + sqlite3_free(aSzDel); + sqlite3Fts3SegmentsClose(p); + return rc; +} - return SQLITE_OK; +/* +** Flush any data in the pending-terms hash table to disk. If successful, +** merge all segments in the database (including the new segment, if +** there was any data to flush) into a single segment. +*/ +SQLITE_PRIVATE int sqlite3Fts3Optimize(Fts3Table *p){ + int rc; + rc = sqlite3_exec(p->db, "SAVEPOINT fts3", 0, 0, 0); + if( rc==SQLITE_OK ){ + rc = fts3DoOptimize(p, 1); + if( rc==SQLITE_OK || rc==SQLITE_DONE ){ + int rc2 = sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0); + if( rc2!=SQLITE_OK ) rc = rc2; + }else{ + sqlite3_exec(p->db, "ROLLBACK TO fts3", 0, 0, 0); + sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0); + } + } + sqlite3Fts3SegmentsClose(p); + return rc; } +#endif + +/************** End of fts3_write.c ******************************************/ +/************** Begin file fts3_snippet.c ************************************/ /* -** This function is called on a MultiSegReader that has been started using -** sqlite3Fts3MsrIncrStart(). One or more calls to MsrIncrNext() may also -** have been made. Calling this function puts the MultiSegReader in such -** a state that if the next two calls are: +** 2009 Oct 23 ** -** sqlite3Fts3SegReaderStart() -** sqlite3Fts3SegReaderStep() +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: ** -** then the entire doclist for the term is available in -** MultiSegReader.aDoclist/nDoclist. +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** */ -SQLITE_PRIVATE int sqlite3Fts3MsrIncrRestart(Fts3MultiSegReader *pCsr){ - int i; /* Used to iterate through segment-readers */ - assert( pCsr->zTerm==0 ); - assert( pCsr->nTerm==0 ); - assert( pCsr->aDoclist==0 ); - assert( pCsr->nDoclist==0 ); +/* #include "fts3Int.h" */ +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) - pCsr->nAdvance = 0; - pCsr->bRestart = 1; - for(i=0; inSegment; i++){ - pCsr->apSegment[i]->pOffsetList = 0; - pCsr->apSegment[i]->nOffsetList = 0; - pCsr->apSegment[i]->iDocid = 0; - } +/* #include */ +/* #include */ - return SQLITE_OK; -} +/* +** Characters that may appear in the second argument to matchinfo(). +*/ +#define FTS3_MATCHINFO_NPHRASE 'p' /* 1 value */ +#define FTS3_MATCHINFO_NCOL 'c' /* 1 value */ +#define FTS3_MATCHINFO_NDOC 'n' /* 1 value */ +#define FTS3_MATCHINFO_AVGLENGTH 'a' /* nCol values */ +#define FTS3_MATCHINFO_LENGTH 'l' /* nCol values */ +#define FTS3_MATCHINFO_LCS 's' /* nCol values */ +#define FTS3_MATCHINFO_HITS 'x' /* 3*nCol*nPhrase values */ +#define FTS3_MATCHINFO_LHITS 'y' /* nCol*nPhrase values */ +#define FTS3_MATCHINFO_LHITS_BM 'b' /* nCol*nPhrase values */ +/* +** The default value for the second argument to matchinfo(). +*/ +#define FTS3_MATCHINFO_DEFAULT "pcx" -SQLITE_PRIVATE int sqlite3Fts3SegReaderStep( - Fts3Table *p, /* Virtual table handle */ - Fts3MultiSegReader *pCsr /* Cursor object */ -){ - int rc = SQLITE_OK; - int isIgnoreEmpty = (pCsr->pFilter->flags & FTS3_SEGMENT_IGNORE_EMPTY); - int isRequirePos = (pCsr->pFilter->flags & FTS3_SEGMENT_REQUIRE_POS); - int isColFilter = (pCsr->pFilter->flags & FTS3_SEGMENT_COLUMN_FILTER); - int isPrefix = (pCsr->pFilter->flags & FTS3_SEGMENT_PREFIX); - int isScan = (pCsr->pFilter->flags & FTS3_SEGMENT_SCAN); - int isFirst = (pCsr->pFilter->flags & FTS3_SEGMENT_FIRST); +/* +** Used as an fts3ExprIterate() context when loading phrase doclists to +** Fts3Expr.aDoclist[]/nDoclist. +*/ +typedef struct LoadDoclistCtx LoadDoclistCtx; +struct LoadDoclistCtx { + Fts3Cursor *pCsr; /* FTS3 Cursor */ + int nPhrase; /* Number of phrases seen so far */ + int nToken; /* Number of tokens seen so far */ +}; - Fts3SegReader **apSegment = pCsr->apSegment; - int nSegment = pCsr->nSegment; - Fts3SegFilter *pFilter = pCsr->pFilter; - int (*xCmp)(Fts3SegReader *, Fts3SegReader *) = ( - p->bDescIdx ? fts3SegReaderDoclistCmpRev : fts3SegReaderDoclistCmp - ); +/* +** The following types are used as part of the implementation of the +** fts3BestSnippet() routine. +*/ +typedef struct SnippetIter SnippetIter; +typedef struct SnippetPhrase SnippetPhrase; +typedef struct SnippetFragment SnippetFragment; - if( pCsr->nSegment==0 ) return SQLITE_OK; +struct SnippetIter { + Fts3Cursor *pCsr; /* Cursor snippet is being generated from */ + int iCol; /* Extract snippet from this column */ + int nSnippet; /* Requested snippet length (in tokens) */ + int nPhrase; /* Number of phrases in query */ + SnippetPhrase *aPhrase; /* Array of size nPhrase */ + int iCurrent; /* First token of current snippet */ +}; - do { - int nMerge; - int i; - - /* Advance the first pCsr->nAdvance entries in the apSegment[] array - ** forward. Then sort the list in order of current term again. - */ - for(i=0; inAdvance; i++){ - Fts3SegReader *pSeg = apSegment[i]; - if( pSeg->bLookup ){ - fts3SegReaderSetEof(pSeg); - }else{ - rc = fts3SegReaderNext(p, pSeg, 0); - } - if( rc!=SQLITE_OK ) return rc; - } - fts3SegReaderSort(apSegment, nSegment, pCsr->nAdvance, fts3SegReaderCmp); - pCsr->nAdvance = 0; +struct SnippetPhrase { + int nToken; /* Number of tokens in phrase */ + char *pList; /* Pointer to start of phrase position list */ + int iHead; /* Next value in position list */ + char *pHead; /* Position list data following iHead */ + int iTail; /* Next value in trailing position list */ + char *pTail; /* Position list data following iTail */ +}; - /* If all the seg-readers are at EOF, we're finished. return SQLITE_OK. */ - assert( rc==SQLITE_OK ); - if( apSegment[0]->aNode==0 ) break; +struct SnippetFragment { + int iCol; /* Column snippet is extracted from */ + int iPos; /* Index of first token in snippet */ + u64 covered; /* Mask of query phrases covered */ + u64 hlmask; /* Mask of snippet terms to highlight */ +}; - pCsr->nTerm = apSegment[0]->nTerm; - pCsr->zTerm = apSegment[0]->zTerm; +/* +** This type is used as an fts3ExprIterate() context object while +** accumulating the data returned by the matchinfo() function. +*/ +typedef struct MatchInfo MatchInfo; +struct MatchInfo { + Fts3Cursor *pCursor; /* FTS3 Cursor */ + int nCol; /* Number of columns in table */ + int nPhrase; /* Number of matchable phrases in query */ + sqlite3_int64 nDoc; /* Number of docs in database */ + char flag; + u32 *aMatchinfo; /* Pre-allocated buffer */ +}; - /* If this is a prefix-search, and if the term that apSegment[0] points - ** to does not share a suffix with pFilter->zTerm/nTerm, then all - ** required callbacks have been made. In this case exit early. - ** - ** Similarly, if this is a search for an exact match, and the first term - ** of segment apSegment[0] is not a match, exit early. - */ - if( pFilter->zTerm && !isScan ){ - if( pCsr->nTermnTerm - || (!isPrefix && pCsr->nTerm>pFilter->nTerm) - || memcmp(pCsr->zTerm, pFilter->zTerm, pFilter->nTerm) - ){ - break; - } - } +/* +** An instance of this structure is used to manage a pair of buffers, each +** (nElem * sizeof(u32)) bytes in size. See the MatchinfoBuffer code below +** for details. +*/ +struct MatchinfoBuffer { + u8 aRef[3]; + int nElem; + int bGlobal; /* Set if global data is loaded */ + char *zMatchinfo; + u32 aMatchinfo[1]; +}; - nMerge = 1; - while( nMergeaNode - && apSegment[nMerge]->nTerm==pCsr->nTerm - && 0==memcmp(pCsr->zTerm, apSegment[nMerge]->zTerm, pCsr->nTerm) - ){ - nMerge++; - } - assert( isIgnoreEmpty || (isRequirePos && !isColFilter) ); - if( nMerge==1 - && !isIgnoreEmpty - && !isFirst - && (p->bDescIdx==0 || fts3SegReaderIsPending(apSegment[0])==0) - ){ - pCsr->nDoclist = apSegment[0]->nDoclist; - if( fts3SegReaderIsPending(apSegment[0]) ){ - rc = fts3MsrBufferData(pCsr, apSegment[0]->aDoclist, pCsr->nDoclist); - pCsr->aDoclist = pCsr->aBuffer; - }else{ - pCsr->aDoclist = apSegment[0]->aDoclist; - } - if( rc==SQLITE_OK ) rc = SQLITE_ROW; - }else{ - int nDoclist = 0; /* Size of doclist */ - sqlite3_int64 iPrev = 0; /* Previous docid stored in doclist */ +/* +** The snippet() and offsets() functions both return text values. An instance +** of the following structure is used to accumulate those values while the +** functions are running. See fts3StringAppend() for details. +*/ +typedef struct StrBuffer StrBuffer; +struct StrBuffer { + char *z; /* Pointer to buffer containing string */ + int n; /* Length of z in bytes (excl. nul-term) */ + int nAlloc; /* Allocated size of buffer z in bytes */ +}; - /* The current term of the first nMerge entries in the array - ** of Fts3SegReader objects is the same. The doclists must be merged - ** and a single term returned with the merged doclist. - */ - for(i=0; ipOffsetList ){ - int j; /* Number of segments that share a docid */ - char *pList = 0; - int nList = 0; - int nByte; - sqlite3_int64 iDocid = apSegment[0]->iDocid; - fts3SegReaderNextDocid(p, apSegment[0], &pList, &nList); - j = 1; - while( jpOffsetList - && apSegment[j]->iDocid==iDocid - ){ - fts3SegReaderNextDocid(p, apSegment[j], 0, 0); - j++; - } - if( isColFilter ){ - fts3ColumnFilter(pFilter->iCol, 0, &pList, &nList); - } +/************************************************************************* +** Start of MatchinfoBuffer code. +*/ - if( !isIgnoreEmpty || nList>0 ){ +/* +** Allocate a two-slot MatchinfoBuffer object. +*/ +static MatchinfoBuffer *fts3MIBufferNew(size_t nElem, const char *zMatchinfo){ + MatchinfoBuffer *pRet; + sqlite3_int64 nByte = sizeof(u32) * (2*(sqlite3_int64)nElem + 1) + + sizeof(MatchinfoBuffer); + sqlite3_int64 nStr = strlen(zMatchinfo); - /* Calculate the 'docid' delta value to write into the merged - ** doclist. */ - sqlite3_int64 iDelta; - if( p->bDescIdx && nDoclist>0 ){ - iDelta = iPrev - iDocid; - }else{ - iDelta = iDocid - iPrev; - } - assert( iDelta>0 || (nDoclist==0 && iDelta==iDocid) ); - assert( nDoclist>0 || iDelta==iDocid ); + pRet = sqlite3_malloc64(nByte + nStr+1); + if( pRet ){ + memset(pRet, 0, nByte); + pRet->aMatchinfo[0] = (u8*)(&pRet->aMatchinfo[1]) - (u8*)pRet; + pRet->aMatchinfo[1+nElem] = pRet->aMatchinfo[0] + + sizeof(u32)*((int)nElem+1); + pRet->nElem = (int)nElem; + pRet->zMatchinfo = ((char*)pRet) + nByte; + memcpy(pRet->zMatchinfo, zMatchinfo, nStr+1); + pRet->aRef[0] = 1; + } - nByte = sqlite3Fts3VarintLen(iDelta) + (isRequirePos?nList+1:0); - if( nDoclist+nByte>pCsr->nBuffer ){ - char *aNew; - pCsr->nBuffer = (nDoclist+nByte)*2; - aNew = sqlite3_realloc(pCsr->aBuffer, pCsr->nBuffer); - if( !aNew ){ - return SQLITE_NOMEM; - } - pCsr->aBuffer = aNew; - } + return pRet; +} - if( isFirst ){ - char *a = &pCsr->aBuffer[nDoclist]; - int nWrite; - - nWrite = sqlite3Fts3FirstFilter(iDelta, pList, nList, a); - if( nWrite ){ - iPrev = iDocid; - nDoclist += nWrite; - } - }else{ - nDoclist += sqlite3Fts3PutVarint(&pCsr->aBuffer[nDoclist], iDelta); - iPrev = iDocid; - if( isRequirePos ){ - memcpy(&pCsr->aBuffer[nDoclist], pList, nList); - nDoclist += nList; - pCsr->aBuffer[nDoclist++] = '\0'; - } - } - } +static void fts3MIBufferFree(void *p){ + MatchinfoBuffer *pBuf = (MatchinfoBuffer*)((u8*)p - ((u32*)p)[-1]); - fts3SegReaderSort(apSegment, nMerge, j, xCmp); - } - if( nDoclist>0 ){ - pCsr->aDoclist = pCsr->aBuffer; - pCsr->nDoclist = nDoclist; - rc = SQLITE_ROW; - } - } - pCsr->nAdvance = nMerge; - }while( rc==SQLITE_OK ); + assert( (u32*)p==&pBuf->aMatchinfo[1] + || (u32*)p==&pBuf->aMatchinfo[pBuf->nElem+2] + ); + if( (u32*)p==&pBuf->aMatchinfo[1] ){ + pBuf->aRef[1] = 0; + }else{ + pBuf->aRef[2] = 0; + } - return rc; + if( pBuf->aRef[0]==0 && pBuf->aRef[1]==0 && pBuf->aRef[2]==0 ){ + sqlite3_free(pBuf); + } } +static void (*fts3MIBufferAlloc(MatchinfoBuffer *p, u32 **paOut))(void*){ + void (*xRet)(void*) = 0; + u32 *aOut = 0; -SQLITE_PRIVATE void sqlite3Fts3SegReaderFinish( - Fts3MultiSegReader *pCsr /* Cursor object */ -){ - if( pCsr ){ - int i; - for(i=0; inSegment; i++){ - sqlite3Fts3SegReaderFree(pCsr->apSegment[i]); + if( p->aRef[1]==0 ){ + p->aRef[1] = 1; + aOut = &p->aMatchinfo[1]; + xRet = fts3MIBufferFree; + } + else if( p->aRef[2]==0 ){ + p->aRef[2] = 1; + aOut = &p->aMatchinfo[p->nElem+2]; + xRet = fts3MIBufferFree; + }else{ + aOut = (u32*)sqlite3_malloc64(p->nElem * sizeof(u32)); + if( aOut ){ + xRet = sqlite3_free; + if( p->bGlobal ) memcpy(aOut, &p->aMatchinfo[1], p->nElem*sizeof(u32)); } - sqlite3_free(pCsr->apSegment); - sqlite3_free(pCsr->aBuffer); - - pCsr->nSegment = 0; - pCsr->apSegment = 0; - pCsr->aBuffer = 0; } + + *paOut = aOut; + return xRet; +} + +static void fts3MIBufferSetGlobal(MatchinfoBuffer *p){ + p->bGlobal = 1; + memcpy(&p->aMatchinfo[2+p->nElem], &p->aMatchinfo[1], p->nElem*sizeof(u32)); } /* -** Decode the "end_block" field, selected by column iCol of the SELECT -** statement passed as the first argument. -** -** The "end_block" field may contain either an integer, or a text field -** containing the text representation of two non-negative integers separated -** by one or more space (0x20) characters. In the first case, set *piEndBlock -** to the integer value and *pnByte to zero before returning. In the second, -** set *piEndBlock to the first value and *pnByte to the second. +** Free a MatchinfoBuffer object allocated using fts3MIBufferNew() */ -static void fts3ReadEndBlockField( - sqlite3_stmt *pStmt, - int iCol, - i64 *piEndBlock, - i64 *pnByte -){ - const unsigned char *zText = sqlite3_column_text(pStmt, iCol); - if( zText ){ - int i; - int iMul = 1; - i64 iVal = 0; - for(i=0; zText[i]>='0' && zText[i]<='9'; i++){ - iVal = iVal*10 + (zText[i] - '0'); - } - *piEndBlock = iVal; - while( zText[i]==' ' ) i++; - iVal = 0; - if( zText[i]=='-' ){ - i++; - iMul = -1; - } - for(/* no-op */; zText[i]>='0' && zText[i]<='9'; i++){ - iVal = iVal*10 + (zText[i] - '0'); +SQLITE_PRIVATE void sqlite3Fts3MIBufferFree(MatchinfoBuffer *p){ + if( p ){ + assert( p->aRef[0]==1 ); + p->aRef[0] = 0; + if( p->aRef[0]==0 && p->aRef[1]==0 && p->aRef[2]==0 ){ + sqlite3_free(p); } - *pnByte = (iVal * (i64)iMul); } } +/* +** End of MatchinfoBuffer code. +*************************************************************************/ + /* -** A segment of size nByte bytes has just been written to absolute level -** iAbsLevel. Promote any segments that should be promoted as a result. +** This function is used to help iterate through a position-list. A position +** list is a list of unique integers, sorted from smallest to largest. Each +** element of the list is represented by an FTS3 varint that takes the value +** of the difference between the current element and the previous one plus +** two. For example, to store the position-list: +** +** 4 9 113 +** +** the three varints: +** +** 6 7 106 +** +** are encoded. +** +** When this function is called, *pp points to the start of an element of +** the list. *piPos contains the value of the previous entry in the list. +** After it returns, *piPos contains the value of the next element of the +** list and *pp is advanced to the following varint. */ -static int fts3PromoteSegments( - Fts3Table *p, /* FTS table handle */ - sqlite3_int64 iAbsLevel, /* Absolute level just updated */ - sqlite3_int64 nByte /* Size of new segment at iAbsLevel */ -){ - int rc = SQLITE_OK; - sqlite3_stmt *pRange; - - rc = fts3SqlStmt(p, SQL_SELECT_LEVEL_RANGE2, &pRange, 0); +static void fts3GetDeltaPosition(char **pp, int *piPos){ + int iVal; + *pp += fts3GetVarint32(*pp, &iVal); + *piPos += (iVal-2); +} - if( rc==SQLITE_OK ){ - int bOk = 0; - i64 iLast = (iAbsLevel/FTS3_SEGDIR_MAXLEVEL + 1) * FTS3_SEGDIR_MAXLEVEL - 1; - i64 nLimit = (nByte*3)/2; +/* +** Helper function for fts3ExprIterate() (see below). +*/ +static int fts3ExprIterate2( + Fts3Expr *pExpr, /* Expression to iterate phrases of */ + int *piPhrase, /* Pointer to phrase counter */ + int (*x)(Fts3Expr*,int,void*), /* Callback function to invoke for phrases */ + void *pCtx /* Second argument to pass to callback */ +){ + int rc; /* Return code */ + int eType = pExpr->eType; /* Type of expression node pExpr */ - /* Loop through all entries in the %_segdir table corresponding to - ** segments in this index on levels greater than iAbsLevel. If there is - ** at least one such segment, and it is possible to determine that all - ** such segments are smaller than nLimit bytes in size, they will be - ** promoted to level iAbsLevel. */ - sqlite3_bind_int64(pRange, 1, iAbsLevel+1); - sqlite3_bind_int64(pRange, 2, iLast); - while( SQLITE_ROW==sqlite3_step(pRange) ){ - i64 nSize = 0, dummy; - fts3ReadEndBlockField(pRange, 2, &dummy, &nSize); - if( nSize<=0 || nSize>nLimit ){ - /* If nSize==0, then the %_segdir.end_block field does not not - ** contain a size value. This happens if it was written by an - ** old version of FTS. In this case it is not possible to determine - ** the size of the segment, and so segment promotion does not - ** take place. */ - bOk = 0; - break; - } - bOk = 1; + if( eType!=FTSQUERY_PHRASE ){ + assert( pExpr->pLeft && pExpr->pRight ); + rc = fts3ExprIterate2(pExpr->pLeft, piPhrase, x, pCtx); + if( rc==SQLITE_OK && eType!=FTSQUERY_NOT ){ + rc = fts3ExprIterate2(pExpr->pRight, piPhrase, x, pCtx); } - rc = sqlite3_reset(pRange); - - if( bOk ){ - int iIdx = 0; - sqlite3_stmt *pUpdate1 = 0; - sqlite3_stmt *pUpdate2 = 0; + }else{ + rc = x(pExpr, *piPhrase, pCtx); + (*piPhrase)++; + } + return rc; +} - if( rc==SQLITE_OK ){ - rc = fts3SqlStmt(p, SQL_UPDATE_LEVEL_IDX, &pUpdate1, 0); - } - if( rc==SQLITE_OK ){ - rc = fts3SqlStmt(p, SQL_UPDATE_LEVEL, &pUpdate2, 0); - } +/* +** Iterate through all phrase nodes in an FTS3 query, except those that +** are part of a sub-tree that is the right-hand-side of a NOT operator. +** For each phrase node found, the supplied callback function is invoked. +** +** If the callback function returns anything other than SQLITE_OK, +** the iteration is abandoned and the error code returned immediately. +** Otherwise, SQLITE_OK is returned after a callback has been made for +** all eligible phrase nodes. +*/ +static int fts3ExprIterate( + Fts3Expr *pExpr, /* Expression to iterate phrases of */ + int (*x)(Fts3Expr*,int,void*), /* Callback function to invoke for phrases */ + void *pCtx /* Second argument to pass to callback */ +){ + int iPhrase = 0; /* Variable used as the phrase counter */ + return fts3ExprIterate2(pExpr, &iPhrase, x, pCtx); +} - if( rc==SQLITE_OK ){ - /* Loop through all %_segdir entries for segments in this index with - ** levels equal to or greater than iAbsLevel. As each entry is visited, - ** updated it to set (level = -1) and (idx = N), where N is 0 for the - ** oldest segment in the range, 1 for the next oldest, and so on. - ** - ** In other words, move all segments being promoted to level -1, - ** setting the "idx" fields as appropriate to keep them in the same - ** order. The contents of level -1 (which is never used, except - ** transiently here), will be moved back to level iAbsLevel below. */ - sqlite3_bind_int64(pRange, 1, iAbsLevel); - while( SQLITE_ROW==sqlite3_step(pRange) ){ - sqlite3_bind_int(pUpdate1, 1, iIdx++); - sqlite3_bind_int(pUpdate1, 2, sqlite3_column_int(pRange, 0)); - sqlite3_bind_int(pUpdate1, 3, sqlite3_column_int(pRange, 1)); - sqlite3_step(pUpdate1); - rc = sqlite3_reset(pUpdate1); - if( rc!=SQLITE_OK ){ - sqlite3_reset(pRange); - break; - } - } - } - if( rc==SQLITE_OK ){ - rc = sqlite3_reset(pRange); - } +/* +** This is an fts3ExprIterate() callback used while loading the doclists +** for each phrase into Fts3Expr.aDoclist[]/nDoclist. See also +** fts3ExprLoadDoclists(). +*/ +static int fts3ExprLoadDoclistsCb(Fts3Expr *pExpr, int iPhrase, void *ctx){ + int rc = SQLITE_OK; + Fts3Phrase *pPhrase = pExpr->pPhrase; + LoadDoclistCtx *p = (LoadDoclistCtx *)ctx; - /* Move level -1 to level iAbsLevel */ - if( rc==SQLITE_OK ){ - sqlite3_bind_int64(pUpdate2, 1, iAbsLevel); - sqlite3_step(pUpdate2); - rc = sqlite3_reset(pUpdate2); - } - } - } + UNUSED_PARAMETER(iPhrase); + p->nPhrase++; + p->nToken += pPhrase->nToken; return rc; } /* -** Merge all level iLevel segments in the database into a single -** iLevel+1 segment. Or, if iLevel<0, merge all segments into a -** single segment with a level equal to the numerically largest level -** currently present in the database. +** Load the doclists for each phrase in the query associated with FTS3 cursor +** pCsr. ** -** If this function is called with iLevel<0, but there is only one -** segment in the database, SQLITE_DONE is returned immediately. -** Otherwise, if successful, SQLITE_OK is returned. If an error occurs, -** an SQLite error code is returned. +** If pnPhrase is not NULL, then *pnPhrase is set to the number of matchable +** phrases in the expression (all phrases except those directly or +** indirectly descended from the right-hand-side of a NOT operator). If +** pnToken is not NULL, then it is set to the number of tokens in all +** matchable phrases of the expression. */ -static int fts3SegmentMerge( - Fts3Table *p, - int iLangid, /* Language id to merge */ - int iIndex, /* Index in p->aIndex[] to merge */ - int iLevel /* Level to merge */ +static int fts3ExprLoadDoclists( + Fts3Cursor *pCsr, /* Fts3 cursor for current query */ + int *pnPhrase, /* OUT: Number of phrases in query */ + int *pnToken /* OUT: Number of tokens in query */ ){ - int rc; /* Return code */ - int iIdx = 0; /* Index of new segment */ - sqlite3_int64 iNewLevel = 0; /* Level/index to create new segment at */ - SegmentWriter *pWriter = 0; /* Used to write the new, merged, segment */ - Fts3SegFilter filter; /* Segment term filter condition */ - Fts3MultiSegReader csr; /* Cursor to iterate through level(s) */ - int bIgnoreEmpty = 0; /* True to ignore empty segments */ - i64 iMaxLevel = 0; /* Max level number for this index/langid */ - - assert( iLevel==FTS3_SEGCURSOR_ALL - || iLevel==FTS3_SEGCURSOR_PENDING - || iLevel>=0 - ); - assert( iLevel=0 && iIndexnIndex ); + int rc; /* Return Code */ + LoadDoclistCtx sCtx = {0,0,0}; /* Context for fts3ExprIterate() */ + sCtx.pCsr = pCsr; + rc = fts3ExprIterate(pCsr->pExpr, fts3ExprLoadDoclistsCb, (void *)&sCtx); + if( pnPhrase ) *pnPhrase = sCtx.nPhrase; + if( pnToken ) *pnToken = sCtx.nToken; + return rc; +} - rc = sqlite3Fts3SegReaderCursor(p, iLangid, iIndex, iLevel, 0, 0, 1, 0, &csr); - if( rc!=SQLITE_OK || csr.nSegment==0 ) goto finished; +static int fts3ExprPhraseCountCb(Fts3Expr *pExpr, int iPhrase, void *ctx){ + (*(int *)ctx)++; + pExpr->iPhrase = iPhrase; + return SQLITE_OK; +} +static int fts3ExprPhraseCount(Fts3Expr *pExpr){ + int nPhrase = 0; + (void)fts3ExprIterate(pExpr, fts3ExprPhraseCountCb, (void *)&nPhrase); + return nPhrase; +} - if( iLevel!=FTS3_SEGCURSOR_PENDING ){ - rc = fts3SegmentMaxLevel(p, iLangid, iIndex, &iMaxLevel); - if( rc!=SQLITE_OK ) goto finished; - } +/* +** Advance the position list iterator specified by the first two +** arguments so that it points to the first element with a value greater +** than or equal to parameter iNext. +*/ +static void fts3SnippetAdvance(char **ppIter, int *piIter, int iNext){ + char *pIter = *ppIter; + if( pIter ){ + int iIter = *piIter; - if( iLevel==FTS3_SEGCURSOR_ALL ){ - /* This call is to merge all segments in the database to a single - ** segment. The level of the new segment is equal to the numerically - ** greatest segment level currently present in the database for this - ** index. The idx of the new segment is always 0. */ - if( csr.nSegment==1 && 0==fts3SegReaderIsPending(csr.apSegment[0]) ){ - rc = SQLITE_DONE; - goto finished; + while( iIteriMaxLevel); + *piIter = iIter; + *ppIter = pIter; } - if( rc!=SQLITE_OK ) goto finished; +} - assert( csr.nSegment>0 ); - assert( iNewLevel>=getAbsoluteLevel(p, iLangid, iIndex, 0) ); - assert( iNewLeveliCurrent<0 ){ + /* The SnippetIter object has just been initialized. The first snippet + ** candidate always starts at offset 0 (even if this candidate has a + ** score of 0.0). + */ + pIter->iCurrent = 0; - rc = sqlite3Fts3SegReaderStart(p, &csr, &filter); - while( SQLITE_OK==rc ){ - rc = sqlite3Fts3SegReaderStep(p, &csr); - if( rc!=SQLITE_ROW ) break; - rc = fts3SegWriterAdd(p, &pWriter, 1, - csr.zTerm, csr.nTerm, csr.aDoclist, csr.nDoclist); - } - if( rc!=SQLITE_OK ) goto finished; - assert( pWriter || bIgnoreEmpty ); + /* Advance the 'head' iterator of each phrase to the first offset that + ** is greater than or equal to (iNext+nSnippet). + */ + for(i=0; inPhrase; i++){ + SnippetPhrase *pPhrase = &pIter->aPhrase[i]; + fts3SnippetAdvance(&pPhrase->pHead, &pPhrase->iHead, pIter->nSnippet); + } + }else{ + int iStart; + int iEnd = 0x7FFFFFFF; - if( iLevel!=FTS3_SEGCURSOR_PENDING ){ - rc = fts3DeleteSegdir( - p, iLangid, iIndex, iLevel, csr.apSegment, csr.nSegment - ); - if( rc!=SQLITE_OK ) goto finished; - } - if( pWriter ){ - rc = fts3SegWriterFlush(p, pWriter, iNewLevel, iIdx); - if( rc==SQLITE_OK ){ - if( iLevel==FTS3_SEGCURSOR_PENDING || iNewLevelnLeafData); + for(i=0; inPhrase; i++){ + SnippetPhrase *pPhrase = &pIter->aPhrase[i]; + if( pPhrase->pHead && pPhrase->iHeadiHead; } } - } - - finished: - fts3SegWriterFree(pWriter); - sqlite3Fts3SegReaderFinish(&csr); - return rc; -} - - -/* -** Flush the contents of pendingTerms to level 0 segments. -*/ -SQLITE_PRIVATE int sqlite3Fts3PendingTermsFlush(Fts3Table *p){ - int rc = SQLITE_OK; - int i; - - for(i=0; rc==SQLITE_OK && inIndex; i++){ - rc = fts3SegmentMerge(p, p->iPrevLangid, i, FTS3_SEGCURSOR_PENDING); - if( rc==SQLITE_DONE ) rc = SQLITE_OK; - } - sqlite3Fts3PendingTermsClear(p); + if( iEnd==0x7FFFFFFF ){ + return 1; + } - /* Determine the auto-incr-merge setting if unknown. If enabled, - ** estimate the number of leaf blocks of content to be written - */ - if( rc==SQLITE_OK && p->bHasStat - && p->nAutoincrmerge==0xff && p->nLeafAdd>0 - ){ - sqlite3_stmt *pStmt = 0; - rc = fts3SqlStmt(p, SQL_SELECT_STAT, &pStmt, 0); - if( rc==SQLITE_OK ){ - sqlite3_bind_int(pStmt, 1, FTS_STAT_AUTOINCRMERGE); - rc = sqlite3_step(pStmt); - if( rc==SQLITE_ROW ){ - p->nAutoincrmerge = sqlite3_column_int(pStmt, 0); - if( p->nAutoincrmerge==1 ) p->nAutoincrmerge = 8; - }else if( rc==SQLITE_DONE ){ - p->nAutoincrmerge = 0; - } - rc = sqlite3_reset(pStmt); + pIter->iCurrent = iStart = iEnd - pIter->nSnippet + 1; + for(i=0; inPhrase; i++){ + SnippetPhrase *pPhrase = &pIter->aPhrase[i]; + fts3SnippetAdvance(&pPhrase->pHead, &pPhrase->iHead, iEnd+1); + fts3SnippetAdvance(&pPhrase->pTail, &pPhrase->iTail, iStart); } } - return rc; -} -/* -** Encode N integers as varints into a blob. -*/ -static void fts3EncodeIntArray( - int N, /* The number of integers to encode */ - u32 *a, /* The integer values */ - char *zBuf, /* Write the BLOB here */ - int *pNBuf /* Write number of bytes if zBuf[] used here */ -){ - int i, j; - for(i=j=0; iiCurrent; /* First token of snippet */ + int iScore = 0; /* Score of this snippet */ + int i; /* Loop counter */ + u64 mCover = 0; /* Mask of phrases covered by this snippet */ + u64 mHighlight = 0; /* Mask of tokens to highlight in snippet */ + + for(i=0; inPhrase; i++){ + SnippetPhrase *pPhrase = &pIter->aPhrase[i]; + if( pPhrase->pTail ){ + char *pCsr = pPhrase->pTail; + int iCsr = pPhrase->iTail; + + while( iCsr<(iStart+pIter->nSnippet) && iCsr>=iStart ){ + int j; + u64 mPhrase = (u64)1 << (i%64); + u64 mPos = (u64)1 << (iCsr - iStart); + assert( iCsr>=iStart && (iCsr - iStart)<=64 ); + assert( i>=0 ); + if( (mCover|mCovered)&mPhrase ){ + iScore++; + }else{ + iScore += 1000; + } + mCover |= mPhrase; + + for(j=0; jnToken; j++){ + mHighlight |= (mPos>>j); + } + + if( 0==(*pCsr & 0x0FE) ) break; + fts3GetDeltaPosition(&pCsr, &iCsr); + } + } } + + /* Set the output variables before returning. */ + *piToken = iStart; + *piScore = iScore; + *pmCover = mCover; + *pmHighlight = mHighlight; } /* -** Insert the sizes (in tokens) for each column of the document -** with docid equal to p->iPrevDocid. The sizes are encoded as -** a blob of varints. +** This function is an fts3ExprIterate() callback used by fts3BestSnippet(). +** Each invocation populates an element of the SnippetIter.aPhrase[] array. */ -static void fts3InsertDocsize( - int *pRC, /* Result code */ - Fts3Table *p, /* Table into which to insert */ - u32 *aSz /* Sizes of each column, in tokens */ -){ - char *pBlob; /* The BLOB encoding of the document size */ - int nBlob; /* Number of bytes in the BLOB */ - sqlite3_stmt *pStmt; /* Statement used to insert the encoding */ - int rc; /* Result code from subfunctions */ +static int fts3SnippetFindPositions(Fts3Expr *pExpr, int iPhrase, void *ctx){ + SnippetIter *p = (SnippetIter *)ctx; + SnippetPhrase *pPhrase = &p->aPhrase[iPhrase]; + char *pCsr; + int rc; - if( *pRC ) return; - pBlob = sqlite3_malloc( 10*p->nColumn ); - if( pBlob==0 ){ - *pRC = SQLITE_NOMEM; - return; - } - fts3EncodeIntArray(p->nColumn, aSz, pBlob, &nBlob); - rc = fts3SqlStmt(p, SQL_REPLACE_DOCSIZE, &pStmt, 0); - if( rc ){ - sqlite3_free(pBlob); - *pRC = rc; - return; + pPhrase->nToken = pExpr->pPhrase->nToken; + rc = sqlite3Fts3EvalPhrasePoslist(p->pCsr, pExpr, p->iCol, &pCsr); + assert( rc==SQLITE_OK || pCsr==0 ); + if( pCsr ){ + int iFirst = 0; + pPhrase->pList = pCsr; + fts3GetDeltaPosition(&pCsr, &iFirst); + if( iFirst<0 ){ + rc = FTS_CORRUPT_VTAB; + }else{ + pPhrase->pHead = pCsr; + pPhrase->pTail = pCsr; + pPhrase->iHead = iFirst; + pPhrase->iTail = iFirst; + } + }else{ + assert( rc!=SQLITE_OK || ( + pPhrase->pList==0 && pPhrase->pHead==0 && pPhrase->pTail==0 + )); } - sqlite3_bind_int64(pStmt, 1, p->iPrevDocid); - sqlite3_bind_blob(pStmt, 2, pBlob, nBlob, sqlite3_free); - sqlite3_step(pStmt); - *pRC = sqlite3_reset(pStmt); + + return rc; } /* -** Record 0 of the %_stat table contains a blob consisting of N varints, -** where N is the number of user defined columns in the fts3 table plus -** two. If nCol is the number of user defined columns, then values of the -** varints are set as follows: -** -** Varint 0: Total number of rows in the table. +** Select the fragment of text consisting of nFragment contiguous tokens +** from column iCol that represent the "best" snippet. The best snippet +** is the snippet with the highest score, where scores are calculated +** by adding: ** -** Varint 1..nCol: For each column, the total number of tokens stored in -** the column for all rows of the table. +** (a) +1 point for each occurrence of a matchable phrase in the snippet. ** -** Varint 1+nCol: The total size, in bytes, of all text values in all -** columns of all rows of the table. +** (b) +1000 points for the first occurrence of each matchable phrase in +** the snippet for which the corresponding mCovered bit is not set. ** +** The selected snippet parameters are stored in structure *pFragment before +** returning. The score of the selected snippet is stored in *piScore +** before returning. */ -static void fts3UpdateDocTotals( - int *pRC, /* The result code */ - Fts3Table *p, /* Table being updated */ - u32 *aSzIns, /* Size increases */ - u32 *aSzDel, /* Size decreases */ - int nChng /* Change in the number of documents */ +static int fts3BestSnippet( + int nSnippet, /* Desired snippet length */ + Fts3Cursor *pCsr, /* Cursor to create snippet for */ + int iCol, /* Index of column to create snippet from */ + u64 mCovered, /* Mask of phrases already covered */ + u64 *pmSeen, /* IN/OUT: Mask of phrases seen */ + SnippetFragment *pFragment, /* OUT: Best snippet found */ + int *piScore /* OUT: Score of snippet pFragment */ ){ - char *pBlob; /* Storage for BLOB written into %_stat */ - int nBlob; /* Size of BLOB written into %_stat */ - u32 *a; /* Array of integers that becomes the BLOB */ - sqlite3_stmt *pStmt; /* Statement for reading and writing */ - int i; /* Loop counter */ - int rc; /* Result code from subfunctions */ + int rc; /* Return Code */ + int nList; /* Number of phrases in expression */ + SnippetIter sIter; /* Iterates through snippet candidates */ + sqlite3_int64 nByte; /* Number of bytes of space to allocate */ + int iBestScore = -1; /* Best snippet score found so far */ + int i; /* Loop counter */ - const int nStat = p->nColumn+2; + memset(&sIter, 0, sizeof(sIter)); - if( *pRC ) return; - a = sqlite3_malloc( (sizeof(u32)+10)*nStat ); - if( a==0 ){ - *pRC = SQLITE_NOMEM; - return; - } - pBlob = (char*)&a[nStat]; - rc = fts3SqlStmt(p, SQL_SELECT_STAT, &pStmt, 0); - if( rc ){ - sqlite3_free(a); - *pRC = rc; - return; - } - sqlite3_bind_int(pStmt, 1, FTS_STAT_DOCTOTAL); - if( sqlite3_step(pStmt)==SQLITE_ROW ){ - fts3DecodeIntArray(nStat, a, - sqlite3_column_blob(pStmt, 0), - sqlite3_column_bytes(pStmt, 0)); - }else{ - memset(a, 0, sizeof(u32)*(nStat) ); - } - rc = sqlite3_reset(pStmt); + /* Iterate through the phrases in the expression to count them. The same + ** callback makes sure the doclists are loaded for each phrase. + */ + rc = fts3ExprLoadDoclists(pCsr, &nList, 0); if( rc!=SQLITE_OK ){ - sqlite3_free(a); - *pRC = rc; - return; + return rc; } - if( nChng<0 && a[0]<(u32)(-nChng) ){ - a[0] = 0; - }else{ - a[0] += nChng; + + /* Now that it is known how many phrases there are, allocate and zero + ** the required space using malloc(). + */ + nByte = sizeof(SnippetPhrase) * nList; + sIter.aPhrase = (SnippetPhrase *)sqlite3_malloc64(nByte); + if( !sIter.aPhrase ){ + return SQLITE_NOMEM; } - for(i=0; inColumn+1; i++){ - u32 x = a[i+1]; - if( x+aSzIns[i] < aSzDel[i] ){ - x = 0; - }else{ - x = x + aSzIns[i] - aSzDel[i]; + memset(sIter.aPhrase, 0, nByte); + + /* Initialize the contents of the SnippetIter object. Then iterate through + ** the set of phrases in the expression to populate the aPhrase[] array. + */ + sIter.pCsr = pCsr; + sIter.iCol = iCol; + sIter.nSnippet = nSnippet; + sIter.nPhrase = nList; + sIter.iCurrent = -1; + rc = fts3ExprIterate(pCsr->pExpr, fts3SnippetFindPositions, (void*)&sIter); + if( rc==SQLITE_OK ){ + + /* Set the *pmSeen output variable. */ + for(i=0; iiCol = iCol; + while( !fts3SnippetNextCandidate(&sIter) ){ + int iPos; + int iScore; + u64 mCover; + u64 mHighlite; + fts3SnippetDetails(&sIter, mCovered, &iPos, &iScore, &mCover,&mHighlite); + assert( iScore>=0 ); + if( iScore>iBestScore ){ + pFragment->iPos = iPos; + pFragment->hlmask = mHighlite; + pFragment->covered = mCover; + iBestScore = iScore; + } + } + + *piScore = iBestScore; } - sqlite3_bind_int(pStmt, 1, FTS_STAT_DOCTOTAL); - sqlite3_bind_blob(pStmt, 2, pBlob, nBlob, SQLITE_STATIC); - sqlite3_step(pStmt); - *pRC = sqlite3_reset(pStmt); - sqlite3_bind_null(pStmt, 2); - sqlite3_free(a); + sqlite3_free(sIter.aPhrase); + return rc; } + /* -** Merge the entire database so that there is one segment for each -** iIndex/iLangid combination. +** Append a string to the string-buffer passed as the first argument. +** +** If nAppend is negative, then the length of the string zAppend is +** determined using strlen(). */ -static int fts3DoOptimize(Fts3Table *p, int bReturnDone){ - int bSeenDone = 0; - int rc; - sqlite3_stmt *pAllLangid = 0; +static int fts3StringAppend( + StrBuffer *pStr, /* Buffer to append to */ + const char *zAppend, /* Pointer to data to append to buffer */ + int nAppend /* Size of zAppend in bytes (or -1) */ +){ + if( nAppend<0 ){ + nAppend = (int)strlen(zAppend); + } - rc = fts3SqlStmt(p, SQL_SELECT_ALL_LANGID, &pAllLangid, 0); - if( rc==SQLITE_OK ){ - int rc2; - sqlite3_bind_int(pAllLangid, 1, p->iPrevLangid); - sqlite3_bind_int(pAllLangid, 2, p->nIndex); - while( sqlite3_step(pAllLangid)==SQLITE_ROW ){ - int i; - int iLangid = sqlite3_column_int(pAllLangid, 0); - for(i=0; rc==SQLITE_OK && inIndex; i++){ - rc = fts3SegmentMerge(p, iLangid, i, FTS3_SEGCURSOR_ALL); - if( rc==SQLITE_DONE ){ - bSeenDone = 1; - rc = SQLITE_OK; - } - } + /* If there is insufficient space allocated at StrBuffer.z, use realloc() + ** to grow the buffer until so that it is big enough to accomadate the + ** appended data. + */ + if( pStr->n+nAppend+1>=pStr->nAlloc ){ + sqlite3_int64 nAlloc = pStr->nAlloc+(sqlite3_int64)nAppend+100; + char *zNew = sqlite3_realloc64(pStr->z, nAlloc); + if( !zNew ){ + return SQLITE_NOMEM; } - rc2 = sqlite3_reset(pAllLangid); - if( rc==SQLITE_OK ) rc = rc2; + pStr->z = zNew; + pStr->nAlloc = nAlloc; } + assert( pStr->z!=0 && (pStr->nAlloc >= pStr->n+nAppend+1) ); - sqlite3Fts3SegmentsClose(p); - sqlite3Fts3PendingTermsClear(p); + /* Append the data to the string buffer. */ + memcpy(&pStr->z[pStr->n], zAppend, nAppend); + pStr->n += nAppend; + pStr->z[pStr->n] = '\0'; - return (rc==SQLITE_OK && bReturnDone && bSeenDone) ? SQLITE_DONE : rc; + return SQLITE_OK; } /* -** This function is called when the user executes the following statement: +** The fts3BestSnippet() function often selects snippets that end with a +** query term. That is, the final term of the snippet is always a term +** that requires highlighting. For example, if 'X' is a highlighted term +** and '.' is a non-highlighted term, BestSnippet() may select: ** -** INSERT INTO () VALUES('rebuild'); +** ........X.....X ** -** The entire FTS index is discarded and rebuilt. If the table is one -** created using the content=xxx option, then the new index is based on -** the current contents of the xxx table. Otherwise, it is rebuilt based -** on the contents of the %_content table. +** This function "shifts" the beginning of the snippet forward in the +** document so that there are approximately the same number of +** non-highlighted terms to the right of the final highlighted term as there +** are to the left of the first highlighted term. For example, to this: +** +** ....X.....X.... +** +** This is done as part of extracting the snippet text, not when selecting +** the snippet. Snippet selection is done based on doclists only, so there +** is no way for fts3BestSnippet() to know whether or not the document +** actually contains terms that follow the final highlighted term. */ -static int fts3DoRebuild(Fts3Table *p){ - int rc; /* Return Code */ +static int fts3SnippetShift( + Fts3Table *pTab, /* FTS3 table snippet comes from */ + int iLangid, /* Language id to use in tokenizing */ + int nSnippet, /* Number of tokens desired for snippet */ + const char *zDoc, /* Document text to extract snippet from */ + int nDoc, /* Size of buffer zDoc in bytes */ + int *piPos, /* IN/OUT: First token of snippet */ + u64 *pHlmask /* IN/OUT: Mask of tokens to highlight */ +){ + u64 hlmask = *pHlmask; /* Local copy of initial highlight-mask */ - rc = fts3DeleteAll(p, 0); - if( rc==SQLITE_OK ){ - u32 *aSz = 0; - u32 *aSzIns = 0; - u32 *aSzDel = 0; - sqlite3_stmt *pStmt = 0; - int nEntry = 0; + if( hlmask ){ + int nLeft; /* Tokens to the left of first highlight */ + int nRight; /* Tokens to the right of last highlight */ + int nDesired; /* Ideal number of tokens to shift forward */ - /* Compose and prepare an SQL statement to loop through the content table */ - char *zSql = sqlite3_mprintf("SELECT %s" , p->zReadExprlist); - if( !zSql ){ - rc = SQLITE_NOMEM; - }else{ - rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); - sqlite3_free(zSql); - } + for(nLeft=0; !(hlmask & ((u64)1 << nLeft)); nLeft++); + for(nRight=0; !(hlmask & ((u64)1 << (nSnippet-1-nRight))); nRight++); + assert( (nSnippet-1-nRight)<=63 && (nSnippet-1-nRight)>=0 ); + nDesired = (nLeft-nRight)/2; - if( rc==SQLITE_OK ){ - int nByte = sizeof(u32) * (p->nColumn+1)*3; - aSz = (u32 *)sqlite3_malloc(nByte); - if( aSz==0 ){ - rc = SQLITE_NOMEM; - }else{ - memset(aSz, 0, nByte); - aSzIns = &aSz[p->nColumn+1]; - aSzDel = &aSzIns[p->nColumn+1]; - } - } + /* Ideally, the start of the snippet should be pushed forward in the + ** document nDesired tokens. This block checks if there are actually + ** nDesired tokens to the right of the snippet. If so, *piPos and + ** *pHlMask are updated to shift the snippet nDesired tokens to the + ** right. Otherwise, the snippet is shifted by the number of tokens + ** available. + */ + if( nDesired>0 ){ + int nShift; /* Number of tokens to shift snippet by */ + int iCurrent = 0; /* Token counter */ + int rc; /* Return Code */ + sqlite3_tokenizer_module *pMod; + sqlite3_tokenizer_cursor *pC; + pMod = (sqlite3_tokenizer_module *)pTab->pTokenizer->pModule; - while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ - int iCol; - int iLangid = langidFromSelect(p, pStmt); - rc = fts3PendingTermsDocid(p, 0, iLangid, sqlite3_column_int64(pStmt, 0)); - memset(aSz, 0, sizeof(aSz[0]) * (p->nColumn+1)); - for(iCol=0; rc==SQLITE_OK && iColnColumn; iCol++){ - if( p->abNotindexed[iCol]==0 ){ - const char *z = (const char *) sqlite3_column_text(pStmt, iCol+1); - rc = fts3PendingTermsAdd(p, iLangid, z, iCol, &aSz[iCol]); - aSz[p->nColumn] += sqlite3_column_bytes(pStmt, iCol+1); - } - } - if( p->bHasDocsize ){ - fts3InsertDocsize(&rc, p, aSz); - } + /* Open a cursor on zDoc/nDoc. Check if there are (nSnippet+nDesired) + ** or more tokens in zDoc/nDoc. + */ + rc = sqlite3Fts3OpenTokenizer(pTab->pTokenizer, iLangid, zDoc, nDoc, &pC); if( rc!=SQLITE_OK ){ - sqlite3_finalize(pStmt); - pStmt = 0; - }else{ - nEntry++; - for(iCol=0; iCol<=p->nColumn; iCol++){ - aSzIns[iCol] += aSz[iCol]; - } + return rc; } - } - if( p->bFts4 ){ - fts3UpdateDocTotals(&rc, p, aSzIns, aSzDel, nEntry); - } - sqlite3_free(aSz); + while( rc==SQLITE_OK && iCurrent<(nSnippet+nDesired) ){ + const char *ZDUMMY; int DUMMY1 = 0, DUMMY2 = 0, DUMMY3 = 0; + rc = pMod->xNext(pC, &ZDUMMY, &DUMMY1, &DUMMY2, &DUMMY3, &iCurrent); + } + pMod->xClose(pC); + if( rc!=SQLITE_OK && rc!=SQLITE_DONE ){ return rc; } - if( pStmt ){ - int rc2 = sqlite3_finalize(pStmt); - if( rc==SQLITE_OK ){ - rc = rc2; + nShift = (rc==SQLITE_DONE)+iCurrent-nSnippet; + assert( nShift<=nDesired ); + if( nShift>0 ){ + *piPos += nShift; + *pHlmask = hlmask >> nShift; } } } - - return rc; + return SQLITE_OK; } - /* -** This function opens a cursor used to read the input data for an -** incremental merge operation. Specifically, it opens a cursor to scan -** the oldest nSeg segments (idx=0 through idx=(nSeg-1)) in absolute -** level iAbsLevel. +** Extract the snippet text for fragment pFragment from cursor pCsr and +** append it to string buffer pOut. */ -static int fts3IncrmergeCsr( - Fts3Table *p, /* FTS3 table handle */ - sqlite3_int64 iAbsLevel, /* Absolute level to open */ - int nSeg, /* Number of segments to merge */ - Fts3MultiSegReader *pCsr /* Cursor object to populate */ +static int fts3SnippetText( + Fts3Cursor *pCsr, /* FTS3 Cursor */ + SnippetFragment *pFragment, /* Snippet to extract */ + int iFragment, /* Fragment number */ + int isLast, /* True for final fragment in snippet */ + int nSnippet, /* Number of tokens in extracted snippet */ + const char *zOpen, /* String inserted before highlighted term */ + const char *zClose, /* String inserted after highlighted term */ + const char *zEllipsis, /* String inserted between snippets */ + StrBuffer *pOut /* Write output here */ ){ - int rc; /* Return Code */ - sqlite3_stmt *pStmt = 0; /* Statement used to read %_segdir entry */ - int nByte; /* Bytes allocated at pCsr->apSegment[] */ - - /* Allocate space for the Fts3MultiSegReader.aCsr[] array */ - memset(pCsr, 0, sizeof(*pCsr)); - nByte = sizeof(Fts3SegReader *) * nSeg; - pCsr->apSegment = (Fts3SegReader **)sqlite3_malloc(nByte); - - if( pCsr->apSegment==0 ){ - rc = SQLITE_NOMEM; - }else{ - memset(pCsr->apSegment, 0, nByte); - rc = fts3SqlStmt(p, SQL_SELECT_LEVEL, &pStmt, 0); - } - if( rc==SQLITE_OK ){ - int i; - int rc2; - sqlite3_bind_int64(pStmt, 1, iAbsLevel); - assert( pCsr->nSegment==0 ); - for(i=0; rc==SQLITE_OK && sqlite3_step(pStmt)==SQLITE_ROW && iapSegment[i] - ); - pCsr->nSegment++; + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + int rc; /* Return code */ + const char *zDoc; /* Document text to extract snippet from */ + int nDoc; /* Size of zDoc in bytes */ + int iCurrent = 0; /* Current token number of document */ + int iEnd = 0; /* Byte offset of end of current token */ + int isShiftDone = 0; /* True after snippet is shifted */ + int iPos = pFragment->iPos; /* First token of snippet */ + u64 hlmask = pFragment->hlmask; /* Highlight-mask for snippet */ + int iCol = pFragment->iCol+1; /* Query column to extract text from */ + sqlite3_tokenizer_module *pMod; /* Tokenizer module methods object */ + sqlite3_tokenizer_cursor *pC; /* Tokenizer cursor open on zDoc/nDoc */ + + zDoc = (const char *)sqlite3_column_text(pCsr->pStmt, iCol); + if( zDoc==0 ){ + if( sqlite3_column_type(pCsr->pStmt, iCol)!=SQLITE_NULL ){ + return SQLITE_NOMEM; } - rc2 = sqlite3_reset(pStmt); - if( rc==SQLITE_OK ) rc = rc2; + return SQLITE_OK; } + nDoc = sqlite3_column_bytes(pCsr->pStmt, iCol); - return rc; -} - -typedef struct IncrmergeWriter IncrmergeWriter; -typedef struct NodeWriter NodeWriter; -typedef struct Blob Blob; -typedef struct NodeReader NodeReader; - -/* -** An instance of the following structure is used as a dynamic buffer -** to build up nodes or other blobs of data in. -** -** The function blobGrowBuffer() is used to extend the allocation. -*/ -struct Blob { - char *a; /* Pointer to allocation */ - int n; /* Number of valid bytes of data in a[] */ - int nAlloc; /* Allocated size of a[] (nAlloc>=n) */ -}; - -/* -** This structure is used to build up buffers containing segment b-tree -** nodes (blocks). -*/ -struct NodeWriter { - sqlite3_int64 iBlock; /* Current block id */ - Blob key; /* Last key written to the current block */ - Blob block; /* Current block image */ -}; - -/* -** An object of this type contains the state required to create or append -** to an appendable b-tree segment. -*/ -struct IncrmergeWriter { - int nLeafEst; /* Space allocated for leaf blocks */ - int nWork; /* Number of leaf pages flushed */ - sqlite3_int64 iAbsLevel; /* Absolute level of input segments */ - int iIdx; /* Index of *output* segment in iAbsLevel+1 */ - sqlite3_int64 iStart; /* Block number of first allocated block */ - sqlite3_int64 iEnd; /* Block number of last allocated block */ - sqlite3_int64 nLeafData; /* Bytes of leaf page data so far */ - u8 bNoLeafData; /* If true, store 0 for segment size */ - NodeWriter aNodeWriter[FTS_MAX_APPENDABLE_HEIGHT]; -}; - -/* -** An object of the following type is used to read data from a single -** FTS segment node. See the following functions: -** -** nodeReaderInit() -** nodeReaderNext() -** nodeReaderRelease() -*/ -struct NodeReader { - const char *aNode; - int nNode; - int iOff; /* Current offset within aNode[] */ + /* Open a token cursor on the document. */ + pMod = (sqlite3_tokenizer_module *)pTab->pTokenizer->pModule; + rc = sqlite3Fts3OpenTokenizer(pTab->pTokenizer, pCsr->iLangid, zDoc,nDoc,&pC); + if( rc!=SQLITE_OK ){ + return rc; + } - /* Output variables. Containing the current node entry. */ - sqlite3_int64 iChild; /* Pointer to child node */ - Blob term; /* Current term */ - const char *aDoclist; /* Pointer to doclist */ - int nDoclist; /* Size of doclist in bytes */ -}; + while( rc==SQLITE_OK ){ + const char *ZDUMMY; /* Dummy argument used with tokenizer */ + int DUMMY1 = -1; /* Dummy argument used with tokenizer */ + int iBegin = 0; /* Offset in zDoc of start of token */ + int iFin = 0; /* Offset in zDoc of end of token */ + int isHighlight = 0; /* True for highlighted terms */ -/* -** If *pRc is not SQLITE_OK when this function is called, it is a no-op. -** Otherwise, if the allocation at pBlob->a is not already at least nMin -** bytes in size, extend (realloc) it to be so. -** -** If an OOM error occurs, set *pRc to SQLITE_NOMEM and leave pBlob->a -** unmodified. Otherwise, if the allocation succeeds, update pBlob->nAlloc -** to reflect the new size of the pBlob->a[] buffer. -*/ -static void blobGrowBuffer(Blob *pBlob, int nMin, int *pRc){ - if( *pRc==SQLITE_OK && nMin>pBlob->nAlloc ){ - int nAlloc = nMin; - char *a = (char *)sqlite3_realloc(pBlob->a, nAlloc); - if( a ){ - pBlob->nAlloc = nAlloc; - pBlob->a = a; - }else{ - *pRc = SQLITE_NOMEM; + /* Variable DUMMY1 is initialized to a negative value above. Elsewhere + ** in the FTS code the variable that the third argument to xNext points to + ** is initialized to zero before the first (*but not necessarily + ** subsequent*) call to xNext(). This is done for a particular application + ** that needs to know whether or not the tokenizer is being used for + ** snippet generation or for some other purpose. + ** + ** Extreme care is required when writing code to depend on this + ** initialization. It is not a documented part of the tokenizer interface. + ** If a tokenizer is used directly by any code outside of FTS, this + ** convention might not be respected. */ + rc = pMod->xNext(pC, &ZDUMMY, &DUMMY1, &iBegin, &iFin, &iCurrent); + if( rc!=SQLITE_OK ){ + if( rc==SQLITE_DONE ){ + /* Special case - the last token of the snippet is also the last token + ** of the column. Append any punctuation that occurred between the end + ** of the previous token and the end of the document to the output. + ** Then break out of the loop. */ + rc = fts3StringAppend(pOut, &zDoc[iEnd], -1); + } + break; } - } -} + if( iCurrentaNode to -** NULL to indicate EOF. Otherwise, populate the NodeReader structure output -** variables for the new entry. -*/ -static int nodeReaderNext(NodeReader *p){ - int bFirst = (p->term.n==0); /* True for first term on the node */ - int nPrefix = 0; /* Bytes to copy from previous term */ - int nSuffix = 0; /* Bytes to append to the prefix */ - int rc = SQLITE_OK; /* Return code */ + if( !isShiftDone ){ + int n = nDoc - iBegin; + rc = fts3SnippetShift( + pTab, pCsr->iLangid, nSnippet, &zDoc[iBegin], n, &iPos, &hlmask + ); + isShiftDone = 1; - assert( p->aNode ); - if( p->iChild && bFirst==0 ) p->iChild++; - if( p->iOff>=p->nNode ){ - /* EOF */ - p->aNode = 0; - }else{ - if( bFirst==0 ){ - p->iOff += fts3GetVarint32(&p->aNode[p->iOff], &nPrefix); + /* Now that the shift has been done, check if the initial "..." are + ** required. They are required if (a) this is not the first fragment, + ** or (b) this fragment does not begin at position 0 of its column. + */ + if( rc==SQLITE_OK ){ + if( iPos>0 || iFragment>0 ){ + rc = fts3StringAppend(pOut, zEllipsis, -1); + }else if( iBegin ){ + rc = fts3StringAppend(pOut, zDoc, iBegin); + } + } + if( rc!=SQLITE_OK || iCurrentiOff += fts3GetVarint32(&p->aNode[p->iOff], &nSuffix); - blobGrowBuffer(&p->term, nPrefix+nSuffix, &rc); - if( rc==SQLITE_OK ){ - memcpy(&p->term.a[nPrefix], &p->aNode[p->iOff], nSuffix); - p->term.n = nPrefix+nSuffix; - p->iOff += nSuffix; - if( p->iChild==0 ){ - p->iOff += fts3GetVarint32(&p->aNode[p->iOff], &p->nDoclist); - p->aDoclist = &p->aNode[p->iOff]; - p->iOff += p->nDoclist; + if( iCurrent>=(iPos+nSnippet) ){ + if( isLast ){ + rc = fts3StringAppend(pOut, zEllipsis, -1); } + break; } - } - assert( p->iOff<=p->nNode ); + /* Set isHighlight to true if this term should be highlighted. */ + isHighlight = (hlmask & ((u64)1 << (iCurrent-iPos)))!=0; + + if( iCurrent>iPos ) rc = fts3StringAppend(pOut, &zDoc[iEnd], iBegin-iEnd); + if( rc==SQLITE_OK && isHighlight ) rc = fts3StringAppend(pOut, zOpen, -1); + if( rc==SQLITE_OK ) rc = fts3StringAppend(pOut, &zDoc[iBegin], iFin-iBegin); + if( rc==SQLITE_OK && isHighlight ) rc = fts3StringAppend(pOut, zClose, -1); + + iEnd = iFin; + } + pMod->xClose(pC); return rc; } -/* -** Release all dynamic resources held by node-reader object *p. -*/ -static void nodeReaderRelease(NodeReader *p){ - sqlite3_free(p->term.a); -} /* -** Initialize a node-reader object to read the node in buffer aNode/nNode. +** This function is used to count the entries in a column-list (a +** delta-encoded list of term offsets within a single column of a single +** row). When this function is called, *ppCollist should point to the +** beginning of the first varint in the column-list (the varint that +** contains the position of the first matching term in the column data). +** Before returning, *ppCollist is set to point to the first byte after +** the last varint in the column-list (either the 0x00 signifying the end +** of the position-list, or the 0x01 that precedes the column number of +** the next column in the position-list). ** -** If successful, SQLITE_OK is returned and the NodeReader object set to -** point to the first entry on the node (if any). Otherwise, an SQLite -** error code is returned. +** The number of elements in the column-list is returned. */ -static int nodeReaderInit(NodeReader *p, const char *aNode, int nNode){ - memset(p, 0, sizeof(NodeReader)); - p->aNode = aNode; - p->nNode = nNode; +static int fts3ColumnlistCount(char **ppCollist){ + char *pEnd = *ppCollist; + char c = 0; + int nEntry = 0; - /* Figure out if this is a leaf or an internal node. */ - if( p->aNode[0] ){ - /* An internal node. */ - p->iOff = 1 + sqlite3Fts3GetVarint(&p->aNode[1], &p->iChild); - }else{ - p->iOff = 1; + /* A column-list is terminated by either a 0x01 or 0x00. */ + while( 0xFE & (*pEnd | c) ){ + c = *pEnd++ & 0x80; + if( !c ) nEntry++; } - return nodeReaderNext(p); + *ppCollist = pEnd; + return nEntry; } /* -** This function is called while writing an FTS segment each time a leaf o -** node is finished and written to disk. The key (zTerm/nTerm) is guaranteed -** to be greater than the largest key on the node just written, but smaller -** than or equal to the first key that will be written to the next leaf -** node. -** -** The block id of the leaf node just written to disk may be found in -** (pWriter->aNodeWriter[0].iBlock) when this function is called. +** This function gathers 'y' or 'b' data for a single phrase. */ -static int fts3IncrmergePush( - Fts3Table *p, /* Fts3 table handle */ - IncrmergeWriter *pWriter, /* Writer object */ - const char *zTerm, /* Term to write to internal node */ - int nTerm /* Bytes at zTerm */ +static int fts3ExprLHits( + Fts3Expr *pExpr, /* Phrase expression node */ + MatchInfo *p /* Matchinfo context */ ){ - sqlite3_int64 iPtr = pWriter->aNodeWriter[0].iBlock; - int iLayer; - - assert( nTerm>0 ); - for(iLayer=1; ALWAYS(iLayeraNodeWriter[iLayer]; - int rc = SQLITE_OK; - int nPrefix; - int nSuffix; - int nSpace; - - /* Figure out how much space the key will consume if it is written to - ** the current node of layer iLayer. Due to the prefix compression, - ** the space required changes depending on which node the key is to - ** be added to. */ - nPrefix = fts3PrefixCompress(pNode->key.a, pNode->key.n, zTerm, nTerm); - nSuffix = nTerm - nPrefix; - nSpace = sqlite3Fts3VarintLen(nPrefix); - nSpace += sqlite3Fts3VarintLen(nSuffix) + nSuffix; + Fts3Table *pTab = (Fts3Table *)p->pCursor->base.pVtab; + int iStart; + Fts3Phrase *pPhrase = pExpr->pPhrase; + char *pIter = pPhrase->doclist.pList; + int iCol = 0; - if( pNode->key.n==0 || (pNode->block.n + nSpace)<=p->nNodeSize ){ - /* If the current node of layer iLayer contains zero keys, or if adding - ** the key to it will not cause it to grow to larger than nNodeSize - ** bytes in size, write the key here. */ + assert( p->flag==FTS3_MATCHINFO_LHITS_BM || p->flag==FTS3_MATCHINFO_LHITS ); + if( p->flag==FTS3_MATCHINFO_LHITS ){ + iStart = pExpr->iPhrase * p->nCol; + }else{ + iStart = pExpr->iPhrase * ((p->nCol + 31) / 32); + } - Blob *pBlk = &pNode->block; - if( pBlk->n==0 ){ - blobGrowBuffer(pBlk, p->nNodeSize, &rc); - if( rc==SQLITE_OK ){ - pBlk->a[0] = (char)iLayer; - pBlk->n = 1 + sqlite3Fts3PutVarint(&pBlk->a[1], iPtr); - } + if( pIter ) while( 1 ){ + int nHit = fts3ColumnlistCount(&pIter); + if( (pPhrase->iColumn>=pTab->nColumn || pPhrase->iColumn==iCol) ){ + if( p->flag==FTS3_MATCHINFO_LHITS ){ + p->aMatchinfo[iStart + iCol] = (u32)nHit; + }else if( nHit ){ + p->aMatchinfo[iStart + (iCol+1)/32] |= (1 << (iCol&0x1F)); } - blobGrowBuffer(pBlk, pBlk->n + nSpace, &rc); - blobGrowBuffer(&pNode->key, nTerm, &rc); - - if( rc==SQLITE_OK ){ - if( pNode->key.n ){ - pBlk->n += sqlite3Fts3PutVarint(&pBlk->a[pBlk->n], nPrefix); - } - pBlk->n += sqlite3Fts3PutVarint(&pBlk->a[pBlk->n], nSuffix); - memcpy(&pBlk->a[pBlk->n], &zTerm[nPrefix], nSuffix); - pBlk->n += nSuffix; + } + assert( *pIter==0x00 || *pIter==0x01 ); + if( *pIter!=0x01 ) break; + pIter++; + pIter += fts3GetVarint32(pIter, &iCol); + if( iCol>=p->nCol ) return FTS_CORRUPT_VTAB; + } + return SQLITE_OK; +} - memcpy(pNode->key.a, zTerm, nTerm); - pNode->key.n = nTerm; - } +/* +** Gather the results for matchinfo directives 'y' and 'b'. +*/ +static int fts3ExprLHitGather( + Fts3Expr *pExpr, + MatchInfo *p +){ + int rc = SQLITE_OK; + assert( (pExpr->pLeft==0)==(pExpr->pRight==0) ); + if( pExpr->bEof==0 && pExpr->iDocid==p->pCursor->iPrevId ){ + if( pExpr->pLeft ){ + rc = fts3ExprLHitGather(pExpr->pLeft, p); + if( rc==SQLITE_OK ) rc = fts3ExprLHitGather(pExpr->pRight, p); }else{ - /* Otherwise, flush the current node of layer iLayer to disk. - ** Then allocate a new, empty sibling node. The key will be written - ** into the parent of this node. */ - rc = fts3WriteSegment(p, pNode->iBlock, pNode->block.a, pNode->block.n); - - assert( pNode->block.nAlloc>=p->nNodeSize ); - pNode->block.a[0] = (char)iLayer; - pNode->block.n = 1 + sqlite3Fts3PutVarint(&pNode->block.a[1], iPtr+1); - - iNextPtr = pNode->iBlock; - pNode->iBlock++; - pNode->key.n = 0; + rc = fts3ExprLHits(pExpr, p); } - - if( rc!=SQLITE_OK || iNextPtr==0 ) return rc; - iPtr = iNextPtr; } - - assert( 0 ); - return 0; + return rc; } /* -** Append a term and (optionally) doclist to the FTS segment node currently -** stored in blob *pNode. The node need not contain any terms, but the -** header must be written before this function is called. +** fts3ExprIterate() callback used to collect the "global" matchinfo stats +** for a single query. ** -** A node header is a single 0x00 byte for a leaf node, or a height varint -** followed by the left-hand-child varint for an internal node. +** fts3ExprIterate() callback to load the 'global' elements of a +** FTS3_MATCHINFO_HITS matchinfo array. The global stats are those elements +** of the matchinfo array that are constant for all rows returned by the +** current query. ** -** The term to be appended is passed via arguments zTerm/nTerm. For a -** leaf node, the doclist is passed as aDoclist/nDoclist. For an internal -** node, both aDoclist and nDoclist must be passed 0. +** Argument pCtx is actually a pointer to a struct of type MatchInfo. This +** function populates Matchinfo.aMatchinfo[] as follows: ** -** If the size of the value in blob pPrev is zero, then this is the first -** term written to the node. Otherwise, pPrev contains a copy of the -** previous term. Before this function returns, it is updated to contain a -** copy of zTerm/nTerm. +** for(iCol=0; iColn==0); /* True if this is the first term written */ - int nPrefix; /* Size of term prefix in bytes */ - int nSuffix; /* Size of term suffix in bytes */ - - /* Node must have already been started. There must be a doclist for a - ** leaf node, and there must not be a doclist for an internal node. */ - assert( pNode->n>0 ); - assert( (pNode->a[0]=='\0')==(aDoclist!=0) ); - - blobGrowBuffer(pPrev, nTerm, &rc); - if( rc!=SQLITE_OK ) return rc; - - nPrefix = fts3PrefixCompress(pPrev->a, pPrev->n, zTerm, nTerm); - nSuffix = nTerm - nPrefix; - memcpy(pPrev->a, zTerm, nTerm); - pPrev->n = nTerm; + MatchInfo *p = (MatchInfo *)pCtx; + return sqlite3Fts3EvalPhraseStats( + p->pCursor, pExpr, &p->aMatchinfo[3*iPhrase*p->nCol] + ); +} - if( bFirst==0 ){ - pNode->n += sqlite3Fts3PutVarint(&pNode->a[pNode->n], nPrefix); - } - pNode->n += sqlite3Fts3PutVarint(&pNode->a[pNode->n], nSuffix); - memcpy(&pNode->a[pNode->n], &zTerm[nPrefix], nSuffix); - pNode->n += nSuffix; +/* +** fts3ExprIterate() callback used to collect the "local" part of the +** FTS3_MATCHINFO_HITS array. The local stats are those elements of the +** array that are different for each row returned by the query. +*/ +static int fts3ExprLocalHitsCb( + Fts3Expr *pExpr, /* Phrase expression node */ + int iPhrase, /* Phrase number */ + void *pCtx /* Pointer to MatchInfo structure */ +){ + int rc = SQLITE_OK; + MatchInfo *p = (MatchInfo *)pCtx; + int iStart = iPhrase * p->nCol * 3; + int i; - if( aDoclist ){ - pNode->n += sqlite3Fts3PutVarint(&pNode->a[pNode->n], nDoclist); - memcpy(&pNode->a[pNode->n], aDoclist, nDoclist); - pNode->n += nDoclist; + for(i=0; inCol && rc==SQLITE_OK; i++){ + char *pCsr; + rc = sqlite3Fts3EvalPhrasePoslist(p->pCursor, pExpr, i, &pCsr); + if( pCsr ){ + p->aMatchinfo[iStart+i*3] = fts3ColumnlistCount(&pCsr); + }else{ + p->aMatchinfo[iStart+i*3] = 0; + } } - assert( pNode->n<=pNode->nAlloc ); - - return SQLITE_OK; + return rc; } -/* -** Append the current term and doclist pointed to by cursor pCsr to the -** appendable b-tree segment opened for writing by pWriter. -** -** Return SQLITE_OK if successful, or an SQLite error code otherwise. -*/ -static int fts3IncrmergeAppend( - Fts3Table *p, /* Fts3 table handle */ - IncrmergeWriter *pWriter, /* Writer object */ - Fts3MultiSegReader *pCsr /* Cursor containing term and doclist */ +static int fts3MatchinfoCheck( + Fts3Table *pTab, + char cArg, + char **pzErr ){ - const char *zTerm = pCsr->zTerm; - int nTerm = pCsr->nTerm; - const char *aDoclist = pCsr->aDoclist; - int nDoclist = pCsr->nDoclist; - int rc = SQLITE_OK; /* Return code */ - int nSpace; /* Total space in bytes required on leaf */ - int nPrefix; /* Size of prefix shared with previous term */ - int nSuffix; /* Size of suffix (nTerm - nPrefix) */ - NodeWriter *pLeaf; /* Object used to write leaf nodes */ - - pLeaf = &pWriter->aNodeWriter[0]; - nPrefix = fts3PrefixCompress(pLeaf->key.a, pLeaf->key.n, zTerm, nTerm); - nSuffix = nTerm - nPrefix; + if( (cArg==FTS3_MATCHINFO_NPHRASE) + || (cArg==FTS3_MATCHINFO_NCOL) + || (cArg==FTS3_MATCHINFO_NDOC && pTab->bFts4) + || (cArg==FTS3_MATCHINFO_AVGLENGTH && pTab->bFts4) + || (cArg==FTS3_MATCHINFO_LENGTH && pTab->bHasDocsize) + || (cArg==FTS3_MATCHINFO_LCS) + || (cArg==FTS3_MATCHINFO_HITS) + || (cArg==FTS3_MATCHINFO_LHITS) + || (cArg==FTS3_MATCHINFO_LHITS_BM) + ){ + return SQLITE_OK; + } + sqlite3Fts3ErrMsg(pzErr, "unrecognized matchinfo request: %c", cArg); + return SQLITE_ERROR; +} - nSpace = sqlite3Fts3VarintLen(nPrefix); - nSpace += sqlite3Fts3VarintLen(nSuffix) + nSuffix; - nSpace += sqlite3Fts3VarintLen(nDoclist) + nDoclist; +static size_t fts3MatchinfoSize(MatchInfo *pInfo, char cArg){ + size_t nVal; /* Number of integers output by cArg */ - /* If the current block is not empty, and if adding this term/doclist - ** to the current block would make it larger than Fts3Table.nNodeSize - ** bytes, write this block out to the database. */ - if( pLeaf->block.n>0 && (pLeaf->block.n + nSpace)>p->nNodeSize ){ - rc = fts3WriteSegment(p, pLeaf->iBlock, pLeaf->block.a, pLeaf->block.n); - pWriter->nWork++; + switch( cArg ){ + case FTS3_MATCHINFO_NDOC: + case FTS3_MATCHINFO_NPHRASE: + case FTS3_MATCHINFO_NCOL: + nVal = 1; + break; - /* Add the current term to the parent node. The term added to the - ** parent must: - ** - ** a) be greater than the largest term on the leaf node just written - ** to the database (still available in pLeaf->key), and - ** - ** b) be less than or equal to the term about to be added to the new - ** leaf node (zTerm/nTerm). - ** - ** In other words, it must be the prefix of zTerm 1 byte longer than - ** the common prefix (if any) of zTerm and pWriter->zTerm. - */ - if( rc==SQLITE_OK ){ - rc = fts3IncrmergePush(p, pWriter, zTerm, nPrefix+1); - } + case FTS3_MATCHINFO_AVGLENGTH: + case FTS3_MATCHINFO_LENGTH: + case FTS3_MATCHINFO_LCS: + nVal = pInfo->nCol; + break; - /* Advance to the next output block */ - pLeaf->iBlock++; - pLeaf->key.n = 0; - pLeaf->block.n = 0; + case FTS3_MATCHINFO_LHITS: + nVal = pInfo->nCol * pInfo->nPhrase; + break; - nSuffix = nTerm; - nSpace = 1; - nSpace += sqlite3Fts3VarintLen(nSuffix) + nSuffix; - nSpace += sqlite3Fts3VarintLen(nDoclist) + nDoclist; - } + case FTS3_MATCHINFO_LHITS_BM: + nVal = pInfo->nPhrase * ((pInfo->nCol + 31) / 32); + break; - pWriter->nLeafData += nSpace; - blobGrowBuffer(&pLeaf->block, pLeaf->block.n + nSpace, &rc); - if( rc==SQLITE_OK ){ - if( pLeaf->block.n==0 ){ - pLeaf->block.n = 1; - pLeaf->block.a[0] = '\0'; - } - rc = fts3AppendToNode( - &pLeaf->block, &pLeaf->key, zTerm, nTerm, aDoclist, nDoclist - ); + default: + assert( cArg==FTS3_MATCHINFO_HITS ); + nVal = pInfo->nCol * pInfo->nPhrase * 3; + break; } - return rc; + return nVal; } -/* -** This function is called to release all dynamic resources held by the -** merge-writer object pWriter, and if no error has occurred, to flush -** all outstanding node buffers held by pWriter to disk. -** -** If *pRc is not SQLITE_OK when this function is called, then no attempt -** is made to write any data to disk. Instead, this function serves only -** to release outstanding resources. -** -** Otherwise, if *pRc is initially SQLITE_OK and an error occurs while -** flushing buffers to disk, *pRc is set to an SQLite error code before -** returning. -*/ -static void fts3IncrmergeRelease( - Fts3Table *p, /* FTS3 table handle */ - IncrmergeWriter *pWriter, /* Merge-writer object */ - int *pRc /* IN/OUT: Error code */ +static int fts3MatchinfoSelectDoctotal( + Fts3Table *pTab, + sqlite3_stmt **ppStmt, + sqlite3_int64 *pnDoc, + const char **paLen, + const char **ppEnd ){ - int i; /* Used to iterate through non-root layers */ - int iRoot; /* Index of root in pWriter->aNodeWriter */ - NodeWriter *pRoot; /* NodeWriter for root node */ - int rc = *pRc; /* Error code */ - - /* Set iRoot to the index in pWriter->aNodeWriter[] of the output segment - ** root node. If the segment fits entirely on a single leaf node, iRoot - ** will be set to 0. If the root node is the parent of the leaves, iRoot - ** will be 1. And so on. */ - for(iRoot=FTS_MAX_APPENDABLE_HEIGHT-1; iRoot>=0; iRoot--){ - NodeWriter *pNode = &pWriter->aNodeWriter[iRoot]; - if( pNode->block.n>0 ) break; - assert( *pRc || pNode->block.nAlloc==0 ); - assert( *pRc || pNode->key.nAlloc==0 ); - sqlite3_free(pNode->block.a); - sqlite3_free(pNode->key.a); - } + sqlite3_stmt *pStmt; + const char *a; + const char *pEnd; + sqlite3_int64 nDoc; + int n; - /* Empty output segment. This is a no-op. */ - if( iRoot<0 ) return; - /* The entire output segment fits on a single node. Normally, this means - ** the node would be stored as a blob in the "root" column of the %_segdir - ** table. However, this is not permitted in this case. The problem is that - ** space has already been reserved in the %_segments table, and so the - ** start_block and end_block fields of the %_segdir table must be populated. - ** And, by design or by accident, released versions of FTS cannot handle - ** segments that fit entirely on the root node with start_block!=0. - ** - ** Instead, create a synthetic root node that contains nothing but a - ** pointer to the single content node. So that the segment consists of a - ** single leaf and a single interior (root) node. - ** - ** Todo: Better might be to defer allocating space in the %_segments - ** table until we are sure it is needed. - */ - if( iRoot==0 ){ - Blob *pBlock = &pWriter->aNodeWriter[1].block; - blobGrowBuffer(pBlock, 1 + FTS3_VARINT_MAX, &rc); - if( rc==SQLITE_OK ){ - pBlock->a[0] = 0x01; - pBlock->n = 1 + sqlite3Fts3PutVarint( - &pBlock->a[1], pWriter->aNodeWriter[0].iBlock - ); - } - iRoot = 1; + if( !*ppStmt ){ + int rc = sqlite3Fts3SelectDoctotal(pTab, ppStmt); + if( rc!=SQLITE_OK ) return rc; } - pRoot = &pWriter->aNodeWriter[iRoot]; + pStmt = *ppStmt; + assert( sqlite3_data_count(pStmt)==1 ); - /* Flush all currently outstanding nodes to disk. */ - for(i=0; iaNodeWriter[i]; - if( pNode->block.n>0 && rc==SQLITE_OK ){ - rc = fts3WriteSegment(p, pNode->iBlock, pNode->block.a, pNode->block.n); - } - sqlite3_free(pNode->block.a); - sqlite3_free(pNode->key.a); + n = sqlite3_column_bytes(pStmt, 0); + a = sqlite3_column_blob(pStmt, 0); + if( a==0 ){ + return FTS_CORRUPT_VTAB; } - - /* Write the %_segdir record. */ - if( rc==SQLITE_OK ){ - rc = fts3WriteSegdir(p, - pWriter->iAbsLevel+1, /* level */ - pWriter->iIdx, /* idx */ - pWriter->iStart, /* start_block */ - pWriter->aNodeWriter[0].iBlock, /* leaves_end_block */ - pWriter->iEnd, /* end_block */ - (pWriter->bNoLeafData==0 ? pWriter->nLeafData : 0), /* end_block */ - pRoot->block.a, pRoot->block.n /* root */ - ); + pEnd = a + n; + a += sqlite3Fts3GetVarintBounded(a, pEnd, &nDoc); + if( nDoc<=0 || a>pEnd ){ + return FTS_CORRUPT_VTAB; } - sqlite3_free(pRoot->block.a); - sqlite3_free(pRoot->key.a); + *pnDoc = nDoc; - *pRc = rc; + if( paLen ) *paLen = a; + if( ppEnd ) *ppEnd = pEnd; + return SQLITE_OK; } /* -** Compare the term in buffer zLhs (size in bytes nLhs) with that in -** zRhs (size in bytes nRhs) using memcmp. If one term is a prefix of -** the other, it is considered to be smaller than the other. -** -** Return -ve if zLhs is smaller than zRhs, 0 if it is equal, or +ve -** if it is greater. +** An instance of the following structure is used to store state while +** iterating through a multi-column position-list corresponding to the +** hits for a single phrase on a single row in order to calculate the +** values for a matchinfo() FTS3_MATCHINFO_LCS request. */ -static int fts3TermCmp( - const char *zLhs, int nLhs, /* LHS of comparison */ - const char *zRhs, int nRhs /* RHS of comparison */ -){ - int nCmp = MIN(nLhs, nRhs); - int res; +typedef struct LcsIterator LcsIterator; +struct LcsIterator { + Fts3Expr *pExpr; /* Pointer to phrase expression */ + int iPosOffset; /* Tokens count up to end of this phrase */ + char *pRead; /* Cursor used to iterate through aDoclist */ + int iPos; /* Current position */ +}; - res = memcmp(zLhs, zRhs, nCmp); - if( res==0 ) res = nLhs - nRhs; +/* +** If LcsIterator.iCol is set to the following value, the iterator has +** finished iterating through all offsets for all columns. +*/ +#define LCS_ITERATOR_FINISHED 0x7FFFFFFF; - return res; +static int fts3MatchinfoLcsCb( + Fts3Expr *pExpr, /* Phrase expression node */ + int iPhrase, /* Phrase number (numbered from zero) */ + void *pCtx /* Pointer to MatchInfo structure */ +){ + LcsIterator *aIter = (LcsIterator *)pCtx; + aIter[iPhrase].pExpr = pExpr; + return SQLITE_OK; } - /* -** Query to see if the entry in the %_segments table with blockid iEnd is -** NULL. If no error occurs and the entry is NULL, set *pbRes 1 before -** returning. Otherwise, set *pbRes to 0. -** -** Or, if an error occurs while querying the database, return an SQLite -** error code. The final value of *pbRes is undefined in this case. -** -** This is used to test if a segment is an "appendable" segment. If it -** is, then a NULL entry has been inserted into the %_segments table -** with blockid %_segdir.end_block. +** Advance the iterator passed as an argument to the next position. Return +** 1 if the iterator is at EOF or if it now points to the start of the +** position list for the next column. */ -static int fts3IsAppendable(Fts3Table *p, sqlite3_int64 iEnd, int *pbRes){ - int bRes = 0; /* Result to set *pbRes to */ - sqlite3_stmt *pCheck = 0; /* Statement to query database with */ - int rc; /* Return code */ +static int fts3LcsIteratorAdvance(LcsIterator *pIter){ + char *pRead = pIter->pRead; + sqlite3_int64 iRead; + int rc = 0; - rc = fts3SqlStmt(p, SQL_SEGMENT_IS_APPENDABLE, &pCheck, 0); - if( rc==SQLITE_OK ){ - sqlite3_bind_int64(pCheck, 1, iEnd); - if( SQLITE_ROW==sqlite3_step(pCheck) ) bRes = 1; - rc = sqlite3_reset(pCheck); + pRead += sqlite3Fts3GetVarint(pRead, &iRead); + if( iRead==0 || iRead==1 ){ + pRead = 0; + rc = 1; + }else{ + pIter->iPos += (int)(iRead-2); } - - *pbRes = bRes; + + pIter->pRead = pRead; return rc; } - + /* -** This function is called when initializing an incremental-merge operation. -** It checks if the existing segment with index value iIdx at absolute level -** (iAbsLevel+1) can be appended to by the incremental merge. If it can, the -** merge-writer object *pWriter is initialized to write to it. -** -** An existing segment can be appended to by an incremental merge if: +** This function implements the FTS3_MATCHINFO_LCS matchinfo() flag. ** -** * It was initially created as an appendable segment (with all required -** space pre-allocated), and +** If the call is successful, the longest-common-substring lengths for each +** column are written into the first nCol elements of the pInfo->aMatchinfo[] +** array before returning. SQLITE_OK is returned in this case. ** -** * The first key read from the input (arguments zKey and nKey) is -** greater than the largest key currently stored in the potential -** output segment. +** Otherwise, if an error occurs, an SQLite error code is returned and the +** data written to the first nCol elements of pInfo->aMatchinfo[] is +** undefined. */ -static int fts3IncrmergeLoad( - Fts3Table *p, /* Fts3 table handle */ - sqlite3_int64 iAbsLevel, /* Absolute level of input segments */ - int iIdx, /* Index of candidate output segment */ - const char *zKey, /* First key to write */ - int nKey, /* Number of bytes in nKey */ - IncrmergeWriter *pWriter /* Populate this object */ -){ - int rc; /* Return code */ - sqlite3_stmt *pSelect = 0; /* SELECT to read %_segdir entry */ - - rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR, &pSelect, 0); - if( rc==SQLITE_OK ){ - sqlite3_int64 iStart = 0; /* Value of %_segdir.start_block */ - sqlite3_int64 iLeafEnd = 0; /* Value of %_segdir.leaves_end_block */ - sqlite3_int64 iEnd = 0; /* Value of %_segdir.end_block */ - const char *aRoot = 0; /* Pointer to %_segdir.root buffer */ - int nRoot = 0; /* Size of aRoot[] in bytes */ - int rc2; /* Return code from sqlite3_reset() */ - int bAppendable = 0; /* Set to true if segment is appendable */ +static int fts3MatchinfoLcs(Fts3Cursor *pCsr, MatchInfo *pInfo){ + LcsIterator *aIter; + int i; + int iCol; + int nToken = 0; + int rc = SQLITE_OK; - /* Read the %_segdir entry for index iIdx absolute level (iAbsLevel+1) */ - sqlite3_bind_int64(pSelect, 1, iAbsLevel+1); - sqlite3_bind_int(pSelect, 2, iIdx); - if( sqlite3_step(pSelect)==SQLITE_ROW ){ - iStart = sqlite3_column_int64(pSelect, 1); - iLeafEnd = sqlite3_column_int64(pSelect, 2); - fts3ReadEndBlockField(pSelect, 3, &iEnd, &pWriter->nLeafData); - if( pWriter->nLeafData<0 ){ - pWriter->nLeafData = pWriter->nLeafData * -1; - } - pWriter->bNoLeafData = (pWriter->nLeafData==0); - nRoot = sqlite3_column_bytes(pSelect, 4); - aRoot = sqlite3_column_blob(pSelect, 4); - }else{ - return sqlite3_reset(pSelect); - } + /* Allocate and populate the array of LcsIterator objects. The array + ** contains one element for each matchable phrase in the query. + **/ + aIter = sqlite3_malloc64(sizeof(LcsIterator) * pCsr->nPhrase); + if( !aIter ) return SQLITE_NOMEM; + memset(aIter, 0, sizeof(LcsIterator) * pCsr->nPhrase); + (void)fts3ExprIterate(pCsr->pExpr, fts3MatchinfoLcsCb, (void*)aIter); - /* Check for the zero-length marker in the %_segments table */ - rc = fts3IsAppendable(p, iEnd, &bAppendable); + for(i=0; inPhrase; i++){ + LcsIterator *pIter = &aIter[i]; + nToken -= pIter->pExpr->pPhrase->nToken; + pIter->iPosOffset = nToken; + } - /* Check that zKey/nKey is larger than the largest key the candidate */ - if( rc==SQLITE_OK && bAppendable ){ - char *aLeaf = 0; - int nLeaf = 0; + for(iCol=0; iColnCol; iCol++){ + int nLcs = 0; /* LCS value for this column */ + int nLive = 0; /* Number of iterators in aIter not at EOF */ - rc = sqlite3Fts3ReadBlock(p, iLeafEnd, &aLeaf, &nLeaf, 0); - if( rc==SQLITE_OK ){ - NodeReader reader; - for(rc = nodeReaderInit(&reader, aLeaf, nLeaf); - rc==SQLITE_OK && reader.aNode; - rc = nodeReaderNext(&reader) - ){ - assert( reader.aNode ); - } - if( fts3TermCmp(zKey, nKey, reader.term.a, reader.term.n)<=0 ){ - bAppendable = 0; + for(i=0; inPhrase; i++){ + LcsIterator *pIt = &aIter[i]; + rc = sqlite3Fts3EvalPhrasePoslist(pCsr, pIt->pExpr, iCol, &pIt->pRead); + if( rc!=SQLITE_OK ) goto matchinfo_lcs_out; + if( pIt->pRead ){ + pIt->iPos = pIt->iPosOffset; + fts3LcsIteratorAdvance(pIt); + if( pIt->pRead==0 ){ + rc = FTS_CORRUPT_VTAB; + goto matchinfo_lcs_out; } - nodeReaderRelease(&reader); + nLive++; } - sqlite3_free(aLeaf); } - if( rc==SQLITE_OK && bAppendable ){ - /* It is possible to append to this segment. Set up the IncrmergeWriter - ** object to do so. */ - int i; - int nHeight = (int)aRoot[0]; - NodeWriter *pNode; - - pWriter->nLeafEst = (int)((iEnd - iStart) + 1)/FTS_MAX_APPENDABLE_HEIGHT; - pWriter->iStart = iStart; - pWriter->iEnd = iEnd; - pWriter->iAbsLevel = iAbsLevel; - pWriter->iIdx = iIdx; - - for(i=nHeight+1; iaNodeWriter[i].iBlock = pWriter->iStart + i*pWriter->nLeafEst; - } - - pNode = &pWriter->aNodeWriter[nHeight]; - pNode->iBlock = pWriter->iStart + pWriter->nLeafEst*nHeight; - blobGrowBuffer(&pNode->block, MAX(nRoot, p->nNodeSize), &rc); - if( rc==SQLITE_OK ){ - memcpy(pNode->block.a, aRoot, nRoot); - pNode->block.n = nRoot; - } - - for(i=nHeight; i>=0 && rc==SQLITE_OK; i--){ - NodeReader reader; - pNode = &pWriter->aNodeWriter[i]; + while( nLive>0 ){ + LcsIterator *pAdv = 0; /* The iterator to advance by one position */ + int nThisLcs = 0; /* LCS for the current iterator positions */ - rc = nodeReaderInit(&reader, pNode->block.a, pNode->block.n); - while( reader.aNode && rc==SQLITE_OK ) rc = nodeReaderNext(&reader); - blobGrowBuffer(&pNode->key, reader.term.n, &rc); - if( rc==SQLITE_OK ){ - memcpy(pNode->key.a, reader.term.a, reader.term.n); - pNode->key.n = reader.term.n; - if( i>0 ){ - char *aBlock = 0; - int nBlock = 0; - pNode = &pWriter->aNodeWriter[i-1]; - pNode->iBlock = reader.iChild; - rc = sqlite3Fts3ReadBlock(p, reader.iChild, &aBlock, &nBlock, 0); - blobGrowBuffer(&pNode->block, MAX(nBlock, p->nNodeSize), &rc); - if( rc==SQLITE_OK ){ - memcpy(pNode->block.a, aBlock, nBlock); - pNode->block.n = nBlock; - } - sqlite3_free(aBlock); + for(i=0; inPhrase; i++){ + LcsIterator *pIter = &aIter[i]; + if( pIter->pRead==0 ){ + /* This iterator is already at EOF for this column. */ + nThisLcs = 0; + }else{ + if( pAdv==0 || pIter->iPosiPos ){ + pAdv = pIter; + } + if( nThisLcs==0 || pIter->iPos==pIter[-1].iPos ){ + nThisLcs++; + }else{ + nThisLcs = 1; } + if( nThisLcs>nLcs ) nLcs = nThisLcs; } - nodeReaderRelease(&reader); } + if( fts3LcsIteratorAdvance(pAdv) ) nLive--; } - rc2 = sqlite3_reset(pSelect); - if( rc==SQLITE_OK ) rc = rc2; + pInfo->aMatchinfo[iCol] = nLcs; } + matchinfo_lcs_out: + sqlite3_free(aIter); return rc; } /* -** Determine the largest segment index value that exists within absolute -** level iAbsLevel+1. If no error occurs, set *piIdx to this value plus -** one before returning SQLITE_OK. Or, if there are no segments at all -** within level iAbsLevel, set *piIdx to zero. -** -** If an error occurs, return an SQLite error code. The final value of -** *piIdx is undefined in this case. -*/ -static int fts3IncrmergeOutputIdx( - Fts3Table *p, /* FTS Table handle */ - sqlite3_int64 iAbsLevel, /* Absolute index of input segments */ - int *piIdx /* OUT: Next free index at iAbsLevel+1 */ -){ - int rc; - sqlite3_stmt *pOutputIdx = 0; /* SQL used to find output index */ - - rc = fts3SqlStmt(p, SQL_NEXT_SEGMENT_INDEX, &pOutputIdx, 0); - if( rc==SQLITE_OK ){ - sqlite3_bind_int64(pOutputIdx, 1, iAbsLevel+1); - sqlite3_step(pOutputIdx); - *piIdx = sqlite3_column_int(pOutputIdx, 0); - rc = sqlite3_reset(pOutputIdx); - } - - return rc; -} - -/* -** Allocate an appendable output segment on absolute level iAbsLevel+1 -** with idx value iIdx. -** -** In the %_segdir table, a segment is defined by the values in three -** columns: -** -** start_block -** leaves_end_block -** end_block -** -** When an appendable segment is allocated, it is estimated that the -** maximum number of leaf blocks that may be required is the sum of the -** number of leaf blocks consumed by the input segments, plus the number -** of input segments, multiplied by two. This value is stored in stack -** variable nLeafEst. +** Populate the buffer pInfo->aMatchinfo[] with an array of integers to +** be returned by the matchinfo() function. Argument zArg contains the +** format string passed as the second argument to matchinfo (or the +** default value "pcx" if no second argument was specified). The format +** string has already been validated and the pInfo->aMatchinfo[] array +** is guaranteed to be large enough for the output. ** -** A total of 16*nLeafEst blocks are allocated when an appendable segment -** is created ((1 + end_block - start_block)==16*nLeafEst). The contiguous -** array of leaf nodes starts at the first block allocated. The array -** of interior nodes that are parents of the leaf nodes start at block -** (start_block + (1 + end_block - start_block) / 16). And so on. +** If bGlobal is true, then populate all fields of the matchinfo() output. +** If it is false, then assume that those fields that do not change between +** rows (i.e. FTS3_MATCHINFO_NPHRASE, NCOL, NDOC, AVGLENGTH and part of HITS) +** have already been populated. ** -** In the actual code below, the value "16" is replaced with the -** pre-processor macro FTS_MAX_APPENDABLE_HEIGHT. +** Return SQLITE_OK if successful, or an SQLite error code if an error +** occurs. If a value other than SQLITE_OK is returned, the state the +** pInfo->aMatchinfo[] buffer is left in is undefined. */ -static int fts3IncrmergeWriter( - Fts3Table *p, /* Fts3 table handle */ - sqlite3_int64 iAbsLevel, /* Absolute level of input segments */ - int iIdx, /* Index of new output segment */ - Fts3MultiSegReader *pCsr, /* Cursor that data will be read from */ - IncrmergeWriter *pWriter /* Populate this object */ +static int fts3MatchinfoValues( + Fts3Cursor *pCsr, /* FTS3 cursor object */ + int bGlobal, /* True to grab the global stats */ + MatchInfo *pInfo, /* Matchinfo context object */ + const char *zArg /* Matchinfo format string */ ){ - int rc; /* Return Code */ - int i; /* Iterator variable */ - int nLeafEst = 0; /* Blocks allocated for leaf nodes */ - sqlite3_stmt *pLeafEst = 0; /* SQL used to determine nLeafEst */ - sqlite3_stmt *pFirstBlock = 0; /* SQL used to determine first block */ - - /* Calculate nLeafEst. */ - rc = fts3SqlStmt(p, SQL_MAX_LEAF_NODE_ESTIMATE, &pLeafEst, 0); - if( rc==SQLITE_OK ){ - sqlite3_bind_int64(pLeafEst, 1, iAbsLevel); - sqlite3_bind_int64(pLeafEst, 2, pCsr->nSegment); - if( SQLITE_ROW==sqlite3_step(pLeafEst) ){ - nLeafEst = sqlite3_column_int(pLeafEst, 0); - } - rc = sqlite3_reset(pLeafEst); - } - if( rc!=SQLITE_OK ) return rc; - - /* Calculate the first block to use in the output segment */ - rc = fts3SqlStmt(p, SQL_NEXT_SEGMENTS_ID, &pFirstBlock, 0); - if( rc==SQLITE_OK ){ - if( SQLITE_ROW==sqlite3_step(pFirstBlock) ){ - pWriter->iStart = sqlite3_column_int64(pFirstBlock, 0); - pWriter->iEnd = pWriter->iStart - 1; - pWriter->iEnd += nLeafEst * FTS_MAX_APPENDABLE_HEIGHT; - } - rc = sqlite3_reset(pFirstBlock); - } - if( rc!=SQLITE_OK ) return rc; + int rc = SQLITE_OK; + int i; + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + sqlite3_stmt *pSelect = 0; - /* Insert the marker in the %_segments table to make sure nobody tries - ** to steal the space just allocated. This is also used to identify - ** appendable segments. */ - rc = fts3WriteSegment(p, pWriter->iEnd, 0, 0); - if( rc!=SQLITE_OK ) return rc; + for(i=0; rc==SQLITE_OK && zArg[i]; i++){ + pInfo->flag = zArg[i]; + switch( zArg[i] ){ + case FTS3_MATCHINFO_NPHRASE: + if( bGlobal ) pInfo->aMatchinfo[0] = pInfo->nPhrase; + break; - pWriter->iAbsLevel = iAbsLevel; - pWriter->nLeafEst = nLeafEst; - pWriter->iIdx = iIdx; + case FTS3_MATCHINFO_NCOL: + if( bGlobal ) pInfo->aMatchinfo[0] = pInfo->nCol; + break; + + case FTS3_MATCHINFO_NDOC: + if( bGlobal ){ + sqlite3_int64 nDoc = 0; + rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &nDoc, 0, 0); + pInfo->aMatchinfo[0] = (u32)nDoc; + } + break; - /* Set up the array of NodeWriter objects */ - for(i=0; iaNodeWriter[i].iBlock = pWriter->iStart + i*pWriter->nLeafEst; - } - return SQLITE_OK; -} + case FTS3_MATCHINFO_AVGLENGTH: + if( bGlobal ){ + sqlite3_int64 nDoc; /* Number of rows in table */ + const char *a; /* Aggregate column length array */ + const char *pEnd; /* First byte past end of length array */ -/* -** Remove an entry from the %_segdir table. This involves running the -** following two statements: -** -** DELETE FROM %_segdir WHERE level = :iAbsLevel AND idx = :iIdx -** UPDATE %_segdir SET idx = idx - 1 WHERE level = :iAbsLevel AND idx > :iIdx -** -** The DELETE statement removes the specific %_segdir level. The UPDATE -** statement ensures that the remaining segments have contiguously allocated -** idx values. -*/ -static int fts3RemoveSegdirEntry( - Fts3Table *p, /* FTS3 table handle */ - sqlite3_int64 iAbsLevel, /* Absolute level to delete from */ - int iIdx /* Index of %_segdir entry to delete */ -){ - int rc; /* Return code */ - sqlite3_stmt *pDelete = 0; /* DELETE statement */ + rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &nDoc, &a, &pEnd); + if( rc==SQLITE_OK ){ + int iCol; + for(iCol=0; iColnCol; iCol++){ + u32 iVal; + sqlite3_int64 nToken; + a += sqlite3Fts3GetVarint(a, &nToken); + if( a>pEnd ){ + rc = SQLITE_CORRUPT_VTAB; + break; + } + iVal = (u32)(((u32)(nToken&0xffffffff)+nDoc/2)/nDoc); + pInfo->aMatchinfo[iCol] = iVal; + } + } + } + break; - rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_ENTRY, &pDelete, 0); - if( rc==SQLITE_OK ){ - sqlite3_bind_int64(pDelete, 1, iAbsLevel); - sqlite3_bind_int(pDelete, 2, iIdx); - sqlite3_step(pDelete); - rc = sqlite3_reset(pDelete); - } + case FTS3_MATCHINFO_LENGTH: { + sqlite3_stmt *pSelectDocsize = 0; + rc = sqlite3Fts3SelectDocsize(pTab, pCsr->iPrevId, &pSelectDocsize); + if( rc==SQLITE_OK ){ + int iCol; + const char *a = sqlite3_column_blob(pSelectDocsize, 0); + const char *pEnd = a + sqlite3_column_bytes(pSelectDocsize, 0); + for(iCol=0; iColnCol; iCol++){ + sqlite3_int64 nToken; + a += sqlite3Fts3GetVarintBounded(a, pEnd, &nToken); + if( a>pEnd ){ + rc = SQLITE_CORRUPT_VTAB; + break; + } + pInfo->aMatchinfo[iCol] = (u32)nToken; + } + } + sqlite3_reset(pSelectDocsize); + break; + } - return rc; -} + case FTS3_MATCHINFO_LCS: + rc = fts3ExprLoadDoclists(pCsr, 0, 0); + if( rc==SQLITE_OK ){ + rc = fts3MatchinfoLcs(pCsr, pInfo); + } + break; -/* -** One or more segments have just been removed from absolute level iAbsLevel. -** Update the 'idx' values of the remaining segments in the level so that -** the idx values are a contiguous sequence starting from 0. -*/ -static int fts3RepackSegdirLevel( - Fts3Table *p, /* FTS3 table handle */ - sqlite3_int64 iAbsLevel /* Absolute level to repack */ -){ - int rc; /* Return code */ - int *aIdx = 0; /* Array of remaining idx values */ - int nIdx = 0; /* Valid entries in aIdx[] */ - int nAlloc = 0; /* Allocated size of aIdx[] */ - int i; /* Iterator variable */ - sqlite3_stmt *pSelect = 0; /* Select statement to read idx values */ - sqlite3_stmt *pUpdate = 0; /* Update statement to modify idx values */ + case FTS3_MATCHINFO_LHITS_BM: + case FTS3_MATCHINFO_LHITS: { + size_t nZero = fts3MatchinfoSize(pInfo, zArg[i]) * sizeof(u32); + memset(pInfo->aMatchinfo, 0, nZero); + rc = fts3ExprLHitGather(pCsr->pExpr, pInfo); + break; + } - rc = fts3SqlStmt(p, SQL_SELECT_INDEXES, &pSelect, 0); - if( rc==SQLITE_OK ){ - int rc2; - sqlite3_bind_int64(pSelect, 1, iAbsLevel); - while( SQLITE_ROW==sqlite3_step(pSelect) ){ - if( nIdx>=nAlloc ){ - int *aNew; - nAlloc += 16; - aNew = sqlite3_realloc(aIdx, nAlloc*sizeof(int)); - if( !aNew ){ - rc = SQLITE_NOMEM; - break; + default: { + Fts3Expr *pExpr; + assert( zArg[i]==FTS3_MATCHINFO_HITS ); + pExpr = pCsr->pExpr; + rc = fts3ExprLoadDoclists(pCsr, 0, 0); + if( rc!=SQLITE_OK ) break; + if( bGlobal ){ + if( pCsr->pDeferred ){ + rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &pInfo->nDoc,0,0); + if( rc!=SQLITE_OK ) break; + } + rc = fts3ExprIterate(pExpr, fts3ExprGlobalHitsCb,(void*)pInfo); + sqlite3Fts3EvalTestDeferred(pCsr, &rc); + if( rc!=SQLITE_OK ) break; } - aIdx = aNew; + (void)fts3ExprIterate(pExpr, fts3ExprLocalHitsCb,(void*)pInfo); + break; } - aIdx[nIdx++] = sqlite3_column_int(pSelect, 0); } - rc2 = sqlite3_reset(pSelect); - if( rc==SQLITE_OK ) rc = rc2; - } - - if( rc==SQLITE_OK ){ - rc = fts3SqlStmt(p, SQL_SHIFT_SEGDIR_ENTRY, &pUpdate, 0); - } - if( rc==SQLITE_OK ){ - sqlite3_bind_int64(pUpdate, 2, iAbsLevel); - } - assert( p->bIgnoreSavepoint==0 ); - p->bIgnoreSavepoint = 1; - for(i=0; rc==SQLITE_OK && iaMatchinfo += fts3MatchinfoSize(pInfo, zArg[i]); } - p->bIgnoreSavepoint = 0; - sqlite3_free(aIdx); + sqlite3_reset(pSelect); return rc; } -static void fts3StartNode(Blob *pNode, int iHeight, sqlite3_int64 iChild){ - pNode->a[0] = (char)iHeight; - if( iChild ){ - assert( pNode->nAlloc>=1+sqlite3Fts3VarintLen(iChild) ); - pNode->n = 1 + sqlite3Fts3PutVarint(&pNode->a[1], iChild); - }else{ - assert( pNode->nAlloc>=1 ); - pNode->n = 1; - } -} /* -** The first two arguments are a pointer to and the size of a segment b-tree -** node. The node may be a leaf or an internal node. -** -** This function creates a new node image in blob object *pNew by copying -** all terms that are greater than or equal to zTerm/nTerm (for leaf nodes) -** or greater than zTerm/nTerm (for internal nodes) from aNode/nNode. +** Populate pCsr->aMatchinfo[] with data for the current row. The +** 'matchinfo' data is an array of 32-bit unsigned integers (C type u32). */ -static int fts3TruncateNode( - const char *aNode, /* Current node image */ - int nNode, /* Size of aNode in bytes */ - Blob *pNew, /* OUT: Write new node image here */ - const char *zTerm, /* Omit all terms smaller than this */ - int nTerm, /* Size of zTerm in bytes */ - sqlite3_int64 *piBlock /* OUT: Block number in next layer down */ +static void fts3GetMatchinfo( + sqlite3_context *pCtx, /* Return results here */ + Fts3Cursor *pCsr, /* FTS3 Cursor object */ + const char *zArg /* Second argument to matchinfo() function */ ){ - NodeReader reader; /* Reader object */ - Blob prev = {0, 0, 0}; /* Previous term written to new node */ - int rc = SQLITE_OK; /* Return code */ - int bLeaf = aNode[0]=='\0'; /* True for a leaf node */ + MatchInfo sInfo; + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + int rc = SQLITE_OK; + int bGlobal = 0; /* Collect 'global' stats as well as local */ - /* Allocate required output space */ - blobGrowBuffer(pNew, nNode, &rc); - if( rc!=SQLITE_OK ) return rc; - pNew->n = 0; + u32 *aOut = 0; + void (*xDestroyOut)(void*) = 0; - /* Populate new node buffer */ - for(rc = nodeReaderInit(&reader, aNode, nNode); - rc==SQLITE_OK && reader.aNode; - rc = nodeReaderNext(&reader) - ){ - if( pNew->n==0 ){ - int res = fts3TermCmp(reader.term.a, reader.term.n, zTerm, nTerm); - if( res<0 || (bLeaf==0 && res==0) ) continue; - fts3StartNode(pNew, (int)aNode[0], reader.iChild); - *piBlock = reader.iChild; - } - rc = fts3AppendToNode( - pNew, &prev, reader.term.a, reader.term.n, - reader.aDoclist, reader.nDoclist - ); - if( rc!=SQLITE_OK ) break; - } - if( pNew->n==0 ){ - fts3StartNode(pNew, (int)aNode[0], reader.iChild); - *piBlock = reader.iChild; + memset(&sInfo, 0, sizeof(MatchInfo)); + sInfo.pCursor = pCsr; + sInfo.nCol = pTab->nColumn; + + /* If there is cached matchinfo() data, but the format string for the + ** cache does not match the format string for this request, discard + ** the cached data. */ + if( pCsr->pMIBuffer && strcmp(pCsr->pMIBuffer->zMatchinfo, zArg) ){ + sqlite3Fts3MIBufferFree(pCsr->pMIBuffer); + pCsr->pMIBuffer = 0; } - assert( pNew->n<=pNew->nAlloc ); - nodeReaderRelease(&reader); - sqlite3_free(prev.a); - return rc; -} + /* If Fts3Cursor.pMIBuffer is NULL, then this is the first time the + ** matchinfo function has been called for this query. In this case + ** allocate the array used to accumulate the matchinfo data and + ** initialize those elements that are constant for every row. + */ + if( pCsr->pMIBuffer==0 ){ + size_t nMatchinfo = 0; /* Number of u32 elements in match-info */ + int i; /* Used to iterate through zArg */ -/* -** Remove all terms smaller than zTerm/nTerm from segment iIdx in absolute -** level iAbsLevel. This may involve deleting entries from the %_segments -** table, and modifying existing entries in both the %_segments and %_segdir -** tables. -** -** SQLITE_OK is returned if the segment is updated successfully. Or an -** SQLite error code otherwise. -*/ -static int fts3TruncateSegment( - Fts3Table *p, /* FTS3 table handle */ - sqlite3_int64 iAbsLevel, /* Absolute level of segment to modify */ - int iIdx, /* Index within level of segment to modify */ - const char *zTerm, /* Remove terms smaller than this */ - int nTerm /* Number of bytes in buffer zTerm */ -){ - int rc = SQLITE_OK; /* Return code */ - Blob root = {0,0,0}; /* New root page image */ - Blob block = {0,0,0}; /* Buffer used for any other block */ - sqlite3_int64 iBlock = 0; /* Block id */ - sqlite3_int64 iNewStart = 0; /* New value for iStartBlock */ - sqlite3_int64 iOldStart = 0; /* Old value for iStartBlock */ - sqlite3_stmt *pFetch = 0; /* Statement used to fetch segdir */ + /* Determine the number of phrases in the query */ + pCsr->nPhrase = fts3ExprPhraseCount(pCsr->pExpr); + sInfo.nPhrase = pCsr->nPhrase; - rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR, &pFetch, 0); - if( rc==SQLITE_OK ){ - int rc2; /* sqlite3_reset() return code */ - sqlite3_bind_int64(pFetch, 1, iAbsLevel); - sqlite3_bind_int(pFetch, 2, iIdx); - if( SQLITE_ROW==sqlite3_step(pFetch) ){ - const char *aRoot = sqlite3_column_blob(pFetch, 4); - int nRoot = sqlite3_column_bytes(pFetch, 4); - iOldStart = sqlite3_column_int64(pFetch, 1); - rc = fts3TruncateNode(aRoot, nRoot, &root, zTerm, nTerm, &iBlock); + /* Determine the number of integers in the buffer returned by this call. */ + for(i=0; zArg[i]; i++){ + char *zErr = 0; + if( fts3MatchinfoCheck(pTab, zArg[i], &zErr) ){ + sqlite3_result_error(pCtx, zErr, -1); + sqlite3_free(zErr); + return; + } + nMatchinfo += fts3MatchinfoSize(&sInfo, zArg[i]); } - rc2 = sqlite3_reset(pFetch); - if( rc==SQLITE_OK ) rc = rc2; - } - while( rc==SQLITE_OK && iBlock ){ - char *aBlock = 0; - int nBlock = 0; - iNewStart = iBlock; + /* Allocate space for Fts3Cursor.aMatchinfo[] and Fts3Cursor.zMatchinfo. */ + pCsr->pMIBuffer = fts3MIBufferNew(nMatchinfo, zArg); + if( !pCsr->pMIBuffer ) rc = SQLITE_NOMEM; - rc = sqlite3Fts3ReadBlock(p, iBlock, &aBlock, &nBlock, 0); - if( rc==SQLITE_OK ){ - rc = fts3TruncateNode(aBlock, nBlock, &block, zTerm, nTerm, &iBlock); - } - if( rc==SQLITE_OK ){ - rc = fts3WriteSegment(p, iNewStart, block.a, block.n); - } - sqlite3_free(aBlock); + pCsr->isMatchinfoNeeded = 1; + bGlobal = 1; } - /* Variable iNewStart now contains the first valid leaf node. */ - if( rc==SQLITE_OK && iNewStart ){ - sqlite3_stmt *pDel = 0; - rc = fts3SqlStmt(p, SQL_DELETE_SEGMENTS_RANGE, &pDel, 0); - if( rc==SQLITE_OK ){ - sqlite3_bind_int64(pDel, 1, iOldStart); - sqlite3_bind_int64(pDel, 2, iNewStart-1); - sqlite3_step(pDel); - rc = sqlite3_reset(pDel); + if( rc==SQLITE_OK ){ + xDestroyOut = fts3MIBufferAlloc(pCsr->pMIBuffer, &aOut); + if( xDestroyOut==0 ){ + rc = SQLITE_NOMEM; } } if( rc==SQLITE_OK ){ - sqlite3_stmt *pChomp = 0; - rc = fts3SqlStmt(p, SQL_CHOMP_SEGDIR, &pChomp, 0); - if( rc==SQLITE_OK ){ - sqlite3_bind_int64(pChomp, 1, iNewStart); - sqlite3_bind_blob(pChomp, 2, root.a, root.n, SQLITE_STATIC); - sqlite3_bind_int64(pChomp, 3, iAbsLevel); - sqlite3_bind_int(pChomp, 4, iIdx); - sqlite3_step(pChomp); - rc = sqlite3_reset(pChomp); - sqlite3_bind_null(pChomp, 2); + sInfo.aMatchinfo = aOut; + sInfo.nPhrase = pCsr->nPhrase; + rc = fts3MatchinfoValues(pCsr, bGlobal, &sInfo, zArg); + if( bGlobal ){ + fts3MIBufferSetGlobal(pCsr->pMIBuffer); } } - sqlite3_free(root.a); - sqlite3_free(block.a); - return rc; + if( rc!=SQLITE_OK ){ + sqlite3_result_error_code(pCtx, rc); + if( xDestroyOut ) xDestroyOut(aOut); + }else{ + int n = pCsr->pMIBuffer->nElem * sizeof(u32); + sqlite3_result_blob(pCtx, aOut, n, xDestroyOut); + } } /* -** This function is called after an incrmental-merge operation has run to -** merge (or partially merge) two or more segments from absolute level -** iAbsLevel. -** -** Each input segment is either removed from the db completely (if all of -** its data was copied to the output segment by the incrmerge operation) -** or modified in place so that it no longer contains those entries that -** have been duplicated in the output segment. +** Implementation of snippet() function. */ -static int fts3IncrmergeChomp( - Fts3Table *p, /* FTS table handle */ - sqlite3_int64 iAbsLevel, /* Absolute level containing segments */ - Fts3MultiSegReader *pCsr, /* Chomp all segments opened by this cursor */ - int *pnRem /* Number of segments not deleted */ +SQLITE_PRIVATE void sqlite3Fts3Snippet( + sqlite3_context *pCtx, /* SQLite function call context */ + Fts3Cursor *pCsr, /* Cursor object */ + const char *zStart, /* Snippet start text - "" */ + const char *zEnd, /* Snippet end text - "" */ + const char *zEllipsis, /* Snippet ellipsis text - "..." */ + int iCol, /* Extract snippet from this column */ + int nToken /* Approximate number of tokens in snippet */ ){ - int i; - int nRem = 0; + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; int rc = SQLITE_OK; + int i; + StrBuffer res = {0, 0, 0}; - for(i=pCsr->nSegment-1; i>=0 && rc==SQLITE_OK; i--){ - Fts3SegReader *pSeg = 0; - int j; + /* The returned text includes up to four fragments of text extracted from + ** the data in the current row. The first iteration of the for(...) loop + ** below attempts to locate a single fragment of text nToken tokens in + ** size that contains at least one instance of all phrases in the query + ** expression that appear in the current row. If such a fragment of text + ** cannot be found, the second iteration of the loop attempts to locate + ** a pair of fragments, and so on. + */ + int nSnippet = 0; /* Number of fragments in this snippet */ + SnippetFragment aSnippet[4]; /* Maximum of 4 fragments per snippet */ + int nFToken = -1; /* Number of tokens in each fragment */ - /* Find the Fts3SegReader object with Fts3SegReader.iIdx==i. It is hiding - ** somewhere in the pCsr->apSegment[] array. */ - for(j=0; ALWAYS(jnSegment); j++){ - pSeg = pCsr->apSegment[j]; - if( pSeg->iIdx==i ) break; + if( !pCsr->pExpr ){ + sqlite3_result_text(pCtx, "", 0, SQLITE_STATIC); + return; + } + + /* Limit the snippet length to 64 tokens. */ + if( nToken<-64 ) nToken = -64; + if( nToken>+64 ) nToken = +64; + + for(nSnippet=1; 1; nSnippet++){ + + int iSnip; /* Loop counter 0..nSnippet-1 */ + u64 mCovered = 0; /* Bitmask of phrases covered by snippet */ + u64 mSeen = 0; /* Bitmask of phrases seen by BestSnippet() */ + + if( nToken>=0 ){ + nFToken = (nToken+nSnippet-1) / nSnippet; + }else{ + nFToken = -1 * nToken; } - assert( jnSegment && pSeg->iIdx==i ); - if( pSeg->aNode==0 ){ - /* Seg-reader is at EOF. Remove the entire input segment. */ - rc = fts3DeleteSegment(p, pSeg); - if( rc==SQLITE_OK ){ - rc = fts3RemoveSegdirEntry(p, iAbsLevel, pSeg->iIdx); + for(iSnip=0; iSnipnColumn; iRead++){ + SnippetFragment sF = {0, 0, 0, 0}; + int iS = 0; + if( iCol>=0 && iRead!=iCol ) continue; + + /* Find the best snippet of nFToken tokens in column iRead. */ + rc = fts3BestSnippet(nFToken, pCsr, iRead, mCovered, &mSeen, &sF, &iS); + if( rc!=SQLITE_OK ){ + goto snippet_out; + } + if( iS>iBestScore ){ + *pFragment = sF; + iBestScore = iS; + } } - *pnRem = 0; - }else{ - /* The incremental merge did not copy all the data from this - ** segment to the upper level. The segment is modified in place - ** so that it contains no keys smaller than zTerm/nTerm. */ - const char *zTerm = pSeg->zTerm; - int nTerm = pSeg->nTerm; - rc = fts3TruncateSegment(p, iAbsLevel, pSeg->iIdx, zTerm, nTerm); - nRem++; + + mCovered |= pFragment->covered; } + + /* If all query phrases seen by fts3BestSnippet() are present in at least + ** one of the nSnippet snippet fragments, break out of the loop. + */ + assert( (mCovered&mSeen)==mCovered ); + if( mSeen==mCovered || nSnippet==SizeofArray(aSnippet) ) break; } - if( rc==SQLITE_OK && nRem!=pCsr->nSegment ){ - rc = fts3RepackSegdirLevel(p, iAbsLevel); + assert( nFToken>0 ); + + for(i=0; ia, pHint->n, SQLITE_STATIC); - sqlite3_step(pReplace); - rc = sqlite3_reset(pReplace); - sqlite3_bind_null(pReplace, 2); - } +typedef struct TermOffset TermOffset; +typedef struct TermOffsetCtx TermOffsetCtx; - return rc; -} +struct TermOffset { + char *pList; /* Position-list */ + int iPos; /* Position just read from pList */ + int iOff; /* Offset of this term from read positions */ +}; + +struct TermOffsetCtx { + Fts3Cursor *pCsr; + int iCol; /* Column of table to populate aTerm for */ + int iTerm; + sqlite3_int64 iDocid; + TermOffset *aTerm; +}; /* -** Load an incr-merge hint from the database. The incr-merge hint, if one -** exists, is stored in the rowid==1 row of the %_stat table. -** -** If successful, populate blob *pHint with the value read from the %_stat -** table and return SQLITE_OK. Otherwise, if an error occurs, return an -** SQLite error code. +** This function is an fts3ExprIterate() callback used by sqlite3Fts3Offsets(). */ -static int fts3IncrmergeHintLoad(Fts3Table *p, Blob *pHint){ - sqlite3_stmt *pSelect = 0; +static int fts3ExprTermOffsetInit(Fts3Expr *pExpr, int iPhrase, void *ctx){ + TermOffsetCtx *p = (TermOffsetCtx *)ctx; + int nTerm; /* Number of tokens in phrase */ + int iTerm; /* For looping through nTerm phrase terms */ + char *pList; /* Pointer to position list for phrase */ + int iPos = 0; /* First position in position-list */ int rc; - pHint->n = 0; - rc = fts3SqlStmt(p, SQL_SELECT_STAT, &pSelect, 0); - if( rc==SQLITE_OK ){ - int rc2; - sqlite3_bind_int(pSelect, 1, FTS_STAT_INCRMERGEHINT); - if( SQLITE_ROW==sqlite3_step(pSelect) ){ - const char *aHint = sqlite3_column_blob(pSelect, 0); - int nHint = sqlite3_column_bytes(pSelect, 0); - if( aHint ){ - blobGrowBuffer(pHint, nHint, &rc); - if( rc==SQLITE_OK ){ - memcpy(pHint->a, aHint, nHint); - pHint->n = nHint; - } - } - } - rc2 = sqlite3_reset(pSelect); - if( rc==SQLITE_OK ) rc = rc2; + UNUSED_PARAMETER(iPhrase); + rc = sqlite3Fts3EvalPhrasePoslist(p->pCsr, pExpr, p->iCol, &pList); + nTerm = pExpr->pPhrase->nToken; + if( pList ){ + fts3GetDeltaPosition(&pList, &iPos); + assert_fts3_nc( iPos>=0 ); + } + + for(iTerm=0; iTermaTerm[p->iTerm++]; + pT->iOff = nTerm-iTerm-1; + pT->pList = pList; + pT->iPos = iPos; } return rc; } /* -** If *pRc is not SQLITE_OK when this function is called, it is a no-op. -** Otherwise, append an entry to the hint stored in blob *pHint. Each entry -** consists of two varints, the absolute level number of the input segments -** and the number of input segments. -** -** If successful, leave *pRc set to SQLITE_OK and return. If an error occurs, -** set *pRc to an SQLite error code before returning. +** Implementation of offsets() function. */ -static void fts3IncrmergeHintPush( - Blob *pHint, /* Hint blob to append to */ - i64 iAbsLevel, /* First varint to store in hint */ - int nInput, /* Second varint to store in hint */ - int *pRc /* IN/OUT: Error code */ +SQLITE_PRIVATE void sqlite3Fts3Offsets( + sqlite3_context *pCtx, /* SQLite function call context */ + Fts3Cursor *pCsr /* Cursor object */ ){ - blobGrowBuffer(pHint, pHint->n + 2*FTS3_VARINT_MAX, pRc); - if( *pRc==SQLITE_OK ){ - pHint->n += sqlite3Fts3PutVarint(&pHint->a[pHint->n], iAbsLevel); - pHint->n += sqlite3Fts3PutVarint(&pHint->a[pHint->n], (i64)nInput); - } -} - -/* -** Read the last entry (most recently pushed) from the hint blob *pHint -** and then remove the entry. Write the two values read to *piAbsLevel and -** *pnInput before returning. -** -** If no error occurs, return SQLITE_OK. If the hint blob in *pHint does -** not contain at least two valid varints, return SQLITE_CORRUPT_VTAB. -*/ -static int fts3IncrmergeHintPop(Blob *pHint, i64 *piAbsLevel, int *pnInput){ - const int nHint = pHint->n; - int i; - - i = pHint->n-2; - while( i>0 && (pHint->a[i-1] & 0x80) ) i--; - while( i>0 && (pHint->a[i-1] & 0x80) ) i--; - - pHint->n = i; - i += sqlite3Fts3GetVarint(&pHint->a[i], piAbsLevel); - i += fts3GetVarint32(&pHint->a[i], pnInput); - if( i!=nHint ) return FTS_CORRUPT_VTAB; + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + sqlite3_tokenizer_module const *pMod = pTab->pTokenizer->pModule; + int rc; /* Return Code */ + int nToken; /* Number of tokens in query */ + int iCol; /* Column currently being processed */ + StrBuffer res = {0, 0, 0}; /* Result string */ + TermOffsetCtx sCtx; /* Context for fts3ExprTermOffsetInit() */ - return SQLITE_OK; -} + if( !pCsr->pExpr ){ + sqlite3_result_text(pCtx, "", 0, SQLITE_STATIC); + return; + } + memset(&sCtx, 0, sizeof(sCtx)); + assert( pCsr->isRequireSeek==0 ); -/* -** Attempt an incremental merge that writes nMerge leaf blocks. -** -** Incremental merges happen nMin segments at a time. The segments -** to be merged are the nMin oldest segments (the ones with the smallest -** values for the _segdir.idx field) in the highest level that contains -** at least nMin segments. Multiple merges might occur in an attempt to -** write the quota of nMerge leaf blocks. -*/ -SQLITE_PRIVATE int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ - int rc; /* Return code */ - int nRem = nMerge; /* Number of leaf pages yet to be written */ - Fts3MultiSegReader *pCsr; /* Cursor used to read input data */ - Fts3SegFilter *pFilter; /* Filter used with cursor pCsr */ - IncrmergeWriter *pWriter; /* Writer object */ - int nSeg = 0; /* Number of input segments */ - sqlite3_int64 iAbsLevel = 0; /* Absolute level number to work on */ - Blob hint = {0, 0, 0}; /* Hint read from %_stat table */ - int bDirtyHint = 0; /* True if blob 'hint' has been modified */ + /* Count the number of terms in the query */ + rc = fts3ExprLoadDoclists(pCsr, 0, &nToken); + if( rc!=SQLITE_OK ) goto offsets_out; - /* Allocate space for the cursor, filter and writer objects */ - const int nAlloc = sizeof(*pCsr) + sizeof(*pFilter) + sizeof(*pWriter); - pWriter = (IncrmergeWriter *)sqlite3_malloc(nAlloc); - if( !pWriter ) return SQLITE_NOMEM; - pFilter = (Fts3SegFilter *)&pWriter[1]; - pCsr = (Fts3MultiSegReader *)&pFilter[1]; + /* Allocate the array of TermOffset iterators. */ + sCtx.aTerm = (TermOffset *)sqlite3_malloc64(sizeof(TermOffset)*nToken); + if( 0==sCtx.aTerm ){ + rc = SQLITE_NOMEM; + goto offsets_out; + } + sCtx.iDocid = pCsr->iPrevId; + sCtx.pCsr = pCsr; - rc = fts3IncrmergeHintLoad(p, &hint); - while( rc==SQLITE_OK && nRem>0 ){ - const i64 nMod = FTS3_SEGDIR_MAXLEVEL * p->nIndex; - sqlite3_stmt *pFindLevel = 0; /* SQL used to determine iAbsLevel */ - int bUseHint = 0; /* True if attempting to append */ - int iIdx = 0; /* Largest idx in level (iAbsLevel+1) */ + /* Loop through the table columns, appending offset information to + ** string-buffer res for each column. + */ + for(iCol=0; iColnColumn; iCol++){ + sqlite3_tokenizer_cursor *pC; /* Tokenizer cursor */ + const char *ZDUMMY; /* Dummy argument used with xNext() */ + int NDUMMY = 0; /* Dummy argument used with xNext() */ + int iStart = 0; + int iEnd = 0; + int iCurrent = 0; + const char *zDoc; + int nDoc; - /* Search the %_segdir table for the absolute level with the smallest - ** relative level number that contains at least nMin segments, if any. - ** If one is found, set iAbsLevel to the absolute level number and - ** nSeg to nMin. If no level with at least nMin segments can be found, - ** set nSeg to -1. + /* Initialize the contents of sCtx.aTerm[] for column iCol. There is + ** no way that this operation can fail, so the return code from + ** fts3ExprIterate() can be discarded. */ - rc = fts3SqlStmt(p, SQL_FIND_MERGE_LEVEL, &pFindLevel, 0); - sqlite3_bind_int(pFindLevel, 1, MAX(2, nMin)); - if( sqlite3_step(pFindLevel)==SQLITE_ROW ){ - iAbsLevel = sqlite3_column_int64(pFindLevel, 0); - nSeg = sqlite3_column_int(pFindLevel, 1); - assert( nSeg>=2 ); - }else{ - nSeg = -1; - } - rc = sqlite3_reset(pFindLevel); + sCtx.iCol = iCol; + sCtx.iTerm = 0; + (void)fts3ExprIterate(pCsr->pExpr, fts3ExprTermOffsetInit, (void*)&sCtx); - /* If the hint read from the %_stat table is not empty, check if the - ** last entry in it specifies a relative level smaller than or equal - ** to the level identified by the block above (if any). If so, this - ** iteration of the loop will work on merging at the hinted level. + /* Retreive the text stored in column iCol. If an SQL NULL is stored + ** in column iCol, jump immediately to the next iteration of the loop. + ** If an OOM occurs while retrieving the data (this can happen if SQLite + ** needs to transform the data from utf-16 to utf-8), return SQLITE_NOMEM + ** to the caller. */ - if( rc==SQLITE_OK && hint.n ){ - int nHint = hint.n; - sqlite3_int64 iHintAbsLevel = 0; /* Hint level */ - int nHintSeg = 0; /* Hint number of segments */ - - rc = fts3IncrmergeHintPop(&hint, &iHintAbsLevel, &nHintSeg); - if( nSeg<0 || (iAbsLevel % nMod) >= (iHintAbsLevel % nMod) ){ - iAbsLevel = iHintAbsLevel; - nSeg = nHintSeg; - bUseHint = 1; - bDirtyHint = 1; - }else{ - /* This undoes the effect of the HintPop() above - so that no entry - ** is removed from the hint blob. */ - hint.n = nHint; + zDoc = (const char *)sqlite3_column_text(pCsr->pStmt, iCol+1); + nDoc = sqlite3_column_bytes(pCsr->pStmt, iCol+1); + if( zDoc==0 ){ + if( sqlite3_column_type(pCsr->pStmt, iCol+1)==SQLITE_NULL ){ + continue; } + rc = SQLITE_NOMEM; + goto offsets_out; } - /* If nSeg is less that zero, then there is no level with at least - ** nMin segments and no hint in the %_stat table. No work to do. - ** Exit early in this case. */ - if( nSeg<0 ) break; + /* Initialize a tokenizer iterator to iterate through column iCol. */ + rc = sqlite3Fts3OpenTokenizer(pTab->pTokenizer, pCsr->iLangid, + zDoc, nDoc, &pC + ); + if( rc!=SQLITE_OK ) goto offsets_out; - /* Open a cursor to iterate through the contents of the oldest nSeg - ** indexes of absolute level iAbsLevel. If this cursor is opened using - ** the 'hint' parameters, it is possible that there are less than nSeg - ** segments available in level iAbsLevel. In this case, no work is - ** done on iAbsLevel - fall through to the next iteration of the loop - ** to start work on some other level. */ - memset(pWriter, 0, nAlloc); - pFilter->flags = FTS3_SEGMENT_REQUIRE_POS; + rc = pMod->xNext(pC, &ZDUMMY, &NDUMMY, &iStart, &iEnd, &iCurrent); + while( rc==SQLITE_OK ){ + int i; /* Used to loop through terms */ + int iMinPos = 0x7FFFFFFF; /* Position of next token */ + TermOffset *pTerm = 0; /* TermOffset associated with next token */ - if( rc==SQLITE_OK ){ - rc = fts3IncrmergeOutputIdx(p, iAbsLevel, &iIdx); - assert( bUseHint==1 || bUseHint==0 ); - if( iIdx==0 || (bUseHint && iIdx==1) ){ - int bIgnore = 0; - rc = fts3SegmentIsMaxLevel(p, iAbsLevel+1, &bIgnore); - if( bIgnore ){ - pFilter->flags |= FTS3_SEGMENT_IGNORE_EMPTY; + for(i=0; ipList && (pT->iPos-pT->iOff)iPos-pT->iOff; + pTerm = pT; } } - } - if( rc==SQLITE_OK ){ - rc = fts3IncrmergeCsr(p, iAbsLevel, nSeg, pCsr); - } - if( SQLITE_OK==rc && pCsr->nSegment==nSeg - && SQLITE_OK==(rc = sqlite3Fts3SegReaderStart(p, pCsr, pFilter)) - && SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, pCsr)) - ){ - if( bUseHint && iIdx>0 ){ - const char *zKey = pCsr->zTerm; - int nKey = pCsr->nTerm; - rc = fts3IncrmergeLoad(p, iAbsLevel, iIdx-1, zKey, nKey, pWriter); + if( !pTerm ){ + /* All offsets for this column have been gathered. */ + rc = SQLITE_DONE; }else{ - rc = fts3IncrmergeWriter(p, iAbsLevel, iIdx, pCsr, pWriter); - } - - if( rc==SQLITE_OK && pWriter->nLeafEst ){ - fts3LogMerge(nSeg, iAbsLevel); - do { - rc = fts3IncrmergeAppend(p, pWriter, pCsr); - if( rc==SQLITE_OK ) rc = sqlite3Fts3SegReaderStep(p, pCsr); - if( pWriter->nWork>=nRem && rc==SQLITE_ROW ) rc = SQLITE_OK; - }while( rc==SQLITE_ROW ); - - /* Update or delete the input segments */ + assert_fts3_nc( iCurrent<=iMinPos ); + if( 0==(0xFE&*pTerm->pList) ){ + pTerm->pList = 0; + }else{ + fts3GetDeltaPosition(&pTerm->pList, &pTerm->iPos); + } + while( rc==SQLITE_OK && iCurrentxNext(pC, &ZDUMMY, &NDUMMY, &iStart, &iEnd, &iCurrent); + } if( rc==SQLITE_OK ){ - nRem -= (1 + pWriter->nWork); - rc = fts3IncrmergeChomp(p, iAbsLevel, pCsr, &nSeg); - if( nSeg!=0 ){ - bDirtyHint = 1; - fts3IncrmergeHintPush(&hint, iAbsLevel, nSeg, &rc); - } + char aBuffer[64]; + sqlite3_snprintf(sizeof(aBuffer), aBuffer, + "%d %d %d %d ", iCol, pTerm-sCtx.aTerm, iStart, iEnd-iStart + ); + rc = fts3StringAppend(&res, aBuffer, -1); + }else if( rc==SQLITE_DONE && pTab->zContentTbl==0 ){ + rc = FTS_CORRUPT_VTAB; } } - - if( nSeg!=0 ){ - pWriter->nLeafData = pWriter->nLeafData * -1; - } - fts3IncrmergeRelease(p, pWriter, &rc); - if( nSeg==0 && pWriter->bNoLeafData==0 ){ - fts3PromoteSegments(p, iAbsLevel+1, pWriter->nLeafData); - } + } + if( rc==SQLITE_DONE ){ + rc = SQLITE_OK; } - sqlite3Fts3SegReaderFinish(pCsr); + pMod->xClose(pC); + if( rc!=SQLITE_OK ) goto offsets_out; } - /* Write the hint values into the %_stat table for the next incr-merger */ - if( bDirtyHint && rc==SQLITE_OK ){ - rc = fts3IncrmergeHintStore(p, &hint); + offsets_out: + sqlite3_free(sCtx.aTerm); + assert( rc!=SQLITE_DONE ); + sqlite3Fts3SegmentsClose(pTab); + if( rc!=SQLITE_OK ){ + sqlite3_result_error_code(pCtx, rc); + sqlite3_free(res.z); + }else{ + sqlite3_result_text(pCtx, res.z, res.n-1, sqlite3_free); } - - sqlite3_free(pWriter); - sqlite3_free(hint.a); - return rc; -} - -/* -** Convert the text beginning at *pz into an integer and return -** its value. Advance *pz to point to the first character past -** the integer. -** -** This function used for parameters to merge= and incrmerge= -** commands. -*/ -static int fts3Getint(const char **pz){ - const char *z = *pz; - int i = 0; - while( (*z)>='0' && (*z)<='9' && i<214748363 ) i = 10*i + *(z++) - '0'; - *pz = z; - return i; + return; } /* -** Process statements of the form: -** -** INSERT INTO table(table) VALUES('merge=A,B'); -** -** A and B are integers that decode to be the number of leaf pages -** written for the merge, and the minimum number of segments on a level -** before it will be selected for a merge, respectively. +** Implementation of matchinfo() function. */ -static int fts3DoIncrmerge( - Fts3Table *p, /* FTS3 table handle */ - const char *zParam /* Nul-terminated string containing "A,B" */ +SQLITE_PRIVATE void sqlite3Fts3Matchinfo( + sqlite3_context *pContext, /* Function call context */ + Fts3Cursor *pCsr, /* FTS3 table cursor */ + const char *zArg /* Second arg to matchinfo() function */ ){ - int rc; - int nMin = (FTS3_MERGE_COUNT / 2); - int nMerge = 0; - const char *z = zParam; - - /* Read the first integer value */ - nMerge = fts3Getint(&z); + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + const char *zFormat; - /* If the first integer value is followed by a ',', read the second - ** integer value. */ - if( z[0]==',' && z[1]!='\0' ){ - z++; - nMin = fts3Getint(&z); + if( zArg ){ + zFormat = zArg; + }else{ + zFormat = FTS3_MATCHINFO_DEFAULT; } - if( z[0]!='\0' || nMin<2 ){ - rc = SQLITE_ERROR; + if( !pCsr->pExpr ){ + sqlite3_result_blob(pContext, "", 0, SQLITE_STATIC); + return; }else{ - rc = SQLITE_OK; - if( !p->bHasStat ){ - assert( p->bFts4==0 ); - sqlite3Fts3CreateStatTable(&rc, p); - } - if( rc==SQLITE_OK ){ - rc = sqlite3Fts3Incrmerge(p, nMerge, nMin); - } - sqlite3Fts3SegmentsClose(p); + /* Retrieve matchinfo() data. */ + fts3GetMatchinfo(pContext, pCsr, zFormat); + sqlite3Fts3SegmentsClose(pTab); } - return rc; } +#endif + +/************** End of fts3_snippet.c ****************************************/ +/************** Begin file fts3_unicode.c ************************************/ /* -** Process statements of the form: +** 2012 May 24 ** -** INSERT INTO table(table) VALUES('automerge=X'); +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: ** -** where X is an integer. X==0 means to turn automerge off. X!=0 means -** turn it on. The setting is persistent. -*/ -static int fts3DoAutoincrmerge( - Fts3Table *p, /* FTS3 table handle */ - const char *zParam /* Nul-terminated string containing boolean */ -){ - int rc = SQLITE_OK; - sqlite3_stmt *pStmt = 0; - p->nAutoincrmerge = fts3Getint(&zParam); - if( p->nAutoincrmerge==1 || p->nAutoincrmerge>FTS3_MERGE_COUNT ){ - p->nAutoincrmerge = 8; - } - if( !p->bHasStat ){ - assert( p->bFts4==0 ); - sqlite3Fts3CreateStatTable(&rc, p); - if( rc ) return rc; - } - rc = fts3SqlStmt(p, SQL_REPLACE_STAT, &pStmt, 0); - if( rc ) return rc; - sqlite3_bind_int(pStmt, 1, FTS_STAT_AUTOINCRMERGE); - sqlite3_bind_int(pStmt, 2, p->nAutoincrmerge); - sqlite3_step(pStmt); - rc = sqlite3_reset(pStmt); - return rc; -} - -/* -** Return a 64-bit checksum for the FTS index entry specified by the -** arguments to this function. -*/ -static u64 fts3ChecksumEntry( - const char *zTerm, /* Pointer to buffer containing term */ - int nTerm, /* Size of zTerm in bytes */ - int iLangid, /* Language id for current row */ - int iIndex, /* Index (0..Fts3Table.nIndex-1) */ - i64 iDocid, /* Docid for current row. */ - int iCol, /* Column number */ - int iPos /* Position */ -){ - int i; - u64 ret = (u64)iDocid; - - ret += (ret<<3) + iLangid; - ret += (ret<<3) + iIndex; - ret += (ret<<3) + iCol; - ret += (ret<<3) + iPos; - for(i=0; inIndex-1) */ - int *pRc /* OUT: Return code */ -){ - Fts3SegFilter filter; - Fts3MultiSegReader csr; - int rc; - u64 cksum = 0; - - assert( *pRc==SQLITE_OK ); - - memset(&filter, 0, sizeof(filter)); - memset(&csr, 0, sizeof(csr)); - filter.flags = FTS3_SEGMENT_REQUIRE_POS|FTS3_SEGMENT_IGNORE_EMPTY; - filter.flags |= FTS3_SEGMENT_SCAN; - - rc = sqlite3Fts3SegReaderCursor( - p, iLangid, iIndex, FTS3_SEGCURSOR_ALL, 0, 0, 0, 1,&csr - ); - if( rc==SQLITE_OK ){ - rc = sqlite3Fts3SegReaderStart(p, &csr, &filter); - } - if( rc==SQLITE_OK ){ - while( SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, &csr)) ){ - char *pCsr = csr.aDoclist; - char *pEnd = &pCsr[csr.nDoclist]; +#ifndef SQLITE_DISABLE_FTS3_UNICODE - i64 iDocid = 0; - i64 iCol = 0; - i64 iPos = 0; +/* #include "fts3Int.h" */ +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) - pCsr += sqlite3Fts3GetVarint(pCsr, &iDocid); - while( pCsr */ +/* #include */ +/* #include */ +/* #include */ - *pRc = rc; - return cksum; -} +/* #include "fts3_tokenizer.h" */ /* -** Check if the contents of the FTS index match the current contents of the -** content table. If no error occurs and the contents do match, set *pbOk -** to true and return SQLITE_OK. Or if the contents do not match, set *pbOk -** to false before returning. -** -** If an error occurs (e.g. an OOM or IO error), return an SQLite error -** code. The final value of *pbOk is undefined in this case. +** The following two macros - READ_UTF8 and WRITE_UTF8 - have been copied +** from the sqlite3 source file utf.c. If this file is compiled as part +** of the amalgamation, they are not required. */ -static int fts3IntegrityCheck(Fts3Table *p, int *pbOk){ - int rc = SQLITE_OK; /* Return code */ - u64 cksum1 = 0; /* Checksum based on FTS index contents */ - u64 cksum2 = 0; /* Checksum based on %_content contents */ - sqlite3_stmt *pAllLangid = 0; /* Statement to return all language-ids */ +#ifndef SQLITE_AMALGAMATION - /* This block calculates the checksum according to the FTS index. */ - rc = fts3SqlStmt(p, SQL_SELECT_ALL_LANGID, &pAllLangid, 0); - if( rc==SQLITE_OK ){ - int rc2; - sqlite3_bind_int(pAllLangid, 1, p->iPrevLangid); - sqlite3_bind_int(pAllLangid, 2, p->nIndex); - while( rc==SQLITE_OK && sqlite3_step(pAllLangid)==SQLITE_ROW ){ - int iLangid = sqlite3_column_int(pAllLangid, 0); - int i; - for(i=0; inIndex; i++){ - cksum1 = cksum1 ^ fts3ChecksumIndex(p, iLangid, i, &rc); - } - } - rc2 = sqlite3_reset(pAllLangid); - if( rc==SQLITE_OK ) rc = rc2; - } +static const unsigned char sqlite3Utf8Trans1[] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x00, 0x00, +}; - /* This block calculates the checksum according to the %_content table */ - if( rc==SQLITE_OK ){ - sqlite3_tokenizer_module const *pModule = p->pTokenizer->pModule; - sqlite3_stmt *pStmt = 0; - char *zSql; - - zSql = sqlite3_mprintf("SELECT %s" , p->zReadExprlist); - if( !zSql ){ - rc = SQLITE_NOMEM; - }else{ - rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); - sqlite3_free(zSql); - } +#define READ_UTF8(zIn, zTerm, c) \ + c = *(zIn++); \ + if( c>=0xc0 ){ \ + c = sqlite3Utf8Trans1[c-0xc0]; \ + while( zIn!=zTerm && (*zIn & 0xc0)==0x80 ){ \ + c = (c<<6) + (0x3f & *(zIn++)); \ + } \ + if( c<0x80 \ + || (c&0xFFFFF800)==0xD800 \ + || (c&0xFFFFFFFE)==0xFFFE ){ c = 0xFFFD; } \ + } - while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ - i64 iDocid = sqlite3_column_int64(pStmt, 0); - int iLang = langidFromSelect(p, pStmt); - int iCol; +#define WRITE_UTF8(zOut, c) { \ + if( c<0x00080 ){ \ + *zOut++ = (u8)(c&0xFF); \ + } \ + else if( c<0x00800 ){ \ + *zOut++ = 0xC0 + (u8)((c>>6)&0x1F); \ + *zOut++ = 0x80 + (u8)(c & 0x3F); \ + } \ + else if( c<0x10000 ){ \ + *zOut++ = 0xE0 + (u8)((c>>12)&0x0F); \ + *zOut++ = 0x80 + (u8)((c>>6) & 0x3F); \ + *zOut++ = 0x80 + (u8)(c & 0x3F); \ + }else{ \ + *zOut++ = 0xF0 + (u8)((c>>18) & 0x07); \ + *zOut++ = 0x80 + (u8)((c>>12) & 0x3F); \ + *zOut++ = 0x80 + (u8)((c>>6) & 0x3F); \ + *zOut++ = 0x80 + (u8)(c & 0x3F); \ + } \ +} - for(iCol=0; rc==SQLITE_OK && iColnColumn; iCol++){ - if( p->abNotindexed[iCol]==0 ){ - const char *zText = (const char *)sqlite3_column_text(pStmt, iCol+1); - int nText = sqlite3_column_bytes(pStmt, iCol+1); - sqlite3_tokenizer_cursor *pT = 0; +#endif /* ifndef SQLITE_AMALGAMATION */ - rc = sqlite3Fts3OpenTokenizer(p->pTokenizer, iLang, zText, nText,&pT); - while( rc==SQLITE_OK ){ - char const *zToken; /* Buffer containing token */ - int nToken = 0; /* Number of bytes in token */ - int iDum1 = 0, iDum2 = 0; /* Dummy variables */ - int iPos = 0; /* Position of token in zText */ +typedef struct unicode_tokenizer unicode_tokenizer; +typedef struct unicode_cursor unicode_cursor; - rc = pModule->xNext(pT, &zToken, &nToken, &iDum1, &iDum2, &iPos); - if( rc==SQLITE_OK ){ - int i; - cksum2 = cksum2 ^ fts3ChecksumEntry( - zToken, nToken, iLang, 0, iDocid, iCol, iPos - ); - for(i=1; inIndex; i++){ - if( p->aIndex[i].nPrefix<=nToken ){ - cksum2 = cksum2 ^ fts3ChecksumEntry( - zToken, p->aIndex[i].nPrefix, iLang, i, iDocid, iCol, iPos - ); - } - } - } - } - if( pT ) pModule->xClose(pT); - if( rc==SQLITE_DONE ) rc = SQLITE_OK; - } - } - } +struct unicode_tokenizer { + sqlite3_tokenizer base; + int eRemoveDiacritic; + int nException; + int *aiException; +}; - sqlite3_finalize(pStmt); - } +struct unicode_cursor { + sqlite3_tokenizer_cursor base; + const unsigned char *aInput; /* Input text being tokenized */ + int nInput; /* Size of aInput[] in bytes */ + int iOff; /* Current offset within aInput[] */ + int iToken; /* Index of next token to be returned */ + char *zToken; /* storage for current token */ + int nAlloc; /* space allocated at zToken */ +}; - *pbOk = (cksum1==cksum2); - return rc; -} /* -** Run the integrity-check. If no error occurs and the current contents of -** the FTS index are correct, return SQLITE_OK. Or, if the contents of the -** FTS index are incorrect, return SQLITE_CORRUPT_VTAB. -** -** Or, if an error (e.g. an OOM or IO error) occurs, return an SQLite -** error code. -** -** The integrity-check works as follows. For each token and indexed token -** prefix in the document set, a 64-bit checksum is calculated (by code -** in fts3ChecksumEntry()) based on the following: -** -** + The index number (0 for the main index, 1 for the first prefix -** index etc.), -** + The token (or token prefix) text itself, -** + The language-id of the row it appears in, -** + The docid of the row it appears in, -** + The column it appears in, and -** + The tokens position within that column. -** -** The checksums for all entries in the index are XORed together to create -** a single checksum for the entire index. -** -** The integrity-check code calculates the same checksum in two ways: -** -** 1. By scanning the contents of the FTS index, and -** 2. By scanning and tokenizing the content table. -** -** If the two checksums are identical, the integrity-check is deemed to have -** passed. +** Destroy a tokenizer allocated by unicodeCreate(). */ -static int fts3DoIntegrityCheck( - Fts3Table *p /* FTS3 table handle */ -){ - int rc; - int bOk = 0; - rc = fts3IntegrityCheck(p, &bOk); - if( rc==SQLITE_OK && bOk==0 ) rc = FTS_CORRUPT_VTAB; - return rc; +static int unicodeDestroy(sqlite3_tokenizer *pTokenizer){ + if( pTokenizer ){ + unicode_tokenizer *p = (unicode_tokenizer *)pTokenizer; + sqlite3_free(p->aiException); + sqlite3_free(p); + } + return SQLITE_OK; } /* -** Handle a 'special' INSERT of the form: +** As part of a tokenchars= or separators= option, the CREATE VIRTUAL TABLE +** statement has specified that the tokenizer for this table shall consider +** all characters in string zIn/nIn to be separators (if bAlnum==0) or +** token characters (if bAlnum==1). ** -** "INSERT INTO tbl(tbl) VALUES()" +** For each codepoint in the zIn/nIn string, this function checks if the +** sqlite3FtsUnicodeIsalnum() function already returns the desired result. +** If so, no action is taken. Otherwise, the codepoint is added to the +** unicode_tokenizer.aiException[] array. For the purposes of tokenization, +** the return value of sqlite3FtsUnicodeIsalnum() is inverted for all +** codepoints in the aiException[] array. ** -** Argument pVal contains the result of . Currently the only -** meaningful value to insert is the text 'optimize'. +** If a standalone diacritic mark (one that sqlite3FtsUnicodeIsdiacritic() +** identifies as a diacritic) occurs in the zIn/nIn string it is ignored. +** It is not possible to change the behavior of the tokenizer with respect +** to these codepoints. */ -static int fts3SpecialInsert(Fts3Table *p, sqlite3_value *pVal){ - int rc; /* Return Code */ - const char *zVal = (const char *)sqlite3_value_text(pVal); - int nVal = sqlite3_value_bytes(pVal); +static int unicodeAddExceptions( + unicode_tokenizer *p, /* Tokenizer to add exceptions to */ + int bAlnum, /* Replace Isalnum() return value with this */ + const char *zIn, /* Array of characters to make exceptions */ + int nIn /* Length of z in bytes */ +){ + const unsigned char *z = (const unsigned char *)zIn; + const unsigned char *zTerm = &z[nIn]; + unsigned int iCode; + int nEntry = 0; - if( !zVal ){ - return SQLITE_NOMEM; - }else if( nVal==8 && 0==sqlite3_strnicmp(zVal, "optimize", 8) ){ - rc = fts3DoOptimize(p, 0); - }else if( nVal==7 && 0==sqlite3_strnicmp(zVal, "rebuild", 7) ){ - rc = fts3DoRebuild(p); - }else if( nVal==15 && 0==sqlite3_strnicmp(zVal, "integrity-check", 15) ){ - rc = fts3DoIntegrityCheck(p); - }else if( nVal>6 && 0==sqlite3_strnicmp(zVal, "merge=", 6) ){ - rc = fts3DoIncrmerge(p, &zVal[6]); - }else if( nVal>10 && 0==sqlite3_strnicmp(zVal, "automerge=", 10) ){ - rc = fts3DoAutoincrmerge(p, &zVal[10]); -#ifdef SQLITE_TEST - }else if( nVal>9 && 0==sqlite3_strnicmp(zVal, "nodesize=", 9) ){ - p->nNodeSize = atoi(&zVal[9]); - rc = SQLITE_OK; - }else if( nVal>11 && 0==sqlite3_strnicmp(zVal, "maxpending=", 9) ){ - p->nMaxPendingData = atoi(&zVal[11]); - rc = SQLITE_OK; - }else if( nVal>21 && 0==sqlite3_strnicmp(zVal, "test-no-incr-doclist=", 21) ){ - p->bNoIncrDoclist = atoi(&zVal[21]); - rc = SQLITE_OK; -#endif - }else{ - rc = SQLITE_ERROR; + assert( bAlnum==0 || bAlnum==1 ); + + while( zaiException,(p->nException+nEntry)*sizeof(int)); + if( aNew==0 ) return SQLITE_NOMEM; + nNew = p->nException; + + z = (const unsigned char *)zIn; + while( zi; j--) aNew[j] = aNew[j-1]; + aNew[i] = (int)iCode; + nNew++; + } + } + p->aiException = aNew; + p->nException = nNew; + } + + return SQLITE_OK; } -#ifndef SQLITE_DISABLE_FTS4_DEFERRED /* -** Delete all cached deferred doclists. Deferred doclists are cached -** (allocated) by the sqlite3Fts3CacheDeferredDoclists() function. +** Return true if the p->aiException[] array contains the value iCode. */ -SQLITE_PRIVATE void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *pCsr){ - Fts3DeferredToken *pDef; - for(pDef=pCsr->pDeferred; pDef; pDef=pDef->pNext){ - fts3PendingListDelete(pDef->pList); - pDef->pList = 0; +static int unicodeIsException(unicode_tokenizer *p, int iCode){ + if( p->nException>0 ){ + int *a = p->aiException; + int iLo = 0; + int iHi = p->nException-1; + + while( iHi>=iLo ){ + int iTest = (iHi + iLo) / 2; + if( iCode==a[iTest] ){ + return 1; + }else if( iCode>a[iTest] ){ + iLo = iTest+1; + }else{ + iHi = iTest-1; + } + } } + + return 0; } /* -** Free all entries in the pCsr->pDeffered list. Entries are added to -** this list using sqlite3Fts3DeferToken(). +** Return true if, for the purposes of tokenization, codepoint iCode is +** considered a token character (not a separator). */ -SQLITE_PRIVATE void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *pCsr){ - Fts3DeferredToken *pDef; - Fts3DeferredToken *pNext; - for(pDef=pCsr->pDeferred; pDef; pDef=pNext){ - pNext = pDef->pNext; - fts3PendingListDelete(pDef->pList); - sqlite3_free(pDef); - } - pCsr->pDeferred = 0; +static int unicodeIsAlnum(unicode_tokenizer *p, int iCode){ + assert( (sqlite3FtsUnicodeIsalnum(iCode) & 0xFFFFFFFE)==0 ); + return sqlite3FtsUnicodeIsalnum(iCode) ^ unicodeIsException(p, iCode); } /* -** Generate deferred-doclists for all tokens in the pCsr->pDeferred list -** based on the row that pCsr currently points to. -** -** A deferred-doclist is like any other doclist with position information -** included, except that it only contains entries for a single row of the -** table, not for all rows. +** Create a new tokenizer instance. */ -SQLITE_PRIVATE int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *pCsr){ - int rc = SQLITE_OK; /* Return code */ - if( pCsr->pDeferred ){ - int i; /* Used to iterate through table columns */ - sqlite3_int64 iDocid; /* Docid of the row pCsr points to */ - Fts3DeferredToken *pDef; /* Used to iterate through deferred tokens */ - - Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; - sqlite3_tokenizer *pT = p->pTokenizer; - sqlite3_tokenizer_module const *pModule = pT->pModule; - - assert( pCsr->isRequireSeek==0 ); - iDocid = sqlite3_column_int64(pCsr->pStmt, 0); - - for(i=0; inColumn && rc==SQLITE_OK; i++){ - if( p->abNotindexed[i]==0 ){ - const char *zText = (const char *)sqlite3_column_text(pCsr->pStmt, i+1); - sqlite3_tokenizer_cursor *pTC = 0; +static int unicodeCreate( + int nArg, /* Size of array argv[] */ + const char * const *azArg, /* Tokenizer creation arguments */ + sqlite3_tokenizer **pp /* OUT: New tokenizer handle */ +){ + unicode_tokenizer *pNew; /* New tokenizer object */ + int i; + int rc = SQLITE_OK; - rc = sqlite3Fts3OpenTokenizer(pT, pCsr->iLangid, zText, -1, &pTC); - while( rc==SQLITE_OK ){ - char const *zToken; /* Buffer containing token */ - int nToken = 0; /* Number of bytes in token */ - int iDum1 = 0, iDum2 = 0; /* Dummy variables */ - int iPos = 0; /* Position of token in zText */ + pNew = (unicode_tokenizer *) sqlite3_malloc(sizeof(unicode_tokenizer)); + if( pNew==NULL ) return SQLITE_NOMEM; + memset(pNew, 0, sizeof(unicode_tokenizer)); + pNew->eRemoveDiacritic = 1; - rc = pModule->xNext(pTC, &zToken, &nToken, &iDum1, &iDum2, &iPos); - for(pDef=pCsr->pDeferred; pDef && rc==SQLITE_OK; pDef=pDef->pNext){ - Fts3PhraseToken *pPT = pDef->pToken; - if( (pDef->iCol>=p->nColumn || pDef->iCol==i) - && (pPT->bFirst==0 || iPos==0) - && (pPT->n==nToken || (pPT->isPrefix && pPT->nz, pPT->n)) - ){ - fts3PendingListAppend(&pDef->pList, iDocid, i, iPos, &rc); - } - } - } - if( pTC ) pModule->xClose(pTC); - if( rc==SQLITE_DONE ) rc = SQLITE_OK; - } - } + for(i=0; rc==SQLITE_OK && ipDeferred; pDef && rc==SQLITE_OK; pDef=pDef->pNext){ - if( pDef->pList ){ - rc = fts3PendingListAppendVarint(&pDef->pList, 0); - } + if( n==19 && memcmp("remove_diacritics=1", z, 19)==0 ){ + pNew->eRemoveDiacritic = 1; + } + else if( n==19 && memcmp("remove_diacritics=0", z, 19)==0 ){ + pNew->eRemoveDiacritic = 0; + } + else if( n==19 && memcmp("remove_diacritics=2", z, 19)==0 ){ + pNew->eRemoveDiacritic = 2; + } + else if( n>=11 && memcmp("tokenchars=", z, 11)==0 ){ + rc = unicodeAddExceptions(pNew, 1, &z[11], n-11); + } + else if( n>=11 && memcmp("separators=", z, 11)==0 ){ + rc = unicodeAddExceptions(pNew, 0, &z[11], n-11); + } + else{ + /* Unrecognized argument */ + rc = SQLITE_ERROR; } } + if( rc!=SQLITE_OK ){ + unicodeDestroy((sqlite3_tokenizer *)pNew); + pNew = 0; + } + *pp = (sqlite3_tokenizer *)pNew; return rc; } -SQLITE_PRIVATE int sqlite3Fts3DeferredTokenList( - Fts3DeferredToken *p, - char **ppData, - int *pnData +/* +** Prepare to begin tokenizing a particular string. The input +** string to be tokenized is pInput[0..nBytes-1]. A cursor +** used to incrementally tokenize this string is returned in +** *ppCursor. +*/ +static int unicodeOpen( + sqlite3_tokenizer *p, /* The tokenizer */ + const char *aInput, /* Input string */ + int nInput, /* Size of string aInput in bytes */ + sqlite3_tokenizer_cursor **pp /* OUT: New cursor object */ ){ - char *pRet; - int nSkip; - sqlite3_int64 dummy; + unicode_cursor *pCsr; - *ppData = 0; - *pnData = 0; + pCsr = (unicode_cursor *)sqlite3_malloc(sizeof(unicode_cursor)); + if( pCsr==0 ){ + return SQLITE_NOMEM; + } + memset(pCsr, 0, sizeof(unicode_cursor)); - if( p->pList==0 ){ - return SQLITE_OK; + pCsr->aInput = (const unsigned char *)aInput; + if( aInput==0 ){ + pCsr->nInput = 0; + }else if( nInput<0 ){ + pCsr->nInput = (int)strlen(aInput); + }else{ + pCsr->nInput = nInput; } - pRet = (char *)sqlite3_malloc(p->pList->nData); - if( !pRet ) return SQLITE_NOMEM; + *pp = &pCsr->base; + UNUSED_PARAMETER(p); + return SQLITE_OK; +} - nSkip = sqlite3Fts3GetVarint(p->pList->aData, &dummy); - *pnData = p->pList->nData - nSkip; - *ppData = pRet; - - memcpy(pRet, &p->pList->aData[nSkip], *pnData); +/* +** Close a tokenization cursor previously opened by a call to +** simpleOpen() above. +*/ +static int unicodeClose(sqlite3_tokenizer_cursor *pCursor){ + unicode_cursor *pCsr = (unicode_cursor *) pCursor; + sqlite3_free(pCsr->zToken); + sqlite3_free(pCsr); return SQLITE_OK; } /* -** Add an entry for token pToken to the pCsr->pDeferred list. +** Extract the next token from a tokenization cursor. The cursor must +** have been opened by a prior call to simpleOpen(). */ -SQLITE_PRIVATE int sqlite3Fts3DeferToken( - Fts3Cursor *pCsr, /* Fts3 table cursor */ - Fts3PhraseToken *pToken, /* Token to defer */ - int iCol /* Column that token must appear in (or -1) */ +static int unicodeNext( + sqlite3_tokenizer_cursor *pC, /* Cursor returned by simpleOpen */ + const char **paToken, /* OUT: Token text */ + int *pnToken, /* OUT: Number of bytes at *paToken */ + int *piStart, /* OUT: Starting offset of token */ + int *piEnd, /* OUT: Ending offset of token */ + int *piPos /* OUT: Position integer of token */ ){ - Fts3DeferredToken *pDeferred; - pDeferred = sqlite3_malloc(sizeof(*pDeferred)); - if( !pDeferred ){ - return SQLITE_NOMEM; + unicode_cursor *pCsr = (unicode_cursor *)pC; + unicode_tokenizer *p = ((unicode_tokenizer *)pCsr->base.pTokenizer); + unsigned int iCode = 0; + char *zOut; + const unsigned char *z = &pCsr->aInput[pCsr->iOff]; + const unsigned char *zStart = z; + const unsigned char *zEnd; + const unsigned char *zTerm = &pCsr->aInput[pCsr->nInput]; + + /* Scan past any delimiter characters before the start of the next token. + ** Return SQLITE_DONE early if this takes us all the way to the end of + ** the input. */ + while( zpToken = pToken; - pDeferred->pNext = pCsr->pDeferred; - pDeferred->iCol = iCol; - pCsr->pDeferred = pDeferred; + if( zStart>=zTerm ) return SQLITE_DONE; - assert( pToken->pDeferred==0 ); - pToken->pDeferred = pDeferred; + zOut = pCsr->zToken; + do { + int iOut; + + /* Grow the output buffer if required. */ + if( (zOut-pCsr->zToken)>=(pCsr->nAlloc-4) ){ + char *zNew = sqlite3_realloc64(pCsr->zToken, pCsr->nAlloc+64); + if( !zNew ) return SQLITE_NOMEM; + zOut = &zNew[zOut - pCsr->zToken]; + pCsr->zToken = zNew; + pCsr->nAlloc += 64; + } + + /* Write the folded case of the last character read to the output */ + zEnd = z; + iOut = sqlite3FtsUnicodeFold((int)iCode, p->eRemoveDiacritic); + if( iOut ){ + WRITE_UTF8(zOut, iOut); + } + + /* If the cursor is not at EOF, read the next character */ + if( z>=zTerm ) break; + READ_UTF8(z, zTerm, iCode); + }while( unicodeIsAlnum(p, (int)iCode) + || sqlite3FtsUnicodeIsdiacritic((int)iCode) + ); + /* Set the output variables and return. */ + pCsr->iOff = (int)(z - pCsr->aInput); + *paToken = pCsr->zToken; + *pnToken = (int)(zOut - pCsr->zToken); + *piStart = (int)(zStart - pCsr->aInput); + *piEnd = (int)(zEnd - pCsr->aInput); + *piPos = pCsr->iToken++; return SQLITE_OK; } -#endif /* -** SQLite value pRowid contains the rowid of a row that may or may not be -** present in the FTS3 table. If it is, delete it and adjust the contents -** of subsiduary data structures accordingly. +** Set *ppModule to a pointer to the sqlite3_tokenizer_module +** structure for the unicode tokenizer. */ -static int fts3DeleteByRowid( - Fts3Table *p, - sqlite3_value *pRowid, - int *pnChng, /* IN/OUT: Decrement if row is deleted */ - u32 *aSzDel -){ - int rc = SQLITE_OK; /* Return code */ - int bFound = 0; /* True if *pRowid really is in the table */ - - fts3DeleteTerms(&rc, p, pRowid, aSzDel, &bFound); - if( bFound && rc==SQLITE_OK ){ - int isEmpty = 0; /* Deleting *pRowid leaves the table empty */ - rc = fts3IsEmpty(p, pRowid, &isEmpty); - if( rc==SQLITE_OK ){ - if( isEmpty ){ - /* Deleting this row means the whole table is empty. In this case - ** delete the contents of all three tables and throw away any - ** data in the pendingTerms hash table. */ - rc = fts3DeleteAll(p, 1); - *pnChng = 0; - memset(aSzDel, 0, sizeof(u32) * (p->nColumn+1) * 2); - }else{ - *pnChng = *pnChng - 1; - if( p->zContentTbl==0 ){ - fts3SqlExec(&rc, p, SQL_DELETE_CONTENT, &pRowid); - } - if( p->bHasDocsize ){ - fts3SqlExec(&rc, p, SQL_DELETE_DOCSIZE, &pRowid); - } - } - } - } - - return rc; +SQLITE_PRIVATE void sqlite3Fts3UnicodeTokenizer(sqlite3_tokenizer_module const **ppModule){ + static const sqlite3_tokenizer_module module = { + 0, + unicodeCreate, + unicodeDestroy, + unicodeOpen, + unicodeClose, + unicodeNext, + 0, + }; + *ppModule = &module; } +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ +#endif /* ifndef SQLITE_DISABLE_FTS3_UNICODE */ + +/************** End of fts3_unicode.c ****************************************/ +/************** Begin file fts3_unicode2.c ***********************************/ /* -** This function does the work for the xUpdate method of FTS3 virtual -** tables. The schema of the virtual table being: +** 2012-05-25 ** -** CREATE TABLE
        ( -** , -**
        HIDDEN, -** docid HIDDEN, -** HIDDEN -** ); +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: ** -** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** */ -SQLITE_PRIVATE int sqlite3Fts3UpdateMethod( - sqlite3_vtab *pVtab, /* FTS3 vtab object */ - int nArg, /* Size of argument array */ - sqlite3_value **apVal, /* Array of arguments */ - sqlite_int64 *pRowid /* OUT: The affected (or effected) rowid */ -){ - Fts3Table *p = (Fts3Table *)pVtab; - int rc = SQLITE_OK; /* Return Code */ - u32 *aSzIns = 0; /* Sizes of inserted documents */ - u32 *aSzDel = 0; /* Sizes of deleted documents */ - int nChng = 0; /* Net change in number of documents */ - int bInsertDone = 0; - /* At this point it must be known if the %_stat table exists or not. - ** So bHasStat may not be 2. */ - assert( p->bHasStat==0 || p->bHasStat==1 ); +/* +** DO NOT EDIT THIS MACHINE GENERATED FILE. +*/ - assert( p->pSegments==0 ); - assert( - nArg==1 /* DELETE operations */ - || nArg==(2 + p->nColumn + 3) /* INSERT or UPDATE operations */ - ); +#ifndef SQLITE_DISABLE_FTS3_UNICODE +#if defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4) - /* Check for a "special" INSERT operation. One of the form: +/* #include */ + +/* +** Return true if the argument corresponds to a unicode codepoint +** classified as either a letter or a number. Otherwise false. +** +** The results are undefined if the value passed to this function +** is less than zero. +*/ +SQLITE_PRIVATE int sqlite3FtsUnicodeIsalnum(int c){ + /* Each unsigned integer in the following array corresponds to a contiguous + ** range of unicode codepoints that are not either letters or numbers (i.e. + ** codepoints for which this function should return 0). ** - ** INSERT INTO xyz(xyz) VALUES('command'); + ** The most significant 22 bits in each 32-bit value contain the first + ** codepoint in the range. The least significant 10 bits are used to store + ** the size of the range (always at least 1). In other words, the value + ** ((C<<22) + N) represents a range of N codepoints starting with codepoint + ** C. It is not possible to represent a range larger than 1023 codepoints + ** using this format. */ - if( nArg>1 - && sqlite3_value_type(apVal[0])==SQLITE_NULL - && sqlite3_value_type(apVal[p->nColumn+2])!=SQLITE_NULL - ){ - rc = fts3SpecialInsert(p, apVal[p->nColumn+2]); - goto update_out; - } + static const unsigned int aEntry[] = { + 0x00000030, 0x0000E807, 0x00016C06, 0x0001EC2F, 0x0002AC07, + 0x0002D001, 0x0002D803, 0x0002EC01, 0x0002FC01, 0x00035C01, + 0x0003DC01, 0x000B0804, 0x000B480E, 0x000B9407, 0x000BB401, + 0x000BBC81, 0x000DD401, 0x000DF801, 0x000E1002, 0x000E1C01, + 0x000FD801, 0x00120808, 0x00156806, 0x00162402, 0x00163C01, + 0x00164437, 0x0017CC02, 0x00180005, 0x00181816, 0x00187802, + 0x00192C15, 0x0019A804, 0x0019C001, 0x001B5001, 0x001B580F, + 0x001B9C07, 0x001BF402, 0x001C000E, 0x001C3C01, 0x001C4401, + 0x001CC01B, 0x001E980B, 0x001FAC09, 0x001FD804, 0x00205804, + 0x00206C09, 0x00209403, 0x0020A405, 0x0020C00F, 0x00216403, + 0x00217801, 0x0023901B, 0x00240004, 0x0024E803, 0x0024F812, + 0x00254407, 0x00258804, 0x0025C001, 0x00260403, 0x0026F001, + 0x0026F807, 0x00271C02, 0x00272C03, 0x00275C01, 0x00278802, + 0x0027C802, 0x0027E802, 0x00280403, 0x0028F001, 0x0028F805, + 0x00291C02, 0x00292C03, 0x00294401, 0x0029C002, 0x0029D401, + 0x002A0403, 0x002AF001, 0x002AF808, 0x002B1C03, 0x002B2C03, + 0x002B8802, 0x002BC002, 0x002C0403, 0x002CF001, 0x002CF807, + 0x002D1C02, 0x002D2C03, 0x002D5802, 0x002D8802, 0x002DC001, + 0x002E0801, 0x002EF805, 0x002F1803, 0x002F2804, 0x002F5C01, + 0x002FCC08, 0x00300403, 0x0030F807, 0x00311803, 0x00312804, + 0x00315402, 0x00318802, 0x0031FC01, 0x00320802, 0x0032F001, + 0x0032F807, 0x00331803, 0x00332804, 0x00335402, 0x00338802, + 0x00340802, 0x0034F807, 0x00351803, 0x00352804, 0x00355C01, + 0x00358802, 0x0035E401, 0x00360802, 0x00372801, 0x00373C06, + 0x00375801, 0x00376008, 0x0037C803, 0x0038C401, 0x0038D007, + 0x0038FC01, 0x00391C09, 0x00396802, 0x003AC401, 0x003AD006, + 0x003AEC02, 0x003B2006, 0x003C041F, 0x003CD00C, 0x003DC417, + 0x003E340B, 0x003E6424, 0x003EF80F, 0x003F380D, 0x0040AC14, + 0x00412806, 0x00415804, 0x00417803, 0x00418803, 0x00419C07, + 0x0041C404, 0x0042080C, 0x00423C01, 0x00426806, 0x0043EC01, + 0x004D740C, 0x004E400A, 0x00500001, 0x0059B402, 0x005A0001, + 0x005A6C02, 0x005BAC03, 0x005C4803, 0x005CC805, 0x005D4802, + 0x005DC802, 0x005ED023, 0x005F6004, 0x005F7401, 0x0060000F, + 0x0062A401, 0x0064800C, 0x0064C00C, 0x00650001, 0x00651002, + 0x0066C011, 0x00672002, 0x00677822, 0x00685C05, 0x00687802, + 0x0069540A, 0x0069801D, 0x0069FC01, 0x006A8007, 0x006AA006, + 0x006C0005, 0x006CD011, 0x006D6823, 0x006E0003, 0x006E840D, + 0x006F980E, 0x006FF004, 0x00709014, 0x0070EC05, 0x0071F802, + 0x00730008, 0x00734019, 0x0073B401, 0x0073C803, 0x00770027, + 0x0077F004, 0x007EF401, 0x007EFC03, 0x007F3403, 0x007F7403, + 0x007FB403, 0x007FF402, 0x00800065, 0x0081A806, 0x0081E805, + 0x00822805, 0x0082801A, 0x00834021, 0x00840002, 0x00840C04, + 0x00842002, 0x00845001, 0x00845803, 0x00847806, 0x00849401, + 0x00849C01, 0x0084A401, 0x0084B801, 0x0084E802, 0x00850005, + 0x00852804, 0x00853C01, 0x00864264, 0x00900027, 0x0091000B, + 0x0092704E, 0x00940200, 0x009C0475, 0x009E53B9, 0x00AD400A, + 0x00B39406, 0x00B3BC03, 0x00B3E404, 0x00B3F802, 0x00B5C001, + 0x00B5FC01, 0x00B7804F, 0x00B8C00C, 0x00BA001A, 0x00BA6C59, + 0x00BC00D6, 0x00BFC00C, 0x00C00005, 0x00C02019, 0x00C0A807, + 0x00C0D802, 0x00C0F403, 0x00C26404, 0x00C28001, 0x00C3EC01, + 0x00C64002, 0x00C6580A, 0x00C70024, 0x00C8001F, 0x00C8A81E, + 0x00C94001, 0x00C98020, 0x00CA2827, 0x00CB003F, 0x00CC0100, + 0x01370040, 0x02924037, 0x0293F802, 0x02983403, 0x0299BC10, + 0x029A7C01, 0x029BC008, 0x029C0017, 0x029C8002, 0x029E2402, + 0x02A00801, 0x02A01801, 0x02A02C01, 0x02A08C09, 0x02A0D804, + 0x02A1D004, 0x02A20002, 0x02A2D011, 0x02A33802, 0x02A38012, + 0x02A3E003, 0x02A4980A, 0x02A51C0D, 0x02A57C01, 0x02A60004, + 0x02A6CC1B, 0x02A77802, 0x02A8A40E, 0x02A90C01, 0x02A93002, + 0x02A97004, 0x02A9DC03, 0x02A9EC01, 0x02AAC001, 0x02AAC803, + 0x02AADC02, 0x02AAF802, 0x02AB0401, 0x02AB7802, 0x02ABAC07, + 0x02ABD402, 0x02AF8C0B, 0x03600001, 0x036DFC02, 0x036FFC02, + 0x037FFC01, 0x03EC7801, 0x03ECA401, 0x03EEC810, 0x03F4F802, + 0x03F7F002, 0x03F8001A, 0x03F88007, 0x03F8C023, 0x03F95013, + 0x03F9A004, 0x03FBFC01, 0x03FC040F, 0x03FC6807, 0x03FCEC06, + 0x03FD6C0B, 0x03FF8007, 0x03FFA007, 0x03FFE405, 0x04040003, + 0x0404DC09, 0x0405E411, 0x0406400C, 0x0407402E, 0x040E7C01, + 0x040F4001, 0x04215C01, 0x04247C01, 0x0424FC01, 0x04280403, + 0x04281402, 0x04283004, 0x0428E003, 0x0428FC01, 0x04294009, + 0x0429FC01, 0x042CE407, 0x04400003, 0x0440E016, 0x04420003, + 0x0442C012, 0x04440003, 0x04449C0E, 0x04450004, 0x04460003, + 0x0446CC0E, 0x04471404, 0x045AAC0D, 0x0491C004, 0x05BD442E, + 0x05BE3C04, 0x074000F6, 0x07440027, 0x0744A4B5, 0x07480046, + 0x074C0057, 0x075B0401, 0x075B6C01, 0x075BEC01, 0x075C5401, + 0x075CD401, 0x075D3C01, 0x075DBC01, 0x075E2401, 0x075EA401, + 0x075F0C01, 0x07BBC002, 0x07C0002C, 0x07C0C064, 0x07C2800F, + 0x07C2C40E, 0x07C3040F, 0x07C3440F, 0x07C4401F, 0x07C4C03C, + 0x07C5C02B, 0x07C7981D, 0x07C8402B, 0x07C90009, 0x07C94002, + 0x07CC0021, 0x07CCC006, 0x07CCDC46, 0x07CE0014, 0x07CE8025, + 0x07CF1805, 0x07CF8011, 0x07D0003F, 0x07D10001, 0x07D108B6, + 0x07D3E404, 0x07D4003E, 0x07D50004, 0x07D54018, 0x07D7EC46, + 0x07D9140B, 0x07DA0046, 0x07DC0074, 0x38000401, 0x38008060, + 0x380400F0, + }; + static const unsigned int aAscii[4] = { + 0xFFFFFFFF, 0xFC00FFFF, 0xF8000001, 0xF8000001, + }; - if( nArg>1 && sqlite3_value_int(apVal[2 + p->nColumn + 2])<0 ){ - rc = SQLITE_CONSTRAINT; - goto update_out; + if( (unsigned int)c<128 ){ + return ( (aAscii[c >> 5] & ((unsigned int)1 << (c & 0x001F)))==0 ); + }else if( (unsigned int)c<(1<<22) ){ + unsigned int key = (((unsigned int)c)<<10) | 0x000003FF; + int iRes = 0; + int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1; + int iLo = 0; + while( iHi>=iLo ){ + int iTest = (iHi + iLo) / 2; + if( key >= aEntry[iTest] ){ + iRes = iTest; + iLo = iTest+1; + }else{ + iHi = iTest-1; + } + } + assert( aEntry[0]=aEntry[iRes] ); + return (((unsigned int)c) >= ((aEntry[iRes]>>10) + (aEntry[iRes]&0x3FF))); } + return 1; +} - /* Allocate space to hold the change in document sizes */ - aSzDel = sqlite3_malloc( sizeof(aSzDel[0])*(p->nColumn+1)*2 ); - if( aSzDel==0 ){ - rc = SQLITE_NOMEM; - goto update_out; + +/* +** If the argument is a codepoint corresponding to a lowercase letter +** in the ASCII range with a diacritic added, return the codepoint +** of the ASCII letter only. For example, if passed 235 - "LATIN +** SMALL LETTER E WITH DIAERESIS" - return 65 ("LATIN SMALL LETTER +** E"). The resuls of passing a codepoint that corresponds to an +** uppercase letter are undefined. +*/ +static int remove_diacritic(int c, int bComplex){ + unsigned short aDia[] = { + 0, 1797, 1848, 1859, 1891, 1928, 1940, 1995, + 2024, 2040, 2060, 2110, 2168, 2206, 2264, 2286, + 2344, 2383, 2472, 2488, 2516, 2596, 2668, 2732, + 2782, 2842, 2894, 2954, 2984, 3000, 3028, 3336, + 3456, 3696, 3712, 3728, 3744, 3766, 3832, 3896, + 3912, 3928, 3944, 3968, 4008, 4040, 4056, 4106, + 4138, 4170, 4202, 4234, 4266, 4296, 4312, 4344, + 4408, 4424, 4442, 4472, 4488, 4504, 6148, 6198, + 6264, 6280, 6360, 6429, 6505, 6529, 61448, 61468, + 61512, 61534, 61592, 61610, 61642, 61672, 61688, 61704, + 61726, 61784, 61800, 61816, 61836, 61880, 61896, 61914, + 61948, 61998, 62062, 62122, 62154, 62184, 62200, 62218, + 62252, 62302, 62364, 62410, 62442, 62478, 62536, 62554, + 62584, 62604, 62640, 62648, 62656, 62664, 62730, 62766, + 62830, 62890, 62924, 62974, 63032, 63050, 63082, 63118, + 63182, 63242, 63274, 63310, 63368, 63390, + }; +#define HIBIT ((unsigned char)0x80) + unsigned char aChar[] = { + '\0', 'a', 'c', 'e', 'i', 'n', + 'o', 'u', 'y', 'y', 'a', 'c', + 'd', 'e', 'e', 'g', 'h', 'i', + 'j', 'k', 'l', 'n', 'o', 'r', + 's', 't', 'u', 'u', 'w', 'y', + 'z', 'o', 'u', 'a', 'i', 'o', + 'u', 'u'|HIBIT, 'a'|HIBIT, 'g', 'k', 'o', + 'o'|HIBIT, 'j', 'g', 'n', 'a'|HIBIT, 'a', + 'e', 'i', 'o', 'r', 'u', 's', + 't', 'h', 'a', 'e', 'o'|HIBIT, 'o', + 'o'|HIBIT, 'y', '\0', '\0', '\0', '\0', + '\0', '\0', '\0', '\0', 'a', 'b', + 'c'|HIBIT, 'd', 'd', 'e'|HIBIT, 'e', 'e'|HIBIT, + 'f', 'g', 'h', 'h', 'i', 'i'|HIBIT, + 'k', 'l', 'l'|HIBIT, 'l', 'm', 'n', + 'o'|HIBIT, 'p', 'r', 'r'|HIBIT, 'r', 's', + 's'|HIBIT, 't', 'u', 'u'|HIBIT, 'v', 'w', + 'w', 'x', 'y', 'z', 'h', 't', + 'w', 'y', 'a', 'a'|HIBIT, 'a'|HIBIT, 'a'|HIBIT, + 'e', 'e'|HIBIT, 'e'|HIBIT, 'i', 'o', 'o'|HIBIT, + 'o'|HIBIT, 'o'|HIBIT, 'u', 'u'|HIBIT, 'u'|HIBIT, 'y', + }; + + unsigned int key = (((unsigned int)c)<<3) | 0x00000007; + int iRes = 0; + int iHi = sizeof(aDia)/sizeof(aDia[0]) - 1; + int iLo = 0; + while( iHi>=iLo ){ + int iTest = (iHi + iLo) / 2; + if( key >= aDia[iTest] ){ + iRes = iTest; + iLo = iTest+1; + }else{ + iHi = iTest-1; + } } - aSzIns = &aSzDel[p->nColumn+1]; - memset(aSzDel, 0, sizeof(aSzDel[0])*(p->nColumn+1)*2); + assert( key>=aDia[iRes] ); + if( bComplex==0 && (aChar[iRes] & 0x80) ) return c; + return (c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : ((int)aChar[iRes] & 0x7F); +} - rc = fts3Writelock(p); - if( rc!=SQLITE_OK ) goto update_out; - /* If this is an INSERT operation, or an UPDATE that modifies the rowid - ** value, then this operation requires constraint handling. +/* +** Return true if the argument interpreted as a unicode codepoint +** is a diacritical modifier character. +*/ +SQLITE_PRIVATE int sqlite3FtsUnicodeIsdiacritic(int c){ + unsigned int mask0 = 0x08029FDF; + unsigned int mask1 = 0x000361F8; + if( c<768 || c>817 ) return 0; + return (c < 768+32) ? + (mask0 & ((unsigned int)1 << (c-768))) : + (mask1 & ((unsigned int)1 << (c-768-32))); +} + + +/* +** Interpret the argument as a unicode codepoint. If the codepoint +** is an upper case character that has a lower case equivalent, +** return the codepoint corresponding to the lower case version. +** Otherwise, return a copy of the argument. +** +** The results are undefined if the value passed to this function +** is less than zero. +*/ +SQLITE_PRIVATE int sqlite3FtsUnicodeFold(int c, int eRemoveDiacritic){ + /* Each entry in the following array defines a rule for folding a range + ** of codepoints to lower case. The rule applies to a range of nRange + ** codepoints starting at codepoint iCode. ** - ** If the on-conflict mode is REPLACE, this means that the existing row - ** should be deleted from the database before inserting the new row. Or, - ** if the on-conflict mode is other than REPLACE, then this method must - ** detect the conflict and return SQLITE_CONSTRAINT before beginning to - ** modify the database file. + ** If the least significant bit in flags is clear, then the rule applies + ** to all nRange codepoints (i.e. all nRange codepoints are upper case and + ** need to be folded). Or, if it is set, then the rule only applies to + ** every second codepoint in the range, starting with codepoint C. + ** + ** The 7 most significant bits in flags are an index into the aiOff[] + ** array. If a specific codepoint C does require folding, then its lower + ** case equivalent is ((C + aiOff[flags>>1]) & 0xFFFF). + ** + ** The contents of this array are generated by parsing the CaseFolding.txt + ** file distributed as part of the "Unicode Character Database". See + ** http://www.unicode.org for details. */ - if( nArg>1 && p->zContentTbl==0 ){ - /* Find the value object that holds the new rowid value. */ - sqlite3_value *pNewRowid = apVal[3+p->nColumn]; - if( sqlite3_value_type(pNewRowid)==SQLITE_NULL ){ - pNewRowid = apVal[1]; - } + static const struct TableEntry { + unsigned short iCode; + unsigned char flags; + unsigned char nRange; + } aEntry[] = { + {65, 14, 26}, {181, 64, 1}, {192, 14, 23}, + {216, 14, 7}, {256, 1, 48}, {306, 1, 6}, + {313, 1, 16}, {330, 1, 46}, {376, 116, 1}, + {377, 1, 6}, {383, 104, 1}, {385, 50, 1}, + {386, 1, 4}, {390, 44, 1}, {391, 0, 1}, + {393, 42, 2}, {395, 0, 1}, {398, 32, 1}, + {399, 38, 1}, {400, 40, 1}, {401, 0, 1}, + {403, 42, 1}, {404, 46, 1}, {406, 52, 1}, + {407, 48, 1}, {408, 0, 1}, {412, 52, 1}, + {413, 54, 1}, {415, 56, 1}, {416, 1, 6}, + {422, 60, 1}, {423, 0, 1}, {425, 60, 1}, + {428, 0, 1}, {430, 60, 1}, {431, 0, 1}, + {433, 58, 2}, {435, 1, 4}, {439, 62, 1}, + {440, 0, 1}, {444, 0, 1}, {452, 2, 1}, + {453, 0, 1}, {455, 2, 1}, {456, 0, 1}, + {458, 2, 1}, {459, 1, 18}, {478, 1, 18}, + {497, 2, 1}, {498, 1, 4}, {502, 122, 1}, + {503, 134, 1}, {504, 1, 40}, {544, 110, 1}, + {546, 1, 18}, {570, 70, 1}, {571, 0, 1}, + {573, 108, 1}, {574, 68, 1}, {577, 0, 1}, + {579, 106, 1}, {580, 28, 1}, {581, 30, 1}, + {582, 1, 10}, {837, 36, 1}, {880, 1, 4}, + {886, 0, 1}, {902, 18, 1}, {904, 16, 3}, + {908, 26, 1}, {910, 24, 2}, {913, 14, 17}, + {931, 14, 9}, {962, 0, 1}, {975, 4, 1}, + {976, 140, 1}, {977, 142, 1}, {981, 146, 1}, + {982, 144, 1}, {984, 1, 24}, {1008, 136, 1}, + {1009, 138, 1}, {1012, 130, 1}, {1013, 128, 1}, + {1015, 0, 1}, {1017, 152, 1}, {1018, 0, 1}, + {1021, 110, 3}, {1024, 34, 16}, {1040, 14, 32}, + {1120, 1, 34}, {1162, 1, 54}, {1216, 6, 1}, + {1217, 1, 14}, {1232, 1, 88}, {1329, 22, 38}, + {4256, 66, 38}, {4295, 66, 1}, {4301, 66, 1}, + {7680, 1, 150}, {7835, 132, 1}, {7838, 96, 1}, + {7840, 1, 96}, {7944, 150, 8}, {7960, 150, 6}, + {7976, 150, 8}, {7992, 150, 8}, {8008, 150, 6}, + {8025, 151, 8}, {8040, 150, 8}, {8072, 150, 8}, + {8088, 150, 8}, {8104, 150, 8}, {8120, 150, 2}, + {8122, 126, 2}, {8124, 148, 1}, {8126, 100, 1}, + {8136, 124, 4}, {8140, 148, 1}, {8152, 150, 2}, + {8154, 120, 2}, {8168, 150, 2}, {8170, 118, 2}, + {8172, 152, 1}, {8184, 112, 2}, {8186, 114, 2}, + {8188, 148, 1}, {8486, 98, 1}, {8490, 92, 1}, + {8491, 94, 1}, {8498, 12, 1}, {8544, 8, 16}, + {8579, 0, 1}, {9398, 10, 26}, {11264, 22, 47}, + {11360, 0, 1}, {11362, 88, 1}, {11363, 102, 1}, + {11364, 90, 1}, {11367, 1, 6}, {11373, 84, 1}, + {11374, 86, 1}, {11375, 80, 1}, {11376, 82, 1}, + {11378, 0, 1}, {11381, 0, 1}, {11390, 78, 2}, + {11392, 1, 100}, {11499, 1, 4}, {11506, 0, 1}, + {42560, 1, 46}, {42624, 1, 24}, {42786, 1, 14}, + {42802, 1, 62}, {42873, 1, 4}, {42877, 76, 1}, + {42878, 1, 10}, {42891, 0, 1}, {42893, 74, 1}, + {42896, 1, 4}, {42912, 1, 10}, {42922, 72, 1}, + {65313, 14, 26}, + }; + static const unsigned short aiOff[] = { + 1, 2, 8, 15, 16, 26, 28, 32, + 37, 38, 40, 48, 63, 64, 69, 71, + 79, 80, 116, 202, 203, 205, 206, 207, + 209, 210, 211, 213, 214, 217, 218, 219, + 775, 7264, 10792, 10795, 23228, 23256, 30204, 54721, + 54753, 54754, 54756, 54787, 54793, 54809, 57153, 57274, + 57921, 58019, 58363, 61722, 65268, 65341, 65373, 65406, + 65408, 65410, 65415, 65424, 65436, 65439, 65450, 65462, + 65472, 65476, 65478, 65480, 65482, 65488, 65506, 65511, + 65514, 65521, 65527, 65528, 65529, + }; - if( sqlite3_value_type(pNewRowid)!=SQLITE_NULL && ( - sqlite3_value_type(apVal[0])==SQLITE_NULL - || sqlite3_value_int64(apVal[0])!=sqlite3_value_int64(pNewRowid) - )){ - /* The new rowid is not NULL (in this case the rowid will be - ** automatically assigned and there is no chance of a conflict), and - ** the statement is either an INSERT or an UPDATE that modifies the - ** rowid column. So if the conflict mode is REPLACE, then delete any - ** existing row with rowid=pNewRowid. - ** - ** Or, if the conflict mode is not REPLACE, insert the new record into - ** the %_content table. If we hit the duplicate rowid constraint (or any - ** other error) while doing so, return immediately. - ** - ** This branch may also run if pNewRowid contains a value that cannot - ** be losslessly converted to an integer. In this case, the eventual - ** call to fts3InsertData() (either just below or further on in this - ** function) will return SQLITE_MISMATCH. If fts3DeleteByRowid is - ** invoked, it will delete zero rows (since no row will have - ** docid=$pNewRowid if $pNewRowid is not an integer value). - */ - if( sqlite3_vtab_on_conflict(p->db)==SQLITE_REPLACE ){ - rc = fts3DeleteByRowid(p, pNewRowid, &nChng, aSzDel); + int ret = c; + + assert( sizeof(unsigned short)==2 && sizeof(unsigned char)==1 ); + + if( c<128 ){ + if( c>='A' && c<='Z' ) ret = c + ('a' - 'A'); + }else if( c<65536 ){ + const struct TableEntry *p; + int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1; + int iLo = 0; + int iRes = -1; + + assert( c>aEntry[0].iCode ); + while( iHi>=iLo ){ + int iTest = (iHi + iLo) / 2; + int cmp = (c - aEntry[iTest].iCode); + if( cmp>=0 ){ + iRes = iTest; + iLo = iTest+1; }else{ - rc = fts3InsertData(p, apVal, pRowid); - bInsertDone = 1; + iHi = iTest-1; } } - } - if( rc!=SQLITE_OK ){ - goto update_out; - } - /* If this is a DELETE or UPDATE operation, remove the old record. */ - if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){ - assert( sqlite3_value_type(apVal[0])==SQLITE_INTEGER ); - rc = fts3DeleteByRowid(p, apVal[0], &nChng, aSzDel); - } - - /* If this is an INSERT or UPDATE operation, insert the new record. */ - if( nArg>1 && rc==SQLITE_OK ){ - int iLangid = sqlite3_value_int(apVal[2 + p->nColumn + 2]); - if( bInsertDone==0 ){ - rc = fts3InsertData(p, apVal, pRowid); - if( rc==SQLITE_CONSTRAINT && p->zContentTbl==0 ){ - rc = FTS_CORRUPT_VTAB; - } - } - if( rc==SQLITE_OK ){ - rc = fts3PendingTermsDocid(p, 0, iLangid, *pRowid); - } - if( rc==SQLITE_OK ){ - assert( p->iPrevDocid==*pRowid ); - rc = fts3InsertTerms(p, iLangid, apVal, aSzIns); + assert( iRes>=0 && c>=aEntry[iRes].iCode ); + p = &aEntry[iRes]; + if( c<(p->iCode + p->nRange) && 0==(0x01 & p->flags & (p->iCode ^ c)) ){ + ret = (c + (aiOff[p->flags>>1])) & 0x0000FFFF; + assert( ret>0 ); } - if( p->bHasDocsize ){ - fts3InsertDocsize(&rc, p, aSzIns); + + if( eRemoveDiacritic ){ + ret = remove_diacritic(ret, eRemoveDiacritic==2); } - nChng++; } - - if( p->bFts4 ){ - fts3UpdateDocTotals(&rc, p, aSzIns, aSzDel, nChng); + + else if( c>=66560 && c<66600 ){ + ret = c + 40; } - update_out: - sqlite3_free(aSzDel); - sqlite3Fts3SegmentsClose(p); - return rc; -} - -/* -** Flush any data in the pending-terms hash table to disk. If successful, -** merge all segments in the database (including the new segment, if -** there was any data to flush) into a single segment. -*/ -SQLITE_PRIVATE int sqlite3Fts3Optimize(Fts3Table *p){ - int rc; - rc = sqlite3_exec(p->db, "SAVEPOINT fts3", 0, 0, 0); - if( rc==SQLITE_OK ){ - rc = fts3DoOptimize(p, 1); - if( rc==SQLITE_OK || rc==SQLITE_DONE ){ - int rc2 = sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0); - if( rc2!=SQLITE_OK ) rc = rc2; - }else{ - sqlite3_exec(p->db, "ROLLBACK TO fts3", 0, 0, 0); - sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0); - } - } - sqlite3Fts3SegmentsClose(p); - return rc; + return ret; } +#endif /* defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4) */ +#endif /* !defined(SQLITE_DISABLE_FTS3_UNICODE) */ -#endif - -/************** End of fts3_write.c ******************************************/ -/************** Begin file fts3_snippet.c ************************************/ +/************** End of fts3_unicode2.c ***************************************/ +/************** Begin file json1.c *******************************************/ /* -** 2009 Oct 23 +** 2015-08-12 ** ** The author disclaims copyright to this source code. In place of ** a legal notice, here is a blessing: @@ -168058,2466 +184568,2623 @@ SQLITE_PRIVATE int sqlite3Fts3Optimize(Fts3Table *p){ ** May you share freely, never taking more than you give. ** ****************************************************************************** +** +** This SQLite extension implements JSON functions. The interface is +** modeled after MySQL JSON functions: +** +** https://dev.mysql.com/doc/refman/5.7/en/json.html +** +** For the time being, all JSON is stored as pure text. (We might add +** a JSONB type in the future which stores a binary encoding of JSON in +** a BLOB, but there is no support for JSONB in the current implementation. +** This implementation parses JSON text at 250 MB/s, so it is hard to see +** how JSONB might improve on that.) */ +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_JSON1) +#if !defined(SQLITEINT_H) +/* #include "sqlite3ext.h" */ +#endif +SQLITE_EXTENSION_INIT1 +/* #include */ +/* #include */ +/* #include */ +/* #include */ -/* #include "fts3Int.h" */ -#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) +/* Mark a function parameter as unused, to suppress nuisance compiler +** warnings. */ +#ifndef UNUSED_PARAM +# define UNUSED_PARAM(X) (void)(X) +#endif -/* #include */ -/* #include */ +#ifndef LARGEST_INT64 +# define LARGEST_INT64 (0xffffffff|(((sqlite3_int64)0x7fffffff)<<32)) +# define SMALLEST_INT64 (((sqlite3_int64)-1) - LARGEST_INT64) +#endif /* -** Characters that may appear in the second argument to matchinfo(). +** Versions of isspace(), isalnum() and isdigit() to which it is safe +** to pass signed char values. */ -#define FTS3_MATCHINFO_NPHRASE 'p' /* 1 value */ -#define FTS3_MATCHINFO_NCOL 'c' /* 1 value */ -#define FTS3_MATCHINFO_NDOC 'n' /* 1 value */ -#define FTS3_MATCHINFO_AVGLENGTH 'a' /* nCol values */ -#define FTS3_MATCHINFO_LENGTH 'l' /* nCol values */ -#define FTS3_MATCHINFO_LCS 's' /* nCol values */ -#define FTS3_MATCHINFO_HITS 'x' /* 3*nCol*nPhrase values */ -#define FTS3_MATCHINFO_LHITS 'y' /* nCol*nPhrase values */ -#define FTS3_MATCHINFO_LHITS_BM 'b' /* nCol*nPhrase values */ +#ifdef sqlite3Isdigit + /* Use the SQLite core versions if this routine is part of the + ** SQLite amalgamation */ +# define safe_isdigit(x) sqlite3Isdigit(x) +# define safe_isalnum(x) sqlite3Isalnum(x) +# define safe_isxdigit(x) sqlite3Isxdigit(x) +#else + /* Use the standard library for separate compilation */ +#include /* amalgamator: keep */ +# define safe_isdigit(x) isdigit((unsigned char)(x)) +# define safe_isalnum(x) isalnum((unsigned char)(x)) +# define safe_isxdigit(x) isxdigit((unsigned char)(x)) +#endif + +/* +** Growing our own isspace() routine this way is twice as fast as +** the library isspace() function, resulting in a 7% overall performance +** increase for the parser. (Ubuntu14.10 gcc 4.8.4 x64 with -Os). +*/ +static const char jsonIsSpace[] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +}; +#define safe_isspace(x) (jsonIsSpace[(unsigned char)x]) -/* -** The default value for the second argument to matchinfo(). -*/ -#define FTS3_MATCHINFO_DEFAULT "pcx" +#ifndef SQLITE_AMALGAMATION + /* Unsigned integer types. These are already defined in the sqliteInt.h, + ** but the definitions need to be repeated for separate compilation. */ + typedef sqlite3_uint64 u64; + typedef unsigned int u32; + typedef unsigned short int u16; + typedef unsigned char u8; +#endif +/* Objects */ +typedef struct JsonString JsonString; +typedef struct JsonNode JsonNode; +typedef struct JsonParse JsonParse; -/* -** Used as an fts3ExprIterate() context when loading phrase doclists to -** Fts3Expr.aDoclist[]/nDoclist. +/* An instance of this object represents a JSON string +** under construction. Really, this is a generic string accumulator +** that can be and is used to create strings other than JSON. */ -typedef struct LoadDoclistCtx LoadDoclistCtx; -struct LoadDoclistCtx { - Fts3Cursor *pCsr; /* FTS3 Cursor */ - int nPhrase; /* Number of phrases seen so far */ - int nToken; /* Number of tokens seen so far */ +struct JsonString { + sqlite3_context *pCtx; /* Function context - put error messages here */ + char *zBuf; /* Append JSON content here */ + u64 nAlloc; /* Bytes of storage available in zBuf[] */ + u64 nUsed; /* Bytes of zBuf[] currently used */ + u8 bStatic; /* True if zBuf is static space */ + u8 bErr; /* True if an error has been encountered */ + char zSpace[100]; /* Initial static space */ }; -/* -** The following types are used as part of the implementation of the -** fts3BestSnippet() routine. +/* JSON type values */ -typedef struct SnippetIter SnippetIter; -typedef struct SnippetPhrase SnippetPhrase; -typedef struct SnippetFragment SnippetFragment; - -struct SnippetIter { - Fts3Cursor *pCsr; /* Cursor snippet is being generated from */ - int iCol; /* Extract snippet from this column */ - int nSnippet; /* Requested snippet length (in tokens) */ - int nPhrase; /* Number of phrases in query */ - SnippetPhrase *aPhrase; /* Array of size nPhrase */ - int iCurrent; /* First token of current snippet */ -}; - -struct SnippetPhrase { - int nToken; /* Number of tokens in phrase */ - char *pList; /* Pointer to start of phrase position list */ - int iHead; /* Next value in position list */ - char *pHead; /* Position list data following iHead */ - int iTail; /* Next value in trailing position list */ - char *pTail; /* Position list data following iTail */ -}; +#define JSON_NULL 0 +#define JSON_TRUE 1 +#define JSON_FALSE 2 +#define JSON_INT 3 +#define JSON_REAL 4 +#define JSON_STRING 5 +#define JSON_ARRAY 6 +#define JSON_OBJECT 7 -struct SnippetFragment { - int iCol; /* Column snippet is extracted from */ - int iPos; /* Index of first token in snippet */ - u64 covered; /* Mask of query phrases covered */ - u64 hlmask; /* Mask of snippet terms to highlight */ -}; +/* The "subtype" set for JSON values */ +#define JSON_SUBTYPE 74 /* Ascii for "J" */ /* -** This type is used as an fts3ExprIterate() context object while -** accumulating the data returned by the matchinfo() function. +** Names of the various JSON types: */ -typedef struct MatchInfo MatchInfo; -struct MatchInfo { - Fts3Cursor *pCursor; /* FTS3 Cursor */ - int nCol; /* Number of columns in table */ - int nPhrase; /* Number of matchable phrases in query */ - sqlite3_int64 nDoc; /* Number of docs in database */ - char flag; - u32 *aMatchinfo; /* Pre-allocated buffer */ +static const char * const jsonType[] = { + "null", "true", "false", "integer", "real", "text", "array", "object" }; -/* -** An instance of this structure is used to manage a pair of buffers, each -** (nElem * sizeof(u32)) bytes in size. See the MatchinfoBuffer code below -** for details. +/* Bit values for the JsonNode.jnFlag field */ -struct MatchinfoBuffer { - u8 aRef[3]; - int nElem; - int bGlobal; /* Set if global data is loaded */ - char *zMatchinfo; - u32 aMatchinfo[1]; -}; +#define JNODE_RAW 0x01 /* Content is raw, not JSON encoded */ +#define JNODE_ESCAPE 0x02 /* Content is text with \ escapes */ +#define JNODE_REMOVE 0x04 /* Do not output */ +#define JNODE_REPLACE 0x08 /* Replace with JsonNode.u.iReplace */ +#define JNODE_PATCH 0x10 /* Patch with JsonNode.u.pPatch */ +#define JNODE_APPEND 0x20 /* More ARRAY/OBJECT entries at u.iAppend */ +#define JNODE_LABEL 0x40 /* Is a label of an object */ -/* -** The snippet() and offsets() functions both return text values. An instance -** of the following structure is used to accumulate those values while the -** functions are running. See fts3StringAppend() for details. +/* A single node of parsed JSON */ -typedef struct StrBuffer StrBuffer; -struct StrBuffer { - char *z; /* Pointer to buffer containing string */ - int n; /* Length of z in bytes (excl. nul-term) */ - int nAlloc; /* Allocated size of buffer z in bytes */ +struct JsonNode { + u8 eType; /* One of the JSON_ type values */ + u8 jnFlags; /* JNODE flags */ + u32 n; /* Bytes of content, or number of sub-nodes */ + union { + const char *zJContent; /* Content for INT, REAL, and STRING */ + u32 iAppend; /* More terms for ARRAY and OBJECT */ + u32 iKey; /* Key for ARRAY objects in json_tree() */ + u32 iReplace; /* Replacement content for JNODE_REPLACE */ + JsonNode *pPatch; /* Node chain of patch for JNODE_PATCH */ + } u; }; - -/************************************************************************* -** Start of MatchinfoBuffer code. +/* A completely parsed JSON string */ +struct JsonParse { + u32 nNode; /* Number of slots of aNode[] used */ + u32 nAlloc; /* Number of slots of aNode[] allocated */ + JsonNode *aNode; /* Array of nodes containing the parse */ + const char *zJson; /* Original JSON string */ + u32 *aUp; /* Index of parent of each node */ + u8 oom; /* Set to true if out of memory */ + u8 nErr; /* Number of errors seen */ + u16 iDepth; /* Nesting depth */ + int nJson; /* Length of the zJson string in bytes */ + u32 iHold; /* Replace cache line with the lowest iHold value */ +}; /* -** Allocate a two-slot MatchinfoBuffer object. +** Maximum nesting depth of JSON for this implementation. +** +** This limit is needed to avoid a stack overflow in the recursive +** descent parser. A depth of 2000 is far deeper than any sane JSON +** should go. */ -static MatchinfoBuffer *fts3MIBufferNew(int nElem, const char *zMatchinfo){ - MatchinfoBuffer *pRet; - int nByte = sizeof(u32) * (2*nElem + 1) + sizeof(MatchinfoBuffer); - int nStr = (int)strlen(zMatchinfo); - - pRet = sqlite3_malloc(nByte + nStr+1); - if( pRet ){ - memset(pRet, 0, nByte); - pRet->aMatchinfo[0] = (u8*)(&pRet->aMatchinfo[1]) - (u8*)pRet; - pRet->aMatchinfo[1+nElem] = pRet->aMatchinfo[0] + sizeof(u32)*(nElem+1); - pRet->nElem = nElem; - pRet->zMatchinfo = ((char*)pRet) + nByte; - memcpy(pRet->zMatchinfo, zMatchinfo, nStr+1); - pRet->aRef[0] = 1; - } - - return pRet; -} - -static void fts3MIBufferFree(void *p){ - MatchinfoBuffer *pBuf = (MatchinfoBuffer*)((u8*)p - ((u32*)p)[-1]); +#define JSON_MAX_DEPTH 2000 - assert( (u32*)p==&pBuf->aMatchinfo[1] - || (u32*)p==&pBuf->aMatchinfo[pBuf->nElem+2] - ); - if( (u32*)p==&pBuf->aMatchinfo[1] ){ - pBuf->aRef[1] = 0; - }else{ - pBuf->aRef[2] = 0; - } +/************************************************************************** +** Utility routines for dealing with JsonString objects +**************************************************************************/ - if( pBuf->aRef[0]==0 && pBuf->aRef[1]==0 && pBuf->aRef[2]==0 ){ - sqlite3_free(pBuf); - } +/* Set the JsonString object to an empty string +*/ +static void jsonZero(JsonString *p){ + p->zBuf = p->zSpace; + p->nAlloc = sizeof(p->zSpace); + p->nUsed = 0; + p->bStatic = 1; } -static void (*fts3MIBufferAlloc(MatchinfoBuffer *p, u32 **paOut))(void*){ - void (*xRet)(void*) = 0; - u32 *aOut = 0; - - if( p->aRef[1]==0 ){ - p->aRef[1] = 1; - aOut = &p->aMatchinfo[1]; - xRet = fts3MIBufferFree; - } - else if( p->aRef[2]==0 ){ - p->aRef[2] = 1; - aOut = &p->aMatchinfo[p->nElem+2]; - xRet = fts3MIBufferFree; - }else{ - aOut = (u32*)sqlite3_malloc(p->nElem * sizeof(u32)); - if( aOut ){ - xRet = sqlite3_free; - if( p->bGlobal ) memcpy(aOut, &p->aMatchinfo[1], p->nElem*sizeof(u32)); - } - } - - *paOut = aOut; - return xRet; +/* Initialize the JsonString object +*/ +static void jsonInit(JsonString *p, sqlite3_context *pCtx){ + p->pCtx = pCtx; + p->bErr = 0; + jsonZero(p); } -static void fts3MIBufferSetGlobal(MatchinfoBuffer *p){ - p->bGlobal = 1; - memcpy(&p->aMatchinfo[2+p->nElem], &p->aMatchinfo[1], p->nElem*sizeof(u32)); -} -/* -** Free a MatchinfoBuffer object allocated using fts3MIBufferNew() +/* Free all allocated memory and reset the JsonString object back to its +** initial state. */ -SQLITE_PRIVATE void sqlite3Fts3MIBufferFree(MatchinfoBuffer *p){ - if( p ){ - assert( p->aRef[0]==1 ); - p->aRef[0] = 0; - if( p->aRef[0]==0 && p->aRef[1]==0 && p->aRef[2]==0 ){ - sqlite3_free(p); - } - } +static void jsonReset(JsonString *p){ + if( !p->bStatic ) sqlite3_free(p->zBuf); + jsonZero(p); } -/* -** End of MatchinfoBuffer code. -*************************************************************************/ - -/* -** This function is used to help iterate through a position-list. A position -** list is a list of unique integers, sorted from smallest to largest. Each -** element of the list is represented by an FTS3 varint that takes the value -** of the difference between the current element and the previous one plus -** two. For example, to store the position-list: -** -** 4 9 113 -** -** the three varints: -** -** 6 7 106 -** -** are encoded. -** -** When this function is called, *pp points to the start of an element of -** the list. *piPos contains the value of the previous entry in the list. -** After it returns, *piPos contains the value of the next element of the -** list and *pp is advanced to the following varint. +/* Report an out-of-memory (OOM) condition */ -static void fts3GetDeltaPosition(char **pp, int *piPos){ - int iVal; - *pp += fts3GetVarint32(*pp, &iVal); - *piPos += (iVal-2); +static void jsonOom(JsonString *p){ + p->bErr = 1; + sqlite3_result_error_nomem(p->pCtx); + jsonReset(p); } -/* -** Helper function for fts3ExprIterate() (see below). +/* Enlarge pJson->zBuf so that it can hold at least N more bytes. +** Return zero on success. Return non-zero on an OOM error */ -static int fts3ExprIterate2( - Fts3Expr *pExpr, /* Expression to iterate phrases of */ - int *piPhrase, /* Pointer to phrase counter */ - int (*x)(Fts3Expr*,int,void*), /* Callback function to invoke for phrases */ - void *pCtx /* Second argument to pass to callback */ -){ - int rc; /* Return code */ - int eType = pExpr->eType; /* Type of expression node pExpr */ - - if( eType!=FTSQUERY_PHRASE ){ - assert( pExpr->pLeft && pExpr->pRight ); - rc = fts3ExprIterate2(pExpr->pLeft, piPhrase, x, pCtx); - if( rc==SQLITE_OK && eType!=FTSQUERY_NOT ){ - rc = fts3ExprIterate2(pExpr->pRight, piPhrase, x, pCtx); +static int jsonGrow(JsonString *p, u32 N){ + u64 nTotal = NnAlloc ? p->nAlloc*2 : p->nAlloc+N+10; + char *zNew; + if( p->bStatic ){ + if( p->bErr ) return 1; + zNew = sqlite3_malloc64(nTotal); + if( zNew==0 ){ + jsonOom(p); + return SQLITE_NOMEM; } + memcpy(zNew, p->zBuf, (size_t)p->nUsed); + p->zBuf = zNew; + p->bStatic = 0; }else{ - rc = x(pExpr, *piPhrase, pCtx); - (*piPhrase)++; + zNew = sqlite3_realloc64(p->zBuf, nTotal); + if( zNew==0 ){ + jsonOom(p); + return SQLITE_NOMEM; + } + p->zBuf = zNew; } - return rc; + p->nAlloc = nTotal; + return SQLITE_OK; } -/* -** Iterate through all phrase nodes in an FTS3 query, except those that -** are part of a sub-tree that is the right-hand-side of a NOT operator. -** For each phrase node found, the supplied callback function is invoked. -** -** If the callback function returns anything other than SQLITE_OK, -** the iteration is abandoned and the error code returned immediately. -** Otherwise, SQLITE_OK is returned after a callback has been made for -** all eligible phrase nodes. +/* Append N bytes from zIn onto the end of the JsonString string. */ -static int fts3ExprIterate( - Fts3Expr *pExpr, /* Expression to iterate phrases of */ - int (*x)(Fts3Expr*,int,void*), /* Callback function to invoke for phrases */ - void *pCtx /* Second argument to pass to callback */ -){ - int iPhrase = 0; /* Variable used as the phrase counter */ - return fts3ExprIterate2(pExpr, &iPhrase, x, pCtx); +static void jsonAppendRaw(JsonString *p, const char *zIn, u32 N){ + if( N==0 ) return; + if( (N+p->nUsed >= p->nAlloc) && jsonGrow(p,N)!=0 ) return; + memcpy(p->zBuf+p->nUsed, zIn, N); + p->nUsed += N; } - -/* -** This is an fts3ExprIterate() callback used while loading the doclists -** for each phrase into Fts3Expr.aDoclist[]/nDoclist. See also -** fts3ExprLoadDoclists(). +/* Append formatted text (not to exceed N bytes) to the JsonString. */ -static int fts3ExprLoadDoclistsCb(Fts3Expr *pExpr, int iPhrase, void *ctx){ - int rc = SQLITE_OK; - Fts3Phrase *pPhrase = pExpr->pPhrase; - LoadDoclistCtx *p = (LoadDoclistCtx *)ctx; - - UNUSED_PARAMETER(iPhrase); - - p->nPhrase++; - p->nToken += pPhrase->nToken; - - return rc; +static void jsonPrintf(int N, JsonString *p, const char *zFormat, ...){ + va_list ap; + if( (p->nUsed + N >= p->nAlloc) && jsonGrow(p, N) ) return; + va_start(ap, zFormat); + sqlite3_vsnprintf(N, p->zBuf+p->nUsed, zFormat, ap); + va_end(ap); + p->nUsed += (int)strlen(p->zBuf+p->nUsed); } -/* -** Load the doclists for each phrase in the query associated with FTS3 cursor -** pCsr. -** -** If pnPhrase is not NULL, then *pnPhrase is set to the number of matchable -** phrases in the expression (all phrases except those directly or -** indirectly descended from the right-hand-side of a NOT operator). If -** pnToken is not NULL, then it is set to the number of tokens in all -** matchable phrases of the expression. +/* Append a single character */ -static int fts3ExprLoadDoclists( - Fts3Cursor *pCsr, /* Fts3 cursor for current query */ - int *pnPhrase, /* OUT: Number of phrases in query */ - int *pnToken /* OUT: Number of tokens in query */ -){ - int rc; /* Return Code */ - LoadDoclistCtx sCtx = {0,0,0}; /* Context for fts3ExprIterate() */ - sCtx.pCsr = pCsr; - rc = fts3ExprIterate(pCsr->pExpr, fts3ExprLoadDoclistsCb, (void *)&sCtx); - if( pnPhrase ) *pnPhrase = sCtx.nPhrase; - if( pnToken ) *pnToken = sCtx.nToken; - return rc; +static void jsonAppendChar(JsonString *p, char c){ + if( p->nUsed>=p->nAlloc && jsonGrow(p,1)!=0 ) return; + p->zBuf[p->nUsed++] = c; } -static int fts3ExprPhraseCountCb(Fts3Expr *pExpr, int iPhrase, void *ctx){ - (*(int *)ctx)++; - pExpr->iPhrase = iPhrase; - return SQLITE_OK; -} -static int fts3ExprPhraseCount(Fts3Expr *pExpr){ - int nPhrase = 0; - (void)fts3ExprIterate(pExpr, fts3ExprPhraseCountCb, (void *)&nPhrase); - return nPhrase; +/* Append a comma separator to the output buffer, if the previous +** character is not '[' or '{'. +*/ +static void jsonAppendSeparator(JsonString *p){ + char c; + if( p->nUsed==0 ) return; + c = p->zBuf[p->nUsed-1]; + if( c!='[' && c!='{' ) jsonAppendChar(p, ','); } -/* -** Advance the position list iterator specified by the first two -** arguments so that it points to the first element with a value greater -** than or equal to parameter iNext. +/* Append the N-byte string in zIn to the end of the JsonString string +** under construction. Enclose the string in "..." and escape +** any double-quotes or backslash characters contained within the +** string. */ -static void fts3SnippetAdvance(char **ppIter, int *piIter, int iNext){ - char *pIter = *ppIter; - if( pIter ){ - int iIter = *piIter; - - while( iIternUsed+2 >= p->nAlloc) && jsonGrow(p,N+2)!=0 ) return; + p->zBuf[p->nUsed++] = '"'; + for(i=0; inUsed+N+3-i > p->nAlloc) && jsonGrow(p,N+3-i)!=0 ) return; + p->zBuf[p->nUsed++] = '\\'; + }else if( c<=0x1f ){ + static const char aSpecial[] = { + 0, 0, 0, 0, 0, 0, 0, 0, 'b', 't', 'n', 0, 'f', 'r', 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + }; + assert( sizeof(aSpecial)==32 ); + assert( aSpecial['\b']=='b' ); + assert( aSpecial['\f']=='f' ); + assert( aSpecial['\n']=='n' ); + assert( aSpecial['\r']=='r' ); + assert( aSpecial['\t']=='t' ); + if( aSpecial[c] ){ + c = aSpecial[c]; + goto json_simple_escape; } - fts3GetDeltaPosition(&pIter, &iIter); + if( (p->nUsed+N+7+i > p->nAlloc) && jsonGrow(p,N+7-i)!=0 ) return; + p->zBuf[p->nUsed++] = '\\'; + p->zBuf[p->nUsed++] = 'u'; + p->zBuf[p->nUsed++] = '0'; + p->zBuf[p->nUsed++] = '0'; + p->zBuf[p->nUsed++] = '0' + (c>>4); + c = "0123456789abcdef"[c&0xf]; } - - *piIter = iIter; - *ppIter = pIter; + p->zBuf[p->nUsed++] = c; } + p->zBuf[p->nUsed++] = '"'; + assert( p->nUsednAlloc ); } /* -** Advance the snippet iterator to the next candidate snippet. +** Append a function parameter value to the JSON string under +** construction. */ -static int fts3SnippetNextCandidate(SnippetIter *pIter){ - int i; /* Loop counter */ - - if( pIter->iCurrent<0 ){ - /* The SnippetIter object has just been initialized. The first snippet - ** candidate always starts at offset 0 (even if this candidate has a - ** score of 0.0). - */ - pIter->iCurrent = 0; - - /* Advance the 'head' iterator of each phrase to the first offset that - ** is greater than or equal to (iNext+nSnippet). - */ - for(i=0; inPhrase; i++){ - SnippetPhrase *pPhrase = &pIter->aPhrase[i]; - fts3SnippetAdvance(&pPhrase->pHead, &pPhrase->iHead, pIter->nSnippet); +static void jsonAppendValue( + JsonString *p, /* Append to this JSON string */ + sqlite3_value *pValue /* Value to append */ +){ + switch( sqlite3_value_type(pValue) ){ + case SQLITE_NULL: { + jsonAppendRaw(p, "null", 4); + break; } - }else{ - int iStart; - int iEnd = 0x7FFFFFFF; - - for(i=0; inPhrase; i++){ - SnippetPhrase *pPhrase = &pIter->aPhrase[i]; - if( pPhrase->pHead && pPhrase->iHeadiHead; - } + case SQLITE_INTEGER: + case SQLITE_FLOAT: { + const char *z = (const char*)sqlite3_value_text(pValue); + u32 n = (u32)sqlite3_value_bytes(pValue); + jsonAppendRaw(p, z, n); + break; } - if( iEnd==0x7FFFFFFF ){ - return 1; + case SQLITE_TEXT: { + const char *z = (const char*)sqlite3_value_text(pValue); + u32 n = (u32)sqlite3_value_bytes(pValue); + if( sqlite3_value_subtype(pValue)==JSON_SUBTYPE ){ + jsonAppendRaw(p, z, n); + }else{ + jsonAppendString(p, z, n); + } + break; } - - pIter->iCurrent = iStart = iEnd - pIter->nSnippet + 1; - for(i=0; inPhrase; i++){ - SnippetPhrase *pPhrase = &pIter->aPhrase[i]; - fts3SnippetAdvance(&pPhrase->pHead, &pPhrase->iHead, iEnd+1); - fts3SnippetAdvance(&pPhrase->pTail, &pPhrase->iTail, iStart); + default: { + if( p->bErr==0 ){ + sqlite3_result_error(p->pCtx, "JSON cannot hold BLOB values", -1); + p->bErr = 2; + jsonReset(p); + } + break; } } - - return 0; } -/* -** Retrieve information about the current candidate snippet of snippet -** iterator pIter. -*/ -static void fts3SnippetDetails( - SnippetIter *pIter, /* Snippet iterator */ - u64 mCovered, /* Bitmask of phrases already covered */ - int *piToken, /* OUT: First token of proposed snippet */ - int *piScore, /* OUT: "Score" for this snippet */ - u64 *pmCover, /* OUT: Bitmask of phrases covered */ - u64 *pmHighlight /* OUT: Bitmask of terms to highlight */ -){ - int iStart = pIter->iCurrent; /* First token of snippet */ - int iScore = 0; /* Score of this snippet */ - int i; /* Loop counter */ - u64 mCover = 0; /* Mask of phrases covered by this snippet */ - u64 mHighlight = 0; /* Mask of tokens to highlight in snippet */ - - for(i=0; inPhrase; i++){ - SnippetPhrase *pPhrase = &pIter->aPhrase[i]; - if( pPhrase->pTail ){ - char *pCsr = pPhrase->pTail; - int iCsr = pPhrase->iTail; - - while( iCsr<(iStart+pIter->nSnippet) ){ - int j; - u64 mPhrase = (u64)1 << i; - u64 mPos = (u64)1 << (iCsr - iStart); - assert( iCsr>=iStart ); - if( (mCover|mCovered)&mPhrase ){ - iScore++; - }else{ - iScore += 1000; - } - mCover |= mPhrase; - - for(j=0; jnToken; j++){ - mHighlight |= (mPos>>j); - } - if( 0==(*pCsr & 0x0FE) ) break; - fts3GetDeltaPosition(&pCsr, &iCsr); - } - } +/* Make the JSON in p the result of the SQL function. +*/ +static void jsonResult(JsonString *p){ + if( p->bErr==0 ){ + sqlite3_result_text64(p->pCtx, p->zBuf, p->nUsed, + p->bStatic ? SQLITE_TRANSIENT : sqlite3_free, + SQLITE_UTF8); + jsonZero(p); } - - /* Set the output variables before returning. */ - *piToken = iStart; - *piScore = iScore; - *pmCover = mCover; - *pmHighlight = mHighlight; + assert( p->bStatic ); } +/************************************************************************** +** Utility routines for dealing with JsonNode and JsonParse objects +**************************************************************************/ + /* -** This function is an fts3ExprIterate() callback used by fts3BestSnippet(). -** Each invocation populates an element of the SnippetIter.aPhrase[] array. +** Return the number of consecutive JsonNode slots need to represent +** the parsed JSON at pNode. The minimum answer is 1. For ARRAY and +** OBJECT types, the number might be larger. +** +** Appended elements are not counted. The value returned is the number +** by which the JsonNode counter should increment in order to go to the +** next peer value. */ -static int fts3SnippetFindPositions(Fts3Expr *pExpr, int iPhrase, void *ctx){ - SnippetIter *p = (SnippetIter *)ctx; - SnippetPhrase *pPhrase = &p->aPhrase[iPhrase]; - char *pCsr; - int rc; +static u32 jsonNodeSize(JsonNode *pNode){ + return pNode->eType>=JSON_ARRAY ? pNode->n+1 : 1; +} - pPhrase->nToken = pExpr->pPhrase->nToken; - rc = sqlite3Fts3EvalPhrasePoslist(p->pCsr, pExpr, p->iCol, &pCsr); - assert( rc==SQLITE_OK || pCsr==0 ); - if( pCsr ){ - int iFirst = 0; - pPhrase->pList = pCsr; - fts3GetDeltaPosition(&pCsr, &iFirst); - assert( iFirst>=0 ); - pPhrase->pHead = pCsr; - pPhrase->pTail = pCsr; - pPhrase->iHead = iFirst; - pPhrase->iTail = iFirst; - }else{ - assert( rc!=SQLITE_OK || ( - pPhrase->pList==0 && pPhrase->pHead==0 && pPhrase->pTail==0 - )); - } +/* +** Reclaim all memory allocated by a JsonParse object. But do not +** delete the JsonParse object itself. +*/ +static void jsonParseReset(JsonParse *pParse){ + sqlite3_free(pParse->aNode); + pParse->aNode = 0; + pParse->nNode = 0; + pParse->nAlloc = 0; + sqlite3_free(pParse->aUp); + pParse->aUp = 0; +} - return rc; +/* +** Free a JsonParse object that was obtained from sqlite3_malloc(). +*/ +static void jsonParseFree(JsonParse *pParse){ + jsonParseReset(pParse); + sqlite3_free(pParse); } /* -** Select the fragment of text consisting of nFragment contiguous tokens -** from column iCol that represent the "best" snippet. The best snippet -** is the snippet with the highest score, where scores are calculated -** by adding: -** -** (a) +1 point for each occurrence of a matchable phrase in the snippet. -** -** (b) +1000 points for the first occurrence of each matchable phrase in -** the snippet for which the corresponding mCovered bit is not set. -** -** The selected snippet parameters are stored in structure *pFragment before -** returning. The score of the selected snippet is stored in *piScore -** before returning. +** Convert the JsonNode pNode into a pure JSON string and +** append to pOut. Subsubstructure is also included. Return +** the number of JsonNode objects that are encoded. */ -static int fts3BestSnippet( - int nSnippet, /* Desired snippet length */ - Fts3Cursor *pCsr, /* Cursor to create snippet for */ - int iCol, /* Index of column to create snippet from */ - u64 mCovered, /* Mask of phrases already covered */ - u64 *pmSeen, /* IN/OUT: Mask of phrases seen */ - SnippetFragment *pFragment, /* OUT: Best snippet found */ - int *piScore /* OUT: Score of snippet pFragment */ +static void jsonRenderNode( + JsonNode *pNode, /* The node to render */ + JsonString *pOut, /* Write JSON here */ + sqlite3_value **aReplace /* Replacement values */ ){ - int rc; /* Return Code */ - int nList; /* Number of phrases in expression */ - SnippetIter sIter; /* Iterates through snippet candidates */ - int nByte; /* Number of bytes of space to allocate */ - int iBestScore = -1; /* Best snippet score found so far */ - int i; /* Loop counter */ - - memset(&sIter, 0, sizeof(sIter)); - - /* Iterate through the phrases in the expression to count them. The same - ** callback makes sure the doclists are loaded for each phrase. - */ - rc = fts3ExprLoadDoclists(pCsr, &nList, 0); - if( rc!=SQLITE_OK ){ - return rc; - } - - /* Now that it is known how many phrases there are, allocate and zero - ** the required space using malloc(). - */ - nByte = sizeof(SnippetPhrase) * nList; - sIter.aPhrase = (SnippetPhrase *)sqlite3_malloc(nByte); - if( !sIter.aPhrase ){ - return SQLITE_NOMEM; + if( pNode->jnFlags & (JNODE_REPLACE|JNODE_PATCH) ){ + if( pNode->jnFlags & JNODE_REPLACE ){ + jsonAppendValue(pOut, aReplace[pNode->u.iReplace]); + return; + } + pNode = pNode->u.pPatch; } - memset(sIter.aPhrase, 0, nByte); - - /* Initialize the contents of the SnippetIter object. Then iterate through - ** the set of phrases in the expression to populate the aPhrase[] array. - */ - sIter.pCsr = pCsr; - sIter.iCol = iCol; - sIter.nSnippet = nSnippet; - sIter.nPhrase = nList; - sIter.iCurrent = -1; - rc = fts3ExprIterate(pCsr->pExpr, fts3SnippetFindPositions, (void*)&sIter); - if( rc==SQLITE_OK ){ - - /* Set the *pmSeen output variable. */ - for(i=0; ieType ){ + default: { + assert( pNode->eType==JSON_NULL ); + jsonAppendRaw(pOut, "null", 4); + break; + } + case JSON_TRUE: { + jsonAppendRaw(pOut, "true", 4); + break; + } + case JSON_FALSE: { + jsonAppendRaw(pOut, "false", 5); + break; + } + case JSON_STRING: { + if( pNode->jnFlags & JNODE_RAW ){ + jsonAppendString(pOut, pNode->u.zJContent, pNode->n); + break; } + /* Fall through into the next case */ } - - /* Loop through all candidate snippets. Store the best snippet in - ** *pFragment. Store its associated 'score' in iBestScore. - */ - pFragment->iCol = iCol; - while( !fts3SnippetNextCandidate(&sIter) ){ - int iPos; - int iScore; - u64 mCover; - u64 mHighlite; - fts3SnippetDetails(&sIter, mCovered, &iPos, &iScore, &mCover,&mHighlite); - assert( iScore>=0 ); - if( iScore>iBestScore ){ - pFragment->iPos = iPos; - pFragment->hlmask = mHighlite; - pFragment->covered = mCover; - iBestScore = iScore; + case JSON_REAL: + case JSON_INT: { + jsonAppendRaw(pOut, pNode->u.zJContent, pNode->n); + break; + } + case JSON_ARRAY: { + u32 j = 1; + jsonAppendChar(pOut, '['); + for(;;){ + while( j<=pNode->n ){ + if( (pNode[j].jnFlags & JNODE_REMOVE)==0 ){ + jsonAppendSeparator(pOut); + jsonRenderNode(&pNode[j], pOut, aReplace); + } + j += jsonNodeSize(&pNode[j]); + } + if( (pNode->jnFlags & JNODE_APPEND)==0 ) break; + pNode = &pNode[pNode->u.iAppend]; + j = 1; + } + jsonAppendChar(pOut, ']'); + break; + } + case JSON_OBJECT: { + u32 j = 1; + jsonAppendChar(pOut, '{'); + for(;;){ + while( j<=pNode->n ){ + if( (pNode[j+1].jnFlags & JNODE_REMOVE)==0 ){ + jsonAppendSeparator(pOut); + jsonRenderNode(&pNode[j], pOut, aReplace); + jsonAppendChar(pOut, ':'); + jsonRenderNode(&pNode[j+1], pOut, aReplace); + } + j += 1 + jsonNodeSize(&pNode[j+1]); + } + if( (pNode->jnFlags & JNODE_APPEND)==0 ) break; + pNode = &pNode[pNode->u.iAppend]; + j = 1; } + jsonAppendChar(pOut, '}'); + break; } - - *piScore = iBestScore; } - sqlite3_free(sIter.aPhrase); - return rc; } - /* -** Append a string to the string-buffer passed as the first argument. -** -** If nAppend is negative, then the length of the string zAppend is -** determined using strlen(). +** Return a JsonNode and all its descendents as a JSON string. */ -static int fts3StringAppend( - StrBuffer *pStr, /* Buffer to append to */ - const char *zAppend, /* Pointer to data to append to buffer */ - int nAppend /* Size of zAppend in bytes (or -1) */ +static void jsonReturnJson( + JsonNode *pNode, /* Node to return */ + sqlite3_context *pCtx, /* Return value for this function */ + sqlite3_value **aReplace /* Array of replacement values */ ){ - if( nAppend<0 ){ - nAppend = (int)strlen(zAppend); - } - - /* If there is insufficient space allocated at StrBuffer.z, use realloc() - ** to grow the buffer until so that it is big enough to accomadate the - ** appended data. - */ - if( pStr->n+nAppend+1>=pStr->nAlloc ){ - int nAlloc = pStr->nAlloc+nAppend+100; - char *zNew = sqlite3_realloc(pStr->z, nAlloc); - if( !zNew ){ - return SQLITE_NOMEM; - } - pStr->z = zNew; - pStr->nAlloc = nAlloc; - } - assert( pStr->z!=0 && (pStr->nAlloc >= pStr->n+nAppend+1) ); - - /* Append the data to the string buffer. */ - memcpy(&pStr->z[pStr->n], zAppend, nAppend); - pStr->n += nAppend; - pStr->z[pStr->n] = '\0'; - - return SQLITE_OK; + JsonString s; + jsonInit(&s, pCtx); + jsonRenderNode(pNode, &s, aReplace); + jsonResult(&s); + sqlite3_result_subtype(pCtx, JSON_SUBTYPE); } /* -** The fts3BestSnippet() function often selects snippets that end with a -** query term. That is, the final term of the snippet is always a term -** that requires highlighting. For example, if 'X' is a highlighted term -** and '.' is a non-highlighted term, BestSnippet() may select: -** -** ........X.....X -** -** This function "shifts" the beginning of the snippet forward in the -** document so that there are approximately the same number of -** non-highlighted terms to the right of the final highlighted term as there -** are to the left of the first highlighted term. For example, to this: -** -** ....X.....X.... -** -** This is done as part of extracting the snippet text, not when selecting -** the snippet. Snippet selection is done based on doclists only, so there -** is no way for fts3BestSnippet() to know whether or not the document -** actually contains terms that follow the final highlighted term. +** Translate a single byte of Hex into an integer. +** This routine only works if h really is a valid hexadecimal +** character: 0..9a..fA..F */ -static int fts3SnippetShift( - Fts3Table *pTab, /* FTS3 table snippet comes from */ - int iLangid, /* Language id to use in tokenizing */ - int nSnippet, /* Number of tokens desired for snippet */ - const char *zDoc, /* Document text to extract snippet from */ - int nDoc, /* Size of buffer zDoc in bytes */ - int *piPos, /* IN/OUT: First token of snippet */ - u64 *pHlmask /* IN/OUT: Mask of tokens to highlight */ -){ - u64 hlmask = *pHlmask; /* Local copy of initial highlight-mask */ - - if( hlmask ){ - int nLeft; /* Tokens to the left of first highlight */ - int nRight; /* Tokens to the right of last highlight */ - int nDesired; /* Ideal number of tokens to shift forward */ - - for(nLeft=0; !(hlmask & ((u64)1 << nLeft)); nLeft++); - for(nRight=0; !(hlmask & ((u64)1 << (nSnippet-1-nRight))); nRight++); - nDesired = (nLeft-nRight)/2; - - /* Ideally, the start of the snippet should be pushed forward in the - ** document nDesired tokens. This block checks if there are actually - ** nDesired tokens to the right of the snippet. If so, *piPos and - ** *pHlMask are updated to shift the snippet nDesired tokens to the - ** right. Otherwise, the snippet is shifted by the number of tokens - ** available. - */ - if( nDesired>0 ){ - int nShift; /* Number of tokens to shift snippet by */ - int iCurrent = 0; /* Token counter */ - int rc; /* Return Code */ - sqlite3_tokenizer_module *pMod; - sqlite3_tokenizer_cursor *pC; - pMod = (sqlite3_tokenizer_module *)pTab->pTokenizer->pModule; - - /* Open a cursor on zDoc/nDoc. Check if there are (nSnippet+nDesired) - ** or more tokens in zDoc/nDoc. - */ - rc = sqlite3Fts3OpenTokenizer(pTab->pTokenizer, iLangid, zDoc, nDoc, &pC); - if( rc!=SQLITE_OK ){ - return rc; - } - while( rc==SQLITE_OK && iCurrent<(nSnippet+nDesired) ){ - const char *ZDUMMY; int DUMMY1 = 0, DUMMY2 = 0, DUMMY3 = 0; - rc = pMod->xNext(pC, &ZDUMMY, &DUMMY1, &DUMMY2, &DUMMY3, &iCurrent); - } - pMod->xClose(pC); - if( rc!=SQLITE_OK && rc!=SQLITE_DONE ){ return rc; } +static u8 jsonHexToInt(int h){ + assert( (h>='0' && h<='9') || (h>='a' && h<='f') || (h>='A' && h<='F') ); +#ifdef SQLITE_EBCDIC + h += 9*(1&~(h>>4)); +#else + h += 9*(1&(h>>6)); +#endif + return (u8)(h & 0xf); +} - nShift = (rc==SQLITE_DONE)+iCurrent-nSnippet; - assert( nShift<=nDesired ); - if( nShift>0 ){ - *piPos += nShift; - *pHlmask = hlmask >> nShift; - } - } - } - return SQLITE_OK; +/* +** Convert a 4-byte hex string into an integer +*/ +static u32 jsonHexToInt4(const char *z){ + u32 v; + assert( safe_isxdigit(z[0]) ); + assert( safe_isxdigit(z[1]) ); + assert( safe_isxdigit(z[2]) ); + assert( safe_isxdigit(z[3]) ); + v = (jsonHexToInt(z[0])<<12) + + (jsonHexToInt(z[1])<<8) + + (jsonHexToInt(z[2])<<4) + + jsonHexToInt(z[3]); + return v; } /* -** Extract the snippet text for fragment pFragment from cursor pCsr and -** append it to string buffer pOut. +** Make the JsonNode the return value of the function. */ -static int fts3SnippetText( - Fts3Cursor *pCsr, /* FTS3 Cursor */ - SnippetFragment *pFragment, /* Snippet to extract */ - int iFragment, /* Fragment number */ - int isLast, /* True for final fragment in snippet */ - int nSnippet, /* Number of tokens in extracted snippet */ - const char *zOpen, /* String inserted before highlighted term */ - const char *zClose, /* String inserted after highlighted term */ - const char *zEllipsis, /* String inserted between snippets */ - StrBuffer *pOut /* Write output here */ +static void jsonReturn( + JsonNode *pNode, /* Node to return */ + sqlite3_context *pCtx, /* Return value for this function */ + sqlite3_value **aReplace /* Array of replacement values */ ){ - Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; - int rc; /* Return code */ - const char *zDoc; /* Document text to extract snippet from */ - int nDoc; /* Size of zDoc in bytes */ - int iCurrent = 0; /* Current token number of document */ - int iEnd = 0; /* Byte offset of end of current token */ - int isShiftDone = 0; /* True after snippet is shifted */ - int iPos = pFragment->iPos; /* First token of snippet */ - u64 hlmask = pFragment->hlmask; /* Highlight-mask for snippet */ - int iCol = pFragment->iCol+1; /* Query column to extract text from */ - sqlite3_tokenizer_module *pMod; /* Tokenizer module methods object */ - sqlite3_tokenizer_cursor *pC; /* Tokenizer cursor open on zDoc/nDoc */ - - zDoc = (const char *)sqlite3_column_text(pCsr->pStmt, iCol); - if( zDoc==0 ){ - if( sqlite3_column_type(pCsr->pStmt, iCol)!=SQLITE_NULL ){ - return SQLITE_NOMEM; + switch( pNode->eType ){ + default: { + assert( pNode->eType==JSON_NULL ); + sqlite3_result_null(pCtx); + break; } - return SQLITE_OK; - } - nDoc = sqlite3_column_bytes(pCsr->pStmt, iCol); - - /* Open a token cursor on the document. */ - pMod = (sqlite3_tokenizer_module *)pTab->pTokenizer->pModule; - rc = sqlite3Fts3OpenTokenizer(pTab->pTokenizer, pCsr->iLangid, zDoc,nDoc,&pC); - if( rc!=SQLITE_OK ){ - return rc; - } - - while( rc==SQLITE_OK ){ - const char *ZDUMMY; /* Dummy argument used with tokenizer */ - int DUMMY1 = -1; /* Dummy argument used with tokenizer */ - int iBegin = 0; /* Offset in zDoc of start of token */ - int iFin = 0; /* Offset in zDoc of end of token */ - int isHighlight = 0; /* True for highlighted terms */ - - /* Variable DUMMY1 is initialized to a negative value above. Elsewhere - ** in the FTS code the variable that the third argument to xNext points to - ** is initialized to zero before the first (*but not necessarily - ** subsequent*) call to xNext(). This is done for a particular application - ** that needs to know whether or not the tokenizer is being used for - ** snippet generation or for some other purpose. - ** - ** Extreme care is required when writing code to depend on this - ** initialization. It is not a documented part of the tokenizer interface. - ** If a tokenizer is used directly by any code outside of FTS, this - ** convention might not be respected. */ - rc = pMod->xNext(pC, &ZDUMMY, &DUMMY1, &iBegin, &iFin, &iCurrent); - if( rc!=SQLITE_OK ){ - if( rc==SQLITE_DONE ){ - /* Special case - the last token of the snippet is also the last token - ** of the column. Append any punctuation that occurred between the end - ** of the previous token and the end of the document to the output. - ** Then break out of the loop. */ - rc = fts3StringAppend(pOut, &zDoc[iEnd], -1); + case JSON_TRUE: { + sqlite3_result_int(pCtx, 1); + break; + } + case JSON_FALSE: { + sqlite3_result_int(pCtx, 0); + break; + } + case JSON_INT: { + sqlite3_int64 i = 0; + const char *z = pNode->u.zJContent; + if( z[0]=='-' ){ z++; } + while( z[0]>='0' && z[0]<='9' ){ + unsigned v = *(z++) - '0'; + if( i>=LARGEST_INT64/10 ){ + if( i>LARGEST_INT64/10 ) goto int_as_real; + if( z[0]>='0' && z[0]<='9' ) goto int_as_real; + if( v==9 ) goto int_as_real; + if( v==8 ){ + if( pNode->u.zJContent[0]=='-' ){ + sqlite3_result_int64(pCtx, SMALLEST_INT64); + goto int_done; + }else{ + goto int_as_real; + } + } + } + i = i*10 + v; } + if( pNode->u.zJContent[0]=='-' ){ i = -i; } + sqlite3_result_int64(pCtx, i); + int_done: break; + int_as_real: /* fall through to real */; } - if( iCurrentiLangid, nSnippet, &zDoc[iBegin], n, &iPos, &hlmask - ); - isShiftDone = 1; - - /* Now that the shift has been done, check if the initial "..." are - ** required. They are required if (a) this is not the first fragment, - ** or (b) this fragment does not begin at position 0 of its column. - */ - if( rc==SQLITE_OK ){ - if( iPos>0 || iFragment>0 ){ - rc = fts3StringAppend(pOut, zEllipsis, -1); - }else if( iBegin ){ - rc = fts3StringAppend(pOut, zDoc, iBegin); + case JSON_REAL: { + double r; +#ifdef SQLITE_AMALGAMATION + const char *z = pNode->u.zJContent; + sqlite3AtoF(z, &r, sqlite3Strlen30(z), SQLITE_UTF8); +#else + r = strtod(pNode->u.zJContent, 0); +#endif + sqlite3_result_double(pCtx, r); + break; + } + case JSON_STRING: { +#if 0 /* Never happens because JNODE_RAW is only set by json_set(), + ** json_insert() and json_replace() and those routines do not + ** call jsonReturn() */ + if( pNode->jnFlags & JNODE_RAW ){ + sqlite3_result_text(pCtx, pNode->u.zJContent, pNode->n, + SQLITE_TRANSIENT); + }else +#endif + assert( (pNode->jnFlags & JNODE_RAW)==0 ); + if( (pNode->jnFlags & JNODE_ESCAPE)==0 ){ + /* JSON formatted without any backslash-escapes */ + sqlite3_result_text(pCtx, pNode->u.zJContent+1, pNode->n-2, + SQLITE_TRANSIENT); + }else{ + /* Translate JSON formatted string into raw text */ + u32 i; + u32 n = pNode->n; + const char *z = pNode->u.zJContent; + char *zOut; + u32 j; + zOut = sqlite3_malloc( n+1 ); + if( zOut==0 ){ + sqlite3_result_error_nomem(pCtx); + break; + } + for(i=1, j=0; i>6)); + zOut[j++] = 0x80 | (v&0x3f); + }else{ + u32 vlo; + if( (v&0xfc00)==0xd800 + && i>18); + zOut[j++] = 0x80 | ((v>>12)&0x3f); + zOut[j++] = 0x80 | ((v>>6)&0x3f); + zOut[j++] = 0x80 | (v&0x3f); + }else{ + zOut[j++] = 0xe0 | (v>>12); + zOut[j++] = 0x80 | ((v>>6)&0x3f); + zOut[j++] = 0x80 | (v&0x3f); + } + } + }else{ + if( c=='b' ){ + c = '\b'; + }else if( c=='f' ){ + c = '\f'; + }else if( c=='n' ){ + c = '\n'; + }else if( c=='r' ){ + c = '\r'; + }else if( c=='t' ){ + c = '\t'; + } + zOut[j++] = c; + } + } } + zOut[j] = 0; + sqlite3_result_text(pCtx, zOut, j, sqlite3_free); } - if( rc!=SQLITE_OK || iCurrent=(iPos+nSnippet) ){ - if( isLast ){ - rc = fts3StringAppend(pOut, zEllipsis, -1); - } + case JSON_ARRAY: + case JSON_OBJECT: { + jsonReturnJson(pNode, pCtx, aReplace); break; } - - /* Set isHighlight to true if this term should be highlighted. */ - isHighlight = (hlmask & ((u64)1 << (iCurrent-iPos)))!=0; - - if( iCurrent>iPos ) rc = fts3StringAppend(pOut, &zDoc[iEnd], iBegin-iEnd); - if( rc==SQLITE_OK && isHighlight ) rc = fts3StringAppend(pOut, zOpen, -1); - if( rc==SQLITE_OK ) rc = fts3StringAppend(pOut, &zDoc[iBegin], iFin-iBegin); - if( rc==SQLITE_OK && isHighlight ) rc = fts3StringAppend(pOut, zClose, -1); - - iEnd = iFin; } - - pMod->xClose(pC); - return rc; } +/* Forward reference */ +static int jsonParseAddNode(JsonParse*,u32,u32,const char*); /* -** This function is used to count the entries in a column-list (a -** delta-encoded list of term offsets within a single column of a single -** row). When this function is called, *ppCollist should point to the -** beginning of the first varint in the column-list (the varint that -** contains the position of the first matching term in the column data). -** Before returning, *ppCollist is set to point to the first byte after -** the last varint in the column-list (either the 0x00 signifying the end -** of the position-list, or the 0x01 that precedes the column number of -** the next column in the position-list). -** -** The number of elements in the column-list is returned. +** A macro to hint to the compiler that a function should not be +** inlined. */ -static int fts3ColumnlistCount(char **ppCollist){ - char *pEnd = *ppCollist; - char c = 0; - int nEntry = 0; +#if defined(__GNUC__) +# define JSON_NOINLINE __attribute__((noinline)) +#elif defined(_MSC_VER) && _MSC_VER>=1310 +# define JSON_NOINLINE __declspec(noinline) +#else +# define JSON_NOINLINE +#endif - /* A column-list is terminated by either a 0x01 or 0x00. */ - while( 0xFE & (*pEnd | c) ){ - c = *pEnd++ & 0x80; - if( !c ) nEntry++; - } - *ppCollist = pEnd; - return nEntry; +static JSON_NOINLINE int jsonParseAddNodeExpand( + JsonParse *pParse, /* Append the node to this object */ + u32 eType, /* Node type */ + u32 n, /* Content size or sub-node count */ + const char *zContent /* Content */ +){ + u32 nNew; + JsonNode *pNew; + assert( pParse->nNode>=pParse->nAlloc ); + if( pParse->oom ) return -1; + nNew = pParse->nAlloc*2 + 10; + pNew = sqlite3_realloc64(pParse->aNode, sizeof(JsonNode)*nNew); + if( pNew==0 ){ + pParse->oom = 1; + return -1; + } + pParse->nAlloc = nNew; + pParse->aNode = pNew; + assert( pParse->nNodenAlloc ); + return jsonParseAddNode(pParse, eType, n, zContent); } /* -** This function gathers 'y' or 'b' data for a single phrase. +** Create a new JsonNode instance based on the arguments and append that +** instance to the JsonParse. Return the index in pParse->aNode[] of the +** new node, or -1 if a memory allocation fails. */ -static void fts3ExprLHits( - Fts3Expr *pExpr, /* Phrase expression node */ - MatchInfo *p /* Matchinfo context */ +static int jsonParseAddNode( + JsonParse *pParse, /* Append the node to this object */ + u32 eType, /* Node type */ + u32 n, /* Content size or sub-node count */ + const char *zContent /* Content */ ){ - Fts3Table *pTab = (Fts3Table *)p->pCursor->base.pVtab; - int iStart; - Fts3Phrase *pPhrase = pExpr->pPhrase; - char *pIter = pPhrase->doclist.pList; - int iCol = 0; - - assert( p->flag==FTS3_MATCHINFO_LHITS_BM || p->flag==FTS3_MATCHINFO_LHITS ); - if( p->flag==FTS3_MATCHINFO_LHITS ){ - iStart = pExpr->iPhrase * p->nCol; - }else{ - iStart = pExpr->iPhrase * ((p->nCol + 31) / 32); - } - - while( 1 ){ - int nHit = fts3ColumnlistCount(&pIter); - if( (pPhrase->iColumn>=pTab->nColumn || pPhrase->iColumn==iCol) ){ - if( p->flag==FTS3_MATCHINFO_LHITS ){ - p->aMatchinfo[iStart + iCol] = (u32)nHit; - }else if( nHit ){ - p->aMatchinfo[iStart + (iCol+1)/32] |= (1 << (iCol&0x1F)); - } - } - assert( *pIter==0x00 || *pIter==0x01 ); - if( *pIter!=0x01 ) break; - pIter++; - pIter += fts3GetVarint32(pIter, &iCol); + JsonNode *p; + if( pParse->nNode>=pParse->nAlloc ){ + return jsonParseAddNodeExpand(pParse, eType, n, zContent); } + p = &pParse->aNode[pParse->nNode]; + p->eType = (u8)eType; + p->jnFlags = 0; + p->n = n; + p->u.zJContent = zContent; + return pParse->nNode++; } /* -** Gather the results for matchinfo directives 'y' and 'b'. +** Return true if z[] begins with 4 (or more) hexadecimal digits */ -static void fts3ExprLHitGather( - Fts3Expr *pExpr, - MatchInfo *p -){ - assert( (pExpr->pLeft==0)==(pExpr->pRight==0) ); - if( pExpr->bEof==0 && pExpr->iDocid==p->pCursor->iPrevId ){ - if( pExpr->pLeft ){ - fts3ExprLHitGather(pExpr->pLeft, p); - fts3ExprLHitGather(pExpr->pRight, p); - }else{ - fts3ExprLHits(pExpr, p); - } - } +static int jsonIs4Hex(const char *z){ + int i; + for(i=0; i<4; i++) if( !safe_isxdigit(z[i]) ) return 0; + return 1; } /* -** fts3ExprIterate() callback used to collect the "global" matchinfo stats -** for a single query. -** -** fts3ExprIterate() callback to load the 'global' elements of a -** FTS3_MATCHINFO_HITS matchinfo array. The global stats are those elements -** of the matchinfo array that are constant for all rows returned by the -** current query. -** -** Argument pCtx is actually a pointer to a struct of type MatchInfo. This -** function populates Matchinfo.aMatchinfo[] as follows: -** -** for(iCol=0; iColzJson[i]. Return the +** index of the first character past the end of the value parsed. ** -** If the phrase pExpr consists entirely of deferred tokens, then all X and -** Y values are set to nDoc, where nDoc is the number of documents in the -** file system. This is done because the full-text index doclist is required -** to calculate these values properly, and the full-text index doclist is -** not available for deferred tokens. +** Return negative for a syntax error. Special cases: return -2 if the +** first non-whitespace character is '}' and return -3 if the first +** non-whitespace character is ']'. */ -static int fts3ExprGlobalHitsCb( - Fts3Expr *pExpr, /* Phrase expression node */ - int iPhrase, /* Phrase number (numbered from zero) */ - void *pCtx /* Pointer to MatchInfo structure */ -){ - MatchInfo *p = (MatchInfo *)pCtx; - return sqlite3Fts3EvalPhraseStats( - p->pCursor, pExpr, &p->aMatchinfo[3*iPhrase*p->nCol] - ); +static int jsonParseValue(JsonParse *pParse, u32 i){ + char c; + u32 j; + int iThis; + int x; + JsonNode *pNode; + const char *z = pParse->zJson; + while( safe_isspace(z[i]) ){ i++; } + if( (c = z[i])=='{' ){ + /* Parse object */ + iThis = jsonParseAddNode(pParse, JSON_OBJECT, 0, 0); + if( iThis<0 ) return -1; + for(j=i+1;;j++){ + while( safe_isspace(z[j]) ){ j++; } + if( ++pParse->iDepth > JSON_MAX_DEPTH ) return -1; + x = jsonParseValue(pParse, j); + if( x<0 ){ + pParse->iDepth--; + if( x==(-2) && pParse->nNode==(u32)iThis+1 ) return j+1; + return -1; + } + if( pParse->oom ) return -1; + pNode = &pParse->aNode[pParse->nNode-1]; + if( pNode->eType!=JSON_STRING ) return -1; + pNode->jnFlags |= JNODE_LABEL; + j = x; + while( safe_isspace(z[j]) ){ j++; } + if( z[j]!=':' ) return -1; + j++; + x = jsonParseValue(pParse, j); + pParse->iDepth--; + if( x<0 ) return -1; + j = x; + while( safe_isspace(z[j]) ){ j++; } + c = z[j]; + if( c==',' ) continue; + if( c!='}' ) return -1; + break; + } + pParse->aNode[iThis].n = pParse->nNode - (u32)iThis - 1; + return j+1; + }else if( c=='[' ){ + /* Parse array */ + iThis = jsonParseAddNode(pParse, JSON_ARRAY, 0, 0); + if( iThis<0 ) return -1; + for(j=i+1;;j++){ + while( safe_isspace(z[j]) ){ j++; } + if( ++pParse->iDepth > JSON_MAX_DEPTH ) return -1; + x = jsonParseValue(pParse, j); + pParse->iDepth--; + if( x<0 ){ + if( x==(-3) && pParse->nNode==(u32)iThis+1 ) return j+1; + return -1; + } + j = x; + while( safe_isspace(z[j]) ){ j++; } + c = z[j]; + if( c==',' ) continue; + if( c!=']' ) return -1; + break; + } + pParse->aNode[iThis].n = pParse->nNode - (u32)iThis - 1; + return j+1; + }else if( c=='"' ){ + /* Parse string */ + u8 jnFlags = 0; + j = i+1; + for(;;){ + c = z[j]; + if( (c & ~0x1f)==0 ){ + /* Control characters are not allowed in strings */ + return -1; + } + if( c=='\\' ){ + c = z[++j]; + if( c=='"' || c=='\\' || c=='/' || c=='b' || c=='f' + || c=='n' || c=='r' || c=='t' + || (c=='u' && jsonIs4Hex(z+j+1)) ){ + jnFlags = JNODE_ESCAPE; + }else{ + return -1; + } + }else if( c=='"' ){ + break; + } + j++; + } + jsonParseAddNode(pParse, JSON_STRING, j+1-i, &z[i]); + if( !pParse->oom ) pParse->aNode[pParse->nNode-1].jnFlags = jnFlags; + return j+1; + }else if( c=='n' + && strncmp(z+i,"null",4)==0 + && !safe_isalnum(z[i+4]) ){ + jsonParseAddNode(pParse, JSON_NULL, 0, 0); + return i+4; + }else if( c=='t' + && strncmp(z+i,"true",4)==0 + && !safe_isalnum(z[i+4]) ){ + jsonParseAddNode(pParse, JSON_TRUE, 0, 0); + return i+4; + }else if( c=='f' + && strncmp(z+i,"false",5)==0 + && !safe_isalnum(z[i+5]) ){ + jsonParseAddNode(pParse, JSON_FALSE, 0, 0); + return i+5; + }else if( c=='-' || (c>='0' && c<='9') ){ + /* Parse number */ + u8 seenDP = 0; + u8 seenE = 0; + assert( '-' < '0' ); + if( c<='0' ){ + j = c=='-' ? i+1 : i; + if( z[j]=='0' && z[j+1]>='0' && z[j+1]<='9' ) return -1; + } + j = i+1; + for(;; j++){ + c = z[j]; + if( c>='0' && c<='9' ) continue; + if( c=='.' ){ + if( z[j-1]=='-' ) return -1; + if( seenDP ) return -1; + seenDP = 1; + continue; + } + if( c=='e' || c=='E' ){ + if( z[j-1]<'0' ) return -1; + if( seenE ) return -1; + seenDP = seenE = 1; + c = z[j+1]; + if( c=='+' || c=='-' ){ + j++; + c = z[j+1]; + } + if( c<'0' || c>'9' ) return -1; + continue; + } + break; + } + if( z[j-1]<'0' ) return -1; + jsonParseAddNode(pParse, seenDP ? JSON_REAL : JSON_INT, + j - i, &z[i]); + return j; + }else if( c=='}' ){ + return -2; /* End of {...} */ + }else if( c==']' ){ + return -3; /* End of [...] */ + }else if( c==0 ){ + return 0; /* End of file */ + }else{ + return -1; /* Syntax error */ + } } /* -** fts3ExprIterate() callback used to collect the "local" part of the -** FTS3_MATCHINFO_HITS array. The local stats are those elements of the -** array that are different for each row returned by the query. +** Parse a complete JSON string. Return 0 on success or non-zero if there +** are any errors. If an error occurs, free all memory associated with +** pParse. +** +** pParse is uninitialized when this routine is called. */ -static int fts3ExprLocalHitsCb( - Fts3Expr *pExpr, /* Phrase expression node */ - int iPhrase, /* Phrase number */ - void *pCtx /* Pointer to MatchInfo structure */ +static int jsonParse( + JsonParse *pParse, /* Initialize and fill this JsonParse object */ + sqlite3_context *pCtx, /* Report errors here */ + const char *zJson /* Input JSON text to be parsed */ ){ - int rc = SQLITE_OK; - MatchInfo *p = (MatchInfo *)pCtx; - int iStart = iPhrase * p->nCol * 3; int i; - - for(i=0; inCol && rc==SQLITE_OK; i++){ - char *pCsr; - rc = sqlite3Fts3EvalPhrasePoslist(p->pCursor, pExpr, i, &pCsr); - if( pCsr ){ - p->aMatchinfo[iStart+i*3] = fts3ColumnlistCount(&pCsr); - }else{ - p->aMatchinfo[iStart+i*3] = 0; - } + memset(pParse, 0, sizeof(*pParse)); + if( zJson==0 ) return 1; + pParse->zJson = zJson; + i = jsonParseValue(pParse, 0); + if( pParse->oom ) i = -1; + if( i>0 ){ + assert( pParse->iDepth==0 ); + while( safe_isspace(zJson[i]) ) i++; + if( zJson[i] ) i = -1; } - - return rc; -} - -static int fts3MatchinfoCheck( - Fts3Table *pTab, - char cArg, - char **pzErr -){ - if( (cArg==FTS3_MATCHINFO_NPHRASE) - || (cArg==FTS3_MATCHINFO_NCOL) - || (cArg==FTS3_MATCHINFO_NDOC && pTab->bFts4) - || (cArg==FTS3_MATCHINFO_AVGLENGTH && pTab->bFts4) - || (cArg==FTS3_MATCHINFO_LENGTH && pTab->bHasDocsize) - || (cArg==FTS3_MATCHINFO_LCS) - || (cArg==FTS3_MATCHINFO_HITS) - || (cArg==FTS3_MATCHINFO_LHITS) - || (cArg==FTS3_MATCHINFO_LHITS_BM) - ){ - return SQLITE_OK; + if( i<=0 ){ + if( pCtx!=0 ){ + if( pParse->oom ){ + sqlite3_result_error_nomem(pCtx); + }else{ + sqlite3_result_error(pCtx, "malformed JSON", -1); + } + } + jsonParseReset(pParse); + return 1; } - sqlite3Fts3ErrMsg(pzErr, "unrecognized matchinfo request: %c", cArg); - return SQLITE_ERROR; + return 0; } -static int fts3MatchinfoSize(MatchInfo *pInfo, char cArg){ - int nVal; /* Number of integers output by cArg */ - - switch( cArg ){ - case FTS3_MATCHINFO_NDOC: - case FTS3_MATCHINFO_NPHRASE: - case FTS3_MATCHINFO_NCOL: - nVal = 1; - break; - - case FTS3_MATCHINFO_AVGLENGTH: - case FTS3_MATCHINFO_LENGTH: - case FTS3_MATCHINFO_LCS: - nVal = pInfo->nCol; - break; - - case FTS3_MATCHINFO_LHITS: - nVal = pInfo->nCol * pInfo->nPhrase; +/* Mark node i of pParse as being a child of iParent. Call recursively +** to fill in all the descendants of node i. +*/ +static void jsonParseFillInParentage(JsonParse *pParse, u32 i, u32 iParent){ + JsonNode *pNode = &pParse->aNode[i]; + u32 j; + pParse->aUp[i] = iParent; + switch( pNode->eType ){ + case JSON_ARRAY: { + for(j=1; j<=pNode->n; j += jsonNodeSize(pNode+j)){ + jsonParseFillInParentage(pParse, i+j, i); + } break; - - case FTS3_MATCHINFO_LHITS_BM: - nVal = pInfo->nPhrase * ((pInfo->nCol + 31) / 32); + } + case JSON_OBJECT: { + for(j=1; j<=pNode->n; j += jsonNodeSize(pNode+j+1)+1){ + pParse->aUp[i+j] = i; + jsonParseFillInParentage(pParse, i+j+1, i); + } break; - - default: - assert( cArg==FTS3_MATCHINFO_HITS ); - nVal = pInfo->nCol * pInfo->nPhrase * 3; + } + default: { break; + } } - - return nVal; } -static int fts3MatchinfoSelectDoctotal( - Fts3Table *pTab, - sqlite3_stmt **ppStmt, - sqlite3_int64 *pnDoc, - const char **paLen -){ - sqlite3_stmt *pStmt; - const char *a; - sqlite3_int64 nDoc; - - if( !*ppStmt ){ - int rc = sqlite3Fts3SelectDoctotal(pTab, ppStmt); - if( rc!=SQLITE_OK ) return rc; +/* +** Compute the parentage of all nodes in a completed parse. +*/ +static int jsonParseFindParents(JsonParse *pParse){ + u32 *aUp; + assert( pParse->aUp==0 ); + aUp = pParse->aUp = sqlite3_malloc64( sizeof(u32)*pParse->nNode ); + if( aUp==0 ){ + pParse->oom = 1; + return SQLITE_NOMEM; } - pStmt = *ppStmt; - assert( sqlite3_data_count(pStmt)==1 ); - - a = sqlite3_column_blob(pStmt, 0); - a += sqlite3Fts3GetVarint(a, &nDoc); - if( nDoc==0 ) return FTS_CORRUPT_VTAB; - *pnDoc = (u32)nDoc; - - if( paLen ) *paLen = a; + jsonParseFillInParentage(pParse, 0, 0); return SQLITE_OK; } /* -** An instance of the following structure is used to store state while -** iterating through a multi-column position-list corresponding to the -** hits for a single phrase on a single row in order to calculate the -** values for a matchinfo() FTS3_MATCHINFO_LCS request. +** Magic number used for the JSON parse cache in sqlite3_get_auxdata() */ -typedef struct LcsIterator LcsIterator; -struct LcsIterator { - Fts3Expr *pExpr; /* Pointer to phrase expression */ - int iPosOffset; /* Tokens count up to end of this phrase */ - char *pRead; /* Cursor used to iterate through aDoclist */ - int iPos; /* Current position */ -}; +#define JSON_CACHE_ID (-429938) /* First cache entry */ +#define JSON_CACHE_SZ 4 /* Max number of cache entries */ -/* -** If LcsIterator.iCol is set to the following value, the iterator has -** finished iterating through all offsets for all columns. +/* +** Obtain a complete parse of the JSON found in the first argument +** of the argv array. Use the sqlite3_get_auxdata() cache for this +** parse if it is available. If the cache is not available or if it +** is no longer valid, parse the JSON again and return the new parse, +** and also register the new parse so that it will be available for +** future sqlite3_get_auxdata() calls. */ -#define LCS_ITERATOR_FINISHED 0x7FFFFFFF; - -static int fts3MatchinfoLcsCb( - Fts3Expr *pExpr, /* Phrase expression node */ - int iPhrase, /* Phrase number (numbered from zero) */ - void *pCtx /* Pointer to MatchInfo structure */ +static JsonParse *jsonParseCached( + sqlite3_context *pCtx, + sqlite3_value **argv, + sqlite3_context *pErrCtx ){ - LcsIterator *aIter = (LcsIterator *)pCtx; - aIter[iPhrase].pExpr = pExpr; - return SQLITE_OK; + const char *zJson = (const char*)sqlite3_value_text(argv[0]); + int nJson = sqlite3_value_bytes(argv[0]); + JsonParse *p; + JsonParse *pMatch = 0; + int iKey; + int iMinKey = 0; + u32 iMinHold = 0xffffffff; + u32 iMaxHold = 0; + if( zJson==0 ) return 0; + for(iKey=0; iKeynJson==nJson + && memcmp(p->zJson,zJson,nJson)==0 + ){ + p->nErr = 0; + pMatch = p; + }else if( p->iHoldiHold; + iMinKey = iKey; + } + if( p->iHold>iMaxHold ){ + iMaxHold = p->iHold; + } + } + if( pMatch ){ + pMatch->nErr = 0; + pMatch->iHold = iMaxHold+1; + return pMatch; + } + p = sqlite3_malloc64( sizeof(*p) + nJson + 1 ); + if( p==0 ){ + sqlite3_result_error_nomem(pCtx); + return 0; + } + memset(p, 0, sizeof(*p)); + p->zJson = (char*)&p[1]; + memcpy((char*)p->zJson, zJson, nJson+1); + if( jsonParse(p, pErrCtx, p->zJson) ){ + sqlite3_free(p); + return 0; + } + p->nJson = nJson; + p->iHold = iMaxHold+1; + sqlite3_set_auxdata(pCtx, JSON_CACHE_ID+iMinKey, p, + (void(*)(void*))jsonParseFree); + return (JsonParse*)sqlite3_get_auxdata(pCtx, JSON_CACHE_ID+iMinKey); } /* -** Advance the iterator passed as an argument to the next position. Return -** 1 if the iterator is at EOF or if it now points to the start of the -** position list for the next column. +** Compare the OBJECT label at pNode against zKey,nKey. Return true on +** a match. */ -static int fts3LcsIteratorAdvance(LcsIterator *pIter){ - char *pRead = pIter->pRead; - sqlite3_int64 iRead; - int rc = 0; - - pRead += sqlite3Fts3GetVarint(pRead, &iRead); - if( iRead==0 || iRead==1 ){ - pRead = 0; - rc = 1; +static int jsonLabelCompare(JsonNode *pNode, const char *zKey, u32 nKey){ + if( pNode->jnFlags & JNODE_RAW ){ + if( pNode->n!=nKey ) return 0; + return strncmp(pNode->u.zJContent, zKey, nKey)==0; }else{ - pIter->iPos += (int)(iRead-2); + if( pNode->n!=nKey+2 ) return 0; + return strncmp(pNode->u.zJContent+1, zKey, nKey)==0; } - - pIter->pRead = pRead; - return rc; } - + +/* forward declaration */ +static JsonNode *jsonLookupAppend(JsonParse*,const char*,int*,const char**); + /* -** This function implements the FTS3_MATCHINFO_LCS matchinfo() flag. -** -** If the call is successful, the longest-common-substring lengths for each -** column are written into the first nCol elements of the pInfo->aMatchinfo[] -** array before returning. SQLITE_OK is returned in this case. +** Search along zPath to find the node specified. Return a pointer +** to that node, or NULL if zPath is malformed or if there is no such +** node. ** -** Otherwise, if an error occurs, an SQLite error code is returned and the -** data written to the first nCol elements of pInfo->aMatchinfo[] is -** undefined. +** If pApnd!=0, then try to append new nodes to complete zPath if it is +** possible to do so and if no existing node corresponds to zPath. If +** new nodes are appended *pApnd is set to 1. */ -static int fts3MatchinfoLcs(Fts3Cursor *pCsr, MatchInfo *pInfo){ - LcsIterator *aIter; - int i; - int iCol; - int nToken = 0; - - /* Allocate and populate the array of LcsIterator objects. The array - ** contains one element for each matchable phrase in the query. - **/ - aIter = sqlite3_malloc(sizeof(LcsIterator) * pCsr->nPhrase); - if( !aIter ) return SQLITE_NOMEM; - memset(aIter, 0, sizeof(LcsIterator) * pCsr->nPhrase); - (void)fts3ExprIterate(pCsr->pExpr, fts3MatchinfoLcsCb, (void*)aIter); - - for(i=0; inPhrase; i++){ - LcsIterator *pIter = &aIter[i]; - nToken -= pIter->pExpr->pPhrase->nToken; - pIter->iPosOffset = nToken; - } - - for(iCol=0; iColnCol; iCol++){ - int nLcs = 0; /* LCS value for this column */ - int nLive = 0; /* Number of iterators in aIter not at EOF */ - - for(i=0; inPhrase; i++){ - int rc; - LcsIterator *pIt = &aIter[i]; - rc = sqlite3Fts3EvalPhrasePoslist(pCsr, pIt->pExpr, iCol, &pIt->pRead); - if( rc!=SQLITE_OK ) return rc; - if( pIt->pRead ){ - pIt->iPos = pIt->iPosOffset; - fts3LcsIteratorAdvance(&aIter[i]); - nLive++; +static JsonNode *jsonLookupStep( + JsonParse *pParse, /* The JSON to search */ + u32 iRoot, /* Begin the search at this node */ + const char *zPath, /* The path to search */ + int *pApnd, /* Append nodes to complete path if not NULL */ + const char **pzErr /* Make *pzErr point to any syntax error in zPath */ +){ + u32 i, j, nKey; + const char *zKey; + JsonNode *pRoot = &pParse->aNode[iRoot]; + if( zPath[0]==0 ) return pRoot; + if( pRoot->jnFlags & JNODE_REPLACE ) return 0; + if( zPath[0]=='.' ){ + if( pRoot->eType!=JSON_OBJECT ) return 0; + zPath++; + if( zPath[0]=='"' ){ + zKey = zPath + 1; + for(i=1; zPath[i] && zPath[i]!='"'; i++){} + nKey = i-1; + if( zPath[i] ){ + i++; + }else{ + *pzErr = zPath; + return 0; } + }else{ + zKey = zPath; + for(i=0; zPath[i] && zPath[i]!='.' && zPath[i]!='['; i++){} + nKey = i; } - - while( nLive>0 ){ - LcsIterator *pAdv = 0; /* The iterator to advance by one position */ - int nThisLcs = 0; /* LCS for the current iterator positions */ - - for(i=0; inPhrase; i++){ - LcsIterator *pIter = &aIter[i]; - if( pIter->pRead==0 ){ - /* This iterator is already at EOF for this column. */ - nThisLcs = 0; - }else{ - if( pAdv==0 || pIter->iPosiPos ){ - pAdv = pIter; - } - if( nThisLcs==0 || pIter->iPos==pIter[-1].iPos ){ - nThisLcs++; - }else{ - nThisLcs = 1; + if( nKey==0 ){ + *pzErr = zPath; + return 0; + } + j = 1; + for(;;){ + while( j<=pRoot->n ){ + if( jsonLabelCompare(pRoot+j, zKey, nKey) ){ + return jsonLookupStep(pParse, iRoot+j+1, &zPath[i], pApnd, pzErr); + } + j++; + j += jsonNodeSize(&pRoot[j]); + } + if( (pRoot->jnFlags & JNODE_APPEND)==0 ) break; + iRoot += pRoot->u.iAppend; + pRoot = &pParse->aNode[iRoot]; + j = 1; + } + if( pApnd ){ + u32 iStart, iLabel; + JsonNode *pNode; + iStart = jsonParseAddNode(pParse, JSON_OBJECT, 2, 0); + iLabel = jsonParseAddNode(pParse, JSON_STRING, nKey, zKey); + zPath += i; + pNode = jsonLookupAppend(pParse, zPath, pApnd, pzErr); + if( pParse->oom ) return 0; + if( pNode ){ + pRoot = &pParse->aNode[iRoot]; + pRoot->u.iAppend = iStart - iRoot; + pRoot->jnFlags |= JNODE_APPEND; + pParse->aNode[iLabel].jnFlags |= JNODE_RAW; + } + return pNode; + } + }else if( zPath[0]=='[' ){ + i = 0; + j = 1; + while( safe_isdigit(zPath[j]) ){ + i = i*10 + zPath[j] - '0'; + j++; + } + if( j<2 || zPath[j]!=']' ){ + if( zPath[1]=='#' ){ + JsonNode *pBase = pRoot; + int iBase = iRoot; + if( pRoot->eType!=JSON_ARRAY ) return 0; + for(;;){ + while( j<=pBase->n ){ + if( (pBase[j].jnFlags & JNODE_REMOVE)==0 ) i++; + j += jsonNodeSize(&pBase[j]); } - if( nThisLcs>nLcs ) nLcs = nThisLcs; + if( (pBase->jnFlags & JNODE_APPEND)==0 ) break; + iBase += pBase->u.iAppend; + pBase = &pParse->aNode[iBase]; + j = 1; + } + j = 2; + if( zPath[2]=='-' && safe_isdigit(zPath[3]) ){ + unsigned int x = 0; + j = 3; + do{ + x = x*10 + zPath[j] - '0'; + j++; + }while( safe_isdigit(zPath[j]) ); + if( x>i ) return 0; + i -= x; + } + if( zPath[j]!=']' ){ + *pzErr = zPath; + return 0; } + }else{ + *pzErr = zPath; + return 0; } - if( fts3LcsIteratorAdvance(pAdv) ) nLive--; } + if( pRoot->eType!=JSON_ARRAY ) return 0; + zPath += j + 1; + j = 1; + for(;;){ + while( j<=pRoot->n && (i>0 || (pRoot[j].jnFlags & JNODE_REMOVE)!=0) ){ + if( (pRoot[j].jnFlags & JNODE_REMOVE)==0 ) i--; + j += jsonNodeSize(&pRoot[j]); + } + if( (pRoot->jnFlags & JNODE_APPEND)==0 ) break; + iRoot += pRoot->u.iAppend; + pRoot = &pParse->aNode[iRoot]; + j = 1; + } + if( j<=pRoot->n ){ + return jsonLookupStep(pParse, iRoot+j, zPath, pApnd, pzErr); + } + if( i==0 && pApnd ){ + u32 iStart; + JsonNode *pNode; + iStart = jsonParseAddNode(pParse, JSON_ARRAY, 1, 0); + pNode = jsonLookupAppend(pParse, zPath, pApnd, pzErr); + if( pParse->oom ) return 0; + if( pNode ){ + pRoot = &pParse->aNode[iRoot]; + pRoot->u.iAppend = iStart - iRoot; + pRoot->jnFlags |= JNODE_APPEND; + } + return pNode; + } + }else{ + *pzErr = zPath; + } + return 0; +} - pInfo->aMatchinfo[iCol] = nLcs; +/* +** Append content to pParse that will complete zPath. Return a pointer +** to the inserted node, or return NULL if the append fails. +*/ +static JsonNode *jsonLookupAppend( + JsonParse *pParse, /* Append content to the JSON parse */ + const char *zPath, /* Description of content to append */ + int *pApnd, /* Set this flag to 1 */ + const char **pzErr /* Make this point to any syntax error */ +){ + *pApnd = 1; + if( zPath[0]==0 ){ + jsonParseAddNode(pParse, JSON_NULL, 0, 0); + return pParse->oom ? 0 : &pParse->aNode[pParse->nNode-1]; } + if( zPath[0]=='.' ){ + jsonParseAddNode(pParse, JSON_OBJECT, 0, 0); + }else if( strncmp(zPath,"[0]",3)==0 ){ + jsonParseAddNode(pParse, JSON_ARRAY, 0, 0); + }else{ + return 0; + } + if( pParse->oom ) return 0; + return jsonLookupStep(pParse, pParse->nNode-1, zPath, pApnd, pzErr); +} - sqlite3_free(aIter); - return SQLITE_OK; +/* +** Return the text of a syntax error message on a JSON path. Space is +** obtained from sqlite3_malloc(). +*/ +static char *jsonPathSyntaxError(const char *zErr){ + return sqlite3_mprintf("JSON path error near '%q'", zErr); } /* -** Populate the buffer pInfo->aMatchinfo[] with an array of integers to -** be returned by the matchinfo() function. Argument zArg contains the -** format string passed as the second argument to matchinfo (or the -** default value "pcx" if no second argument was specified). The format -** string has already been validated and the pInfo->aMatchinfo[] array -** is guaranteed to be large enough for the output. +** Do a node lookup using zPath. Return a pointer to the node on success. +** Return NULL if not found or if there is an error. ** -** If bGlobal is true, then populate all fields of the matchinfo() output. -** If it is false, then assume that those fields that do not change between -** rows (i.e. FTS3_MATCHINFO_NPHRASE, NCOL, NDOC, AVGLENGTH and part of HITS) -** have already been populated. +** On an error, write an error message into pCtx and increment the +** pParse->nErr counter. ** -** Return SQLITE_OK if successful, or an SQLite error code if an error -** occurs. If a value other than SQLITE_OK is returned, the state the -** pInfo->aMatchinfo[] buffer is left in is undefined. +** If pApnd!=NULL then try to append missing nodes and set *pApnd = 1 if +** nodes are appended. */ -static int fts3MatchinfoValues( - Fts3Cursor *pCsr, /* FTS3 cursor object */ - int bGlobal, /* True to grab the global stats */ - MatchInfo *pInfo, /* Matchinfo context object */ - const char *zArg /* Matchinfo format string */ +static JsonNode *jsonLookup( + JsonParse *pParse, /* The JSON to search */ + const char *zPath, /* The path to search */ + int *pApnd, /* Append nodes to complete path if not NULL */ + sqlite3_context *pCtx /* Report errors here, if not NULL */ ){ - int rc = SQLITE_OK; - int i; - Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; - sqlite3_stmt *pSelect = 0; + const char *zErr = 0; + JsonNode *pNode = 0; + char *zMsg; - for(i=0; rc==SQLITE_OK && zArg[i]; i++){ - pInfo->flag = zArg[i]; - switch( zArg[i] ){ - case FTS3_MATCHINFO_NPHRASE: - if( bGlobal ) pInfo->aMatchinfo[0] = pInfo->nPhrase; - break; + if( zPath==0 ) return 0; + if( zPath[0]!='$' ){ + zErr = zPath; + goto lookup_err; + } + zPath++; + pNode = jsonLookupStep(pParse, 0, zPath, pApnd, &zErr); + if( zErr==0 ) return pNode; - case FTS3_MATCHINFO_NCOL: - if( bGlobal ) pInfo->aMatchinfo[0] = pInfo->nCol; - break; - - case FTS3_MATCHINFO_NDOC: - if( bGlobal ){ - sqlite3_int64 nDoc = 0; - rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &nDoc, 0); - pInfo->aMatchinfo[0] = (u32)nDoc; - } - break; +lookup_err: + pParse->nErr++; + assert( zErr!=0 && pCtx!=0 ); + zMsg = jsonPathSyntaxError(zErr); + if( zMsg ){ + sqlite3_result_error(pCtx, zMsg, -1); + sqlite3_free(zMsg); + }else{ + sqlite3_result_error_nomem(pCtx); + } + return 0; +} - case FTS3_MATCHINFO_AVGLENGTH: - if( bGlobal ){ - sqlite3_int64 nDoc; /* Number of rows in table */ - const char *a; /* Aggregate column length array */ - rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &nDoc, &a); - if( rc==SQLITE_OK ){ - int iCol; - for(iCol=0; iColnCol; iCol++){ - u32 iVal; - sqlite3_int64 nToken; - a += sqlite3Fts3GetVarint(a, &nToken); - iVal = (u32)(((u32)(nToken&0xffffffff)+nDoc/2)/nDoc); - pInfo->aMatchinfo[iCol] = iVal; - } - } - } - break; +/* +** Report the wrong number of arguments for json_insert(), json_replace() +** or json_set(). +*/ +static void jsonWrongNumArgs( + sqlite3_context *pCtx, + const char *zFuncName +){ + char *zMsg = sqlite3_mprintf("json_%s() needs an odd number of arguments", + zFuncName); + sqlite3_result_error(pCtx, zMsg, -1); + sqlite3_free(zMsg); +} - case FTS3_MATCHINFO_LENGTH: { - sqlite3_stmt *pSelectDocsize = 0; - rc = sqlite3Fts3SelectDocsize(pTab, pCsr->iPrevId, &pSelectDocsize); - if( rc==SQLITE_OK ){ - int iCol; - const char *a = sqlite3_column_blob(pSelectDocsize, 0); - for(iCol=0; iColnCol; iCol++){ - sqlite3_int64 nToken; - a += sqlite3Fts3GetVarint(a, &nToken); - pInfo->aMatchinfo[iCol] = (u32)nToken; - } - } - sqlite3_reset(pSelectDocsize); +/* +** Mark all NULL entries in the Object passed in as JNODE_REMOVE. +*/ +static void jsonRemoveAllNulls(JsonNode *pNode){ + int i, n; + assert( pNode->eType==JSON_OBJECT ); + n = pNode->n; + for(i=2; i<=n; i += jsonNodeSize(&pNode[i])+1){ + switch( pNode[i].eType ){ + case JSON_NULL: + pNode[i].jnFlags |= JNODE_REMOVE; break; - } - - case FTS3_MATCHINFO_LCS: - rc = fts3ExprLoadDoclists(pCsr, 0, 0); - if( rc==SQLITE_OK ){ - rc = fts3MatchinfoLcs(pCsr, pInfo); - } + case JSON_OBJECT: + jsonRemoveAllNulls(&pNode[i]); break; + } + } +} - case FTS3_MATCHINFO_LHITS_BM: - case FTS3_MATCHINFO_LHITS: { - int nZero = fts3MatchinfoSize(pInfo, zArg[i]) * sizeof(u32); - memset(pInfo->aMatchinfo, 0, nZero); - fts3ExprLHitGather(pCsr->pExpr, pInfo); - break; - } - default: { - Fts3Expr *pExpr; - assert( zArg[i]==FTS3_MATCHINFO_HITS ); - pExpr = pCsr->pExpr; - rc = fts3ExprLoadDoclists(pCsr, 0, 0); - if( rc!=SQLITE_OK ) break; - if( bGlobal ){ - if( pCsr->pDeferred ){ - rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &pInfo->nDoc, 0); - if( rc!=SQLITE_OK ) break; - } - rc = fts3ExprIterate(pExpr, fts3ExprGlobalHitsCb,(void*)pInfo); - sqlite3Fts3EvalTestDeferred(pCsr, &rc); - if( rc!=SQLITE_OK ) break; - } - (void)fts3ExprIterate(pExpr, fts3ExprLocalHitsCb,(void*)pInfo); - break; - } - } +/**************************************************************************** +** SQL functions used for testing and debugging +****************************************************************************/ + +#ifdef SQLITE_DEBUG +/* +** The json_parse(JSON) function returns a string which describes +** a parse of the JSON provided. Or it returns NULL if JSON is not +** well-formed. +*/ +static void jsonParseFunc( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + JsonString s; /* Output string - not real JSON */ + JsonParse x; /* The parse */ + u32 i; - pInfo->aMatchinfo += fts3MatchinfoSize(pInfo, zArg[i]); + assert( argc==1 ); + if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; + jsonParseFindParents(&x); + jsonInit(&s, ctx); + for(i=0; iaMatchinfo[] with data for the current row. The -** 'matchinfo' data is an array of 32-bit unsigned integers (C type u32). +** Implementation of the json_QUOTE(VALUE) function. Return a JSON value +** corresponding to the SQL value input. Mostly this means putting +** double-quotes around strings and returning the unquoted string "null" +** when given a NULL input. */ -static void fts3GetMatchinfo( - sqlite3_context *pCtx, /* Return results here */ - Fts3Cursor *pCsr, /* FTS3 Cursor object */ - const char *zArg /* Second argument to matchinfo() function */ +static void jsonQuoteFunc( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv ){ - MatchInfo sInfo; - Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; - int rc = SQLITE_OK; - int bGlobal = 0; /* Collect 'global' stats as well as local */ + JsonString jx; + UNUSED_PARAM(argc); - u32 *aOut = 0; - void (*xDestroyOut)(void*) = 0; + jsonInit(&jx, ctx); + jsonAppendValue(&jx, argv[0]); + jsonResult(&jx); + sqlite3_result_subtype(ctx, JSON_SUBTYPE); +} - memset(&sInfo, 0, sizeof(MatchInfo)); - sInfo.pCursor = pCsr; - sInfo.nCol = pTab->nColumn; +/* +** Implementation of the json_array(VALUE,...) function. Return a JSON +** array that contains all values given in arguments. Or if any argument +** is a BLOB, throw an error. +*/ +static void jsonArrayFunc( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + int i; + JsonString jx; - /* If there is cached matchinfo() data, but the format string for the - ** cache does not match the format string for this request, discard - ** the cached data. */ - if( pCsr->pMIBuffer && strcmp(pCsr->pMIBuffer->zMatchinfo, zArg) ){ - sqlite3Fts3MIBufferFree(pCsr->pMIBuffer); - pCsr->pMIBuffer = 0; + jsonInit(&jx, ctx); + jsonAppendChar(&jx, '['); + for(i=0; ipMIBuffer==0 ){ - int nMatchinfo = 0; /* Number of u32 elements in match-info */ - int i; /* Used to iterate through zArg */ - - /* Determine the number of phrases in the query */ - pCsr->nPhrase = fts3ExprPhraseCount(pCsr->pExpr); - sInfo.nPhrase = pCsr->nPhrase; - - /* Determine the number of integers in the buffer returned by this call. */ - for(i=0; zArg[i]; i++){ - char *zErr = 0; - if( fts3MatchinfoCheck(pTab, zArg[i], &zErr) ){ - sqlite3_result_error(pCtx, zErr, -1); - sqlite3_free(zErr); - return; - } - nMatchinfo += fts3MatchinfoSize(&sInfo, zArg[i]); - } - /* Allocate space for Fts3Cursor.aMatchinfo[] and Fts3Cursor.zMatchinfo. */ - pCsr->pMIBuffer = fts3MIBufferNew(nMatchinfo, zArg); - if( !pCsr->pMIBuffer ) rc = SQLITE_NOMEM; +/* +** json_array_length(JSON) +** json_array_length(JSON, PATH) +** +** Return the number of elements in the top-level JSON array. +** Return 0 if the input is not a well-formed JSON array. +*/ +static void jsonArrayLengthFunc( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + JsonParse *p; /* The parse */ + sqlite3_int64 n = 0; + u32 i; + JsonNode *pNode; - pCsr->isMatchinfoNeeded = 1; - bGlobal = 1; + p = jsonParseCached(ctx, argv, ctx); + if( p==0 ) return; + assert( p->nNode ); + if( argc==2 ){ + const char *zPath = (const char*)sqlite3_value_text(argv[1]); + pNode = jsonLookup(p, zPath, 0, ctx); + }else{ + pNode = p->aNode; } - - if( rc==SQLITE_OK ){ - xDestroyOut = fts3MIBufferAlloc(pCsr->pMIBuffer, &aOut); - if( xDestroyOut==0 ){ - rc = SQLITE_NOMEM; - } + if( pNode==0 ){ + return; } - - if( rc==SQLITE_OK ){ - sInfo.aMatchinfo = aOut; - sInfo.nPhrase = pCsr->nPhrase; - rc = fts3MatchinfoValues(pCsr, bGlobal, &sInfo, zArg); - if( bGlobal ){ - fts3MIBufferSetGlobal(pCsr->pMIBuffer); + if( pNode->eType==JSON_ARRAY ){ + assert( (pNode->jnFlags & JNODE_APPEND)==0 ); + for(i=1; i<=pNode->n; n++){ + i += jsonNodeSize(&pNode[i]); } } - - if( rc!=SQLITE_OK ){ - sqlite3_result_error_code(pCtx, rc); - if( xDestroyOut ) xDestroyOut(aOut); - }else{ - int n = pCsr->pMIBuffer->nElem * sizeof(u32); - sqlite3_result_blob(pCtx, aOut, n, xDestroyOut); - } + sqlite3_result_int64(ctx, n); } /* -** Implementation of snippet() function. +** json_extract(JSON, PATH, ...) +** +** Return the element described by PATH. Return NULL if there is no +** PATH element. If there are multiple PATHs, then return a JSON array +** with the result from each path. Throw an error if the JSON or any PATH +** is malformed. */ -SQLITE_PRIVATE void sqlite3Fts3Snippet( - sqlite3_context *pCtx, /* SQLite function call context */ - Fts3Cursor *pCsr, /* Cursor object */ - const char *zStart, /* Snippet start text - "" */ - const char *zEnd, /* Snippet end text - "" */ - const char *zEllipsis, /* Snippet ellipsis text - "..." */ - int iCol, /* Extract snippet from this column */ - int nToken /* Approximate number of tokens in snippet */ +static void jsonExtractFunc( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv ){ - Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; - int rc = SQLITE_OK; + JsonParse *p; /* The parse */ + JsonNode *pNode; + const char *zPath; + JsonString jx; int i; - StrBuffer res = {0, 0, 0}; - - /* The returned text includes up to four fragments of text extracted from - ** the data in the current row. The first iteration of the for(...) loop - ** below attempts to locate a single fragment of text nToken tokens in - ** size that contains at least one instance of all phrases in the query - ** expression that appear in the current row. If such a fragment of text - ** cannot be found, the second iteration of the loop attempts to locate - ** a pair of fragments, and so on. - */ - int nSnippet = 0; /* Number of fragments in this snippet */ - SnippetFragment aSnippet[4]; /* Maximum of 4 fragments per snippet */ - int nFToken = -1; /* Number of tokens in each fragment */ - - if( !pCsr->pExpr ){ - sqlite3_result_text(pCtx, "", 0, SQLITE_STATIC); - return; - } - - for(nSnippet=1; 1; nSnippet++){ - int iSnip; /* Loop counter 0..nSnippet-1 */ - u64 mCovered = 0; /* Bitmask of phrases covered by snippet */ - u64 mSeen = 0; /* Bitmask of phrases seen by BestSnippet() */ - - if( nToken>=0 ){ - nFToken = (nToken+nSnippet-1) / nSnippet; - }else{ - nFToken = -1 * nToken; + if( argc<2 ) return; + p = jsonParseCached(ctx, argv, ctx); + if( p==0 ) return; + jsonInit(&jx, ctx); + jsonAppendChar(&jx, '['); + for(i=1; inErr ) break; + if( argc>2 ){ + jsonAppendSeparator(&jx); + if( pNode ){ + jsonRenderNode(pNode, &jx, 0); + }else{ + jsonAppendRaw(&jx, "null", 4); + } + }else if( pNode ){ + jsonReturn(pNode, ctx, 0); } + } + if( argc>2 && i==argc ){ + jsonAppendChar(&jx, ']'); + jsonResult(&jx); + sqlite3_result_subtype(ctx, JSON_SUBTYPE); + } + jsonReset(&jx); +} - for(iSnip=0; iSnipnColumn; iRead++){ - SnippetFragment sF = {0, 0, 0, 0}; - int iS = 0; - if( iCol>=0 && iRead!=iCol ) continue; - - /* Find the best snippet of nFToken tokens in column iRead. */ - rc = fts3BestSnippet(nFToken, pCsr, iRead, mCovered, &mSeen, &sF, &iS); - if( rc!=SQLITE_OK ){ - goto snippet_out; - } - if( iS>iBestScore ){ - *pFragment = sF; - iBestScore = iS; +/* This is the RFC 7396 MergePatch algorithm. +*/ +static JsonNode *jsonMergePatch( + JsonParse *pParse, /* The JSON parser that contains the TARGET */ + u32 iTarget, /* Node of the TARGET in pParse */ + JsonNode *pPatch /* The PATCH */ +){ + u32 i, j; + u32 iRoot; + JsonNode *pTarget; + if( pPatch->eType!=JSON_OBJECT ){ + return pPatch; + } + assert( iTarget>=0 && iTargetnNode ); + pTarget = &pParse->aNode[iTarget]; + assert( (pPatch->jnFlags & JNODE_APPEND)==0 ); + if( pTarget->eType!=JSON_OBJECT ){ + jsonRemoveAllNulls(pPatch); + return pPatch; + } + iRoot = iTarget; + for(i=1; in; i += jsonNodeSize(&pPatch[i+1])+1){ + u32 nKey; + const char *zKey; + assert( pPatch[i].eType==JSON_STRING ); + assert( pPatch[i].jnFlags & JNODE_LABEL ); + nKey = pPatch[i].n; + zKey = pPatch[i].u.zJContent; + assert( (pPatch[i].jnFlags & JNODE_RAW)==0 ); + for(j=1; jn; j += jsonNodeSize(&pTarget[j+1])+1 ){ + assert( pTarget[j].eType==JSON_STRING ); + assert( pTarget[j].jnFlags & JNODE_LABEL ); + assert( (pPatch[i].jnFlags & JNODE_RAW)==0 ); + if( pTarget[j].n==nKey && strncmp(pTarget[j].u.zJContent,zKey,nKey)==0 ){ + if( pTarget[j+1].jnFlags & (JNODE_REMOVE|JNODE_PATCH) ) break; + if( pPatch[i+1].eType==JSON_NULL ){ + pTarget[j+1].jnFlags |= JNODE_REMOVE; + }else{ + JsonNode *pNew = jsonMergePatch(pParse, iTarget+j+1, &pPatch[i+1]); + if( pNew==0 ) return 0; + pTarget = &pParse->aNode[iTarget]; + if( pNew!=&pTarget[j+1] ){ + pTarget[j+1].u.pPatch = pNew; + pTarget[j+1].jnFlags |= JNODE_PATCH; + } } + break; } - - mCovered |= pFragment->covered; } - - /* If all query phrases seen by fts3BestSnippet() are present in at least - ** one of the nSnippet snippet fragments, break out of the loop. - */ - assert( (mCovered&mSeen)==mCovered ); - if( mSeen==mCovered || nSnippet==SizeofArray(aSnippet) ) break; + if( j>=pTarget->n && pPatch[i+1].eType!=JSON_NULL ){ + int iStart, iPatch; + iStart = jsonParseAddNode(pParse, JSON_OBJECT, 2, 0); + jsonParseAddNode(pParse, JSON_STRING, nKey, zKey); + iPatch = jsonParseAddNode(pParse, JSON_TRUE, 0, 0); + if( pParse->oom ) return 0; + jsonRemoveAllNulls(pPatch); + pTarget = &pParse->aNode[iTarget]; + pParse->aNode[iRoot].jnFlags |= JNODE_APPEND; + pParse->aNode[iRoot].u.iAppend = iStart - iRoot; + iRoot = iStart; + pParse->aNode[iPatch].jnFlags |= JNODE_PATCH; + pParse->aNode[iPatch].u.pPatch = &pPatch[i+1]; + } } + return pTarget; +} - assert( nFToken>0 ); +/* +** Implementation of the json_mergepatch(JSON1,JSON2) function. Return a JSON +** object that is the result of running the RFC 7396 MergePatch() algorithm +** on the two arguments. +*/ +static void jsonPatchFunc( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + JsonParse x; /* The JSON that is being patched */ + JsonParse y; /* The patch */ + JsonNode *pResult; /* The result of the merge */ - for(i=0; ipCsr, pExpr, p->iCol, &pList); - nTerm = pExpr->pPhrase->nToken; - if( pList ){ - fts3GetDeltaPosition(&pList, &iPos); - assert( iPos>=0 ); + if( argc<1 ) return; + if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; + assert( x.nNode ); + for(i=1; i<(u32)argc; i++){ + zPath = (const char*)sqlite3_value_text(argv[i]); + if( zPath==0 ) goto remove_done; + pNode = jsonLookup(&x, zPath, 0, ctx); + if( x.nErr ) goto remove_done; + if( pNode ) pNode->jnFlags |= JNODE_REMOVE; } - - for(iTerm=0; iTermaTerm[p->iTerm++]; - pT->iOff = nTerm-iTerm-1; - pT->pList = pList; - pT->iPos = iPos; + if( (x.aNode[0].jnFlags & JNODE_REMOVE)==0 ){ + jsonReturnJson(x.aNode, ctx, 0); } - - return rc; +remove_done: + jsonParseReset(&x); } /* -** Implementation of offsets() function. +** json_replace(JSON, PATH, VALUE, ...) +** +** Replace the value at PATH with VALUE. If PATH does not already exist, +** this routine is a no-op. If JSON or PATH is malformed, throw an error. */ -SQLITE_PRIVATE void sqlite3Fts3Offsets( - sqlite3_context *pCtx, /* SQLite function call context */ - Fts3Cursor *pCsr /* Cursor object */ +static void jsonReplaceFunc( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv ){ - Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; - sqlite3_tokenizer_module const *pMod = pTab->pTokenizer->pModule; - int rc; /* Return Code */ - int nToken; /* Number of tokens in query */ - int iCol; /* Column currently being processed */ - StrBuffer res = {0, 0, 0}; /* Result string */ - TermOffsetCtx sCtx; /* Context for fts3ExprTermOffsetInit() */ + JsonParse x; /* The parse */ + JsonNode *pNode; + const char *zPath; + u32 i; - if( !pCsr->pExpr ){ - sqlite3_result_text(pCtx, "", 0, SQLITE_STATIC); + if( argc<1 ) return; + if( (argc&1)==0 ) { + jsonWrongNumArgs(ctx, "replace"); return; } - - memset(&sCtx, 0, sizeof(sCtx)); - assert( pCsr->isRequireSeek==0 ); - - /* Count the number of terms in the query */ - rc = fts3ExprLoadDoclists(pCsr, 0, &nToken); - if( rc!=SQLITE_OK ) goto offsets_out; - - /* Allocate the array of TermOffset iterators. */ - sCtx.aTerm = (TermOffset *)sqlite3_malloc(sizeof(TermOffset)*nToken); - if( 0==sCtx.aTerm ){ - rc = SQLITE_NOMEM; - goto offsets_out; - } - sCtx.iDocid = pCsr->iPrevId; - sCtx.pCsr = pCsr; - - /* Loop through the table columns, appending offset information to - ** string-buffer res for each column. - */ - for(iCol=0; iColnColumn; iCol++){ - sqlite3_tokenizer_cursor *pC; /* Tokenizer cursor */ - const char *ZDUMMY; /* Dummy argument used with xNext() */ - int NDUMMY = 0; /* Dummy argument used with xNext() */ - int iStart = 0; - int iEnd = 0; - int iCurrent = 0; - const char *zDoc; - int nDoc; - - /* Initialize the contents of sCtx.aTerm[] for column iCol. There is - ** no way that this operation can fail, so the return code from - ** fts3ExprIterate() can be discarded. - */ - sCtx.iCol = iCol; - sCtx.iTerm = 0; - (void)fts3ExprIterate(pCsr->pExpr, fts3ExprTermOffsetInit, (void*)&sCtx); - - /* Retreive the text stored in column iCol. If an SQL NULL is stored - ** in column iCol, jump immediately to the next iteration of the loop. - ** If an OOM occurs while retrieving the data (this can happen if SQLite - ** needs to transform the data from utf-16 to utf-8), return SQLITE_NOMEM - ** to the caller. - */ - zDoc = (const char *)sqlite3_column_text(pCsr->pStmt, iCol+1); - nDoc = sqlite3_column_bytes(pCsr->pStmt, iCol+1); - if( zDoc==0 ){ - if( sqlite3_column_type(pCsr->pStmt, iCol+1)==SQLITE_NULL ){ - continue; - } - rc = SQLITE_NOMEM; - goto offsets_out; + if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; + assert( x.nNode ); + for(i=1; i<(u32)argc; i+=2){ + zPath = (const char*)sqlite3_value_text(argv[i]); + pNode = jsonLookup(&x, zPath, 0, ctx); + if( x.nErr ) goto replace_err; + if( pNode ){ + pNode->jnFlags |= (u8)JNODE_REPLACE; + pNode->u.iReplace = i + 1; } + } + if( x.aNode[0].jnFlags & JNODE_REPLACE ){ + sqlite3_result_value(ctx, argv[x.aNode[0].u.iReplace]); + }else{ + jsonReturnJson(x.aNode, ctx, argv); + } +replace_err: + jsonParseReset(&x); +} - /* Initialize a tokenizer iterator to iterate through column iCol. */ - rc = sqlite3Fts3OpenTokenizer(pTab->pTokenizer, pCsr->iLangid, - zDoc, nDoc, &pC - ); - if( rc!=SQLITE_OK ) goto offsets_out; - - rc = pMod->xNext(pC, &ZDUMMY, &NDUMMY, &iStart, &iEnd, &iCurrent); - while( rc==SQLITE_OK ){ - int i; /* Used to loop through terms */ - int iMinPos = 0x7FFFFFFF; /* Position of next token */ - TermOffset *pTerm = 0; /* TermOffset associated with next token */ - - for(i=0; ipList && (pT->iPos-pT->iOff)iPos-pT->iOff; - pTerm = pT; - } - } +/* +** json_set(JSON, PATH, VALUE, ...) +** +** Set the value at PATH to VALUE. Create the PATH if it does not already +** exist. Overwrite existing values that do exist. +** If JSON or PATH is malformed, throw an error. +** +** json_insert(JSON, PATH, VALUE, ...) +** +** Create PATH and initialize it to VALUE. If PATH already exists, this +** routine is a no-op. If JSON or PATH is malformed, throw an error. +*/ +static void jsonSetFunc( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + JsonParse x; /* The parse */ + JsonNode *pNode; + const char *zPath; + u32 i; + int bApnd; + int bIsSet = *(int*)sqlite3_user_data(ctx); - if( !pTerm ){ - /* All offsets for this column have been gathered. */ - rc = SQLITE_DONE; - }else{ - assert( iCurrent<=iMinPos ); - if( 0==(0xFE&*pTerm->pList) ){ - pTerm->pList = 0; - }else{ - fts3GetDeltaPosition(&pTerm->pList, &pTerm->iPos); - } - while( rc==SQLITE_OK && iCurrentxNext(pC, &ZDUMMY, &NDUMMY, &iStart, &iEnd, &iCurrent); - } - if( rc==SQLITE_OK ){ - char aBuffer[64]; - sqlite3_snprintf(sizeof(aBuffer), aBuffer, - "%d %d %d %d ", iCol, pTerm-sCtx.aTerm, iStart, iEnd-iStart - ); - rc = fts3StringAppend(&res, aBuffer, -1); - }else if( rc==SQLITE_DONE && pTab->zContentTbl==0 ){ - rc = FTS_CORRUPT_VTAB; - } - } - } - if( rc==SQLITE_DONE ){ - rc = SQLITE_OK; + if( argc<1 ) return; + if( (argc&1)==0 ) { + jsonWrongNumArgs(ctx, bIsSet ? "set" : "insert"); + return; + } + if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; + assert( x.nNode ); + for(i=1; i<(u32)argc; i+=2){ + zPath = (const char*)sqlite3_value_text(argv[i]); + bApnd = 0; + pNode = jsonLookup(&x, zPath, &bApnd, ctx); + if( x.oom ){ + sqlite3_result_error_nomem(ctx); + goto jsonSetDone; + }else if( x.nErr ){ + goto jsonSetDone; + }else if( pNode && (bApnd || bIsSet) ){ + pNode->jnFlags |= (u8)JNODE_REPLACE; + pNode->u.iReplace = i + 1; } - - pMod->xClose(pC); - if( rc!=SQLITE_OK ) goto offsets_out; } - - offsets_out: - sqlite3_free(sCtx.aTerm); - assert( rc!=SQLITE_DONE ); - sqlite3Fts3SegmentsClose(pTab); - if( rc!=SQLITE_OK ){ - sqlite3_result_error_code(pCtx, rc); - sqlite3_free(res.z); + if( x.aNode[0].jnFlags & JNODE_REPLACE ){ + sqlite3_result_value(ctx, argv[x.aNode[0].u.iReplace]); }else{ - sqlite3_result_text(pCtx, res.z, res.n-1, sqlite3_free); + jsonReturnJson(x.aNode, ctx, argv); } - return; +jsonSetDone: + jsonParseReset(&x); } /* -** Implementation of matchinfo() function. +** json_type(JSON) +** json_type(JSON, PATH) +** +** Return the top-level "type" of a JSON string. Throw an error if +** either the JSON or PATH inputs are not well-formed. */ -SQLITE_PRIVATE void sqlite3Fts3Matchinfo( - sqlite3_context *pContext, /* Function call context */ - Fts3Cursor *pCsr, /* FTS3 table cursor */ - const char *zArg /* Second arg to matchinfo() function */ +static void jsonTypeFunc( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv ){ - Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; - const char *zFormat; + JsonParse *p; /* The parse */ + const char *zPath; + JsonNode *pNode; - if( zArg ){ - zFormat = zArg; + p = jsonParseCached(ctx, argv, ctx); + if( p==0 ) return; + if( argc==2 ){ + zPath = (const char*)sqlite3_value_text(argv[1]); + pNode = jsonLookup(p, zPath, 0, ctx); }else{ - zFormat = FTS3_MATCHINFO_DEFAULT; + pNode = p->aNode; } - - if( !pCsr->pExpr ){ - sqlite3_result_blob(pContext, "", 0, SQLITE_STATIC); - return; - }else{ - /* Retrieve matchinfo() data. */ - fts3GetMatchinfo(pContext, pCsr, zFormat); - sqlite3Fts3SegmentsClose(pTab); + if( pNode ){ + sqlite3_result_text(ctx, jsonType[pNode->eType], -1, SQLITE_STATIC); } } -#endif - -/************** End of fts3_snippet.c ****************************************/ -/************** Begin file fts3_unicode.c ************************************/ /* -** 2012 May 24 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -****************************************************************************** +** json_valid(JSON) ** -** Implementation of the "unicode" full-text-search tokenizer. +** Return 1 if JSON is a well-formed JSON string according to RFC-7159. +** Return 0 otherwise. */ +static void jsonValidFunc( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + JsonParse *p; /* The parse */ + UNUSED_PARAM(argc); + p = jsonParseCached(ctx, argv, 0); + sqlite3_result_int(ctx, p!=0); +} -#ifndef SQLITE_DISABLE_FTS3_UNICODE - -/* #include "fts3Int.h" */ -#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) - -/* #include */ -/* #include */ -/* #include */ -/* #include */ - -/* #include "fts3_tokenizer.h" */ +/**************************************************************************** +** Aggregate SQL function implementations +****************************************************************************/ /* -** The following two macros - READ_UTF8 and WRITE_UTF8 - have been copied -** from the sqlite3 source file utf.c. If this file is compiled as part -** of the amalgamation, they are not required. +** json_group_array(VALUE) +** +** Return a JSON array composed of all values in the aggregate. */ -#ifndef SQLITE_AMALGAMATION - -static const unsigned char sqlite3Utf8Trans1[] = { - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, - 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, - 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, - 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, - 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, - 0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x00, 0x00, -}; - -#define READ_UTF8(zIn, zTerm, c) \ - c = *(zIn++); \ - if( c>=0xc0 ){ \ - c = sqlite3Utf8Trans1[c-0xc0]; \ - while( zIn!=zTerm && (*zIn & 0xc0)==0x80 ){ \ - c = (c<<6) + (0x3f & *(zIn++)); \ - } \ - if( c<0x80 \ - || (c&0xFFFFF800)==0xD800 \ - || (c&0xFFFFFFFE)==0xFFFE ){ c = 0xFFFD; } \ +static void jsonArrayStep( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + JsonString *pStr; + UNUSED_PARAM(argc); + pStr = (JsonString*)sqlite3_aggregate_context(ctx, sizeof(*pStr)); + if( pStr ){ + if( pStr->zBuf==0 ){ + jsonInit(pStr, ctx); + jsonAppendChar(pStr, '['); + }else if( pStr->nUsed>1 ){ + jsonAppendChar(pStr, ','); + pStr->pCtx = ctx; + } + jsonAppendValue(pStr, argv[0]); } - -#define WRITE_UTF8(zOut, c) { \ - if( c<0x00080 ){ \ - *zOut++ = (u8)(c&0xFF); \ - } \ - else if( c<0x00800 ){ \ - *zOut++ = 0xC0 + (u8)((c>>6)&0x1F); \ - *zOut++ = 0x80 + (u8)(c & 0x3F); \ - } \ - else if( c<0x10000 ){ \ - *zOut++ = 0xE0 + (u8)((c>>12)&0x0F); \ - *zOut++ = 0x80 + (u8)((c>>6) & 0x3F); \ - *zOut++ = 0x80 + (u8)(c & 0x3F); \ - }else{ \ - *zOut++ = 0xF0 + (u8)((c>>18) & 0x07); \ - *zOut++ = 0x80 + (u8)((c>>12) & 0x3F); \ - *zOut++ = 0x80 + (u8)((c>>6) & 0x3F); \ - *zOut++ = 0x80 + (u8)(c & 0x3F); \ - } \ +} +static void jsonArrayCompute(sqlite3_context *ctx, int isFinal){ + JsonString *pStr; + pStr = (JsonString*)sqlite3_aggregate_context(ctx, 0); + if( pStr ){ + pStr->pCtx = ctx; + jsonAppendChar(pStr, ']'); + if( pStr->bErr ){ + if( pStr->bErr==1 ) sqlite3_result_error_nomem(ctx); + assert( pStr->bStatic ); + }else if( isFinal ){ + sqlite3_result_text(ctx, pStr->zBuf, (int)pStr->nUsed, + pStr->bStatic ? SQLITE_TRANSIENT : sqlite3_free); + pStr->bStatic = 1; + }else{ + sqlite3_result_text(ctx, pStr->zBuf, (int)pStr->nUsed, SQLITE_TRANSIENT); + pStr->nUsed--; + } + }else{ + sqlite3_result_text(ctx, "[]", 2, SQLITE_STATIC); + } + sqlite3_result_subtype(ctx, JSON_SUBTYPE); +} +static void jsonArrayValue(sqlite3_context *ctx){ + jsonArrayCompute(ctx, 0); +} +static void jsonArrayFinal(sqlite3_context *ctx){ + jsonArrayCompute(ctx, 1); } -#endif /* ifndef SQLITE_AMALGAMATION */ - -typedef struct unicode_tokenizer unicode_tokenizer; -typedef struct unicode_cursor unicode_cursor; - -struct unicode_tokenizer { - sqlite3_tokenizer base; - int bRemoveDiacritic; - int nException; - int *aiException; -}; - -struct unicode_cursor { - sqlite3_tokenizer_cursor base; - const unsigned char *aInput; /* Input text being tokenized */ - int nInput; /* Size of aInput[] in bytes */ - int iOff; /* Current offset within aInput[] */ - int iToken; /* Index of next token to be returned */ - char *zToken; /* storage for current token */ - int nAlloc; /* space allocated at zToken */ -}; - - +#ifndef SQLITE_OMIT_WINDOWFUNC /* -** Destroy a tokenizer allocated by unicodeCreate(). +** This method works for both json_group_array() and json_group_object(). +** It works by removing the first element of the group by searching forward +** to the first comma (",") that is not within a string and deleting all +** text through that comma. */ -static int unicodeDestroy(sqlite3_tokenizer *pTokenizer){ - if( pTokenizer ){ - unicode_tokenizer *p = (unicode_tokenizer *)pTokenizer; - sqlite3_free(p->aiException); - sqlite3_free(p); +static void jsonGroupInverse( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + unsigned int i; + int inStr = 0; + int nNest = 0; + char *z; + char c; + JsonString *pStr; + UNUSED_PARAM(argc); + UNUSED_PARAM(argv); + pStr = (JsonString*)sqlite3_aggregate_context(ctx, 0); +#ifdef NEVER + /* pStr is always non-NULL since jsonArrayStep() or jsonObjectStep() will + ** always have been called to initalize it */ + if( NEVER(!pStr) ) return; +#endif + z = pStr->zBuf; + for(i=1; (c = z[i])!=',' || inStr || nNest; i++){ + if( i>=pStr->nUsed ){ + pStr->nUsed = 1; + return; + } + if( c=='"' ){ + inStr = !inStr; + }else if( c=='\\' ){ + i++; + }else if( !inStr ){ + if( c=='{' || c=='[' ) nNest++; + if( c=='}' || c==']' ) nNest--; + } } - return SQLITE_OK; + pStr->nUsed -= i; + memmove(&z[1], &z[i+1], (size_t)pStr->nUsed-1); } +#else +# define jsonGroupInverse 0 +#endif + /* -** As part of a tokenchars= or separators= option, the CREATE VIRTUAL TABLE -** statement has specified that the tokenizer for this table shall consider -** all characters in string zIn/nIn to be separators (if bAlnum==0) or -** token characters (if bAlnum==1). -** -** For each codepoint in the zIn/nIn string, this function checks if the -** sqlite3FtsUnicodeIsalnum() function already returns the desired result. -** If so, no action is taken. Otherwise, the codepoint is added to the -** unicode_tokenizer.aiException[] array. For the purposes of tokenization, -** the return value of sqlite3FtsUnicodeIsalnum() is inverted for all -** codepoints in the aiException[] array. +** json_group_obj(NAME,VALUE) ** -** If a standalone diacritic mark (one that sqlite3FtsUnicodeIsdiacritic() -** identifies as a diacritic) occurs in the zIn/nIn string it is ignored. -** It is not possible to change the behavior of the tokenizer with respect -** to these codepoints. +** Return a JSON object composed of all names and values in the aggregate. */ -static int unicodeAddExceptions( - unicode_tokenizer *p, /* Tokenizer to add exceptions to */ - int bAlnum, /* Replace Isalnum() return value with this */ - const char *zIn, /* Array of characters to make exceptions */ - int nIn /* Length of z in bytes */ +static void jsonObjectStep( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv ){ - const unsigned char *z = (const unsigned char *)zIn; - const unsigned char *zTerm = &z[nIn]; - unsigned int iCode; - int nEntry = 0; - - assert( bAlnum==0 || bAlnum==1 ); - - while( zzBuf==0 ){ + jsonInit(pStr, ctx); + jsonAppendChar(pStr, '{'); + }else if( pStr->nUsed>1 ){ + jsonAppendChar(pStr, ','); + pStr->pCtx = ctx; + } + z = (const char*)sqlite3_value_text(argv[0]); + n = (u32)sqlite3_value_bytes(argv[0]); + jsonAppendString(pStr, z, n); + jsonAppendChar(pStr, ':'); + jsonAppendValue(pStr, argv[1]); + } +} +static void jsonObjectCompute(sqlite3_context *ctx, int isFinal){ + JsonString *pStr; + pStr = (JsonString*)sqlite3_aggregate_context(ctx, 0); + if( pStr ){ + jsonAppendChar(pStr, '}'); + if( pStr->bErr ){ + if( pStr->bErr==1 ) sqlite3_result_error_nomem(ctx); + assert( pStr->bStatic ); + }else if( isFinal ){ + sqlite3_result_text(ctx, pStr->zBuf, (int)pStr->nUsed, + pStr->bStatic ? SQLITE_TRANSIENT : sqlite3_free); + pStr->bStatic = 1; + }else{ + sqlite3_result_text(ctx, pStr->zBuf, (int)pStr->nUsed, SQLITE_TRANSIENT); + pStr->nUsed--; } + }else{ + sqlite3_result_text(ctx, "{}", 2, SQLITE_STATIC); } + sqlite3_result_subtype(ctx, JSON_SUBTYPE); +} +static void jsonObjectValue(sqlite3_context *ctx){ + jsonObjectCompute(ctx, 0); +} +static void jsonObjectFinal(sqlite3_context *ctx){ + jsonObjectCompute(ctx, 1); +} - if( nEntry ){ - int *aNew; /* New aiException[] array */ - int nNew; /* Number of valid entries in array aNew[] */ - aNew = sqlite3_realloc(p->aiException, (p->nException+nEntry)*sizeof(int)); - if( aNew==0 ) return SQLITE_NOMEM; - nNew = p->nException; - z = (const unsigned char *)zIn; - while( zi; j--) aNew[j] = aNew[j-1]; - aNew[i] = (int)iCode; - nNew++; - } - } - p->aiException = aNew; - p->nException = nNew; +#ifndef SQLITE_OMIT_VIRTUALTABLE +/**************************************************************************** +** The json_each virtual table +****************************************************************************/ +typedef struct JsonEachCursor JsonEachCursor; +struct JsonEachCursor { + sqlite3_vtab_cursor base; /* Base class - must be first */ + u32 iRowid; /* The rowid */ + u32 iBegin; /* The first node of the scan */ + u32 i; /* Index in sParse.aNode[] of current row */ + u32 iEnd; /* EOF when i equals or exceeds this value */ + u8 eType; /* Type of top-level element */ + u8 bRecursive; /* True for json_tree(). False for json_each() */ + char *zJson; /* Input JSON */ + char *zRoot; /* Path by which to filter zJson */ + JsonParse sParse; /* Parse of the input JSON */ +}; + +/* Constructor for the json_each virtual table */ +static int jsonEachConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + sqlite3_vtab *pNew; + int rc; + +/* Column numbers */ +#define JEACH_KEY 0 +#define JEACH_VALUE 1 +#define JEACH_TYPE 2 +#define JEACH_ATOM 3 +#define JEACH_ID 4 +#define JEACH_PARENT 5 +#define JEACH_FULLKEY 6 +#define JEACH_PATH 7 +/* The xBestIndex method assumes that the JSON and ROOT columns are +** the last two columns in the table. Should this ever changes, be +** sure to update the xBestIndex method. */ +#define JEACH_JSON 8 +#define JEACH_ROOT 9 + + UNUSED_PARAM(pzErr); + UNUSED_PARAM(argv); + UNUSED_PARAM(argc); + UNUSED_PARAM(pAux); + rc = sqlite3_declare_vtab(db, + "CREATE TABLE x(key,value,type,atom,id,parent,fullkey,path," + "json HIDDEN,root HIDDEN)"); + if( rc==SQLITE_OK ){ + pNew = *ppVtab = sqlite3_malloc( sizeof(*pNew) ); + if( pNew==0 ) return SQLITE_NOMEM; + memset(pNew, 0, sizeof(*pNew)); + sqlite3_vtab_config(db, SQLITE_VTAB_INNOCUOUS); } + return rc; +} +/* destructor for json_each virtual table */ +static int jsonEachDisconnect(sqlite3_vtab *pVtab){ + sqlite3_free(pVtab); return SQLITE_OK; } -/* -** Return true if the p->aiException[] array contains the value iCode. -*/ -static int unicodeIsException(unicode_tokenizer *p, int iCode){ - if( p->nException>0 ){ - int *a = p->aiException; - int iLo = 0; - int iHi = p->nException-1; - - while( iHi>=iLo ){ - int iTest = (iHi + iLo) / 2; - if( iCode==a[iTest] ){ - return 1; - }else if( iCode>a[iTest] ){ - iLo = iTest+1; - }else{ - iHi = iTest-1; - } - } - } +/* constructor for a JsonEachCursor object for json_each(). */ +static int jsonEachOpenEach(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ + JsonEachCursor *pCur; - return 0; + UNUSED_PARAM(p); + pCur = sqlite3_malloc( sizeof(*pCur) ); + if( pCur==0 ) return SQLITE_NOMEM; + memset(pCur, 0, sizeof(*pCur)); + *ppCursor = &pCur->base; + return SQLITE_OK; } -/* -** Return true if, for the purposes of tokenization, codepoint iCode is -** considered a token character (not a separator). -*/ -static int unicodeIsAlnum(unicode_tokenizer *p, int iCode){ - assert( (sqlite3FtsUnicodeIsalnum(iCode) & 0xFFFFFFFE)==0 ); - return sqlite3FtsUnicodeIsalnum(iCode) ^ unicodeIsException(p, iCode); +/* constructor for a JsonEachCursor object for json_tree(). */ +static int jsonEachOpenTree(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ + int rc = jsonEachOpenEach(p, ppCursor); + if( rc==SQLITE_OK ){ + JsonEachCursor *pCur = (JsonEachCursor*)*ppCursor; + pCur->bRecursive = 1; + } + return rc; } -/* -** Create a new tokenizer instance. -*/ -static int unicodeCreate( - int nArg, /* Size of array argv[] */ - const char * const *azArg, /* Tokenizer creation arguments */ - sqlite3_tokenizer **pp /* OUT: New tokenizer handle */ -){ - unicode_tokenizer *pNew; /* New tokenizer object */ - int i; - int rc = SQLITE_OK; +/* Reset a JsonEachCursor back to its original state. Free any memory +** held. */ +static void jsonEachCursorReset(JsonEachCursor *p){ + sqlite3_free(p->zJson); + sqlite3_free(p->zRoot); + jsonParseReset(&p->sParse); + p->iRowid = 0; + p->i = 0; + p->iEnd = 0; + p->eType = 0; + p->zJson = 0; + p->zRoot = 0; +} - pNew = (unicode_tokenizer *) sqlite3_malloc(sizeof(unicode_tokenizer)); - if( pNew==NULL ) return SQLITE_NOMEM; - memset(pNew, 0, sizeof(unicode_tokenizer)); - pNew->bRemoveDiacritic = 1; +/* Destructor for a jsonEachCursor object */ +static int jsonEachClose(sqlite3_vtab_cursor *cur){ + JsonEachCursor *p = (JsonEachCursor*)cur; + jsonEachCursorReset(p); + sqlite3_free(cur); + return SQLITE_OK; +} - for(i=0; rc==SQLITE_OK && ii >= p->iEnd; +} - if( n==19 && memcmp("remove_diacritics=1", z, 19)==0 ){ - pNew->bRemoveDiacritic = 1; - } - else if( n==19 && memcmp("remove_diacritics=0", z, 19)==0 ){ - pNew->bRemoveDiacritic = 0; - } - else if( n>=11 && memcmp("tokenchars=", z, 11)==0 ){ - rc = unicodeAddExceptions(pNew, 1, &z[11], n-11); - } - else if( n>=11 && memcmp("separators=", z, 11)==0 ){ - rc = unicodeAddExceptions(pNew, 0, &z[11], n-11); +/* Advance the cursor to the next element for json_tree() */ +static int jsonEachNext(sqlite3_vtab_cursor *cur){ + JsonEachCursor *p = (JsonEachCursor*)cur; + if( p->bRecursive ){ + if( p->sParse.aNode[p->i].jnFlags & JNODE_LABEL ) p->i++; + p->i++; + p->iRowid++; + if( p->iiEnd ){ + u32 iUp = p->sParse.aUp[p->i]; + JsonNode *pUp = &p->sParse.aNode[iUp]; + p->eType = pUp->eType; + if( pUp->eType==JSON_ARRAY ){ + if( iUp==p->i-1 ){ + pUp->u.iKey = 0; + }else{ + pUp->u.iKey++; + } + } } - else{ - /* Unrecognized argument */ - rc = SQLITE_ERROR; + }else{ + switch( p->eType ){ + case JSON_ARRAY: { + p->i += jsonNodeSize(&p->sParse.aNode[p->i]); + p->iRowid++; + break; + } + case JSON_OBJECT: { + p->i += 1 + jsonNodeSize(&p->sParse.aNode[p->i+1]); + p->iRowid++; + break; + } + default: { + p->i = p->iEnd; + break; + } } } - - if( rc!=SQLITE_OK ){ - unicodeDestroy((sqlite3_tokenizer *)pNew); - pNew = 0; - } - *pp = (sqlite3_tokenizer *)pNew; - return rc; + return SQLITE_OK; } -/* -** Prepare to begin tokenizing a particular string. The input -** string to be tokenized is pInput[0..nBytes-1]. A cursor -** used to incrementally tokenize this string is returned in -** *ppCursor. +/* Append the name of the path for element i to pStr */ -static int unicodeOpen( - sqlite3_tokenizer *p, /* The tokenizer */ - const char *aInput, /* Input string */ - int nInput, /* Size of string aInput in bytes */ - sqlite3_tokenizer_cursor **pp /* OUT: New cursor object */ +static void jsonEachComputePath( + JsonEachCursor *p, /* The cursor */ + JsonString *pStr, /* Write the path here */ + u32 i /* Path to this element */ ){ - unicode_cursor *pCsr; - - pCsr = (unicode_cursor *)sqlite3_malloc(sizeof(unicode_cursor)); - if( pCsr==0 ){ - return SQLITE_NOMEM; + JsonNode *pNode, *pUp; + u32 iUp; + if( i==0 ){ + jsonAppendChar(pStr, '$'); + return; } - memset(pCsr, 0, sizeof(unicode_cursor)); - - pCsr->aInput = (const unsigned char *)aInput; - if( aInput==0 ){ - pCsr->nInput = 0; - }else if( nInput<0 ){ - pCsr->nInput = (int)strlen(aInput); + iUp = p->sParse.aUp[i]; + jsonEachComputePath(p, pStr, iUp); + pNode = &p->sParse.aNode[i]; + pUp = &p->sParse.aNode[iUp]; + if( pUp->eType==JSON_ARRAY ){ + jsonPrintf(30, pStr, "[%d]", pUp->u.iKey); }else{ - pCsr->nInput = nInput; + assert( pUp->eType==JSON_OBJECT ); + if( (pNode->jnFlags & JNODE_LABEL)==0 ) pNode--; + assert( pNode->eType==JSON_STRING ); + assert( pNode->jnFlags & JNODE_LABEL ); + jsonPrintf(pNode->n+1, pStr, ".%.*s", pNode->n-2, pNode->u.zJContent+1); } +} - *pp = &pCsr->base; - UNUSED_PARAMETER(p); +/* Return the value of a column */ +static int jsonEachColumn( + sqlite3_vtab_cursor *cur, /* The cursor */ + sqlite3_context *ctx, /* First argument to sqlite3_result_...() */ + int i /* Which column to return */ +){ + JsonEachCursor *p = (JsonEachCursor*)cur; + JsonNode *pThis = &p->sParse.aNode[p->i]; + switch( i ){ + case JEACH_KEY: { + if( p->i==0 ) break; + if( p->eType==JSON_OBJECT ){ + jsonReturn(pThis, ctx, 0); + }else if( p->eType==JSON_ARRAY ){ + u32 iKey; + if( p->bRecursive ){ + if( p->iRowid==0 ) break; + iKey = p->sParse.aNode[p->sParse.aUp[p->i]].u.iKey; + }else{ + iKey = p->iRowid; + } + sqlite3_result_int64(ctx, (sqlite3_int64)iKey); + } + break; + } + case JEACH_VALUE: { + if( pThis->jnFlags & JNODE_LABEL ) pThis++; + jsonReturn(pThis, ctx, 0); + break; + } + case JEACH_TYPE: { + if( pThis->jnFlags & JNODE_LABEL ) pThis++; + sqlite3_result_text(ctx, jsonType[pThis->eType], -1, SQLITE_STATIC); + break; + } + case JEACH_ATOM: { + if( pThis->jnFlags & JNODE_LABEL ) pThis++; + if( pThis->eType>=JSON_ARRAY ) break; + jsonReturn(pThis, ctx, 0); + break; + } + case JEACH_ID: { + sqlite3_result_int64(ctx, + (sqlite3_int64)p->i + ((pThis->jnFlags & JNODE_LABEL)!=0)); + break; + } + case JEACH_PARENT: { + if( p->i>p->iBegin && p->bRecursive ){ + sqlite3_result_int64(ctx, (sqlite3_int64)p->sParse.aUp[p->i]); + } + break; + } + case JEACH_FULLKEY: { + JsonString x; + jsonInit(&x, ctx); + if( p->bRecursive ){ + jsonEachComputePath(p, &x, p->i); + }else{ + if( p->zRoot ){ + jsonAppendRaw(&x, p->zRoot, (int)strlen(p->zRoot)); + }else{ + jsonAppendChar(&x, '$'); + } + if( p->eType==JSON_ARRAY ){ + jsonPrintf(30, &x, "[%d]", p->iRowid); + }else if( p->eType==JSON_OBJECT ){ + jsonPrintf(pThis->n, &x, ".%.*s", pThis->n-2, pThis->u.zJContent+1); + } + } + jsonResult(&x); + break; + } + case JEACH_PATH: { + if( p->bRecursive ){ + JsonString x; + jsonInit(&x, ctx); + jsonEachComputePath(p, &x, p->sParse.aUp[p->i]); + jsonResult(&x); + break; + } + /* For json_each() path and root are the same so fall through + ** into the root case */ + } + default: { + const char *zRoot = p->zRoot; + if( zRoot==0 ) zRoot = "$"; + sqlite3_result_text(ctx, zRoot, -1, SQLITE_STATIC); + break; + } + case JEACH_JSON: { + assert( i==JEACH_JSON ); + sqlite3_result_text(ctx, p->sParse.zJson, -1, SQLITE_STATIC); + break; + } + } return SQLITE_OK; } -/* -** Close a tokenization cursor previously opened by a call to -** simpleOpen() above. -*/ -static int unicodeClose(sqlite3_tokenizer_cursor *pCursor){ - unicode_cursor *pCsr = (unicode_cursor *) pCursor; - sqlite3_free(pCsr->zToken); - sqlite3_free(pCsr); +/* Return the current rowid value */ +static int jsonEachRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ + JsonEachCursor *p = (JsonEachCursor*)cur; + *pRowid = p->iRowid; return SQLITE_OK; } -/* -** Extract the next token from a tokenization cursor. The cursor must -** have been opened by a prior call to simpleOpen(). +/* The query strategy is to look for an equality constraint on the json +** column. Without such a constraint, the table cannot operate. idxNum is +** 1 if the constraint is found, 3 if the constraint and zRoot are found, +** and 0 otherwise. */ -static int unicodeNext( - sqlite3_tokenizer_cursor *pC, /* Cursor returned by simpleOpen */ - const char **paToken, /* OUT: Token text */ - int *pnToken, /* OUT: Number of bytes at *paToken */ - int *piStart, /* OUT: Starting offset of token */ - int *piEnd, /* OUT: Ending offset of token */ - int *piPos /* OUT: Position integer of token */ +static int jsonEachBestIndex( + sqlite3_vtab *tab, + sqlite3_index_info *pIdxInfo ){ - unicode_cursor *pCsr = (unicode_cursor *)pC; - unicode_tokenizer *p = ((unicode_tokenizer *)pCsr->base.pTokenizer); - unsigned int iCode = 0; - char *zOut; - const unsigned char *z = &pCsr->aInput[pCsr->iOff]; - const unsigned char *zStart = z; - const unsigned char *zEnd; - const unsigned char *zTerm = &pCsr->aInput[pCsr->nInput]; - - /* Scan past any delimiter characters before the start of the next token. - ** Return SQLITE_DONE early if this takes us all the way to the end of - ** the input. */ - while( z=zTerm ) return SQLITE_DONE; - - zOut = pCsr->zToken; - do { - int iOut; - - /* Grow the output buffer if required. */ - if( (zOut-pCsr->zToken)>=(pCsr->nAlloc-4) ){ - char *zNew = sqlite3_realloc(pCsr->zToken, pCsr->nAlloc+64); - if( !zNew ) return SQLITE_NOMEM; - zOut = &zNew[zOut - pCsr->zToken]; - pCsr->zToken = zNew; - pCsr->nAlloc += 64; - } + int i; /* Loop counter or computed array index */ + int aIdx[2]; /* Index of constraints for JSON and ROOT */ + int unusableMask = 0; /* Mask of unusable JSON and ROOT constraints */ + int idxMask = 0; /* Mask of usable == constraints JSON and ROOT */ + const struct sqlite3_index_constraint *pConstraint; - /* Write the folded case of the last character read to the output */ - zEnd = z; - iOut = sqlite3FtsUnicodeFold((int)iCode, p->bRemoveDiacritic); - if( iOut ){ - WRITE_UTF8(zOut, iOut); + /* This implementation assumes that JSON and ROOT are the last two + ** columns in the table */ + assert( JEACH_ROOT == JEACH_JSON+1 ); + UNUSED_PARAM(tab); + aIdx[0] = aIdx[1] = -1; + pConstraint = pIdxInfo->aConstraint; + for(i=0; inConstraint; i++, pConstraint++){ + int iCol; + int iMask; + if( pConstraint->iColumn < JEACH_JSON ) continue; + iCol = pConstraint->iColumn - JEACH_JSON; + assert( iCol==0 || iCol==1 ); + iMask = 1 << iCol; + if( pConstraint->usable==0 ){ + unusableMask |= iMask; + }else if( pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ ){ + aIdx[iCol] = i; + idxMask |= iMask; + } + } + if( (unusableMask & ~idxMask)!=0 ){ + /* If there are any unusable constraints on JSON or ROOT, then reject + ** this entire plan */ + return SQLITE_CONSTRAINT; + } + if( aIdx[0]<0 ){ + /* No JSON input. Leave estimatedCost at the huge value that it was + ** initialized to to discourage the query planner from selecting this + ** plan. */ + pIdxInfo->idxNum = 0; + }else{ + pIdxInfo->estimatedCost = 1.0; + i = aIdx[0]; + pIdxInfo->aConstraintUsage[i].argvIndex = 1; + pIdxInfo->aConstraintUsage[i].omit = 1; + if( aIdx[1]<0 ){ + pIdxInfo->idxNum = 1; /* Only JSON supplied. Plan 1 */ + }else{ + i = aIdx[1]; + pIdxInfo->aConstraintUsage[i].argvIndex = 2; + pIdxInfo->aConstraintUsage[i].omit = 1; + pIdxInfo->idxNum = 3; /* Both JSON and ROOT are supplied. Plan 3 */ } - - /* If the cursor is not at EOF, read the next character */ - if( z>=zTerm ) break; - READ_UTF8(z, zTerm, iCode); - }while( unicodeIsAlnum(p, (int)iCode) - || sqlite3FtsUnicodeIsdiacritic((int)iCode) - ); - - /* Set the output variables and return. */ - pCsr->iOff = (int)(z - pCsr->aInput); - *paToken = pCsr->zToken; - *pnToken = (int)(zOut - pCsr->zToken); - *piStart = (int)(zStart - pCsr->aInput); - *piEnd = (int)(zEnd - pCsr->aInput); - *piPos = pCsr->iToken++; + } return SQLITE_OK; } -/* -** Set *ppModule to a pointer to the sqlite3_tokenizer_module -** structure for the unicode tokenizer. -*/ -SQLITE_PRIVATE void sqlite3Fts3UnicodeTokenizer(sqlite3_tokenizer_module const **ppModule){ - static const sqlite3_tokenizer_module module = { - 0, - unicodeCreate, - unicodeDestroy, - unicodeOpen, - unicodeClose, - unicodeNext, - 0, - }; - *ppModule = &module; -} - -#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ -#endif /* ifndef SQLITE_DISABLE_FTS3_UNICODE */ - -/************** End of fts3_unicode.c ****************************************/ -/************** Begin file fts3_unicode2.c ***********************************/ -/* -** 2012 May 25 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -****************************************************************************** -*/ - -/* -** DO NOT EDIT THIS MACHINE GENERATED FILE. -*/ - -#ifndef SQLITE_DISABLE_FTS3_UNICODE -#if defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4) - -/* #include */ - -/* -** Return true if the argument corresponds to a unicode codepoint -** classified as either a letter or a number. Otherwise false. -** -** The results are undefined if the value passed to this function -** is less than zero. -*/ -SQLITE_PRIVATE int sqlite3FtsUnicodeIsalnum(int c){ - /* Each unsigned integer in the following array corresponds to a contiguous - ** range of unicode codepoints that are not either letters or numbers (i.e. - ** codepoints for which this function should return 0). - ** - ** The most significant 22 bits in each 32-bit value contain the first - ** codepoint in the range. The least significant 10 bits are used to store - ** the size of the range (always at least 1). In other words, the value - ** ((C<<22) + N) represents a range of N codepoints starting with codepoint - ** C. It is not possible to represent a range larger than 1023 codepoints - ** using this format. - */ - static const unsigned int aEntry[] = { - 0x00000030, 0x0000E807, 0x00016C06, 0x0001EC2F, 0x0002AC07, - 0x0002D001, 0x0002D803, 0x0002EC01, 0x0002FC01, 0x00035C01, - 0x0003DC01, 0x000B0804, 0x000B480E, 0x000B9407, 0x000BB401, - 0x000BBC81, 0x000DD401, 0x000DF801, 0x000E1002, 0x000E1C01, - 0x000FD801, 0x00120808, 0x00156806, 0x00162402, 0x00163C01, - 0x00164437, 0x0017CC02, 0x00180005, 0x00181816, 0x00187802, - 0x00192C15, 0x0019A804, 0x0019C001, 0x001B5001, 0x001B580F, - 0x001B9C07, 0x001BF402, 0x001C000E, 0x001C3C01, 0x001C4401, - 0x001CC01B, 0x001E980B, 0x001FAC09, 0x001FD804, 0x00205804, - 0x00206C09, 0x00209403, 0x0020A405, 0x0020C00F, 0x00216403, - 0x00217801, 0x0023901B, 0x00240004, 0x0024E803, 0x0024F812, - 0x00254407, 0x00258804, 0x0025C001, 0x00260403, 0x0026F001, - 0x0026F807, 0x00271C02, 0x00272C03, 0x00275C01, 0x00278802, - 0x0027C802, 0x0027E802, 0x00280403, 0x0028F001, 0x0028F805, - 0x00291C02, 0x00292C03, 0x00294401, 0x0029C002, 0x0029D401, - 0x002A0403, 0x002AF001, 0x002AF808, 0x002B1C03, 0x002B2C03, - 0x002B8802, 0x002BC002, 0x002C0403, 0x002CF001, 0x002CF807, - 0x002D1C02, 0x002D2C03, 0x002D5802, 0x002D8802, 0x002DC001, - 0x002E0801, 0x002EF805, 0x002F1803, 0x002F2804, 0x002F5C01, - 0x002FCC08, 0x00300403, 0x0030F807, 0x00311803, 0x00312804, - 0x00315402, 0x00318802, 0x0031FC01, 0x00320802, 0x0032F001, - 0x0032F807, 0x00331803, 0x00332804, 0x00335402, 0x00338802, - 0x00340802, 0x0034F807, 0x00351803, 0x00352804, 0x00355C01, - 0x00358802, 0x0035E401, 0x00360802, 0x00372801, 0x00373C06, - 0x00375801, 0x00376008, 0x0037C803, 0x0038C401, 0x0038D007, - 0x0038FC01, 0x00391C09, 0x00396802, 0x003AC401, 0x003AD006, - 0x003AEC02, 0x003B2006, 0x003C041F, 0x003CD00C, 0x003DC417, - 0x003E340B, 0x003E6424, 0x003EF80F, 0x003F380D, 0x0040AC14, - 0x00412806, 0x00415804, 0x00417803, 0x00418803, 0x00419C07, - 0x0041C404, 0x0042080C, 0x00423C01, 0x00426806, 0x0043EC01, - 0x004D740C, 0x004E400A, 0x00500001, 0x0059B402, 0x005A0001, - 0x005A6C02, 0x005BAC03, 0x005C4803, 0x005CC805, 0x005D4802, - 0x005DC802, 0x005ED023, 0x005F6004, 0x005F7401, 0x0060000F, - 0x0062A401, 0x0064800C, 0x0064C00C, 0x00650001, 0x00651002, - 0x0066C011, 0x00672002, 0x00677822, 0x00685C05, 0x00687802, - 0x0069540A, 0x0069801D, 0x0069FC01, 0x006A8007, 0x006AA006, - 0x006C0005, 0x006CD011, 0x006D6823, 0x006E0003, 0x006E840D, - 0x006F980E, 0x006FF004, 0x00709014, 0x0070EC05, 0x0071F802, - 0x00730008, 0x00734019, 0x0073B401, 0x0073C803, 0x00770027, - 0x0077F004, 0x007EF401, 0x007EFC03, 0x007F3403, 0x007F7403, - 0x007FB403, 0x007FF402, 0x00800065, 0x0081A806, 0x0081E805, - 0x00822805, 0x0082801A, 0x00834021, 0x00840002, 0x00840C04, - 0x00842002, 0x00845001, 0x00845803, 0x00847806, 0x00849401, - 0x00849C01, 0x0084A401, 0x0084B801, 0x0084E802, 0x00850005, - 0x00852804, 0x00853C01, 0x00864264, 0x00900027, 0x0091000B, - 0x0092704E, 0x00940200, 0x009C0475, 0x009E53B9, 0x00AD400A, - 0x00B39406, 0x00B3BC03, 0x00B3E404, 0x00B3F802, 0x00B5C001, - 0x00B5FC01, 0x00B7804F, 0x00B8C00C, 0x00BA001A, 0x00BA6C59, - 0x00BC00D6, 0x00BFC00C, 0x00C00005, 0x00C02019, 0x00C0A807, - 0x00C0D802, 0x00C0F403, 0x00C26404, 0x00C28001, 0x00C3EC01, - 0x00C64002, 0x00C6580A, 0x00C70024, 0x00C8001F, 0x00C8A81E, - 0x00C94001, 0x00C98020, 0x00CA2827, 0x00CB003F, 0x00CC0100, - 0x01370040, 0x02924037, 0x0293F802, 0x02983403, 0x0299BC10, - 0x029A7C01, 0x029BC008, 0x029C0017, 0x029C8002, 0x029E2402, - 0x02A00801, 0x02A01801, 0x02A02C01, 0x02A08C09, 0x02A0D804, - 0x02A1D004, 0x02A20002, 0x02A2D011, 0x02A33802, 0x02A38012, - 0x02A3E003, 0x02A4980A, 0x02A51C0D, 0x02A57C01, 0x02A60004, - 0x02A6CC1B, 0x02A77802, 0x02A8A40E, 0x02A90C01, 0x02A93002, - 0x02A97004, 0x02A9DC03, 0x02A9EC01, 0x02AAC001, 0x02AAC803, - 0x02AADC02, 0x02AAF802, 0x02AB0401, 0x02AB7802, 0x02ABAC07, - 0x02ABD402, 0x02AF8C0B, 0x03600001, 0x036DFC02, 0x036FFC02, - 0x037FFC01, 0x03EC7801, 0x03ECA401, 0x03EEC810, 0x03F4F802, - 0x03F7F002, 0x03F8001A, 0x03F88007, 0x03F8C023, 0x03F95013, - 0x03F9A004, 0x03FBFC01, 0x03FC040F, 0x03FC6807, 0x03FCEC06, - 0x03FD6C0B, 0x03FF8007, 0x03FFA007, 0x03FFE405, 0x04040003, - 0x0404DC09, 0x0405E411, 0x0406400C, 0x0407402E, 0x040E7C01, - 0x040F4001, 0x04215C01, 0x04247C01, 0x0424FC01, 0x04280403, - 0x04281402, 0x04283004, 0x0428E003, 0x0428FC01, 0x04294009, - 0x0429FC01, 0x042CE407, 0x04400003, 0x0440E016, 0x04420003, - 0x0442C012, 0x04440003, 0x04449C0E, 0x04450004, 0x04460003, - 0x0446CC0E, 0x04471404, 0x045AAC0D, 0x0491C004, 0x05BD442E, - 0x05BE3C04, 0x074000F6, 0x07440027, 0x0744A4B5, 0x07480046, - 0x074C0057, 0x075B0401, 0x075B6C01, 0x075BEC01, 0x075C5401, - 0x075CD401, 0x075D3C01, 0x075DBC01, 0x075E2401, 0x075EA401, - 0x075F0C01, 0x07BBC002, 0x07C0002C, 0x07C0C064, 0x07C2800F, - 0x07C2C40E, 0x07C3040F, 0x07C3440F, 0x07C4401F, 0x07C4C03C, - 0x07C5C02B, 0x07C7981D, 0x07C8402B, 0x07C90009, 0x07C94002, - 0x07CC0021, 0x07CCC006, 0x07CCDC46, 0x07CE0014, 0x07CE8025, - 0x07CF1805, 0x07CF8011, 0x07D0003F, 0x07D10001, 0x07D108B6, - 0x07D3E404, 0x07D4003E, 0x07D50004, 0x07D54018, 0x07D7EC46, - 0x07D9140B, 0x07DA0046, 0x07DC0074, 0x38000401, 0x38008060, - 0x380400F0, - }; - static const unsigned int aAscii[4] = { - 0xFFFFFFFF, 0xFC00FFFF, 0xF8000001, 0xF8000001, - }; +/* Start a search on a new JSON string */ +static int jsonEachFilter( + sqlite3_vtab_cursor *cur, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + JsonEachCursor *p = (JsonEachCursor*)cur; + const char *z; + const char *zRoot = 0; + sqlite3_int64 n; - if( (unsigned int)c<128 ){ - return ( (aAscii[c >> 5] & ((unsigned int)1 << (c & 0x001F)))==0 ); - }else if( (unsigned int)c<(1<<22) ){ - unsigned int key = (((unsigned int)c)<<10) | 0x000003FF; - int iRes = 0; - int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1; - int iLo = 0; - while( iHi>=iLo ){ - int iTest = (iHi + iLo) / 2; - if( key >= aEntry[iTest] ){ - iRes = iTest; - iLo = iTest+1; + UNUSED_PARAM(idxStr); + UNUSED_PARAM(argc); + jsonEachCursorReset(p); + if( idxNum==0 ) return SQLITE_OK; + z = (const char*)sqlite3_value_text(argv[0]); + if( z==0 ) return SQLITE_OK; + n = sqlite3_value_bytes(argv[0]); + p->zJson = sqlite3_malloc64( n+1 ); + if( p->zJson==0 ) return SQLITE_NOMEM; + memcpy(p->zJson, z, (size_t)n+1); + if( jsonParse(&p->sParse, 0, p->zJson) ){ + int rc = SQLITE_NOMEM; + if( p->sParse.oom==0 ){ + sqlite3_free(cur->pVtab->zErrMsg); + cur->pVtab->zErrMsg = sqlite3_mprintf("malformed JSON"); + if( cur->pVtab->zErrMsg ) rc = SQLITE_ERROR; + } + jsonEachCursorReset(p); + return rc; + }else if( p->bRecursive && jsonParseFindParents(&p->sParse) ){ + jsonEachCursorReset(p); + return SQLITE_NOMEM; + }else{ + JsonNode *pNode = 0; + if( idxNum==3 ){ + const char *zErr = 0; + zRoot = (const char*)sqlite3_value_text(argv[1]); + if( zRoot==0 ) return SQLITE_OK; + n = sqlite3_value_bytes(argv[1]); + p->zRoot = sqlite3_malloc64( n+1 ); + if( p->zRoot==0 ) return SQLITE_NOMEM; + memcpy(p->zRoot, zRoot, (size_t)n+1); + if( zRoot[0]!='$' ){ + zErr = zRoot; }else{ - iHi = iTest-1; + pNode = jsonLookupStep(&p->sParse, 0, p->zRoot+1, 0, &zErr); + } + if( zErr ){ + sqlite3_free(cur->pVtab->zErrMsg); + cur->pVtab->zErrMsg = jsonPathSyntaxError(zErr); + jsonEachCursorReset(p); + return cur->pVtab->zErrMsg ? SQLITE_ERROR : SQLITE_NOMEM; + }else if( pNode==0 ){ + return SQLITE_OK; } + }else{ + pNode = p->sParse.aNode; } - assert( aEntry[0]=aEntry[iRes] ); - return (((unsigned int)c) >= ((aEntry[iRes]>>10) + (aEntry[iRes]&0x3FF))); - } - return 1; -} - - -/* -** If the argument is a codepoint corresponding to a lowercase letter -** in the ASCII range with a diacritic added, return the codepoint -** of the ASCII letter only. For example, if passed 235 - "LATIN -** SMALL LETTER E WITH DIAERESIS" - return 65 ("LATIN SMALL LETTER -** E"). The resuls of passing a codepoint that corresponds to an -** uppercase letter are undefined. -*/ -static int remove_diacritic(int c){ - unsigned short aDia[] = { - 0, 1797, 1848, 1859, 1891, 1928, 1940, 1995, - 2024, 2040, 2060, 2110, 2168, 2206, 2264, 2286, - 2344, 2383, 2472, 2488, 2516, 2596, 2668, 2732, - 2782, 2842, 2894, 2954, 2984, 3000, 3028, 3336, - 3456, 3696, 3712, 3728, 3744, 3896, 3912, 3928, - 3968, 4008, 4040, 4106, 4138, 4170, 4202, 4234, - 4266, 4296, 4312, 4344, 4408, 4424, 4472, 4504, - 6148, 6198, 6264, 6280, 6360, 6429, 6505, 6529, - 61448, 61468, 61534, 61592, 61642, 61688, 61704, 61726, - 61784, 61800, 61836, 61880, 61914, 61948, 61998, 62122, - 62154, 62200, 62218, 62302, 62364, 62442, 62478, 62536, - 62554, 62584, 62604, 62640, 62648, 62656, 62664, 62730, - 62924, 63050, 63082, 63274, 63390, - }; - char aChar[] = { - '\0', 'a', 'c', 'e', 'i', 'n', 'o', 'u', 'y', 'y', 'a', 'c', - 'd', 'e', 'e', 'g', 'h', 'i', 'j', 'k', 'l', 'n', 'o', 'r', - 's', 't', 'u', 'u', 'w', 'y', 'z', 'o', 'u', 'a', 'i', 'o', - 'u', 'g', 'k', 'o', 'j', 'g', 'n', 'a', 'e', 'i', 'o', 'r', - 'u', 's', 't', 'h', 'a', 'e', 'o', 'y', '\0', '\0', '\0', '\0', - '\0', '\0', '\0', '\0', 'a', 'b', 'd', 'd', 'e', 'f', 'g', 'h', - 'h', 'i', 'k', 'l', 'l', 'm', 'n', 'p', 'r', 'r', 's', 't', - 'u', 'v', 'w', 'w', 'x', 'y', 'z', 'h', 't', 'w', 'y', 'a', - 'e', 'i', 'o', 'u', 'y', - }; - - unsigned int key = (((unsigned int)c)<<3) | 0x00000007; - int iRes = 0; - int iHi = sizeof(aDia)/sizeof(aDia[0]) - 1; - int iLo = 0; - while( iHi>=iLo ){ - int iTest = (iHi + iLo) / 2; - if( key >= aDia[iTest] ){ - iRes = iTest; - iLo = iTest+1; + p->iBegin = p->i = (int)(pNode - p->sParse.aNode); + p->eType = pNode->eType; + if( p->eType>=JSON_ARRAY ){ + pNode->u.iKey = 0; + p->iEnd = p->i + pNode->n + 1; + if( p->bRecursive ){ + p->eType = p->sParse.aNode[p->sParse.aUp[p->i]].eType; + if( p->i>0 && (p->sParse.aNode[p->i-1].jnFlags & JNODE_LABEL)!=0 ){ + p->i--; + } + }else{ + p->i++; + } }else{ - iHi = iTest-1; + p->iEnd = p->i+1; } } - assert( key>=aDia[iRes] ); - return ((c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : (int)aChar[iRes]); + return SQLITE_OK; } +/* The methods of the json_each virtual table */ +static sqlite3_module jsonEachModule = { + 0, /* iVersion */ + 0, /* xCreate */ + jsonEachConnect, /* xConnect */ + jsonEachBestIndex, /* xBestIndex */ + jsonEachDisconnect, /* xDisconnect */ + 0, /* xDestroy */ + jsonEachOpenEach, /* xOpen - open a cursor */ + jsonEachClose, /* xClose - close a cursor */ + jsonEachFilter, /* xFilter - configure scan constraints */ + jsonEachNext, /* xNext - advance a cursor */ + jsonEachEof, /* xEof - check for end of scan */ + jsonEachColumn, /* xColumn - read data */ + jsonEachRowid, /* xRowid - read data */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindMethod */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0, /* xRollbackTo */ + 0 /* xShadowName */ +}; -/* -** Return true if the argument interpreted as a unicode codepoint -** is a diacritical modifier character. -*/ -SQLITE_PRIVATE int sqlite3FtsUnicodeIsdiacritic(int c){ - unsigned int mask0 = 0x08029FDF; - unsigned int mask1 = 0x000361F8; - if( c<768 || c>817 ) return 0; - return (c < 768+32) ? - (mask0 & (1 << (c-768))) : - (mask1 & (1 << (c-768-32))); -} +/* The methods of the json_tree virtual table. */ +static sqlite3_module jsonTreeModule = { + 0, /* iVersion */ + 0, /* xCreate */ + jsonEachConnect, /* xConnect */ + jsonEachBestIndex, /* xBestIndex */ + jsonEachDisconnect, /* xDisconnect */ + 0, /* xDestroy */ + jsonEachOpenTree, /* xOpen - open a cursor */ + jsonEachClose, /* xClose - close a cursor */ + jsonEachFilter, /* xFilter - configure scan constraints */ + jsonEachNext, /* xNext - advance a cursor */ + jsonEachEof, /* xEof - check for end of scan */ + jsonEachColumn, /* xColumn - read data */ + jsonEachRowid, /* xRowid - read data */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindMethod */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0, /* xRollbackTo */ + 0 /* xShadowName */ +}; +#endif /* SQLITE_OMIT_VIRTUALTABLE */ +/**************************************************************************** +** The following routines are the only publically visible identifiers in this +** file. Call the following routines in order to register the various SQL +** functions and the virtual table implemented by this file. +****************************************************************************/ -/* -** Interpret the argument as a unicode codepoint. If the codepoint -** is an upper case character that has a lower case equivalent, -** return the codepoint corresponding to the lower case version. -** Otherwise, return a copy of the argument. -** -** The results are undefined if the value passed to this function -** is less than zero. -*/ -SQLITE_PRIVATE int sqlite3FtsUnicodeFold(int c, int bRemoveDiacritic){ - /* Each entry in the following array defines a rule for folding a range - ** of codepoints to lower case. The rule applies to a range of nRange - ** codepoints starting at codepoint iCode. - ** - ** If the least significant bit in flags is clear, then the rule applies - ** to all nRange codepoints (i.e. all nRange codepoints are upper case and - ** need to be folded). Or, if it is set, then the rule only applies to - ** every second codepoint in the range, starting with codepoint C. - ** - ** The 7 most significant bits in flags are an index into the aiOff[] - ** array. If a specific codepoint C does require folding, then its lower - ** case equivalent is ((C + aiOff[flags>>1]) & 0xFFFF). - ** - ** The contents of this array are generated by parsing the CaseFolding.txt - ** file distributed as part of the "Unicode Character Database". See - ** http://www.unicode.org for details. - */ - static const struct TableEntry { - unsigned short iCode; - unsigned char flags; - unsigned char nRange; - } aEntry[] = { - {65, 14, 26}, {181, 64, 1}, {192, 14, 23}, - {216, 14, 7}, {256, 1, 48}, {306, 1, 6}, - {313, 1, 16}, {330, 1, 46}, {376, 116, 1}, - {377, 1, 6}, {383, 104, 1}, {385, 50, 1}, - {386, 1, 4}, {390, 44, 1}, {391, 0, 1}, - {393, 42, 2}, {395, 0, 1}, {398, 32, 1}, - {399, 38, 1}, {400, 40, 1}, {401, 0, 1}, - {403, 42, 1}, {404, 46, 1}, {406, 52, 1}, - {407, 48, 1}, {408, 0, 1}, {412, 52, 1}, - {413, 54, 1}, {415, 56, 1}, {416, 1, 6}, - {422, 60, 1}, {423, 0, 1}, {425, 60, 1}, - {428, 0, 1}, {430, 60, 1}, {431, 0, 1}, - {433, 58, 2}, {435, 1, 4}, {439, 62, 1}, - {440, 0, 1}, {444, 0, 1}, {452, 2, 1}, - {453, 0, 1}, {455, 2, 1}, {456, 0, 1}, - {458, 2, 1}, {459, 1, 18}, {478, 1, 18}, - {497, 2, 1}, {498, 1, 4}, {502, 122, 1}, - {503, 134, 1}, {504, 1, 40}, {544, 110, 1}, - {546, 1, 18}, {570, 70, 1}, {571, 0, 1}, - {573, 108, 1}, {574, 68, 1}, {577, 0, 1}, - {579, 106, 1}, {580, 28, 1}, {581, 30, 1}, - {582, 1, 10}, {837, 36, 1}, {880, 1, 4}, - {886, 0, 1}, {902, 18, 1}, {904, 16, 3}, - {908, 26, 1}, {910, 24, 2}, {913, 14, 17}, - {931, 14, 9}, {962, 0, 1}, {975, 4, 1}, - {976, 140, 1}, {977, 142, 1}, {981, 146, 1}, - {982, 144, 1}, {984, 1, 24}, {1008, 136, 1}, - {1009, 138, 1}, {1012, 130, 1}, {1013, 128, 1}, - {1015, 0, 1}, {1017, 152, 1}, {1018, 0, 1}, - {1021, 110, 3}, {1024, 34, 16}, {1040, 14, 32}, - {1120, 1, 34}, {1162, 1, 54}, {1216, 6, 1}, - {1217, 1, 14}, {1232, 1, 88}, {1329, 22, 38}, - {4256, 66, 38}, {4295, 66, 1}, {4301, 66, 1}, - {7680, 1, 150}, {7835, 132, 1}, {7838, 96, 1}, - {7840, 1, 96}, {7944, 150, 8}, {7960, 150, 6}, - {7976, 150, 8}, {7992, 150, 8}, {8008, 150, 6}, - {8025, 151, 8}, {8040, 150, 8}, {8072, 150, 8}, - {8088, 150, 8}, {8104, 150, 8}, {8120, 150, 2}, - {8122, 126, 2}, {8124, 148, 1}, {8126, 100, 1}, - {8136, 124, 4}, {8140, 148, 1}, {8152, 150, 2}, - {8154, 120, 2}, {8168, 150, 2}, {8170, 118, 2}, - {8172, 152, 1}, {8184, 112, 2}, {8186, 114, 2}, - {8188, 148, 1}, {8486, 98, 1}, {8490, 92, 1}, - {8491, 94, 1}, {8498, 12, 1}, {8544, 8, 16}, - {8579, 0, 1}, {9398, 10, 26}, {11264, 22, 47}, - {11360, 0, 1}, {11362, 88, 1}, {11363, 102, 1}, - {11364, 90, 1}, {11367, 1, 6}, {11373, 84, 1}, - {11374, 86, 1}, {11375, 80, 1}, {11376, 82, 1}, - {11378, 0, 1}, {11381, 0, 1}, {11390, 78, 2}, - {11392, 1, 100}, {11499, 1, 4}, {11506, 0, 1}, - {42560, 1, 46}, {42624, 1, 24}, {42786, 1, 14}, - {42802, 1, 62}, {42873, 1, 4}, {42877, 76, 1}, - {42878, 1, 10}, {42891, 0, 1}, {42893, 74, 1}, - {42896, 1, 4}, {42912, 1, 10}, {42922, 72, 1}, - {65313, 14, 26}, +SQLITE_PRIVATE int sqlite3Json1Init(sqlite3 *db){ + int rc = SQLITE_OK; + unsigned int i; + static const struct { + const char *zName; + int nArg; + int flag; + void (*xFunc)(sqlite3_context*,int,sqlite3_value**); + } aFunc[] = { + { "json", 1, 0, jsonRemoveFunc }, + { "json_array", -1, 0, jsonArrayFunc }, + { "json_array_length", 1, 0, jsonArrayLengthFunc }, + { "json_array_length", 2, 0, jsonArrayLengthFunc }, + { "json_extract", -1, 0, jsonExtractFunc }, + { "json_insert", -1, 0, jsonSetFunc }, + { "json_object", -1, 0, jsonObjectFunc }, + { "json_patch", 2, 0, jsonPatchFunc }, + { "json_quote", 1, 0, jsonQuoteFunc }, + { "json_remove", -1, 0, jsonRemoveFunc }, + { "json_replace", -1, 0, jsonReplaceFunc }, + { "json_set", -1, 1, jsonSetFunc }, + { "json_type", 1, 0, jsonTypeFunc }, + { "json_type", 2, 0, jsonTypeFunc }, + { "json_valid", 1, 0, jsonValidFunc }, + +#if SQLITE_DEBUG + /* DEBUG and TESTING functions */ + { "json_parse", 1, 0, jsonParseFunc }, + { "json_test1", 1, 0, jsonTest1Func }, +#endif }; - static const unsigned short aiOff[] = { - 1, 2, 8, 15, 16, 26, 28, 32, - 37, 38, 40, 48, 63, 64, 69, 71, - 79, 80, 116, 202, 203, 205, 206, 207, - 209, 210, 211, 213, 214, 217, 218, 219, - 775, 7264, 10792, 10795, 23228, 23256, 30204, 54721, - 54753, 54754, 54756, 54787, 54793, 54809, 57153, 57274, - 57921, 58019, 58363, 61722, 65268, 65341, 65373, 65406, - 65408, 65410, 65415, 65424, 65436, 65439, 65450, 65462, - 65472, 65476, 65478, 65480, 65482, 65488, 65506, 65511, - 65514, 65521, 65527, 65528, 65529, + static const struct { + const char *zName; + int nArg; + void (*xStep)(sqlite3_context*,int,sqlite3_value**); + void (*xFinal)(sqlite3_context*); + void (*xValue)(sqlite3_context*); + } aAgg[] = { + { "json_group_array", 1, + jsonArrayStep, jsonArrayFinal, jsonArrayValue }, + { "json_group_object", 2, + jsonObjectStep, jsonObjectFinal, jsonObjectValue }, }; - - int ret = c; - - assert( sizeof(unsigned short)==2 && sizeof(unsigned char)==1 ); - - if( c<128 ){ - if( c>='A' && c<='Z' ) ret = c + ('a' - 'A'); - }else if( c<65536 ){ - const struct TableEntry *p; - int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1; - int iLo = 0; - int iRes = -1; - - assert( c>aEntry[0].iCode ); - while( iHi>=iLo ){ - int iTest = (iHi + iLo) / 2; - int cmp = (c - aEntry[iTest].iCode); - if( cmp>=0 ){ - iRes = iTest; - iLo = iTest+1; - }else{ - iHi = iTest-1; - } - } - - assert( iRes>=0 && c>=aEntry[iRes].iCode ); - p = &aEntry[iRes]; - if( c<(p->iCode + p->nRange) && 0==(0x01 & p->flags & (p->iCode ^ c)) ){ - ret = (c + (aiOff[p->flags>>1])) & 0x0000FFFF; - assert( ret>0 ); - } - - if( bRemoveDiacritic ) ret = remove_diacritic(ret); +#ifndef SQLITE_OMIT_VIRTUALTABLE + static const struct { + const char *zName; + sqlite3_module *pModule; + } aMod[] = { + { "json_each", &jsonEachModule }, + { "json_tree", &jsonTreeModule }, + }; +#endif + static const int enc = + SQLITE_UTF8 | + SQLITE_DETERMINISTIC | + SQLITE_INNOCUOUS; + for(i=0; i=66560 && c<66600 ){ - ret = c + 40; +#ifndef SQLITE_OMIT_WINDOWFUNC + for(i=0; i */ -/* #include */ -/* #include */ +SQLITE_PRIVATE int sqlite3GetToken(const unsigned char*,int*); /* In the SQLite core */ #ifndef SQLITE_AMALGAMATION #include "sqlite3rtree.h" @@ -170595,8 +187259,18 @@ typedef sqlite3_uint64 u64; typedef unsigned char u8; typedef unsigned short u16; typedef unsigned int u32; +#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) +# define NDEBUG 1 +#endif +#if defined(NDEBUG) && defined(SQLITE_DEBUG) +# undef NDEBUG +#endif #endif +/* #include */ +/* #include */ +/* #include */ + /* The following macro is used to suppress compiler warnings. */ #ifndef UNUSED_PARAMETER @@ -170648,6 +187322,10 @@ struct Rtree { u8 nBytesPerCell; /* Bytes consumed per cell */ u8 inWrTrans; /* True if inside write transaction */ u8 nAux; /* # of auxiliary columns in %_rowid */ + u8 nAuxNotNull; /* Number of initial not-null aux columns */ +#ifdef SQLITE_DEBUG + u8 bCorrupt; /* Shadow table corruption detected */ +#endif int iDepth; /* Current depth of the r-tree structure */ char *zDb; /* Name of database containing r-tree table */ char *zName; /* Name of r-tree table */ @@ -170707,6 +187385,15 @@ struct Rtree { # define RTREE_ZERO 0.0 #endif +/* +** Set the Rtree.bCorrupt flag +*/ +#ifdef SQLITE_DEBUG +# define RTREE_IS_CORRUPT(X) ((X)->bCorrupt = 1) +#else +# define RTREE_IS_CORRUPT(X) +#endif + /* ** When doing a search of an r-tree, instances of the following structure ** record intermediate results from the tree walk. @@ -170827,6 +187514,12 @@ struct RtreeConstraint { #define RTREE_MATCH 0x46 /* F: Old-style sqlite3_rtree_geometry_callback() */ #define RTREE_QUERY 0x47 /* G: New-style sqlite3_rtree_query_callback() */ +/* Special operators available only on cursors. Needs to be consecutive +** with the normal values above, but must be less than RTREE_MATCH. These +** are used in the cursor for contraints such as x=NULL (RTREE_FALSE) or +** x<'xyz' (RTREE_TRUE) */ +#define RTREE_TRUE 0x3f /* ? */ +#define RTREE_FALSE 0x40 /* @ */ /* ** An rtree structure node. @@ -171073,8 +187766,8 @@ static void nodeZero(Rtree *pRtree, RtreeNode *p){ ** Given a node number iNode, return the corresponding key to use ** in the Rtree.aHash table. */ -static int nodeHash(i64 iNode){ - return iNode % HASHSIZE; +static unsigned int nodeHash(i64 iNode){ + return ((unsigned)iNode) % HASHSIZE; } /* @@ -171119,7 +187812,7 @@ static void nodeHashDelete(Rtree *pRtree, RtreeNode *pNode){ */ static RtreeNode *nodeNew(Rtree *pRtree, RtreeNode *pParent){ RtreeNode *pNode; - pNode = (RtreeNode *)sqlite3_malloc(sizeof(RtreeNode) + pRtree->iNodeSize); + pNode = (RtreeNode *)sqlite3_malloc64(sizeof(RtreeNode) + pRtree->iNodeSize); if( pNode ){ memset(pNode, 0, sizeof(RtreeNode) + pRtree->iNodeSize); pNode->zData = (u8 *)&pNode[1]; @@ -171143,6 +187836,18 @@ static void nodeBlobReset(Rtree *pRtree){ } } +/* +** Check to see if pNode is the same as pParent or any of the parents +** of pParent. +*/ +static int nodeInParentChain(const RtreeNode *pNode, const RtreeNode *pParent){ + do{ + if( pNode==pParent ) return 1; + pParent = pParent->pParent; + }while( pParent ); + return 0; +} + /* ** Obtain a reference to an r-tree node. */ @@ -171159,10 +187864,16 @@ static int nodeAcquire( ** increase its reference count and return it. */ if( (pNode = nodeHashLookup(pRtree, iNode))!=0 ){ - assert( !pParent || !pNode->pParent || pNode->pParent==pParent ); if( pParent && !pNode->pParent ){ + if( nodeInParentChain(pNode, pParent) ){ + RTREE_IS_CORRUPT(pRtree); + return SQLITE_CORRUPT_VTAB; + } pParent->nRef++; pNode->pParent = pParent; + }else if( pParent && pNode->pParent && pParent!=pNode->pParent ){ + RTREE_IS_CORRUPT(pRtree); + return SQLITE_CORRUPT_VTAB; } pNode->nRef++; *ppNode = pNode; @@ -171191,9 +187902,12 @@ static int nodeAcquire( *ppNode = 0; /* If unable to open an sqlite3_blob on the desired row, that can only ** be because the shadow tables hold erroneous data. */ - if( rc==SQLITE_ERROR ) rc = SQLITE_CORRUPT_VTAB; + if( rc==SQLITE_ERROR ){ + rc = SQLITE_CORRUPT_VTAB; + RTREE_IS_CORRUPT(pRtree); + } }else if( pRtree->iNodeSize==sqlite3_blob_bytes(pRtree->pNodeBlob) ){ - pNode = (RtreeNode *)sqlite3_malloc(sizeof(RtreeNode)+pRtree->iNodeSize); + pNode = (RtreeNode *)sqlite3_malloc64(sizeof(RtreeNode)+pRtree->iNodeSize); if( !pNode ){ rc = SQLITE_NOMEM; }else{ @@ -171206,7 +187920,6 @@ static int nodeAcquire( pNode->pNext = 0; rc = sqlite3_blob_read(pRtree->pNodeBlob, pNode->zData, pRtree->iNodeSize, 0); - nodeReference(pParent); } } @@ -171220,6 +187933,7 @@ static int nodeAcquire( pRtree->iDepth = readInt16(pNode->zData); if( pRtree->iDepth>RTREE_MAX_DEPTH ){ rc = SQLITE_CORRUPT_VTAB; + RTREE_IS_CORRUPT(pRtree); } } @@ -171230,14 +187944,17 @@ static int nodeAcquire( if( pNode && rc==SQLITE_OK ){ if( NCELL(pNode)>((pRtree->iNodeSize-4)/pRtree->nBytesPerCell) ){ rc = SQLITE_CORRUPT_VTAB; + RTREE_IS_CORRUPT(pRtree); } } if( rc==SQLITE_OK ){ if( pNode!=0 ){ + nodeReference(pParent); nodeHashInsert(pRtree, pNode); }else{ rc = SQLITE_CORRUPT_VTAB; + RTREE_IS_CORRUPT(pRtree); } *ppNode = pNode; }else{ @@ -171463,7 +188180,7 @@ static void rtreeRelease(Rtree *pRtree){ pRtree->inWrTrans = 0; assert( pRtree->nCursor==0 ); nodeBlobReset(pRtree); - assert( pRtree->nNodeRef==0 ); + assert( pRtree->nNodeRef==0 || pRtree->bCorrupt ); sqlite3_finalize(pRtree->pWriteNode); sqlite3_finalize(pRtree->pDeleteNode); sqlite3_finalize(pRtree->pReadRowid); @@ -171522,7 +188239,7 @@ static int rtreeOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ Rtree *pRtree = (Rtree *)pVTab; RtreeCursor *pCsr; - pCsr = (RtreeCursor *)sqlite3_malloc(sizeof(RtreeCursor)); + pCsr = (RtreeCursor *)sqlite3_malloc64(sizeof(RtreeCursor)); if( pCsr ){ memset(pCsr, 0, sizeof(RtreeCursor)); pCsr->base.pVtab = pVTab; @@ -171536,9 +188253,12 @@ static int rtreeOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ /* -** Free the RtreeCursor.aConstraint[] array and its contents. +** Reset a cursor back to its initial state. */ -static void freeCursorConstraints(RtreeCursor *pCsr){ +static void resetCursor(RtreeCursor *pCsr){ + Rtree *pRtree = (Rtree *)(pCsr->base.pVtab); + int ii; + sqlite3_stmt *pStmt; if( pCsr->aConstraint ){ int i; /* Used to iterate through constraint array */ for(i=0; inConstraint; i++){ @@ -171551,6 +188271,13 @@ static void freeCursorConstraints(RtreeCursor *pCsr){ sqlite3_free(pCsr->aConstraint); pCsr->aConstraint = 0; } + for(ii=0; iiaNode[ii]); + sqlite3_free(pCsr->aPoint); + pStmt = pCsr->pReadAux; + memset(pCsr, 0, sizeof(RtreeCursor)); + pCsr->base.pVtab = (sqlite3_vtab*)pRtree; + pCsr->pReadAux = pStmt; + } /* @@ -171558,13 +188285,10 @@ static void freeCursorConstraints(RtreeCursor *pCsr){ */ static int rtreeClose(sqlite3_vtab_cursor *cur){ Rtree *pRtree = (Rtree *)(cur->pVtab); - int ii; RtreeCursor *pCsr = (RtreeCursor *)cur; assert( pRtree->nCursor>0 ); - freeCursorConstraints(pCsr); + resetCursor(pCsr); sqlite3_finalize(pCsr->pReadAux); - sqlite3_free(pCsr->aPoint); - for(ii=0; iiaNode[ii]); sqlite3_free(pCsr); pRtree->nCursor--; nodeBlobReset(pRtree); @@ -171722,9 +188446,12 @@ static void rtreeNonleafConstraint( pCellData += 8 + 4*(p->iCoord&0xfe); assert(p->op==RTREE_LE || p->op==RTREE_LT || p->op==RTREE_GE - || p->op==RTREE_GT || p->op==RTREE_EQ ); + || p->op==RTREE_GT || p->op==RTREE_EQ || p->op==RTREE_TRUE + || p->op==RTREE_FALSE ); assert( ((((char*)pCellData) - (char*)0)&3)==0 ); /* 4-byte aligned */ switch( p->op ){ + case RTREE_TRUE: return; /* Always satisfied */ + case RTREE_FALSE: break; /* Never satisfied */ case RTREE_LE: case RTREE_LT: case RTREE_EQ: @@ -171762,16 +188489,19 @@ static void rtreeLeafConstraint( RtreeDValue xN; /* Coordinate value converted to a double */ assert(p->op==RTREE_LE || p->op==RTREE_LT || p->op==RTREE_GE - || p->op==RTREE_GT || p->op==RTREE_EQ ); + || p->op==RTREE_GT || p->op==RTREE_EQ || p->op==RTREE_TRUE + || p->op==RTREE_FALSE ); pCellData += 8 + p->iCoord*4; assert( ((((char*)pCellData) - (char*)0)&3)==0 ); /* 4-byte aligned */ RTREE_DECODE_COORD(eInt, pCellData, xN); switch( p->op ){ - case RTREE_LE: if( xN <= p->u.rValue ) return; break; - case RTREE_LT: if( xN < p->u.rValue ) return; break; - case RTREE_GE: if( xN >= p->u.rValue ) return; break; - case RTREE_GT: if( xN > p->u.rValue ) return; break; - default: if( xN == p->u.rValue ) return; break; + case RTREE_TRUE: return; /* Always satisfied */ + case RTREE_FALSE: break; /* Never satisfied */ + case RTREE_LE: if( xN <= p->u.rValue ) return; break; + case RTREE_LT: if( xN < p->u.rValue ) return; break; + case RTREE_GE: if( xN >= p->u.rValue ) return; break; + case RTREE_GT: if( xN > p->u.rValue ) return; break; + default: if( xN == p->u.rValue ) return; break; } *peWithin = NOT_WITHIN; } @@ -171795,6 +188525,7 @@ static int nodeRowidIndex( return SQLITE_OK; } } + RTREE_IS_CORRUPT(pRtree); return SQLITE_CORRUPT_VTAB; } @@ -171888,7 +188619,7 @@ static RtreeSearchPoint *rtreeEnqueue( RtreeSearchPoint *pNew; if( pCur->nPoint>=pCur->nPointAlloc ){ int nNew = pCur->nPointAlloc*2 + 8; - pNew = sqlite3_realloc(pCur->aPoint, nNew*sizeof(pCur->aPoint[0])); + pNew = sqlite3_realloc64(pCur->aPoint, nNew*sizeof(pCur->aPoint[0])); if( pNew==0 ) return 0; pCur->aPoint = pNew; pCur->nPointAlloc = nNew; @@ -172043,13 +188774,14 @@ static int rtreeStepToLeaf(RtreeCursor *pCur){ eInt = pRtree->eCoordType==RTREE_COORD_INT32; while( (p = rtreeSearchPointFirst(pCur))!=0 && p->iLevel>0 ){ + u8 *pCellData; pNode = rtreeNodeOfFirstSearchPoint(pCur, &rc); if( rc ) return rc; nCell = NCELL(pNode); assert( nCell<200 ); + pCellData = pNode->zData + (4+pRtree->nBytesPerCell*p->iCell); while( p->iCellzData + (4+pRtree->nBytesPerCell*p->iCell); eWithin = FULLY_WITHIN; for(ii=0; iiaConstraint + ii; @@ -172062,13 +188794,23 @@ static int rtreeStepToLeaf(RtreeCursor *pCur){ }else{ rtreeNonleafConstraint(pConstraint, eInt, pCellData, &eWithin); } - if( eWithin==NOT_WITHIN ) break; + if( eWithin==NOT_WITHIN ){ + p->iCell++; + pCellData += pRtree->nBytesPerCell; + break; + } } - p->iCell++; if( eWithin==NOT_WITHIN ) continue; + p->iCell++; x.iLevel = p->iLevel - 1; if( x.iLevel ){ x.id = readInt64(pCellData); + for(ii=0; iinPoint; ii++){ + if( pCur->aPoint[ii].id==x.id ){ + RTREE_IS_CORRUPT(pRtree); + return SQLITE_CORRUPT_VTAB; + } + } x.iCell = 0; }else{ x.id = p->id; @@ -172252,17 +188994,11 @@ static int rtreeFilter( int ii; int rc = SQLITE_OK; int iCell = 0; - sqlite3_stmt *pStmt; rtreeReference(pRtree); /* Reset the cursor to the same state as rtreeOpen() leaves it in. */ - freeCursorConstraints(pCsr); - sqlite3_free(pCsr->aPoint); - pStmt = pCsr->pReadAux; - memset(pCsr, 0, sizeof(RtreeCursor)); - pCsr->base.pVtab = (sqlite3_vtab*)pRtree; - pCsr->pReadAux = pStmt; + resetCursor(pCsr); pCsr->iStrategy = idxNum; if( idxNum==1 ){ @@ -172271,7 +189007,15 @@ static int rtreeFilter( RtreeSearchPoint *p; /* Search point for the leaf */ i64 iRowid = sqlite3_value_int64(argv[0]); i64 iNode = 0; - rc = findLeafNode(pRtree, iRowid, &pLeaf, &iNode); + int eType = sqlite3_value_numeric_type(argv[0]); + if( eType==SQLITE_INTEGER + || (eType==SQLITE_FLOAT && sqlite3_value_double(argv[0])==iRowid) + ){ + rc = findLeafNode(pRtree, iRowid, &pLeaf, &iNode); + }else{ + rc = SQLITE_OK; + pLeaf = 0; + } if( rc==SQLITE_OK && pLeaf!=0 ){ p = rtreeSearchPointNew(pCsr, RTREE_ZERO, 0); assert( p!=0 ); /* Always returns pCsr->sPoint */ @@ -172290,7 +189034,7 @@ static int rtreeFilter( */ rc = nodeAcquire(pRtree, 1, 0, &pRoot); if( rc==SQLITE_OK && argc>0 ){ - pCsr->aConstraint = sqlite3_malloc(sizeof(RtreeConstraint)*argc); + pCsr->aConstraint = sqlite3_malloc64(sizeof(RtreeConstraint)*argc); pCsr->nConstraint = argc; if( !pCsr->aConstraint ){ rc = SQLITE_NOMEM; @@ -172301,6 +189045,7 @@ static int rtreeFilter( || (idxStr && (int)strlen(idxStr)==argc*2) ); for(ii=0; iiaConstraint[ii]; + int eType = sqlite3_value_numeric_type(argv[ii]); p->op = idxStr[ii*2]; p->iCoord = idxStr[ii*2+1]-'0'; if( p->op>=RTREE_MATCH ){ @@ -172315,12 +189060,21 @@ static int rtreeFilter( p->pInfo->nCoord = pRtree->nDim2; p->pInfo->anQueue = pCsr->anQueue; p->pInfo->mxLevel = pRtree->iDepth + 1; - }else{ + }else if( eType==SQLITE_INTEGER || eType==SQLITE_FLOAT ){ #ifdef SQLITE_RTREE_INT_ONLY p->u.rValue = sqlite3_value_int64(argv[ii]); #else p->u.rValue = sqlite3_value_double(argv[ii]); #endif + }else{ + p->u.rValue = RTREE_ZERO; + if( eType==SQLITE_NULL ){ + p->op = RTREE_FALSE; + }else if( p->op==RTREE_LT || p->op==RTREE_LE ){ + p->op = RTREE_TRUE; + }else{ + p->op = RTREE_FALSE; + } } } } @@ -172435,20 +189189,20 @@ static int rtreeBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ ){ u8 op; switch( p->op ){ - case SQLITE_INDEX_CONSTRAINT_EQ: op = RTREE_EQ; break; - case SQLITE_INDEX_CONSTRAINT_GT: op = RTREE_GT; break; - case SQLITE_INDEX_CONSTRAINT_LE: op = RTREE_LE; break; - case SQLITE_INDEX_CONSTRAINT_LT: op = RTREE_LT; break; - case SQLITE_INDEX_CONSTRAINT_GE: op = RTREE_GE; break; - default: - assert( p->op==SQLITE_INDEX_CONSTRAINT_MATCH ); - op = RTREE_MATCH; - break; + case SQLITE_INDEX_CONSTRAINT_EQ: op = RTREE_EQ; break; + case SQLITE_INDEX_CONSTRAINT_GT: op = RTREE_GT; break; + case SQLITE_INDEX_CONSTRAINT_LE: op = RTREE_LE; break; + case SQLITE_INDEX_CONSTRAINT_LT: op = RTREE_LT; break; + case SQLITE_INDEX_CONSTRAINT_GE: op = RTREE_GE; break; + case SQLITE_INDEX_CONSTRAINT_MATCH: op = RTREE_MATCH; break; + default: op = 0; break; + } + if( op ){ + zIdxStr[iIdx++] = op; + zIdxStr[iIdx++] = (char)(p->iColumn - 1 + '0'); + pIdxInfo->aConstraintUsage[ii].argvIndex = (iIdx/2); + pIdxInfo->aConstraintUsage[ii].omit = 1; } - zIdxStr[iIdx++] = op; - zIdxStr[iIdx++] = (char)(p->iColumn - 1 + '0'); - pIdxInfo->aConstraintUsage[ii].argvIndex = (iIdx/2); - pIdxInfo->aConstraintUsage[ii].omit = 1; } } @@ -172484,11 +189238,11 @@ static RtreeDValue cellArea(Rtree *pRtree, RtreeCell *p){ #endif { switch( pRtree->nDim ){ - case 5: area = p->aCoord[9].i - p->aCoord[8].i; - case 4: area *= p->aCoord[7].i - p->aCoord[6].i; - case 3: area *= p->aCoord[5].i - p->aCoord[4].i; - case 2: area *= p->aCoord[3].i - p->aCoord[2].i; - default: area *= p->aCoord[1].i - p->aCoord[0].i; + case 5: area = (i64)p->aCoord[9].i - (i64)p->aCoord[8].i; + case 4: area *= (i64)p->aCoord[7].i - (i64)p->aCoord[6].i; + case 3: area *= (i64)p->aCoord[5].i - (i64)p->aCoord[4].i; + case 2: area *= (i64)p->aCoord[3].i - (i64)p->aCoord[2].i; + default: area *= (i64)p->aCoord[1].i - (i64)p->aCoord[0].i; } } return area; @@ -172657,12 +189411,14 @@ static int AdjustTree( RtreeCell *pCell /* This cell was just inserted */ ){ RtreeNode *p = pNode; + int cnt = 0; while( p->pParent ){ RtreeNode *pParent = p->pParent; RtreeCell cell; int iCell; - if( nodeParentIndex(pRtree, p, &iCell) ){ + if( (++cnt)>1000 || nodeParentIndex(pRtree, p, &iCell) ){ + RTREE_IS_CORRUPT(pRtree); return SQLITE_CORRUPT_VTAB; } @@ -172859,9 +189615,9 @@ static int splitNodeStartree( int iBestSplit = 0; RtreeDValue fBestMargin = RTREE_ZERO; - int nByte = (pRtree->nDim+1)*(sizeof(int*)+nCell*sizeof(int)); + sqlite3_int64 nByte = (pRtree->nDim+1)*(sizeof(int*)+nCell*sizeof(int)); - aaSorted = (int **)sqlite3_malloc(nByte); + aaSorted = (int **)sqlite3_malloc64(nByte); if( !aaSorted ){ return SQLITE_NOMEM; } @@ -172982,7 +189738,7 @@ static int SplitNode( /* Allocate an array and populate it with a copy of pCell and ** all cells from node pLeft. Then zero the original node. */ - aCell = sqlite3_malloc((sizeof(RtreeCell)+sizeof(int))*(nCell+1)); + aCell = sqlite3_malloc64((sizeof(RtreeCell)+sizeof(int))*(nCell+1)); if( !aCell ){ rc = SQLITE_NOMEM; goto splitnode_out; @@ -173130,7 +189886,10 @@ static int fixLeafParent(Rtree *pRtree, RtreeNode *pLeaf){ } rc = sqlite3_reset(pRtree->pReadParent); if( rc==SQLITE_OK ) rc = rc2; - if( rc==SQLITE_OK && !pChild->pParent ) rc = SQLITE_CORRUPT_VTAB; + if( rc==SQLITE_OK && !pChild->pParent ){ + RTREE_IS_CORRUPT(pRtree); + rc = SQLITE_CORRUPT_VTAB; + } pChild = pChild->pParent; } return rc; @@ -173270,7 +190029,7 @@ static int Reinsert( /* Allocate the buffers used by this operation. The allocation is ** relinquished before this function returns. */ - aCell = (RtreeCell *)sqlite3_malloc(n * ( + aCell = (RtreeCell *)sqlite3_malloc64(n * ( sizeof(RtreeCell) + /* aCell array */ sizeof(int) + /* aOrder array */ sizeof(int) + /* aSpare array */ @@ -173414,7 +190173,7 @@ static int reinsertNodeContent(Rtree *pRtree, RtreeNode *pNode){ /* ** Select a currently unused rowid for a new r-tree record. */ -static int newRowid(Rtree *pRtree, i64 *piRowid){ +static int rtreeNewRowid(Rtree *pRtree, i64 *piRowid){ int rc; sqlite3_bind_null(pRtree->pWriteRowid, 1); sqlite3_bind_null(pRtree->pWriteRowid, 2); @@ -173444,8 +190203,12 @@ static int rtreeDeleteRowid(Rtree *pRtree, sqlite3_int64 iDelete){ rc = findLeafNode(pRtree, iDelete, &pLeaf, 0); } +#ifdef CORRUPT_DB + assert( pLeaf!=0 || rc!=SQLITE_OK || CORRUPT_DB ); +#endif + /* Delete the cell in question from the leaf node. */ - if( rc==SQLITE_OK ){ + if( rc==SQLITE_OK && pLeaf ){ int rc2; rc = nodeRowidIndex(pRtree, pLeaf, iDelete, &iCell); if( rc==SQLITE_OK ){ @@ -173701,7 +190464,7 @@ static int rtreeUpdate( /* Figure out the rowid of the new row. */ if( bHaveRowid==0 ){ - rc = newRowid(pRtree, &cell.iRowid); + rc = rtreeNewRowid(pRtree, &cell.iRowid); } *pRowid = cell.iRowid; @@ -173717,7 +190480,7 @@ static int rtreeUpdate( rc = rc2; } } - if( pRtree->nAux ){ + if( rc==SQLITE_OK && pRtree->nAux ){ sqlite3_stmt *pUp = pRtree->pWriteAux; int jj; sqlite3_bind_int64(pUp, 1, *pRowid); @@ -173793,7 +190556,7 @@ static int rtreeRename(sqlite3_vtab *pVtab, const char *zNewName){ */ static int rtreeSavepoint(sqlite3_vtab *pVtab, int iSavepoint){ Rtree *pRtree = (Rtree *)pVtab; - int iwt = pRtree->inWrTrans; + u8 iwt = pRtree->inWrTrans; UNUSED_PARAMETER(iSavepoint); pRtree->inWrTrans = 0; nodeBlobReset(pRtree); @@ -173845,8 +190608,24 @@ static int rtreeQueryStat1(sqlite3 *db, Rtree *pRtree){ return rc; } + +/* +** Return true if zName is the extension on one of the shadow tables used +** by this module. +*/ +static int rtreeShadowName(const char *zName){ + static const char *azName[] = { + "node", "parent", "rowid" + }; + unsigned int i; + for(i=0; idb = db; @@ -173954,8 +190735,7 @@ static int rtreeSqlInit( } zSql = sqlite3_mprintf(zFormat, zDb, zPrefix); if( zSql ){ - rc = sqlite3_prepare_v3(db, zSql, -1, SQLITE_PREPARE_PERSISTENT, - appStmt[i], 0); + rc = sqlite3_prepare_v3(db, zSql, -1, f, appStmt[i], 0); }else{ rc = SQLITE_NOMEM; } @@ -173974,15 +190754,18 @@ static int rtreeSqlInit( sqlite3_str_appendf(p, "UPDATE \"%w\".\"%w_rowid\"SET ", zDb, zPrefix); for(ii=0; iinAux; ii++){ if( ii ) sqlite3_str_append(p, ",", 1); - sqlite3_str_appendf(p,"a%d=?%d",ii,ii+2); + if( iinAuxNotNull ){ + sqlite3_str_appendf(p,"a%d=coalesce(?%d,a%d)",ii,ii+2,ii); + }else{ + sqlite3_str_appendf(p,"a%d=?%d",ii,ii+2); + } } sqlite3_str_appendf(p, " WHERE rowid=?1"); zSql = sqlite3_str_finish(p); if( zSql==0 ){ rc = SQLITE_NOMEM; }else{ - rc = sqlite3_prepare_v3(db, zSql, -1, SQLITE_PREPARE_PERSISTENT, - &pRtree->pWriteAux, 0); + rc = sqlite3_prepare_v3(db, zSql, -1, f, &pRtree->pWriteAux, 0); sqlite3_free(zSql); } } @@ -174058,6 +190841,7 @@ static int getNodeSize( *pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(db)); }else if( pRtree->iNodeSize<(512-64) ){ rc = SQLITE_CORRUPT_VTAB; + RTREE_IS_CORRUPT(pRtree); *pzErr = sqlite3_mprintf("undersize RTree blobs in \"%q_node\"", pRtree->zName); } @@ -174067,6 +190851,14 @@ static int getNodeSize( return rc; } +/* +** Return the length of a token +*/ +static int rtreeTokenLength(const char *z){ + int dummy = 0; + return sqlite3GetToken((const unsigned char*)z,&dummy); +} + /* ** This function is the implementation of both the xConnect and xCreate ** methods of the r-tree virtual table. @@ -174103,8 +190895,8 @@ static int rtreeInit( }; assert( RTREE_MAX_AUX_COLUMN<256 ); /* Aux columns counted by a u8 */ - if( argc>RTREE_MAX_AUX_COLUMN+3 ){ - *pzErr = sqlite3_mprintf("%s", aErrMsg[3]); + if( argc<6 || argc>RTREE_MAX_AUX_COLUMN+3 ){ + *pzErr = sqlite3_mprintf("%s", aErrMsg[2 + (argc>=6)]); return SQLITE_ERROR; } @@ -174113,7 +190905,7 @@ static int rtreeInit( /* Allocate the sqlite3_vtab structure */ nDb = (int)strlen(argv[1]); nName = (int)strlen(argv[2]); - pRtree = (Rtree *)sqlite3_malloc(sizeof(Rtree)+nDb+nName+2); + pRtree = (Rtree *)sqlite3_malloc64(sizeof(Rtree)+nDb+nName+2); if( !pRtree ){ return SQLITE_NOMEM; } @@ -174132,16 +190924,20 @@ static int rtreeInit( ** the r-tree table schema. */ pSql = sqlite3_str_new(db); - sqlite3_str_appendf(pSql, "CREATE TABLE x(%s", argv[3]); + sqlite3_str_appendf(pSql, "CREATE TABLE x(%.*s INT", + rtreeTokenLength(argv[3]), argv[3]); for(ii=4; iinAux++; - sqlite3_str_appendf(pSql, ",%s", argv[ii]+1); + sqlite3_str_appendf(pSql, ",%.*s", rtreeTokenLength(zArg+1), zArg+1); }else if( pRtree->nAux>0 ){ break; }else{ + static const char *azFormat[] = {",%.*s REAL", ",%.*s INT"}; pRtree->nDim2++; - sqlite3_str_appendf(pSql, ",%s", argv[ii]); + sqlite3_str_appendf(pSql, azFormat[eCoordType], + rtreeTokenLength(zArg), zArg); } } sqlite3_str_appendf(pSql, ");"); @@ -174210,49 +191006,45 @@ static int rtreeInit( ** *2 coordinates. */ static void rtreenode(sqlite3_context *ctx, int nArg, sqlite3_value **apArg){ - char *zText = 0; RtreeNode node; Rtree tree; int ii; + int nData; + int errCode; + sqlite3_str *pOut; UNUSED_PARAMETER(nArg); memset(&node, 0, sizeof(RtreeNode)); memset(&tree, 0, sizeof(Rtree)); tree.nDim = (u8)sqlite3_value_int(apArg[0]); + if( tree.nDim<1 || tree.nDim>5 ) return; tree.nDim2 = tree.nDim*2; tree.nBytesPerCell = 8 + 8 * tree.nDim; node.zData = (u8 *)sqlite3_value_blob(apArg[1]); + nData = sqlite3_value_bytes(apArg[1]); + if( nData<4 ) return; + if( nData0 ) sqlite3_str_append(pOut, " ", 1); + sqlite3_str_appendf(pOut, "{%lld", cell.iRowid); for(jj=0; jjrc==SQLITE_OK ); - if( pCheck->pGetNode==0 ){ + if( pCheck->rc==SQLITE_OK && pCheck->pGetNode==0 ){ pCheck->pGetNode = rtreeCheckPrepare(pCheck, "SELECT data FROM %Q.'%q_node' WHERE nodeno=?", pCheck->zDb, pCheck->zTab @@ -174394,7 +191185,7 @@ static u8 *rtreeCheckGetNode(RtreeCheck *pCheck, i64 iNode, int *pnNode){ if( sqlite3_step(pCheck->pGetNode)==SQLITE_ROW ){ int nNode = sqlite3_column_bytes(pCheck->pGetNode, 0); const u8 *pNode = (const u8*)sqlite3_column_blob(pCheck->pGetNode, 0); - pRet = sqlite3_malloc(nNode); + pRet = sqlite3_malloc64(nNode); if( pRet==0 ){ pCheck->rc = SQLITE_NOMEM; }else{ @@ -174737,12 +191528,1828 @@ static void rtreecheck( if( rc==SQLITE_OK ){ sqlite3_result_text(ctx, zReport ? zReport : "ok", -1, SQLITE_TRANSIENT); }else{ - sqlite3_result_error_code(ctx, rc); + sqlite3_result_error_code(ctx, rc); + } + sqlite3_free(zReport); + } +} + +/* Conditionally include the geopoly code */ +#ifdef SQLITE_ENABLE_GEOPOLY +/************** Include geopoly.c in the middle of rtree.c *******************/ +/************** Begin file geopoly.c *****************************************/ +/* +** 2018-05-25 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file implements an alternative R-Tree virtual table that +** uses polygons to express the boundaries of 2-dimensional objects. +** +** This file is #include-ed onto the end of "rtree.c" so that it has +** access to all of the R-Tree internals. +*/ +/* #include */ + +/* Enable -DGEOPOLY_ENABLE_DEBUG for debugging facilities */ +#ifdef GEOPOLY_ENABLE_DEBUG + static int geo_debug = 0; +# define GEODEBUG(X) if(geo_debug)printf X +#else +# define GEODEBUG(X) +#endif + +#ifndef JSON_NULL /* The following stuff repeats things found in json1 */ +/* +** Versions of isspace(), isalnum() and isdigit() to which it is safe +** to pass signed char values. +*/ +#ifdef sqlite3Isdigit + /* Use the SQLite core versions if this routine is part of the + ** SQLite amalgamation */ +# define safe_isdigit(x) sqlite3Isdigit(x) +# define safe_isalnum(x) sqlite3Isalnum(x) +# define safe_isxdigit(x) sqlite3Isxdigit(x) +#else + /* Use the standard library for separate compilation */ +#include /* amalgamator: keep */ +# define safe_isdigit(x) isdigit((unsigned char)(x)) +# define safe_isalnum(x) isalnum((unsigned char)(x)) +# define safe_isxdigit(x) isxdigit((unsigned char)(x)) +#endif + +/* +** Growing our own isspace() routine this way is twice as fast as +** the library isspace() function. +*/ +static const char geopolyIsSpace[] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +}; +#define safe_isspace(x) (geopolyIsSpace[(unsigned char)x]) +#endif /* JSON NULL - back to original code */ + +/* Compiler and version */ +#ifndef GCC_VERSION +#if defined(__GNUC__) && !defined(SQLITE_DISABLE_INTRINSIC) +# define GCC_VERSION (__GNUC__*1000000+__GNUC_MINOR__*1000+__GNUC_PATCHLEVEL__) +#else +# define GCC_VERSION 0 +#endif +#endif +#ifndef MSVC_VERSION +#if defined(_MSC_VER) && !defined(SQLITE_DISABLE_INTRINSIC) +# define MSVC_VERSION _MSC_VER +#else +# define MSVC_VERSION 0 +#endif +#endif + +/* Datatype for coordinates +*/ +typedef float GeoCoord; + +/* +** Internal representation of a polygon. +** +** The polygon consists of a sequence of vertexes. There is a line +** segment between each pair of vertexes, and one final segment from +** the last vertex back to the first. (This differs from the GeoJSON +** standard in which the final vertex is a repeat of the first.) +** +** The polygon follows the right-hand rule. The area to the right of +** each segment is "outside" and the area to the left is "inside". +** +** The on-disk representation consists of a 4-byte header followed by +** the values. The 4-byte header is: +** +** encoding (1 byte) 0=big-endian, 1=little-endian +** nvertex (3 bytes) Number of vertexes as a big-endian integer +** +** Enough space is allocated for 4 coordinates, to work around over-zealous +** warnings coming from some compiler (notably, clang). In reality, the size +** of each GeoPoly memory allocate is adjusted as necessary so that the +** GeoPoly.a[] array at the end is the appropriate size. +*/ +typedef struct GeoPoly GeoPoly; +struct GeoPoly { + int nVertex; /* Number of vertexes */ + unsigned char hdr[4]; /* Header for on-disk representation */ + GeoCoord a[8]; /* 2*nVertex values. X (longitude) first, then Y */ +}; + +/* The size of a memory allocation needed for a GeoPoly object sufficient +** to hold N coordinate pairs. +*/ +#define GEOPOLY_SZ(N) (sizeof(GeoPoly) + sizeof(GeoCoord)*2*((N)-4)) + +/* Macros to access coordinates of a GeoPoly. +** We have to use these macros, rather than just say p->a[i] in order +** to silence (incorrect) UBSAN warnings if the array index is too large. +*/ +#define GeoX(P,I) (((GeoCoord*)(P)->a)[(I)*2]) +#define GeoY(P,I) (((GeoCoord*)(P)->a)[(I)*2+1]) + + +/* +** State of a parse of a GeoJSON input. +*/ +typedef struct GeoParse GeoParse; +struct GeoParse { + const unsigned char *z; /* Unparsed input */ + int nVertex; /* Number of vertexes in a[] */ + int nAlloc; /* Space allocated to a[] */ + int nErr; /* Number of errors encountered */ + GeoCoord *a; /* Array of vertexes. From sqlite3_malloc64() */ +}; + +/* Do a 4-byte byte swap */ +static void geopolySwab32(unsigned char *a){ + unsigned char t = a[0]; + a[0] = a[3]; + a[3] = t; + t = a[1]; + a[1] = a[2]; + a[2] = t; +} + +/* Skip whitespace. Return the next non-whitespace character. */ +static char geopolySkipSpace(GeoParse *p){ + while( safe_isspace(p->z[0]) ) p->z++; + return p->z[0]; +} + +/* Parse out a number. Write the value into *pVal if pVal!=0. +** return non-zero on success and zero if the next token is not a number. +*/ +static int geopolyParseNumber(GeoParse *p, GeoCoord *pVal){ + char c = geopolySkipSpace(p); + const unsigned char *z = p->z; + int j = 0; + int seenDP = 0; + int seenE = 0; + if( c=='-' ){ + j = 1; + c = z[j]; + } + if( c=='0' && z[j+1]>='0' && z[j+1]<='9' ) return 0; + for(;; j++){ + c = z[j]; + if( safe_isdigit(c) ) continue; + if( c=='.' ){ + if( z[j-1]=='-' ) return 0; + if( seenDP ) return 0; + seenDP = 1; + continue; + } + if( c=='e' || c=='E' ){ + if( z[j-1]<'0' ) return 0; + if( seenE ) return -1; + seenDP = seenE = 1; + c = z[j+1]; + if( c=='+' || c=='-' ){ + j++; + c = z[j+1]; + } + if( c<'0' || c>'9' ) return 0; + continue; + } + break; + } + if( z[j-1]<'0' ) return 0; + if( pVal ){ +#ifdef SQLITE_AMALGAMATION + /* The sqlite3AtoF() routine is much much faster than atof(), if it + ** is available */ + double r; + (void)sqlite3AtoF((const char*)p->z, &r, j, SQLITE_UTF8); + *pVal = r; +#else + *pVal = (GeoCoord)atof((const char*)p->z); +#endif + } + p->z += j; + return 1; +} + +/* +** If the input is a well-formed JSON array of coordinates with at least +** four coordinates and where each coordinate is itself a two-value array, +** then convert the JSON into a GeoPoly object and return a pointer to +** that object. +** +** If any error occurs, return NULL. +*/ +static GeoPoly *geopolyParseJson(const unsigned char *z, int *pRc){ + GeoParse s; + int rc = SQLITE_OK; + memset(&s, 0, sizeof(s)); + s.z = z; + if( geopolySkipSpace(&s)=='[' ){ + s.z++; + while( geopolySkipSpace(&s)=='[' ){ + int ii = 0; + char c; + s.z++; + if( s.nVertex>=s.nAlloc ){ + GeoCoord *aNew; + s.nAlloc = s.nAlloc*2 + 16; + aNew = sqlite3_realloc64(s.a, s.nAlloc*sizeof(GeoCoord)*2 ); + if( aNew==0 ){ + rc = SQLITE_NOMEM; + s.nErr++; + break; + } + s.a = aNew; + } + while( geopolyParseNumber(&s, ii<=1 ? &s.a[s.nVertex*2+ii] : 0) ){ + ii++; + if( ii==2 ) s.nVertex++; + c = geopolySkipSpace(&s); + s.z++; + if( c==',' ) continue; + if( c==']' && ii>=2 ) break; + s.nErr++; + rc = SQLITE_ERROR; + goto parse_json_err; + } + if( geopolySkipSpace(&s)==',' ){ + s.z++; + continue; + } + break; + } + if( geopolySkipSpace(&s)==']' + && s.nVertex>=4 + && s.a[0]==s.a[s.nVertex*2-2] + && s.a[1]==s.a[s.nVertex*2-1] + && (s.z++, geopolySkipSpace(&s)==0) + ){ + GeoPoly *pOut; + int x = 1; + s.nVertex--; /* Remove the redundant vertex at the end */ + pOut = sqlite3_malloc64( GEOPOLY_SZ((sqlite3_int64)s.nVertex) ); + x = 1; + if( pOut==0 ) goto parse_json_err; + pOut->nVertex = s.nVertex; + memcpy(pOut->a, s.a, s.nVertex*2*sizeof(GeoCoord)); + pOut->hdr[0] = *(unsigned char*)&x; + pOut->hdr[1] = (s.nVertex>>16)&0xff; + pOut->hdr[2] = (s.nVertex>>8)&0xff; + pOut->hdr[3] = s.nVertex&0xff; + sqlite3_free(s.a); + if( pRc ) *pRc = SQLITE_OK; + return pOut; + }else{ + s.nErr++; + rc = SQLITE_ERROR; + } + } +parse_json_err: + if( pRc ) *pRc = rc; + sqlite3_free(s.a); + return 0; +} + +/* +** Given a function parameter, try to interpret it as a polygon, either +** in the binary format or JSON text. Compute a GeoPoly object and +** return a pointer to that object. Or if the input is not a well-formed +** polygon, put an error message in sqlite3_context and return NULL. +*/ +static GeoPoly *geopolyFuncParam( + sqlite3_context *pCtx, /* Context for error messages */ + sqlite3_value *pVal, /* The value to decode */ + int *pRc /* Write error here */ +){ + GeoPoly *p = 0; + int nByte; + if( sqlite3_value_type(pVal)==SQLITE_BLOB + && (nByte = sqlite3_value_bytes(pVal))>=(4+6*sizeof(GeoCoord)) + ){ + const unsigned char *a = sqlite3_value_blob(pVal); + int nVertex; + nVertex = (a[1]<<16) + (a[2]<<8) + a[3]; + if( (a[0]==0 || a[0]==1) + && (nVertex*2*sizeof(GeoCoord) + 4)==(unsigned int)nByte + ){ + p = sqlite3_malloc64( sizeof(*p) + (nVertex-1)*2*sizeof(GeoCoord) ); + if( p==0 ){ + if( pRc ) *pRc = SQLITE_NOMEM; + if( pCtx ) sqlite3_result_error_nomem(pCtx); + }else{ + int x = 1; + p->nVertex = nVertex; + memcpy(p->hdr, a, nByte); + if( a[0] != *(unsigned char*)&x ){ + int ii; + for(ii=0; iihdr[0] ^= 1; + } + } + } + if( pRc ) *pRc = SQLITE_OK; + return p; + }else if( sqlite3_value_type(pVal)==SQLITE_TEXT ){ + const unsigned char *zJson = sqlite3_value_text(pVal); + if( zJson==0 ){ + if( pRc ) *pRc = SQLITE_NOMEM; + return 0; + } + return geopolyParseJson(zJson, pRc); + }else{ + if( pRc ) *pRc = SQLITE_ERROR; + return 0; + } +} + +/* +** Implementation of the geopoly_blob(X) function. +** +** If the input is a well-formed Geopoly BLOB or JSON string +** then return the BLOB representation of the polygon. Otherwise +** return NULL. +*/ +static void geopolyBlobFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + GeoPoly *p = geopolyFuncParam(context, argv[0], 0); + if( p ){ + sqlite3_result_blob(context, p->hdr, + 4+8*p->nVertex, SQLITE_TRANSIENT); + sqlite3_free(p); + } +} + +/* +** SQL function: geopoly_json(X) +** +** Interpret X as a polygon and render it as a JSON array +** of coordinates. Or, if X is not a valid polygon, return NULL. +*/ +static void geopolyJsonFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + GeoPoly *p = geopolyFuncParam(context, argv[0], 0); + if( p ){ + sqlite3 *db = sqlite3_context_db_handle(context); + sqlite3_str *x = sqlite3_str_new(db); + int i; + sqlite3_str_append(x, "[", 1); + for(i=0; inVertex; i++){ + sqlite3_str_appendf(x, "[%!g,%!g],", GeoX(p,i), GeoY(p,i)); + } + sqlite3_str_appendf(x, "[%!g,%!g]]", GeoX(p,0), GeoY(p,0)); + sqlite3_result_text(context, sqlite3_str_finish(x), -1, sqlite3_free); + sqlite3_free(p); + } +} + +/* +** SQL function: geopoly_svg(X, ....) +** +** Interpret X as a polygon and render it as a SVG . +** Additional arguments are added as attributes to the . +*/ +static void geopolySvgFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + GeoPoly *p; + if( argc<1 ) return; + p = geopolyFuncParam(context, argv[0], 0); + if( p ){ + sqlite3 *db = sqlite3_context_db_handle(context); + sqlite3_str *x = sqlite3_str_new(db); + int i; + char cSep = '\''; + sqlite3_str_appendf(x, ""); + sqlite3_result_text(context, sqlite3_str_finish(x), -1, sqlite3_free); + sqlite3_free(p); + } +} + +/* +** SQL Function: geopoly_xform(poly, A, B, C, D, E, F) +** +** Transform and/or translate a polygon as follows: +** +** x1 = A*x0 + B*y0 + E +** y1 = C*x0 + D*y0 + F +** +** For a translation: +** +** geopoly_xform(poly, 1, 0, 0, 1, x-offset, y-offset) +** +** Rotate by R around the point (0,0): +** +** geopoly_xform(poly, cos(R), sin(R), -sin(R), cos(R), 0, 0) +*/ +static void geopolyXformFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + GeoPoly *p = geopolyFuncParam(context, argv[0], 0); + double A = sqlite3_value_double(argv[1]); + double B = sqlite3_value_double(argv[2]); + double C = sqlite3_value_double(argv[3]); + double D = sqlite3_value_double(argv[4]); + double E = sqlite3_value_double(argv[5]); + double F = sqlite3_value_double(argv[6]); + GeoCoord x1, y1, x0, y0; + int ii; + if( p ){ + for(ii=0; iinVertex; ii++){ + x0 = GeoX(p,ii); + y0 = GeoY(p,ii); + x1 = (GeoCoord)(A*x0 + B*y0 + E); + y1 = (GeoCoord)(C*x0 + D*y0 + F); + GeoX(p,ii) = x1; + GeoY(p,ii) = y1; + } + sqlite3_result_blob(context, p->hdr, + 4+8*p->nVertex, SQLITE_TRANSIENT); + sqlite3_free(p); + } +} + +/* +** Compute the area enclosed by the polygon. +** +** This routine can also be used to detect polygons that rotate in +** the wrong direction. Polygons are suppose to be counter-clockwise (CCW). +** This routine returns a negative value for clockwise (CW) polygons. +*/ +static double geopolyArea(GeoPoly *p){ + double rArea = 0.0; + int ii; + for(ii=0; iinVertex-1; ii++){ + rArea += (GeoX(p,ii) - GeoX(p,ii+1)) /* (x0 - x1) */ + * (GeoY(p,ii) + GeoY(p,ii+1)) /* (y0 + y1) */ + * 0.5; + } + rArea += (GeoX(p,ii) - GeoX(p,0)) /* (xN - x0) */ + * (GeoY(p,ii) + GeoY(p,0)) /* (yN + y0) */ + * 0.5; + return rArea; +} + +/* +** Implementation of the geopoly_area(X) function. +** +** If the input is a well-formed Geopoly BLOB then return the area +** enclosed by the polygon. If the polygon circulates clockwise instead +** of counterclockwise (as it should) then return the negative of the +** enclosed area. Otherwise return NULL. +*/ +static void geopolyAreaFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + GeoPoly *p = geopolyFuncParam(context, argv[0], 0); + if( p ){ + sqlite3_result_double(context, geopolyArea(p)); + sqlite3_free(p); + } +} + +/* +** Implementation of the geopoly_ccw(X) function. +** +** If the rotation of polygon X is clockwise (incorrect) instead of +** counter-clockwise (the correct winding order according to RFC7946) +** then reverse the order of the vertexes in polygon X. +** +** In other words, this routine returns a CCW polygon regardless of the +** winding order of its input. +** +** Use this routine to sanitize historical inputs that that sometimes +** contain polygons that wind in the wrong direction. +*/ +static void geopolyCcwFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + GeoPoly *p = geopolyFuncParam(context, argv[0], 0); + if( p ){ + if( geopolyArea(p)<0.0 ){ + int ii, jj; + for(ii=1, jj=p->nVertex-1; iihdr, + 4+8*p->nVertex, SQLITE_TRANSIENT); + sqlite3_free(p); + } +} + +#define GEOPOLY_PI 3.1415926535897932385 + +/* Fast approximation for sine(X) for X between -0.5*pi and 2*pi +*/ +static double geopolySine(double r){ + assert( r>=-0.5*GEOPOLY_PI && r<=2.0*GEOPOLY_PI ); + if( r>=1.5*GEOPOLY_PI ){ + r -= 2.0*GEOPOLY_PI; + } + if( r>=0.5*GEOPOLY_PI ){ + return -geopolySine(r-GEOPOLY_PI); + }else{ + double r2 = r*r; + double r3 = r2*r; + double r5 = r3*r2; + return 0.9996949*r - 0.1656700*r3 + 0.0075134*r5; + } +} + +/* +** Function: geopoly_regular(X,Y,R,N) +** +** Construct a simple, convex, regular polygon centered at X, Y +** with circumradius R and with N sides. +*/ +static void geopolyRegularFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + double x = sqlite3_value_double(argv[0]); + double y = sqlite3_value_double(argv[1]); + double r = sqlite3_value_double(argv[2]); + int n = sqlite3_value_int(argv[3]); + int i; + GeoPoly *p; + + if( n<3 || r<=0.0 ) return; + if( n>1000 ) n = 1000; + p = sqlite3_malloc64( sizeof(*p) + (n-1)*2*sizeof(GeoCoord) ); + if( p==0 ){ + sqlite3_result_error_nomem(context); + return; + } + i = 1; + p->hdr[0] = *(unsigned char*)&i; + p->hdr[1] = 0; + p->hdr[2] = (n>>8)&0xff; + p->hdr[3] = n&0xff; + for(i=0; ihdr, 4+8*n, SQLITE_TRANSIENT); + sqlite3_free(p); +} + +/* +** If pPoly is a polygon, compute its bounding box. Then: +** +** (1) if aCoord!=0 store the bounding box in aCoord, returning NULL +** (2) otherwise, compute a GeoPoly for the bounding box and return the +** new GeoPoly +** +** If pPoly is NULL but aCoord is not NULL, then compute a new GeoPoly from +** the bounding box in aCoord and return a pointer to that GeoPoly. +*/ +static GeoPoly *geopolyBBox( + sqlite3_context *context, /* For recording the error */ + sqlite3_value *pPoly, /* The polygon */ + RtreeCoord *aCoord, /* Results here */ + int *pRc /* Error code here */ +){ + GeoPoly *pOut = 0; + GeoPoly *p; + float mnX, mxX, mnY, mxY; + if( pPoly==0 && aCoord!=0 ){ + p = 0; + mnX = aCoord[0].f; + mxX = aCoord[1].f; + mnY = aCoord[2].f; + mxY = aCoord[3].f; + goto geopolyBboxFill; + }else{ + p = geopolyFuncParam(context, pPoly, pRc); + } + if( p ){ + int ii; + mnX = mxX = GeoX(p,0); + mnY = mxY = GeoY(p,0); + for(ii=1; iinVertex; ii++){ + double r = GeoX(p,ii); + if( rmxX ) mxX = (float)r; + r = GeoY(p,ii); + if( rmxY ) mxY = (float)r; + } + if( pRc ) *pRc = SQLITE_OK; + if( aCoord==0 ){ + geopolyBboxFill: + pOut = sqlite3_realloc64(p, GEOPOLY_SZ(4)); + if( pOut==0 ){ + sqlite3_free(p); + if( context ) sqlite3_result_error_nomem(context); + if( pRc ) *pRc = SQLITE_NOMEM; + return 0; + } + pOut->nVertex = 4; + ii = 1; + pOut->hdr[0] = *(unsigned char*)ⅈ + pOut->hdr[1] = 0; + pOut->hdr[2] = 0; + pOut->hdr[3] = 4; + GeoX(pOut,0) = mnX; + GeoY(pOut,0) = mnY; + GeoX(pOut,1) = mxX; + GeoY(pOut,1) = mnY; + GeoX(pOut,2) = mxX; + GeoY(pOut,2) = mxY; + GeoX(pOut,3) = mnX; + GeoY(pOut,3) = mxY; + }else{ + sqlite3_free(p); + aCoord[0].f = mnX; + aCoord[1].f = mxX; + aCoord[2].f = mnY; + aCoord[3].f = mxY; + } + } + return pOut; +} + +/* +** Implementation of the geopoly_bbox(X) SQL function. +*/ +static void geopolyBBoxFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + GeoPoly *p = geopolyBBox(context, argv[0], 0, 0); + if( p ){ + sqlite3_result_blob(context, p->hdr, + 4+8*p->nVertex, SQLITE_TRANSIENT); + sqlite3_free(p); + } +} + +/* +** State vector for the geopoly_group_bbox() aggregate function. +*/ +typedef struct GeoBBox GeoBBox; +struct GeoBBox { + int isInit; + RtreeCoord a[4]; +}; + + +/* +** Implementation of the geopoly_group_bbox(X) aggregate SQL function. +*/ +static void geopolyBBoxStep( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + RtreeCoord a[4]; + int rc = SQLITE_OK; + (void)geopolyBBox(context, argv[0], a, &rc); + if( rc==SQLITE_OK ){ + GeoBBox *pBBox; + pBBox = (GeoBBox*)sqlite3_aggregate_context(context, sizeof(*pBBox)); + if( pBBox==0 ) return; + if( pBBox->isInit==0 ){ + pBBox->isInit = 1; + memcpy(pBBox->a, a, sizeof(RtreeCoord)*4); + }else{ + if( a[0].f < pBBox->a[0].f ) pBBox->a[0] = a[0]; + if( a[1].f > pBBox->a[1].f ) pBBox->a[1] = a[1]; + if( a[2].f < pBBox->a[2].f ) pBBox->a[2] = a[2]; + if( a[3].f > pBBox->a[3].f ) pBBox->a[3] = a[3]; + } + } +} +static void geopolyBBoxFinal( + sqlite3_context *context +){ + GeoPoly *p; + GeoBBox *pBBox; + pBBox = (GeoBBox*)sqlite3_aggregate_context(context, 0); + if( pBBox==0 ) return; + p = geopolyBBox(context, 0, pBBox->a, 0); + if( p ){ + sqlite3_result_blob(context, p->hdr, + 4+8*p->nVertex, SQLITE_TRANSIENT); + sqlite3_free(p); + } +} + + +/* +** Determine if point (x0,y0) is beneath line segment (x1,y1)->(x2,y2). +** Returns: +** +** +2 x0,y0 is on the line segement +** +** +1 x0,y0 is beneath line segment +** +** 0 x0,y0 is not on or beneath the line segment or the line segment +** is vertical and x0,y0 is not on the line segment +** +** The left-most coordinate min(x1,x2) is not considered to be part of +** the line segment for the purposes of this analysis. +*/ +static int pointBeneathLine( + double x0, double y0, + double x1, double y1, + double x2, double y2 +){ + double y; + if( x0==x1 && y0==y1 ) return 2; + if( x1x2 ) return 0; + }else if( x1>x2 ){ + if( x0<=x2 || x0>x1 ) return 0; + }else{ + /* Vertical line segment */ + if( x0!=x1 ) return 0; + if( y0y1 && y0>y2 ) return 0; + return 2; + } + y = y1 + (y2-y1)*(x0-x1)/(x2-x1); + if( y0==y ) return 2; + if( y0nVertex-1; ii++){ + v = pointBeneathLine(x0,y0,GeoX(p1,ii), GeoY(p1,ii), + GeoX(p1,ii+1),GeoY(p1,ii+1)); + if( v==2 ) break; + cnt += v; + } + if( v!=2 ){ + v = pointBeneathLine(x0,y0,GeoX(p1,ii), GeoY(p1,ii), + GeoX(p1,0), GeoY(p1,0)); + } + if( v==2 ){ + sqlite3_result_int(context, 1); + }else if( ((v+cnt)&1)==0 ){ + sqlite3_result_int(context, 0); + }else{ + sqlite3_result_int(context, 2); + } + sqlite3_free(p1); +} + +/* Forward declaration */ +static int geopolyOverlap(GeoPoly *p1, GeoPoly *p2); + +/* +** SQL function: geopoly_within(P1,P2) +** +** Return +2 if P1 and P2 are the same polygon +** Return +1 if P2 is contained within P1 +** Return 0 if any part of P2 is on the outside of P1 +** +*/ +static void geopolyWithinFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + GeoPoly *p1 = geopolyFuncParam(context, argv[0], 0); + GeoPoly *p2 = geopolyFuncParam(context, argv[1], 0); + if( p1 && p2 ){ + int x = geopolyOverlap(p1, p2); + if( x<0 ){ + sqlite3_result_error_nomem(context); + }else{ + sqlite3_result_int(context, x==2 ? 1 : x==4 ? 2 : 0); + } + } + sqlite3_free(p1); + sqlite3_free(p2); +} + +/* Objects used by the overlap algorihm. */ +typedef struct GeoEvent GeoEvent; +typedef struct GeoSegment GeoSegment; +typedef struct GeoOverlap GeoOverlap; +struct GeoEvent { + double x; /* X coordinate at which event occurs */ + int eType; /* 0 for ADD, 1 for REMOVE */ + GeoSegment *pSeg; /* The segment to be added or removed */ + GeoEvent *pNext; /* Next event in the sorted list */ +}; +struct GeoSegment { + double C, B; /* y = C*x + B */ + double y; /* Current y value */ + float y0; /* Initial y value */ + unsigned char side; /* 1 for p1, 2 for p2 */ + unsigned int idx; /* Which segment within the side */ + GeoSegment *pNext; /* Next segment in a list sorted by y */ +}; +struct GeoOverlap { + GeoEvent *aEvent; /* Array of all events */ + GeoSegment *aSegment; /* Array of all segments */ + int nEvent; /* Number of events */ + int nSegment; /* Number of segments */ +}; + +/* +** Add a single segment and its associated events. +*/ +static void geopolyAddOneSegment( + GeoOverlap *p, + GeoCoord x0, + GeoCoord y0, + GeoCoord x1, + GeoCoord y1, + unsigned char side, + unsigned int idx +){ + GeoSegment *pSeg; + GeoEvent *pEvent; + if( x0==x1 ) return; /* Ignore vertical segments */ + if( x0>x1 ){ + GeoCoord t = x0; + x0 = x1; + x1 = t; + t = y0; + y0 = y1; + y1 = t; + } + pSeg = p->aSegment + p->nSegment; + p->nSegment++; + pSeg->C = (y1-y0)/(x1-x0); + pSeg->B = y1 - x1*pSeg->C; + pSeg->y0 = y0; + pSeg->side = side; + pSeg->idx = idx; + pEvent = p->aEvent + p->nEvent; + p->nEvent++; + pEvent->x = x0; + pEvent->eType = 0; + pEvent->pSeg = pSeg; + pEvent = p->aEvent + p->nEvent; + p->nEvent++; + pEvent->x = x1; + pEvent->eType = 1; + pEvent->pSeg = pSeg; +} + + + +/* +** Insert all segments and events for polygon pPoly. +*/ +static void geopolyAddSegments( + GeoOverlap *p, /* Add segments to this Overlap object */ + GeoPoly *pPoly, /* Take all segments from this polygon */ + unsigned char side /* The side of pPoly */ +){ + unsigned int i; + GeoCoord *x; + for(i=0; i<(unsigned)pPoly->nVertex-1; i++){ + x = &GeoX(pPoly,i); + geopolyAddOneSegment(p, x[0], x[1], x[2], x[3], side, i); + } + x = &GeoX(pPoly,i); + geopolyAddOneSegment(p, x[0], x[1], pPoly->a[0], pPoly->a[1], side, i); +} + +/* +** Merge two lists of sorted events by X coordinate +*/ +static GeoEvent *geopolyEventMerge(GeoEvent *pLeft, GeoEvent *pRight){ + GeoEvent head, *pLast; + head.pNext = 0; + pLast = &head; + while( pRight && pLeft ){ + if( pRight->x <= pLeft->x ){ + pLast->pNext = pRight; + pLast = pRight; + pRight = pRight->pNext; + }else{ + pLast->pNext = pLeft; + pLast = pLeft; + pLeft = pLeft->pNext; + } + } + pLast->pNext = pRight ? pRight : pLeft; + return head.pNext; +} + +/* +** Sort an array of nEvent event objects into a list. +*/ +static GeoEvent *geopolySortEventsByX(GeoEvent *aEvent, int nEvent){ + int mx = 0; + int i, j; + GeoEvent *p; + GeoEvent *a[50]; + for(i=0; ipNext = 0; + for(j=0; j=mx ) mx = j+1; + } + p = 0; + for(i=0; iy - pLeft->y; + if( r==0.0 ) r = pRight->C - pLeft->C; + if( r<0.0 ){ + pLast->pNext = pRight; + pLast = pRight; + pRight = pRight->pNext; + }else{ + pLast->pNext = pLeft; + pLast = pLeft; + pLeft = pLeft->pNext; + } + } + pLast->pNext = pRight ? pRight : pLeft; + return head.pNext; +} + +/* +** Sort a list of GeoSegments in order of increasing Y and in the event of +** a tie, increasing C (slope). +*/ +static GeoSegment *geopolySortSegmentsByYAndC(GeoSegment *pList){ + int mx = 0; + int i; + GeoSegment *p; + GeoSegment *a[50]; + while( pList ){ + p = pList; + pList = pList->pNext; + p->pNext = 0; + for(i=0; i=mx ) mx = i+1; + } + p = 0; + for(i=0; inVertex + p2->nVertex + 2; + GeoOverlap *p; + sqlite3_int64 nByte; + GeoEvent *pThisEvent; + double rX; + int rc = 0; + int needSort = 0; + GeoSegment *pActive = 0; + GeoSegment *pSeg; + unsigned char aOverlap[4]; + + nByte = sizeof(GeoEvent)*nVertex*2 + + sizeof(GeoSegment)*nVertex + + sizeof(GeoOverlap); + p = sqlite3_malloc64( nByte ); + if( p==0 ) return -1; + p->aEvent = (GeoEvent*)&p[1]; + p->aSegment = (GeoSegment*)&p->aEvent[nVertex*2]; + p->nEvent = p->nSegment = 0; + geopolyAddSegments(p, p1, 1); + geopolyAddSegments(p, p2, 2); + pThisEvent = geopolySortEventsByX(p->aEvent, p->nEvent); + rX = pThisEvent->x==0.0 ? -1.0 : 0.0; + memset(aOverlap, 0, sizeof(aOverlap)); + while( pThisEvent ){ + if( pThisEvent->x!=rX ){ + GeoSegment *pPrev = 0; + int iMask = 0; + GEODEBUG(("Distinct X: %g\n", pThisEvent->x)); + rX = pThisEvent->x; + if( needSort ){ + GEODEBUG(("SORT\n")); + pActive = geopolySortSegmentsByYAndC(pActive); + needSort = 0; + } + for(pSeg=pActive; pSeg; pSeg=pSeg->pNext){ + if( pPrev ){ + if( pPrev->y!=pSeg->y ){ + GEODEBUG(("MASK: %d\n", iMask)); + aOverlap[iMask] = 1; + } + } + iMask ^= pSeg->side; + pPrev = pSeg; + } + pPrev = 0; + for(pSeg=pActive; pSeg; pSeg=pSeg->pNext){ + double y = pSeg->C*rX + pSeg->B; + GEODEBUG(("Segment %d.%d %g->%g\n", pSeg->side, pSeg->idx, pSeg->y, y)); + pSeg->y = y; + if( pPrev ){ + if( pPrev->y>pSeg->y && pPrev->side!=pSeg->side ){ + rc = 1; + GEODEBUG(("Crossing: %d.%d and %d.%d\n", + pPrev->side, pPrev->idx, + pSeg->side, pSeg->idx)); + goto geopolyOverlapDone; + }else if( pPrev->y!=pSeg->y ){ + GEODEBUG(("MASK: %d\n", iMask)); + aOverlap[iMask] = 1; + } + } + iMask ^= pSeg->side; + pPrev = pSeg; + } + } + GEODEBUG(("%s %d.%d C=%g B=%g\n", + pThisEvent->eType ? "RM " : "ADD", + pThisEvent->pSeg->side, pThisEvent->pSeg->idx, + pThisEvent->pSeg->C, + pThisEvent->pSeg->B)); + if( pThisEvent->eType==0 ){ + /* Add a segment */ + pSeg = pThisEvent->pSeg; + pSeg->y = pSeg->y0; + pSeg->pNext = pActive; + pActive = pSeg; + needSort = 1; + }else{ + /* Remove a segment */ + if( pActive==pThisEvent->pSeg ){ + pActive = pActive->pNext; + }else{ + for(pSeg=pActive; pSeg; pSeg=pSeg->pNext){ + if( pSeg->pNext==pThisEvent->pSeg ){ + pSeg->pNext = pSeg->pNext->pNext; + break; + } + } + } + } + pThisEvent = pThisEvent->pNext; + } + if( aOverlap[3]==0 ){ + rc = 0; + }else if( aOverlap[1]!=0 && aOverlap[2]==0 ){ + rc = 3; + }else if( aOverlap[1]==0 && aOverlap[2]!=0 ){ + rc = 2; + }else if( aOverlap[1]==0 && aOverlap[2]==0 ){ + rc = 4; + }else{ + rc = 1; + } + +geopolyOverlapDone: + sqlite3_free(p); + return rc; +} + +/* +** SQL function: geopoly_overlap(P1,P2) +** +** Determine whether or not P1 and P2 overlap. Return value: +** +** 0 The two polygons are disjoint +** 1 They overlap +** 2 P1 is completely contained within P2 +** 3 P2 is completely contained within P1 +** 4 P1 and P2 are the same polygon +** NULL Either P1 or P2 or both are not valid polygons +*/ +static void geopolyOverlapFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + GeoPoly *p1 = geopolyFuncParam(context, argv[0], 0); + GeoPoly *p2 = geopolyFuncParam(context, argv[1], 0); + if( p1 && p2 ){ + int x = geopolyOverlap(p1, p2); + if( x<0 ){ + sqlite3_result_error_nomem(context); + }else{ + sqlite3_result_int(context, x); + } + } + sqlite3_free(p1); + sqlite3_free(p2); +} + +/* +** Enable or disable debugging output +*/ +static void geopolyDebugFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ +#ifdef GEOPOLY_ENABLE_DEBUG + geo_debug = sqlite3_value_int(argv[0]); +#endif +} + +/* +** This function is the implementation of both the xConnect and xCreate +** methods of the geopoly virtual table. +** +** argv[0] -> module name +** argv[1] -> database name +** argv[2] -> table name +** argv[...] -> column names... +*/ +static int geopolyInit( + sqlite3 *db, /* Database connection */ + void *pAux, /* One of the RTREE_COORD_* constants */ + int argc, const char *const*argv, /* Parameters to CREATE TABLE statement */ + sqlite3_vtab **ppVtab, /* OUT: New virtual table */ + char **pzErr, /* OUT: Error message, if any */ + int isCreate /* True for xCreate, false for xConnect */ +){ + int rc = SQLITE_OK; + Rtree *pRtree; + sqlite3_int64 nDb; /* Length of string argv[1] */ + sqlite3_int64 nName; /* Length of string argv[2] */ + sqlite3_str *pSql; + char *zSql; + int ii; + + sqlite3_vtab_config(db, SQLITE_VTAB_CONSTRAINT_SUPPORT, 1); + + /* Allocate the sqlite3_vtab structure */ + nDb = strlen(argv[1]); + nName = strlen(argv[2]); + pRtree = (Rtree *)sqlite3_malloc64(sizeof(Rtree)+nDb+nName+2); + if( !pRtree ){ + return SQLITE_NOMEM; + } + memset(pRtree, 0, sizeof(Rtree)+nDb+nName+2); + pRtree->nBusy = 1; + pRtree->base.pModule = &rtreeModule; + pRtree->zDb = (char *)&pRtree[1]; + pRtree->zName = &pRtree->zDb[nDb+1]; + pRtree->eCoordType = RTREE_COORD_REAL32; + pRtree->nDim = 2; + pRtree->nDim2 = 4; + memcpy(pRtree->zDb, argv[1], nDb); + memcpy(pRtree->zName, argv[2], nName); + + + /* Create/Connect to the underlying relational database schema. If + ** that is successful, call sqlite3_declare_vtab() to configure + ** the r-tree table schema. + */ + pSql = sqlite3_str_new(db); + sqlite3_str_appendf(pSql, "CREATE TABLE x(_shape"); + pRtree->nAux = 1; /* Add one for _shape */ + pRtree->nAuxNotNull = 1; /* The _shape column is always not-null */ + for(ii=3; iinAux++; + sqlite3_str_appendf(pSql, ",%s", argv[ii]); + } + sqlite3_str_appendf(pSql, ");"); + zSql = sqlite3_str_finish(pSql); + if( !zSql ){ + rc = SQLITE_NOMEM; + }else if( SQLITE_OK!=(rc = sqlite3_declare_vtab(db, zSql)) ){ + *pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(db)); + } + sqlite3_free(zSql); + if( rc ) goto geopolyInit_fail; + pRtree->nBytesPerCell = 8 + pRtree->nDim2*4; + + /* Figure out the node size to use. */ + rc = getNodeSize(db, pRtree, isCreate, pzErr); + if( rc ) goto geopolyInit_fail; + rc = rtreeSqlInit(pRtree, db, argv[1], argv[2], isCreate); + if( rc ){ + *pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(db)); + goto geopolyInit_fail; + } + + *ppVtab = (sqlite3_vtab *)pRtree; + return SQLITE_OK; + +geopolyInit_fail: + if( rc==SQLITE_OK ) rc = SQLITE_ERROR; + assert( *ppVtab==0 ); + assert( pRtree->nBusy==1 ); + rtreeRelease(pRtree); + return rc; +} + + +/* +** GEOPOLY virtual table module xCreate method. +*/ +static int geopolyCreate( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + return geopolyInit(db, pAux, argc, argv, ppVtab, pzErr, 1); +} + +/* +** GEOPOLY virtual table module xConnect method. +*/ +static int geopolyConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + return geopolyInit(db, pAux, argc, argv, ppVtab, pzErr, 0); +} + + +/* +** GEOPOLY virtual table module xFilter method. +** +** Query plans: +** +** 1 rowid lookup +** 2 search for objects overlapping the same bounding box +** that contains polygon argv[0] +** 3 search for objects overlapping the same bounding box +** that contains polygon argv[0] +** 4 full table scan +*/ +static int geopolyFilter( + sqlite3_vtab_cursor *pVtabCursor, /* The cursor to initialize */ + int idxNum, /* Query plan */ + const char *idxStr, /* Not Used */ + int argc, sqlite3_value **argv /* Parameters to the query plan */ +){ + Rtree *pRtree = (Rtree *)pVtabCursor->pVtab; + RtreeCursor *pCsr = (RtreeCursor *)pVtabCursor; + RtreeNode *pRoot = 0; + int rc = SQLITE_OK; + int iCell = 0; + + rtreeReference(pRtree); + + /* Reset the cursor to the same state as rtreeOpen() leaves it in. */ + resetCursor(pCsr); + + pCsr->iStrategy = idxNum; + if( idxNum==1 ){ + /* Special case - lookup by rowid. */ + RtreeNode *pLeaf; /* Leaf on which the required cell resides */ + RtreeSearchPoint *p; /* Search point for the leaf */ + i64 iRowid = sqlite3_value_int64(argv[0]); + i64 iNode = 0; + rc = findLeafNode(pRtree, iRowid, &pLeaf, &iNode); + if( rc==SQLITE_OK && pLeaf!=0 ){ + p = rtreeSearchPointNew(pCsr, RTREE_ZERO, 0); + assert( p!=0 ); /* Always returns pCsr->sPoint */ + pCsr->aNode[0] = pLeaf; + p->id = iNode; + p->eWithin = PARTLY_WITHIN; + rc = nodeRowidIndex(pRtree, pLeaf, iRowid, &iCell); + p->iCell = (u8)iCell; + RTREE_QUEUE_TRACE(pCsr, "PUSH-F1:"); + }else{ + pCsr->atEOF = 1; + } + }else{ + /* Normal case - r-tree scan. Set up the RtreeCursor.aConstraint array + ** with the configured constraints. + */ + rc = nodeAcquire(pRtree, 1, 0, &pRoot); + if( rc==SQLITE_OK && idxNum<=3 ){ + RtreeCoord bbox[4]; + RtreeConstraint *p; + assert( argc==1 ); + geopolyBBox(0, argv[0], bbox, &rc); + if( rc ){ + goto geopoly_filter_end; + } + pCsr->aConstraint = p = sqlite3_malloc(sizeof(RtreeConstraint)*4); + pCsr->nConstraint = 4; + if( p==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(pCsr->aConstraint, 0, sizeof(RtreeConstraint)*4); + memset(pCsr->anQueue, 0, sizeof(u32)*(pRtree->iDepth + 1)); + if( idxNum==2 ){ + /* Overlap query */ + p->op = 'B'; + p->iCoord = 0; + p->u.rValue = bbox[1].f; + p++; + p->op = 'D'; + p->iCoord = 1; + p->u.rValue = bbox[0].f; + p++; + p->op = 'B'; + p->iCoord = 2; + p->u.rValue = bbox[3].f; + p++; + p->op = 'D'; + p->iCoord = 3; + p->u.rValue = bbox[2].f; + }else{ + /* Within query */ + p->op = 'D'; + p->iCoord = 0; + p->u.rValue = bbox[0].f; + p++; + p->op = 'B'; + p->iCoord = 1; + p->u.rValue = bbox[1].f; + p++; + p->op = 'D'; + p->iCoord = 2; + p->u.rValue = bbox[2].f; + p++; + p->op = 'B'; + p->iCoord = 3; + p->u.rValue = bbox[3].f; + } + } + } + if( rc==SQLITE_OK ){ + RtreeSearchPoint *pNew; + pNew = rtreeSearchPointNew(pCsr, RTREE_ZERO, (u8)(pRtree->iDepth+1)); + if( pNew==0 ){ + rc = SQLITE_NOMEM; + goto geopoly_filter_end; + } + pNew->id = 1; + pNew->iCell = 0; + pNew->eWithin = PARTLY_WITHIN; + assert( pCsr->bPoint==1 ); + pCsr->aNode[0] = pRoot; + pRoot = 0; + RTREE_QUEUE_TRACE(pCsr, "PUSH-Fm:"); + rc = rtreeStepToLeaf(pCsr); + } + } + +geopoly_filter_end: + nodeRelease(pRtree, pRoot); + rtreeRelease(pRtree); + return rc; +} + +/* +** Rtree virtual table module xBestIndex method. There are three +** table scan strategies to choose from (in order from most to +** least desirable): +** +** idxNum idxStr Strategy +** ------------------------------------------------ +** 1 "rowid" Direct lookup by rowid. +** 2 "rtree" R-tree overlap query using geopoly_overlap() +** 3 "rtree" R-tree within query using geopoly_within() +** 4 "fullscan" full-table scan. +** ------------------------------------------------ +*/ +static int geopolyBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ + int ii; + int iRowidTerm = -1; + int iFuncTerm = -1; + int idxNum = 0; + + for(ii=0; iinConstraint; ii++){ + struct sqlite3_index_constraint *p = &pIdxInfo->aConstraint[ii]; + if( !p->usable ) continue; + if( p->iColumn<0 && p->op==SQLITE_INDEX_CONSTRAINT_EQ ){ + iRowidTerm = ii; + break; + } + if( p->iColumn==0 && p->op>=SQLITE_INDEX_CONSTRAINT_FUNCTION ){ + /* p->op==SQLITE_INDEX_CONSTRAINT_FUNCTION for geopoly_overlap() + ** p->op==(SQLITE_INDEX_CONTRAINT_FUNCTION+1) for geopoly_within(). + ** See geopolyFindFunction() */ + iFuncTerm = ii; + idxNum = p->op - SQLITE_INDEX_CONSTRAINT_FUNCTION + 2; + } + } + + if( iRowidTerm>=0 ){ + pIdxInfo->idxNum = 1; + pIdxInfo->idxStr = "rowid"; + pIdxInfo->aConstraintUsage[iRowidTerm].argvIndex = 1; + pIdxInfo->aConstraintUsage[iRowidTerm].omit = 1; + pIdxInfo->estimatedCost = 30.0; + pIdxInfo->estimatedRows = 1; + pIdxInfo->idxFlags = SQLITE_INDEX_SCAN_UNIQUE; + return SQLITE_OK; + } + if( iFuncTerm>=0 ){ + pIdxInfo->idxNum = idxNum; + pIdxInfo->idxStr = "rtree"; + pIdxInfo->aConstraintUsage[iFuncTerm].argvIndex = 1; + pIdxInfo->aConstraintUsage[iFuncTerm].omit = 0; + pIdxInfo->estimatedCost = 300.0; + pIdxInfo->estimatedRows = 10; + return SQLITE_OK; + } + pIdxInfo->idxNum = 4; + pIdxInfo->idxStr = "fullscan"; + pIdxInfo->estimatedCost = 3000000.0; + pIdxInfo->estimatedRows = 100000; + return SQLITE_OK; +} + + +/* +** GEOPOLY virtual table module xColumn method. +*/ +static int geopolyColumn(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int i){ + Rtree *pRtree = (Rtree *)cur->pVtab; + RtreeCursor *pCsr = (RtreeCursor *)cur; + RtreeSearchPoint *p = rtreeSearchPointFirst(pCsr); + int rc = SQLITE_OK; + RtreeNode *pNode = rtreeNodeOfFirstSearchPoint(pCsr, &rc); + + if( rc ) return rc; + if( p==0 ) return SQLITE_OK; + if( i==0 && sqlite3_vtab_nochange(ctx) ) return SQLITE_OK; + if( i<=pRtree->nAux ){ + if( !pCsr->bAuxValid ){ + if( pCsr->pReadAux==0 ){ + rc = sqlite3_prepare_v3(pRtree->db, pRtree->zReadAuxSql, -1, 0, + &pCsr->pReadAux, 0); + if( rc ) return rc; + } + sqlite3_bind_int64(pCsr->pReadAux, 1, + nodeGetRowid(pRtree, pNode, p->iCell)); + rc = sqlite3_step(pCsr->pReadAux); + if( rc==SQLITE_ROW ){ + pCsr->bAuxValid = 1; + }else{ + sqlite3_reset(pCsr->pReadAux); + if( rc==SQLITE_DONE ) rc = SQLITE_OK; + return rc; + } + } + sqlite3_result_value(ctx, sqlite3_column_value(pCsr->pReadAux, i+2)); + } + return SQLITE_OK; +} + + +/* +** The xUpdate method for GEOPOLY module virtual tables. +** +** For DELETE: +** +** argv[0] = the rowid to be deleted +** +** For INSERT: +** +** argv[0] = SQL NULL +** argv[1] = rowid to insert, or an SQL NULL to select automatically +** argv[2] = _shape column +** argv[3] = first application-defined column.... +** +** For UPDATE: +** +** argv[0] = rowid to modify. Never NULL +** argv[1] = rowid after the change. Never NULL +** argv[2] = new value for _shape +** argv[3] = new value for first application-defined column.... +*/ +static int geopolyUpdate( + sqlite3_vtab *pVtab, + int nData, + sqlite3_value **aData, + sqlite_int64 *pRowid +){ + Rtree *pRtree = (Rtree *)pVtab; + int rc = SQLITE_OK; + RtreeCell cell; /* New cell to insert if nData>1 */ + i64 oldRowid; /* The old rowid */ + int oldRowidValid; /* True if oldRowid is valid */ + i64 newRowid; /* The new rowid */ + int newRowidValid; /* True if newRowid is valid */ + int coordChange = 0; /* Change in coordinates */ + + if( pRtree->nNodeRef ){ + /* Unable to write to the btree while another cursor is reading from it, + ** since the write might do a rebalance which would disrupt the read + ** cursor. */ + return SQLITE_LOCKED_VTAB; + } + rtreeReference(pRtree); + assert(nData>=1); + + oldRowidValid = sqlite3_value_type(aData[0])!=SQLITE_NULL;; + oldRowid = oldRowidValid ? sqlite3_value_int64(aData[0]) : 0; + newRowidValid = nData>1 && sqlite3_value_type(aData[1])!=SQLITE_NULL; + newRowid = newRowidValid ? sqlite3_value_int64(aData[1]) : 0; + cell.iRowid = newRowid; + + if( nData>1 /* not a DELETE */ + && (!oldRowidValid /* INSERT */ + || !sqlite3_value_nochange(aData[2]) /* UPDATE _shape */ + || oldRowid!=newRowid) /* Rowid change */ + ){ + geopolyBBox(0, aData[2], cell.aCoord, &rc); + if( rc ){ + if( rc==SQLITE_ERROR ){ + pVtab->zErrMsg = + sqlite3_mprintf("_shape does not contain a valid polygon"); + } + goto geopoly_update_end; + } + coordChange = 1; + + /* If a rowid value was supplied, check if it is already present in + ** the table. If so, the constraint has failed. */ + if( newRowidValid && (!oldRowidValid || oldRowid!=newRowid) ){ + int steprc; + sqlite3_bind_int64(pRtree->pReadRowid, 1, cell.iRowid); + steprc = sqlite3_step(pRtree->pReadRowid); + rc = sqlite3_reset(pRtree->pReadRowid); + if( SQLITE_ROW==steprc ){ + if( sqlite3_vtab_on_conflict(pRtree->db)==SQLITE_REPLACE ){ + rc = rtreeDeleteRowid(pRtree, cell.iRowid); + }else{ + rc = rtreeConstraintError(pRtree, 0); + } + } + } + } + + /* If aData[0] is not an SQL NULL value, it is the rowid of a + ** record to delete from the r-tree table. The following block does + ** just that. + */ + if( rc==SQLITE_OK && (nData==1 || (coordChange && oldRowidValid)) ){ + rc = rtreeDeleteRowid(pRtree, oldRowid); + } + + /* If the aData[] array contains more than one element, elements + ** (aData[2]..aData[argc-1]) contain a new record to insert into + ** the r-tree structure. + */ + if( rc==SQLITE_OK && nData>1 && coordChange ){ + /* Insert the new record into the r-tree */ + RtreeNode *pLeaf = 0; + if( !newRowidValid ){ + rc = rtreeNewRowid(pRtree, &cell.iRowid); + } + *pRowid = cell.iRowid; + if( rc==SQLITE_OK ){ + rc = ChooseLeaf(pRtree, &cell, 0, &pLeaf); + } + if( rc==SQLITE_OK ){ + int rc2; + pRtree->iReinsertHeight = -1; + rc = rtreeInsertCell(pRtree, pLeaf, &cell, 0); + rc2 = nodeRelease(pRtree, pLeaf); + if( rc==SQLITE_OK ){ + rc = rc2; + } + } + } + + /* Change the data */ + if( rc==SQLITE_OK && nData>1 ){ + sqlite3_stmt *pUp = pRtree->pWriteAux; + int jj; + int nChange = 0; + sqlite3_bind_int64(pUp, 1, cell.iRowid); + assert( pRtree->nAux>=1 ); + if( sqlite3_value_nochange(aData[2]) ){ + sqlite3_bind_null(pUp, 2); + }else{ + GeoPoly *p = 0; + if( sqlite3_value_type(aData[2])==SQLITE_TEXT + && (p = geopolyFuncParam(0, aData[2], &rc))!=0 + && rc==SQLITE_OK + ){ + sqlite3_bind_blob(pUp, 2, p->hdr, 4+8*p->nVertex, SQLITE_TRANSIENT); + }else{ + sqlite3_bind_value(pUp, 2, aData[2]); + } + sqlite3_free(p); + nChange = 1; + } + for(jj=1; jjnAux; jj++){ + nChange++; + sqlite3_bind_value(pUp, jj+2, aData[jj+2]); + } + if( nChange ){ + sqlite3_step(pUp); + rc = sqlite3_reset(pUp); + } + } + +geopoly_update_end: + rtreeRelease(pRtree); + return rc; +} + +/* +** Report that geopoly_overlap() is an overloaded function suitable +** for use in xBestIndex. +*/ +static int geopolyFindFunction( + sqlite3_vtab *pVtab, + int nArg, + const char *zName, + void (**pxFunc)(sqlite3_context*,int,sqlite3_value**), + void **ppArg +){ + if( sqlite3_stricmp(zName, "geopoly_overlap")==0 ){ + *pxFunc = geopolyOverlapFunc; + *ppArg = 0; + return SQLITE_INDEX_CONSTRAINT_FUNCTION; + } + if( sqlite3_stricmp(zName, "geopoly_within")==0 ){ + *pxFunc = geopolyWithinFunc; + *ppArg = 0; + return SQLITE_INDEX_CONSTRAINT_FUNCTION+1; + } + return 0; +} + + +static sqlite3_module geopolyModule = { + 3, /* iVersion */ + geopolyCreate, /* xCreate - create a table */ + geopolyConnect, /* xConnect - connect to an existing table */ + geopolyBestIndex, /* xBestIndex - Determine search strategy */ + rtreeDisconnect, /* xDisconnect - Disconnect from a table */ + rtreeDestroy, /* xDestroy - Drop a table */ + rtreeOpen, /* xOpen - open a cursor */ + rtreeClose, /* xClose - close a cursor */ + geopolyFilter, /* xFilter - configure scan constraints */ + rtreeNext, /* xNext - advance a cursor */ + rtreeEof, /* xEof */ + geopolyColumn, /* xColumn - read data */ + rtreeRowid, /* xRowid - read data */ + geopolyUpdate, /* xUpdate - write data */ + rtreeBeginTransaction, /* xBegin - begin transaction */ + rtreeEndTransaction, /* xSync - sync transaction */ + rtreeEndTransaction, /* xCommit - commit transaction */ + rtreeEndTransaction, /* xRollback - rollback transaction */ + geopolyFindFunction, /* xFindFunction - function overloading */ + rtreeRename, /* xRename - rename the table */ + rtreeSavepoint, /* xSavepoint */ + 0, /* xRelease */ + 0, /* xRollbackTo */ + rtreeShadowName /* xShadowName */ +}; + +static int sqlite3_geopoly_init(sqlite3 *db){ + int rc = SQLITE_OK; + static const struct { + void (*xFunc)(sqlite3_context*,int,sqlite3_value**); + signed char nArg; + unsigned char bPure; + const char *zName; + } aFunc[] = { + { geopolyAreaFunc, 1, 1, "geopoly_area" }, + { geopolyBlobFunc, 1, 1, "geopoly_blob" }, + { geopolyJsonFunc, 1, 1, "geopoly_json" }, + { geopolySvgFunc, -1, 1, "geopoly_svg" }, + { geopolyWithinFunc, 2, 1, "geopoly_within" }, + { geopolyContainsPointFunc, 3, 1, "geopoly_contains_point" }, + { geopolyOverlapFunc, 2, 1, "geopoly_overlap" }, + { geopolyDebugFunc, 1, 0, "geopoly_debug" }, + { geopolyBBoxFunc, 1, 1, "geopoly_bbox" }, + { geopolyXformFunc, 7, 1, "geopoly_xform" }, + { geopolyRegularFunc, 4, 1, "geopoly_regular" }, + { geopolyCcwFunc, 1, 1, "geopoly_ccw" }, + }; + static const struct { + void (*xStep)(sqlite3_context*,int,sqlite3_value**); + void (*xFinal)(sqlite3_context*); + const char *zName; + } aAgg[] = { + { geopolyBBoxStep, geopolyBBoxFinal, "geopoly_group_bbox" }, + }; + int i; + for(i=0; i0 ){ n = strlen(argv[0])+1; } - p = (IcuTokenizer *)sqlite3_malloc(sizeof(IcuTokenizer)+n); + p = (IcuTokenizer *)sqlite3_malloc64(sizeof(IcuTokenizer)+n); if( !p ){ return SQLITE_NOMEM; } @@ -175591,7 +194204,7 @@ static int icuOpen( nInput = strlen(zInput); } nChar = nInput+1; - pCsr = (IcuCursor *)sqlite3_malloc( + pCsr = (IcuCursor *)sqlite3_malloc64( sizeof(IcuCursor) + /* IcuCursor */ ((nChar+3)&~3) * sizeof(UChar) + /* IcuCursor.aChar[] */ (nChar+1) * sizeof(int) /* IcuCursor.aOffset[] */ @@ -176082,10489 +194695,9176 @@ SQLITE_PRIVATE void sqlite3Fts3IcuTokenizerModule( ** ** DATABASE LOCKING ** -** An RBU update may not be applied to a database in WAL mode. Attempting -** to do so is an error (SQLITE_ERROR). -** -** While an RBU handle is open, a SHARED lock may be held on the target -** database file. This means it is possible for other clients to read the -** database, but not to write it. -** -** If an RBU update is started and then suspended before it is completed, -** then an external client writes to the database, then attempting to resume -** the suspended RBU update is also an error (SQLITE_BUSY). -*/ - -#ifndef _SQLITE3RBU_H -#define _SQLITE3RBU_H - -/* #include "sqlite3.h" ** Required for error code definitions ** */ - -#if 0 -extern "C" { -#endif - -typedef struct sqlite3rbu sqlite3rbu; - -/* -** Open an RBU handle. -** -** Argument zTarget is the path to the target database. Argument zRbu is -** the path to the RBU database. Each call to this function must be matched -** by a call to sqlite3rbu_close(). When opening the databases, RBU passes -** the SQLITE_CONFIG_URI flag to sqlite3_open_v2(). So if either zTarget -** or zRbu begin with "file:", it will be interpreted as an SQLite -** database URI, not a regular file name. -** -** If the zState argument is passed a NULL value, the RBU extension stores -** the current state of the update (how many rows have been updated, which -** indexes are yet to be updated etc.) within the RBU database itself. This -** can be convenient, as it means that the RBU application does not need to -** organize removing a separate state file after the update is concluded. -** Or, if zState is non-NULL, it must be a path to a database file in which -** the RBU extension can store the state of the update. -** -** When resuming an RBU update, the zState argument must be passed the same -** value as when the RBU update was started. -** -** Once the RBU update is finished, the RBU extension does not -** automatically remove any zState database file, even if it created it. -** -** By default, RBU uses the default VFS to access the files on disk. To -** use a VFS other than the default, an SQLite "file:" URI containing a -** "vfs=..." option may be passed as the zTarget option. -** -** IMPORTANT NOTE FOR ZIPVFS USERS: The RBU extension works with all of -** SQLite's built-in VFSs, including the multiplexor VFS. However it does -** not work out of the box with zipvfs. Refer to the comment describing -** the zipvfs_create_vfs() API below for details on using RBU with zipvfs. -*/ -SQLITE_API sqlite3rbu *sqlite3rbu_open( - const char *zTarget, - const char *zRbu, - const char *zState -); - -/* -** Open an RBU handle to perform an RBU vacuum on database file zTarget. -** An RBU vacuum is similar to SQLite's built-in VACUUM command, except -** that it can be suspended and resumed like an RBU update. -** -** The second argument to this function identifies a database in which -** to store the state of the RBU vacuum operation if it is suspended. The -** first time sqlite3rbu_vacuum() is called, to start an RBU vacuum -** operation, the state database should either not exist or be empty -** (contain no tables). If an RBU vacuum is suspended by calling -** sqlite3rbu_close() on the RBU handle before sqlite3rbu_step() has -** returned SQLITE_DONE, the vacuum state is stored in the state database. -** The vacuum can be resumed by calling this function to open a new RBU -** handle specifying the same target and state databases. -** -** If the second argument passed to this function is NULL, then the -** name of the state database is "-vacuum", where -** is the name of the target database file. In this case, on UNIX, if the -** state database is not already present in the file-system, it is created -** with the same permissions as the target db is made. -** -** This function does not delete the state database after an RBU vacuum -** is completed, even if it created it. However, if the call to -** sqlite3rbu_close() returns any value other than SQLITE_OK, the contents -** of the state tables within the state database are zeroed. This way, -** the next call to sqlite3rbu_vacuum() opens a handle that starts a -** new RBU vacuum operation. -** -** As with sqlite3rbu_open(), Zipvfs users should rever to the comment -** describing the sqlite3rbu_create_vfs() API function below for -** a description of the complications associated with using RBU with -** zipvfs databases. -*/ -SQLITE_API sqlite3rbu *sqlite3rbu_vacuum( - const char *zTarget, - const char *zState -); - -/* -** Configure a limit for the amount of temp space that may be used by -** the RBU handle passed as the first argument. The new limit is specified -** in bytes by the second parameter. If it is positive, the limit is updated. -** If the second parameter to this function is passed zero, then the limit -** is removed entirely. If the second parameter is negative, the limit is -** not modified (this is useful for querying the current limit). -** -** In all cases the returned value is the current limit in bytes (zero -** indicates unlimited). -** -** If the temp space limit is exceeded during operation, an SQLITE_FULL -** error is returned. -*/ -SQLITE_API sqlite3_int64 sqlite3rbu_temp_size_limit(sqlite3rbu*, sqlite3_int64); - -/* -** Return the current amount of temp file space, in bytes, currently used by -** the RBU handle passed as the only argument. -*/ -SQLITE_API sqlite3_int64 sqlite3rbu_temp_size(sqlite3rbu*); - -/* -** Internally, each RBU connection uses a separate SQLite database -** connection to access the target and rbu update databases. This -** API allows the application direct access to these database handles. -** -** The first argument passed to this function must be a valid, open, RBU -** handle. The second argument should be passed zero to access the target -** database handle, or non-zero to access the rbu update database handle. -** Accessing the underlying database handles may be useful in the -** following scenarios: -** -** * If any target tables are virtual tables, it may be necessary to -** call sqlite3_create_module() on the target database handle to -** register the required virtual table implementations. -** -** * If the data_xxx tables in the RBU source database are virtual -** tables, the application may need to call sqlite3_create_module() on -** the rbu update db handle to any required virtual table -** implementations. -** -** * If the application uses the "rbu_delta()" feature described above, -** it must use sqlite3_create_function() or similar to register the -** rbu_delta() implementation with the target database handle. -** -** If an error has occurred, either while opening or stepping the RBU object, -** this function may return NULL. The error code and message may be collected -** when sqlite3rbu_close() is called. -** -** Database handles returned by this function remain valid until the next -** call to any sqlite3rbu_xxx() function other than sqlite3rbu_db(). -*/ -SQLITE_API sqlite3 *sqlite3rbu_db(sqlite3rbu*, int bRbu); - -/* -** Do some work towards applying the RBU update to the target db. -** -** Return SQLITE_DONE if the update has been completely applied, or -** SQLITE_OK if no error occurs but there remains work to do to apply -** the RBU update. If an error does occur, some other error code is -** returned. -** -** Once a call to sqlite3rbu_step() has returned a value other than -** SQLITE_OK, all subsequent calls on the same RBU handle are no-ops -** that immediately return the same value. -*/ -SQLITE_API int sqlite3rbu_step(sqlite3rbu *pRbu); - -/* -** Force RBU to save its state to disk. -** -** If a power failure or application crash occurs during an update, following -** system recovery RBU may resume the update from the point at which the state -** was last saved. In other words, from the most recent successful call to -** sqlite3rbu_close() or this function. -** -** SQLITE_OK is returned if successful, or an SQLite error code otherwise. -*/ -SQLITE_API int sqlite3rbu_savestate(sqlite3rbu *pRbu); - -/* -** Close an RBU handle. -** -** If the RBU update has been completely applied, mark the RBU database -** as fully applied. Otherwise, assuming no error has occurred, save the -** current state of the RBU update appliation to the RBU database. -** -** If an error has already occurred as part of an sqlite3rbu_step() -** or sqlite3rbu_open() call, or if one occurs within this function, an -** SQLite error code is returned. Additionally, if pzErrmsg is not NULL, -** *pzErrmsg may be set to point to a buffer containing a utf-8 formatted -** English language error message. It is the responsibility of the caller to -** eventually free any such buffer using sqlite3_free(). -** -** Otherwise, if no error occurs, this function returns SQLITE_OK if the -** update has been partially applied, or SQLITE_DONE if it has been -** completely applied. -*/ -SQLITE_API int sqlite3rbu_close(sqlite3rbu *pRbu, char **pzErrmsg); - -/* -** Return the total number of key-value operations (inserts, deletes or -** updates) that have been performed on the target database since the -** current RBU update was started. -*/ -SQLITE_API sqlite3_int64 sqlite3rbu_progress(sqlite3rbu *pRbu); - -/* -** Obtain permyriadage (permyriadage is to 10000 as percentage is to 100) -** progress indications for the two stages of an RBU update. This API may -** be useful for driving GUI progress indicators and similar. -** -** An RBU update is divided into two stages: -** -** * Stage 1, in which changes are accumulated in an oal/wal file, and -** * Stage 2, in which the contents of the wal file are copied into the -** main database. -** -** The update is visible to non-RBU clients during stage 2. During stage 1 -** non-RBU reader clients may see the original database. -** -** If this API is called during stage 2 of the update, output variable -** (*pnOne) is set to 10000 to indicate that stage 1 has finished and (*pnTwo) -** to a value between 0 and 10000 to indicate the permyriadage progress of -** stage 2. A value of 5000 indicates that stage 2 is half finished, -** 9000 indicates that it is 90% finished, and so on. -** -** If this API is called during stage 1 of the update, output variable -** (*pnTwo) is set to 0 to indicate that stage 2 has not yet started. The -** value to which (*pnOne) is set depends on whether or not the RBU -** database contains an "rbu_count" table. The rbu_count table, if it -** exists, must contain the same columns as the following: -** -** CREATE TABLE rbu_count(tbl TEXT PRIMARY KEY, cnt INTEGER) WITHOUT ROWID; -** -** There must be one row in the table for each source (data_xxx) table within -** the RBU database. The 'tbl' column should contain the name of the source -** table. The 'cnt' column should contain the number of rows within the -** source table. -** -** If the rbu_count table is present and populated correctly and this -** API is called during stage 1, the *pnOne output variable is set to the -** permyriadage progress of the same stage. If the rbu_count table does -** not exist, then (*pnOne) is set to -1 during stage 1. If the rbu_count -** table exists but is not correctly populated, the value of the *pnOne -** output variable during stage 1 is undefined. -*/ -SQLITE_API void sqlite3rbu_bp_progress(sqlite3rbu *pRbu, int *pnOne, int*pnTwo); - -/* -** Obtain an indication as to the current stage of an RBU update or vacuum. -** This function always returns one of the SQLITE_RBU_STATE_XXX constants -** defined in this file. Return values should be interpreted as follows: -** -** SQLITE_RBU_STATE_OAL: -** RBU is currently building a *-oal file. The next call to sqlite3rbu_step() -** may either add further data to the *-oal file, or compute data that will -** be added by a subsequent call. -** -** SQLITE_RBU_STATE_MOVE: -** RBU has finished building the *-oal file. The next call to sqlite3rbu_step() -** will move the *-oal file to the equivalent *-wal path. If the current -** operation is an RBU update, then the updated version of the database -** file will become visible to ordinary SQLite clients following the next -** call to sqlite3rbu_step(). -** -** SQLITE_RBU_STATE_CHECKPOINT: -** RBU is currently performing an incremental checkpoint. The next call to -** sqlite3rbu_step() will copy a page of data from the *-wal file into -** the target database file. -** -** SQLITE_RBU_STATE_DONE: -** The RBU operation has finished. Any subsequent calls to sqlite3rbu_step() -** will immediately return SQLITE_DONE. -** -** SQLITE_RBU_STATE_ERROR: -** An error has occurred. Any subsequent calls to sqlite3rbu_step() will -** immediately return the SQLite error code associated with the error. -*/ -#define SQLITE_RBU_STATE_OAL 1 -#define SQLITE_RBU_STATE_MOVE 2 -#define SQLITE_RBU_STATE_CHECKPOINT 3 -#define SQLITE_RBU_STATE_DONE 4 -#define SQLITE_RBU_STATE_ERROR 5 - -SQLITE_API int sqlite3rbu_state(sqlite3rbu *pRbu); - -/* -** Create an RBU VFS named zName that accesses the underlying file-system -** via existing VFS zParent. Or, if the zParent parameter is passed NULL, -** then the new RBU VFS uses the default system VFS to access the file-system. -** The new object is registered as a non-default VFS with SQLite before -** returning. -** -** Part of the RBU implementation uses a custom VFS object. Usually, this -** object is created and deleted automatically by RBU. -** -** The exception is for applications that also use zipvfs. In this case, -** the custom VFS must be explicitly created by the user before the RBU -** handle is opened. The RBU VFS should be installed so that the zipvfs -** VFS uses the RBU VFS, which in turn uses any other VFS layers in use -** (for example multiplexor) to access the file-system. For example, -** to assemble an RBU enabled VFS stack that uses both zipvfs and -** multiplexor (error checking omitted): -** -** // Create a VFS named "multiplex" (not the default). -** sqlite3_multiplex_initialize(0, 0); -** -** // Create an rbu VFS named "rbu" that uses multiplexor. If the -** // second argument were replaced with NULL, the "rbu" VFS would -** // access the file-system via the system default VFS, bypassing the -** // multiplexor. -** sqlite3rbu_create_vfs("rbu", "multiplex"); -** -** // Create a zipvfs VFS named "zipvfs" that uses rbu. -** zipvfs_create_vfs_v3("zipvfs", "rbu", 0, xCompressorAlgorithmDetector); -** -** // Make zipvfs the default VFS. -** sqlite3_vfs_register(sqlite3_vfs_find("zipvfs"), 1); -** -** Because the default VFS created above includes a RBU functionality, it -** may be used by RBU clients. Attempting to use RBU with a zipvfs VFS stack -** that does not include the RBU layer results in an error. -** -** The overhead of adding the "rbu" VFS to the system is negligible for -** non-RBU users. There is no harm in an application accessing the -** file-system via "rbu" all the time, even if it only uses RBU functionality -** occasionally. -*/ -SQLITE_API int sqlite3rbu_create_vfs(const char *zName, const char *zParent); - -/* -** Deregister and destroy an RBU vfs created by an earlier call to -** sqlite3rbu_create_vfs(). -** -** VFS objects are not reference counted. If a VFS object is destroyed -** before all database handles that use it have been closed, the results -** are undefined. -*/ -SQLITE_API void sqlite3rbu_destroy_vfs(const char *zName); - -#if 0 -} /* end of the 'extern "C"' block */ -#endif - -#endif /* _SQLITE3RBU_H */ - -/************** End of sqlite3rbu.h ******************************************/ -/************** Continuing where we left off in sqlite3rbu.c *****************/ - -#if defined(_WIN32_WCE) -/* #include "windows.h" */ -#endif - -/* Maximum number of prepared UPDATE statements held by this module */ -#define SQLITE_RBU_UPDATE_CACHESIZE 16 - -/* Delta checksums disabled by default. Compile with -DRBU_ENABLE_DELTA_CKSUM -** to enable checksum verification. -*/ -#ifndef RBU_ENABLE_DELTA_CKSUM -# define RBU_ENABLE_DELTA_CKSUM 0 -#endif - -/* -** Swap two objects of type TYPE. -*/ -#if !defined(SQLITE_AMALGAMATION) -# define SWAP(TYPE,A,B) {TYPE t=A; A=B; B=t;} -#endif - -/* -** The rbu_state table is used to save the state of a partially applied -** update so that it can be resumed later. The table consists of integer -** keys mapped to values as follows: -** -** RBU_STATE_STAGE: -** May be set to integer values 1, 2, 4 or 5. As follows: -** 1: the *-rbu file is currently under construction. -** 2: the *-rbu file has been constructed, but not yet moved -** to the *-wal path. -** 4: the checkpoint is underway. -** 5: the rbu update has been checkpointed. -** -** RBU_STATE_TBL: -** Only valid if STAGE==1. The target database name of the table -** currently being written. -** -** RBU_STATE_IDX: -** Only valid if STAGE==1. The target database name of the index -** currently being written, or NULL if the main table is currently being -** updated. -** -** RBU_STATE_ROW: -** Only valid if STAGE==1. Number of rows already processed for the current -** table/index. -** -** RBU_STATE_PROGRESS: -** Trbul number of sqlite3rbu_step() calls made so far as part of this -** rbu update. -** -** RBU_STATE_CKPT: -** Valid if STAGE==4. The 64-bit checksum associated with the wal-index -** header created by recovering the *-wal file. This is used to detect -** cases when another client appends frames to the *-wal file in the -** middle of an incremental checkpoint (an incremental checkpoint cannot -** be continued if this happens). -** -** RBU_STATE_COOKIE: -** Valid if STAGE==1. The current change-counter cookie value in the -** target db file. -** -** RBU_STATE_OALSZ: -** Valid if STAGE==1. The size in bytes of the *-oal file. -** -** RBU_STATE_DATATBL: -** Only valid if STAGE==1. The RBU database name of the table -** currently being read. -*/ -#define RBU_STATE_STAGE 1 -#define RBU_STATE_TBL 2 -#define RBU_STATE_IDX 3 -#define RBU_STATE_ROW 4 -#define RBU_STATE_PROGRESS 5 -#define RBU_STATE_CKPT 6 -#define RBU_STATE_COOKIE 7 -#define RBU_STATE_OALSZ 8 -#define RBU_STATE_PHASEONESTEP 9 -#define RBU_STATE_DATATBL 10 - -#define RBU_STAGE_OAL 1 -#define RBU_STAGE_MOVE 2 -#define RBU_STAGE_CAPTURE 3 -#define RBU_STAGE_CKPT 4 -#define RBU_STAGE_DONE 5 - - -#define RBU_CREATE_STATE \ - "CREATE TABLE IF NOT EXISTS %s.rbu_state(k INTEGER PRIMARY KEY, v)" - -typedef struct RbuFrame RbuFrame; -typedef struct RbuObjIter RbuObjIter; -typedef struct RbuState RbuState; -typedef struct rbu_vfs rbu_vfs; -typedef struct rbu_file rbu_file; -typedef struct RbuUpdateStmt RbuUpdateStmt; - -#if !defined(SQLITE_AMALGAMATION) -typedef unsigned int u32; -typedef unsigned short u16; -typedef unsigned char u8; -typedef sqlite3_int64 i64; -#endif - -/* -** These values must match the values defined in wal.c for the equivalent -** locks. These are not magic numbers as they are part of the SQLite file -** format. -*/ -#define WAL_LOCK_WRITE 0 -#define WAL_LOCK_CKPT 1 -#define WAL_LOCK_READ0 3 - -#define SQLITE_FCNTL_RBUCNT 5149216 - -/* -** A structure to store values read from the rbu_state table in memory. -*/ -struct RbuState { - int eStage; - char *zTbl; - char *zDataTbl; - char *zIdx; - i64 iWalCksum; - int nRow; - i64 nProgress; - u32 iCookie; - i64 iOalSz; - i64 nPhaseOneStep; -}; - -struct RbuUpdateStmt { - char *zMask; /* Copy of update mask used with pUpdate */ - sqlite3_stmt *pUpdate; /* Last update statement (or NULL) */ - RbuUpdateStmt *pNext; -}; - -/* -** An iterator of this type is used to iterate through all objects in -** the target database that require updating. For each such table, the -** iterator visits, in order: -** -** * the table itself, -** * each index of the table (zero or more points to visit), and -** * a special "cleanup table" state. -** -** abIndexed: -** If the table has no indexes on it, abIndexed is set to NULL. Otherwise, -** it points to an array of flags nTblCol elements in size. The flag is -** set for each column that is either a part of the PK or a part of an -** index. Or clear otherwise. -** -*/ -struct RbuObjIter { - sqlite3_stmt *pTblIter; /* Iterate through tables */ - sqlite3_stmt *pIdxIter; /* Index iterator */ - int nTblCol; /* Size of azTblCol[] array */ - char **azTblCol; /* Array of unquoted target column names */ - char **azTblType; /* Array of target column types */ - int *aiSrcOrder; /* src table col -> target table col */ - u8 *abTblPk; /* Array of flags, set on target PK columns */ - u8 *abNotNull; /* Array of flags, set on NOT NULL columns */ - u8 *abIndexed; /* Array of flags, set on indexed & PK cols */ - int eType; /* Table type - an RBU_PK_XXX value */ - - /* Output variables. zTbl==0 implies EOF. */ - int bCleanup; /* True in "cleanup" state */ - const char *zTbl; /* Name of target db table */ - const char *zDataTbl; /* Name of rbu db table (or null) */ - const char *zIdx; /* Name of target db index (or null) */ - int iTnum; /* Root page of current object */ - int iPkTnum; /* If eType==EXTERNAL, root of PK index */ - int bUnique; /* Current index is unique */ - int nIndex; /* Number of aux. indexes on table zTbl */ - - /* Statements created by rbuObjIterPrepareAll() */ - int nCol; /* Number of columns in current object */ - sqlite3_stmt *pSelect; /* Source data */ - sqlite3_stmt *pInsert; /* Statement for INSERT operations */ - sqlite3_stmt *pDelete; /* Statement for DELETE ops */ - sqlite3_stmt *pTmpInsert; /* Insert into rbu_tmp_$zDataTbl */ - - /* Last UPDATE used (for PK b-tree updates only), or NULL. */ - RbuUpdateStmt *pRbuUpdate; -}; - -/* -** Values for RbuObjIter.eType -** -** 0: Table does not exist (error) -** 1: Table has an implicit rowid. -** 2: Table has an explicit IPK column. -** 3: Table has an external PK index. -** 4: Table is WITHOUT ROWID. -** 5: Table is a virtual table. -*/ -#define RBU_PK_NOTABLE 0 -#define RBU_PK_NONE 1 -#define RBU_PK_IPK 2 -#define RBU_PK_EXTERNAL 3 -#define RBU_PK_WITHOUT_ROWID 4 -#define RBU_PK_VTAB 5 - - -/* -** Within the RBU_STAGE_OAL stage, each call to sqlite3rbu_step() performs -** one of the following operations. -*/ -#define RBU_INSERT 1 /* Insert on a main table b-tree */ -#define RBU_DELETE 2 /* Delete a row from a main table b-tree */ -#define RBU_REPLACE 3 /* Delete and then insert a row */ -#define RBU_IDX_DELETE 4 /* Delete a row from an aux. index b-tree */ -#define RBU_IDX_INSERT 5 /* Insert on an aux. index b-tree */ - -#define RBU_UPDATE 6 /* Update a row in a main table b-tree */ - -/* -** A single step of an incremental checkpoint - frame iWalFrame of the wal -** file should be copied to page iDbPage of the database file. -*/ -struct RbuFrame { - u32 iDbPage; - u32 iWalFrame; -}; - -/* -** RBU handle. -** -** nPhaseOneStep: -** If the RBU database contains an rbu_count table, this value is set to -** a running estimate of the number of b-tree operations required to -** finish populating the *-oal file. This allows the sqlite3_bp_progress() -** API to calculate the permyriadage progress of populating the *-oal file -** using the formula: -** -** permyriadage = (10000 * nProgress) / nPhaseOneStep -** -** nPhaseOneStep is initialized to the sum of: -** -** nRow * (nIndex + 1) -** -** for all source tables in the RBU database, where nRow is the number -** of rows in the source table and nIndex the number of indexes on the -** corresponding target database table. -** -** This estimate is accurate if the RBU update consists entirely of -** INSERT operations. However, it is inaccurate if: -** -** * the RBU update contains any UPDATE operations. If the PK specified -** for an UPDATE operation does not exist in the target table, then -** no b-tree operations are required on index b-trees. Or if the -** specified PK does exist, then (nIndex*2) such operations are -** required (one delete and one insert on each index b-tree). -** -** * the RBU update contains any DELETE operations for which the specified -** PK does not exist. In this case no operations are required on index -** b-trees. -** -** * the RBU update contains REPLACE operations. These are similar to -** UPDATE operations. -** -** nPhaseOneStep is updated to account for the conditions above during the -** first pass of each source table. The updated nPhaseOneStep value is -** stored in the rbu_state table if the RBU update is suspended. -*/ -struct sqlite3rbu { - int eStage; /* Value of RBU_STATE_STAGE field */ - sqlite3 *dbMain; /* target database handle */ - sqlite3 *dbRbu; /* rbu database handle */ - char *zTarget; /* Path to target db */ - char *zRbu; /* Path to rbu db */ - char *zState; /* Path to state db (or NULL if zRbu) */ - char zStateDb[5]; /* Db name for state ("stat" or "main") */ - int rc; /* Value returned by last rbu_step() call */ - char *zErrmsg; /* Error message if rc!=SQLITE_OK */ - int nStep; /* Rows processed for current object */ - int nProgress; /* Rows processed for all objects */ - RbuObjIter objiter; /* Iterator for skipping through tbl/idx */ - const char *zVfsName; /* Name of automatically created rbu vfs */ - rbu_file *pTargetFd; /* File handle open on target db */ - int nPagePerSector; /* Pages per sector for pTargetFd */ - i64 iOalSz; - i64 nPhaseOneStep; - - /* The following state variables are used as part of the incremental - ** checkpoint stage (eStage==RBU_STAGE_CKPT). See comments surrounding - ** function rbuSetupCheckpoint() for details. */ - u32 iMaxFrame; /* Largest iWalFrame value in aFrame[] */ - u32 mLock; - int nFrame; /* Entries in aFrame[] array */ - int nFrameAlloc; /* Allocated size of aFrame[] array */ - RbuFrame *aFrame; - int pgsz; - u8 *aBuf; - i64 iWalCksum; - i64 szTemp; /* Current size of all temp files in use */ - i64 szTempLimit; /* Total size limit for temp files */ - - /* Used in RBU vacuum mode only */ - int nRbu; /* Number of RBU VFS in the stack */ - rbu_file *pRbuFd; /* Fd for main db of dbRbu */ -}; - -/* -** An rbu VFS is implemented using an instance of this structure. -** -** Variable pRbu is only non-NULL for automatically created RBU VFS objects. -** It is NULL for RBU VFS objects created explicitly using -** sqlite3rbu_create_vfs(). It is used to track the total amount of temp -** space used by the RBU handle. -*/ -struct rbu_vfs { - sqlite3_vfs base; /* rbu VFS shim methods */ - sqlite3_vfs *pRealVfs; /* Underlying VFS */ - sqlite3_mutex *mutex; /* Mutex to protect pMain */ - sqlite3rbu *pRbu; /* Owner RBU object */ - rbu_file *pMain; /* Linked list of main db files */ -}; - -/* -** Each file opened by an rbu VFS is represented by an instance of -** the following structure. -** -** If this is a temporary file (pRbu!=0 && flags&DELETE_ON_CLOSE), variable -** "sz" is set to the current size of the database file. -*/ -struct rbu_file { - sqlite3_file base; /* sqlite3_file methods */ - sqlite3_file *pReal; /* Underlying file handle */ - rbu_vfs *pRbuVfs; /* Pointer to the rbu_vfs object */ - sqlite3rbu *pRbu; /* Pointer to rbu object (rbu target only) */ - i64 sz; /* Size of file in bytes (temp only) */ - - int openFlags; /* Flags this file was opened with */ - u32 iCookie; /* Cookie value for main db files */ - u8 iWriteVer; /* "write-version" value for main db files */ - u8 bNolock; /* True to fail EXCLUSIVE locks */ - - int nShm; /* Number of entries in apShm[] array */ - char **apShm; /* Array of mmap'd *-shm regions */ - char *zDel; /* Delete this when closing file */ - - const char *zWal; /* Wal filename for this main db file */ - rbu_file *pWalFd; /* Wal file descriptor for this main db */ - rbu_file *pMainNext; /* Next MAIN_DB file */ -}; - -/* -** True for an RBU vacuum handle, or false otherwise. -*/ -#define rbuIsVacuum(p) ((p)->zTarget==0) - - -/************************************************************************* -** The following three functions, found below: -** -** rbuDeltaGetInt() -** rbuDeltaChecksum() -** rbuDeltaApply() -** -** are lifted from the fossil source code (http://fossil-scm.org). They -** are used to implement the scalar SQL function rbu_fossil_delta(). -*/ - -/* -** Read bytes from *pz and convert them into a positive integer. When -** finished, leave *pz pointing to the first character past the end of -** the integer. The *pLen parameter holds the length of the string -** in *pz and is decremented once for each character in the integer. -*/ -static unsigned int rbuDeltaGetInt(const char **pz, int *pLen){ - static const signed char zValue[] = { - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, -1, -1, -1, -1, -1, -1, - -1, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, - 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, -1, -1, -1, -1, 36, - -1, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, - 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, -1, -1, -1, 63, -1, - }; - unsigned int v = 0; - int c; - unsigned char *z = (unsigned char*)*pz; - unsigned char *zStart = z; - while( (c = zValue[0x7f&*(z++)])>=0 ){ - v = (v<<6) + c; - } - z--; - *pLen -= z - zStart; - *pz = (char*)z; - return v; -} - -#if RBU_ENABLE_DELTA_CKSUM -/* -** Compute a 32-bit checksum on the N-byte buffer. Return the result. -*/ -static unsigned int rbuDeltaChecksum(const char *zIn, size_t N){ - const unsigned char *z = (const unsigned char *)zIn; - unsigned sum0 = 0; - unsigned sum1 = 0; - unsigned sum2 = 0; - unsigned sum3 = 0; - while(N >= 16){ - sum0 += ((unsigned)z[0] + z[4] + z[8] + z[12]); - sum1 += ((unsigned)z[1] + z[5] + z[9] + z[13]); - sum2 += ((unsigned)z[2] + z[6] + z[10]+ z[14]); - sum3 += ((unsigned)z[3] + z[7] + z[11]+ z[15]); - z += 16; - N -= 16; - } - while(N >= 4){ - sum0 += z[0]; - sum1 += z[1]; - sum2 += z[2]; - sum3 += z[3]; - z += 4; - N -= 4; - } - sum3 += (sum2 << 8) + (sum1 << 16) + (sum0 << 24); - switch(N){ - case 3: sum3 += (z[2] << 8); - case 2: sum3 += (z[1] << 16); - case 1: sum3 += (z[0] << 24); - default: ; - } - return sum3; -} -#endif - -/* -** Apply a delta. -** -** The output buffer should be big enough to hold the whole output -** file and a NUL terminator at the end. The delta_output_size() -** routine will determine this size for you. -** -** The delta string should be null-terminated. But the delta string -** may contain embedded NUL characters (if the input and output are -** binary files) so we also have to pass in the length of the delta in -** the lenDelta parameter. -** -** This function returns the size of the output file in bytes (excluding -** the final NUL terminator character). Except, if the delta string is -** malformed or intended for use with a source file other than zSrc, -** then this routine returns -1. -** -** Refer to the delta_create() documentation above for a description -** of the delta file format. -*/ -static int rbuDeltaApply( - const char *zSrc, /* The source or pattern file */ - int lenSrc, /* Length of the source file */ - const char *zDelta, /* Delta to apply to the pattern */ - int lenDelta, /* Length of the delta */ - char *zOut /* Write the output into this preallocated buffer */ -){ - unsigned int limit; - unsigned int total = 0; -#if RBU_ENABLE_DELTA_CKSUM - char *zOrigOut = zOut; -#endif - - limit = rbuDeltaGetInt(&zDelta, &lenDelta); - if( *zDelta!='\n' ){ - /* ERROR: size integer not terminated by "\n" */ - return -1; - } - zDelta++; lenDelta--; - while( *zDelta && lenDelta>0 ){ - unsigned int cnt, ofst; - cnt = rbuDeltaGetInt(&zDelta, &lenDelta); - switch( zDelta[0] ){ - case '@': { - zDelta++; lenDelta--; - ofst = rbuDeltaGetInt(&zDelta, &lenDelta); - if( lenDelta>0 && zDelta[0]!=',' ){ - /* ERROR: copy command not terminated by ',' */ - return -1; - } - zDelta++; lenDelta--; - total += cnt; - if( total>limit ){ - /* ERROR: copy exceeds output file size */ - return -1; - } - if( (int)(ofst+cnt) > lenSrc ){ - /* ERROR: copy extends past end of input */ - return -1; - } - memcpy(zOut, &zSrc[ofst], cnt); - zOut += cnt; - break; - } - case ':': { - zDelta++; lenDelta--; - total += cnt; - if( total>limit ){ - /* ERROR: insert command gives an output larger than predicted */ - return -1; - } - if( (int)cnt>lenDelta ){ - /* ERROR: insert count exceeds size of delta */ - return -1; - } - memcpy(zOut, zDelta, cnt); - zOut += cnt; - zDelta += cnt; - lenDelta -= cnt; - break; - } - case ';': { - zDelta++; lenDelta--; - zOut[0] = 0; -#if RBU_ENABLE_DELTA_CKSUM - if( cnt!=rbuDeltaChecksum(zOrigOut, total) ){ - /* ERROR: bad checksum */ - return -1; - } -#endif - if( total!=limit ){ - /* ERROR: generated size does not match predicted size */ - return -1; - } - return total; - } - default: { - /* ERROR: unknown delta operator */ - return -1; - } - } - } - /* ERROR: unterminated delta */ - return -1; -} - -static int rbuDeltaOutputSize(const char *zDelta, int lenDelta){ - int size; - size = rbuDeltaGetInt(&zDelta, &lenDelta); - if( *zDelta!='\n' ){ - /* ERROR: size integer not terminated by "\n" */ - return -1; - } - return size; -} - -/* -** End of code taken from fossil. -*************************************************************************/ - -/* -** Implementation of SQL scalar function rbu_fossil_delta(). -** -** This function applies a fossil delta patch to a blob. Exactly two -** arguments must be passed to this function. The first is the blob to -** patch and the second the patch to apply. If no error occurs, this -** function returns the patched blob. -*/ -static void rbuFossilDeltaFunc( - sqlite3_context *context, - int argc, - sqlite3_value **argv -){ - const char *aDelta; - int nDelta; - const char *aOrig; - int nOrig; - - int nOut; - int nOut2; - char *aOut; - - assert( argc==2 ); - - nOrig = sqlite3_value_bytes(argv[0]); - aOrig = (const char*)sqlite3_value_blob(argv[0]); - nDelta = sqlite3_value_bytes(argv[1]); - aDelta = (const char*)sqlite3_value_blob(argv[1]); - - /* Figure out the size of the output */ - nOut = rbuDeltaOutputSize(aDelta, nDelta); - if( nOut<0 ){ - sqlite3_result_error(context, "corrupt fossil delta", -1); - return; - } - - aOut = sqlite3_malloc(nOut+1); - if( aOut==0 ){ - sqlite3_result_error_nomem(context); - }else{ - nOut2 = rbuDeltaApply(aOrig, nOrig, aDelta, nDelta, aOut); - if( nOut2!=nOut ){ - sqlite3_result_error(context, "corrupt fossil delta", -1); - }else{ - sqlite3_result_blob(context, aOut, nOut, sqlite3_free); - } - } -} - - -/* -** Prepare the SQL statement in buffer zSql against database handle db. -** If successful, set *ppStmt to point to the new statement and return -** SQLITE_OK. -** -** Otherwise, if an error does occur, set *ppStmt to NULL and return -** an SQLite error code. Additionally, set output variable *pzErrmsg to -** point to a buffer containing an error message. It is the responsibility -** of the caller to (eventually) free this buffer using sqlite3_free(). -*/ -static int prepareAndCollectError( - sqlite3 *db, - sqlite3_stmt **ppStmt, - char **pzErrmsg, - const char *zSql -){ - int rc = sqlite3_prepare_v2(db, zSql, -1, ppStmt, 0); - if( rc!=SQLITE_OK ){ - *pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(db)); - *ppStmt = 0; - } - return rc; -} - -/* -** Reset the SQL statement passed as the first argument. Return a copy -** of the value returned by sqlite3_reset(). -** -** If an error has occurred, then set *pzErrmsg to point to a buffer -** containing an error message. It is the responsibility of the caller -** to eventually free this buffer using sqlite3_free(). -*/ -static int resetAndCollectError(sqlite3_stmt *pStmt, char **pzErrmsg){ - int rc = sqlite3_reset(pStmt); - if( rc!=SQLITE_OK ){ - *pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(sqlite3_db_handle(pStmt))); - } - return rc; -} - -/* -** Unless it is NULL, argument zSql points to a buffer allocated using -** sqlite3_malloc containing an SQL statement. This function prepares the SQL -** statement against database db and frees the buffer. If statement -** compilation is successful, *ppStmt is set to point to the new statement -** handle and SQLITE_OK is returned. +** An RBU update may not be applied to a database in WAL mode. Attempting +** to do so is an error (SQLITE_ERROR). ** -** Otherwise, if an error occurs, *ppStmt is set to NULL and an error code -** returned. In this case, *pzErrmsg may also be set to point to an error -** message. It is the responsibility of the caller to free this error message -** buffer using sqlite3_free(). +** While an RBU handle is open, a SHARED lock may be held on the target +** database file. This means it is possible for other clients to read the +** database, but not to write it. ** -** If argument zSql is NULL, this function assumes that an OOM has occurred. -** In this case SQLITE_NOMEM is returned and *ppStmt set to NULL. +** If an RBU update is started and then suspended before it is completed, +** then an external client writes to the database, then attempting to resume +** the suspended RBU update is also an error (SQLITE_BUSY). */ -static int prepareFreeAndCollectError( - sqlite3 *db, - sqlite3_stmt **ppStmt, - char **pzErrmsg, - char *zSql -){ - int rc; - assert( *pzErrmsg==0 ); - if( zSql==0 ){ - rc = SQLITE_NOMEM; - *ppStmt = 0; - }else{ - rc = prepareAndCollectError(db, ppStmt, pzErrmsg, zSql); - sqlite3_free(zSql); - } - return rc; -} -/* -** Free the RbuObjIter.azTblCol[] and RbuObjIter.abTblPk[] arrays allocated -** by an earlier call to rbuObjIterCacheTableInfo(). -*/ -static void rbuObjIterFreeCols(RbuObjIter *pIter){ - int i; - for(i=0; inTblCol; i++){ - sqlite3_free(pIter->azTblCol[i]); - sqlite3_free(pIter->azTblType[i]); - } - sqlite3_free(pIter->azTblCol); - pIter->azTblCol = 0; - pIter->azTblType = 0; - pIter->aiSrcOrder = 0; - pIter->abTblPk = 0; - pIter->abNotNull = 0; - pIter->nTblCol = 0; - pIter->eType = 0; /* Invalid value */ -} +#ifndef _SQLITE3RBU_H +#define _SQLITE3RBU_H -/* -** Finalize all statements and free all allocations that are specific to -** the current object (table/index pair). -*/ -static void rbuObjIterClearStatements(RbuObjIter *pIter){ - RbuUpdateStmt *pUp; +/* #include "sqlite3.h" ** Required for error code definitions ** */ - sqlite3_finalize(pIter->pSelect); - sqlite3_finalize(pIter->pInsert); - sqlite3_finalize(pIter->pDelete); - sqlite3_finalize(pIter->pTmpInsert); - pUp = pIter->pRbuUpdate; - while( pUp ){ - RbuUpdateStmt *pTmp = pUp->pNext; - sqlite3_finalize(pUp->pUpdate); - sqlite3_free(pUp); - pUp = pTmp; - } - - pIter->pSelect = 0; - pIter->pInsert = 0; - pIter->pDelete = 0; - pIter->pRbuUpdate = 0; - pIter->pTmpInsert = 0; - pIter->nCol = 0; -} +#if 0 +extern "C" { +#endif -/* -** Clean up any resources allocated as part of the iterator object passed -** as the only argument. -*/ -static void rbuObjIterFinalize(RbuObjIter *pIter){ - rbuObjIterClearStatements(pIter); - sqlite3_finalize(pIter->pTblIter); - sqlite3_finalize(pIter->pIdxIter); - rbuObjIterFreeCols(pIter); - memset(pIter, 0, sizeof(RbuObjIter)); -} +typedef struct sqlite3rbu sqlite3rbu; /* -** Advance the iterator to the next position. +** Open an RBU handle. ** -** If no error occurs, SQLITE_OK is returned and the iterator is left -** pointing to the next entry. Otherwise, an error code and message is -** left in the RBU handle passed as the first argument. A copy of the -** error code is returned. +** Argument zTarget is the path to the target database. Argument zRbu is +** the path to the RBU database. Each call to this function must be matched +** by a call to sqlite3rbu_close(). When opening the databases, RBU passes +** the SQLITE_CONFIG_URI flag to sqlite3_open_v2(). So if either zTarget +** or zRbu begin with "file:", it will be interpreted as an SQLite +** database URI, not a regular file name. +** +** If the zState argument is passed a NULL value, the RBU extension stores +** the current state of the update (how many rows have been updated, which +** indexes are yet to be updated etc.) within the RBU database itself. This +** can be convenient, as it means that the RBU application does not need to +** organize removing a separate state file after the update is concluded. +** Or, if zState is non-NULL, it must be a path to a database file in which +** the RBU extension can store the state of the update. +** +** When resuming an RBU update, the zState argument must be passed the same +** value as when the RBU update was started. +** +** Once the RBU update is finished, the RBU extension does not +** automatically remove any zState database file, even if it created it. +** +** By default, RBU uses the default VFS to access the files on disk. To +** use a VFS other than the default, an SQLite "file:" URI containing a +** "vfs=..." option may be passed as the zTarget option. +** +** IMPORTANT NOTE FOR ZIPVFS USERS: The RBU extension works with all of +** SQLite's built-in VFSs, including the multiplexor VFS. However it does +** not work out of the box with zipvfs. Refer to the comment describing +** the zipvfs_create_vfs() API below for details on using RBU with zipvfs. */ -static int rbuObjIterNext(sqlite3rbu *p, RbuObjIter *pIter){ - int rc = p->rc; - if( rc==SQLITE_OK ){ - - /* Free any SQLite statements used while processing the previous object */ - rbuObjIterClearStatements(pIter); - if( pIter->zIdx==0 ){ - rc = sqlite3_exec(p->dbMain, - "DROP TRIGGER IF EXISTS temp.rbu_insert_tr;" - "DROP TRIGGER IF EXISTS temp.rbu_update1_tr;" - "DROP TRIGGER IF EXISTS temp.rbu_update2_tr;" - "DROP TRIGGER IF EXISTS temp.rbu_delete_tr;" - , 0, 0, &p->zErrmsg - ); - } - - if( rc==SQLITE_OK ){ - if( pIter->bCleanup ){ - rbuObjIterFreeCols(pIter); - pIter->bCleanup = 0; - rc = sqlite3_step(pIter->pTblIter); - if( rc!=SQLITE_ROW ){ - rc = resetAndCollectError(pIter->pTblIter, &p->zErrmsg); - pIter->zTbl = 0; - }else{ - pIter->zTbl = (const char*)sqlite3_column_text(pIter->pTblIter, 0); - pIter->zDataTbl = (const char*)sqlite3_column_text(pIter->pTblIter,1); - rc = (pIter->zDataTbl && pIter->zTbl) ? SQLITE_OK : SQLITE_NOMEM; - } - }else{ - if( pIter->zIdx==0 ){ - sqlite3_stmt *pIdx = pIter->pIdxIter; - rc = sqlite3_bind_text(pIdx, 1, pIter->zTbl, -1, SQLITE_STATIC); - } - if( rc==SQLITE_OK ){ - rc = sqlite3_step(pIter->pIdxIter); - if( rc!=SQLITE_ROW ){ - rc = resetAndCollectError(pIter->pIdxIter, &p->zErrmsg); - pIter->bCleanup = 1; - pIter->zIdx = 0; - }else{ - pIter->zIdx = (const char*)sqlite3_column_text(pIter->pIdxIter, 0); - pIter->iTnum = sqlite3_column_int(pIter->pIdxIter, 1); - pIter->bUnique = sqlite3_column_int(pIter->pIdxIter, 2); - rc = pIter->zIdx ? SQLITE_OK : SQLITE_NOMEM; - } - } - } - } - } - - if( rc!=SQLITE_OK ){ - rbuObjIterFinalize(pIter); - p->rc = rc; - } - return rc; -} - +SQLITE_API sqlite3rbu *sqlite3rbu_open( + const char *zTarget, + const char *zRbu, + const char *zState +); /* -** The implementation of the rbu_target_name() SQL function. This function -** accepts one or two arguments. The first argument is the name of a table - -** the name of a table in the RBU database. The second, if it is present, is 1 -** for a view or 0 for a table. +** Open an RBU handle to perform an RBU vacuum on database file zTarget. +** An RBU vacuum is similar to SQLite's built-in VACUUM command, except +** that it can be suspended and resumed like an RBU update. ** -** For a non-vacuum RBU handle, if the table name matches the pattern: +** The second argument to this function identifies a database in which +** to store the state of the RBU vacuum operation if it is suspended. The +** first time sqlite3rbu_vacuum() is called, to start an RBU vacuum +** operation, the state database should either not exist or be empty +** (contain no tables). If an RBU vacuum is suspended by calling +** sqlite3rbu_close() on the RBU handle before sqlite3rbu_step() has +** returned SQLITE_DONE, the vacuum state is stored in the state database. +** The vacuum can be resumed by calling this function to open a new RBU +** handle specifying the same target and state databases. ** -** data[0-9]_ +** If the second argument passed to this function is NULL, then the +** name of the state database is "-vacuum", where +** is the name of the target database file. In this case, on UNIX, if the +** state database is not already present in the file-system, it is created +** with the same permissions as the target db is made. ** -** where is any sequence of 1 or more characters, is returned. -** Otherwise, if the only argument does not match the above pattern, an SQL -** NULL is returned. +** With an RBU vacuum, it is an SQLITE_MISUSE error if the name of the +** state database ends with "-vactmp". This name is reserved for internal +** use. ** -** "data_t1" -> "t1" -** "data0123_t2" -> "t2" -** "dataAB_t3" -> NULL +** This function does not delete the state database after an RBU vacuum +** is completed, even if it created it. However, if the call to +** sqlite3rbu_close() returns any value other than SQLITE_OK, the contents +** of the state tables within the state database are zeroed. This way, +** the next call to sqlite3rbu_vacuum() opens a handle that starts a +** new RBU vacuum operation. ** -** For an rbu vacuum handle, a copy of the first argument is returned if -** the second argument is either missing or 0 (not a view). +** As with sqlite3rbu_open(), Zipvfs users should rever to the comment +** describing the sqlite3rbu_create_vfs() API function below for +** a description of the complications associated with using RBU with +** zipvfs databases. */ -static void rbuTargetNameFunc( - sqlite3_context *pCtx, - int argc, - sqlite3_value **argv -){ - sqlite3rbu *p = sqlite3_user_data(pCtx); - const char *zIn; - assert( argc==1 || argc==2 ); - - zIn = (const char*)sqlite3_value_text(argv[0]); - if( zIn ){ - if( rbuIsVacuum(p) ){ - if( argc==1 || 0==sqlite3_value_int(argv[1]) ){ - sqlite3_result_text(pCtx, zIn, -1, SQLITE_STATIC); - } - }else{ - if( strlen(zIn)>4 && memcmp("data", zIn, 4)==0 ){ - int i; - for(i=4; zIn[i]>='0' && zIn[i]<='9'; i++); - if( zIn[i]=='_' && zIn[i+1] ){ - sqlite3_result_text(pCtx, &zIn[i+1], -1, SQLITE_STATIC); - } - } - } - } -} +SQLITE_API sqlite3rbu *sqlite3rbu_vacuum( + const char *zTarget, + const char *zState +); /* -** Initialize the iterator structure passed as the second argument. +** Configure a limit for the amount of temp space that may be used by +** the RBU handle passed as the first argument. The new limit is specified +** in bytes by the second parameter. If it is positive, the limit is updated. +** If the second parameter to this function is passed zero, then the limit +** is removed entirely. If the second parameter is negative, the limit is +** not modified (this is useful for querying the current limit). ** -** If no error occurs, SQLITE_OK is returned and the iterator is left -** pointing to the first entry. Otherwise, an error code and message is -** left in the RBU handle passed as the first argument. A copy of the -** error code is returned. +** In all cases the returned value is the current limit in bytes (zero +** indicates unlimited). +** +** If the temp space limit is exceeded during operation, an SQLITE_FULL +** error is returned. */ -static int rbuObjIterFirst(sqlite3rbu *p, RbuObjIter *pIter){ - int rc; - memset(pIter, 0, sizeof(RbuObjIter)); - - rc = prepareFreeAndCollectError(p->dbRbu, &pIter->pTblIter, &p->zErrmsg, - sqlite3_mprintf( - "SELECT rbu_target_name(name, type='view') AS target, name " - "FROM sqlite_master " - "WHERE type IN ('table', 'view') AND target IS NOT NULL " - " %s " - "ORDER BY name" - , rbuIsVacuum(p) ? "AND rootpage!=0 AND rootpage IS NOT NULL" : "")); - - if( rc==SQLITE_OK ){ - rc = prepareAndCollectError(p->dbMain, &pIter->pIdxIter, &p->zErrmsg, - "SELECT name, rootpage, sql IS NULL OR substr(8, 6)=='UNIQUE' " - " FROM main.sqlite_master " - " WHERE type='index' AND tbl_name = ?" - ); - } - - pIter->bCleanup = 1; - p->rc = rc; - return rbuObjIterNext(p, pIter); -} +SQLITE_API sqlite3_int64 sqlite3rbu_temp_size_limit(sqlite3rbu*, sqlite3_int64); /* -** This is a wrapper around "sqlite3_mprintf(zFmt, ...)". If an OOM occurs, -** an error code is stored in the RBU handle passed as the first argument. -** -** If an error has already occurred (p->rc is already set to something other -** than SQLITE_OK), then this function returns NULL without modifying the -** stored error code. In this case it still calls sqlite3_free() on any -** printf() parameters associated with %z conversions. +** Return the current amount of temp file space, in bytes, currently used by +** the RBU handle passed as the only argument. */ -static char *rbuMPrintf(sqlite3rbu *p, const char *zFmt, ...){ - char *zSql = 0; - va_list ap; - va_start(ap, zFmt); - zSql = sqlite3_vmprintf(zFmt, ap); - if( p->rc==SQLITE_OK ){ - if( zSql==0 ) p->rc = SQLITE_NOMEM; - }else{ - sqlite3_free(zSql); - zSql = 0; - } - va_end(ap); - return zSql; -} +SQLITE_API sqlite3_int64 sqlite3rbu_temp_size(sqlite3rbu*); /* -** Argument zFmt is a sqlite3_mprintf() style format string. The trailing -** arguments are the usual subsitution values. This function performs -** the printf() style substitutions and executes the result as an SQL -** statement on the RBU handles database. +** Internally, each RBU connection uses a separate SQLite database +** connection to access the target and rbu update databases. This +** API allows the application direct access to these database handles. ** -** If an error occurs, an error code and error message is stored in the -** RBU handle. If an error has already occurred when this function is -** called, it is a no-op. +** The first argument passed to this function must be a valid, open, RBU +** handle. The second argument should be passed zero to access the target +** database handle, or non-zero to access the rbu update database handle. +** Accessing the underlying database handles may be useful in the +** following scenarios: +** +** * If any target tables are virtual tables, it may be necessary to +** call sqlite3_create_module() on the target database handle to +** register the required virtual table implementations. +** +** * If the data_xxx tables in the RBU source database are virtual +** tables, the application may need to call sqlite3_create_module() on +** the rbu update db handle to any required virtual table +** implementations. +** +** * If the application uses the "rbu_delta()" feature described above, +** it must use sqlite3_create_function() or similar to register the +** rbu_delta() implementation with the target database handle. +** +** If an error has occurred, either while opening or stepping the RBU object, +** this function may return NULL. The error code and message may be collected +** when sqlite3rbu_close() is called. +** +** Database handles returned by this function remain valid until the next +** call to any sqlite3rbu_xxx() function other than sqlite3rbu_db(). */ -static int rbuMPrintfExec(sqlite3rbu *p, sqlite3 *db, const char *zFmt, ...){ - va_list ap; - char *zSql; - va_start(ap, zFmt); - zSql = sqlite3_vmprintf(zFmt, ap); - if( p->rc==SQLITE_OK ){ - if( zSql==0 ){ - p->rc = SQLITE_NOMEM; - }else{ - p->rc = sqlite3_exec(db, zSql, 0, 0, &p->zErrmsg); - } - } - sqlite3_free(zSql); - va_end(ap); - return p->rc; -} +SQLITE_API sqlite3 *sqlite3rbu_db(sqlite3rbu*, int bRbu); /* -** Attempt to allocate and return a pointer to a zeroed block of nByte -** bytes. +** Do some work towards applying the RBU update to the target db. ** -** If an error (i.e. an OOM condition) occurs, return NULL and leave an -** error code in the rbu handle passed as the first argument. Or, if an -** error has already occurred when this function is called, return NULL -** immediately without attempting the allocation or modifying the stored -** error code. +** Return SQLITE_DONE if the update has been completely applied, or +** SQLITE_OK if no error occurs but there remains work to do to apply +** the RBU update. If an error does occur, some other error code is +** returned. +** +** Once a call to sqlite3rbu_step() has returned a value other than +** SQLITE_OK, all subsequent calls on the same RBU handle are no-ops +** that immediately return the same value. */ -static void *rbuMalloc(sqlite3rbu *p, int nByte){ - void *pRet = 0; - if( p->rc==SQLITE_OK ){ - assert( nByte>0 ); - pRet = sqlite3_malloc64(nByte); - if( pRet==0 ){ - p->rc = SQLITE_NOMEM; - }else{ - memset(pRet, 0, nByte); - } - } - return pRet; -} - +SQLITE_API int sqlite3rbu_step(sqlite3rbu *pRbu); /* -** Allocate and zero the pIter->azTblCol[] and abTblPk[] arrays so that -** there is room for at least nCol elements. If an OOM occurs, store an -** error code in the RBU handle passed as the first argument. +** Force RBU to save its state to disk. +** +** If a power failure or application crash occurs during an update, following +** system recovery RBU may resume the update from the point at which the state +** was last saved. In other words, from the most recent successful call to +** sqlite3rbu_close() or this function. +** +** SQLITE_OK is returned if successful, or an SQLite error code otherwise. */ -static void rbuAllocateIterArrays(sqlite3rbu *p, RbuObjIter *pIter, int nCol){ - int nByte = (2*sizeof(char*) + sizeof(int) + 3*sizeof(u8)) * nCol; - char **azNew; - - azNew = (char**)rbuMalloc(p, nByte); - if( azNew ){ - pIter->azTblCol = azNew; - pIter->azTblType = &azNew[nCol]; - pIter->aiSrcOrder = (int*)&pIter->azTblType[nCol]; - pIter->abTblPk = (u8*)&pIter->aiSrcOrder[nCol]; - pIter->abNotNull = (u8*)&pIter->abTblPk[nCol]; - pIter->abIndexed = (u8*)&pIter->abNotNull[nCol]; - } -} +SQLITE_API int sqlite3rbu_savestate(sqlite3rbu *pRbu); /* -** The first argument must be a nul-terminated string. This function -** returns a copy of the string in memory obtained from sqlite3_malloc(). -** It is the responsibility of the caller to eventually free this memory -** using sqlite3_free(). +** Close an RBU handle. ** -** If an OOM condition is encountered when attempting to allocate memory, -** output variable (*pRc) is set to SQLITE_NOMEM before returning. Otherwise, -** if the allocation succeeds, (*pRc) is left unchanged. +** If the RBU update has been completely applied, mark the RBU database +** as fully applied. Otherwise, assuming no error has occurred, save the +** current state of the RBU update appliation to the RBU database. +** +** If an error has already occurred as part of an sqlite3rbu_step() +** or sqlite3rbu_open() call, or if one occurs within this function, an +** SQLite error code is returned. Additionally, if pzErrmsg is not NULL, +** *pzErrmsg may be set to point to a buffer containing a utf-8 formatted +** English language error message. It is the responsibility of the caller to +** eventually free any such buffer using sqlite3_free(). +** +** Otherwise, if no error occurs, this function returns SQLITE_OK if the +** update has been partially applied, or SQLITE_DONE if it has been +** completely applied. */ -static char *rbuStrndup(const char *zStr, int *pRc){ - char *zRet = 0; - - assert( *pRc==SQLITE_OK ); - if( zStr ){ - size_t nCopy = strlen(zStr) + 1; - zRet = (char*)sqlite3_malloc64(nCopy); - if( zRet ){ - memcpy(zRet, zStr, nCopy); - }else{ - *pRc = SQLITE_NOMEM; - } - } +SQLITE_API int sqlite3rbu_close(sqlite3rbu *pRbu, char **pzErrmsg); - return zRet; -} +/* +** Return the total number of key-value operations (inserts, deletes or +** updates) that have been performed on the target database since the +** current RBU update was started. +*/ +SQLITE_API sqlite3_int64 sqlite3rbu_progress(sqlite3rbu *pRbu); /* -** Finalize the statement passed as the second argument. +** Obtain permyriadage (permyriadage is to 10000 as percentage is to 100) +** progress indications for the two stages of an RBU update. This API may +** be useful for driving GUI progress indicators and similar. ** -** If the sqlite3_finalize() call indicates that an error occurs, and the -** rbu handle error code is not already set, set the error code and error -** message accordingly. +** An RBU update is divided into two stages: +** +** * Stage 1, in which changes are accumulated in an oal/wal file, and +** * Stage 2, in which the contents of the wal file are copied into the +** main database. +** +** The update is visible to non-RBU clients during stage 2. During stage 1 +** non-RBU reader clients may see the original database. +** +** If this API is called during stage 2 of the update, output variable +** (*pnOne) is set to 10000 to indicate that stage 1 has finished and (*pnTwo) +** to a value between 0 and 10000 to indicate the permyriadage progress of +** stage 2. A value of 5000 indicates that stage 2 is half finished, +** 9000 indicates that it is 90% finished, and so on. +** +** If this API is called during stage 1 of the update, output variable +** (*pnTwo) is set to 0 to indicate that stage 2 has not yet started. The +** value to which (*pnOne) is set depends on whether or not the RBU +** database contains an "rbu_count" table. The rbu_count table, if it +** exists, must contain the same columns as the following: +** +** CREATE TABLE rbu_count(tbl TEXT PRIMARY KEY, cnt INTEGER) WITHOUT ROWID; +** +** There must be one row in the table for each source (data_xxx) table within +** the RBU database. The 'tbl' column should contain the name of the source +** table. The 'cnt' column should contain the number of rows within the +** source table. +** +** If the rbu_count table is present and populated correctly and this +** API is called during stage 1, the *pnOne output variable is set to the +** permyriadage progress of the same stage. If the rbu_count table does +** not exist, then (*pnOne) is set to -1 during stage 1. If the rbu_count +** table exists but is not correctly populated, the value of the *pnOne +** output variable during stage 1 is undefined. */ -static void rbuFinalize(sqlite3rbu *p, sqlite3_stmt *pStmt){ - sqlite3 *db = sqlite3_db_handle(pStmt); - int rc = sqlite3_finalize(pStmt); - if( p->rc==SQLITE_OK && rc!=SQLITE_OK ){ - p->rc = rc; - p->zErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(db)); - } -} +SQLITE_API void sqlite3rbu_bp_progress(sqlite3rbu *pRbu, int *pnOne, int*pnTwo); -/* Determine the type of a table. +/* +** Obtain an indication as to the current stage of an RBU update or vacuum. +** This function always returns one of the SQLITE_RBU_STATE_XXX constants +** defined in this file. Return values should be interpreted as follows: ** -** peType is of type (int*), a pointer to an output parameter of type -** (int). This call sets the output parameter as follows, depending -** on the type of the table specified by parameters dbName and zTbl. +** SQLITE_RBU_STATE_OAL: +** RBU is currently building a *-oal file. The next call to sqlite3rbu_step() +** may either add further data to the *-oal file, or compute data that will +** be added by a subsequent call. ** -** RBU_PK_NOTABLE: No such table. -** RBU_PK_NONE: Table has an implicit rowid. -** RBU_PK_IPK: Table has an explicit IPK column. -** RBU_PK_EXTERNAL: Table has an external PK index. -** RBU_PK_WITHOUT_ROWID: Table is WITHOUT ROWID. -** RBU_PK_VTAB: Table is a virtual table. +** SQLITE_RBU_STATE_MOVE: +** RBU has finished building the *-oal file. The next call to sqlite3rbu_step() +** will move the *-oal file to the equivalent *-wal path. If the current +** operation is an RBU update, then the updated version of the database +** file will become visible to ordinary SQLite clients following the next +** call to sqlite3rbu_step(). ** -** Argument *piPk is also of type (int*), and also points to an output -** parameter. Unless the table has an external primary key index -** (i.e. unless *peType is set to 3), then *piPk is set to zero. Or, -** if the table does have an external primary key index, then *piPk -** is set to the root page number of the primary key index before -** returning. +** SQLITE_RBU_STATE_CHECKPOINT: +** RBU is currently performing an incremental checkpoint. The next call to +** sqlite3rbu_step() will copy a page of data from the *-wal file into +** the target database file. ** -** ALGORITHM: +** SQLITE_RBU_STATE_DONE: +** The RBU operation has finished. Any subsequent calls to sqlite3rbu_step() +** will immediately return SQLITE_DONE. ** -** if( no entry exists in sqlite_master ){ -** return RBU_PK_NOTABLE -** }else if( sql for the entry starts with "CREATE VIRTUAL" ){ -** return RBU_PK_VTAB -** }else if( "PRAGMA index_list()" for the table contains a "pk" index ){ -** if( the index that is the pk exists in sqlite_master ){ -** *piPK = rootpage of that index. -** return RBU_PK_EXTERNAL -** }else{ -** return RBU_PK_WITHOUT_ROWID -** } -** }else if( "PRAGMA table_info()" lists one or more "pk" columns ){ -** return RBU_PK_IPK -** }else{ -** return RBU_PK_NONE -** } +** SQLITE_RBU_STATE_ERROR: +** An error has occurred. Any subsequent calls to sqlite3rbu_step() will +** immediately return the SQLite error code associated with the error. */ -static void rbuTableType( - sqlite3rbu *p, - const char *zTab, - int *peType, - int *piTnum, - int *piPk -){ - /* - ** 0) SELECT count(*) FROM sqlite_master where name=%Q AND IsVirtual(%Q) - ** 1) PRAGMA index_list = ? - ** 2) SELECT count(*) FROM sqlite_master where name=%Q - ** 3) PRAGMA table_info = ? - */ - sqlite3_stmt *aStmt[4] = {0, 0, 0, 0}; - - *peType = RBU_PK_NOTABLE; - *piPk = 0; - - assert( p->rc==SQLITE_OK ); - p->rc = prepareFreeAndCollectError(p->dbMain, &aStmt[0], &p->zErrmsg, - sqlite3_mprintf( - "SELECT (sql LIKE 'create virtual%%'), rootpage" - " FROM sqlite_master" - " WHERE name=%Q", zTab - )); - if( p->rc!=SQLITE_OK || sqlite3_step(aStmt[0])!=SQLITE_ROW ){ - /* Either an error, or no such table. */ - goto rbuTableType_end; - } - if( sqlite3_column_int(aStmt[0], 0) ){ - *peType = RBU_PK_VTAB; /* virtual table */ - goto rbuTableType_end; - } - *piTnum = sqlite3_column_int(aStmt[0], 1); - - p->rc = prepareFreeAndCollectError(p->dbMain, &aStmt[1], &p->zErrmsg, - sqlite3_mprintf("PRAGMA index_list=%Q",zTab) - ); - if( p->rc ) goto rbuTableType_end; - while( sqlite3_step(aStmt[1])==SQLITE_ROW ){ - const u8 *zOrig = sqlite3_column_text(aStmt[1], 3); - const u8 *zIdx = sqlite3_column_text(aStmt[1], 1); - if( zOrig && zIdx && zOrig[0]=='p' ){ - p->rc = prepareFreeAndCollectError(p->dbMain, &aStmt[2], &p->zErrmsg, - sqlite3_mprintf( - "SELECT rootpage FROM sqlite_master WHERE name = %Q", zIdx - )); - if( p->rc==SQLITE_OK ){ - if( sqlite3_step(aStmt[2])==SQLITE_ROW ){ - *piPk = sqlite3_column_int(aStmt[2], 0); - *peType = RBU_PK_EXTERNAL; - }else{ - *peType = RBU_PK_WITHOUT_ROWID; - } - } - goto rbuTableType_end; - } - } - - p->rc = prepareFreeAndCollectError(p->dbMain, &aStmt[3], &p->zErrmsg, - sqlite3_mprintf("PRAGMA table_info=%Q",zTab) - ); - if( p->rc==SQLITE_OK ){ - while( sqlite3_step(aStmt[3])==SQLITE_ROW ){ - if( sqlite3_column_int(aStmt[3],5)>0 ){ - *peType = RBU_PK_IPK; /* explicit IPK column */ - goto rbuTableType_end; - } - } - *peType = RBU_PK_NONE; - } +#define SQLITE_RBU_STATE_OAL 1 +#define SQLITE_RBU_STATE_MOVE 2 +#define SQLITE_RBU_STATE_CHECKPOINT 3 +#define SQLITE_RBU_STATE_DONE 4 +#define SQLITE_RBU_STATE_ERROR 5 -rbuTableType_end: { - unsigned int i; - for(i=0; iabIndexed[] array. +** Create an RBU VFS named zName that accesses the underlying file-system +** via existing VFS zParent. Or, if the zParent parameter is passed NULL, +** then the new RBU VFS uses the default system VFS to access the file-system. +** The new object is registered as a non-default VFS with SQLite before +** returning. +** +** Part of the RBU implementation uses a custom VFS object. Usually, this +** object is created and deleted automatically by RBU. +** +** The exception is for applications that also use zipvfs. In this case, +** the custom VFS must be explicitly created by the user before the RBU +** handle is opened. The RBU VFS should be installed so that the zipvfs +** VFS uses the RBU VFS, which in turn uses any other VFS layers in use +** (for example multiplexor) to access the file-system. For example, +** to assemble an RBU enabled VFS stack that uses both zipvfs and +** multiplexor (error checking omitted): +** +** // Create a VFS named "multiplex" (not the default). +** sqlite3_multiplex_initialize(0, 0); +** +** // Create an rbu VFS named "rbu" that uses multiplexor. If the +** // second argument were replaced with NULL, the "rbu" VFS would +** // access the file-system via the system default VFS, bypassing the +** // multiplexor. +** sqlite3rbu_create_vfs("rbu", "multiplex"); +** +** // Create a zipvfs VFS named "zipvfs" that uses rbu. +** zipvfs_create_vfs_v3("zipvfs", "rbu", 0, xCompressorAlgorithmDetector); +** +** // Make zipvfs the default VFS. +** sqlite3_vfs_register(sqlite3_vfs_find("zipvfs"), 1); +** +** Because the default VFS created above includes a RBU functionality, it +** may be used by RBU clients. Attempting to use RBU with a zipvfs VFS stack +** that does not include the RBU layer results in an error. +** +** The overhead of adding the "rbu" VFS to the system is negligible for +** non-RBU users. There is no harm in an application accessing the +** file-system via "rbu" all the time, even if it only uses RBU functionality +** occasionally. */ -static void rbuObjIterCacheIndexedCols(sqlite3rbu *p, RbuObjIter *pIter){ - sqlite3_stmt *pList = 0; - int bIndex = 0; - - if( p->rc==SQLITE_OK ){ - memcpy(pIter->abIndexed, pIter->abTblPk, sizeof(u8)*pIter->nTblCol); - p->rc = prepareFreeAndCollectError(p->dbMain, &pList, &p->zErrmsg, - sqlite3_mprintf("PRAGMA main.index_list = %Q", pIter->zTbl) - ); - } - - pIter->nIndex = 0; - while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pList) ){ - const char *zIdx = (const char*)sqlite3_column_text(pList, 1); - sqlite3_stmt *pXInfo = 0; - if( zIdx==0 ) break; - p->rc = prepareFreeAndCollectError(p->dbMain, &pXInfo, &p->zErrmsg, - sqlite3_mprintf("PRAGMA main.index_xinfo = %Q", zIdx) - ); - while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXInfo) ){ - int iCid = sqlite3_column_int(pXInfo, 1); - if( iCid>=0 ) pIter->abIndexed[iCid] = 1; - } - rbuFinalize(p, pXInfo); - bIndex = 1; - pIter->nIndex++; - } - - if( pIter->eType==RBU_PK_WITHOUT_ROWID ){ - /* "PRAGMA index_list" includes the main PK b-tree */ - pIter->nIndex--; - } - - rbuFinalize(p, pList); - if( bIndex==0 ) pIter->abIndexed = 0; -} - +SQLITE_API int sqlite3rbu_create_vfs(const char *zName, const char *zParent); /* -** If they are not already populated, populate the pIter->azTblCol[], -** pIter->abTblPk[], pIter->nTblCol and pIter->bRowid variables according to -** the table (not index) that the iterator currently points to. +** Deregister and destroy an RBU vfs created by an earlier call to +** sqlite3rbu_create_vfs(). ** -** Return SQLITE_OK if successful, or an SQLite error code otherwise. If -** an error does occur, an error code and error message are also left in -** the RBU handle. +** VFS objects are not reference counted. If a VFS object is destroyed +** before all database handles that use it have been closed, the results +** are undefined. */ -static int rbuObjIterCacheTableInfo(sqlite3rbu *p, RbuObjIter *pIter){ - if( pIter->azTblCol==0 ){ - sqlite3_stmt *pStmt = 0; - int nCol = 0; - int i; /* for() loop iterator variable */ - int bRbuRowid = 0; /* If input table has column "rbu_rowid" */ - int iOrder = 0; - int iTnum = 0; - - /* Figure out the type of table this step will deal with. */ - assert( pIter->eType==0 ); - rbuTableType(p, pIter->zTbl, &pIter->eType, &iTnum, &pIter->iPkTnum); - if( p->rc==SQLITE_OK && pIter->eType==RBU_PK_NOTABLE ){ - p->rc = SQLITE_ERROR; - p->zErrmsg = sqlite3_mprintf("no such table: %s", pIter->zTbl); - } - if( p->rc ) return p->rc; - if( pIter->zIdx==0 ) pIter->iTnum = iTnum; - - assert( pIter->eType==RBU_PK_NONE || pIter->eType==RBU_PK_IPK - || pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_WITHOUT_ROWID - || pIter->eType==RBU_PK_VTAB - ); - - /* Populate the azTblCol[] and nTblCol variables based on the columns - ** of the input table. Ignore any input table columns that begin with - ** "rbu_". */ - p->rc = prepareFreeAndCollectError(p->dbRbu, &pStmt, &p->zErrmsg, - sqlite3_mprintf("SELECT * FROM '%q'", pIter->zDataTbl) - ); - if( p->rc==SQLITE_OK ){ - nCol = sqlite3_column_count(pStmt); - rbuAllocateIterArrays(p, pIter, nCol); - } - for(i=0; p->rc==SQLITE_OK && irc); - pIter->aiSrcOrder[pIter->nTblCol] = pIter->nTblCol; - pIter->azTblCol[pIter->nTblCol++] = zCopy; - } - else if( 0==sqlite3_stricmp("rbu_rowid", zName) ){ - bRbuRowid = 1; - } - } - sqlite3_finalize(pStmt); - pStmt = 0; +SQLITE_API void sqlite3rbu_destroy_vfs(const char *zName); - if( p->rc==SQLITE_OK - && rbuIsVacuum(p)==0 - && bRbuRowid!=(pIter->eType==RBU_PK_VTAB || pIter->eType==RBU_PK_NONE) - ){ - p->rc = SQLITE_ERROR; - p->zErrmsg = sqlite3_mprintf( - "table %q %s rbu_rowid column", pIter->zDataTbl, - (bRbuRowid ? "may not have" : "requires") - ); - } +#if 0 +} /* end of the 'extern "C"' block */ +#endif - /* Check that all non-HIDDEN columns in the destination table are also - ** present in the input table. Populate the abTblPk[], azTblType[] and - ** aiTblOrder[] arrays at the same time. */ - if( p->rc==SQLITE_OK ){ - p->rc = prepareFreeAndCollectError(p->dbMain, &pStmt, &p->zErrmsg, - sqlite3_mprintf("PRAGMA table_info(%Q)", pIter->zTbl) - ); - } - while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ - const char *zName = (const char*)sqlite3_column_text(pStmt, 1); - if( zName==0 ) break; /* An OOM - finalize() below returns S_NOMEM */ - for(i=iOrder; inTblCol; i++){ - if( 0==strcmp(zName, pIter->azTblCol[i]) ) break; - } - if( i==pIter->nTblCol ){ - p->rc = SQLITE_ERROR; - p->zErrmsg = sqlite3_mprintf("column missing from %q: %s", - pIter->zDataTbl, zName - ); - }else{ - int iPk = sqlite3_column_int(pStmt, 5); - int bNotNull = sqlite3_column_int(pStmt, 3); - const char *zType = (const char*)sqlite3_column_text(pStmt, 2); +#endif /* _SQLITE3RBU_H */ - if( i!=iOrder ){ - SWAP(int, pIter->aiSrcOrder[i], pIter->aiSrcOrder[iOrder]); - SWAP(char*, pIter->azTblCol[i], pIter->azTblCol[iOrder]); - } +/************** End of sqlite3rbu.h ******************************************/ +/************** Continuing where we left off in sqlite3rbu.c *****************/ - pIter->azTblType[iOrder] = rbuStrndup(zType, &p->rc); - pIter->abTblPk[iOrder] = (iPk!=0); - pIter->abNotNull[iOrder] = (u8)bNotNull || (iPk!=0); - iOrder++; - } - } +#if defined(_WIN32_WCE) +/* #include "windows.h" */ +#endif - rbuFinalize(p, pStmt); - rbuObjIterCacheIndexedCols(p, pIter); - assert( pIter->eType!=RBU_PK_VTAB || pIter->abIndexed==0 ); - assert( pIter->eType!=RBU_PK_VTAB || pIter->nIndex==0 ); - } +/* Maximum number of prepared UPDATE statements held by this module */ +#define SQLITE_RBU_UPDATE_CACHESIZE 16 - return p->rc; -} +/* Delta checksums disabled by default. Compile with -DRBU_ENABLE_DELTA_CKSUM +** to enable checksum verification. +*/ +#ifndef RBU_ENABLE_DELTA_CKSUM +# define RBU_ENABLE_DELTA_CKSUM 0 +#endif /* -** This function constructs and returns a pointer to a nul-terminated -** string containing some SQL clause or list based on one or more of the -** column names currently stored in the pIter->azTblCol[] array. +** Swap two objects of type TYPE. */ -static char *rbuObjIterGetCollist( - sqlite3rbu *p, /* RBU object */ - RbuObjIter *pIter /* Object iterator for column names */ -){ - char *zList = 0; - const char *zSep = ""; - int i; - for(i=0; inTblCol; i++){ - const char *z = pIter->azTblCol[i]; - zList = rbuMPrintf(p, "%z%s\"%w\"", zList, zSep, z); - zSep = ", "; - } - return zList; -} +#if !defined(SQLITE_AMALGAMATION) +# define SWAP(TYPE,A,B) {TYPE t=A; A=B; B=t;} +#endif /* -** This function is used to create a SELECT list (the list of SQL -** expressions that follows a SELECT keyword) for a SELECT statement -** used to read from an data_xxx or rbu_tmp_xxx table while updating the -** index object currently indicated by the iterator object passed as the -** second argument. A "PRAGMA index_xinfo = " statement is used -** to obtain the required information. +** The rbu_state table is used to save the state of a partially applied +** update so that it can be resumed later. The table consists of integer +** keys mapped to values as follows: ** -** If the index is of the following form: +** RBU_STATE_STAGE: +** May be set to integer values 1, 2, 4 or 5. As follows: +** 1: the *-rbu file is currently under construction. +** 2: the *-rbu file has been constructed, but not yet moved +** to the *-wal path. +** 4: the checkpoint is underway. +** 5: the rbu update has been checkpointed. ** -** CREATE INDEX i1 ON t1(c, b COLLATE nocase); +** RBU_STATE_TBL: +** Only valid if STAGE==1. The target database name of the table +** currently being written. ** -** and "t1" is a table with an explicit INTEGER PRIMARY KEY column -** "ipk", the returned string is: +** RBU_STATE_IDX: +** Only valid if STAGE==1. The target database name of the index +** currently being written, or NULL if the main table is currently being +** updated. ** -** "`c` COLLATE 'BINARY', `b` COLLATE 'NOCASE', `ipk` COLLATE 'BINARY'" +** RBU_STATE_ROW: +** Only valid if STAGE==1. Number of rows already processed for the current +** table/index. ** -** As well as the returned string, three other malloc'd strings are -** returned via output parameters. As follows: +** RBU_STATE_PROGRESS: +** Trbul number of sqlite3rbu_step() calls made so far as part of this +** rbu update. ** -** pzImposterCols: ... -** pzImposterPk: ... -** pzWhere: ... +** RBU_STATE_CKPT: +** Valid if STAGE==4. The 64-bit checksum associated with the wal-index +** header created by recovering the *-wal file. This is used to detect +** cases when another client appends frames to the *-wal file in the +** middle of an incremental checkpoint (an incremental checkpoint cannot +** be continued if this happens). +** +** RBU_STATE_COOKIE: +** Valid if STAGE==1. The current change-counter cookie value in the +** target db file. +** +** RBU_STATE_OALSZ: +** Valid if STAGE==1. The size in bytes of the *-oal file. +** +** RBU_STATE_DATATBL: +** Only valid if STAGE==1. The RBU database name of the table +** currently being read. */ -static char *rbuObjIterGetIndexCols( - sqlite3rbu *p, /* RBU object */ - RbuObjIter *pIter, /* Object iterator for column names */ - char **pzImposterCols, /* OUT: Columns for imposter table */ - char **pzImposterPk, /* OUT: Imposter PK clause */ - char **pzWhere, /* OUT: WHERE clause */ - int *pnBind /* OUT: Trbul number of columns */ -){ - int rc = p->rc; /* Error code */ - int rc2; /* sqlite3_finalize() return code */ - char *zRet = 0; /* String to return */ - char *zImpCols = 0; /* String to return via *pzImposterCols */ - char *zImpPK = 0; /* String to return via *pzImposterPK */ - char *zWhere = 0; /* String to return via *pzWhere */ - int nBind = 0; /* Value to return via *pnBind */ - const char *zCom = ""; /* Set to ", " later on */ - const char *zAnd = ""; /* Set to " AND " later on */ - sqlite3_stmt *pXInfo = 0; /* PRAGMA index_xinfo = ? */ - - if( rc==SQLITE_OK ){ - assert( p->zErrmsg==0 ); - rc = prepareFreeAndCollectError(p->dbMain, &pXInfo, &p->zErrmsg, - sqlite3_mprintf("PRAGMA main.index_xinfo = %Q", pIter->zIdx) - ); - } +#define RBU_STATE_STAGE 1 +#define RBU_STATE_TBL 2 +#define RBU_STATE_IDX 3 +#define RBU_STATE_ROW 4 +#define RBU_STATE_PROGRESS 5 +#define RBU_STATE_CKPT 6 +#define RBU_STATE_COOKIE 7 +#define RBU_STATE_OALSZ 8 +#define RBU_STATE_PHASEONESTEP 9 +#define RBU_STATE_DATATBL 10 - while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXInfo) ){ - int iCid = sqlite3_column_int(pXInfo, 1); - int bDesc = sqlite3_column_int(pXInfo, 3); - const char *zCollate = (const char*)sqlite3_column_text(pXInfo, 4); - const char *zCol; - const char *zType; +#define RBU_STAGE_OAL 1 +#define RBU_STAGE_MOVE 2 +#define RBU_STAGE_CAPTURE 3 +#define RBU_STAGE_CKPT 4 +#define RBU_STAGE_DONE 5 - if( iCid<0 ){ - /* An integer primary key. If the table has an explicit IPK, use - ** its name. Otherwise, use "rbu_rowid". */ - if( pIter->eType==RBU_PK_IPK ){ - int i; - for(i=0; pIter->abTblPk[i]==0; i++); - assert( inTblCol ); - zCol = pIter->azTblCol[i]; - }else if( rbuIsVacuum(p) ){ - zCol = "_rowid_"; - }else{ - zCol = "rbu_rowid"; - } - zType = "INTEGER"; - }else{ - zCol = pIter->azTblCol[iCid]; - zType = pIter->azTblType[iCid]; - } - zRet = sqlite3_mprintf("%z%s\"%w\" COLLATE %Q", zRet, zCom, zCol, zCollate); - if( pIter->bUnique==0 || sqlite3_column_int(pXInfo, 5) ){ - const char *zOrder = (bDesc ? " DESC" : ""); - zImpPK = sqlite3_mprintf("%z%s\"rbu_imp_%d%w\"%s", - zImpPK, zCom, nBind, zCol, zOrder - ); - } - zImpCols = sqlite3_mprintf("%z%s\"rbu_imp_%d%w\" %s COLLATE %Q", - zImpCols, zCom, nBind, zCol, zType, zCollate - ); - zWhere = sqlite3_mprintf( - "%z%s\"rbu_imp_%d%w\" IS ?", zWhere, zAnd, nBind, zCol - ); - if( zRet==0 || zImpPK==0 || zImpCols==0 || zWhere==0 ) rc = SQLITE_NOMEM; - zCom = ", "; - zAnd = " AND "; - nBind++; - } +#define RBU_CREATE_STATE \ + "CREATE TABLE IF NOT EXISTS %s.rbu_state(k INTEGER PRIMARY KEY, v)" - rc2 = sqlite3_finalize(pXInfo); - if( rc==SQLITE_OK ) rc = rc2; +typedef struct RbuFrame RbuFrame; +typedef struct RbuObjIter RbuObjIter; +typedef struct RbuState RbuState; +typedef struct RbuSpan RbuSpan; +typedef struct rbu_vfs rbu_vfs; +typedef struct rbu_file rbu_file; +typedef struct RbuUpdateStmt RbuUpdateStmt; - if( rc!=SQLITE_OK ){ - sqlite3_free(zRet); - sqlite3_free(zImpCols); - sqlite3_free(zImpPK); - sqlite3_free(zWhere); - zRet = 0; - zImpCols = 0; - zImpPK = 0; - zWhere = 0; - p->rc = rc; - } +#if !defined(SQLITE_AMALGAMATION) +typedef unsigned int u32; +typedef unsigned short u16; +typedef unsigned char u8; +typedef sqlite3_int64 i64; +#endif - *pzImposterCols = zImpCols; - *pzImposterPk = zImpPK; - *pzWhere = zWhere; - *pnBind = nBind; - return zRet; -} +/* +** These values must match the values defined in wal.c for the equivalent +** locks. These are not magic numbers as they are part of the SQLite file +** format. +*/ +#define WAL_LOCK_WRITE 0 +#define WAL_LOCK_CKPT 1 +#define WAL_LOCK_READ0 3 + +#define SQLITE_FCNTL_RBUCNT 5149216 /* -** Assuming the current table columns are "a", "b" and "c", and the zObj -** paramter is passed "old", return a string of the form: -** -** "old.a, old.b, old.b" -** -** With the column names escaped. -** -** For tables with implicit rowids - RBU_PK_EXTERNAL and RBU_PK_NONE, append -** the text ", old._rowid_" to the returned value. +** A structure to store values read from the rbu_state table in memory. */ -static char *rbuObjIterGetOldlist( - sqlite3rbu *p, - RbuObjIter *pIter, - const char *zObj -){ - char *zList = 0; - if( p->rc==SQLITE_OK && pIter->abIndexed ){ - const char *zS = ""; - int i; - for(i=0; inTblCol; i++){ - if( pIter->abIndexed[i] ){ - const char *zCol = pIter->azTblCol[i]; - zList = sqlite3_mprintf("%z%s%s.\"%w\"", zList, zS, zObj, zCol); - }else{ - zList = sqlite3_mprintf("%z%sNULL", zList, zS); - } - zS = ", "; - if( zList==0 ){ - p->rc = SQLITE_NOMEM; - break; - } - } +struct RbuState { + int eStage; + char *zTbl; + char *zDataTbl; + char *zIdx; + i64 iWalCksum; + int nRow; + i64 nProgress; + u32 iCookie; + i64 iOalSz; + i64 nPhaseOneStep; +}; - /* For a table with implicit rowids, append "old._rowid_" to the list. */ - if( pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_NONE ){ - zList = rbuMPrintf(p, "%z, %s._rowid_", zList, zObj); - } - } - return zList; -} +struct RbuUpdateStmt { + char *zMask; /* Copy of update mask used with pUpdate */ + sqlite3_stmt *pUpdate; /* Last update statement (or NULL) */ + RbuUpdateStmt *pNext; +}; + +struct RbuSpan { + const char *zSpan; + int nSpan; +}; /* -** Return an expression that can be used in a WHERE clause to match the -** primary key of the current table. For example, if the table is: +** An iterator of this type is used to iterate through all objects in +** the target database that require updating. For each such table, the +** iterator visits, in order: ** -** CREATE TABLE t1(a, b, c, PRIMARY KEY(b, c)); +** * the table itself, +** * each index of the table (zero or more points to visit), and +** * a special "cleanup table" state. ** -** Return the string: +** abIndexed: +** If the table has no indexes on it, abIndexed is set to NULL. Otherwise, +** it points to an array of flags nTblCol elements in size. The flag is +** set for each column that is either a part of the PK or a part of an +** index. Or clear otherwise. ** -** "b = ?1 AND c = ?2" +** If there are one or more partial indexes on the table, all fields of +** this array set set to 1. This is because in that case, the module has +** no way to tell which fields will be required to add and remove entries +** from the partial indexes. +** */ -static char *rbuObjIterGetWhere( - sqlite3rbu *p, - RbuObjIter *pIter -){ - char *zList = 0; - if( pIter->eType==RBU_PK_VTAB || pIter->eType==RBU_PK_NONE ){ - zList = rbuMPrintf(p, "_rowid_ = ?%d", pIter->nTblCol+1); - }else if( pIter->eType==RBU_PK_EXTERNAL ){ - const char *zSep = ""; - int i; - for(i=0; inTblCol; i++){ - if( pIter->abTblPk[i] ){ - zList = rbuMPrintf(p, "%z%sc%d=?%d", zList, zSep, i, i+1); - zSep = " AND "; - } - } - zList = rbuMPrintf(p, - "_rowid_ = (SELECT id FROM rbu_imposter2 WHERE %z)", zList - ); +struct RbuObjIter { + sqlite3_stmt *pTblIter; /* Iterate through tables */ + sqlite3_stmt *pIdxIter; /* Index iterator */ + int nTblCol; /* Size of azTblCol[] array */ + char **azTblCol; /* Array of unquoted target column names */ + char **azTblType; /* Array of target column types */ + int *aiSrcOrder; /* src table col -> target table col */ + u8 *abTblPk; /* Array of flags, set on target PK columns */ + u8 *abNotNull; /* Array of flags, set on NOT NULL columns */ + u8 *abIndexed; /* Array of flags, set on indexed & PK cols */ + int eType; /* Table type - an RBU_PK_XXX value */ - }else{ - const char *zSep = ""; - int i; - for(i=0; inTblCol; i++){ - if( pIter->abTblPk[i] ){ - const char *zCol = pIter->azTblCol[i]; - zList = rbuMPrintf(p, "%z%s\"%w\"=?%d", zList, zSep, zCol, i+1); - zSep = " AND "; - } - } - } - return zList; -} + /* Output variables. zTbl==0 implies EOF. */ + int bCleanup; /* True in "cleanup" state */ + const char *zTbl; /* Name of target db table */ + const char *zDataTbl; /* Name of rbu db table (or null) */ + const char *zIdx; /* Name of target db index (or null) */ + int iTnum; /* Root page of current object */ + int iPkTnum; /* If eType==EXTERNAL, root of PK index */ + int bUnique; /* Current index is unique */ + int nIndex; /* Number of aux. indexes on table zTbl */ -/* -** The SELECT statement iterating through the keys for the current object -** (p->objiter.pSelect) currently points to a valid row. However, there -** is something wrong with the rbu_control value in the rbu_control value -** stored in the (p->nCol+1)'th column. Set the error code and error message -** of the RBU handle to something reflecting this. -*/ -static void rbuBadControlError(sqlite3rbu *p){ - p->rc = SQLITE_ERROR; - p->zErrmsg = sqlite3_mprintf("invalid rbu_control value"); -} + /* Statements created by rbuObjIterPrepareAll() */ + int nCol; /* Number of columns in current object */ + sqlite3_stmt *pSelect; /* Source data */ + sqlite3_stmt *pInsert; /* Statement for INSERT operations */ + sqlite3_stmt *pDelete; /* Statement for DELETE ops */ + sqlite3_stmt *pTmpInsert; /* Insert into rbu_tmp_$zDataTbl */ + int nIdxCol; + RbuSpan *aIdxCol; + char *zIdxSql; + /* Last UPDATE used (for PK b-tree updates only), or NULL. */ + RbuUpdateStmt *pRbuUpdate; +}; /* -** Return a nul-terminated string containing the comma separated list of -** assignments that should be included following the "SET" keyword of -** an UPDATE statement used to update the table object that the iterator -** passed as the second argument currently points to if the rbu_control -** column of the data_xxx table entry is set to zMask. -** -** The memory for the returned string is obtained from sqlite3_malloc(). -** It is the responsibility of the caller to eventually free it using -** sqlite3_free(). +** Values for RbuObjIter.eType ** -** If an OOM error is encountered when allocating space for the new -** string, an error code is left in the rbu handle passed as the first -** argument and NULL is returned. Or, if an error has already occurred -** when this function is called, NULL is returned immediately, without -** attempting the allocation or modifying the stored error code. +** 0: Table does not exist (error) +** 1: Table has an implicit rowid. +** 2: Table has an explicit IPK column. +** 3: Table has an external PK index. +** 4: Table is WITHOUT ROWID. +** 5: Table is a virtual table. */ -static char *rbuObjIterGetSetlist( - sqlite3rbu *p, - RbuObjIter *pIter, - const char *zMask -){ - char *zList = 0; - if( p->rc==SQLITE_OK ){ - int i; +#define RBU_PK_NOTABLE 0 +#define RBU_PK_NONE 1 +#define RBU_PK_IPK 2 +#define RBU_PK_EXTERNAL 3 +#define RBU_PK_WITHOUT_ROWID 4 +#define RBU_PK_VTAB 5 - if( (int)strlen(zMask)!=pIter->nTblCol ){ - rbuBadControlError(p); - }else{ - const char *zSep = ""; - for(i=0; inTblCol; i++){ - char c = zMask[pIter->aiSrcOrder[i]]; - if( c=='x' ){ - zList = rbuMPrintf(p, "%z%s\"%w\"=?%d", - zList, zSep, pIter->azTblCol[i], i+1 - ); - zSep = ", "; - } - else if( c=='d' ){ - zList = rbuMPrintf(p, "%z%s\"%w\"=rbu_delta(\"%w\", ?%d)", - zList, zSep, pIter->azTblCol[i], pIter->azTblCol[i], i+1 - ); - zSep = ", "; - } - else if( c=='f' ){ - zList = rbuMPrintf(p, "%z%s\"%w\"=rbu_fossil_delta(\"%w\", ?%d)", - zList, zSep, pIter->azTblCol[i], pIter->azTblCol[i], i+1 - ); - zSep = ", "; - } - } - } - } - return zList; -} /* -** Return a nul-terminated string consisting of nByte comma separated -** "?" expressions. For example, if nByte is 3, return a pointer to -** a buffer containing the string "?,?,?". -** -** The memory for the returned string is obtained from sqlite3_malloc(). -** It is the responsibility of the caller to eventually free it using -** sqlite3_free(). -** -** If an OOM error is encountered when allocating space for the new -** string, an error code is left in the rbu handle passed as the first -** argument and NULL is returned. Or, if an error has already occurred -** when this function is called, NULL is returned immediately, without -** attempting the allocation or modifying the stored error code. +** Within the RBU_STAGE_OAL stage, each call to sqlite3rbu_step() performs +** one of the following operations. */ -static char *rbuObjIterGetBindlist(sqlite3rbu *p, int nBind){ - char *zRet = 0; - int nByte = nBind*2 + 1; +#define RBU_INSERT 1 /* Insert on a main table b-tree */ +#define RBU_DELETE 2 /* Delete a row from a main table b-tree */ +#define RBU_REPLACE 3 /* Delete and then insert a row */ +#define RBU_IDX_DELETE 4 /* Delete a row from an aux. index b-tree */ +#define RBU_IDX_INSERT 5 /* Insert on an aux. index b-tree */ - zRet = (char*)rbuMalloc(p, nByte); - if( zRet ){ - int i; - for(i=0; izIdx==0 ); - if( p->rc==SQLITE_OK ){ - const char *zSep = "PRIMARY KEY("; - sqlite3_stmt *pXList = 0; /* PRAGMA index_list = (pIter->zTbl) */ - sqlite3_stmt *pXInfo = 0; /* PRAGMA index_xinfo = */ - - p->rc = prepareFreeAndCollectError(p->dbMain, &pXList, &p->zErrmsg, - sqlite3_mprintf("PRAGMA main.index_list = %Q", pIter->zTbl) - ); - while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXList) ){ - const char *zOrig = (const char*)sqlite3_column_text(pXList,3); - if( zOrig && strcmp(zOrig, "pk")==0 ){ - const char *zIdx = (const char*)sqlite3_column_text(pXList,1); - if( zIdx ){ - p->rc = prepareFreeAndCollectError(p->dbMain, &pXInfo, &p->zErrmsg, - sqlite3_mprintf("PRAGMA main.index_xinfo = %Q", zIdx) - ); - } - break; - } - } - rbuFinalize(p, pXList); - - while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXInfo) ){ - if( sqlite3_column_int(pXInfo, 5) ){ - /* int iCid = sqlite3_column_int(pXInfo, 0); */ - const char *zCol = (const char*)sqlite3_column_text(pXInfo, 2); - const char *zDesc = sqlite3_column_int(pXInfo, 3) ? " DESC" : ""; - z = rbuMPrintf(p, "%z%s\"%w\"%s", z, zSep, zCol, zDesc); - zSep = ", "; - } - } - z = rbuMPrintf(p, "%z)", z); - rbuFinalize(p, pXInfo); - } - return z; -} +struct RbuFrame { + u32 iDbPage; + u32 iWalFrame; +}; /* -** This function creates the second imposter table used when writing to -** a table b-tree where the table has an external primary key. If the -** iterator passed as the second argument does not currently point to -** a table (not index) with an external primary key, this function is a -** no-op. +** RBU handle. ** -** Assuming the iterator does point to a table with an external PK, this -** function creates a WITHOUT ROWID imposter table named "rbu_imposter2" -** used to access that PK index. For example, if the target table is -** declared as follows: +** nPhaseOneStep: +** If the RBU database contains an rbu_count table, this value is set to +** a running estimate of the number of b-tree operations required to +** finish populating the *-oal file. This allows the sqlite3_bp_progress() +** API to calculate the permyriadage progress of populating the *-oal file +** using the formula: ** -** CREATE TABLE t1(a, b TEXT, c REAL, PRIMARY KEY(b, c)); +** permyriadage = (10000 * nProgress) / nPhaseOneStep ** -** then the imposter table schema is: +** nPhaseOneStep is initialized to the sum of: ** -** CREATE TABLE rbu_imposter2(c1 TEXT, c2 REAL, id INTEGER) WITHOUT ROWID; +** nRow * (nIndex + 1) +** +** for all source tables in the RBU database, where nRow is the number +** of rows in the source table and nIndex the number of indexes on the +** corresponding target database table. +** +** This estimate is accurate if the RBU update consists entirely of +** INSERT operations. However, it is inaccurate if: +** +** * the RBU update contains any UPDATE operations. If the PK specified +** for an UPDATE operation does not exist in the target table, then +** no b-tree operations are required on index b-trees. Or if the +** specified PK does exist, then (nIndex*2) such operations are +** required (one delete and one insert on each index b-tree). ** +** * the RBU update contains any DELETE operations for which the specified +** PK does not exist. In this case no operations are required on index +** b-trees. +** +** * the RBU update contains REPLACE operations. These are similar to +** UPDATE operations. +** +** nPhaseOneStep is updated to account for the conditions above during the +** first pass of each source table. The updated nPhaseOneStep value is +** stored in the rbu_state table if the RBU update is suspended. */ -static void rbuCreateImposterTable2(sqlite3rbu *p, RbuObjIter *pIter){ - if( p->rc==SQLITE_OK && pIter->eType==RBU_PK_EXTERNAL ){ - int tnum = pIter->iPkTnum; /* Root page of PK index */ - sqlite3_stmt *pQuery = 0; /* SELECT name ... WHERE rootpage = $tnum */ - const char *zIdx = 0; /* Name of PK index */ - sqlite3_stmt *pXInfo = 0; /* PRAGMA main.index_xinfo = $zIdx */ - const char *zComma = ""; - char *zCols = 0; /* Used to build up list of table cols */ - char *zPk = 0; /* Used to build up table PK declaration */ - - /* Figure out the name of the primary key index for the current table. - ** This is needed for the argument to "PRAGMA index_xinfo". Set - ** zIdx to point to a nul-terminated string containing this name. */ - p->rc = prepareAndCollectError(p->dbMain, &pQuery, &p->zErrmsg, - "SELECT name FROM sqlite_master WHERE rootpage = ?" - ); - if( p->rc==SQLITE_OK ){ - sqlite3_bind_int(pQuery, 1, tnum); - if( SQLITE_ROW==sqlite3_step(pQuery) ){ - zIdx = (const char*)sqlite3_column_text(pQuery, 0); - } - } - if( zIdx ){ - p->rc = prepareFreeAndCollectError(p->dbMain, &pXInfo, &p->zErrmsg, - sqlite3_mprintf("PRAGMA main.index_xinfo = %Q", zIdx) - ); - } - rbuFinalize(p, pQuery); +struct sqlite3rbu { + int eStage; /* Value of RBU_STATE_STAGE field */ + sqlite3 *dbMain; /* target database handle */ + sqlite3 *dbRbu; /* rbu database handle */ + char *zTarget; /* Path to target db */ + char *zRbu; /* Path to rbu db */ + char *zState; /* Path to state db (or NULL if zRbu) */ + char zStateDb[5]; /* Db name for state ("stat" or "main") */ + int rc; /* Value returned by last rbu_step() call */ + char *zErrmsg; /* Error message if rc!=SQLITE_OK */ + int nStep; /* Rows processed for current object */ + int nProgress; /* Rows processed for all objects */ + RbuObjIter objiter; /* Iterator for skipping through tbl/idx */ + const char *zVfsName; /* Name of automatically created rbu vfs */ + rbu_file *pTargetFd; /* File handle open on target db */ + int nPagePerSector; /* Pages per sector for pTargetFd */ + i64 iOalSz; + i64 nPhaseOneStep; - while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXInfo) ){ - int bKey = sqlite3_column_int(pXInfo, 5); - if( bKey ){ - int iCid = sqlite3_column_int(pXInfo, 1); - int bDesc = sqlite3_column_int(pXInfo, 3); - const char *zCollate = (const char*)sqlite3_column_text(pXInfo, 4); - zCols = rbuMPrintf(p, "%z%sc%d %s COLLATE %Q", zCols, zComma, - iCid, pIter->azTblType[iCid], zCollate - ); - zPk = rbuMPrintf(p, "%z%sc%d%s", zPk, zComma, iCid, bDesc?" DESC":""); - zComma = ", "; - } - } - zCols = rbuMPrintf(p, "%z, id INTEGER", zCols); - rbuFinalize(p, pXInfo); + /* The following state variables are used as part of the incremental + ** checkpoint stage (eStage==RBU_STAGE_CKPT). See comments surrounding + ** function rbuSetupCheckpoint() for details. */ + u32 iMaxFrame; /* Largest iWalFrame value in aFrame[] */ + u32 mLock; + int nFrame; /* Entries in aFrame[] array */ + int nFrameAlloc; /* Allocated size of aFrame[] array */ + RbuFrame *aFrame; + int pgsz; + u8 *aBuf; + i64 iWalCksum; + i64 szTemp; /* Current size of all temp files in use */ + i64 szTempLimit; /* Total size limit for temp files */ - sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 1, tnum); - rbuMPrintfExec(p, p->dbMain, - "CREATE TABLE rbu_imposter2(%z, PRIMARY KEY(%z)) WITHOUT ROWID", - zCols, zPk - ); - sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 0, 0); - } -} + /* Used in RBU vacuum mode only */ + int nRbu; /* Number of RBU VFS in the stack */ + rbu_file *pRbuFd; /* Fd for main db of dbRbu */ +}; /* -** If an error has already occurred when this function is called, it -** immediately returns zero (without doing any work). Or, if an error -** occurs during the execution of this function, it sets the error code -** in the sqlite3rbu object indicated by the first argument and returns -** zero. +** An rbu VFS is implemented using an instance of this structure. ** -** The iterator passed as the second argument is guaranteed to point to -** a table (not an index) when this function is called. This function -** attempts to create any imposter table required to write to the main -** table b-tree of the table before returning. Non-zero is returned if -** an imposter table are created, or zero otherwise. +** Variable pRbu is only non-NULL for automatically created RBU VFS objects. +** It is NULL for RBU VFS objects created explicitly using +** sqlite3rbu_create_vfs(). It is used to track the total amount of temp +** space used by the RBU handle. +*/ +struct rbu_vfs { + sqlite3_vfs base; /* rbu VFS shim methods */ + sqlite3_vfs *pRealVfs; /* Underlying VFS */ + sqlite3_mutex *mutex; /* Mutex to protect pMain */ + sqlite3rbu *pRbu; /* Owner RBU object */ + rbu_file *pMain; /* List of main db files */ + rbu_file *pMainRbu; /* List of main db files with pRbu!=0 */ +}; + +/* +** Each file opened by an rbu VFS is represented by an instance of +** the following structure. ** -** An imposter table is required in all cases except RBU_PK_VTAB. Only -** virtual tables are written to directly. The imposter table has the -** same schema as the actual target table (less any UNIQUE constraints). -** More precisely, the "same schema" means the same columns, types, -** collation sequences. For tables that do not have an external PRIMARY -** KEY, it also means the same PRIMARY KEY declaration. +** If this is a temporary file (pRbu!=0 && flags&DELETE_ON_CLOSE), variable +** "sz" is set to the current size of the database file. */ -static void rbuCreateImposterTable(sqlite3rbu *p, RbuObjIter *pIter){ - if( p->rc==SQLITE_OK && pIter->eType!=RBU_PK_VTAB ){ - int tnum = pIter->iTnum; - const char *zComma = ""; - char *zSql = 0; - int iCol; - sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 0, 1); +struct rbu_file { + sqlite3_file base; /* sqlite3_file methods */ + sqlite3_file *pReal; /* Underlying file handle */ + rbu_vfs *pRbuVfs; /* Pointer to the rbu_vfs object */ + sqlite3rbu *pRbu; /* Pointer to rbu object (rbu target only) */ + i64 sz; /* Size of file in bytes (temp only) */ - for(iCol=0; p->rc==SQLITE_OK && iColnTblCol; iCol++){ - const char *zPk = ""; - const char *zCol = pIter->azTblCol[iCol]; - const char *zColl = 0; + int openFlags; /* Flags this file was opened with */ + u32 iCookie; /* Cookie value for main db files */ + u8 iWriteVer; /* "write-version" value for main db files */ + u8 bNolock; /* True to fail EXCLUSIVE locks */ - p->rc = sqlite3_table_column_metadata( - p->dbMain, "main", pIter->zTbl, zCol, 0, &zColl, 0, 0, 0 - ); + int nShm; /* Number of entries in apShm[] array */ + char **apShm; /* Array of mmap'd *-shm regions */ + char *zDel; /* Delete this when closing file */ - if( pIter->eType==RBU_PK_IPK && pIter->abTblPk[iCol] ){ - /* If the target table column is an "INTEGER PRIMARY KEY", add - ** "PRIMARY KEY" to the imposter table column declaration. */ - zPk = "PRIMARY KEY "; - } - zSql = rbuMPrintf(p, "%z%s\"%w\" %s %sCOLLATE %Q%s", - zSql, zComma, zCol, pIter->azTblType[iCol], zPk, zColl, - (pIter->abNotNull[iCol] ? " NOT NULL" : "") - ); - zComma = ", "; - } + const char *zWal; /* Wal filename for this main db file */ + rbu_file *pWalFd; /* Wal file descriptor for this main db */ + rbu_file *pMainNext; /* Next MAIN_DB file */ + rbu_file *pMainRbuNext; /* Next MAIN_DB file with pRbu!=0 */ +}; - if( pIter->eType==RBU_PK_WITHOUT_ROWID ){ - char *zPk = rbuWithoutRowidPK(p, pIter); - if( zPk ){ - zSql = rbuMPrintf(p, "%z, %z", zSql, zPk); - } - } +/* +** True for an RBU vacuum handle, or false otherwise. +*/ +#define rbuIsVacuum(p) ((p)->zTarget==0) - sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 1, tnum); - rbuMPrintfExec(p, p->dbMain, "CREATE TABLE \"rbu_imp_%w\"(%z)%s", - pIter->zTbl, zSql, - (pIter->eType==RBU_PK_WITHOUT_ROWID ? " WITHOUT ROWID" : "") - ); - sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 0, 0); - } -} -/* -** Prepare a statement used to insert rows into the "rbu_tmp_xxx" table. -** Specifically a statement of the form: +/************************************************************************* +** The following three functions, found below: ** -** INSERT INTO rbu_tmp_xxx VALUES(?, ?, ? ...); +** rbuDeltaGetInt() +** rbuDeltaChecksum() +** rbuDeltaApply() ** -** The number of bound variables is equal to the number of columns in -** the target table, plus one (for the rbu_control column), plus one more -** (for the rbu_rowid column) if the target table is an implicit IPK or -** virtual table. +** are lifted from the fossil source code (http://fossil-scm.org). They +** are used to implement the scalar SQL function rbu_fossil_delta(). */ -static void rbuObjIterPrepareTmpInsert( - sqlite3rbu *p, - RbuObjIter *pIter, - const char *zCollist, - const char *zRbuRowid -){ - int bRbuRowid = (pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_NONE); - char *zBind = rbuObjIterGetBindlist(p, pIter->nTblCol + 1 + bRbuRowid); - if( zBind ){ - assert( pIter->pTmpInsert==0 ); - p->rc = prepareFreeAndCollectError( - p->dbRbu, &pIter->pTmpInsert, &p->zErrmsg, sqlite3_mprintf( - "INSERT INTO %s.'rbu_tmp_%q'(rbu_control,%s%s) VALUES(%z)", - p->zStateDb, pIter->zDataTbl, zCollist, zRbuRowid, zBind - )); - } -} - -static void rbuTmpInsertFunc( - sqlite3_context *pCtx, - int nVal, - sqlite3_value **apVal -){ - sqlite3rbu *p = sqlite3_user_data(pCtx); - int rc = SQLITE_OK; - int i; - assert( sqlite3_value_int(apVal[0])!=0 - || p->objiter.eType==RBU_PK_EXTERNAL - || p->objiter.eType==RBU_PK_NONE - ); - if( sqlite3_value_int(apVal[0])!=0 ){ - p->nPhaseOneStep += p->objiter.nIndex; +/* +** Read bytes from *pz and convert them into a positive integer. When +** finished, leave *pz pointing to the first character past the end of +** the integer. The *pLen parameter holds the length of the string +** in *pz and is decremented once for each character in the integer. +*/ +static unsigned int rbuDeltaGetInt(const char **pz, int *pLen){ + static const signed char zValue[] = { + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, -1, -1, -1, -1, -1, -1, + -1, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, + 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, -1, -1, -1, -1, 36, + -1, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, + 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, -1, -1, -1, 63, -1, + }; + unsigned int v = 0; + int c; + unsigned char *z = (unsigned char*)*pz; + unsigned char *zStart = z; + while( (c = zValue[0x7f&*(z++)])>=0 ){ + v = (v<<6) + c; } + z--; + *pLen -= z - zStart; + *pz = (char*)z; + return v; +} - for(i=0; rc==SQLITE_OK && iobjiter.pTmpInsert, i+1, apVal[i]); +#if RBU_ENABLE_DELTA_CKSUM +/* +** Compute a 32-bit checksum on the N-byte buffer. Return the result. +*/ +static unsigned int rbuDeltaChecksum(const char *zIn, size_t N){ + const unsigned char *z = (const unsigned char *)zIn; + unsigned sum0 = 0; + unsigned sum1 = 0; + unsigned sum2 = 0; + unsigned sum3 = 0; + while(N >= 16){ + sum0 += ((unsigned)z[0] + z[4] + z[8] + z[12]); + sum1 += ((unsigned)z[1] + z[5] + z[9] + z[13]); + sum2 += ((unsigned)z[2] + z[6] + z[10]+ z[14]); + sum3 += ((unsigned)z[3] + z[7] + z[11]+ z[15]); + z += 16; + N -= 16; } - if( rc==SQLITE_OK ){ - sqlite3_step(p->objiter.pTmpInsert); - rc = sqlite3_reset(p->objiter.pTmpInsert); + while(N >= 4){ + sum0 += z[0]; + sum1 += z[1]; + sum2 += z[2]; + sum3 += z[3]; + z += 4; + N -= 4; } - - if( rc!=SQLITE_OK ){ - sqlite3_result_error_code(pCtx, rc); + sum3 += (sum2 << 8) + (sum1 << 16) + (sum0 << 24); + switch(N){ + case 3: sum3 += (z[2] << 8); + case 2: sum3 += (z[1] << 16); + case 1: sum3 += (z[0] << 24); + default: ; } + return sum3; } +#endif /* -** Ensure that the SQLite statement handles required to update the -** target database object currently indicated by the iterator passed -** as the second argument are available. +** Apply a delta. +** +** The output buffer should be big enough to hold the whole output +** file and a NUL terminator at the end. The delta_output_size() +** routine will determine this size for you. +** +** The delta string should be null-terminated. But the delta string +** may contain embedded NUL characters (if the input and output are +** binary files) so we also have to pass in the length of the delta in +** the lenDelta parameter. +** +** This function returns the size of the output file in bytes (excluding +** the final NUL terminator character). Except, if the delta string is +** malformed or intended for use with a source file other than zSrc, +** then this routine returns -1. +** +** Refer to the delta_create() documentation above for a description +** of the delta file format. */ -static int rbuObjIterPrepareAll( - sqlite3rbu *p, - RbuObjIter *pIter, - int nOffset /* Add "LIMIT -1 OFFSET $nOffset" to SELECT */ +static int rbuDeltaApply( + const char *zSrc, /* The source or pattern file */ + int lenSrc, /* Length of the source file */ + const char *zDelta, /* Delta to apply to the pattern */ + int lenDelta, /* Length of the delta */ + char *zOut /* Write the output into this preallocated buffer */ ){ - assert( pIter->bCleanup==0 ); - if( pIter->pSelect==0 && rbuObjIterCacheTableInfo(p, pIter)==SQLITE_OK ){ - const int tnum = pIter->iTnum; - char *zCollist = 0; /* List of indexed columns */ - char **pz = &p->zErrmsg; - const char *zIdx = pIter->zIdx; - char *zLimit = 0; - - if( nOffset ){ - zLimit = sqlite3_mprintf(" LIMIT -1 OFFSET %d", nOffset); - if( !zLimit ) p->rc = SQLITE_NOMEM; - } - - if( zIdx ){ - const char *zTbl = pIter->zTbl; - char *zImposterCols = 0; /* Columns for imposter table */ - char *zImposterPK = 0; /* Primary key declaration for imposter */ - char *zWhere = 0; /* WHERE clause on PK columns */ - char *zBind = 0; - int nBind = 0; - - assert( pIter->eType!=RBU_PK_VTAB ); - zCollist = rbuObjIterGetIndexCols( - p, pIter, &zImposterCols, &zImposterPK, &zWhere, &nBind - ); - zBind = rbuObjIterGetBindlist(p, nBind); - - /* Create the imposter table used to write to this index. */ - sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 0, 1); - sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 1,tnum); - rbuMPrintfExec(p, p->dbMain, - "CREATE TABLE \"rbu_imp_%w\"( %s, PRIMARY KEY( %s ) ) WITHOUT ROWID", - zTbl, zImposterCols, zImposterPK - ); - sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 0, 0); - - /* Create the statement to insert index entries */ - pIter->nCol = nBind; - if( p->rc==SQLITE_OK ){ - p->rc = prepareFreeAndCollectError( - p->dbMain, &pIter->pInsert, &p->zErrmsg, - sqlite3_mprintf("INSERT INTO \"rbu_imp_%w\" VALUES(%s)", zTbl, zBind) - ); - } - - /* And to delete index entries */ - if( rbuIsVacuum(p)==0 && p->rc==SQLITE_OK ){ - p->rc = prepareFreeAndCollectError( - p->dbMain, &pIter->pDelete, &p->zErrmsg, - sqlite3_mprintf("DELETE FROM \"rbu_imp_%w\" WHERE %s", zTbl, zWhere) - ); - } - - /* Create the SELECT statement to read keys in sorted order */ - if( p->rc==SQLITE_OK ){ - char *zSql; - if( rbuIsVacuum(p) ){ - zSql = sqlite3_mprintf( - "SELECT %s, 0 AS rbu_control FROM '%q' ORDER BY %s%s", - zCollist, - pIter->zDataTbl, - zCollist, zLimit - ); - }else + unsigned int limit; + unsigned int total = 0; +#if RBU_ENABLE_DELTA_CKSUM + char *zOrigOut = zOut; +#endif - if( pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_NONE ){ - zSql = sqlite3_mprintf( - "SELECT %s, rbu_control FROM %s.'rbu_tmp_%q' ORDER BY %s%s", - zCollist, p->zStateDb, pIter->zDataTbl, - zCollist, zLimit - ); - }else{ - zSql = sqlite3_mprintf( - "SELECT %s, rbu_control FROM %s.'rbu_tmp_%q' " - "UNION ALL " - "SELECT %s, rbu_control FROM '%q' " - "WHERE typeof(rbu_control)='integer' AND rbu_control!=1 " - "ORDER BY %s%s", - zCollist, p->zStateDb, pIter->zDataTbl, - zCollist, pIter->zDataTbl, - zCollist, zLimit - ); + limit = rbuDeltaGetInt(&zDelta, &lenDelta); + if( *zDelta!='\n' ){ + /* ERROR: size integer not terminated by "\n" */ + return -1; + } + zDelta++; lenDelta--; + while( *zDelta && lenDelta>0 ){ + unsigned int cnt, ofst; + cnt = rbuDeltaGetInt(&zDelta, &lenDelta); + switch( zDelta[0] ){ + case '@': { + zDelta++; lenDelta--; + ofst = rbuDeltaGetInt(&zDelta, &lenDelta); + if( lenDelta>0 && zDelta[0]!=',' ){ + /* ERROR: copy command not terminated by ',' */ + return -1; } - p->rc = prepareFreeAndCollectError(p->dbRbu, &pIter->pSelect, pz, zSql); - } - - sqlite3_free(zImposterCols); - sqlite3_free(zImposterPK); - sqlite3_free(zWhere); - sqlite3_free(zBind); - }else{ - int bRbuRowid = (pIter->eType==RBU_PK_VTAB) - ||(pIter->eType==RBU_PK_NONE) - ||(pIter->eType==RBU_PK_EXTERNAL && rbuIsVacuum(p)); - const char *zTbl = pIter->zTbl; /* Table this step applies to */ - const char *zWrite; /* Imposter table name */ - - char *zBindings = rbuObjIterGetBindlist(p, pIter->nTblCol + bRbuRowid); - char *zWhere = rbuObjIterGetWhere(p, pIter); - char *zOldlist = rbuObjIterGetOldlist(p, pIter, "old"); - char *zNewlist = rbuObjIterGetOldlist(p, pIter, "new"); - - zCollist = rbuObjIterGetCollist(p, pIter); - pIter->nCol = pIter->nTblCol; - - /* Create the imposter table or tables (if required). */ - rbuCreateImposterTable(p, pIter); - rbuCreateImposterTable2(p, pIter); - zWrite = (pIter->eType==RBU_PK_VTAB ? "" : "rbu_imp_"); - - /* Create the INSERT statement to write to the target PK b-tree */ - if( p->rc==SQLITE_OK ){ - p->rc = prepareFreeAndCollectError(p->dbMain, &pIter->pInsert, pz, - sqlite3_mprintf( - "INSERT INTO \"%s%w\"(%s%s) VALUES(%s)", - zWrite, zTbl, zCollist, (bRbuRowid ? ", _rowid_" : ""), zBindings - ) - ); - } - - /* Create the DELETE statement to write to the target PK b-tree. - ** Because it only performs INSERT operations, this is not required for - ** an rbu vacuum handle. */ - if( rbuIsVacuum(p)==0 && p->rc==SQLITE_OK ){ - p->rc = prepareFreeAndCollectError(p->dbMain, &pIter->pDelete, pz, - sqlite3_mprintf( - "DELETE FROM \"%s%w\" WHERE %s", zWrite, zTbl, zWhere - ) - ); + zDelta++; lenDelta--; + total += cnt; + if( total>limit ){ + /* ERROR: copy exceeds output file size */ + return -1; + } + if( (int)(ofst+cnt) > lenSrc ){ + /* ERROR: copy extends past end of input */ + return -1; + } + memcpy(zOut, &zSrc[ofst], cnt); + zOut += cnt; + break; } - - if( rbuIsVacuum(p)==0 && pIter->abIndexed ){ - const char *zRbuRowid = ""; - if( pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_NONE ){ - zRbuRowid = ", rbu_rowid"; + case ':': { + zDelta++; lenDelta--; + total += cnt; + if( total>limit ){ + /* ERROR: insert command gives an output larger than predicted */ + return -1; } - - /* Create the rbu_tmp_xxx table and the triggers to populate it. */ - rbuMPrintfExec(p, p->dbRbu, - "CREATE TABLE IF NOT EXISTS %s.'rbu_tmp_%q' AS " - "SELECT *%s FROM '%q' WHERE 0;" - , p->zStateDb, pIter->zDataTbl - , (pIter->eType==RBU_PK_EXTERNAL ? ", 0 AS rbu_rowid" : "") - , pIter->zDataTbl - ); - - rbuMPrintfExec(p, p->dbMain, - "CREATE TEMP TRIGGER rbu_delete_tr BEFORE DELETE ON \"%s%w\" " - "BEGIN " - " SELECT rbu_tmp_insert(3, %s);" - "END;" - - "CREATE TEMP TRIGGER rbu_update1_tr BEFORE UPDATE ON \"%s%w\" " - "BEGIN " - " SELECT rbu_tmp_insert(3, %s);" - "END;" - - "CREATE TEMP TRIGGER rbu_update2_tr AFTER UPDATE ON \"%s%w\" " - "BEGIN " - " SELECT rbu_tmp_insert(4, %s);" - "END;", - zWrite, zTbl, zOldlist, - zWrite, zTbl, zOldlist, - zWrite, zTbl, zNewlist - ); - - if( pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_NONE ){ - rbuMPrintfExec(p, p->dbMain, - "CREATE TEMP TRIGGER rbu_insert_tr AFTER INSERT ON \"%s%w\" " - "BEGIN " - " SELECT rbu_tmp_insert(0, %s);" - "END;", - zWrite, zTbl, zNewlist - ); + if( (int)cnt>lenDelta ){ + /* ERROR: insert count exceeds size of delta */ + return -1; } - - rbuObjIterPrepareTmpInsert(p, pIter, zCollist, zRbuRowid); + memcpy(zOut, zDelta, cnt); + zOut += cnt; + zDelta += cnt; + lenDelta -= cnt; + break; } - - /* Create the SELECT statement to read keys from data_xxx */ - if( p->rc==SQLITE_OK ){ - const char *zRbuRowid = ""; - if( bRbuRowid ){ - zRbuRowid = rbuIsVacuum(p) ? ",_rowid_ " : ",rbu_rowid"; + case ';': { + zDelta++; lenDelta--; + zOut[0] = 0; +#if RBU_ENABLE_DELTA_CKSUM + if( cnt!=rbuDeltaChecksum(zOrigOut, total) ){ + /* ERROR: bad checksum */ + return -1; } - p->rc = prepareFreeAndCollectError(p->dbRbu, &pIter->pSelect, pz, - sqlite3_mprintf( - "SELECT %s,%s rbu_control%s FROM '%q'%s", - zCollist, - (rbuIsVacuum(p) ? "0 AS " : ""), - zRbuRowid, - pIter->zDataTbl, zLimit - ) - ); +#endif + if( total!=limit ){ + /* ERROR: generated size does not match predicted size */ + return -1; + } + return total; + } + default: { + /* ERROR: unknown delta operator */ + return -1; } - - sqlite3_free(zWhere); - sqlite3_free(zOldlist); - sqlite3_free(zNewlist); - sqlite3_free(zBindings); } - sqlite3_free(zCollist); - sqlite3_free(zLimit); } - - return p->rc; + /* ERROR: unterminated delta */ + return -1; +} + +static int rbuDeltaOutputSize(const char *zDelta, int lenDelta){ + int size; + size = rbuDeltaGetInt(&zDelta, &lenDelta); + if( *zDelta!='\n' ){ + /* ERROR: size integer not terminated by "\n" */ + return -1; + } + return size; } /* -** Set output variable *ppStmt to point to an UPDATE statement that may -** be used to update the imposter table for the main table b-tree of the -** table object that pIter currently points to, assuming that the -** rbu_control column of the data_xyz table contains zMask. -** -** If the zMask string does not specify any columns to update, then this -** is not an error. Output variable *ppStmt is set to NULL in this case. +** End of code taken from fossil. +*************************************************************************/ + +/* +** Implementation of SQL scalar function rbu_fossil_delta(). +** +** This function applies a fossil delta patch to a blob. Exactly two +** arguments must be passed to this function. The first is the blob to +** patch and the second the patch to apply. If no error occurs, this +** function returns the patched blob. */ -static int rbuGetUpdateStmt( - sqlite3rbu *p, /* RBU handle */ - RbuObjIter *pIter, /* Object iterator */ - const char *zMask, /* rbu_control value ('x.x.') */ - sqlite3_stmt **ppStmt /* OUT: UPDATE statement handle */ +static void rbuFossilDeltaFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv ){ - RbuUpdateStmt **pp; - RbuUpdateStmt *pUp = 0; - int nUp = 0; - - /* In case an error occurs */ - *ppStmt = 0; - - /* Search for an existing statement. If one is found, shift it to the front - ** of the LRU queue and return immediately. Otherwise, leave nUp pointing - ** to the number of statements currently in the cache and pUp to the - ** last object in the list. */ - for(pp=&pIter->pRbuUpdate; *pp; pp=&((*pp)->pNext)){ - pUp = *pp; - if( strcmp(pUp->zMask, zMask)==0 ){ - *pp = pUp->pNext; - pUp->pNext = pIter->pRbuUpdate; - pIter->pRbuUpdate = pUp; - *ppStmt = pUp->pUpdate; - return SQLITE_OK; - } - nUp++; - } - assert( pUp==0 || pUp->pNext==0 ); + const char *aDelta; + int nDelta; + const char *aOrig; + int nOrig; - if( nUp>=SQLITE_RBU_UPDATE_CACHESIZE ){ - for(pp=&pIter->pRbuUpdate; *pp!=pUp; pp=&((*pp)->pNext)); - *pp = 0; - sqlite3_finalize(pUp->pUpdate); - pUp->pUpdate = 0; - }else{ - pUp = (RbuUpdateStmt*)rbuMalloc(p, sizeof(RbuUpdateStmt)+pIter->nTblCol+1); - } + int nOut; + int nOut2; + char *aOut; - if( pUp ){ - char *zWhere = rbuObjIterGetWhere(p, pIter); - char *zSet = rbuObjIterGetSetlist(p, pIter, zMask); - char *zUpdate = 0; + assert( argc==2 ); - pUp->zMask = (char*)&pUp[1]; - memcpy(pUp->zMask, zMask, pIter->nTblCol); - pUp->pNext = pIter->pRbuUpdate; - pIter->pRbuUpdate = pUp; + nOrig = sqlite3_value_bytes(argv[0]); + aOrig = (const char*)sqlite3_value_blob(argv[0]); + nDelta = sqlite3_value_bytes(argv[1]); + aDelta = (const char*)sqlite3_value_blob(argv[1]); - if( zSet ){ - const char *zPrefix = ""; + /* Figure out the size of the output */ + nOut = rbuDeltaOutputSize(aDelta, nDelta); + if( nOut<0 ){ + sqlite3_result_error(context, "corrupt fossil delta", -1); + return; + } - if( pIter->eType!=RBU_PK_VTAB ) zPrefix = "rbu_imp_"; - zUpdate = sqlite3_mprintf("UPDATE \"%s%w\" SET %s WHERE %s", - zPrefix, pIter->zTbl, zSet, zWhere - ); - p->rc = prepareFreeAndCollectError( - p->dbMain, &pUp->pUpdate, &p->zErrmsg, zUpdate - ); - *ppStmt = pUp->pUpdate; + aOut = sqlite3_malloc(nOut+1); + if( aOut==0 ){ + sqlite3_result_error_nomem(context); + }else{ + nOut2 = rbuDeltaApply(aOrig, nOrig, aDelta, nDelta, aOut); + if( nOut2!=nOut ){ + sqlite3_free(aOut); + sqlite3_result_error(context, "corrupt fossil delta", -1); + }else{ + sqlite3_result_blob(context, aOut, nOut, sqlite3_free); } - sqlite3_free(zWhere); - sqlite3_free(zSet); } - - return p->rc; } -static sqlite3 *rbuOpenDbhandle( - sqlite3rbu *p, - const char *zName, - int bUseVfs + +/* +** Prepare the SQL statement in buffer zSql against database handle db. +** If successful, set *ppStmt to point to the new statement and return +** SQLITE_OK. +** +** Otherwise, if an error does occur, set *ppStmt to NULL and return +** an SQLite error code. Additionally, set output variable *pzErrmsg to +** point to a buffer containing an error message. It is the responsibility +** of the caller to (eventually) free this buffer using sqlite3_free(). +*/ +static int prepareAndCollectError( + sqlite3 *db, + sqlite3_stmt **ppStmt, + char **pzErrmsg, + const char *zSql ){ - sqlite3 *db = 0; - if( p->rc==SQLITE_OK ){ - const int flags = SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|SQLITE_OPEN_URI; - p->rc = sqlite3_open_v2(zName, &db, flags, bUseVfs ? p->zVfsName : 0); - if( p->rc ){ - p->zErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(db)); - sqlite3_close(db); - db = 0; - } + int rc = sqlite3_prepare_v2(db, zSql, -1, ppStmt, 0); + if( rc!=SQLITE_OK ){ + *pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(db)); + *ppStmt = 0; } - return db; + return rc; } /* -** Free an RbuState object allocated by rbuLoadState(). +** Reset the SQL statement passed as the first argument. Return a copy +** of the value returned by sqlite3_reset(). +** +** If an error has occurred, then set *pzErrmsg to point to a buffer +** containing an error message. It is the responsibility of the caller +** to eventually free this buffer using sqlite3_free(). */ -static void rbuFreeState(RbuState *p){ - if( p ){ - sqlite3_free(p->zTbl); - sqlite3_free(p->zDataTbl); - sqlite3_free(p->zIdx); - sqlite3_free(p); +static int resetAndCollectError(sqlite3_stmt *pStmt, char **pzErrmsg){ + int rc = sqlite3_reset(pStmt); + if( rc!=SQLITE_OK ){ + *pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(sqlite3_db_handle(pStmt))); } + return rc; } /* -** Allocate an RbuState object and load the contents of the rbu_state -** table into it. Return a pointer to the new object. It is the -** responsibility of the caller to eventually free the object using -** sqlite3_free(). +** Unless it is NULL, argument zSql points to a buffer allocated using +** sqlite3_malloc containing an SQL statement. This function prepares the SQL +** statement against database db and frees the buffer. If statement +** compilation is successful, *ppStmt is set to point to the new statement +** handle and SQLITE_OK is returned. ** -** If an error occurs, leave an error code and message in the rbu handle -** and return NULL. +** Otherwise, if an error occurs, *ppStmt is set to NULL and an error code +** returned. In this case, *pzErrmsg may also be set to point to an error +** message. It is the responsibility of the caller to free this error message +** buffer using sqlite3_free(). +** +** If argument zSql is NULL, this function assumes that an OOM has occurred. +** In this case SQLITE_NOMEM is returned and *ppStmt set to NULL. */ -static RbuState *rbuLoadState(sqlite3rbu *p){ - RbuState *pRet = 0; - sqlite3_stmt *pStmt = 0; +static int prepareFreeAndCollectError( + sqlite3 *db, + sqlite3_stmt **ppStmt, + char **pzErrmsg, + char *zSql +){ int rc; - int rc2; - - pRet = (RbuState*)rbuMalloc(p, sizeof(RbuState)); - if( pRet==0 ) return 0; - - rc = prepareFreeAndCollectError(p->dbRbu, &pStmt, &p->zErrmsg, - sqlite3_mprintf("SELECT k, v FROM %s.rbu_state", p->zStateDb) - ); - while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ - switch( sqlite3_column_int(pStmt, 0) ){ - case RBU_STATE_STAGE: - pRet->eStage = sqlite3_column_int(pStmt, 1); - if( pRet->eStage!=RBU_STAGE_OAL - && pRet->eStage!=RBU_STAGE_MOVE - && pRet->eStage!=RBU_STAGE_CKPT - ){ - p->rc = SQLITE_CORRUPT; - } - break; - - case RBU_STATE_TBL: - pRet->zTbl = rbuStrndup((char*)sqlite3_column_text(pStmt, 1), &rc); - break; - - case RBU_STATE_IDX: - pRet->zIdx = rbuStrndup((char*)sqlite3_column_text(pStmt, 1), &rc); - break; - - case RBU_STATE_ROW: - pRet->nRow = sqlite3_column_int(pStmt, 1); - break; - - case RBU_STATE_PROGRESS: - pRet->nProgress = sqlite3_column_int64(pStmt, 1); - break; - - case RBU_STATE_CKPT: - pRet->iWalCksum = sqlite3_column_int64(pStmt, 1); - break; - - case RBU_STATE_COOKIE: - pRet->iCookie = (u32)sqlite3_column_int64(pStmt, 1); - break; - - case RBU_STATE_OALSZ: - pRet->iOalSz = (u32)sqlite3_column_int64(pStmt, 1); - break; - - case RBU_STATE_PHASEONESTEP: - pRet->nPhaseOneStep = sqlite3_column_int64(pStmt, 1); - break; - - case RBU_STATE_DATATBL: - pRet->zDataTbl = rbuStrndup((char*)sqlite3_column_text(pStmt, 1), &rc); - break; - - default: - rc = SQLITE_CORRUPT; - break; - } + assert( *pzErrmsg==0 ); + if( zSql==0 ){ + rc = SQLITE_NOMEM; + *ppStmt = 0; + }else{ + rc = prepareAndCollectError(db, ppStmt, pzErrmsg, zSql); + sqlite3_free(zSql); } - rc2 = sqlite3_finalize(pStmt); - if( rc==SQLITE_OK ) rc = rc2; - - p->rc = rc; - return pRet; + return rc; } - /* -** Open the database handle and attach the RBU database as "rbu". If an -** error occurs, leave an error code and message in the RBU handle. +** Free the RbuObjIter.azTblCol[] and RbuObjIter.abTblPk[] arrays allocated +** by an earlier call to rbuObjIterCacheTableInfo(). */ -static void rbuOpenDatabase(sqlite3rbu *p, int *pbRetry){ - assert( p->rc || (p->dbMain==0 && p->dbRbu==0) ); - assert( p->rc || rbuIsVacuum(p) || p->zTarget!=0 ); - - /* Open the RBU database */ - p->dbRbu = rbuOpenDbhandle(p, p->zRbu, 1); - - if( p->rc==SQLITE_OK && rbuIsVacuum(p) ){ - sqlite3_file_control(p->dbRbu, "main", SQLITE_FCNTL_RBUCNT, (void*)p); - if( p->zState==0 ){ - const char *zFile = sqlite3_db_filename(p->dbRbu, "main"); - p->zState = rbuMPrintf(p, "file://%s-vacuum?modeof=%s", zFile, zFile); - } +static void rbuObjIterFreeCols(RbuObjIter *pIter){ + int i; + for(i=0; inTblCol; i++){ + sqlite3_free(pIter->azTblCol[i]); + sqlite3_free(pIter->azTblType[i]); } + sqlite3_free(pIter->azTblCol); + pIter->azTblCol = 0; + pIter->azTblType = 0; + pIter->aiSrcOrder = 0; + pIter->abTblPk = 0; + pIter->abNotNull = 0; + pIter->nTblCol = 0; + pIter->eType = 0; /* Invalid value */ +} - /* If using separate RBU and state databases, attach the state database to - ** the RBU db handle now. */ - if( p->zState ){ - rbuMPrintfExec(p, p->dbRbu, "ATTACH %Q AS stat", p->zState); - memcpy(p->zStateDb, "stat", 4); - }else{ - memcpy(p->zStateDb, "main", 4); - } +/* +** Finalize all statements and free all allocations that are specific to +** the current object (table/index pair). +*/ +static void rbuObjIterClearStatements(RbuObjIter *pIter){ + RbuUpdateStmt *pUp; -#if 0 - if( p->rc==SQLITE_OK && rbuIsVacuum(p) ){ - p->rc = sqlite3_exec(p->dbRbu, "BEGIN", 0, 0, 0); + sqlite3_finalize(pIter->pSelect); + sqlite3_finalize(pIter->pInsert); + sqlite3_finalize(pIter->pDelete); + sqlite3_finalize(pIter->pTmpInsert); + pUp = pIter->pRbuUpdate; + while( pUp ){ + RbuUpdateStmt *pTmp = pUp->pNext; + sqlite3_finalize(pUp->pUpdate); + sqlite3_free(pUp); + pUp = pTmp; } -#endif + sqlite3_free(pIter->aIdxCol); + sqlite3_free(pIter->zIdxSql); + + pIter->pSelect = 0; + pIter->pInsert = 0; + pIter->pDelete = 0; + pIter->pRbuUpdate = 0; + pIter->pTmpInsert = 0; + pIter->nCol = 0; + pIter->nIdxCol = 0; + pIter->aIdxCol = 0; + pIter->zIdxSql = 0; +} - /* If it has not already been created, create the rbu_state table */ - rbuMPrintfExec(p, p->dbRbu, RBU_CREATE_STATE, p->zStateDb); +/* +** Clean up any resources allocated as part of the iterator object passed +** as the only argument. +*/ +static void rbuObjIterFinalize(RbuObjIter *pIter){ + rbuObjIterClearStatements(pIter); + sqlite3_finalize(pIter->pTblIter); + sqlite3_finalize(pIter->pIdxIter); + rbuObjIterFreeCols(pIter); + memset(pIter, 0, sizeof(RbuObjIter)); +} -#if 0 - if( rbuIsVacuum(p) ){ - if( p->rc==SQLITE_OK ){ - int rc2; - int bOk = 0; - sqlite3_stmt *pCnt = 0; - p->rc = prepareAndCollectError(p->dbRbu, &pCnt, &p->zErrmsg, - "SELECT count(*) FROM stat.sqlite_master" +/* +** Advance the iterator to the next position. +** +** If no error occurs, SQLITE_OK is returned and the iterator is left +** pointing to the next entry. Otherwise, an error code and message is +** left in the RBU handle passed as the first argument. A copy of the +** error code is returned. +*/ +static int rbuObjIterNext(sqlite3rbu *p, RbuObjIter *pIter){ + int rc = p->rc; + if( rc==SQLITE_OK ){ + + /* Free any SQLite statements used while processing the previous object */ + rbuObjIterClearStatements(pIter); + if( pIter->zIdx==0 ){ + rc = sqlite3_exec(p->dbMain, + "DROP TRIGGER IF EXISTS temp.rbu_insert_tr;" + "DROP TRIGGER IF EXISTS temp.rbu_update1_tr;" + "DROP TRIGGER IF EXISTS temp.rbu_update2_tr;" + "DROP TRIGGER IF EXISTS temp.rbu_delete_tr;" + , 0, 0, &p->zErrmsg ); - if( p->rc==SQLITE_OK - && sqlite3_step(pCnt)==SQLITE_ROW - && 1==sqlite3_column_int(pCnt, 0) - ){ - bOk = 1; - } - rc2 = sqlite3_finalize(pCnt); - if( p->rc==SQLITE_OK ) p->rc = rc2; + } - if( p->rc==SQLITE_OK && bOk==0 ){ - p->rc = SQLITE_ERROR; - p->zErrmsg = sqlite3_mprintf("invalid state database"); - } - - if( p->rc==SQLITE_OK ){ - p->rc = sqlite3_exec(p->dbRbu, "COMMIT", 0, 0, 0); + if( rc==SQLITE_OK ){ + if( pIter->bCleanup ){ + rbuObjIterFreeCols(pIter); + pIter->bCleanup = 0; + rc = sqlite3_step(pIter->pTblIter); + if( rc!=SQLITE_ROW ){ + rc = resetAndCollectError(pIter->pTblIter, &p->zErrmsg); + pIter->zTbl = 0; + }else{ + pIter->zTbl = (const char*)sqlite3_column_text(pIter->pTblIter, 0); + pIter->zDataTbl = (const char*)sqlite3_column_text(pIter->pTblIter,1); + rc = (pIter->zDataTbl && pIter->zTbl) ? SQLITE_OK : SQLITE_NOMEM; + } + }else{ + if( pIter->zIdx==0 ){ + sqlite3_stmt *pIdx = pIter->pIdxIter; + rc = sqlite3_bind_text(pIdx, 1, pIter->zTbl, -1, SQLITE_STATIC); + } + if( rc==SQLITE_OK ){ + rc = sqlite3_step(pIter->pIdxIter); + if( rc!=SQLITE_ROW ){ + rc = resetAndCollectError(pIter->pIdxIter, &p->zErrmsg); + pIter->bCleanup = 1; + pIter->zIdx = 0; + }else{ + pIter->zIdx = (const char*)sqlite3_column_text(pIter->pIdxIter, 0); + pIter->iTnum = sqlite3_column_int(pIter->pIdxIter, 1); + pIter->bUnique = sqlite3_column_int(pIter->pIdxIter, 2); + rc = pIter->zIdx ? SQLITE_OK : SQLITE_NOMEM; + } + } } } } -#endif - if( p->rc==SQLITE_OK && rbuIsVacuum(p) ){ - int bOpen = 0; - int rc; - p->nRbu = 0; - p->pRbuFd = 0; - rc = sqlite3_file_control(p->dbRbu, "main", SQLITE_FCNTL_RBUCNT, (void*)p); - if( rc!=SQLITE_NOTFOUND ) p->rc = rc; - if( p->eStage>=RBU_STAGE_MOVE ){ - bOpen = 1; - }else{ - RbuState *pState = rbuLoadState(p); - if( pState ){ - bOpen = (pState->eStage>=RBU_STAGE_MOVE); - rbuFreeState(pState); - } - } - if( bOpen ) p->dbMain = rbuOpenDbhandle(p, p->zRbu, p->nRbu<=1); + if( rc!=SQLITE_OK ){ + rbuObjIterFinalize(pIter); + p->rc = rc; } + return rc; +} - p->eStage = 0; - if( p->rc==SQLITE_OK && p->dbMain==0 ){ - if( !rbuIsVacuum(p) ){ - p->dbMain = rbuOpenDbhandle(p, p->zTarget, 1); - }else if( p->pRbuFd->pWalFd ){ - if( pbRetry ){ - p->pRbuFd->bNolock = 0; - sqlite3_close(p->dbRbu); - sqlite3_close(p->dbMain); - p->dbMain = 0; - p->dbRbu = 0; - *pbRetry = 1; - return; + +/* +** The implementation of the rbu_target_name() SQL function. This function +** accepts one or two arguments. The first argument is the name of a table - +** the name of a table in the RBU database. The second, if it is present, is 1 +** for a view or 0 for a table. +** +** For a non-vacuum RBU handle, if the table name matches the pattern: +** +** data[0-9]_ +** +** where is any sequence of 1 or more characters, is returned. +** Otherwise, if the only argument does not match the above pattern, an SQL +** NULL is returned. +** +** "data_t1" -> "t1" +** "data0123_t2" -> "t2" +** "dataAB_t3" -> NULL +** +** For an rbu vacuum handle, a copy of the first argument is returned if +** the second argument is either missing or 0 (not a view). +*/ +static void rbuTargetNameFunc( + sqlite3_context *pCtx, + int argc, + sqlite3_value **argv +){ + sqlite3rbu *p = sqlite3_user_data(pCtx); + const char *zIn; + assert( argc==1 || argc==2 ); + + zIn = (const char*)sqlite3_value_text(argv[0]); + if( zIn ){ + if( rbuIsVacuum(p) ){ + assert( argc==2 || argc==1 ); + if( argc==1 || 0==sqlite3_value_int(argv[1]) ){ + sqlite3_result_text(pCtx, zIn, -1, SQLITE_STATIC); } - p->rc = SQLITE_ERROR; - p->zErrmsg = sqlite3_mprintf("cannot vacuum wal mode database"); }else{ - char *zTarget; - char *zExtra = 0; - if( strlen(p->zRbu)>=5 && 0==memcmp("file:", p->zRbu, 5) ){ - zExtra = &p->zRbu[5]; - while( *zExtra ){ - if( *zExtra++=='?' ) break; + if( strlen(zIn)>4 && memcmp("data", zIn, 4)==0 ){ + int i; + for(i=4; zIn[i]>='0' && zIn[i]<='9'; i++); + if( zIn[i]=='_' && zIn[i+1] ){ + sqlite3_result_text(pCtx, &zIn[i+1], -1, SQLITE_STATIC); } - if( *zExtra=='\0' ) zExtra = 0; - } - - zTarget = sqlite3_mprintf("file:%s-vacuum?rbu_memory=1%s%s", - sqlite3_db_filename(p->dbRbu, "main"), - (zExtra==0 ? "" : "&"), (zExtra==0 ? "" : zExtra) - ); - - if( zTarget==0 ){ - p->rc = SQLITE_NOMEM; - return; } - p->dbMain = rbuOpenDbhandle(p, zTarget, p->nRbu<=1); - sqlite3_free(zTarget); } } +} - if( p->rc==SQLITE_OK ){ - p->rc = sqlite3_create_function(p->dbMain, - "rbu_tmp_insert", -1, SQLITE_UTF8, (void*)p, rbuTmpInsertFunc, 0, 0 - ); - } +/* +** Initialize the iterator structure passed as the second argument. +** +** If no error occurs, SQLITE_OK is returned and the iterator is left +** pointing to the first entry. Otherwise, an error code and message is +** left in the RBU handle passed as the first argument. A copy of the +** error code is returned. +*/ +static int rbuObjIterFirst(sqlite3rbu *p, RbuObjIter *pIter){ + int rc; + memset(pIter, 0, sizeof(RbuObjIter)); - if( p->rc==SQLITE_OK ){ - p->rc = sqlite3_create_function(p->dbMain, - "rbu_fossil_delta", 2, SQLITE_UTF8, 0, rbuFossilDeltaFunc, 0, 0 + rc = prepareFreeAndCollectError(p->dbRbu, &pIter->pTblIter, &p->zErrmsg, + sqlite3_mprintf( + "SELECT rbu_target_name(name, type='view') AS target, name " + "FROM sqlite_master " + "WHERE type IN ('table', 'view') AND target IS NOT NULL " + " %s " + "ORDER BY name" + , rbuIsVacuum(p) ? "AND rootpage!=0 AND rootpage IS NOT NULL" : "")); + + if( rc==SQLITE_OK ){ + rc = prepareAndCollectError(p->dbMain, &pIter->pIdxIter, &p->zErrmsg, + "SELECT name, rootpage, sql IS NULL OR substr(8, 6)=='UNIQUE' " + " FROM main.sqlite_master " + " WHERE type='index' AND tbl_name = ?" ); } + pIter->bCleanup = 1; + p->rc = rc; + return rbuObjIterNext(p, pIter); +} + +/* +** This is a wrapper around "sqlite3_mprintf(zFmt, ...)". If an OOM occurs, +** an error code is stored in the RBU handle passed as the first argument. +** +** If an error has already occurred (p->rc is already set to something other +** than SQLITE_OK), then this function returns NULL without modifying the +** stored error code. In this case it still calls sqlite3_free() on any +** printf() parameters associated with %z conversions. +*/ +static char *rbuMPrintf(sqlite3rbu *p, const char *zFmt, ...){ + char *zSql = 0; + va_list ap; + va_start(ap, zFmt); + zSql = sqlite3_vmprintf(zFmt, ap); if( p->rc==SQLITE_OK ){ - p->rc = sqlite3_create_function(p->dbRbu, - "rbu_target_name", -1, SQLITE_UTF8, (void*)p, rbuTargetNameFunc, 0, 0 - ); + if( zSql==0 ) p->rc = SQLITE_NOMEM; + }else{ + sqlite3_free(zSql); + zSql = 0; } + va_end(ap); + return zSql; +} +/* +** Argument zFmt is a sqlite3_mprintf() style format string. The trailing +** arguments are the usual subsitution values. This function performs +** the printf() style substitutions and executes the result as an SQL +** statement on the RBU handles database. +** +** If an error occurs, an error code and error message is stored in the +** RBU handle. If an error has already occurred when this function is +** called, it is a no-op. +*/ +static int rbuMPrintfExec(sqlite3rbu *p, sqlite3 *db, const char *zFmt, ...){ + va_list ap; + char *zSql; + va_start(ap, zFmt); + zSql = sqlite3_vmprintf(zFmt, ap); if( p->rc==SQLITE_OK ){ - p->rc = sqlite3_file_control(p->dbMain, "main", SQLITE_FCNTL_RBU, (void*)p); + if( zSql==0 ){ + p->rc = SQLITE_NOMEM; + }else{ + p->rc = sqlite3_exec(db, zSql, 0, 0, &p->zErrmsg); + } } - rbuMPrintfExec(p, p->dbMain, "SELECT * FROM sqlite_master"); + sqlite3_free(zSql); + va_end(ap); + return p->rc; +} - /* Mark the database file just opened as an RBU target database. If - ** this call returns SQLITE_NOTFOUND, then the RBU vfs is not in use. - ** This is an error. */ +/* +** Attempt to allocate and return a pointer to a zeroed block of nByte +** bytes. +** +** If an error (i.e. an OOM condition) occurs, return NULL and leave an +** error code in the rbu handle passed as the first argument. Or, if an +** error has already occurred when this function is called, return NULL +** immediately without attempting the allocation or modifying the stored +** error code. +*/ +static void *rbuMalloc(sqlite3rbu *p, sqlite3_int64 nByte){ + void *pRet = 0; if( p->rc==SQLITE_OK ){ - p->rc = sqlite3_file_control(p->dbMain, "main", SQLITE_FCNTL_RBU, (void*)p); + assert( nByte>0 ); + pRet = sqlite3_malloc64(nByte); + if( pRet==0 ){ + p->rc = SQLITE_NOMEM; + }else{ + memset(pRet, 0, nByte); + } } + return pRet; +} - if( p->rc==SQLITE_NOTFOUND ){ - p->rc = SQLITE_ERROR; - p->zErrmsg = sqlite3_mprintf("rbu vfs not found"); + +/* +** Allocate and zero the pIter->azTblCol[] and abTblPk[] arrays so that +** there is room for at least nCol elements. If an OOM occurs, store an +** error code in the RBU handle passed as the first argument. +*/ +static void rbuAllocateIterArrays(sqlite3rbu *p, RbuObjIter *pIter, int nCol){ + sqlite3_int64 nByte = (2*sizeof(char*) + sizeof(int) + 3*sizeof(u8)) * nCol; + char **azNew; + + azNew = (char**)rbuMalloc(p, nByte); + if( azNew ){ + pIter->azTblCol = azNew; + pIter->azTblType = &azNew[nCol]; + pIter->aiSrcOrder = (int*)&pIter->azTblType[nCol]; + pIter->abTblPk = (u8*)&pIter->aiSrcOrder[nCol]; + pIter->abNotNull = (u8*)&pIter->abTblPk[nCol]; + pIter->abIndexed = (u8*)&pIter->abNotNull[nCol]; } } /* -** This routine is a copy of the sqlite3FileSuffix3() routine from the core. -** It is a no-op unless SQLITE_ENABLE_8_3_NAMES is defined. -** -** If SQLITE_ENABLE_8_3_NAMES is set at compile-time and if the database -** filename in zBaseFilename is a URI with the "8_3_names=1" parameter and -** if filename in z[] has a suffix (a.k.a. "extension") that is longer than -** three characters, then shorten the suffix on z[] to be the last three -** characters of the original suffix. -** -** If SQLITE_ENABLE_8_3_NAMES is set to 2 at compile-time, then always -** do the suffix shortening regardless of URI parameter. -** -** Examples: +** The first argument must be a nul-terminated string. This function +** returns a copy of the string in memory obtained from sqlite3_malloc(). +** It is the responsibility of the caller to eventually free this memory +** using sqlite3_free(). ** -** test.db-journal => test.nal -** test.db-wal => test.wal -** test.db-shm => test.shm -** test.db-mj7f3319fa => test.9fa +** If an OOM condition is encountered when attempting to allocate memory, +** output variable (*pRc) is set to SQLITE_NOMEM before returning. Otherwise, +** if the allocation succeeds, (*pRc) is left unchanged. */ -static void rbuFileSuffix3(const char *zBase, char *z){ -#ifdef SQLITE_ENABLE_8_3_NAMES -#if SQLITE_ENABLE_8_3_NAMES<2 - if( sqlite3_uri_boolean(zBase, "8_3_names", 0) ) -#endif - { - int i, sz; - sz = (int)strlen(z)&0xffffff; - for(i=sz-1; i>0 && z[i]!='/' && z[i]!='.'; i--){} - if( z[i]=='.' && sz>i+4 ) memmove(&z[i+1], &z[sz-3], 4); +static char *rbuStrndup(const char *zStr, int *pRc){ + char *zRet = 0; + + if( *pRc==SQLITE_OK ){ + if( zStr ){ + size_t nCopy = strlen(zStr) + 1; + zRet = (char*)sqlite3_malloc64(nCopy); + if( zRet ){ + memcpy(zRet, zStr, nCopy); + }else{ + *pRc = SQLITE_NOMEM; + } + } } -#endif + + return zRet; } /* -** Return the current wal-index header checksum for the target database -** as a 64-bit integer. +** Finalize the statement passed as the second argument. ** -** The checksum is store in the first page of xShmMap memory as an 8-byte -** blob starting at byte offset 40. +** If the sqlite3_finalize() call indicates that an error occurs, and the +** rbu handle error code is not already set, set the error code and error +** message accordingly. */ -static i64 rbuShmChecksum(sqlite3rbu *p){ - i64 iRet = 0; - if( p->rc==SQLITE_OK ){ - sqlite3_file *pDb = p->pTargetFd->pReal; - u32 volatile *ptr; - p->rc = pDb->pMethods->xShmMap(pDb, 0, 32*1024, 0, (void volatile**)&ptr); - if( p->rc==SQLITE_OK ){ - iRet = ((i64)ptr[10] << 32) + ptr[11]; - } +static void rbuFinalize(sqlite3rbu *p, sqlite3_stmt *pStmt){ + sqlite3 *db = sqlite3_db_handle(pStmt); + int rc = sqlite3_finalize(pStmt); + if( p->rc==SQLITE_OK && rc!=SQLITE_OK ){ + p->rc = rc; + p->zErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(db)); } - return iRet; } -/* -** This function is called as part of initializing or reinitializing an -** incremental checkpoint. +/* Determine the type of a table. ** -** It populates the sqlite3rbu.aFrame[] array with the set of -** (wal frame -> db page) copy operations required to checkpoint the -** current wal file, and obtains the set of shm locks required to safely -** perform the copy operations directly on the file-system. +** peType is of type (int*), a pointer to an output parameter of type +** (int). This call sets the output parameter as follows, depending +** on the type of the table specified by parameters dbName and zTbl. ** -** If argument pState is not NULL, then the incremental checkpoint is -** being resumed. In this case, if the checksum of the wal-index-header -** following recovery is not the same as the checksum saved in the RbuState -** object, then the rbu handle is set to DONE state. This occurs if some -** other client appends a transaction to the wal file in the middle of -** an incremental checkpoint. +** RBU_PK_NOTABLE: No such table. +** RBU_PK_NONE: Table has an implicit rowid. +** RBU_PK_IPK: Table has an explicit IPK column. +** RBU_PK_EXTERNAL: Table has an external PK index. +** RBU_PK_WITHOUT_ROWID: Table is WITHOUT ROWID. +** RBU_PK_VTAB: Table is a virtual table. +** +** Argument *piPk is also of type (int*), and also points to an output +** parameter. Unless the table has an external primary key index +** (i.e. unless *peType is set to 3), then *piPk is set to zero. Or, +** if the table does have an external primary key index, then *piPk +** is set to the root page number of the primary key index before +** returning. +** +** ALGORITHM: +** +** if( no entry exists in sqlite_master ){ +** return RBU_PK_NOTABLE +** }else if( sql for the entry starts with "CREATE VIRTUAL" ){ +** return RBU_PK_VTAB +** }else if( "PRAGMA index_list()" for the table contains a "pk" index ){ +** if( the index that is the pk exists in sqlite_master ){ +** *piPK = rootpage of that index. +** return RBU_PK_EXTERNAL +** }else{ +** return RBU_PK_WITHOUT_ROWID +** } +** }else if( "PRAGMA table_info()" lists one or more "pk" columns ){ +** return RBU_PK_IPK +** }else{ +** return RBU_PK_NONE +** } */ -static void rbuSetupCheckpoint(sqlite3rbu *p, RbuState *pState){ +static void rbuTableType( + sqlite3rbu *p, + const char *zTab, + int *peType, + int *piTnum, + int *piPk +){ + /* + ** 0) SELECT count(*) FROM sqlite_master where name=%Q AND IsVirtual(%Q) + ** 1) PRAGMA index_list = ? + ** 2) SELECT count(*) FROM sqlite_master where name=%Q + ** 3) PRAGMA table_info = ? + */ + sqlite3_stmt *aStmt[4] = {0, 0, 0, 0}; - /* If pState is NULL, then the wal file may not have been opened and - ** recovered. Running a read-statement here to ensure that doing so - ** does not interfere with the "capture" process below. */ - if( pState==0 ){ - p->eStage = 0; - if( p->rc==SQLITE_OK ){ - p->rc = sqlite3_exec(p->dbMain, "SELECT * FROM sqlite_master", 0, 0, 0); - } - } + *peType = RBU_PK_NOTABLE; + *piPk = 0; - /* Assuming no error has occurred, run a "restart" checkpoint with the - ** sqlite3rbu.eStage variable set to CAPTURE. This turns on the following - ** special behaviour in the rbu VFS: - ** - ** * If the exclusive shm WRITER or READ0 lock cannot be obtained, - ** the checkpoint fails with SQLITE_BUSY (normally SQLite would - ** proceed with running a passive checkpoint instead of failing). - ** - ** * Attempts to read from the *-wal file or write to the database file - ** do not perform any IO. Instead, the frame/page combinations that - ** would be read/written are recorded in the sqlite3rbu.aFrame[] - ** array. - ** - ** * Calls to xShmLock(UNLOCK) to release the exclusive shm WRITER, - ** READ0 and CHECKPOINT locks taken as part of the checkpoint are - ** no-ops. These locks will not be released until the connection - ** is closed. - ** - ** * Attempting to xSync() the database file causes an SQLITE_INTERNAL - ** error. - ** - ** As a result, unless an error (i.e. OOM or SQLITE_BUSY) occurs, the - ** checkpoint below fails with SQLITE_INTERNAL, and leaves the aFrame[] - ** array populated with a set of (frame -> page) mappings. Because the - ** WRITER, CHECKPOINT and READ0 locks are still held, it is safe to copy - ** data from the wal file into the database file according to the - ** contents of aFrame[]. - */ - if( p->rc==SQLITE_OK ){ - int rc2; - p->eStage = RBU_STAGE_CAPTURE; - rc2 = sqlite3_exec(p->dbMain, "PRAGMA main.wal_checkpoint=restart", 0, 0,0); - if( rc2!=SQLITE_INTERNAL ) p->rc = rc2; + assert( p->rc==SQLITE_OK ); + p->rc = prepareFreeAndCollectError(p->dbMain, &aStmt[0], &p->zErrmsg, + sqlite3_mprintf( + "SELECT (sql LIKE 'create virtual%%'), rootpage" + " FROM sqlite_master" + " WHERE name=%Q", zTab + )); + if( p->rc!=SQLITE_OK || sqlite3_step(aStmt[0])!=SQLITE_ROW ){ + /* Either an error, or no such table. */ + goto rbuTableType_end; + } + if( sqlite3_column_int(aStmt[0], 0) ){ + *peType = RBU_PK_VTAB; /* virtual table */ + goto rbuTableType_end; } + *piTnum = sqlite3_column_int(aStmt[0], 1); - if( p->rc==SQLITE_OK && p->nFrame>0 ){ - p->eStage = RBU_STAGE_CKPT; - p->nStep = (pState ? pState->nRow : 0); - p->aBuf = rbuMalloc(p, p->pgsz); - p->iWalCksum = rbuShmChecksum(p); + p->rc = prepareFreeAndCollectError(p->dbMain, &aStmt[1], &p->zErrmsg, + sqlite3_mprintf("PRAGMA index_list=%Q",zTab) + ); + if( p->rc ) goto rbuTableType_end; + while( sqlite3_step(aStmt[1])==SQLITE_ROW ){ + const u8 *zOrig = sqlite3_column_text(aStmt[1], 3); + const u8 *zIdx = sqlite3_column_text(aStmt[1], 1); + if( zOrig && zIdx && zOrig[0]=='p' ){ + p->rc = prepareFreeAndCollectError(p->dbMain, &aStmt[2], &p->zErrmsg, + sqlite3_mprintf( + "SELECT rootpage FROM sqlite_master WHERE name = %Q", zIdx + )); + if( p->rc==SQLITE_OK ){ + if( sqlite3_step(aStmt[2])==SQLITE_ROW ){ + *piPk = sqlite3_column_int(aStmt[2], 0); + *peType = RBU_PK_EXTERNAL; + }else{ + *peType = RBU_PK_WITHOUT_ROWID; + } + } + goto rbuTableType_end; + } } + p->rc = prepareFreeAndCollectError(p->dbMain, &aStmt[3], &p->zErrmsg, + sqlite3_mprintf("PRAGMA table_info=%Q",zTab) + ); if( p->rc==SQLITE_OK ){ - if( p->nFrame==0 || (pState && pState->iWalCksum!=p->iWalCksum) ){ - p->rc = SQLITE_DONE; - p->eStage = RBU_STAGE_DONE; - }else{ - int nSectorSize; - sqlite3_file *pDb = p->pTargetFd->pReal; - sqlite3_file *pWal = p->pTargetFd->pWalFd->pReal; - assert( p->nPagePerSector==0 ); - nSectorSize = pDb->pMethods->xSectorSize(pDb); - if( nSectorSize>p->pgsz ){ - p->nPagePerSector = nSectorSize / p->pgsz; - }else{ - p->nPagePerSector = 1; + while( sqlite3_step(aStmt[3])==SQLITE_ROW ){ + if( sqlite3_column_int(aStmt[3],5)>0 ){ + *peType = RBU_PK_IPK; /* explicit IPK column */ + goto rbuTableType_end; } + } + *peType = RBU_PK_NONE; + } - /* Call xSync() on the wal file. This causes SQLite to sync the - ** directory in which the target database and the wal file reside, in - ** case it has not been synced since the rename() call in - ** rbuMoveOalFile(). */ - p->rc = pWal->pMethods->xSync(pWal, SQLITE_SYNC_NORMAL); +rbuTableType_end: { + unsigned int i; + for(i=0; iabIndexed[] array. */ -static int rbuCaptureWalRead(sqlite3rbu *pRbu, i64 iOff, int iAmt){ - const u32 mReq = (1<mLock!=mReq ){ - pRbu->rc = SQLITE_BUSY; - return SQLITE_INTERNAL; + if( p->rc==SQLITE_OK ){ + memcpy(pIter->abIndexed, pIter->abTblPk, sizeof(u8)*pIter->nTblCol); + p->rc = prepareFreeAndCollectError(p->dbMain, &pList, &p->zErrmsg, + sqlite3_mprintf("PRAGMA main.index_list = %Q", pIter->zTbl) + ); } - pRbu->pgsz = iAmt; - if( pRbu->nFrame==pRbu->nFrameAlloc ){ - int nNew = (pRbu->nFrameAlloc ? pRbu->nFrameAlloc : 64) * 2; - RbuFrame *aNew; - aNew = (RbuFrame*)sqlite3_realloc64(pRbu->aFrame, nNew * sizeof(RbuFrame)); - if( aNew==0 ) return SQLITE_NOMEM; - pRbu->aFrame = aNew; - pRbu->nFrameAlloc = nNew; + pIter->nIndex = 0; + while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pList) ){ + const char *zIdx = (const char*)sqlite3_column_text(pList, 1); + int bPartial = sqlite3_column_int(pList, 4); + sqlite3_stmt *pXInfo = 0; + if( zIdx==0 ) break; + if( bPartial ){ + memset(pIter->abIndexed, 0x01, sizeof(u8)*pIter->nTblCol); + } + p->rc = prepareFreeAndCollectError(p->dbMain, &pXInfo, &p->zErrmsg, + sqlite3_mprintf("PRAGMA main.index_xinfo = %Q", zIdx) + ); + while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXInfo) ){ + int iCid = sqlite3_column_int(pXInfo, 1); + if( iCid>=0 ) pIter->abIndexed[iCid] = 1; + if( iCid==-2 ){ + memset(pIter->abIndexed, 0x01, sizeof(u8)*pIter->nTblCol); + } + } + rbuFinalize(p, pXInfo); + bIndex = 1; + pIter->nIndex++; } - iFrame = (u32)((iOff-32) / (i64)(iAmt+24)) + 1; - if( pRbu->iMaxFrame