sql: BREAKING CHANGE -- create new escape syntax for the "@" characte…

…r in prepared Statements. (#51)

* sql: create improved escape syntax for prepared statements

* sql: fix typo in Tokenizer fandoc

src/sql/fan/Tokenizer.fan

@@ -0,0 +1,171 @@
+//
+// Copyright (c) 2024, Brian Frank and Andy Frank
+// Licensed under the Academic Free License version 3.0
+//
+// History:
+//   12 July 2024  Mike Jarmy   Creation
+//
+
+**************************************************************************
+**
+** Tokenizer transforms a parameterized SQL string into JDBC SQL, using escape
+** syntax to transform "\@" and "\\" into "@" and "\" respectively.
+**
+**************************************************************************
+
+internal class Tokenizer
+{
+  internal new make(Str origSql)
+  {
+    this.origSql = origSql
+
+    next := nextToken()
+    while (true)
+    {
+      switch (next)
+      {
+        case Token.text:   next = text()
+        case Token.param:  next = param()
+        case Token.escape: next = escape()
+        case Token.quoted: next = quoted()
+
+        case Token.end:
+          this.sql = sqlBuf.toStr
+          return
+
+        default: throw Err("unreachable")
+      }
+    }
+  }
+
+  ** Process a text token.
+  private Token text()
+  {
+    start := cur++
+    tok := nextToken()
+    while (tok == Token.text)
+    {
+      cur++
+      tok = nextToken()
+    }
+
+    sqlBuf.add(origSql[start..<cur])
+    return tok
+  }
+
+  ** Process a parameter token: @foo
+  private Token param()
+  {
+    start := cur++
+    while (cur < origSql.size && isIdent(origSql[cur]))
+      cur++
+
+    // add the JDBC placeholder
+    sqlBuf.add("?")
+
+    // remove the leading '@' from the param name
+    name := origSql[(start+1)..<cur]
+
+    // save the parameter's location
+    locs := params.getOrAdd(name, |k->Int[]| {Int[,]})
+    locs.add(++numParams)
+
+    return nextToken()
+  }
+
+  ** Process a escaped "@" or "\"
+  private Token escape()
+  {
+    sqlBuf.addChar(origSql[cur+1])
+    cur += 2
+
+    return nextToken()
+  }
+
+  ** Process a quoted token
+  private Token quoted()
+  {
+    start := cur++
+    while (cur < origSql.size)
+    {
+      if (origSql[cur] == '\'')
+      {
+        sqlBuf.add(origSql[start..(cur++)])
+        return nextToken()
+      }
+      cur++
+    }
+    throw SqlErr("Unterminated quoted text.")
+  }
+
+  ** Figure out the next token
+  private Token nextToken()
+  {
+    if (cur >= origSql.size)
+      return Token.end
+
+    switch(origSql[cur])
+    {
+      case '@':
+
+        if (isIdent(lookahead(1)))
+          return Token.param // @foo
+        else
+          return Token.text
+
+      case '\\':
+
+        look := lookahead(1)
+        if ((look == '@') || (look == '\\'))
+          return Token.escape
+        else
+          throw SqlErr("Invalid escape sequence '${origSql[cur..(cur+1)]}'.")
+
+      case '\'':
+          return Token.quoted
+
+      default:
+          return Token.text
+    }
+  }
+
+  ** Is the character part of a valid identifier?
+  private static Bool isIdent(Int ch)
+  {
+    return ((ch >= 'a') && (ch <= 'z')) ||
+           ((ch >= 'A') && (ch <= 'Z')) ||
+           ((ch >= '0') && (ch <= '9')) ||
+           (ch == '_')
+  }
+
+  ** Look ahead by n chars, or return -1 if past the end.
+  private Int lookahead(Int n)
+  {
+    return ((cur+n) < origSql.size) ? origSql[cur+n] : -1
+  }
+
+//////////////////////////////////////////////////////////////////////////
+// Fields
+//////////////////////////////////////////////////////////////////////////
+
+  private Str origSql
+  private Int cur := 0
+  private Int numParams := 0
+  private StrBuf sqlBuf := StrBuf()
+
+  internal Str? sql
+  internal Str:Int[] params := Str:Int[][:]
+}
+
+**************************************************************************
+** Fields
+**************************************************************************
+
+internal enum class Token
+{
+  text,
+  param,
+  escape,
+  quoted,
+  end
+}

src/sql/java/StatementPeer.java

@@ -39,14 +39,26 @@ public Statement prepare(Statement self)
     // replace though because we need to keep the key/value
     // map.
 
-    // If there is an @, then there might be a parameter
-    if (self.sql.contains("@"))
+    // Maybe there is a parameter or an escape.
+    if ((self.sql.indexOf('@') != -1) || (self.sql.indexOf('\\') != -1))
     {
-      DeprecatedTokenizer t = DeprecatedTokenizer.make(self.sql);
-      this.translated = t.sql;
-      this.paramMap = t.params;
+      // Check for deprecated escape: "@@foo"
+      String depEsc = self.typeof().pod().config("deprecatedEscape");
+
+      if ((depEsc != null) && depEsc.equals("true"))
+      {
+        DeprecatedTokenizer t = DeprecatedTokenizer.make(self.sql);
+        this.translated = t.sql;
+        this.paramMap = t.params;
+      }
+      else
+      {
+        Tokenizer t = Tokenizer.make(self.sql);
+        this.translated = t.sql;
+        this.paramMap = t.params;
+      }
     }
-    // No parameters, so we don't need to tokenize.
+    // No parameters or escapes, so we don't need to tokenize.
     else
     {
       this.translated = self.sql;

src/sql/test/SqlServiceTest.fan

@@ -60,6 +60,8 @@ class SqlServiceTest : Test
     uri  = pod.config("test.uri")      ?: throw Err("Missing 'sql::test.uri' config prop")
     user = pod.config("test.username") ?: throw Err("Missing 'sql::test.username' config prop")
     pass = pod.config("test.password") ?: throw Err("Missing 'sql::test.password' config prop")
+
+    Log.get("sql").info("SqlServiceTest.open " + uri);
     db = SqlConn.open(uri, user, pass)
     verifyEq(db.isClosed, false)
 
@@ -539,19 +541,36 @@ class SqlServiceTest : Test
     if (dbType != DbType.mysql) return;
 
     // We aren't preparing the statement,
-    // so we must not escape the user variable.
+    // so we cannot escape the user variable.
     db.sql("set @v1 = 42").execute
 
-    // We are preparing the statement,
-    // so we must escape the user variable.
-    stmt := db.sql("select name, @@v1 from farmers where farmer_id = @farmerId")
-    stmt.prepare
+    if (typeof.pod.config("deprecatedEscape") == "true")
+    {
+      // We are preparing the statement,
+      // so we must escape the user variable.
+      stmt := db.sql("select name, @@v1 from farmers where farmer_id = @farmerId")
+      stmt.prepare
+
+      rows := stmt.query(["farmerId":1])
+      verifyEq(rows.size, 1)
+      r := rows[0]
+      verifyEq(r.get(r.col("name")), "Alice")
+      verifyEq(r.get(r.col("@v1")),  42)
+    }
+    else
+    {
+      // We are preparing the statement,
+      // so we must escape the user variable.
+      stmt := db.sql("select name, \\@v1 from farmers where farmer_id = @farmerId")
+      stmt.prepare
+
+      rows := stmt.query(["farmerId":1])
+      verifyEq(rows.size, 1)
+      r := rows[0]
+      verifyEq(r.get(r.col("name")), "Alice")
+      verifyEq(r.get(r.col("@v1")),  42)
 
-    rows := stmt.query(["farmerId":1])
-    verifyEq(rows.size, 1)
-    r := rows[0]
-    verifyEq(r.get(r.col("name")), "Alice")
-    verifyEq(r.get(r.col("@v1")),  42)
+    }
   }
 
 //////////////////////////////////////////////////////////////////////////

src/sql/test/TokenizerTest.fan

@@ -0,0 +1,128 @@
+//
+// Copyright (c) 2024, Brian Frank and Andy Frank
+// Licensed under the Academic Free License version 3.0
+//
+// History:
+//   12 July 2024  Mike Jarmy   Creation
+//
+
+**
+** TokenizerTest
+**
+class TokenizerTest : Test
+{
+  Void test()
+  {
+    //---------------------------------------------------------------
+    // Tokenizer doesn't know anything about real SQL syntax, so lets
+    // test against all kinds of cases that it can process successfully even if
+    // the SQL is dubious or obviously bogus.
+
+    doVerify("", "", Str:Int[][:])
+    doVerify("x", "x", Str:Int[][:])
+
+    // params
+    doVerify("@a", "?", Str:Int[]["a": [1]])
+    doVerify("@a @b @a", "? ? ?", Str:Int[]["a": [1,3], "b": [2]])
+    doVerify("@a @b @a @a @c", "? ? ? ? ?", Str:Int[]["a": [1,3,4], "b": [2], "c": [5]])
+    doVerify("@", "@", Str:Int[][:])
+    doVerify("@@ x", "@@ x", Str:Int[][:])
+
+    // params and normal
+    doVerify("-@a-", "-?-", Str:Int[]["a": [1]])
+    doVerify("@a@>-@a@@@>", "?@>-?@@@>", Str:Int[]["a": [1,2]])
+
+    // params and quoted
+    doVerify("'x'@a", "'x'?", Str:Int[]["a": [1]])
+    doVerify("'x'y@a", "'x'y?", Str:Int[]["a": [1]])
+    doVerify("@a'@b'", "?'@b'", Str:Int[]["a": [1]])
+    doVerify("x'123'@a", "x'123'?", Str:Int[]["a": [1]])
+
+    // escape
+    doVerify("\\@b", "@b", Str:Int[][:])
+    doVerify("\\\\", "\\", Str:Int[][:])
+    doVerify("\\@\\\\\\@", "@\\@", Str:Int[][:])
+    doVerify("@a\\@b", "?@b", Str:Int[]["a": [1]])
+    doVerify("@a\\@\\@b", "?@@b", Str:Int[]["a": [1]])
+    doVerify("\\@b@a", "@b?", Str:Int[]["a": [1]])
+    doVerify("x\\@b@a", "x@b?", Str:Int[]["a": [1]])
+    doVerify("x\\@b'123'@a", "x@b'123'?", Str:Int[]["a": [1]])
+
+    // invalid escape
+    verifyErr(SqlErr#) { p := Tokenizer("\\^") }
+
+    // unterminated
+    verifyErr(SqlErr#) { p := Tokenizer("'") }
+    verifyErr(SqlErr#) { p := Tokenizer("@a'") }
+    verifyErr(SqlErr#) { p := Tokenizer("'x'@a'y") }
+
+    //--------------------------------------------------------
+    // Now lets go ahead and do some syntactically correct sql
+
+    doVerify(
+      "select * from foo",
+      "select * from foo",
+      Str:Int[][:])
+
+    // one param
+    doVerify(
+      "select name, age from farmers where name = @name",
+      "select name, age from farmers where name = ?",
+      Str:Int[]["name":[1]])
+
+    // repeated param
+    doVerify(
+      "select * from foo where @a = 1 or @b = 2 or @a = 3",
+      "select * from foo where ? = 1 or ? = 2 or ? = 3",
+      Str:Int[]["a":[1,3], "b":[2]])
+
+    // escaped mysql user variable
+    doVerify(
+      "select \\@bar",
+      "select @bar",
+      Str:Int[][:])
+    doVerify(
+      "select \\@bar from foo where @a = 1",
+      "select @bar from foo where ? = 1",
+      Str:Int[]["a":[1]])
+
+    // escaped mysql system variable
+    doVerify(
+      "select \\@\\@bar",
+      "select @@bar",
+      Str:Int[][:])
+    doVerify(
+      "select \\@\\@bar from foo where @a = 1",
+      "select @@bar from foo where ? = 1",
+      Str:Int[]["a":[1]])
+
+    // postgres operators that start with '@'
+    doVerify(
+      "select * from foo where @a \\@> 1",
+      "select * from foo where ? @> 1",
+      Str:Int[]["a":[1]])
+    doVerify(
+      "select * from foo where @a \\@\\@ 1",
+      "select * from foo where ? @@ 1",
+      Str:Int[]["a":[1]])
+
+    // quoted string
+    doVerify(
+      "select 'abc' from foo where @a = 1",
+      "select 'abc' from foo where ? = 1",
+      Str:Int[]["a":[1]])
+    doVerify(
+      "select '@x \\@y \\@ \\\\@>' from foo where @a = 1",
+      "select '@x \\@y \\@ \\\\@>' from foo where ? = 1",
+      Str:Int[]["a":[1]])
+  }
+
+  private Void doVerify(Str sql, Str expected, Str:Int[] params)
+  {
+    //echo("------------------------------------------")
+    //echo(sql)
+    t := Tokenizer(sql)
+    verifyEq(t.sql, expected)
+    verifyEq(t.params, params)
+  }
+}