SQLite

Check-in [27160df7b3]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Change the fts3_tokenizer() function to always return the pointer as a BLOB as long as the first argument is a bound parameter, regardless of the SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER setting.
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | value_frombind
Files: files | file ages | folders
SHA3-256: 27160df7b3a04ac59d06013ede1d2ee27a4ace86a525ecc333bf7f8b5c580afb
User & Date: drh 2019-03-29 17:26:44.232
Context
2019-04-01
20:57
Performance improvement on the OP_Variable opcode. (Closed-Leaf check-in: 1dc7993bb6 user: drh tags: value_frombind)
2019-03-29
17:26
Change the fts3_tokenizer() function to always return the pointer as a BLOB as long as the first argument is a bound parameter, regardless of the SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER setting. (check-in: 27160df7b3 user: drh tags: value_frombind)
11:39
The two-argument version of fts3_tokenizer() works regardless of the value of SQLITE_DBCONFIG_ENABLE_FT3_TOKENIZER as long as the second argument is a bind parameter. (check-in: ab76e3a90e user: drh tags: value_frombind)
Changes
Unified Diff Ignore Whitespace Patch
Changes to ext/fts3/fts3_tokenizer.c.
︙︙
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116

    if( !pPtr ){
      char *zErr = sqlite3_mprintf("unknown tokenizer: %s", zName);
      sqlite3_result_error(context, zErr, -1);
      sqlite3_free(zErr);
      return;
    }
  }
  if( fts3TokenizerEnabled(context) ){
    sqlite3_result_blob(context, (void *)&pPtr, sizeof(pPtr), SQLITE_TRANSIENT);
  }
}

int sqlite3Fts3IsIdChar(char c){
  static const char isFtsIdChar[] = {
      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,  /* 0x */








|








102
103
104
105
106
107
108
109
110
111
112
113
114
115
116

    if( !pPtr ){
      char *zErr = sqlite3_mprintf("unknown tokenizer: %s", zName);
      sqlite3_result_error(context, zErr, -1);
      sqlite3_free(zErr);
      return;
    }
  }
  if( fts3TokenizerEnabled(context) || sqlite3_value_frombind(argv[0]) ){
    sqlite3_result_blob(context, (void *)&pPtr, sizeof(pPtr), SQLITE_TRANSIENT);
  }
}

int sqlite3Fts3IsIdChar(char c){
  static const char isFtsIdChar[] = {
      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,  /* 0x */
︙︙
Changes to test/fts3atoken.test.
︙︙
82
83
84
85
86
87
88
89


90
91
92
93
94
95
96
97
98
99

100




101
102


103


104

105


106








107

108
109
110
111
112
113
114

    INSERT INTO t1(content) VALUES('There was movement at the station');
    INSERT INTO t1(content) VALUES('For the word has passed around');
    INSERT INTO t1(content) VALUES('That the colt from ol regret had got');
    SELECT content FROM t1 WHERE content MATCH 'movement'
  }
} {{There was movement at the station}}

unset -nocomplain simple


set simple [db one {SELECT fts3_tokenizer('simple')}]
sqlite3_db_config db SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER 0
do_catchsql_test 1.6 {
  SELECT fts3_tokenizer('blah', fts3_tokenizer('simple')) IS NULL;
} {1 {fts3tokenize disabled}}
do_test fts3atoken-1.7 {
  execsql {
    SELECT fts3_tokenizer('blah2', $simple) IS NULL;
  }
} {1}

sqlite3_db_config db SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER 1




do_test fts3atoken-1.8 {
  execsql {


    SELECT fts3_tokenizer('blah2') == fts3_tokenizer('simple');


  }

} {1}














#--------------------------------------------------------------------------
# Test cases fts3atoken-2.* test error cases in the scalar function based
# API for getting and setting tokenizers.
#
do_test fts3atoken-2.1 {
  catchsql {








|
>
>










>
|
>
>
>
>


>
>
|
>
>

>
|
>
>
|
>
>
>
>
>
>
>
>
|
>








82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137

    INSERT INTO t1(content) VALUES('There was movement at the station');
    INSERT INTO t1(content) VALUES('For the word has passed around');
    INSERT INTO t1(content) VALUES('That the colt from ol regret had got');
    SELECT content FROM t1 WHERE content MATCH 'movement'
  }
} {{There was movement at the station}}

unset -nocomplain simple blah2name simplename
set simplename "simple"
set blah2name "blah2"
set simple [db one {SELECT fts3_tokenizer('simple')}]
sqlite3_db_config db SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER 0
do_catchsql_test 1.6 {
  SELECT fts3_tokenizer('blah', fts3_tokenizer('simple')) IS NULL;
} {1 {fts3tokenize disabled}}
do_test fts3atoken-1.7 {
  execsql {
    SELECT fts3_tokenizer('blah2', $simple) IS NULL;
  }
} {1}

# With ENABLE_FTS3_TOKENIZER off, the fts3_tokenzer(1) function
# returns NULL unless the first parameter is a bound parameter.
# If the first parameter is a bound parameter, then fts3_tokenizer(1)
# returns the actual pointer value as a BLOB.
#
do_test fts3atoken-1.8 {
  execsql {
    SELECT fts3_tokenizer($blah2name) == fts3_tokenizer($simplename),
           typeof(fts3_tokenizer($blah2name)),
           typeof(fts3_tokenizer('blah2')),
           typeof(fts3_tokenizer($simplename)),
           typeof(fts3_tokenizer('simple'));
  }
} {1 blob null blob null}

# With ENABLE_FTS3_TOKENIZER on, fts3_tokenizer() always returns
# the BLOB pointer, regardless the parameter
#
sqlite3_db_config db SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER 1
do_test fts3atoken-1.9 {
  execsql {
    SELECT fts3_tokenizer('blah2') == fts3_tokenizer('simple'),
           typeof(fts3_tokenizer($blah2name)),
           typeof(fts3_tokenizer('blah2')),
           typeof(fts3_tokenizer($simplename)),
           typeof(fts3_tokenizer('simple'));
  }
} {1 blob blob blob blob}

#--------------------------------------------------------------------------
# Test cases fts3atoken-2.* test error cases in the scalar function based
# API for getting and setting tokenizers.
#
do_test fts3atoken-2.1 {
  catchsql {
︙︙