SQLite

Check-in [de9f8ef6eb]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Fix a memory leak that could follow an OOM condition in fts5.
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | fts5
Files: files | file ages | folders
SHA1: de9f8ef6ebf036df5a558cd78fb4927da2d83ce8
User & Date: dan 2015-05-19 11:32:01.082
Context
2015-05-19
11:38
Update Makefile.in so that the amalgamation files built by "make sqlite3.c" include fts5. (check-in: 2870a80593 user: dan tags: fts5)
11:32
Fix a memory leak that could follow an OOM condition in fts5. (check-in: de9f8ef6eb user: dan tags: fts5)
2015-05-18
20:34
Improve test coverage of fts5_storage.c. (check-in: 4dfe2e5871 user: dan tags: fts5)
Changes
Unified Diff Ignore Whitespace Patch
Changes to ext/fts5/fts5_tcl.c.
︙︙
150
151
152
153
154
155
156



157
158
159
160
161
162
163

  Tcl_IncrRefCount(pEval);
  Tcl_ListObjAppendElement(p->interp, pEval, Tcl_NewStringObj(zToken, nToken));
  Tcl_ListObjAppendElement(p->interp, pEval, Tcl_NewIntObj(iStart));
  Tcl_ListObjAppendElement(p->interp, pEval, Tcl_NewIntObj(iEnd));

  rc = Tcl_EvalObjEx(p->interp, pEval, 0);
  Tcl_DecrRefCount(pEval);




  return rc;
}

static int xF5tApi(void*, Tcl_Interp*, int, Tcl_Obj *CONST []);

static int xQueryPhraseCb(








>
>
>








150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166

  Tcl_IncrRefCount(pEval);
  Tcl_ListObjAppendElement(p->interp, pEval, Tcl_NewStringObj(zToken, nToken));
  Tcl_ListObjAppendElement(p->interp, pEval, Tcl_NewIntObj(iStart));
  Tcl_ListObjAppendElement(p->interp, pEval, Tcl_NewIntObj(iEnd));

  rc = Tcl_EvalObjEx(p->interp, pEval, 0);
  Tcl_DecrRefCount(pEval);
  if( rc==TCL_OK ){
    rc = f5tResultToErrorCode(Tcl_GetStringResult(p->interp));
  }

  return rc;
}

static int xF5tApi(void*, Tcl_Interp*, int, Tcl_Obj *CONST []);

static int xQueryPhraseCb(
︙︙
Changes to ext/fts5/fts5_tokenize.c.
︙︙
51
52
53
54
55
56
57
58







59
60
61
62
63
64
65

    if( (zArg[i] & 0x80)==0 ){
      p->aTokenChar[(int)zArg[i]] = (unsigned char)bTokenChars;
    }
  }
}

/*
** Create a "ascii" tokenizer.







*/
static int fts5AsciiCreate(
  void *pCtx, 
  const char **azArg, int nArg,
  Fts5Tokenizer **ppOut
){
  int rc = SQLITE_OK;








|
>
>
>
>
>
>
>








51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72

    if( (zArg[i] & 0x80)==0 ){
      p->aTokenChar[(int)zArg[i]] = (unsigned char)bTokenChars;
    }
  }
}

/*
** Delete a "ascii" tokenizer.
*/
static void fts5AsciiDelete(Fts5Tokenizer *p){
  sqlite3_free(p);
}

/*
** Create an "ascii" tokenizer.
*/
static int fts5AsciiCreate(
  void *pCtx, 
  const char **azArg, int nArg,
  Fts5Tokenizer **ppOut
){
  int rc = SQLITE_OK;
︙︙
81
82
83
84
85
86
87




88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108

        }else
        if( 0==sqlite3_stricmp(azArg[i], "separators") ){
          fts5AsciiAddExceptions(p, zArg, 0);
        }else{
          rc = SQLITE_ERROR;
        }
      }




    }
  }

  *ppOut = (Fts5Tokenizer*)p;
  return rc;
}

/*
** Delete a "ascii" tokenizer.
*/
static void fts5AsciiDelete(Fts5Tokenizer *p){
  sqlite3_free(p);
}


static void asciiFold(char *aOut, const char *aIn, int nByte){
  int i;
  for(i=0; i<nByte; i++){
    char c = aIn[i];
    if( c>='A' && c<='Z' ) c += 32;
    aOut[i] = c;








>
>
>
>







<
<
<
<
<
<
<








88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105







106
107
108
109
110
111
112

        }else
        if( 0==sqlite3_stricmp(azArg[i], "separators") ){
          fts5AsciiAddExceptions(p, zArg, 0);
        }else{
          rc = SQLITE_ERROR;
        }
      }
      if( rc!=SQLITE_OK ){
        fts5AsciiDelete((Fts5Tokenizer*)p);
        p = 0;
      }
    }
  }

  *ppOut = (Fts5Tokenizer*)p;
  return rc;
}









static void asciiFold(char *aOut, const char *aIn, int nByte){
  int i;
  for(i=0; i<nByte; i++){
    char c = aIn[i];
    if( c>='A' && c<='Z' ) c += 32;
    aOut[i] = c;
︙︙
Changes to ext/fts5/test/fts5_common.tcl.
︙︙
47
48
49
50
51
52
53

54
55
56
57
58
59
60

  }
  set res
}

proc test_append_token {varname token iStart iEnd} {
  upvar $varname var
  lappend var $token

}
proc fts5_test_tokenize {cmd} {
  set res [list]
  for {set i 0} {$i < [$cmd xColumnCount]} {incr i} {
    set tokens [list]
    $cmd xTokenize [$cmd xColumnText $i] [list test_append_token tokens]
    lappend res $tokens








>








47
48
49
50
51
52
53
54
55
56
57
58
59
60
61

  }
  set res
}

proc test_append_token {varname token iStart iEnd} {
  upvar $varname var
  lappend var $token
  return "SQLITE_OK"
}
proc fts5_test_tokenize {cmd} {
  set res [list]
  for {set i 0} {$i < [$cmd xColumnCount]} {incr i} {
    set tokens [list]
    $cmd xTokenize [$cmd xColumnText $i] [list test_append_token tokens]
    lappend res $tokens
︙︙
Changes to ext/fts5/test/fts5tokenizer.test. 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19

# 2014 Dec 20
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
#
# Tests focusing on the fts5 tokenizers
#

source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5tokenizer


do_execsql_test 1.0 {












|








1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19

# 2014 Dec 20
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
#
# Tests focusing on the built-in fts5 tokenizers. 
#

source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5tokenizer


do_execsql_test 1.0 {
︙︙
96
97
98
99
100
101
102
103

































104
105

    9 vw  1
  } {
    do_execsql_test 5.$tn.2.$tn2 "
      SELECT rowid FROM t1 WHERE t1 MATCH '\"$token\"'
    " $res
  }
}


































finish_test










>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>



96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138

    9 vw  1
  } {
    do_execsql_test 5.$tn.2.$tn2 "
      SELECT rowid FROM t1 WHERE t1 MATCH '\"$token\"'
    " $res
  }
}

#-------------------------------------------------------------------------
# Miscellaneous tests for the ascii tokenizer.
#
# 5.1.*: Test that the ascii tokenizer ignores non-ASCII characters in the
#        'separators' option. But unicode61 does not.
#
# 5.2.*: An option without an argument is an error.
#

do_test 5.1.1 {
  execsql "
    CREATE VIRTUAL TABLE a1 USING fts5(x, tokenize=`ascii separators '\u1234'`);
    INSERT INTO a1 VALUES('abc\u1234def');
  "
  execsql { SELECT rowid FROM a1 WHERE a1 MATCH 'def' } 
} {}

do_test 5.1.2 {
  execsql "
    CREATE VIRTUAL TABLE a2 USING fts5(
        x, tokenize=`unicode61 separators '\u1234'`);
    INSERT INTO a2 VALUES('abc\u1234def');
  "
  execsql { SELECT rowid FROM a2 WHERE a2 MATCH 'def' } 
} {1}

do_catchsql_test 5.2 {
  CREATE VIRTUAL TABLE a3 USING fts5(x, y, tokenize = 'ascii tokenchars');
} {1 {error in tokenizer constructor}}
do_catchsql_test 5.3 {
  CREATE VIRTUAL TABLE a3 USING fts5(x, y, tokenize = 'ascii opt arg');
} {1 {error in tokenizer constructor}}

finish_test