Index: ext/fts3/fts3.c ================================================================== --- ext/fts3/fts3.c +++ ext/fts3/fts3.c @@ -1079,22 +1079,34 @@ char *zPrefix = 0; /* Prefix parameter value (or NULL) */ char *zCompress = 0; /* compress=? parameter (or NULL) */ char *zUncompress = 0; /* uncompress=? parameter (or NULL) */ char *zContent = 0; /* content=? parameter (or NULL) */ char *zLanguageid = 0; /* languageid=? parameter (or NULL) */ + char **azNotindexed = 0; /* The set of notindexed= columns */ + int nNotindexed = 0; /* Size of azNotindexed[] array */ assert( strlen(argv[0])==4 ); assert( (sqlite3_strnicmp(argv[0], "fts4", 4)==0 && isFts4) || (sqlite3_strnicmp(argv[0], "fts3", 4)==0 && !isFts4) ); nDb = (int)strlen(argv[1]) + 1; nName = (int)strlen(argv[2]) + 1; - aCol = (const char **)sqlite3_malloc(sizeof(const char *) * (argc-2) ); - if( !aCol ) return SQLITE_NOMEM; - memset((void *)aCol, 0, sizeof(const char *) * (argc-2)); + nByte = sizeof(const char *) * (argc-2); + aCol = (const char **)sqlite3_malloc(nByte); + if( aCol ){ + memset(aCol, 0, nByte); + azNotindexed = (char **)sqlite3_malloc(nByte); + } + if( azNotindexed ){ + memset(azNotindexed, 0, nByte); + } + if( !aCol || !azNotindexed ){ + rc = SQLITE_NOMEM; + goto fts3_init_out; + } /* Loop through all of the arguments passed by the user to the FTS3/4 ** module (i.e. all the column names and special arguments). This loop ** does the following: ** @@ -1129,11 +1141,12 @@ { "prefix", 6 }, /* 1 -> PREFIX */ { "compress", 8 }, /* 2 -> COMPRESS */ { "uncompress", 10 }, /* 3 -> UNCOMPRESS */ { "order", 5 }, /* 4 -> ORDER */ { "content", 7 }, /* 5 -> CONTENT */ - { "languageid", 10 } /* 6 -> LANGUAGEID */ + { "languageid", 10 }, /* 6 -> LANGUAGEID */ + { "notindexed", 10 } /* 7 -> NOTINDEXED */ }; int iOpt; if( !zVal ){ rc = SQLITE_NOMEM; @@ -1195,10 +1208,15 @@ assert( iOpt==6 ); sqlite3_free(zLanguageid); zLanguageid = zVal; zVal = 0; break; + + case 7: /* NOTINDEXED */ + azNotindexed[nNotindexed++] = zVal; + zVal = 0; + break; } } sqlite3_free(zVal); } } @@ -1266,10 +1284,11 @@ /* Allocate and populate the Fts3Table structure. */ nByte = sizeof(Fts3Table) + /* Fts3Table */ nCol * sizeof(char *) + /* azColumn */ nIndex * sizeof(struct Fts3Index) + /* aIndex */ + nCol * sizeof(u8) + /* abNotindexed */ nName + /* zName */ nDb + /* zDb */ nString; /* Space for azColumn strings */ p = (Fts3Table*)sqlite3_malloc(nByte); if( p==0 ){ @@ -1299,13 +1318,14 @@ memcpy(p->aIndex, aIndex, sizeof(struct Fts3Index) * nIndex); p->nIndex = nIndex; for(i=0; iaIndex[i].hPending, FTS3_HASH_STRING, 1); } + p->abNotindexed = (u8 *)&p->aIndex[nIndex]; /* Fill in the zName and zDb fields of the vtab structure. */ - zCsr = (char *)&p->aIndex[nIndex]; + zCsr = (char *)&p->abNotindexed[nCol]; p->zName = zCsr; memcpy(zCsr, argv[2], nName); zCsr += nName; p->zDb = zCsr; memcpy(zCsr, argv[1], nDb); @@ -1322,11 +1342,30 @@ p->azColumn[iCol] = zCsr; zCsr += n+1; assert( zCsr <= &((char *)p)[nByte] ); } - if( (zCompress==0)!=(zUncompress==0) ){ + /* Fill in the abNotindexed array */ + for(iCol=0; iColazColumn[iCol]); + for(i=0; iazColumn[iCol], zNot, n) ){ + p->abNotindexed[iCol] = 1; + sqlite3_free(zNot); + azNotindexed[i] = 0; + } + } + } + for(i=0; izReadExprlist = fts3ReadExprList(p, zUncompress, &rc); @@ -1363,11 +1402,13 @@ sqlite3_free(aIndex); sqlite3_free(zCompress); sqlite3_free(zUncompress); sqlite3_free(zContent); sqlite3_free(zLanguageid); + for(i=0; ipModule->xDestroy(pTokenizer); Index: ext/fts3/fts3Int.h ================================================================== --- ext/fts3/fts3Int.h +++ ext/fts3/fts3Int.h @@ -204,10 +204,11 @@ sqlite3 *db; /* The database connection */ const char *zDb; /* logical database name */ const char *zName; /* virtual table name */ int nColumn; /* number of named columns in virtual table */ char **azColumn; /* column names. malloced */ + u8 *abNotindexed; /* True for 'notindexed' columns */ sqlite3_tokenizer *pTokenizer; /* tokenizer for inserts and queries */ char *zContentTbl; /* content=xxx option, or NULL */ char *zLanguageid; /* languageid=xxx option, or NULL */ u8 bAutoincrmerge; /* True if automerge=1 */ u32 nLeafAdd; /* Number of leaf blocks added this trans */ Index: ext/fts3/fts3_write.c ================================================================== --- ext/fts3/fts3_write.c +++ ext/fts3/fts3_write.c @@ -898,16 +898,19 @@ sqlite3_value **apVal, u32 *aSz ){ int i; /* Iterator variable */ for(i=2; inColumn+2; i++){ - const char *zText = (const char *)sqlite3_value_text(apVal[i]); - int rc = fts3PendingTermsAdd(p, iLangid, zText, i-2, &aSz[i-2]); - if( rc!=SQLITE_OK ){ - return rc; + int iCol = i-2; + if( p->abNotindexed[iCol]==0 ){ + const char *zText = (const char *)sqlite3_value_text(apVal[i]); + int rc = fts3PendingTermsAdd(p, iLangid, zText, iCol, &aSz[iCol]); + if( rc!=SQLITE_OK ){ + return rc; + } + aSz[p->nColumn] += sqlite3_value_bytes(apVal[i]); } - aSz[p->nColumn] += sqlite3_value_bytes(apVal[i]); } return SQLITE_OK; } /* @@ -1050,13 +1053,16 @@ if( SQLITE_ROW==sqlite3_step(pSelect) ){ int i; int iLangid = langidFromSelect(p, pSelect); rc = fts3PendingTermsDocid(p, iLangid, sqlite3_column_int64(pSelect, 0)); for(i=1; rc==SQLITE_OK && i<=p->nColumn; i++){ - const char *zText = (const char *)sqlite3_column_text(pSelect, i); - rc = fts3PendingTermsAdd(p, iLangid, zText, -1, &aSz[i-1]); - aSz[p->nColumn] += sqlite3_column_bytes(pSelect, i); + int iCol = i-1; + if( p->abNotindexed[iCol]==0 ){ + const char *zText = (const char *)sqlite3_column_text(pSelect, i); + rc = fts3PendingTermsAdd(p, iLangid, zText, -1, &aSz[iCol]); + aSz[p->nColumn] += sqlite3_column_bytes(pSelect, i); + } } if( rc!=SQLITE_OK ){ sqlite3_reset(pSelect); *pRC = rc; return; @@ -3294,13 +3300,15 @@ int iCol; int iLangid = langidFromSelect(p, pStmt); rc = fts3PendingTermsDocid(p, iLangid, sqlite3_column_int64(pStmt, 0)); memset(aSz, 0, sizeof(aSz[0]) * (p->nColumn+1)); for(iCol=0; rc==SQLITE_OK && iColnColumn; iCol++){ - const char *z = (const char *) sqlite3_column_text(pStmt, iCol+1); - rc = fts3PendingTermsAdd(p, iLangid, z, iCol, &aSz[iCol]); - aSz[p->nColumn] += sqlite3_column_bytes(pStmt, iCol+1); + if( p->abNotindexed[iCol]==0 ){ + const char *z = (const char *) sqlite3_column_text(pStmt, iCol+1); + rc = fts3PendingTermsAdd(p, iLangid, z, iCol, &aSz[iCol]); + aSz[p->nColumn] += sqlite3_column_bytes(pStmt, iCol+1); + } } if( p->bHasDocsize ){ fts3InsertDocsize(&rc, p, aSz); } if( rc!=SQLITE_OK ){ @@ -5099,39 +5107,41 @@ assert( pCsr->isRequireSeek==0 ); iDocid = sqlite3_column_int64(pCsr->pStmt, 0); for(i=0; inColumn && rc==SQLITE_OK; i++){ - const char *zText = (const char *)sqlite3_column_text(pCsr->pStmt, i+1); - sqlite3_tokenizer_cursor *pTC = 0; - - rc = sqlite3Fts3OpenTokenizer(pT, pCsr->iLangid, zText, -1, &pTC); - while( rc==SQLITE_OK ){ - char const *zToken; /* Buffer containing token */ - int nToken = 0; /* Number of bytes in token */ - int iDum1 = 0, iDum2 = 0; /* Dummy variables */ - int iPos = 0; /* Position of token in zText */ - - rc = pModule->xNext(pTC, &zToken, &nToken, &iDum1, &iDum2, &iPos); - for(pDef=pCsr->pDeferred; pDef && rc==SQLITE_OK; pDef=pDef->pNext){ - Fts3PhraseToken *pPT = pDef->pToken; - if( (pDef->iCol>=p->nColumn || pDef->iCol==i) - && (pPT->bFirst==0 || iPos==0) - && (pPT->n==nToken || (pPT->isPrefix && pPT->nz, pPT->n)) - ){ - fts3PendingListAppend(&pDef->pList, iDocid, i, iPos, &rc); - } - } - } - if( pTC ) pModule->xClose(pTC); - if( rc==SQLITE_DONE ) rc = SQLITE_OK; - } - - for(pDef=pCsr->pDeferred; pDef && rc==SQLITE_OK; pDef=pDef->pNext){ - if( pDef->pList ){ - rc = fts3PendingListAppendVarint(&pDef->pList, 0); + if( p->abNotindexed[i]==0 ){ + const char *zText = (const char *)sqlite3_column_text(pCsr->pStmt, i+1); + sqlite3_tokenizer_cursor *pTC = 0; + + rc = sqlite3Fts3OpenTokenizer(pT, pCsr->iLangid, zText, -1, &pTC); + while( rc==SQLITE_OK ){ + char const *zToken; /* Buffer containing token */ + int nToken = 0; /* Number of bytes in token */ + int iDum1 = 0, iDum2 = 0; /* Dummy variables */ + int iPos = 0; /* Position of token in zText */ + + rc = pModule->xNext(pTC, &zToken, &nToken, &iDum1, &iDum2, &iPos); + for(pDef=pCsr->pDeferred; pDef && rc==SQLITE_OK; pDef=pDef->pNext){ + Fts3PhraseToken *pPT = pDef->pToken; + if( (pDef->iCol>=p->nColumn || pDef->iCol==i) + && (pPT->bFirst==0 || iPos==0) + && (pPT->n==nToken || (pPT->isPrefix && pPT->nz, pPT->n)) + ){ + fts3PendingListAppend(&pDef->pList, iDocid, i, iPos, &rc); + } + } + } + if( pTC ) pModule->xClose(pTC); + if( rc==SQLITE_DONE ) rc = SQLITE_OK; + } + + for(pDef=pCsr->pDeferred; pDef && rc==SQLITE_OK; pDef=pDef->pNext){ + if( pDef->pList ){ + rc = fts3PendingListAppendVarint(&pDef->pList, 0); + } } } } return rc; Index: test/fts3malloc.test ================================================================== --- test/fts3malloc.test +++ test/fts3malloc.test @@ -60,10 +60,13 @@ CREATE VIRTUAL TABLE ft5 USING fts3(a, b, tokenize unknown) } {unknown tokenizer: unknown} do_write_test fts3_malloc-1.6 sqlite_master { CREATE VIRTUAL TABLE ft6 USING fts3(a, b, tokenize porter) } +do_write_test fts3_malloc-1.7 sqlite_master { + CREATE VIRTUAL TABLE ft7 USING fts4(a, b, notindexed=b) +} # Test the xConnect/xDisconnect methods: #db eval { ATTACH 'test2.db' AS aux } #do_write_test fts3_malloc-1.6 aux.sqlite_master { # CREATE VIRTUAL TABLE aux.ft7 USING fts3(a, b, c); @@ -79,10 +82,11 @@ DROP TABLE ft1; DROP TABLE ft2; DROP TABLE ft3; DROP TABLE ft4; DROP TABLE ft6; + DROP TABLE ft7; } execsql { CREATE VIRTUAL TABLE ft USING fts3(a, b) } for {set ii 1} {$ii < 32} {incr ii} { set a [list] set b [list] ADDED test/fts4noti.test Index: test/fts4noti.test ================================================================== --- /dev/null +++ test/fts4noti.test @@ -0,0 +1,171 @@ +# 2013 June 21 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The +# focus of this script is testing the notindexed=xxx FTS4 option. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set ::testprefix fts4noti + +# If SQLITE_ENABLE_FTS3 is defined, omit this file. +ifcapable !fts3 { + finish_test + return +} + + +#------------------------------------------------------------------------- +# Test that typos in "notindexed=" column names are detected. +# +do_execsql_test 1.0 { + CREATE TABLE cc(a, b, c); +} +foreach {tn arg res} { + 1 "(b, c, notindexed=a)" {1 {no such column: a}} + 2 "(a, b, notindexed=a)" {0 {}} + 3 "(a, b, notindexed=a, notindexed=a)" {0 {}} + 4 "(notindexed=a, a, b)" {0 {}} + 5 "(notindexed=a, notindexed=b, notindexed=c, a, b, c, d)" {0 {}} + 6 "(notindexed=a, notindexed=B, notindexed=c, a, b, c, d)" {0 {}} + 7 "(notindexed=a, notindexed=b, notindexed=c, a, B, c, d)" {0 {}} + 8 "(notindexed=d, content=cc)" {1 {no such column: d}} + 9 "(notindexed=a, content=cc)" {0 {}} + 10 "(notindexed=a, notindexed=b, a)" {1 {no such column: b}} + 11 "(notindexed=a, notindexed=b, b)" {1 {no such column: a}} +} { + do_catchsql_test 1.$tn "CREATE VIRTUAL TABLE t1 USING fts4 $arg" $res + if {[lindex $res 0]==0} { execsql "DROP TABLE t1" } +} + +do_execsql_test 1.x { SELECT name FROM sqlite_master } {cc} + + +#------------------------------------------------------------------------- +# Test that notindexed columns are not indexed. +# +foreach {tn schema} { + 1 { + CREATE VIRTUAL TABLE t1 USING fts4(a, b, c, notindexed=b); + } + 2 { + CREATE TABLE c1(a, b, c); + INSERT INTO c1 VALUES('one two', 'three four', 'five six'); + INSERT INTO c1 VALUES('three four', 'five six', 'one two'); + CREATE VIRTUAL TABLE t1 USING fts4(content=c1, notindexed=b); + } + 3 { + CREATE VIRTUAL TABLE t1 USING fts4(content="", a, b, c, notindexed=b); + } +} { + execsql $schema + + do_execsql_test 2.$tn.1 { + INSERT INTO t1(docid,a,b,c) VALUES(1, 'one two', 'three four', 'five six'); + INSERT INTO t1(docid,a,b,c) VALUES(2, 'three four', 'five six', 'one two'); + } + + do_execsql_test 2.$tn.2 { SELECT docid FROM t1 WHERE t1 MATCH 'one' } {1 2} + do_execsql_test 2.$tn.3 { SELECT docid FROM t1 WHERE t1 MATCH 'three' } {2} + do_execsql_test 2.$tn.4 { SELECT docid FROM t1 WHERE t1 MATCH 'five' } {1} + + do_execsql_test 2.$tn.5 { INSERT INTO t1(t1) VALUES('optimize') } + + do_execsql_test 2.$tn.6 { SELECT docid FROM t1 WHERE t1 MATCH 'one' } {1 2} + do_execsql_test 2.$tn.7 { SELECT docid FROM t1 WHERE t1 MATCH 'three' } {2} + do_execsql_test 2.$tn.8 { SELECT docid FROM t1 WHERE t1 MATCH 'five' } {1} + + if {$tn!=3} { + do_execsql_test 2.$tn.9 { INSERT INTO t1(t1) VALUES('rebuild') } + + do_execsql_test 2.$tn.10 { SELECT docid FROM t1 WHERE t1 MATCH 'one' } {1 2} + do_execsql_test 2.$tn.11 { SELECT docid FROM t1 WHERE t1 MATCH 'three' } {2} + do_execsql_test 2.$tn.12 { SELECT docid FROM t1 WHERE t1 MATCH 'five' } {1} + + do_execsql_test 2.$tn.13 { + SELECT a,b,c FROM t1 WHERE docid=1 + } {{one two} {three four} {five six}} + do_execsql_test 2.$tn.14 { + SELECT a,b,c FROM t1 WHERE docid=2 + } {{three four} {five six} {one two}} + } + + do_execsql_test 2.x { DROP TABLE t1 } +} + +#------------------------------------------------------------------------- +# Test that notindexed columns are not scanned for deferred tokens. +# + +do_execsql_test 3.1 { + CREATE VIRTUAL TABLE t2 USING fts4(x, y, notindexed=x); +} +do_test 3.2 { + set v [string repeat " 1" 50000] + set v1 "x $v" + set v2 "y $v" + execsql { + INSERT INTO t2 VALUES(1, 'x y z'); + INSERT INTO t2 VALUES(2, $v1); + INSERT INTO t2 VALUES(3, $v2); + INSERT INTO t2 VALUES(4, $v2); + INSERT INTO t2 VALUES(5, $v2); + INSERT INTO t2 VALUES(6, $v2); + } +} {} + +do_execsql_test 3.3 { SELECT x FROM t2 WHERE t2 MATCH '2' } {} +do_execsql_test 3.4 { SELECT x FROM t2 WHERE t2 MATCH '1' } {2 3 4 5 6} +do_execsql_test 3.5 { SELECT x FROM t2 WHERE t2 MATCH 'x' } {1 2} +do_execsql_test 3.6 { SELECT x FROM t2 WHERE t2 MATCH 'x 1' } {2} + +do_execsql_test 3.x { DROP TABLE t2 } + +#------------------------------------------------------------------------- +# Test that the types of notindexed columns are not modified. +# +do_execsql_test 4.1 { + CREATE VIRTUAL TABLE t2 USING fts4(poi, addr, notindexed=poi); + INSERT INTO t2 VALUES(114, 'x x x'); + INSERT INTO t2 VALUES(X'1234', 'y y y'); + INSERT INTO t2 VALUES(NULL, 'z z z'); + INSERT INTO t2 VALUES(113.2, 'w w w'); + INSERT INTO t2 VALUES('poi', 'v v v'); +} +do_execsql_test 4.2 { SELECT typeof(poi) FROM t2 WHERE t2 MATCH 'x' } {integer} +do_execsql_test 4.3 { SELECT typeof(poi) FROM t2 WHERE t2 MATCH 'y' } {blob} +do_execsql_test 4.4 { SELECT typeof(poi) FROM t2 WHERE t2 MATCH 'z' } {null} +do_execsql_test 4.5 { SELECT typeof(poi) FROM t2 WHERE t2 MATCH 'w' } {real} +do_execsql_test 4.6 { SELECT typeof(poi) FROM t2 WHERE t2 MATCH 'v' } {text} + +do_execsql_test 4.x { DROP TABLE t2 } + +#------------------------------------------------------------------------- +# Test that multiple notindexed options on a single table work as expected. +# +do_execsql_test 5.1 { + CREATE VIRTUAL TABLE t2 USING fts4( + notindexed="three", one, two, three, notindexed="one", + ); + INSERT INTO t2 VALUES('a', 'b', 'c'); + INSERT INTO t2 VALUES('c', 'a', 'b'); + INSERT INTO t2 VALUES('b', 'c', 'a'); +} +do_execsql_test 5.2 { SELECT docid FROM t2 WHERE t2 MATCH 'a' } {2} +do_execsql_test 5.3 { SELECT docid FROM t2 WHERE t2 MATCH 'b' } {1} +do_execsql_test 5.4 { SELECT docid FROM t2 WHERE t2 MATCH 'c' } {3} + +do_execsql_test 5.x { DROP TABLE t2 } + +finish_test + + + Index: test/permutations.test ================================================================== --- test/permutations.test +++ test/permutations.test @@ -192,11 +192,11 @@ fts3fault.test fts3malloc.test fts3matchinfo.test fts3aux1.test fts3comp1.test fts3auto.test fts4aa.test fts4content.test fts3conf.test fts3prefix.test fts3fault2.test fts3corrupt.test fts3corrupt2.test fts3first.test fts4langid.test fts4merge.test - fts4check.test fts4unicode.test + fts4check.test fts4unicode.test fts4noti.test } test_suite "nofaultsim" -prefix "" -description { "Very" quick test suite. Runs in less than 5 minutes on a workstation. This test suite is the same as the "quick" tests, except that some files