Index: ext/fts3/fts3.c ================================================================== --- ext/fts3/fts3.c +++ ext/fts3/fts3.c @@ -446,10 +446,12 @@ /* Free any prepared statements held */ for(i=0; iaStmt); i++){ sqlite3_finalize(p->aStmt[i]); } sqlite3_free(p->zSegmentsTbl); + sqlite3_free(p->zReadExprlist); + sqlite3_free(p->zWriteExprlist); /* Invoke the tokenizer destructor to free the tokenizer. */ p->pTokenizer->pModule->xDestroy(p->pTokenizer); sqlite3_free(p); @@ -662,10 +664,100 @@ sqlite3Fts3Dequote(zValue); } *pzValue = zValue; return 1; } + +/* +** Append the output of a printf() style formatting to an existing string. +*/ +static void fts3Appendf( + int *pRc, /* IN/OUT: Error code */ + char **pz, /* IN/OUT: Pointer to string buffer */ + const char *zFormat, /* Printf format string to append */ + ... /* Arguments for printf format string */ +){ + if( *pRc==SQLITE_OK ){ + va_list ap; + char *z; + va_start(ap, zFormat); + z = sqlite3_vmprintf(zFormat, ap); + if( z && *pz ){ + char *z2 = sqlite3_mprintf("%s%s", *pz, z); + sqlite3_free(z); + z = z2; + } + if( z==0 ) *pRc = SQLITE_NOMEM; + sqlite3_free(*pz); + *pz = z; + } +} + +/* +** Return a list of comma separated SQL expressions that could be used +** in a SELECT statement such as the following: +** +** SELECT FROM %_content AS x ... +** +** to return the docid, followed by each column of text data in order +** from left to write. If parameter zFunc is not NULL, then instead of +** being returned directly each column of text data is passed to an SQL +** function named zFunc first. For example, if zFunc is "unzip" and the +** table has the three user-defined columns "a", "b", and "c", the following +** string is returned: +** +** "docid, unzip(x.'a'), unzip(x.'b'), unzip(x.'c')" +** +** The pointer returned points to a buffer allocated by sqlite3_malloc(). It +** is the responsibility of the caller to eventually free it. +** +** If *pRc is not SQLITE_OK when this function is called, it is a no-op (and +** a NULL pointer is returned). Otherwise, if an OOM error is encountered +** by this function, NULL is returned and *pRc is set to SQLITE_NOMEM. If +** no error occurs, *pRc is left unmodified. +*/ +static char *fts3ReadExprList(Fts3Table *p, const char *zFunc, int *pRc){ + char *zRet = 0; + int i; + if( !zFunc ) zFunc = ""; + fts3Appendf(pRc, &zRet, "docid"); + for(i=0; inColumn; i++){ + fts3Appendf(pRc, &zRet, ",%s(x.'c%d%q')", zFunc, i, p->azColumn[i]); + } + return zRet; +} + +/* +** Return a list of N comma separated question marks, where N is the number +** of columns in the %_content table (one for the docid plus one for each +** user-defined text column). +** +** If argument zFunc is not NULL, then all but the first question mark +** is preceded by zFunc and an open bracket, and followed by a closed +** bracket. For example, if zFunc is "zip" and the FTS3 table has three +** user-defined text columns, the following string is returned: +** +** "?, zip(?), zip(?), zip(?)" +** +** The pointer returned points to a buffer allocated by sqlite3_malloc(). It +** is the responsibility of the caller to eventually free it. +** +** If *pRc is not SQLITE_OK when this function is called, it is a no-op (and +** a NULL pointer is returned). Otherwise, if an OOM error is encountered +** by this function, NULL is returned and *pRc is set to SQLITE_NOMEM. If +** no error occurs, *pRc is left unmodified. +*/ +static char *fts3WriteExprList(Fts3Table *p, const char *zFunc, int *pRc){ + char *zRet = 0; + int i; + if( !zFunc ) zFunc = ""; + fts3Appendf(pRc, &zRet, "?"); + for(i=0; inColumn; i++){ + fts3Appendf(pRc, &zRet, ",%s(?)", zFunc); + } + return zRet; +} /* ** This function is the implementation of both the xConnect and xCreate ** methods of the FTS3 virtual table. ** @@ -699,10 +791,13 @@ int isFts4 = (argv[0][3]=='4'); /* True for FTS4, false for FTS3 */ int bNoDocsize = 0; /* True to omit %_docsize table */ const char **aCol; /* Array of column names */ sqlite3_tokenizer *pTokenizer = 0; /* Tokenizer for this table */ + char *zCompress = 0; + char *zUncompress = 0; + assert( strlen(argv[0])==4 ); assert( (sqlite3_strnicmp(argv[0], "fts4", 4)==0 && isFts4) || (sqlite3_strnicmp(argv[0], "fts3", 4)==0 && !isFts4) ); @@ -749,10 +844,16 @@ bNoDocsize = 1; }else{ *pzErr = sqlite3_mprintf("unrecognized matchinfo: %s", zVal); rc = SQLITE_ERROR; } + }else if( nKey==8 && 0==sqlite3_strnicmp(z, "compress", 8) ){ + zCompress = zVal; + zVal = 0; + }else if( nKey==10 && 0==sqlite3_strnicmp(z, "uncompress", 10) ){ + zUncompress = zVal; + zVal = 0; }else{ *pzErr = sqlite3_mprintf("unrecognized parameter: %s", z); rc = SQLITE_ERROR; } sqlite3_free(zVal); @@ -822,10 +923,19 @@ sqlite3Fts3Dequote(zCsr); p->azColumn[iCol] = zCsr; zCsr += n+1; assert( zCsr <= &((char *)p)[nByte] ); } + + if( (zCompress==0)!=(zUncompress==0) ){ + char const *zMissing = (zCompress==0 ? "compress" : "uncompress"); + rc = SQLITE_ERROR; + *pzErr = sqlite3_mprintf("missing %s parameter", zMissing); + } + p->zReadExprlist = fts3ReadExprList(p, zUncompress, &rc); + p->zWriteExprlist = fts3WriteExprList(p, zCompress, &rc); + if( rc!=SQLITE_OK ) goto fts3_init_out; /* If this is an xCreate call, create the underlying tables in the ** database. TODO: For xConnect(), it could verify that said tables exist. */ if( isCreate ){ @@ -840,11 +950,12 @@ /* Declare the table schema to SQLite. */ fts3DeclareVtab(&rc, p); fts3_init_out: - + sqlite3_free(zCompress); + sqlite3_free(zUncompress); sqlite3_free((void *)aCol); if( rc!=SQLITE_OK ){ if( p ){ fts3DisconnectMethod((sqlite3_vtab *)p); }else if( pTokenizer ){ @@ -1933,135 +2044,134 @@ } return SQLITE_OK; } -/* -** An Fts3SegReaderArray is used to store an array of Fts3SegReader objects. -** Elements are added to the array using fts3SegReaderArrayAdd(). -*/ -struct Fts3SegReaderArray { - int nSegment; /* Number of valid entries in apSegment[] */ - int nAlloc; /* Allocated size of apSegment[] */ - int nCost; /* The cost of executing SegReaderIterate() */ - Fts3SegReader *apSegment[1]; /* Array of seg-reader objects */ -}; - - -/* -** Free an Fts3SegReaderArray object. Also free all seg-readers in the -** array (using sqlite3Fts3SegReaderFree()). -*/ -static void fts3SegReaderArrayFree(Fts3SegReaderArray *pArray){ - if( pArray ){ - int i; - for(i=0; inSegment; i++){ - sqlite3Fts3SegReaderFree(pArray->apSegment[i]); - } - sqlite3_free(pArray); - } -} - -static int fts3SegReaderArrayAdd( - Fts3SegReaderArray **ppArray, - Fts3SegReader *pNew -){ - Fts3SegReaderArray *pArray = *ppArray; - - if( !pArray || pArray->nAlloc==pArray->nSegment ){ - int nNew = (pArray ? pArray->nAlloc+16 : 16); - pArray = (Fts3SegReaderArray *)sqlite3_realloc(pArray, - sizeof(Fts3SegReaderArray) + (nNew-1) * sizeof(Fts3SegReader*) - ); - if( !pArray ){ - sqlite3Fts3SegReaderFree(pNew); - return SQLITE_NOMEM; - } - if( nNew==16 ){ - pArray->nSegment = 0; - pArray->nCost = 0; - } - pArray->nAlloc = nNew; - *ppArray = pArray; - } - - pArray->apSegment[pArray->nSegment++] = pNew; - return SQLITE_OK; -} - -static int fts3TermSegReaderArray( +int sqlite3Fts3SegReaderCursor( + Fts3Table *p, /* FTS3 table handle */ + int iLevel, /* Level of segments to scan */ + const char *zTerm, /* Term to query for */ + int nTerm, /* Size of zTerm in bytes */ + int isPrefix, /* True for a prefix search */ + Fts3SegReaderCursor *pCsr /* Cursor object to populate */ +){ + int rc = SQLITE_OK; + int rc2; + int iAge = 0; + sqlite3_stmt *pStmt = 0; + Fts3SegReader *pPending = 0; + + assert( iLevel==FTS3_SEGCURSOR_ALL + || iLevel==FTS3_SEGCURSOR_PENDING + || iLevel>=0 + ); + assert( FTS3_SEGCURSOR_PENDING<0 ); + assert( FTS3_SEGCURSOR_ALL<0 ); + assert( iLevel==FTS3_SEGCURSOR_ALL || (zTerm==0 && isPrefix==1) ); + + memset(pCsr, 0, sizeof(Fts3SegReaderCursor)); + + /* If iLevel is less than 0, include a seg-reader for the pending-terms. */ + if( iLevel<0 ){ + rc = sqlite3Fts3SegReaderPending(p, zTerm, nTerm, isPrefix, &pPending); + if( rc==SQLITE_OK && pPending ){ + int nByte = (sizeof(Fts3SegReader *) * 16); + pCsr->apSegment = (Fts3SegReader **)sqlite3_malloc(nByte); + if( pCsr->apSegment==0 ){ + rc = SQLITE_NOMEM; + }else{ + pCsr->apSegment[0] = pPending; + pCsr->nSegment = 1; + pPending = 0; + } + } + } + + if( iLevel!=FTS3_SEGCURSOR_PENDING ){ + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3AllSegdirs(p, iLevel, &pStmt); + } + while( rc==SQLITE_OK && SQLITE_ROW==(rc = sqlite3_step(pStmt)) ){ + + /* Read the values returned by the SELECT into local variables. */ + sqlite3_int64 iStartBlock = sqlite3_column_int64(pStmt, 1); + sqlite3_int64 iLeavesEndBlock = sqlite3_column_int64(pStmt, 2); + sqlite3_int64 iEndBlock = sqlite3_column_int64(pStmt, 3); + int nRoot = sqlite3_column_bytes(pStmt, 4); + char const *zRoot = sqlite3_column_blob(pStmt, 4); + + /* If nSegment is a multiple of 16 the array needs to be extended. */ + if( (pCsr->nSegment%16)==0 ){ + Fts3SegReader **apNew; + int nByte = (pCsr->nSegment + 16)*sizeof(Fts3SegReader*); + apNew = (Fts3SegReader **)sqlite3_realloc(pCsr->apSegment, nByte); + if( !apNew ){ + rc = SQLITE_NOMEM; + goto finished; + } + pCsr->apSegment = apNew; + } + + /* If zTerm is not NULL, and this segment is not stored entirely on its + ** root node, the range of leaves scanned can be reduced. Do this. */ + if( iStartBlock && zTerm ){ + sqlite3_int64 *pi = (isPrefix ? &iLeavesEndBlock : 0); + rc = fts3SelectLeaf(p, zTerm, nTerm, zRoot, nRoot, &iStartBlock, pi); + if( rc!=SQLITE_OK ) goto finished; + if( isPrefix==0 ) iLeavesEndBlock = iStartBlock; + } + + rc = sqlite3Fts3SegReaderNew(iAge, iStartBlock, iLeavesEndBlock, + iEndBlock, zRoot, nRoot, &pCsr->apSegment[pCsr->nSegment] + ); + if( rc!=SQLITE_OK ) goto finished; + pCsr->nSegment++; + iAge++; + } + } + + finished: + rc2 = sqlite3_reset(pStmt); + if( rc==SQLITE_DONE ) rc = rc2; + sqlite3Fts3SegReaderFree(pPending); + + return rc; +} + + +static int fts3TermSegReaderCursor( Fts3Cursor *pCsr, /* Virtual table cursor handle */ const char *zTerm, /* Term to query for */ int nTerm, /* Size of zTerm in bytes */ int isPrefix, /* True for a prefix search */ - Fts3SegReaderArray **ppArray /* OUT: Allocated seg-reader array */ -){ - Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; - int rc; /* Return code */ - Fts3SegReaderArray *pArray = 0; /* Array object to build */ - Fts3SegReader *pReader = 0; /* Seg-reader to add to pArray */ - sqlite3_stmt *pStmt = 0; /* SQL statement to scan %_segdir table */ - int iAge = 0; /* Used to assign ages to segments */ - - /* Allocate a seg-reader to scan the pending terms, if any. */ - rc = sqlite3Fts3SegReaderPending(p, zTerm, nTerm, isPrefix, &pReader); - if( rc==SQLITE_OK && pReader ) { - rc = fts3SegReaderArrayAdd(&pArray, pReader); - } - - /* Loop through the entire %_segdir table. For each segment, create a - ** Fts3SegReader to iterate through the subset of the segment leaves - ** that may contain a term that matches zTerm/nTerm. For non-prefix - ** searches, this is always a single leaf. For prefix searches, this - ** may be a contiguous block of leaves. - */ - if( rc==SQLITE_OK ){ - rc = sqlite3Fts3AllSegdirs(p, &pStmt); - } - while( rc==SQLITE_OK && SQLITE_ROW==(rc = sqlite3_step(pStmt)) ){ - Fts3SegReader *pNew = 0; - int nRoot = sqlite3_column_bytes(pStmt, 4); - char const *zRoot = sqlite3_column_blob(pStmt, 4); - if( sqlite3_column_int64(pStmt, 1)==0 ){ - /* The entire segment is stored on the root node (which must be a - ** leaf). Do not bother inspecting any data in this case, just - ** create a Fts3SegReader to scan the single leaf. - */ - rc = sqlite3Fts3SegReaderNew(iAge, 0, 0, 0, zRoot, nRoot, &pNew); - }else{ - sqlite3_int64 i1; /* First leaf that may contain zTerm */ - sqlite3_int64 i2; /* Final leaf that may contain zTerm */ - rc = fts3SelectLeaf(p, zTerm, nTerm, zRoot, nRoot, &i1, (isPrefix?&i2:0)); - if( isPrefix==0 ) i2 = i1; - if( rc==SQLITE_OK ){ - rc = sqlite3Fts3SegReaderNew(iAge, i1, i2, 0, 0, 0, &pNew); - } - } - assert( (pNew==0)==(rc!=SQLITE_OK) ); - - /* If a new Fts3SegReader was allocated, add it to the array. */ - if( rc==SQLITE_OK ){ - rc = fts3SegReaderArrayAdd(&pArray, pNew); - } - if( rc==SQLITE_OK ){ - rc = sqlite3Fts3SegReaderCost(pCsr, pNew, &pArray->nCost); - } - iAge++; - } - - if( rc==SQLITE_DONE ){ - rc = sqlite3_reset(pStmt); - }else{ - sqlite3_reset(pStmt); - } - if( rc!=SQLITE_OK ){ - fts3SegReaderArrayFree(pArray); - pArray = 0; - } - *ppArray = pArray; + Fts3SegReaderCursor **ppSegcsr /* OUT: Allocated seg-reader cursor */ +){ + Fts3SegReaderCursor *pSegcsr; /* Object to allocate and return */ + int rc = SQLITE_NOMEM; /* Return code */ + + pSegcsr = sqlite3_malloc(sizeof(Fts3SegReaderCursor)); + if( pSegcsr ){ + Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; + int i; + int nCost = 0; + rc = sqlite3Fts3SegReaderCursor( + p, FTS3_SEGCURSOR_ALL, zTerm, nTerm, isPrefix, pSegcsr); + + for(i=0; rc==SQLITE_OK && inSegment; i++){ + rc = sqlite3Fts3SegReaderCost(pCsr, pSegcsr->apSegment[i], &nCost); + } + pSegcsr->nCost = nCost; + } + + *ppSegcsr = pSegcsr; return rc; } + +static void fts3SegReaderCursorFree(Fts3SegReaderCursor *pSegcsr){ + sqlite3Fts3SegReaderFinish(pSegcsr); + sqlite3_free(pSegcsr); +} /* ** This function retreives the doclist for the specified term (or term ** prefix) from the database. ** @@ -2079,15 +2189,15 @@ int isReqPos, /* True to include position lists in output */ int *pnOut, /* OUT: Size of buffer at *ppOut */ char **ppOut /* OUT: Malloced result buffer */ ){ int rc; /* Return code */ - Fts3SegReaderArray *pArray; /* Seg-reader array for this term */ - TermSelect tsc; /* Context object for fts3TermSelectCb() */ - Fts3SegFilter filter; /* Segment term filter configuration */ + Fts3SegReaderCursor *pSegcsr; /* Seg-reader cursor for this term */ + TermSelect tsc; /* Context object for fts3TermSelectCb() */ + Fts3SegFilter filter; /* Segment term filter configuration */ - pArray = pTok->pArray; + pSegcsr = pTok->pSegcsr; memset(&tsc, 0, sizeof(TermSelect)); tsc.isReqPos = isReqPos; filter.flags = FTS3_SEGMENT_IGNORE_EMPTY | (pTok->isPrefix ? FTS3_SEGMENT_PREFIX : 0) @@ -2095,17 +2205,22 @@ | (iColumnnColumn ? FTS3_SEGMENT_COLUMN_FILTER : 0); filter.iCol = iColumn; filter.zTerm = pTok->z; filter.nTerm = pTok->n; - rc = sqlite3Fts3SegReaderIterate(p, pArray->apSegment, pArray->nSegment, - &filter, fts3TermSelectCb, (void *)&tsc - ); + rc = sqlite3Fts3SegReaderStart(p, pSegcsr, &filter); + while( SQLITE_OK==rc + && SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, pSegcsr)) + ){ + rc = fts3TermSelectCb(p, (void *)&tsc, + pSegcsr->zTerm, pSegcsr->nTerm, pSegcsr->aDoclist, pSegcsr->nDoclist + ); + } + if( rc==SQLITE_OK ){ rc = fts3TermSelectMerge(&tsc); } - if( rc==SQLITE_OK ){ *ppOut = tsc.aaOutput[0]; *pnOut = tsc.anOutput[0]; }else{ int i; @@ -2112,12 +2227,12 @@ for(i=0; ipArray = 0; + fts3SegReaderCursorFree(pSegcsr); + pTok->pSegcsr = 0; return rc; } /* ** This function counts the total number of docids in the doclist stored @@ -2236,17 +2351,17 @@ ** evaluation, only create segment-readers if there are no Fts3DeferredToken ** objects attached to the phrase-tokens. */ for(ii=0; iinToken; ii++){ Fts3PhraseToken *pTok = &pPhrase->aToken[ii]; - if( pTok->pArray==0 ){ + if( pTok->pSegcsr==0 ){ if( (pCsr->eEvalmode==FTS3_EVAL_FILTER) || (pCsr->eEvalmode==FTS3_EVAL_NEXT && pCsr->pDeferred==0) || (pCsr->eEvalmode==FTS3_EVAL_MATCHINFO && pTok->bFulltext) ){ - rc = fts3TermSegReaderArray( - pCsr, pTok->z, pTok->n, pTok->isPrefix, &pTok->pArray + rc = fts3TermSegReaderCursor( + pCsr, pTok->z, pTok->n, pTok->isPrefix, &pTok->pSegcsr ); if( rc!=SQLITE_OK ) return rc; } } } @@ -2273,14 +2388,14 @@ int nMinCost = 0x7FFFFFFF; int jj; /* Find the remaining token with the lowest cost. */ for(jj=0; jjnToken; jj++){ - Fts3SegReaderArray *pArray = pPhrase->aToken[jj].pArray; - if( pArray && pArray->nCostaToken[jj].pSegcsr; + if( pSegcsr && pSegcsr->nCostnCost; + nMinCost = pSegcsr->nCost; } } pTok = &pPhrase->aToken[iTok]; /* This branch is taken if it is determined that loading the doclist @@ -2295,16 +2410,16 @@ } if( pCsr->eEvalmode==FTS3_EVAL_NEXT && pTok->pDeferred ){ rc = fts3DeferredTermSelect(pTok->pDeferred, isTermPos, &nList, &pList); }else{ - if( pTok->pArray ){ + if( pTok->pSegcsr ){ rc = fts3TermSelect(p, pTok, iCol, isTermPos, &nList, &pList); } pTok->bFulltext = 1; } - assert( rc!=SQLITE_OK || pCsr->eEvalmode || pTok->pArray==0 ); + assert( rc!=SQLITE_OK || pCsr->eEvalmode || pTok->pSegcsr==0 ); if( rc!=SQLITE_OK ) break; if( isFirst ){ pOut = pList; nOut = nList; @@ -2478,13 +2593,13 @@ Fts3Phrase *pPhrase = pExpr->pPhrase; int ii; for(ii=0; rc==SQLITE_OK && iinToken; ii++){ Fts3PhraseToken *pTok = &pPhrase->aToken[ii]; - if( pTok->pArray==0 ){ - rc = fts3TermSegReaderArray( - pCsr, pTok->z, pTok->n, pTok->isPrefix, &pTok->pArray + if( pTok->pSegcsr==0 ){ + rc = fts3TermSegReaderCursor( + pCsr, pTok->z, pTok->n, pTok->isPrefix, &pTok->pSegcsr ); } } }else{ rc = fts3ExprAllocateSegReaders(pCsr, pExpr->pLeft, pnExpr); @@ -2504,12 +2619,12 @@ if( pExpr ){ Fts3Phrase *pPhrase = pExpr->pPhrase; if( pPhrase ){ int kk; for(kk=0; kknToken; kk++){ - fts3SegReaderArrayFree(pPhrase->aToken[kk].pArray); - pPhrase->aToken[kk].pArray = 0; + fts3SegReaderCursorFree(pPhrase->aToken[kk].pSegcsr); + pPhrase->aToken[kk].pSegcsr = 0; } } fts3ExprFreeSegReaders(pExpr->pLeft); fts3ExprFreeSegReaders(pExpr->pRight); } @@ -2525,14 +2640,12 @@ if( pExpr->eType==FTSQUERY_PHRASE ){ Fts3Phrase *pPhrase = pExpr->pPhrase; int ii; nCost = 0; for(ii=0; iinToken; ii++){ - Fts3SegReaderArray *pArray = pPhrase->aToken[ii].pArray; - if( pArray ){ - nCost += pPhrase->aToken[ii].pArray->nCost; - } + Fts3SegReaderCursor *pSegcsr = pPhrase->aToken[ii].pSegcsr; + if( pSegcsr ) nCost += pSegcsr->nCost; } }else{ nCost = fts3ExprCost(pExpr->pLeft) + fts3ExprCost(pExpr->pRight); } return nCost; @@ -2870,12 +2983,12 @@ const char *idxStr, /* Unused */ int nVal, /* Number of elements in apVal */ sqlite3_value **apVal /* Arguments for the indexing scheme */ ){ const char *azSql[] = { - "SELECT * FROM %Q.'%q_content' WHERE docid = ?", /* non-full-table-scan */ - "SELECT * FROM %Q.'%q_content'", /* full-table-scan */ + "SELECT %s FROM %Q.'%q_content' AS x WHERE docid = ?", /* non-full-scan */ + "SELECT %s FROM %Q.'%q_content' AS x ", /* full-scan */ }; int rc; /* Return code */ char *zSql; /* SQL statement used to access %_content */ Fts3Table *p = (Fts3Table *)pCursor->pVtab; Fts3Cursor *pCsr = (Fts3Cursor *)pCursor; @@ -2926,11 +3039,12 @@ /* Compile a SELECT statement for this cursor. For a full-table-scan, the ** statement loops through all rows of the %_content table. For a ** full-text query or docid lookup, the statement retrieves a single ** row by docid. */ - zSql = sqlite3_mprintf(azSql[idxNum==FTS3_FULLSCAN_SEARCH], p->zDb, p->zName); + zSql = (char *)azSql[idxNum==FTS3_FULLSCAN_SEARCH]; + zSql = sqlite3_mprintf(zSql, p->zReadExprlist, p->zDb, p->zName); if( !zSql ){ rc = SQLITE_NOMEM; }else{ rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0); sqlite3_free(zSql); @@ -3444,10 +3558,13 @@ #ifdef SQLITE_ENABLE_ICU const sqlite3_tokenizer_module *pIcu = 0; sqlite3Fts3IcuTokenizerModule(&pIcu); #endif + rc = sqlite3Fts3InitAux(db); + if( rc!=SQLITE_OK ) return rc; + sqlite3Fts3SimpleTokenizerModule(&pSimple); sqlite3Fts3PorterTokenizerModule(&pPorter); /* Allocate and initialise the hash-table used to store tokenizers. */ pHash = sqlite3_malloc(sizeof(Fts3Hash)); Index: ext/fts3/fts3Int.h ================================================================== --- ext/fts3/fts3Int.h +++ ext/fts3/fts3Int.h @@ -105,11 +105,11 @@ typedef struct Fts3PhraseToken Fts3PhraseToken; typedef struct Fts3SegFilter Fts3SegFilter; typedef struct Fts3DeferredToken Fts3DeferredToken; typedef struct Fts3SegReader Fts3SegReader; -typedef struct Fts3SegReaderArray Fts3SegReaderArray; +typedef struct Fts3SegReaderCursor Fts3SegReaderCursor; /* ** A connection to a fulltext index is an instance of the following ** structure. The xCreate and xConnect methods create an instance ** of this structure and xDestroy and xDisconnect free that instance. @@ -128,10 +128,13 @@ /* Precompiled statements used by the implementation. Each of these ** statements is run and reset within a single virtual table API call. */ sqlite3_stmt *aStmt[24]; + char *zReadExprlist; + char *zWriteExprlist; + int nNodeSize; /* Soft limit for node size */ u8 bHasStat; /* True if %_stat table exists */ u8 bHasDocsize; /* True if %_docsize table exists */ int nPgsz; /* Page size for host database */ char *zSegmentsTbl; /* Name of %_segments table */ @@ -215,11 +218,11 @@ struct Fts3PhraseToken { char *z; /* Text of the token */ int n; /* Number of bytes in buffer z */ int isPrefix; /* True if token ends with a "*" character */ int bFulltext; /* True if full-text index was used */ - Fts3SegReaderArray *pArray; /* Segment-reader for this token */ + Fts3SegReaderCursor *pSegcsr; /* Segment-reader for this token */ Fts3DeferredToken *pDeferred; /* Deferred token object for this token */ }; struct Fts3Phrase { /* Variables populated by fts3_expr.c when parsing a MATCH expression */ @@ -283,16 +286,12 @@ int sqlite3Fts3Optimize(Fts3Table *); int sqlite3Fts3SegReaderNew(int, sqlite3_int64, sqlite3_int64, sqlite3_int64, const char *, int, Fts3SegReader**); int sqlite3Fts3SegReaderPending(Fts3Table*,const char*,int,int,Fts3SegReader**); void sqlite3Fts3SegReaderFree(Fts3SegReader *); -int sqlite3Fts3SegReaderIterate( - Fts3Table *, Fts3SegReader **, int, Fts3SegFilter *, - int (*)(Fts3Table *, void *, char *, int, char *, int), void * -); int sqlite3Fts3SegReaderCost(Fts3Cursor *, Fts3SegReader *, int *); -int sqlite3Fts3AllSegdirs(Fts3Table*, sqlite3_stmt **); +int sqlite3Fts3AllSegdirs(Fts3Table*, int, sqlite3_stmt **); int sqlite3Fts3ReadLock(Fts3Table *); int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char **, int*); int sqlite3Fts3SelectDoctotal(Fts3Table *, sqlite3_stmt **); int sqlite3Fts3SelectDocsize(Fts3Table *, sqlite3_int64, sqlite3_stmt **); @@ -300,13 +299,21 @@ void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *); int sqlite3Fts3DeferToken(Fts3Cursor *, Fts3PhraseToken *, int); int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *); void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *); char *sqlite3Fts3DeferredDoclist(Fts3DeferredToken *, int *); - void sqlite3Fts3SegmentsClose(Fts3Table *); +#define FTS3_SEGCURSOR_PENDING -1 +#define FTS3_SEGCURSOR_ALL -2 + +int sqlite3Fts3SegReaderStart(Fts3Table*, Fts3SegReaderCursor*, Fts3SegFilter*); +int sqlite3Fts3SegReaderStep(Fts3Table *, Fts3SegReaderCursor *); +void sqlite3Fts3SegReaderFinish(Fts3SegReaderCursor *); +int sqlite3Fts3SegReaderCursor( + Fts3Table *, int, const char *, int, int, Fts3SegReaderCursor *); + /* Flags allowed as part of the 4th argument to SegmentReaderIterate() */ #define FTS3_SEGMENT_REQUIRE_POS 0x00000001 #define FTS3_SEGMENT_IGNORE_EMPTY 0x00000002 #define FTS3_SEGMENT_COLUMN_FILTER 0x00000004 #define FTS3_SEGMENT_PREFIX 0x00000008 @@ -316,10 +323,29 @@ const char *zTerm; int nTerm; int iCol; int flags; }; + +struct Fts3SegReaderCursor { + /* Used internally by sqlite3Fts3SegReaderXXX() calls */ + Fts3SegReader **apSegment; /* Array of Fts3SegReader objects */ + int nSegment; /* Size of apSegment array */ + int nAdvance; /* How many seg-readers to advance */ + Fts3SegFilter *pFilter; /* Pointer to filter object */ + char *aBuffer; /* Buffer to merge doclists in */ + int nBuffer; /* Allocated size of aBuffer[] in bytes */ + + /* Cost of running this iterator. Used by fts3.c only. */ + int nCost; + + /* Output values. Valid only after Fts3SegReaderStep() returns SQLITE_ROW. */ + char *zTerm; /* Pointer to term buffer */ + int nTerm; /* Size of zTerm in bytes */ + char *aDoclist; /* Pointer to doclist buffer */ + int nDoclist; /* Size of aDoclist[] in bytes */ +}; /* fts3.c */ int sqlite3Fts3PutVarint(char *, sqlite3_int64); int sqlite3Fts3GetVarint(const char *, sqlite_int64 *); int sqlite3Fts3GetVarint32(const char *, int *); @@ -353,6 +379,9 @@ void sqlite3Fts3ExprFree(Fts3Expr *); #ifdef SQLITE_TEST int sqlite3Fts3ExprInitTestInterface(sqlite3 *db); #endif +/* fts3_aux.c */ +int sqlite3Fts3InitAux(sqlite3 *db); + #endif /* _FTSINT_H */ ADDED ext/fts3/fts3_aux.c Index: ext/fts3/fts3_aux.c ================================================================== --- /dev/null +++ ext/fts3/fts3_aux.c @@ -0,0 +1,310 @@ +/* +** 2011 Jan 27 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +*/ + +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + +#include "fts3Int.h" +#include +#include + +typedef struct Fts3auxTable Fts3auxTable; +typedef struct Fts3auxCursor Fts3auxCursor; + +struct Fts3auxTable { + sqlite3_vtab base; /* Base class used by SQLite core */ + Fts3Table *pFts3Tab; +}; + +struct Fts3auxCursor { + sqlite3_vtab_cursor base; /* Base class used by SQLite core */ + + Fts3SegFilter filter; + Fts3SegReaderCursor csr; + int isEof; + + sqlite3_int64 iRowid; + sqlite3_int64 nDoc; + sqlite3_int64 nOcc; +}; + +/* +** Schema of the terms table. +*/ +#define FTS3_TERMS_SCHEMA "CREATE TABLE x(term, documents, occurrences)" + +/* +** This function does all the work for both the xConnect and xCreate methods. +** These tables have no persistent representation of their own, so xConnect +** and xCreate are identical operations. +*/ +static int fts3auxConnectMethod( + sqlite3 *db, /* Database connection */ + void *pUnused, /* Unused */ + int argc, /* Number of elements in argv array */ + const char * const *argv, /* xCreate/xConnect argument array */ + sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */ + char **pzErr /* OUT: sqlite3_malloc'd error message */ +){ + char const *zDb; /* Name of database (e.g. "main") */ + char const *zFts3; /* Name of fts3 table */ + int nDb; /* Result of strlen(zDb) */ + int nFts3; /* Result of strlen(zFts3) */ + int nByte; /* Bytes of space to allocate here */ + int rc; /* value returned by declare_vtab() */ + Fts3auxTable *p; /* Virtual table object to return */ + + /* The user should specify a single argument - the name of an fts3 table. */ + if( argc!=4 ){ + *pzErr = sqlite3_mprintf("wrong number of arguments"); + return SQLITE_ERROR; + } + + zDb = argv[1]; + nDb = strlen(zDb); + zFts3 = argv[3]; + nFts3 = strlen(zFts3); + + rc = sqlite3_declare_vtab(db, FTS3_TERMS_SCHEMA); + if( rc!=SQLITE_OK ) return rc; + + nByte = sizeof(Fts3auxTable) + sizeof(Fts3Table) + nDb + nFts3 + 2; + p = (Fts3auxTable *)sqlite3_malloc(nByte); + if( !p ) return SQLITE_NOMEM; + memset(p, 0, nByte); + + p->pFts3Tab = (Fts3Table *)&p[1]; + p->pFts3Tab->zDb = (char *)&p->pFts3Tab[1]; + p->pFts3Tab->zName = &p->pFts3Tab->zDb[nDb+1]; + p->pFts3Tab->db = db; + + memcpy((char *)p->pFts3Tab->zDb, zDb, nDb); + memcpy((char *)p->pFts3Tab->zName, zFts3, nFts3); + + *ppVtab = (sqlite3_vtab *)p; + return SQLITE_OK; +} + +/* +** This function does the work for both the xDisconnect and xDestroy methods. +** These tables have no persistent representation of their own, so xDisconnect +** and xDestroy are identical operations. +*/ +static int fts3auxDisconnectMethod(sqlite3_vtab *pVtab){ + Fts3auxTable *p = (Fts3auxTable *)pVtab; + Fts3Table *pFts3 = p->pFts3Tab; + int i; + + /* Free any prepared statements held */ + for(i=0; iaStmt); i++){ + sqlite3_finalize(pFts3->aStmt[i]); + } + sqlite3_free(pFts3->zSegmentsTbl); + sqlite3_free(p); + return SQLITE_OK; +} + +/* +** xBestIndex - Analyze a WHERE and ORDER BY clause. +*/ +static int fts3auxBestIndexMethod( + sqlite3_vtab *pVTab, + sqlite3_index_info *pInfo +){ + + /* This vtab delivers always results in "ORDER BY term ASC" order. */ + if( pInfo->nOrderBy==1 + && pInfo->aOrderBy[0].iColumn==0 + && pInfo->aOrderBy[0].desc==0 + ){ + pInfo->orderByConsumed = 1; + } + + pInfo->estimatedCost = 20000; + return SQLITE_OK; +} + +/* +** xOpen - Open a cursor. +*/ +static int fts3auxOpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){ + Fts3auxCursor *pCsr; /* Pointer to cursor object to return */ + + pCsr = (Fts3auxCursor *)sqlite3_malloc(sizeof(Fts3auxCursor)); + if( !pCsr ) return SQLITE_NOMEM; + memset(pCsr, 0, sizeof(Fts3auxCursor)); + + *ppCsr = (sqlite3_vtab_cursor *)pCsr; + return SQLITE_OK; +} + +/* +** xClose - Close a cursor. +*/ +static int fts3auxCloseMethod(sqlite3_vtab_cursor *pCursor){ + Fts3Table *pFts3 = ((Fts3auxTable *)pCursor->pVtab)->pFts3Tab; + Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor; + + sqlite3Fts3SegmentsClose(pFts3); + sqlite3Fts3SegReaderFinish(&pCsr->csr); + sqlite3_free(pCsr); + return SQLITE_OK; +} + +/* +** xNext - Advance the cursor to the next row, if any. +*/ +static int fts3auxNextMethod(sqlite3_vtab_cursor *pCursor){ + Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor; + Fts3Table *pFts3 = ((Fts3auxTable *)pCursor->pVtab)->pFts3Tab; + int rc; + + rc = sqlite3Fts3SegReaderStep(pFts3, &pCsr->csr); + if( rc==SQLITE_ROW ){ + int i; + int isIgnore = 1; + int nDoclist = pCsr->csr.nDoclist; + char *aDoclist = pCsr->csr.aDoclist; + + /* Now count the number of documents and positions in the doclist + ** in pCsr->csr.aDoclist[]. Store the number of documents in pCsr->nDoc + ** and the number of occurrences in pCsr->nOcc. */ + pCsr->nDoc = 0; + pCsr->nOcc = 0; + i = 0; + while( i1 ){ + pCsr->nOcc++; + }else{ + if( v==0 ) pCsr->nDoc++; + isIgnore = 1; + } + } + + rc = SQLITE_OK; + pCsr->iRowid++; + }else{ + pCsr->isEof = 1; + } + return rc; +} + +/* +** xFilter - Initialize a cursor to point at the start of its data. +*/ +static int fts3auxFilterMethod( + sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */ + int idxNum, /* Strategy index */ + const char *idxStr, /* Unused */ + int nVal, /* Number of elements in apVal */ + sqlite3_value **apVal /* Arguments for the indexing scheme */ +){ + Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor; + Fts3Table *pFts3 = ((Fts3auxTable *)pCursor->pVtab)->pFts3Tab; + int rc; + + sqlite3Fts3SegReaderFinish(&pCsr->csr); + memset(&pCsr->csr, 0, sizeof(Fts3SegReaderCursor)); + pCsr->isEof = 0; + pCsr->iRowid = 0; + pCsr->filter.flags = FTS3_SEGMENT_REQUIRE_POS|FTS3_SEGMENT_IGNORE_EMPTY; + + rc = sqlite3Fts3SegReaderCursor(pFts3, FTS3_SEGCURSOR_ALL, 0, 0,1,&pCsr->csr); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3SegReaderStart(pFts3, &pCsr->csr, &pCsr->filter); + } + + if( rc==SQLITE_OK ) rc = fts3auxNextMethod(pCursor); + return rc; +} + +/* +** xEof - Return true if the cursor is at EOF, or false otherwise. +*/ +static int fts3auxEofMethod(sqlite3_vtab_cursor *pCursor){ + Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor; + return pCsr->isEof; +} + +/* +** xColumn - Return a column value. +*/ +static int fts3auxColumnMethod( + sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ + sqlite3_context *pContext, /* Context for sqlite3_result_xxx() calls */ + int iCol /* Index of column to read value from */ +){ + Fts3auxCursor *p = (Fts3auxCursor *)pCursor; + + assert( p->isEof==0 ); + if( iCol==0 ){ /* Column "term" */ + sqlite3_result_text(pContext, p->csr.zTerm, p->csr.nTerm, SQLITE_TRANSIENT); + }else if( iCol==1 ){ /* Column "documents" */ + sqlite3_result_int64(pContext, p->nDoc); + }else{ /* Column "occurrences" */ + sqlite3_result_int64(pContext, p->nOcc); + } + + return SQLITE_OK; +} + +/* +** xRowid - Return the current rowid for the cursor. +*/ +static int fts3auxRowidMethod( + sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ + sqlite_int64 *pRowid /* OUT: Rowid value */ +){ + Fts3auxCursor *pCsr = (Fts3auxCursor *)pCursor; + *pRowid = pCsr->iRowid; + return SQLITE_OK; +} + +/* +** Register the fts3aux module with database connection db. Return SQLITE_OK +** if successful or an error code if sqlite3_create_module() fails. +*/ +int sqlite3Fts3InitAux(sqlite3 *db){ + static const sqlite3_module fts3aux_module = { + 0, /* iVersion */ + fts3auxConnectMethod, /* xCreate */ + fts3auxConnectMethod, /* xConnect */ + fts3auxBestIndexMethod, /* xBestIndex */ + fts3auxDisconnectMethod, /* xDisconnect */ + fts3auxDisconnectMethod, /* xDestroy */ + fts3auxOpenMethod, /* xOpen */ + fts3auxCloseMethod, /* xClose */ + fts3auxFilterMethod, /* xFilter */ + fts3auxNextMethod, /* xNext */ + fts3auxEofMethod, /* xEof */ + fts3auxColumnMethod, /* xColumn */ + fts3auxRowidMethod, /* xRowid */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindFunction */ + 0 /* xRename */ + }; + int rc; /* Return code */ + + rc = sqlite3_create_module(db, "fts4aux", &fts3aux_module, 0); + return rc; +} + +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ Index: ext/fts3/fts3_snippet.c ================================================================== --- ext/fts3/fts3_snippet.c +++ ext/fts3/fts3_snippet.c @@ -958,10 +958,11 @@ pStmt = *ppStmt; assert( sqlite3_data_count(pStmt)==1 ); a = sqlite3_column_blob(pStmt, 0); a += sqlite3Fts3GetVarint(a, &nDoc); + if( nDoc==0 ) return SQLITE_CORRUPT; *pnDoc = (u32)nDoc; if( paLen ) *paLen = a; return SQLITE_OK; } @@ -1164,13 +1165,15 @@ rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &nDoc, &a); if( rc==SQLITE_OK ){ int iCol; for(iCol=0; iColnCol; iCol++){ + u32 iVal; sqlite3_int64 nToken; a += sqlite3Fts3GetVarint(a, &nToken); - pInfo->aMatchinfo[iCol] = (u32)(((u32)(nToken&0xffffffff)+nDoc/2)/nDoc); + iVal = (u32)(((u32)(nToken&0xffffffff)+nDoc/2)/nDoc); + pInfo->aMatchinfo[iCol] = iVal; } } } break; Index: ext/fts3/fts3_write.c ================================================================== --- ext/fts3/fts3_write.c +++ ext/fts3/fts3_write.c @@ -210,11 +210,11 @@ /* 2 */ "DELETE FROM %Q.'%q_content'", /* 3 */ "DELETE FROM %Q.'%q_segments'", /* 4 */ "DELETE FROM %Q.'%q_segdir'", /* 5 */ "DELETE FROM %Q.'%q_docsize'", /* 6 */ "DELETE FROM %Q.'%q_stat'", -/* 7 */ "SELECT * FROM %Q.'%q_content' WHERE rowid=?", +/* 7 */ "SELECT %s FROM %Q.'%q_content' AS x WHERE rowid=?", /* 8 */ "SELECT (SELECT max(idx) FROM %Q.'%q_segdir' WHERE level = ?) + 1", /* 9 */ "INSERT INTO %Q.'%q_segments'(blockid, block) VALUES(?, ?)", /* 10 */ "SELECT coalesce((SELECT max(blockid) FROM %Q.'%q_segments') + 1, 1)", /* 11 */ "INSERT INTO %Q.'%q_segdir' VALUES(?,?,?,?,?,?)", @@ -227,11 +227,11 @@ /* 14 */ "SELECT count(*) FROM %Q.'%q_segdir' WHERE level = ?", /* 15 */ "SELECT count(*), max(level) FROM %Q.'%q_segdir'", /* 16 */ "DELETE FROM %Q.'%q_segdir' WHERE level = ?", /* 17 */ "DELETE FROM %Q.'%q_segments' WHERE blockid BETWEEN ? AND ?", -/* 18 */ "INSERT INTO %Q.'%q_content' VALUES(%z)", +/* 18 */ "INSERT INTO %Q.'%q_content' VALUES(%s)", /* 19 */ "DELETE FROM %Q.'%q_docsize' WHERE docid = ?", /* 20 */ "REPLACE INTO %Q.'%q_docsize' VALUES(?,?)", /* 21 */ "SELECT size FROM %Q.'%q_docsize' WHERE docid=?", /* 22 */ "SELECT value FROM %Q.'%q_stat' WHERE id=0", /* 23 */ "REPLACE INTO %Q.'%q_stat' VALUES(0,?)", @@ -244,24 +244,13 @@ pStmt = p->aStmt[eStmt]; if( !pStmt ){ char *zSql; if( eStmt==SQL_CONTENT_INSERT ){ - int i; /* Iterator variable */ - char *zVarlist; /* The "?, ?, ..." string */ - zVarlist = (char *)sqlite3_malloc(2*p->nColumn+2); - if( !zVarlist ){ - *pp = 0; - return SQLITE_NOMEM; - } - zVarlist[0] = '?'; - zVarlist[p->nColumn*2+1] = '\0'; - for(i=1; i<=p->nColumn; i++){ - zVarlist[i*2-1] = ','; - zVarlist[i*2] = '?'; - } - zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName, zVarlist); + zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName, p->zWriteExprlist); + }else if( eStmt==SQL_SELECT_CONTENT_BY_ROWID ){ + zSql = sqlite3_mprintf(azSql[eStmt], p->zReadExprlist, p->zDb, p->zName); }else{ zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName); } if( !zSql ){ rc = SQLITE_NOMEM; @@ -298,11 +287,11 @@ if( rc==SQLITE_OK ){ if( eStmt==SQL_SELECT_DOCSIZE ){ sqlite3_bind_int64(pStmt, 1, iDocid); } rc = sqlite3_step(pStmt); - if( rc!=SQLITE_ROW ){ + if( rc!=SQLITE_ROW || sqlite3_column_type(pStmt, 0)!=SQLITE_BLOB ){ rc = sqlite3_reset(pStmt); if( rc==SQLITE_OK ) rc = SQLITE_CORRUPT; pStmt = 0; }else{ rc = SQLITE_OK; @@ -399,12 +388,21 @@ ** 1: start_block ** 2: leaves_end_block ** 3: end_block ** 4: root */ -int sqlite3Fts3AllSegdirs(Fts3Table *p, sqlite3_stmt **ppStmt){ - return fts3SqlStmt(p, SQL_SELECT_ALL_LEVEL, ppStmt, 0); +int sqlite3Fts3AllSegdirs(Fts3Table *p, int iLevel, sqlite3_stmt **ppStmt){ + int rc; + sqlite3_stmt *pStmt = 0; + if( iLevel<0 ){ + rc = fts3SqlStmt(p, SQL_SELECT_ALL_LEVEL, &pStmt, 0); + }else{ + rc = fts3SqlStmt(p, SQL_SELECT_LEVEL, &pStmt, 0); + if( rc==SQLITE_OK ) sqlite3_bind_int(pStmt, 1, iLevel); + } + *ppStmt = pStmt; + return rc; } /* ** Append a single varint to a PendingList buffer. SQLITE_OK is returned @@ -1102,20 +1100,22 @@ ** to right. */ sqlite3_stmt *pStmt; sqlite3_int64 nDoc = 0; sqlite3_int64 nByte = 0; + const char *pEnd; const char *a; + rc = sqlite3Fts3SelectDoctotal(p, &pStmt); - if( rc ) return rc; + if( rc!=SQLITE_OK ) return rc; a = sqlite3_column_blob(pStmt, 0); - if( a ){ - const char *pEnd = &a[sqlite3_column_bytes(pStmt, 0)]; - a += sqlite3Fts3GetVarint(a, &nDoc); - while( a=0 ); - rc = fts3SqlStmt(p, SQL_SELECT_LEVEL_COUNT, &pStmt, 0); - if( rc!=SQLITE_OK ) return rc; - sqlite3_bind_int(pStmt, 1, iLevel); - if( SQLITE_ROW==sqlite3_step(pStmt) ){ - *pnSegment = sqlite3_column_int(pStmt, 0); - } - return sqlite3_reset(pStmt); -} - /* ** Set *pnSegment to the total number of segments in the database. Set ** *pnMax to the largest segment level in the database (segment levels ** are stored in the 'level' column of the %_segdir table). ** @@ -2018,19 +1963,22 @@ } if( rc!=SQLITE_OK ){ return rc; } - if( iLevel>=0 ){ + if( iLevel==FTS3_SEGCURSOR_ALL ){ + fts3SqlExec(&rc, p, SQL_DELETE_ALL_SEGDIR, 0); + }else if( iLevel==FTS3_SEGCURSOR_PENDING ){ + sqlite3Fts3PendingTermsClear(p); + }else{ + assert( iLevel>=0 ); rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_BY_LEVEL, &pDelete, 0); if( rc==SQLITE_OK ){ sqlite3_bind_int(pDelete, 1, iLevel); sqlite3_step(pDelete); rc = sqlite3_reset(pDelete); } - }else{ - fts3SqlExec(&rc, p, SQL_DELETE_ALL_SEGDIR, 0); } return rc; } @@ -2075,148 +2023,116 @@ *ppList = pList; *pnList = nList; } -/* -** sqlite3Fts3SegReaderIterate() callback used when merging multiple -** segments to create a single, larger segment. -*/ -static int fts3MergeCallback( - Fts3Table *p, /* FTS3 Virtual table handle */ - void *pContext, /* Pointer to SegmentWriter* to write with */ - char *zTerm, /* Term to write to the db */ - int nTerm, /* Number of bytes in zTerm */ - char *aDoclist, /* Doclist associated with zTerm */ - int nDoclist /* Number of bytes in doclist */ -){ - SegmentWriter **ppW = (SegmentWriter **)pContext; - return fts3SegWriterAdd(p, ppW, 1, zTerm, nTerm, aDoclist, nDoclist); -} - -/* -** sqlite3Fts3SegReaderIterate() callback used when flushing the contents -** of the pending-terms hash table to the database. -*/ -static int fts3FlushCallback( - Fts3Table *p, /* FTS3 Virtual table handle */ - void *pContext, /* Pointer to SegmentWriter* to write with */ - char *zTerm, /* Term to write to the db */ - int nTerm, /* Number of bytes in zTerm */ - char *aDoclist, /* Doclist associated with zTerm */ - int nDoclist /* Number of bytes in doclist */ -){ - SegmentWriter **ppW = (SegmentWriter **)pContext; - return fts3SegWriterAdd(p, ppW, 0, zTerm, nTerm, aDoclist, nDoclist); -} - -/* -** This function is used to iterate through a contiguous set of terms -** stored in the full-text index. It merges data contained in one or -** more segments to support this. -** -** The second argument is passed an array of pointers to SegReader objects -** allocated with sqlite3Fts3SegReaderNew(). This function merges the range -** of terms selected by each SegReader. If a single term is present in -** more than one segment, the associated doclists are merged. For each -** term and (possibly merged) doclist in the merged range, the callback -** function xFunc is invoked with its arguments set as follows. -** -** arg 0: Copy of 'p' parameter passed to this function -** arg 1: Copy of 'pContext' parameter passed to this function -** arg 2: Pointer to buffer containing term -** arg 3: Size of arg 2 buffer in bytes -** arg 4: Pointer to buffer containing doclist -** arg 5: Size of arg 2 buffer in bytes -** -** The 4th argument to this function is a pointer to a structure of type -** Fts3SegFilter, defined in fts3Int.h. The contents of this structure -** further restrict the range of terms that callbacks are made for and -** modify the behaviour of this function. See comments above structure -** definition for details. -*/ -int sqlite3Fts3SegReaderIterate( +int sqlite3Fts3SegReaderStart( Fts3Table *p, /* Virtual table handle */ - Fts3SegReader **apSegment, /* Array of Fts3SegReader objects */ - int nSegment, /* Size of apSegment array */ - Fts3SegFilter *pFilter, /* Restrictions on range of iteration */ - int (*xFunc)(Fts3Table *, void *, char *, int, char *, int), /* Callback */ - void *pContext /* Callback context (2nd argument) */ + Fts3SegReaderCursor *pCsr, /* Cursor object */ + Fts3SegFilter *pFilter /* Restrictions on range of iteration */ ){ - int i; /* Iterator variable */ - char *aBuffer = 0; /* Buffer to merge doclists in */ - int nAlloc = 0; /* Allocated size of aBuffer buffer */ - int rc = SQLITE_OK; /* Return code */ - - int isIgnoreEmpty = (pFilter->flags & FTS3_SEGMENT_IGNORE_EMPTY); - int isRequirePos = (pFilter->flags & FTS3_SEGMENT_REQUIRE_POS); - int isColFilter = (pFilter->flags & FTS3_SEGMENT_COLUMN_FILTER); - int isPrefix = (pFilter->flags & FTS3_SEGMENT_PREFIX); - - /* If there are zero segments, this function is a no-op. This scenario - ** comes about only when reading from an empty database. - */ - if( nSegment==0 ) goto finished; + int i; + + /* Initialize the cursor object */ + pCsr->pFilter = pFilter; /* If the Fts3SegFilter defines a specific term (or term prefix) to search ** for, then advance each segment iterator until it points to a term of ** equal or greater value than the specified term. This prevents many ** unnecessary merge/sort operations for the case where single segment ** b-tree leaf nodes contain more than one term. */ - for(i=0; inSegment; i++){ int nTerm = pFilter->nTerm; const char *zTerm = pFilter->zTerm; - Fts3SegReader *pSeg = apSegment[i]; + Fts3SegReader *pSeg = pCsr->apSegment[i]; do { - rc = fts3SegReaderNext(p, pSeg); - if( rc!=SQLITE_OK ) goto finished; + int rc = fts3SegReaderNext(p, pSeg); + if( rc!=SQLITE_OK ) return rc; }while( zTerm && fts3SegReaderTermCmp(pSeg, zTerm, nTerm)<0 ); } + fts3SegReaderSort( + pCsr->apSegment, pCsr->nSegment, pCsr->nSegment, fts3SegReaderCmp); + + return SQLITE_OK; +} + +int sqlite3Fts3SegReaderStep( + Fts3Table *p, /* Virtual table handle */ + Fts3SegReaderCursor *pCsr /* Cursor object */ +){ + int rc = SQLITE_OK; + + int isIgnoreEmpty = (pCsr->pFilter->flags & FTS3_SEGMENT_IGNORE_EMPTY); + int isRequirePos = (pCsr->pFilter->flags & FTS3_SEGMENT_REQUIRE_POS); + int isColFilter = (pCsr->pFilter->flags & FTS3_SEGMENT_COLUMN_FILTER); + int isPrefix = (pCsr->pFilter->flags & FTS3_SEGMENT_PREFIX); + + Fts3SegReader **apSegment = pCsr->apSegment; + int nSegment = pCsr->nSegment; + Fts3SegFilter *pFilter = pCsr->pFilter; + + if( pCsr->nSegment==0 ) return SQLITE_OK; + + do { + int nMerge; + int i; + + /* Advance the first pCsr->nAdvance entries in the apSegment[] array + ** forward. Then sort the list in order of current term again. + */ + for(i=0; inAdvance; i++){ + rc = fts3SegReaderNext(p, apSegment[i]); + if( rc!=SQLITE_OK ) return rc; + } + fts3SegReaderSort(apSegment, nSegment, pCsr->nAdvance, fts3SegReaderCmp); + pCsr->nAdvance = 0; + + /* If all the seg-readers are at EOF, we're finished. return SQLITE_OK. */ + assert( rc==SQLITE_OK ); + if( apSegment[0]->aNode==0 ) break; - fts3SegReaderSort(apSegment, nSegment, nSegment, fts3SegReaderCmp); - while( apSegment[0]->aNode ){ - int nTerm = apSegment[0]->nTerm; - char *zTerm = apSegment[0]->zTerm; - int nMerge = 1; + pCsr->nTerm = apSegment[0]->nTerm; + pCsr->zTerm = apSegment[0]->zTerm; /* If this is a prefix-search, and if the term that apSegment[0] points ** to does not share a suffix with pFilter->zTerm/nTerm, then all ** required callbacks have been made. In this case exit early. ** ** Similarly, if this is a search for an exact match, and the first term ** of segment apSegment[0] is not a match, exit early. */ if( pFilter->zTerm ){ - if( nTermnTerm - || (!isPrefix && nTerm>pFilter->nTerm) - || memcmp(zTerm, pFilter->zTerm, pFilter->nTerm) - ){ - goto finished; + if( pCsr->nTermnTerm + || (!isPrefix && pCsr->nTerm>pFilter->nTerm) + || memcmp(pCsr->zTerm, pFilter->zTerm, pFilter->nTerm) + ){ + break; } } + nMerge = 1; while( nMergeaNode - && apSegment[nMerge]->nTerm==nTerm - && 0==memcmp(zTerm, apSegment[nMerge]->zTerm, nTerm) + && apSegment[nMerge]->nTerm==pCsr->nTerm + && 0==memcmp(pCsr->zTerm, apSegment[nMerge]->zTerm, pCsr->nTerm) ){ nMerge++; } assert( isIgnoreEmpty || (isRequirePos && !isColFilter) ); if( nMerge==1 && !isIgnoreEmpty ){ - Fts3SegReader *p0 = apSegment[0]; - rc = xFunc(p, pContext, zTerm, nTerm, p0->aDoclist, p0->nDoclist); - if( rc!=SQLITE_OK ) goto finished; + pCsr->aDoclist = apSegment[0]->aDoclist; + pCsr->nDoclist = apSegment[0]->nDoclist; + rc = SQLITE_ROW; }else{ int nDoclist = 0; /* Size of doclist */ sqlite3_int64 iPrev = 0; /* Previous docid stored in doclist */ /* The current term of the first nMerge entries in the array ** of Fts3SegReader objects is the same. The doclists must be merged - ** and a single term added to the new segment. + ** and a single term returned with the merged doclist. */ for(i=0; iiCol, &pList, &nList); } if( !isIgnoreEmpty || nList>0 ){ nByte = sqlite3Fts3VarintLen(iDocid-iPrev) + (isRequirePos?nList+1:0); - if( nDoclist+nByte>nAlloc ){ + if( nDoclist+nByte>pCsr->nBuffer ){ char *aNew; - nAlloc = (nDoclist+nByte)*2; - aNew = sqlite3_realloc(aBuffer, nAlloc); + pCsr->nBuffer = (nDoclist+nByte)*2; + aNew = sqlite3_realloc(pCsr->aBuffer, pCsr->nBuffer); if( !aNew ){ - rc = SQLITE_NOMEM; - goto finished; + return SQLITE_NOMEM; } - aBuffer = aNew; + pCsr->aBuffer = aNew; } - nDoclist += sqlite3Fts3PutVarint(&aBuffer[nDoclist], iDocid-iPrev); + nDoclist += sqlite3Fts3PutVarint( + &pCsr->aBuffer[nDoclist], iDocid-iPrev + ); iPrev = iDocid; if( isRequirePos ){ - memcpy(&aBuffer[nDoclist], pList, nList); + memcpy(&pCsr->aBuffer[nDoclist], pList, nList); nDoclist += nList; - aBuffer[nDoclist++] = '\0'; + pCsr->aBuffer[nDoclist++] = '\0'; } } fts3SegReaderSort(apSegment, nMerge, j, fts3SegReaderDoclistCmp); } - - if( nDoclist>0 ){ - rc = xFunc(p, pContext, zTerm, nTerm, aBuffer, nDoclist); - if( rc!=SQLITE_OK ) goto finished; - } - } - - /* If there is a term specified to filter on, and this is not a prefix - ** search, return now. The callback that corresponds to the required - ** term (if such a term exists in the index) has already been made. - */ - if( pFilter->zTerm && !isPrefix ){ - goto finished; - } - - for(i=0; i0 ){ + pCsr->aDoclist = pCsr->aBuffer; + pCsr->nDoclist = nDoclist; + rc = SQLITE_ROW; + } + } + pCsr->nAdvance = nMerge; + }while( rc==SQLITE_OK ); + + return rc; +} + +void sqlite3Fts3SegReaderFinish( + Fts3SegReaderCursor *pCsr /* Cursor object */ +){ + if( pCsr ){ + int i; + for(i=0; inSegment; i++){ + sqlite3Fts3SegReaderFree(pCsr->apSegment[i]); + } + sqlite3_free(pCsr->apSegment); + sqlite3_free(pCsr->aBuffer); + + pCsr->nSegment = 0; + pCsr->apSegment = 0; + pCsr->aBuffer = 0; + } } /* ** Merge all level iLevel segments in the database into a single ** iLevel+1 segment. Or, if iLevel<0, merge all segments into a @@ -2300,161 +2219,74 @@ ** segment in the database, SQLITE_DONE is returned immediately. ** Otherwise, if successful, SQLITE_OK is returned. If an error occurs, ** an SQLite error code is returned. */ static int fts3SegmentMerge(Fts3Table *p, int iLevel){ - int i; /* Iterator variable */ int rc; /* Return code */ int iIdx; /* Index of new segment */ int iNewLevel = 0; /* Level to create new segment at */ - sqlite3_stmt *pStmt = 0; - SegmentWriter *pWriter = 0; - int nSegment = 0; /* Number of segments being merged */ - Fts3SegReader **apSegment = 0; /* Array of Segment iterators */ - Fts3SegReader *pPending = 0; /* Iterator for pending-terms */ + SegmentWriter *pWriter = 0; /* Used to write the new, merged, segment */ Fts3SegFilter filter; /* Segment term filter condition */ + Fts3SegReaderCursor csr; /* Cursor to iterate through level(s) */ + + rc = sqlite3Fts3SegReaderCursor(p, iLevel, 0, 0, 1, &csr); + if( rc!=SQLITE_OK || csr.nSegment==0 ) goto finished; - if( iLevel<0 ){ + if( iLevel==FTS3_SEGCURSOR_ALL ){ /* This call is to merge all segments in the database to a single ** segment. The level of the new segment is equal to the the numerically ** greatest segment level currently present in the database. The index - ** of the new segment is always 0. - */ + ** of the new segment is always 0. */ + int nDummy; /* TODO: Remove this */ + if( csr.nSegment==1 ){ + rc = SQLITE_DONE; + goto finished; + } iIdx = 0; - rc = sqlite3Fts3SegReaderPending(p, 0, 0, 1, &pPending); - if( rc!=SQLITE_OK ) goto finished; - rc = fts3SegmentCountMax(p, &nSegment, &iNewLevel); - if( rc!=SQLITE_OK ) goto finished; - nSegment += (pPending!=0); - if( nSegment<=1 ){ - return SQLITE_DONE; - } + rc = fts3SegmentCountMax(p, &nDummy, &iNewLevel); }else{ /* This call is to merge all segments at level iLevel. Find the next ** available segment index at level iLevel+1. The call to ** fts3AllocateSegdirIdx() will merge the segments at level iLevel+1 to - ** a single iLevel+2 segment if necessary. - */ + ** a single iLevel+2 segment if necessary. */ iNewLevel = iLevel+1; rc = fts3AllocateSegdirIdx(p, iNewLevel, &iIdx); - if( rc!=SQLITE_OK ) goto finished; - rc = fts3SegmentCount(p, iLevel, &nSegment); - if( rc!=SQLITE_OK ) goto finished; - } - assert( nSegment>0 ); - assert( iNewLevel>=0 ); - - /* Allocate space for an array of pointers to segment iterators. */ - apSegment = (Fts3SegReader**)sqlite3_malloc(sizeof(Fts3SegReader *)*nSegment); - if( !apSegment ){ - rc = SQLITE_NOMEM; - goto finished; - } - memset(apSegment, 0, sizeof(Fts3SegReader *)*nSegment); - - /* Allocate a Fts3SegReader structure for each segment being merged. A - ** Fts3SegReader stores the state data required to iterate through all - ** entries on all leaves of a single segment. - */ - assert( SQL_SELECT_LEVEL+1==SQL_SELECT_ALL_LEVEL); - rc = fts3SqlStmt(p, SQL_SELECT_LEVEL+(iLevel<0), &pStmt, 0); - if( rc!=SQLITE_OK ) goto finished; - sqlite3_bind_int(pStmt, 1, iLevel); - for(i=0; SQLITE_ROW==(sqlite3_step(pStmt)); i++){ - rc = fts3SegReaderNew(pStmt, i, &apSegment[i]); - if( rc!=SQLITE_OK ){ - goto finished; - } - } - rc = sqlite3_reset(pStmt); - if( pPending ){ - apSegment[i] = pPending; - pPending = 0; - } - pStmt = 0; - if( rc!=SQLITE_OK ) goto finished; + } + if( rc!=SQLITE_OK ) goto finished; + assert( csr.nSegment>0 ); + assert( iNewLevel>=0 ); memset(&filter, 0, sizeof(Fts3SegFilter)); filter.flags = FTS3_SEGMENT_REQUIRE_POS; - filter.flags |= (iLevel<0 ? FTS3_SEGMENT_IGNORE_EMPTY : 0); - rc = sqlite3Fts3SegReaderIterate(p, apSegment, nSegment, - &filter, fts3MergeCallback, (void *)&pWriter - ); + filter.flags |= (iLevel==FTS3_SEGCURSOR_ALL ? FTS3_SEGMENT_IGNORE_EMPTY : 0); + + rc = sqlite3Fts3SegReaderStart(p, &csr, &filter); + while( SQLITE_OK==rc ){ + rc = sqlite3Fts3SegReaderStep(p, &csr); + if( rc!=SQLITE_ROW ) break; + rc = fts3SegWriterAdd(p, &pWriter, 1, + csr.zTerm, csr.nTerm, csr.aDoclist, csr.nDoclist); + } + if( rc!=SQLITE_OK ) goto finished; + assert( pWriter ); + + rc = fts3DeleteSegdir(p, iLevel, csr.apSegment, csr.nSegment); if( rc!=SQLITE_OK ) goto finished; - - rc = fts3DeleteSegdir(p, iLevel, apSegment, nSegment); - if( rc==SQLITE_OK ){ - rc = fts3SegWriterFlush(p, pWriter, iNewLevel, iIdx); - } + rc = fts3SegWriterFlush(p, pWriter, iNewLevel, iIdx); finished: fts3SegWriterFree(pWriter); - if( apSegment ){ - for(i=0; idb, "SAVEPOINT fts3", 0, 0, 0); if( rc==SQLITE_OK ){ - rc = fts3SegmentMerge(p, -1); + rc = fts3SegmentMerge(p, FTS3_SEGCURSOR_ALL); if( rc==SQLITE_OK ){ rc = sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0); if( rc==SQLITE_OK ){ sqlite3Fts3PendingTermsClear(p); } Index: main.mk ================================================================== --- main.mk +++ main.mk @@ -51,11 +51,11 @@ # Object files for the SQLite library. # LIBOBJ+= alter.o analyze.o attach.o auth.o \ backup.o bitvec.o btmutex.o btree.o build.o \ callback.o complete.o ctime.o date.o delete.o expr.o fault.o fkey.o \ - fts3.o fts3_expr.o fts3_hash.o fts3_icu.o fts3_porter.o \ + fts3.o fts3_aux.o fts3_expr.o fts3_hash.o fts3_icu.o fts3_porter.o \ fts3_snippet.o fts3_tokenizer.o fts3_tokenizer1.o fts3_write.o \ func.o global.o hash.o \ icu.o insert.o journal.o legacy.o loadext.o \ main.o malloc.o mem0.o mem1.o mem2.o mem3.o mem5.o \ memjournal.o \ @@ -185,10 +185,11 @@ $(TOP)/ext/fts2/fts2_tokenizer1.c SRC += \ $(TOP)/ext/fts3/fts3.c \ $(TOP)/ext/fts3/fts3.h \ $(TOP)/ext/fts3/fts3Int.h \ + $(TOP)/ext/fts3/fts3_aux.c \ $(TOP)/ext/fts3/fts3_expr.c \ $(TOP)/ext/fts3/fts3_hash.c \ $(TOP)/ext/fts3/fts3_hash.h \ $(TOP)/ext/fts3/fts3_icu.c \ $(TOP)/ext/fts3/fts3_porter.c \ @@ -291,10 +292,11 @@ $(TOP)/src/vdbe.c \ $(TOP)/src/vdbemem.c \ $(TOP)/src/where.c \ parse.c \ $(TOP)/ext/fts3/fts3.c \ + $(TOP)/ext/fts3/fts3_aux.c \ $(TOP)/ext/fts3/fts3_expr.c \ $(TOP)/ext/fts3/fts3_tokenizer.c \ $(TOP)/ext/fts3/fts3_write.c \ $(TOP)/ext/async/sqlite3async.c @@ -459,10 +461,13 @@ fts2_tokenizer1.o: $(TOP)/ext/fts2/fts2_tokenizer1.c $(HDR) $(EXTHDR) $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts2/fts2_tokenizer1.c fts3.o: $(TOP)/ext/fts3/fts3.c $(HDR) $(EXTHDR) $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3.c + +fts3_aux.o: $(TOP)/ext/fts3/fts3_aux.c $(HDR) $(EXTHDR) + $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_aux.c fts3_expr.o: $(TOP)/ext/fts3/fts3_expr.c $(HDR) $(EXTHDR) $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_expr.c fts3_hash.o: $(TOP)/ext/fts3/fts3_hash.c $(HDR) $(EXTHDR) Index: src/pager.c ================================================================== --- src/pager.c +++ src/pager.c @@ -2848,10 +2848,32 @@ PAGERTRACE(("FETCH %d page %d hash(%08x)\n", PAGERID(pPager), pgno, pager_pagehash(pPg))); return rc; } + +/* +** Update the value of the change-counter at offsets 24 and 92 in +** the header and the sqlite version number at offset 96. +** +** This is an unconditional update. See also the pager_incr_changecounter() +** routine which only updates the change-counter if the update is actually +** needed, as determined by the pPager->changeCountDone state variable. +*/ +static void pager_write_changecounter(PgHdr *pPg){ + u32 change_counter; + + /* Increment the value just read and write it back to byte 24. */ + change_counter = sqlite3Get4byte((u8*)pPg->pPager->dbFileVers)+1; + put32bits(((char*)pPg->pData)+24, change_counter); + + /* Also store the SQLite version number in bytes 96..99 and in + ** bytes 92..95 store the change counter for which the version number + ** is valid. */ + put32bits(((char*)pPg->pData)+92, change_counter); + put32bits(((char*)pPg->pData)+96, SQLITE_VERSION_NUMBER); +} #ifndef SQLITE_OMIT_WAL /* ** This function is invoked once for each page that has already been ** written into the log file when a WAL transaction is rolled back. @@ -2919,33 +2941,10 @@ } return rc; } - -/* -** Update the value of the change-counter at offsets 24 and 92 in -** the header and the sqlite version number at offset 96. -** -** This is an unconditional update. See also the pager_incr_changecounter() -** routine which only updates the change-counter if the update is actually -** needed, as determined by the pPager->changeCountDone state variable. -*/ -static void pager_write_changecounter(PgHdr *pPg){ - u32 change_counter; - - /* Increment the value just read and write it back to byte 24. */ - change_counter = sqlite3Get4byte((u8*)pPg->pPager->dbFileVers)+1; - put32bits(((char*)pPg->pData)+24, change_counter); - - /* Also store the SQLite version number in bytes 96..99 and in - ** bytes 92..95 store the change counter for which the version number - ** is valid. */ - put32bits(((char*)pPg->pData)+92, change_counter); - put32bits(((char*)pPg->pData)+96, SQLITE_VERSION_NUMBER); -} - /* ** This function is a wrapper around sqlite3WalFrames(). As well as logging ** the contents of the list of pages headed by pList (connected by pDirty), ** this function notifies any active backup processes that the pages have ** changed. Index: src/sqlite.h.in ================================================================== --- src/sqlite.h.in +++ src/sqlite.h.in @@ -5532,28 +5532,25 @@ ** checked out.)^ ** ** ^(
SQLITE_DBSTATUS_LOOKASIDE_HIT
**
This parameter returns the number malloc attempts that were ** satisfied using lookaside memory. Only the high-water value is meaningful; -** the current value is always zero. -** checked out.
)^ +** the current value is always zero.)^ ** ** ^(
SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE
**
This parameter returns the number malloc attempts that might have ** been satisfied using lookaside memory but failed due to the amount of ** memory requested being larger than the lookaside slot size. ** Only the high-water value is meaningful; -** the current value is always zero. -** checked out.
)^ +** the current value is always zero.)^ ** ** ^(
SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL
**
This parameter returns the number malloc attempts that might have ** been satisfied using lookaside memory but failed due to all lookaside ** memory already being in use. ** Only the high-water value is meaningful; -** the current value is always zero. -** checked out.
)^ +** the current value is always zero.)^ ** ** ^(
SQLITE_DBSTATUS_CACHE_USED
**
This parameter returns the approximate number of of bytes of heap ** memory used by all pager caches associated with the database connection.)^ ** ^The highwater mark associated with SQLITE_DBSTATUS_CACHE_USED is always 0. ADDED test/fts3aux1.test Index: test/fts3aux1.test ================================================================== --- /dev/null +++ test/fts3aux1.test @@ -0,0 +1,59 @@ +# 2011 January 27 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The +# focus of this script is testing the FTS3 module. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +ifcapable !fts3 { finish_test ; return } +set ::testprefix fts3aux1 + +do_execsql_test 1.1 { + CREATE VIRTUAL TABLE t1 USING fts4; + INSERT INTO t1 VALUES('one two three four'); + INSERT INTO t1 VALUES('three four five six'); + INSERT INTO t1 VALUES('one three five seven'); + + CREATE VIRTUAL TABLE terms USING fts4aux(t1); + SELECT * FROM terms; +} { + five 2 2 four 2 2 one 2 2 seven 1 1 + six 1 1 three 3 3 two 1 1 +} + +do_execsql_test 1.2 { + INSERT INTO t1 VALUES('one one one three three three'); + SELECT * FROM terms; +} { + five 2 2 four 2 2 one 3 5 seven 1 1 + six 1 1 three 4 6 two 1 1 +} + +do_execsql_test 1.3 { + DELETE FROM t1; + SELECT * FROM terms; +} {} + +do_execsql_test 1.4 { + INSERT INTO t1 VALUES('a b a b a b a'); + INSERT INTO t1 SELECT * FROM t1; + INSERT INTO t1 SELECT * FROM t1; + INSERT INTO t1 SELECT * FROM t1; + INSERT INTO t1 SELECT * FROM t1; + INSERT INTO t1 SELECT * FROM t1; + INSERT INTO t1 SELECT * FROM t1; + INSERT INTO t1 SELECT * FROM t1; + INSERT INTO t1 SELECT * FROM t1; + SELECT * FROM terms; +} {a 256 1024 b 256 768} + +finish_test ADDED test/fts3comp1.test Index: test/fts3comp1.test ================================================================== --- /dev/null +++ test/fts3comp1.test @@ -0,0 +1,78 @@ +# 2011 January 27 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The +# focus of this script is testing the FTS3 module. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +ifcapable !fts3 { finish_test ; return } +set ::testprefix fts3comp1 + +set next_x 0 +proc zip {x} { + incr ::next_x + set ::strings($::next_x) $x + return $::next_x +} +proc unzip {x} { + return $::strings($x) +} + +db func zip zip +db func unzip unzip + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts4( + a, b, + compress=zip, uncompress=unzip + ); + INSERT INTO t1 VALUES('one two three', 'two four six'); +} + +do_execsql_test 1.1 { + SELECT a, b FROM t1; +} {{one two three} {two four six}} + +do_execsql_test 1.2 { + SELECT c0a, c1b FROM t1_content; +} {1 2} + +do_execsql_test 1.3 { + INSERT INTO t1 VALUES('three six nine', 'four eight twelve'); + SELECT a, b FROM t1; +} {{one two three} {two four six} {three six nine} {four eight twelve}} + +do_execsql_test 1.4 { + SELECT c0a, c1b FROM t1_content; +} {1 2 3 4} + +do_execsql_test 1.5 { + CREATE VIRTUAL TABLE terms USING fts4aux(t1); + SELECT * FROM terms; +} { + eight 1 1 four 2 2 nine 1 1 one 1 1 + six 2 2 three 2 2 twelve 1 1 two 1 2 +} + +do_execsql_test 1.6 { + DELETE FROM t1 WHERE docid = 1; + SELECT * FROM terms; +} { + eight 1 1 four 1 1 nine 1 1 + six 1 1 three 1 1 twelve 1 1 +} + +do_execsql_test 1.7 { + SELECT c0a, c1b FROM t1_content; +} {3 4} + +finish_test Index: test/fts3corrupt.test ================================================================== --- test/fts3corrupt.test +++ test/fts3corrupt.test @@ -128,8 +128,35 @@ do_catchsql_test 4.3 { UPDATE t1_segdir SET root = $blob; SELECT rowid FROM t1 WHERE t1 MATCH 'world'; } {1 {database disk image is malformed}} + +# Test a special kind of corruption, where the %_stat table contains +# an invalid entry. At one point this could lead to a division-by-zero +# error in fts4. +# +do_execsql_test 5.0 { + DROP TABLE t1; + CREATE VIRTUAL TABLE t1 USING fts4; +} +do_test 5.1 { + db func nn nn + execsql BEGIN + execsql { INSERT INTO t1 VALUES('one') } + execsql { INSERT INTO t1 VALUES('two') } + execsql { INSERT INTO t1 VALUES('three') } + execsql { INSERT INTO t1 VALUES('four') } + execsql COMMIT +} {} +do_catchsql_test 5.2 { + UPDATE t1_stat SET value = X'0000'; + SELECT matchinfo(t1, 'nxa') FROM t1 WHERE t1 MATCH 't*'; +} {1 {database disk image is malformed}} +do_catchsql_test 5.3 { + UPDATE t1_stat SET value = NULL; + SELECT matchinfo(t1, 'nxa') FROM t1 WHERE t1 MATCH 't*'; +} {1 {database disk image is malformed}} + finish_test Index: test/permutations.test ================================================================== --- test/permutations.test +++ test/permutations.test @@ -179,10 +179,12 @@ fts3atoken.test fts3b.test fts3c.test fts3cov.test fts3d.test fts3defer.test fts3defer2.test fts3e.test fts3expr.test fts3expr2.test fts3near.test fts3query.test fts3shared.test fts3snippet.test fts3fault.test fts3malloc.test fts3matchinfo.test + + fts3aux1.test fts3comp1.test } lappend ::testsuitelist xxx #------------------------------------------------------------------------- Index: test/trace2.test ================================================================== --- test/trace2.test +++ test/trace2.test @@ -126,11 +126,11 @@ do_trace_test 2.2 { INSERT INTO x1 VALUES('North northwest wind between 8 and 14 mph'); } { "INSERT INTO x1 VALUES('North northwest wind between 8 and 14 mph');" - "-- INSERT INTO 'main'.'x1_content' VALUES(?,?)" + "-- INSERT INTO 'main'.'x1_content' VALUES(?,(?))" "-- REPLACE INTO 'main'.'x1_docsize' VALUES(?,?)" "-- SELECT value FROM 'main'.'x1_stat' WHERE id=0" "-- REPLACE INTO 'main'.'x1_stat' VALUES(0,?)" "-- SELECT (SELECT max(idx) FROM 'main'.'x1_segdir' WHERE level = ?) + 1" "-- SELECT coalesce((SELECT max(blockid) FROM 'main'.'x1_segments') + 1, 1)" @@ -139,14 +139,14 @@ do_trace_test 2.3 { INSERT INTO x1(x1) VALUES('optimize'); } { "INSERT INTO x1(x1) VALUES('optimize');" - "-- SELECT count(*), max(level) FROM 'main'.'x1_segdir'" "-- SELECT idx, start_block, leaves_end_block, end_block, root FROM 'main'.'x1_segdir' ORDER BY level DESC, idx ASC" + "-- SELECT count(*), max(level) FROM 'main'.'x1_segdir'" "-- SELECT coalesce((SELECT max(blockid) FROM 'main'.'x1_segments') + 1, 1)" "-- DELETE FROM 'main'.'x1_segdir'" "-- INSERT INTO 'main'.'x1_segdir' VALUES(?,?,?,?,?,?)" } } finish_test Index: tool/mksqlite3c.tcl ================================================================== --- tool/mksqlite3c.tcl +++ tool/mksqlite3c.tcl @@ -292,10 +292,11 @@ main.c notify.c fts3.c + fts3_aux.c fts3_expr.c fts3_hash.c fts3_porter.c fts3_tokenizer.c fts3_tokenizer1.c