Index: ext/fts3/fts3.c ================================================================== --- ext/fts3/fts3.c +++ ext/fts3/fts3.c @@ -1274,10 +1274,11 @@ p->pTokenizer = pTokenizer; p->nMaxPendingData = FTS3_MAX_PENDING_DATA; p->bHasDocsize = (isFts4 && bNoDocsize==0); p->bHasStat = isFts4; p->bDescIdx = bDescIdx; + p->bAutoincrmerge = 0xff; /* 0xff means setting unknown */ p->zContentTbl = zContent; p->zLanguageid = zLanguageid; zContent = 0; zLanguageid = 0; TESTONLY( p->inTransaction = -1 ); @@ -3100,12 +3101,25 @@ /* ** Implementation of xSync() method. Flush the contents of the pending-terms ** hash-table to the database. */ static int fts3SyncMethod(sqlite3_vtab *pVtab){ - int rc = sqlite3Fts3PendingTermsFlush((Fts3Table *)pVtab); - sqlite3Fts3SegmentsClose((Fts3Table *)pVtab); + const int nMinMerge = 64; /* Minimum amount of incr-merge work to do */ + Fts3Table *p = (Fts3Table*)pVtab; + int rc = sqlite3Fts3PendingTermsFlush(p); + + if( rc==SQLITE_OK && p->bAutoincrmerge==1 && p->nLeafAdd>(nMinMerge/16) ){ + int mxLevel = 0; /* Maximum relative level value in db */ + int A; /* Incr-merge parameter A */ + + rc = sqlite3Fts3MaxLevel(p, &mxLevel); + assert( rc==SQLITE_OK || mxLevel==0 ); + A = p->nLeafAdd * p->mxLevel; + A += (A/2); + if( A>nMinMerge ) rc = sqlite3Fts3Incrmerge(p, A, 8); + } + sqlite3Fts3SegmentsClose(p); return rc; } /* ** Implementation of xBegin() method. This is a no-op. @@ -3116,10 +3130,11 @@ assert( p->pSegments==0 ); assert( p->nPendingData==0 ); assert( p->inTransaction!=1 ); TESTONLY( p->inTransaction = 1 ); TESTONLY( p->mxSavepoint = -1; ); + p->nLeafAdd = 0; return SQLITE_OK; } /* ** Implementation of xCommit() method. This is a no-op. The contents of Index: ext/fts3/fts3Int.h ================================================================== --- ext/fts3/fts3Int.h +++ ext/fts3/fts3Int.h @@ -194,15 +194,18 @@ int nColumn; /* number of named columns in virtual table */ char **azColumn; /* column names. malloced */ sqlite3_tokenizer *pTokenizer; /* tokenizer for inserts and queries */ char *zContentTbl; /* content=xxx option, or NULL */ char *zLanguageid; /* languageid=xxx option, or NULL */ + u8 bAutoincrmerge; /* True if automerge=1 */ + int mxLevel; /* Maximum level seen on this transaction */ + u32 nLeafAdd; /* Number of leaf blocks added this trans */ /* Precompiled statements used by the implementation. Each of these ** statements is run and reset within a single virtual table API call. */ - sqlite3_stmt *aStmt[36]; + sqlite3_stmt *aStmt[37]; char *zReadExprlist; char *zWriteExprlist; int nNodeSize; /* Soft limit for node size */ @@ -426,10 +429,11 @@ void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *); int sqlite3Fts3DeferToken(Fts3Cursor *, Fts3PhraseToken *, int); int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *); void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *); void sqlite3Fts3SegmentsClose(Fts3Table *); +int sqlite3Fts3MaxLevel(Fts3Table *, int *); /* Special values interpreted by sqlite3SegReaderCursor() */ #define FTS3_SEGCURSOR_PENDING -1 #define FTS3_SEGCURSOR_ALL -2 @@ -476,10 +480,12 @@ char *zTerm; /* Pointer to term buffer */ int nTerm; /* Size of zTerm in bytes */ char *aDoclist; /* Pointer to doclist buffer */ int nDoclist; /* Size of aDoclist[] in bytes */ }; + +int sqlite3Fts3Incrmerge(Fts3Table*,int,int); /* fts3.c */ int sqlite3Fts3PutVarint(char *, sqlite3_int64); int sqlite3Fts3GetVarint(const char *, sqlite_int64 *); int sqlite3Fts3GetVarint32(const char *, int *); Index: ext/fts3/fts3_write.c ================================================================== --- ext/fts3/fts3_write.c +++ ext/fts3/fts3_write.c @@ -70,10 +70,11 @@ ** The two values that may be meaningfully bound to the :1 parameter in ** statements SQL_REPLACE_STAT and SQL_SELECT_STAT. */ #define FTS_STAT_DOCTOTAL 0 #define FTS_STAT_INCRMERGEHINT 1 +#define FTS_STAT_AUTOINCRMERGE 2 /* ** If FTS_LOG_MERGES is defined, call sqlite3_log() to report each automatic ** and incremental merge operation that takes place. This is used for ** debugging FTS only, it should not usually be turned on in production @@ -262,10 +263,11 @@ #define SQL_SHIFT_SEGDIR_ENTRY 31 #define SQL_SELECT_SEGDIR 32 #define SQL_CHOMP_SEGDIR 33 #define SQL_SEGMENT_IS_APPENDABLE 34 #define SQL_SELECT_INDEXES 35 +#define SQL_SELECT_MXLEVEL 36 /* ** This function is used to obtain an SQLite prepared statement handle ** for the statement identified by the second argument. If successful, ** *pp is set to the requested statement handle and SQLITE_OK returned. @@ -328,11 +330,11 @@ /* 28 */ "SELECT level FROM %Q.'%q_segdir' GROUP BY level HAVING count(*)>=?" " ORDER BY (level %% 1024) DESC LIMIT 1", /* Estimate the upper limit on the number of leaf nodes in a new segment ** created by merging the oldest :2 segments from absolute level :1. See -** function fts3Incrmerge() for details. */ +** function sqlite3Fts3Incrmerge() for details. */ /* 29 */ "SELECT 2 * total(1 + leaves_end_block - start_block) " " FROM %Q.'%q_segdir' WHERE level = ? AND idx < ?", /* SQL_DELETE_SEGDIR_ENTRY ** Delete the %_segdir entry on absolute level :1 with index :2. */ @@ -360,11 +362,15 @@ ** no rows otherwise. */ /* 34 */ "SELECT 1 FROM %Q.'%q_segments' WHERE blockid=? AND block IS NULL", /* SQL_SELECT_INDEXES ** Return the list of valid segment indexes for absolute level ? */ -/* 35 */ "SELECT idx FROM %Q.'%q_segdir' WHERE level=? ORDER BY 1 ASC" +/* 35 */ "SELECT idx FROM %Q.'%q_segdir' WHERE level=? ORDER BY 1 ASC", + +/* SQL_SELECT_MXLEVEL +** Return the largest relative level in the FTS index or indexes. */ +/* 36 */ "SELECT max( level %% 1024 ) FROM %Q.'%q_segdir'" }; int rc = SQLITE_OK; sqlite3_stmt *pStmt; assert( SizeofArray(azSql)==SizeofArray(p->aStmt) ); @@ -1865,10 +1871,39 @@ sqlite3_step(pStmt); rc = sqlite3_reset(pStmt); } return rc; } + +/* +** Update the Fts3Table.mxLevel field, if appropriate +*/ +static void fts3UpdateMaxLevel(Fts3Table *p, sqlite3_int64 iLevel){ + iLevel %= FTS3_SEGDIR_MAXLEVEL; + if( iLevel>p->mxLevel ) p->mxLevel = iLevel; +} + +/* +** Find the largest relative level number in the table. If successful, set +** *pnMax to this value and return SQLITE_OK. Otherwise, if an error occurs, +** set *pnMax to zero and return an SQLite error code. +*/ +int sqlite3Fts3MaxLevel(Fts3Table *p, int *pnMax){ + int rc; + int mxLevel = 0; + sqlite3_stmt *pStmt = 0; + + rc = fts3SqlStmt(p, SQL_SELECT_MXLEVEL, &pStmt, 0); + if( rc==SQLITE_OK ){ + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + mxLevel = sqlite3_column_int(pStmt, 0); + } + rc = sqlite3_reset(pStmt); + } + *pnMax = mxLevel; + return rc; +} /* ** Insert a record into the %_segdir table. */ static int fts3WriteSegdir( @@ -1883,10 +1918,11 @@ ){ sqlite3_stmt *pStmt; int rc = fts3SqlStmt(p, SQL_INSERT_SEGDIR, &pStmt, 0); if( rc==SQLITE_OK ){ sqlite3_bind_int64(pStmt, 1, iLevel); + fts3UpdateMaxLevel(p, iLevel); sqlite3_bind_int(pStmt, 2, iIdx); sqlite3_bind_int64(pStmt, 3, iStartBlock); sqlite3_bind_int64(pStmt, 4, iLeafEndBlock); sqlite3_bind_int64(pStmt, 5, iEndBlock); sqlite3_bind_blob(pStmt, 6, zRoot, nRoot, SQLITE_STATIC); @@ -2182,10 +2218,11 @@ int rc; /* The current leaf node is full. Write it out to the database. */ rc = fts3WriteSegment(p, pWriter->iFree++, pWriter->aData, nData); if( rc!=SQLITE_OK ) return rc; + p->nLeafAdd++; /* Add the current term to the interior node tree. The term added to ** the interior tree must: ** ** a) be greater than the largest term on the leaf node just written @@ -2290,10 +2327,11 @@ }else{ /* The entire tree fits on the root node. Write it to the segdir table. */ rc = fts3WriteSegdir( p, iLevel, iIdx, 0, 0, 0, pWriter->aData, pWriter->nData); } + p->nLeafAdd++; return rc; } /* ** Release all memory held by the SegmentWriter object passed as the @@ -2367,10 +2405,11 @@ sqlite3_bind_int64(pStmt, 2, getAbsoluteLevel(p, iLangid, iIndex, FTS3_SEGDIR_MAXLEVEL-1) ); if( SQLITE_ROW==sqlite3_step(pStmt) ){ *pnMax = sqlite3_column_int64(pStmt, 0); + fts3UpdateMaxLevel(p, *pnMax); } return sqlite3_reset(pStmt); } /* @@ -2997,15 +3036,32 @@ ** Flush the contents of pendingTerms to level 0 segments. */ int sqlite3Fts3PendingTermsFlush(Fts3Table *p){ int rc = SQLITE_OK; int i; + for(i=0; rc==SQLITE_OK && inIndex; i++){ rc = fts3SegmentMerge(p, p->iPrevLangid, i, FTS3_SEGCURSOR_PENDING); if( rc==SQLITE_DONE ) rc = SQLITE_OK; } sqlite3Fts3PendingTermsClear(p); + + /* Determine the auto-incr-merge setting if unknown. If enabled, + ** estimate the number of leaf blocks of content to be written + */ + if( rc==SQLITE_OK && p->bHasStat + && p->bAutoincrmerge==0xff && p->nLeafAdd>0 + ){ + sqlite3_stmt *pStmt = 0; + rc = fts3SqlStmt(p, SQL_SELECT_STAT, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int(pStmt, 1, FTS_STAT_AUTOINCRMERGE); + rc = sqlite3_step(pStmt); + p->bAutoincrmerge = (rc==SQLITE_ROW && sqlite3_column_int(pStmt, 0)); + rc = sqlite3_reset(pStmt); + } + } return rc; } /* ** Encode N integers as varints into a blob. @@ -4470,11 +4526,11 @@ ** segments to be merged are the nMin oldest segments (the ones with ** the smallest indexes) in the highest level that contains at least ** nMin segments. Multiple merges might occur in an attempt to write the ** quota of nMerge leaf blocks. */ -static int fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ +int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ int rc; /* Return code */ int nRem = nMerge; /* Number of leaf pages yet to be written */ int bUseHint = 1; /* True if hint has not yet been attempted */ sqlite3_int64 iHintAbsLevel = 0;/* Hint level */ int nHintSeg = 0; /* Hint number of segments */ @@ -4575,10 +4631,23 @@ rc = fts3IncrmergeHintStore(p, iAbsLevel, nSeg); } return rc; } + +/* +** Convert the text beginning at *pz into an integer and return +** its value. Advance *pz to point to the first character past +** the integer. +*/ +static int fts3Getint(const char **pz){ + const char *z = *pz; + int i = 0; + while( (*z)>='0' && (*z)<='9' ) i = 10*i + *(z++) - '0'; + *pz = z; + return i; +} /* ** Process statements of the form: ** ** INSERT INTO table(table) VALUES('merge=A,B'); @@ -4595,33 +4664,52 @@ int nMin = (FTS3_MERGE_COUNT / 2); int nMerge = 0; const char *z = zParam; /* Read the first integer value */ - for(z=zParam; z[0]>='0' && z[0]<='9'; z++){ - nMerge = nMerge * 10 + (z[0] - '0'); - } + nMerge = fts3Getint(&z); /* If the first integer value is followed by a ',', read the second ** integer value. */ if( z[0]==',' && z[1]!='\0' ){ z++; - nMin = 0; - while( z[0]>='0' && z[0]<='9' ){ - nMin = nMin * 10 + (z[0] - '0'); - z++; - } + nMin = fts3Getint(&z); } if( z[0]!='\0' || nMin<2 ){ rc = SQLITE_ERROR; }else{ - rc = fts3Incrmerge(p, nMerge, nMin); + rc = sqlite3Fts3Incrmerge(p, nMerge, nMin); sqlite3Fts3SegmentsClose(p); } return rc; } + +/* +** Process statements of the form: +** +** INSERT INTO table(table) VALUES('automerge=X'); +** +** where X is an integer. X==0 means to turn automerge off. X!=0 means +** turn it on. The setting is persistent. +*/ +static int fts3DoAutoincrmerge( + Fts3Table *p, /* FTS3 table handle */ + const char *zParam /* Nul-terminated string containing boolean */ +){ + int rc; + sqlite3_stmt *pStmt = 0; + p->bAutoincrmerge = fts3Getint(&zParam)!=0; + rc = fts3SqlStmt(p, SQL_REPLACE_STAT, &pStmt, 0); + if( rc ) return rc;; + sqlite3_bind_int(pStmt, 1, FTS_STAT_AUTOINCRMERGE); + sqlite3_bind_int(pStmt, 2, p->bAutoincrmerge); + sqlite3_step(pStmt); + rc = sqlite3_reset(pStmt); + return rc; +} + /* ** Handle a 'special' INSERT of the form: ** ** "INSERT INTO tbl(tbl) VALUES()" @@ -4640,10 +4728,12 @@ rc = fts3DoOptimize(p, 0); }else if( nVal==7 && 0==sqlite3_strnicmp(zVal, "rebuild", 7) ){ rc = fts3DoRebuild(p); }else if( nVal>6 && 0==sqlite3_strnicmp(zVal, "merge=", 6) ){ rc = fts3DoIncrmerge(p, &zVal[6]); + }else if( nVal>10 && 0==sqlite3_strnicmp(zVal, "automerge=", 10) ){ + rc = fts3DoAutoincrmerge(p, &zVal[10]); #ifdef SQLITE_TEST }else if( nVal>9 && 0==sqlite3_strnicmp(zVal, "nodesize=", 9) ){ p->nNodeSize = atoi(&zVal[9]); rc = SQLITE_OK; }else if( nVal>11 && 0==sqlite3_strnicmp(zVal, "maxpending=", 9) ){ Index: test/bc_common.tcl ================================================================== --- test/bc_common.tcl +++ test/bc_common.tcl @@ -15,11 +15,11 @@ if {[file executable $file] && [file isfile $file]} {lappend binaries $file} } if {[llength $binaries]==0} { puts "WARNING: No historical binaries to test against." - puts "WARNING: Omitting backwards-compatibility tests $zFile" + puts "WARNING: Omitting backwards-compatibility tests" } foreach bin $binaries { puts -nonewline "Testing against $bin - " flush stdout @@ -68,9 +68,5 @@ proc do_all_bc_test {script} { foreach bin $::BC(binaries) { uplevel [list do_bc_test $bin $script] } } - - - -