Index: ext/fts3/fts3.c ================================================================== --- ext/fts3/fts3.c +++ ext/fts3/fts3.c @@ -1331,11 +1331,11 @@ p->nMaxPendingData = FTS3_MAX_PENDING_DATA; p->bHasDocsize = (isFts4 && bNoDocsize==0); p->bHasStat = isFts4; p->bFts4 = isFts4; p->bDescIdx = bDescIdx; - p->bAutoincrmerge = 0xff; /* 0xff means setting unknown */ + p->nAutoincrmerge = 0xff; /* 0xff means setting unknown */ p->zContentTbl = zContent; p->zLanguageid = zLanguageid; zContent = 0; zLanguageid = 0; TESTONLY( p->inTransaction = -1 ); @@ -3300,19 +3300,22 @@ const u32 nMinMerge = 64; /* Minimum amount of incr-merge work to do */ Fts3Table *p = (Fts3Table*)pVtab; int rc = sqlite3Fts3PendingTermsFlush(p); - if( rc==SQLITE_OK && p->bAutoincrmerge==1 && p->nLeafAdd>(nMinMerge/16) ){ + if( rc==SQLITE_OK + && p->nLeafAdd>(nMinMerge/16) + && p->nAutoincrmerge && p->nAutoincrmerge!=0xff + ){ int mxLevel = 0; /* Maximum relative level value in db */ int A; /* Incr-merge parameter A */ rc = sqlite3Fts3MaxLevel(p, &mxLevel); assert( rc==SQLITE_OK || mxLevel==0 ); A = p->nLeafAdd * mxLevel; A += (A/2); - if( A>(int)nMinMerge ) rc = sqlite3Fts3Incrmerge(p, A, 8); + if( A>(int)nMinMerge ) rc = sqlite3Fts3Incrmerge(p, A, p->nAutoincrmerge); } sqlite3Fts3SegmentsClose(p); return rc; } Index: ext/fts3/fts3Int.h ================================================================== --- ext/fts3/fts3Int.h +++ ext/fts3/fts3Int.h @@ -208,17 +208,17 @@ char **azColumn; /* column names. malloced */ u8 *abNotindexed; /* True for 'notindexed' columns */ sqlite3_tokenizer *pTokenizer; /* tokenizer for inserts and queries */ char *zContentTbl; /* content=xxx option, or NULL */ char *zLanguageid; /* languageid=xxx option, or NULL */ - u8 bAutoincrmerge; /* True if automerge=1 */ + int nAutoincrmerge; /* Value configured by 'automerge' */ u32 nLeafAdd; /* Number of leaf blocks added this trans */ /* Precompiled statements used by the implementation. Each of these ** statements is run and reset within a single virtual table API call. */ - sqlite3_stmt *aStmt[37]; + sqlite3_stmt *aStmt[40]; char *zReadExprlist; char *zWriteExprlist; int nNodeSize; /* Soft limit for node size */ Index: ext/fts3/fts3_write.c ================================================================== --- ext/fts3/fts3_write.c +++ ext/fts3/fts3_write.c @@ -191,10 +191,11 @@ int nMalloc; /* Size of malloc'd buffer at zMalloc */ char *zMalloc; /* Malloc'd space (possibly) used for zTerm */ int nSize; /* Size of allocation at aData */ int nData; /* Bytes of data in aData */ char *aData; /* Pointer to block from malloc() */ + i64 nLeafData; /* Number of bytes of leaf data written */ }; /* ** Type SegmentNode is used by the following three functions to create ** the interior part of the segment b+-tree structures (everything except @@ -265,10 +266,14 @@ #define SQL_SELECT_SEGDIR 32 #define SQL_CHOMP_SEGDIR 33 #define SQL_SEGMENT_IS_APPENDABLE 34 #define SQL_SELECT_INDEXES 35 #define SQL_SELECT_MXLEVEL 36 + +#define SQL_SELECT_LEVEL_RANGE2 37 +#define SQL_UPDATE_LEVEL_IDX 38 +#define SQL_UPDATE_LEVEL 39 /* ** This function is used to obtain an SQLite prepared statement handle ** for the statement identified by the second argument. If successful, ** *pp is set to the requested statement handle and SQLITE_OK returned. @@ -367,11 +372,22 @@ ** Return the list of valid segment indexes for absolute level ? */ /* 35 */ "SELECT idx FROM %Q.'%q_segdir' WHERE level=? ORDER BY 1 ASC", /* SQL_SELECT_MXLEVEL ** Return the largest relative level in the FTS index or indexes. */ -/* 36 */ "SELECT max( level %% 1024 ) FROM %Q.'%q_segdir'" +/* 36 */ "SELECT max( level %% 1024 ) FROM %Q.'%q_segdir'", + + /* Return segments in order from oldest to newest.*/ +/* 37 */ "SELECT level, idx, end_block " + "FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ? " + "ORDER BY level DESC, idx ASC", + + /* Update statements used while promoting segments */ +/* 38 */ "UPDATE OR FAIL %Q.'%q_segdir' SET level=-1,idx=? " + "WHERE level=? AND idx=?", +/* 39 */ "UPDATE OR FAIL %Q.'%q_segdir' SET level=? WHERE level=-1" + }; int rc = SQLITE_OK; sqlite3_stmt *pStmt; assert( SizeofArray(azSql)==SizeofArray(p->aStmt) ); @@ -1908,10 +1924,11 @@ sqlite3_int64 iLevel, /* Value for "level" field (absolute level) */ int iIdx, /* Value for "idx" field */ sqlite3_int64 iStartBlock, /* Value for "start_block" field */ sqlite3_int64 iLeafEndBlock, /* Value for "leaves_end_block" field */ sqlite3_int64 iEndBlock, /* Value for "end_block" field */ + sqlite3_int64 nLeafData, /* Bytes of leaf data in segment */ char *zRoot, /* Blob value for "root" field */ int nRoot /* Number of bytes in buffer zRoot */ ){ sqlite3_stmt *pStmt; int rc = fts3SqlStmt(p, SQL_INSERT_SEGDIR, &pStmt, 0); @@ -1918,11 +1935,17 @@ if( rc==SQLITE_OK ){ sqlite3_bind_int64(pStmt, 1, iLevel); sqlite3_bind_int(pStmt, 2, iIdx); sqlite3_bind_int64(pStmt, 3, iStartBlock); sqlite3_bind_int64(pStmt, 4, iLeafEndBlock); - sqlite3_bind_int64(pStmt, 5, iEndBlock); + if( nLeafData==0 ){ + sqlite3_bind_int64(pStmt, 5, iEndBlock); + }else{ + char *zEnd = sqlite3_mprintf("%lld %lld", iEndBlock, nLeafData); + if( !zEnd ) return SQLITE_NOMEM; + sqlite3_bind_text(pStmt, 5, zEnd, -1, sqlite3_free); + } sqlite3_bind_blob(pStmt, 6, zRoot, nRoot, SQLITE_STATIC); sqlite3_step(pStmt); rc = sqlite3_reset(pStmt); } return rc; @@ -2243,10 +2266,13 @@ sqlite3Fts3VarintLen(nTerm) + /* varint containing suffix size */ nTerm + /* Term suffix */ sqlite3Fts3VarintLen(nDoclist) + /* Size of doclist */ nDoclist; /* Doclist data */ } + + /* Increase the total number of bytes written to account for the new entry. */ + pWriter->nLeafData += nReq; /* If the buffer currently allocated is too small for this entry, realloc ** the buffer to make it large enough. */ if( nReq>pWriter->nSize ){ @@ -2315,17 +2341,17 @@ if( rc==SQLITE_OK ){ rc = fts3NodeWrite(p, pWriter->pTree, 1, pWriter->iFirst, pWriter->iFree, &iLast, &zRoot, &nRoot); } if( rc==SQLITE_OK ){ - rc = fts3WriteSegdir( - p, iLevel, iIdx, pWriter->iFirst, iLastLeaf, iLast, zRoot, nRoot); + rc = fts3WriteSegdir(p, iLevel, iIdx, + pWriter->iFirst, iLastLeaf, iLast, pWriter->nLeafData, zRoot, nRoot); } }else{ /* The entire tree fits on the root node. Write it to the segdir table. */ - rc = fts3WriteSegdir( - p, iLevel, iIdx, 0, 0, 0, pWriter->aData, pWriter->nData); + rc = fts3WriteSegdir(p, iLevel, iIdx, + 0, 0, 0, pWriter->nLeafData, pWriter->aData, pWriter->nData); } p->nLeafAdd++; return rc; } @@ -2404,10 +2430,41 @@ if( SQLITE_ROW==sqlite3_step(pStmt) ){ *pnMax = sqlite3_column_int64(pStmt, 0); } return sqlite3_reset(pStmt); } + +/* +** iAbsLevel is an absolute level that may be assumed to exist within +** the database. This function checks if it is the largest level number +** within its index. Assuming no error occurs, *pbMax is set to 1 if +** iAbsLevel is indeed the largest level, or 0 otherwise, and SQLITE_OK +** is returned. If an error occurs, an error code is returned and the +** final value of *pbMax is undefined. +*/ +static int fts3SegmentIsMaxLevel(Fts3Table *p, i64 iAbsLevel, int *pbMax){ + + /* Set pStmt to the compiled version of: + ** + ** SELECT max(level) FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ? + ** + ** (1024 is actually the value of macro FTS3_SEGDIR_PREFIXLEVEL_STR). + */ + sqlite3_stmt *pStmt; + int rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR_MAX_LEVEL, &pStmt, 0); + if( rc!=SQLITE_OK ) return rc; + sqlite3_bind_int64(pStmt, 1, iAbsLevel+1); + sqlite3_bind_int64(pStmt, 2, + ((iAbsLevel/FTS3_SEGDIR_MAXLEVEL)+1) * FTS3_SEGDIR_MAXLEVEL + ); + + *pbMax = 0; + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + *pbMax = sqlite3_column_type(pStmt, 0)==SQLITE_NULL; + } + return sqlite3_reset(pStmt); +} /* ** Delete all entries in the %_segments table associated with the segment ** opened with seg-reader pSeg. This function does not affect the contents ** of the %_segdir table. @@ -2939,10 +2996,144 @@ pCsr->nSegment = 0; pCsr->apSegment = 0; pCsr->aBuffer = 0; } } + +/* +** Decode the "end_block" field, selected by column iCol of the SELECT +** statement passed as the first argument. +** +** The "end_block" field may contain either an integer, or a text field +** containing the text representation of two non-negative integers separated +** by one or more space (0x20) characters. In the first case, set *piEndBlock +** to the integer value and *pnByte to zero before returning. In the second, +** set *piEndBlock to the first value and *pnByte to the second. +*/ +static void fts3ReadEndBlockField( + sqlite3_stmt *pStmt, + int iCol, + i64 *piEndBlock, + i64 *pnByte +){ + const unsigned char *zText = sqlite3_column_text(pStmt, iCol); + if( zText ){ + int i; + int iMul = 1; + i64 iVal = 0; + for(i=0; zText[i]>='0' && zText[i]<='9'; i++){ + iVal = iVal*10 + (zText[i] - '0'); + } + *piEndBlock = iVal; + while( zText[i]==' ' ) i++; + iVal = 0; + if( zText[i]=='-' ){ + i++; + iMul = -1; + } + for(/* no-op */; zText[i]>='0' && zText[i]<='9'; i++){ + iVal = iVal*10 + (zText[i] - '0'); + } + *pnByte = (iVal * (i64)iMul); + } +} + + +/* +** A segment of size nByte bytes has just been written to absolute level +** iAbsLevel. Promote any segments that should be promoted as a result. +*/ +static int fts3PromoteSegments( + Fts3Table *p, /* FTS table handle */ + int iAbsLevel, /* Absolute level just updated */ + sqlite3_int64 nByte /* Size of new segment at iAbsLevel */ +){ + int rc = SQLITE_OK; + sqlite3_stmt *pRange; + + rc = fts3SqlStmt(p, SQL_SELECT_LEVEL_RANGE2, &pRange, 0); + + if( rc==SQLITE_OK ){ + int bOk = 0; + int iLast = (iAbsLevel/FTS3_SEGDIR_MAXLEVEL + 1) * FTS3_SEGDIR_MAXLEVEL - 1; + i64 nLimit = (nByte*3)/2; + + /* Loop through all entries in the %_segdir table corresponding to + ** segments in this index on levels greater than iAbsLevel. If there is + ** at least one such segment, and it is possible to determine that all + ** such segments are smaller than nLimit bytes in size, they will be + ** promoted to level iAbsLevel. */ + sqlite3_bind_int(pRange, 1, iAbsLevel+1); + sqlite3_bind_int(pRange, 2, iLast); + while( SQLITE_ROW==sqlite3_step(pRange) ){ + i64 nSize, dummy; + fts3ReadEndBlockField(pRange, 2, &dummy, &nSize); + if( nSize<=0 || nSize>nLimit ){ + /* If nSize==0, then the %_segdir.end_block field does not not + ** contain a size value. This happens if it was written by an + ** old version of FTS. In this case it is not possible to determine + ** the size of the segment, and so segment promotion does not + ** take place. */ + bOk = 0; + break; + } + bOk = 1; + } + rc = sqlite3_reset(pRange); + + if( bOk ){ + int iIdx = 0; + sqlite3_stmt *pUpdate1; + sqlite3_stmt *pUpdate2; + + if( rc==SQLITE_OK ){ + rc = fts3SqlStmt(p, SQL_UPDATE_LEVEL_IDX, &pUpdate1, 0); + } + if( rc==SQLITE_OK ){ + rc = fts3SqlStmt(p, SQL_UPDATE_LEVEL, &pUpdate2, 0); + } + + if( rc==SQLITE_OK ){ + + /* Loop through all %_segdir entries for segments in this index with + ** levels equal to or greater than iAbsLevel. As each entry is visited, + ** updated it to set (level = -1) and (idx = N), where N is 0 for the + ** oldest segment in the range, 1 for the next oldest, and so on. + ** + ** In other words, move all segments being promoted to level -1, + ** setting the "idx" fields as appropriate to keep them in the same + ** order. The contents of level -1 (which is never used, except + ** transiently here), will be moved back to level iAbsLevel below. */ + sqlite3_bind_int(pRange, 1, iAbsLevel); + while( SQLITE_ROW==sqlite3_step(pRange) ){ + sqlite3_bind_int(pUpdate1, 1, iIdx++); + sqlite3_bind_int(pUpdate1, 2, sqlite3_column_int(pRange, 0)); + sqlite3_bind_int(pUpdate1, 3, sqlite3_column_int(pRange, 1)); + sqlite3_step(pUpdate1); + rc = sqlite3_reset(pUpdate1); + if( rc!=SQLITE_OK ){ + sqlite3_reset(pRange); + break; + } + } + } + if( rc==SQLITE_OK ){ + rc = sqlite3_reset(pRange); + } + + /* Move level -1 to level iAbsLevel */ + if( rc==SQLITE_OK ){ + sqlite3_bind_int(pUpdate2, 1, iAbsLevel); + sqlite3_step(pUpdate2); + rc = sqlite3_reset(pUpdate2); + } + } + } + + + return rc; +} /* ** Merge all level iLevel segments in the database into a single ** iLevel+1 segment. Or, if iLevel<0, merge all segments into a ** single segment with a level equal to the numerically largest level @@ -2964,10 +3155,11 @@ sqlite3_int64 iNewLevel = 0; /* Level/index to create new segment at */ SegmentWriter *pWriter = 0; /* Used to write the new, merged, segment */ Fts3SegFilter filter; /* Segment term filter condition */ Fts3MultiSegReader csr; /* Cursor to iterate through level(s) */ int bIgnoreEmpty = 0; /* True to ignore empty segments */ + i64 iMaxLevel = 0; /* Max level number for this index/langid */ assert( iLevel==FTS3_SEGCURSOR_ALL || iLevel==FTS3_SEGCURSOR_PENDING || iLevel>=0 ); @@ -2974,10 +3166,15 @@ assert( iLevel=0 && iIndexnIndex ); rc = sqlite3Fts3SegReaderCursor(p, iLangid, iIndex, iLevel, 0, 0, 1, 0, &csr); if( rc!=SQLITE_OK || csr.nSegment==0 ) goto finished; + + if( iLevel!=FTS3_SEGCURSOR_PENDING ){ + rc = fts3SegmentMaxLevel(p, iLangid, iIndex, &iMaxLevel); + if( rc!=SQLITE_OK ) goto finished; + } if( iLevel==FTS3_SEGCURSOR_ALL ){ /* This call is to merge all segments in the database to a single ** segment. The level of the new segment is equal to the numerically ** greatest segment level currently present in the database for this @@ -2984,25 +3181,25 @@ ** index. The idx of the new segment is always 0. */ if( csr.nSegment==1 ){ rc = SQLITE_DONE; goto finished; } - rc = fts3SegmentMaxLevel(p, iLangid, iIndex, &iNewLevel); + iNewLevel = iMaxLevel; bIgnoreEmpty = 1; - }else if( iLevel==FTS3_SEGCURSOR_PENDING ){ - iNewLevel = getAbsoluteLevel(p, iLangid, iIndex, 0); - rc = fts3AllocateSegdirIdx(p, iLangid, iIndex, 0, &iIdx); }else{ /* This call is to merge all segments at level iLevel. find the next ** available segment index at level iLevel+1. The call to ** fts3AllocateSegdirIdx() will merge the segments at level iLevel+1 to ** a single iLevel+2 segment if necessary. */ - rc = fts3AllocateSegdirIdx(p, iLangid, iIndex, iLevel+1, &iIdx); + assert( FTS3_SEGCURSOR_PENDING==-1 ); iNewLevel = getAbsoluteLevel(p, iLangid, iIndex, iLevel+1); + rc = fts3AllocateSegdirIdx(p, iLangid, iIndex, iLevel+1, &iIdx); + bIgnoreEmpty = (iLevel!=FTS3_SEGCURSOR_PENDING) && (iNewLevel>iMaxLevel); } if( rc!=SQLITE_OK ) goto finished; + assert( csr.nSegment>0 ); assert( iNewLevel>=getAbsoluteLevel(p, iLangid, iIndex, 0) ); assert( iNewLevelnLeafData); + } + } + } finished: fts3SegWriterFree(pWriter); sqlite3Fts3SegReaderFinish(&csr); return rc; } /* -** Flush the contents of pendingTerms to level 0 segments. +** Flush the contents of pendingTerms to level 0 segments. */ int sqlite3Fts3PendingTermsFlush(Fts3Table *p){ int rc = SQLITE_OK; int i; @@ -3049,18 +3253,23 @@ /* Determine the auto-incr-merge setting if unknown. If enabled, ** estimate the number of leaf blocks of content to be written */ if( rc==SQLITE_OK && p->bHasStat - && p->bAutoincrmerge==0xff && p->nLeafAdd>0 + && p->nAutoincrmerge==0xff && p->nLeafAdd>0 ){ sqlite3_stmt *pStmt = 0; rc = fts3SqlStmt(p, SQL_SELECT_STAT, &pStmt, 0); if( rc==SQLITE_OK ){ sqlite3_bind_int(pStmt, 1, FTS_STAT_AUTOINCRMERGE); rc = sqlite3_step(pStmt); - p->bAutoincrmerge = (rc==SQLITE_ROW && sqlite3_column_int(pStmt, 0)); + if( rc==SQLITE_ROW ){ + p->nAutoincrmerge = sqlite3_column_int(pStmt, 0); + if( p->nAutoincrmerge==1 ) p->nAutoincrmerge = 8; + }else if( rc==SQLITE_DONE ){ + p->nAutoincrmerge = 0; + } rc = sqlite3_reset(pStmt); } } return rc; } @@ -3424,10 +3633,12 @@ int nWork; /* Number of leaf pages flushed */ sqlite3_int64 iAbsLevel; /* Absolute level of input segments */ int iIdx; /* Index of *output* segment in iAbsLevel+1 */ sqlite3_int64 iStart; /* Block number of first allocated block */ sqlite3_int64 iEnd; /* Block number of last allocated block */ + sqlite3_int64 nLeafData; /* Bytes of leaf page data so far */ + u8 bNoLeafData; /* If true, store 0 for segment size */ NodeWriter aNodeWriter[FTS_MAX_APPENDABLE_HEIGHT]; }; /* ** An object of the following type is used to read data from a single @@ -3762,12 +3973,12 @@ nSpace = 1; nSpace += sqlite3Fts3VarintLen(nSuffix) + nSuffix; nSpace += sqlite3Fts3VarintLen(nDoclist) + nDoclist; } + pWriter->nLeafData += nSpace; blobGrowBuffer(&pLeaf->block, pLeaf->block.n + nSpace, &rc); - if( rc==SQLITE_OK ){ if( pLeaf->block.n==0 ){ pLeaf->block.n = 1; pLeaf->block.a[0] = '\0'; } @@ -3862,10 +4073,11 @@ pWriter->iAbsLevel+1, /* level */ pWriter->iIdx, /* idx */ pWriter->iStart, /* start_block */ pWriter->aNodeWriter[0].iBlock, /* leaves_end_block */ pWriter->iEnd, /* end_block */ + (pWriter->bNoLeafData==0 ? pWriter->nLeafData : 0), /* end_block */ pRoot->block.a, pRoot->block.n /* root */ ); } sqlite3_free(pRoot->block.a); sqlite3_free(pRoot->key.a); @@ -3963,11 +4175,15 @@ sqlite3_bind_int64(pSelect, 1, iAbsLevel+1); sqlite3_bind_int(pSelect, 2, iIdx); if( sqlite3_step(pSelect)==SQLITE_ROW ){ iStart = sqlite3_column_int64(pSelect, 1); iLeafEnd = sqlite3_column_int64(pSelect, 2); - iEnd = sqlite3_column_int64(pSelect, 3); + fts3ReadEndBlockField(pSelect, 3, &iEnd, &pWriter->nLeafData); + if( pWriter->nLeafData<0 ){ + pWriter->nLeafData = pWriter->nLeafData * -1; + } + pWriter->bNoLeafData = (pWriter->nLeafData==0); nRoot = sqlite3_column_bytes(pSelect, 4); aRoot = sqlite3_column_blob(pSelect, 4); }else{ return sqlite3_reset(pSelect); } @@ -4564,15 +4780,15 @@ /* ** Attempt an incremental merge that writes nMerge leaf blocks. ** -** Incremental merges happen nMin segments at a time. The two -** segments to be merged are the nMin oldest segments (the ones with -** the smallest indexes) in the highest level that contains at least -** nMin segments. Multiple merges might occur in an attempt to write the -** quota of nMerge leaf blocks. +** Incremental merges happen nMin segments at a time. The segments +** to be merged are the nMin oldest segments (the ones with the smallest +** values for the _segdir.idx field) in the highest level that contains +** at least nMin segments. Multiple merges might occur in an attempt to +** write the quota of nMerge leaf blocks. */ int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ int rc; /* Return code */ int nRem = nMerge; /* Number of leaf pages yet to be written */ Fts3MultiSegReader *pCsr; /* Cursor used to read input data */ @@ -4593,10 +4809,11 @@ rc = fts3IncrmergeHintLoad(p, &hint); while( rc==SQLITE_OK && nRem>0 ){ const i64 nMod = FTS3_SEGDIR_MAXLEVEL * p->nIndex; sqlite3_stmt *pFindLevel = 0; /* SQL used to determine iAbsLevel */ int bUseHint = 0; /* True if attempting to append */ + int iIdx = 0; /* Largest idx in level (iAbsLevel+1) */ /* Search the %_segdir table for the absolute level with the smallest ** relative level number that contains at least nMin segments, if any. ** If one is found, set iAbsLevel to the absolute level number and ** nSeg to nMin. If no level with at least nMin segments can be found, @@ -4646,27 +4863,36 @@ ** segments available in level iAbsLevel. In this case, no work is ** done on iAbsLevel - fall through to the next iteration of the loop ** to start work on some other level. */ memset(pWriter, 0, nAlloc); pFilter->flags = FTS3_SEGMENT_REQUIRE_POS; + + if( rc==SQLITE_OK ){ + rc = fts3IncrmergeOutputIdx(p, iAbsLevel, &iIdx); + assert( bUseHint==1 || bUseHint==0 ); + if( iIdx==0 || (bUseHint && iIdx==1) ){ + int bIgnore; + rc = fts3SegmentIsMaxLevel(p, iAbsLevel+1, &bIgnore); + if( bIgnore ){ + pFilter->flags |= FTS3_SEGMENT_IGNORE_EMPTY; + } + } + } + if( rc==SQLITE_OK ){ rc = fts3IncrmergeCsr(p, iAbsLevel, nSeg, pCsr); } if( SQLITE_OK==rc && pCsr->nSegment==nSeg && SQLITE_OK==(rc = sqlite3Fts3SegReaderStart(p, pCsr, pFilter)) && SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, pCsr)) ){ - int iIdx = 0; /* Largest idx in level (iAbsLevel+1) */ - rc = fts3IncrmergeOutputIdx(p, iAbsLevel, &iIdx); - if( rc==SQLITE_OK ){ - if( bUseHint && iIdx>0 ){ - const char *zKey = pCsr->zTerm; - int nKey = pCsr->nTerm; - rc = fts3IncrmergeLoad(p, iAbsLevel, iIdx-1, zKey, nKey, pWriter); - }else{ - rc = fts3IncrmergeWriter(p, iAbsLevel, iIdx, pCsr, pWriter); - } + if( bUseHint && iIdx>0 ){ + const char *zKey = pCsr->zTerm; + int nKey = pCsr->nTerm; + rc = fts3IncrmergeLoad(p, iAbsLevel, iIdx-1, zKey, nKey, pWriter); + }else{ + rc = fts3IncrmergeWriter(p, iAbsLevel, iIdx, pCsr, pWriter); } if( rc==SQLITE_OK && pWriter->nLeafEst ){ fts3LogMerge(nSeg, iAbsLevel); do { @@ -4684,11 +4910,17 @@ fts3IncrmergeHintPush(&hint, iAbsLevel, nSeg, &rc); } } } + if( nSeg!=0 ){ + pWriter->nLeafData = pWriter->nLeafData * -1; + } fts3IncrmergeRelease(p, pWriter, &rc); + if( nSeg==0 && pWriter->bNoLeafData==0 ){ + fts3PromoteSegments(p, iAbsLevel+1, pWriter->nLeafData); + } } sqlite3Fts3SegReaderFinish(pCsr); } @@ -4771,20 +5003,23 @@ Fts3Table *p, /* FTS3 table handle */ const char *zParam /* Nul-terminated string containing boolean */ ){ int rc = SQLITE_OK; sqlite3_stmt *pStmt = 0; - p->bAutoincrmerge = fts3Getint(&zParam)!=0; + p->nAutoincrmerge = fts3Getint(&zParam); + if( p->nAutoincrmerge==1 || p->nAutoincrmerge>FTS3_MERGE_COUNT ){ + p->nAutoincrmerge = 8; + } if( !p->bHasStat ){ assert( p->bFts4==0 ); sqlite3Fts3CreateStatTable(&rc, p); if( rc ) return rc; } rc = fts3SqlStmt(p, SQL_REPLACE_STAT, &pStmt, 0); if( rc ) return rc; sqlite3_bind_int(pStmt, 1, FTS_STAT_AUTOINCRMERGE); - sqlite3_bind_int(pStmt, 2, p->bAutoincrmerge); + sqlite3_bind_int(pStmt, 2, p->nAutoincrmerge); sqlite3_step(pStmt); rc = sqlite3_reset(pStmt); return rc; } Index: test/backcompat.test ================================================================== --- test/backcompat.test +++ test/backcompat.test @@ -55,17 +55,29 @@ proc sql1 sql { code1 [list db eval $sql] } proc sql2 sql { code2 [list db eval $sql] } code1 { sqlite3 db test.db } code2 { sqlite3 db test.db } + + foreach c {code1 code2} { + $c { + set v [split [db version] .] + if {[llength $v]==3} {lappend v 0} + set ::sqlite_libversion [format \ + "%d%.2d%.2d%2d" [lindex $v 0] [lindex $v 1] [lindex $v 2] [lindex $v 3] + ] + } + } uplevel $script catch { code1 { db close } } catch { code2 { db close } } catch { close $::bc_chan2 } catch { close $::bc_chan1 } + + } array set ::incompatible [list] proc do_allbackcompat_test {script} { @@ -379,10 +391,52 @@ 7 "SELECT offsets(t1) FROM t1 WHERE t1 MATCH '44'" 8 "SELECT offsets(t1) FROM t1 WHERE t1 MATCH 'a*'" } { do_test backcompat-3.7 [list sql1 $q] [sql2 $q] } + + # Now test that an incremental merge can be started by one version + # and finished by another. And that the integrity-check still + # passes. + do_test backcompat-3.8 { + sql1 { + DROP TABLE IF EXISTS t1; + DROP TABLE IF EXISTS t2; + CREATE TABLE t1(docid, words); + CREATE VIRTUAL TABLE t2 USING fts3(words); + } + code1 [list source $testdir/genesis.tcl] + code1 { fts_kjv_genesis } + sql1 { + INSERT INTO t2 SELECT words FROM t1; + INSERT INTO t2 SELECT words FROM t1; + INSERT INTO t2 SELECT words FROM t1; + INSERT INTO t2 SELECT words FROM t1; + INSERT INTO t2 SELECT words FROM t1; + INSERT INTO t2 SELECT words FROM t1; + SELECT level, group_concat(idx, ' ') FROM t2_segdir GROUP BY level; + } + } {0 {0 1 2 3 4 5}} + + if {[code1 { set ::sqlite_libversion }] >=3071200 + && [code2 { set ::sqlite_libversion }] >=3071200 + } { + do_test backcompat-3.9 { + sql1 { INSERT INTO t2(t2) VALUES('merge=100,4'); } + sql2 { INSERT INTO t2(t2) VALUES('merge=100,4'); } + sql1 { INSERT INTO t2(t2) VALUES('merge=100,4'); } + sql2 { INSERT INTO t2(t2) VALUES('merge=2500,4'); } + sql2 { + SELECT level, group_concat(idx, ' ') FROM t2_segdir GROUP BY level; + } + } {0 {0 1} 1 0} + + do_test backcompat-3.10 { + sql1 { INSERT INTO t2(t2) VALUES('integrity-check') } + sql2 { INSERT INTO t2(t2) VALUES('integrity-check') } + } {} + } } } } #------------------------------------------------------------------------- Index: test/fts3d.test ================================================================== --- test/fts3d.test +++ test/fts3d.test @@ -211,20 +211,21 @@ } } [list {0 0 0 4 0 4 5 2 0 3 8 1 0 5 10 4} \ {0 1 0 4 0 2 5 3 0 3 9 1 0 5 11 4} \ {0 0 0 4 0 4 5 2 0 3 8 1 0 5 10 4}] -check_terms_all fts3d-4.1 {a four is one test that this three two was} +puts [db eval {SELECT c FROM t1 } ] +check_terms_all fts3d-4.1 {a four is test that this was} check_doclist_all fts3d-4.1.1 a {[1 0[2]] [2 0[2]] [3 0[2]]} check_doclist_all fts3d-4.1.2 four {} check_doclist_all fts3d-4.1.3 is {[1 0[1]] [3 0[1]]} -check_doclist_all fts3d-4.1.4 one {} +#check_doclist_all fts3d-4.1.4 one {} check_doclist_all fts3d-4.1.5 test {[1 0[3]] [2 0[3]] [3 0[3]]} check_doclist_all fts3d-4.1.6 that {[2 0[0]]} check_doclist_all fts3d-4.1.7 this {[1 0[0]] [3 0[0]]} -check_doclist_all fts3d-4.1.8 three {} -check_doclist_all fts3d-4.1.9 two {} +#check_doclist_all fts3d-4.1.8 three {} +#check_doclist_all fts3d-4.1.9 two {} check_doclist_all fts3d-4.1.10 was {[2 0[1]]} check_terms fts3d-4.2 0 0 {a four test that was} check_doclist fts3d-4.2.1 0 0 a {[2 0[2]]} check_doclist fts3d-4.2.2 0 0 four {[2]} @@ -237,20 +238,20 @@ check_doclist fts3d-4.3.2 0 1 four {[3]} check_doclist fts3d-4.3.3 0 1 is {[3 0[1]]} check_doclist fts3d-4.3.4 0 1 test {[3 0[3]]} check_doclist fts3d-4.3.5 0 1 this {[3 0[0]]} -check_terms fts3d-4.4 1 0 {a four is one test that this three two was} +check_terms fts3d-4.4 1 0 {a four is test that this was} check_doclist fts3d-4.4.1 1 0 a {[1 0[2]] [2 0[2]] [3 0[2]]} -check_doclist fts3d-4.4.2 1 0 four {[1] [2 0[4]] [3 0[4]]} +check_doclist fts3d-4.4.2 1 0 four {[2 0[4]] [3 0[4]]} check_doclist fts3d-4.4.3 1 0 is {[1 0[1]] [3 0[1]]} -check_doclist fts3d-4.4.4 1 0 one {[1] [2] [3]} +#check_doclist fts3d-4.4.4 1 0 one {[1] [2] [3]} check_doclist fts3d-4.4.5 1 0 test {[1 0[3]] [2 0[3]] [3 0[3]]} check_doclist fts3d-4.4.6 1 0 that {[2 0[0]]} check_doclist fts3d-4.4.7 1 0 this {[1 0[0]] [3 0[0]]} -check_doclist fts3d-4.4.8 1 0 three {[1] [2] [3]} -check_doclist fts3d-4.4.9 1 0 two {[1] [2] [3]} +#check_doclist fts3d-4.4.8 1 0 three {[1] [2] [3]} +#check_doclist fts3d-4.4.9 1 0 two {[1] [2] [3]} check_doclist fts3d-4.4.10 1 0 was {[2 0[1]]} # Optimize should leave the result in the level of the highest-level # prior segment. do_test fts3d-4.5 { ADDED test/fts4growth.test Index: test/fts4growth.test ================================================================== --- /dev/null +++ test/fts4growth.test @@ -0,0 +1,437 @@ +# 2014 May 12 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The +# focus of this script is testing the FTS4 module. +# +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix fts4growth + +# If SQLITE_ENABLE_FTS3 is defined, omit this file. +ifcapable !fts3 { + finish_test + return +} + +source $testdir/genesis.tcl + +do_execsql_test 1.1 { CREATE VIRTUAL TABLE x1 USING fts3; } + +do_test 1.2 { + foreach L { + {"See here, young man," said Mulga Bill, "from Walgett to the sea,} + {From Conroy's Gap to Castlereagh, there's none can ride like me.} + {I'm good all round at everything as everybody knows,} + {Although I'm not the one to talk -- I hate a man that blows.} + } { + execsql { INSERT INTO x1 VALUES($L) } + } + execsql { SELECT end_block, length(root) FROM x1_segdir } +} {{0 114} 114 {0 118} 118 {0 95} 95 {0 115} 115} + +do_execsql_test 1.3 { + INSERT INTO x1(x1) VALUES('optimize'); + SELECT level, end_block, length(root) FROM x1_segdir; +} {0 {0 394} 394} + +do_test 1.4 { + foreach L { + {But riding is my special gift, my chiefest, sole delight;} + {Just ask a wild duck can it swim, a wildcat can it fight.} + {There's nothing clothed in hair or hide, or built of flesh or steel,} + {There's nothing walks or jumps, or runs, on axle, hoof, or wheel,} + {But what I'll sit, while hide will hold and girths and straps are tight:} + {I'll ride this here two-wheeled concern right straight away at sight."} + } { + execsql { INSERT INTO x1 VALUES($L) } + } + execsql { + INSERT INTO x1(x1) VALUES('merge=4,4'); + SELECT level, end_block, length(root) FROM x1_segdir; + } +} {0 {0 110} 110 0 {0 132} 132 0 {0 129} 129 1 {128 658} 2} + +do_execsql_test 1.5 { + SELECT length(block) FROM x1_segments; +} {658 {}} + +do_test 1.6 { + foreach L { + {'Twas Mulga Bill, from Eaglehawk, that sought his own abode,} + {That perched above Dead Man's Creek, beside the mountain road.} + {He turned the cycle down the hill and mounted for the fray,} + {But 'ere he'd gone a dozen yards it bolted clean away.} + {It left the track, and through the trees, just like a silver steak,} + {It whistled down the awful slope towards the Dead Man's Creek.} + {It shaved a stump by half an inch, it dodged a big white-box:} + {The very wallaroos in fright went scrambling up the rocks,} + {The wombats hiding in their caves dug deeper underground,} + {As Mulga Bill, as white as chalk, sat tight to every bound.} + {It struck a stone and gave a spring that cleared a fallen tree,} + {It raced beside a precipice as close as close could be;} + {And then as Mulga Bill let out one last despairing shriek} + {It made a leap of twenty feet into the Dead Man's Creek.} + } { + execsql { INSERT INTO x1 VALUES($L) } + } + execsql { + SELECT level, end_block, length(root) FROM x1_segdir; + } +} {1 {128 658} 2 1 {130 1377} 6 0 {0 117} 117} + +do_execsql_test 1.7 { + SELECT sum(length(block)) FROM x1_segments WHERE blockid IN (129, 130); +} {1377} + +#------------------------------------------------------------------------- +# +do_execsql_test 2.1 { + CREATE TABLE t1(docid, words); + CREATE VIRTUAL TABLE x2 USING fts4; +} +fts_kjv_genesis +do_test 2.2 { + foreach id [db eval {SELECT docid FROM t1}] { + execsql { + INSERT INTO x2(docid, content) SELECT $id, words FROM t1 WHERE docid=$id + } + } + foreach id [db eval {SELECT docid FROM t1}] { + execsql { + INSERT INTO x2(docid, content) SELECT NULL, words FROM t1 WHERE docid=$id + } + if {[db one {SELECT count(*) FROM x2_segdir WHERE level<2}]==2} break + } +} {} + +do_execsql_test 2.3 { + SELECT count(*) FROM x2_segdir WHERE level=2; + SELECT count(*) FROM x2_segdir WHERE level=3; +} {6 0} + +do_execsql_test 2.4 { + INSERT INTO x2(x2) VALUES('merge=4,4'); + SELECT count(*) FROM x2_segdir WHERE level=2; + SELECT count(*) FROM x2_segdir WHERE level=3; +} {6 1} + +do_execsql_test 2.5 { + SELECT end_block FROM x2_segdir WHERE level=3; + INSERT INTO x2(x2) VALUES('merge=4,4'); + SELECT end_block FROM x2_segdir WHERE level=3; + INSERT INTO x2(x2) VALUES('merge=4,4'); + SELECT end_block FROM x2_segdir WHERE level=3; +} {{3828 -3430} {3828 -10191} {3828 -14109}} + +do_execsql_test 2.6 { + SELECT sum(length(block)) FROM x2_segdir, x2_segments WHERE + blockid BETWEEN start_block AND leaves_end_block + AND level=3 +} {14109} + +do_execsql_test 2.7 { + INSERT INTO x2(x2) VALUES('merge=1000,4'); + SELECT end_block FROM x2_segdir WHERE level=3; +} {{3828 86120}} + +do_execsql_test 2.8 { + SELECT sum(length(block)) FROM x2_segdir, x2_segments WHERE + blockid BETWEEN start_block AND leaves_end_block + AND level=3 +} {86120} + +#-------------------------------------------------------------------------- +# Test that delete markers are removed from FTS segments when possible. +# It is only possible to remove delete markers when the output of the +# merge operation will become the oldest segment in the index. +# +# 3.1 - when the oldest segment is created by an 'optimize'. +# 3.2 - when the oldest segment is created by an incremental merge. +# 3.3 - by a crisis merge. +# + +proc insert_doc {args} { + foreach iDoc $args { + set L [lindex { + {In your eagerness to engage the Trojans,} + {don’t any of you charge ahead of others,} + {trusting in your strength and horsemanship.} + {And don’t lag behind. That will hurt our charge.} + {Any man whose chariot confronts an enemy’s} + {should thrust with his spear at him from there.} + {That’s the most effective tactic, the way} + {men wiped out city strongholds long ago —} + {their chests full of that style and spirit.} + } [expr $iDoc%9]] + execsql { REPLACE INTO x3(docid, content) VALUES($iDoc, $L) } + } +} + +proc delete_doc {args} { + foreach iDoc $args { + execsql { DELETE FROM x3 WHERE docid = $iDoc } + } +} + +proc second {x} { lindex $x 1 } +db func second second + +do_execsql_test 3.0 { CREATE VIRTUAL TABLE x3 USING fts4 } + +do_test 3.1.1 { + db transaction { insert_doc 1 2 3 4 5 6 } + execsql { SELECT level, idx, second(end_block) FROM x3_segdir } +} {0 0 412} +do_test 3.1.2 { + delete_doc 1 2 3 4 5 6 + execsql { SELECT count(*) FROM x3_segdir } +} {0} +do_test 3.1.3 { + db transaction { + insert_doc 1 2 3 4 5 6 7 8 9 + delete_doc 9 8 7 + } + execsql { SELECT level, idx, second(end_block) FROM x3_segdir } +} {0 0 591 0 1 65 0 2 72 0 3 76} +do_test 3.1.4 { + execsql { INSERT INTO x3(x3) VALUES('optimize') } + execsql { SELECT level, idx, second(end_block) FROM x3_segdir } +} {0 0 412} + +do_test 3.2.1 { + execsql { DELETE FROM x3 } + insert_doc 8 7 6 5 4 3 2 1 + delete_doc 7 8 + execsql { SELECT count(*) FROM x3_segdir } +} {10} +do_test 3.2.2 { + execsql { INSERT INTO x3(x3) VALUES('merge=500,10') } + execsql { SELECT level, idx, second(end_block) FROM x3_segdir } +} {1 0 412} + +# This assumes the crisis merge happens when there are already 16 +# segments and one more is added. +# +do_test 3.3.1 { + execsql { DELETE FROM x3 } + insert_doc 1 2 3 4 5 6 7 8 9 10 11 + delete_doc 11 10 9 8 7 + execsql { SELECT count(*) FROM x3_segdir } +} {16} + +do_test 3.3.2 { + insert_doc 12 + execsql { SELECT level, idx, second(end_block) FROM x3_segdir WHERE level=1 } +} {1 0 412} + +#-------------------------------------------------------------------------- +# Check a theory on a bug in fts4 - that segments with idx==0 were not +# being incrementally merged correctly. Theory turned out to be false. +# +do_execsql_test 4.1 { + DROP TABLE IF EXISTS x4; + DROP TABLE IF EXISTS t1; + CREATE TABLE t1(docid, words); + CREATE VIRTUAL TABLE x4 USING fts4(words); +} +do_test 4.2 { + fts_kjv_genesis + execsql { INSERT INTO x4 SELECT words FROM t1 } + execsql { INSERT INTO x4 SELECT words FROM t1 } +} {} + +do_execsql_test 4.3 { + SELECT level, idx, second(end_block) FROM x4_segdir +} {0 0 117483 0 1 118006} + +do_execsql_test 4.4 { + INSERT INTO x4(x4) VALUES('merge=10,2'); + SELECT count(*) FROM x4_segdir; +} {3} + +do_execsql_test 4.5 { + INSERT INTO x4(x4) VALUES('merge=10,2'); + SELECT count(*) FROM x4_segdir; +} {3} + +do_execsql_test 4.6 { + INSERT INTO x4(x4) VALUES('merge=1000,2'); + SELECT count(*) FROM x4_segdir; +} {1} + + + +#-------------------------------------------------------------------------- +# Check that segments are not promoted if the "end_block" field does not +# contain a size. +# +do_execsql_test 5.1 { + DROP TABLE IF EXISTS x2; + DROP TABLE IF EXISTS t1; + CREATE TABLE t1(docid, words); + CREATE VIRTUAL TABLE x2 USING fts4; +} +fts_kjv_genesis + +proc first {L} {lindex $L 0} +db func first first + +do_test 5.2 { + foreach r [db eval { SELECT rowid FROM t1 }] { + execsql { + INSERT INTO x2(docid, content) SELECT docid, words FROM t1 WHERE rowid=$r + } + } + foreach d [db eval { SELECT docid FROM t1 LIMIT -1 OFFSET 20 }] { + execsql { DELETE FROM x2 WHERE docid = $d } + } + + execsql { + INSERT INTO x2(x2) VALUES('optimize'); + SELECT level, idx, end_block FROM x2_segdir + } +} {2 0 {752 1926}} + +do_execsql_test 5.3 { + UPDATE x2_segdir SET end_block = CAST( first(end_block) AS INTEGER ); + SELECT end_block, typeof(end_block) FROM x2_segdir; +} {752 integer} + +do_execsql_test 5.4 { + INSERT INTO x2 SELECT words FROM t1 LIMIT 50; + SELECT level, idx, end_block FROM x2_segdir +} {2 0 752 0 0 {758 5174}} + +do_execsql_test 5.5 { + UPDATE x2_segdir SET end_block = end_block || ' 1926' WHERE level=2; + INSERT INTO x2 SELECT words FROM t1 LIMIT 40; + SELECT level, idx, end_block FROM x2_segdir +} {0 0 {752 1926} 0 1 {758 5174} 0 2 {763 4170}} + +proc t1_to_x2 {} { + foreach id [db eval {SELECT docid FROM t1 LIMIT 2}] { + execsql { + DELETE FROM x2 WHERE docid=$id; + INSERT INTO x2(docid, content) SELECT $id, words FROM t1 WHERE docid=$id; + } + } +} + +#-------------------------------------------------------------------------- +# Check that segments created by auto-merge are not promoted until they +# are completed. +# + +do_execsql_test 6.1 { + CREATE VIRTUAL TABLE x5 USING fts4; + INSERT INTO x5 SELECT words FROM t1 LIMIT 100 OFFSET 0; + INSERT INTO x5 SELECT words FROM t1 LIMIT 100 OFFSET 25; + INSERT INTO x5 SELECT words FROM t1 LIMIT 100 OFFSET 50; + INSERT INTO x5 SELECT words FROM t1 LIMIT 100 OFFSET 75; + SELECT count(*) FROM x5_segdir +} {4} + +do_execsql_test 6.2 { + INSERT INTO x5(x5) VALUES('merge=2,4'); + SELECT level, idx, end_block FROM x5_segdir; +} {0 0 {10 9216} 0 1 {21 9330} 0 2 {31 8850} 0 3 {40 8689} 1 0 {1320 -3117}} + +do_execsql_test 6.3 { + INSERT INTO x5 SELECT words FROM t1 LIMIT 100 OFFSET 100; + SELECT level, idx, end_block FROM x5_segdir; +} { + 0 0 {10 9216} 0 1 {21 9330} 0 2 {31 8850} + 0 3 {40 8689} 1 0 {1320 -3117} 0 4 {1329 8297} +} + +do_execsql_test 6.4 { + INSERT INTO x5(x5) VALUES('merge=200,4'); + SELECT level, idx, end_block FROM x5_segdir; +} {0 0 {1329 8297} 1 0 {1320 28009}} + +do_execsql_test 6.5 { + INSERT INTO x5 SELECT words FROM t1; + SELECT level, idx, end_block FROM x5_segdir; +} { + 0 1 {1329 8297} 0 0 {1320 28009} 0 2 {1449 118006} +} + +#-------------------------------------------------------------------------- +# Ensure that if part of an incremental merge is performed by an old +# version that does not support storing segment sizes in the end_block +# field, no size is stored in the final segment (as it would be incorrect). +# +do_execsql_test 7.1 { + CREATE VIRTUAL TABLE x6 USING fts4; + INSERT INTO x6 SELECT words FROM t1; + INSERT INTO x6 SELECT words FROM t1; + INSERT INTO x6 SELECT words FROM t1; + INSERT INTO x6 SELECT words FROM t1; + INSERT INTO x6 SELECT words FROM t1; + INSERT INTO x6 SELECT words FROM t1; + SELECT level, idx, end_block FROM x6_segdir; +} { + 0 0 {118 117483} 0 1 {238 118006} 0 2 {358 118006} + 0 3 {478 118006} 0 4 {598 118006} 0 5 {718 118006} +} + +do_execsql_test 7.2 { + INSERT INTO x6(x6) VALUES('merge=25,4'); + SELECT level, idx, end_block FROM x6_segdir; +} { + 0 0 {118 117483} 0 1 {238 118006} 0 2 {358 118006} + 0 3 {478 118006} 0 4 {598 118006} 0 5 {718 118006} + 1 0 {16014 -51226} +} + +do_execsql_test 7.3 { + UPDATE x6_segdir SET end_block = first(end_block) WHERE level=1; + SELECT level, idx, end_block FROM x6_segdir; +} { + 0 0 {118 117483} 0 1 {238 118006} 0 2 {358 118006} + 0 3 {478 118006} 0 4 {598 118006} 0 5 {718 118006} + 1 0 16014 +} + +do_execsql_test 7.4 { + INSERT INTO x6(x6) VALUES('merge=25,4'); + SELECT level, idx, end_block FROM x6_segdir; +} { + 0 0 {118 117483} 0 1 {238 118006} 0 2 {358 118006} + 0 3 {478 118006} 0 4 {598 118006} 0 5 {718 118006} + 1 0 16014 +} + +do_execsql_test 7.5 { + INSERT INTO x6(x6) VALUES('merge=2500,4'); + SELECT level, idx, end_block FROM x6_segdir; +} { + 0 0 {598 118006} 0 1 {718 118006} 1 0 16014 +} + +do_execsql_test 7.6 { + INSERT INTO x6(x6) VALUES('merge=2500,2'); + SELECT level, idx, start_block, leaves_end_block, end_block FROM x6_segdir; +} { + 2 0 23695 24147 {41262 633507} +} + +do_execsql_test 7.7 { + SELECT sum(length(block)) FROM x6_segments + WHERE blockid BETWEEN 23695 AND 24147 +} {633507} + + + +finish_test + ADDED test/fts4growth2.test Index: test/fts4growth2.test ================================================================== --- /dev/null +++ test/fts4growth2.test @@ -0,0 +1,88 @@ +# 2014 May 12 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The +# focus of this script is testing the FTS4 module. +# +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix fts4growth2 + +# If SQLITE_ENABLE_FTS3 is defined, omit this file. +ifcapable !fts3 { + finish_test + return +} + +source $testdir/genesis.tcl + +do_execsql_test 1.0 { CREATE TABLE t1(docid, words); } +fts_kjv_genesis + +proc tt {val} { + execsql { + DELETE FROM x1 + WHERE docid IN (SELECT docid FROM t1 WHERE (rowid-1)%4==$val+0); + INSERT INTO x1(docid, content) + SELECT docid, words FROM t1 WHERE (rowid%4)==$val+0; + } + #puts [db eval {SELECT level, idx, end_block FROM x1_segdir}] +} + +do_execsql_test 1.1 { + CREATE VIRTUAL TABLE x1 USING fts4; + INSERT INTO x1(x1) VALUES('automerge=2'); +} + +do_test 1.2 { + for {set i 0} {$i < 40} {incr i} { + tt 0 ; tt 1 ; tt 2 ; tt 3 + } + execsql { + SELECT max(level) FROM x1_segdir; + SELECT count(*) FROM x1_segdir WHERE level=3; + } +} {3 1} + +do_test 1.3 { + for {set i 0} {$i < 40} {incr i} { + tt 0 ; tt 1 ; tt 2 ; tt 3 + } + execsql { + SELECT max(level) FROM x1_segdir; + SELECT count(*) FROM x1_segdir WHERE level=2; + } +} {2 1} + +#------------------------------------------------------------------------- +# +do_execsql_test 2.1 { + DELETE FROM t1 WHERE rowid>16; + DROP TABLE IF EXISTS x1; + CREATE VIRTUAL TABLE x1 USING fts4; +} + +db func second second +proc second {L} {lindex $L 1} + +for {set tn 0} {$tn < 40} {incr tn} { + do_test 2.2.$tn { + for {set i 0} {$i < 100} {incr i} { + tt 0 ; tt 1 ; tt 2 ; tt 3 + } + execsql { SELECT max(level) FROM x1_segdir } + } {1} +} + + +finish_test + Index: test/fts4merge4.test ================================================================== --- test/fts4merge4.test +++ test/fts4merge4.test @@ -51,8 +51,52 @@ } {} do_execsql_test 2.2 { SELECT count(*) FROM t1_segdir; } 35 do_execsql_test 2.3 { INSERT INTO t1(t1) VALUES('optimize') } {} do_execsql_test 2.4 { SELECT count(*) FROM t1_segdir; } 1 +#------------------------------------------------------------------------- +# Now test that the automerge=? option appears to work. +# +do_execsql_test 2.1 { CREATE VIRTUAL TABLE t2 USING fts4; } + +set doc "" +foreach c1 "a b c d e f g h i j" { + foreach c2 "a b c d e f g h i j" { + foreach c3 "a b c d e f g h i j" { + lappend doc "$c1$c2$c3" + } + } +} +set doc [string repeat $doc 10] + +foreach {tn am expected} { + 1 {automerge=2} {1 1 2 1 4 1 6 1} + 2 {automerge=4} {1 2 2 1 3 1} + 3 {automerge=8} {0 4 1 3 2 1} + 4 {automerge=1} {0 4 1 3 2 1} +} { + foreach {tn2 openclose} {1 {} 2 { db close ; sqlite3 db test.db }} { + do_test 2.2.$tn.$tn2 { + execsql { DELETE FROM t2 } + execsql { INSERT INTO t2(t2) VALUES($am) }; + + eval $openclose + + for {set i 0} {$i < 100} {incr i} { + execsql { + BEGIN; + INSERT INTO t2 VALUES($doc); + INSERT INTO t2 VALUES($doc); + INSERT INTO t2 VALUES($doc); + INSERT INTO t2 VALUES($doc); + INSERT INTO t2 VALUES($doc); + COMMIT; + } + } + + execsql { SELECT level, count(*) FROM t2_segdir GROUP BY level } + } [list {*}$expected] + } +} sqlite3_enable_shared_cache $::enable_shared_cache finish_test Index: test/permutations.test ================================================================== --- test/permutations.test +++ test/permutations.test @@ -110,10 +110,11 @@ speed4p.test sqllimits1.test tkt2686.test thread001.test thread002.test thread003.test thread004.test thread005.test trans2.test vacuum3.test incrvacuum_ioerr.test autovacuum_crash.test btree8.test shared_err.test vtab_err.test walslow.test walcrash.test walcrash3.test walthread.test rtree3.test indexfault.test securedel2.test + fts4growth.test fts4growth2.test }] if {[info exists ::env(QUICKTEST_INCLUDE)]} { set allquicktests [concat $allquicktests $::env(QUICKTEST_INCLUDE)] } @@ -194,10 +195,11 @@ fts4aa.test fts4content.test fts3conf.test fts3prefix.test fts3fault2.test fts3corrupt.test fts3corrupt2.test fts3first.test fts4langid.test fts4merge.test fts4check.test fts4unicode.test fts4noti.test fts3varint.test + fts4growth.test fts4growth2.test } test_suite "nofaultsim" -prefix "" -description { "Very" quick test suite. Runs in less than 5 minutes on a workstation. This test suite is the same as the "quick" tests, except that some files Index: test/trace2.test ================================================================== --- test/trace2.test +++ test/trace2.test @@ -134,10 +134,11 @@ "-- SELECT value FROM 'main'.'x1_stat' WHERE id=?" "-- REPLACE INTO 'main'.'x1_stat' VALUES(?,?)" "-- SELECT (SELECT max(idx) FROM 'main'.'x1_segdir' WHERE level = ?) + 1" "-- SELECT coalesce((SELECT max(blockid) FROM 'main'.'x1_segments') + 1, 1)" "-- REPLACE INTO 'main'.'x1_segdir' VALUES(?,?,?,?,?,?)" + "-- SELECT level, idx, end_block FROM 'main'.'x1_segdir' WHERE level BETWEEN ? AND ? ORDER BY level DESC, idx ASC" } do_trace_test 2.3 { INSERT INTO x1(x1) VALUES('optimize'); } {