Index: Makefile.msc ================================================================== --- Makefile.msc +++ Makefile.msc @@ -74,11 +74,11 @@ # cross-compiling, it is suggested that this macro be modified via the command # line (since nmake itself does not provide a built-in method to guess it). # For example, to use the x86 compiler when cross-compiling for x64, a command # line similar to the following could be used (all on one line): # -# nmake /f Makefile.msc +# nmake /f Makefile.msc sqlite3.dll # "NCC=""%VCINSTALLDIR%\bin\cl.exe""" # USE_NATIVE_LIBPATHS=1 # !IFDEF NCC NCC = $(NCC:\\=\) Index: ext/fts3/fts3.c ================================================================== --- ext/fts3/fts3.c +++ ext/fts3/fts3.c @@ -4434,10 +4434,11 @@ /* Allocate a MultiSegReader for each token in the expression. */ fts3EvalAllocateReaders(pCsr, pCsr->pExpr, &nToken, &nOr, &rc); /* Determine which, if any, tokens in the expression should be deferred. */ +#ifndef SQLITE_DISABLE_FTS4_DEFERRED if( rc==SQLITE_OK && nToken>1 && pTab->bFts4 ){ Fts3TokenAndCost *aTC; Fts3Expr **apOr; aTC = (Fts3TokenAndCost *)sqlite3_malloc( sizeof(Fts3TokenAndCost) * nToken @@ -4464,10 +4465,11 @@ } sqlite3_free(aTC); } } +#endif fts3EvalStartReaders(pCsr, pCsr->pExpr, 1, &rc); return rc; } @@ -4847,10 +4849,11 @@ && !fts3EvalTestExpr(pCsr, pExpr->pRight, pRc) ); break; default: { +#ifndef SQLITE_DISABLE_FTS4_DEFERRED if( pCsr->pDeferred && (pExpr->iDocid==pCsr->iPrevId || pExpr->bDeferred) ){ Fts3Phrase *pPhrase = pExpr->pPhrase; assert( pExpr->bDeferred || pPhrase->doclist.bFreeList==0 ); @@ -4858,11 +4861,13 @@ fts3EvalInvalidatePoslist(pPhrase); } *pRc = fts3EvalDeferredPhrase(pCsr, pPhrase); bHit = (pPhrase->doclist.pList!=0); pExpr->iDocid = pCsr->iPrevId; - }else{ + }else +#endif + { bHit = (pExpr->bEof==0 && pExpr->iDocid==pCsr->iPrevId); } break; } } Index: ext/fts3/fts3Int.h ================================================================== --- ext/fts3/fts3Int.h +++ ext/fts3/fts3Int.h @@ -425,14 +425,24 @@ int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char **, int*, int*); int sqlite3Fts3SelectDoctotal(Fts3Table *, sqlite3_stmt **); int sqlite3Fts3SelectDocsize(Fts3Table *, sqlite3_int64, sqlite3_stmt **); +#ifndef SQLITE_DISABLE_FTS4_DEFERRED void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *); int sqlite3Fts3DeferToken(Fts3Cursor *, Fts3PhraseToken *, int); int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *); void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *); +int sqlite3Fts3DeferredTokenList(Fts3DeferredToken *, char **, int *); +#else +# define sqlite3Fts3FreeDeferredTokens(x) +# define sqlite3Fts3DeferToken(x,y,z) SQLITE_OK +# define sqlite3Fts3CacheDeferredDoclists(x) SQLITE_OK +# define sqlite3Fts3FreeDeferredDoclists(x) +# define sqlite3Fts3DeferredTokenList(x,y,z) SQLITE_OK +#endif + void sqlite3Fts3SegmentsClose(Fts3Table *); int sqlite3Fts3MaxLevel(Fts3Table *, int *); /* Special values interpreted by sqlite3SegReaderCursor() */ #define FTS3_SEGCURSOR_PENDING -1 @@ -537,16 +547,14 @@ Fts3Table *, Fts3MultiSegReader *, sqlite3_int64 *, char **, int *); int sqlite3Fts3EvalPhrasePoslist(Fts3Cursor *, Fts3Expr *, int iCol, char **); int sqlite3Fts3MsrOvfl(Fts3Cursor *, Fts3MultiSegReader *, int *); int sqlite3Fts3MsrIncrRestart(Fts3MultiSegReader *pCsr); -int sqlite3Fts3DeferredTokenList(Fts3DeferredToken *, char **, int *); - /* fts3_unicode2.c (functions generated by parsing unicode text files) */ #ifdef SQLITE_ENABLE_FTS4_UNICODE61 int sqlite3FtsUnicodeFold(int, int); int sqlite3FtsUnicodeIsalnum(int); int sqlite3FtsUnicodeIsdiacritic(int); #endif #endif /* !SQLITE_CORE || SQLITE_ENABLE_FTS3 */ #endif /* _FTSINT_H */ Index: ext/fts3/fts3_write.c ================================================================== --- ext/fts3/fts3_write.c +++ ext/fts3/fts3_write.c @@ -5044,10 +5044,11 @@ } return rc; } +#ifndef SQLITE_DISABLE_FTS4_DEFERRED /* ** Delete all cached deferred doclists. Deferred doclists are cached ** (allocated) by the sqlite3Fts3CacheDeferredDoclists() function. */ void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *pCsr){ @@ -5181,10 +5182,11 @@ assert( pToken->pDeferred==0 ); pToken->pDeferred = pDeferred; return SQLITE_OK; } +#endif /* ** SQLite value pRowid contains the rowid of a row that may or may not be ** present in the FTS3 table. If it is, delete it and adjust the contents ** of subsiduary data structures accordingly. Index: src/btree.c ================================================================== --- src/btree.c +++ src/btree.c @@ -1459,11 +1459,11 @@ } next = get2byte(&data[pc]); size = get2byte(&data[pc+2]); if( (next>0 && next<=pc+size+3) || pc+size>usableSize ){ /* Free blocks must be in ascending order. And the last byte of - ** the free-block must lie on the database page. */ + ** the free-block must lie on the database page. */ return SQLITE_CORRUPT_BKPT; } nFree = nFree + size; pc = next; } @@ -2633,11 +2633,11 @@ if( rc==SQLITE_OK ){ if( p->inTrans==TRANS_NONE ){ pBt->nTransaction++; #ifndef SQLITE_OMIT_SHARED_CACHE if( p->sharable ){ - assert( p->lock.pBtree==p && p->lock.iTable==1 ); + assert( p->lock.pBtree==p && p->lock.iTable==1 ); p->lock.eLock = READ_LOCK; p->lock.pNext = pBt->pLock; pBt->pLock = &p->lock; } #endif Index: src/build.c ================================================================== --- src/build.c +++ src/build.c @@ -532,11 +532,11 @@ pNext = pIndex->pNext; assert( pIndex->pSchema==pTable->pSchema ); if( !db || db->pnBytesFreed==0 ){ char *zName = pIndex->zName; TESTONLY ( Index *pOld = ) sqlite3HashInsert( - &pIndex->pSchema->idxHash, zName, sqlite3Strlen30(zName), 0 + &pIndex->pSchema->idxHash, zName, sqlite3Strlen30(zName), 0 ); assert( db==0 || sqlite3SchemaMutexHeld(db, 0, pIndex->pSchema) ); assert( pOld==pIndex || pOld==0 ); } freeIndex(db, pIndex); Index: src/mem1.c ================================================================== --- src/mem1.c +++ src/mem1.c @@ -229,18 +229,18 @@ /* defer MT decisions to system malloc */ _sqliteZone_ = malloc_default_zone(); }else{ /* only 1 core, use our own zone to contention over global locks, ** e.g. we have our own dedicated locks */ - bool success; + bool success; malloc_zone_t* newzone = malloc_create_zone(4096, 0); malloc_set_zone_name(newzone, "Sqlite_Heap"); do{ success = OSAtomicCompareAndSwapPtrBarrier(NULL, newzone, (void * volatile *)&_sqliteZone_); }while(!_sqliteZone_); - if( !success ){ + if( !success ){ /* somebody registered a zone first */ malloc_destroy_zone(newzone); } } #endif Index: src/os_unix.c ================================================================== --- src/os_unix.c +++ src/os_unix.c @@ -713,13 +713,13 @@ return SQLITE_BUSY; case EACCES: /* EACCES is like EAGAIN during locking operations, but not any other time*/ if( (sqliteIOErr == SQLITE_IOERR_LOCK) || - (sqliteIOErr == SQLITE_IOERR_UNLOCK) || - (sqliteIOErr == SQLITE_IOERR_RDLOCK) || - (sqliteIOErr == SQLITE_IOERR_CHECKRESERVEDLOCK) ){ + (sqliteIOErr == SQLITE_IOERR_UNLOCK) || + (sqliteIOErr == SQLITE_IOERR_RDLOCK) || + (sqliteIOErr == SQLITE_IOERR_CHECKRESERVEDLOCK) ){ return SQLITE_BUSY; } /* else fall through */ case EPERM: return SQLITE_PERM; @@ -1762,11 +1762,11 @@ lock.l_start = lock.l_len = 0L; if( unixFileLock(pFile, &lock)==0 ){ pInode->eFileLock = NO_LOCK; }else{ rc = SQLITE_IOERR_UNLOCK; - pFile->lastErrno = errno; + pFile->lastErrno = errno; pInode->eFileLock = NO_LOCK; pFile->eFileLock = NO_LOCK; } } @@ -1778,11 +1778,11 @@ assert( pInode->nLock>=0 ); if( pInode->nLock==0 ){ closePendingFds(pFile); } } - + end_unlock: unixLeaveMutex(); if( rc==SQLITE_OK ) pFile->eFileLock = eFileLock; return rc; } @@ -2045,11 +2045,11 @@ char *zLockFile = (char *)pFile->lockingContext; int rc; assert( pFile ); OSTRACE(("UNLOCK %d %d was %d pid=%d (dotlock)\n", pFile->h, eFileLock, - pFile->eFileLock, getpid())); + pFile->eFileLock, getpid())); assert( eFileLock<=SHARED_LOCK ); /* no-op if possible */ if( pFile->eFileLock==eFileLock ){ return SQLITE_OK; @@ -2432,11 +2432,11 @@ sem_t *pSem = pFile->pInode->pSem; assert( pFile ); assert( pSem ); OSTRACE(("UNLOCK %d %d was %d pid=%d (sem)\n", pFile->h, eFileLock, - pFile->eFileLock, getpid())); + pFile->eFileLock, getpid())); assert( eFileLock<=SHARED_LOCK ); /* no-op if possible */ if( pFile->eFileLock==eFileLock ){ return SQLITE_OK; @@ -3022,11 +3022,11 @@ SimulateIOError( newOffset-- ); if( newOffset!=offset ){ if( newOffset == -1 ){ ((unixFile*)id)->lastErrno = errno; }else{ - ((unixFile*)id)->lastErrno = 0; + ((unixFile*)id)->lastErrno = 0; } return -1; } got = osRead(id->h, pBuf, cnt); #endif @@ -3110,11 +3110,11 @@ SimulateIOError( newOffset-- ); if( newOffset!=offset ){ if( newOffset == -1 ){ ((unixFile*)id)->lastErrno = errno; }else{ - ((unixFile*)id)->lastErrno = 0; + ((unixFile*)id)->lastErrno = 0; } return -1; } got = osWrite(id->h, pBuf, cnt); }while( got<0 && errno==EINTR ); @@ -5624,11 +5624,11 @@ ** with _IOWR('z', 23, struct ByteRangeLockPB2) to track the same 5 states. ** To simulate a F_RDLCK on the shared range, on AFP a randomly selected ** address in the shared range is taken for a SHARED lock, the entire ** shared range is taken for an EXCLUSIVE lock): ** -** PENDING_BYTE 0x40000000 +** PENDING_BYTE 0x40000000 ** RESERVED_BYTE 0x40000001 ** SHARED_RANGE 0x40000002 -> 0x40000200 ** ** This works well on the local file system, but shows a nearly 100x ** slowdown in read performance on AFP because the AFP client disables Index: src/os_win.c ================================================================== --- src/os_win.c +++ src/os_win.c @@ -44,13 +44,15 @@ #ifndef FILE_ATTRIBUTE_MASK # define FILE_ATTRIBUTE_MASK (0x0003FFF7) #endif +#ifndef SQLITE_OMIT_WAL /* Forward references */ typedef struct winShm winShm; /* A connection to shared-memory */ typedef struct winShmNode winShmNode; /* A region of shared-memory */ +#endif /* ** WinCE lacks native support for file locking so we have to fake it ** with some code of our own. */ @@ -74,11 +76,13 @@ HANDLE h; /* Handle for accessing the file */ u8 locktype; /* Type of lock currently held on this file */ short sharedLockByte; /* Randomly chosen byte used as a shared lock */ u8 ctrlFlags; /* Flags. See WINFILE_* below */ DWORD lastErrno; /* The Windows errno from the last I/O error */ +#ifndef SQLITE_OMIT_WAL winShm *pShm; /* Instance of shared memory on this file */ +#endif const char *zPath; /* Full pathname of this file */ int szChunk; /* Chunk size configured by FCNTL_CHUNK_SIZE */ #if SQLITE_OS_WINCE LPWSTR zDeleteOnClose; /* Name of file to delete when closing */ HANDLE hMutex; /* Mutex used to control access to shared lock */ @@ -1935,11 +1939,13 @@ static int winClose(sqlite3_file *id){ int rc, cnt = 0; winFile *pFile = (winFile*)id; assert( id!=0 ); +#ifndef SQLITE_OMIT_WAL assert( pFile->pShm==0 ); +#endif OSTRACE(("CLOSE %d\n", pFile->h)); do{ rc = osCloseHandle(pFile->h); /* SimulateIOError( rc=0; cnt=MX_CLOSE_ATTEMPT; ); */ }while( rc==0 && ++cnt < MX_CLOSE_ATTEMPT && (sqlite3_win32_sleep(100), 1) ); @@ -3685,11 +3691,13 @@ memset(pFile, 0, sizeof(*pFile)); pFile->pMethod = &winIoMethod; pFile->h = h; pFile->lastErrno = NO_ERROR; pFile->pVfs = pVfs; +#ifndef SQLITE_OMIT_WAL pFile->pShm = 0; +#endif pFile->zPath = zName; if( sqlite3_uri_boolean(zName, "psow", SQLITE_POWERSAFE_OVERWRITE) ){ pFile->ctrlFlags |= WINFILE_PSOW; } Index: src/sqlite.h.in ================================================================== --- src/sqlite.h.in +++ src/sqlite.h.in @@ -3152,12 +3152,15 @@ ** ^The third argument is the value to bind to the parameter. ** ** ^(In those routines that have a fourth argument, its value is the ** number of bytes in the parameter. To be clear: the value is the ** number of bytes in the value, not the number of characters.)^ -** ^If the fourth parameter is negative, the length of the string is +** ^If the fourth parameter to sqlite3_bind_text() or sqlite3_bind_text16() +** is negative, then the length of the string is ** the number of bytes up to the first zero terminator. +** If the fourth parameter to sqlite3_bind_blob() is negative, then +** the behavior is undefined. ** If a non-negative fourth parameter is provided to sqlite3_bind_text() ** or sqlite3_bind_text16() then that parameter must be the byte offset ** where the NUL terminator would occur assuming the string were NUL ** terminated. If any NUL characters occur at byte offsets less than ** the value of the fourth parameter then the resulting string value will Index: src/test1.c ================================================================== --- src/test1.c +++ src/test1.c @@ -5926,11 +5926,11 @@ ){ int i; sqlite3 *db; const char *zOpt; int onoff; - int mask; + int mask = 0; static const struct { const char *zOptName; int mask; } aOpt[] = { { "all", SQLITE_OptMask }, Index: src/test_config.c ================================================================== --- src/test_config.c +++ src/test_config.c @@ -316,10 +316,16 @@ #if defined(SQLITE_ENABLE_FTS3) && defined(SQLITE_ENABLE_FTS4_UNICODE61) Tcl_SetVar2(interp, "sqlite_options", "fts3_unicode", "1", TCL_GLOBAL_ONLY); #else Tcl_SetVar2(interp, "sqlite_options", "fts3_unicode", "0", TCL_GLOBAL_ONLY); #endif + +#ifdef SQLITE_DISABLE_FTS4_DEFERRED + Tcl_SetVar2(interp, "sqlite_options", "fts4_deferred", "0", TCL_GLOBAL_ONLY); +#else + Tcl_SetVar2(interp, "sqlite_options", "fts4_deferred", "1", TCL_GLOBAL_ONLY); +#endif #ifdef SQLITE_OMIT_GET_TABLE Tcl_SetVar2(interp, "sqlite_options", "gettable", "0", TCL_GLOBAL_ONLY); #else Tcl_SetVar2(interp, "sqlite_options", "gettable", "1", TCL_GLOBAL_ONLY); Index: src/test_multiplex.c ================================================================== --- src/test_multiplex.c +++ src/test_multiplex.c @@ -500,15 +500,15 @@ int flags, /* Flags to control the opening */ int *pOutFlags /* Flags showing results of opening */ ){ int rc = SQLITE_OK; /* Result code */ multiplexConn *pMultiplexOpen; /* The new multiplex file descriptor */ - multiplexGroup *pGroup; /* Corresponding multiplexGroup object */ + multiplexGroup *pGroup = 0; /* Corresponding multiplexGroup object */ sqlite3_file *pSubOpen = 0; /* Real file descriptor */ sqlite3_vfs *pOrigVfs = gMultiplex.pOrigVfs; /* Real VFS */ - int nName; - int sz; + int nName = 0; + int sz = 0; char *zToFree = 0; UNUSED_PARAMETER(pVfs); memset(pConn, 0, pVfs->szOsFile); assert( zName || (flags & SQLITE_OPEN_DELETEONCLOSE) ); Index: src/test_spellfix.c ================================================================== --- src/test_spellfix.c +++ src/test_spellfix.c @@ -657,11 +657,11 @@ ){ sqlite3_stmt *pStmt; int rc, rc2; char *zSql; int iLangPrev = -9999; - EditDist3Lang *pLang; + EditDist3Lang *pLang = 0; zSql = sqlite3_mprintf("SELECT iLang, cFrom, cTo, iCost" " FROM \"%w\" WHERE iLang>=0 ORDER BY iLang", zTable); if( zSql==0 ) return SQLITE_NOMEM; rc = sqlite3_prepare(db, zSql, -1, &pStmt, 0); @@ -678,11 +678,11 @@ assert( zFrom!=0 || nFrom==0 ); assert( zTo!=0 || nTo==0 ); if( nFrom>100 || nTo>100 ) continue; if( iCost<0 ) continue; - if( iLang!=iLangPrev ){ + if( pLang==0 || iLang!=iLangPrev ){ EditDist3Lang *pNew; pNew = sqlite3_realloc(p->a, (p->nLang+1)*sizeof(p->a[0])); if( pNew==0 ){ rc = SQLITE_NOMEM; break; } p->a = pNew; pLang = &p->a[p->nLang]; @@ -861,13 +861,13 @@ unsigned int *m, int i, int j, int iCost ){ - int b; + assert( iCost>=0 ); if( iCost<10000 ){ - b = m[j] + iCost; + unsigned int b = m[j] + iCost; if( biRow < pCur->nRow ){ if( pCur->pFullScan ){ - int rc = sqlite3_step(pCur->pFullScan); + rc = sqlite3_step(pCur->pFullScan); if( rc!=SQLITE_ROW ) pCur->iRow = pCur->nRow; + if( rc==SQLITE_ROW || rc==SQLITE_DONE ) rc = SQLITE_OK; }else{ pCur->iRow++; } } - return SQLITE_OK; + return rc; } /* ** Return TRUE if we are at the end-of-file */ @@ -2771,29 +2773,39 @@ /* ** Register the various functions and the virtual table. */ static int spellfix1Register(sqlite3 *db){ - int nErr = 0; + int rc = SQLITE_OK; int i; - nErr += sqlite3_create_function(db, "spellfix1_translit", 1, SQLITE_UTF8, 0, + rc = sqlite3_create_function(db, "spellfix1_translit", 1, SQLITE_UTF8, 0, transliterateSqlFunc, 0, 0); - nErr += sqlite3_create_function(db, "spellfix1_editdist", 2, SQLITE_UTF8, 0, + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "spellfix1_editdist", 2, SQLITE_UTF8, 0, editdistSqlFunc, 0, 0); - nErr += sqlite3_create_function(db, "spellfix1_phonehash", 1, SQLITE_UTF8, 0, + } + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "spellfix1_phonehash", 1, SQLITE_UTF8, 0, phoneticHashSqlFunc, 0, 0); - nErr += sqlite3_create_function(db, "spellfix1_scriptcode", 1, SQLITE_UTF8, 0, + } + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "spellfix1_scriptcode", 1, SQLITE_UTF8, 0, scriptCodeSqlFunc, 0, 0); - nErr += sqlite3_create_module(db, "spellfix1", &spellfix1Module, 0); - nErr += editDist3Install(db); + } + if( rc==SQLITE_OK ){ + rc = sqlite3_create_module(db, "spellfix1", &spellfix1Module, 0); + } + if( rc==SQLITE_OK ){ + rc = editDist3Install(db); + } /* Verify sanity of the translit[] table */ for(i=0; i /* A running thread */ struct SQLiteThread { pthread_t tid; + int done; + void *pOut; }; /* Create a new thread */ int sqlite3ThreadCreate( SQLiteThread **ppThread, /* OUT: Write the thread object here */ void *(*xTask)(void*), /* Routine to run in a separate thread */ void *pIn /* Argument passed into xTask() */ ){ SQLiteThread *p; - int rc; assert( ppThread!=0 ); assert( xTask!=0 ); *ppThread = 0; p = sqlite3Malloc(sizeof(*p)); if( p==0 ) return SQLITE_NOMEM; - rc = pthread_create(&p->tid, 0, xTask, pIn); - if( rc ){ - sqlite3_free(p); - return SQLITE_ERROR; + memset(p, 0, sizeof(*p)); + if( sqlite3GlobalConfig.bCoreMutex==0 + || pthread_create(&p->tid, 0, xTask, pIn)!=0 + ){ + p->done = 1; + p->pOut = xTask(pIn); } *ppThread = p; return SQLITE_OK; } @@ -65,11 +68,16 @@ int sqlite3ThreadJoin(SQLiteThread *p, void **ppOut){ int rc; assert( ppOut!=0 ); if( p==0 ) return SQLITE_NOMEM; - rc = pthread_join(p->tid, ppOut); + if( p->done ){ + *ppOut = p->pOut; + rc = SQLITE_OK; + }else{ + rc = pthread_join(p->tid, ppOut); + } sqlite3_free(p); return rc ? SQLITE_ERROR : SQLITE_OK; } #endif /* SQLITE_OS_UNIX && defined(SQLITE_MUTEX_PTHREADS) */ @@ -113,15 +121,22 @@ assert( ppThread!=0 ); assert( xTask!=0 ); *ppThread = 0; p = sqlite3Malloc(sizeof(*p)); if( p==0 ) return SQLITE_NOMEM; - p->xTask = xTask; p->pIn = pIn; - p->tid = _beginthread(sqlite3ThreadProc, 0, p); - if( p->tid==(uintptr_t)-1 ){ - sqlite3_free(p); - return SQLITE_ERROR; + if( sqlite3GlobalConfig.bCoreMutex==0 ){ + memset(p, 0, sizeof(*p)); + }else{ + p->xTask = xTask; + p->pIn = pIn; + p->tid = _beginthread(sqlite3ThreadProc, 0, p); + if( p->tid==(uintptr_t)-1 ){ + memset(p, 0, sizeof(*p)); + } + } + if( p->xTask==0 ){ + p->pResult = xTask(pIn); } *ppThread = p; return SQLITE_OK; } @@ -129,12 +144,16 @@ int sqlite3ThreadJoin(SQLiteThread *p, void **ppOut){ DWORD rc; assert( ppOut!=0 ); if( p==0 ) return SQLITE_NOMEM; - rc = sqlite3Win32Wait((HANDLE)p->tid); - assert( rc!=WAIT_IO_COMPLETION ); + if( p->xTask==0 ){ + rc = WAIT_OBJECT_O; + }else{ + rc = sqlite3Win32Wait((HANDLE)p->tid); + assert( rc!=WAIT_IO_COMPLETION ); + } if( rc==WAIT_OBJECT_0 ) *ppOut = p->pResult; sqlite3_free(p); return (rc==WAIT_OBJECT_0) ? SQLITE_OK : SQLITE_ERROR; } Index: src/vdbe.c ================================================================== --- src/vdbe.c +++ src/vdbe.c @@ -4403,11 +4403,11 @@ pC = p->apCsr[pOp->p1]; assert( pC!=0 ); assert( pC->isSorter==(pOp->opcode==OP_SorterSort) ); res = 1; if( isSorter(pC) ){ - rc = sqlite3VdbeSorterRewind(db, pC, &res); + rc = sqlite3VdbeSorterRewind(pC, &res); }else{ pCrsr = pC->pCursor; assert( pCrsr ); rc = sqlite3BtreeFirst(pCrsr, &res); pC->atFirst = res==0 ?1:0; @@ -4472,11 +4472,11 @@ break; /* See ticket #2273 */ } assert( pC->isSorter==(pOp->opcode==OP_SorterNext) ); if( isSorter(pC) ){ assert( pOp->opcode==OP_SorterNext ); - rc = sqlite3VdbeSorterNext(db, pC, &res); + rc = sqlite3VdbeSorterNext(pC, &res); }else{ res = 1; assert( pC->deferredMoveto==0 ); assert( pC->pCursor ); assert( pOp->opcode!=OP_Next || pOp->p4.xAdvance==sqlite3BtreeNext ); @@ -4528,11 +4528,11 @@ if( ALWAYS(pCrsr!=0) ){ assert( pC->isTable==0 ); rc = ExpandBlob(pIn2); if( rc==SQLITE_OK ){ if( isSorter(pC) ){ - rc = sqlite3VdbeSorterWrite(db, pC, pIn2); + rc = sqlite3VdbeSorterWrite(pC, pIn2); }else{ nKey = pIn2->n; zKey = pIn2->z; rc = sqlite3BtreeInsert(pCrsr, zKey, nKey, "", 0, 0, pOp->p3, ((pOp->p5 & OPFLAG_USESEEKRESULT) ? pC->seekResult : 0) Index: src/vdbeInt.h ================================================================== --- src/vdbeInt.h +++ src/vdbeInt.h @@ -420,23 +420,23 @@ void sqlite3VdbeMemStoreType(Mem *pMem); int sqlite3VdbeTransferError(Vdbe *p); #ifdef SQLITE_OMIT_MERGE_SORT # define sqlite3VdbeSorterInit(Y,Z) SQLITE_OK -# define sqlite3VdbeSorterWrite(X,Y,Z) SQLITE_OK -# define sqlite3VdbeSorterClose(Y,Z) +# define sqlite3VdbeSorterWrite(Y,Z) SQLITE_OK +# define sqlite3VdbeSorterClose(Z) # define sqlite3VdbeSorterRowkey(Y,Z) SQLITE_OK -# define sqlite3VdbeSorterRewind(X,Y,Z) SQLITE_OK -# define sqlite3VdbeSorterNext(X,Y,Z) SQLITE_OK -# define sqlite3VdbeSorterCompare(X,Y,Z) SQLITE_OK +# define sqlite3VdbeSorterRewind(Y,Z) SQLITE_OK +# define sqlite3VdbeSorterNext(Y,Z) SQLITE_OK +# define sqlite3VdbeSorterCompare(Y,Z) SQLITE_OK #else int sqlite3VdbeSorterInit(sqlite3 *, VdbeCursor *); -void sqlite3VdbeSorterClose(sqlite3 *, VdbeCursor *); +void sqlite3VdbeSorterClose(VdbeCursor *); int sqlite3VdbeSorterRowkey(const VdbeCursor *, Mem *); -int sqlite3VdbeSorterNext(sqlite3 *, const VdbeCursor *, int *); -int sqlite3VdbeSorterRewind(sqlite3 *, const VdbeCursor *, int *); -int sqlite3VdbeSorterWrite(sqlite3 *, const VdbeCursor *, Mem *); +int sqlite3VdbeSorterNext(const VdbeCursor *, int *); +int sqlite3VdbeSorterRewind(const VdbeCursor *, int *); +int sqlite3VdbeSorterWrite(const VdbeCursor *, Mem *); int sqlite3VdbeSorterCompare(const VdbeCursor *, Mem *, int *); #endif #if !defined(SQLITE_OMIT_SHARED_CACHE) && SQLITE_THREADSAFE>0 void sqlite3VdbeEnter(Vdbe*); Index: src/vdbeaux.c ================================================================== --- src/vdbeaux.c +++ src/vdbeaux.c @@ -1573,11 +1573,11 @@ */ void sqlite3VdbeFreeCursor(Vdbe *p, VdbeCursor *pCx){ if( pCx==0 ){ return; } - sqlite3VdbeSorterClose(p->db, pCx); + sqlite3VdbeSorterClose(pCx); if( pCx->pBt ){ sqlite3BtreeClose(pCx->pBt); /* The pCx->pCursor will be close automatically, if it exists, by ** the call above. */ }else if( pCx->pCursor ){ Index: src/vdbesort.c ================================================================== --- src/vdbesort.c +++ src/vdbesort.c @@ -103,20 +103,24 @@ int mxPmaSize; /* Maximum PMA size, in bytes. 0==no limit */ VdbeSorterIter *aIter; /* Array of iterators to merge */ int *aTree; /* Current state of incremental merge */ sqlite3_file *pTemp1; /* PMA file 1 */ SorterRecord *pRecord; /* Head of in-memory record list */ + int nRecord; /* Number of elements on the pRecord list */ UnpackedRecord *pUnpacked; /* Used to unpack keys */ + KeyInfo *pKeyInfo; /* Copy of cursor KeyInfo without db ptr */ + sqlite3 *db; /* Database connection */ }; /* ** The following type is an iterator for a PMA. It caches the current key in ** variables nKey/aKey. If the iterator is at EOF, pFile==0. */ struct VdbeSorterIter { i64 iReadOff; /* Current read offset */ i64 iEof; /* 1 byte past EOF for this iterator */ + sqlite3 *db; /* Corresponding database connection */ int nAlloc; /* Bytes of space at aAlloc */ int nKey; /* Number of bytes in key */ sqlite3_file *pFile; /* File iterator is reading from */ u8 *aAlloc; /* Allocated space */ u8 *aKey; /* Pointer to current key */ @@ -175,11 +179,10 @@ ** ** The buffer indicated by *ppOut may only be considered valid until the ** next call to this function. */ static int vdbeSorterIterRead( - sqlite3 *db, /* Database handle (for malloc) */ VdbeSorterIter *p, /* Iterator */ int nByte, /* Bytes of data to read */ u8 **ppOut /* OUT: Pointer to buffer containing data */ ){ int iBuf; /* Offset within buffer to read from */ @@ -220,11 +223,11 @@ /* Extend the p->aAlloc[] allocation if required. */ if( p->nAllocnAlloc*2; while( nByte>nNew ) nNew = nNew*2; - p->aAlloc = sqlite3DbReallocOrFree(db, p->aAlloc, nNew); + p->aAlloc = sqlite3DbReallocOrFree(p->db, p->aAlloc, nNew); if( !p->aAlloc ) return SQLITE_NOMEM; p->nAlloc = nNew; } /* Copy as much data as is available in the buffer into the start of @@ -240,11 +243,11 @@ int nCopy; /* Number of bytes to copy */ u8 *aNext; /* Pointer to buffer to copy data from */ nCopy = nRem; if( nRem>p->nBuffer ) nCopy = p->nBuffer; - rc = vdbeSorterIterRead(db, p, nCopy, &aNext); + rc = vdbeSorterIterRead(p, nCopy, &aNext); if( rc!=SQLITE_OK ) return rc; assert( aNext!=p->aAlloc ); memcpy(&p->aAlloc[nByte - nRem], aNext, nCopy); nRem -= nCopy; } @@ -257,21 +260,21 @@ /* ** Read a varint from the stream of data accessed by p. Set *pnOut to ** the value read. */ -static int vdbeSorterIterVarint(sqlite3 *db, VdbeSorterIter *p, u64 *pnOut){ +static int vdbeSorterIterVarint(VdbeSorterIter *p, u64 *pnOut){ int iBuf; iBuf = p->iReadOff % p->nBuffer; if( iBuf && (p->nBuffer-iBuf)>=9 ){ p->iReadOff += sqlite3GetVarint(&p->aBuffer[iBuf], pnOut); }else{ u8 aVarint[16], *a; int i = 0, rc; do{ - rc = vdbeSorterIterRead(db, p, 1, &a); + rc = vdbeSorterIterRead(p, 1, &a); if( rc ) return rc; aVarint[(i++)&0xf] = a[0]; }while( (a[0]&0x80)!=0 ); sqlite3GetVarint(aVarint, pnOut); } @@ -283,26 +286,26 @@ /* ** Advance iterator pIter to the next key in its PMA. Return SQLITE_OK if ** no error occurs, or an SQLite error code if one does. */ static int vdbeSorterIterNext( - sqlite3 *db, /* Database handle (for sqlite3DbMalloc() ) */ VdbeSorterIter *pIter /* Iterator to advance */ ){ int rc; /* Return Code */ u64 nRec = 0; /* Size of record in bytes */ + sqlite3 *db = pIter->db; /* Database connection */ if( pIter->iReadOff>=pIter->iEof ){ /* This is an EOF condition */ vdbeSorterIterZero(db, pIter); return SQLITE_OK; } - rc = vdbeSorterIterVarint(db, pIter, &nRec); + rc = vdbeSorterIterVarint(pIter, &nRec); if( rc==SQLITE_OK ){ pIter->nKey = (int)nRec; - rc = vdbeSorterIterRead(db, pIter, (int)nRec, &pIter->aKey); + rc = vdbeSorterIterRead(pIter, (int)nRec, &pIter->aKey); } return rc; } @@ -311,26 +314,27 @@ ** starting at offset iStart and ending at offset iEof-1. This function ** leaves the iterator pointing to the first key in the PMA (or EOF if the ** PMA is empty). */ static int vdbeSorterIterInit( - sqlite3 *db, /* Database handle */ const VdbeSorter *pSorter, /* Sorter object */ i64 iStart, /* Start offset in pFile */ VdbeSorterIter *pIter, /* Iterator to populate */ i64 *pnByte /* IN/OUT: Increment this value by PMA size */ ){ int rc = SQLITE_OK; int nBuf; + sqlite3 *db = pSorter->db; nBuf = sqlite3BtreeGetPageSize(db->aDb[0].pBt); assert( pSorter->iWriteOff>iStart ); assert( pIter->aAlloc==0 ); assert( pIter->aBuffer==0 ); pIter->pFile = pSorter->pTemp1; pIter->iReadOff = iStart; + pIter->db = db; pIter->nAlloc = 128; pIter->aAlloc = (u8 *)sqlite3DbMallocRaw(db, pIter->nAlloc); pIter->nBuffer = nBuf; pIter->aBuffer = (u8 *)sqlite3DbMallocRaw(db, nBuf); @@ -352,18 +356,18 @@ } if( rc==SQLITE_OK ){ u64 nByte; /* Size of PMA in bytes */ pIter->iEof = pSorter->iWriteOff; - rc = vdbeSorterIterVarint(db, pIter, &nByte); + rc = vdbeSorterIterVarint(pIter, &nByte); pIter->iEof = pIter->iReadOff + nByte; *pnByte += nByte; } } if( rc==SQLITE_OK ){ - rc = vdbeSorterIterNext(db, pIter); + rc = vdbeSorterIterNext(pIter); } return rc; } @@ -381,22 +385,22 @@ ** ** If pKey2 is passed a NULL pointer, then it is assumed that the pCsr->aSpace ** has been allocated and contains an unpacked record that is used as key2. */ static void vdbeSorterCompare( - const VdbeCursor *pCsr, /* Cursor object (for pKeyInfo) */ + VdbeSorter *pSorter, /* The sorter */ int bOmitRowid, /* Ignore rowid field at end of keys */ const void *pKey1, int nKey1, /* Left side of comparison */ const void *pKey2, int nKey2, /* Right side of comparison */ - int *pRes /* OUT: Result of comparison */ + int *pRes, /* OUT: Result of comparison */ + UnpackedRecord *r2 /* Space to hold the unpacked Key2 record */ ){ - KeyInfo *pKeyInfo = pCsr->pKeyInfo; - VdbeSorter *pSorter = pCsr->pSorter; - UnpackedRecord *r2 = pSorter->pUnpacked; + KeyInfo *pKeyInfo = pSorter->pKeyInfo; int i; if( pKey2 ){ + assert( r2!=0 ); sqlite3VdbeRecordUnpack(pKeyInfo, nKey2, pKey2, r2); } if( bOmitRowid ){ r2->nField = pKeyInfo->nField; @@ -416,12 +420,11 @@ /* ** This function is called to compare two iterator keys when merging ** multiple b-tree segments. Parameter iOut is the index of the aTree[] ** value to recalculate. */ -static int vdbeSorterDoCompare(const VdbeCursor *pCsr, int iOut){ - VdbeSorter *pSorter = pCsr->pSorter; +static int vdbeSorterDoCompare(VdbeSorter *pSorter, int iOut){ int i1; int i2; int iRes; VdbeSorterIter *p1; VdbeSorterIter *p2; @@ -443,13 +446,13 @@ iRes = i2; }else if( p2->pFile==0 ){ iRes = i1; }else{ int res; - assert( pCsr->pSorter->pUnpacked!=0 ); /* allocated in vdbeSorterMerge() */ vdbeSorterCompare( - pCsr, 0, p1->aKey, p1->nKey, p2->aKey, p2->nKey, &res + pSorter, 0, p1->aKey, p1->nKey, p2->aKey, p2->nKey, &res, + pSorter->pUnpacked ); if( res<=0 ){ iRes = i1; }else{ iRes = i2; @@ -466,20 +469,35 @@ int sqlite3VdbeSorterInit(sqlite3 *db, VdbeCursor *pCsr){ int pgsz; /* Page size of main database */ int mxCache; /* Cache size */ VdbeSorter *pSorter; /* The new sorter */ char *d; /* Dummy */ + int nByte; /* Bytes in pKeyInfo */ assert( pCsr->pKeyInfo && pCsr->pBt==0 ); pCsr->pSorter = pSorter = sqlite3DbMallocZero(db, sizeof(VdbeSorter)); if( pSorter==0 ){ return SQLITE_NOMEM; } + pSorter->db = db; pSorter->pUnpacked = sqlite3VdbeAllocUnpackedRecord(pCsr->pKeyInfo, 0, 0, &d); if( pSorter->pUnpacked==0 ) return SQLITE_NOMEM; assert( pSorter->pUnpacked==(UnpackedRecord *)d ); + + /* pSorter->pKeyInfo is a copy of pCsr->pKeyInfo with the db field set to + ** zero. We use this modified pKeyInfo for sorting so that no lookaside + ** memory will be used, so that sorting can proceed in parallel in multiple + ** threads. + */ + nByte = sizeof(KeyInfo) + (pCsr->pKeyInfo->nField - 1)*sizeof(CollSeq*); + pSorter->pKeyInfo = sqlite3DbMallocRaw(db, nByte); + if( pSorter->pKeyInfo==0 ){ + return SQLITE_NOMEM; + } + memcpy(pSorter->pKeyInfo, pCsr->pKeyInfo, nByte); + pSorter->pKeyInfo->db = 0; if( !sqlite3TempInMemory(db) ){ pgsz = sqlite3BtreeGetPageSize(db->aDb[0].pBt); pSorter->mnPmaSize = SORTER_MIN_WORKING * pgsz; mxCache = db->aDb[0].pSchema->cache_size; @@ -503,13 +521,14 @@ } /* ** Free any cursor components allocated by sqlite3VdbeSorterXXX routines. */ -void sqlite3VdbeSorterClose(sqlite3 *db, VdbeCursor *pCsr){ +void sqlite3VdbeSorterClose(VdbeCursor *pCsr){ VdbeSorter *pSorter = pCsr->pSorter; if( pSorter ){ + sqlite3 *db = pSorter->db; if( pSorter->aIter ){ int i; for(i=0; inTree; i++){ vdbeSorterIterZero(db, &pSorter->aIter[i]); } @@ -518,10 +537,11 @@ if( pSorter->pTemp1 ){ sqlite3OsCloseFree(pSorter->pTemp1); } vdbeSorterRecordFree(db, pSorter->pRecord); sqlite3DbFree(db, pSorter->pUnpacked); + sqlite3DbFree(db, pSorter->pKeyInfo); sqlite3DbFree(db, pSorter); pCsr->pSorter = 0; } } @@ -542,22 +562,24 @@ /* ** Merge the two sorted lists p1 and p2 into a single list. ** Set *ppOut to the head of the new list. */ static void vdbeSorterMerge( - const VdbeCursor *pCsr, /* For pKeyInfo */ + VdbeSorter *pSorter, /* The sorter object */ SorterRecord *p1, /* First list to merge */ SorterRecord *p2, /* Second list to merge */ - SorterRecord **ppOut /* OUT: Head of merged list */ + SorterRecord **ppOut, /* OUT: Head of merged list */ + UnpackedRecord *pUnpacked /* Space to hold an unpacked record */ ){ SorterRecord *pFinal = 0; SorterRecord **pp = &pFinal; void *pVal2 = p2 ? p2->pVal : 0; while( p1 && p2 ){ int res; - vdbeSorterCompare(pCsr, 0, p1->pVal, p1->nVal, pVal2, p2->nVal, &res); + vdbeSorterCompare(pSorter, 0, p1->pVal, p1->nVal, pVal2, p2->nVal, &res, + pUnpacked); if( res<=0 ){ *pp = p1; pp = &p1->pNext; p1 = p1->pNext; pVal2 = 0; @@ -570,47 +592,117 @@ } } *pp = p1 ? p1 : p2; *ppOut = pFinal; } + +/* +** Background sorting task +*/ +typedef struct SortTask { + VdbeSorter *pSorter; /* The sorter for which this task works */ + UnpackedRecord *pUnpacked; /* Space to hold an unpacked key */ + SorterRecord *pList; /* List of elements to be sorted */ + SorterRecord **apSlot; /* Temp memory for the merge sort */ +} SortTask; + +/* +** Do a sort in a background thread +*/ +void *vdbeSorterBackgroundSort(SortTask *pTask){ + SorterRecord *p = pTask->pList; + SorterRecord **a = pTask->apSlot; + int i; + for(i=0; i<64; i++) a[i] = 0; + while( p ){ + SorterRecord *pNext = p->pNext; + p->pNext = 0; + for(i=0; a[i]; i++){ + if( a[i]==0 ) break; + vdbeSorterMerge(pTask->pSorter, a[i], p, &p, pTask->pUnpacked); + a[i] = 0; + } + a[i] = p; + p = pNext; + } + p = 0; + for(i=0; i<64; i++){ + vdbeSorterMerge(pTask->pSorter, a[i], p, &p, pTask->pUnpacked); + } + pTask->pList = p; + return p; +} + +/* +** Divide a linked list of SorterRecord objects into two separate +** linked lists +*/ +static void vdbeSorterDivideList( + SorterRecord *pIn, /* The list to be divided */ + SorterRecord **ppOut1, /* Write the first list here */ + SorterRecord **ppOut2 /* Write the second list here */ +){ + int i = 0; + *ppOut1 = *ppOut2 = 0; + while( pIn ){ + SorterRecord *pNext = pIn->pNext; + pIn->pNext = 0; + if( i & 1 ){ + *ppOut1 = pIn; + ppOut1 = &pIn->pNext; + }else{ + *ppOut2 = pIn; + ppOut2 = &pIn->pNext; + } + i++; + pIn = pNext; + } +} /* ** Sort the linked list of records headed at pCsr->pRecord. Return SQLITE_OK ** if successful, or an SQLite error code (i.e. SQLITE_NOMEM) if an error ** occurs. */ -static int vdbeSorterSort(const VdbeCursor *pCsr){ - int i; - SorterRecord **aSlot; - SorterRecord *p; - VdbeSorter *pSorter = pCsr->pSorter; - - aSlot = (SorterRecord **)sqlite3MallocZero(64 * sizeof(SorterRecord *)); - if( !aSlot ){ +static int vdbeSorterSort(VdbeSorter *pSorter){ + int rc; + char *pDummy = 0; + int nByteA, nByteB; + SortTask aTask[2]; + SQLiteThread *pThread; + + nByteA = 64*sizeof(SorterRecord*); + nByteB = ROUND8(sizeof(UnpackedRecord)); + nByteB += sizeof(Mem)*(pSorter->pKeyInfo->nField+1); + + aTask[0].apSlot = (SorterRecord **)sqlite3MallocZero(2*(nByteA + nByteB)); + if( !aTask[0].apSlot ){ return SQLITE_NOMEM; } - - p = pSorter->pRecord; - while( p ){ - SorterRecord *pNext = p->pNext; - p->pNext = 0; - for(i=0; aSlot[i]; i++){ - vdbeSorterMerge(pCsr, p, aSlot[i], &p); - aSlot[i] = 0; - } - aSlot[i] = p; - p = pNext; - } - - p = 0; - for(i=0; i<64; i++){ - vdbeSorterMerge(pCsr, p, aSlot[i], &p); - } - pSorter->pRecord = p; - - sqlite3_free(aSlot); - return SQLITE_OK; + aTask[0].pSorter = pSorter; + aTask[0].pUnpacked = sqlite3VdbeAllocUnpackedRecord(pSorter->pKeyInfo, + (char*)&aTask[0].apSlot[64], nByteB, &pDummy); + assert( pDummy==0 ); + aTask[1].apSlot = (SorterRecord**)((nByteA+nByteB)+(char*)aTask[0].apSlot); + aTask[1].pSorter = pSorter; + aTask[1].pUnpacked = sqlite3VdbeAllocUnpackedRecord(pSorter->pKeyInfo, + (char*)&aTask[1].apSlot[64], nByteB, &pDummy); + assert( pDummy==0 ); + + vdbeSorterDivideList(pSorter->pRecord, &aTask[0].pList, &aTask[1].pList); + rc = sqlite3ThreadCreate(&pThread, + (void*(*)(void*))vdbeSorterBackgroundSort, &aTask[0]); + vdbeSorterBackgroundSort(&aTask[1]); + if( rc==SQLITE_NOMEM ){ + vdbeSorterBackgroundSort(&aTask[0]); + }else{ + rc = sqlite3ThreadJoin(pThread, (void**)&pDummy); + } + vdbeSorterMerge(pSorter, aTask[0].pList, aTask[1].pList, &pSorter->pRecord, + pSorter->pUnpacked); + sqlite3_free(aTask[0].apSlot); + return rc; } /* ** Initialize a file-writer object. */ @@ -708,23 +800,24 @@ ** ** * One or more records packed end-to-end in order of ascending keys. ** Each record consists of a varint followed by a blob of data (the ** key). The varint is the number of bytes in the blob of data. */ -static int vdbeSorterListToPMA(sqlite3 *db, const VdbeCursor *pCsr){ +static int vdbeSorterListToPMA(VdbeSorter *pSorter){ int rc = SQLITE_OK; /* Return code */ - VdbeSorter *pSorter = pCsr->pSorter; FileWriter writer; + sqlite3 *db = pSorter->db; memset(&writer, 0, sizeof(FileWriter)); if( pSorter->nInMemory==0 ){ assert( pSorter->pRecord==0 ); + assert( pSorter->nRecord==0 ); return rc; } - rc = vdbeSorterSort(pCsr); + rc = vdbeSorterSort(pSorter); /* If the first temporary PMA file has not been opened, open it now. */ if( rc==SQLITE_OK && pSorter->pTemp1==0 ){ rc = vdbeSorterOpenTempFile(db, &pSorter->pTemp1); assert( rc!=SQLITE_OK || pSorter->pTemp1 ); @@ -743,11 +836,12 @@ pNext = p->pNext; fileWriterWriteVarint(&writer, p->nVal); fileWriterWrite(&writer, p->pVal, p->nVal); sqlite3DbFree(db, p); } - pSorter->pRecord = p; + pSorter->pRecord = 0; + pSorter->nRecord = 0; rc = fileWriterFinish(db, &writer, &pSorter->iWriteOff); } return rc; } @@ -754,15 +848,15 @@ /* ** Add a record to the sorter. */ int sqlite3VdbeSorterWrite( - sqlite3 *db, /* Database handle */ - const VdbeCursor *pCsr, /* Sorter cursor */ + const VdbeCursor *pCsr, /* Sorter cursor */ Mem *pVal /* Memory cell containing record */ ){ VdbeSorter *pSorter = pCsr->pSorter; + sqlite3 *db = pSorter->db; int rc = SQLITE_OK; /* Return Code */ SorterRecord *pNew; /* New list element */ assert( pSorter ); pSorter->nInMemory += sqlite3VarintLen(pVal->n) + pVal->n; @@ -774,10 +868,11 @@ pNew->pVal = (void *)&pNew[1]; memcpy(pNew->pVal, pVal->z, pVal->n); pNew->nVal = pVal->n; pNew->pNext = pSorter->pRecord; pSorter->pRecord = pNew; + pSorter->nRecord++; } /* See if the contents of the sorter should now be written out. They ** are written out when either of the following are true: ** @@ -794,11 +889,11 @@ #ifdef SQLITE_DEBUG i64 nExpect = pSorter->iWriteOff + sqlite3VarintLen(pSorter->nInMemory) + pSorter->nInMemory; #endif - rc = vdbeSorterListToPMA(db, pCsr); + rc = vdbeSorterListToPMA(pSorter); pSorter->nInMemory = 0; assert( rc!=SQLITE_OK || (nExpect==pSorter->iWriteOff) ); } return rc; @@ -806,31 +901,29 @@ /* ** Helper function for sqlite3VdbeSorterRewind(). */ static int vdbeSorterInitMerge( - sqlite3 *db, /* Database handle */ - const VdbeCursor *pCsr, /* Cursor handle for this sorter */ + VdbeSorter *pSorter, /* The sorter */ i64 *pnByte /* Sum of bytes in all opened PMAs */ ){ - VdbeSorter *pSorter = pCsr->pSorter; int rc = SQLITE_OK; /* Return code */ int i; /* Used to iterator through aIter[] */ i64 nByte = 0; /* Total bytes in all opened PMAs */ /* Initialize the iterators. */ for(i=0; iaIter[i]; - rc = vdbeSorterIterInit(db, pSorter, pSorter->iReadOff, pIter, &nByte); + rc = vdbeSorterIterInit(pSorter, pSorter->iReadOff, pIter, &nByte); pSorter->iReadOff = pIter->iEof; assert( rc!=SQLITE_OK || pSorter->iReadOff<=pSorter->iWriteOff ); if( rc!=SQLITE_OK || pSorter->iReadOff>=pSorter->iWriteOff ) break; } /* Initialize the aTree[] array. */ for(i=pSorter->nTree-1; rc==SQLITE_OK && i>0; i--){ - rc = vdbeSorterDoCompare(pCsr, i); + rc = vdbeSorterDoCompare(pSorter, i); } *pnByte = nByte; return rc; } @@ -837,12 +930,13 @@ /* ** Once the sorter has been populated, this function is called to prepare ** for iterating through its contents in sorted order. */ -int sqlite3VdbeSorterRewind(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ +int sqlite3VdbeSorterRewind(const VdbeCursor *pCsr, int *pbEof){ VdbeSorter *pSorter = pCsr->pSorter; + sqlite3 *db = pSorter->db; int rc; /* Return code */ sqlite3_file *pTemp2 = 0; /* Second temp file to use */ i64 iWrite2 = 0; /* Write offset for pTemp2 */ int nIter; /* Number of iterators used */ int nByte; /* Bytes of space required for aIter/aTree */ @@ -854,15 +948,15 @@ ** sort the VdbeSorter.pRecord list. The vdbe layer will read data directly ** from the in-memory list. */ if( pSorter->nPMA==0 ){ *pbEof = !pSorter->pRecord; assert( pSorter->aTree==0 ); - return vdbeSorterSort(pCsr); + return vdbeSorterSort(pSorter); } /* Write the current in-memory list to a PMA. */ - rc = vdbeSorterListToPMA(db, pCsr); + rc = vdbeSorterListToPMA(pSorter); if( rc!=SQLITE_OK ) return rc; /* Allocate space for aIter[] and aTree[]. */ nIter = pSorter->nPMA; if( nIter>SORTER_MAX_MERGE_COUNT ) nIter = SORTER_MAX_MERGE_COUNT; @@ -894,11 +988,11 @@ ** ** Otherwise, if pTemp1 contains more than SORTER_MAX_MERGE_COUNT PMAs, ** initialize interators for SORTER_MAX_MERGE_COUNT of them. These PMAs ** are merged into a single PMA that is written to file pTemp2. */ - rc = vdbeSorterInitMerge(db, pCsr, &nWrite); + rc = vdbeSorterInitMerge(pSorter, &nWrite); assert( rc!=SQLITE_OK || pSorter->aIter[ pSorter->aTree[1] ].pFile ); if( rc!=SQLITE_OK || pSorter->nPMA<=SORTER_MAX_MERGE_COUNT ){ break; } @@ -916,11 +1010,11 @@ VdbeSorterIter *pIter = &pSorter->aIter[ pSorter->aTree[1] ]; assert( pIter->pFile ); fileWriterWriteVarint(&writer, pIter->nKey); fileWriterWrite(&writer, pIter->aKey, pIter->nKey); - rc = sqlite3VdbeSorterNext(db, pCsr, &bEof); + rc = sqlite3VdbeSorterNext(pCsr, &bEof); } rc2 = fileWriterFinish(db, &writer, &iWrite2); if( rc==SQLITE_OK ) rc = rc2; } } @@ -946,27 +1040,29 @@ } /* ** Advance to the next element in the sorter. */ -int sqlite3VdbeSorterNext(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ +int sqlite3VdbeSorterNext(const VdbeCursor *pCsr, int *pbEof){ VdbeSorter *pSorter = pCsr->pSorter; + sqlite3 *db = pSorter->db; int rc; /* Return code */ if( pSorter->aTree ){ int iPrev = pSorter->aTree[1];/* Index of iterator to advance */ int i; /* Index of aTree[] to recalculate */ - rc = vdbeSorterIterNext(db, &pSorter->aIter[iPrev]); + rc = vdbeSorterIterNext(&pSorter->aIter[iPrev]); for(i=(pSorter->nTree+iPrev)/2; rc==SQLITE_OK && i>0; i=i/2){ - rc = vdbeSorterDoCompare(pCsr, i); + rc = vdbeSorterDoCompare(pSorter, i); } *pbEof = (pSorter->aIter[pSorter->aTree[1]].pFile==0); }else{ SorterRecord *pFree = pSorter->pRecord; pSorter->pRecord = pFree->pNext; + pSorter->nRecord--; pFree->pNext = 0; vdbeSorterRecordFree(db, pFree); *pbEof = !pSorter->pRecord; rc = SQLITE_OK; } @@ -1029,10 +1125,11 @@ ){ VdbeSorter *pSorter = pCsr->pSorter; void *pKey; int nKey; /* Sorter key to compare pVal with */ pKey = vdbeSorterRowkey(pSorter, &nKey); - vdbeSorterCompare(pCsr, 1, pVal->z, pVal->n, pKey, nKey, pRes); + vdbeSorterCompare(pSorter, 1, pVal->z, pVal->n, pKey, nKey, pRes, + pSorter->pUnpacked); return SQLITE_OK; } #endif /* #ifndef SQLITE_OMIT_MERGE_SORT */ Index: test/fts3auto.test ================================================================== --- test/fts3auto.test +++ test/fts3auto.test @@ -65,11 +65,11 @@ set deferred [list] foreach {k v} [lrange $args 0 [expr $nArg-3]] { switch -- $k { -deferred { - set deferred $v + ifcapable fts4_deferred { set deferred $v } } default { error "bad option \"$k\": must be -deferred" } } @@ -507,13 +507,13 @@ set limit [fts3_make_deferrable t1 c] do_fts3query_test 3.$tn.2.1 t1 {a OR c} - do_test 3.$tn.3 { - fts3_zero_long_segments t1 $limit - } {1} + ifcapable fts4_deferred { + do_test 3.$tn.3 { fts3_zero_long_segments t1 $limit } {1} + } foreach {tn2 expr def} { 1 {a NEAR c} {} 2 {a AND c} c 3 {"a c"} c @@ -548,11 +548,15 @@ do_fts3query_test 4.$tn.1.5 t1 {one NEAR/3 five} do_test 4.$tn.2 { set limit [fts3_make_deferrable t1 five] execsql { INSERT INTO t1(t1) VALUES('optimize') } - expr {[fts3_zero_long_segments t1 $limit]>0} + ifcapable fts4_deferred { + expr {[fts3_zero_long_segments t1 $limit]>0} + } else { + expr 1 + } } {1} do_fts3query_test 4.$tn.3.1 -deferred five t1 {one AND five} do_fts3query_test 4.$tn.3.2 -deferred five t1 {one NEAR five} do_fts3query_test 4.$tn.3.3 -deferred five t1 {one NEAR/1 five} Index: test/fts3defer.test ================================================================== --- test/fts3defer.test +++ test/fts3defer.test @@ -11,11 +11,11 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl source $testdir/malloc_common.tcl -ifcapable !fts3 { +ifcapable !fts3||!fts4_deferred { finish_test return } set sqlite_fts3_enable_parentheses 1 Index: test/fts3defer2.test ================================================================== --- test/fts3defer2.test +++ test/fts3defer2.test @@ -11,11 +11,14 @@ # set testdir [file dirname $argv0] source $testdir/tester.tcl source $testdir/malloc_common.tcl -ifcapable !fts3 { finish_test ; return } +ifcapable !fts3||!fts4_deferred { + finish_test + return +} set testprefix fts3defer2 proc mit {blob} { set scan(littleEndian) i* Index: test/fts3matchinfo.test ================================================================== --- test/fts3matchinfo.test +++ test/fts3matchinfo.test @@ -273,15 +273,18 @@ do_matchinfo_test 4.3.3 t5 {t5 MATCH 'a b a'} { s {3} } do_matchinfo_test 4.3.4 t5 {t5 MATCH 'a a a'} { s {3 1} } do_matchinfo_test 4.3.5 t5 {t5 MATCH '"a b" "a b"'} { s {2} } do_matchinfo_test 4.3.6 t5 {t5 MATCH 'a OR b'} { s {1 2 1 1} } -do_execsql_test 4.4.0 { - INSERT INTO t5(t5) VALUES('optimize'); - UPDATE t5_segments - SET block = zeroblob(length(block)) - WHERE length(block)>10000; +do_execsql_test 4.4.0.1 { INSERT INTO t5(t5) VALUES('optimize') } + +ifcapable fts4_deferred { + do_execsql_test 4.4.0.2 { + UPDATE t5_segments + SET block = zeroblob(length(block)) + WHERE length(block)>10000; + } } do_matchinfo_test 4.4.2 t5 {t5 MATCH 'a b'} { s {2} } do_matchinfo_test 4.4.1 t5 {t5 MATCH 'a a'} { s {2 1} } do_matchinfo_test 4.4.2 t5 {t5 MATCH 'a b'} { s {2} } Index: test/fts4aa.test ================================================================== --- test/fts4aa.test +++ test/fts4aa.test @@ -1653,18 +1653,20 @@ do_test fts4aa-1.8 { db eval { SELECT docid FROM t1_docsize EXCEPT SELECT docid FROM t1 } } {} -do_test fts4aa-1.9 { - # Note: Token 'in' is being deferred in the following query. - db eval { - SELECT docid, mit(matchinfo(t1, 'pcxnal')) FROM t1 - WHERE t1 MATCH 'joseph died in egypt' - ORDER BY docid; - } -} {1050026 {4 1 1 1 1 1 1 1 2 1 1 1 1 1 1 23 23}} +ifcapable fts4_deferred { + do_test fts4aa-1.9 { + # Note: Token 'in' is being deferred in the following query. + db eval { + SELECT docid, mit(matchinfo(t1, 'pcxnal')) FROM t1 + WHERE t1 MATCH 'joseph died in egypt' + ORDER BY docid; + } + } {1050026 {4 1 1 1 1 1 1 1 2 1 1 1 1 1 1 23 23}} +} # Should get the same search results from FTS3 # do_test fts4aa-2.0 { db eval { Index: test/releasetest.tcl ================================================================== --- test/releasetest.tcl +++ test/releasetest.tcl @@ -149,10 +149,19 @@ } "Extra-Robustness" { -DSQLITE_ENABLE_OVERSIZE_CELL_CHECK=1 -DSQLITE_MAX_ATTACHED=62 } + "Devkit" { + -DSQLITE_DEFAULT_FILE_FORMAT=4 + -DSQLITE_MAX_ATTACHED=30 + -DSQLITE_ENABLE_COLUMN_METADATA + -DSQLITE_ENABLE_FTS4 + -DSQLITE_ENABLE_FTS4_PARENTHESIS + -DSQLITE_DISABLE_FTS4_DEFERRED + -DSQLITE_ENABLE_RTREE + } } array set ::Platforms { Linux-x86_64 { "Debug-One" "checksymbols test" @@ -164,10 +173,11 @@ "Ftrapv" test "Default" "threadtest test" "Device-One" fulltest } Linux-i686 { + "Devkit" test "Unlock-Notify" "QUICKTEST_INCLUDE=notify2.test test" "Device-One" test "Device-Two" test "Default" "threadtest fulltest" } ADDED tool/checkSpacing.c Index: tool/checkSpacing.c ================================================================== --- /dev/null +++ tool/checkSpacing.c @@ -0,0 +1,84 @@ +/* +** This program checks for formatting problems in source code: +** +** * Any use of tab characters +** * White space at the end of a line +** * Blank lines at the end of a file +** +** Any violations are reported. +*/ +#include +#include +#include + +#define CR_OK 0x001 +#define WSEOL_OK 0x002 + +static void checkSpacing(const char *zFile, unsigned flags){ + FILE *in = fopen(zFile, "rb"); + int i; + int seenSpace; + int seenTab; + int ln = 0; + int lastNonspace = 0; + char zLine[2000]; + if( in==0 ){ + printf("cannot open %s\n", zFile); + return; + } + while( fgets(zLine, sizeof(zLine), in) ){ + seenSpace = 0; + seenTab = 0; + ln++; + for(i=0; zLine[i]; i++){ + if( zLine[i]=='\t' && seenTab==0 ){ + printf("%s:%d: tab (\\t) character\n", zFile, ln); + seenTab = 1; + }else if( zLine[i]=='\r' ){ + if( (flags & CR_OK)==0 ){ + printf("%s:%d: carriage-return (\\r) character\n", zFile, ln); + } + }else if( zLine[i]==' ' ){ + seenSpace = 1; + }else if( zLine[i]!='\n' ){ + lastNonspace = ln; + seenSpace = 0; + } + } + if( seenSpace && (flags & WSEOL_OK)==0 ){ + printf("%s:%d: whitespace at end-of-line\n", zFile, ln); + } + } + fclose(in); + if( lastNonspace #include #ifndef __WIN32__ # if defined(_WIN32) || defined(WIN32) -# define __WIN32__ +# define __WIN32__ # endif #endif #ifdef __WIN32__ #ifdef __cplusplus @@ -651,11 +651,11 @@ break; } } }else if( sp->prec>=0 ){ rp->precsym = rp->rhs[i]; - } + } } } } return; } @@ -709,16 +709,16 @@ }else if( s2->type==MULTITERMINAL ){ for(j=0; jnsubsym; j++){ progress += SetAdd(s1->firstset,s2->subsym[j]->index); } break; - }else if( s1==s2 ){ + }else if( s1==s2 ){ if( s1->lambda==LEMON_FALSE ) break; - }else{ + }else{ progress += SetUnion(s1->firstset,s2->firstset); if( s2->lambda==LEMON_FALSE ) break; - } + } } } }while( progress ); return; } @@ -957,12 +957,12 @@ for(plp=cfp->fplp; plp; plp=plp->next){ change = SetUnion(plp->cfp->fws,cfp->fws); if( change ){ plp->cfp->status = INCOMPLETE; progress = 1; - } - } + } + } cfp->status = COMPLETE; } } }while( progress ); } @@ -991,11 +991,11 @@ if( SetFind(cfp->fws,j) ){ /* Add a reduce action to the state "stp" which will reduce by the ** rule "cfp->rp" if the lookahead symbol is "lemp->symbols[j]" */ Action_add(&stp->ap,REDUCE,lemp->symbols[j],(char *)cfp->rp); } - } + } } } } /* Add the accepting token */ @@ -1260,15 +1260,15 @@ int k; for(k=0; knsubsym; k++){ SetAdd(newcfp->fws, xsp->subsym[k]->index); } break; - }else{ + }else{ SetUnion(newcfp->fws,xsp->firstset); if( xsp->lambda==LEMON_FALSE ) break; - } - } + } + } if( i==rp->nrhs ) Plink_add(&cfp->fplp,newcfp); } } } return; @@ -1994,19 +1994,19 @@ if( psp->prevrule==0 ){ ErrorMsg(psp->filename,psp->tokenlineno, "There is no prior rule upon which to attach the code \ fragment which begins on this line."); psp->errorcnt++; - }else if( psp->prevrule->code!=0 ){ + }else if( psp->prevrule->code!=0 ){ ErrorMsg(psp->filename,psp->tokenlineno, "Code fragment beginning on this line is not the first \ to follow the previous rule."); psp->errorcnt++; }else{ psp->prevrule->line = psp->tokenlineno; psp->prevrule->code = &x[1]; - } + } }else if( x[0]=='[' ){ psp->state = PRECEDENCE_MARK_1; }else{ ErrorMsg(psp->filename,psp->tokenlineno, "Token \"%s\" should be either \"%%\" or a nonterminal name.", @@ -2095,19 +2095,19 @@ if( rp==0 ){ ErrorMsg(psp->filename,psp->tokenlineno, "Can't allocate enough memory for this rule."); psp->errorcnt++; psp->prevrule = 0; - }else{ + }else{ int i; rp->ruleline = psp->tokenlineno; rp->rhs = (struct symbol**)&rp[1]; rp->rhsalias = (const char**)&(rp->rhs[psp->nrhs]); for(i=0; inrhs; i++){ rp->rhs[i] = psp->rhs[i]; rp->rhsalias[i] = psp->alias[i]; - } + } rp->lhs = psp->lhs; rp->lhsalias = psp->lhsalias; rp->nrhs = psp->nrhs; rp->code = 0; rp->precsym = 0; @@ -2115,29 +2115,29 @@ rp->nextlhs = rp->lhs->rule; rp->lhs->rule = rp; rp->next = 0; if( psp->firstrule==0 ){ psp->firstrule = psp->lastrule = rp; - }else{ + }else{ psp->lastrule->next = rp; psp->lastrule = rp; - } + } psp->prevrule = rp; - } + } psp->state = WAITING_FOR_DECL_OR_RULE; }else if( isalpha(x[0]) ){ if( psp->nrhs>=MAXRHS ){ ErrorMsg(psp->filename,psp->tokenlineno, "Too many symbols on RHS of rule beginning at \"%s\".", x); psp->errorcnt++; psp->state = RESYNC_AFTER_RULE_ERROR; - }else{ + }else{ psp->rhs[psp->nrhs] = Symbol_new(x); psp->alias[psp->nrhs] = 0; psp->nrhs++; - } + } }else if( (x[0]=='|' || x[0]=='/') && psp->nrhs>0 ){ struct symbol *msp = psp->rhs[psp->nrhs-1]; if( msp->type!=MULTITERMINAL ){ struct symbol *origsp = msp; msp = (struct symbol *) calloc(1,sizeof(*msp)); @@ -2197,28 +2197,28 @@ psp->insertLineMacro = 1; psp->state = WAITING_FOR_DECL_ARG; if( strcmp(x,"name")==0 ){ psp->declargslot = &(psp->gp->name); psp->insertLineMacro = 0; - }else if( strcmp(x,"include")==0 ){ + }else if( strcmp(x,"include")==0 ){ psp->declargslot = &(psp->gp->include); - }else if( strcmp(x,"code")==0 ){ + }else if( strcmp(x,"code")==0 ){ psp->declargslot = &(psp->gp->extracode); - }else if( strcmp(x,"token_destructor")==0 ){ + }else if( strcmp(x,"token_destructor")==0 ){ psp->declargslot = &psp->gp->tokendest; - }else if( strcmp(x,"default_destructor")==0 ){ + }else if( strcmp(x,"default_destructor")==0 ){ psp->declargslot = &psp->gp->vardest; - }else if( strcmp(x,"token_prefix")==0 ){ + }else if( strcmp(x,"token_prefix")==0 ){ psp->declargslot = &psp->gp->tokenprefix; psp->insertLineMacro = 0; - }else if( strcmp(x,"syntax_error")==0 ){ + }else if( strcmp(x,"syntax_error")==0 ){ psp->declargslot = &(psp->gp->error); - }else if( strcmp(x,"parse_accept")==0 ){ + }else if( strcmp(x,"parse_accept")==0 ){ psp->declargslot = &(psp->gp->accept); - }else if( strcmp(x,"parse_failure")==0 ){ + }else if( strcmp(x,"parse_failure")==0 ){ psp->declargslot = &(psp->gp->failure); - }else if( strcmp(x,"stack_overflow")==0 ){ + }else if( strcmp(x,"stack_overflow")==0 ){ psp->declargslot = &(psp->gp->overflow); }else if( strcmp(x,"extra_argument")==0 ){ psp->declargslot = &(psp->gp->arg); psp->insertLineMacro = 0; }else if( strcmp(x,"token_type")==0 ){ @@ -2243,13 +2243,13 @@ psp->state = WAITING_FOR_PRECEDENCE_SYMBOL; }else if( strcmp(x,"nonassoc")==0 ){ psp->preccounter++; psp->declassoc = NONE; psp->state = WAITING_FOR_PRECEDENCE_SYMBOL; - }else if( strcmp(x,"destructor")==0 ){ + }else if( strcmp(x,"destructor")==0 ){ psp->state = WAITING_FOR_DESTRUCTOR_SYMBOL; - }else if( strcmp(x,"type")==0 ){ + }else if( strcmp(x,"type")==0 ){ psp->state = WAITING_FOR_DATATYPE_SYMBOL; }else if( strcmp(x,"fallback")==0 ){ psp->fallback = 0; psp->state = WAITING_FOR_FALLBACK_ID; }else if( strcmp(x,"wildcard")==0 ){ @@ -2257,11 +2257,11 @@ }else{ ErrorMsg(psp->filename,psp->tokenlineno, "Unknown declaration keyword: \"%%%s\".",x); psp->errorcnt++; psp->state = RESYNC_AFTER_DECL_ERROR; - } + } }else{ ErrorMsg(psp->filename,psp->tokenlineno, "Illegal declaration keyword: \"%s\".",x); psp->errorcnt++; psp->state = RESYNC_AFTER_DECL_ERROR; @@ -2312,14 +2312,14 @@ sp = Symbol_new(x); if( sp->prec>=0 ){ ErrorMsg(psp->filename,psp->tokenlineno, "Symbol \"%s\" has already be given a precedence.",x); psp->errorcnt++; - }else{ + }else{ sp->prec = psp->preccounter; sp->assoc = psp->declassoc; - } + } }else{ ErrorMsg(psp->filename,psp->tokenlineno, "Can't assign a precedence to \"%s\".",x); psp->errorcnt++; } @@ -2585,25 +2585,25 @@ prevc = 0; while( (c= *cp)!=0 && (c!='/' || prevc!='*') ){ if( c=='\n' ) lineno++; prevc = c; cp++; - } - }else if( c=='/' && cp[1]=='/' ){ /* Skip C++ style comments too */ + } + }else if( c=='/' && cp[1]=='/' ){ /* Skip C++ style comments too */ cp = &cp[2]; while( (c= *cp)!=0 && c!='\n' ) cp++; if( c ) lineno++; - }else if( c=='\'' || c=='\"' ){ /* String a character literals */ + }else if( c=='\'' || c=='\"' ){ /* String a character literals */ int startchar, prevc; startchar = c; prevc = 0; for(cp++; (c= *cp)!=0 && (c!=startchar || prevc=='\\'); cp++){ if( c=='\n' ) lineno++; if( prevc=='\\' ) prevc = 0; else prevc = c; - } - } + } + } } if( c==0 ){ ErrorMsg(ps.filename,ps.tokenlineno, "C code starting on this line is not terminated before the end of the file."); ps.errorcnt++;