Index: src/btree.c ================================================================== --- src/btree.c +++ src/btree.c @@ -9117,11 +9117,11 @@ while( i>0 ){ int size, j; assert( (u32)i<=usableSize-4 ); /* Enforced by btreeInitPage() */ size = get2byte(&data[i+2]); assert( (u32)(i+size)<=usableSize ); /* Enforced by btreeInitPage() */ - btreeHeapInsert(heap, (i<<16)|(i+size-1)); + btreeHeapInsert(heap, (((u32)i)<<16)|(i+size-1)); /* EVIDENCE-OF: R-58208-19414 The first 2 bytes of a freeblock are a ** big-endian integer which is the offset in the b-tree page of the next ** freeblock in the chain, or zero if the freeblock is the last on the ** chain. */ j = get2byte(&data[i]); Index: src/global.c ================================================================== --- src/global.c +++ src/global.c @@ -184,11 +184,11 @@ (void*)0, /* pScratch */ 0, /* szScratch */ 0, /* nScratch */ (void*)0, /* pPage */ 0, /* szPage */ - 0, /* nPage */ + SQLITE_DEFAULT_PCACHE_INITSZ, /* nPage */ 0, /* mxParserStack */ 0, /* sharedCacheEnabled */ SQLITE_SORTER_PMASZ, /* szPma */ /* All the rest should always be initialized to zero */ 0, /* isInit */ Index: src/pcache1.c ================================================================== --- src/pcache1.c +++ src/pcache1.c @@ -39,21 +39,49 @@ ** size can vary according to architecture, compile-time options, and ** SQLite library version number. ** ** If SQLITE_PCACHE_SEPARATE_HEADER is defined, then the extension is obtained ** using a separate memory allocation from the database page content. This -** seeks to overcome the "clownfoot" problem of allocating a few bytes more +** seeks to overcome the "clownshoe" problem (also called "internal +** fragmentation" in academic literature) of allocating a few bytes more ** than a power of two with the memory allocator rounding up to the next ** power of two, and leaving the rounded-up space unused. ** ** This module tracks pointers to PgHdr1 objects. Only pcache.c communicates ** with this module. Information is passed back and forth as PgHdr1 pointers. ** ** The pcache.c and pager.c modules deal pointers to PgHdr objects. ** The btree.c module deals with pointers to MemPage objects. +** +** SOURCE OF PAGE CACHE MEMORY: +** +** Memory for a page might come from any of three sources: +** +** (1) The general-purpose memory allocator - sqlite3Malloc() +** (2) Global page-cache memory provided using sqlite3_config() with +** SQLITE_CONFIG_PAGECACHE. +** (3) PCache-local bulk allocation. +** +** The third case is a chunk of heap memory (defaulting to 100 pages worth) +** that is allocated when the page cache is created. The size of the local +** bulk allocation can be adjusted using +** +** sqlite3_config(SQLITE_CONFIG_PCACHE, 0, 0, N). +** +** If N is positive, then N pages worth of memory are allocated using a single +** sqlite3Malloc() call and that memory is used for the first N pages allocated. +** Or if N is negative, then -1024*N bytes of memory are allocated and used +** for as many pages as can be accomodated. +** +** Only one of (2) or (3) can be used. Once the memory available to (2) or +** (3) is exhausted, subsequent allocations fail over to the general-purpose +** memory allocator (1). +** +** Earlier versions of SQLite used only methods (1) and (2). But experiments +** show that method (3) with N==100 provides about a 5% performance boost for +** common workloads. */ - #include "sqliteInt.h" typedef struct PCache1 PCache1; typedef struct PgHdr1 PgHdr1; typedef struct PgFreeslot PgFreeslot; @@ -103,12 +131,13 @@ ** flag (bPurgeable) are set when the cache is created. nMax may be ** modified at any time by a call to the pcache1Cachesize() method. ** The PGroup mutex must be held when accessing nMax. */ PGroup *pGroup; /* PGroup this cache belongs to */ - int szPage; /* Size of allocated pages in bytes */ - int szExtra; /* Size of extra space in bytes */ + int szPage; /* Size of database content section */ + int szExtra; /* sizeof(MemPage)+sizeof(PgHdr) */ + int szAlloc; /* Total size of one pcache line */ int bPurgeable; /* True if cache is purgeable */ unsigned int nMin; /* Minimum number of pages reserved */ unsigned int nMax; /* Configured "cache_size" value */ unsigned int n90pct; /* nMax*9/10 */ unsigned int iMaxKey; /* Largest key seen since xTruncate() */ @@ -118,10 +147,12 @@ */ unsigned int nRecyclable; /* Number of pages in the LRU list */ unsigned int nPage; /* Total number of pages in apHash */ unsigned int nHash; /* Number of slots in apHash[] */ PgHdr1 **apHash; /* Hash table for fast lookup by key */ + PgHdr1 *pFree; /* List of unused pcache-local pages */ + void *pBulk; /* Bulk memory used by pcache-local */ }; /* ** Each cache entry is represented by an instance of the following ** structure. Unless SQLITE_PCACHE_SEPARATE_HEADER is defined, a buffer of @@ -130,19 +161,20 @@ */ struct PgHdr1 { sqlite3_pcache_page page; unsigned int iKey; /* Key value (page number) */ u8 isPinned; /* Page in use, not on the LRU list */ + u8 isBulkLocal; /* This page from bulk local storage */ PgHdr1 *pNext; /* Next in hash table chain */ PCache1 *pCache; /* Cache that currently owns this page */ PgHdr1 *pLruNext; /* Next in LRU list of unpinned pages */ PgHdr1 *pLruPrev; /* Previous in LRU list of unpinned pages */ }; /* -** Free slots in the allocator used to divide up the buffer provided using -** the SQLITE_CONFIG_PAGECACHE mechanism. +** Free slots in the allocator used to divide up the global page cache +** buffer provided using the SQLITE_CONFIG_PAGECACHE mechanism. */ struct PgFreeslot { PgFreeslot *pNext; /* Next free slot */ }; @@ -156,14 +188,15 @@ ** szSlot, nSlot, pStart, pEnd, nReserve, and isInit values are all ** fixed at sqlite3_initialize() time and do not require mutex protection. ** The nFreeSlot and pFree values do require mutex protection. */ int isInit; /* True if initialized */ + int separateCache; /* Use a new PGroup for each PCache */ int szSlot; /* Size of each free slot */ int nSlot; /* The number of pcache slots */ int nReserve; /* Try to keep nFreeSlot above this */ - void *pStart, *pEnd; /* Bounds of pagecache malloc range */ + void *pStart, *pEnd; /* Bounds of global page cache memory */ /* Above requires no mutex. Use mutex below for variable that follow. */ sqlite3_mutex *mutex; /* Mutex for accessing the following: */ PgFreeslot *pFree; /* Free page blocks */ int nFreeSlot; /* Number of unused pcache slots */ /* The following value requires a mutex to change. We skip the mutex on @@ -206,10 +239,11 @@ ** to be serialized already. There is no need for further mutexing. */ void sqlite3PCacheBufferSetup(void *pBuf, int sz, int n){ if( pcache1.isInit ){ PgFreeslot *p; + if( pBuf==0 ) sz = n = 0; sz = ROUNDDOWN8(sz); pcache1.szSlot = sz; pcache1.nSlot = pcache1.nFreeSlot = n; pcache1.nReserve = n>90 ? 10 : (n/10 + 1); pcache1.pStart = pBuf; @@ -270,13 +304,13 @@ } /* ** Free an allocated buffer obtained from pcache1Alloc(). */ -static int pcache1Free(void *p){ +static void pcache1Free(void *p){ int nFreed = 0; - if( p==0 ) return 0; + if( p==0 ) return; if( p>=pcache1.pStart && ppGroup->mutex) ); - pcache1LeaveMutex(pCache->pGroup); + if( pCache->pFree ){ + p = pCache->pFree; + pCache->pFree = p->pNext; + p->pNext = 0; + }else{ +#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT + /* The group mutex must be released before pcache1Alloc() is called. This + ** is because it might call sqlite3_release_memory(), which assumes that + ** this mutex is not held. */ + assert( pcache1.separateCache==0 ); + assert( pCache->pGroup==&pcache1.grp ); + pcache1LeaveMutex(pCache->pGroup); +#endif #ifdef SQLITE_PCACHE_SEPARATE_HEADER - pPg = pcache1Alloc(pCache->szPage); - p = sqlite3Malloc(sizeof(PgHdr1) + pCache->szExtra); - if( !pPg || !p ){ - pcache1Free(pPg); - sqlite3_free(p); - pPg = 0; - } + pPg = pcache1Alloc(pCache->szPage); + p = sqlite3Malloc(sizeof(PgHdr1) + pCache->szExtra); + if( !pPg || !p ){ + pcache1Free(pPg); + sqlite3_free(p); + pPg = 0; + } #else - pPg = pcache1Alloc(ROUND8(sizeof(PgHdr1)) + pCache->szPage + pCache->szExtra); - p = (PgHdr1 *)&((u8 *)pPg)[pCache->szPage]; + pPg = pcache1Alloc(pCache->szAlloc); + p = (PgHdr1 *)&((u8 *)pPg)[pCache->szPage]; +#endif +#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT + pcache1EnterMutex(pCache->pGroup); #endif - pcache1EnterMutex(pCache->pGroup); - - if( pPg ){ + if( pPg==0 ) return 0; p->page.pBuf = pPg; p->page.pExtra = &p[1]; - if( pCache->bPurgeable ){ - pCache->pGroup->nCurrentPage++; - } - return p; + p->isBulkLocal = 0; + } + if( pCache->bPurgeable ){ + pCache->pGroup->nCurrentPage++; } - return 0; + return p; } /* ** Free a page object allocated by pcache1AllocPage(). -** -** The pointer is allowed to be NULL, which is prudent. But it turns out -** that the current implementation happens to never call this routine -** with a NULL pointer, so we mark the NULL test with ALWAYS(). */ static void pcache1FreePage(PgHdr1 *p){ - if( ALWAYS(p) ){ - PCache1 *pCache = p->pCache; - assert( sqlite3_mutex_held(p->pCache->pGroup->mutex) ); + PCache1 *pCache; + assert( p!=0 ); + pCache = p->pCache; + assert( sqlite3_mutex_held(p->pCache->pGroup->mutex) ); + if( p->isBulkLocal ){ + p->pNext = pCache->pFree; + pCache->pFree = p; + }else{ pcache1Free(p->page.pBuf); #ifdef SQLITE_PCACHE_SEPARATE_HEADER sqlite3_free(p); #endif - if( pCache->bPurgeable ){ - pCache->pGroup->nCurrentPage--; - } + } + if( pCache->bPurgeable ){ + pCache->pGroup->nCurrentPage--; } } /* ** Malloc function used by SQLite to obtain space from the buffer configured @@ -570,10 +614,35 @@ */ static int pcache1Init(void *NotUsed){ UNUSED_PARAMETER(NotUsed); assert( pcache1.isInit==0 ); memset(&pcache1, 0, sizeof(pcache1)); + + + /* + ** The pcache1.separateCache variable is true if each PCache has its own + ** private PGroup (mode-1). pcache1.separateCache is false if the single + ** PGroup in pcache1.grp is used for all page caches (mode-2). + ** + ** * Always use a unified cache (mode-2) if ENABLE_MEMORY_MANAGEMENT + ** + ** * Use a unified cache in single-threaded applications that have + ** configured a start-time buffer for use as page-cache memory using + ** sqlite3_config(SQLITE_CONFIG_PAGECACHE, pBuf, sz, N) with non-NULL + ** pBuf argument. + ** + ** * Otherwise use separate caches (mode-1) + */ +#if defined(SQLITE_ENABLE_MEMORY_MANAGEMENT) + pcache1.separateCache = 0; +#elif SQLITE_THREADSAFE + pcache1.separateCache = sqlite3GlobalConfig.pPage==0 + || sqlite3GlobalConfig.bCoreMutex>0; +#else + pcache1.separateCache = sqlite3GlobalConfig.pPage==0; +#endif + #if SQLITE_THREADSAFE if( sqlite3GlobalConfig.bCoreMutex ){ pcache1.grp.mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_LRU); pcache1.mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_PMEM); } @@ -605,52 +674,65 @@ static sqlite3_pcache *pcache1Create(int szPage, int szExtra, int bPurgeable){ PCache1 *pCache; /* The newly created page cache */ PGroup *pGroup; /* The group the new page cache will belong to */ int sz; /* Bytes of memory required to allocate the new cache */ - /* - ** The separateCache variable is true if each PCache has its own private - ** PGroup. In other words, separateCache is true for mode (1) where no - ** mutexing is required. - ** - ** * Always use a unified cache (mode-2) if ENABLE_MEMORY_MANAGEMENT - ** - ** * Always use a unified cache in single-threaded applications - ** - ** * Otherwise (if multi-threaded and ENABLE_MEMORY_MANAGEMENT is off) - ** use separate caches (mode-1) - */ -#if defined(SQLITE_ENABLE_MEMORY_MANAGEMENT) || SQLITE_THREADSAFE==0 - const int separateCache = 0; -#else - int separateCache = sqlite3GlobalConfig.bCoreMutex>0; -#endif - assert( (szPage & (szPage-1))==0 && szPage>=512 && szPage<=65536 ); assert( szExtra < 300 ); - sz = sizeof(PCache1) + sizeof(PGroup)*separateCache; + sz = sizeof(PCache1) + sizeof(PGroup)*pcache1.separateCache; pCache = (PCache1 *)sqlite3MallocZero(sz); if( pCache ){ - if( separateCache ){ + if( pcache1.separateCache ){ pGroup = (PGroup*)&pCache[1]; pGroup->mxPinned = 10; }else{ pGroup = &pcache1.grp; } pCache->pGroup = pGroup; pCache->szPage = szPage; pCache->szExtra = szExtra; + pCache->szAlloc = szPage + szExtra + ROUND8(sizeof(PgHdr1)); pCache->bPurgeable = (bPurgeable ? 1 : 0); pcache1EnterMutex(pGroup); pcache1ResizeHash(pCache); if( bPurgeable ){ pCache->nMin = 10; pGroup->nMinPage += pCache->nMin; pGroup->mxPinned = pGroup->nMaxPage + 10 - pGroup->nMinPage; } pcache1LeaveMutex(pGroup); + /* Try to initialize the local bulk pagecache line allocation if using + ** separate caches and if nPage!=0 */ + if( pcache1.separateCache + && sqlite3GlobalConfig.nPage!=0 + && sqlite3GlobalConfig.pPage==0 + ){ + int szBulk; + char *zBulk; + sqlite3BeginBenignMalloc(); + if( sqlite3GlobalConfig.nPage>0 ){ + szBulk = pCache->szAlloc * sqlite3GlobalConfig.nPage; + }else{ + szBulk = -1024*sqlite3GlobalConfig.nPage; + } + zBulk = pCache->pBulk = sqlite3Malloc( szBulk ); + sqlite3EndBenignMalloc(); + if( zBulk ){ + int nBulk = sqlite3MallocSize(zBulk)/pCache->szAlloc; + int i; + for(i=0; ipage.pBuf = zBulk; + pX->page.pExtra = &pX[1]; + pX->isBulkLocal = 1; + pX->pNext = pCache->pFree; + pCache->pFree = pX; + zBulk += pCache->szAlloc; + } + } + } if( pCache->nHash==0 ){ pcache1Destroy((sqlite3_pcache*)pCache); pCache = 0; } } @@ -740,30 +822,21 @@ if( pCache->nPage>=pCache->nHash ) pcache1ResizeHash(pCache); assert( pCache->nHash>0 && pCache->apHash ); /* Step 4. Try to recycle a page. */ - if( pCache->bPurgeable && pGroup->pLruTail && ( - (pCache->nPage+1>=pCache->nMax) - || pGroup->nCurrentPage>=pGroup->nMaxPage - || pcache1UnderMemoryPressure(pCache) - )){ + if( pCache->bPurgeable + && pGroup->pLruTail + && ((pCache->nPage+1>=pCache->nMax) || pcache1UnderMemoryPressure(pCache)) + ){ PCache1 *pOther; pPage = pGroup->pLruTail; assert( pPage->isPinned==0 ); pcache1RemoveFromHash(pPage, 0); pcache1PinPage(pPage); pOther = pPage->pCache; - - /* We want to verify that szPage and szExtra are the same for pOther - ** and pCache. Assert that we can verify this by comparing sums. */ - assert( (pCache->szPage & (pCache->szPage-1))==0 && pCache->szPage>=512 ); - assert( pCache->szExtra<512 ); - assert( (pOther->szPage & (pOther->szPage-1))==0 && pOther->szPage>=512 ); - assert( pOther->szExtra<512 ); - - if( pOther->szPage+pOther->szExtra != pCache->szPage+pCache->szExtra ){ + if( pOther->szAlloc != pCache->szAlloc ){ pcache1FreePage(pPage); pPage = 0; }else{ pGroup->nCurrentPage -= (pOther->bPurgeable - pCache->bPurgeable); } @@ -1035,10 +1108,11 @@ assert( pGroup->nMinPage >= pCache->nMin ); pGroup->nMinPage -= pCache->nMin; pGroup->mxPinned = pGroup->nMaxPage + 10 - pGroup->nMinPage; pcache1EnforceMaxPage(pGroup); pcache1LeaveMutex(pGroup); + sqlite3_free(pCache->pBulk); sqlite3_free(pCache->apHash); sqlite3_free(pCache); } /* @@ -1090,11 +1164,11 @@ */ int sqlite3PcacheReleaseMemory(int nReq){ int nFree = 0; assert( sqlite3_mutex_notheld(pcache1.grp.mutex) ); assert( sqlite3_mutex_notheld(pcache1.mutex) ); - if( pcache1.pStart==0 ){ + if( sqlite3GlobalConfig.nPage==0 ){ PgHdr1 *p; pcache1EnterMutex(&pcache1.grp); while( (nReq<0 || nFreepage.pBuf); #ifdef SQLITE_PCACHE_SEPARATE_HEADER Index: src/sqliteInt.h ================================================================== --- src/sqliteInt.h +++ src/sqliteInt.h @@ -507,10 +507,20 @@ #if SQLITE_DEFAULT_WORKER_THREADS>SQLITE_MAX_WORKER_THREADS # undef SQLITE_MAX_WORKER_THREADS # define SQLITE_MAX_WORKER_THREADS SQLITE_DEFAULT_WORKER_THREADS #endif +/* +** The default initial allocation for the pagecache when using separate +** pagecaches for each database connection. A positive number is the +** number of pages. A negative number N translations means that a buffer +** of -1024*N bytes is allocated and used for as many pages as it will hold. +*/ +#ifndef SQLITE_DEFAULT_PCACHE_INITSZ +# define SQLITE_DEFAULT_PCACHE_INITSZ 100 +#endif + /* ** GCC does not define the offsetof() macro so we'll have to do it ** ourselves. */ Index: test/memdb.test ================================================================== --- test/memdb.test +++ test/memdb.test @@ -417,15 +417,14 @@ CREATE TABLE t1(a); INSERT INTO t1 VALUES(randstr(1000,1000)); INSERT INTO t1 VALUES(randstr(1000,1000)); INSERT INTO t1 VALUES(randstr(1000,1000)); } - set memused [lindex [sqlite3_status SQLITE_STATUS_MEMORY_USED 0] 1] - set pgovfl [lindex [sqlite3_status SQLITE_STATUS_PAGECACHE_OVERFLOW 0] 1] + set before [db one {PRAGMA page_count}] execsql { DELETE FROM t1 } - set memused2 [lindex [sqlite3_status SQLITE_STATUS_MEMORY_USED 0] 1] - expr {($memused2 + 2048 < $memused) || $pgovfl==0} + set after [db one {PRAGMA page_count}] + expr {$before>$after} } {1} } } ;# ifcapable memorydb Index: test/memsubsys1.test ================================================================== --- test/memsubsys1.test +++ test/memsubsys1.test @@ -73,10 +73,11 @@ # Test 1: Both PAGECACHE and SCRATCH are shut down. # db close sqlite3_shutdown sqlite3_config_lookaside 0 0 +sqlite3_config_pagecache 0 0 sqlite3_initialize reset_highwater_marks build_test_db memsubsys1-1 {PRAGMA page_size=1024} do_test memsubsys1-1.3 { set pg_used [lindex [sqlite3_status SQLITE_STATUS_PAGECACHE_USED 0] 2] @@ -113,14 +114,14 @@ # so that PAGECACHE is not used. # db close sqlite3_shutdown sqlite3_config_pagecache [expr 512+$xtra_size] 20 +sqlite3_config singlethread sqlite3_initialize reset_highwater_marks build_test_db memsubsys1-3.1 {PRAGMA page_size=1024} -#show_memstats do_test memsubsys1-3.1.3 { set pg_used [lindex [sqlite3_status SQLITE_STATUS_PAGECACHE_USED 0] 2] } 0 do_test memsubsys1-3.1.4 { set overflow [lindex [sqlite3_status SQLITE_STATUS_PAGECACHE_OVERFLOW 0] 2] @@ -310,8 +311,9 @@ sqlite3_shutdown sqlite3_config_memstatus 1 sqlite3_config_pagecache 0 0 sqlite3_config_scratch 0 0 sqlite3_config_lookaside 100 500 +sqlite3_config serialized sqlite3_initialize autoinstall_test_functions finish_test Index: test/pcache2.test ================================================================== --- test/pcache2.test +++ test/pcache2.test @@ -22,10 +22,11 @@ do_test pcache2-1.1 { db close sqlite3_reset_auto_extension sqlite3_shutdown sqlite3_config_pagecache 6000 100 + sqlite3_config singlethread sqlite3_initialize autoinstall_test_functions sqlite3_status SQLITE_STATUS_PAGECACHE_USED 1 sqlite3_status SQLITE_STATUS_PAGECACHE_USED 0 } {0 0 0} @@ -71,9 +72,10 @@ db close catch {db2 close} sqlite3_reset_auto_extension sqlite3_shutdown sqlite3_config_pagecache 0 0 +sqlite3_config serialized sqlite3_initialize autoinstall_test_functions finish_test Index: test/speedtest1.c ================================================================== --- test/speedtest1.c +++ test/speedtest1.c @@ -1175,10 +1175,11 @@ const char *zKey = 0; /* Encryption key */ int nLook = 0, szLook = 0; /* --lookaside configuration */ int noSync = 0; /* True for --nosync */ int pageSize = 0; /* Desired page size. 0 means default */ int nPCache = 0, szPCache = 0;/* --pcache configuration */ + int doPCache = 0; /* True if --pcache is seen */ int nScratch = 0, szScratch=0;/* --scratch configuration */ int showStats = 0; /* True for --stats */ int nThread = 0; /* --threads value */ const char *zTSet = "main"; /* Which --testset torun */ int doTrace = 0; /* True for --trace */ @@ -1249,10 +1250,11 @@ pageSize = integerValue(argv[++i]); }else if( strcmp(z,"pcache")==0 ){ if( i>=argc-2 ) fatal_error("missing arguments on %s\n", argv[i]); nPCache = integerValue(argv[i+1]); szPCache = integerValue(argv[i+2]); + doPCache = 1; i += 2; }else if( strcmp(z,"primarykey")==0 ){ g.zPK = "PRIMARY KEY"; }else if( strcmp(z,"reprepare")==0 ){ g.bReprepare = 1; @@ -1315,14 +1317,16 @@ pHeap = malloc( nHeap ); if( pHeap==0 ) fatal_error("cannot allocate %d-byte heap\n", nHeap); rc = sqlite3_config(SQLITE_CONFIG_HEAP, pHeap, nHeap, mnHeap); if( rc ) fatal_error("heap configuration failed: %d\n", rc); } - if( nPCache>0 && szPCache>0 ){ - pPCache = malloc( nPCache*(sqlite3_int64)szPCache ); - if( pPCache==0 ) fatal_error("cannot allocate %lld-byte pcache\n", - nPCache*(sqlite3_int64)szPCache); + if( doPCache ){ + if( nPCache>0 && szPCache>0 ){ + pPCache = malloc( nPCache*(sqlite3_int64)szPCache ); + if( pPCache==0 ) fatal_error("cannot allocate %lld-byte pcache\n", + nPCache*(sqlite3_int64)szPCache); + } rc = sqlite3_config(SQLITE_CONFIG_PAGECACHE, pPCache, szPCache, nPCache); if( rc ) fatal_error("pcache configuration failed: %d\n", rc); } if( nScratch>0 && szScratch>0 ){ pScratch = malloc( nScratch*(sqlite3_int64)szScratch );