SQLite

/*
** 2008 August 05
**
** The author disclaims copyright to this source code.  In place of
** a legal notice, here is a blessing:
**
**    May you do good and not evil.
**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
*************************************************************************
** This file implements that page cache.
**
** @(#) $Id: pcache.c,v 1.6 2008/08/21 15:54:01 danielk1977 Exp $
*/
#include "sqliteInt.h"

/*
** A complete page cache is an instance of this structure.
*/
struct PCache {

** from within a call to sqlite3PcacheFetch() on the same pager. A call
** to sqlite3PcacheLock() may block if such an xStress() call is currently 
** underway.
**
** Before the xStress callback of a pager-cache (PCache) is invoked, the
** SQLITE_MUTEX_STATIC_MEM2 mutex is obtained and the SQLITE_MUTEX_STATIC_LRU 
** mutex released (in that order) before making the call.
**
** Deadlock within the module is avoided by never blocking on the MEM2 
** mutex while the LRU mutex is held.
*/

#define pcacheEnterGlobal() sqlite3_mutex_enter(pcache.mutex_lru)
#define pcacheExitGlobal()  sqlite3_mutex_leave(pcache.mutex_lru)

/*
** Increment the reference count on both page p and its cache by n.

}

/*
** Allocate a page cache line.  Look in the page cache memory pool first
** and use an element from it first if available.  If nothing is available
** in the page cache memory pool, go to the general purpose memory allocator.
*/
void *pcacheMalloc(int sz, PCache *pCache){
  assert( sqlite3_mutex_held(pcache.mutex_lru) );
  if( sz<=pcache.szSlot && pcache.pFree ){
    PgFreeslot *p = pcache.pFree;
    pcache.pFree = p->pNext;
    sqlite3StatusSet(SQLITE_STATUS_PAGECACHE_SIZE, sz);
    sqlite3StatusAdd(SQLITE_STATUS_PAGECACHE_USED, 1);
    return (void*)p;
  }else{
    void *p;

    /* Allocate a new buffer using sqlite3Malloc. Before doing so, exit the
    ** global pcache mutex and unlock the pager-cache object pCache. This is 
    ** so that if the attempt to allocate a new buffer causes the the 
    ** configured soft-heap-limit to be breached, it will be possible to
    ** reclaim memory from this pager-cache. Because sqlite3PcacheLock() 
    ** might block on the MEM2 mutex, it has to be called before re-entering
    ** the global LRU mutex.
    */
    pcacheExitGlobal();
    sqlite3PcacheUnlock(pCache);
    p = sqlite3Malloc(sz);
    sqlite3PcacheLock(pCache);
    pcacheEnterGlobal();

    if( p ){
      sz = sqlite3MallocSize(p);
      sqlite3StatusAdd(SQLITE_STATUS_PAGECACHE_OVERFLOW, sz);
    }
    return p;
  }
}
void *sqlite3PageMalloc(sz){
  void *p;
  pcacheEnterGlobal();
  p = pcacheMalloc(sz, 0);
  pcacheExitGlobal();
  return p;
}

/*
** Release a pager memory allocation
*/

  pcacheFree(p);
  pcacheExitGlobal();
}

/*
** Allocate a new page.
*/
static PgHdr *pcachePageAlloc(PCache *pCache){
  PgHdr *p;
  int sz = sizeof(*p) + pCache->szPage + pCache->szExtra;
  assert( sqlite3_mutex_held(pcache.mutex_lru) );
  p = pcacheMalloc(sz, pCache);
  if( p==0 ) return 0;
  memset(p, 0, sizeof(PgHdr));
  p->pData = (void*)&p[1];
  p->pExtra = (void*)&((char*)p->pData)[pCache->szPage];

  pcache.nPage++;
  if( pCache->bPurgeable ){
    pcache.nPurgeable++;
  }

  return p;
}

/*

  if( p && (p->pCache->szPage!=szPage || p->pCache->szExtra!=szExtra) ){
    pcachePageFree(p);
    p = 0;
  }

  if( !p ){

    p = pcachePageAlloc(pCache);
  }

  pcacheExitGlobal();
  return p;
}

/*************************************************** General Interfaces ******

  if( bPurgeable ){
    pcacheEnterGlobal();
    pcache.mxPagePurgeable += p->nMax;
    pcacheExitGlobal();
  }

  /* Add the new pager-cache to the list of caches starting at pcache.pAll */
  assert( sqlite3_mutex_notheld(pcache.mutex_lru) );
  sqlite3_mutex_enter(pcache.mutex_mem2);
  p->pNextAll = pcache.pAll;
  if( pcache.pAll ){
    pcache.pAll->pPrevAll = p;
  }
  p->pPrevAll = 0;
  pcache.pAll = p;

  sqlite3_free(pCache->apHash);
  pcacheExitGlobal();

  /* Now remove the pager-cache structure itself from the list of
  ** all such structures headed by pcache.pAll. This required the
  ** MUTEX_STATIC_MEM2 mutex.
  */
  assert( sqlite3_mutex_notheld(pcache.mutex_lru) );
  sqlite3_mutex_enter(pcache.mutex_mem2);
  assert(pCache==pcache.pAll || pCache->pPrevAll);
  assert(pCache->pNextAll==0 || pCache->pNextAll->pPrevAll==pCache);
  assert(pCache->pPrevAll==0 || pCache->pPrevAll->pNextAll==pCache);
  if( pCache->pPrevAll ){
    pCache->pPrevAll->pNextAll = pCache->pNextAll;
  }else{

  pCache->nMax = mxPage;
}

/*
** Lock a pager-cache.
*/
void sqlite3PcacheLock(PCache *pCache){
  if( pCache ){
    assert( sqlite3_mutex_notheld(pcache.mutex_lru) );
    pCache->iInUseDB++;
    if( pCache->iInUseMM && pCache->iInUseDB==1 ){
      pCache->iInUseDB = 0;
      sqlite3_mutex_enter(pcache.mutex_mem2);
      assert( pCache->iInUseMM==0 && pCache->iInUseDB==0 );
      pCache->iInUseDB = 1;
      sqlite3_mutex_leave(pcache.mutex_mem2);
    }
  }
}

/*
** Unlock a pager-cache.
*/
void sqlite3PcacheUnlock(PCache *pCache){
  if( pCache ){
    pCache->iInUseDB--;
    assert( pCache->iInUseDB>=0 );
  }
}

#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT
/*
** This function is called to free superfluous dynamically allocated memory
** held by the pager system. Memory in use by any SQLite pager allocated
** by the current thread may be sqlite3_free()ed.

#    May you share freely, never taking more than you give.
#
#***********************************************************************
#
# This file contains test cases focused on the two memory-management APIs, 
# sqlite3_soft_heap_limit() and sqlite3_release_memory().
#
# $Id: malloc5.test,v 1.19 2008/08/21 15:54:01 danielk1977 Exp $

#---------------------------------------------------------------------------
# NOTES ON EXPECTED BEHAVIOUR
#
#---------------------------------------------------------------------------

  execsql {COMMIT;}
  set nMaxBytes [sqlite3_memory_highwater 1]
  puts -nonewline " (Highwater mark: $nMaxBytes) "
  expr $nMaxBytes > 1000000
} {1}
do_test malloc5-4.2 {
  sqlite3_release_memory
  sqlite3_soft_heap_limit 110000
  sqlite3_memory_highwater 1
  execsql {BEGIN;}
  for {set i 0} {$i < 10000} {incr i} {
    execsql "INSERT INTO abc VALUES($i, $i, '[string repeat X 100]');"
  }
  execsql {COMMIT;}
  set nMaxBytes [sqlite3_memory_highwater 1]

  # used to store cached page data are both small and transient.
  #
  # Summary: the actual high-water mark for memory usage may be slightly 
  # higher than the soft-heap-limit. The specific allocations that cause
  # the problem are the calls to sqlite3_malloc() inserted into selected
  # sqlite3OsXXX() functions in test builds.
  #
  expr $nMaxBytes <= 110100
} {1}
do_test malloc5-4.3 {
  # Check that the content of table abc is at least roughly as expected.
  execsql {
    SELECT count(*), sum(a), sum(b) FROM abc;
  }
} [list 20000 [expr int(20000.0 * 4999.5)] [expr int(20000.0 * 4999.5)]]

# This block of test-cases (malloc5-6.1.*) prepares two database files
# for the subsequent tests.
do_test malloc5-6.1.1 {
  sqlite3 db test.db
  execsql {
    PRAGMA page_size=1024;
    PRAGMA default_cache_size=10;
  }
  execsql {
    PRAGMA temp_store = memory;
    BEGIN;
    CREATE TABLE abc(a PRIMARY KEY, b, c);
    INSERT INTO abc VALUES(randstr(50,50), randstr(75,75), randstr(100,100));
    INSERT INTO abc 
        SELECT randstr(50,50), randstr(75,75), randstr(100,100) FROM abc;
    INSERT INTO abc 
        SELECT randstr(50,50), randstr(75,75), randstr(100,100) FROM abc;

do_test malloc5-6.1.2 {
  list [execsql {PRAGMA cache_size}] [execsql {PRAGMA cache_size} db2]
} {10 10}

do_test malloc5-6.2.1 {
  execsql { SELECT * FROM abc } db2
  execsql {SELECT * FROM abc} db
  expr [nPage db] + [nPage db2]
} {20}

do_test malloc5-6.2.2 {
  # If we now try to reclaim some memory, it should come from the db2 cache.
  sqlite3_release_memory 3000
  expr [nPage db] + [nPage db2]
} {17}
do_test malloc5-6.2.3 {
  # Access the db2 cache again, so that all the db2 pages have been used
  # more recently than all the db pages. Then try to reclaim 3000 bytes.
  # This time, 3 pages should be pulled from the db cache.
  execsql { SELECT * FROM abc } db2
  sqlite3_release_memory 3000
  expr [nPage db] + [nPage db2]
} {17}


do_test malloc5-6.3.1 {
  # Now open a transaction and update 2 pages in the db2 cache. Then
  # do a SELECT on the db cache so that all the db pages are more recently
  # used than the db2 pages. When we try to free memory, SQLite should
  # free the non-dirty db2 pages, then the db pages, then finally use
  # sync() to free up the dirty db2 pages. The only page that cannot be
  # freed is page1 of db2. Because there is an open transaction, the
  # btree layer holds a reference to page 1 in the db2 cache.
  execsql {
    BEGIN;
    UPDATE abc SET c = randstr(100,100) 
    WHERE rowid = 1 OR rowid = (SELECT max(rowid) FROM abc);
  } db2
  execsql { SELECT * FROM abc } db
  expr [nPage db] + [nPage db2]
} {20}
do_test malloc5-6.3.2 {
  # Try to release 7700 bytes. This should release all the 
  # non-dirty pages held by db2.
  sqlite3_release_memory [expr 7*1100]
  list [nPage db] [nPage db2]
} {10 3}
do_test malloc5-6.3.3 {

# 2008 June 21
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
#
# $Id: permutations.test,v 1.21 2008/08/21 15:54:01 danielk1977 Exp $

set testdir [file dirname $argv0]
source $testdir/tester.tcl

# Argument processing.
#
set ::testmode [lindex $argv 0]

#############################################################################
# Start of tests

# Run some tests using pre-allocated page and scratch blocks.
#
run_tests "memsubsys1" -description {
  Tests using pre-allocated page and scratch blocks
} -exclude {
  ioerr5.test
  malloc5.test
} -initialize {
  catch {db close}
  sqlite3_shutdown
  sqlite3_config_pagecache 4096 24
  sqlite3_config_scratch 25000 1
  sqlite3_initialize
} -shutdown {