Index: src/main.c ================================================================== --- src/main.c +++ src/main.c @@ -1312,11 +1312,11 @@ case SQLITE_LOCKED: zName = "SQLITE_LOCKED"; break; case SQLITE_LOCKED_SHAREDCACHE: zName = "SQLITE_LOCKED_SHAREDCACHE";break; case SQLITE_NOMEM: zName = "SQLITE_NOMEM"; break; case SQLITE_READONLY: zName = "SQLITE_READONLY"; break; case SQLITE_READONLY_RECOVERY: zName = "SQLITE_READONLY_RECOVERY"; break; - case SQLITE_READONLY_CANTLOCK: zName = "SQLITE_READONLY_CANTLOCK"; break; + case SQLITE_READONLY_CANTINIT: zName = "SQLITE_READONLY_CANTINIT"; break; case SQLITE_READONLY_ROLLBACK: zName = "SQLITE_READONLY_ROLLBACK"; break; case SQLITE_READONLY_DBMOVED: zName = "SQLITE_READONLY_DBMOVED"; break; case SQLITE_INTERRUPT: zName = "SQLITE_INTERRUPT"; break; case SQLITE_IOERR: zName = "SQLITE_IOERR"; break; case SQLITE_IOERR_READ: zName = "SQLITE_IOERR_READ"; break; Index: src/os_unix.c ================================================================== --- src/os_unix.c +++ src/os_unix.c @@ -4106,10 +4106,11 @@ char *zFilename; /* Name of the mmapped file */ int h; /* Open file descriptor */ int szRegion; /* Size of shared-memory regions */ u16 nRegion; /* Size of array apRegion */ u8 isReadonly; /* True if read-only */ + u8 isUnlocked; /* True if no DMS lock held */ char **apRegion; /* Array of mapped shared-memory regions */ int nRef; /* Number of unixShm objects pointing to this */ unixShm *pFirst; /* All unixShm objects pointing to this */ #ifdef SQLITE_DEBUG u8 exclMask; /* Mask of exclusive locks held */ @@ -4267,10 +4268,68 @@ } p->pInode->pShmNode = 0; sqlite3_free(p); } } + +/* +** The DMS lock has not yet been taken on shm file pShmNode. Attempt to +** take it now. Return SQLITE_OK if successful, or an SQLite error +** code otherwise. +** +** If the DMS cannot be locked because this is a readonly_shm=1 +** connection and no other process already holds a lock, return +** SQLITE_READONLY_CANTINIT and set pShmNode->isUnlocked=1. +*/ +static int unixLockSharedMemory(unixFile *pDbFd, unixShmNode *pShmNode){ + struct flock lock; + int rc = SQLITE_OK; + + /* Use F_GETLK to determine the locks other processes are holding + ** on the DMS byte. If it indicates that another process is holding + ** a SHARED lock, then this process may also take a SHARED lock + ** and proceed with opening the *-shm file. + ** + ** Or, if no other process is holding any lock, then this process + ** is the first to open it. In this case take an EXCLUSIVE lock on the + ** DMS byte and truncate the *-shm file to zero bytes in size. Then + ** downgrade to a SHARED lock on the DMS byte. + ** + ** If another process is holding an EXCLUSIVE lock on the DMS byte, + ** return SQLITE_BUSY to the caller (it will try again). An earlier + ** version of this code attempted the SHARED lock at this point. But + ** this introduced a subtle race condition: if the process holding + ** EXCLUSIVE failed just before truncating the *-shm file, then this + ** process might open and use the *-shm file without truncating it. + ** And if the *-shm file has been corrupted by a power failure or + ** system crash, the database itself may also become corrupt. */ + lock.l_whence = SEEK_SET; + lock.l_start = UNIX_SHM_DMS; + lock.l_len = 1; + lock.l_type = F_WRLCK; + if( osFcntl(pShmNode->h, F_GETLK, &lock)!=0 ) { + rc = SQLITE_IOERR_LOCK; + }else if( lock.l_type==F_UNLCK ){ + if( pShmNode->isReadonly ){ + pShmNode->isUnlocked = 1; + rc = SQLITE_READONLY_CANTINIT; + }else{ + rc = unixShmSystemLock(pDbFd, F_WRLCK, UNIX_SHM_DMS, 1); + if( rc==SQLITE_OK && robust_ftruncate(pShmNode->h, 0) ){ + rc = unixLogError(SQLITE_IOERR_SHMOPEN,"ftruncate",pShmNode->zFilename); + } + } + }else if( lock.l_type==F_WRLCK ){ + rc = SQLITE_BUSY; + } + + if( rc==SQLITE_OK ){ + assert( lock.l_type==F_UNLCK || lock.l_type==F_RDLCK ); + rc = unixShmSystemLock(pDbFd, F_RDLCK, UNIX_SHM_DMS, 1); + } + return rc; +} /* ** Open a shared-memory area associated with open database file pDbFd. ** This particular implementation uses mmapped files. ** @@ -4306,13 +4365,13 @@ ** file is created. The shared memory will be simulated with heap memory. */ static int unixOpenSharedMemory(unixFile *pDbFd){ struct unixShm *p = 0; /* The connection to be opened */ struct unixShmNode *pShmNode; /* The underlying mmapped file */ - int rc; /* Result code */ + int rc = SQLITE_OK; /* Result code */ unixInodeInfo *pInode; /* The inode of fd */ - char *zShmFilename; /* Name of the file used for SHM */ + char *zShm; /* Name of the file used for SHM */ int nShmFilename; /* Size of the SHM filename in bytes */ /* Allocate space for the new unixShm object. */ p = sqlite3_malloc64( sizeof(*p) ); if( p==0 ) return SQLITE_NOMEM_BKPT; @@ -4349,18 +4408,18 @@ if( pShmNode==0 ){ rc = SQLITE_NOMEM_BKPT; goto shm_open_err; } memset(pShmNode, 0, sizeof(*pShmNode)+nShmFilename); - zShmFilename = pShmNode->zFilename = (char*)&pShmNode[1]; + zShm = pShmNode->zFilename = (char*)&pShmNode[1]; #ifdef SQLITE_SHM_DIRECTORY - sqlite3_snprintf(nShmFilename, zShmFilename, + sqlite3_snprintf(nShmFilename, zShm, SQLITE_SHM_DIRECTORY "/sqlite-shm-%x-%x", (u32)sStat.st_ino, (u32)sStat.st_dev); #else - sqlite3_snprintf(nShmFilename, zShmFilename, "%s-shm", zBasePath); - sqlite3FileSuffix3(pDbFd->zPath, zShmFilename); + sqlite3_snprintf(nShmFilename, zShm, "%s-shm", zBasePath); + sqlite3FileSuffix3(pDbFd->zPath, zShm); #endif pShmNode->h = -1; pDbFd->pInode->pShmNode = pShmNode; pShmNode->pInode = pDbFd->pInode; if( sqlite3GlobalConfig.bCoreMutex ){ @@ -4370,40 +4429,30 @@ goto shm_open_err; } } if( pInode->bProcessLock==0 ){ - int openFlags = O_RDWR | O_CREAT; - if( sqlite3_uri_boolean(pDbFd->zPath, "readonly_shm", 0) ){ - openFlags = O_RDONLY; - pShmNode->isReadonly = 1; + if( 0==sqlite3_uri_boolean(pDbFd->zPath, "readonly_shm", 0) ){ + pShmNode->h = robust_open(zShm, O_RDWR|O_CREAT, (sStat.st_mode&0777)); } - pShmNode->h = robust_open(zShmFilename, openFlags, (sStat.st_mode&0777)); if( pShmNode->h<0 ){ - rc = unixLogError(SQLITE_CANTOPEN_BKPT, "open", zShmFilename); - goto shm_open_err; + pShmNode->h = robust_open(zShm, O_RDONLY, (sStat.st_mode&0777)); + if( pShmNode->h<0 ){ + rc = unixLogError(SQLITE_CANTOPEN_BKPT, "open", zShm); + goto shm_open_err; + } + pShmNode->isReadonly = 1; } /* If this process is running as root, make sure that the SHM file ** is owned by the same user that owns the original database. Otherwise, ** the original owner will not be able to connect. */ robustFchown(pShmNode->h, sStat.st_uid, sStat.st_gid); - - /* Check to see if another process is holding the dead-man switch. - ** If not, truncate the file to zero length. - */ - rc = SQLITE_OK; - if( unixShmSystemLock(pDbFd, F_WRLCK, UNIX_SHM_DMS, 1)==SQLITE_OK ){ - if( robust_ftruncate(pShmNode->h, 0) ){ - rc = unixLogError(SQLITE_IOERR_SHMOPEN, "ftruncate", zShmFilename); - } - } - if( rc==SQLITE_OK ){ - rc = unixShmSystemLock(pDbFd, F_RDLCK, UNIX_SHM_DMS, 1); - } - if( rc ) goto shm_open_err; + + rc = unixLockSharedMemory(pDbFd, pShmNode); + if( rc!=SQLITE_OK && rc!=SQLITE_READONLY_CANTINIT ) goto shm_open_err; } } /* Make the new connection a child of the unixShmNode */ p->pShmNode = pShmNode; @@ -4423,11 +4472,11 @@ */ sqlite3_mutex_enter(pShmNode->mutex); p->pNext = pShmNode->pFirst; pShmNode->pFirst = p; sqlite3_mutex_leave(pShmNode->mutex); - return SQLITE_OK; + return rc; /* Jump here on any error */ shm_open_err: unixShmPurge(pDbFd); /* This call frees pShmNode if required */ sqlite3_free(p); @@ -4475,10 +4524,15 @@ } p = pDbFd->pShm; pShmNode = p->pShmNode; sqlite3_mutex_enter(pShmNode->mutex); + if( pShmNode->isUnlocked ){ + rc = unixLockSharedMemory(pDbFd, pShmNode); + if( rc!=SQLITE_OK ) goto shmpage_out; + pShmNode->isUnlocked = 0; + } assert( szRegion==pShmNode->szRegion || pShmNode->nRegion==0 ); assert( pShmNode->pInode==pDbFd->pInode ); assert( pShmNode->h>=0 || pDbFd->pInode->bProcessLock==1 ); assert( pShmNode->h<0 || pDbFd->pInode->bProcessLock==0 ); Index: src/os_win.c ================================================================== --- src/os_win.c +++ src/os_win.c @@ -3671,10 +3671,13 @@ char *zFilename; /* Name of the file */ winFile hFile; /* File handle from winOpen */ int szRegion; /* Size of shared-memory regions */ int nRegion; /* Size of array apRegion */ + u8 isReadonly; /* True if read-only */ + u8 isUnlocked; /* True if no DMS lock held */ + struct ShmRegion { HANDLE hMap; /* File handle from CreateFileMapping */ void *pMap; } *aRegion; DWORD lastErrno; /* The Windows errno from the last I/O error */ @@ -3817,10 +3820,41 @@ }else{ pp = &p->pNext; } } } + +/* +** The DMS lock has not yet been taken on shm file pShmNode. Attempt to +** take it now. Return SQLITE_OK if successful, or an SQLite error +** code otherwise. +** +** If the DMS cannot be locked because this is a readonly_shm=1 +** connection and no other process already holds a lock, return +** SQLITE_READONLY_CANTINIT and set pShmNode->isUnlocked=1. +*/ +static int winLockSharedMemory(winShmNode *pShmNode){ + int rc = winShmSystemLock(pShmNode, WINSHM_WRLCK, WIN_SHM_DMS, 1); + + if( rc==SQLITE_OK ){ + if( pShmNode->isReadonly ){ + pShmNode->isUnlocked = 1; + winShmSystemLock(pShmNode, WINSHM_UNLCK, WIN_SHM_DMS, 1); + return SQLITE_READONLY_CANTINIT; + }else if( winTruncate((sqlite3_file*)&pShmNode->hFile, 0) ){ + winShmSystemLock(pShmNode, WINSHM_UNLCK, WIN_SHM_DMS, 1); + return winLogError(SQLITE_IOERR_SHMOPEN, osGetLastError(), + "winLockSharedMemory", pShmNode->zFilename); + } + } + + if( rc==SQLITE_OK ){ + winShmSystemLock(pShmNode, WINSHM_UNLCK, WIN_SHM_DMS, 1); + } + + return winShmSystemLock(pShmNode, WINSHM_RDLCK, WIN_SHM_DMS, 1); +} /* ** Open the shared-memory area associated with database file pDbFd. ** ** When opening a new shared-memory file, if no other instances of that @@ -3827,13 +3861,14 @@ ** file are currently open, in this process or in other processes, then ** the file must be truncated to zero length or have its header cleared. */ static int winOpenSharedMemory(winFile *pDbFd){ struct winShm *p; /* The connection to be opened */ - struct winShmNode *pShmNode = 0; /* The underlying mmapped file */ - int rc; /* Result code */ - struct winShmNode *pNew; /* Newly allocated winShmNode */ + winShmNode *pShmNode = 0; /* The underlying mmapped file */ + int rc = SQLITE_OK; /* Result code */ + int rc2 = SQLITE_ERROR; /* winOpen result code */ + winShmNode *pNew; /* Newly allocated winShmNode */ int nName; /* Size of zName in bytes */ assert( pDbFd->pShm==0 ); /* Not previously opened */ /* Allocate space for the new sqlite3_shm object. Also speculatively @@ -3876,34 +3911,33 @@ rc = SQLITE_IOERR_NOMEM_BKPT; goto shm_open_err; } } - rc = winOpen(pDbFd->pVfs, - pShmNode->zFilename, /* Name of the file (UTF-8) */ - (sqlite3_file*)&pShmNode->hFile, /* File handle here */ - SQLITE_OPEN_WAL | SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, - 0); - if( SQLITE_OK!=rc ){ - goto shm_open_err; - } - - /* Check to see if another process is holding the dead-man switch. - ** If not, truncate the file to zero length. - */ - if( winShmSystemLock(pShmNode, WINSHM_WRLCK, WIN_SHM_DMS, 1)==SQLITE_OK ){ - rc = winTruncate((sqlite3_file *)&pShmNode->hFile, 0); - if( rc!=SQLITE_OK ){ - rc = winLogError(SQLITE_IOERR_SHMOPEN, osGetLastError(), - "winOpenShm", pDbFd->zPath); - } - } - if( rc==SQLITE_OK ){ - winShmSystemLock(pShmNode, WINSHM_UNLCK, WIN_SHM_DMS, 1); - rc = winShmSystemLock(pShmNode, WINSHM_RDLCK, WIN_SHM_DMS, 1); - } - if( rc ) goto shm_open_err; + if( 0==sqlite3_uri_boolean(pDbFd->zPath, "readonly_shm", 0) ){ + rc2 = winOpen(pDbFd->pVfs, + pShmNode->zFilename, + (sqlite3_file*)&pShmNode->hFile, + SQLITE_OPEN_WAL|SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE, + 0); + } + if( rc2!=SQLITE_OK ){ + rc2 = winOpen(pDbFd->pVfs, + pShmNode->zFilename, + (sqlite3_file*)&pShmNode->hFile, + SQLITE_OPEN_WAL|SQLITE_OPEN_READONLY, + 0); + if( rc2!=SQLITE_OK ){ + rc = winLogError(rc2, osGetLastError(), "winOpenShm", + pShmNode->zFilename); + goto shm_open_err; + } + pShmNode->isReadonly = 1; + } + + rc = winLockSharedMemory(pShmNode); + if( rc!=SQLITE_OK && rc!=SQLITE_READONLY_CANTINIT ) goto shm_open_err; } /* Make the new connection a child of the winShmNode */ p->pShmNode = pShmNode; #if defined(SQLITE_DEBUG) || defined(SQLITE_HAVE_OS_TRACE) @@ -3922,11 +3956,11 @@ */ sqlite3_mutex_enter(pShmNode->mutex); p->pNext = pShmNode->pFirst; pShmNode->pFirst = p; sqlite3_mutex_leave(pShmNode->mutex); - return SQLITE_OK; + return rc; /* Jump here on any error */ shm_open_err: winShmSystemLock(pShmNode, WINSHM_UNLCK, WIN_SHM_DMS, 1); winShmPurge(pDbFd->pVfs, 0); /* This call frees pShmNode if required */ @@ -4126,10 +4160,12 @@ void volatile **pp /* OUT: Mapped memory */ ){ winFile *pDbFd = (winFile*)fd; winShm *pShm = pDbFd->pShm; winShmNode *pShmNode; + DWORD protect = PAGE_READWRITE; + DWORD flags = FILE_MAP_WRITE | FILE_MAP_READ; int rc = SQLITE_OK; if( !pShm ){ rc = winOpenSharedMemory(pDbFd); if( rc!=SQLITE_OK ) return rc; @@ -4136,10 +4172,15 @@ pShm = pDbFd->pShm; } pShmNode = pShm->pShmNode; sqlite3_mutex_enter(pShmNode->mutex); + if( pShmNode->isUnlocked ){ + rc = winLockSharedMemory(pShmNode); + if( rc!=SQLITE_OK ) goto shmpage_out; + pShmNode->isUnlocked = 0; + } assert( szRegion==pShmNode->szRegion || pShmNode->nRegion==0 ); if( pShmNode->nRegion<=iRegion ){ struct ShmRegion *apNew; /* New aRegion[] array */ int nByte = (iRegion+1)*szRegion; /* Minimum required file size */ @@ -4181,40 +4222,45 @@ if( !apNew ){ rc = SQLITE_IOERR_NOMEM_BKPT; goto shmpage_out; } pShmNode->aRegion = apNew; + + if( pShmNode->isReadonly ){ + protect = PAGE_READONLY; + flags = FILE_MAP_READ; + } while( pShmNode->nRegion<=iRegion ){ HANDLE hMap = NULL; /* file-mapping handle */ void *pMap = 0; /* Mapped memory region */ #if SQLITE_OS_WINRT hMap = osCreateFileMappingFromApp(pShmNode->hFile.h, - NULL, PAGE_READWRITE, nByte, NULL + NULL, protect, nByte, NULL ); #elif defined(SQLITE_WIN32_HAS_WIDE) hMap = osCreateFileMappingW(pShmNode->hFile.h, - NULL, PAGE_READWRITE, 0, nByte, NULL + NULL, protect, 0, nByte, NULL ); #elif defined(SQLITE_WIN32_HAS_ANSI) && SQLITE_WIN32_CREATEFILEMAPPINGA hMap = osCreateFileMappingA(pShmNode->hFile.h, - NULL, PAGE_READWRITE, 0, nByte, NULL + NULL, protect, 0, nByte, NULL ); #endif OSTRACE(("SHM-MAP-CREATE pid=%lu, region=%d, size=%d, rc=%s\n", osGetCurrentProcessId(), pShmNode->nRegion, nByte, hMap ? "ok" : "failed")); if( hMap ){ int iOffset = pShmNode->nRegion*szRegion; int iOffsetShift = iOffset % winSysInfo.dwAllocationGranularity; #if SQLITE_OS_WINRT - pMap = osMapViewOfFileFromApp(hMap, FILE_MAP_WRITE | FILE_MAP_READ, + pMap = osMapViewOfFileFromApp(hMap, flags, iOffset - iOffsetShift, szRegion + iOffsetShift ); #else - pMap = osMapViewOfFile(hMap, FILE_MAP_WRITE | FILE_MAP_READ, + pMap = osMapViewOfFile(hMap, flags, 0, iOffset - iOffsetShift, szRegion + iOffsetShift ); #endif OSTRACE(("SHM-MAP-MAP pid=%lu, region=%d, offset=%d, size=%d, rc=%s\n", osGetCurrentProcessId(), pShmNode->nRegion, iOffset, @@ -4241,10 +4287,11 @@ char *p = (char *)pShmNode->aRegion[iRegion].pMap; *pp = (void *)&p[iOffsetShift]; }else{ *pp = 0; } + if( pShmNode->isReadonly && rc==SQLITE_OK ) rc = SQLITE_READONLY; sqlite3_mutex_leave(pShmNode->mutex); return rc; } #else Index: src/printf.c ================================================================== --- src/printf.c +++ src/printf.c @@ -1090,12 +1090,19 @@ sqlite3StrAccumInit(&acc, 0, zBuf, sizeof(zBuf), 0); va_start(ap,zFormat); sqlite3VXPrintf(&acc, zFormat, ap); va_end(ap); sqlite3StrAccumFinish(&acc); +#ifdef SQLITE_OS_TRACE_PROC + { + extern void SQLITE_OS_TRACE_PROC(const char *zBuf, int nBuf); + SQLITE_OS_TRACE_PROC(zBuf, sizeof(zBuf)); + } +#else fprintf(stdout,"%s", zBuf); fflush(stdout); +#endif } #endif /* Index: src/sqlite.h.in ================================================================== --- src/sqlite.h.in +++ src/sqlite.h.in @@ -506,15 +506,17 @@ #define SQLITE_BUSY_SNAPSHOT (SQLITE_BUSY | (2<<8)) #define SQLITE_CANTOPEN_NOTEMPDIR (SQLITE_CANTOPEN | (1<<8)) #define SQLITE_CANTOPEN_ISDIR (SQLITE_CANTOPEN | (2<<8)) #define SQLITE_CANTOPEN_FULLPATH (SQLITE_CANTOPEN | (3<<8)) #define SQLITE_CANTOPEN_CONVPATH (SQLITE_CANTOPEN | (4<<8)) +#define SQLITE_CANTOPEN_DIRTYWAL (SQLITE_CANTOPEN | (5<<8)) #define SQLITE_CORRUPT_VTAB (SQLITE_CORRUPT | (1<<8)) #define SQLITE_READONLY_RECOVERY (SQLITE_READONLY | (1<<8)) #define SQLITE_READONLY_CANTLOCK (SQLITE_READONLY | (2<<8)) #define SQLITE_READONLY_ROLLBACK (SQLITE_READONLY | (3<<8)) #define SQLITE_READONLY_DBMOVED (SQLITE_READONLY | (4<<8)) +#define SQLITE_READONLY_CANTINIT (SQLITE_READONLY | (5<<8)) #define SQLITE_ABORT_ROLLBACK (SQLITE_ABORT | (2<<8)) #define SQLITE_CONSTRAINT_CHECK (SQLITE_CONSTRAINT | (1<<8)) #define SQLITE_CONSTRAINT_COMMITHOOK (SQLITE_CONSTRAINT | (2<<8)) #define SQLITE_CONSTRAINT_FOREIGNKEY (SQLITE_CONSTRAINT | (3<<8)) #define SQLITE_CONSTRAINT_FUNCTION (SQLITE_CONSTRAINT | (4<<8)) Index: src/wal.c ================================================================== --- src/wal.c +++ src/wal.c @@ -453,10 +453,11 @@ u8 ckptLock; /* True if holding a checkpoint lock */ u8 readOnly; /* WAL_RDWR, WAL_RDONLY, or WAL_SHM_RDONLY */ u8 truncateOnCommit; /* True to truncate WAL file on commit */ u8 syncHeader; /* Fsync the WAL header if true */ u8 padToSectorBoundary; /* Pad transactions out to the next sector */ + u8 bShmUnreliable; /* SHM content is read-only and unreliable */ WalIndexHdr hdr; /* Wal-index header for current transaction */ u32 minFrame; /* Ignore wal frames before this one */ u32 iReCksum; /* On commit, recalculate checksums from here */ const char *zWalName; /* Name of WAL file */ u32 nCkpt; /* Checkpoint sequence counter in the wal-header */ @@ -541,10 +542,15 @@ /* ** Obtain a pointer to the iPage'th page of the wal-index. The wal-index ** is broken into pages of WALINDEX_PGSZ bytes. Wal-index pages are ** numbered from zero. +** +** If the wal-index is currently smaller the iPage pages then the size +** of the wal-index might be increased, but only if it is safe to do +** so. It is safe to enlarge the wal-index if pWal->writeLock is true +** or pWal->exclusiveMode==WAL_HEAPMEMORY_MODE. ** ** If this call is successful, *ppPage is set to point to the wal-index ** page and SQLITE_OK is returned. If an error (an OOM or VFS error) occurs, ** then an SQLite error code is returned and *ppPage is set to 0. */ @@ -573,13 +579,17 @@ if( !pWal->apWiData[iPage] ) rc = SQLITE_NOMEM_BKPT; }else{ rc = sqlite3OsShmMap(pWal->pDbFd, iPage, WALINDEX_PGSZ, pWal->writeLock, (void volatile **)&pWal->apWiData[iPage] ); - if( rc==SQLITE_READONLY ){ + assert( pWal->apWiData[iPage]!=0 || rc!=SQLITE_OK || pWal->writeLock==0 ); + testcase( pWal->apWiData[iPage]==0 && rc==SQLITE_OK ); + if( (rc&0xff)==SQLITE_READONLY ){ pWal->readOnly |= WAL_SHM_RDONLY; - rc = SQLITE_OK; + if( rc==SQLITE_READONLY ){ + rc = SQLITE_OK; + } } } } *ppPage = pWal->apWiData[iPage]; @@ -1097,11 +1107,10 @@ static int walIndexRecover(Wal *pWal){ int rc; /* Return Code */ i64 nSize; /* Size of log file */ u32 aFrameCksum[2] = {0, 0}; int iLock; /* Lock offset to lock for checkpoint */ - int nLock; /* Number of locks to hold */ /* Obtain an exclusive lock on all byte in the locking range not already ** locked by the caller. The caller is guaranteed to have locked the ** WAL_WRITE_LOCK byte, and may have also locked the WAL_CKPT_LOCK byte. ** If successful, the same bytes that are locked here are unlocked before @@ -1110,15 +1119,21 @@ assert( pWal->ckptLock==1 || pWal->ckptLock==0 ); assert( WAL_ALL_BUT_WRITE==WAL_WRITE_LOCK+1 ); assert( WAL_CKPT_LOCK==WAL_ALL_BUT_WRITE ); assert( pWal->writeLock ); iLock = WAL_ALL_BUT_WRITE + pWal->ckptLock; - nLock = SQLITE_SHM_NLOCK - iLock; - rc = walLockExclusive(pWal, iLock, nLock); + rc = walLockExclusive(pWal, iLock, WAL_READ_LOCK(0)-iLock); + if( rc==SQLITE_OK ){ + rc = walLockExclusive(pWal, WAL_READ_LOCK(1), WAL_NREADER-1); + if( rc!=SQLITE_OK ){ + walUnlockExclusive(pWal, iLock, WAL_READ_LOCK(0)-iLock); + } + } if( rc ){ return rc; } + WALTRACE(("WAL%p: recovery begin...\n", pWal)); memset(&pWal->hdr, 0, sizeof(WalIndexHdr)); rc = sqlite3OsFileSize(pWal->pWalFd, &nSize); @@ -1252,25 +1267,27 @@ } } recovery_error: WALTRACE(("WAL%p: recovery %s\n", pWal, rc ? "failed" : "ok")); - walUnlockExclusive(pWal, iLock, nLock); + walUnlockExclusive(pWal, iLock, WAL_READ_LOCK(0)-iLock); + walUnlockExclusive(pWal, WAL_READ_LOCK(1), WAL_NREADER-1); return rc; } /* ** Close an open wal-index. */ static void walIndexClose(Wal *pWal, int isDelete){ - if( pWal->exclusiveMode==WAL_HEAPMEMORY_MODE ){ + if( pWal->exclusiveMode==WAL_HEAPMEMORY_MODE || pWal->bShmUnreliable ){ int i; for(i=0; inWiData; i++){ sqlite3_free((void *)pWal->apWiData[i]); pWal->apWiData[i] = 0; } - }else{ + } + if( pWal->exclusiveMode!=WAL_HEAPMEMORY_MODE ){ sqlite3OsShmUnmap(pWal->pDbFd, isDelete); } } /* @@ -2059,10 +2076,16 @@ /* The header was successfully read. Return zero. */ return 0; } +/* +** This is the value that walTryBeginRead returns when it needs to +** be retried. +*/ +#define WAL_RETRY (-1) + /* ** Read the wal-index header from the wal-index and into pWal->hdr. ** If the wal-header appears to be corrupt, try to reconstruct the ** wal-index from the WAL before returning. ** @@ -2082,13 +2105,33 @@ ** wal-index header) is mapped. Return early if an error occurs here. */ assert( pChanged ); rc = walIndexPage(pWal, 0, &page0); if( rc!=SQLITE_OK ){ - return rc; - }; - assert( page0 || pWal->writeLock==0 ); + assert( rc!=SQLITE_READONLY ); /* READONLY changed to OK in walIndexPage */ + if( rc==SQLITE_READONLY_CANTINIT ){ + /* The SQLITE_READONLY_CANTINIT return means that the shared-memory + ** was openable but is not writable, and this thread is unable to + ** confirm that another write-capable connection has the shared-memory + ** open, and hence the content of the shared-memory is unreliable, + ** since the shared-memory might be inconsistent with the WAL file + ** and there is no writer on hand to fix it. */ + assert( page0==0 ); + assert( pWal->writeLock==0 ); + assert( pWal->readOnly & WAL_SHM_RDONLY ); + pWal->bShmUnreliable = 1; + pWal->exclusiveMode = WAL_HEAPMEMORY_MODE; + *pChanged = 1; + }else{ + return rc; /* Any other non-OK return is just an error */ + } + }else{ + /* page0 can be NULL if the SHM is zero bytes in size and pWal->writeLock + ** is zero, which prevents the SHM from growing */ + testcase( page0!=0 ); + } + assert( page0!=0 || pWal->writeLock==0 ); /* If the first page of the wal-index has been mapped, try to read the ** wal-index header immediately, without holding any lock. This usually ** works, but may fail if the wal-index header is corrupt or currently ** being modified by another thread or process. @@ -2098,11 +2141,11 @@ /* If the first attempt failed, it might have been due to a race ** with a writer. So get a WRITE lock and try again. */ assert( badHdr==0 || pWal->writeLock==0 ); if( badHdr ){ - if( pWal->readOnly & WAL_SHM_RDONLY ){ + if( pWal->bShmUnreliable==0 && (pWal->readOnly & WAL_SHM_RDONLY) ){ if( SQLITE_OK==(rc = walLockShared(pWal, WAL_WRITE_LOCK)) ){ walUnlockShared(pWal, WAL_WRITE_LOCK); rc = SQLITE_READONLY_RECOVERY; } }else if( SQLITE_OK==(rc = walLockExclusive(pWal, WAL_WRITE_LOCK, 1)) ){ @@ -2128,19 +2171,197 @@ ** this version of SQLite cannot understand. */ if( badHdr==0 && pWal->hdr.iVersion!=WALINDEX_MAX_VERSION ){ rc = SQLITE_CANTOPEN_BKPT; } + if( pWal->bShmUnreliable ){ + if( rc!=SQLITE_OK ){ + walIndexClose(pWal, 0); + pWal->bShmUnreliable = 0; + assert( pWal->nWiData>0 && pWal->apWiData[0]==0 ); + /* walIndexRecover() might have returned SHORT_READ if a concurrent + ** writer truncated the WAL out from under it. If that happens, it + ** indicates that a writer has fixed the SHM file for us, so retry */ + if( rc==SQLITE_IOERR_SHORT_READ ) rc = WAL_RETRY; + } + pWal->exclusiveMode = WAL_NORMAL_MODE; + } return rc; } /* -** This is the value that walTryBeginRead returns when it needs to -** be retried. +** Open a transaction in a connection where the shared-memory is read-only +** and where we cannot verify that there is a separate write-capable connection +** on hand to keep the shared-memory up-to-date with the WAL file. +** +** This can happen, for example, when the shared-memory is implemented by +** memory-mapping a *-shm file, where a prior writer has shut down and +** left the *-shm file on disk, and now the present connection is trying +** to use that database but lacks write permission on the *-shm file. +** Other scenarios are also possible, depending on the VFS implementation. +** +** Precondition: +** +** The *-wal file has been read and an appropriate wal-index has been +** constructed in pWal->apWiData[] using heap memory instead of shared +** memory. +** +** If this function returns SQLITE_OK, then the read transaction has +** been successfully opened. In this case output variable (*pChanged) +** is set to true before returning if the caller should discard the +** contents of the page cache before proceeding. Or, if it returns +** WAL_RETRY, then the heap memory wal-index has been discarded and +** the caller should retry opening the read transaction from the +** beginning (including attempting to map the *-shm file). +** +** If an error occurs, an SQLite error code is returned. */ -#define WAL_RETRY (-1) +static int walBeginShmUnreliable(Wal *pWal, int *pChanged){ + i64 szWal; /* Size of wal file on disk in bytes */ + i64 iOffset; /* Current offset when reading wal file */ + u8 aBuf[WAL_HDRSIZE]; /* Buffer to load WAL header into */ + u8 *aFrame = 0; /* Malloc'd buffer to load entire frame */ + int szFrame; /* Number of bytes in buffer aFrame[] */ + u8 *aData; /* Pointer to data part of aFrame buffer */ + volatile void *pDummy; /* Dummy argument for xShmMap */ + int rc; /* Return code */ + u32 aSaveCksum[2]; /* Saved copy of pWal->hdr.aFrameCksum */ + + assert( pWal->bShmUnreliable ); + assert( pWal->readOnly & WAL_SHM_RDONLY ); + assert( pWal->nWiData>0 && pWal->apWiData[0] ); + + /* Take WAL_READ_LOCK(0). This has the effect of preventing any + ** writers from running a checkpoint, but does not stop them + ** from running recovery. */ + rc = walLockShared(pWal, WAL_READ_LOCK(0)); + if( rc!=SQLITE_OK ){ + if( rc==SQLITE_BUSY ) rc = WAL_RETRY; + goto begin_unreliable_shm_out; + } + pWal->readLock = 0; + + /* Check to see if a separate writer has attached to the shared-memory area, + ** thus making the shared-memory "reliable" again. Do this by invoking + ** the xShmMap() routine of the VFS and looking to see if the return + ** is SQLITE_READONLY instead of SQLITE_READONLY_CANTINIT. + ** + ** If the shared-memory is now "reliable" return WAL_RETRY, which will + ** cause the heap-memory WAL-index to be discarded and the actual + ** shared memory to be used in its place. + ** + ** This step is important because, even though this connection is holding + ** the WAL_READ_LOCK(0) which prevents a checkpoint, a writer might + ** have already checkpointed the WAL file and, while the current + ** is active, wrap the WAL and start overwriting frames that this + ** process wants to use. + ** + ** Once sqlite3OsShmMap() has been called for an sqlite3_file and has + ** returned any SQLITE_READONLY value, it must return only SQLITE_READONLY + ** or SQLITE_READONLY_CANTINIT or some error for all subsequent invocations, + ** even if some external agent does a "chmod" to make the shared-memory + ** writable by us, until sqlite3OsShmUnmap() has been called. + ** This is a requirement on the VFS implementation. + */ + rc = sqlite3OsShmMap(pWal->pDbFd, 0, WALINDEX_PGSZ, 0, &pDummy); + assert( rc!=SQLITE_OK ); /* SQLITE_OK not possible for read-only connection */ + if( rc!=SQLITE_READONLY_CANTINIT ){ + rc = (rc==SQLITE_READONLY ? WAL_RETRY : rc); + goto begin_unreliable_shm_out; + } + + /* We reach this point only if the real shared-memory is still unreliable. + ** Assume the in-memory WAL-index substitute is correct and load it + ** into pWal->hdr. + */ + memcpy(&pWal->hdr, (void*)walIndexHdr(pWal), sizeof(WalIndexHdr)); + + /* Make sure some writer hasn't come in and changed the WAL file out + ** from under us, then disconnected, while we were not looking. + */ + rc = sqlite3OsFileSize(pWal->pWalFd, &szWal); + if( rc!=SQLITE_OK ){ + goto begin_unreliable_shm_out; + } + if( szWalhdr.mxFrame==0 ? SQLITE_OK : WAL_RETRY); + goto begin_unreliable_shm_out; + } + + /* Check the salt keys at the start of the wal file still match. */ + rc = sqlite3OsRead(pWal->pWalFd, aBuf, WAL_HDRSIZE, 0); + if( rc!=SQLITE_OK ){ + goto begin_unreliable_shm_out; + } + if( memcmp(&pWal->hdr.aSalt, &aBuf[16], 8) ){ + /* Some writer has wrapped the WAL file while we were not looking. + ** Return WAL_RETRY which will cause the in-memory WAL-index to be + ** rebuilt. */ + rc = WAL_RETRY; + goto begin_unreliable_shm_out; + } + + /* Allocate a buffer to read frames into */ + szFrame = pWal->hdr.szPage + WAL_FRAME_HDRSIZE; + aFrame = (u8 *)sqlite3_malloc64(szFrame); + if( aFrame==0 ){ + rc = SQLITE_NOMEM_BKPT; + goto begin_unreliable_shm_out; + } + aData = &aFrame[WAL_FRAME_HDRSIZE]; + + /* Check to see if a complete transaction has been appended to the + ** wal file since the heap-memory wal-index was created. If so, the + ** heap-memory wal-index is discarded and WAL_RETRY returned to + ** the caller. */ + aSaveCksum[0] = pWal->hdr.aFrameCksum[0]; + aSaveCksum[1] = pWal->hdr.aFrameCksum[1]; + for(iOffset=walFrameOffset(pWal->hdr.mxFrame+1, pWal->hdr.szPage); + iOffset+szFrame<=szWal; + iOffset+=szFrame + ){ + u32 pgno; /* Database page number for frame */ + u32 nTruncate; /* dbsize field from frame header */ + + /* Read and decode the next log frame. */ + rc = sqlite3OsRead(pWal->pWalFd, aFrame, szFrame, iOffset); + if( rc!=SQLITE_OK ) break; + if( !walDecodeFrame(pWal, &pgno, &nTruncate, aData, aFrame) ) break; + + /* If nTruncate is non-zero, then a complete transaction has been + ** appended to this wal file. Set rc to WAL_RETRY and break out of + ** the loop. */ + if( nTruncate ){ + rc = WAL_RETRY; + break; + } + } + pWal->hdr.aFrameCksum[0] = aSaveCksum[0]; + pWal->hdr.aFrameCksum[1] = aSaveCksum[1]; + + begin_unreliable_shm_out: + sqlite3_free(aFrame); + if( rc!=SQLITE_OK ){ + int i; + for(i=0; inWiData; i++){ + sqlite3_free((void*)pWal->apWiData[i]); + pWal->apWiData[i] = 0; + } + pWal->bShmUnreliable = 0; + sqlite3WalEndReadTransaction(pWal); + *pChanged = 1; + } + return rc; +} /* ** Attempt to start a read transaction. This might fail due to a race or ** other transient condition. When that happens, it returns WAL_RETRY to ** indicate to the caller that it is safe to retry immediately. @@ -2198,10 +2419,13 @@ int rc = SQLITE_OK; /* Return code */ u32 mxFrame; /* Wal frame to lock to */ assert( pWal->readLock<0 ); /* Not currently locked */ + /* useWal may only be set for read/write connections */ + assert( (pWal->readOnly & WAL_SHM_RDONLY)==0 || useWal==0 ); + /* Take steps to avoid spinning forever if there is a protocol error. ** ** Circumstances that cause a RETRY should only last for the briefest ** instances of time. No I/O or other system calls are done while the ** locks are held, so the locks should not be held for very long. But @@ -2226,11 +2450,14 @@ if( cnt>=10 ) nDelay = (cnt-9)*(cnt-9)*39; sqlite3OsSleep(pWal->pVfs, nDelay); } if( !useWal ){ - rc = walIndexReadHdr(pWal, pChanged); + assert( rc==SQLITE_OK ); + if( pWal->bShmUnreliable==0 ){ + rc = walIndexReadHdr(pWal, pChanged); + } if( rc==SQLITE_BUSY ){ /* If there is not a recovery running in another thread or process ** then convert BUSY errors to WAL_RETRY. If recovery is known to ** be running, convert BUSY to BUSY_RECOVERY. There is a race here ** which might cause WAL_RETRY to be returned even if BUSY_RECOVERY @@ -2255,14 +2482,19 @@ } } if( rc!=SQLITE_OK ){ return rc; } + else if( pWal->bShmUnreliable ){ + return walBeginShmUnreliable(pWal, pChanged); + } } + assert( pWal->nWiData>0 ); + assert( pWal->apWiData[0]!=0 ); pInfo = walCkptInfo(pWal); - if( !useWal && pInfo->nBackfill==pWal->hdr.mxFrame + if( !useWal && pInfo->nBackfill==pWal->hdr.mxFrame #ifdef SQLITE_ENABLE_SNAPSHOT && (pWal->pSnapshot==0 || pWal->hdr.mxFrame==0 || 0==memcmp(&pWal->hdr, pWal->pSnapshot, sizeof(WalIndexHdr))) #endif ){ @@ -2332,11 +2564,11 @@ } } } if( mxI==0 ){ assert( rc==SQLITE_BUSY || (pWal->readOnly & WAL_SHM_RDONLY)!=0 ); - return rc==SQLITE_BUSY ? WAL_RETRY : SQLITE_READONLY_CANTLOCK; + return rc==SQLITE_BUSY ? WAL_RETRY : SQLITE_READONLY_CANTINIT; } rc = walLockShared(pWal, WAL_READ_LOCK(mxI)); if( rc ){ return rc==SQLITE_BUSY ? WAL_RETRY : rc; @@ -2604,11 +2836,11 @@ ** no data will be read from the wal under any circumstances. Return early ** in this case as an optimization. Likewise, if pWal->readLock==0, ** then the WAL is ignored by the reader so return early, as if the ** WAL were empty. */ - if( iLast==0 || pWal->readLock==0 ){ + if( iLast==0 || (pWal->readLock==0 && pWal->bShmUnreliable==0) ){ *piRead = 0; return SQLITE_OK; } /* Search the hash table or tables for an entry matching page number @@ -2667,12 +2899,12 @@ ** of the wal-index file content. Make sure the results agree with the ** result obtained using the hash indexes above. */ { u32 iRead2 = 0; u32 iTest; - assert( pWal->minFrame>0 ); - for(iTest=iLast; iTest>=pWal->minFrame; iTest--){ + assert( pWal->bShmUnreliable || pWal->minFrame>0 ); + for(iTest=iLast; iTest>=pWal->minFrame && iTest>0; iTest--){ if( walFramePgno(pWal, iTest)==pgno ){ iRead2 = iTest; break; } } @@ -3444,28 +3676,28 @@ */ assert( pWal->readLock>=0 || pWal->lockError ); assert( pWal->readLock>=0 || (op<=0 && pWal->exclusiveMode==0) ); if( op==0 ){ - if( pWal->exclusiveMode ){ - pWal->exclusiveMode = 0; + if( pWal->exclusiveMode!=WAL_NORMAL_MODE ){ + pWal->exclusiveMode = WAL_NORMAL_MODE; if( walLockShared(pWal, WAL_READ_LOCK(pWal->readLock))!=SQLITE_OK ){ - pWal->exclusiveMode = 1; + pWal->exclusiveMode = WAL_EXCLUSIVE_MODE; } - rc = pWal->exclusiveMode==0; + rc = pWal->exclusiveMode==WAL_NORMAL_MODE; }else{ /* Already in locking_mode=NORMAL */ rc = 0; } }else if( op>0 ){ - assert( pWal->exclusiveMode==0 ); + assert( pWal->exclusiveMode==WAL_NORMAL_MODE ); assert( pWal->readLock>=0 ); walUnlockShared(pWal, WAL_READ_LOCK(pWal->readLock)); - pWal->exclusiveMode = 1; + pWal->exclusiveMode = WAL_EXCLUSIVE_MODE; rc = 1; }else{ - rc = pWal->exclusiveMode==0; + rc = pWal->exclusiveMode==WAL_NORMAL_MODE; } return rc; } /* Index: test/wal2.test ================================================================== --- test/wal2.test +++ test/wal2.test @@ -120,12 +120,12 @@ do_test wal2-1.1 { execsql { SELECT count(a), sum(a) FROM t1 } db2 } {4 10} set RECOVER [list \ - {0 1 lock exclusive} {1 7 lock exclusive} \ - {1 7 unlock exclusive} {0 1 unlock exclusive} \ + {0 1 lock exclusive} {1 2 lock exclusive} {4 4 lock exclusive} \ + {1 2 unlock exclusive} {4 4 unlock exclusive} {0 1 unlock exclusive} \ ] set READ [list \ {4 1 lock shared} {4 1 unlock shared} \ ] set INITSLOT [list \ @@ -391,12 +391,14 @@ # recovery performed as a pre-cursor to a normal database transaction). # set expected_locks [list] lappend expected_locks {1 1 lock exclusive} ;# Lock checkpoint lappend expected_locks {0 1 lock exclusive} ;# Lock writer -lappend expected_locks {2 6 lock exclusive} ;# Lock recovery & all aReadMark[] -lappend expected_locks {2 6 unlock exclusive} ;# Unlock recovery & aReadMark[] +lappend expected_locks {2 1 lock exclusive} ;# Lock recovery +lappend expected_locks {4 4 lock exclusive} ;# Lock all aReadMark[] +lappend expected_locks {2 1 unlock exclusive} ;# Unlock recovery +lappend expected_locks {4 4 unlock exclusive} ;# Unlock all aReadMark[] lappend expected_locks {0 1 unlock exclusive} ;# Unlock writer lappend expected_locks {3 1 lock exclusive} ;# Lock aReadMark[0] lappend expected_locks {3 1 unlock exclusive} ;# Unlock aReadMark[0] lappend expected_locks {1 1 unlock exclusive} ;# Unlock checkpoint do_test wal2-5.1 { @@ -613,12 +615,12 @@ sqlite3 db test.db -vfs tvfs set {} {} } {} set RECOVERY { - {0 1 lock exclusive} {1 7 lock exclusive} - {1 7 unlock exclusive} {0 1 unlock exclusive} + {0 1 lock exclusive} {1 2 lock exclusive} {4 4 lock exclusive} + {1 2 unlock exclusive} {4 4 unlock exclusive} {0 1 unlock exclusive} } set READMARK0_READ { {3 1 lock shared} {3 1 unlock shared} } set READMARK0_WRITE { @@ -1126,11 +1128,11 @@ faultsim_save_and_close foreach {tn db_perm wal_perm shm_perm can_open can_read can_write} { 2 00644 00644 00644 1 1 1 3 00644 00400 00644 1 1 0 - 4 00644 00644 00400 1 0 0 + 4 00644 00644 00400 1 1 0 5 00400 00644 00644 1 1 0 7 00644 00000 00644 1 0 0 8 00644 00644 00000 1 0 0 9 00000 00644 00644 0 0 0 Index: test/walro.test ================================================================== --- test/walro.test +++ test/walro.test @@ -99,26 +99,27 @@ do_test 1.2.1 { code2 { db2 close } code1 { db close } list [file exists test.db-wal] [file exists test.db-shm] } {1 1} + do_test 1.2.2 { code1 { sqlite3 db file:test.db?readonly_shm=1 } - sql1 { SELECT * FROM t1 } - } {a b c d e f g h i j} + list [catch { sql1 { SELECT * FROM t1 } } msg] $msg + } {0 {a b c d e f g h i j}} do_test 1.2.3 { code1 { db close } file attributes test.db-shm -permissions rw-r--r-- hexio_write test.db-shm 0 01020304 file attributes test.db-shm -permissions r--r--r-- code1 { sqlite3 db file:test.db?readonly_shm=1 } csql1 { SELECT * FROM t1 } - } {1 {attempt to write a readonly database}} + } {0 {a b c d e f g h i j}} do_test 1.2.4 { code1 { sqlite3_extended_errcode db } - } {SQLITE_READONLY_RECOVERY} + } {SQLITE_OK} do_test 1.2.5 { file attributes test.db-shm -permissions rw-r--r-- code2 { sqlite3 db2 test.db } sql2 "SELECT * FROM t1" @@ -136,15 +137,19 @@ do_test 1.2.8 { sql1 "SELECT * FROM t1" } {a b c d e f g h i j k l} # Now check that if the readonly_shm option is not supplied, or if it # is set to zero, it is not possible to connect to the database without # read-write access to the shm. + # + # UPDATE: os_unix.c now opens the *-shm file in readonly mode + # automatically. + # do_test 1.3.1 { code1 { db close } code1 { sqlite3 db test.db } csql1 { SELECT * FROM t1 } - } {1 {unable to open database file}} + } {0 {a b c d e f g h i j k l}} # Also test that if the -shm file can be opened for read/write access, # it is not if readonly_shm=1 is present in the URI. do_test 1.3.2.1 { code1 { db close } @@ -159,14 +164,14 @@ code1 { db close } close [open test.db-shm w] file attributes test.db-shm -permissions r--r--r-- code1 { sqlite3 db file:test.db?readonly_shm=1 } csql1 { SELECT * FROM t1 } - } {1 {attempt to write a readonly database}} + } {0 {a b c d e f g h i j k l}} do_test 1.3.2.4 { code1 { sqlite3_extended_errcode db } - } {SQLITE_READONLY_RECOVERY} + } {SQLITE_OK} #----------------------------------------------------------------------- # Test cases 1.4.* check that checkpoints and log wraps don't prevent # read-only connections from reading the database. do_test 1.4.1 { ADDED test/walro2.test Index: test/walro2.test ================================================================== --- /dev/null +++ test/walro2.test @@ -0,0 +1,393 @@ +# 2011 May 09 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file contains tests for using WAL databases in read-only mode. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/lock_common.tcl +source $testdir/wal_common.tcl +set ::testprefix walro2 + +# And only if the build is WAL-capable. +# +ifcapable !wal { + finish_test + return +} + +proc copy_to_test2 {bZeroShm} { + forcecopy test.db test.db2 + forcecopy test.db-wal test.db2-wal + if {$bZeroShm} { + forcedelete test.db2-shm + set fd [open test.db2-shm w] + seek $fd [expr [file size test.db-shm]-1] + puts -nonewline $fd "\0" + close $fd + } else { + forcecopy test.db-shm test.db2-shm + } +} + +foreach bZeroShm {0 1} { +set TN [expr $bZeroShm+1] +do_multiclient_test tn { + + # Close all connections and delete the database. + # + code1 { db close } + code2 { db2 close } + code3 { db3 close } + forcedelete test.db + + # Do not run tests with the connections in the same process. + # + if {$tn==2} continue + + foreach c {code1 code2 code3} { + $c { + sqlite3_shutdown + sqlite3_config_uri 1 + } + } + + do_test $TN.1.1 { + code2 { sqlite3 db2 test.db } + sql2 { + CREATE TABLE t1(x, y); + PRAGMA journal_mode = WAL; + INSERT INTO t1 VALUES('a', 'b'); + INSERT INTO t1 VALUES('c', 'd'); + } + file exists test.db-shm + } {1} + + do_test $TN.1.2.1 { + copy_to_test2 $bZeroShm + code1 { + sqlite3 db file:test.db2?readonly_shm=1 + } + + sql1 { SELECT * FROM t1 } + } {a b c d} + do_test $TN.1.2.2 { + sql1 { SELECT * FROM t1 } + } {a b c d} + + do_test $TN.1.3.1 { + code3 { sqlite3 db3 test.db2 } + sql3 { SELECT * FROM t1 } + } {a b c d} + + do_test $TN.1.3.2 { + sql1 { SELECT * FROM t1 } + } {a b c d} + + code1 { db close } + code2 { db2 close } + code3 { db3 close } + + do_test $TN.2.1 { + code2 { sqlite3 db2 test.db } + sql2 { + INSERT INTO t1 VALUES('e', 'f'); + INSERT INTO t1 VALUES('g', 'h'); + } + file exists test.db-shm + } {1} + + do_test $TN.2.2 { + copy_to_test2 $bZeroShm + code1 { + sqlite3 db file:test.db2?readonly_shm=1 + } + sql1 { + BEGIN; + SELECT * FROM t1; + } + } {a b c d e f g h} + + do_test $TN.2.3.1 { + code3 { sqlite3 db3 test.db2 } + sql3 { SELECT * FROM t1 } + } {a b c d e f g h} + do_test $TN.2.3.2 { + sql3 { INSERT INTO t1 VALUES('i', 'j') } + code3 { db3 close } + sql1 { COMMIT } + } {} + do_test $TN.2.3.3 { + sql1 { SELECT * FROM t1 } + } {a b c d e f g h i j} + + + #----------------------------------------------------------------------- + # 3.1.*: That a readonly_shm connection can read a database file if both + # the *-wal and *-shm files are zero bytes in size. + # + # 3.2.*: That it flushes the cache if, between transactions on a db with a + # zero byte *-wal file, some other connection modifies the db, then + # does "PRAGMA wal_checkpoint=truncate" to truncate the wal file + # back to zero bytes in size. + # + # 3.3.*: That, if between transactions some other process wraps the wal + # file, the readonly_shm client reruns recovery. + # + catch { code1 { db close } } + catch { code2 { db2 close } } + catch { code3 { db3 close } } + do_test $TN.3.1.0 { + list [file exists test.db-wal] [file exists test.db-shm] + } {0 0} + do_test $TN.3.1.1 { + close [open test.db-wal w] + close [open test.db-shm w] + code1 { + sqlite3 db file:test.db?readonly_shm=1 + } + sql1 { SELECT * FROM t1 } + } {a b c d e f g h} + + do_test $TN.3.2.0 { + list [file size test.db-wal] [file size test.db-shm] + } {0 0} + do_test $TN.3.2.1 { + code2 { sqlite3 db2 test.db } + sql2 { INSERT INTO t1 VALUES(1, 2) ; PRAGMA wal_checkpoint=truncate } + code2 { db2 close } + sql1 { SELECT * FROM t1 } + } {a b c d e f g h 1 2} + do_test $TN.3.2.2 { + list [file size test.db-wal] [file size test.db-shm] + } {0 32768} + + do_test $TN.3.3.0 { + code2 { sqlite3 db2 test.db } + sql2 { + INSERT INTO t1 VALUES(3, 4); + INSERT INTO t1 VALUES(5, 6); + INSERT INTO t1 VALUES(7, 8); + INSERT INTO t1 VALUES(9, 10); + } + code2 { db2 close } + code1 { db close } + list [file size test.db-wal] [file size test.db-shm] + } [list [wal_file_size 4 1024] 32768] + do_test $TN.3.3.1 { + code1 { sqlite3 db file:test.db?readonly_shm=1 } + sql1 { SELECT * FROM t1 } + } {a b c d e f g h 1 2 3 4 5 6 7 8 9 10} + do_test $TN.3.3.2 { + code2 { sqlite3 db2 test.db } + sql2 { + PRAGMA wal_checkpoint; + DELETE FROM t1; + INSERT INTO t1 VALUES('i', 'ii'); + } + code2 { db2 close } + list [file size test.db-wal] [file size test.db-shm] + } [list [wal_file_size 4 1024] 32768] + do_test $TN.3.3.3 { + sql1 { SELECT * FROM t1 } + } {i ii} + + #----------------------------------------------------------------------- + # + # + catch { code1 { db close } } + catch { code2 { db2 close } } + catch { code3 { db3 close } } + + do_test $TN.4.0 { + code1 { forcedelete test.db } + code1 { sqlite3 db test.db } + sql1 { + PRAGMA journal_mode = wal; + CREATE TABLE t1(x); + INSERT INTO t1 VALUES('hello'); + INSERT INTO t1 VALUES('world'); + } + + copy_to_test2 $bZeroShm + + code1 { db close } + } {} + + do_test $TN.4.1.1 { + code2 { sqlite3 db2 file:test.db2?readonly_shm=1 } + sql2 { SELECT * FROM t1 } + } {hello world} + + do_test $TN.4.1.2 { + code3 { sqlite3 db3 test.db2 } + sql3 { + INSERT INTO t1 VALUES('!'); + PRAGMA wal_checkpoint = truncate; + } + code3 { db3 close } + } {} + do_test $TN.4.1.3 { + sql2 { SELECT * FROM t1 } + } {hello world !} + + catch { code1 { db close } } + catch { code2 { db2 close } } + catch { code3 { db3 close } } + + do_test $TN.4.2.1 { + code1 { sqlite3 db test.db } + sql1 { + INSERT INTO t1 VALUES('!'); + INSERT INTO t1 VALUES('!'); + + PRAGMA cache_size = 10; + CREATE TABLE t2(x); + + BEGIN; + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<500 + ) + INSERT INTO t2 SELECT randomblob(500) FROM s; + SELECT count(*) FROM t2; + } + } {500} + do_test $TN.4.2.2 { + file size test.db-wal + } {461152} + do_test $TN.4.2.4 { + file_control_persist_wal db 1; db close + + copy_to_test2 $bZeroShm + code2 { sqlite3 db2 file:test.db2?readonly_shm=1 } + sql2 { + SELECT * FROM t1; + SELECT count(*) FROM t2; + } + } {hello world ! ! 0} + + #----------------------------------------------------------------------- + # + # + catch { code1 { db close } } + catch { code2 { db2 close } } + catch { code3 { db3 close } } + + do_test $TN.5.0 { + code1 { forcedelete test.db } + code1 { sqlite3 db test.db } + sql1 { + PRAGMA journal_mode = wal; + CREATE TABLE t1(x); + INSERT INTO t1 VALUES('hello'); + INSERT INTO t1 VALUES('world'); + INSERT INTO t1 VALUES('!'); + INSERT INTO t1 VALUES('world'); + INSERT INTO t1 VALUES('hello'); + } + + copy_to_test2 $bZeroShm + + code1 { db close } + } {} + + do_test $TN.5.1 { + code2 { sqlite3 db2 file:test.db2?readonly_shm=1 } + sql2 { + SELECT * FROM t1; + } + } {hello world ! world hello} + + do_test $TN.5.2 { + code1 { + proc handle_read {op args} { + if {$op=="xRead" && [file tail [lindex $args 0]]=="test.db2-wal"} { + set ::res2 [sql2 { SELECT * FROM t1 }] + } + puts "$msg xRead $args" + return "SQLITE_OK" + } + testvfs tvfs -fullshm 1 + + sqlite3 db file:test.db2?vfs=tvfs + db eval { SELECT * FROM sqlite_master } + + tvfs filter xRead + tvfs script handle_read + } + sql1 { + PRAGMA wal_checkpoint = truncate; + } + code1 { set ::res2 } + } {hello world ! world hello} + + do_test $TN.5.3 { + code1 { db close } + code1 { tvfs delete } + } {} + + #----------------------------------------------------------------------- + # + # + catch { code1 { db close } } + catch { code2 { db2 close } } + catch { code3 { db3 close } } + + do_test $TN.6.1 { + code1 { forcedelete test.db } + code1 { sqlite3 db test.db } + sql1 { + PRAGMA journal_mode = wal; + CREATE TABLE t1(x); + INSERT INTO t1 VALUES('hello'); + INSERT INTO t1 VALUES('world'); + INSERT INTO t1 VALUES('!'); + INSERT INTO t1 VALUES('world'); + INSERT INTO t1 VALUES('hello'); + } + + copy_to_test2 $bZeroShm + + code1 { db close } + } {} + + do_test $TN.6.2 { + code1 { + set ::nRem 5 + proc handle_read {op args} { + if {$op=="xRead" && [file tail [lindex $args 0]]=="test.db2-wal"} { + incr ::nRem -1 + if {$::nRem==0} { + code2 { sqlite3 db2 test.db2 } + sql2 { PRAGMA wal_checkpoint = truncate } + } + } + return "SQLITE_OK" + } + testvfs tvfs -fullshm 1 + + tvfs filter xRead + tvfs script handle_read + + sqlite3 db file:test.db2?readonly_shm=1&vfs=tvfs + db eval { SELECT * FROM t1 } + } + } {hello world ! world hello} + + do_test $TN.6.3 { + code1 { db close } + code1 { tvfs delete } + } {} +} +} ;# foreach bZeroShm + +finish_test ADDED test/walrofault.test Index: test/walrofault.test ================================================================== --- /dev/null +++ test/walrofault.test @@ -0,0 +1,60 @@ +# 2011 May 09 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file contains tests for using WAL databases in read-only mode. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/malloc_common.tcl +set ::testprefix walro2 + +# And only if the build is WAL-capable. +# +ifcapable !wal { + finish_test + return +} + +db close +sqlite3_shutdown +sqlite3_config_uri 1 +sqlite3 db test.db + +do_execsql_test 1.0 { + CREATE TABLE t1(b); + PRAGMA journal_mode = wal; + INSERT INTO t1 VALUES('hello'); + INSERT INTO t1 VALUES('world'); + INSERT INTO t1 VALUES('!'); + INSERT INTO t1 VALUES('world'); + INSERT INTO t1 VALUES('hello'); + PRAGMA cache_size = 10; + BEGIN; + WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<30 ) + INSERT INTO t1(b) SELECT randomblob(800) FROM s; +} {wal} +file_control_persist_wal db 1; db close +faultsim_save_and_close + +do_faultsim_test 1 -faults oom* -prep { + catch { db close } + faultsim_restore + sqlite3 db file:test.db?readonly_shm=1 +} -body { + execsql { SELECT * FROM t1 } +} -test { + faultsim_test_result {0 {hello world ! world hello}} +} + + + +finish_test