Index: src/backup.c ================================================================== --- src/backup.c +++ src/backup.c @@ -415,10 +415,19 @@ ){ int nDestTruncate; if( p->pDestDb ){ sqlite3ResetInternalSchema(p->pDestDb, -1); + } + + if( destMode==PAGER_JOURNALMODE_WAL ){ + /* This call cannot fail. The success of the BtreeUpdateMeta() + ** method above indicates that a write transaction has been opened + ** and page 1 is already dirty. Therefore this always succeeds. + */ + TESTONLY(int rc2 =) sqlite3BtreeSetVersion(p->pDest, 2); + assert( rc2==SQLITE_OK ); } /* Set nDestTruncate to the final number of pages in the destination ** database. The complication here is that the destination page ** size may be different to the source page size. Index: src/btree.c ================================================================== --- src/btree.c +++ src/btree.c @@ -8167,11 +8167,10 @@ */ int sqlite3BtreeSetVersion(Btree *pBtree, int iVersion){ BtShared *pBt = pBtree->pBt; int rc; /* Return code */ - assert( pBtree->inTrans==TRANS_NONE ); assert( iVersion==1 || iVersion==2 ); /* If setting the version fields to 1, do not automatically open the ** WAL connection, even if the version fields are currently set to 2. */ Index: src/main.c ================================================================== --- src/main.c +++ src/main.c @@ -2977,146 +2977,28 @@ zFilename += sqlite3Strlen30(zFilename) + 1; } return 0; } -#if (SQLITE_ENABLE_APPLE_SPI>0) -#define SQLITE_FILE_HEADER_LEN 16 -#include -#include "sqlite3_private.h" -#include "btreeInt.h" -#include -#include - -/* Check for a conflicting lock. If one is found, print an this - ** on standard output using the format string given and return 1. - ** If there are no conflicting locks, return 0. - */ -static int isLocked( - pid_t pid, /* PID to test for lock owner */ - int h, /* File descriptor to check */ - int type, /* F_RDLCK or F_WRLCK */ - unsigned int iOfst, /* First byte of the lock */ - unsigned int iCnt, /* Number of bytes in the lock range */ - const char *zType /* Type of lock */ -){ - struct flock lk; - int err; - - memset(&lk, 0, sizeof(lk)); - lk.l_type = type; - lk.l_whence = SEEK_SET; - lk.l_start = iOfst; - lk.l_len = iCnt; - - if( pid!=SQLITE_LOCKSTATE_ANYPID ){ -#ifndef F_GETLKPID -# warning F_GETLKPID undefined, _sqlite3_lockstate falling back to F_GETLK - err = fcntl(h, F_GETLK, &lk); -#else - lk.l_pid = pid; - err = fcntl(h, F_GETLKPID, &lk); -#endif - }else{ - err = fcntl(h, F_GETLK, &lk); - } - - if( err==(-1) ){ - fprintf(stderr, "fcntl(%d) failed: errno=%d\n", h, errno); - return -1; - } - - if( lk.l_type!=F_UNLCK && (pid==SQLITE_LOCKSTATE_ANYPID || lk.l_pid==pid) ){ -#ifdef SQLITE_DEBUG - fprintf(stderr, "%s lock held by %d\n", zType, (int)lk.l_pid); -#endif - return 1; - } - return 0; -} - -/* - ** Location of locking bytes in the database file - */ -#ifndef PENDING_BYTE -# define PENDING_BYTE (0x40000000) -# define RESERVED_BYTE (PENDING_BYTE+1) -# define SHARED_FIRST (PENDING_BYTE+2) -# define SHARED_SIZE 510 -#endif /* PENDING_BYTE */ - -/* - ** Lock locations for shared-memory locks used by WAL mode. - */ -#ifndef SHM_BASE -# define SHM_BASE 120 -# define SHM_WRITE SHM_BASE -# define SHM_CHECKPOINT (SHM_BASE+1) -# define SHM_RECOVER (SHM_BASE+2) -# define SHM_READ_FIRST (SHM_BASE+3) -# define SHM_READ_SIZE 5 -#endif /* SHM_BASE */ +#if (SQLITE_ENABLE_APPLE_SPI>0) && defined(__APPLE__) + +#include "sqlite3_private.h" /* ** Testing a file path for sqlite locks held by a process ID. ** Returns SQLITE_LOCKSTATE_ON if locks are present on path ** that would prevent writing to the database. -** -** This test only works for lock testing on unix/posix VFS. -** Adapted from tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce */ int _sqlite3_lockstate(const char *path, pid_t pid){ - int hDb; /* File descriptor for the open database file */ - int hShm; /* File descriptor for WAL shared-memory file */ - ssize_t got; /* Bytes read from header */ - int isWal; /* True if in WAL mode */ - int nLock = 0; /* Number of locks held */ - unsigned char aHdr[100]; /* Database header */ - - /* Open the file at path and make sure we are dealing with a database file */ - hDb = open(path, O_RDONLY | O_NOCTTY); - if( hDb<0 ){ - return SQLITE_LOCKSTATE_ERROR; - } - assert( (strlen(SQLITE_FILE_HEADER)+1)==SQLITE_FILE_HEADER_LEN ); - got = pread(hDb, aHdr, 100, 0); - if( got<0 ){ - close(hDb); - return SQLITE_LOCKSTATE_ERROR; - } - if( got!=100 || memcmp(aHdr, SQLITE_FILE_HEADER, SQLITE_FILE_HEADER_LEN)!=0 ){ - close(hDb); - return SQLITE_LOCKSTATE_NOTADB; - } - - /* First check for an exclusive lock */ - nLock += isLocked(pid, hDb, F_RDLCK, SHARED_FIRST, SHARED_SIZE, "EXCLUSIVE"); - isWal = aHdr[18]==2; - if( nLock==0 && isWal==0 ){ - /* Rollback mode */ - nLock += isLocked(pid, hDb, F_WRLCK, PENDING_BYTE, SHARED_SIZE+2, "PENDING|RESERVED|SHARED"); - } - close(hDb); - if( nLock==0 && isWal!=0 ){ - char zShm[MAXPATHLEN]; - - close(hDb); - /* WAL mode */ - strlcpy(zShm, path, MAXPATHLEN); - strlcat(zShm, "-shm", MAXPATHLEN); - hShm = open(zShm, O_RDONLY, 0); - if( hShm<0 ){ - return SQLITE_LOCKSTATE_OFF; - } - if( isLocked(pid, hShm, F_RDLCK, SHM_RECOVER, 1, "WAL-RECOVERY") || - isLocked(pid, hShm, F_RDLCK, SHM_WRITE, 1, "WAL-WRITE") ){ - nLock = 1; - } - close(hShm); - } - if( nLock>0 ){ - return SQLITE_LOCKSTATE_ON; - } - return SQLITE_LOCKSTATE_OFF; + sqlite3 *db = NULL; + + if( sqlite3_open_v2(path, &db, SQLITE_OPEN_READONLY, NULL) == SQLITE_OK ){ + LockstatePID lockstate = {pid, -1}; + sqlite3_file_control(db, NULL, SQLITE_FCNTL_LOCKSTATE_PID, &lockstate); + sqlite3_close(db); + int state = lockstate.state; + return state; + } + return SQLITE_LOCKSTATE_ERROR; } #endif /* SQLITE_ENABLE_APPLE_SPI */ Index: src/mem1.c ================================================================== --- src/mem1.c +++ src/mem1.c @@ -30,11 +30,10 @@ #define SQLITE_MALLOC(x) malloc(x) #define SQLITE_FREE(x) free(x) #define SQLITE_REALLOC(x,y) realloc((x),(y)) - #else #include #include @@ -43,10 +42,11 @@ static malloc_zone_t* _sqliteZone_; #define SQLITE_MALLOC(x) malloc_zone_malloc(_sqliteZone_, (x)) #define SQLITE_FREE(x) malloc_zone_free(_sqliteZone_, (x)); #define SQLITE_REALLOC(x,y) malloc_zone_realloc(_sqliteZone_, (x), (y)) +#define SQLITE_MALLOCSIZE(x) (_sqliteZone_ ? _sqliteZone_->size(_sqliteZone_,x) : malloc_size(x)) #endif /* ** Like malloc(), but remember the size of the allocation @@ -58,18 +58,26 @@ */ static void *sqlite3MemMalloc(int nByte){ sqlite3_int64 *p; assert( nByte>0 ); nByte = ROUND8(nByte); - p = SQLITE_MALLOC( nByte+8 ); +#ifndef SQLITE_MALLOCSIZE + p = SQLITE_MALLOC( nByte + 8 ); if( p ){ p[0] = nByte; p++; }else{ testcase( sqlite3GlobalConfig.xLog!=0 ); sqlite3_log(SQLITE_NOMEM, "failed to allocate %u bytes of memory", nByte); } +#else + p = SQLITE_MALLOC( nByte ); + if( !p ){ + testcase( sqlite3GlobalConfig.xLog!=0 ); + sqlite3_log(SQLITE_NOMEM, "failed to allocate %u bytes of memory", nByte); + } +#endif return (void *)p; } /* ** Like free() but works for allocations obtained from sqlite3MemMalloc() @@ -80,24 +88,30 @@ ** by higher-level routines. */ static void sqlite3MemFree(void *pPrior){ sqlite3_int64 *p = (sqlite3_int64*)pPrior; assert( pPrior!=0 ); +#ifndef SQLITE_MALLOCSIZE p--; +#endif SQLITE_FREE(p); } /* ** Report the allocated size of a prior return from xMalloc() ** or xRealloc(). */ static int sqlite3MemSize(void *pPrior){ +#ifndef SQLITE_MALLOCSIZE sqlite3_int64 *p; if( pPrior==0 ) return 0; p = (sqlite3_int64*)pPrior; p--; return (int)p[0]; +#else + return (int)SQLITE_MALLOCSIZE(pPrior); +#endif } /* ** Like realloc(). Resize an allocation previously obtained from ** sqlite3MemMalloc(). @@ -110,21 +124,31 @@ */ static void *sqlite3MemRealloc(void *pPrior, int nByte){ sqlite3_int64 *p = (sqlite3_int64*)pPrior; assert( pPrior!=0 && nByte>0 ); assert( nByte==ROUND8(nByte) ); /* EV: R-46199-30249 */ +#ifndef SQLITE_MALLOCSIZE p--; p = SQLITE_REALLOC(p, nByte+8 ); if( p ){ p[0] = nByte; p++; }else{ + testcase( sqlite3GlobalConfig.xLog!=0 ); + sqlite3_log(SQLITE_NOMEM, + "failed memory resize %u to %u bytes", + sqlite3MemSize(pPrior), nByte); + } +#else + p = SQLITE_REALLOC(p, nByte ); + if( !p ){ testcase( sqlite3GlobalConfig.xLog!=0 ); sqlite3_log(SQLITE_NOMEM, "failed memory resize %u to %u bytes", sqlite3MemSize(pPrior), nByte); } +#endif return (void*)p; } /* ** Round up a request size to the next valid allocation size. Index: src/os_unix.c ================================================================== --- src/os_unix.c +++ src/os_unix.c @@ -256,12 +256,13 @@ }; /* ** Allowed values for the unixFile.ctrlFlags bitmask: */ -#define UNIXFILE_EXCL 0x01 /* Connections from one process only */ -#define UNIXFILE_RDONLY 0x02 /* Connection is read only */ +#define UNIXFILE_EXCL 0x01 /* Connections from one process only */ +#define UNIXFILE_RDONLY 0x02 /* Connection is read only */ +#define UNIXFILE_PERSIST_WAL 0x04 /* Persistent WAL mode */ /* ** Include code that is common to all os_*.c files */ #include "os_common.h" @@ -1537,10 +1538,12 @@ *pResOut = reserved; return rc; } +static int _unixFileLock(unixFile *pFile, struct flock *pLock, int retry); + /* ** Attempt to set a system-lock on the file pFile. The lock is ** described by pLock. ** ** If the pFile was opened read/write from unix-excl, then the only lock @@ -1557,11 +1560,19 @@ ** ** Zero is returned if the call completes successfully, or -1 if a call ** to fcntl() fails. In this case, errno is set appropriately (by fcntl()). */ static int unixFileLock(unixFile *pFile, struct flock *pLock){ - int rc; + return _unixFileLock(pFile, pLock, 0); +} + +static int unixFileLock2(unixFile *pFile, struct flock *pLock){ + return _unixFileLock(pFile, pLock, 10); +} + +static int _unixFileLock(unixFile *pFile, struct flock *pLock, int retry) { + int rc = 0; unixInodeInfo *pInode = pFile->pInode; assert( unixMutexHeld() ); assert( pInode!=0 ); if( ((pFile->ctrlFlags & UNIXFILE_EXCL)!=0 || pInode->bProcessLock) && ((pFile->ctrlFlags & UNIXFILE_RDONLY)==0) @@ -1579,11 +1590,17 @@ pInode->nLock++; }else{ rc = 0; } }else{ - rc = osFcntl(pFile->h, F_SETLK, pLock); + int i = 0; + do { + rc = osFcntl(pFile->h, F_SETLK, pLock); + if (rc && retry) { + usleep(100 * (++i)); + } + } while (!rc && retry--); } return rc; } /* @@ -1750,11 +1767,11 @@ /* Drop the temporary PENDING lock */ lock.l_start = PENDING_BYTE; lock.l_len = 1L; lock.l_type = F_UNLCK; - if( unixFileLock(pFile, &lock) && rc==SQLITE_OK ){ + if( unixFileLock2(pFile, &lock) && rc==SQLITE_OK ){ /* This could happen with a network mount */ tErrno = errno; #if OSLOCKING_CHECK_BUSY_IOERR rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_UNLOCK); #else @@ -1912,25 +1929,25 @@ ** 2: [....W] ** 3: [RRRRW] ** 4: [RRRR.] */ if( eFileLock==SHARED_LOCK ){ + int tErrno; /* Error code from system call errors */ #if !defined(__APPLE__) || !SQLITE_ENABLE_LOCKING_STYLE (void)handleNFSUnlock; assert( handleNFSUnlock==0 ); #endif #if defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE if( handleNFSUnlock ){ - int tErrno; /* Error code from system call errors */ off_t divSize = SHARED_SIZE - 1; lock.l_type = F_UNLCK; lock.l_whence = SEEK_SET; lock.l_start = SHARED_FIRST; lock.l_len = divSize; - if( unixFileLock(pFile, &lock)==(-1) ){ + if( unixFileLock2(pFile, &lock)==(-1) ){ tErrno = errno; #if OSLOCKING_CHECK_BUSY_IOERR rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_UNLOCK); #else rc = SQLITE_IOERR_UNLOCK; @@ -1942,23 +1959,27 @@ } lock.l_type = F_RDLCK; lock.l_whence = SEEK_SET; lock.l_start = SHARED_FIRST; lock.l_len = divSize; - if( unixFileLock(pFile, &lock)==(-1) ){ + if( unixFileLock2(pFile, &lock)==(-1) ){ tErrno = errno; +#if OSLOCKING_CHECK_BUSY_IOERR rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_RDLOCK); +#else + rc = SQLITE_IOERR_UNLOCK; +#endif if( IS_LOCK_ERROR(rc) ){ pFile->lastErrno = tErrno; } goto end_unlock; } lock.l_type = F_UNLCK; lock.l_whence = SEEK_SET; lock.l_start = SHARED_FIRST+divSize; lock.l_len = SHARED_SIZE-divSize; - if( unixFileLock(pFile, &lock)==(-1) ){ + if( unixFileLock2(pFile, &lock)==(-1) ){ tErrno = errno; #if OSLOCKING_CHECK_BUSY_IOERR rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_UNLOCK); #else rc = SQLITE_IOERR_UNLOCK; @@ -1973,36 +1994,36 @@ { lock.l_type = F_RDLCK; lock.l_whence = SEEK_SET; lock.l_start = SHARED_FIRST; lock.l_len = SHARED_SIZE; - if( unixFileLock(pFile, &lock) ){ -#if OSLOCKING_CHECK_BUSY_IOERR + if( unixFileLock2(pFile, &lock) ){ tErrno = errno; +#if OSLOCKING_CHECK_BUSY_IOERR rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_RDLOCK); - if( IS_LOCK_ERROR(rc) ){ - pFile->lastErrno = tErrno; - } #else /* In theory, the call to unixFileLock() cannot fail because another ** process is holding an incompatible lock. If it does, this ** indicates that the other process is not following the locking ** protocol. If this happens, return SQLITE_IOERR_RDLOCK. Returning ** SQLITE_BUSY would confuse the upper layer (in practice it causes ** an assert to fail). */ rc = SQLITE_IOERR_RDLOCK; - pFile->lastErrno = errno; + pFile->lastErrno = tErrno; #endif + if( IS_LOCK_ERROR(rc) ){ + pFile->lastErrno = tErrno; + } goto end_unlock; } } } lock.l_type = F_UNLCK; lock.l_whence = SEEK_SET; lock.l_start = PENDING_BYTE; lock.l_len = 2L; assert( PENDING_BYTE+1==RESERVED_BYTE ); - if( unixFileLock(pFile, &lock)==0 ){ + if( unixFileLock2(pFile, &lock)==0 ){ pInode->eFileLock = SHARED_LOCK; }else{ #if OSLOCKING_CHECK_BUSY_IOERR tErrno = errno; rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_UNLOCK); @@ -2027,11 +2048,11 @@ lock.l_whence = SEEK_SET; lock.l_start = lock.l_len = 0L; SimulateIOErrorBenign(1); SimulateIOError( h=(-1) ) SimulateIOErrorBenign(0); - if( unixFileLock(pFile, &lock)==0 ){ + if( unixFileLock2(pFile, &lock)==0 ){ pInode->eFileLock = NO_LOCK; }else{ #if OSLOCKING_CHECK_BUSY_IOERR tErrno = errno; rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_UNLOCK); @@ -3843,29 +3864,275 @@ #include static int getDbPathForUnixFile(unixFile *pFile, char *dbPath); #endif static int isProxyLockingMode(unixFile *); +#if (SQLITE_ENABLE_APPLE_SPI>0) && defined(__APPLE__) +static int unixTruncateDatabase(unixFile *, int); + +static int unixInvalidateSupportFiles(unixFile *, int); + +static int unixReplaceDatabase(unixFile *pFile, sqlite3 *srcdb) { + sqlite3_file *id = (sqlite3_file *)pFile; + Btree *pSrcBtree = NULL; + sqlite3_file *src_file = NULL; + unixFile *pSrcFile = NULL; + char srcWalPath[MAXPATHLEN+5]; + int srcWalFD = -1; + int rc = SQLITE_OK; + void *pLock = NULL; + int flags = 0; + sqlite3 *srcdb2 = NULL; + copyfile_state_t s; + int corruptSrcFileLock = 0; + int corruptDstFileLock = 0; + int isSrcCorrupt = 0; + int isDstCorrupt = 0; + + if( !sqlite3SafetyCheckOk(srcdb) ){ + return SQLITE_MISUSE; + } + +#if SQLITE_ENABLE_DATA_PROTECTION + flags |= pFile->protFlags; +#endif +#if SQLITE_ENABLE_LOCKING_STYLE + if( isProxyLockingMode(pFile) ){ + flags |= SQLITE_OPEN_AUTOPROXY; + } +#endif + + rc = sqlite3demo_superlock(pFile->zPath, 0, flags, 0, 0, &pLock); + if( rc ){ + if( rc==SQLITE_CORRUPT || rc==SQLITE_NOTADB ){ + isDstCorrupt = 1; + rc = sqlite3demo_superlock_corrupt(id, SQLITE_LOCK_EXCLUSIVE, &corruptDstFileLock); + } + if( rc ){ + return rc; + } + } + /* get the src file descriptor adhering to the db struct access rules + ** this code is modeled after sqlite3_file_control() in main.c + */ + sqlite3_mutex_enter(srcdb->mutex); + if( srcdb->nDb>0 ){ + pSrcBtree = srcdb->aDb[0].pBt; + } + if( pSrcBtree ){ + Pager *pSrcPager; + sqlite3BtreeEnter(pSrcBtree); + pSrcPager = sqlite3BtreePager(pSrcBtree); + assert( pSrcPager!=0 ); + src_file = sqlite3PagerFile(pSrcPager); + assert( src_file!=0 ); + if( src_file->pMethods ){ + int srcFlags = 0; + pSrcFile = (unixFile *)src_file; + /* wal mode db cannot be opened readonly */ + if ((pSrcFile->openFlags & O_RDWR) == O_RDWR) { + srcFlags = SQLITE_OPEN_READWRITE; + } else { + srcFlags = SQLITE_OPEN_READONLY; + } +#if SQLITE_ENABLE_DATA_PROTECTION + srcFlags |= pSrcFile->protFlags; +#endif +#if SQLITE_ENABLE_LOCKING_STYLE + if( isProxyLockingMode(pSrcFile) ){ + srcFlags |= SQLITE_OPEN_AUTOPROXY; + } +#endif + rc = sqlite3_open_v2(pSrcFile->zPath, &srcdb2, srcFlags, 0); + if( rc==SQLITE_OK ){ + /* start a deferred transaction and read to establish a read lock */ + rc = sqlite3_exec(srcdb2, "BEGIN DEFERRED; PRAGMA schema_version", 0, 0, 0); + if( rc==SQLITE_CORRUPT || rc==SQLITE_NOTADB ){ + isSrcCorrupt = 1; + rc = sqlite3demo_superlock_corrupt(src_file, SQLITE_LOCK_SHARED, &corruptSrcFileLock); + } + } + } + } + if( !srcdb2 || pSrcFile==NULL || pSrcFile->h<0){ + rc = SQLITE_INTERNAL; + } + if( rc!=SQLITE_OK ){ + goto end_replace_database; + } + /* both databases are locked appropriately, copy the src wal journal if + ** one exists and then the actual database file + */ + strlcpy(srcWalPath, pSrcFile->zPath, MAXPATHLEN+5); + strlcat(srcWalPath, "-wal", MAXPATHLEN+5); + srcWalFD = open(srcWalPath, O_RDONLY); + if( !(srcWalFD<0) ){ + char dstWalPath[MAXPATHLEN+5]; + int dstWalFD = -1; + strlcpy(dstWalPath, pFile->zPath, MAXPATHLEN+5); + strlcat(dstWalPath, "-wal", MAXPATHLEN+5); + dstWalFD = open(dstWalPath, O_RDWR|O_CREAT, SQLITE_DEFAULT_FILE_PERMISSIONS); + if( !(dstWalFD<0) ){ + s = copyfile_state_alloc(); + lseek(srcWalFD, 0, SEEK_SET); + lseek(dstWalFD, 0, SEEK_SET); + if( fcopyfile(srcWalFD, dstWalFD, s, COPYFILE_ALL) ){ + int err=errno; + switch(err) { + case ENOMEM: + rc = SQLITE_NOMEM; + break; + default: + pFile->lastErrno = err; + rc = SQLITE_IOERR; + } + } + copyfile_state_free(s); + close(dstWalFD); + } + close(srcWalFD); + } + if( rc==SQLITE_OK ){ + /* before we copy, ensure that the file change counter will be modified */ + uint32_t srcChange = 0; + uint32_t dstChange = 0; + pread(pSrcFile->h, &srcChange, 4, 24); + pread(pFile->h, &dstChange, 4, 24); + + /* copy the actual database */ + s = copyfile_state_alloc(); + lseek(pSrcFile->h, 0, SEEK_SET); + lseek(pFile->h, 0, SEEK_SET); + if( fcopyfile(pSrcFile->h, pFile->h, s, COPYFILE_ALL) ){ + int err=errno; + switch(err) { + case ENOMEM: + rc = SQLITE_NOMEM; + break; + default: + pFile->lastErrno = err; + rc = SQLITE_IOERR; + } + } + copyfile_state_free(s); + + if (srcChange == dstChange) { + /* modify the change counter to force page zero to be reloaded */ + dstChange ++; + pwrite(pFile->h, &dstChange, 4, 24); + } + } + if( isSrcCorrupt ){ + sqlite3demo_superunlock_corrupt(src_file, corruptSrcFileLock); + }else{ + /* done with the source db so end the transaction */ + sqlite3_exec(srcdb2, "COMMIT", 0, 0, 0); + } + /* zero out any old journal clutter */ + if( rc==SQLITE_OK ){ + int skipWAL = (srcWalFD<0)?0:1; + unixInvalidateSupportFiles(pFile, skipWAL); + } + +end_replace_database: + if( pSrcBtree ){ + sqlite3_close(srcdb2); + sqlite3BtreeLeave(pSrcBtree); + } + sqlite3_mutex_leave(srcdb->mutex); + if( isDstCorrupt ){ + sqlite3demo_superunlock_corrupt(id, corruptDstFileLock); + }else{ + sqlite3demo_superunlock(pLock); + } + return rc; +} + +#define SQLITE_FILE_HEADER_LEN 16 +#include "btreeInt.h" +/* Check for a conflicting lock. If one is found, print an this + ** on standard output using the format string given and return 1. + ** If there are no conflicting locks, return 0. + */ +static int unixIsLocked( + pid_t pid, /* PID to test for lock owner */ + int h, /* File descriptor to check */ + int type, /* F_RDLCK or F_WRLCK */ + unsigned int iOfst, /* First byte of the lock */ + unsigned int iCnt, /* Number of bytes in the lock range */ + const char *zType /* Type of lock */ +){ + struct flock lk; + int err; + + memset(&lk, 0, sizeof(lk)); + lk.l_type = type; + lk.l_whence = SEEK_SET; + lk.l_start = iOfst; + lk.l_len = iCnt; + + if( pid!=SQLITE_LOCKSTATE_ANYPID ){ +#ifndef F_GETLKPID +# warning F_GETLKPID undefined, _sqlite3_lockstate falling back to F_GETLK + err = fcntl(h, F_GETLK, &lk); +#else + lk.l_pid = pid; + err = fcntl(h, F_GETLKPID, &lk); +#endif + }else{ + err = fcntl(h, F_GETLK, &lk); + } + + if( err==(-1) ){ + fprintf(stderr, "fcntl(%d) failed: errno=%d\n", h, errno); + return -1; + } + + if( lk.l_type!=F_UNLCK && (pid==SQLITE_LOCKSTATE_ANYPID || lk.l_pid==pid) ){ +#ifdef SQLITE_DEBUG + fprintf(stderr, "%s lock held by %d\n", zType, (int)lk.l_pid); +#endif + return 1; + } + return 0; +} + +static int unixLockstatePid(unixFile *, pid_t, int *); + +#endif /* (SQLITE_ENABLE_APPLE_SPI>0) && defined(__APPLE__) */ + /* ** Information and control of an open file handle. */ static int unixFileControl(sqlite3_file *id, int op, void *pArg){ + unixFile *pFile = (unixFile*)id; switch( op ){ case SQLITE_FCNTL_LOCKSTATE: { - *(int*)pArg = ((unixFile*)id)->eFileLock; + *(int*)pArg = pFile->eFileLock; return SQLITE_OK; } - case SQLITE_LAST_ERRNO: { - *(int*)pArg = ((unixFile*)id)->lastErrno; + case SQLITE_FCNTL_LAST_ERRNO: { + *(int*)pArg = pFile->lastErrno; return SQLITE_OK; } case SQLITE_FCNTL_CHUNK_SIZE: { - ((unixFile*)id)->szChunk = *(int *)pArg; + pFile->szChunk = *(int *)pArg; return SQLITE_OK; } case SQLITE_FCNTL_SIZE_HINT: { - return fcntlSizeHint((unixFile *)id, *(i64 *)pArg); + return fcntlSizeHint(pFile, *(i64 *)pArg); + } + case SQLITE_FCNTL_PERSIST_WAL: { + int bPersist = *(int*)pArg; + if( bPersist<0 ){ + *(int*)pArg = (pFile->ctrlFlags & UNIXFILE_PERSIST_WAL)!=0; + }else if( bPersist==0 ){ + pFile->ctrlFlags &= ~UNIXFILE_PERSIST_WAL; + }else{ + pFile->ctrlFlags |= UNIXFILE_PERSIST_WAL; + } + return SQLITE_OK; } #ifndef NDEBUG /* The pager calls this method to signal that it has done ** a rollback and that the database is therefore unchanged and ** it hence it is OK for the transaction change counter to be @@ -3875,271 +4142,31 @@ ((unixFile*)id)->dbUpdate = 0; return SQLITE_OK; } #endif #if SQLITE_ENABLE_LOCKING_STYLE && defined(__APPLE__) - case SQLITE_SET_LOCKPROXYFILE: - case SQLITE_GET_LOCKPROXYFILE: { + case SQLITE_FCNTL_SET_LOCKPROXYFILE: + case SQLITE_FCNTL_GET_LOCKPROXYFILE: { return proxyFileControl(id,op,pArg); } #endif /* SQLITE_ENABLE_LOCKING_STYLE && defined(__APPLE__) */ #if (SQLITE_ENABLE_APPLE_SPI>0) && defined(__APPLE__) - case SQLITE_TRUNCATE_DATABASE: { - unixFile *pFile = (unixFile*)id; - int rc = SQLITE_OK; - void *pLock = NULL; - int flags = 0; - int corruptFileLock = 0; - int isCorrupt = 0; - -#if SQLITE_ENABLE_DATA_PROTECTION - flags |= pFile->protFlags; -#endif -#if SQLITE_ENABLE_LOCKING_STYLE - if( isProxyLockingMode(pFile) ){ - flags |= SQLITE_OPEN_AUTOPROXY; - } -#endif - - rc = sqlite3demo_superlock(pFile->zPath, 0, flags, 0, 0, &pLock); - if( rc ){ - if( rc==SQLITE_CORRUPT || rc==SQLITE_NOTADB ){ - isCorrupt = 1; - rc = sqlite3demo_superlock_corrupt(id, SQLITE_LOCK_EXCLUSIVE, &corruptFileLock); - } - if( rc ){ - return rc; - } - } - rc = pFile->pMethod->xTruncate(id, ((pFile->fsFlags & SQLITE_FSFLAGS_IS_MSDOS) != 0) ? 1L : 0L); - - if( rc==SQLITE_OK ){ - char jPath[MAXPATHLEN+9]; - int zLen = strlcpy(jPath, pFile->zPath, MAXPATHLEN+9); - if( zLenpMethod->xSync(id, SQLITE_SYNC_FULL); - } - if( isCorrupt ){ - sqlite3demo_superunlock_corrupt(id, corruptFileLock); - }else{ - sqlite3demo_superunlock(pLock); - } - return rc; - } - - case SQLITE_REPLACE_DATABASE: { - unixFile *pFile = (unixFile*)id; - sqlite3 *srcdb = (sqlite3 *)pArg; - Btree *pSrcBtree = NULL; - sqlite3_file *src_file = NULL; - unixFile *pSrcFile = NULL; - char srcWalPath[MAXPATHLEN+5]; - int srcWalFD = -1; - int rc = SQLITE_OK; - void *pLock = NULL; - int flags = 0; - sqlite3 *srcdb2 = NULL; - copyfile_state_t s; - int corruptSrcFileLock = 0; - int corruptDstFileLock = 0; - int isSrcCorrupt = 0; - int isDstCorrupt = 0; - - if( !sqlite3SafetyCheckOk(srcdb) ){ + case SQLITE_FCNTL_TRUNCATE_DATABASE: { + return unixTruncateDatabase(pFile, (pArg ? (*(int *)pArg) : 0)); + } + case SQLITE_FCNTL_REPLACE_DATABASE: { + return unixReplaceDatabase(pFile, (sqlite3 *)pArg); + } + case SQLITE_FCNTL_LOCKSTATE_PID: { + LockstatePID *pLockstate; + int rc; + + if( pArg==NULL ){ return SQLITE_MISUSE; } - -#if SQLITE_ENABLE_DATA_PROTECTION - flags |= pFile->protFlags; -#endif -#if SQLITE_ENABLE_LOCKING_STYLE - if( isProxyLockingMode(pFile) ){ - flags |= SQLITE_OPEN_AUTOPROXY; - } -#endif - - rc = sqlite3demo_superlock(pFile->zPath, 0, flags, 0, 0, &pLock); - if( rc ){ - if( rc==SQLITE_CORRUPT || rc==SQLITE_NOTADB ){ - isDstCorrupt = 1; - rc = sqlite3demo_superlock_corrupt(id, SQLITE_LOCK_EXCLUSIVE, &corruptDstFileLock); - } - if( rc ){ - return rc; - } - } - /* get the src file descriptor adhering to the db struct access rules - ** this code is modeled after sqlite3_file_control() in main.c - */ - sqlite3_mutex_enter(srcdb->mutex); - if( srcdb->nDb>0 ){ - pSrcBtree = srcdb->aDb[0].pBt; - } - if( pSrcBtree ){ - Pager *pSrcPager; - sqlite3BtreeEnter(pSrcBtree); - pSrcPager = sqlite3BtreePager(pSrcBtree); - assert( pSrcPager!=0 ); - src_file = sqlite3PagerFile(pSrcPager); - assert( src_file!=0 ); - if( src_file->pMethods ){ - int srcFlags = 0; - pSrcFile = (unixFile *)src_file; - /* wal mode db cannot be opened readonly */ - if ((pSrcFile->openFlags & O_RDWR) == O_RDWR) { - srcFlags = SQLITE_OPEN_READWRITE; - } else { - srcFlags = SQLITE_OPEN_READONLY; - } -#if SQLITE_ENABLE_DATA_PROTECTION - srcFlags |= pSrcFile->protFlags; -#endif -#if SQLITE_ENABLE_LOCKING_STYLE - if( isProxyLockingMode(pSrcFile) ){ - srcFlags |= SQLITE_OPEN_AUTOPROXY; - } -#endif - rc = sqlite3_open_v2(pSrcFile->zPath, &srcdb2, srcFlags, 0); - if( rc==SQLITE_OK ){ - /* start a deferred transaction and read to establish a read lock */ - rc = sqlite3_exec(srcdb2, "BEGIN DEFERRED; PRAGMA schema_version", 0, 0, 0); - if( rc==SQLITE_CORRUPT || rc==SQLITE_NOTADB ){ - isSrcCorrupt = 1; - rc = sqlite3demo_superlock_corrupt(src_file, SQLITE_LOCK_SHARED, &corruptSrcFileLock); - } - } - } - } - if( !srcdb2 || pSrcFile==NULL || pSrcFile->h<0){ - rc = SQLITE_INTERNAL; - } - if( rc!=SQLITE_OK ){ - goto end_replace_database; - } - /* both databases are locked appropriately, copy the src wal journal if - ** one exists and then the actual database file - */ - strlcpy(srcWalPath, pSrcFile->zPath, MAXPATHLEN+5); - strlcat(srcWalPath, "-wal", MAXPATHLEN+5); - srcWalFD = open(srcWalPath, O_RDONLY); - if( !(srcWalFD<0) ){ - char dstWalPath[MAXPATHLEN+5]; - int dstWalFD = -1; - strlcpy(dstWalPath, pFile->zPath, MAXPATHLEN+5); - strlcat(dstWalPath, "-wal", MAXPATHLEN+5); - dstWalFD = open(dstWalPath, O_RDWR|O_CREAT, SQLITE_DEFAULT_FILE_PERMISSIONS); - if( !(dstWalFD<0) ){ - s = copyfile_state_alloc(); - lseek(srcWalFD, 0, SEEK_SET); - lseek(dstWalFD, 0, SEEK_SET); - if( fcopyfile(srcWalFD, dstWalFD, s, COPYFILE_ALL) ){ - int err=errno; - switch(err) { - case ENOMEM: - rc = SQLITE_NOMEM; - break; - default: - pFile->lastErrno = err; - rc = SQLITE_IOERR; - } - } - copyfile_state_free(s); - close(dstWalFD); - } - close(srcWalFD); - } - if( rc==SQLITE_OK ){ - /* before we copy, ensure that the file change counter will be modified */ - uint32_t srcChange = 0; - uint32_t dstChange = 0; - pread(pSrcFile->h, &srcChange, 4, 24); - pread(pFile->h, &dstChange, 4, 24); - - /* copy the actual database */ - s = copyfile_state_alloc(); - lseek(pSrcFile->h, 0, SEEK_SET); - lseek(pFile->h, 0, SEEK_SET); - if( fcopyfile(pSrcFile->h, pFile->h, s, COPYFILE_ALL) ){ - int err=errno; - switch(err) { - case ENOMEM: - rc = SQLITE_NOMEM; - break; - default: - pFile->lastErrno = err; - rc = SQLITE_IOERR; - } - } - copyfile_state_free(s); - - if (srcChange == dstChange) { - /* modify the change counter to force page zero to be reloaded */ - dstChange ++; - pwrite(pFile->h, &dstChange, 4, 24); - } - } - if( isSrcCorrupt ){ - sqlite3demo_superunlock_corrupt(src_file, corruptSrcFileLock); - }else{ - /* done with the source db so end the transaction */ - sqlite3_exec(srcdb2, "COMMIT", 0, 0, 0); - } - /* zero out any old journal clutter */ - if( rc==SQLITE_OK ){ - char jPath[MAXPATHLEN+9]; - int zLen = strlcpy(jPath, pFile->zPath, MAXPATHLEN+9); - if( zLenpMethod->xSync(id, SQLITE_SYNC_FULL); - } - - end_replace_database: - if( pSrcBtree ){ - sqlite3_close(srcdb2); - sqlite3BtreeLeave(pSrcBtree); - } - sqlite3_mutex_leave(srcdb->mutex); - if( isDstCorrupt ){ - sqlite3demo_superunlock_corrupt(id, corruptDstFileLock); - }else{ - sqlite3demo_superunlock(pLock); - } + pLockstate = (LockstatePID *)pArg; + rc = unixLockstatePid(pFile, pLockstate->pid, &(pLockstate->state)); return rc; } #endif /* (SQLITE_ENABLE_APPLE_SPI>0) && defined(__APPLE__) */ case SQLITE_FCNTL_SYNC_OMITTED: { return SQLITE_OK; /* A no-op */ @@ -4468,12 +4495,20 @@ rc = SQLITE_NOMEM; goto shm_open_err; } if( pInode->bProcessLock==0 ){ - pShmNode->h = robust_open(zShmFilename, O_RDWR|O_CREAT, + const char *zRO; + zRO = sqlite3_uri_parameter(pDbFd->zPath, "readonly_shm"); + if( zRO && sqlite3GetBoolean(zRO) ){ + pShmNode->h = robust_open(zShmFilename, O_RDONLY, + (sStat.st_mode & 0777)); + pShmNode->isReadonly = 1; + }else{ + pShmNode->h = robust_open(zShmFilename, O_RDWR|O_CREAT, (sStat.st_mode & 0777)); + } if( pShmNode->h<0 ){ const char *zRO; zRO = sqlite3_uri_parameter(pDbFd->zPath, "readonly_shm"); if( zRO && sqlite3GetBoolean(zRO) ){ pShmNode->h = robust_open(zShmFilename, O_RDONLY, @@ -4844,10 +4879,214 @@ # define unixShmMap 0 # define unixShmLock 0 # define unixShmBarrier 0 # define unixShmUnmap 0 #endif /* #ifndef SQLITE_OMIT_WAL */ + +#if (SQLITE_ENABLE_APPLE_SPI>0) && defined(__APPLE__) +static const char *unixTempFileDir(void); + +static int unixInvalidateSupportFiles(unixFile *pFile, int skipWAL) { + char jPath[MAXPATHLEN+9]; + int zLen = strlcpy(jPath, pFile->zPath, MAXPATHLEN+9); + if( zLenpInode is shared across threads */ + unixShmNode *pShmNode = pFile->pInode->pShmNode; + if( pShmNode && !pShmNode->isReadonly ){ + struct stat sStat; + sqlite3_mutex_enter(pShmNode->mutex); + + if( pShmNode->h>=0 && !osFstat(pShmNode->h, &sStat) ){ + unsigned long size = (sStat.st_size<4) ? sStat.st_size : 4; + if( size>0 ){ + bzero(pShmNode->apRegion[0], size); + sqlite3_mutex_leave(pShmNode->mutex); + unixLeaveMutex(); + continue; + } + } + sqlite3_mutex_leave(pShmNode->mutex); + } + unixLeaveMutex(); + } + jLen = strlcpy(&jPath[zLen], extensions[j], 9); + if( jLen < 9 ){ + int jflags = (j<2) ? O_TRUNC : O_RDWR; + int jfd = open(jPath, jflags); + if( jfd==(-1) ){ + if( errno!=ENOENT ){ + perror(jPath); + } + } else { + if( j==2 ){ + struct stat sStat; + if( !osFstat(jfd, &sStat) ){ + unsigned long size = (sStat.st_size<4) ? sStat.st_size : 4; + if( size>0 ){ + uint32_t zero = 0; + pwrite(jfd, &zero, (size_t)size, 0); + } + } + } + fsync(jfd); + close(jfd); + } + } + } + } + return SQLITE_OK; +} + +static int unixTruncateDatabase(unixFile *pFile, int bFlags) { + sqlite3_file *id = (sqlite3_file *)pFile; + int rc = SQLITE_OK; + void *pLock = NULL; + int flags = 0; + int corruptFileLock = 0; + int isCorrupt = 0; + +#if SQLITE_ENABLE_DATA_PROTECTION + flags |= pFile->protFlags; +#endif +#if SQLITE_ENABLE_LOCKING_STYLE + if( isProxyLockingMode(pFile) ){ + flags |= SQLITE_OPEN_AUTOPROXY; + } +#endif + + rc = sqlite3demo_superlock(pFile->zPath, 0, flags, 0, 0, &pLock); + if( rc ){ + if( rc==SQLITE_CORRUPT || rc==SQLITE_NOTADB ){ + isCorrupt = 1; + rc = sqlite3demo_superlock_corrupt(id, SQLITE_LOCK_EXCLUSIVE, &corruptFileLock); + } + if( rc ){ + return rc; + } + } + rc = pFile->pMethod->xTruncate(id, ((pFile->fsFlags & SQLITE_FSFLAGS_IS_MSDOS) != 0) ? 1L : 0L); + if( rc==SQLITE_OK ){ + unixInvalidateSupportFiles(pFile, 0); + } + pFile->pMethod->xSync(id, SQLITE_SYNC_FULL); + + + if( isCorrupt ){ + sqlite3demo_superunlock_corrupt(id, corruptFileLock); + }else{ + sqlite3demo_superunlock(pLock); + } + return rc; +} + +/* + ** Lock locations for shared-memory locks used by WAL mode. + */ +#ifndef SHM_BASE +# define SHM_BASE 120 +# define SHM_WRITE SHM_BASE +# define SHM_CHECKPOINT (SHM_BASE+1) +# define SHM_RECOVER (SHM_BASE+2) +# define SHM_READ_FIRST (SHM_BASE+3) +# define SHM_READ_SIZE 5 +#endif /* SHM_BASE */ + +/* +** This test only works for lock testing on unix/posix VFS. +** Adapted from tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce +*/ +static int unixLockstatePid(unixFile *pFile, pid_t pid, int *pLockstate){ + int hDb; /* File descriptor for the open database file */ + int hShm = -1; /* File descriptor for WAL shared-memory file */ + ssize_t got; /* Bytes read from header */ + int isWal; /* True if in WAL mode */ + int nLock = 0; /* Number of locks held */ + unsigned char aHdr[100]; /* Database header */ + + assert(pLockstate); + + /* make sure we are dealing with a database file */ + hDb = pFile->h; + if( hDb<0 ){ + *pLockstate = SQLITE_LOCKSTATE_ERROR; + return SQLITE_ERROR; + } + assert( (strlen(SQLITE_FILE_HEADER)+1)==SQLITE_FILE_HEADER_LEN ); + got = pread(hDb, aHdr, 100, 0); + if( got<0 ){ + *pLockstate = SQLITE_LOCKSTATE_ERROR; + return SQLITE_ERROR; + } + if( got!=100 || memcmp(aHdr, SQLITE_FILE_HEADER, SQLITE_FILE_HEADER_LEN)!=0 ){ + *pLockstate = SQLITE_LOCKSTATE_NOTADB; + return SQLITE_NOTADB; + } + + /* First check for an exclusive lock */ + nLock += unixIsLocked(pid, hDb, F_RDLCK, SHARED_FIRST, SHARED_SIZE, "EXCLUSIVE"); + isWal = aHdr[18]==2; + if( nLock==0 && isWal==0 ){ + /* Rollback mode */ + nLock += unixIsLocked(pid, hDb, F_WRLCK, PENDING_BYTE, SHARED_SIZE+2, "PENDING|RESERVED|SHARED"); + } + if( nLock==0 && isWal!=0 ){ + /* lookup the file descriptor for the shared memory file if we have it open in this process */ + unixEnterMutex(); /* Because pFile->pInode is shared across threads */ + unixShmNode *pShmNode = pFile->pInode->pShmNode; + if( pShmNode ){ + sqlite3_mutex_enter(pShmNode->mutex); + + hShm = pShmNode->h; + if( hShm >= 0){ + if( unixIsLocked(pid, hShm, F_RDLCK, SHM_RECOVER, 1, "WAL-RECOVERY") || + unixIsLocked(pid, hShm, F_RDLCK, SHM_WRITE, 1, "WAL-WRITE") ){ + nLock = 1; + } + } + + sqlite3_mutex_leave(pShmNode->mutex); + } + + if( hShm<0 ){ + /* the shared memory file isn't open in this process space, open our own FD */ + char zShm[MAXPATHLEN]; + + /* WAL mode */ + strlcpy(zShm, pFile->zPath, MAXPATHLEN); + strlcat(zShm, "-shm", MAXPATHLEN); + hShm = open(zShm, O_RDONLY, 0); + if( hShm<0 ){ + *pLockstate = SQLITE_LOCKSTATE_OFF; + unixLeaveMutex(); + return SQLITE_OK; + } + if( unixIsLocked(pid, hShm, F_RDLCK, SHM_RECOVER, 1, "WAL-RECOVERY") || + unixIsLocked(pid, hShm, F_RDLCK, SHM_WRITE, 1, "WAL-WRITE") ){ + nLock = 1; + } + close(hShm); + } + unixLeaveMutex(); + } + if( nLock>0 ){ + *pLockstate = SQLITE_LOCKSTATE_ON; + } else { + *pLockstate = SQLITE_LOCKSTATE_OFF; + } + return SQLITE_OK; +} + +#endif /* (SQLITE_ENABLE_APPLE_SPI>0) && defined(__APPLE__) */ + + /* ** Here ends the implementation of all sqlite3_file methods. ** ********************** End sqlite3_file Methods ******************************* Index: src/os_win.c ================================================================== --- src/os_win.c +++ src/os_win.c @@ -100,12 +100,13 @@ typedef struct winFile winFile; struct winFile { const sqlite3_io_methods *pMethod; /*** Must be first ***/ sqlite3_vfs *pVfs; /* The VFS used to open this file */ HANDLE h; /* Handle for accessing the file */ - unsigned char locktype; /* Type of lock currently held on this file */ + u8 locktype; /* Type of lock currently held on this file */ short sharedLockByte; /* Randomly chosen byte used as a shared lock */ + u8 bPersistWal; /* True to persist WAL files */ DWORD lastErrno; /* The Windows errno from the last I/O error */ DWORD sectorSize; /* Sector size of the device file is on */ winShm *pShm; /* Instance of shared memory on this file */ const char *zPath; /* Full pathname of this file */ int szChunk; /* Chunk size configured by FCNTL_CHUNK_SIZE */ @@ -1274,29 +1275,39 @@ /* ** Control and query of the open file handle. */ static int winFileControl(sqlite3_file *id, int op, void *pArg){ + winFile *pFile = (winFile*)id; switch( op ){ case SQLITE_FCNTL_LOCKSTATE: { - *(int*)pArg = ((winFile*)id)->locktype; + *(int*)pArg = pFile->locktype; return SQLITE_OK; } case SQLITE_LAST_ERRNO: { - *(int*)pArg = (int)((winFile*)id)->lastErrno; + *(int*)pArg = (int)pFile->lastErrno; return SQLITE_OK; } case SQLITE_FCNTL_CHUNK_SIZE: { - ((winFile*)id)->szChunk = *(int *)pArg; + pFile->szChunk = *(int *)pArg; return SQLITE_OK; } case SQLITE_FCNTL_SIZE_HINT: { sqlite3_int64 sz = *(sqlite3_int64*)pArg; SimulateIOErrorBenign(1); winTruncate(id, sz); SimulateIOErrorBenign(0); return SQLITE_OK; + } + case SQLITE_FCNTL_PERSIST_WAL: { + int bPersist = *(int*)pArg; + if( bPersist<0 ){ + *(int*)pArg = pFile->bPersistWal; + }else{ + pFile->bPersistWal = bPersist!=0; + } + return SQLITE_OK; } case SQLITE_FCNTL_SYNC_OMITTED: { return SQLITE_OK; } } Index: src/pager.c ================================================================== --- src/pager.c +++ src/pager.c @@ -6737,15 +6737,15 @@ ** (e.g. due to malloc() failure), return an error code. */ if( rc==SQLITE_OK ){ #if SQLITE_ENABLE_DATA_PROTECTION rc = sqlite3WalOpen(pPager->pVfs, pPager->fd, pPager->zWal, pPager->exclusiveMode, - pPager->journalSizeLimit, (pPager->vfsFlags & SQLITE_OPEN_FILEPROTECTION_MASK), + pPager->journalSizeLimit, (pPager->vfsFlags & (SQLITE_OPEN_FILEPROTECTION_MASK | SQLITE_OPEN_READONLY)), &pPager->pWal); #else rc = sqlite3WalOpen(pPager->pVfs, pPager->fd, pPager->zWal, pPager->exclusiveMode, - pPager->journalSizeLimit, 0, &pPager->pWal); + pPager->journalSizeLimit, (pPager->vfsFlags & SQLITE_OPEN_READONLY), &pPager->pWal); #endif } return rc; } Index: src/sqlite.h.in ================================================================== --- src/sqlite.h.in +++ src/sqlite.h.in @@ -735,20 +735,39 @@ ** when [PRAGMA synchronous | PRAGMA synchronous=OFF] is set, but most ** VFSes do not need this signal and should silently ignore this opcode. ** Applications should not call [sqlite3_file_control()] with this ** opcode as doing so may disrupt the operation of the specialized VFSes ** that do require it. +** +** ^The [SQLITE_FCNTL_PERSIST_WAL] opcode is used to set or query the +** persistent [WAL | Write AHead Log] setting. By default, the auxiliary +** write ahead log and shared memory files used for transaction control +** are automatically deleted when the latest connection to the database +** closes. Setting persistent WAL mode causes those files to persist after +** close. Persisting the files is useful when other processes that do not +** have write permission on the directory containing the database file want +** to read the database file, as the WAL and shared memory files must exist +** in order for the database to be readable. The fourth parameter to +** [sqlite3_file_control()] for this opcode should be a pointer to an integer. +** That integer is 0 to disable persistent WAL mode or 1 to enable persistent +** WAL mode. If the integer is -1, then it is overwritten with the current +** WAL persistence setting. +** */ -#define SQLITE_FCNTL_LOCKSTATE 1 -#define SQLITE_GET_LOCKPROXYFILE 2 -#define SQLITE_SET_LOCKPROXYFILE 3 -#define SQLITE_LAST_ERRNO 4 -#define SQLITE_FCNTL_SIZE_HINT 5 -#define SQLITE_FCNTL_CHUNK_SIZE 6 -#define SQLITE_FCNTL_FILE_POINTER 7 -#define SQLITE_FCNTL_SYNC_OMITTED 8 - +#define SQLITE_FCNTL_LOCKSTATE 1 +#define SQLITE_FCNTL_GET_LOCKPROXYFILE 2 +#define SQLITE_FCNTL_SET_LOCKPROXYFILE 3 +#define SQLITE_FCNTL_LAST_ERRNO 4 +#define SQLITE_FCNTL_SIZE_HINT 5 +#define SQLITE_FCNTL_CHUNK_SIZE 6 +#define SQLITE_FCNTL_FILE_POINTER 7 +#define SQLITE_FCNTL_SYNC_OMITTED 8 +#define SQLITE_FCNTL_PERSIST_WAL 10 +/* deprecated names */ +#define SQLITE_GET_LOCKPROXYFILE SQLITE_FCNTL_GET_LOCKPROXYFILE +#define SQLITE_SET_LOCKPROXYFILE SQLITE_FCNTL_SET_LOCKPROXYFILE +#define SQLITE_LAST_ERRNO SQLITE_FCNTL_LAST_ERRNO /* ** CAPI3REF: Mutex Handle ** ** The mutex module within SQLite defines [sqlite3_mutex] to be an @@ -3321,10 +3340,16 @@ ** current row of the result set of [prepared statement] P. ** ^If prepared statement P does not have results ready to return ** (via calls to the [sqlite3_column_int | sqlite3_column_*()] of ** interfaces) then sqlite3_data_count(P) returns 0. ** ^The sqlite3_data_count(P) routine also returns 0 if P is a NULL pointer. +** ^The sqlite3_data_count(P) routine returns 0 if the previous call to +** [sqlite3_step](P) returned [SQLITE_DONE]. ^The sqlite3_data_count(P) +** will return non-zero if previous call to [sqlite3_step](P) returned +** [SQLITE_ROW], except in the case of the [PRAGMA incremental_vacuum] +** where it always returns zero since each step of that multi-step +** pragma returns 0 columns of data. ** ** See also: [sqlite3_column_count()] */ int sqlite3_data_count(sqlite3_stmt *pStmt); Index: src/sqlite3_private.h ================================================================== --- src/sqlite3_private.h +++ src/sqlite3_private.h @@ -25,19 +25,29 @@ ** currently. Zero byte files are tested for sqlite locks, but if no sqlite ** locks are present then SQLITE_LOCKSTATE_NOTADB is returned. */ extern int _sqlite3_lockstate(const char *path, pid_t pid); +/* +** Test an open database connection for sqlite locks held by a process ID, +** if a process has an open database connection this will avoid trashing file +** locks by re-using open file descriptors for the database file and support +** files (-shm) +*/ +#define SQLITE_FCNTL_LOCKSTATE_PID 103 + /* ** Pass the SQLITE_TRUNCATE_DATABASE operation code to sqlite3_file_control() ** to truncate a database and its associated journal file to zero length. */ -#define SQLITE_TRUNCATE_DATABASE 101 +#define SQLITE_FCNTL_TRUNCATE_DATABASE 101 +#define SQLITE_TRUNCATE_DATABASE SQLITE_FCNTL_TRUNCATE_DATABASE /* ** Pass the SQLITE_REPLACE_DATABASE operation code to sqlite3_file_control() ** and a sqlite3 pointer to another open database file to safely copy the ** contents of that database file into the receiving database. */ -#define SQLITE_REPLACE_DATABASE 102 +#define SQLITE_FCNTL_REPLACE_DATABASE 102 +#define SQLITE_REPLACE_DATABASE SQLITE_FCNTL_REPLACE_DATABASE #endif Index: src/sqliteInt.h ================================================================== --- src/sqliteInt.h +++ src/sqliteInt.h @@ -3248,6 +3248,31 @@ #define MEMTYPE_LOOKASIDE 0x02 /* Might have been lookaside memory */ #define MEMTYPE_SCRATCH 0x04 /* Scratch allocations */ #define MEMTYPE_PCACHE 0x08 /* Page cache allocations */ #define MEMTYPE_DB 0x10 /* Uses sqlite3DbMalloc, not sqlite_malloc */ + +#if (SQLITE_ENABLE_APPLE_SPI>0) && defined(__APPLE__) + +/* +** An instance of the following structure is used to hold the process ID +** and return-by-reference lockstate value. The SQLITE_FCNTL_LOCKSTATE_PID +** requires the 4th argument to sqlite3_file_control to be a pointer to an +** instance of LockstatePID initialized with a LockstatePID.pid value equal +** to a process ID to be tested, or the special value SQLITE_LOCKSTATE_ANYPID +** The Lockstate.state value is always set to one of the following values +** when sqlite3_file_control returns: +** +** SQLITE_LOCKSTATE_OFF no active sqlite file locks match the specified pid +** SQLITE_LOCKSTATE_ON active sqlite file locks match the specified pid +** SQLITE_LOCKSTATE_NOTADB path points to a file that is not an sqlite db file +** SQLITE_LOCKSTATE_ERROR path was not vaild or was unreadable +*/ +typedef struct LockstatePID LockstatePID; +struct LockstatePID { + pid_t pid; /* Process ID to test */ + int state; /* The state of the lock (return value) */ +}; + +#endif + #endif /* _SQLITEINT_H_ */ Index: src/test1.c ================================================================== --- src/test1.c +++ src/test1.c @@ -4988,21 +4988,25 @@ Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ int objc, /* Number of arguments */ Tcl_Obj *CONST objv[] /* Command arguments */ ){ sqlite3 *db; + int flags; int rc; - if( objc!=2 ){ + if( objc!=3 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", - Tcl_GetStringFromObj(objv[0], 0), " DB", 0); + Tcl_GetStringFromObj(objv[0], 0), " DB FLAGS", 0); return TCL_ERROR; } if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ){ return TCL_ERROR; } - rc = sqlite3_file_control(db, NULL, SQLITE_TRUNCATE_DATABASE, 0); + if( Tcl_GetIntFromObj(interp, objv[2], &flags) ){ + return TCL_ERROR; + } + rc = sqlite3_file_control(db, NULL, SQLITE_TRUNCATE_DATABASE, &flags); if( rc ){ Tcl_SetObjResult(interp, Tcl_NewIntObj(rc)); return TCL_ERROR; } return TCL_OK; @@ -5204,11 +5208,10 @@ ClientData clientData, /* Pointer to sqlite3_enable_XXX function */ Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ int objc, /* Number of arguments */ Tcl_Obj *CONST objv[] /* Command arguments */ ){ - sqlite3 *db; const char *zPath; int nPath; if( objc!=2 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", @@ -5246,11 +5249,10 @@ ClientData clientData, /* Pointer to sqlite3_enable_XXX function */ Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ int objc, /* Number of arguments */ Tcl_Obj *CONST objv[] /* Command arguments */ ){ - sqlite3 *db; const char *zPath; int nPath; if( objc!=2 ){ Tcl_AppendResult(interp, "wrong # args: should be \"", @@ -5280,10 +5282,43 @@ Tcl_SetObjResult(interp, Tcl_NewIntObj(0)); #endif return TCL_OK; } + +/* +** tclcmd: file_control_persist_wal DB PERSIST-FLAG +** +** This TCL command runs the sqlite3_file_control interface with +** the SQLITE_FCNTL_PERSIST_WAL opcode. +*/ +static int file_control_persist_wal( + ClientData clientData, /* Pointer to sqlite3_enable_XXX function */ + Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ + int objc, /* Number of arguments */ + Tcl_Obj *CONST objv[] /* Command arguments */ +){ + sqlite3 *db; + int rc; + int bPersist; + char z[100]; + + if( objc!=3 ){ + Tcl_AppendResult(interp, "wrong # args: should be \"", + Tcl_GetStringFromObj(objv[0], 0), " DB FLAG", 0); + return TCL_ERROR; + } + if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ){ + return TCL_ERROR; + } + if( Tcl_GetIntFromObj(interp, objv[2], &bPersist) ) return TCL_ERROR; + rc = sqlite3_file_control(db, NULL, SQLITE_FCNTL_PERSIST_WAL, (void*)&bPersist); + sqlite3_snprintf(sizeof(z), z, "%d %d", rc, bPersist); + Tcl_AppendResult(interp, z, (char*)0); + return TCL_OK; +} + /* ** tclcmd: sqlite3_vfs_list ** ** Return a tcl list containing the names of all registered vfs's. @@ -5992,10 +6027,11 @@ { "file_control_truncate_test", file_control_truncate_test, 0 }, { "file_control_replace_test", file_control_replace_test, 0 }, #endif { "file_control_chunksize_test", file_control_chunksize_test, 0 }, { "file_control_sizehint_test", file_control_sizehint_test, 0 }, + { "file_control_persist_wal", file_control_persist_wal, 0 }, { "sqlite3_vfs_list", vfs_list, 0 }, { "sqlite3_create_function_v2", test_create_function_v2, 0 }, { "path_is_local", path_is_local, 0 }, { "path_is_dos", path_is_dos, 0 }, Index: src/vdbeaux.c ================================================================== --- src/vdbeaux.c +++ src/vdbeaux.c @@ -1136,11 +1136,11 @@ SubProgram **apSub = 0; /* Array of sub-vdbes */ Mem *pSub = 0; /* Memory cell hold array of subprogs */ sqlite3 *db = p->db; /* The database connection */ int i; /* Loop counter */ int rc = SQLITE_OK; /* Return code */ - Mem *pMem = p->pResultSet = &p->aMem[1]; /* First Mem of result set */ + Mem *pMem = &p->aMem[1]; /* First Mem of result set */ assert( p->explain ); assert( p->magic==VDBE_MAGIC_RUN ); assert( p->rc==SQLITE_OK || p->rc==SQLITE_BUSY || p->rc==SQLITE_NOMEM ); @@ -1147,10 +1147,11 @@ /* Even though this opcode does not use dynamic strings for ** the result, result columns may become dynamic if the user calls ** sqlite3_column_text16(), causing a translation to UTF-16 encoding. */ releaseMemArray(pMem, 8); + p->pResultSet = 0; if( p->rc==SQLITE_NOMEM ){ /* This happens if a malloc() inside a call to sqlite3_column_text() or ** sqlite3_column_text16() failed. */ db->mallocFailed = 1; @@ -1301,10 +1302,11 @@ pMem->type = SQLITE_NULL; } } p->nResColumn = 8 - 4*(p->explain-1); + p->pResultSet = &p->aMem[1]; p->rc = SQLITE_OK; rc = SQLITE_ROW; } return rc; } @@ -1504,10 +1506,12 @@ } memset(zCsr, 0, zEnd-zCsr); zCsr += (zCsr - (u8*)0)&7; assert( EIGHT_BYTE_ALIGNMENT(zCsr) ); + p->expired = 0; + /* Memory for registers, parameters, cursor, etc, is allocated in two ** passes. On the first pass, we try to reuse unused space at the ** end of the opcode array. If we are unable to satisfy all memory ** requirements by reusing the opcode array tail, then the second ** pass will fill in the rest using a fresh allocation. Index: src/wal.c ================================================================== --- src/wal.c +++ src/wal.c @@ -1283,11 +1283,15 @@ pRet->mxWalSize = mxWalSize; pRet->zWalName = zWalName; pRet->exclusiveMode = (bNoShm ? WAL_HEAPMEMORY_MODE: WAL_NORMAL_MODE); /* Open file handle on the write-ahead log file. */ - vfsFlags = flags | (SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|SQLITE_OPEN_WAL); + if( flags&SQLITE_OPEN_READONLY ){ + vfsFlags = flags | SQLITE_OPEN_WAL; + } else { + vfsFlags = flags | (SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|SQLITE_OPEN_WAL); + } rc = sqlite3OsOpen(pVfs, zWalName, pRet->pWalFd, vfsFlags, &vfsFlags); if( rc==SQLITE_OK && vfsFlags&SQLITE_OPEN_READONLY ){ pRet->readOnly = WAL_RDONLY; } @@ -1803,17 +1807,19 @@ ** ** The EXCLUSIVE lock is not released before returning. */ rc = sqlite3OsLock(pWal->pDbFd, SQLITE_LOCK_EXCLUSIVE); if( rc==SQLITE_OK ){ + int bPersistWal = -1; if( pWal->exclusiveMode==WAL_NORMAL_MODE ){ pWal->exclusiveMode = WAL_EXCLUSIVE_MODE; } rc = sqlite3WalCheckpoint( pWal, SQLITE_CHECKPOINT_PASSIVE, 0, 0, sync_flags, nBuf, zBuf, 0, 0 ); - if( rc==SQLITE_OK ){ + sqlite3OsFileControl(pWal->pDbFd, SQLITE_FCNTL_PERSIST_WAL, &bPersistWal); + if( rc==SQLITE_OK && bPersistWal!=1 ){ isDelete = 1; } } walIndexClose(pWal, isDelete); @@ -1862,10 +1868,13 @@ ** When reading, read [0] first then [1]. Writes are in the reverse order. ** Memory barriers are used to prevent the compiler or the hardware from ** reordering the reads and writes. */ aHdr = walIndexHdr(pWal); + if( aHdr==NULL ){ + return 1; /* Shouldn't be getting NULL from walIndexHdr, but we are */ + } memcpy(&h1, (void *)&aHdr[0], sizeof(h1)); walShmBarrier(pWal); memcpy(&h2, (void *)&aHdr[1], sizeof(h2)); if( memcmp(&h1, &h2, sizeof(h1))!=0 ){ @@ -2046,11 +2055,11 @@ ** so that on the 100th (and last) RETRY we delay for 21 milliseconds. ** The total delay time before giving up is less than 1 second. */ if( cnt>5 ){ int nDelay = 1; /* Pause time in microseconds */ - if( cnt>100 ){ + if( cnt>500 ){ VVA_ONLY( pWal->lockError = 1; ) return SQLITE_PROTOCOL; } if( cnt>=10 ) nDelay = (cnt-9)*238; /* Max delay 21ms. Total delay 996ms */ sqlite3OsSleep(pWal->pVfs, nDelay); ADDED test/walpersist.test Index: test/walpersist.test ================================================================== --- /dev/null +++ test/walpersist.test @@ -0,0 +1,68 @@ +# 2011 July 26 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file contains tests for using WAL with persistent WAL file mode. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/lock_common.tcl +set ::testprefix walpersist + +do_test walpersist-1.0 { + db eval { + PRAGMA journal_mode=WAL; + CREATE TABLE t1(a); + INSERT INTO t1 VALUES(randomblob(5000)); + } + file exists test.db-wal +} {1} +do_test walpersist-1.1 { + file exists test.db-shm +} {1} +do_test walpersist-1.2 { + db close + list [file exists test.db] [file exists test.db-wal] [file exists test.db-shm] +} {1 0 0} +do_test walpersist-1.3 { + sqlite3 db test.db + db eval {SELECT length(a) FROM t1} +} {5000} +do_test walpersist-1.4 { + list [file exists test.db] [file exists test.db-wal] [file exists test.db-shm] +} {1 1 1} +do_test walpersist-1.5 { + file_control_persist_wal db -1 +} {0 0} +do_test walpersist-1.6 { + file_control_persist_wal db 1 +} {0 1} +do_test walpersist-1.7 { + file_control_persist_wal db -1 +} {0 1} +do_test walpersist-1.8 { + file_control_persist_wal db 0 +} {0 0} +do_test walpersist-1.9 { + file_control_persist_wal db -1 +} {0 0} +do_test walpersist-1.10 { + file_control_persist_wal db 1 +} {0 1} +do_test walpersist-1.11 { + db close + list [file exists test.db] [file exists test.db-wal] [file exists test.db-shm] +} {1 1 1} + + + + +finish_test