Index: Makefile.in ================================================================== --- Makefile.in +++ Makefile.in @@ -173,11 +173,11 @@ icu.lo insert.lo journal.lo legacy.lo loadext.lo \ main.lo malloc.lo mem0.lo mem1.lo mem2.lo mem3.lo mem5.lo \ memjournal.lo \ mutex.lo mutex_noop.lo mutex_unix.lo mutex_w32.lo \ notify.lo opcodes.lo os.lo os_unix.lo os_win.lo \ - pager.lo parse.lo pcache.lo pcache1.lo pragma.lo prepare.lo printf.lo \ + pager.lo parse.lo pcache.lo pcache1.lo pcache_purgeable.lo pragma.lo prepare.lo printf.lo \ random.lo resolve.lo rowset.lo rtree.lo select.lo status.lo \ table.lo tokenize.lo trigger.lo \ update.lo util.lo vacuum.lo \ vdbe.lo vdbeapi.lo vdbeaux.lo vdbeblob.lo vdbemem.lo vdbesort.lo \ vdbetrace.lo wal.lo walker.lo where.lo utf.lo vtab.lo @@ -247,10 +247,11 @@ $(TOP)/src/pager.h \ $(TOP)/src/parse.y \ $(TOP)/src/pcache.c \ $(TOP)/src/pcache.h \ $(TOP)/src/pcache1.c \ + $(TOP)/src/pcache_purgeable.c \ $(TOP)/src/pragma.c \ $(TOP)/src/prepare.c \ $(TOP)/src/printf.c \ $(TOP)/src/random.c \ $(TOP)/src/resolve.c \ @@ -262,10 +263,11 @@ $(TOP)/src/sqlite3ext.h \ $(TOP)/src/sqliteInt.h \ $(TOP)/src/sqliteLimit.h \ $(TOP)/src/table.c \ $(TOP)/src/tclsqlite.c \ + $(TOP)/src/test_intarray.h \ $(TOP)/src/tokenize.c \ $(TOP)/src/trigger.c \ $(TOP)/src/utf.c \ $(TOP)/src/update.c \ $(TOP)/src/util.c \ @@ -331,10 +333,14 @@ $(TOP)/ext/rtree/rtree.h \ $(TOP)/ext/rtree/rtree.c SRC += \ $(TOP)/ext/sqlrr/sqlrr.h \ $(TOP)/ext/sqlrr/sqlrr.c +SRC += \ + $(TOP)/src/test_intarray.c +SRC += \ + $(TOP)/src/sqlite3_private.h # Generated source code files # SRC += \ @@ -367,11 +373,10 @@ $(TOP)/src/test_devsym.c \ $(TOP)/src/test_fs.c \ $(TOP)/src/test_func.c \ $(TOP)/src/test_hexio.c \ $(TOP)/src/test_init.c \ - $(TOP)/src/test_intarray.c \ $(TOP)/src/test_journal.c \ $(TOP)/src/test_malloc.c \ $(TOP)/src/test_multiplex.c \ $(TOP)/src/test_mutex.c \ $(TOP)/src/test_onefile.c \ @@ -469,10 +474,11 @@ parse.h \ sqlite3.h \ $(TOP)/src/sqlite3ext.h \ $(TOP)/src/sqliteInt.h \ $(TOP)/src/sqliteLimit.h \ + $(TOP)/src/test_intarray.h \ $(TOP)/src/vdbe.h \ $(TOP)/src/vdbeInt.h \ $(TOP)/src/whereInt.h \ config.h @@ -710,10 +716,13 @@ pcache.lo: $(TOP)/src/pcache.c $(HDR) $(TOP)/src/pcache.h $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/pcache.c pcache1.lo: $(TOP)/src/pcache1.c $(HDR) $(TOP)/src/pcache.h $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/pcache1.c + +pcache_purgeable.lo: $(TOP)/src/pcache_purgeable.c $(HDR) $(TOP)/src/pcache.h + $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/pcache_purgeable.c os.lo: $(TOP)/src/os.c $(HDR) $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/os.c os_unix.lo: $(TOP)/src/os_unix.c $(HDR) Index: config.h.in ================================================================== --- config.h.in +++ config.h.in @@ -32,14 +32,14 @@ /* Define to 1 if you have the `localtime_s' function. */ #undef HAVE_LOCALTIME_S /* Define to 1 if you have the header file. */ -#undef HAVE_MALLOC_H +/* #undef HAVE_MALLOC_H */ /* Define to 1 if you have the `malloc_usable_size' function. */ -#undef HAVE_MALLOC_USABLE_SIZE +/* #undef HAVE_MALLOC_USABLE_SIZE */ /* Define to 1 if you have the header file. */ #undef HAVE_MEMORY_H /* Define to 1 if you have the header file. */ @@ -80,11 +80,11 @@ /* Define to 1 if you have the `usleep' function. */ #undef HAVE_USLEEP /* Define to 1 if you have the utime() library function. */ -#undef HAVE_UTIME +/* #undef HAVE_UTIME */ /* Define to the sub-directory in which libtool stores uninstalled libraries. */ #undef LT_OBJDIR Index: src/btree.c ================================================================== --- src/btree.c +++ src/btree.c @@ -12,16 +12,33 @@ ** This file implements a external (disk-based) database using BTrees. ** See the header comment on "btreeInt.h" for additional information. ** Including a description of file format and an overview of operation. */ #include "btreeInt.h" +#include /* ** The header string that appears at the beginning of every ** SQLite database. */ static const char zMagicHeader[] = SQLITE_FILE_HEADER; + +#if defined(SQLITE_ENABLE_FLOCKTIMEOUT)&&(SQLITE_ENABLE_FLOCKTIMEOUT>0) +# if (defined(__APPLE__) && (!defined(TARGET_IPHONE_SIMULATOR)||(TARGET_IPHONE_SIMULATOR==0)) && ((TARGET_OS_EMBEDDED>0)||(MAC_OS_X_VERSION_10_9>0))) +# define SQLITE_USE_FLOCKTIMEOUT 1 +# endif +#endif /* SQLITE_ENABLE_FLOCKTIMEOUT */ + +/* + ** Operator used with sqlite3_file_control to set the timeout for the next call + ** to lock the database file via fcntl with F_SETLKWTIMEOUT. The timeout should + ** be passed as a pointer to a long representing the timeout in milliseconds + */ +#ifndef SQLITE_FCNTL_SET_NEXTFLOCKTIMEOUT +#define SQLITE_FCNTL_SET_NEXTFLOCKTIMEOUT 104 +#define SQLITE_FCNTL_GET_NEXTFLOCKTIMEOUT 105 +#endif /* ** Set this global variable to 1 to enable tracing using the TRACE ** macro. */ @@ -2766,12 +2783,26 @@ ** may return SQLITE_OK but leave pBt->pPage1 set to 0 if after ** reading page 1 it discovers that the page-size of the database ** file is not pBt->pageSize. In this case lockBtree() will update ** pBt->pageSize to the page-size of the file on disk. */ + +#ifdef SQLITE_USE_FLOCKTIMEOUT + int existingNextTimeout = 0; + if (wrflag) { + sqlite3_file_control(p->db, NULL, SQLITE_FCNTL_GET_NEXTFLOCKTIMEOUT, &existingNextTimeout); + } + + while( pBt->pPage1==0 && SQLITE_OK==(rc = lockBtree(pBt)) ); + + if (wrflag) { + sqlite3_file_control(p->db, NULL, SQLITE_FCNTL_SET_NEXTFLOCKTIMEOUT, &existingNextTimeout); + } +#else while( pBt->pPage1==0 && SQLITE_OK==(rc = lockBtree(pBt)) ); - +#endif + if( rc==SQLITE_OK && wrflag ){ if( (pBt->btsFlags & BTS_READ_ONLY)!=0 ){ rc = SQLITE_READONLY; }else{ rc = sqlite3PagerBegin(pBt->pPager,wrflag>1,sqlite3TempInMemory(p->db)); Index: src/ctime.c ================================================================== --- src/ctime.c +++ src/ctime.c @@ -45,10 +45,13 @@ #ifdef SQLITE_CHECK_PAGES "CHECK_PAGES", #endif #ifdef SQLITE_COVERAGE_TEST "COVERAGE_TEST", +#endif +#ifdef SQLITE_CURDIR + "CURDIR", #endif #ifdef SQLITE_DEBUG "DEBUG", #endif #ifdef SQLITE_DEFAULT_LOCKING_MODE Index: src/loadext.c ================================================================== --- src/loadext.c +++ src/loadext.c @@ -747,12 +747,11 @@ wsdAutoext.aExt[i]; } sqlite3_mutex_leave(mutex); zErrmsg = 0; if( xInit && (rc = xInit(db, &zErrmsg, &sqlite3Apis))!=0 ){ - sqlite3Error(db, rc, - "automatic extension loading failed: %s", zErrmsg); - go = 0; +// sqlite3Error(db, rc, "automatic extension loading failed: %s", zErrmsg); +// go = 0; } sqlite3_free(zErrmsg); } } Index: src/main.c ================================================================== --- src/main.c +++ src/main.c @@ -13,10 +13,11 @@ ** implement the programmer interface to the library. Routines in ** other files are for internal use by SQLite and should not be ** accessed by users of the library. */ #include "sqliteInt.h" +#include #ifdef SQLITE_ENABLE_SQLRR # include "sqlrr.h" #endif #ifdef SQLITE_ENABLE_FTS3 @@ -27,10 +28,42 @@ #endif #ifdef SQLITE_ENABLE_ICU # include "sqliteicu.h" #endif +#if defined(SQLITE_ENABLE_FLOCKTIMEOUT)&&(SQLITE_ENABLE_FLOCKTIMEOUT>0) +# if (defined(__APPLE__) && (!defined(TARGET_IPHONE_SIMULATOR)||(TARGET_IPHONE_SIMULATOR==0)) && ((TARGET_OS_EMBEDDED>0)||(MAC_OS_X_VERSION_10_9>0))) +# include "sqlite3_private.h" +# define SQLITE_USE_FLOCKTIMEOUT 1 +# endif +#endif /* SQLITE_ENABLE_FLOCKTIMEOUT */ + +#if defined(__APPLE__) +# include +# include +# include +extern void _sqlite3_purgeEligiblePagerCacheMemory(); +# if defined(DISPATCH_API_VERSION)&&(DISPATCH_API_VERSION>=20100226) && \ + defined(SQLITE_ENABLE_PURGEABLE_PCACHE)&&(SQLITE_ENABLE_PURGEABLE_PCACHE>0) +# include /* amalgamator: keep */ +static dispatch_source_t _cache_event_source = NULL; +# endif +#endif + +/* + ** Operator used with sqlite3_file_control to set the timeout for the next call + ** to lock the database file via fcntl with F_SETLKWTIMEOUT. The timeout should + ** be passed as a pointer to a long representing the timeout in milliseconds + */ +#ifndef SQLITE_FCNTL_SET_NEXTFLOCKTIMEOUT +#define SQLITE_FCNTL_SET_NEXTFLOCKTIMEOUT 104 +#define SQLITE_FCNTL_GET_NEXTFLOCKTIMEOUT 105 +#endif + +/* +** The version of the library +*/ #ifndef SQLITE_AMALGAMATION /* IMPLEMENTATION-OF: R-46656-45156 The sqlite3_version[] string constant ** contains the text of SQLITE_VERSION macro. */ const char sqlite3_version[] = SQLITE_VERSION; @@ -215,10 +248,50 @@ sqlite3GlobalConfig.isInit = 1; #ifdef SQLITE_EXTRA_INIT bRunExtraInit = 1; #endif } + +#if defined(__APPLE__) && defined(USETHREADASSERTS) + if ((0 != access("/var/db/disableAppleInternal", R_OK))) { +// fprintf(stderr, "SQLite multi-threading assertions ENABLED.\n"); + + if (sqlite3GlobalConfig.bCoreMutex != 0) { + sqlite3GlobalConfig.bFullMutex = 1; + } + } else { + sqlite3_mutex_methods *pTo = &sqlite3GlobalConfig.mutex; + pTo->xMutexHeld = NULL; + pTo->xMutexNotheld = NULL; +// fprintf(stderr, "SQLite multi-threading assertions DISABLED.\n"); + } +#endif + + +#if defined(DISPATCH_API_VERSION)&&(DISPATCH_API_VERSION>=20100226) && \ + defined(SQLITE_ENABLE_PURGEABLE_PCACHE)&&(SQLITE_ENABLE_PURGEABLE_PCACHE>0) + + dispatch_queue_t mq = dispatch_get_main_queue(); + if (mq) { + if( NULL != dlopen("/usr/lib/libsqlite3.dylib", RTLD_LAZY) ){ + /* never dlclose to avoid dylib unloading */ + dispatch_async(mq, ^{ + dispatch_queue_t queue = dispatch_get_global_queue(0, 0); + if (queue) { + _cache_event_source = dispatch_source_create(DISPATCH_SOURCE_TYPE_VM, 0, DISPATCH_VM_PRESSURE, queue); + if (_cache_event_source != NULL) { + dispatch_source_set_event_handler(_cache_event_source, ^{ + _sqlite3_purgeEligiblePagerCacheMemory(); + }); + dispatch_resume(_cache_event_source); + } + } + }); + } + } +#endif + sqlite3GlobalConfig.inProgress = 0; } sqlite3_mutex_leave(sqlite3GlobalConfig.pInitMutex); /* Go back under the static mutex and clean up the recursive @@ -1229,14 +1302,22 @@ static int sqliteDefaultBusyCallback( void *ptr, /* Database connection */ int count /* Number of times table has been busy */ ){ #if SQLITE_OS_WIN || (defined(HAVE_USLEEP) && HAVE_USLEEP) +# ifdef SQLITE_USE_FLOCKTIMEOUT + static const u16 delays[] = + { 1, 2, 5, 10, 15, 20, 25, 25, 25, 50, 50, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 200, 200, 200, 200, 200, 250, 250, 250, 250, 333 }; // 333, 333, 500, 500, 500, 500, 1000, 1000, 1000, 10000 }; + static const u16 totals[] = + { 0, 1, 3, 8, 18, 33, 53, 78, 103, 128, 178, 228, 328, 428, 528, 628, 728, 828, 928, 1028, 1128, 1228, 1328, 1428, 1528, 1628, 1728, 1828, 1928, 2128, 2228, 2428, 2628, 2828, 3028, 3228, 3478, 3728, 3978, 4228, 4561, 4894, 5227, 5727, 6227, 6727, 7227, 8227, 9227 }; +#else static const u8 delays[] = { 1, 2, 5, 10, 15, 20, 25, 25, 25, 50, 50, 100 }; static const u8 totals[] = { 0, 1, 3, 8, 18, 33, 53, 78, 103, 128, 178, 228 }; +#endif + # define NDELAY ArraySize(delays) sqlite3 *db = (sqlite3 *)ptr; int timeout = db->busyTimeout; int delay, prior; @@ -1248,13 +1329,23 @@ delay = delays[NDELAY-1]; prior = totals[NDELAY-1] + delay*(count-(NDELAY-1)); } if( prior + delay > timeout ){ delay = timeout - prior; - if( delay<=0 ) return 0; + if( delay<=0 ){ +# ifdef SQLITE_USE_FLOCKTIMEOUT + delay = 0; /* reset the delay */ + sqlite3_file_control(db, NULL, SQLITE_FCNTL_SET_NEXTFLOCKTIMEOUT, &delay); +# endif + return 0; + } } +# ifdef SQLITE_USE_FLOCKTIMEOUT + sqlite3_file_control(db, NULL, SQLITE_FCNTL_SET_NEXTFLOCKTIMEOUT, &delay); +# else sqlite3OsSleep(db->pVfs, delay*1000); +# endif return 1; #else sqlite3 *db = (sqlite3 *)ptr; int timeout = ((sqlite3 *)ptr)->busyTimeout; if( (count+1)*1000 > timeout ){ @@ -2189,23 +2280,23 @@ autolog_client = NULL; } } static void _open_asl_log() { if( NULL==autolog_client ){ - autolog_client = asl_open("SQLite", NULL, 0); + autolog_client = asl_open(NULL, NULL, 0); atexit(_close_asl_log); } } void _sqlite_auto_profile_syslog(void *aux, const char *sql, u64 ns); void _sqlite_auto_trace_syslog(void *aux, const char *sql); void _sqlite_auto_profile_syslog(void *aux, const char *sql, u64 ns) { #pragma unused(aux) - asl_log(autolog_client, NULL, ASL_LEVEL_NOTICE, "Query: %s\n Execution Time: %llu ms\n", sql, ns / 1000000); + asl_log(autolog_client, NULL, ASL_LEVEL_NOTICE, "Query: %s\n Execution Time: %llu ms\n", sql ? sql : "", ns / 1000000); } void _sqlite_auto_trace_syslog(void *aux, const char *sql) { - asl_log(autolog_client, NULL, ASL_LEVEL_NOTICE, "TraceSQL(%p): %s\n", aux, sql); + asl_log(autolog_client, NULL, ASL_LEVEL_NOTICE, "TraceSQL(%p): %s\n", aux, sql ? sql : ""); } #endif /* ** This function is used to parse both URIs and non-URI filenames passed by the @@ -2505,10 +2596,13 @@ sqlite3 *db; /* Store allocated handle here */ int rc; /* Return code */ int isThreadsafe; /* True for threadsafe connections */ char *zOpen = 0; /* Filename argument to pass to BtreeOpen() */ char *zErrMsg = 0; /* Error message from sqlite3ParseUri() */ +#if SQLITE_ENABLE_DATA_PROTECTION + int protFlags = flags & SQLITE_OPEN_FILEPROTECTION_MASK; +#endif *ppDb = 0; #ifndef SQLITE_OMIT_AUTOINIT rc = sqlite3_initialize(); if( rc ) return rc; @@ -2567,10 +2661,13 @@ SQLITE_OPEN_MASTER_JOURNAL | SQLITE_OPEN_NOMUTEX | SQLITE_OPEN_FULLMUTEX | SQLITE_OPEN_WAL ); +#if SQLITE_ENABLE_DATA_PROTECTION + flags |= protFlags; +#endif /* Allocate the sqlite data structure */ db = sqlite3MallocZero( sizeof(sqlite3) ); if( db==0 ) goto opendb_out; if( isThreadsafe ){ @@ -2677,21 +2774,10 @@ ** is accessed. */ sqlite3Error(db, SQLITE_OK, 0); sqlite3RegisterBuiltinFunctions(db); - /* Load automatic extensions - extensions that have been registered - ** using the sqlite3_automatic_extension() API. - */ - rc = sqlite3_errcode(db); - if( rc==SQLITE_OK ){ - sqlite3AutoLoadExtensions(db); - rc = sqlite3_errcode(db); - if( rc!=SQLITE_OK ){ - goto opendb_out; - } - } #ifdef SQLITE_ENABLE_FTS1 if( !db->mallocFailed ){ extern int sqlite3Fts1Init(sqlite3*); rc = sqlite3Fts1Init(db); @@ -2720,10 +2806,22 @@ #ifdef SQLITE_ENABLE_RTREE if( !db->mallocFailed && rc==SQLITE_OK){ rc = sqlite3RtreeInit(db); } #endif + + /* Load automatic extensions - extensions that have been registered + ** using the sqlite3_automatic_extension() API. + */ + rc = sqlite3_errcode(db); + if( rc==SQLITE_OK ){ + sqlite3AutoLoadExtensions(db); + rc = sqlite3_errcode(db); + if( rc!=SQLITE_OK ){ + goto opendb_out; + } + } /* -DSQLITE_DEFAULT_LOCKING_MODE=1 makes EXCLUSIVE the default locking ** mode. -DSQLITE_DEFAULT_LOCKING_MODE=0 make NORMAL the default locking ** mode. Doing nothing at all also makes NORMAL the default. */ @@ -3601,7 +3699,15 @@ if( NULL!=db ){ sqlite3_close(db); /* need to close even if open returns an error */ } return SQLITE_LOCKSTATE_ERROR; } + +/* +** Returns the system defined default sqlite3_busy_handler function +** +*/ +int (*_sqlite3_system_busy_handler(void))(void*,int) { + return &sqliteDefaultBusyCallback; +} #endif /* SQLITE_ENABLE_APPLE_SPI */ Index: src/mutex.c ================================================================== --- src/mutex.c +++ src/mutex.c @@ -135,19 +135,19 @@ if( p ){ sqlite3GlobalConfig.mutex.xMutexLeave(p); } } -#ifndef NDEBUG +#if defined(USETHREADASSERTS) || defined(SQLITE_DEBUG) /* ** The sqlite3_mutex_held() and sqlite3_mutex_notheld() routine are ** intended for use inside assert() statements. */ int sqlite3_mutex_held(sqlite3_mutex *p){ - return p==0 || sqlite3GlobalConfig.mutex.xMutexHeld(p); + return p==0 || (sqlite3GlobalConfig.mutex.xMutexHeld && sqlite3GlobalConfig.mutex.xMutexHeld(p)); } int sqlite3_mutex_notheld(sqlite3_mutex *p){ - return p==0 || sqlite3GlobalConfig.mutex.xMutexNotheld(p); + return p==0 || (sqlite3GlobalConfig.mutex.xMutexNotheld && sqlite3GlobalConfig.mutex.xMutexNotheld(p)); } #endif #endif /* !defined(SQLITE_MUTEX_OMIT) */ Index: src/mutex_unix.c ================================================================== --- src/mutex_unix.c +++ src/mutex_unix.c @@ -27,11 +27,11 @@ /* ** The sqlite3_mutex.id, sqlite3_mutex.nRef, and sqlite3_mutex.owner fields ** are necessary under two condidtions: (1) Debug builds and (2) using ** home-grown mutexes. Encapsulate these conditions into a single #define. */ -#if defined(SQLITE_DEBUG) || defined(SQLITE_HOMEGROWN_RECURSIVE_MUTEX) +#if defined(SQLITE_DEBUG) || defined(SQLITE_HOMEGROWN_RECURSIVE_MUTEX) || defined(USETHREADASSERTS) # define SQLITE_MUTEX_NREF 1 #else # define SQLITE_MUTEX_NREF 0 #endif @@ -67,11 +67,11 @@ ** On those platforms where pthread_equal() is not atomic, SQLite ** should be compiled without -DSQLITE_DEBUG and with -DNDEBUG to ** make sure no assert() statements are evaluated and hence these ** routines are never called. */ -#if !defined(NDEBUG) || defined(SQLITE_DEBUG) +#if defined(USETHREADASSERTS) || defined(SQLITE_DEBUG) static int pthreadMutexHeld(sqlite3_mutex *p){ return (p->nRef!=0 && pthread_equal(p->owner, pthread_self())); } static int pthreadMutexNotheld(sqlite3_mutex *p){ return p->nRef==0 || pthread_equal(p->owner, pthread_self())==0; @@ -334,11 +334,11 @@ pthreadMutexAlloc, pthreadMutexFree, pthreadMutexEnter, pthreadMutexTry, pthreadMutexLeave, -#ifdef SQLITE_DEBUG +#if defined(USETHREADASSERTS) || defined(SQLITE_DEBUG) pthreadMutexHeld, pthreadMutexNotheld #else 0, 0 Index: src/os_unix.c ================================================================== --- src/os_unix.c +++ src/os_unix.c @@ -45,10 +45,11 @@ */ #include "sqliteInt.h" #if SQLITE_OS_UNIX /* This file is used on unix only */ #include "btreeInt.h" /* Use by Apple extensions only */ +#include /* ** There are various methods for file locking used for concurrency ** control: @@ -93,10 +94,11 @@ #include #include #include #include #include +#include #include #if !defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0 #include #endif @@ -104,11 +106,16 @@ #if SQLITE_ENABLE_LOCKING_STYLE # include # include # if defined(__APPLE__) && ((__MAC_OS_X_VERSION_MIN_REQUIRED > 1050) || \ (__IPHONE_OS_VERSION_MIN_REQUIRED > 2000)) +#if (!defined(TARGET_OS_EMBEDDED)||(TARGET_OS_EMBEDDED==0))&&(!defined(TARGET_IPHONE_SIMULATOR)||(TARGET_IPHONE_SIMULATOR==0)) # define HAVE_GETHOSTUUID 1 +#warning "gethostuuid() is enabled." +#else +#warning "gethostuuid() is disabled." +#endif # endif # if OS_VXWORKS # include # include # else @@ -118,19 +125,29 @@ #endif /* SQLITE_ENABLE_LOCKING_STYLE */ #if defined(__APPLE__) || (SQLITE_ENABLE_LOCKING_STYLE && !OS_VXWORKS) # include #endif +#if defined(__APPLE__) && defined(SQLITE_ENABLE_LOCKING_STYLE) +# include +#endif #ifdef HAVE_UTIME # include #endif + +#if defined(__APPLE__) +#include +#include +#include +#endif /* ** Allowed values of unixFile.fsFlags */ #define SQLITE_FSFLAGS_IS_MSDOS 0x1 +#define SQLITE_FSFLAGS_IS_MMAP_SAFE 0x2 /* ** If we are to be thread-safe, include the pthreads header and define ** the SQLITE_UNIX_THREADS macro. */ @@ -137,10 +154,140 @@ #if SQLITE_THREADSAFE # include # define SQLITE_UNIX_THREADS 1 #endif +/* + ** Operator used with sqlite3_file_control to set the timeout for the next call + ** to lock the database file via fcntl with F_SETLKWTIMEOUT. The timeout should + ** be passed as a pointer to a long representing the timeout in milliseconds + */ +#ifndef SQLITE_FCNTL_SET_NEXTFLOCKTIMEOUT +#define SQLITE_FCNTL_SET_NEXTFLOCKTIMEOUT 104 +#define SQLITE_FCNTL_GET_NEXTFLOCKTIMEOUT 105 +#endif + +#if defined(__APPLE__) +#include +#include +static aslclient errorlog_client = NULL; + +static void _close_error_asl_log() { + if( NULL!=errorlog_client ){ + asl_close(errorlog_client); + errorlog_client = NULL; + } +} +static void _open_error_asl_log() { + OSMemoryBarrier(); + if( NULL==errorlog_client ){ + aslclient aclient = asl_open(NULL, NULL, ASL_OPT_STDERR); + + if (OSAtomicCompareAndSwapPtrBarrier(NULL, (void*)aclient, (void* volatile*)&errorlog_client)) { + atexit(_close_error_asl_log); + } else { + asl_close(aclient); + } + } +} + +#endif + +#if defined(USE_GUARDED_FD)&&defined(__APPLE__) +# include +# include + +/* taken from */ +# ifndef _GUARDID_T +typedef __uint64_t guardid_t; +# define GUARD_CLOSE (1u << 0) +# define GUARD_DUP (1u << 1) +# define GUARD_SOCKET_IPC (1u << 2) +# define GUARD_FILEPORT (1u << 3) +# endif /* _GUARDID_T */ + +#ifndef GUARD_WRITE +#define GUARD_WRITE (1u << 4) +#endif + +#ifndef GUARD_MMAP +#define GUARD_MMAP (1u << 5) +#endif + +#ifndef PROTECTION_CLASS_A +#define PROTECTION_CLASS_A 1 +#define PROTECTION_CLASS_B 2 +#define PROTECTION_CLASS_C 3 +#define PROTECTION_CLASS_D 4 +#endif + +#ifndef PROTECTION_CLASS_SYSTEM_DEFAULT +#define PROTECTION_CLASS_SYSTEM_DEFAULT 0 +#endif + +static int unixProtectionClassForVFSProtectionFlags(int protFlags) { + protFlags &= SQLITE_OPEN_FILEPROTECTION_MASK; + + switch (protFlags) { + case SQLITE_OPEN_FILEPROTECTION_COMPLETE: + return PROTECTION_CLASS_A; + case SQLITE_OPEN_FILEPROTECTION_COMPLETEUNLESSOPEN: + return PROTECTION_CLASS_B; + case SQLITE_OPEN_FILEPROTECTION_COMPLETEUNTILFIRSTUSERAUTHENTICATION: + return PROTECTION_CLASS_C; + case SQLITE_OPEN_FILEPROTECTION_NONE: + return PROTECTION_CLASS_D; + default: + return PROTECTION_CLASS_SYSTEM_DEFAULT; + } +} + +static int shmUnixProtectionClassForVFSProtectionFlags(int protFlags) { + protFlags &= SQLITE_OPEN_FILEPROTECTION_MASK; + + switch (protFlags) { + case SQLITE_OPEN_FILEPROTECTION_COMPLETE: + return PROTECTION_CLASS_C; + case SQLITE_OPEN_FILEPROTECTION_COMPLETEUNLESSOPEN: + return PROTECTION_CLASS_C; + case SQLITE_OPEN_FILEPROTECTION_COMPLETEUNTILFIRSTUSERAUTHENTICATION: + return PROTECTION_CLASS_C; + case SQLITE_OPEN_FILEPROTECTION_NONE: + return PROTECTION_CLASS_D; + default: + return PROTECTION_CLASS_D; + } +} + +static guardid_t guard = 0x8fd4dbfade2deadull; +static dispatch_once_t guard_once_token; + +// extern user_ssize_t guarded_pwrite_np(int fd, const guardid_t *guard, user_addr_t buf, user_size_t nbyte, off_t offset); +// extern user_addr_t guarded_mmap_np(caddr_t addr, const guardid_t *guard, size_t len, int prot, int flags, int fd, off_t pos); +// extern user_ssize_t guarded_write_np(int fd, const guardid_t *guard, user_addr_t cbuf, user_size_t nbyte); + + +static int (*dl_guarded_open_dprotected_np)(const char *path, const guardid_t *guard, u_int guardflags, int flags, int dpclass, int dpflags, ...); + +static int (*dl_guarded_open_np)(const char *path, const guardid_t *guard, u_int guardflags, int flags, ...); + +static int (*dl_guarded_close_np)(int fd, const guardid_t *guard); + +static int (*dl_guarded_pwrite_np)(int fd, const guardid_t *guard, user_addr_t buf, user_size_t nbyte, off_t offset); + +static int (*dl_guarded_write_np)(int fd, const guardid_t *guard, user_addr_t cbuf, user_size_t nbyte); + +static int (*dl_guarded_mmap_np)(caddr_t addr, const guardid_t *guard, size_t len, int prot, int flags, int fd, off_t pos); + +#endif /* USE_GUARDED_FD */ + +#if defined(SQLITE_ENABLE_FLOCKTIMEOUT)&&(SQLITE_ENABLE_FLOCKTIMEOUT>0) +# if (defined(__APPLE__) && (!defined(TARGET_IPHONE_SIMULATOR)||(TARGET_IPHONE_SIMULATOR==0)) && ((TARGET_OS_EMBEDDED>0)||(MAC_OS_X_VERSION_10_9>0))) +# define SQLITE_USE_FLOCKTIMEOUT 1 +# endif +#endif /* SQLITE_ENABLE_FLOCKTIMEOUT */ + /* ** Default permissions when creating a new file */ #ifndef SQLITE_DEFAULT_FILE_PERMISSIONS # define SQLITE_DEFAULT_FILE_PERMISSIONS 0644 @@ -154,11 +301,11 @@ #endif /* ** Maximum supported path-length. */ -#define MAX_PATHNAME 512 +#define MAX_PATHNAME PATH_MAX /* ** Only set the lastErrno if the error code is a real error and not ** a normal expected return code of SQLITE_BUSY or SQLITE_OK */ @@ -218,10 +365,13 @@ int protFlags; /* Data protection flags from unixOpen */ #endif #if SQLITE_ENABLE_LOCKING_STYLE || defined(__APPLE__) unsigned fsFlags; /* cached details from statfs() */ #endif +#ifdef SQLITE_USE_FLOCKTIMEOUT + int nextFlockTimeout; +#endif #if OS_VXWORKS struct vxworksFileId *pId; /* Unique file ID */ #endif #ifdef SQLITE_DEBUG /* The next group of variables are used to track whether or not the @@ -240,10 +390,18 @@ #ifdef SQLITE_TEST /* In test mode, increase the size of this structure a bit so that ** it is larger than the struct CrashFile defined in test6.c. */ char aPadding[32]; +#endif +#if defined(__APPLE__) + /* */ + /* This variable is used to sanity check reads and writes against + ** corrupt offsets or lengths. No reads or writes should ever exceed + ** the total capacity of the volume. + */ + i64 nFsBytes; #endif }; /* This variable holds the process id (pid) from when the xRandomness() ** method was called. If xOpen() is called from a different process id, @@ -601,11 +759,100 @@ ** The difference is important when using a pointer to the function. ** ** The safest way to deal with the problem is to always use this wrapper ** which always has the same well-defined interface. */ -static int posixOpen(const char *zFile, int flags, int mode){ +static int posixOpen(const char *zFile, int flags, int mode, int protectionFlags, int allowmmap){ +#if defined(USE_GUARDED_FD)&&defined(__APPLE__) + dispatch_once(&guard_once_token, ^{ + void *kernellib = (void *)dlopen("/usr/lib/libSystem.dylib", RTLD_GLOBAL); + assert(kernellib); + if (kernellib) { +#if SQLITE_ENABLE_DATA_PROTECTION + dl_guarded_open_dprotected_np = dlsym(kernellib, "guarded_open_dprotected_np"); +#else + dl_guarded_open_dprotected_np = NULL; +#endif + dl_guarded_open_np = dlsym(kernellib, "guarded_open_np"); + + if (dl_guarded_open_dprotected_np || dl_guarded_open_np) { + dl_guarded_close_np = dlsym(kernellib, "guarded_close_np"); + dl_guarded_pwrite_np = dlsym(kernellib, "guarded_pwrite_np"); + dl_guarded_write_np = dlsym(kernellib, "guarded_write_np"); + dl_guarded_mmap_np = NULL; // dlsym(kernellib, "guarded_mmap_np"); + +#if ((!defined(TARGET_IPHONE_SIMULATOR)||(TARGET_IPHONE_SIMULATOR==0)) && (!defined(__i386__))) + char* isbandi = getenv("RC_XBS"); + if (isbandi && (strlen(isbandi) > 0)) { + dl_guarded_pwrite_np = NULL; + dl_guarded_write_np = NULL; + dl_guarded_mmap_np = NULL; + } +#else + dl_guarded_open_dprotected_np = NULL; + dl_guarded_pwrite_np = NULL; + dl_guarded_write_np = NULL; + dl_guarded_mmap_np = NULL; +#endif + +#if defined(SQLITE_DEBUG) && !defined(SQLITE_TEST) + _open_error_asl_log(); + if (dl_guarded_open_dprotected_np) { + asl_log(errorlog_client, NULL, ASL_LEVEL_NOTICE, "SQLite: Found guarded_open_dprotected_np\n"); + } else if (dl_guarded_open_np) { + asl_log(errorlog_client, NULL, ASL_LEVEL_NOTICE, "SQLite: Using guarded_open_np\n"); + } + if (dl_guarded_pwrite_np) { + asl_log(errorlog_client, NULL, ASL_LEVEL_NOTICE, "SQLite: Using guarded_pwrite_np\n"); + } else { + asl_log(errorlog_client, NULL, ASL_LEVEL_WARNING, "SQLite: Failed to find guarded_pwrite_np!\n"); + } + if (dl_guarded_write_np) { + asl_log(errorlog_client, NULL, ASL_LEVEL_NOTICE, "SQLite: Using guarded_write_np\n"); + if (!dl_guarded_pwrite_np) { + dl_guarded_write_np = NULL; + } + } else { + asl_log(errorlog_client, NULL, ASL_LEVEL_WARNING, "SQLite: Failed to find guarded_write_np!\n"); + dl_guarded_pwrite_np = NULL; + } + + if( dl_guarded_close_np==NULL|| (dl_guarded_open_np == NULL && dl_guarded_open_dprotected_np == NULL) ){ + _open_error_asl_log(); + asl_log(errorlog_client, NULL, ASL_LEVEL_ERR, "SQLite: Failed to find guarded open/close\n"); + } +#else + if (dl_guarded_pwrite_np == NULL || dl_guarded_write_np == NULL) { + dl_guarded_pwrite_np = NULL; + dl_guarded_write_np = NULL; + } +#endif + } + } + }); + + if(dl_guarded_open_np || dl_guarded_open_dprotected_np){ + int guardflags = GUARD_CLOSE | GUARD_DUP | GUARD_SOCKET_IPC | GUARD_FILEPORT; + if (dl_guarded_pwrite_np && dl_guarded_write_np) { + if (!allowmmap) { + guardflags |= (GUARD_WRITE ); + } + } + if (dl_guarded_mmap_np) { + guardflags |= ( GUARD_MMAP ); + } + int dpc = 0; + if (flags & O_CREAT) { + dpc = protectionFlags ? protectionFlags : 0; + } + if (dl_guarded_open_dprotected_np && dpc) { + return dl_guarded_open_dprotected_np(zFile, &guard, guardflags, flags, dpc, 0, mode); + } else { + return dl_guarded_open_np(zFile, &guard, guardflags, flags, mode); + } + } +#endif return open(zFile, flags, mode); } /* ** On some systems, calls to fchown() will trigger a message in a security @@ -613,10 +860,45 @@ ** we are not running as root. */ static int posixFchown(int fd, uid_t uid, gid_t gid){ return geteuid() ? 0 : fchown(fd,uid,gid); } + +static ssize_t apple_write(int fildes, const void *buf, size_t nbyte, off_t offset) { +#if defined(USE_GUARDED_FD)&&defined(__APPLE__) + if( dl_guarded_write_np ){ + return dl_guarded_write_np(fildes, &guard, buf, nbyte); + } +#endif + return write(fildes, buf, nbyte); +} + +static ssize_t apple_Pwrite(int fildes, const void *buf, size_t nbyte, off_t offset) { +#if defined(USE_GUARDED_FD)&&defined(__APPLE__) + if( dl_guarded_pwrite_np ){ + return dl_guarded_pwrite_np(fildes, &guard, buf, nbyte, offset); + } +#endif + return pwrite(fildes, buf, nbyte, offset);; +} + +static ssize_t os_rawPwrite(int fildes, const void *buf, size_t nbyte, off_t offset) { + return pwrite(fildes, buf, nbyte, offset); +} + +static int posixOpen2(const char *zFile, int flags, int mode){ + return posixOpen(zFile, flags, mode, 0, 1); +} + +static int posixClose(int fd){ +#if defined(USE_GUARDED_FD)&&defined(__APPLE__) + if( dl_guarded_close_np ){ + return dl_guarded_close_np(fd, &guard); + } +#endif + return close(fd); +} /* Forward reference */ static int openDirectory(const char*, int*); static int unixGetpagesize(void); @@ -629,14 +911,14 @@ static struct unix_syscall { const char *zName; /* Name of the system call */ sqlite3_syscall_ptr pCurrent; /* Current value of the system call */ sqlite3_syscall_ptr pDefault; /* Default value */ } aSyscall[] = { - { "open", (sqlite3_syscall_ptr)posixOpen, 0 }, + { "open", (sqlite3_syscall_ptr)posixOpen2, 0 }, #define osOpen ((int(*)(const char*,int,int))aSyscall[0].pCurrent) - { "close", (sqlite3_syscall_ptr)close, 0 }, + { "close", (sqlite3_syscall_ptr)posixClose, 0 }, #define osClose ((int(*)(int))aSyscall[1].pCurrent) { "access", (sqlite3_syscall_ptr)access, 0 }, #define osAccess ((int(*)(const char*,int))aSyscall[2].pCurrent) @@ -681,15 +963,15 @@ #else { "pread64", (sqlite3_syscall_ptr)0, 0 }, #endif #define osPread64 ((ssize_t(*)(int,void*,size_t,off_t))aSyscall[10].pCurrent) - { "write", (sqlite3_syscall_ptr)write, 0 }, + { "write", (sqlite3_syscall_ptr)apple_write, 0 }, #define osWrite ((ssize_t(*)(int,const void*,size_t))aSyscall[11].pCurrent) #if defined(USE_PREAD) || SQLITE_ENABLE_LOCKING_STYLE - { "pwrite", (sqlite3_syscall_ptr)pwrite, 0 }, + { "pwrite", (sqlite3_syscall_ptr)apple_Pwrite, 0 }, #else { "pwrite", (sqlite3_syscall_ptr)0, 0 }, #endif #define osPwrite ((ssize_t(*)(int,const void*,size_t,off_t))\ aSyscall[12].pCurrent) @@ -731,10 +1013,12 @@ { "mmap", (sqlite3_syscall_ptr)mmap, 0 }, #define osMmap ((void*(*)(void*,size_t,int,int,int,off_t))aSyscall[21].pCurrent) { "munmap", (sqlite3_syscall_ptr)munmap, 0 }, #define osMunmap ((void*(*)(void*,size_t))aSyscall[22].pCurrent) + +{ "umask", (sqlite3_syscall_ptr)0, 0 }, #if HAVE_MREMAP { "mremap", (sqlite3_syscall_ptr)mremap, 0 }, #else { "mremap", (sqlite3_syscall_ptr)0, 0 }, @@ -854,34 +1138,38 @@ ** In that way, if a database file is -rw-rw-rw or -rw-rw-r-, and a ** transaction crashes and leaves behind hot journals, then any ** process that is able to write to the database will also be able to ** recover the hot journals. */ -static int robust_open(const char *z, int f, mode_t m){ - int fd; + +static int robust_open2(const char *z, int f, mode_t m, int protectionFlags, int allowmmap){ + int fd = -1; mode_t m2 = m ? m : SQLITE_DEFAULT_FILE_PERMISSIONS; - while(1){ + #if defined(O_CLOEXEC) - fd = osOpen(z,f|O_CLOEXEC,m2); -#else - fd = osOpen(z,f,m2); + f = f|O_CLOEXEC; #endif - if( fd<0 ){ - if( errno==EINTR ) continue; - break; + + do{ + if (osOpen == posixOpen2) { + fd = posixOpen(z,f,m2, protectionFlags, allowmmap); + } else { + fd = osOpen(z,f,m2); } if( fd>=SQLITE_MINIMUM_FILE_DESCRIPTOR ) break; osClose(fd); sqlite3_log(SQLITE_WARNING, "attempt to open \"%s\" as file descriptor %d", z, fd); fd = -1; + // numist: is this ok with sandboxing? if( osOpen("/dev/null", f, m)<0 ) break; - } - if( fd>=0 ){ - if( m!=0 ){ - struct stat statbuf; - if( osFstat(fd, &statbuf)==0 + }while( fd<0 && errno==EINTR ); + + if( fd>=0 ) { + if( m!=0 ){ + struct stat statbuf; + if( osFstat(fd, &statbuf)==0 && statbuf.st_size==0 && (statbuf.st_mode&0777)!=m ){ osFchmod(fd, m); } @@ -890,10 +1178,14 @@ osFcntl(fd, F_SETFD, osFcntl(fd, F_GETFD, 0) | FD_CLOEXEC); #endif } return fd; } + +static int robust_open(const char *z, int f, mode_t m, int protectionFlags) { + return robust_open2(z, f, m, protectionFlags, 0); +} /* ** Helper functions to obtain and relinquish the global mutex. The ** global mutex is used to protect the unixInodeInfo and ** vxworksFileId objects used by this file, all of which may be @@ -1471,11 +1763,11 @@ UnixUnusedFd *p; UnixUnusedFd *pNext; for(p=pInode->pUnused; p; p=pNext){ pNext = p->pNext; #if OSCLOSE_CHECK_CLOSE_IOERR - if( close(p->fd) ){ + if( osClose(p->fd) ){ storeLastErrno(pFile, errno); rc = SQLITE_IOERR_CLOSE; p->pNext = pError; pError = p; }else{ @@ -1564,11 +1856,11 @@ ** in the header of every SQLite database. In this way, if there ** is a race condition such that another thread has already populated ** the first page of the database, no damage is done. */ if( statbuf.st_size==0 && (pFile->fsFlags & SQLITE_FSFLAGS_IS_MSDOS)!=0 ){ - do{ rc = osWrite(fd, "S", 1); }while( rc<0 && errno==EINTR ); + do{ rc = (int)osWrite(fd, "S", 1); }while( rc<0 && errno==EINTR ); if( rc!=1 ){ storeLastErrno(pFile, errno); return SQLITE_IOERR; } rc = osFstat(fd, &statbuf); @@ -1744,24 +2036,72 @@ assert( pInode->nLock==0 ); lock.l_whence = SEEK_SET; lock.l_start = SHARED_FIRST; lock.l_len = SHARED_SIZE; lock.l_type = F_WRLCK; +#ifdef SQLITE_USE_FLOCKTIMEOUT + if( pFile->nextFlockTimeout>0 ){ + struct timespec ts; + struct flocktimeout fltimeout; + + ts.tv_sec = pFile->nextFlockTimeout / 1000; + ts.tv_nsec = (((long)pFile->nextFlockTimeout) % 1000L) * 1000000L; + fltimeout.fl = lock; + fltimeout.timeout = ts; + + rc = osFcntl(pFile->h, F_SETLKWTIMEOUT, &fltimeout); + if( !rc ){ + /* reset the timeout on success */ + pFile->nextFlockTimeout = 0; + } + }else{ + rc = osFcntl(pFile->h, F_SETLK, &lock); + } +#else rc = osFcntl(pFile->h, F_SETLK, &lock); +#endif if( rc<0 ) return rc; pInode->bProcessLock = 1; pInode->nLock++; }else{ rc = 0; } }else{ int i = 0; do { +#ifdef SQLITE_USE_FLOCKTIMEOUT + if( (i>0)||(pFile->nextFlockTimeout>0) ){ + struct timespec ts; + struct flocktimeout fltimeout; + + if( i==0 ){ + ts.tv_sec = pFile->nextFlockTimeout / 1000; + ts.tv_nsec = (((long)pFile->nextFlockTimeout) % 1000L) * 1000000L; + }else{ + long usTimeout = (i*100); + assert(i<10000); + ts.tv_sec = 0; + ts.tv_nsec = usTimeout * 1000; + } + fltimeout.fl = *pLock; + fltimeout.timeout = ts; + + rc = osFcntl(pFile->h, F_SETLKWTIMEOUT, &fltimeout); + if( !rc ){ + /* reset the timeout on success */ + pFile->nextFlockTimeout = 0; + } + }else{ + rc = osFcntl(pFile->h, F_SETLK, pLock); + } + i++; +#else rc = osFcntl(pFile->h, F_SETLK, pLock); if( rc && nRetry ){ usleep(100 * (++i)); } +#endif }while( !rc && nRetry-- ); } return rc; } @@ -1868,10 +2208,15 @@ */ if( (pFile->eFileLock!=pInode->eFileLock && (pInode->eFileLock>=PENDING_LOCK || eFileLock>SHARED_LOCK)) ){ rc = SQLITE_BUSY; +#ifdef SQLITE_USE_FLOCKTIMEOUT + if( pFile->nextFlockTimeout>0 ){ + usleep(((int)pFile->nextFlockTimeout) * 1000L); + } +#endif goto end_lock; } /* If a SHARED lock is requested, and some thread using this PID already ** has a SHARED or RESERVED lock, then increment reference counts and @@ -1953,10 +2298,15 @@ } }else if( eFileLock==EXCLUSIVE_LOCK && pInode->nShared>1 ){ /* We are trying for an exclusive lock but another thread in this ** same process is still holding a shared lock. */ rc = SQLITE_BUSY; +#ifdef SQLITE_USE_FLOCKTIMEOUT + if( pFile->nextFlockTimeout>0 ){ + usleep(((int)pFile->nextFlockTimeout) * 1000L); + } +#endif }else{ /* The request was for a RESERVED or EXCLUSIVE lock. It is ** assumed that there is a SHARED or greater lock on the file ** already. */ @@ -2266,14 +2616,26 @@ static int closeUnixFile(sqlite3_file *id){ unixFile *pFile = (unixFile*)id; #if SQLITE_MAX_MMAP_SIZE>0 unixUnmapfile(pFile); #endif +#if OSCLOSE_CHECK_CLOSE_IOERR + if( pFile->h>=0 ){ + int err = osClose(pFile->h); + if( err ){ + storeLastErrno(pFile, errno); + return SQLITE_IOERR_CLOSE; + }else{ + pFile->h=-1; + } + } +#else if( pFile->h>=0 ){ robust_close(pFile, pFile->h, __LINE__); pFile->h = -1; } +#endif #if OS_VXWORKS if( pFile->pId ){ if( pFile->ctrlFlags & UNIXFILE_DELETE ){ osUnlink(pFile->pId->zCanonicalName); } @@ -2493,10 +2855,15 @@ if( rc<0 ){ /* failed to open/create the lock directory */ int tErrno = errno; if( EEXIST == tErrno ){ rc = SQLITE_BUSY; +#ifdef SQLITE_USE_FLOCKTIMEOUT + if( pFile->nextFlockTimeout>0 ){ + usleep(((int)pFile->nextFlockTimeout) * 1000L); + } +#endif } else { rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_LOCK); if( IS_LOCK_ERROR(rc) ){ storeLastErrno(pFile, tErrno); } @@ -2732,10 +3099,15 @@ OSTRACE(("LOCK %d %s %s (flock)\n", pFile->h, azFileLock(eFileLock), rc==SQLITE_OK ? "ok" : "failed")); #ifdef SQLITE_IGNORE_FLOCK_LOCK_ERRORS if( (rc & SQLITE_IOERR) == SQLITE_IOERR ){ rc = SQLITE_BUSY; +#ifdef SQLITE_USE_FLOCKTIMEOUT + if( pFile->nextFlockTimeout>0 ){ + usleep(((long)pFile->nextFlockTimeout) * 1000L); + } +#endif } #endif /* SQLITE_IGNORE_FLOCK_LOCK_ERRORS */ return rc; } @@ -2896,10 +3268,15 @@ } /* lock semaphore now but bail out when already locked. */ if( sem_trywait(pSem)==-1 ){ rc = SQLITE_BUSY; +#ifdef SQLITE_USE_FLOCKTIMEOUT + if( pFile->nextFlockTimeout>0 ){ + usleep(((long)pFile->nextFlockTimeout) * 1000L); + } +#endif goto sem_end_lock; } /* got it, set the type and return ok */ pFile->eFileLock = eFileLock; @@ -3043,10 +3420,15 @@ setLockFlag ? SQLITE_IOERR_LOCK : SQLITE_IOERR_UNLOCK); #endif /* SQLITE_IGNORE_AFP_LOCK_ERRORS */ if( IS_LOCK_ERROR(rc) ){ storeLastErrno(pFile, tErrno); } +#ifdef SQLITE_USE_FLOCKTIMEOUT + if( (rc==SQLITE_BUSY)&&(pFile->nextFlockTimeout>0) ){ + usleep(((int)pFile->nextFlockTimeout) * 1000L); + } +#endif return rc; } else { return SQLITE_OK; } } @@ -3167,10 +3549,15 @@ */ if( (pFile->eFileLock!=pInode->eFileLock && (pInode->eFileLock>=PENDING_LOCK || eFileLock>SHARED_LOCK)) ){ rc = SQLITE_BUSY; +#ifdef SQLITE_USE_FLOCKTIMEOUT + if( pFile->nextFlockTimeout>0 ){ + usleep(((int)pFile->nextFlockTimeout) * 1000L); + } +#endif goto afp_end_lock; } /* If a SHARED lock is requested, and some thread using this PID already ** has a SHARED or RESERVED lock, then increment reference counts and @@ -3241,10 +3628,15 @@ } }else if( eFileLock==EXCLUSIVE_LOCK && pInode->nShared>1 ){ /* We are trying for an exclusive lock but another thread in this ** same process is still holding a shared lock. */ rc = SQLITE_BUSY; +#ifdef SQLITE_USE_FLOCKTIMEOUT + if( pFile->nextFlockTimeout>0 ){ + usleep(((int)pFile->nextFlockTimeout) * 1000L); + } +#endif }else{ /* The request was for a RESERVED or EXCLUSIVE lock. It is ** assumed that there is a SHARED or greater lock on the file ** already. */ @@ -3354,11 +3746,11 @@ if( pFile->eFileLock==EXCLUSIVE_LOCK ){ rc = afpSetLock(context->dbPath, pFile, SHARED_FIRST, SHARED_SIZE, 0); if( rc==SQLITE_OK && (eFileLock==SHARED_LOCK || pInode->nShared>1) ){ /* only re-establish the shared lock if necessary */ - int sharedLockByte = SHARED_FIRST+pInode->sharedByte; + int sharedLockByte = (int) (SHARED_FIRST+pInode->sharedByte); rc = afpSetLock(context->dbPath, pFile, sharedLockByte, 1, 1); } else { skipShared = 1; } } @@ -3495,17 +3887,24 @@ int got; int prior = 0; #if (!defined(USE_PREAD) && !defined(USE_PREAD64)) i64 newOffset; #endif + +#if defined(__APPLE__) + /* */ + if( id->nFsBytes > 0 && offset + cnt > id->nFsBytes ){ + __builtin_trap(); + } +#endif TIMER_START; assert( cnt==(cnt&0x1ffff) ); assert( id->h>2 ); cnt &= 0x1ffff; do{ #if defined(USE_PREAD) - got = osPread(id->h, pBuf, cnt, offset); + got = (int)osPread(id->h, pBuf, cnt, offset); SimulateIOError( got = -1 ); #elif defined(USE_PREAD64) got = osPread64(id->h, pBuf, cnt, offset); SimulateIOError( got = -1 ); #else @@ -3609,10 +4008,13 @@ const void *pBuf, /* Copy data from this buffer to the file */ int nBuf, /* Size of buffer pBuf in bytes */ int *piErrno /* OUT: Error number if error occurs */ ){ int rc = 0; /* Value returned by system call */ +#if (!defined(USE_PREAD) && !defined(USE_PREAD64)) + i64 newOffset; +#endif assert( nBuf==(nBuf&0x1ffff) ); assert( fd>2 ); nBuf &= 0x1ffff; TIMER_START; @@ -3648,10 +4050,16 @@ ** ** To avoid stomping the errno value on a failed write the lastErrno value ** is set before returning. */ static int seekAndWrite(unixFile *id, i64 offset, const void *pBuf, int cnt){ +#if defined(__APPLE__) + /* */ + if( id->nFsBytes > 0 && offset + cnt > id->nFsBytes ){ + __builtin_trap(); + } +#endif return seekAndWriteFd(id->h, offset, pBuf, cnt, &id->lastErrno); } /* @@ -3905,11 +4313,11 @@ sqlite3_snprintf(MAX_PATHNAME, zDirname, "%s", zFilename); for(ii=(int)strlen(zDirname); ii>1 && zDirname[ii]!='/'; ii--); if( ii>0 ){ zDirname[ii] = '\0'; - fd = robust_open(zDirname, O_RDONLY|O_BINARY, 0); + fd = robust_open(zDirname, O_RDONLY|O_BINARY, 0, 0); if( fd>=0 ){ OSTRACE(("OPENDIR %-3d %s\n", fd, zDirname)); } } *pFd = fd; @@ -3967,11 +4375,11 @@ HAVE_FULLFSYNC, isFullsync)); rc = osOpenDirectory(pFile->zPath, &dirfd); if( rc==SQLITE_OK && dirfd>=0 ){ full_fsync(dirfd, 0, 0); #if OSCLOSE_CHECK_CLOSE_IOERR - if( close(pFile->dirfd) ){ + if( osClose(pFile->dirfd) ){ storeLastErrno(pFile, errno); rc = SQLITE_IOERR_DIR_CLOSE; } #else robust_close(pFile, dirfd, __LINE__); @@ -4079,11 +4487,14 @@ static int fcntlSizeHint(unixFile *pFile, i64 nByte){ if( pFile->szChunk>0 ){ i64 nSize; /* Required file size */ struct stat buf; /* Used to hold return values of fstat() */ - if( osFstat(pFile->h, &buf) ) return SQLITE_IOERR_FSTAT; + if( osFstat(pFile->h, &buf) ) { + storeLastErrno(pFile, errno); + return SQLITE_IOERR_FSTAT; + } nSize = ((nByte+pFile->szChunk-1) / pFile->szChunk) * pFile->szChunk; if( nSize>(i64)buf.st_size ){ #if defined(HAVE_POSIX_FALLOCATE) && HAVE_POSIX_FALLOCATE @@ -4135,10 +4546,50 @@ #endif return SQLITE_OK; } +#if SQLITE_ENABLE_DATA_PROTECTION +static int unixSetFileProtection(int fd, int protFlags, int isShm){ + int protClass; + int currClass; + + if( 0==protFlags ){ + return SQLITE_OK; + } + assert(fd>=0); + assert((protFlags&SQLITE_OPEN_FILEPROTECTION_MASK)==protFlags); + + protClass = (!isShm) ? unixProtectionClassForVFSProtectionFlags(protFlags) : shmUnixProtectionClassForVFSProtectionFlags(protFlags); + +# if defined(__APPLE__)&&(TARGET_OS_EMBEDDED!=0)&&\ +(!defined(TARGET_IPHONE_SIMULATOR)||(TARGET_IPHONE_SIMULATOR==0)) + + currClass = fcntl(fd, F_GETPROTECTIONCLASS); + if( currClass<1 && protClass==4 ){ + /* F_GETPROTECTIONCLASS will fail if the file system doesn't have content protection enabled, + * if the protClass is 4/NONE that's OK */ + return SQLITE_OK; + } + if( protClass!=currClass ){ + if (0 != fcntl(fd, F_SETPROTECTIONCLASS, protClass)) { + if( errno == EPERM ){ + /* Test device locked state when error reporting for content protection issues */ + return SQLITE_AUTH; + } + return SQLITE_IOERR; + } + + rc = unixMapfile(pFile, nByte); + return rc; + } +# else +# warning SQLITE_ENABLE_DATA_PROTECTION is defined but could not be enabled for this deployment target +# endif + return SQLITE_OK; +} +#endif /* SQLITE_ENABLE_DATA_PROTECTION */ #if (SQLITE_ENABLE_APPLE_SPI>0) && defined(__APPLE__) #include "sqlite3_private.h" #include static int getDbPathForUnixFile(unixFile *pFile, char *dbPath); @@ -4148,10 +4599,11 @@ static int isProxyLockingMode(unixFile *); #endif #if (SQLITE_ENABLE_APPLE_SPI>0) && defined(__APPLE__) static int unixTruncateDatabase(unixFile *, int); +static const char *unixTempFileDir(void); static int unixInvalidateSupportFiles(unixFile *, int); static int findCreateFileMode(const char *, int, mode_t*, uid_t *,gid_t *); @@ -4169,20 +4621,39 @@ int fd = -1; mode_t openMode; /* Permissions to create file with */ uid_t uid; /* Userid for the file */ gid_t gid; /* Groupid for the file */ int rc; + int isShm = 0; + int zNameLen = zName ? (int)strlen(zName) : 0; + if (zNameLen > 4) { + const char* suffix = &(zName[zNameLen-4]); + if (strncmp("-shm", suffix, 4) == 0) { + isShm = 1; + } + } assert(pFd!=NULL); rc = findCreateFileMode(zName, dbOpenFlags, &openMode, &uid, &gid); if( rc!=SQLITE_OK ){ return rc; } - fd = robust_open(zName, openFlags, openMode); + int unixProtFlags = (!isShm) ? unixProtectionClassForVFSProtectionFlags(protFlags) : shmUnixProtectionClassForVFSProtectionFlags(protFlags); + + fd = robust_open2(zName, openFlags, openMode, unixProtFlags, isShm); OSTRACE(("OPENX %-3d %s 0%o\n", fd, zName, openFlags)); if( fd<0 ){ +#if SQLITE_ENABLE_DATA_PROTECTION + if( errno==EPERM ){ + /* Test device locked state when error reporting for content protection issues */ + rc = unixLogError(SQLITE_AUTH, "open", zName); + }else{ + rc = unixLogError(SQLITE_CANTOPEN_BKPT, "open", zName); + } +#else rc = unixLogError(SQLITE_CANTOPEN_BKPT, "open", zName); +#endif return rc; } /* if we're opening the wal or journal and running as root, set ** the journal uid/gid */ if( dbOpenFlags & (SQLITE_OPEN_WAL|SQLITE_OPEN_MAIN_JOURNAL) ){ @@ -4191,59 +4662,219 @@ if( fchown(fd, uid, gid) ){ rc = SQLITE_CANTOPEN_BKPT; } } } +#if SQLITE_ENABLE_DATA_PROTECTION + if( rc==SQLITE_OK ){ + rc = unixSetFileProtection(fd, protFlags, isShm); + } +#endif if( rc==SQLITE_OK ){ *pFd = fd; } else { *pFd = -1; - close(fd); + osClose(fd); } return rc; } + +static int readWALSettingFromFile(unixFile *pFile) { + char magicword[20]; + int result = -1; + sqlite3_file *id = (sqlite3_file *)pFile; + if (id != NULL) { + int sqlitecode = id->pMethods->xRead(id, magicword, sizeof(magicword), 0l); + + if (SQLITE_OK == sqlitecode) { + if (0 == memcmp(magicword, "SQLite format 3", 15)) { + result = 0; + if ((magicword[18] == 2) && (magicword[19] == 2)) { + result = 1; + } + } + } + } + return result; +} + +static int sqlite_guarded_fcopyfile(int src_fd, int dst_fd) { + size_t blen; + char *bp = 0; + ssize_t nread; + int ret = 0; + size_t iBlocksize = 0; + size_t oBlocksize = 0; + size_t totalCopied = 0; + const size_t twomeg = (1024 * 1024 * 2); + struct statfs sfs; + bzero(&sfs, sizeof(struct statfs)); + struct stat sbuf; + bzero(&sbuf, sizeof(struct stat)); + int err = -1; + + fstat(src_fd, &sbuf); + + if (fstatfs(src_fd, &sfs) == -1) { + iBlocksize = sbuf.st_blksize; + } else { + iBlocksize = sfs.f_iosize; + } + + if (iBlocksize > twomeg) { + iBlocksize = twomeg; + } + + if ((bp = malloc(iBlocksize)) == NULL) + return -1; + + oBlocksize = iBlocksize; + + blen = iBlocksize; + + /* If supported, do preallocation for Xsan / HFS volumes */ +#ifdef F_PREALLOCATE + { + fstore_t fst; + + fst.fst_flags = 0; + fst.fst_posmode = F_PEOFPOSMODE; + fst.fst_offset = 0; + fst.fst_length = sbuf.st_size; + /* Ignore errors; this is merely advisory. */ + (void)fcntl(dst_fd, F_PREALLOCATE, &fst); + } +#endif + + while ((nread = osRead(src_fd, bp, blen)) > 0) + { + ssize_t nwritten; + size_t left = nread; + void *ptr = bp; + int loop = 0; + + while (left > 0) { + nwritten = osWrite(dst_fd, ptr, MIN(left, oBlocksize)); + switch (nwritten) { + case 0: + if (++loop > 5) { + ret = -1; + err = EAGAIN; + goto exit; + } + break; + case -1: + ret = -1; + goto exit; + default: + left -= nwritten; + ptr = ((char*)ptr) + nwritten; + loop = 0; + break; + } + totalCopied += nwritten; + } + } + if (nread < 0) + { + ret = -1; + goto exit; + } + + if (ftruncate(dst_fd, totalCopied) < 0) + { + ret = -1; + goto exit; + } + +exit: + if (ret == -1) + { + if (err != -1) { + errno = err; + } else { + err = errno; + } + } + free(bp); + return (ret != -1) ? 0 : err; +} + static int unixReplaceDatabase(unixFile *pFile, sqlite3 *srcdb) { sqlite3_file *id = (sqlite3_file *)pFile; Btree *pSrcBtree = NULL; sqlite3_file *src_file = NULL; unixFile *pSrcFile = NULL; char srcWalPath[MAXPATHLEN+5]; int srcWalFD = -1; int rc = SQLITE_OK; - void *pLock = NULL; + Superlock *pLock = NULL; int flags = 0; sqlite3 *srcdb2 = NULL; copyfile_state_t s; int corruptSrcFileLock = 0; int corruptDstFileLock = 0; int isSrcCorrupt = 0; int isDstCorrupt = 0; + int shouldCopySrc = 0; + int isDstWAL = -1; + int isSrcWAL = -1; + int extraFDtoClose = -1; + char* replacementSrcPath = NULL; + char* replacementSrcWALPath = NULL; + char* replacementSrcShmPath = NULL; + int notGuardedFileDescriptors = 0; + int dstPreExists = 1; + struct stat checkBuffer; + int srcFlags = 0; if( !sqlite3SafetyCheckOk(srcdb) ){ return SQLITE_MISUSE; } + const char* realSrcPath = sqlite3_db_filename(srcdb, NULL); + + const char *tDir = unixTempFileDir(); + int tDirLen = tDir ? (int)strlen(tDir) : 0; + if (tDirLen < 1) { + _open_error_asl_log(); + asl_log(errorlog_client, NULL, ASL_LEVEL_CRIT, "SQLite: replace database failed because TMPDIR is not set correctly\n"); + + return SQLITE_PERM; + } + + if (0 == stat(pFile->zPath, &checkBuffer)) { + if (checkBuffer.st_size == 0l) { + dstPreExists = 0; + } + } else { + if (errno == ENOENT) { + dstPreExists = 0; + } + } #if SQLITE_ENABLE_DATA_PROTECTION flags |= pFile->protFlags; #endif #if SQLITE_ENABLE_LOCKING_STYLE if( isProxyLockingMode(pFile) ){ flags |= SQLITE_OPEN_AUTOPROXY; } #endif - rc = sqlite3demo_superlock(pFile->zPath, 0, flags, 0, 0, &pLock); + rc = sqlite3demo_superlock(pFile->zPath, 0, flags, 0, 0, (void**)&pLock); if( rc ){ if( rc==SQLITE_CORRUPT || rc==SQLITE_NOTADB ){ isDstCorrupt = 1; rc = sqlite3demo_superlock_corrupt(id, SQLITE_LOCK_EXCLUSIVE, &corruptDstFileLock); } if( rc ){ return rc; } + } else { + isDstWAL = ((pLock->bWal) ? 1 : 0); } /* get the src file descriptor adhering to the db struct access rules ** this code is modeled after sqlite3_file_control() in main.c */ sqlite3_mutex_enter(srcdb->mutex); @@ -4256,11 +4887,10 @@ pSrcPager = sqlite3BtreePager(pSrcBtree); assert( pSrcPager!=0 ); src_file = sqlite3PagerFile(pSrcPager); assert( src_file!=0 ); if( src_file->pMethods ){ - int srcFlags = 0; pSrcFile = (unixFile *)src_file; #if SQLITE_ENABLE_LOCKING_STYLE || defined(__APPLE__) if ((pSrcFile->openFlags & O_RDWR) == O_RDWR) { srcFlags = SQLITE_OPEN_READWRITE; } else { @@ -4275,49 +4905,257 @@ #if SQLITE_ENABLE_LOCKING_STYLE if( isProxyLockingMode(pSrcFile) ){ srcFlags |= SQLITE_OPEN_AUTOPROXY; } #endif + rc = sqlite3_open_v2(pSrcFile->zPath, &srcdb2, srcFlags, 0); if( rc==SQLITE_OK ){ + sqlite3_file* tempSrcFile = NULL; + int anotherError = sqlite3_file_control(srcdb2, NULL, SQLITE_FCNTL_FILE_POINTER, &tempSrcFile); + + if (anotherError == SQLITE_OK && tempSrcFile) { + isSrcWAL = (1== readWALSettingFromFile((unixFile*)tempSrcFile)) ? 1 : 0; + } else { + isSrcWAL = (1== readWALSettingFromFile(pSrcFile)) ? 1 : 0; + } + + sqlite3_busy_timeout(srcdb2, srcdb->busyTimeout); /* start a deferred transaction and read to establish a read lock */ - rc = sqlite3_exec(srcdb2, "BEGIN DEFERRED; PRAGMA schema_version", - 0, 0, 0); + rc = sqlite3_exec(srcdb2, "BEGIN DEFERRED; PRAGMA schema_version", 0, 0, 0); + + if (rc == SQLITE_CANTOPEN) { + if (isSrcWAL) { + // try opening writeable and again + sqlite3_close(srcdb2); + srcdb2 = NULL; + rc = sqlite3_open_v2(pSrcFile->zPath, &srcdb2, ((srcFlags & (~SQLITE_OPEN_READONLY))| SQLITE_OPEN_READWRITE), 0); + if( rc==SQLITE_OK ){ + sqlite3_busy_timeout(srcdb2, srcdb->busyTimeout); + /* start a deferred transaction and read to establish a read lock */ + rc = sqlite3_exec(srcdb2, "BEGIN DEFERRED; PRAGMA schema_version", 0, 0, 0); + } + } + } + + if (isSrcWAL && (rc == SQLITE_CANTOPEN)) { + // try copying to temp and opening writeable + if (srcdb2) { + sqlite3_close(srcdb2); + srcdb2 = NULL; + } + rc = SQLITE_OK; // well, maybe + shouldCopySrc = 1; + } + if (dstPreExists && (isSrcWAL != isDstWAL) && (!isDstCorrupt)) { + shouldCopySrc = 1; + } + if( rc==SQLITE_CORRUPT || rc==SQLITE_NOTADB ){ isSrcCorrupt = 1; rc = sqlite3demo_superlock_corrupt(src_file, SQLITE_LOCK_SHARED, &corruptSrcFileLock); } } } } - if( !srcdb2 || pSrcFile==NULL || pSrcFile->h<0){ + if( pSrcFile==NULL || pSrcFile->h<0){ + rc = SQLITE_INTERNAL; + } + if( rc!=SQLITE_OK ){ + goto end_replace_database; + } + + int srcFD = pSrcFile->h; + const char* srcPath = pSrcFile->zPath; + + if (shouldCopySrc) { + sqlite3* newSrcDB = NULL; + int shouldUnlinkWAL = 0; + + /* initialize a new database in TMPDIR and copy the contents over */ + int tLen = sizeof(char) * (tDirLen + 25); + replacementSrcPath = (char *)calloc(1, tLen); + + int tsrcFd = -1; + + strlcpy(replacementSrcPath, tDir, tLen); + /* create a temporary database with the prefix tmpsqlitereplacedb */ + if( replacementSrcPath[(tDirLen-1)] != '/' ){ + strlcat(replacementSrcPath, "/", tLen); + } + strlcat(replacementSrcPath, "tmpsqlitereplacedbXXXXXX", tLen); + tsrcFd = mkstemp(replacementSrcPath); + if( tsrcFd==-1 ){ + storeLastErrno(pFile, errno); + rc = SQLITE_IOERR; + } else{ + extraFDtoClose = tsrcFd; + + replacementSrcWALPath = (char*)calloc(1,tLen+5); + strlcpy(replacementSrcWALPath, replacementSrcPath, tLen+5); + strlcat(replacementSrcWALPath, "-wal", tLen+5); + + replacementSrcShmPath = (char*)calloc(1,tLen+5); + strlcpy(replacementSrcShmPath, replacementSrcPath, tLen+5); + strlcat(replacementSrcShmPath, "-shm", tLen+5); + + // copy src wal & db + s = copyfile_state_alloc(); + lseek(srcFD, 0, SEEK_SET); + lseek(tsrcFd, 0, SEEK_SET); + + if( fcopyfile(srcFD, tsrcFd, s, COPYFILE_DATA)){ + int err=errno; + switch(err) { + case ENOMEM: + rc = SQLITE_NOMEM; + break; + default: + storeLastErrno(pFile, err); + rc = SQLITE_IOERR; + } + } + copyfile_state_free(s); + fsync(tsrcFd); + + if (rc == SQLITE_OK) { + int tSrcWALFD = -1; + strlcpy(srcWalPath, srcPath, MAXPATHLEN+5); + strlcat(srcWalPath, "-wal", MAXPATHLEN+5); + tSrcWALFD = robust_open2(srcWalPath, O_RDONLY, 0, 0, 0); + if (tSrcWALFD >=0) { + srcWalFD = robust_open2(replacementSrcWALPath, O_RDWR | O_CREAT | O_TRUNC, 0, 0, 0); + if ((srcWalFD < 0) && (errno != ENOENT)) { + storeLastErrno(pFile, errno); + rc = SQLITE_IOERR; + osClose(tSrcWALFD); + tSrcWALFD = -1; + } + } else { + if (errno != ENOENT) { + storeLastErrno(pFile, errno); + rc = SQLITE_IOERR; + } + } + + if (rc == SQLITE_OK) { + if (srcWalFD >= 0) { + s = copyfile_state_alloc(); + lseek(srcWalFD, 0, SEEK_SET); + lseek(tSrcWALFD, 0, SEEK_SET); + if( sqlite_guarded_fcopyfile(tSrcWALFD, srcWalFD) != 0){ + int err=errno; + switch(err) { + case ENOMEM: + rc = SQLITE_NOMEM; + break; + default: + storeLastErrno(pFile, err); + rc = SQLITE_IOERR; + } + } + copyfile_state_free(s); + fsync(srcWalFD); + } + if (tSrcWALFD >= 0) { + osClose(tSrcWALFD); + } + + // open src as db + rc = sqlite3_open_v2(replacementSrcPath, &newSrcDB, ((srcFlags & (~SQLITE_OPEN_READONLY))| SQLITE_OPEN_READWRITE), 0); + if (rc != SQLITE_OK) { + } else { + sqlite3_busy_timeout(newSrcDB, srcdb->busyTimeout); + } + + + } + } + + if (rc != SQLITE_OK) { + if (newSrcDB) { + sqlite3_close(newSrcDB); + newSrcDB = NULL; + } + } + + if (rc == SQLITE_OK) { + rc = sqlite3_exec(newSrcDB, "BEGIN DEFERRED; PRAGMA schema_version; COMMIT;", 0, 0, 0); + + if (isSrcWAL != isDstWAL) { + if (isDstWAL) { + rc = sqlite3_exec(newSrcDB, "PRAGMA journal_mode=WAL", 0, 0, 0); + } else { + rc = sqlite3_exec(newSrcDB, "PRAGMA journal_mode=DELETE", 0, 0, 0); + shouldUnlinkWAL = 1; + } + } + + if (srcdb2 == NULL) { + srcdb2 = newSrcDB; + sqlite3_exec(newSrcDB, "BEGIN DEFERRED; PRAGMA schema_version;", 0, 0, 0); + } else { + sqlite3_close(newSrcDB); + newSrcDB = NULL; + } + } + + if (shouldUnlinkWAL) { + if (srcWalFD >= 0) { + osClose(srcWalFD); + srcWalFD = -1; + } + unlink(replacementSrcWALPath); + free(replacementSrcWALPath); + replacementSrcWALPath = NULL; + + unlink(replacementSrcShmPath); + free(replacementSrcShmPath); + replacementSrcShmPath = NULL; + } + + if (rc == SQLITE_OK) { + notGuardedFileDescriptors = 1; + srcFD = tsrcFd; + srcPath = replacementSrcPath; + } + } + + } + + if (!srcdb2) { rc = SQLITE_INTERNAL; } if( rc!=SQLITE_OK ){ goto end_replace_database; } /* both databases are locked appropriately, copy the src wal journal if ** one exists and then the actual database file */ - strlcpy(srcWalPath, pSrcFile->zPath, MAXPATHLEN+5); + strlcpy(srcWalPath, srcPath, MAXPATHLEN+5); strlcat(srcWalPath, "-wal", MAXPATHLEN+5); - srcWalFD = open(srcWalPath, O_RDONLY); + if (srcWalFD < 0) { + srcWalFD = robust_open2(srcWalPath, O_RDONLY, 0, 0, 0); + } if( !(srcWalFD<0) ){ char dstWalPath[MAXPATHLEN+5]; int dstWalFD = -1; int protFlags = 0; strlcpy(dstWalPath, pFile->zPath, MAXPATHLEN+5); strlcat(dstWalPath, "-wal", MAXPATHLEN+5); +#if SQLITE_ENABLE_DATA_PROTECTION + protFlags = pFile->protFlags; +#endif rc = unixOpenChildFile(dstWalPath, O_RDWR|O_CREAT, SQLITE_OPEN_WAL, protFlags, &dstWalFD); if( rc==SQLITE_OK ){ s = copyfile_state_alloc(); lseek(srcWalFD, 0, SEEK_SET); lseek(dstWalFD, 0, SEEK_SET); - if( fcopyfile(srcWalFD, dstWalFD, s, COPYFILE_DATA) ){ + if( sqlite_guarded_fcopyfile(srcWalFD, dstWalFD) != 0){ int err=errno; switch(err) { case ENOMEM: rc = SQLITE_NOMEM; break; @@ -4324,27 +5162,29 @@ default: storeLastErrno(pFile, err); rc = SQLITE_IOERR; } } + copyfile_state_free(s); - close(dstWalFD); + fsync(dstWalFD); + osClose(dstWalFD); } - close(srcWalFD); + osClose(srcWalFD); } if( rc==SQLITE_OK ){ /* before we copy, ensure that the file change counter will be modified */ uint32_t srcChange = 0; uint32_t dstChange = 0; - pread(pSrcFile->h, &srcChange, 4, 24); + pread(srcFD, &srcChange, 4, 24); pread(pFile->h, &dstChange, 4, 24); /* copy the actual database */ s = copyfile_state_alloc(); - lseek(pSrcFile->h, 0, SEEK_SET); + lseek(srcFD, 0, SEEK_SET); lseek(pFile->h, 0, SEEK_SET); - if( fcopyfile(pSrcFile->h, pFile->h, s, COPYFILE_DATA) ){ + if( sqlite_guarded_fcopyfile(srcFD, pFile->h) != 0 ){ int err=errno; switch(err) { case ENOMEM: rc = SQLITE_NOMEM; break; @@ -4356,11 +5196,11 @@ copyfile_state_free(s); if (srcChange == dstChange) { /* modify the change counter to force page zero to be reloaded */ dstChange ++; - pwrite(pFile->h, &dstChange, 4, 24); + osPwrite(pFile->h, &dstChange, 4, 24); } } if( isSrcCorrupt ){ sqlite3demo_superunlock_corrupt(src_file, corruptSrcFileLock); }else{ @@ -4369,23 +5209,70 @@ } /* zero out any old journal clutter */ if( rc==SQLITE_OK ){ int skipWAL = (srcWalFD<0)?0:1; unixInvalidateSupportFiles(pFile, skipWAL); + pFile->pMethod->xSync((sqlite3_file*)pFile, SQLITE_SYNC_FULL); } end_replace_database: - if( pSrcBtree ){ + if (extraFDtoClose >= 0) { + /* extraFDtoClose is created with mkstemp, can't use guarded close() */ + close(extraFDtoClose); + extraFDtoClose = -1; + } + if (replacementSrcPath) { + unlink(replacementSrcPath); + free(replacementSrcPath); + } + if (replacementSrcWALPath) { + unlink(replacementSrcWALPath); + free(replacementSrcWALPath); + } + if (replacementSrcShmPath) { + unlink(replacementSrcShmPath); + free(replacementSrcShmPath); + } + if (srcdb2) { sqlite3_close(srcdb2); + srcdb2 = NULL; + } + if( pSrcBtree ){ sqlite3BtreeLeave(pSrcBtree); } sqlite3_mutex_leave(srcdb->mutex); if( isDstCorrupt ){ sqlite3demo_superunlock_corrupt(id, corruptDstFileLock); }else{ sqlite3demo_superunlock(pLock); } + if( rc==SQLITE_OK && (srcWalFD>=0) && !isSrcCorrupt ){ + /* sync up the -shm file */ + sqlite3 *tdb = NULL; + int trc = sqlite3_open_v2(pFile->zPath, &tdb, SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|flags, 0); + if( trc==SQLITE_OK ){ + sqlite3_busy_timeout(tdb, srcdb->busyTimeout); + sqlite3_exec(tdb, "PRAGMA schema_version", 0, 0, 0); + } + if( tdb!=NULL ){ + sqlite3_close(tdb); + } + } + + if (rc) { + _open_error_asl_log(); + const char* dstPath = pFile ? pFile->zPath : ""; + + if (dstPath == NULL) { + dstPath = ""; + } + if (realSrcPath == NULL) { + realSrcPath = ""; + } + + asl_log(errorlog_client, NULL, ASL_LEVEL_ERR, "SQLite: unixReplaceDatabase() failed with error %d for replacing '%s' with '%s' \n", (int)rc, dstPath, realSrcPath); + } return rc; } #define SQLITE_FILE_HEADER_LEN 16 /* Check for a conflicting lock. If one is found, print an this ** on standard output using the format string given and return 1. @@ -4427,16 +5314,63 @@ if( lk.l_type!=F_UNLCK && (pid==SQLITE_LOCKSTATE_ANYPID || lk.l_pid==pid) ){ #ifdef SQLITE_DEBUG fprintf(stderr, "%s lock held by %d\n", zType, (int)lk.l_pid); #endif + OSTRACE(("GETLOCKSTATE fd-%d type-%s, loc-0x%x len-%d pid-(%d) state=ON\n", + h, zType, iOfst, iCnt, pid)); return 1; } + OSTRACE(("GETLOCKSTATE fd-%d type-%s, loc-0x%x len-%d pid-(%d) state=OFF\n", + h, zType, iOfst, iCnt, pid)); return 0; } static int unixLockstatePid(unixFile *, pid_t, int *); + +/* + ** Copy data from shared-memory into a buffer. + ** + ** Returns an error if the mapping has disappeared, or if the memory cannot + ** be accessed safely. +*/ +static int unixShmRead(sqlite3_file *fd, /* The underlying database file */ + void *dst, /* The destination memory */ + volatile void *src, /* The source (mapped) memory */ + sqlite3_int64 nByte /* Size of the destination */ +){ +#if defined(__APPLE__) + unixFile *pDbFd; /* The underlying database file */ + + pDbFd = (unixFile*)fd; + + if ( pDbFd->fsFlags & SQLITE_FSFLAGS_IS_MMAP_SAFE ) { + memcpy(dst, (void *)src, nByte); + } else { + mach_vm_size_t aSz; + mach_vm_read_overwrite(mach_task_self(), src, nByte, dst, &aSz); + if ( aSz!=nByte ) { + return SQLITE_IOERR; + } + } +#else + memcpy(dst, (void *)src, nByte); +#endif + + return SQLITE_OK; +} + +int sqlite30sShmRead(sqlite3_file *id, void *dst, volatile void *src, sqlite3_int64 size); + +int sqlite30sShmRead(sqlite3_file *id, void *dst, volatile void *src, sqlite3_int64 size){ +#if (defined(__APPLE__) && ((!defined(TARGET_OS_EMBEDDED)||(TARGET_OS_EMBEDDED==0)))) + return unixShmRead(id, dst, src, size); +#else + memcpy(dst, (void *)src, size); + return SQLITE_OK; +#endif +} #endif /* (SQLITE_ENABLE_APPLE_SPI>0) && defined(__APPLE__) */ /* @@ -4560,10 +5494,24 @@ rc = unixLockstatePid(pFile, pLockstate->pid, &(pLockstate->state)); return rc; } #endif /* (SQLITE_ENABLE_APPLE_SPI>0) && defined(__APPLE__) */ + + case SQLITE_FCNTL_SET_NEXTFLOCKTIMEOUT: { +#ifdef SQLITE_USE_FLOCKTIMEOUT + pFile->nextFlockTimeout = *(int*)pArg; +#endif + return SQLITE_OK; + } + case SQLITE_FCNTL_GET_NEXTFLOCKTIMEOUT: { +#ifdef SQLITE_USE_FLOCKTIMEOUT + *(int*)pArg = pFile->nextFlockTimeout; +#endif + return SQLITE_OK; + } + } return SQLITE_NOTFOUND; } /* @@ -4776,10 +5724,13 @@ static int unixShmSystemLock( unixShmNode *pShmNode, /* Apply locks to this open shared-memory segment */ int lockType, /* F_UNLCK, F_RDLCK, or F_WRLCK */ int ofst, /* First byte of the locking range */ int n /* Number of bytes to lock */ +#ifdef SQLITE_USE_FLOCKTIMEOUT + , int timeout /* Lock timeout in milliseconds */ +#endif ){ struct flock f; /* The posix advisory locking structure */ int rc = SQLITE_OK; /* Result code form fcntl() */ /* Access to the unixShmNode object is serialized by the caller */ @@ -4797,11 +5748,27 @@ f.l_type = lockType; f.l_whence = SEEK_SET; f.l_start = ofst; f.l_len = n; +#ifdef SQLITE_USE_FLOCKTIMEOUT + if( timeout>0 ){ + struct timespec ts; + struct flocktimeout fltimeout; + + ts.tv_sec = timeout / 1000; + ts.tv_nsec = (((long)timeout) % 1000L) * 1000000L; + fltimeout.fl = f; + fltimeout.timeout = ts; + + rc = osFcntl(pShmNode->h, F_SETLKWTIMEOUT, &fltimeout); + }else{ + rc = osFcntl(pShmNode->h, F_SETLK, &f); + } +#else rc = osFcntl(pShmNode->h, F_SETLK, &f); +#endif rc = (rc!=(-1)) ? SQLITE_OK : SQLITE_BUSY; } /* Update the global lock state and do debug tracing */ #ifdef SQLITE_DEBUG @@ -4969,10 +5936,11 @@ /* Call fstat() to figure out the permissions on the database file. If ** a new *-shm file is created, an attempt will be made to create it ** with the same permissions. */ if( osFstat(pDbFd->h, &sStat) && pInode->bProcessLock==0 ){ + storeLastErrno(pDbFd, errno); rc = SQLITE_IOERR_FSTAT; goto shm_open_err; } #if defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE @@ -5026,21 +5994,53 @@ #endif ){ openFlags = O_RDONLY; pShmNode->isReadonly = 1; } - pShmNode->h = robust_open(zShmFilename, openFlags, (sStat.st_mode&0777)); + int protFlags = 0; +#if SQLITE_ENABLE_DATA_PROTECTION + protFlags = shmUnixProtectionClassForVFSProtectionFlags(pDbFd->protFlags); +#endif + pShmNode->h = robust_open2(zShmFilename, openFlags, (sStat.st_mode&0777), protFlags, 1); + if( pShmNode->h<0 ){ +#if SQLITE_ENABLE_DATA_PROTECTION + if( errno==EPERM ) { + rc = unixLogError(SQLITE_AUTH, "open", zShmFilename); + }else{ + rc = unixLogError(SQLITE_CANTOPEN_BKPT, "open", zShmFilename); + } +#else rc = unixLogError(SQLITE_CANTOPEN_BKPT, "open", zShmFilename); +#endif goto shm_open_err; } +#if SQLITE_ENABLE_DATA_PROTECTION + if( (pDbFd->ctrlFlags&UNIXFILE_RDONLY) == 0 ){ + int shmProtFlags = (pDbFd->protFlags)&SQLITE_OPEN_FILEPROTECTION_MASK; + + rc = unixSetFileProtection(pShmNode->h, shmProtFlags, 1); + if( rc ){ + if( rc==SQLITE_IOERR ){ + storeLastErrno(pDbFd, errno); + } + osClose(pShmNode->h); + goto shm_open_err; + } + } +#endif /* Check to see if another process is holding the dead-man switch. ** If not, truncate the file to zero length. */ rc = SQLITE_OK; +#ifdef SQLITE_USE_FLOCKTIMEOUT + if( unixShmSystemLock(pShmNode, F_WRLCK, UNIX_SHM_DMS, 1, pDbFd->nextFlockTimeout)==SQLITE_OK ){ + pDbFd->nextFlockTimeout = 0; +#else if( unixShmSystemLock(pShmNode, F_WRLCK, UNIX_SHM_DMS, 1)==SQLITE_OK ){ +#endif if( robust_ftruncate(pShmNode->h, 0) ){ rc = unixLogError(SQLITE_IOERR_SHMOPEN, "ftruncate", zShmFilename); }else{ /* If running as root set the uid/gid of the shm file to match ** the database */ @@ -5051,11 +6051,18 @@ } } } } if( rc==SQLITE_OK ){ +#ifdef SQLITE_USE_FLOCKTIMEOUT + rc = unixShmSystemLock(pShmNode, F_RDLCK, UNIX_SHM_DMS, 1, pDbFd->nextFlockTimeout); + if( rc==SQLITE_OK ){ + pDbFd->nextFlockTimeout = 0; + } +#else rc = unixShmSystemLock(pShmNode, F_RDLCK, UNIX_SHM_DMS, 1); +#endif } if( rc ) goto shm_open_err; } } @@ -5113,10 +6120,12 @@ int iRegion, /* Region to retrieve */ int szRegion, /* Size of regions */ int bExtend, /* True to extend file if necessary */ void volatile **pp /* OUT: Mapped memory */ ){ + static int oncethingie = 0; + unixFile *pDbFd = (unixFile*)fd; unixShm *p; unixShmNode *pShmNode; int rc = SQLITE_OK; int nShmPerMap = unixShmRegionPerMap(); @@ -5198,16 +6207,17 @@ } pShmNode->apRegion = apNew; while( pShmNode->nRegionh>=0 ){ pMem = osMmap(0, nMap, pShmNode->isReadonly ? PROT_READ : PROT_READ|PROT_WRITE, MAP_SHARED, pShmNode->h, szRegion*(i64)pShmNode->nRegion ); + if( pMem==MAP_FAILED ){ rc = unixLogError(SQLITE_IOERR_SHMMAP, "mmap", pShmNode->zFilename); goto shmpage_out; } }else{ @@ -5283,11 +6293,18 @@ allMask |= pX->sharedMask; } /* Unlock the system-level locks */ if( (mask & allMask)==0 ){ +#ifdef SQLITE_USE_FLOCKTIMEOUT + rc = unixShmSystemLock(pShmNode, F_UNLCK, ofst+UNIX_SHM_BASE, n, pDbFd->nextFlockTimeout); + if( rc==SQLITE_OK ){ + pDbFd->nextFlockTimeout = 0; + } +#else rc = unixShmSystemLock(pShmNode, F_UNLCK, ofst+UNIX_SHM_BASE, n); +#endif }else{ rc = SQLITE_OK; } /* Undo the local locks */ @@ -5303,19 +6320,31 @@ ** SQLITE_BUSY. */ for(pX=pShmNode->pFirst; pX; pX=pX->pNext){ if( (pX->exclMask & mask)!=0 ){ rc = SQLITE_BUSY; +#ifdef SQLITE_USE_FLOCKTIMEOUT + if( pDbFd->nextFlockTimeout>0 ){ + usleep(((int)pDbFd->nextFlockTimeout) * 1000L); + } +#endif break; } allShared |= pX->sharedMask; } /* Get shared locks at the system level, if necessary */ if( rc==SQLITE_OK ){ if( (allShared & mask)==0 ){ +#ifdef SQLITE_USE_FLOCKTIMEOUT + rc = unixShmSystemLock(pShmNode, F_RDLCK, ofst+UNIX_SHM_BASE, n, pDbFd->nextFlockTimeout); + if( rc==SQLITE_OK ){ + pDbFd->nextFlockTimeout = 0; + } +#else rc = unixShmSystemLock(pShmNode, F_RDLCK, ofst+UNIX_SHM_BASE, n); +#endif }else{ rc = SQLITE_OK; } } @@ -5328,20 +6357,31 @@ ** lock. If any do, return SQLITE_BUSY right away. */ for(pX=pShmNode->pFirst; pX; pX=pX->pNext){ if( (pX->exclMask & mask)!=0 || (pX->sharedMask & mask)!=0 ){ rc = SQLITE_BUSY; +#ifdef SQLITE_USE_FLOCKTIMEOUT + if( pDbFd->nextFlockTimeout>0 ){ + usleep(((int)pDbFd->nextFlockTimeout) * 1000L); + } +#endif break; } } /* Get the exclusive locks at the system level. Then if successful ** also mark the local connection as being locked. */ if( rc==SQLITE_OK ){ +#ifdef SQLITE_USE_FLOCKTIMEOUT + rc = unixShmSystemLock(pShmNode, F_WRLCK, ofst+UNIX_SHM_BASE, n, pDbFd->nextFlockTimeout); + if( rc==SQLITE_OK ){ + pDbFd->nextFlockTimeout = 0; +#else rc = unixShmSystemLock(pShmNode, F_WRLCK, ofst+UNIX_SHM_BASE, n); if( rc==SQLITE_OK ){ +#endif assert( (p->sharedMask & mask)==0 ); p->exclMask |= mask; } } } @@ -5359,12 +6399,16 @@ */ static void unixShmBarrier( sqlite3_file *fd /* Database file holding the shared memory */ ){ UNUSED_PARAMETER(fd); +#if defined(__APPLE__) + OSMemoryBarrier(); +#else unixEnterMutex(); unixLeaveMutex(); +#endif } /* ** Close a connection to shared-memory. Delete the underlying ** storage if deleteFlag is true. @@ -5432,19 +6476,22 @@ #if (SQLITE_ENABLE_APPLE_SPI>0) && defined(__APPLE__) static const char *unixTempFileDir(void); static int unixInvalidateSupportFiles(unixFile *pFile, int skipWAL) { char jPath[MAXPATHLEN+9]; - int zLen = strlcpy(jPath, pFile->zPath, MAXPATHLEN+9); + int zLen = (int)strlcpy(jPath, pFile->zPath, MAXPATHLEN+9); if( zLenpInode is shared across threads */ unixShmNode *pShmNode = pFile->pInode->pShmNode; if( pShmNode && !pShmNode->isReadonly ){ struct stat sStat; sqlite3_mutex_enter(pShmNode->mutex); @@ -5463,11 +6510,11 @@ unixLeaveMutex(); } jLen = strlcpy(&jPath[zLen], extensions[j], 9); if( jLen < 9 ){ int jflags = (j<2) ? O_TRUNC : O_RDWR; - int jfd = open(jPath, jflags); + int jfd = robust_open2(jPath, jflags, 0, 0, isShm); if( jfd==(-1) ){ if( errno!=ENOENT ){ perror(jPath); } } else { @@ -5475,16 +6522,17 @@ struct stat sStat; if( !osFstat(jfd, &sStat) ){ unsigned long size = (sStat.st_size<4) ? sStat.st_size : 4; if( size>0 ){ uint32_t zero = 0; - pwrite(jfd, &zero, (size_t)size, 0); + + os_rawPwrite(jfd, &zero, (size_t)size, 0); } } } fsync(jfd); - close(jfd); + osClose(jfd); } } } } return SQLITE_OK; @@ -5515,11 +6563,11 @@ if (result) { rc = SQLITE_IOERR; storeLastErrno(pFile, result); } - int fd2 = open(journalPath, O_RDWR); + int fd2 = robust_open2(journalPath, O_RDWR, 0, 0, 0); int result2 = 0; if (fd2 < 0) { if (errno != ENOENT) { result2 = errno; } else { @@ -5534,11 +6582,11 @@ if (result2 && !result) { rc = SQLITE_IOERR; storeLastErrno(pFile, result2); } - int fd3 = open(walPath, O_RDWR); + int fd3 = robust_open2(walPath, O_RDWR, 0, 0, 0); int result3 = 0; if (fd3 < 0) { if (errno != ENOENT) { result3 = errno; } else { @@ -5555,28 +6603,30 @@ storeLastErrno(pFile, result2); } if (fd3 >= 0) { fsync(fd3); - close(fd3); + osClose(fd3); } if (fd2 >= 0) { fsync(fd2); - close(fd2); + osClose(fd2); } - fsync(fd1); + + pFile->pMethod->xSync((sqlite3_file*)pFile, SQLITE_SYNC_FULL); return rc; } static int unixTruncateDatabase(unixFile *pFile, int bFlags) { sqlite3_file *id = (sqlite3_file *)pFile; int rc = SQLITE_OK; - void *pLock = NULL; + Superlock *pLock = NULL; int flags = 0; int corruptFileLock = 0; int isCorrupt = 0; + int isWal = (bFlags & SQLITE_TRUNCATE_JOURNALMODE_WAL); int force = (bFlags & SQLITE_TRUNCATE_FORCE); int safeFailed = 0; #if SQLITE_ENABLE_DATA_PROTECTION flags |= pFile->protFlags; @@ -5584,37 +6634,55 @@ #if SQLITE_ENABLE_LOCKING_STYLE if( isProxyLockingMode(pFile) ){ flags |= SQLITE_OPEN_AUTOPROXY; } #endif + + const char *tDir = NULL; + int tDirLen = 0; + + if( (bFlags&SQLITE_TRUNCATE_INITIALIZE_HEADER_MASK)!=0 ){ + tDir = unixTempFileDir(); + tDirLen = tDir ? (int)strlen(tDir) : 0; + if (tDirLen < 1) { + _open_error_asl_log(); + asl_log(errorlog_client, NULL, ASL_LEVEL_CRIT, "SQLite: truncate database failed because TMPDIR is not set correctly\n"); + + return SQLITE_PERM; + } + } - rc = sqlite3demo_superlock(pFile->zPath, 0, flags, 0, 0, &pLock); + rc = sqlite3demo_superlock(pFile->zPath, 0, flags, 0, 0, (void**)&pLock); if( rc ){ if( rc==SQLITE_CORRUPT || rc==SQLITE_NOTADB ){ isCorrupt = 1; rc = sqlite3demo_superlock_corrupt(id, SQLITE_LOCK_EXCLUSIVE, &corruptFileLock); } if( rc && !force ){ return rc; } + if (!isWal) { + isWal = (1 == readWALSettingFromFile(pFile)); + } rc = SQLITE_OK; /* Ignore the locking failure if force is true */ + } else if (!isWal && (pLock != NULL)) { + isWal = pLock->bWal; } + if( (bFlags&SQLITE_TRUNCATE_INITIALIZE_HEADER_MASK)!=0 ){ /* initialize a new database in TMPDIR and copy the contents over */ - const char *tDir = unixTempFileDir(); - int tDirLen = strlen(tDir); - int tLen = sizeof(char) * (tDirLen + 12); - char *tDbPath = (char *)malloc(tLen); + int tLen = sizeof(char) * (tDirLen + 26); + char *tDbPath = (char *)calloc(1, tLen); int tFd = -1; strlcpy(tDbPath, tDir, tLen); + /* create a temporary database with the prefix tmpsqlitetruncatedb */ if( tDbPath[(tDirLen-1)] != '/' ){ - strlcat(tDbPath, "/tmpdbXXXXX", tLen); - } else { - strlcat(tDbPath, "tmpdbXXXXX", tLen); + strlcat(tDbPath, "/", tLen); } + strlcat(tDbPath, "tmpsqlitetruncatedbXXXXXX", tLen); tFd = mkstemp(tDbPath); if( tFd==-1 ){ storeLastErrno(pFile, errno); rc = SQLITE_IOERR; safeFailed = 1; @@ -5670,14 +6738,15 @@ int off = 0; /* merge the wal into the db */ sqlite3_file_control(tDb, NULL, SQLITE_FCNTL_PERSIST_WAL, &off); sqlite3_close(tDb); } + s = copyfile_state_alloc(); lseek(tFd, 0, SEEK_SET); lseek(pFile->h, 0, SEEK_SET); - if( fcopyfile(tFd, pFile->h, s, COPYFILE_DATA) ){ + if( sqlite_guarded_fcopyfile(tFd, pFile->h) != 0 ){ int err=errno; switch(err) { case ENOMEM: trc = SQLITE_NOMEM; break; @@ -5686,18 +6755,21 @@ trc = SQLITE_IOERR; } } copyfile_state_free(s); fsync(pFile->h); + /* tFd is created with mkstemp, can't use guarded close() */ close(tFd); unlink(tDbPath); if( trc!=SQLITE_OK ){ safeFailed = 1; rc = trc; } } - free(tDbPath); + if (tDbPath) { + free(tDbPath); + } } else { rc = pFile->pMethod->xTruncate(id, ((pFile->fsFlags & SQLITE_FSFLAGS_IS_MSDOS) != 0) ? 1L : 0L); if( rc ){ safeFailed = 1; @@ -5722,10 +6794,21 @@ if( force && safeFailed){ rc = unixUnsafeTruncateDatabase(pFile); } + if (rc) { + _open_error_asl_log(); + const char* dstPath = pFile ? pFile->zPath : ""; + + if (dstPath == NULL) { + dstPath = ""; + } + + asl_log(errorlog_client, NULL, ASL_LEVEL_ERR, "SQLite: unixTruncateDatabase() failed with %d for replacing '%s' with flags %x\n", rc, dstPath, flags); + } + return rc; } /* ** Lock locations for shared-memory locks used by WAL mode. @@ -5747,10 +6830,11 @@ int hDb; /* File descriptor for the open database file */ int hShm = -1; /* File descriptor for WAL shared-memory file */ ssize_t got; /* Bytes read from header */ int isWal = 0; /* True if in WAL mode */ int nLock = 0; /* Number of locks held */ + int npLock = 0; /* Number of potential locks held */ int noHdr = 0; /* Zero byte DB has no header */ unsigned char aHdr[100]; /* Database header */ assert(pLockstate); @@ -5758,10 +6842,21 @@ hDb = pFile->h; if( hDb<0 ){ *pLockstate = SQLITE_LOCKSTATE_ERROR; return SQLITE_ERROR; } + + /* First check for an exclusive lock */ + nLock += unixIsLocked(pid, hDb, F_RDLCK, SHARED_FIRST, SHARED_SIZE, + "EXCLUSIVE"); + npLock += unixIsLocked(pid, hDb, F_WRLCK, PENDING_BYTE, SHARED_SIZE+2, + "PENDING|RESERVED|SHARED"); + if( nLock==0 && npLock==0 ){ + *pLockstate = SQLITE_LOCKSTATE_OFF; + return SQLITE_OK; + } + assert( (strlen(SQLITE_FILE_HEADER)+1)==SQLITE_FILE_HEADER_LEN ); got = pread(hDb, aHdr, 100, 0); if( got<0 ){ *pLockstate = SQLITE_LOCKSTATE_ERROR; return SQLITE_ERROR; @@ -5773,20 +6868,16 @@ ){ *pLockstate = SQLITE_LOCKSTATE_NOTADB; return SQLITE_NOTADB; } - /* First check for an exclusive lock */ - nLock += unixIsLocked(pid, hDb, F_RDLCK, SHARED_FIRST, SHARED_SIZE, - "EXCLUSIVE"); if (!noHdr) { isWal = aHdr[18]==2; } if( nLock==0 && isWal==0 ){ /* Rollback mode */ - nLock += unixIsLocked(pid, hDb, F_WRLCK, PENDING_BYTE, SHARED_SIZE+2, - "PENDING|RESERVED|SHARED"); + nLock += npLock; } if( nLock==0 && isWal!=0 ){ /* lookup the file descriptor for the shared memory file if we have it open ** in this process */ unixEnterMutex(); /* Because pFile->pInode is shared across threads */ @@ -5811,21 +6902,21 @@ char zShm[MAXPATHLEN]; /* WAL mode */ strlcpy(zShm, pFile->zPath, MAXPATHLEN); strlcat(zShm, "-shm", MAXPATHLEN); - hShm = open(zShm, O_RDONLY, 0); + hShm = robust_open2(zShm, O_RDONLY, 0, 0, 1); if( hShm<0 ){ *pLockstate = SQLITE_LOCKSTATE_OFF; unixLeaveMutex(); return SQLITE_OK; } if( unixIsLocked(pid, hShm, F_RDLCK, SHM_RECOVER, 1, "WAL-RECOVERY") || unixIsLocked(pid, hShm, F_RDLCK, SHM_WRITE, 1, "WAL-WRITE") ){ nLock = 1; } - close(hShm); + osClose(hShm); } unixLeaveMutex(); } if( nLock>0 ){ *pLockstate = SQLITE_LOCKSTATE_ON; @@ -6551,24 +7642,45 @@ "/var/tmp", "/usr/tmp", "/tmp", 0 /* List terminator */ }; + unsigned int i; struct stat buf; const char *zDir = 0; azDirs[0] = sqlite3_temp_directory; if( !azDirs[1] ) azDirs[1] = getenv("SQLITE_TMPDIR"); - if( !azDirs[2] ) azDirs[2] = getenv("TMPDIR"); + if( !azDirs[2] ){ +#if defined(__APPLE__) + char *path = malloc(PATH_MAX); + size_t pathLen = 0; + path[0] = '\0'; + pathLen = confstr(_CS_DARWIN_USER_TEMP_DIR, path, PATH_MAX); + if (pathLen > 0) { + azDirs[2] = path; + } else { + free(path); + path = NULL; + } +#else + azDirs[2] = getenv("TMPDIR"); +#endif + } for(i=0; i 0) { + homePath = upwd.pw_dir; + } + if (!homePath) { + char* t = getenv("HOME"); + if (t && (strlen(t) > 0)) { + homePath = t; + } + } + + if (homePath) { + strlcpy(userhomedirectory, homePath, bufferSize); + return 1; + } + + return 0; +} + + +static int _canSafelyMapFileDescriptor(struct statfs *fs) { +#if (!defined(TARGET_OS_EMBEDDED)||(TARGET_OS_EMBEDDED==0)) + static int userhomedirinited = 0; + static fsid_t *userhomedirid = NULL; + static fsid_t underlyinghomedirid; + static OSSpinLock underlyinghomelock = OS_SPINLOCK_INIT; + + //Network volumes are never safe to mmap + int isNetworkVolume = (MNT_LOCAL != (fs->f_flags & MNT_LOCAL)); + if (isNetworkVolume) { + return 0; + } + + //The root volume shouldn't be removeable + int isRootVolume = (MNT_ROOTFS == (fs->f_flags & MNT_ROOTFS)); + if (isRootVolume) { + return 1; + } + + OSMemoryBarrier(); + if (userhomedirinited == 0) { + OSSpinLockLock(&underlyinghomelock); + + char apath[MAXPATHLEN+1]; + bzero(apath, MAXPATHLEN+1); + + if (getuid() != 0) { + if (0 != _copyUserHomeDirectory(apath, MAXPATHLEN)) { + if (strlen(apath) > 0) { + struct statfs userfs; + + if (statfs(apath, &userfs) == 0) { + if (0 != (userfs.f_flags & MNT_LOCAL)) { + fsid_t tests; + bzero(&tests, sizeof(fsid_t)); + + if (memcmp(&tests, &(userfs.f_fsid), sizeof(fsid_t)) != 0) { + memcpy(&underlyinghomedirid, &(userfs.f_fsid), sizeof(fsid_t)); + + // if the user is not root, their home directory is local and the f_fsid field is accessible + userhomedirid = &underlyinghomedirid; + } + } + } + } + } + } + + userhomedirinited = 1; + + OSSpinLockUnlock(&underlyinghomelock); + } + + if (userhomedirid != NULL) { + if (memcmp(userhomedirid, &(fs->f_fsid), sizeof(fsid_t)) == 0) { + return 1; + } + } + return 0; +#else + return 1; +#endif + } +#endif + /* ** Open the file zPath. ** ** Previously, the SQLite OS layer used three functions in place of this ** one: @@ -6793,10 +8013,11 @@ int eType = flags&0xFFFFFF00; /* Type of file to open */ #endif int noLock; /* True to omit locking primitives */ int rc = SQLITE_OK; /* Function Return Code */ int ctrlFlags = 0; /* UNIXFILE_* flags */ + int protClass = 0; int isExclusive = (flags & SQLITE_OPEN_EXCLUSIVE); int isDelete = (flags & SQLITE_OPEN_DELETEONCLOSE); int isCreate = (flags & SQLITE_OPEN_CREATE); int isReadonly = (flags & SQLITE_OPEN_READONLY); @@ -6805,11 +8026,13 @@ int isAutoProxy = (flags & SQLITE_OPEN_AUTOPROXY); #endif #if defined(__APPLE__) || SQLITE_ENABLE_LOCKING_STYLE struct statfs fsInfo; #endif - + + int isShm = 0; + /* If creating a master or main-file journal, this function will open ** a file-descriptor on the directory too. The first time unixSync() ** is called the directory file descriptor will be fsync()ed and close()d. */ int syncDir = (isCreate && ( @@ -6904,10 +8127,11 @@ if( isExclusive ) openFlags |= (O_EXCL|O_NOFOLLOW); openFlags |= (O_LARGEFILE|O_BINARY); #if SQLITE_ENABLE_DATA_PROTECTION p->protFlags = (flags & SQLITE_OPEN_FILEPROTECTION_MASK); + protClass = (!isShm) ? unixProtectionClassForVFSProtectionFlags((flags & SQLITE_OPEN_FILEPROTECTION_MASK)) : shmUnixProtectionClassForVFSProtectionFlags((flags & SQLITE_OPEN_FILEPROTECTION_MASK)); #endif if( fd<0 ){ mode_t openMode; /* Permissions to create file with */ uid_t uid; /* Userid for the file */ @@ -6916,23 +8140,32 @@ if( rc!=SQLITE_OK ){ assert( !p->pUnused ); assert( eType==SQLITE_OPEN_WAL || eType==SQLITE_OPEN_MAIN_JOURNAL ); return rc; } - fd = robust_open(zName, openFlags, openMode); + fd = robust_open2(zName, openFlags, openMode, protClass, isShm); OSTRACE(("OPENX %-3d %s 0%o\n", fd, zName, openFlags)); if( fd<0 && errno!=EISDIR && isReadWrite && !isExclusive ){ /* Failed to open the file for read/write access. Try read-only. */ flags &= ~(SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE); openFlags &= ~(O_RDWR|O_CREAT); flags |= SQLITE_OPEN_READONLY; openFlags |= O_RDONLY; isReadonly = 1; - fd = robust_open(zName, openFlags, openMode); + fd = robust_open2(zName, openFlags, openMode, protClass, isShm); } if( fd<0 ){ +#if SQLITE_ENABLE_DATA_PROTECTION + if( errno==EPERM ){ + /* Test device locked state when error reporting for content protection issues */ + rc = unixLogError(SQLITE_AUTH, "open", zName); + }else{ + rc = unixLogError(SQLITE_CANTOPEN_BKPT, "open", zName); + } +#else rc = unixLogError(SQLITE_CANTOPEN_BKPT, "open", zName); +#endif goto open_finished; } /* if we're opening the wal or journal and running as root, set the ** journal uid/gid */ @@ -6944,10 +8177,22 @@ goto open_finished; } } } } +#if SQLITE_ENABLE_DATA_PROTECTION + if( !isReadonly ){ + rc = unixSetFileProtection(fd, p->protFlags, isShm); + if( rc ){ + if( rc!=SQLITE_AUTH ){ + rc =SQLITE_CANTOPEN_BKPT; + } + osClose(fd); + goto open_finished; + } + } +#endif assert( fd>=0 ); if( pOutFlags ){ *pOutFlags = flags; } @@ -6982,10 +8227,23 @@ ((unixFile*)pFile)->fsFlags |= SQLITE_FSFLAGS_IS_MSDOS; } if (0 == strncmp("exfat", fsInfo.f_fstypename, 5)) { ((unixFile*)pFile)->fsFlags |= SQLITE_FSFLAGS_IS_MSDOS; } + if (fsInfo.f_flags & MNT_ROOTFS) { + ((unixFile*)pFile)->fsFlags |= SQLITE_FSFLAGS_IS_MMAP_SAFE; + } else { +#if defined(__APPLE__) + if (_canSafelyMapFileDescriptor(&fsInfo)) { + ((unixFile*)pFile)->fsFlags |= SQLITE_FSFLAGS_IS_MMAP_SAFE; + } +#endif + } +#if defined(__APPLE__) + /* */ + ((unixFile*)pFile)->nFsBytes = (i64)fsInfo.f_blocks * (i64)fsInfo.f_bsize; +#endif #endif /* Set up appropriate ctrlFlags */ if( isDelete ) ctrlFlags |= UNIXFILE_DELETE; if( isReadonly ) ctrlFlags |= UNIXFILE_RDONLY; @@ -7009,11 +8267,11 @@ /* SQLITE_FORCE_PROXY_LOCKING==1 means force always use proxy, 0 means ** never use proxy, NULL means use proxy for non-local files only. */ if( envforce!=NULL ){ useProxy = atoi(envforce)>0; }else{ - useProxy = !(fsInfo.f_flags&MNT_LOCAL); + useProxy = !((fsInfo.f_flags&MNT_LOCAL) || (fsInfo.f_flags&MNT_RDONLY)); } if( useProxy ){ rc = fillInUnixFile(pVfs, fd, pFile, zPath, ctrlFlags); if( rc==SQLITE_OK ){ /* cache the pMethod in case the transform fails */ @@ -7079,11 +8337,11 @@ #endif { rc = unixLogError(SQLITE_IOERR_DIR_FSYNC, "fsync", zPath); } #if OSCLOSE_CHECK_CLOSE_IOERR - if( close(fd)&&!rc ){ + if( osClose(fd)&&!rc ){ rc = SQLITE_IOERR_DIR_CLOSE; } #else robust_close(0, fd, __LINE__); #endif @@ -7183,11 +8441,11 @@ #ifndef SQLITE_OMIT_LOAD_EXTENSION /* ** Interfaces for opening a shared library, finding entry points ** within the shared library, and closing the shared library. */ -#include +#include /* amalgamator: keep */ static void *unixDlOpen(sqlite3_vfs *NotUsed, const char *zFilename){ UNUSED_PARAMETER(NotUsed); return dlopen(zFilename, RTLD_NOW | RTLD_GLOBAL); } @@ -7264,20 +8522,20 @@ memset(zBuf, 0, nBuf); randomnessPid = getpid(); #if !defined(SQLITE_TEST) { int fd, got; - fd = robust_open("/dev/urandom", O_RDONLY, 0); + fd = robust_open("/dev/urandom", O_RDONLY, 0, 0); if( fd<0 ){ time_t t; time(&t); memcpy(zBuf, &t, sizeof(t)); memcpy(&zBuf[sizeof(t)], &randomnessPid, sizeof(randomnessPid)); assert( sizeof(t)+sizeof(randomnessPid)<=(size_t)nBuf ); nBuf = sizeof(t) + sizeof(randomnessPid); }else{ - do{ got = osRead(fd, zBuf, nBuf); }while( got<0 && errno==EINTR ); + do{ got = (int)osRead(fd, zBuf, nBuf); }while( got<0 && errno==EINTR ); robust_close(0, fd, __LINE__); } } #endif return nBuf; @@ -7568,45 +8826,109 @@ /* ** The proxy lock file path for the database at dbPath is written into lPath, ** which must point to valid, writable memory large enough for a maxLen length ** file path. */ -static int proxyGetLockPath(const char *dbPath, char *lPath, size_t maxLen){ +static int proxyGetLockPath(const char *dbPath, char *lockPath, size_t maxLen){ int len; int dbLen; int i; - + int tooBig; + int nameMax; #ifdef LOCKPROXYDIR - len = strlcpy(lPath, LOCKPROXYDIR, maxLen); + len = strlcpy(lockPath, LOCKPROXYDIR, maxLen); #else # ifdef _CS_DARWIN_USER_TEMP_DIR { - if( !confstr(_CS_DARWIN_USER_TEMP_DIR, lPath, maxLen) ){ + if( !confstr(_CS_DARWIN_USER_TEMP_DIR, lockPath, maxLen) ){ OSTRACE(("GETLOCKPATH failed %s errno=%d pid=%d\n", - lPath, errno, getpid())); + lockPath, errno, getpid())); return SQLITE_IOERR_LOCK; } - len = strlcat(lPath, "sqliteplocks", maxLen); + len = (int)strlcat(lockPath, "sqliteplocks", maxLen); } # else - len = strlcpy(lPath, "/tmp/", maxLen); + len = strlcpy(lockPath, "/tmp/", maxLen); # endif #endif - if( lPath[len-1]!='/' ){ - len = strlcat(lPath, "/", maxLen); + if( lockPath[len-1]!='/' ){ + len = (int)strlcat(lockPath, "/", maxLen); } - - /* transform the db path to a unique cache name */ + + /* + ** transform the db path to a unique cache name. + ** By default this simply turns '/'s into '_'s, + ** but if the resulting path would exceed maxLen, hash the + ** path and create a subpath like: + ** -:auto: + */ dbLen = (int)strlen(dbPath); - for( i=0; i maxLen ) ? 1 : 0; +#if defined(NAME_MAX) + nameMax = NAME_MAX; +#else + nameMax = 255; +#endif + +#if defined(__APPLE__) + if( tooBig || (nameMax<(dbLen+7)) ){ + CC_SHA256_CTX ctx; + unsigned char shabytes[CC_SHA256_DIGEST_LENGTH]; + char shahex[CC_SHA256_DIGEST_LENGTH * 2 + 2]; + + CC_SHA256_Init (&ctx); + CC_SHA256_Update (&ctx, dbPath, dbLen); + CC_SHA256_Final (shabytes, &ctx); + + /* render the hash into the path as chars [0-9][a-f] */ + for( i=0; i> 4); + if (shahex[i*2] > '9'){ + shahex[i*2] += 'a' - '9' - 1; + } + shahex[i*2+1] = '0' + (byte & 0xf); + if (shahex[i*2+1] > '9'){ + shahex[i*2+1] += 'a' - '9' - 1; + } + } + shahex[CC_SHA256_DIGEST_LENGTH * 2] = '-'; + shahex[CC_SHA256_DIGEST_LENGTH * 2 + 1] = 0; + + strlcat(lockPath, shahex, maxLen); + + /* tack on the actual database name for discoverability */ + const char *dbName = strrchr (dbPath, '/'); + if (!dbName){ + dbName = dbPath; + }else{ + ++dbName; + } + + strlcat(lockPath, dbName, maxLen); + }else{ +#endif + int pos = len; + for( i=0; i(i+2)) && (dbPath[i]=='.') && (dbPath[i+1]=='/') && + ((i==0) || dbPath[i-1]=='/') ){ + i ++; /* skips 2 */ + continue; + } + + lockPath[pos] = (c=='/')?'_':c; + pos++; + } + lockPath[pos]='\0'; +#if defined(__APPLE__) } - lPath[i+len]='\0'; - strlcat(lPath, ":auto:", maxLen); - OSTRACE(("GETLOCKPATH proxy lock path=%s pid=%d\n", lPath, getpid())); +#endif + strlcat(lockPath, ":auto:", maxLen); + OSTRACE(("GETLOCKPATH proxy lock path=%s pid=%d\n", lockPath, getpid())); return SQLITE_OK; } /* ** Creates the lock file and any missing directories in lockPath @@ -7661,13 +8983,17 @@ assert(pFile!=NULL && pFile->lockingContext!=NULL); assert(pFile->pMethod == &proxyIoMethods); pCtx = ((proxyLockingContext *)(pFile->lockingContext)); pLockFile = pCtx->lockProxy; - if( pLockFile->pMethod->iVersion>=2 && pLockFile->pMethod->xShmMap!=0 ){ - return pCtx->lockProxyPath; + + if (pLockFile) { + if( pLockFile->pMethod->iVersion>=2 && pLockFile->pMethod->xShmMap!=0 ){ + return pCtx->lockProxyPath; + } } + return NULL; } #endif /* @@ -7684,10 +9010,11 @@ ) { int fd = -1; unixFile *pNew; int rc = SQLITE_OK; int openFlags = O_RDWR | O_CREAT; + int sqliteFlags = SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE; sqlite3_vfs dummyVfs; int terrno = 0; UnixUnusedFd *pUnused = NULL; /* 1. first try to open/create the file @@ -7694,31 +9021,32 @@ ** 2. if that fails, and this is a lock file (not-conch), try creating ** the parent directories and then try again. ** 3. if that fails, try to open the file read-only ** otherwise return BUSY (if lock file) or CANTOPEN for the conch file */ - pUnused = findReusableFd(path, openFlags); + pUnused = findReusableFd(path, sqliteFlags); if( pUnused ){ fd = pUnused->fd; }else{ pUnused = sqlite3_malloc(sizeof(*pUnused)); if( !pUnused ){ return SQLITE_NOMEM; } } if( fd<0 ){ - fd = robust_open(path, openFlags, 0); + fd = robust_open2(path, openFlags, 0, 0,0); terrno = errno; if( fd<0 && errno==ENOENT && islockfile ){ if( proxyCreateLockPath(path) == SQLITE_OK ){ - fd = robust_open(path, openFlags, 0); + fd = robust_open2(path, openFlags, 0, 0, 0); } } } if( fd<0 ){ openFlags = O_RDONLY; - fd = robust_open(path, openFlags, 0); + sqliteFlags = SQLITE_OPEN_READONLY; + fd = robust_open2(path, openFlags, 0, 0, 0); terrno = errno; } if( fd<0 ){ sqlite3_free(pUnused); if( islockfile ){ @@ -7743,11 +9071,11 @@ pNew->openFlags = openFlags; memset(&dummyVfs, 0, sizeof(dummyVfs)); dummyVfs.pAppData = (void*)&autolockIoFinder; dummyVfs.zName = "dummy"; pUnused->fd = fd; - pUnused->flags = openFlags; + pUnused->flags = sqliteFlags; pNew->pUnused = pUnused; rc = fillInUnixFile(&dummyVfs, fd, (sqlite3_file*)pNew, path, 0); if( rc==SQLITE_OK ){ *ppFile = pNew; @@ -7765,13 +9093,15 @@ int sqlite3_hostid_num = 0; #endif #define PROXY_HOSTIDLEN 16 /* conch file host id length */ +#if HAVE_GETHOSTUUID /* Not always defined in the headers as it ought to be */ extern int gethostuuid(uuid_t id, const struct timespec *wait); - +#endif + /* get the host ID via gethostuuid(), pHostID must point to PROXY_HOSTIDLEN ** bytes of writable memory. */ static int proxyGetHostID(unsigned char *pHostID, int *pError){ assert(PROXY_HOSTIDLEN == sizeof(uuid_t)); @@ -7839,11 +9169,11 @@ if( readLennFails==2 ){ char tBuf[PROXY_MAXCONCHLEN]; - int len = osPread(conchFile->h, tBuf, PROXY_MAXCONCHLEN, 0); + int len = (int)osPread(conchFile->h, tBuf, PROXY_MAXCONCHLEN, 0); if( len<0 ){ storeLastErrno(pFile, errno); return SQLITE_IOERR_LOCK; } if( len>PROXY_PATHINDEX && tBuf[0]==(char)PROXY_CONCHVERSION){ @@ -8068,10 +9398,15 @@ futimes(conchFile->h, NULL); if( hostIdMatch && !createConch ){ if( conchFile->pInode && conchFile->pInode->nShared>1 ){ /* We are trying for an exclusive lock but another thread in this ** same process is still holding a shared lock. */ +#ifdef SQLITE_USE_FLOCKTIMEOUT + if( pFile->nextFlockTimeout>0 ){ + usleep(((int)pFile->nextFlockTimeout) * 1000L); + } +#endif rc = SQLITE_BUSY; } else { rc = proxyConchLock(pFile, myHostID, EXCLUSIVE_LOCK); } }else{ @@ -8087,11 +9422,11 @@ strlcpy(&writeBuffer[PROXY_PATHINDEX], pCtx->lockProxyPath, MAXPATHLEN); }else{ strlcpy(&writeBuffer[PROXY_PATHINDEX], tempLockPath, MAXPATHLEN); } - writeSize = PROXY_PATHINDEX + strlen(&writeBuffer[PROXY_PATHINDEX]); + writeSize = PROXY_PATHINDEX + (int)strlen(&writeBuffer[PROXY_PATHINDEX]); robust_ftruncate(conchFile->h, writeSize); rc = unixWrite((sqlite3_file *)conchFile, writeBuffer, writeSize, 0); fsync(conchFile->h); /* If we created a new conch file (not just updated the contents of a ** valid conch file), try to match the permissions of the database @@ -8130,20 +9465,20 @@ OSTRACE(("TRANSPROXY: CLOSE %d\n", pFile->h)); if( rc==SQLITE_OK && pFile->openFlags ){ int fd; if( pFile->h>=0 ){ #if defined(STRICT_CLOSE_ERROR) && OSCLOSE_CHECK_CLOSE_IOERR - if( close(pFile->h) ){ + if( osClose(pFile->h) ){ storeLastErrno(pFile, errno); return SQLITE_IOERR_CLOSE; } #else robust_close(pFile, pFile->h, __LINE__); #endif } pFile->h = -1; - fd = robust_open(pCtx->dbPath, pFile->openFlags, 0); + fd = robust_open2(pCtx->dbPath, pFile->openFlags, 0, 0, 0); OSTRACE(("TRANSPROXY: OPEN %d\n", fd)); if( fd>=0 ){ pFile->h = fd; }else{ rc=SQLITE_CANTOPEN_BKPT; /* SQLITE_BUSY? proxyTakeConch called @@ -8266,10 +9601,15 @@ proxyLockingContext *pCtx = (proxyLockingContext*)pFile->lockingContext; char *oldPath = pCtx->lockProxyPath; int rc = SQLITE_OK; if( pFile->eFileLock!=NO_LOCK ){ +#ifdef SQLITE_USE_FLOCKTIMEOUT + if( pFile->nextFlockTimeout>0 ){ + usleep(((int)pFile->nextFlockTimeout) * 1000L); + } +#endif return SQLITE_BUSY; } /* nothing to do if the path is NULL, :auto: or matches the existing path */ if( !path || path[0]=='\0' || !strcmp(path, ":auto:") || @@ -8309,11 +9649,11 @@ } else #endif if( pFile->pMethod == &dotlockIoMethods ){ /* dot lock style uses the locking context to store the dot lock ** file path */ - int len = strlen((char *)pFile->lockingContext) - strlen(DOTLOCK_SUFFIX); + long len = strlen((char *)pFile->lockingContext) - strlen(DOTLOCK_SUFFIX); memcpy(dbPath, (char *)pFile->lockingContext, len + 1); }else{ /* all other styles use the locking context to store the db file path */ assert( strlen((char*)pFile->lockingContext)<=MAXPATHLEN ); strlcpy(dbPath, (char *)pFile->lockingContext, MAXPATHLEN); @@ -8334,10 +9674,15 @@ char dbPath[MAXPATHLEN+1]; /* Name of the database file */ char *lockPath=NULL; int rc = SQLITE_OK; if( pFile->eFileLock!=NO_LOCK ){ +#ifdef SQLITE_USE_FLOCKTIMEOUT + if( pFile->nextFlockTimeout>0 ){ + usleep(((int)pFile->nextFlockTimeout) * 1000L); + } +#endif return SQLITE_BUSY; } getDbPathForUnixFile(pFile, dbPath); if( !path || path[0]=='\0' || !strcmp(path, ":auto:") ){ lockPath=NULL; Index: src/pager.c ================================================================== --- src/pager.c +++ src/pager.c @@ -20,10 +20,11 @@ */ #ifndef SQLITE_OMIT_DISKIO #include "sqliteInt.h" #include "wal.h" +extern void pthread_yield_np(void); /******************* NOTES ON THE DESIGN OF THE PAGER ************************ ** ** This comment block describes invariants that hold when using a rollback ** journal. These invariants do not apply for journal_mode=WAL, @@ -3790,13 +3791,21 @@ assert( (pPager->eLock>=locktype) || (pPager->eLock==NO_LOCK && locktype==SHARED_LOCK) || (pPager->eLock==RESERVED_LOCK && locktype==EXCLUSIVE_LOCK) ); + int i = 0; + int c = 0; do { rc = pagerLockDb(pPager, locktype); - }while( rc==SQLITE_BUSY && pPager->xBusyHandler(pPager->pBusyHandlerArg) ); + c = ( rc==SQLITE_BUSY && pPager->xBusyHandler(pPager->pBusyHandlerArg) ); + if (c && (i > 0)) { + pthread_yield_np(); + } + i++; + }while( c ); + return rc; } /* ** Function assertTruncateConstraint(pPager) checks that one of the Index: src/pcache1.c ================================================================== --- src/pcache1.c +++ src/pcache1.c @@ -17,10 +17,12 @@ ** these two features are available. */ #include "sqliteInt.h" +#if !defined(SQLITE_OMIT_PCACHE1_PCACHE) + typedef struct PCache1 PCache1; typedef struct PgHdr1 PgHdr1; typedef struct PgFreeslot PgFreeslot; typedef struct PGroup PGroup; @@ -1027,5 +1029,7 @@ *pnMax = (int)pcache1.grp.nMaxPage; *pnMin = (int)pcache1.grp.nMinPage; *pnRecyclable = nRecyclable; } #endif + +#endif /* SQLITE_OMIT_PCACHE1_PCACHE */ Index: src/pragma.c ================================================================== --- src/pragma.c +++ src/pragma.c @@ -1323,10 +1323,17 @@ } break; } #endif +#if !defined(SQLITE_ENABLE_LOCKING_STYLE) +# if defined(__APPLE__) +# define SQLITE_ENABLE_LOCKING_STYLE 1 +# else +# define SQLITE_ENABLE_LOCKING_STYLE 0 +# endif +#endif #if SQLITE_ENABLE_LOCKING_STYLE /* ** PRAGMA [database.]lock_proxy_file ** PRAGMA [database.]lock_proxy_file = ":auto:"|"lock_file_path" ** Index: src/shell.c ================================================================== --- src/shell.c +++ src/shell.c @@ -3718,13 +3718,14 @@ int rc = 0; int warnInmemoryDb = 0; #if USE_SYSTEM_SQLITE+0!=1 if( strcmp(sqlite3_sourceid(),SQLITE_SOURCE_ID)!=0 ){ - fprintf(stderr, "SQLite header and source version mismatch\n%s\n%s\n", + fprintf(stderr, "\nWARNING! SQLite header and source version mismatch" + "\n source: %s\n header: %s\n\n", sqlite3_sourceid(), SQLITE_SOURCE_ID); - exit(1); + /* exit(1); */ } #endif Argv0 = argv[0]; main_init(&data); stdin_is_interactive = isatty(0); Index: src/sqlite.h.in ================================================================== --- src/sqlite.h.in +++ src/sqlite.h.in @@ -62,10 +62,13 @@ ** that we have taken it all out and gone back to using simple ** noop macros. */ #define SQLITE_DEPRECATED #define SQLITE_EXPERIMENTAL +#ifndef __OSX_AVAILABLE_BUT_DEPRECATED +#define __OSX_AVAILABLE_BUT_DEPRECATED(MacOSAvailable, MacOSDeprecated, iPhoneOSAvailable, iPhoneOSDeprecated) +#endif /* ** Ensure these symbols were not defined by some previous header file. */ #ifdef SQLITE_VERSION @@ -527,10 +530,14 @@ #define SQLITE_OPEN_NOMUTEX 0x00008000 /* Ok for sqlite3_open_v2() */ #define SQLITE_OPEN_FULLMUTEX 0x00010000 /* Ok for sqlite3_open_v2() */ #define SQLITE_OPEN_SHAREDCACHE 0x00020000 /* Ok for sqlite3_open_v2() */ #define SQLITE_OPEN_PRIVATECACHE 0x00040000 /* Ok for sqlite3_open_v2() */ #define SQLITE_OPEN_WAL 0x00080000 /* VFS only */ +#define SQLITE_OPEN_FILEPROTECTION_COMPLETE 0x00100000 +#define SQLITE_OPEN_FILEPROTECTION_COMPLETEUNLESSOPEN 0x00200000 +#define SQLITE_OPEN_FILEPROTECTION_COMPLETEUNTILFIRSTUSERAUTHENTICATION 0x00300000 +#define SQLITE_OPEN_FILEPROTECTION_NONE 0x00400000 #define SQLITE_OPEN_FILEPROTECTION_MASK 0x00700000 /* Reserved: 0x00F00000 */ /* @@ -4989,11 +4996,11 @@ ** This interface is threadsafe on processors where writing a ** 32-bit integer is atomic. ** ** See Also: [SQLite Shared-Cache Mode] */ -int sqlite3_enable_shared_cache(int); +int sqlite3_enable_shared_cache(int) __OSX_AVAILABLE_BUT_DEPRECATED(__MAC_10_0, __MAC_10_7, __IPHONE_2_0, __IPHONE_5_0); /* ** CAPI3REF: Attempt To Free Heap Memory ** ** ^The sqlite3_release_memory() interface attempts to free N bytes Index: src/sqlite3_private.h ================================================================== --- src/sqlite3_private.h +++ src/sqlite3_private.h @@ -3,10 +3,13 @@ */ #ifndef _SQLITE3_PRIVATE_H #define _SQLITE3_PRIVATE_H +#include +#include + #define SQLITE_LOCKSTATE_OFF 0 #define SQLITE_LOCKSTATE_ON 1 #define SQLITE_LOCKSTATE_NOTADB 2 #define SQLITE_LOCKSTATE_ERROR -1 @@ -33,10 +36,22 @@ ** locks by re-using open file descriptors for the database file and support ** files (-shm) */ #define SQLITE_FCNTL_LOCKSTATE_PID 103 +/* +** Purges eligible purgable memory regions (holding only unpinned pages) from +** the page cache +*/ +extern void _sqlite3_purgeEligiblePagerCacheMemory(void); + +/* +** Returns the system defined default sqlite3_busy_handler function +** +*/ +extern int (*_sqlite3_system_busy_handler(void))(void*,int); + /* ** Pass the SQLITE_TRUNCATE_DATABASE operation code to sqlite3_file_control() ** to truncate a database and its associated journal file to zero length. The ** SQLITE_TRUNCATE_* flags represent optional flags to safely initialize an ** empty database in the place of the truncated database, the flags are passed Index: src/sqliteInt.h ================================================================== --- src/sqliteInt.h +++ src/sqliteInt.h @@ -88,11 +88,15 @@ /* ** Include the configuration header output by 'configure' if we're using the ** autoconf-based build */ #ifdef _HAVE_SQLITE_CONFIG_H +# ifdef _MSC_VER +#include "win_config.h" /* hand-mantained windows variant */ +# else #include "config.h" +# endif #endif #include "sqliteLimit.h" /* Disable nuisance warnings on Borland compilers */ @@ -188,10 +192,20 @@ */ #if !defined(SQLITE_DEFAULT_MEMSTATUS) # define SQLITE_DEFAULT_MEMSTATUS 1 #endif +/* +** The SQLITE_ENABLE_PURGEABLE_PCACHE sets the default pcache to +** pcache_purgeable, otherwise pcache1 is used. +*/ +#if !defined(SQLITE_ENABLE_PURGEABLE_PCACHE) +# define SQLITE_OMIT_PURGEABLE_PCACHE +#else +# define SQLITE_OMIT_PCACHE1_PCACHE +#endif + /* ** Exactly one of the following macros must be defined in order to ** specify which memory allocation subsystem to use. ** ** SQLITE_SYSTEM_MALLOC // Use normal system malloc() Index: src/test1.c ================================================================== --- src/test1.c +++ src/test1.c @@ -5092,11 +5092,11 @@ } return TCL_OK; } #ifdef __APPLE__ -/* From sqlite3_priavet.h */ +/* From sqlite3_private.h */ # ifndef SQLITE_TRUNCATE_DATABASE # define SQLITE_TRUNCATE_DATABASE 101 # define SQLITE_TRUNCATE_JOURNALMODE_WAL (0x1<<0) # define SQLITE_TRUNCATE_AUTOVACUUM_MASK (0x3<<2) # define SQLITE_TRUNCATE_AUTOVACUUM_OFF (0x1<<2) @@ -5109,10 +5109,19 @@ # define SQLITE_TRUNCATE_PAGESIZE_8192 (0x4<<4) # endif # ifndef SQLITE_REPLACE_DATABASE # define SQLITE_REPLACE_DATABASE 102 # endif +# ifndef SQLITE_FCNTL_LOCKSTATE_PID +# define SQLITE_FCNTL_LOCKSTATE_PID 103 +# define SQLITE_LOCKSTATE_OFF 0 +# define SQLITE_LOCKSTATE_ON 1 +# define SQLITE_LOCKSTATE_NOTADB 2 +# define SQLITE_LOCKSTATE_ERROR -1 +# define SQLITE_LOCKSTATE_ANYPID -1 +extern int _sqlite3_lockstate(const char *path, pid_t pid); +# endif /* ** tclcmd: file_control_truncate_test DB ** ** This TCL command runs the sqlite3_file_control interface and @@ -5157,11 +5166,10 @@ ClientData clientData, /* Pointer to sqlite3_enable_XXX function */ Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ int objc, /* Number of arguments */ Tcl_Obj *CONST objv[] /* Command arguments */ ){ - int iArg = 0; sqlite3 *src_db; sqlite3 *dst_db; int rc; if( objc!=3 ){ @@ -5180,10 +5188,44 @@ Tcl_SetObjResult(interp, Tcl_NewIntObj(rc)); return TCL_ERROR; } return TCL_OK; } + +/* +** tclcmd: file_control_lockstate_test DBNAME PID +** +** This TCL command runs the _sqlite3_lockstate interface and +** verifies correct operation of the SQLITE_FCNTL_LOCKSTATE_PID verb. +*/ +static int file_control_lockstate_test( + ClientData clientData, /* Pointer to sqlite3_enable_XXX function */ + Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ + int objc, /* Number of arguments */ + Tcl_Obj *CONST objv[] /* Command arguments */ +){ + char *zDb; + int pid = -1; + int state; + + if( objc!=3 ){ + Tcl_AppendResult(interp, "wrong # args: should be \"", + Tcl_GetStringFromObj(objv[0], 0), " DBNAME PID", 0); + return TCL_ERROR; + } + if( Tcl_GetIntFromObj(interp, objv[2], &pid) ){ + return TCL_ERROR; + } + zDb = Tcl_GetString(objv[1]); + if( zDb[0]=='\0' ) zDb = NULL; + if( !zDb ){ + return TCL_ERROR; + } + state = _sqlite3_lockstate(zDb, pid); + Tcl_SetObjResult(interp, Tcl_NewIntObj(state)); + return TCL_OK; +} #endif /* __APPLE__ */ /* ** tclcmd: file_control_chunksize_test DB DBNAME SIZE ** @@ -6711,10 +6753,11 @@ { "file_control_lasterrno_test", file_control_lasterrno_test, 0 }, { "file_control_lockproxy_test", file_control_lockproxy_test, 0 }, #ifdef __APPLE__ { "file_control_truncate_test", file_control_truncate_test, 0 }, { "file_control_replace_test", file_control_replace_test, 0 }, + { "file_control_lockstate_test", file_control_lockstate_test, 0 }, #endif { "file_control_chunksize_test", file_control_chunksize_test, 0 }, { "file_control_sizehint_test", file_control_sizehint_test, 0 }, #if SQLITE_OS_WIN { "file_control_win32_av_retry", file_control_win32_av_retry, 0 }, Index: src/test_config.c ================================================================== --- src/test_config.c +++ src/test_config.c @@ -539,10 +539,16 @@ Tcl_SetVar2(interp, "sqlite_options", "threadsafe", STRINGVALUE(SQLITE_THREADSAFE), TCL_GLOBAL_ONLY); assert( sqlite3_threadsafe()==SQLITE_THREADSAFE ); +#if defined(USETHREADASSERTS) + Tcl_SetVar2(interp,"sqlite_options","use_thread_asserts","1",TCL_GLOBAL_ONLY); +#else + Tcl_SetVar2(interp,"sqlite_options","use_thread_asserts","0",TCL_GLOBAL_ONLY); +#endif + #ifdef SQLITE_OMIT_TEMPDB Tcl_SetVar2(interp, "sqlite_options", "tempdb", "0", TCL_GLOBAL_ONLY); #else Tcl_SetVar2(interp, "sqlite_options", "tempdb", "1", TCL_GLOBAL_ONLY); #endif Index: src/test_intarray.c ================================================================== --- src/test_intarray.c +++ src/test_intarray.c @@ -56,12 +56,13 @@ /* ** Free an sqlite3_intarray object. */ static void intarrayFree(sqlite3_intarray *p){ - if( p->xFree ){ + if( p->xFree && p->a){ p->xFree(p->a); + p->a = NULL; } sqlite3_free(p); } /* @@ -256,10 +257,11 @@ sqlite3_int64 *aElements, /* Content of the intarray */ void (*xFree)(void*) /* How to dispose of the intarray when done */ ){ if( pIntArray->xFree ){ pIntArray->xFree(pIntArray->a); + pIntArray->a = NULL; } pIntArray->n = nElements; pIntArray->a = aElements; pIntArray->xFree = xFree; return SQLITE_OK; Index: src/util.c ================================================================== --- src/util.c +++ src/util.c @@ -1068,10 +1068,18 @@ } return zBlob; } #endif /* !SQLITE_OMIT_BLOB_LITERAL || SQLITE_HAS_CODEC */ +#if defined(__APPLE__) && (defined(USETHREADASSERTS) || defined(SQLITE_DEBUG)) +__private_extern__ char* __crashreporter_info__ = NULL; +static const char* _defaultMessage_ = "\n SQLite: Fatal multithreading error in caller. Illegal multi-threaded access to the same database connection.\n"; +#define HALT_CATCH_FIRE() { __crashreporter_info__ = (char*)_defaultMessage_; __builtin_trap(); } +#else +#define HALT_CATCH_FIRE() +#endif + /* ** Log an error that is an API call on a connection pointer that should ** not have been used. The "type" of connection pointer is given as the ** argument. The zType is a word like "NULL" or "closed" or "invalid". */ @@ -1106,10 +1114,15 @@ if( magic!=SQLITE_MAGIC_OPEN ){ if( sqlite3SafetyCheckSickOrOk(db) ){ testcase( sqlite3GlobalConfig.xLog!=0 ); logBadConnection("unopened"); } +#if defined(USETHREADASSERTS) || defined(SQLITE_DEBUG) + if (magic==SQLITE_MAGIC_BUSY) { + HALT_CATCH_FIRE(); + } +#endif return 0; }else{ return 1; } } @@ -1163,10 +1176,11 @@ #define TWOPOWER32 (((i64)1)<<32) #define TWOPOWER31 (((i64)1)<<31) int sqlite3MulInt64(i64 *pA, i64 iB){ i64 iA = *pA; i64 iA1, iA0, iB1, iB0, r; +#undef HALT_CATCH_FIRE iA1 = iA/TWOPOWER32; iA0 = iA % TWOPOWER32; iB1 = iB/TWOPOWER32; iB0 = iB % TWOPOWER32; Index: src/vtab.c ================================================================== --- src/vtab.c +++ src/vtab.c @@ -43,10 +43,15 @@ sqlite3_mutex_enter(db->mutex); nName = sqlite3Strlen30(zName); if( sqlite3HashFind(&db->aModule, zName, nName) ){ rc = SQLITE_MISUSE_BKPT; + // + if (strncmp("fts", zName, 3) == 0) { + xDestroy = NULL; + rc = SQLITE_OK; + } }else{ Module *pMod; pMod = (Module *)sqlite3DbMallocRaw(db, sizeof(Module) + nName + 1); if( pMod ){ Module *pDel; @@ -324,11 +329,15 @@ pTable->tabFlags |= TF_Virtual; pTable->nModuleArg = 0; addModuleArgument(db, pTable, sqlite3NameFromToken(db, pModuleName)); addModuleArgument(db, pTable, 0); addModuleArgument(db, pTable, sqlite3DbStrDup(db, pTable->zName)); - pParse->sNameToken.n = (int)(&pModuleName->z[pModuleName->n] - pName1->z); + if (pName2 && pName2->z) { + pParse->sNameToken.n = (int)(&pModuleName->z[pModuleName->n] - pName2->z); + } else { + pParse->sNameToken.n = (int)(&pModuleName->z[pModuleName->n] - pName1->z); + } #ifndef SQLITE_OMIT_AUTHORIZATION /* Creating a virtual table invokes the authorization callback twice. ** The first invocation, to obtain permission to INSERT a row into the ** sqlite_master table, has already been made by sqlite3StartTable(). Index: src/wal.c ================================================================== --- src/wal.c +++ src/wal.c @@ -1675,16 +1675,34 @@ u32 mxSafeFrame; /* Max frame that can be backfilled */ u32 mxPage; /* Max database page to write */ int i; /* Loop counter */ volatile WalCkptInfo *pInfo; /* The checkpoint status information */ int (*xBusy)(void*) = 0; /* Function to call when waiting for locks */ +#if defined(__APPLE__) + /* */ + i64 szDbFile; /* File size of the database, in bytes */ + i64 szWalFile; /* File size of the wal, in bytes */ +#endif szPage = walPagesize(pWal); testcase( szPage<=32768 ); testcase( szPage>=65536 ); pInfo = walCkptInfo(pWal); if( pInfo->nBackfill>=pWal->hdr.mxFrame ) return SQLITE_OK; + +#if defined(__APPLE__) + /* */ + /* Get the size of the db and wal files to sanity check write offsets. */ + rc = sqlite3OsFileSize(pWal->pDbFd, &szDbFile); + if( rc!=SQLITE_OK ){ + return rc; + } + rc = sqlite3OsFileSize(pWal->pWalFd, &szWalFile); + if( rc!=SQLITE_OK ){ + return rc; + } +#endif /* Allocate the iterator */ rc = walIteratorInit(pWal, &pIter); if( rc!=SQLITE_OK ){ return rc; @@ -1749,10 +1767,28 @@ /* testcase( IS_BIG_INT(iOffset) ); // requires a 4GiB WAL file */ rc = sqlite3OsRead(pWal->pWalFd, zBuf, szPage, iOffset); if( rc!=SQLITE_OK ) break; iOffset = (iDbpage-1)*(i64)szPage; testcase( IS_BIG_INT(iOffset) ); + +#if defined(__APPLE__) + /* */ + /* DEBUG(numist): this is way, way more conservative than the checks in + ** the seekAndRead/seekAndWrite functions. I'm not even 100% sure that + ** it's not overconservative, but db storage is more efficient than wal + ** storage, so I'm pretty confident that writes to the db should never + ** exceed dbsize +_ walsize. + */ + if( iOffset>(szDbFile+szWalFile) ){ + __builtin_trap(); + } + /* Update the db file size if the file will grow due to the write. */ + if( iOffset+szPage > szDbFile ){ + szDbFile = iOffset+szPage; + } +#endif + rc = sqlite3OsWrite(pWal->pDbFd, zBuf, szPage, iOffset); if( rc!=SQLITE_OK ) break; } /* If work was actually accomplished... */ @@ -1884,10 +1920,14 @@ sqlite3_free(pWal); } return rc; } +#ifndef SQLITE_AMALGAMATION +extern int sqlite30sShmRead(sqlite3_file *id, void *dst, volatile void *src, sqlite3_int64 size); +#endif + /* ** Try to read the wal-index header. Return 0 on success and 1 if ** there is a problem. ** ** The wal-index is in shared memory. Another thread or process might @@ -1905,10 +1945,11 @@ */ static int walIndexTryHdr(Wal *pWal, int *pChanged){ u32 aCksum[2]; /* Checksum on the header content */ WalIndexHdr h1, h2; /* Two copies of the header content */ WalIndexHdr volatile *aHdr; /* Header in shared memory */ + int rc = SQLITE_OK; /* The first page of the wal-index must be mapped at this point. */ assert( pWal->nWiData>0 && pWal->apWiData[0] ); /* Read the header. This might happen concurrently with a write to the @@ -1923,13 +1964,22 @@ */ aHdr = walIndexHdr(pWal); if( aHdr==NULL ){ return 1; /* Shouldn't be getting NULL from walIndexHdr, but we are */ } - memcpy(&h1, (void *)&aHdr[0], sizeof(h1)); + + rc = sqlite30sShmRead(pWal->pDbFd, &h1, &aHdr[0], sizeof(h1)); + if ( rc!=SQLITE_OK ) { + return rc; + } + walShmBarrier(pWal); - memcpy(&h2, (void *)&aHdr[1], sizeof(h2)); + + rc = sqlite30sShmRead(pWal->pDbFd, &h2, &aHdr[1], sizeof(h2)); + if ( rc!=SQLITE_OK ) { + return rc; + } if( memcmp(&h1, &h2, sizeof(h1))!=0 ){ return 1; /* Dirty read */ } if( h1.isInit==0 ){ @@ -2852,11 +2902,17 @@ int nDbSize; /* 0 normally. Positive == commit flag */ iFrame++; assert( iOffset==walFrameOffset(iFrame, szPage) ); nDbSize = (isCommit && p->pDirty==0) ? nTruncate : 0; rc = walWriteOneFrame(&w, p, nDbSize, iOffset); - if( rc ) return rc; + if( rc ) { +#if defined(SQLITE_WRITE_WALFRAME_PREBUFFERED) + free(w.aFrameBuf); + w.aFrameBuf = NULL; +#endif + return rc; + } pLast = p; iOffset += szFrame; } /* If this is the end of a transaction, then we might need to pad @@ -2877,11 +2933,17 @@ if( pWal->padToSectorBoundary ){ int sectorSize = sqlite3SectorSize(pWal->pWalFd); w.iSyncPoint = ((iOffset+sectorSize-1)/sectorSize)*sectorSize; while( iOffset 0} { lappend res $key } + } + ifcapable {use_thread_asserts} { + if { $mode == "multithread" } { + # USETHREADASSERTS changes the multithread mutexes to serialized + set res [lsort $mutexes] + } } lsort $res } [lsort $mutexes] } sqlite3_enable_shared_cache $enable_shared_cache Index: test/permutations.test ================================================================== --- test/permutations.test +++ test/permutations.test @@ -343,13 +343,13 @@ test_suite "coverage-pager" -description { Coverage tests for file pager.c. } -files { pager1.test pager2.test pagerfault.test pagerfault2.test walfault.test walbak.test journal2.test tkt-9d68c883.test +test_suite "coverage-analyze" -description { } -test_suite "coverage-analyze" -description { Coverage tests for file analyze.c. } -files { analyze3.test analyze4.test analyze5.test analyze6.test analyze7.test analyze8.test analyze9.test analyzeA.test analyze.test analyzeB.test mallocA.test Index: test/pragma.test ================================================================== --- test/pragma.test +++ test/pragma.test @@ -1518,10 +1518,33 @@ create table if not exists aa(bb); PRAGMA lock_proxy_file; } db] string match "*proxytest/sub/dir/lock" $lockpath4 } {1} + + # make sure we can deal with long (> NAME_MAX) file paths correctly + db close + set db_longroot [exec mktemp -d -t "longdbpath"] + set db_longpath $db_longroot/123456789/223456789/323456789/423456789/523456789/623456789/723456789/823456789/923456789/a23456789/b23456789/c23456789/d23456789/e23456789/f23456789/g23456789/h23456789/i23456789/j23456789/k23456789/l23456789/m23456789/n23456789/o23456789/p23456789/q23456789 + file mkdir $db_longpath + do_test pragma-16.10.4.1 { + sqlite3 db $db_longpath/longtest.db + execsql "PRAGMA lock_proxy_file=':auto:'" + set lockpath_long [execsql { + create table if not exists aa(bb); + PRAGMA lock_proxy_file; + } db] + #puts $db_longpath + #puts $lockpath_long + set lockPathLen [string len $lockpath_long] + set dbPathLen [string len $db_longpath] + set lockName [string range $lockpath_long [string last / $lockpath_long] $lockPathLen] + set nameLength [string len $lockName] + list [expr "$dbPathLen > 256"] [expr "$lockPathLen < 1024"] [expr "$nameLength < 256"] [expr "$lockPathLen > 7"] [expr "$nameLength > 7"] + } {1 1 1 1 1} + file delete -force $db_longroot + file delete -force $lockpath_long # ensure that if the path can not be created (perm), setting :auto: deals db close file delete -force $lpp5d do_test pragma-16.10.5 { Index: test/superlock.test ================================================================== --- test/superlock.test +++ test/superlock.test @@ -211,13 +211,21 @@ write_content ${file2}-wal $wal1 unlock1 unlock2 } + +sqlite3 db test.db +file_control_persist_wal db 0 +db close +sqlite3 db2 test.db2 +file_control_persist_wal db2 0 +db2 close forcedelete test.db sqlite3 db test.db +file_control_persist_wal db 0 do_execsql_test 6.1 { ATTACH 'test.db2' AS aux; PRAGMA aux.journal_mode = wal; CREATE TABLE aux.t2(x, y); INSERT INTO aux.t2 VALUES('a', 'b'); @@ -229,10 +237,13 @@ INSERT INTO t1 VALUES(1, 2); INSERT INTO t1 VALUES(3, 4); SELECT * FROM t1; } {wal wal 1 2 3 4} +sqlite3 db2 test.db2 +file_control_persist_wal db2 0 +db2 close db_swap test.db2 test.db do_catchsql_test 6.2 { SELECT * FROM t1 } {1 {no such table: t1}} do_catchsql_test 6.3 { SELECT * FROM t2 } {0 {a b}} Index: test/walro.test ================================================================== --- test/walro.test +++ test/walro.test @@ -44,11 +44,11 @@ # Close all connections and delete the database. # code1 { db close } code2 { db2 close } code3 { db3 close } - forcedelete test.db + forcedelete test.db $shmpath forcedelete walro # Do not run tests with the connections in the same process. # if {$tn==2} continue @@ -181,12 +181,12 @@ #----------------------------------------------------------------------- # Test cases 1.4.* check that checkpoints and log wraps don't prevent # read-only connections from reading the database. do_test 1.4.1 { code1 { db close } - forcedelete test.db-shm - file exists test.db-shm + forcedelete $shmpath + file exists $shmpath } {0} # Open one read-only and one read-write connection. Write some data # and then run a checkpoint using the read-write connection. Then # check the read-only connection can still read. @@ -243,10 +243,13 @@ #----------------------------------------------------------------------- # Test cases 2.* check that a read-only connection may read the # database file while a checkpoint operation is ongoing. # do_multiclient_test tn { + # Do not run tests with the connections in the same process. + # + if {$tn==2} continue # Close all connections and delete the database. # code1 { db close } code2 { db2 close } Index: tool/mksqlite3c.tcl ================================================================== --- tool/mksqlite3c.tcl +++ tool/mksqlite3c.tcl @@ -114,14 +114,16 @@ sqlite3.h sqliteicu.h sqliteInt.h sqliteLimit.h sqlrr.h + test_intarray.h vdbe.h vdbeInt.h wal.h whereInt.h + sqlite3_private.h } { set available_hdr($hdr) 1 } set available_hdr(sqliteInt.h) 0 @@ -263,10 +265,11 @@ os_win.c bitvec.c pcache.c pcache1.c + pcache_purgeable.c rowset.c pager.c wal.c btmutex.c @@ -331,10 +334,11 @@ rtree.c icu.c fts3_icu.c sqlrr.c + test_intarray.c } { copy_file tsrc/$file } close $out Index: tool/warnings.sh ================================================================== --- tool/warnings.sh +++ tool/warnings.sh @@ -3,17 +3,21 @@ # Run this script in a directory with a working makefile to check for # compiler warnings in SQLite. # rm -f sqlite3.c make sqlite3.c + +# Apple-specific defines used for platform specific feature support +DEFINES="-DSQLITE_OMIT_LOAD_EXTENSION=1 -DSQLITE_ENABLE_LOCKING_STYLE=1 -DSQLITE_ENABLE_AUTO_PROFILE=1 -DUSE_PREAD=1 -DSQLITE_THREADSAFE=2 -DSQLITE_MAX_LENGTH=2147483645 -DSQLITE_MAX_VARIABLE_NUMBER=500000 -DSQLITE_ENABLE_APPLE_SPI=1 -DHAVE_USLEEP=1 -DSQLITE_OMIT_AUTORESET=1 -DSQLITE_ENABLE_PURGEABLE_PCACHE=1 -DSQLITE_DEFAULT_CKPTFULLFSYNC=1 -DSQLITE_DEFAULT_WAL_SAFETYLEVEL=2 -DSQLITE_USE_URI=1 -DSQLITE_WRITE_WALFRAME_PREBUFFERED=1 -DSQLITE_ENABLE_PERSIST_WAL=1 -DUSE_GUARDED_FD=1 -DSQLITE_ENABLE_FLOCKTIMEOUT=1 -DSQLITE_OMIT_BUILTIN_TEST=1 -DSQLITE_DEFAULT_MEMSTATUS=0 -DSQLITE_DEFAULT_PAGE_SIZE=4096 -DSQLITE_DEFAULT_JOURNAL_SIZE_LIMIT=32768 -DSQLITE_DEFAULT_CACHE_SIZE=500 -DSQLITE_ENABLE_RTREE=1 -DSQLITE_ENABLE_FTS3=1 -DSQLITE_ENABLE_FTS3_PARENTHESIS=1" + echo '********** No optimizations. Includes FTS4 and RTREE *********' -gcc -c -Wshadow -Wall -Wextra -pedantic-errors -Wno-long-long -std=c89 \ - -ansi -DHAVE_STDINT_H -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_RTREE \ +gcc -c -Wshadow -Wall -Wextra -Weverything -Wno-long-long -std=c89 \ + -ansi $DEFINES -DHAVE_STDINT_H -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_RTREE \ sqlite3.c echo '********** No optimizations. ENABLE_STAT4. THREADSAFE=0 *******' -gcc -c -Wshadow -Wall -Wextra -pedantic-errors -Wno-long-long -std=c89 \ - -ansi -DSQLITE_ENABLE_STAT4 -DSQLITE_THREADSAFE=0 \ +gcc -c -Wshadow -Wall -Wextra -Weverything -Wno-long-long -std=c89 \ + -ansi $DEFINES -DSQLITE_ENABLE_STAT4 -DSQLITE_THREADSAFE=0 \ sqlite3.c echo '********** Optimized -O3. Includes FTS4 and RTREE ************' -gcc -O3 -c -Wshadow -Wall -Wextra -pedantic-errors -Wno-long-long -std=c89 \ - -ansi -DHAVE_STDINT_H -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_RTREE \ +gcc -O3 -c -Wshadow -Wall -Wextra -Weverything -Wno-long-long -std=c89 \ + -ansi $DEFINES -DHAVE_STDINT_H -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_RTREE \ sqlite3.c