Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Changes In Branch shared-mapping-hack Excluding Merge-Ins
This is equivalent to a diff from 42f0777555 to 2e57335081
2017-09-18
| ||
16:28 | Add extension "mmapwarm.c". Provides function sqlite3_mmap_warm(), used to "warm up" the memory mapping used by SQLite in mmap mode to access db file content. (check-in: d4a30b91f9 user: dan tags: mmap-warm) | |
09:40 | The out-of-bounds read on recovery fix of check-in [378afa16381a222a] caused problems for some corner-case error conditions. This alternative fix appears to work better. (check-in: 74f399d81f user: drh tags: trunk) | |
08:51 | Merge latest trunk changes with this branch. (Leaf check-in: 2e57335081 user: dan tags: shared-mapping-hack) | |
00:18 | Fix the CSV virtual table extension so that it works when the default character is unsigned. (check-in: 42f0777555 user: drh tags: trunk) | |
2017-09-17
| ||
19:45 | Do not make the assumption (as check-in [4da49a95c0f07] incorrectly did) that the ExprList returned by sqlite3ExprListDup() would never be passed into sqlite3ExprListAppend(). Include a new test case that shows this sometimes does happen. (check-in: 29227d00a9 user: drh tags: trunk) | |
2017-09-12
| ||
18:11 | Merge the mutex-free PRNG change into this branch. (check-in: 8b1fc4b9f3 user: dan tags: shared-mapping-hack) | |
Changes to src/func.c.
︙ | |||
477 478 479 480 481 482 483 484 | 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 | + - + | */ static void randomFunc( sqlite3_context *context, int NotUsed, sqlite3_value **NotUsed2 ){ sqlite_int64 r; sqlite3 *db = sqlite3_context_db_handle(context); UNUSED_PARAMETER2(NotUsed, NotUsed2); |
︙ | |||
504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 | 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 | + - + | static void randomBlob( sqlite3_context *context, int argc, sqlite3_value **argv ){ int n; unsigned char *p; sqlite3 *db = sqlite3_context_db_handle(context); assert( argc==1 ); UNUSED_PARAMETER(argc); n = sqlite3_value_int(argv[0]); if( n<1 ){ n = 1; } p = contextMalloc(context, n); if( p ){ |
︙ |
Changes to src/main.c.
︙ | |||
2861 2862 2863 2864 2865 2866 2867 2868 2869 2870 2871 2872 2873 2874 | 2861 2862 2863 2864 2865 2866 2867 2868 2869 2870 2871 2872 2873 2874 2875 | + | } } sqlite3_mutex_enter(db->mutex); db->errMask = 0xff; db->nDb = 2; db->magic = SQLITE_MAGIC_BUSY; db->aDb = db->aDbStatic; sqlite3FastPrngInit(&db->sPrng); assert( sizeof(db->aLimit)==sizeof(aHardLimit) ); memcpy(db->aLimit, aHardLimit, sizeof(db->aLimit)); db->aLimit[SQLITE_LIMIT_WORKER_THREADS] = SQLITE_DEFAULT_WORKER_THREADS; db->autoCommit = 1; db->nextAutovac = -1; db->szMmap = sqlite3GlobalConfig.szMmap; |
︙ |
Changes to src/os_unix.c.
︙ | |||
42 43 44 45 46 47 48 49 50 51 52 53 54 55 | 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 | + + | ** * Locking primitives for the proxy uber-locking-method. (MacOSX only) ** * Definitions of sqlite3_vfs objects for all locking methods ** plus implementations of sqlite3_os_init() and sqlite3_os_end(). */ #include "sqliteInt.h" #if SQLITE_OS_UNIX /* This file is used on unix only */ /* #define SQLITE_SHARED_MAPPING 1 */ /* ** There are various methods for file locking used for concurrency ** control: ** ** 1. POSIX locking (the default), ** 2. No locking, ** 3. Dot-file locking, |
︙ | |||
1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 | 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 | + + + + | #if SQLITE_ENABLE_LOCKING_STYLE unsigned long long sharedByte; /* for AFP simulated shared lock */ #endif #if OS_VXWORKS sem_t *pSem; /* Named POSIX semaphore */ char aSemName[MAX_PATHNAME+2]; /* Name of that semaphore */ #endif #ifdef SQLITE_SHARED_MAPPING sqlite3_int64 nSharedMapping; /* Size of mapped region in bytes */ void *pSharedMapping; /* Memory mapped region */ #endif }; /* ** A lists of all unixInodeInfo objects. */ static unixInodeInfo *inodeList = 0; /* All unixInodeInfo objects */ static unsigned int nUnusedFd = 0; /* Total unused file descriptors */ |
︙ | |||
1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 | 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 | + + + + + + + | static void releaseInodeInfo(unixFile *pFile){ unixInodeInfo *pInode = pFile->pInode; assert( unixMutexHeld() ); if( ALWAYS(pInode) ){ pInode->nRef--; if( pInode->nRef==0 ){ assert( pInode->pShmNode==0 ); #ifdef SQLITE_SHARED_MAPPING if( pInode->pSharedMapping ){ osMunmap(pInode->pSharedMapping, pInode->nSharedMapping); pInode->pSharedMapping = 0; pInode->nSharedMapping = 0; } #endif closePendingFds(pFile); if( pInode->pPrev ){ assert( pInode->pPrev->pNext==pInode ); pInode->pPrev->pNext = pInode->pNext; }else{ assert( inodeList==pInode ); inodeList = pInode->pNext; |
︙ | |||
2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 | 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084 2085 | + + + + + + + + | return SQLITE_OK; } /* ** Close the file. */ static int nolockClose(sqlite3_file *id) { #ifdef SQLITE_SHARED_MAPPING unixFile *pFd = (unixFile*)id; if( pFd->pInode ){ unixEnterMutex(); releaseInodeInfo(pFd); unixLeaveMutex(); } #endif return closeUnixFile(id); } /******************* End of the no-op lock implementation ********************* ******************************************************************************/ /****************************************************************************** |
︙ | |||
3870 3871 3872 3873 3874 3875 3876 3877 3878 3879 3880 3881 3882 3883 | 3891 3892 3893 3894 3895 3896 3897 3898 3899 3900 3901 3902 3903 3904 3905 3906 3907 | + + + | if( newLimit>0 && sizeof(size_t)<8 ){ newLimit = (newLimit & 0x7FFFFFFF); } *(i64*)pArg = pFile->mmapSizeMax; if( newLimit>=0 && newLimit!=pFile->mmapSizeMax && pFile->nFetchOut==0 ){ pFile->mmapSizeMax = newLimit; #ifdef SQLITE_SHARED_MAPPING if( pFile->pInode==0 ) #endif if( pFile->mmapSize>0 ){ unixUnmapfile(pFile); rc = unixMapfile(pFile, -1); } } return rc; } |
︙ | |||
4770 4771 4772 4773 4774 4775 4776 4777 4778 4779 4780 4781 4782 4783 | 4794 4795 4796 4797 4798 4799 4800 4801 4802 4803 4804 4805 4806 4807 4808 4809 4810 | + + + | #if SQLITE_MAX_MMAP_SIZE>0 /* ** If it is currently memory mapped, unmap file pFd. */ static void unixUnmapfile(unixFile *pFd){ assert( pFd->nFetchOut==0 ); #ifdef SQLITE_SHARED_MAPPING if( pFd->pInode ) return; #endif if( pFd->pMapRegion ){ osMunmap(pFd->pMapRegion, pFd->mmapSizeActual); pFd->pMapRegion = 0; pFd->mmapSize = 0; pFd->mmapSizeActual = 0; } } |
︙ | |||
4900 4901 4902 4903 4904 4905 4906 4907 4908 4909 4910 4911 4912 4913 | 4927 4928 4929 4930 4931 4932 4933 4934 4935 4936 4937 4938 4939 4940 4941 4942 4943 4944 4945 4946 4947 4948 4949 4950 4951 4952 4953 4954 4955 4956 4957 4958 4959 4960 4961 4962 | + + + + + + + + + + + + + + + + + + + + + + | return SQLITE_IOERR_FSTAT; } nMap = statbuf.st_size; } if( nMap>pFd->mmapSizeMax ){ nMap = pFd->mmapSizeMax; } #ifdef SQLITE_SHARED_MAPPING if( pFd->pInode ){ unixInodeInfo *pInode = pFd->pInode; if( pFd->pMapRegion ) return SQLITE_OK; unixEnterMutex(); if( pInode->pSharedMapping==0 ){ u8 *pNew = osMmap(0, nMap, PROT_READ, MAP_SHARED, pFd->h, 0); if( pNew==MAP_FAILED ){ unixLogError(SQLITE_OK, "mmap", pFd->zPath); pFd->mmapSizeMax = 0; }else{ pInode->pSharedMapping = pNew; pInode->nSharedMapping = nMap; } } pFd->pMapRegion = pInode->pSharedMapping; pFd->mmapSizeActual = pFd->mmapSize = pInode->nSharedMapping; unixLeaveMutex(); return SQLITE_OK; } #endif assert( nMap>0 || (pFd->mmapSize==0 && pFd->pMapRegion==0) ); if( nMap!=pFd->mmapSize ){ unixRemapfile(pFd, nMap); } return SQLITE_OK; |
︙ | |||
5339 5340 5341 5342 5343 5344 5345 5346 5347 5348 5349 5350 5351 5352 | 5388 5389 5390 5391 5392 5393 5394 5395 5396 5397 5398 5399 5400 5401 5402 5403 5404 | + + + | #endif } if( pLockingStyle == &posixIoMethods #if defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE || pLockingStyle == &nfsIoMethods #endif #ifdef SQLITE_SHARED_MAPPING || pLockingStyle == &nolockIoMethods #endif ){ unixEnterMutex(); rc = findInodeInfo(pNew, &pNew->pInode); if( rc!=SQLITE_OK ){ /* If an error occurred in findInodeInfo(), close the file descriptor ** immediately, before releasing the mutex. findInodeInfo() may fail ** in two scenarios: |
︙ |
Changes to src/random.c.
︙ | |||
101 102 103 104 105 106 107 108 109 110 111 112 113 114 | 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 | + + + + + + + + + + + + + + + + + + + + + + | wsdPrng.s[wsdPrng.i] = wsdPrng.s[wsdPrng.j]; wsdPrng.s[wsdPrng.j] = t; t += wsdPrng.s[wsdPrng.i]; *(zBuf++) = wsdPrng.s[t]; }while( --N ); sqlite3_mutex_leave(mutex); } /* ** Initialize a fast PRNG. A Fast PRNG is called "fast" because it does ** not need a mutex to operate, though it does use a mutex to initialize. ** The quality of the randomness is not as good as the global PRNG. */ void sqlite3FastPrngInit(FastPrng *pPrng){ sqlite3_randomness(sizeof(*pPrng), pPrng); pPrng->x |= 1; } /* ** Generate N bytes of pseudo-randomness using a FastPrng */ void sqlite3FastRandomness(FastPrng *pPrng, int N, void *P){ unsigned char *pOut = (unsigned char*)P; while( N-->0 ){ pPrng->x = ((pPrng->x)>>1) ^ ((1+~((pPrng->x)&1)) & 0xd0000001); pPrng->y = (pPrng->y)*1103515245 + 12345; *(pOut++) = (pPrng->x ^ pPrng->y) & 0xff; } } #ifndef SQLITE_UNTESTABLE /* ** For testing purposes, we sometimes want to preserve the state of ** PRNG and restore the PRNG to its saved state at a later time, or ** to reset the PRNG to its initial state. These routines accomplish ** those tasks. |
︙ |
Changes to src/select.c.
︙ | |||
1740 1741 1742 1743 1744 1745 1746 | 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 | - + | while( zName && sqlite3HashFind(&ht, zName)!=0 ){ nName = sqlite3Strlen30(zName); if( nName>0 ){ for(j=nName-1; j>0 && sqlite3Isdigit(zName[j]); j--){} if( zName[j]==':' ) nName = j; } zName = sqlite3MPrintf(db, "%.*z:%u", nName, zName, ++cnt); |
︙ |
Changes to src/sqliteInt.h.
︙ | |||
1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 | 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 | + | typedef struct Lookaside Lookaside; typedef struct LookasideSlot LookasideSlot; typedef struct Module Module; typedef struct NameContext NameContext; typedef struct Parse Parse; typedef struct PreUpdate PreUpdate; typedef struct PrintfArguments PrintfArguments; typedef struct FastPrng FastPrng; typedef struct RowSet RowSet; typedef struct Savepoint Savepoint; typedef struct Select Select; typedef struct SQLiteThread SQLiteThread; typedef struct SelectDest SelectDest; typedef struct SrcList SrcList; typedef struct StrAccum StrAccum; |
︙ | |||
1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 | 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 | + + + + + + + + | */ #ifndef SQLITE_DEFAULT_SYNCHRONOUS # define SQLITE_DEFAULT_SYNCHRONOUS 2 #endif #ifndef SQLITE_DEFAULT_WAL_SYNCHRONOUS # define SQLITE_DEFAULT_WAL_SYNCHRONOUS SQLITE_DEFAULT_SYNCHRONOUS #endif /* ** State of a simple PRNG used for the per-connection and per-pager ** pseudo-random number generators. */ struct FastPrng { unsigned int x, y; }; /* ** Each database file to be accessed by the system is an instance ** of the following structure. There are normally two of these structures ** in the sqlite.aDb[] array. aDb[0] is the main database file and ** aDb[1] is the database file used to hold temporary tables. Additional ** databases may be attached. |
︙ | |||
1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 | 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 | + | u8 vtabOnConflict; /* Value to return for s3_vtab_on_conflict() */ u8 isTransactionSavepoint; /* True if the outermost savepoint is a TS */ u8 mTrace; /* zero or more SQLITE_TRACE flags */ u8 skipBtreeMutex; /* True if no shared-cache backends */ u8 nSqlExec; /* Number of pending OP_SqlExec opcodes */ int nextPagesize; /* Pagesize after VACUUM if >0 */ u32 magic; /* Magic number for detect library misuse */ FastPrng sPrng; /* State of the per-connection PRNG */ int nChange; /* Value returned by sqlite3_changes() */ int nTotalChange; /* Value returned by sqlite3_total_changes() */ int aLimit[SQLITE_N_LIMIT]; /* Limits */ int nMaxSorterMmap; /* Maximum size of regions mapped by sorter */ struct sqlite3InitInfo { /* Information used during initialization */ int newTnum; /* Rootpage of table being initialized */ u8 iDb; /* Which db file is being initialized */ |
︙ | |||
3808 3809 3810 3811 3812 3813 3814 3815 3816 3817 3818 3819 3820 3821 | 3818 3819 3820 3821 3822 3823 3824 3825 3826 3827 3828 3829 3830 3831 3832 3833 | + + | int sqlite3ExprCoveredByIndex(Expr*, int iCur, Index *pIdx); int sqlite3FunctionUsesThisSrc(Expr*, SrcList*); Vdbe *sqlite3GetVdbe(Parse*); #ifndef SQLITE_UNTESTABLE void sqlite3PrngSaveState(void); void sqlite3PrngRestoreState(void); #endif void sqlite3FastPrngInit(FastPrng*); void sqlite3FastRandomness(FastPrng*, int N, void *P); void sqlite3RollbackAll(sqlite3*,int); void sqlite3CodeVerifySchema(Parse*, int); void sqlite3CodeVerifyNamedSchema(Parse*, const char *zDb); void sqlite3BeginTransaction(Parse*, int); void sqlite3EndTransaction(Parse*,int); void sqlite3Savepoint(Parse*, int, Token*); void sqlite3CloseSavepoints(sqlite3 *); |
︙ |
Changes to src/wal.c.
︙ | |||
444 445 446 447 448 449 450 451 452 453 454 455 456 457 | 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 | + | u8 syncHeader; /* Fsync the WAL header if true */ u8 padToSectorBoundary; /* Pad transactions out to the next sector */ WalIndexHdr hdr; /* Wal-index header for current transaction */ u32 minFrame; /* Ignore wal frames before this one */ u32 iReCksum; /* On commit, recalculate checksums from here */ const char *zWalName; /* Name of WAL file */ u32 nCkpt; /* Checkpoint sequence counter in the wal-header */ FastPrng sPrng; /* Random number generator */ #ifdef SQLITE_DEBUG u8 lockError; /* True if a locking error has occurred */ #endif #ifdef SQLITE_ENABLE_SNAPSHOT WalIndexHdr *pSnapshot; /* Start transaction here if not NULL */ #endif }; |
︙ | |||
1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 | 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 | + | pRet->pDbFd = pDbFd; pRet->readLock = -1; pRet->mxWalSize = mxWalSize; pRet->zWalName = zWalName; pRet->syncHeader = 1; pRet->padToSectorBoundary = 1; pRet->exclusiveMode = (bNoShm ? WAL_HEAPMEMORY_MODE: WAL_NORMAL_MODE); sqlite3FastPrngInit(&pRet->sPrng); /* Open file handle on the write-ahead log file. */ flags = (SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|SQLITE_OPEN_WAL); rc = sqlite3OsOpen(pVfs, zWalName, pRet->pWalFd, flags, &flags); if( rc==SQLITE_OK && flags&SQLITE_OPEN_READONLY ){ pRet->readOnly = WAL_RDONLY; } |
︙ | |||
1866 1867 1868 1869 1870 1871 1872 | 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 | - + | */ if( rc==SQLITE_OK && eMode!=SQLITE_CHECKPOINT_PASSIVE ){ assert( pWal->writeLock ); if( pInfo->nBackfill<pWal->hdr.mxFrame ){ rc = SQLITE_BUSY; }else if( eMode>=SQLITE_CHECKPOINT_RESTART ){ u32 salt1; |
︙ | |||
2874 2875 2876 2877 2878 2879 2880 | 2876 2877 2878 2879 2880 2881 2882 2883 2884 2885 2886 2887 2888 2889 2890 | - + | int cnt; if( pWal->readLock==0 ){ volatile WalCkptInfo *pInfo = walCkptInfo(pWal); assert( pInfo->nBackfill==pWal->hdr.mxFrame ); if( pInfo->nBackfill>0 ){ u32 salt1; |
︙ | |||
3090 3091 3092 3093 3094 3095 3096 | 3092 3093 3094 3095 3096 3097 3098 3099 3100 3101 3102 3103 3104 3105 3106 | - + | u8 aWalHdr[WAL_HDRSIZE]; /* Buffer to assemble wal-header in */ u32 aCksum[2]; /* Checksum for wal-header */ sqlite3Put4byte(&aWalHdr[0], (WAL_MAGIC | SQLITE_BIGENDIAN)); sqlite3Put4byte(&aWalHdr[4], WAL_MAX_VERSION); sqlite3Put4byte(&aWalHdr[8], szPage); sqlite3Put4byte(&aWalHdr[12], pWal->nCkpt); |
︙ |