Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Changes In Branch checkpoint-truncate Excluding Merge-Ins
This is equivalent to a diff from 61b31e7714 to 6ee08769f0
2014-12-02
| ||
20:51 | Add support for SQLITE_CHECKPOINT_TRUNCATE. (check-in: edda2b9e7a user: drh tags: trunk) | |
19:35 | When attempting to restart a wal file, make any required calls to sqlite3_randomness() before waiting on or checking for wal file readers. This restores the behaviour exhibited by the trunk. (Closed-Leaf check-in: 6ee08769f0 user: dan tags: checkpoint-truncate) | |
19:04 | Add the SQLITE_CHECKPOINT_TRUNCATE option. (check-in: 8e20a43419 user: dan tags: checkpoint-truncate) | |
16:38 | Merge all recent enhancements and bug fixes from trunk into the apple-osx branch. (check-in: 43c4ba26a2 user: drh tags: apple-osx) | |
16:31 | Merge all recent fixes and enhancements from trunk into sessions. (check-in: 2617d93713 user: drh tags: sessions) | |
16:16 | Convert two unreachable branches into assert() statements. (check-in: 61b31e7714 user: drh tags: trunk) | |
13:46 | Work around overzealous NULL pointer checking in memcpy() and memset() for some systems. (check-in: 0d04f380e1 user: drh tags: trunk) | |
Changes to src/main.c.
︙ | ︙ | |||
1932 1933 1934 1935 1936 1937 1938 | if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; #endif /* Initialize the output variables to -1 in case an error occurs. */ if( pnLog ) *pnLog = -1; if( pnCkpt ) *pnCkpt = -1; | | | | > | | 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 | if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; #endif /* Initialize the output variables to -1 in case an error occurs. */ if( pnLog ) *pnLog = -1; if( pnCkpt ) *pnCkpt = -1; assert( SQLITE_CHECKPOINT_PASSIVE==0 ); assert( SQLITE_CHECKPOINT_FULL==1 ); assert( SQLITE_CHECKPOINT_RESTART==2 ); assert( SQLITE_CHECKPOINT_TRUNCATE==3 ); if( eMode<SQLITE_CHECKPOINT_PASSIVE || eMode>SQLITE_CHECKPOINT_TRUNCATE ){ return SQLITE_MISUSE; } sqlite3_mutex_enter(db->mutex); if( zDb && zDb[0] ){ iDb = sqlite3FindDbName(db, zDb); } |
︙ | ︙ |
Changes to src/pragma.c.
︙ | ︙ | |||
2191 2192 2193 2194 2195 2196 2197 | } } break; #endif /* SQLITE_OMIT_COMPILEOPTION_DIAGS */ #ifndef SQLITE_OMIT_WAL /* | | > > | 2191 2192 2193 2194 2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211 2212 2213 2214 2215 2216 2217 2218 | } } break; #endif /* SQLITE_OMIT_COMPILEOPTION_DIAGS */ #ifndef SQLITE_OMIT_WAL /* ** PRAGMA [database.]wal_checkpoint = passive|full|restart|truncate ** ** Checkpoint the database. */ case PragTyp_WAL_CHECKPOINT: { int iBt = (pId2->z?iDb:SQLITE_MAX_ATTACHED); int eMode = SQLITE_CHECKPOINT_PASSIVE; if( zRight ){ if( sqlite3StrICmp(zRight, "full")==0 ){ eMode = SQLITE_CHECKPOINT_FULL; }else if( sqlite3StrICmp(zRight, "restart")==0 ){ eMode = SQLITE_CHECKPOINT_RESTART; }else if( sqlite3StrICmp(zRight, "truncate")==0 ){ eMode = SQLITE_CHECKPOINT_TRUNCATE; } } sqlite3VdbeSetNumCols(v, 3); pParse->nMem = 3; sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "busy", SQLITE_STATIC); sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "log", SQLITE_STATIC); sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "checkpointed", SQLITE_STATIC); |
︙ | ︙ |
Changes to src/sqlite.h.in.
︙ | ︙ | |||
7287 7288 7289 7290 7291 7292 7293 7294 7295 7296 7297 7298 7299 7300 7301 7302 7303 7304 7305 7306 7307 7308 | ** This mode works the same way as SQLITE_CHECKPOINT_FULL, except after ** checkpointing the log file it blocks (calls the ** [sqlite3_busy_handler|busy-handler callback]) ** until all readers are reading from the database file only. This ensures ** that the next client to write to the database file restarts the log file ** from the beginning. This call blocks database writers while it is running, ** but not database readers. ** </dl> ** ** If pnLog is not NULL, then *pnLog is set to the total number of frames in ** the log file before returning. If pnCkpt is not NULL, then *pnCkpt is set to ** the total number of checkpointed frames (including any that were already ** checkpointed when this function is called). *pnLog and *pnCkpt may be ** populated even if sqlite3_wal_checkpoint_v2() returns other than SQLITE_OK. ** If no values are available because of an error, they are both set to -1 ** before returning to communicate this to the caller. ** ** All calls obtain an exclusive "checkpoint" lock on the database file. If ** any other process is running a checkpoint operation at the same time, the ** lock cannot be obtained and SQLITE_BUSY is returned. Even if there is a ** busy-handler configured, it will not be invoked in this case. ** | > > > > | | | | | | 7287 7288 7289 7290 7291 7292 7293 7294 7295 7296 7297 7298 7299 7300 7301 7302 7303 7304 7305 7306 7307 7308 7309 7310 7311 7312 7313 7314 7315 7316 7317 7318 7319 7320 7321 7322 7323 7324 | ** This mode works the same way as SQLITE_CHECKPOINT_FULL, except after ** checkpointing the log file it blocks (calls the ** [sqlite3_busy_handler|busy-handler callback]) ** until all readers are reading from the database file only. This ensures ** that the next client to write to the database file restarts the log file ** from the beginning. This call blocks database writers while it is running, ** but not database readers. ** ** <dt>SQLITE_CHECKPOINT_TRUNCATE<dd> ** This mode works the same way as SQLITE_CHECKPOINT_RESTART except that, ** if successful, it also truncates the log file to zero bytes in size. ** </dl> ** ** If pnLog is not NULL, then *pnLog is set to the total number of frames in ** the log file before returning. If pnCkpt is not NULL, then *pnCkpt is set to ** the total number of checkpointed frames (including any that were already ** checkpointed when this function is called). *pnLog and *pnCkpt may be ** populated even if sqlite3_wal_checkpoint_v2() returns other than SQLITE_OK. ** If no values are available because of an error, they are both set to -1 ** before returning to communicate this to the caller. ** ** All calls obtain an exclusive "checkpoint" lock on the database file. If ** any other process is running a checkpoint operation at the same time, the ** lock cannot be obtained and SQLITE_BUSY is returned. Even if there is a ** busy-handler configured, it will not be invoked in this case. ** ** The SQLITE_CHECKPOINT_FULL, RESTART and TRUNCATE modes also obtain the ** exclusive "writer" lock on the database file. If the writer lock cannot be ** obtained immediately, and a busy-handler is configured, it is invoked and ** the writer lock retried until either the busy-handler returns 0 or the lock ** is successfully obtained. The busy-handler is also invoked while waiting for ** database readers as described above. If the busy-handler returns 0 before ** the writer lock is obtained or while waiting for database readers, the ** checkpoint operation proceeds from that point in the same way as ** SQLITE_CHECKPOINT_PASSIVE - checkpointing as many frames as possible ** without blocking any further. SQLITE_BUSY is returned in this case. ** ** If parameter zDb is NULL or points to a zero length string, then the |
︙ | ︙ | |||
7345 7346 7347 7348 7349 7350 7351 | ** CAPI3REF: Checkpoint operation parameters ** ** These constants can be used as the 3rd parameter to ** [sqlite3_wal_checkpoint_v2()]. See the [sqlite3_wal_checkpoint_v2()] ** documentation for additional information about the meaning and use of ** each of these values. */ | | | | > | 7349 7350 7351 7352 7353 7354 7355 7356 7357 7358 7359 7360 7361 7362 7363 7364 7365 7366 | ** CAPI3REF: Checkpoint operation parameters ** ** These constants can be used as the 3rd parameter to ** [sqlite3_wal_checkpoint_v2()]. See the [sqlite3_wal_checkpoint_v2()] ** documentation for additional information about the meaning and use of ** each of these values. */ #define SQLITE_CHECKPOINT_PASSIVE 0 #define SQLITE_CHECKPOINT_FULL 1 #define SQLITE_CHECKPOINT_RESTART 2 #define SQLITE_CHECKPOINT_TRUNCATE 3 /* ** CAPI3REF: Virtual Table Interface Configuration ** ** This function may be called by either the [xConnect] or [xCreate] method ** of a [virtual table] implementation to configure ** various facets of the virtual table interface. |
︙ | ︙ |
Changes to src/test1.c.
︙ | ︙ | |||
5686 5687 5688 5689 5690 5691 5692 | int rc; int eMode; int nLog = -555; int nCkpt = -555; Tcl_Obj *pRet; | | > | 5686 5687 5688 5689 5690 5691 5692 5693 5694 5695 5696 5697 5698 5699 5700 5701 5702 5703 5704 | int rc; int eMode; int nLog = -555; int nCkpt = -555; Tcl_Obj *pRet; const char * aMode[] = { "passive", "full", "restart", "truncate", 0 }; assert( SQLITE_CHECKPOINT_PASSIVE==0 ); assert( SQLITE_CHECKPOINT_FULL==1 ); assert( SQLITE_CHECKPOINT_RESTART==2 ); assert( SQLITE_CHECKPOINT_TRUNCATE==3 ); if( objc!=3 && objc!=4 ){ Tcl_WrongNumArgs(interp, 1, objv, "DB MODE ?NAME?"); return TCL_ERROR; } if( objc==4 ){ |
︙ | ︙ |
Changes to src/vdbe.c.
︙ | ︙ | |||
5720 5721 5722 5723 5724 5725 5726 5727 5728 5729 5730 5731 5732 5733 | assert( p->readOnly==0 ); aRes[0] = 0; aRes[1] = aRes[2] = -1; assert( pOp->p2==SQLITE_CHECKPOINT_PASSIVE || pOp->p2==SQLITE_CHECKPOINT_FULL || pOp->p2==SQLITE_CHECKPOINT_RESTART ); rc = sqlite3Checkpoint(db, pOp->p1, pOp->p2, &aRes[1], &aRes[2]); if( rc==SQLITE_BUSY ){ rc = SQLITE_OK; aRes[0] = 1; } for(i=0, pMem = &aMem[pOp->p3]; i<3; i++, pMem++){ | > | 5720 5721 5722 5723 5724 5725 5726 5727 5728 5729 5730 5731 5732 5733 5734 | assert( p->readOnly==0 ); aRes[0] = 0; aRes[1] = aRes[2] = -1; assert( pOp->p2==SQLITE_CHECKPOINT_PASSIVE || pOp->p2==SQLITE_CHECKPOINT_FULL || pOp->p2==SQLITE_CHECKPOINT_RESTART || pOp->p2==SQLITE_CHECKPOINT_TRUNCATE ); rc = sqlite3Checkpoint(db, pOp->p1, pOp->p2, &aRes[1], &aRes[2]); if( rc==SQLITE_BUSY ){ rc = SQLITE_OK; aRes[0] = 1; } for(i=0, pMem = &aMem[pOp->p3]; i<3; i++, pMem++){ |
︙ | ︙ |
Changes to src/wal.c.
︙ | ︙ | |||
1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 | /* ** The cache of the wal-index header must be valid to call this function. ** Return the page-size in bytes used by the database. */ static int walPagesize(Wal *pWal){ return (pWal->hdr.szPage&0xfe00) + ((pWal->hdr.szPage&0x0001)<<16); } /* ** Copy as much content as we can from the WAL back into the database file ** in response to an sqlite3_wal_checkpoint() request or the equivalent. ** ** The amount of information copies from WAL to database might be limited ** by active readers. This routine will never overwrite a database page | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 | /* ** The cache of the wal-index header must be valid to call this function. ** Return the page-size in bytes used by the database. */ static int walPagesize(Wal *pWal){ return (pWal->hdr.szPage&0xfe00) + ((pWal->hdr.szPage&0x0001)<<16); } /* ** The following is guaranteed when this function is called: ** ** a) the WRITER lock is held, ** b) the entire log file has been checkpointed, and ** c) any existing readers are reading exclusively from the database ** file - there are no readers that may attempt to read a frame from ** the log file. ** ** This function updates the shared-memory structures so that the next ** client to write to the database (which may be this one) does so by ** writing frames into the start of the log file. ** ** The value of parameter salt1 is used as the aSalt[1] value in the ** new wal-index header. It should be passed a pseudo-random value (i.e. ** one obtained from sqlite3_randomness()). */ static void walRestartHdr(Wal *pWal, u32 salt1){ volatile WalCkptInfo *pInfo = walCkptInfo(pWal); int i; /* Loop counter */ u32 *aSalt = pWal->hdr.aSalt; /* Big-endian salt values */ pWal->nCkpt++; pWal->hdr.mxFrame = 0; sqlite3Put4byte((u8*)&aSalt[0], 1 + sqlite3Get4byte((u8*)&aSalt[0])); memcpy(&pWal->hdr.aSalt[1], &salt1, 4); walIndexWriteHdr(pWal); pInfo->nBackfill = 0; pInfo->aReadMark[1] = 0; for(i=2; i<WAL_NREADER; i++) pInfo->aReadMark[i] = READMARK_NOT_USED; assert( pInfo->aReadMark[0]==0 ); } /* ** Copy as much content as we can from the WAL back into the database file ** in response to an sqlite3_wal_checkpoint() request or the equivalent. ** ** The amount of information copies from WAL to database might be limited ** by active readers. This routine will never overwrite a database page |
︙ | ︙ | |||
1771 1772 1773 1774 1775 1776 1777 | if( rc==SQLITE_BUSY ){ /* Reset the return code so as not to report a checkpoint failure ** just because there are active readers. */ rc = SQLITE_OK; } | | | | | | > > > > > > > > > > > > > > > > > | 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 | if( rc==SQLITE_BUSY ){ /* Reset the return code so as not to report a checkpoint failure ** just because there are active readers. */ rc = SQLITE_OK; } /* If this is an SQLITE_CHECKPOINT_RESTART or TRUNCATE operation, and the ** entire wal file has been copied into the database file, then block ** until all readers have finished using the wal file. This ensures that ** the next process to write to the database restarts the wal file. */ if( rc==SQLITE_OK && eMode!=SQLITE_CHECKPOINT_PASSIVE ){ assert( pWal->writeLock ); if( pInfo->nBackfill<pWal->hdr.mxFrame ){ rc = SQLITE_BUSY; }else if( eMode>=SQLITE_CHECKPOINT_RESTART ){ u32 salt1; sqlite3_randomness(4, &salt1); assert( mxSafeFrame==pWal->hdr.mxFrame ); rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(1), WAL_NREADER-1); if( rc==SQLITE_OK ){ if( eMode==SQLITE_CHECKPOINT_TRUNCATE ){ /* If this is a TRUNCATE checkpoint, also truncate the wal file ** to zero bytes in size on disk. ** ** In theory, it might be safe to do this without updating the ** wal-index header in shared memory, as all subsequent reader or ** writer clients should see that the entire log file has been ** checkpointed and behave accordingly. This seems unsafe though, ** as it would leave the system in a state where the contents of ** the wal-index header do not match the contents of the ** file-system. To avoid this, update the wal-index header to ** indicate that the log file contains zero valid frames. */ walRestartHdr(pWal, salt1); rc = sqlite3OsTruncate(pWal->pWalFd, 0); } walUnlockExclusive(pWal, WAL_READ_LOCK(1), WAL_NREADER-1); } } } walcheckpoint_out: walIteratorFree(pIter); |
︙ | ︙ | |||
2569 2570 2571 2572 2573 2574 2575 | pWal->hdr.aFrameCksum[1] = aWalData[2]; walCleanupHash(pWal); } return rc; } | < | 2618 2619 2620 2621 2622 2623 2624 2625 2626 2627 2628 2629 2630 2631 | pWal->hdr.aFrameCksum[1] = aWalData[2]; walCleanupHash(pWal); } return rc; } /* ** This function is called just before writing a set of frames to the log ** file (see sqlite3WalFrames()). It checks to see if, instead of appending ** to the current log file, it is possible to overwrite the start of the ** existing log file with the new frames (i.e. "reset" the log). If so, ** it sets pWal->hdr.mxFrame to 0. Otherwise, pWal->hdr.mxFrame is left ** unchanged. |
︙ | ︙ | |||
2602 2603 2604 2605 2606 2607 2608 | ** readers are currently using the WAL), then the transactions ** frames will overwrite the start of the existing log. Update the ** wal-index header to reflect this. ** ** In theory it would be Ok to update the cache of the header only ** at this point. But updating the actual wal-index header is also ** safe and means there is no special case for sqlite3WalUndo() | | < < < | < < < < < < < < < | 2650 2651 2652 2653 2654 2655 2656 2657 2658 2659 2660 2661 2662 2663 2664 2665 | ** readers are currently using the WAL), then the transactions ** frames will overwrite the start of the existing log. Update the ** wal-index header to reflect this. ** ** In theory it would be Ok to update the cache of the header only ** at this point. But updating the actual wal-index header is also ** safe and means there is no special case for sqlite3WalUndo() ** to handle if this transaction is rolled back. */ walRestartHdr(pWal, salt1); walUnlockExclusive(pWal, WAL_READ_LOCK(1), WAL_NREADER-1); }else if( rc!=SQLITE_BUSY ){ return rc; } } walUnlockShared(pWal, WAL_READ_LOCK(0)); pWal->readLock = -1; |
︙ | ︙ |
Changes to test/wal5.test.
︙ | ︙ | |||
51 52 53 54 55 56 57 | proc do_wal_checkpoint { dbhandle args } { set a(-mode) passive array set a $args foreach key [array names a] { if {[lsearch {-mode -db} $key]<0} { error "unknown switch: $key" } } | > | | 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 | proc do_wal_checkpoint { dbhandle args } { set a(-mode) passive array set a $args foreach key [array names a] { if {[lsearch {-mode -db} $key]<0} { error "unknown switch: $key" } } set vals {restart full truncate} if {[lsearch -exact $vals $a(-mode)]<0} { set a(-mode) passive } set cmd [list sqlite3_wal_checkpoint_v2 $dbhandle $a(-mode)] if {[info exists a(-db)]} { lappend sql $a(-db) } uplevel $cmd } } |
︙ | ︙ | |||
274 275 276 277 278 279 280 281 282 283 284 285 286 287 | 6 FULL 3 {0 4 4} 2 7 RESTART - {0 4 4} 3 8 RESTART 1 {1 3 3} 1 9 RESTART 2 {1 4 3} 2 10 RESTART 3 {1 4 4} 3 } { do_multiclient_test tn { setup_and_attach_aux proc busyhandler {x} { set ::max_busyhandler $x if {$::busy_on!="-" && $x==$::busy_on} { return 1 } | > > > > > | 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 | 6 FULL 3 {0 4 4} 2 7 RESTART - {0 4 4} 3 8 RESTART 1 {1 3 3} 1 9 RESTART 2 {1 4 3} 2 10 RESTART 3 {1 4 4} 3 11 TRUNCATE - {0 0 0} 3 12 TRUNCATE 1 {1 3 3} 1 13 TRUNCATE 2 {1 4 3} 2 14 TRUNCATE 3 {1 4 4} 3 } { do_multiclient_test tn { setup_and_attach_aux proc busyhandler {x} { set ::max_busyhandler $x if {$::busy_on!="-" && $x==$::busy_on} { return 1 } |
︙ | ︙ | |||
343 344 345 346 347 348 349 350 351 352 353 354 | code1 {sqlite3 db test.db} code2 {sqlite3 db2 test.db} code3 {sqlite3 db3 test.db} do_test 3.$tn.5 { sql3 { PRAGMA journal_mode } } {wal} do_test 3.$tn.6 { code3 { do_wal_checkpoint db3 } } {0 0 0} } } finish_test | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 | code1 {sqlite3 db test.db} code2 {sqlite3 db2 test.db} code3 {sqlite3 db3 test.db} do_test 3.$tn.5 { sql3 { PRAGMA journal_mode } } {wal} do_test 3.$tn.6 { code3 { do_wal_checkpoint db3 } } {0 0 0} } # Test SQLITE_CHECKPOINT_TRUNCATE. # do_multiclient_test tn { code1 $do_wal_checkpoint code2 $do_wal_checkpoint code3 $do_wal_checkpoint do_test 3.$tn.1 { sql1 { PRAGMA page_size = 1024; PRAGMA journal_mode = WAL; PRAGMA synchronous = normal; CREATE TABLE t1(x, y); CREATE INDEX i1 ON t1(x, y); INSERT INTO t1 VALUES(1, 2); INSERT INTO t1 VALUES(3, 4); } file size test.db-wal } [wal_file_size 8 1024] do_test 3.$tn.2 { do_wal_checkpoint db -mode truncate } {0 0 0} do_test 3.$tn.3 { file size test.db-wal } 0 do_test 3.$tn.4 { sql2 { SELECT * FROM t1 } } {1 2 3 4} do_test 3.$tn.5 { sql2 { INSERT INTO t1 VALUES('a', 'b') } file size test.db-wal } [wal_file_size 2 1024] } } finish_test |