Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Changes In Branch zipvfs-multifile-commit Excluding Merge-Ins
This is equivalent to a diff from 5e239ecda0 to 9ff4dfe1e3
2013-12-11
| ||
15:47 | Add the SQLITE_FCNTL_SYNC and SQLITE_FCNTL_COMMIT_PHASETWO file-controls and have the pager call them at appropriate times. This is needed in order to enable ZIPVFS to do multi-file atomic commits. (check-in: 552f94d50f user: drh tags: trunk) | |
14:17 | Merge in the latest trunk changes. (Closed-Leaf check-in: 9ff4dfe1e3 user: drh tags: zipvfs-multifile-commit) | |
14:00 | Fix a bug in the shell ".import" command: Do not end the field when an escaped double-quote occurs at the end of a CRNL line. (check-in: 5e239ecda0 user: drh tags: trunk) | |
12:02 | Remove an unreachable conditional inserted by the previous check-in. (check-in: 3e1d55f0bd user: drh tags: trunk) | |
11:03 | Have the crash-test infrastructure code in test6.c read and write the 512-byte block containing the pending-byte lock as normal. Earlier versions did not access this part of the file in order to avoid triggering an assert in os_unix.c. But the assert() has since been removed. And not reading/writing this part of the file causes problems for multiplexor tests. (check-in: fdc3f3fa3d user: dan tags: zipvfs-multifile-commit) | |
Changes to src/backup.c.
︙ | ︙ | |||
522 523 524 525 526 527 528 | } if( rc==SQLITE_OK ){ rc = backupTruncateFile(pFile, iSize); } /* Sync the database file to disk. */ if( rc==SQLITE_OK ){ | | | 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 | } if( rc==SQLITE_OK ){ rc = backupTruncateFile(pFile, iSize); } /* Sync the database file to disk. */ if( rc==SQLITE_OK ){ rc = sqlite3PagerSync(pDestPager, 0); } }else{ sqlite3PagerTruncateImage(pDestPager, nDestTruncate); rc = sqlite3PagerCommitPhaseOne(pDestPager, 0, 0); } /* Finish committing the transaction to the destination database. */ |
︙ | ︙ |
Changes to src/pager.c.
︙ | ︙ | |||
1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 | unsigned char aMagic[8]; /* A buffer to hold the magic header */ zMaster[0] = '\0'; if( SQLITE_OK!=(rc = sqlite3OsFileSize(pJrnl, &szJ)) || szJ<16 || SQLITE_OK!=(rc = read32bits(pJrnl, szJ-16, &len)) || len>=nMaster || SQLITE_OK!=(rc = read32bits(pJrnl, szJ-12, &cksum)) || SQLITE_OK!=(rc = sqlite3OsRead(pJrnl, aMagic, 8, szJ-8)) || memcmp(aMagic, aJournalMagic, 8) || SQLITE_OK!=(rc = sqlite3OsRead(pJrnl, zMaster, len, szJ-16-len)) ){ return rc; } | > | 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 | unsigned char aMagic[8]; /* A buffer to hold the magic header */ zMaster[0] = '\0'; if( SQLITE_OK!=(rc = sqlite3OsFileSize(pJrnl, &szJ)) || szJ<16 || SQLITE_OK!=(rc = read32bits(pJrnl, szJ-16, &len)) || len>=nMaster || len==0 || SQLITE_OK!=(rc = read32bits(pJrnl, szJ-12, &cksum)) || SQLITE_OK!=(rc = sqlite3OsRead(pJrnl, aMagic, 8, szJ-8)) || memcmp(aMagic, aJournalMagic, 8) || SQLITE_OK!=(rc = sqlite3OsRead(pJrnl, zMaster, len, szJ-16-len)) ){ return rc; } |
︙ | ︙ | |||
2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 | ** At this point the journal has been finalized and the transaction ** successfully committed, but the EXCLUSIVE lock is still held on the ** file. So it is safe to truncate the database file to its minimum ** required size. */ assert( pPager->eLock==EXCLUSIVE_LOCK ); rc = pager_truncate(pPager, pPager->dbSize); } if( !pPager->exclusiveMode && (!pagerUseWal(pPager) || sqlite3WalExclusiveMode(pPager->pWal, 0)) ){ rc2 = pagerUnlockDb(pPager, SHARED_LOCK); pPager->changeCountDone = 0; } | > > > > > | 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 | ** At this point the journal has been finalized and the transaction ** successfully committed, but the EXCLUSIVE lock is still held on the ** file. So it is safe to truncate the database file to its minimum ** required size. */ assert( pPager->eLock==EXCLUSIVE_LOCK ); rc = pager_truncate(pPager, pPager->dbSize); } if( rc==SQLITE_OK && bCommit && isOpen(pPager->fd) ){ rc = sqlite3OsFileControl(pPager->fd, SQLITE_FCNTL_COMMIT_PHASETWO, 0); if( rc==SQLITE_NOTFOUND ) rc = SQLITE_OK; } if( !pPager->exclusiveMode && (!pagerUseWal(pPager) || sqlite3WalExclusiveMode(pPager->pWal, 0)) ){ rc2 = pagerUnlockDb(pPager, SHARED_LOCK); pPager->changeCountDone = 0; } |
︙ | ︙ | |||
2827 2828 2829 2830 2831 2832 2833 | zMaster = pPager->pTmpSpace; rc = readMasterJournal(pPager->jfd, zMaster, pPager->pVfs->mxPathname+1); testcase( rc!=SQLITE_OK ); } if( rc==SQLITE_OK && (pPager->eState>=PAGER_WRITER_DBMOD || pPager->eState==PAGER_OPEN) ){ | | | 2833 2834 2835 2836 2837 2838 2839 2840 2841 2842 2843 2844 2845 2846 2847 | zMaster = pPager->pTmpSpace; rc = readMasterJournal(pPager->jfd, zMaster, pPager->pVfs->mxPathname+1); testcase( rc!=SQLITE_OK ); } if( rc==SQLITE_OK && (pPager->eState>=PAGER_WRITER_DBMOD || pPager->eState==PAGER_OPEN) ){ rc = sqlite3PagerSync(pPager, 0); } if( rc==SQLITE_OK ){ rc = pager_end_transaction(pPager, zMaster[0]!='\0', 0); testcase( rc!=SQLITE_OK ); } if( rc==SQLITE_OK && zMaster[0] && res ){ /* If there was a master journal and this routine will return success, |
︙ | ︙ | |||
5996 5997 5998 5999 6000 6001 6002 | /* ** Sync the database file to disk. This is a no-op for in-memory databases ** or pages with the Pager.noSync flag set. ** ** If successful, or if called on a pager for which it is a no-op, this ** function returns SQLITE_OK. Otherwise, an IO error code is returned. */ | | | < < | < > | | < | > > > | 6002 6003 6004 6005 6006 6007 6008 6009 6010 6011 6012 6013 6014 6015 6016 6017 6018 6019 6020 6021 6022 6023 6024 6025 6026 | /* ** Sync the database file to disk. This is a no-op for in-memory databases ** or pages with the Pager.noSync flag set. ** ** If successful, or if called on a pager for which it is a no-op, this ** function returns SQLITE_OK. Otherwise, an IO error code is returned. */ int sqlite3PagerSync(Pager *pPager, const char *zMaster){ int rc = SQLITE_OK; if( isOpen(pPager->fd) ){ void *pArg = (void*)zMaster; rc = sqlite3OsFileControl(pPager->fd, SQLITE_FCNTL_SYNC, pArg); if( rc==SQLITE_NOTFOUND ) rc = SQLITE_OK; } if( rc==SQLITE_OK && !pPager->noSync ){ assert( !MEMDB ); rc = sqlite3OsSync(pPager->fd, pPager->syncFlags); } return rc; } /* ** This function may only be called while a write-transaction is active in ** rollback. If the connection is in WAL mode, this call is a no-op. |
︙ | ︙ | |||
6205 6206 6207 6208 6209 6210 6211 | assert( pPager->eState==PAGER_WRITER_DBMOD ); rc = pager_truncate(pPager, nNew); if( rc!=SQLITE_OK ) goto commit_phase_one_exit; } /* Finally, sync the database file. */ if( !noSync ){ | | | 6211 6212 6213 6214 6215 6216 6217 6218 6219 6220 6221 6222 6223 6224 6225 | assert( pPager->eState==PAGER_WRITER_DBMOD ); rc = pager_truncate(pPager, nNew); if( rc!=SQLITE_OK ) goto commit_phase_one_exit; } /* Finally, sync the database file. */ if( !noSync ){ rc = sqlite3PagerSync(pPager, zMaster); } IOTRACE(("DBSYNC %p\n", pPager)) } } commit_phase_one_exit: if( rc==SQLITE_OK && !pagerUseWal(pPager) ){ |
︙ | ︙ | |||
6334 6335 6336 6337 6338 6339 6340 | } }else{ rc = pager_playback(pPager, 0); } assert( pPager->eState==PAGER_READER || rc!=SQLITE_OK ); assert( rc==SQLITE_OK || rc==SQLITE_FULL || rc==SQLITE_CORRUPT | | > > | 6340 6341 6342 6343 6344 6345 6346 6347 6348 6349 6350 6351 6352 6353 6354 6355 6356 | } }else{ rc = pager_playback(pPager, 0); } assert( pPager->eState==PAGER_READER || rc!=SQLITE_OK ); assert( rc==SQLITE_OK || rc==SQLITE_FULL || rc==SQLITE_CORRUPT || rc==SQLITE_NOMEM || (rc&0xFF)==SQLITE_IOERR || rc==SQLITE_CANTOPEN ); /* If an error occurs during a ROLLBACK, we can no longer trust the pager ** cache. So call pager_error() on the way out to make any error persistent. */ return pager_error(pPager, rc); } |
︙ | ︙ |
Changes to src/pager.h.
︙ | ︙ | |||
146 147 148 149 150 151 152 | void *sqlite3PagerGetExtra(DbPage *); /* Functions used to manage pager transactions and savepoints. */ void sqlite3PagerPagecount(Pager*, int*); int sqlite3PagerBegin(Pager*, int exFlag, int); int sqlite3PagerCommitPhaseOne(Pager*,const char *zMaster, int); int sqlite3PagerExclusiveLock(Pager*); | | | 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 | void *sqlite3PagerGetExtra(DbPage *); /* Functions used to manage pager transactions and savepoints. */ void sqlite3PagerPagecount(Pager*, int*); int sqlite3PagerBegin(Pager*, int exFlag, int); int sqlite3PagerCommitPhaseOne(Pager*,const char *zMaster, int); int sqlite3PagerExclusiveLock(Pager*); int sqlite3PagerSync(Pager *pPager, const char *zMaster); int sqlite3PagerCommitPhaseTwo(Pager*); int sqlite3PagerRollback(Pager*); int sqlite3PagerOpenSavepoint(Pager *pPager, int n); int sqlite3PagerSavepoint(Pager *pPager, int op, int iSavepoint); int sqlite3PagerSharedLock(Pager *pPager); #ifndef SQLITE_OMIT_WAL |
︙ | ︙ |
Changes to src/sqlite.h.in.
︙ | ︙ | |||
782 783 784 785 786 787 788 | ** <li>[[SQLITE_FCNTL_FILE_POINTER]] ** The [SQLITE_FCNTL_FILE_POINTER] opcode is used to obtain a pointer ** to the [sqlite3_file] object associated with a particular database ** connection. See the [sqlite3_file_control()] documentation for ** additional information. ** ** <li>[[SQLITE_FCNTL_SYNC_OMITTED]] | > > > | | | < > | > > > > | | | > > > > > > > | | 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 | ** <li>[[SQLITE_FCNTL_FILE_POINTER]] ** The [SQLITE_FCNTL_FILE_POINTER] opcode is used to obtain a pointer ** to the [sqlite3_file] object associated with a particular database ** connection. See the [sqlite3_file_control()] documentation for ** additional information. ** ** <li>[[SQLITE_FCNTL_SYNC_OMITTED]] ** No longer in use. ** ** <li>[[SQLITE_FCNTL_SYNC]] ** The [SQLITE_FCNTL_SYNC] opcode is generated internally by SQLite and ** sent to the VFS immediately before the xSync method is invoked on a ** database file descriptor. Or, if the xSync method is not invoked ** because the user has configured SQLite with ** [PRAGMA synchronous | PRAGMA synchronous=OFF] it is invoked in place ** of the xSync method. In most cases, the pointer argument passed with ** this file-control is NULL. However, if the database file is being synced ** as part of a multi-database commit, the argument points to a nul-terminated ** string containing the transactions master-journal file name. VFSes that ** do not need this signal should silently ignore this opcode. Applications ** should not call [sqlite3_file_control()] with this opcode as doing so may ** disrupt the operation of the specialized VFSes that do require it. ** ** <li>[[SQLITE_FCNTL_COMMIT_PHASETWO]] ** The [SQLITE_FCNTL_COMMIT_PHASETWO] opcode is generated internally by SQLite ** and sent to the VFS after a transaction has been committed immediately ** but before the database is unlocked. VFSes that do not need this signal ** should silently ignore this opcode. Applications should not call ** [sqlite3_file_control()] with this opcode as doing so may disrupt the ** operation of the specialized VFSes that do require it. ** ** <li>[[SQLITE_FCNTL_WIN32_AV_RETRY]] ** ^The [SQLITE_FCNTL_WIN32_AV_RETRY] opcode is used to configure automatic ** retry counts and intervals for certain disk I/O operations for the ** windows [VFS] in order to provide robustness in the presence of ** anti-virus programs. By default, the windows VFS will retry file read, ** file write, and file delete operations up to 10 times, with a delay |
︙ | ︙ | |||
941 942 943 944 945 946 947 948 949 950 951 952 953 954 | #define SQLITE_FCNTL_POWERSAFE_OVERWRITE 13 #define SQLITE_FCNTL_PRAGMA 14 #define SQLITE_FCNTL_BUSYHANDLER 15 #define SQLITE_FCNTL_TEMPFILENAME 16 #define SQLITE_FCNTL_MMAP_SIZE 18 #define SQLITE_FCNTL_TRACE 19 #define SQLITE_FCNTL_HAS_MOVED 20 /* ** CAPI3REF: Mutex Handle ** ** The mutex module within SQLite defines [sqlite3_mutex] to be an ** abstract type for a mutex object. The SQLite core never looks ** at the internal representation of an [sqlite3_mutex]. It only | > > | 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 | #define SQLITE_FCNTL_POWERSAFE_OVERWRITE 13 #define SQLITE_FCNTL_PRAGMA 14 #define SQLITE_FCNTL_BUSYHANDLER 15 #define SQLITE_FCNTL_TEMPFILENAME 16 #define SQLITE_FCNTL_MMAP_SIZE 18 #define SQLITE_FCNTL_TRACE 19 #define SQLITE_FCNTL_HAS_MOVED 20 #define SQLITE_FCNTL_SYNC 21 #define SQLITE_FCNTL_COMMIT_PHASETWO 22 /* ** CAPI3REF: Mutex Handle ** ** The mutex module within SQLite defines [sqlite3_mutex] to be an ** abstract type for a mutex object. The SQLite core never looks ** at the internal representation of an [sqlite3_mutex]. It only |
︙ | ︙ |
Changes to src/test6.c.
︙ | ︙ | |||
169 170 171 172 173 174 175 | /* ** Wrapper around the sqlite3OsWrite() function that avoids writing to the ** 512 byte block begining at offset PENDING_BYTE. */ static int writeDbFile(CrashFile *p, u8 *z, i64 iAmt, i64 iOff){ int rc = SQLITE_OK; int iSkip = 0; | < < < | 169 170 171 172 173 174 175 176 177 178 179 180 181 182 | /* ** Wrapper around the sqlite3OsWrite() function that avoids writing to the ** 512 byte block begining at offset PENDING_BYTE. */ static int writeDbFile(CrashFile *p, u8 *z, i64 iAmt, i64 iOff){ int rc = SQLITE_OK; int iSkip = 0; if( (iAmt-iSkip)>0 ){ rc = sqlite3OsWrite(p->pRealFile, &z[iSkip], (int)(iAmt-iSkip), iOff+iSkip); } return rc; } /* |
︙ | ︙ | |||
405 406 407 408 409 410 411 412 413 | static int cfRead( sqlite3_file *pFile, void *zBuf, int iAmt, sqlite_int64 iOfst ){ CrashFile *pCrash = (CrashFile *)pFile; /* Check the file-size to see if this is a short-read */ | > > > > > | < | 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 | static int cfRead( sqlite3_file *pFile, void *zBuf, int iAmt, sqlite_int64 iOfst ){ CrashFile *pCrash = (CrashFile *)pFile; int nCopy = (int)MIN((i64)iAmt, (pCrash->iSize - iOfst)); if( nCopy>0 ){ memcpy(zBuf, &pCrash->zData[iOfst], nCopy); } /* Check the file-size to see if this is a short-read */ if( nCopy<iAmt ){ return SQLITE_IOERR_SHORT_READ; } return SQLITE_OK; } /* ** Write data to a crash-file. */ static int cfWrite( |
︙ | ︙ | |||
635 636 637 638 639 640 641 | const int isDb = (flags&SQLITE_OPEN_MAIN_DB); i64 iOff; memset(pWrapper->zData, 0, pWrapper->nData); for(iOff=0; iOff<pWrapper->iSize; iOff += 512){ int nRead = (int)(pWrapper->iSize - iOff); if( nRead>512 ) nRead = 512; | < | 636 637 638 639 640 641 642 643 644 645 646 647 648 649 | const int isDb = (flags&SQLITE_OPEN_MAIN_DB); i64 iOff; memset(pWrapper->zData, 0, pWrapper->nData); for(iOff=0; iOff<pWrapper->iSize; iOff += 512){ int nRead = (int)(pWrapper->iSize - iOff); if( nRead>512 ) nRead = 512; rc = sqlite3OsRead(pReal, &pWrapper->zData[iOff], nRead, iOff); } }else{ rc = SQLITE_NOMEM; } } if( rc!=SQLITE_OK && pWrapper->pMethod ){ |
︙ | ︙ |
Changes to test/tester.tcl.
︙ | ︙ | |||
1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 | # crashsql -delay CRASHDELAY -file CRASHFILE ?-blocksize BLOCKSIZE? $sql # proc crashsql {args} { set blocksize "" set crashdelay 1 set prngseed 0 set tclbody {} set crashfile "" set dc "" set sql [lindex $args end] for {set ii 0} {$ii < [llength $args]-1} {incr ii 2} { set z [lindex $args $ii] set n [string length $z] set z2 [lindex $args [expr $ii+1]] if {$n>1 && [string first $z -delay]==0} {set crashdelay $z2} \ elseif {$n>1 && [string first $z -seed]==0} {set prngseed $z2} \ elseif {$n>1 && [string first $z -file]==0} {set crashfile $z2} \ elseif {$n>1 && [string first $z -tclbody]==0} {set tclbody $z2} \ elseif {$n>1 && [string first $z -blocksize]==0} {set blocksize "-s $z2" } \ elseif {$n>1 && [string first $z -characteristics]==0} {set dc "-c {$z2}" } \ else { error "Unrecognized option: $z" } } | > > | 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 | # crashsql -delay CRASHDELAY -file CRASHFILE ?-blocksize BLOCKSIZE? $sql # proc crashsql {args} { set blocksize "" set crashdelay 1 set prngseed 0 set opendb { sqlite3 db test.db -vfs crash } set tclbody {} set crashfile "" set dc "" set sql [lindex $args end] for {set ii 0} {$ii < [llength $args]-1} {incr ii 2} { set z [lindex $args $ii] set n [string length $z] set z2 [lindex $args [expr $ii+1]] if {$n>1 && [string first $z -delay]==0} {set crashdelay $z2} \ elseif {$n>1 && [string first $z -opendb]==0} {set opendb $z2} \ elseif {$n>1 && [string first $z -seed]==0} {set prngseed $z2} \ elseif {$n>1 && [string first $z -file]==0} {set crashfile $z2} \ elseif {$n>1 && [string first $z -tclbody]==0} {set tclbody $z2} \ elseif {$n>1 && [string first $z -blocksize]==0} {set blocksize "-s $z2" } \ elseif {$n>1 && [string first $z -characteristics]==0} {set dc "-c {$z2}" } \ else { error "Unrecognized option: $z" } } |
︙ | ︙ | |||
1279 1280 1281 1282 1283 1284 1285 | # default, so here we force it to the "nativename" format. set cfile [string map {\\ \\\\} [file nativename [file join [get_pwd] $crashfile]]] set f [open crash.tcl w] puts $f "sqlite3_crash_enable 1" puts $f "sqlite3_crashparams $blocksize $dc $crashdelay $cfile" puts $f "sqlite3_test_control_pending_byte $::sqlite_pending_byte" | | > | 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 | # default, so here we force it to the "nativename" format. set cfile [string map {\\ \\\\} [file nativename [file join [get_pwd] $crashfile]]] set f [open crash.tcl w] puts $f "sqlite3_crash_enable 1" puts $f "sqlite3_crashparams $blocksize $dc $crashdelay $cfile" puts $f "sqlite3_test_control_pending_byte $::sqlite_pending_byte" puts $f $opendb # This block sets the cache size of the main database to 10 # pages. This is done in case the build is configured to omit # "PRAGMA cache_size". puts $f {db eval {SELECT * FROM sqlite_master;}} puts $f {set bt [btree_from_db db]} puts $f {btree_set_cache_size $bt 10} if {$prngseed} { set seed [expr {$prngseed%10007+1}] # puts seed=$seed puts $f "db eval {SELECT randomblob($seed)}" } if {[string length $tclbody]>0} { |
︙ | ︙ |