Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Changes In Branch wal-overwrite-frames Excluding Merge-Ins
This is equivalent to a diff from 52c1660398 to 87dae56c32
2016-01-11
| ||
12:13 | If a single page is written to the wal file more than once, instead of appending the second and subsequent copy to the wal file, overwrite the first. Update: See the important bug fix at [f694e60a]! (check-in: d493d4f118 user: dan tags: trunk) | |
08:12 | Add a test case to waloverwrite.test to check that savepoint rollback does not cause a problem. (Closed-Leaf check-in: 87dae56c32 user: dan tags: wal-overwrite-frames) | |
03:48 | Slightly smaller and faster LIKE/GLOB comparison implementation. (check-in: 5dddcc78ee user: drh tags: trunk) | |
00:52 | Improved testability. (check-in: e83d3a2a4e user: drh tags: wal-overwrite-frames) | |
2016-01-09
| ||
16:39 | If a single page is written to the wal file more than once, have each subsequent copy overwrite the original frame. (check-in: 5d113aef2c user: dan tags: wal-overwrite-frames) | |
2016-01-08
| ||
22:31 | Typo fix in the previous commit. (check-in: 52c1660398 user: drh tags: trunk) | |
22:18 | More #ifdef fixes for the unix VFS. (check-in: 122c111e36 user: drh tags: trunk) | |
Changes to src/pcache.h.
︙ | |||
50 51 52 53 54 55 56 57 58 59 60 61 62 63 | 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 | + + | #define PGHDR_DIRTY 0x002 /* Page is on the PCache.pDirty list */ #define PGHDR_WRITEABLE 0x004 /* Journaled and ready to modify */ #define PGHDR_NEED_SYNC 0x008 /* Fsync the rollback journal before ** writing this page to the database */ #define PGHDR_NEED_READ 0x010 /* Content is unread */ #define PGHDR_DONT_WRITE 0x020 /* Do not write content to disk */ #define PGHDR_MMAP 0x040 /* This is an mmap page object */ #define PGHDR_WAL_APPEND 0x080 /* Appended to wal file */ /* Initialize and shutdown the page cache subsystem */ int sqlite3PcacheInitialize(void); void sqlite3PcacheShutdown(void); /* Page cache buffer management: ** These routines implement SQLITE_CONFIG_PAGECACHE. |
︙ |
Changes to src/wal.c.
︙ | |||
441 442 443 444 445 446 447 448 449 450 451 452 453 454 | 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 | + | u8 ckptLock; /* True if holding a checkpoint lock */ u8 readOnly; /* WAL_RDWR, WAL_RDONLY, or WAL_SHM_RDONLY */ u8 truncateOnCommit; /* True to truncate WAL file on commit */ u8 syncHeader; /* Fsync the WAL header if true */ u8 padToSectorBoundary; /* Pad transactions out to the next sector */ WalIndexHdr hdr; /* Wal-index header for current transaction */ u32 minFrame; /* Ignore wal frames before this one */ u32 iReCksum; /* On commit, recalculate checksums from here */ const char *zWalName; /* Name of WAL file */ u32 nCkpt; /* Checkpoint sequence counter in the wal-header */ #ifdef SQLITE_DEBUG u8 lockError; /* True if a locking error has occurred */ #endif #ifdef SQLITE_ENABLE_SNAPSHOT WalIndexHdr *pSnapshot; /* Start transaction here if not NULL */ |
︙ | |||
694 695 696 697 698 699 700 | 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 | + - + - - - + + + - - + + + | u8 *aFrame /* OUT: Write encoded frame here */ ){ int nativeCksum; /* True for native byte-order checksums */ u32 *aCksum = pWal->hdr.aFrameCksum; assert( WAL_FRAME_HDRSIZE==24 ); sqlite3Put4byte(&aFrame[0], iPage); sqlite3Put4byte(&aFrame[4], nTruncate); if( pWal->iReCksum==0 ){ |
︙ | |||
2628 2629 2630 2631 2632 2633 2634 2635 2636 2637 2638 2639 2640 2641 | 2631 2632 2633 2634 2635 2636 2637 2638 2639 2640 2641 2642 2643 2644 2645 | + | */ int sqlite3WalBeginWriteTransaction(Wal *pWal){ int rc; /* Cannot start a write transaction without first holding a read ** transaction. */ assert( pWal->readLock>=0 ); assert( pWal->writeLock==0 && pWal->iReCksum==0 ); if( pWal->readOnly ){ return SQLITE_READONLY; } /* Only one writer allowed at a time. Get the write lock. Return ** SQLITE_BUSY if unable. |
︙ | |||
2663 2664 2665 2666 2667 2668 2669 2670 2671 2672 2673 2674 2675 2676 | 2667 2668 2669 2670 2671 2672 2673 2674 2675 2676 2677 2678 2679 2680 2681 | + | ** End a write transaction. The commit has already been done. This ** routine merely releases the lock. */ int sqlite3WalEndWriteTransaction(Wal *pWal){ if( pWal->writeLock ){ walUnlockExclusive(pWal, WAL_WRITE_LOCK, 1); pWal->writeLock = 0; pWal->iReCksum = 0; pWal->truncateOnCommit = 0; } return SQLITE_OK; } /* ** If any data has been written (but not committed) to the log file, this |
︙ | |||
2880 2881 2882 2883 2884 2885 2886 2887 2888 2889 2890 2891 2892 2893 | 2885 2886 2887 2888 2889 2890 2891 2892 2893 2894 2895 2896 2897 2898 2899 2900 2901 2902 2903 2904 2905 2906 2907 2908 2909 2910 2911 2912 2913 2914 2915 2916 2917 2918 2919 2920 2921 2922 2923 2924 2925 2926 2927 2928 2929 2930 2931 2932 2933 2934 2935 2936 2937 2938 2939 2940 2941 2942 2943 2944 2945 2946 2947 2948 2949 2950 2951 | + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + | walEncodeFrame(p->pWal, pPage->pgno, nTruncate, pData, aFrame); rc = walWriteToLog(p, aFrame, sizeof(aFrame), iOffset); if( rc ) return rc; /* Write the page data */ rc = walWriteToLog(p, pData, p->szPage, iOffset+sizeof(aFrame)); return rc; } /* ** This function is called as part of committing a transaction within which ** one or more frames have been overwritten. It updates the checksums for ** all frames written to the wal file by the current transaction starting ** with the earliest to have been overwritten. ** ** SQLITE_OK is returned if successful, or an SQLite error code otherwise. */ static int walRewriteChecksums(Wal *pWal, u32 iLast){ const int szPage = pWal->szPage;/* Database page size */ int rc = SQLITE_OK; /* Return code */ u8 *aBuf; /* Buffer to load data from wal file into */ u8 aFrame[WAL_FRAME_HDRSIZE]; /* Buffer to assemble frame-headers in */ u32 iRead; /* Next frame to read from wal file */ i64 iCksumOff; aBuf = sqlite3_malloc(szPage + WAL_FRAME_HDRSIZE); if( aBuf==0 ) return SQLITE_NOMEM; /* Find the checksum values to use as input for the recalculating the ** first checksum. If the first frame is frame 1 (implying that the current ** transaction restarted the wal file), these values must be read from the ** wal-file header. Otherwise, read them from the frame header of the ** previous frame. */ assert( pWal->iReCksum>0 ); if( pWal->iReCksum==1 ){ iCksumOff = 24; }else{ iCksumOff = walFrameOffset(pWal->iReCksum-1, szPage) + 16; } rc = sqlite3OsRead(pWal->pWalFd, aBuf, sizeof(u32)*2, iCksumOff); pWal->hdr.aFrameCksum[0] = sqlite3Get4byte(aBuf); pWal->hdr.aFrameCksum[1] = sqlite3Get4byte(&aBuf[sizeof(u32)]); iRead = pWal->iReCksum; pWal->iReCksum = 0; for(; rc==SQLITE_OK && iRead<=iLast; iRead++){ i64 iOff = walFrameOffset(iRead, szPage); rc = sqlite3OsRead(pWal->pWalFd, aBuf, szPage+WAL_FRAME_HDRSIZE, iOff); if( rc==SQLITE_OK ){ u32 iPgno, nDbSize; iPgno = sqlite3Get4byte(aBuf); nDbSize = sqlite3Get4byte(&aBuf[4]); walEncodeFrame(pWal, iPgno, nDbSize, &aBuf[WAL_FRAME_HDRSIZE], aFrame); rc = sqlite3OsWrite(pWal->pWalFd, aFrame, sizeof(aFrame), iOff); } } sqlite3_free(aBuf); return rc; } /* ** Write a set of frames to the log. The caller must hold the write-lock ** on the log file (obtained using sqlite3WalBeginWriteTransaction()). */ int sqlite3WalFrames( Wal *pWal, /* Wal handle to write to */ |
︙ | |||
2901 2902 2903 2904 2905 2906 2907 2908 2909 2910 2911 2912 2913 2914 2915 2916 2917 2918 2919 2920 2921 2922 2923 2924 2925 2926 2927 2928 | 2959 2960 2961 2962 2963 2964 2965 2966 2967 2968 2969 2970 2971 2972 2973 2974 2975 2976 2977 2978 2979 2980 2981 2982 2983 2984 2985 2986 2987 2988 2989 2990 2991 2992 2993 | + + + + + + + | u32 iFrame; /* Next frame address */ PgHdr *p; /* Iterator to run through pList with. */ PgHdr *pLast = 0; /* Last frame in list */ int nExtra = 0; /* Number of extra copies of last page */ int szFrame; /* The size of a single frame */ i64 iOffset; /* Next byte to write in WAL file */ WalWriter w; /* The writer */ u32 iFirst = 0; /* First frame that may be overwritten */ WalIndexHdr *pLive; /* Pointer to shared header */ assert( pList ); assert( pWal->writeLock ); /* If this frame set completes a transaction, then nTruncate>0. If ** nTruncate==0 then this frame set does not complete the transaction. */ assert( (isCommit!=0)==(nTruncate!=0) ); #if defined(SQLITE_TEST) && defined(SQLITE_DEBUG) { int cnt; for(cnt=0, p=pList; p; p=p->pDirty, cnt++){} WALTRACE(("WAL%p: frame write begin. %d frames. mxFrame=%d. %s\n", pWal, cnt, pWal->hdr.mxFrame, isCommit ? "Commit" : "Spill")); } #endif pLive = (WalIndexHdr*)walIndexHdr(pWal); if( memcmp(&pWal->hdr, (void *)pLive, sizeof(WalIndexHdr))!=0 ){ iFirst = pLive->mxFrame+1; } /* See if it is possible to write these frames into the start of the ** log file, instead of appending to it at pWal->hdr.mxFrame. */ if( SQLITE_OK!=(rc = walRestartLog(pWal)) ){ return rc; } |
︙ | |||
2980 2981 2982 2983 2984 2985 2986 2987 2988 2989 2990 2991 2992 2993 2994 2995 2996 2997 2998 2999 3000 | 3045 3046 3047 3048 3049 3050 3051 3052 3053 3054 3055 3056 3057 3058 3059 3060 3061 3062 3063 3064 3065 3066 3067 3068 3069 3070 3071 3072 3073 3074 3075 3076 3077 3078 3079 3080 3081 3082 3083 3084 3085 3086 3087 3088 3089 3090 3091 3092 3093 | + + + + + + + + + + + + + + + + + + + + + + + + + + + + | w.szPage = szPage; iOffset = walFrameOffset(iFrame+1, szPage); szFrame = szPage + WAL_FRAME_HDRSIZE; /* Write all frames into the log file exactly once */ for(p=pList; p; p=p->pDirty){ int nDbSize; /* 0 normally. Positive == commit flag */ /* Check if this page has already been written into the wal file by ** the current transaction. If so, overwrite the existing frame and ** set Wal.writeLock to WAL_WRITELOCK_RECKSUM - indicating that ** checksums must be recomputed when the transaction is committed. */ if( iFirst && (p->pDirty || isCommit==0) ){ u32 iWrite = 0; VVA_ONLY(rc =) sqlite3WalFindFrame(pWal, p->pgno, &iWrite); assert( rc==SQLITE_OK || iWrite==0 ); if( iWrite>=iFirst ){ i64 iOff = walFrameOffset(iWrite, szPage) + WAL_FRAME_HDRSIZE; if( pWal->iReCksum==0 || iWrite<pWal->iReCksum ){ pWal->iReCksum = iWrite; } rc = sqlite3OsWrite(pWal->pWalFd, p->pData, szPage, iOff); if( rc ) return rc; p->flags &= ~PGHDR_WAL_APPEND; continue; } } iFrame++; assert( iOffset==walFrameOffset(iFrame, szPage) ); nDbSize = (isCommit && p->pDirty==0) ? nTruncate : 0; rc = walWriteOneFrame(&w, p, nDbSize, iOffset); if( rc ) return rc; pLast = p; iOffset += szFrame; p->flags |= PGHDR_WAL_APPEND; } /* Recalculate checksums within the wal file if required. */ if( isCommit && pWal->iReCksum ){ rc = walRewriteChecksums(pWal, iFrame); if( rc ) return rc; } /* If this is the end of a transaction, then we might need to pad ** the transaction and/or sync the WAL file. ** ** Padding and syncing only occur if this set of frames complete a ** transaction and if PRAGMA synchronous=FULL. If synchronous==NORMAL |
︙ | |||
3038 3039 3040 3041 3042 3043 3044 3045 3046 3047 3048 3049 3050 3051 | 3131 3132 3133 3134 3135 3136 3137 3138 3139 3140 3141 3142 3143 3144 3145 | + | /* Append data to the wal-index. It is not necessary to lock the ** wal-index to do this as the SQLITE_SHM_WRITE lock held on the wal-index ** guarantees that there are no other writers, and no data that may ** be in use by existing readers is being overwritten. */ iFrame = pWal->hdr.mxFrame; for(p=pList; p && rc==SQLITE_OK; p=p->pDirty){ if( (p->flags & PGHDR_WAL_APPEND)==0 ) continue; iFrame++; rc = walIndexAppend(pWal, iFrame, p->pgno); } while( rc==SQLITE_OK && nExtra>0 ){ iFrame++; nExtra--; rc = walIndexAppend(pWal, iFrame, pLast->pgno); |
︙ | |||
3150 3151 3152 3153 3154 3155 3156 3157 3158 3159 3160 3161 3162 3163 | 3244 3245 3246 3247 3248 3249 3250 3251 3252 3253 3254 3255 3256 3257 3258 | + | if( isChanged && pWal->pDbFd->pMethods->iVersion>=3 ){ sqlite3OsUnfetch(pWal->pDbFd, 0, 0); } } /* Copy data from the log to the database file. */ if( rc==SQLITE_OK ){ if( pWal->hdr.mxFrame && walPagesize(pWal)!=nBuf ){ rc = SQLITE_CORRUPT_BKPT; }else{ rc = walCheckpoint(pWal, eMode2, xBusy2, pBusyArg, sync_flags, zBuf); } /* If no error occurred, set the output variables. */ |
︙ |
Changes to test/wal.test.
︙ | |||
708 709 710 711 712 713 714 | 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 | - + - + + + + + + - - - + + + + + | SELECT count(*) FROM t1; PRAGMA integrity_check; } } {16 ok} do_test wal-11.6 { execsql COMMIT list [expr [file size test.db]/1024] [file size test.db-wal] |
︙ |
Added test/waloverwrite.test.