/ Check-in [756589ad6e]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Add "PRAGMA checkpoint_fullfsync". Similar to "PRAGMA fullfsync", but enables full fsyncs only during checkpoint operations. Update: This change was added to the trunk by check-in [a069867301de3ca2e17]
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | apple-osx-exp
Files: files | file ages | folders
SHA1: 756589ad6e286a898ca34ff1b14d5bb4843529e1
User & Date: dan 2010-11-09 20:08:02
Original Comment: Add "PRAGMA checkpoint_fullfsync". Similar to "PRAGMA fullfsync", but enables full fsyncs only during checkpoint operations.
Context
2010-11-09
20:08
Add "PRAGMA checkpoint_fullfsync". Similar to "PRAGMA fullfsync", but enables full fsyncs only during checkpoint operations. Update: This change was added to the trunk by check-in [a069867301de3ca2e17] Closed-Leaf check-in: 756589ad6e user: dan tags: apple-osx-exp
01:53
Back out the prior attempt to enable full-fsync for WAL and attempt the same thing using a completely different approach. check-in: f59949fac1 user: drh tags: apple-osx-exp
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to src/pager.c.

615
616
617
618
619
620
621

622
623
624
625
626
627
628
....
3078
3079
3080
3081
3082
3083
3084













3085
3086
3087
3088
3089
3090
3091
....
3258
3259
3260
3261
3262
3263
3264
3265

3266
3267

3268




3269
3270
3271
3272
3273
3274
3275
....
3647
3648
3649
3650
3651
3652
3653
3654
3655
3656
3657
3658
3659
3660
3661
3662
....
6517
6518
6519
6520
6521
6522
6523

6524
6525
6526
6527
6528
6529
6530
6531
6532
6533
6534
....
6625
6626
6627
6628
6629
6630
6631
6632
6633
6634
6635
6636
6637
6638
6639
6640
6641
  u8 noReadlock;              /* Do not bother to obtain readlocks */
  u8 noSync;                  /* Do not sync the journal if true */
  u8 fullSync;                /* Do extra syncs of the journal for robustness */
  u8 sync_flags;              /* One of SYNC_NORMAL or SYNC_FULL */
  u8 tempFile;                /* zFilename is a temporary file */
  u8 readOnly;                /* True for a read-only database */
  u8 memDb;                   /* True to inhibit all file I/O */


  /**************************************************************************
  ** The following block contains those class members that change during
  ** routine opertion.  Class members not in this block are either fixed
  ** when the pager is first created or else only change when there is a
  ** significant mode change (such as changing the page_size, locking_mode,
  ** or the journal_mode).  From another view, these class members describe
................................................................................
      }else if( pPager->journalMode==PAGER_JOURNALMODE_WAL ){
        pPager->journalMode = PAGER_JOURNALMODE_DELETE;
      }
    }
  }
  return rc;
}













#endif

/*
** Playback savepoint pSavepoint. Or, if pSavepoint==NULL, then playback
** the entire master journal file. The case pSavepoint==NULL occurs when 
** a ROLLBACK TO command is invoked on a SAVEPOINT that is a transaction 
** savepoint.
................................................................................
**              assurance that the journal will not be corrupted to the
**              point of causing damage to the database during rollback.
**
** Numeric values associated with these states are OFF==1, NORMAL=2,
** and FULL=3.
*/
#ifndef SQLITE_OMIT_PAGER_PRAGMAS
void sqlite3PagerSetSafetyLevel(Pager *pPager, int level, int bFullFsync){

  pPager->noSync =  (level==1 || pPager->tempFile) ?1:0;
  pPager->fullSync = (level==3 && !pPager->tempFile) ?1:0;

  pPager->sync_flags = (bFullFsync?SQLITE_SYNC_FULL:SQLITE_SYNC_NORMAL);




}
#endif

/*
** The following global variable is incremented whenever the library
** attempts to open a temporary file.  This information is used for
** testing and analysis only.  
................................................................................
  u8 *pTmp = (u8 *)pPager->pTmpSpace;

  disable_simulated_io_errors();
  sqlite3BeginBenignMalloc();
  /* pPager->errCode = 0; */
  pPager->exclusiveMode = 0;
#ifndef SQLITE_OMIT_WAL
  sqlite3WalClose(pPager->pWal,
    (pPager->noSync ? 0 : pPager->sync_flags), 
    pPager->pageSize, pTmp
  );
  pPager->pWal = 0;
#endif
  pager_reset(pPager);
  if( MEMDB ){
    pager_unlock(pPager);
................................................................................
/*
** This function is called when the user invokes "PRAGMA checkpoint".
*/
int sqlite3PagerCheckpoint(Pager *pPager){
  int rc = SQLITE_OK;
  if( pPager->pWal ){
    u8 *zBuf = (u8 *)pPager->pTmpSpace;

    rc = sqlite3WalCheckpoint(pPager->pWal,
        (pPager->noSync ? 0 : pPager->sync_flags),
        pPager->pageSize, zBuf
    );
  }
  return rc;
}

int sqlite3PagerWalCallback(Pager *pPager){
  return sqlite3WalCallback(pPager->pWal);
}
................................................................................
    
  /* Checkpoint and close the log. Because an EXCLUSIVE lock is held on
  ** the database file, the log and log-summary files will be deleted.
  */
  if( rc==SQLITE_OK && pPager->pWal ){
    rc = pagerLockDb(pPager, EXCLUSIVE_LOCK);
    if( rc==SQLITE_OK ){
      rc = sqlite3WalClose(pPager->pWal,
                           (pPager->noSync ? 0 : pPager->sync_flags), 
        pPager->pageSize, (u8*)pPager->pTmpSpace
      );
      pPager->pWal = 0;
    }else{
      /* If we cannot get an EXCLUSIVE lock, downgrade the PENDING lock
      ** that we did get back to SHARED. */
      pagerUnlockDb(pPager, SQLITE_LOCK_SHARED);
    }







>







 







>
>
>
>
>
>
>
>
>
>
>
>
>







 







|
>


>
|
>
>
>
>







 







|
<







 







>
|
<
<
<







 







|
<
|







615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
....
3079
3080
3081
3082
3083
3084
3085
3086
3087
3088
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
....
3272
3273
3274
3275
3276
3277
3278
3279
3280
3281
3282
3283
3284
3285
3286
3287
3288
3289
3290
3291
3292
3293
3294
3295
....
3667
3668
3669
3670
3671
3672
3673
3674

3675
3676
3677
3678
3679
3680
3681
....
6536
6537
6538
6539
6540
6541
6542
6543
6544



6545
6546
6547
6548
6549
6550
6551
....
6642
6643
6644
6645
6646
6647
6648
6649

6650
6651
6652
6653
6654
6655
6656
6657
  u8 noReadlock;              /* Do not bother to obtain readlocks */
  u8 noSync;                  /* Do not sync the journal if true */
  u8 fullSync;                /* Do extra syncs of the journal for robustness */
  u8 sync_flags;              /* One of SYNC_NORMAL or SYNC_FULL */
  u8 tempFile;                /* zFilename is a temporary file */
  u8 readOnly;                /* True for a read-only database */
  u8 memDb;                   /* True to inhibit all file I/O */
  u8 ckptFullSync;            /* True to pass SYNC_FULL to WalCheckpoint() */

  /**************************************************************************
  ** The following block contains those class members that change during
  ** routine opertion.  Class members not in this block are either fixed
  ** when the pager is first created or else only change when there is a
  ** significant mode change (such as changing the page_size, locking_mode,
  ** or the journal_mode).  From another view, these class members describe
................................................................................
      }else if( pPager->journalMode==PAGER_JOURNALMODE_WAL ){
        pPager->journalMode = PAGER_JOURNALMODE_DELETE;
      }
    }
  }
  return rc;
}

/*
** Return the value of the flags parameter that should be passed to
** sqlite3OsSync() when checkpointing a WAL file.
*/
static int walCkptSyncFlags(Pager *pPager){
  int flags = 0;
  if( pPager->noSync==0 ){
    flags = pPager->sync_flags | (pPager->ckptFullSync?SQLITE_SYNC_FULL:0);
  }
  return flags;
}

#endif

/*
** Playback savepoint pSavepoint. Or, if pSavepoint==NULL, then playback
** the entire master journal file. The case pSavepoint==NULL occurs when 
** a ROLLBACK TO command is invoked on a SAVEPOINT that is a transaction 
** savepoint.
................................................................................
**              assurance that the journal will not be corrupted to the
**              point of causing damage to the database during rollback.
**
** Numeric values associated with these states are OFF==1, NORMAL=2,
** and FULL=3.
*/
#ifndef SQLITE_OMIT_PAGER_PRAGMAS
void sqlite3PagerSetSafetyLevel(Pager *pPager, int level, int fullFsync){
  assert( 0==(fullFsync & ~(SQLITE_FullFSync|SQLITE_CkptFullFSync)) );
  pPager->noSync =  (level==1 || pPager->tempFile) ?1:0;
  pPager->fullSync = (level==3 && !pPager->tempFile) ?1:0;
  if( fullFsync & SQLITE_FullFSync ){
    pPager->sync_flags = SQLITE_SYNC_FULL;
  }else{
    pPager->sync_flags = SQLITE_SYNC_NORMAL;
  }
  pPager->ckptFullSync = (fullFsync & SQLITE_CkptFullFSync)!=0;
}
#endif

/*
** The following global variable is incremented whenever the library
** attempts to open a temporary file.  This information is used for
** testing and analysis only.  
................................................................................
  u8 *pTmp = (u8 *)pPager->pTmpSpace;

  disable_simulated_io_errors();
  sqlite3BeginBenignMalloc();
  /* pPager->errCode = 0; */
  pPager->exclusiveMode = 0;
#ifndef SQLITE_OMIT_WAL
  sqlite3WalClose(pPager->pWal, walCkptSyncFlags(pPager),

    pPager->pageSize, pTmp
  );
  pPager->pWal = 0;
#endif
  pager_reset(pPager);
  if( MEMDB ){
    pager_unlock(pPager);
................................................................................
/*
** This function is called when the user invokes "PRAGMA checkpoint".
*/
int sqlite3PagerCheckpoint(Pager *pPager){
  int rc = SQLITE_OK;
  if( pPager->pWal ){
    u8 *zBuf = (u8 *)pPager->pTmpSpace;
    int flags = walCkptSyncFlags(pPager);
    rc = sqlite3WalCheckpoint(pPager->pWal, flags, pPager->pageSize, zBuf);



  }
  return rc;
}

int sqlite3PagerWalCallback(Pager *pPager){
  return sqlite3WalCallback(pPager->pWal);
}
................................................................................
    
  /* Checkpoint and close the log. Because an EXCLUSIVE lock is held on
  ** the database file, the log and log-summary files will be deleted.
  */
  if( rc==SQLITE_OK && pPager->pWal ){
    rc = pagerLockDb(pPager, EXCLUSIVE_LOCK);
    if( rc==SQLITE_OK ){
      rc = sqlite3WalClose(pPager->pWal, walCkptSyncFlags(pPager),

          pPager->pageSize, (u8*)pPager->pTmpSpace
      );
      pPager->pWal = 0;
    }else{
      /* If we cannot get an EXCLUSIVE lock, downgrade the PENDING lock
      ** that we did get back to SHARED. */
      pagerUnlockDb(pPager, SQLITE_LOCK_SHARED);
    }

Changes to src/pragma.c.

168
169
170
171
172
173
174

175
176
177
178
179
180
181
....
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
  } aPragma[] = {
    { "full_column_names",        SQLITE_FullColNames  },
    { "short_column_names",       SQLITE_ShortColNames },
    { "count_changes",            SQLITE_CountRows     },
    { "empty_result_callbacks",   SQLITE_NullCallback  },
    { "legacy_file_format",       SQLITE_LegacyFileFmt },
    { "fullfsync",                SQLITE_FullFSync     },

    { "reverse_unordered_selects", SQLITE_ReverseOrder  },
#ifndef SQLITE_OMIT_AUTOMATIC_INDEX
    { "automatic_index",          SQLITE_AutoIndex     },
#endif
#ifdef SQLITE_DEBUG
    { "sql_trace",                SQLITE_SqlTrace      },
    { "vdbe_listing",             SQLITE_VdbeListing   },
................................................................................
  /*
  ** Reset the safety level, in case the fullfsync flag or synchronous
  ** setting changed.
  */
#ifndef SQLITE_OMIT_PAGER_PRAGMAS
  if( db->autoCommit ){
    sqlite3BtreeSetSafetyLevel(pDb->pBt, pDb->safety_level,
               (db->flags&SQLITE_FullFSync)!=0);
  }
#endif
pragma_out:
  sqlite3DbFree(db, zLeft);
  sqlite3DbFree(db, zRight);
}

#endif /* SQLITE_OMIT_PRAGMA */







>







 







|








168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
....
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
  } aPragma[] = {
    { "full_column_names",        SQLITE_FullColNames  },
    { "short_column_names",       SQLITE_ShortColNames },
    { "count_changes",            SQLITE_CountRows     },
    { "empty_result_callbacks",   SQLITE_NullCallback  },
    { "legacy_file_format",       SQLITE_LegacyFileFmt },
    { "fullfsync",                SQLITE_FullFSync     },
    { "checkpoint_fullfsync",     SQLITE_CkptFullFSync },
    { "reverse_unordered_selects", SQLITE_ReverseOrder  },
#ifndef SQLITE_OMIT_AUTOMATIC_INDEX
    { "automatic_index",          SQLITE_AutoIndex     },
#endif
#ifdef SQLITE_DEBUG
    { "sql_trace",                SQLITE_SqlTrace      },
    { "vdbe_listing",             SQLITE_VdbeListing   },
................................................................................
  /*
  ** Reset the safety level, in case the fullfsync flag or synchronous
  ** setting changed.
  */
#ifndef SQLITE_OMIT_PAGER_PRAGMAS
  if( db->autoCommit ){
    sqlite3BtreeSetSafetyLevel(pDb->pBt, pDb->safety_level,
               db->flags&(SQLITE_FullFSync|SQLITE_CkptFullFSync));
  }
#endif
pragma_out:
  sqlite3DbFree(db, zLeft);
  sqlite3DbFree(db, zRight);
}

#endif /* SQLITE_OMIT_PRAGMA */

Changes to src/sqliteInt.h.

910
911
912
913
914
915
916

917
918
919
920
921
922
923
#define SQLITE_LoadExtension  0x00400000  /* Enable load_extension */
#define SQLITE_RecoveryMode   0x00800000  /* Ignore schema errors */
#define SQLITE_ReverseOrder   0x01000000  /* Reverse unordered SELECTs */
#define SQLITE_RecTriggers    0x02000000  /* Enable recursive triggers */
#define SQLITE_ForeignKeys    0x04000000  /* Enforce foreign key constraints  */
#define SQLITE_AutoIndex      0x08000000  /* Enable automatic indexes */
#define SQLITE_PreferBuiltin  0x10000000  /* Preference to built-in funcs */


/*
** Bits of the sqlite3.flags field that are used by the
** sqlite3_test_control(SQLITE_TESTCTRL_OPTIMIZATIONS,...) interface.
** These must be the low-order bits of the flags field.
*/
#define SQLITE_QueryFlattener 0x01        /* Disable query flattening */







>







910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
#define SQLITE_LoadExtension  0x00400000  /* Enable load_extension */
#define SQLITE_RecoveryMode   0x00800000  /* Ignore schema errors */
#define SQLITE_ReverseOrder   0x01000000  /* Reverse unordered SELECTs */
#define SQLITE_RecTriggers    0x02000000  /* Enable recursive triggers */
#define SQLITE_ForeignKeys    0x04000000  /* Enforce foreign key constraints  */
#define SQLITE_AutoIndex      0x08000000  /* Enable automatic indexes */
#define SQLITE_PreferBuiltin  0x10000000  /* Preference to built-in funcs */
#define SQLITE_CkptFullFSync  0x20000000  /* Use full fsync on checkpoint */

/*
** Bits of the sqlite3.flags field that are used by the
** sqlite3_test_control(SQLITE_TESTCTRL_OPTIMIZATIONS,...) interface.
** These must be the low-order bits of the flags field.
*/
#define SQLITE_QueryFlattener 0x01        /* Disable query flattening */

Changes to src/wal.c.

1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
....
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
   && (rc = walLockExclusive(pWal, WAL_READ_LOCK(0), 1))==SQLITE_OK
  ){
    i64 nSize;                    /* Current size of database file */
    u32 nBackfill = pInfo->nBackfill;

    /* Sync the WAL to disk */
    if( sync_flags ){
      rc = sqlite3OsSync(pWal->pWalFd, sync_flags | SQLITE_SYNC_FULL );
    }

    /* If the database file may grow as a result of this checkpoint, hint
    ** about the eventual size of the db file to the VFS layer. 
    */
    if( rc==SQLITE_OK ){
      i64 nReq = ((i64)mxPage * szPage);
................................................................................
    /* If work was actually accomplished... */
    if( rc==SQLITE_OK ){
      if( mxSafeFrame==walIndexHdr(pWal)->mxFrame ){
        i64 szDb = pWal->hdr.nPage*(i64)szPage;
        testcase( IS_BIG_INT(szDb) );
        rc = sqlite3OsTruncate(pWal->pDbFd, szDb);
        if( rc==SQLITE_OK && sync_flags ){
          rc = sqlite3OsSync(pWal->pDbFd, sync_flags | SQLITE_SYNC_FULL);
        }
      }
      if( rc==SQLITE_OK ){
        pInfo->nBackfill = mxSafeFrame;
      }
    }








|







 







|







1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
....
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
   && (rc = walLockExclusive(pWal, WAL_READ_LOCK(0), 1))==SQLITE_OK
  ){
    i64 nSize;                    /* Current size of database file */
    u32 nBackfill = pInfo->nBackfill;

    /* Sync the WAL to disk */
    if( sync_flags ){
      rc = sqlite3OsSync(pWal->pWalFd, sync_flags);
    }

    /* If the database file may grow as a result of this checkpoint, hint
    ** about the eventual size of the db file to the VFS layer. 
    */
    if( rc==SQLITE_OK ){
      i64 nReq = ((i64)mxPage * szPage);
................................................................................
    /* If work was actually accomplished... */
    if( rc==SQLITE_OK ){
      if( mxSafeFrame==walIndexHdr(pWal)->mxFrame ){
        i64 szDb = pWal->hdr.nPage*(i64)szPage;
        testcase( IS_BIG_INT(szDb) );
        rc = sqlite3OsTruncate(pWal->pDbFd, szDb);
        if( rc==SQLITE_OK && sync_flags ){
          rc = sqlite3OsSync(pWal->pDbFd, sync_flags);
        }
      }
      if( rc==SQLITE_OK ){
        pInfo->nBackfill = mxSafeFrame;
      }
    }

Changes to test/pragma.test.

1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303

# Reset the sqlite3_temp_directory variable for the next run of tests:
sqlite3 dbX :memory:
dbX eval {PRAGMA temp_store_directory = ""}
dbX close

set skip_lock_proxy_tests [path_is_dos "."]
ifcapable !lock_proxy_pragmas&&prefer_proxy_locking {
  set skip_lock_proxy_tests 1
}

if !$skip_lock_proxy_tests {
  set sqlite_hostid_num 1

  set using_proxy 0







|







1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303

# Reset the sqlite3_temp_directory variable for the next run of tests:
sqlite3 dbX :memory:
dbX eval {PRAGMA temp_store_directory = ""}
dbX close

set skip_lock_proxy_tests [path_is_dos "."]
ifcapable !lock_proxy_pragmas||prefer_proxy_locking {
  set skip_lock_proxy_tests 1
}

if !$skip_lock_proxy_tests {
  set sqlite_hostid_num 1

  set using_proxy 0

Changes to test/wal2.test.

1148
1149
1150
1151
1152
1153
1154
1155























































1156
      do_test wal2-13.$tn.4 {
        catchsql { INSERT INTO t1 DEFAULT VALUES }
      } $b($can_read,$can_write)
    }
    catch { db close }
  }
}
























































finish_test








>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>

1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
      do_test wal2-13.$tn.4 {
        catchsql { INSERT INTO t1 DEFAULT VALUES }
      } $b($can_read,$can_write)
    }
    catch { db close }
  }
}

#-------------------------------------------------------------------------
# Test that "PRAGMA checkpoint_fullsync" appears to be working.
#
foreach {tn sql reslist} {
  1 { }                                 {8 0 3 0 5 0}
  2 { PRAGMA checkpoint_fullfsync = 1 } {8 4 3 2 5 2}
  3 { PRAGMA checkpoint_fullfsync = 0 } {8 0 3 0 5 0}
} {
  faultsim_delete_and_reopen

  execsql $sql
  do_execsql_test wal2-14.$tn.1 { PRAGMA journal_mode = WAL } {wal}

  set sqlite_sync_count 0
  set sqlite_fullsync_count 0

  do_execsql_test wal2-14.$tn.2 {
    PRAGMA wal_autocheckpoint = 10;
    CREATE TABLE t1(a, b);                -- 2 wal syncs
    INSERT INTO t1 VALUES(1, 2);          -- 1 wal sync
    PRAGMA wal_checkpoint;                -- 1 wal sync, 1 db sync
    BEGIN;
      INSERT INTO t1 VALUES(3, 4);
      INSERT INTO t1 VALUES(5, 6);
    COMMIT;                               -- 1 wal sync
    PRAGMA wal_checkpoint;                -- 1 wal sync, 1 db sync
  } {10}

  do_test wal2-14.$tn.3 {
    list $sqlite_sync_count $sqlite_fullsync_count
  } [lrange $reslist 0 1]

  set sqlite_sync_count 0
  set sqlite_fullsync_count 0

  do_test wal2-14.$tn.4 {
    execsql { INSERT INTO t1 VALUES(7, zeroblob(12*4096)) }
    list $sqlite_sync_count $sqlite_fullsync_count
  } [lrange $reslist 2 3]

  set sqlite_sync_count 0
  set sqlite_fullsync_count 0

  do_test wal2-14.$tn.5 {
    execsql { PRAGMA wal_autocheckpoint = 1000 }
    execsql { INSERT INTO t1 VALUES(9, 10) }
    execsql { INSERT INTO t1 VALUES(11, 12) }
    execsql { INSERT INTO t1 VALUES(13, 14) }
    db close
    list $sqlite_sync_count $sqlite_fullsync_count
  } [lrange $reslist 4 5]
}



finish_test