SQLite

Check-in [e3d2be3ba4]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Add a test to ensure os_unix.c works with 64KiB OS pages.
Downloads: Tarball | ZIP archive
Timelines: family | ancestors | descendants | both | shm-mapping-fix
Files: files | file ages | folders
SHA1: e3d2be3ba47cdaafd26347620ae3bc2813203f16
User & Date: dan 2014-03-20 09:42:09.744
Context
2014-03-24
11:23
Avoid attempting to mmap memory from an offset that is not a multiple of the system page size on systems with page sizes larger than 32KB. (check-in: db7d62c8d5 user: dan tags: trunk)
2014-03-20
09:42
Add a test to ensure os_unix.c works with 64KiB OS pages. (Closed-Leaf check-in: e3d2be3ba4 user: dan tags: shm-mapping-fix)
08:59
Add an experimental fix to avoid attempting to mmap memory from an offset that is not a multiple of the system page size on systems with page sizes larger than 32KB. (check-in: 6f3a5c24d2 user: dan tags: shm-mapping-fix)
Changes
Unified Diff Ignore Whitespace Patch
Changes to src/os_unix.c.
319
320
321
322
323
324
325

326
327
328
329
330
331
332
*/
static int posixFchown(int fd, uid_t uid, gid_t gid){
  return geteuid() ? 0 : fchown(fd,uid,gid);
}

/* Forward reference */
static int openDirectory(const char*, int*);


/*
** Many system calls are accessed through pointer-to-functions so that
** they may be overridden at runtime to facilitate fault injection during
** testing and sandboxing.  The following array holds the names and pointers
** to all overrideable system calls.
*/







>







319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
*/
static int posixFchown(int fd, uid_t uid, gid_t gid){
  return geteuid() ? 0 : fchown(fd,uid,gid);
}

/* Forward reference */
static int openDirectory(const char*, int*);
static int unixGetpagesize(void);

/*
** Many system calls are accessed through pointer-to-functions so that
** they may be overridden at runtime to facilitate fault injection during
** testing and sandboxing.  The following array holds the names and pointers
** to all overrideable system calls.
*/
441
442
443
444
445
446
447



448
449
450
451
452
453
454
#if HAVE_MREMAP
  { "mremap",       (sqlite3_syscall_ptr)mremap,          0 },
#else
  { "mremap",       (sqlite3_syscall_ptr)0,               0 },
#endif
#define osMremap ((void*(*)(void*,size_t,size_t,int,...))aSyscall[23].pCurrent)
#endif




}; /* End of the overrideable system calls */

/*
** This is the xSetSystemCall() method of sqlite3_vfs for all of the
** "unix" VFSes.  Return SQLITE_OK opon successfully updating the
** system call pointer, or SQLITE_NOTFOUND if there is no configurable







>
>
>







442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
#if HAVE_MREMAP
  { "mremap",       (sqlite3_syscall_ptr)mremap,          0 },
#else
  { "mremap",       (sqlite3_syscall_ptr)0,               0 },
#endif
#define osMremap ((void*(*)(void*,size_t,size_t,int,...))aSyscall[23].pCurrent)
#endif

  { "getpagesize",  (sqlite3_syscall_ptr)unixGetpagesize, 0 },
#define osGetpagesize ((int(*)(void))aSyscall[24].pCurrent)

}; /* End of the overrideable system calls */

/*
** This is the xSetSystemCall() method of sqlite3_vfs for all of the
** "unix" VFSes.  Return SQLITE_OK opon successfully updating the
** system call pointer, or SQLITE_NOTFOUND if there is no configurable
4103
4104
4105
4106
4107
4108
4109



4110
4111
4112
4113
4114
4115
4116
4117
4118
4119
4120
4121
4122
4123
4124
4125
4126
4127
4128
4129
4130
4131
4132
4133
4134
4135
4136
4137
#endif

  return rc;        
}

/*
** Return the system page size.



*/
static int unixGetPagesize(void){
#if defined(_BSD_SOURCE)
  return getpagesize();
#else
  return (int)sysconf(_SC_PAGESIZE);
#endif
}

/*
** Return the minimum number of 32KB shm regions that should be mapped at
** a time, assuming that each mapping must be an integer multiple of the
** current system page-size.
**
** Usually, this is 1. The exception seems to be systems that are configured
** to use 64KB pages - in this case each mapping must cover at least two
** shm regions.
*/
static int unixShmRegionPerMap(void){
  int shmsz = 32*1024;            /* SHM region size */
  int pgsz = unixGetPagesize();   /* System page size */
  assert( ((pgsz-1)&pgsz)==0 );   /* Page size must be a power of 2 */
  if( pgsz<shmsz ) return 1;
  return pgsz/shmsz;
}

/*
** Purge the unixShmNodeList list of all entries with unixShmNode.nRef==0.







>
>
>

|


















|







4107
4108
4109
4110
4111
4112
4113
4114
4115
4116
4117
4118
4119
4120
4121
4122
4123
4124
4125
4126
4127
4128
4129
4130
4131
4132
4133
4134
4135
4136
4137
4138
4139
4140
4141
4142
4143
4144
#endif

  return rc;        
}

/*
** Return the system page size.
**
** This function should not be called directly by other code in this file. 
** Instead, it should be called via macro osGetpagesize().
*/
static int unixGetpagesize(void){
#if defined(_BSD_SOURCE)
  return getpagesize();
#else
  return (int)sysconf(_SC_PAGESIZE);
#endif
}

/*
** Return the minimum number of 32KB shm regions that should be mapped at
** a time, assuming that each mapping must be an integer multiple of the
** current system page-size.
**
** Usually, this is 1. The exception seems to be systems that are configured
** to use 64KB pages - in this case each mapping must cover at least two
** shm regions.
*/
static int unixShmRegionPerMap(void){
  int shmsz = 32*1024;            /* SHM region size */
  int pgsz = osGetpagesize();   /* System page size */
  assert( ((pgsz-1)&pgsz)==0 );   /* Page size must be a power of 2 */
  if( pgsz<shmsz ) return 1;
  return pgsz/shmsz;
}

/*
** Purge the unixShmNodeList list of all entries with unixShmNode.nRef==0.
4706
4707
4708
4709
4710
4711
4712
4713
4714
4715
4716
4717
4718
4719
4720

  if( (pFd->ctrlFlags & UNIXFILE_RDONLY)==0 ) flags |= PROT_WRITE;

  if( pOrig ){
#if HAVE_MREMAP
    i64 nReuse = pFd->mmapSize;
#else
    const int szSyspage = unixGetPagesize();
    i64 nReuse = (pFd->mmapSize & ~(szSyspage-1));
#endif
    u8 *pReq = &pOrig[nReuse];

    /* Unmap any pages of the existing mapping that cannot be reused. */
    if( nReuse!=nOrig ){
      osMunmap(pReq, nOrig-nReuse);







|







4713
4714
4715
4716
4717
4718
4719
4720
4721
4722
4723
4724
4725
4726
4727

  if( (pFd->ctrlFlags & UNIXFILE_RDONLY)==0 ) flags |= PROT_WRITE;

  if( pOrig ){
#if HAVE_MREMAP
    i64 nReuse = pFd->mmapSize;
#else
    const int szSyspage = osGetpagesize();
    i64 nReuse = (pFd->mmapSize & ~(szSyspage-1));
#endif
    u8 *pReq = &pOrig[nReuse];

    /* Unmap any pages of the existing mapping that cannot be reused. */
    if( nReuse!=nOrig ){
      osMunmap(pReq, nOrig-nReuse);
7454
7455
7456
7457
7458
7459
7460
7461
7462
7463
7464
7465
7466
7467
7468
    UNIXVFS("unix-proxy",    proxyIoFinder ),
#endif
  };
  unsigned int i;          /* Loop counter */

  /* Double-check that the aSyscall[] array has been constructed
  ** correctly.  See ticket [bb3a86e890c8e96ab] */
  assert( ArraySize(aSyscall)==24 );

  /* Register all VFSes defined in the aVfs[] array */
  for(i=0; i<(sizeof(aVfs)/sizeof(sqlite3_vfs)); i++){
    sqlite3_vfs_register(&aVfs[i], i==0);
  }
  return SQLITE_OK; 
}







|







7461
7462
7463
7464
7465
7466
7467
7468
7469
7470
7471
7472
7473
7474
7475
    UNIXVFS("unix-proxy",    proxyIoFinder ),
#endif
  };
  unsigned int i;          /* Loop counter */

  /* Double-check that the aSyscall[] array has been constructed
  ** correctly.  See ticket [bb3a86e890c8e96ab] */
  assert( ArraySize(aSyscall)==25 );

  /* Register all VFSes defined in the aVfs[] array */
  for(i=0; i<(sizeof(aVfs)/sizeof(sqlite3_vfs)); i++){
    sqlite3_vfs_register(&aVfs[i], i==0);
  }
  return SQLITE_OK; 
}
Changes to src/test_syscall.c.
63
64
65
66
67
68
69





70
71
72
73
74
75
76
**
**   test_syscall exists SYSTEM-CALL
**     Return true if the named system call exists. Or false otherwise.
**
**   test_syscall list
**     Return a list of all system calls. The list is constructed using
**     the xNextSystemCall() VFS method.





*/

#include "sqliteInt.h"
#include "sqlite3.h"
#include "tcl.h"
#include <stdlib.h>
#include <string.h>







>
>
>
>
>







63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
**
**   test_syscall exists SYSTEM-CALL
**     Return true if the named system call exists. Or false otherwise.
**
**   test_syscall list
**     Return a list of all system calls. The list is constructed using
**     the xNextSystemCall() VFS method.
**
**   test_syscall pagesize PGSZ
**     If PGSZ is a power of two greater than 256, install a wrapper around
**     OS function getpagesize() that reports the system page size as PGSZ.
**     Or, if PGSZ is less than zero, remove any wrapper already installed.
*/

#include "sqliteInt.h"
#include "sqlite3.h"
#include "tcl.h"
#include <stdlib.h>
#include <string.h>
85
86
87
88
89
90
91


92
93
94
95
96
97
98
99
#include <sys/types.h>
#include <errno.h>

static struct TestSyscallGlobal {
  int bPersist;                   /* 1 for persistent errors, 0 for transient */
  int nCount;                     /* Fail after this many more calls */
  int nFail;                      /* Number of failures that have occurred */


} gSyscall = { 0, 0 };

static int ts_open(const char *, int, int);
static int ts_close(int fd);
static int ts_access(const char *zPath, int mode);
static char *ts_getcwd(char *zPath, size_t nPath);
static int ts_stat(const char *zPath, struct stat *p);
static int ts_fstat(int fd, struct stat *p);







>
>
|







90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
#include <sys/types.h>
#include <errno.h>

static struct TestSyscallGlobal {
  int bPersist;                   /* 1 for persistent errors, 0 for transient */
  int nCount;                     /* Fail after this many more calls */
  int nFail;                      /* Number of failures that have occurred */
  int pgsz;
  sqlite3_syscall_ptr orig_getpagesize;
} gSyscall = { 0, 0, 0, 0, 0 };

static int ts_open(const char *, int, int);
static int ts_close(int fd);
static int ts_access(const char *zPath, int mode);
static char *ts_getcwd(char *zPath, size_t nPath);
static int ts_stat(const char *zPath, struct stat *p);
static int ts_fstat(int fd, struct stat *p);
645
646
647
648
649
650
651







































652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670

671
672
673
674
675
676
677
    return TCL_ERROR;
  }

  pVfs = sqlite3_vfs_find(0);
  Tcl_SetObjResult(interp, Tcl_NewStringObj(pVfs->zName, -1));
  return TCL_OK;
}








































static int test_syscall(
  void * clientData,
  Tcl_Interp *interp,
  int objc,
  Tcl_Obj *CONST objv[]
){
  struct SyscallCmd {
    const char *zName;
    Tcl_ObjCmdProc *xCmd;
  } aCmd[] = {
    { "fault",      test_syscall_fault },
    { "install",    test_syscall_install },
    { "uninstall",  test_syscall_uninstall },
    { "reset",      test_syscall_reset },
    { "errno",      test_syscall_errno },
    { "exists",     test_syscall_exists },
    { "list",       test_syscall_list },
    { "defaultvfs", test_syscall_defaultvfs },

    { 0, 0 }
  };
  int iCmd;
  int rc;

  if( objc<2 ){
    Tcl_WrongNumArgs(interp, 1, objv, "SUB-COMMAND ...");







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>



















>







652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
    return TCL_ERROR;
  }

  pVfs = sqlite3_vfs_find(0);
  Tcl_SetObjResult(interp, Tcl_NewStringObj(pVfs->zName, -1));
  return TCL_OK;
}

static int ts_getpagesize(void){
  return gSyscall.pgsz;
}

static int test_syscall_pagesize(
  void * clientData,
  Tcl_Interp *interp,
  int objc,
  Tcl_Obj *CONST objv[]
){
  sqlite3_vfs *pVfs = sqlite3_vfs_find(0);
  int pgsz;
  if( objc!=3 ){
    Tcl_WrongNumArgs(interp, 2, objv, "PGSZ");
    return TCL_ERROR;
  }
  if( Tcl_GetIntFromObj(interp, objv[2], &pgsz) ){
    return TCL_ERROR;
  }

  if( pgsz<0 ){
    if( gSyscall.orig_getpagesize ){
      pVfs->xSetSystemCall(pVfs, "getpagesize", gSyscall.orig_getpagesize);
    }
  }else{
    if( pgsz<512 || (pgsz & (pgsz-1)) ){
      Tcl_AppendResult(interp, "pgsz out of range", 0);
      return TCL_ERROR;
    }
    gSyscall.orig_getpagesize = pVfs->xGetSystemCall(pVfs, "getpagesize");
    gSyscall.pgsz = pgsz;
    pVfs->xSetSystemCall(
        pVfs, "getpagesize", (sqlite3_syscall_ptr)ts_getpagesize
    );
  }

  return TCL_OK;
}

static int test_syscall(
  void * clientData,
  Tcl_Interp *interp,
  int objc,
  Tcl_Obj *CONST objv[]
){
  struct SyscallCmd {
    const char *zName;
    Tcl_ObjCmdProc *xCmd;
  } aCmd[] = {
    { "fault",      test_syscall_fault },
    { "install",    test_syscall_install },
    { "uninstall",  test_syscall_uninstall },
    { "reset",      test_syscall_reset },
    { "errno",      test_syscall_errno },
    { "exists",     test_syscall_exists },
    { "list",       test_syscall_list },
    { "defaultvfs", test_syscall_defaultvfs },
    { "pagesize",   test_syscall_pagesize },
    { 0, 0 }
  };
  int iCmd;
  int rc;

  if( objc<2 ){
    Tcl_WrongNumArgs(interp, 1, objv, "SUB-COMMAND ...");
Changes to test/syscall.test.
57
58
59
60
61
62
63

64
65
66
67
68
69
70
# Tests for the xNextSystemCall method.
#
foreach s {
    open close access getcwd stat fstat ftruncate
    fcntl read pread write pwrite fchmod fallocate
    pread64 pwrite64 unlink openDirectory mkdir rmdir 
    statvfs fchown umask mmap munmap mremap

} {
  if {[test_syscall exists $s]} {lappend syscall_list $s}
}
do_test 3.1 { lsort [test_syscall list] } [lsort $syscall_list]

#-------------------------------------------------------------------------
# This test verifies that if a call to open() fails and errno is set to







>







57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
# Tests for the xNextSystemCall method.
#
foreach s {
    open close access getcwd stat fstat ftruncate
    fcntl read pread write pwrite fchmod fallocate
    pread64 pwrite64 unlink openDirectory mkdir rmdir 
    statvfs fchown umask mmap munmap mremap
    getpagesize
} {
  if {[test_syscall exists $s]} {lappend syscall_list $s}
}
do_test 3.1 { lsort [test_syscall list] } [lsort $syscall_list]

#-------------------------------------------------------------------------
# This test verifies that if a call to open() fails and errno is set to
Added test/wal64k.test.






























































































>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
# 2010 April 13
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
# This file implements regression tests for SQLite library.  The
# focus of this file is testing the operation of the library in
# "PRAGMA journal_mode=WAL" mode.
#

set testdir [file dirname $argv0]
source $testdir/tester.tcl
set testprefix wal64k

ifcapable !wal {finish_test ; return }

db close
test_syscall pagesize 65536
sqlite3 db test.db

do_execsql_test 1.0 { 
  PRAGMA journal_mode = WAL;
  CREATE TABLE t1(x);
  CREATE INDEX i1 ON t1(x);
} {wal}
do_test 1.1 { file size test.db-shm } {65536}

do_test 1.2 {
  execsql BEGIN
  while {[file size test.db-shm]==65536} {
    execsql { INSERT INTO t1 VALUES( randstr(900,1100) ) }
  }
  execsql COMMIT
  file size test.db-shm
} {131072}

integrity_check 1.3

db close
test_syscall pagesize -1
finish_test