SQLite

/* 
** The "Set Error Line" macro.
*/
#define SEL(e) ((e)->iLine = ((e)->rc ? (e)->iLine : __LINE__))

/* Database functions */
#define opendb(w,x,y,z,f)       (SEL(w), opendb_x(w,x,y,z,f))
#define closedb(y,z)            (SEL(y), closedb_x(y,z))

/* Functions to execute SQL */
#define sql_script(x,y,z)       (SEL(x), sql_script_x(x,y,z))
#define integrity_check(x,y)    (SEL(x), integrity_check_x(x,y))
#define execsql_i64(x,y,...)    (SEL(x), execsql_i64_x(x,y,__VA_ARGS__))
#define execsql_text(x,y,z,...) (SEL(x), execsql_text_x(x,y,z,__VA_ARGS__))

  return 1;
}

static void opendb_x(
  Error *pErr,                    /* IN/OUT: Error code */
  Sqlite *pDb,                    /* OUT: Database handle */
  const char *zFile,              /* Database file name */
  int bDelete,                    /* True to delete db file before opening */
  int flags
){
  if( pErr->rc==SQLITE_OK ){
    int rc;
    if( flags==0 ){
      flags = SQLITE_OPEN_CREATE | SQLITE_OPEN_READWRITE | SQLITE_OPEN_URI;
    }
    if( bDelete ) unlink(zFile);
    rc = sqlite3_open_v2(zFile, &pDb->db, flags, 0);
    if( rc ){
      sqlite_error(pErr, pDb, "open");
      sqlite3_close(pDb->db);
      pDb->db = 0;
    }else{

#define WALTHREAD3_NTHREAD  6

static char *walthread1_thread(int iTid, void *pArg){
  Error err = {0};                /* Error code and message */
  Sqlite db = {0};                /* SQLite database connection */
  int nIter = 0;                  /* Iterations so far */

  opendb(&err, &db, "test.db", 0, 0);
  while( !timetostop(&err) ){
    const char *azSql[] = {
      "SELECT md5sum(x) FROM t1 WHERE rowid != (SELECT max(rowid) FROM t1)",
      "SELECT x FROM t1 WHERE rowid = (SELECT max(rowid) FROM t1)",
    };
    char *z1, *z2, *z3;

}

static char *walthread1_ckpt_thread(int iTid, void *pArg){
  Error err = {0};                /* Error code and message */
  Sqlite db = {0};                /* SQLite database connection */
  int nCkpt = 0;                  /* Checkpoints so far */

  opendb(&err, &db, "test.db", 0, 0);
  while( !timetostop(&err) ){
    usleep(500*1000);
    execsql(&err, &db, "PRAGMA wal_checkpoint");
    if( err.rc==SQLITE_OK ) nCkpt++;
    clear_error(&err, SQLITE_BUSY);
  }
  closedb(&err, &db);

  print_and_free_err(&err);
  return sqlite3_mprintf("%d checkpoints", nCkpt);
}

static void walthread1(int nMs){
  Error err = {0};                /* Error code and message */
  Sqlite db = {0};                /* SQLite database connection */
  Threadset threads = {0};        /* Test threads */
  int i;                          /* Iterator variable */

  opendb(&err, &db, "test.db", 1, 0);
  sql_script(&err, &db,
      "PRAGMA journal_mode = WAL;"
      "CREATE TABLE t1(x PRIMARY KEY);"
      "INSERT INTO t1 VALUES(randomblob(100));"
      "INSERT INTO t1 VALUES(randomblob(100));"
      "INSERT INTO t1 SELECT md5sum(x) FROM t1;"
  );

  const char *zJournal = "PRAGMA journal_mode = WAL";
  if( iArg ){ zJournal = "PRAGMA journal_mode = DELETE"; }

  while( !timetostop(&err) ){
    int journal_exists = 0;
    int wal_exists = 0;

    opendb(&err, &db, "test.db", 0, 0);

    sql_script(&err, &db, zJournal);
    clear_error(&err, SQLITE_BUSY);
    sql_script(&err, &db, "BEGIN");
    sql_script(&err, &db, "INSERT INTO t1 VALUES(NULL, randomblob(100))");

    journal_exists = (filesize(&err, "test.db-journal") >= 0);

}

static void walthread2(int nMs){
  Error err = {0};
  Sqlite db = {0};
  Threadset threads = {0};

  opendb(&err, &db, "test.db", 1, 0);
  sql_script(&err, &db, "CREATE TABLE t1(x INTEGER PRIMARY KEY, y UNIQUE)");
  closedb(&err, &db);

  setstoptime(&err, nMs);
  launch_thread(&err, &threads, walthread2_thread, 0);
  launch_thread(&err, &threads, walthread2_thread, 0);
  launch_thread(&err, &threads, walthread2_thread, (void*)1);
  launch_thread(&err, &threads, walthread2_thread, (void*)1);
  join_all_threads(&err, &threads);

  print_and_free_err(&err);
}

static char *walthread3_thread(int iTid, void *pArg){
  Error err = {0};                /* Error code and message */
  Sqlite db = {0};                /* SQLite database connection */
  i64 iNextWrite;                 /* Next value this thread will write */
  int iArg = PTR2INT(pArg);

  opendb(&err, &db, "test.db", 0, 0);
  sql_script(&err, &db, "PRAGMA wal_autocheckpoint = 10");

  iNextWrite = iArg+1;
  while( 1 ){
    i64 sum1;
    i64 sum2;
    int stop = 0;                 /* True to stop executing (test timed out) */

static void walthread3(int nMs){
  Error err = {0};
  Sqlite db = {0};
  Threadset threads = {0};
  int i;

  opendb(&err, &db, "test.db", 1, 0);
  sql_script(&err, &db, 
      "PRAGMA journal_mode = WAL;"
      "CREATE TABLE t1(cnt PRIMARY KEY, sum1, sum2);"
      "CREATE INDEX i1 ON t1(sum1);"
      "CREATE INDEX i2 ON t1(sum2);"
      "INSERT INTO t1 VALUES(0, 0, 0);"
  );

  print_and_free_err(&err);
}

static char *walthread4_reader_thread(int iTid, void *pArg){
  Error err = {0};                /* Error code and message */
  Sqlite db = {0};                /* SQLite database connection */

  opendb(&err, &db, "test.db", 0, 0);
  while( !timetostop(&err) ){
    integrity_check(&err, &db);
  }
  closedb(&err, &db);

  print_and_free_err(&err);
  return 0;
}

static char *walthread4_writer_thread(int iTid, void *pArg){
  Error err = {0};                /* Error code and message */
  Sqlite db = {0};                /* SQLite database connection */
  i64 iRow = 1;

  opendb(&err, &db, "test.db", 0, 0);
  sql_script(&err, &db, "PRAGMA wal_autocheckpoint = 15;");
  while( !timetostop(&err) ){
    execsql_i64(
        &err, &db, "REPLACE INTO t1 VALUES(:iRow, randomblob(300))", &iRow
    );
    iRow++;
    if( iRow==10 ) iRow = 0;
  }
  closedb(&err, &db);

  print_and_free_err(&err);
  return 0;
}

static void walthread4(int nMs){
  Error err = {0};
  Sqlite db = {0};
  Threadset threads = {0};

  opendb(&err, &db, "test.db", 1, 0);
  sql_script(&err, &db, 
      "PRAGMA journal_mode = WAL;"
      "CREATE TABLE t1(a INTEGER PRIMARY KEY, b UNIQUE);"
  );
  closedb(&err, &db);

  setstoptime(&err, nMs);
  launch_thread(&err, &threads, walthread4_reader_thread, 0);
  launch_thread(&err, &threads, walthread4_writer_thread, 0);
  join_all_threads(&err, &threads);

  print_and_free_err(&err);
}

static char *walthread5_thread(int iTid, void *pArg){
  Error err = {0};                /* Error code and message */
  Sqlite db = {0};                /* SQLite database connection */
  i64 nRow;

  opendb(&err, &db, "test.db", 0, 0);
  nRow = execsql_i64(&err, &db, "SELECT count(*) FROM t1");
  closedb(&err, &db);

  if( nRow!=65536 ) test_error(&err, "Bad row count: %d", (int)nRow);
  print_and_free_err(&err);
  return 0;
}
static void walthread5(int nMs){
  Error err = {0};
  Sqlite db = {0};
  Threadset threads = {0};

  opendb(&err, &db, "test.db", 1, 0);
  sql_script(&err, &db, 
      "PRAGMA wal_autocheckpoint = 0;"
      "PRAGMA page_size = 1024;"
      "PRAGMA journal_mode = WAL;"
      "CREATE TABLE t1(x);"
      "BEGIN;"
      "INSERT INTO t1 VALUES(randomblob(900));"

  sql_script(pErr, pDb, "COMMIT");
}
static void cgt_pager_1(int nMs){
  void (*xSub)(Error *, Sqlite *);
  Error err = {0};
  Sqlite db = {0};

  opendb(&err, &db, "test.db", 1, 0);
  sql_script(&err, &db,
      "PRAGMA cache_size = 2000;"
      "PRAGMA page_size = 1024;"
      "CREATE TABLE t1(a INTEGER PRIMARY KEY, b BLOB);"
  );

  xSub = cgt_pager_1_populate; xSub(&err, &db);

static char *dynamic_triggers_1(int iTid, void *pArg){
  Error err = {0};                /* Error code and message */
  Sqlite db = {0};                /* SQLite database connection */
  int nDrop = 0;
  int nCreate = 0;

  opendb(&err, &db, "test.db", 0, 0);
  while( !timetostop(&err) ){
    int i;

    for(i=1; i<9; i++){
      char *zSql = sqlite3_mprintf(
        "CREATE TRIGGER itr%d BEFORE INSERT ON t%d BEGIN "
          "INSERT INTO t%d VALUES(new.x, new.y);"

static char *dynamic_triggers_2(int iTid, void *pArg){
  Error err = {0};                /* Error code and message */
  Sqlite db = {0};                /* SQLite database connection */
  i64 iVal = 0;
  int nInsert = 0;
  int nDelete = 0;

  opendb(&err, &db, "test.db", 0, 0);
  while( !timetostop(&err) ){
    do {
      iVal = (iVal+1)%100;
      execsql(&err, &db, "INSERT INTO t1 VALUES(:iX, :iY+1)", &iVal, &iVal);
      nInsert++;
    } while( iVal );

}

static void dynamic_triggers(int nMs){
  Error err = {0};
  Sqlite db = {0};
  Threadset threads = {0};

  opendb(&err, &db, "test.db", 1, 0);
  sql_script(&err, &db, 
      "PRAGMA page_size = 1024;"
      "PRAGMA journal_mode = WAL;"
      "CREATE TABLE t1(x, y);"
      "CREATE TABLE t2(x, y);"
      "CREATE TABLE t3(x, y);"
      "CREATE TABLE t4(x, y);"

#include "tt3_checkpoint.c"
#include "tt3_index.c"
#include "tt3_lookaside1.c"
#include "tt3_vacuum.c"
#include "tt3_stress.c"
#include "tt3_reuseschema.c"

int main(int argc, char **argv){
  struct ThreadTest {
    void (*xTest)(int);   /* Routine for running this test */
    const char *zTest;    /* Name of this test */
    int nMs;              /* How long to run this test, in milliseconds */
  } aTest[] = {

    { checkpoint_starvation_2, "checkpoint_starvation_2", 10000 },

    { create_drop_index_1, "create_drop_index_1", 10000 },
    { lookaside1,          "lookaside1", 10000 },
    { vacuum1,             "vacuum1", 10000 },
    { stress1,             "stress1", 10000 },
    { stress2,             "stress2", 60000 },
    { reuse_schema_1,      "reuse_schema_1", 20000 },
  };
  static char *substArgv[] = { 0, "*", 0 };
  int i, iArg;
  int nTestfound = 0;

  sqlite3_config(SQLITE_CONFIG_MULTITHREAD);
  if( argc<2 ){

  return SQLITE_OK;
}

static char *checkpoint_starvation_reader(int iTid, void *pArg){
  Error err = {0};
  Sqlite db = {0};

  opendb(&err, &db, "test.db", 0, 0);
  while( !timetostop(&err) ){
    i64 iCount1, iCount2;
    sql_script(&err, &db, "BEGIN");
    iCount1 = execsql_i64(&err, &db, "SELECT count(x) FROM t1");
    usleep(CHECKPOINT_STARVATION_READMS*1000);
    iCount2 = execsql_i64(&err, &db, "SELECT count(x) FROM t1");
    sql_script(&err, &db, "COMMIT");

static void checkpoint_starvation_main(int nMs, CheckpointStarvationCtx *p){
  Error err = {0};
  Sqlite db = {0};
  Threadset threads = {0};
  int nInsert = 0;
  int i;

  opendb(&err, &db, "test.db", 1, 0);
  sql_script(&err, &db, 
      "PRAGMA page_size = 1024;"
      "PRAGMA journal_mode = WAL;"
      "CREATE TABLE t1(x);"
  );

  setstoptime(&err, nMs);

static char *create_drop_index_thread(int iTid, void *pArg){
  Error err = {0};                /* Error code and message */
  Sqlite db = {0};                /* SQLite database connection */

  while( !timetostop(&err) ){
    opendb(&err, &db, "test.db", 0, 0);

    sql_script(&err, &db, 
      "DROP INDEX IF EXISTS i1;"
      "DROP INDEX IF EXISTS i2;"
      "DROP INDEX IF EXISTS i3;"
      "DROP INDEX IF EXISTS i4;"

}

static void create_drop_index_1(int nMs){
  Error err = {0};
  Sqlite db = {0};
  Threadset threads = {0};

  opendb(&err, &db, "test.db", 1, 0);
  sql_script(&err, &db, 
     "CREATE TABLE t11(a, b, c, d);"
     "WITH data(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM data WHERE x<100) "
     "INSERT INTO t11 SELECT x,x,x,x FROM data;"
  );
  closedb(&err, &db);

** that is suspected to exist at time of writing.
*/

static char *lookaside1_thread_reader(int iTid, void *pArg){
  Error err = {0};                /* Error code and message */
  Sqlite db = {0};                /* SQLite database connection */

  opendb(&err, &db, "test.db", 0, 0);

  while( !timetostop(&err) ){
    sqlite3_stmt *pStmt = 0;
    int rc;

    sqlite3_prepare_v2(db.db, "SELECT 1 FROM t1", -1, &pStmt, 0);
    while( sqlite3_step(pStmt)==SQLITE_ROW ){

  return sqlite3_mprintf("ok");
}

static char *lookaside1_thread_writer(int iTid, void *pArg){
  Error err = {0};                /* Error code and message */
  Sqlite db = {0};                /* SQLite database connection */

  opendb(&err, &db, "test.db", 0, 0);

  do{
    sql_script(&err, &db, 
      "BEGIN;"
        "UPDATE t3 SET i=i+1 WHERE x=1;"
      "ROLLBACK;"
    );
  }while( !timetostop(&err) );

  closedb(&err, &db);
  print_and_free_err(&err);
  return sqlite3_mprintf("ok");
}


static void lookaside1(int nMs){
  Error err = {0};
  Sqlite db = {0};
  Threadset threads = {0};

  opendb(&err, &db, "test.db", 1, 0);
  sql_script(&err, &db, 
     "CREATE TABLE t1(x PRIMARY KEY) WITHOUT ROWID;"
     "WITH data(x,y) AS ("
     "  SELECT 1, quote(randomblob(750)) UNION ALL "
     "  SELECT x*2, y||y FROM data WHERE x<5) "
     "INSERT INTO t1 SELECT y FROM data;"

/*
** 2014 December 9
**
** The author disclaims copyright to this source code.  In place of
** a legal notice, here is a blessing:
**
**    May you do good and not evil.
**    May you find forgiveness for yourself and forgive others.
**    May you share freely, never taking more than you give.
**
*************************************************************************
**
**     reuse_schema_1
*/


static char *reuse_schema_thread(int iTid, void *pArg){
  Error err = {0};                /* Error code and message */
  Sqlite db = {0};                /* SQLite database connection */
  int iRep = 0;

  while( !timetostop(&err) ){
    int f = SQLITE_OPEN_READWRITE|SQLITE_OPEN_SHARED_SCHEMA;
    opendb(&err, &db, "test.db", 0, f);

    execsql_i64(&err, &db, "SELECT count(*) FROM t1");
    sql_script(&err, &db, "ATTACH 'test.db2' AS aux");
    execsql_i64(&err, &db, "SELECT count(*) FROM t1");

    closedb(&err, &db);
    iRep++;
  }

  print_and_free_err(&err);
  return sqlite3_mprintf("%d", iRep);
}

static void reuse_schema_1(int nMs){
  Error err = {0};
  Sqlite db = {0};
  Threadset threads = {0};

  opendb(&err, &db, "test.db", 1, 0);
  sql_script(&err, &db, 
     "CREATE TABLE t1(a, b, c, d);"
     "WITH data(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM data WHERE x<100) "
     "INSERT INTO t1 SELECT x,x,x,x FROM data;"
  );
  closedb(&err, &db);
  opendb(&err, &db, "test.db2", 1, 0);
  sql_script(&err, &db, 
#ifdef SQLITE_ENABLE_FTS5
     "CREATE VIRTUAL TABLE t2 USING fts5(a, b, c, d);"
#else
     "CREATE TABLE t2(a, b, c, d);"
#endif
     "WITH data(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM data WHERE x<100) "
     "INSERT INTO t2 SELECT x*2,x*2,x*2,x*2 FROM data;"
  );
  closedb(&err, &db);

  setstoptime(&err, nMs);

  launch_thread(&err, &threads, reuse_schema_thread, 0);
  launch_thread(&err, &threads, reuse_schema_thread, 0);
  launch_thread(&err, &threads, reuse_schema_thread, 0);
  launch_thread(&err, &threads, reuse_schema_thread, 0);
  launch_thread(&err, &threads, reuse_schema_thread, 0);

  join_all_threads(&err, &threads);
  sqlite3_enable_shared_cache(0);
  print_and_free_err(&err);
}

/*
** Thread 1. CREATE and DROP a table.
*/
static char *stress_thread_1(int iTid, void *pArg){
  Error err = {0};                /* Error code and message */
  Sqlite db = {0};                /* SQLite database connection */

  opendb(&err, &db, "test.db", 0, 0);
  while( !timetostop(&err) ){
    sql_script(&err, &db, "CREATE TABLE IF NOT EXISTS t1(a PRIMARY KEY, b)");
    clear_error(&err, SQLITE_LOCKED);
    sql_script(&err, &db, "DROP TABLE IF EXISTS t1");
    clear_error(&err, SQLITE_LOCKED);
  }
  closedb(&err, &db);
  print_and_free_err(&err);
  return sqlite3_mprintf("ok");
}

/*
** Thread 2. Open and close database connections.
*/
static char *stress_thread_2(int iTid, void *pArg){
  Error err = {0};                /* Error code and message */
  Sqlite db = {0};                /* SQLite database connection */
  while( !timetostop(&err) ){
    opendb(&err, &db, "test.db", 0, 0);
    sql_script(&err, &db, "SELECT * FROM sqlite_master;");
    clear_error(&err, SQLITE_LOCKED);
    closedb(&err, &db);
  }
  print_and_free_err(&err);
  return sqlite3_mprintf("ok");
}

/*
** Thread 3. Attempt many small SELECT statements.
*/
static char *stress_thread_3(int iTid, void *pArg){
  Error err = {0};                /* Error code and message */
  Sqlite db = {0};                /* SQLite database connection */

  int i1 = 0;
  int i2 = 0;

  opendb(&err, &db, "test.db", 0, 0);
  while( !timetostop(&err) ){
    sql_script(&err, &db, "SELECT * FROM t1 ORDER BY a;");
    i1++;
    if( err.rc ) i2++;
    clear_error(&err, SQLITE_LOCKED);
    clear_error(&err, SQLITE_ERROR);
  }

static char *stress_thread_4(int iTid, void *pArg){
  Error err = {0};                /* Error code and message */
  Sqlite db = {0};                /* SQLite database connection */
  int i1 = 0;
  int i2 = 0;
  int iArg = PTR2INT(pArg);

  opendb(&err, &db, "test.db", 0, 0);
  while( !timetostop(&err) ){
    if( iArg ){
      closedb(&err, &db);
      opendb(&err, &db, "test.db", 0, 0);
    }
    sql_script(&err, &db, 
        "WITH loop(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM loop LIMIT 200) "
        "INSERT INTO t1 VALUES(randomblob(60), randomblob(60));"
    );
    i1++;
    if( err.rc ) i2++;

  Error err = {0};                /* Error code and message */
  Sqlite db = {0};                /* SQLite database connection */
  int iArg = PTR2INT(pArg);

  int i1 = 0;
  int i2 = 0;

  opendb(&err, &db, "test.db", 0, 0);
  while( !timetostop(&err) ){
    i64 i = (i1 % 4);
    if( iArg ){
      closedb(&err, &db);
      opendb(&err, &db, "test.db", 0, 0);
    }
    execsql(&err, &db, "DELETE FROM t1 WHERE (rowid % 4)==:i", &i);
    i1++;
    if( err.rc ) i2++;
    clear_error(&err, SQLITE_LOCKED);
  }
  closedb(&err, &db);

}

static char *stress2_workload19(int iTid, void *pArg){
  Error err = {0};                /* Error code and message */
  Sqlite db = {0};                /* SQLite database connection */
  const char *zDb = (const char*)pArg;
  while( !timetostop(&err) ){
    opendb(&err, &db, zDb, 0, 0);
    sql_script(&err, &db, "SELECT * FROM sqlite_master;");
    clear_error(&err, SQLITE_LOCKED);
    closedb(&err, &db);
  }
  print_and_free_err(&err);
  return sqlite3_mprintf("ok");
}

  Error err = {0};                /* Error code and message */
  Sqlite db = {0};                /* SQLite database connection */
  int i1 = 0;
  int i2 = 0;

  while( !timetostop(&err) ){
    int cnt;
    opendb(&err, &db, pCtx->zDb, 0, 0);
    for(cnt=0; err.rc==SQLITE_OK && cnt<STRESS2_TABCNT; cnt++){
      pCtx->xProc(&err, &db, i1);
      i2 += (err.rc==SQLITE_OK);
      clear_error(&err, SQLITE_LOCKED);
      i1++;
    }
    closedb(&err, &db);

  int i;
  Error err = {0};
  Sqlite db = {0};
  Threadset threads = {0};

  /* To make sure the db file is empty before commencing */
  opendb(&err, &db, zDb, 1, 0);
  sql_script(&err, &db, 
      "CREATE TABLE IF NOT EXISTS t0(x PRIMARY KEY, y, z);"
      "CREATE INDEX IF NOT EXISTS i0 ON t0(y);"
  );
  closedb(&err, &db);

  setstoptime(&err, nMs);

**
*/


static char *vacuum1_thread_writer(int iTid, void *pArg){
  Error err = {0};                /* Error code and message */
  Sqlite db = {0};                /* SQLite database connection */
  opendb(&err, &db, "test.db", 0, 0);
  i64 i = 0;

  while( !timetostop(&err) ){
    i++;

    /* Insert lots of rows. Then delete some. */
    execsql(&err, &db, 

  print_and_free_err(&err);
  return sqlite3_mprintf("ok");
}

static char *vacuum1_thread_vacuumer(int iTid, void *pArg){
  Error err = {0};                /* Error code and message */
  Sqlite db = {0};                /* SQLite database connection */
  opendb(&err, &db, "test.db", 0, 0);

  do{
    sql_script(&err, &db, "VACUUM");
    clear_error(&err, SQLITE_LOCKED);
  }while( !timetostop(&err) );

  closedb(&err, &db);
  print_and_free_err(&err);
  return sqlite3_mprintf("ok");
}

static void vacuum1(int nMs){
  Error err = {0};
  Sqlite db = {0};
  Threadset threads = {0};

  opendb(&err, &db, "test.db", 1, 0);
  sql_script(&err, &db, 
     "CREATE TABLE t1(x PRIMARY KEY, y BLOB);"
     "CREATE INDEX i1 ON t1(y);"
  );
  closedb(&err, &db);

  setstoptime(&err, nMs);