/ Changes On Branch dump-recovery
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Changes In Branch dump-recovery Excluding Merge-Ins

This is equivalent to a diff from 50eec5d9aa to 5dc82a155d

2017-03-09
18:13
Fix the ".dump" command to correctly extract tail data from corrupt WITHOUT ROWID tables. (check-in: 6c627e5062 user: drh tags: trunk)
16:32
Add the --recovery-mode option to the CLI ".dump" command. This involves enhancing the "PRAGMA reverse_unordered_selects" command to accept a "TOGGLE" option. Recovery mode used to be always on. Now it is opt-in. Not sure this is the correct approach. Also not sure if TOGGLE is a good feature to have on boolean PRAGMA statements. (Closed-Leaf check-in: 5dc82a155d user: drh tags: dump-recovery)
13:50
Begin moving separate boolean variables in the ShellState object of the CLI into the shellFlgs bitmask. (check-in: 50eec5d9aa user: drh tags: trunk)
2017-03-08
18:06
Make sure the database connection is open prior to running the ".sha3sum" command. (check-in: 2ea300fb8f user: drh tags: trunk)

Changes to src/pragma.c.

  1041   1041         }
  1042   1042   #if SQLITE_USER_AUTHENTICATION
  1043   1043         if( db->auth.authLevel==UAUTH_User ){
  1044   1044           /* Do not allow non-admin users to modify the schema arbitrarily */
  1045   1045           mask &= ~(SQLITE_WriteSchema);
  1046   1046         }
  1047   1047   #endif
  1048         -
  1049         -      if( sqlite3GetBoolean(zRight, 0) ){
         1048  +      if( sqlite3StrICmp(zRight, "toggle")==0 ){
         1049  +        db->flags ^= mask;
         1050  +      }else if( sqlite3GetBoolean(zRight, 0) ){
  1050   1051           db->flags |= mask;
  1051   1052         }else{
  1052   1053           db->flags &= ~mask;
  1053         -        if( mask==SQLITE_DeferFKs ) db->nDeferredImmCons = 0;
         1054  +      }
         1055  +      if( mask==SQLITE_DeferFKs && (db->flags & mask)==0 ){
         1056  +        db->nDeferredImmCons = 0;
  1054   1057         }
  1055   1058   
  1056   1059         /* Many of the flag-pragmas modify the code generated by the SQL 
  1057   1060         ** compiler (eg. count_changes). So add an opcode to expire all
  1058   1061         ** compiled SQL statements after modifying a pragma value.
  1059   1062         */
  1060   1063         sqlite3VdbeAddOp0(v, OP_Expire);

Changes to src/shell.c.

  1406   1406   #define SHFLG_Scratch        0x00000001 /* The --scratch option is used */
  1407   1407   #define SHFLG_Pagecache      0x00000002 /* The --pagecache option is used */
  1408   1408   #define SHFLG_Lookaside      0x00000004 /* Lookaside memory is used */
  1409   1409   #define SHFLG_Backslash      0x00000008 /* The --backslash option is used */
  1410   1410   #define SHFLG_PreserveRowid  0x00000010 /* .dump preserves rowid values */
  1411   1411   #define SHFLG_CountChanges   0x00000020 /* .changes setting */
  1412   1412   #define SHFLG_Echo           0x00000040 /* .echo or --echo setting */
         1413  +#define SHFLG_RecoveryMode   0x00000080 /* The --recovery-mode flag for .dump */
  1413   1414   
  1414   1415   /*
  1415   1416   ** Macros for testing and setting shellFlgs
  1416   1417   */
  1417   1418   #define ShellHasFlag(P,X)    (((P)->shellFlgs & (X))!=0)
  1418   1419   #define ShellSetFlag(P,X)    ((P)->shellFlgs|=(X))
  1419   1420   #define ShellClearFlag(P,X)  ((P)->shellFlgs&=(~(X)))
................................................................................
  2215   2216       raw_printf(pArg->out,
  2216   2217               "Memory Used:                         %d (max %d) bytes\n",
  2217   2218               iCur, iHiwtr);
  2218   2219       iHiwtr = iCur = -1;
  2219   2220       sqlite3_status(SQLITE_STATUS_MALLOC_COUNT, &iCur, &iHiwtr, bReset);
  2220   2221       raw_printf(pArg->out, "Number of Outstanding Allocations:   %d (max %d)\n",
  2221   2222               iCur, iHiwtr);
  2222         -    if( pArg->shellFlgs & SHFLG_Pagecache ){
         2223  +    if( ShellHasFlag(pArg, SHFLG_Pagecache) ){
  2223   2224         iHiwtr = iCur = -1;
  2224   2225         sqlite3_status(SQLITE_STATUS_PAGECACHE_USED, &iCur, &iHiwtr, bReset);
  2225   2226         raw_printf(pArg->out,
  2226   2227                 "Number of Pcache Pages Used:         %d (max %d) pages\n",
  2227   2228                 iCur, iHiwtr);
  2228   2229       }
  2229   2230       iHiwtr = iCur = -1;
  2230   2231       sqlite3_status(SQLITE_STATUS_PAGECACHE_OVERFLOW, &iCur, &iHiwtr, bReset);
  2231   2232       raw_printf(pArg->out,
  2232   2233               "Number of Pcache Overflow Bytes:     %d (max %d) bytes\n",
  2233   2234               iCur, iHiwtr);
  2234         -    if( pArg->shellFlgs & SHFLG_Scratch ){
         2235  +    if( ShellHasFlag(pArg, SHFLG_Scratch) ){
  2235   2236         iHiwtr = iCur = -1;
  2236   2237         sqlite3_status(SQLITE_STATUS_SCRATCH_USED, &iCur, &iHiwtr, bReset);
  2237   2238         raw_printf(pArg->out,
  2238   2239                 "Number of Scratch Allocations Used:  %d (max %d)\n",
  2239   2240                 iCur, iHiwtr);
  2240   2241       }
  2241   2242       iHiwtr = iCur = -1;
................................................................................
  2260   2261       sqlite3_status(SQLITE_STATUS_PARSER_STACK, &iCur, &iHiwtr, bReset);
  2261   2262       raw_printf(pArg->out, "Deepest Parser Stack:                %d (max %d)\n",
  2262   2263               iCur, iHiwtr);
  2263   2264   #endif
  2264   2265     }
  2265   2266   
  2266   2267     if( pArg && pArg->out && db ){
  2267         -    if( pArg->shellFlgs & SHFLG_Lookaside ){
         2268  +    if( ShellHasFlag(pArg, SHFLG_Lookaside) ){
  2268   2269         iHiwtr = iCur = -1;
  2269   2270         sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_USED,
  2270   2271                           &iCur, &iHiwtr, bReset);
  2271   2272         raw_printf(pArg->out,
  2272   2273                 "Lookaside Slots Used:                %d (max %d)\n",
  2273   2274                 iCur, iHiwtr);
  2274   2275         sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_HIT,
................................................................................
  2942   2943       appendText(&sSelect, zTable, quoteChar(zTable));
  2943   2944   
  2944   2945       savedDestTable = p->zDestTable;
  2945   2946       savedMode = p->mode;
  2946   2947       p->zDestTable = sTable.z;
  2947   2948       p->mode = p->cMode = MODE_Insert;
  2948   2949       rc = shell_exec(p->db, sSelect.z, shell_callback, p, 0);
         2950  +    if( (rc&0xff)==SQLITE_CORRUPT ){
         2951  +      raw_printf(p->out, "/****** CORRUPTION ERROR *******/\n");
         2952  +      if( ShellHasFlag(p, SHFLG_RecoveryMode) ){
         2953  +        sqlite3_exec(p->db, "PRAGMA reverse_unordered_selects=TOGGLE",0,0,0);
         2954  +        shell_exec(p->db, sSelect.z, shell_callback, p, 0);
         2955  +        sqlite3_exec(p->db, "PRAGMA reverse_unordered_selects=TOGGLE",0,0,0);
         2956  +      }
         2957  +    }
  2949   2958       p->zDestTable = savedDestTable;
  2950   2959       p->mode = savedMode;
  2951   2960       freeText(&sTable);
  2952   2961       freeText(&sSelect);
  2953   2962       if( rc ) p->nErr++;
  2954   2963     }
  2955   2964     return 0;
  2956   2965   }
  2957   2966   
  2958   2967   /*
  2959   2968   ** Run zQuery.  Use dump_callback() as the callback routine so that
  2960   2969   ** the contents of the query are output as SQL statements.
  2961   2970   **
  2962         -** If we get a SQLITE_CORRUPT error, rerun the query after appending
  2963         -** "ORDER BY rowid DESC" to the end.
         2971  +** If we get a SQLITE_CORRUPT error and the --recovery-mode flag is
         2972  +** set, then rerun the query in reverse order to try to get more
         2973  +** data.
  2964   2974   */
  2965   2975   static int run_schema_dump_query(
  2966   2976     ShellState *p,
  2967   2977     const char *zQuery
  2968   2978   ){
  2969   2979     int rc;
  2970   2980     char *zErr = 0;
  2971   2981     rc = sqlite3_exec(p->db, zQuery, dump_callback, p, &zErr);
  2972         -  if( rc==SQLITE_CORRUPT ){
         2982  +  if( rc==SQLITE_CORRUPT && ShellHasFlag(p, SHFLG_RecoveryMode) ){
  2973   2983       char *zQ2;
  2974   2984       int len = strlen30(zQuery);
  2975   2985       raw_printf(p->out, "/****** CORRUPTION ERROR *******/\n");
  2976   2986       if( zErr ){
  2977   2987         utf8_printf(p->out, "/****** %s ******/\n", zErr);
  2978   2988         sqlite3_free(zErr);
  2979   2989         zErr = 0;
................................................................................
  4566   4576     if( c=='d' && strncmp(azArg[0], "dbinfo", n)==0 ){
  4567   4577       rc = shell_dbinfo_command(p, nArg, azArg);
  4568   4578     }else
  4569   4579   
  4570   4580     if( c=='d' && strncmp(azArg[0], "dump", n)==0 ){
  4571   4581       const char *zLike = 0;
  4572   4582       int i;
  4573         -    ShellClearFlag(p, SHFLG_PreserveRowid);
         4583  +    ShellClearFlag(p, SHFLG_PreserveRowid|SHFLG_RecoveryMode);
  4574   4584       for(i=1; i<nArg; i++){
  4575   4585         if( azArg[i][0]=='-' ){
  4576   4586           const char *z = azArg[i]+1;
  4577   4587           if( z[0]=='-' ) z++;
  4578   4588           if( strcmp(z,"preserve-rowids")==0 ){
  4579   4589             ShellSetFlag(p, SHFLG_PreserveRowid);
  4580   4590           }else
         4591  +        if( strcmp(z,"recovery-mode")==0 ){
         4592  +          ShellSetFlag(p, SHFLG_RecoveryMode);
         4593  +        }else
  4581   4594           {
  4582         -          raw_printf(stderr, "Unknown option \"%s\" on \".dump\"\n", azArg[i]);
         4595  +          raw_printf(stderr, "Unknown option \"%s\" on \".dump\"\n"
         4596  +                  "Should be one of: --preserve-rowids --recovery-mode\n",
         4597  +                  azArg[i]);
  4583   4598             rc = 1;
  4584   4599             goto meta_command_exit;
  4585   4600           }
  4586   4601         }else if( zLike ){
  4587   4602           raw_printf(stderr, "Usage: .dump ?--preserve-rowids? ?LIKE-PATTERN?\n");
  4588   4603           rc = 1;
  4589   4604           goto meta_command_exit;
................................................................................
  4594   4609       open_db(p, 0);
  4595   4610       /* When playing back a "dump", the content might appear in an order
  4596   4611       ** which causes immediate foreign key constraints to be violated.
  4597   4612       ** So disable foreign-key constraint enforcement to prevent problems. */
  4598   4613       raw_printf(p->out, "PRAGMA foreign_keys=OFF;\n");
  4599   4614       raw_printf(p->out, "BEGIN TRANSACTION;\n");
  4600   4615       p->writableSchema = 0;
  4601         -    /* Set writable_schema=ON since doing so forces SQLite to initialize
  4602         -    ** as much of the schema as it can even if the sqlite_master table is
  4603         -    ** corrupt. */
  4604         -    sqlite3_exec(p->db, "SAVEPOINT dump; PRAGMA writable_schema=ON", 0, 0, 0);
         4616  +    if( ShellHasFlag(p, SHFLG_RecoveryMode) ){
         4617  +      /* Set writable_schema=ON since doing so forces SQLite to initialize
         4618  +      ** as much of the schema as it can even if the sqlite_master table is
         4619  +      ** corrupt. */
         4620  +      sqlite3_exec(p->db, "SAVEPOINT dump; PRAGMA writable_schema=ON", 0, 0, 0);
         4621  +    }
  4605   4622       p->nErr = 0;
  4606   4623       if( zLike==0 ){
  4607   4624         run_schema_dump_query(p,
  4608   4625           "SELECT name, type, sql FROM sqlite_master "
  4609   4626           "WHERE sql NOT NULL AND type=='table' AND name!='sqlite_sequence'"
  4610   4627         );
  4611   4628         run_schema_dump_query(p,
................................................................................
  4632   4649         run_table_dump_query(p, zSql, 0);
  4633   4650         sqlite3_free(zSql);
  4634   4651       }
  4635   4652       if( p->writableSchema ){
  4636   4653         raw_printf(p->out, "PRAGMA writable_schema=OFF;\n");
  4637   4654         p->writableSchema = 0;
  4638   4655       }
  4639         -    sqlite3_exec(p->db, "PRAGMA writable_schema=OFF;", 0, 0, 0);
  4640         -    sqlite3_exec(p->db, "RELEASE dump;", 0, 0, 0);
         4656  +    if( ShellHasFlag(p, SHFLG_RecoveryMode) ){
         4657  +      sqlite3_exec(p->db, "PRAGMA writable_schema=OFF;", 0, 0, 0);
         4658  +      sqlite3_exec(p->db, "RELEASE dump;", 0, 0, 0);
         4659  +    }
  4641   4660       raw_printf(p->out, p->nErr ? "ROLLBACK; -- due to errors\n" : "COMMIT;\n");
  4642   4661     }else
  4643   4662   
  4644   4663     if( c=='e' && strncmp(azArg[0], "echo", n)==0 ){
  4645   4664       if( nArg==2 ){
  4646   4665         setOrClearFlag(p, SHFLG_Echo, azArg[1]);
  4647   4666       }else{