/ Changes On Branch trim-nulls
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Changes In Branch trim-nulls Excluding Merge-Ins

This is equivalent to a diff from 8cd1a4451c to c7651d21bf

2017-01-25
20:55
Trim NULL values off the end of records when the SQLITE_ENABLE_NULL_TRIM compile-time option is used. Increase the size of the P5 operand to 16 bits. Fix a problem with short records in the sessions extension. (check-in: 4801bd59a0 user: drh tags: trunk)
18:53
Fix another pre-update hook issue, this time in sqlite3preupdate_old(). (Closed-Leaf check-in: c7651d21bf user: dan tags: trim-nulls)
18:12
Fix a test script problem in exclusive2.test causing it to fail on this branch. (check-in: f66614dc78 user: dan tags: trim-nulls)
14:58
Experimental enhancement to automatically trim NULL values from the end of records, for a reduced disk footprint. This change also involves increasing the P5 operand from 8 to 16 bits. (check-in: 118ded403b user: drh tags: trim-nulls)
14:38
Ensure that sqlite3_blob_reopen() correctly handles short rows. Fix for ticket [e6e962d6b0f06f46e]. (check-in: 8cd1a4451c user: drh tags: trunk)
13:54
Fix SQLITEINT_H macro usage in two extensions. (check-in: 0803390c15 user: drh tags: trunk)
04:41
Ensure that sqlite3_blob_reopen() correctly handles short rows. Proposed fix for ticket [e6e962d6b0f06f46e]. Further testing needed. (Closed-Leaf check-in: 57d8dad35c user: drh tags: blob_reopen-fix)

Changes to src/insert.c.

1663
1664
1665
1666
1667
1668
1669

















1670
1671
1672
1673
1674
1675
1676
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693







+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+







    sqlite3VdbeGoto(v, ipkTop+1);
    sqlite3VdbeJumpHere(v, ipkBottom);
  }
  
  *pbMayReplace = seenReplace;
  VdbeModuleComment((v, "END: GenCnstCks(%d)", seenReplace));
}

/*
** Change the P5 operand on the last opcode (which should be an OP_MakeRecord)
** to be the number of columns in table pTab that must not be NULL-trimmed.
**
** Or if no columns of pTab may be NULL-trimmed, leave P5 at zero.
*/
void sqlite3SetMakeRecordP5(Vdbe *v, Table *pTab){
  u16 i;

  /* Records with omitted columns are only allowed for schema format
  ** version 2 and later (SQLite version 3.1.4, 2005-02-20). */
  if( pTab->pSchema->file_format<2 ) return;

  for(i=pTab->nCol; i>1 && pTab->aCol[i-1].pDflt==0; i--){}
  sqlite3VdbeChangeP5(v, i);
}

/*
** This routine generates code to finish the INSERT or UPDATE operation
** that was started by a prior call to sqlite3GenerateConstraintChecks.
** A consecutive range of registers starting at regNewData contains the
** rowid and the content to be inserted.
**
1723
1724
1725
1726
1727
1728
1729

1730
1731
1732
1733
1734
1735
1736
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754







+







    }
    sqlite3VdbeChangeP5(v, pik_flags);
  }
  if( !HasRowid(pTab) ) return;
  regData = regNewData + 1;
  regRec = sqlite3GetTempReg(pParse);
  sqlite3VdbeAddOp3(v, OP_MakeRecord, regData, pTab->nCol, regRec);
  sqlite3SetMakeRecordP5(v, pTab);
  if( !bAffinityDone ){
    sqlite3TableAffinity(v, pTab, 0);
    sqlite3ExprCacheAffinityChange(pParse, regData, pTab->nCol);
  }
  if( pParse->nested ){
    pik_flags = 0;
  }else{

Changes to src/sqliteInt.h.

3786
3787
3788
3789
3790
3791
3792

3793
3794
3795
3796
3797
3798
3799
3786
3787
3788
3789
3790
3791
3792
3793
3794
3795
3796
3797
3798
3799
3800







+







void sqlite3GenerateRowDelete(
    Parse*,Table*,Trigger*,int,int,int,i16,u8,u8,u8,int);
void sqlite3GenerateRowIndexDelete(Parse*, Table*, int, int, int*, int);
int sqlite3GenerateIndexKey(Parse*, Index*, int, int, int, int*,Index*,int);
void sqlite3ResolvePartIdxLabel(Parse*,int);
void sqlite3GenerateConstraintChecks(Parse*,Table*,int*,int,int,int,int,
                                     u8,u8,int,int*,int*);
void sqlite3SetMakeRecordP5(Vdbe*,Table*);
void sqlite3CompleteInsertion(Parse*,Table*,int,int,int,int*,int,int,int);
int sqlite3OpenTableAndIndices(Parse*, Table*, int, u8, int, u8*, int*, int*);
void sqlite3BeginWriteOperation(Parse*, int, int);
void sqlite3MultiWrite(Parse*);
void sqlite3MayAbort(Parse*);
void sqlite3HaltConstraint(Parse*, int, int, char*, i8, u8);
void sqlite3UniqueConstraint(Parse*, int, Index*);

Changes to src/vdbe.c.

2772
2773
2774
2775
2776
2777
2778












2779
2780
2781
2782
2783
2784
2785
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797







+
+
+
+
+
+
+
+
+
+
+
+







  if( zAffinity ){
    pRec = pData0;
    do{
      applyAffinity(pRec++, *(zAffinity++), encoding);
      assert( zAffinity[0]==0 || pRec<=pLast );
    }while( zAffinity[0] );
  }

  /* NULLs can be safely trimmed from the end of the record, as long as
  ** as the schema format is 2 or more and none of the omitted columns
  ** have a non-NULL default value.  Also, the record must be left with
  ** at least one field.  If P5>0 then it will be one more than the
  ** index of the right-most column with a non-NULL default value */
  if( pOp->p5 ){
    while( (pLast->flags & MEM_Null)!=0 && nField>pOp->p5 ){
      pLast--;
      nField--;
    }
  }

  /* Loop through the elements that will make up the record to figure
  ** out how much space is required for the new record.
  */
  pRec = pLast;
  do{
    assert( memIsValid(pRec) );

Changes to src/vdbe.h.

37
38
39
40
41
42
43
44
45

46
47
48
49
50
51
52
37
38
39
40
41
42
43


44
45
46
47
48
49
50
51







-
-
+







** A single instruction of the virtual machine has an opcode
** and as many as three operands.  The instruction is recorded
** as an instance of the following structure:
*/
struct VdbeOp {
  u8 opcode;          /* What operation to perform */
  signed char p4type; /* One of the P4_xxx constants for p4 */
  u8 notUsed1;
  u8 p5;              /* Fifth parameter is an unsigned character */
  u16 p5;             /* Fifth parameter is an unsigned 16-bit integer */
  int p1;             /* First operand */
  int p2;             /* Second parameter (often the jump destination) */
  int p3;             /* The third parameter */
  union p4union {     /* fourth parameter */
    int i;                 /* Integer value if p4type==P4_INT32 */
    void *p;               /* Generic pointer */
    char *z;               /* Pointer to data for string (char array) types */
190
191
192
193
194
195
196
197

198
199
200
201
202
203
204
189
190
191
192
193
194
195

196
197
198
199
200
201
202
203







-
+







#endif
VdbeOp *sqlite3VdbeAddOpList(Vdbe*, int nOp, VdbeOpList const *aOp, int iLineno);
void sqlite3VdbeAddParseSchemaOp(Vdbe*,int,char*);
void sqlite3VdbeChangeOpcode(Vdbe*, u32 addr, u8);
void sqlite3VdbeChangeP1(Vdbe*, u32 addr, int P1);
void sqlite3VdbeChangeP2(Vdbe*, u32 addr, int P2);
void sqlite3VdbeChangeP3(Vdbe*, u32 addr, int P3);
void sqlite3VdbeChangeP5(Vdbe*, u8 P5);
void sqlite3VdbeChangeP5(Vdbe*, u16 P5);
void sqlite3VdbeJumpHere(Vdbe*, int addr);
int sqlite3VdbeChangeToNoop(Vdbe*, int addr);
int sqlite3VdbeDeletePriorOpcode(Vdbe*, u8 op);
void sqlite3VdbeChangeP4(Vdbe*, int addr, const char *zP4, int N);
void sqlite3VdbeAppendP4(Vdbe*, void *pP4, int p4type);
void sqlite3VdbeSetP4KeyInfo(Parse*, Index*);
void sqlite3VdbeUsesBtree(Vdbe*, int);

Changes to src/vdbeapi.c.

1656
1657
1658
1659
1660
1661
1662

1663
1664
1665
1666
1667
1668
1669
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670







+








/*
** This function is called from within a pre-update callback to retrieve
** a field of the row currently being updated or deleted.
*/
int sqlite3_preupdate_old(sqlite3 *db, int iIdx, sqlite3_value **ppValue){
  PreUpdate *p = db->pPreUpdate;
  Mem *pMem;
  int rc = SQLITE_OK;

  /* Test that this call is being made from within an SQLITE_DELETE or
  ** SQLITE_UPDATE pre-update callback, and that iIdx is within range. */
  if( !p || p->op==SQLITE_INSERT ){
    rc = SQLITE_MISUSE_BKPT;
    goto preupdate_old_out;
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699

1700
1701
1702
1703
1704
1705







1706
1707
1708
1709
1710
1711
1712
1713
1690
1691
1692
1693
1694
1695
1696




1697






1698
1699
1700
1701
1702
1703
1704

1705
1706
1707
1708
1709
1710
1711







-
-
-
-
+
-
-
-
-
-
-
+
+
+
+
+
+
+
-







    if( rc!=SQLITE_OK ){
      sqlite3DbFree(db, aRec);
      goto preupdate_old_out;
    }
    p->aRecord = aRec;
  }

  if( iIdx>=p->pUnpacked->nField ){
    *ppValue = (sqlite3_value *)columnNullValue();
  }else{
    Mem *pMem = *ppValue = &p->pUnpacked->aMem[iIdx];
  pMem = *ppValue = &p->pUnpacked->aMem[iIdx];
    *ppValue = &p->pUnpacked->aMem[iIdx];
    if( iIdx==p->pTab->iPKey ){
      sqlite3VdbeMemSetInt64(pMem, p->iKey1);
    }else if( p->pTab->aCol[iIdx].affinity==SQLITE_AFF_REAL ){
      if( pMem->flags & MEM_Int ){
        sqlite3VdbeMemRealify(pMem);
  if( iIdx==p->pTab->iPKey ){
    sqlite3VdbeMemSetInt64(pMem, p->iKey1);
  }else if( iIdx>=p->pUnpacked->nField ){
    *ppValue = (sqlite3_value *)columnNullValue();
  }else if( p->pTab->aCol[iIdx].affinity==SQLITE_AFF_REAL ){
    if( pMem->flags & MEM_Int ){
      sqlite3VdbeMemRealify(pMem);
      }
    }
  }

 preupdate_old_out:
  sqlite3Error(db, rc);
  return sqlite3ApiExit(db, rc);
}
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784



1785


1786
1787
1788
1789
1790
1791
1792
1770
1771
1772
1773
1774
1775
1776






1777
1778
1779

1780
1781
1782
1783
1784
1785
1786
1787
1788







-
-
-
-
-
-
+
+
+
-
+
+







      pUnpack = vdbeUnpackRecord(&p->keyinfo, pData->n, pData->z);
      if( !pUnpack ){
        rc = SQLITE_NOMEM;
        goto preupdate_new_out;
      }
      p->pNewUnpacked = pUnpack;
    }
    if( iIdx>=pUnpack->nField ){
      pMem = (sqlite3_value *)columnNullValue();
    }else{
      pMem = &pUnpack->aMem[iIdx];
      if( iIdx==p->pTab->iPKey ){
        sqlite3VdbeMemSetInt64(pMem, p->iKey2);
    pMem = &pUnpack->aMem[iIdx];
    if( iIdx==p->pTab->iPKey ){
      sqlite3VdbeMemSetInt64(pMem, p->iKey2);
      }
    }else if( iIdx>=pUnpack->nField ){
      pMem = (sqlite3_value *)columnNullValue();
    }
  }else{
    /* For an UPDATE, memory cell (p->iNewReg+1+iIdx) contains the required
    ** value. Make a copy of the cell contents and return a pointer to it.
    ** It is not safe to return a pointer to the memory cell itself as the
    ** caller may modify the value text encoding.
    */

Changes to src/vdbeaux.c.

779
780
781
782
783
784
785
786

787
788
789
790
791
792
793
779
780
781
782
783
784
785

786
787
788
789
790
791
792
793







-
+







}
void sqlite3VdbeChangeP2(Vdbe *p, u32 addr, int val){
  sqlite3VdbeGetOp(p,addr)->p2 = val;
}
void sqlite3VdbeChangeP3(Vdbe *p, u32 addr, int val){
  sqlite3VdbeGetOp(p,addr)->p3 = val;
}
void sqlite3VdbeChangeP5(Vdbe *p, u8 p5){
void sqlite3VdbeChangeP5(Vdbe *p, u16 p5){
  assert( p->nOp>0 || p->db->mallocFailed );
  if( p->nOp>0 ) p->aOp[p->nOp-1].p5 = p5;
}

/*
** Change the P2 operand of instruction addr so that it points to
** the address of the next instruction to be coded.
4581
4582
4583
4584
4585
4586
4587
4588

4589
4590
4591

4592
4593
4594
4595
4596
4597
4598
4581
4582
4583
4584
4585
4586
4587

4588
4589
4590

4591
4592
4593
4594
4595
4596
4597
4598







-
+


-
+







** If the second argument is not NULL, release any allocations associated 
** with the memory cells in the p->aMem[] array. Also free the UnpackedRecord
** structure itself, using sqlite3DbFree().
**
** This function is used to free UnpackedRecord structures allocated by
** the vdbeUnpackRecord() function found in vdbeapi.c.
*/
static void vdbeFreeUnpacked(sqlite3 *db, UnpackedRecord *p){
static void vdbeFreeUnpacked(sqlite3 *db, int nField, UnpackedRecord *p){
  if( p ){
    int i;
    for(i=0; i<p->nField; i++){
    for(i=0; i<nField; i++){
      Mem *pMem = &p->aMem[i];
      if( pMem->zMalloc ) sqlite3VdbeMemRelease(pMem);
    }
    sqlite3DbFree(db, p);
  }
}
#endif /* SQLITE_ENABLE_PREUPDATE_HOOK */
4643
4644
4645
4646
4647
4648
4649
4650
4651


4652
4653
4654
4655
4656
4657
4658
4659
4660
4643
4644
4645
4646
4647
4648
4649


4650
4651
4652
4653
4654
4655
4656
4657
4658
4659
4660







-
-
+
+









  preupdate.iKey2 = iKey2;
  preupdate.pTab = pTab;

  db->pPreUpdate = &preupdate;
  db->xPreUpdateCallback(db->pPreUpdateArg, db, op, zDb, zTbl, iKey1, iKey2);
  db->pPreUpdate = 0;
  sqlite3DbFree(db, preupdate.aRecord);
  vdbeFreeUnpacked(db, preupdate.pUnpacked);
  vdbeFreeUnpacked(db, preupdate.pNewUnpacked);
  vdbeFreeUnpacked(db, preupdate.keyinfo.nField+1, preupdate.pUnpacked);
  vdbeFreeUnpacked(db, preupdate.keyinfo.nField+1, preupdate.pNewUnpacked);
  if( preupdate.aNew ){
    int i;
    for(i=0; i<pCsr->nField; i++){
      sqlite3VdbeMemRelease(&preupdate.aNew[i]);
    }
    sqlite3DbFree(db, preupdate.aNew);
  }
}
#endif /* SQLITE_ENABLE_PREUPDATE_HOOK */

Changes to test/exclusive2.test.

117
118
119
120
121
122
123
124
125
126
127
128
129
130







131
132
133
134
135
136
137
117
118
119
120
121
122
123







124
125
126
127
128
129
130
131
132
133
134
135
136
137







-
-
-
-
-
-
-
+
+
+
+
+
+
+







# allocation of 24 pages (shared between all pagers). This is not enough for
# this test.
#
do_test exclusive2-1.1 {
  execsql {
    BEGIN;
    CREATE TABLE t1(a, b);
    INSERT INTO t1(a) VALUES(randstr(10, 400));
    INSERT INTO t1(a) VALUES(randstr(10, 400));
    INSERT INTO t1(a) SELECT randstr(10, 400) FROM t1;
    INSERT INTO t1(a) SELECT randstr(10, 400) FROM t1;
    INSERT INTO t1(a) SELECT randstr(10, 400) FROM t1;
    INSERT INTO t1(a) SELECT randstr(10, 400) FROM t1;
    INSERT INTO t1(a) SELECT randstr(10, 400) FROM t1;
    INSERT INTO t1(a, b) VALUES(randstr(10, 400), 0);
    INSERT INTO t1(a, b) VALUES(randstr(10, 400), 0);
    INSERT INTO t1(a, b) SELECT randstr(10, 400), 0 FROM t1;
    INSERT INTO t1(a, b) SELECT randstr(10, 400), 0 FROM t1;
    INSERT INTO t1(a, b) SELECT randstr(10, 400), 0 FROM t1;
    INSERT INTO t1(a, b) SELECT randstr(10, 400), 0 FROM t1;
    INSERT INTO t1(a, b) SELECT randstr(10, 400), 0 FROM t1;
    COMMIT;
    SELECT count(*) FROM t1;
  }
} {64}
do_test exclusive2-1.2.1 {
  # Make sure the pager cache is large enough to store the 
  # entire database.
150
151
152
153
154
155
156
157

158
159
160
161
162
163
164
150
151
152
153
154
155
156

157
158
159
160
161
162
163
164







-
+







} $::sig
do_test exclusive2-1.4 {
  sqlite3 db2 test.db
  t1sig db2
} $::sig
do_test exclusive2-1.5 {
  execsql {
    UPDATE t1 SET b=a, a=NULL;
    UPDATE t1 SET b=a, a=0;
  } db2
  expr {[t1sig db2] eq $::sig}
} 0
do_test exclusive2-1.6 {
  readPagerChangeCounter test.db
} {2}
do_test exclusive2-1.7 {

Changes to test/hook.test.

849
850
851
852
853
854
855



















































856
857
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908







+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+


}

# No preupdate callbacks for modifying sqlite_master.
do_preupdate_test 8.1 {
  CREATE TABLE x1(x, y);
} {
}

#-------------------------------------------------------------------------
reset_db
db preupdate hook preupdate_hook
do_execsql_test 9.0 {
  CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c);
  CREATE TABLE t2(a, b INTEGER PRIMARY KEY);
}
do_preupdate_test 9.1 {
  INSERT INTO t1 VALUES(456, NULL, NULL);
} {
  INSERT main t1 456 456  0  456 {} {}
}
do_execsql_test 9.2 {
  ALTER TABLE t1 ADD COLUMN d;
}
do_preupdate_test 9.3 {
  INSERT INTO t1(a, b, c) VALUES(457, NULL, NULL);
} {
  INSERT main t1 457 457  0  457 {} {} {}
}
do_preupdate_test 9.4 {
  DELETE FROM t1 WHERE a=456
} {
  DELETE main t1 456 456  0  456 {} {} {}
}
do_preupdate_test 9.5 {
  INSERT INTO t2 DEFAULT VALUES;
} {
  INSERT main t2 1 1  0  {} 1
} 
do_preupdate_test 9.6 {
  INSERT INTO t1 DEFAULT VALUES;
} {
  INSERT main t1 458 458  0  458 {} {} {}
} 


do_execsql_test 10.0 {
  CREATE TABLE t3(a, b INTEGER PRIMARY KEY);
}
do_preupdate_test 10.1 {
  INSERT INTO t3 DEFAULT VALUES
} {
  INSERT main t3 1 1 0 {} 1
}
do_execsql_test 10.2 { SELECT * FROM t3 } {{} 1}
do_preupdate_test 10.3 {
  DELETE FROM t3 WHERE b=1
} {DELETE main t3 1 1 0 {} 1}


finish_test

Changes to test/tester.tcl.

457
458
459
460
461
462
463
464

465
466
467
468
469
470
471
457
458
459
460
461
462
463

464
465
466
467
468
469
470
471







-
+







        foreach {dummy cmdlinearg(malloctrace)} [split $a =] break
        if {$cmdlinearg(malloctrace)} {
          sqlite3_memdebug_log start
        }
      }
      {^-+backtrace=.+$} {
        foreach {dummy cmdlinearg(backtrace)} [split $a =] break
        sqlite3_memdebug_backtrace $value
        sqlite3_memdebug_backtrace $cmdlinearg(backtrace)
      }
      {^-+binarylog=.+$} {
        foreach {dummy cmdlinearg(binarylog)} [split $a =] break
        set cmdlinearg(binarylog) [file normalize $cmdlinearg(binarylog)]
      }
      {^-+soak=.+$} {
        foreach {dummy cmdlinearg(soak)} [split $a =] break