/ Changes On Branch testFixes
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Changes In Branch testFixes Excluding Merge-Ins

This is equivalent to a diff from d75e67654a to 9a5ef341de

2018-03-20
16:56
For 'zipfile', detect attempts to cause a duplicate entry via UPDATE. Also, fix handling of 'UPDATE OR REPLACE' statements run on zipfile virtual tables. Win32 portability fixes to the 'fileio' extension. Miscellaneous test fixes. (check-in: b36caeca91 user: mistachkin tags: trunk)
13:26
Fix handling of "UPDATE OR REPLACE" statements run on zipfile virtual tables. (Closed-Leaf check-in: 9a5ef341de user: dan tags: testFixes)
12:12
Add a test case for the fix on this branch. (check-in: 7834cf6c28 user: dan tags: testFixes)
2018-03-16
23:59
Detect corruption in the form of the sqlite_sequence table pointing to the wrong type of btree. (check-in: 525deb7a67 user: drh tags: trunk)
23:53
Fix a duplicate test number and cleanup a bit of Makefile whitespace. (check-in: 56d11c2509 user: mistachkin tags: testFixes)
20:23
Detect databases whose schema is corrupted using a CREATE TABLE AS statement and issue an appropriate error message. (check-in: d75e67654a user: drh tags: trunk)
20:15
Better error message text when the schema is corrupted by a CREATE TABLE AS entry. (Closed-Leaf check-in: e13993cf83 user: drh tags: corrupt-schema)
07:48
Fix a problem in test script thread001.test causing a spurious "-1 files were left open" error when run separately. (check-in: 1774f1c3ba user: dan tags: trunk)

Changes to Makefile.msc.

2087
2088
2089
2090
2091
2092
2093
2094

2095
2096
2097
2098
2099
2100
2101
2087
2088
2089
2090
2091
2092
2093

2094
2095
2096
2097
2098
2099
2100
2101







-
+








keywordhash.h:	$(TOP)\tool\mkkeywordhash.c mkkeywordhash.exe
	.\mkkeywordhash.exe > keywordhash.h

# Source files that go into making shell.c
SHELL_SRC = \
	$(TOP)\src\shell.c.in \
        $(TOP)\ext\misc\appendvfs.c \
	$(TOP)\ext\misc\appendvfs.c \
	$(TOP)\ext\misc\shathree.c \
	$(TOP)\ext\misc\fileio.c \
	$(TOP)\ext\misc\completion.c \
	$(TOP)\ext\expert\sqlite3expert.c \
	$(TOP)\ext\expert\sqlite3expert.h \
	$(TOP)\src\test_windirent.c

Changes to ext/misc/fileio.c.

154
155
156
157
158
159
160



























































































161
162
163
164
165
166
167
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258







+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+







  va_list ap;
  va_start(ap, zFmt);
  zMsg = sqlite3_vmprintf(zFmt, ap);
  sqlite3_result_error(ctx, zMsg, -1);
  sqlite3_free(zMsg);
  va_end(ap);
}

#if defined(_WIN32)
/*
** This function is designed to convert a Win32 FILETIME structure into the
** number of seconds since the Unix Epoch (1970-01-01 00:00:00 UTC).
*/
static sqlite3_uint64 fileTimeToUnixTime(
  LPFILETIME pFileTime
){
  SYSTEMTIME epochSystemTime;
  ULARGE_INTEGER epochIntervals;
  FILETIME epochFileTime;
  ULARGE_INTEGER fileIntervals;

  memset(&epochSystemTime, 0, sizeof(SYSTEMTIME));
  epochSystemTime.wYear = 1970;
  epochSystemTime.wMonth = 1;
  epochSystemTime.wDay = 1;
  SystemTimeToFileTime(&epochSystemTime, &epochFileTime);
  epochIntervals.LowPart = epochFileTime.dwLowDateTime;
  epochIntervals.HighPart = epochFileTime.dwHighDateTime;

  fileIntervals.LowPart = pFileTime->dwLowDateTime;
  fileIntervals.HighPart = pFileTime->dwHighDateTime;

  return (fileIntervals.QuadPart - epochIntervals.QuadPart) / 10000000;
}

/*
** This function attempts to normalize the time values found in the stat()
** buffer to UTC.  This is necessary on Win32, where the runtime library
** appears to return these values as local times.
*/
static void statTimesToUtc(
  const char *zPath,
  struct stat *pStatBuf
){
  HANDLE hFindFile;
  WIN32_FIND_DATAW fd;
  LPWSTR zUnicodeName;
  extern LPWSTR sqlite3_win32_utf8_to_unicode(const char*);
  zUnicodeName = sqlite3_win32_utf8_to_unicode(zPath);
  if( zUnicodeName ){
    memset(&fd, 0, sizeof(WIN32_FIND_DATA));
    hFindFile = FindFirstFileW(zUnicodeName, &fd);
    if( hFindFile!=NULL ){
      pStatBuf->st_ctime = (time_t)fileTimeToUnixTime(&fd.ftCreationTime);
      pStatBuf->st_atime = (time_t)fileTimeToUnixTime(&fd.ftLastAccessTime);
      pStatBuf->st_mtime = (time_t)fileTimeToUnixTime(&fd.ftLastWriteTime);
      FindClose(hFindFile);
    }
    sqlite3_free(zUnicodeName);
  }
}
#endif

/*
** This function is used in place of stat().  On Windows, special handling
** is required in order for the included time to be returned as UTC.  On all
** other systems, this function simply calls stat().
*/
static int fileStat(
  const char *zPath,
  struct stat *pStatBuf
){
#if defined(_WIN32)
  int rc = stat(zPath, pStatBuf);
  if( rc==0 ) statTimesToUtc(zPath, pStatBuf);
  return rc;
#else
  return stat(zPath, pStatBuf);
#endif
}

/*
** This function is used in place of lstat().  On Windows, special handling
** is required in order for the included time to be returned as UTC.  On all
** other systems, this function simply calls lstat().
*/
static int fileLinkStat(
  const char *zPath,
  struct stat *pStatBuf
){
#if defined(_WIN32)
  int rc = lstat(zPath, pStatBuf);
  if( rc==0 ) statTimesToUtc(zPath, pStatBuf);
  return rc;
#else
  return lstat(zPath, pStatBuf);
#endif
}

/*
** Argument zFile is the name of a file that will be created and/or written
** by SQL function writefile(). This function ensures that the directory
** zFile will be written to exists, creating it if required. The permissions
** for any path components created by this function are set to (mode&0777).
**
186
187
188
189
190
191
192
193

194
195
196
197
198
199
200
277
278
279
280
281
282
283

284
285
286
287
288
289
290
291







-
+







      struct stat sStat;
      int rc2;

      for(; zCopy[i]!='/' && i<nCopy; i++);
      if( i==nCopy ) break;
      zCopy[i] = '\0';

      rc2 = stat(zCopy, &sStat);
      rc2 = fileStat(zCopy, &sStat);
      if( rc2!=0 ){
        if( mkdir(zCopy, mode & 0777) ) rc = SQLITE_ERROR;
      }else{
        if( !S_ISDIR(sStat.st_mode) ) rc = SQLITE_ERROR;
      }
      zCopy[i] = '/';
      i++;
228
229
230
231
232
233
234
235

236
237
238
239
240
241
242
319
320
321
322
323
324
325

326
327
328
329
330
331
332
333







-
+







      if( mkdir(zFile, mode) ){
        /* The mkdir() call to create the directory failed. This might not
        ** be an error though - if there is already a directory at the same
        ** path and either the permissions already match or can be changed
        ** to do so using chmod(), it is not an error.  */
        struct stat sStat;
        if( errno!=EEXIST
         || 0!=stat(zFile, &sStat)
         || 0!=fileStat(zFile, &sStat)
         || !S_ISDIR(sStat.st_mode)
         || ((sStat.st_mode&0777)!=(mode&0777) && 0!=chmod(zFile, mode&0777))
        ){
          return 1;
        }
      }
    }else{
275
276
277
278
279
280
281



282
283
284
285
286
287
288
289
290
291
292
293
294

295
296
297
298
299
300
301
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387

388
389
390
391
392
393
394
395







+
+
+












-
+








    GetSystemTime(&currentTime);
    SystemTimeToFileTime(&currentTime, &lastAccess);
    intervals = Int32x32To64(mtime, 10000000) + 116444736000000000;
    lastWrite.dwLowDateTime = (DWORD)intervals;
    lastWrite.dwHighDateTime = intervals >> 32;
    zUnicodeName = sqlite3_win32_utf8_to_unicode(zFile);
    if( zUnicodeName==0 ){
      return 1;
    }
    hFile = CreateFileW(
      zUnicodeName, FILE_WRITE_ATTRIBUTES, 0, NULL, OPEN_EXISTING,
      FILE_FLAG_BACKUP_SEMANTICS, NULL
    );
    sqlite3_free(zUnicodeName);
    if( hFile!=INVALID_HANDLE_VALUE ){
      BOOL bResult = SetFileTime(hFile, NULL, &lastAccess, &lastWrite);
      CloseHandle(hFile);
      return !bResult;
    }else{
      return 1;
    }
#elif defined(AT_FDCWD) && 0 /* utimensat() is not univerally available */
#elif defined(AT_FDCWD) && 0 /* utimensat() is not universally available */
    /* Recent unix */
    struct timespec times[2];
    times[0].tv_nsec = times[1].tv_nsec = 0;
    times[0].tv_sec = time(0);
    times[1].tv_sec = mtime;
    if( utimensat(AT_FDCWD, zFile, times, AT_SYMLINK_NOFOLLOW) ){
      return 1;
564
565
566
567
568
569
570
571

572
573
574
575
576
577
578
658
659
660
661
662
663
664

665
666
667
668
669
670
671
672







-
+







      if( pEntry->d_name[0]=='.' ){
       if( pEntry->d_name[1]=='.' && pEntry->d_name[2]=='\0' ) continue;
       if( pEntry->d_name[1]=='\0' ) continue;
      }
      sqlite3_free(pCur->zPath);
      pCur->zPath = sqlite3_mprintf("%s/%s", pLvl->zDir, pEntry->d_name);
      if( pCur->zPath==0 ) return SQLITE_NOMEM;
      if( lstat(pCur->zPath, &pCur->sStat) ){
      if( fileLinkStat(pCur->zPath, &pCur->sStat) ){
        fsdirSetErrmsg(pCur, "cannot stat file: %s", pCur->zPath);
        return SQLITE_ERROR;
      }
      return SQLITE_OK;
    }
    closedir(pLvl->pDir);
    sqlite3_free(pLvl->zDir);
698
699
700
701
702
703
704
705

706
707
708
709
710
711
712
792
793
794
795
796
797
798

799
800
801
802
803
804
805
806







-
+







  }else{
    pCur->zPath = sqlite3_mprintf("%s", zDir);
  }

  if( pCur->zPath==0 ){
    return SQLITE_NOMEM;
  }
  if( lstat(pCur->zPath, &pCur->sStat) ){
  if( fileLinkStat(pCur->zPath, &pCur->sStat) ){
    fsdirSetErrmsg(pCur, "cannot stat file: %s", pCur->zPath);
    return SQLITE_ERROR;
  }

  return SQLITE_OK;
}

Changes to ext/misc/zipfile.c.

1495
1496
1497
1498
1499
1500
1501













1502
1503
1504
1505
1506
1507
1508
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521







+
+
+
+
+
+
+
+
+
+
+
+
+







*/
static u32 zipfileGetTime(sqlite3_value *pVal){
  if( pVal==0 || sqlite3_value_type(pVal)==SQLITE_NULL ){
    return zipfileTime();
  }
  return (u32)sqlite3_value_int64(pVal);
}

/*
** Unless it is NULL, entry pOld is currently part of the pTab->pFirstEntry
** linked list.  Remove it from the list and free the object.
*/
static void zipfileRemoveEntryFromList(ZipfileTab *pTab, ZipfileEntry *pOld){
  if( pOld ){
    ZipfileEntry **pp;
    for(pp=&pTab->pFirstEntry; (*pp)!=pOld; pp=&((*pp)->pNext));
    *pp = (*pp)->pNext;
    zipfileEntryFree(pOld);
  }
}

/*
** xUpdate method.
*/
static int zipfileUpdate(
  sqlite3_vtab *pVtab, 
  int nVal, 
1520
1521
1522
1523
1524
1525
1526


1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538






1539
1540
1541
1542
1543
1544
1545
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566







+
+












+
+
+
+
+
+







  int nPath = 0;                  /* strlen(zPath) */
  const u8 *pData = 0;            /* Pointer to buffer containing content */
  int nData = 0;                  /* Size of pData buffer in bytes */
  int iMethod = 0;                /* Compression method for new entry */
  u8 *pFree = 0;                  /* Free this */
  char *zFree = 0;                /* Also free this */
  ZipfileEntry *pOld = 0;
  ZipfileEntry *pOld2 = 0;
  int bUpdate = 0;                /* True for an update that modifies "name" */
  int bIsDir = 0;
  u32 iCrc32 = 0;

  if( pTab->pWriteFd==0 ){
    rc = zipfileBegin(pVtab);
    if( rc!=SQLITE_OK ) return rc;
  }

  /* If this is a DELETE or UPDATE, find the archive entry to delete. */
  if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){
    const char *zDelete = (const char*)sqlite3_value_text(apVal[0]);
    int nDelete = (int)strlen(zDelete);
    if( nVal>1 ){
      const char *zUpdate = (const char*)sqlite3_value_text(apVal[1]);
      if( zUpdate && zipfileComparePath(zUpdate, zDelete, nDelete)!=0 ){
        bUpdate = 1;
      }
    }
    for(pOld=pTab->pFirstEntry; 1; pOld=pOld->pNext){
      if( zipfileComparePath(pOld->cds.zFile, zDelete, nDelete)==0 ){
        break;
      }
      assert( pOld->pNext );
    }
  }
1609
1610
1611
1612
1613
1614
1615
1616
1617



1618
1619
1620
1621
1622
1623
1624
1625
1626

1627
1628
1629
1630
1631
1632
1633
1630
1631
1632
1633
1634
1635
1636


1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647

1648
1649
1650
1651
1652
1653
1654
1655







-
-
+
+
+








-
+







        zFree = sqlite3_mprintf("%s/", zPath);
        if( zFree==0 ){ rc = SQLITE_NOMEM; }
        zPath = (const char*)zFree;
        nPath++;
      }
    }

    /* Check that we're not inserting a duplicate entry */
    if( pOld==0 && rc==SQLITE_OK ){
    /* Check that we're not inserting a duplicate entry -OR- updating an
    ** entry with a path, thereby making it into a duplicate. */
    if( (pOld==0 || bUpdate) && rc==SQLITE_OK ){
      ZipfileEntry *p;
      for(p=pTab->pFirstEntry; p; p=p->pNext){
        if( zipfileComparePath(p->cds.zFile, zPath, nPath)==0 ){
          switch( sqlite3_vtab_on_conflict(pTab->db) ){
            case SQLITE_IGNORE: {
              goto zipfile_update_done;
            }
            case SQLITE_REPLACE: {
              pOld = p;
              pOld2 = p;
              break;
            }
            default: {
              zipfileTableErr(pTab, "duplicate name: \"%s\"", zPath);
              rc = SQLITE_CONSTRAINT;
              break;
            }
1657
1658
1659
1660
1661
1662
1663
1664

1665
1666
1667
1668
1669


1670
1671
1672
1673
1674
1675



1676
1677
1678
1679
1680
1681
1682
1679
1680
1681
1682
1683
1684
1685

1686

1687
1688


1689
1690
1691
1692
1693



1694
1695
1696
1697
1698
1699
1700
1701
1702
1703







-
+
-


-
-
+
+



-
-
-
+
+
+







        pNew->mUnixTime = (u32)mTime;
        rc = zipfileAppendEntry(pTab, pNew, pData, nData);
        zipfileAddEntry(pTab, pOld, pNew);
      }
    }
  }

  if( rc==SQLITE_OK && pOld ){
  if( rc==SQLITE_OK && (pOld || pOld2) ){
    ZipfileEntry **pp;
    ZipfileCsr *pCsr;
    for(pCsr=pTab->pCsrList; pCsr; pCsr=pCsr->pCsrNext){
      if( pCsr->pCurrent==pOld ){
        pCsr->pCurrent = pOld->pNext;
      if( pCsr->pCurrent && (pCsr->pCurrent==pOld || pCsr->pCurrent==pOld2) ){
        pCsr->pCurrent = pCsr->pCurrent->pNext;
        pCsr->bNoop = 1;
      }
    }
    for(pp=&pTab->pFirstEntry; (*pp)!=pOld; pp=&((*pp)->pNext));
    *pp = (*pp)->pNext;
    zipfileEntryFree(pOld);

    zipfileRemoveEntryFromList(pTab, pOld);
    zipfileRemoveEntryFromList(pTab, pOld2);
  }

zipfile_update_done:
  sqlite3_free(pFree);
  sqlite3_free(zFree);
  return rc;
}

Changes to test/dbstatus2.test.

106
107
108
109
110
111
112
113

114
115
106
107
108
109
110
111
112

113
114
115







-
+


do_test 3.0 { db_spill db 1 } {0 0 0}
do_test 3.1 { db_spill db 0 } {0 0 0}
do_execsql_test 3.2 {
  PRAGMA journal_mode=DELETE;
  PRAGMA cache_size=3;
  UPDATE t1 SET b=randomblob(1000);
} {delete}
do_test 3.2 { db_spill db 0 } {0 8 0}
do_test 3.3 { db_spill db 0 } {0 8 0}
 
finish_test

Changes to test/trace3.test.

125
126
127
128
129
130
131
132

133
134
135
136
137
138
139
140
141
142

143
144
145
146
147
148
149
125
126
127
128
129
130
131

132
133
134
135
136
137
138
139
140
141

142
143
144
145
146
147
148
149







-
+









-
+







  set ::stmtlist(record) {}
  db trace_v2 trace_v2_record profile
  execsql {
    SELECT a, b FROM t1 ORDER BY a;
  }
  set stmt [lindex [lindex $::stmtlist(record) 0] 0]
  set ns [lindex [lindex $::stmtlist(record) 0] 1]
  list $stmt [expr {$ns >= 0 && $ns <= 1000000}]; # less than 0.001 second
  list $stmt [expr {$ns >= 0 && $ns <= 9999999}]; # less than 0.010 seconds
} {/^-?\d+ 1$/}
do_test trace3-4.4 {
  set ::stmtlist(record) {}
  db trace_v2 trace_v2_record 2
  execsql {
    SELECT a, b FROM t1 ORDER BY a;
  }
  set stmt [lindex [lindex $::stmtlist(record) 0] 0]
  set ns [lindex [lindex $::stmtlist(record) 0] 1]
  list $stmt [expr {$ns >= 0 && $ns <= 1000000}]; # less than 0.001 second
  list $stmt [expr {$ns >= 0 && $ns <= 9999999}]; # less than 0.010 seconds
} {/^-?\d+ 1$/}

do_test trace3-5.1 {
  set ::stmtlist(record) {}
  db trace_v2 trace_v2_record row
  execsql {
    SELECT a, b FROM t1 ORDER BY a;

Changes to test/zipfile.test.

1
2
3
4
5
6
7
8
9
10
11


12
13
14
15
16
17
18
19
20
21
22




23
24
25
26
27
28
29
30
31
32


33
34
35






















36
37
38
39
40
41
42






43
44
45
46
47
48
49
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40



41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65




66
67
68
69
70
71
72
73
74
75
76
77
78











+
+











+
+
+
+










+
+
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+



-
-
-
-
+
+
+
+
+
+







# 2017 December 9
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
#

package require Tcl 8.6

set testdir [file dirname $argv0]
source $testdir/tester.tcl
set testprefix zipfile

ifcapable !vtab {
  finish_test; return
}
if {[catch {load_static_extension db zipfile} error]} {
  puts "Skipping zipfile tests, hit load error: $error"
  finish_test; return
}
if {[catch {load_static_extension db fileio} error]} {
  puts "Skipping zipfile tests, hit load error: $error"
  finish_test; return
}

proc readfile {f} {
  set fd [open $f]
  fconfigure $fd -translation binary -encoding binary
  set data [read $fd]
  close $fd
  set data
}

unset -nocomplain ::UNZIP

if {$::tcl_platform(platform)=="unix" && [catch {exec unzip}]==0} {
  set ::UNZIP 1
  load_static_extension db fileio
if {[catch {exec unzip} msg]==0 && \
    [regexp -line {^UnZip \d+\.\d+ .*? Info-ZIP\.} $msg]} {
  set ::UNZIP unzip
  proc fix_stat_mode {name mode} {
    if {$::tcl_platform(platform)=="windows"} {
      #
      # NOTE: Set or unset the write bits of the file permissions
      #       based on the read-only attribute because the Win32
      #       version of UnZip does this.
      #
      set writebits 0x12; # 0o22
      set result $mode
      if {[file attributes $name -readonly]} {
        set result [expr {$result | $writebits}]
      } else {
        set result [expr {$result & ~$writebits}]
      }
      return $result
    } else {
      return $mode
    }
  }
  proc do_unzip {file} {
    forcedelete test_unzip
    file mkdir test_unzip
    exec unzip -d test_unzip $file
  
    set res [db eval { 
      SELECT replace(name,'test_unzip/',''),mode,mtime,data 
    exec $::UNZIP -d test_unzip $file

    db func modefix fix_stat_mode

    set res [db eval {
      SELECT replace(name,'test_unzip/',''),modefix(name,mode),mtime,data
      FROM fsdir('test_unzip') 
      WHERE name!='test_unzip'
      ORDER BY name
    }]
    set res
  }
}
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
133
134
135
136
137
138
139

140
141
142
143
144
145
146







-







#          ( SELECT zipfile(name,mode,mtime,data,method) FROM zipfile($file) )
#      );
#
#      Then tests that unpacking the new archive using [unzip] produces
#      the same results as in (1).
#
proc do_unzip_test {tn file} {
  if {[info vars ::UNZIP]==""} { return }
  db func sss strip_slash

  db eval {
    SELECT writefile('test_unzip.zip',
        ( SELECT zipfile(name,mode,mtime,data,method) FROM zipfile($file) )
    );
  }
242
243
244
245
246
247
248
249








250
251

252
253

254
255

256
257

258
259
260
261
262
263

264
265

266
267

268
269
270
271
272
273

274
275

276
277

278
279
280
281
282

283
284

285
286

287
288
289
290
291
292
293

294
295

296
297

298
299

300
301
302

303
304

305
306

307
308

309
310
311
312
313
314
315
316
317
318

319
320
321
322
323
324
325
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286

287
288

289
290

291
292

293
294
295
296
297
298

299
300

301
302

303
304
305
306
307
308

309
310

311
312

313
314
315
316
317

318
319

320
321

322
323
324
325
326
327
328

329
330

331
332

333
334

335
336
337

338
339

340
341

342
343

344
345
346
347
348
349
350
351
352
353

354
355
356
357
358
359
360
361








+
+
+
+
+
+
+
+

-
+

-
+

-
+

-
+





-
+

-
+

-
+





-
+

-
+

-
+




-
+

-
+

-
+






-
+

-
+

-
+

-
+


-
+

-
+

-
+

-
+









-
+







  UPDATE zz SET mtime=4 WHERE name='i.txt';
  SELECT name, mode, mtime, data, method FROM zipfile('test.zip');
} {
  f.txt 33188 1000000000 abcde 0
  h.txt 33188 1000000004 aaaaaaaaaabbbbbbbbbb 8
  i.txt 33188 4 zxcvb 0
}

if {$::tcl_platform(platform)=="unix"} {
  set modes -rw-r--r-x
  set perms 33189
} else {
  set modes -rw-r--r--; # no execute bits on Win32
  set perms 33188
}

do_execsql_test 1.6.3 {
  UPDATE zz SET mode='-rw-r--r-x' WHERE name='h.txt';
  UPDATE zz SET mode=$modes WHERE name='h.txt';
  SELECT name, mode, mtime, data, method FROM zipfile('test.zip');
} {
} [string map [list %perms% $perms] {
  f.txt 33188 1000000000 abcde 0
  h.txt 33189 1000000004 aaaaaaaaaabbbbbbbbbb 8
  h.txt %perms% 1000000004 aaaaaaaaaabbbbbbbbbb 8
  i.txt 33188 4 zxcvb 0
}
}]
do_zip_tests 1.6.3a test.zip

do_execsql_test 1.6.4 {
  UPDATE zz SET name = 'blue.txt' WHERE name='f.txt';
  SELECT name, mode, mtime, data, method FROM zipfile('test.zip');
} {
} [string map [list %perms% $perms] {
  blue.txt 33188 1000000000 abcde 0
  h.txt 33189 1000000004 aaaaaaaaaabbbbbbbbbb 8
  h.txt %perms% 1000000004 aaaaaaaaaabbbbbbbbbb 8
  i.txt 33188 4 zxcvb 0
}
}]
do_zip_tests 1.6.4a test.zip

do_execsql_test 1.6.5 {
  UPDATE zz SET data = 'edcba' WHERE name='blue.txt';
  SELECT name, mode, mtime, data, method FROM zipfile('test.zip');
} {
} [string map [list %perms% $perms] {
  blue.txt 33188 1000000000 edcba 0
  h.txt 33189 1000000004 aaaaaaaaaabbbbbbbbbb 8
  h.txt %perms% 1000000004 aaaaaaaaaabbbbbbbbbb 8
  i.txt 33188 4 zxcvb 0
}
}]

do_execsql_test 1.6.6 {
  UPDATE zz SET mode=NULL, data = NULL WHERE name='blue.txt';
  SELECT name, mode, mtime, data, method FROM zipfile('test.zip');
} {
} [string map [list %perms% $perms] {
  blue.txt/ 16877 1000000000 {} 0
  h.txt 33189 1000000004 aaaaaaaaaabbbbbbbbbb 8
  h.txt %perms% 1000000004 aaaaaaaaaabbbbbbbbbb 8
  i.txt 33188 4 zxcvb 0
}
}]

do_catchsql_test 1.6.7 {
  UPDATE zz SET data=NULL WHERE name='i.txt'
} {1 {zipfile: mode does not match data}}
do_execsql_test 1.6.8 {
  SELECT name, mode, mtime, data, method FROM zipfile('test.zip');
} {
} [string map [list %perms% $perms] {
  blue.txt/ 16877 1000000000 {} 0
  h.txt 33189 1000000004 aaaaaaaaaabbbbbbbbbb 8
  h.txt %perms% 1000000004 aaaaaaaaaabbbbbbbbbb 8
  i.txt 33188 4 zxcvb 0
}
}]

do_execsql_test 1.6.8 {
do_execsql_test 1.6.9 {
  UPDATE zz SET data = '' WHERE name='i.txt';
  SELECT name,mode,mtime,data,method from zipfile('test.zip');
} {
} [string map [list %perms% $perms] {
  blue.txt/ 16877 1000000000 {} 0
  h.txt 33189 1000000004 aaaaaaaaaabbbbbbbbbb 8
  h.txt %perms% 1000000004 aaaaaaaaaabbbbbbbbbb 8
  i.txt 33188 4 {} 0
}
}]

do_execsql_test 1.6.9 {
do_execsql_test 1.6.10 {
  SELECT a.name, a.data 
  FROM zz AS a, zz AS b 
  WHERE a.name=+b.name AND +a.mode=b.mode
} {
  blue.txt/ {}
  h.txt aaaaaaaaaabbbbbbbbbb
  i.txt {}
}

do_execsql_test 1.6.10 {
do_execsql_test 1.6.11 {
  SELECT name, data FROM zz WHERE name LIKE '%txt'
} {
  h.txt aaaaaaaaaabbbbbbbbbb
  i.txt {}
}

do_execsql_test 1.7 {
356
357
358
359
360
361
362
363

364
365

366
367
368





369

370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386

387
388
389
390
391
392
393
392
393
394
395
396
397
398

399
400

401
402
403
404
405
406
407
408
409

410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435







-
+

-
+



+
+
+
+
+
-
+

















+







} {
  dirname3/ 16877 {}
  dirname2/ 16877 {}
  dirname2/file1.txt 33188 abcdefghijklmnop
}
do_zip_tests 2.4a test.zip

# If on unix, check that the [unzip] utility can unpack our archive.
# Check that the [unzip] utility can unpack our archive.
#
if {$::tcl_platform(platform)=="unix"} {
if {[info exists ::UNZIP]} {
  do_test 2.5.1 {
    forcedelete dirname
    forcedelete dirname2
    if {$::tcl_platform(platform)=="unix"} {
      set null /dev/null
    } else {
      set null NUL
    }
    set rc [catch { exec unzip test.zip > /dev/null } msg]
    set rc [catch { exec $::UNZIP test.zip > $null } msg]
    list $rc $msg
  } {0 {}}
  do_test 2.5.2 { file isdir dirname3 } 1
  do_test 2.5.3 { file isdir dirname2 } 1
  do_test 2.5.4 { file isdir dirname2/file1.txt } 0
  do_test 2.5.5 { 
    set fd [open dirname2/file1.txt]
    set data [read $fd]
    close $fd
    set data
  } {abcdefghijklmnop}
}

#-------------------------------------------------------------------------
reset_db
forcedelete test.zip
load_static_extension db zipfile
load_static_extension db fileio

do_execsql_test 3.0 {
  CREATE VIRTUAL TABLE temp.x1 USING zipfile('test.zip');
  INSERT INTO x1(name, data) VALUES('dir1/', NULL);
  INSERT INTO x1(name, data) VALUES('file1', '1234');
  INSERT INTO x1(name, data) VALUES('dir1/file2', '5678');
}
449
450
451
452
453
454
455
456

457
458
459
460
461
462
463
464
465

466
467
468
469
470
471
472
491
492
493
494
495
496
497

498
499
500
501
502
503
504
505
506

507
508
509
510
511
512
513
514







-
+








-
+







  WITH c(name,data) AS ( 
    SELECT 'a.txt', 'abc' UNION ALL
    SELECT NULL, 'def'
  )
  SELECT zipfile(name,data) FROM c
} {1 {first argument to zipfile() must be non-NULL}}

do_catchsql_test 4.7 {
do_catchsql_test 4.8 {
  WITH c(name,data,method) AS ( 
    SELECT 'a.txt', 'abc', 0
    UNION SELECT 'b.txt', 'def', 8
    UNION SELECT 'c.txt', 'ghi', 16
  )
  SELECT zipfile(name,NULL,NULL,data,method) FROM c
} {1 {illegal method value: 16}}

do_catchsql_test 4.8 {
do_catchsql_test 4.9 {
  WITH c(name,data) AS ( 
    SELECT 'a.txt', 'abc'
    UNION SELECT 'b.txt', 'def'
    UNION SELECT 'c.txt/', 'ghi'
  )
  SELECT zipfile(name,NULL,NULL,data) FROM c
} {1 {non-directory name must not end with /}}
481
482
483
484
485
486
487
488

489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505

506
507
508
509
510
511
512
523
524
525
526
527
528
529

530
531

532
533
534
535
536
537
538
539
540
541
542
543
544
545

546
547
548
549
550
551
552
553







-
+

-














-
+







  SELECT name,mtime,data FROM zipfile(
    ( SELECT rt( zipfile(name,NULL,mtime,data,NULL) ) FROM c )
  )
} {
  a.txt 946684800 abc
}

if {[info vars ::UNZIP]!=""} { 
if {[info exists ::UNZIP]} {
ifcapable datetime {
  load_static_extension db fileio
  forcedelete test1.zip test2.zip
  do_test 6.0 {
    execsql {
      WITH c(name,mtime,data) AS (
        SELECT 'a.txt', 946684800, 'abc' UNION ALL
        SELECT 'b.txt', 1000000000, 'abc' UNION ALL
        SELECT 'c.txt', 1111111000, 'abc'
      )
      SELECT writefile('test1.zip', rt( zipfile(name, NULL, mtime, data) ) ),
             writefile('test2.zip',   ( zipfile(name, NULL, mtime, data) ) ) 
      FROM c;
    }
    forcedelete test_unzip
    file mkdir test_unzip
    exec unzip -d test_unzip test1.zip
    exec $::UNZIP -d test_unzip test1.zip

    db eval {
      SELECT name, strftime('%s', mtime, 'unixepoch', 'localtime') 
      FROM fsdir('test_unzip') WHERE name!='test_unzip'
      ORDER BY name
    }
  } [list {*}{
530
531
532
533
534
535
536
537

538
539
540
541
542
543
544
571
572
573
574
575
576
577

578
579
580
581
582
583
584
585







-
+







    b.txt 1000000000  abc
    c.txt 1111111000  abc
  }

  do_test 6.2 {
    forcedelete test_unzip
    file mkdir test_unzip
    exec unzip -d test_unzip test2.zip
    exec $::UNZIP -d test_unzip test2.zip

    db eval {
      SELECT name, mtime 
      FROM fsdir('test_unzip') WHERE name!='test_unzip'
      ORDER BY name
    }
  } [list {*}{
648
649
650
651
652
653
654


655
656
657
658
659
660
661
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704







+
+







#-------------------------------------------------------------------------
# INSERT OR REPLACE and INSERT OR IGNORE
#
catch {db close}
forcedelete test.zip test.db
sqlite3 db :memory:
load_static_extension db zipfile
load_static_extension db fileio

do_execsql_test 10.0 {
  CREATE VIRTUAL TABLE z USING zipfile('test.zip');
} {}
do_catchsql_test 10.1 {
  INSERT INTO z(name,data) VALUES('a0','one'),('a0','two');
} {1 {duplicate name: "a0"}}
do_execsql_test 10.2 {
670
671
672
673
674
675
676







































677
678
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760







+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+


do_execsql_test 10.5 {
  INSERT OR IGNORE INTO z(name,data) VALUES('a0','five'),('a0','six');
} {}
do_execsql_test 10.6 {
  SELECT name, data FROM z;
} {a0 four}

do_execsql_test 11.1 {
  DELETE FROM z;
} {}
do_execsql_test 11.2 {
  SELECT name, data FROM z;
} {}
do_execsql_test 11.3 {
  INSERT INTO z (name,data) VALUES ('b0','one');
  SELECT name, data FROM z;
} {b0 one}
do_execsql_test 11.4 {
  UPDATE z SET name = 'b1' WHERE name = 'b0';
  SELECT name, data FROM z;
} {b1 one}
do_execsql_test 11.5 {
  INSERT INTO z (name,data) VALUES ('b0','one');
  SELECT name, data FROM z ORDER BY name;
} {b0 one b1 one}
do_catchsql_test 11.6 {
  UPDATE z SET name = 'b1' WHERE name = 'b0';
} {1 {duplicate name: "b1"}}
do_execsql_test 11.7 {
  UPDATE z SET data = 'two' WHERE name = 'b0';
  SELECT name, data FROM z ORDER BY name;
} {b0 two b1 one}
do_catchsql_test 11.8 {
  UPDATE z SET name = 'b1';
} {1 {duplicate name: "b1"}}
do_catchsql_test 11.9 {
  UPDATE z SET name = 'b2';
} {1 {duplicate name: "b2"}}
do_execsql_test 11.10 {
  UPDATE z SET name = name;
  SELECT name, data FROM z ORDER BY name;
} {b0 two b2 one}
do_execsql_test 11.11 {
  UPDATE z SET name = name || 'suffix';
  SELECT name, data FROM z ORDER BY name;
} {b0suffix two b2suffix one}

finish_test

Changes to test/zipfile2.test.

1
2
3
4
5
6
7
8
9
10
11


12
13
14
15
16
17
18
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20











+
+







# 2018 January 30
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#***********************************************************************
#

package require Tcl 8.6

set testdir [file dirname $argv0]
source $testdir/tester.tcl
set testprefix zipfile2

ifcapable !vtab {
  finish_test; return
46
47
48
49
50
51
52





53
54
55
56
57
58

59
60
61
62
63
64
65
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64

65
66
67
68
69
70
71
72







+
+
+
+
+





-
+







  CREATE VIRTUAL TABLE bbb USING zipfile("testzip");
  CREATE VIRTUAL TABLE ccc USING zipfile(`testzip`);
  CREATE VIRTUAL TABLE ddd USING zipfile([testzip]);
  CREATE VIRTUAL TABLE eee USING zipfile(testzip);
  CREATE VIRTUAL TABLE fff USING zipfile('test''zip');
}

if {$::tcl_platform(platform)=="windows"} {
  set res {1 {cannot open file: testdir}}
} else {
  set res {1 {error in fread()}}
}
do_test 2.0 {
  forcedelete testdir
  file mkdir testdir
  execsql { CREATE VIRTUAL TABLE hhh USING zipfile('testdir') }
  catchsql { SELECT * FROM hhh } 
} {1 {error in fread()}}
} $res


set archive {
  504B0304140000080000D4A52BEC09F3B6E0110000001100000005000900612E
  747874555405000140420F00636F6E74656E7473206F6620612E747874504B03
  04140000080000D4A52BECD98916A7110000001100000005000900622E747874
  555405000140420F00636F6E74656E7473206F6620622E747874504B01021E03
199
200
201
202
203
204
205



206





























207
208
206
207
208
209
210
211
212
213
214
215

216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246







+
+
+
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+


  set hex [binary encode hex $blob]
  set hex [string map {6e6f7461646972 6e6f746164692f} $hex] 
  set blob2 [binary decode hex $hex]

  execsql { SELECT name, data IS NULL FROM zipfile($blob2) }
} {notadi/ 1}

#-------------------------------------------------------------------------
# Test that duplicate entries may not be created using UPDATE
# statements.

#
forcedelete test.zip
do_execsql_test 6.0 {
  CREATE VIRTUAL TABLE temp.zip USING zipfile('test.zip'); 
  INSERT INTO temp.zip (name,data) VALUES ('test1','test'); 
  INSERT INTO temp.zip (name,data) VALUES ('test2','test'); 
}
do_catchsql_test 6.1 {
  UPDATE temp.zip SET name='test1' WHERE name='test2'
} {1 {duplicate name: "test1"}}

forcedelete test.zip
do_catchsql_test 6.2 {
  DROP TABLE zip;
  CREATE VIRTUAL TABLE temp.zip USING zipfile('test.zip'); 
  INSERT INTO temp.zip (name,data) VALUES ('test','test'); 
  UPDATE  temp.zip set name=name||'new' where name='test'; 
  INSERT INTO temp.zip (name,data) VALUES ('test','test'); 
  UPDATE  temp.zip set name=name||'new' where name='test'; 
} {1 {duplicate name: "testnew"}}

forcedelete test.zip
do_execsql_test 6.3 {
  INSERT INTO temp.zip (name,data) VALUES ('test1','test'); 
  INSERT INTO temp.zip (name,data) VALUES ('test2','test'); 
  UPDATE OR REPLACE zip SET name='test2' WHERE name='test1';
  SELECT name FROM zip;
} {test2}

finish_test