Index: Makefile.in ================================================================== --- Makefile.in +++ Makefile.in @@ -520,10 +520,16 @@ testfixture$(TEXE) \ sqlite3$(TEXE) \ sqlite3_analyzer$(TEXE) \ sqldiff$(TEXE) +# Databases containing fuzzer test cases +# +FUZZDATA = \ + $(TOP)/test/fuzzdata1.db \ + $(TOP)/test/fuzzdata2.db \ + $(TOP)/test/fuzzdata3.db # This is the default Makefile target. The objects listed here # are what get build when you type just "make" with no arguments. # all: sqlite3.h libsqlite3.la sqlite3$(TEXE) $(HAVE_TCL:1=libtclsqlite3.la) @@ -973,23 +979,20 @@ # A very detailed test running most or all test cases fulltest: $(TESTPROGS) fuzztest ./testfixture$(TEXE) $(TOP)/test/all.test # Really really long testing -soaktest: $(TESTPROGS) fuzzoomtest +soaktest: $(TESTPROGS) ./testfixture$(TEXE) $(TOP)/test/all.test -soak=1 # Do extra testing but not everything. fulltestonly: $(TESTPROGS) ./testfixture$(TEXE) $(TOP)/test/full.test # Fuzz testing -fuzztest: fuzzershell$(TEXE) - ./fuzzershell$(TEXE) $(TOP)/test/fuzzdata1.txt $(TOP)/test/fuzzdata2.txt - -fuzzoomtest: fuzzershell$(TEXE) - ./fuzzershell$(TEXE) -f $(TOP)/test/fuzzdata1.txt --oom +fuzztest: fuzzcheck$(TEXE) + ./fuzzcheck$(TEXE) $(FUZZDATA) # This is the common case. Run many tests but not those that take # a really long time. # test: $(TESTPROGS) fuzztest @@ -996,19 +999,19 @@ ./testfixture$(TEXE) $(TOP)/test/veryquick.test # Run a test using valgrind. This can take a really long time # because valgrind is so much slower than a native machine. # -valgrindtest: $(TESTPROGS) fuzzershell$(TEXE) - valgrind -v ./fuzzershell$(TEXE) -f $(TOP)/test/fuzzdata1.txt +valgrindtest: $(TESTPROGS) fuzzcheck$(TEXE) + valgrind -v ./fuzzcheck$(TEXE) $(FUZZDATA) OMIT_MISUSE=1 valgrind -v ./testfixture$(TEXE) $(TOP)/test/permutations.test valgrind # A very fast test that checks basic sanity. The name comes from # the 60s-era electronics testing: "Turn it on and see if smoke # comes out." # -smoketest: $(TESTPROGS) fuzzershell$(TEXE) +smoketest: $(TESTPROGS) fuzzcheck$(TEXE) ./testfixture$(TEXE) $(TOP)/test/main.test sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl echo "#define TCLSH 2" > $@ echo "#define SQLITE_ENABLE_DBSTAT_VTAB 1" >> $@ Index: Makefile.msc ================================================================== --- Makefile.msc +++ Makefile.msc @@ -1188,10 +1188,17 @@ testfixture.exe \ sqlite3.exe \ sqlite3_analyzer.exe \ sqldiff.exe +# Databases containing fuzzer test cases +# +FUZZDATA = \ + $(TOP)\test\fuzzdata1.db \ + $(TOP)\test\fuzzdata2.db \ + $(TOP)\test\fuzzdata3.db + # This is the default Makefile target. The objects listed here # are what get build when you type just "make" with no arguments. # all: dll libsqlite3.lib sqlite3.exe libtclsqlite3.lib @@ -1658,29 +1665,26 @@ .\testfixture.exe $(TOP)\test\loadext.test fulltest: $(TESTPROGS) fuzztest .\testfixture.exe $(TOP)\test\all.test -soaktest: $(TESTPROGS) fuzzoomtest +soaktest: $(TESTPROGS) .\testfixture.exe $(TOP)\test\all.test -soak=1 fulltestonly: $(TESTPROGS) fuzztest .\testfixture.exe $(TOP)\test\full.test queryplantest: testfixture.exe sqlite3.exe .\testfixture.exe $(TOP)\test\permutations.test queryplanner -fuzztest: fuzzershell.exe - .\fuzzershell.exe $(TOP)\test\fuzzdata1.txt $(TOP)\test\fuzzdata2.txt - -fuzzoomtest: fuzzershell.exe - .\fuzzershell.exe -f $(TOP)\test\fuzzdata1.txt --oom +fuzztest: fuzzcheck.exe + .\fuzzcheck.exe $(FUZZDATA) test: $(TESTPROGS) fuzztest .\testfixture.exe $(TOP)\test\veryquick.test -smoketest: $(TESTPROGS) fuzzershell.exe +smoketest: $(TESTPROGS) .\testfixture.exe $(TOP)\test\main.test sqlite3_analyzer.c: $(SQLITE3C) $(TOP)\src\tclsqlite.c $(TOP)\tool\spaceanal.tcl echo #define TCLSH 2 > $@ echo #define SQLITE_ENABLE_DBSTAT_VTAB 1 >> $@ Index: main.mk ================================================================== --- main.mk +++ main.mk @@ -400,10 +400,17 @@ TESTPROGS = \ testfixture$(EXE) \ sqlite3$(EXE) \ sqlite3_analyzer$(EXE) \ sqldiff$(EXE) + +# Databases containing fuzzer test cases +# +FUZZDATA = \ + $(TOP)/test/fuzzdata1.db \ + $(TOP)/test/fuzzdata2.db \ + $(TOP)/test/fuzzdata3.db # This is the default Makefile target. The objects listed here # are what get build when you type just "make" with no arguments. # all: sqlite3.h libsqlite3.a sqlite3$(EXE) @@ -655,40 +662,37 @@ -o testfixture$(EXE) $(LIBTCL) $(THREADLIB) fulltest: $(TESTPROGS) fuzztest ./testfixture$(EXE) $(TOP)/test/all.test -soaktest: $(TESTPROGS) fuzzoomtest +soaktest: $(TESTPROGS) ./testfixture$(EXE) $(TOP)/test/all.test -soak=1 fulltestonly: $(TESTPROGS) fuzztest ./testfixture$(EXE) $(TOP)/test/full.test queryplantest: testfixture$(EXE) sqlite3$(EXE) ./testfixture$(EXE) $(TOP)/test/permutations.test queryplanner -fuzztest: fuzzershell$(EXE) - ./fuzzershell$(EXE) $(TOP)/test/fuzzdata1.txt $(TOP)/test/fuzzdata2.txt - -fuzzoomtest: fuzzershell$(EXE) - ./fuzzershell$(EXE) -f $(TOP)/test/fuzzdata1.txt --oom +fuzztest: fuzzcheck$(EXE) $(FUZZDATA) + ./fuzzcheck$(EXE) $(FUZZDATA) test: $(TESTPROGS) fuzztest ./testfixture$(EXE) $(TOP)/test/veryquick.test # Run a test using valgrind. This can take a really long time # because valgrind is so much slower than a native machine. # -valgrindtest: $(TESTPROGS) fuzzershell$(EXE) - valgrind -v ./fuzzershell$(EXE) -f $(TOP)/test/fuzzdata1.txt +valgrindtest: $(TESTPROGS) fuzzcheck$(EXE) $(FUZZDATA) + valgrind -v ./fuzzcheck$(EXE) $(FUZZDATA) OMIT_MISUSE=1 valgrind -v ./testfixture$(EXE) $(TOP)/test/permutations.test valgrind # A very fast test that checks basic sanity. The name comes from # the 60s-era electronics testing: "Turn it on and see if smoke # comes out." # -smoketest: $(TESTPROGS) fuzzershell$(EXE) +smoketest: $(TESTPROGS) fuzzcheck$(EXE) ./testfixture$(EXE) $(TOP)/test/main.test # The next two rules are used to support the "threadtest" target. Building # threadtest runs a few thread-safety tests that are implemented in C. This # target is invoked by the releasetest.tcl script. Index: src/tokenize.c ================================================================== --- src/tokenize.c +++ src/tokenize.c @@ -448,11 +448,11 @@ } } } abort_parse: assert( nErr==0 ); - if( zSql[i]==0 && pParse->rc==SQLITE_OK && db->mallocFailed==0 ){ + if( pParse->rc==SQLITE_OK && db->mallocFailed==0 && zSql[i]==0 ){ if( lastTokenParsed!=TK_SEMI ){ sqlite3Parser(pEngine, TK_SEMI, pParse->sLastToken, pParse); pParse->zTail = &zSql[i]; } if( pParse->rc==SQLITE_OK && db->mallocFailed==0 ){ Index: test/fuzzcheck.c ================================================================== --- test/fuzzcheck.c +++ test/fuzzcheck.c @@ -631,13 +631,15 @@ int nSrcDb = 0; /* Number of source databases */ char **azSrcDb = 0; /* Array of source database names */ int iSrcDb; /* Loop over all source databases */ int nTest = 0; /* Total number of tests performed */ char *zDbName = ""; /* Appreviated name of a source database */ + const char *zFailCode = 0; /* Value of the TEST_FAILURE environment variable */ iBegin = timeOfDay(); g.zArgv0 = argv[0]; + zFailCode = getenv("TEST_FAILURE"); for(i=1; i0 ) fatalError("memory leak"); reformatVfs(); nTest++; g.zTestName[0] = 0; + + /* Simulate an error if the TEST_FAILURE environment variable is "5". + ** This is used to verify that automated test script really do spot + ** errors that occur in this test program. + */ + if( zFailCode ){ + if( zFailCode[0]=='5' && zFailCode[1]==0 ){ + fatalError("simulated failure"); + }else if( zFailCode[0]!=0 ){ + /* If TEST_FAILURE is something other than 5, just exit the test + ** early */ + printf("\nExit early due to TEST_FAILURE being set\n"); + iSrcDb = nSrcDb-1; + goto sourcedb_cleanup; + } + } } } if( !quietFlag && !verboseFlag ){ printf(" 100%% - %d tests\n", g.nDb*g.nSql); } /* Clean up at the end of processing a single source database */ + sourcedb_cleanup: blobListFree(g.pFirstSql); blobListFree(g.pFirstDb); reformatVfs(); } /* End loop over all source databases */ ADDED test/fuzzdata1.db Index: test/fuzzdata1.db ================================================================== --- /dev/null +++ test/fuzzdata1.db cannot compute difference between binary files DELETED test/fuzzdata1.txt Index: test/fuzzdata1.txt ================================================================== --- test/fuzzdata1.txt +++ /dev/null cannot compute difference between binary files ADDED test/fuzzdata2.db Index: test/fuzzdata2.db ================================================================== --- /dev/null +++ test/fuzzdata2.db cannot compute difference between binary files DELETED test/fuzzdata2.txt Index: test/fuzzdata2.txt ================================================================== --- test/fuzzdata2.txt +++ /dev/null cannot compute difference between binary files ADDED test/fuzzdata3.db Index: test/fuzzdata3.db ================================================================== --- /dev/null +++ test/fuzzdata3.db cannot compute difference between binary files Index: test/misc1.test ================================================================== --- test/misc1.test +++ test/misc1.test @@ -691,6 +691,14 @@ CREATE TABLE t2(y); ROLLBACK; DROP TABLE IF EXISTS t; } {0 {}} + +# At one point, running this would read one byte passed the end of a +# buffer, upsetting valgrind. +# +do_test misc1-24.0 { + list [catch { sqlite3_prepare_v2 db ! -1 dummy } msg] $msg +} {1 {(1) unrecognized token: "!}} + finish_test DELETED test/mkfuzzdata1.tcl Index: test/mkfuzzdata1.tcl ================================================================== --- test/mkfuzzdata1.tcl +++ /dev/null @@ -1,113 +0,0 @@ -#!/usr/bin/tclsh -# -# Run this script in order to rebuild the fuzzdata1.txt file containing -# fuzzer data for the fuzzershell utility that is create by afl-fuzz. -# -# This script gathers all of the test cases identified by afl-fuzz and -# runs afl-cmin and afl-tmin over them all to try to generate a mimimum -# set of tests that cover all observed behavior. -# -# Options: -# -# --afl-bin DIR1 DIR1 contains the AFL binaries -# --fuzzershell PATH Full pathname of instrumented fuzzershell -# --afl-data DIR3 DIR3 is the "-o" directory from afl-fuzz -# -o FILE Write results into FILE -# -set AFLBIN {} -set FUZZERSHELL {} -set AFLDATA {} -set OUTFILE {} - -proc usage {} { - puts stderr "Usage: $::argv0 --afl-bin DIR --fuzzershell PATH\ - --afl-data DIR -o FILE" - exit 1 -} -proc cmdlineerr {msg} { - puts stderr $msg - usage -} - -for {set i 0} {$i<[llength $argv]} {incr i} { - set x [lindex $argv $i] - if {[string index $x 0]!="-"} {cmdlineerr "illegal argument: $x"} - set x [string trimleft $x -] - incr i - if {$i>=[llength $argv]} {cmdlineerr "no argument on --$x"} - set a [lindex $argv $i] - switch -- $x { - afl-bin {set AFLBIN $a} - afl-data {set AFLDATA $a} - fuzzershell {set FUZZERSHELL $a} - o {set OUTFILE $a} - default {cmdlineerr "unknown option: --$x"} - } -} -proc checkarg {varname option} { - set val [set ::$varname] - if {$val==""} {cmdlineerr "required option missing: --$option"} -} -checkarg AFLBIN afl-bin -checkarg AFLDATA afl-data -checkarg FUZZERSHELL fuzzershell -checkarg OUTFILE o -proc checkexec {x} { - if {![file exec $x]} {cmdlineerr "cannot find $x"} -} -checkexec $AFLBIN/afl-cmin -checkexec $AFLBIN/afl-tmin -checkexec $FUZZERSHELL -proc checkdir {x} { - if {![file isdir $x]} {cmdlineerr "no such directory: $x"} -} -checkdir $AFLDATA/queue - -proc progress {msg} { - puts "******** $msg" - flush stdout -} -progress "mkdir tmp1 tmp2" -file mkdir tmp1 tmp2 -progress "copying test cases from $AFLDATA into tmp1..." -set n 0 -foreach file [glob -nocomplain $AFLDATA/queue/id:*] { - incr n - file copy $file tmp1/$n -} -foreach file [glob -nocomplain $AFLDATA/crash*/id:*] { - incr n - file copy $file tmp1/$n -} -progress "total $n files copied." -progress "running: $AFLBIN/afl-cmin -i tmp1 -o tmp2 $FUZZERSHELL" -exec $AFLBIN/afl-cmin -i tmp1 -o tmp2 $FUZZERSHELL >&@ stdout -progress "afl-cmin complete." -# -# Experiments show that running afl-tmin is too slow for this application. -# And it doesn't really make the test cases that much smaller. So let's -# just skip it. -# -# foreach file [glob tmp2/*] { -# progress "$AFLBIN/afl-tmin -i $file -o tmp3/[file tail $file] $FUZZERSHELL" -# exec $AFLBIN/afl-tmin -i $file -o tmp3/[file tail $file] \ -# $FUZZERSHELL >&@ stdout -# } -progress "generating final output into $OUTFILE" -set out [open $OUTFILE wb] -puts $out "# Test data for use with fuzzershell. Automatically -# generated using $argv0. This file contains binary data -#" -set n 0 -foreach file [glob tmp2/*] { - incr n - puts -nonewline $out "/****<$n>****/" - set in [open $file rb] - puts -nonewline $out [read $in] - close $in -} -close $out -progress "done. $n test cases written to $OUTFILE" -progress "clean-up..." -file delete -force tmp1 -progress "culled test cases left in the tmp2 directory"