/ Changes On Branch test-using-fuzzcheck
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Changes In Branch test-using-fuzzcheck Excluding Merge-Ins

This is equivalent to a diff from 193364c81c to e85628e34a

2015-05-26
19:08
The "make fuzztest" target now uses fuzzcheck instead of fuzzershell. Test data is added for database fuzz testing. (check-in: 5e3e410bf4 user: drh tags: trunk)
19:01
Merge accidental fork. (Closed-Leaf check-in: e85628e34a user: dan tags: test-using-fuzzcheck)
18:58
Fix a one-byte buffer overread that may follow a syntax error while preparing an SQL statement. (check-in: 075003930d user: dan tags: test-using-fuzzcheck)
18:58
Fix fuzzcheck so that it responds correctly to the TEST_FAILURE environment variable. (check-in: 76770c9e06 user: drh tags: test-using-fuzzcheck)
18:15
The "make fuzztest" target now uses fuzzcheck instead of fuzzershell. (check-in: 4a5f6f1f01 user: drh tags: test-using-fuzzcheck)
17:57
Enhance fuzzcheck with the ability to store descriptions in each source database and to run multiple source databases in a single invocation. (check-in: 193364c81c user: drh tags: trunk)
17:33
Add a cast to one side of the test added by [97806a78] in order to avoid a signed/unsigned comparison warning. (check-in: 5b46d2a78a user: dan tags: trunk)

Changes to Makefile.in.

518
519
520
521
522
523
524






525
526
527
528
529
530
531
#
TESTPROGS = \
  testfixture$(TEXE) \
  sqlite3$(TEXE) \
  sqlite3_analyzer$(TEXE) \
  sqldiff$(TEXE)








# This is the default Makefile target.  The objects listed here
# are what get build when you type just "make" with no arguments.
#
all:	sqlite3.h libsqlite3.la sqlite3$(TEXE) $(HAVE_TCL:1=libtclsqlite3.la)

Makefile: $(TOP)/Makefile.in







>
>
>
>
>
>







518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
#
TESTPROGS = \
  testfixture$(TEXE) \
  sqlite3$(TEXE) \
  sqlite3_analyzer$(TEXE) \
  sqldiff$(TEXE)

# Databases containing fuzzer test cases
#
FUZZDATA = \
  $(TOP)/test/fuzzdata1.db \
  $(TOP)/test/fuzzdata2.db \
  $(TOP)/test/fuzzdata3.db

# This is the default Makefile target.  The objects listed here
# are what get build when you type just "make" with no arguments.
#
all:	sqlite3.h libsqlite3.la sqlite3$(TEXE) $(HAVE_TCL:1=libtclsqlite3.la)

Makefile: $(TOP)/Makefile.in
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
		-o $@ $(TESTFIXTURE_SRC) $(LIBTCL) $(TLIBS)

# A very detailed test running most or all test cases
fulltest:	$(TESTPROGS) fuzztest
	./testfixture$(TEXE) $(TOP)/test/all.test

# Really really long testing
soaktest:	$(TESTPROGS) fuzzoomtest
	./testfixture$(TEXE) $(TOP)/test/all.test -soak=1

# Do extra testing but not everything.
fulltestonly:	$(TESTPROGS)
	./testfixture$(TEXE) $(TOP)/test/full.test

# Fuzz testing
fuzztest:	fuzzershell$(TEXE)
	./fuzzershell$(TEXE) $(TOP)/test/fuzzdata1.txt $(TOP)/test/fuzzdata2.txt

fuzzoomtest:	fuzzershell$(TEXE)
	./fuzzershell$(TEXE) -f $(TOP)/test/fuzzdata1.txt --oom

# This is the common case.  Run many tests but not those that take
# a really long time.
#
test:	$(TESTPROGS) fuzztest
	./testfixture$(TEXE) $(TOP)/test/veryquick.test

# Run a test using valgrind.  This can take a really long time
# because valgrind is so much slower than a native machine.
#
valgrindtest:	$(TESTPROGS) fuzzershell$(TEXE)
	valgrind -v ./fuzzershell$(TEXE) -f $(TOP)/test/fuzzdata1.txt
	OMIT_MISUSE=1 valgrind -v ./testfixture$(TEXE) $(TOP)/test/permutations.test valgrind

# A very fast test that checks basic sanity.  The name comes from
# the 60s-era electronics testing:  "Turn it on and see if smoke
# comes out."
#
smoketest:	$(TESTPROGS) fuzzershell$(TEXE)
	./testfixture$(TEXE) $(TOP)/test/main.test

sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl
	echo "#define TCLSH 2" > $@
	echo "#define SQLITE_ENABLE_DBSTAT_VTAB 1" >> $@
	cat sqlite3.c $(TOP)/src/tclsqlite.c >> $@
	echo "static const char *tclsh_main_loop(void){" >> $@







|







|
<
|
<
<










|
|






|







977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992

993


994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
		-o $@ $(TESTFIXTURE_SRC) $(LIBTCL) $(TLIBS)

# A very detailed test running most or all test cases
fulltest:	$(TESTPROGS) fuzztest
	./testfixture$(TEXE) $(TOP)/test/all.test

# Really really long testing
soaktest:	$(TESTPROGS)
	./testfixture$(TEXE) $(TOP)/test/all.test -soak=1

# Do extra testing but not everything.
fulltestonly:	$(TESTPROGS)
	./testfixture$(TEXE) $(TOP)/test/full.test

# Fuzz testing
fuzztest:	fuzzcheck$(TEXE)

	./fuzzcheck$(TEXE) $(FUZZDATA)



# This is the common case.  Run many tests but not those that take
# a really long time.
#
test:	$(TESTPROGS) fuzztest
	./testfixture$(TEXE) $(TOP)/test/veryquick.test

# Run a test using valgrind.  This can take a really long time
# because valgrind is so much slower than a native machine.
#
valgrindtest:	$(TESTPROGS) fuzzcheck$(TEXE)
	valgrind -v ./fuzzcheck$(TEXE) $(FUZZDATA)
	OMIT_MISUSE=1 valgrind -v ./testfixture$(TEXE) $(TOP)/test/permutations.test valgrind

# A very fast test that checks basic sanity.  The name comes from
# the 60s-era electronics testing:  "Turn it on and see if smoke
# comes out."
#
smoketest:	$(TESTPROGS) fuzzcheck$(TEXE)
	./testfixture$(TEXE) $(TOP)/test/main.test

sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl
	echo "#define TCLSH 2" > $@
	echo "#define SQLITE_ENABLE_DBSTAT_VTAB 1" >> $@
	cat sqlite3.c $(TOP)/src/tclsqlite.c >> $@
	echo "static const char *tclsh_main_loop(void){" >> $@

Changes to Makefile.msc.

1186
1187
1188
1189
1190
1191
1192







1193
1194
1195
1196
1197
1198
1199
#
TESTPROGS = \
  testfixture.exe \
  sqlite3.exe \
  sqlite3_analyzer.exe \
  sqldiff.exe









# This is the default Makefile target.  The objects listed here
# are what get build when you type just "make" with no arguments.
#
all:	dll libsqlite3.lib sqlite3.exe libtclsqlite3.lib

libsqlite3.lib:	$(LIBOBJ)







>
>
>
>
>
>
>







1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
#
TESTPROGS = \
  testfixture.exe \
  sqlite3.exe \
  sqlite3_analyzer.exe \
  sqldiff.exe

# Databases containing fuzzer test cases
#
FUZZDATA = \
  $(TOP)\test\fuzzdata1.db \
  $(TOP)\test\fuzzdata2.db \
  $(TOP)\test\fuzzdata3.db


# This is the default Makefile target.  The objects listed here
# are what get build when you type just "make" with no arguments.
#
all:	dll libsqlite3.lib sqlite3.exe libtclsqlite3.lib

libsqlite3.lib:	$(LIBOBJ)
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688

extensiontest: testfixture.exe testloadext.dll
	.\testfixture.exe $(TOP)\test\loadext.test

fulltest:	$(TESTPROGS) fuzztest
	.\testfixture.exe $(TOP)\test\all.test

soaktest:	$(TESTPROGS) fuzzoomtest
	.\testfixture.exe $(TOP)\test\all.test -soak=1

fulltestonly:	$(TESTPROGS) fuzztest
	.\testfixture.exe $(TOP)\test\full.test

queryplantest:	testfixture.exe sqlite3.exe
	.\testfixture.exe $(TOP)\test\permutations.test queryplanner

fuzztest:	fuzzershell.exe
	.\fuzzershell.exe $(TOP)\test\fuzzdata1.txt $(TOP)\test\fuzzdata2.txt

fuzzoomtest:	fuzzershell.exe
	.\fuzzershell.exe -f $(TOP)\test\fuzzdata1.txt --oom

test:	$(TESTPROGS) fuzztest
	.\testfixture.exe $(TOP)\test\veryquick.test

smoketest:	$(TESTPROGS) fuzzershell.exe
	.\testfixture.exe $(TOP)\test\main.test

sqlite3_analyzer.c: $(SQLITE3C) $(TOP)\src\tclsqlite.c $(TOP)\tool\spaceanal.tcl
	echo #define TCLSH 2 > $@
	echo #define SQLITE_ENABLE_DBSTAT_VTAB 1 >> $@
	copy $@ + $(SQLITE3C) + $(TOP)\src\tclsqlite.c $@
	echo static const char *tclsh_main_loop(void){ >> $@







|








|
<
|
<
<




|







1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679

1680


1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692

extensiontest: testfixture.exe testloadext.dll
	.\testfixture.exe $(TOP)\test\loadext.test

fulltest:	$(TESTPROGS) fuzztest
	.\testfixture.exe $(TOP)\test\all.test

soaktest:	$(TESTPROGS)
	.\testfixture.exe $(TOP)\test\all.test -soak=1

fulltestonly:	$(TESTPROGS) fuzztest
	.\testfixture.exe $(TOP)\test\full.test

queryplantest:	testfixture.exe sqlite3.exe
	.\testfixture.exe $(TOP)\test\permutations.test queryplanner

fuzztest:	fuzzcheck.exe

	.\fuzzcheck.exe $(FUZZDATA)



test:	$(TESTPROGS) fuzztest
	.\testfixture.exe $(TOP)\test\veryquick.test

smoketest:	$(TESTPROGS)
	.\testfixture.exe $(TOP)\test\main.test

sqlite3_analyzer.c: $(SQLITE3C) $(TOP)\src\tclsqlite.c $(TOP)\tool\spaceanal.tcl
	echo #define TCLSH 2 > $@
	echo #define SQLITE_ENABLE_DBSTAT_VTAB 1 >> $@
	copy $@ + $(SQLITE3C) + $(TOP)\src\tclsqlite.c $@
	echo static const char *tclsh_main_loop(void){ >> $@

Changes to main.mk.

398
399
400
401
402
403
404







405
406
407
408
409
410
411
# executables needed for testing
#
TESTPROGS = \
  testfixture$(EXE) \
  sqlite3$(EXE) \
  sqlite3_analyzer$(EXE) \
  sqldiff$(EXE)








# This is the default Makefile target.  The objects listed here
# are what get build when you type just "make" with no arguments.
#
all:	sqlite3.h libsqlite3.a sqlite3$(EXE)

libsqlite3.a:	$(LIBOBJ)







>
>
>
>
>
>
>







398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
# executables needed for testing
#
TESTPROGS = \
  testfixture$(EXE) \
  sqlite3$(EXE) \
  sqlite3_analyzer$(EXE) \
  sqldiff$(EXE)

# Databases containing fuzzer test cases
#
FUZZDATA = \
  $(TOP)/test/fuzzdata1.db \
  $(TOP)/test/fuzzdata2.db \
  $(TOP)/test/fuzzdata3.db

# This is the default Makefile target.  The objects listed here
# are what get build when you type just "make" with no arguments.
#
all:	sqlite3.h libsqlite3.a sqlite3$(EXE)

libsqlite3.a:	$(LIBOBJ)
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
	-DSQLITE_ENABLE_FTS3=1                                               \
		$(TESTSRC) $(TOP)/src/tclsqlite.c sqlite3.c fts3amal.c       \
		-o testfixture$(EXE) $(LIBTCL) $(THREADLIB)

fulltest:	$(TESTPROGS) fuzztest
	./testfixture$(EXE) $(TOP)/test/all.test

soaktest:	$(TESTPROGS) fuzzoomtest
	./testfixture$(EXE) $(TOP)/test/all.test -soak=1

fulltestonly:	$(TESTPROGS) fuzztest
	./testfixture$(EXE) $(TOP)/test/full.test

queryplantest:	testfixture$(EXE) sqlite3$(EXE)
	./testfixture$(EXE) $(TOP)/test/permutations.test queryplanner

fuzztest:	fuzzershell$(EXE)
	./fuzzershell$(EXE) $(TOP)/test/fuzzdata1.txt $(TOP)/test/fuzzdata2.txt

fuzzoomtest:	fuzzershell$(EXE)
	./fuzzershell$(EXE) -f $(TOP)/test/fuzzdata1.txt --oom

test:	$(TESTPROGS) fuzztest
	./testfixture$(EXE) $(TOP)/test/veryquick.test

# Run a test using valgrind.  This can take a really long time
# because valgrind is so much slower than a native machine.
#
valgrindtest:	$(TESTPROGS) fuzzershell$(EXE)
	valgrind -v ./fuzzershell$(EXE) -f $(TOP)/test/fuzzdata1.txt
	OMIT_MISUSE=1 valgrind -v ./testfixture$(EXE) $(TOP)/test/permutations.test valgrind

# A very fast test that checks basic sanity.  The name comes from
# the 60s-era electronics testing:  "Turn it on and see if smoke
# comes out."
#
smoketest:	$(TESTPROGS) fuzzershell$(EXE)
	./testfixture$(EXE) $(TOP)/test/main.test

# The next two rules are used to support the "threadtest" target. Building
# threadtest runs a few thread-safety tests that are implemented in C. This
# target is invoked by the releasetest.tcl script.
# 
THREADTEST3_SRC = $(TOP)/test/threadtest3.c    \







|








|
<
|
<
<







|
|






|







660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676

677


678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
	-DSQLITE_ENABLE_FTS3=1                                               \
		$(TESTSRC) $(TOP)/src/tclsqlite.c sqlite3.c fts3amal.c       \
		-o testfixture$(EXE) $(LIBTCL) $(THREADLIB)

fulltest:	$(TESTPROGS) fuzztest
	./testfixture$(EXE) $(TOP)/test/all.test

soaktest:	$(TESTPROGS)
	./testfixture$(EXE) $(TOP)/test/all.test -soak=1

fulltestonly:	$(TESTPROGS) fuzztest
	./testfixture$(EXE) $(TOP)/test/full.test

queryplantest:	testfixture$(EXE) sqlite3$(EXE)
	./testfixture$(EXE) $(TOP)/test/permutations.test queryplanner

fuzztest:	fuzzcheck$(EXE) $(FUZZDATA)

	./fuzzcheck$(EXE) $(FUZZDATA)



test:	$(TESTPROGS) fuzztest
	./testfixture$(EXE) $(TOP)/test/veryquick.test

# Run a test using valgrind.  This can take a really long time
# because valgrind is so much slower than a native machine.
#
valgrindtest:	$(TESTPROGS) fuzzcheck$(EXE) $(FUZZDATA)
	valgrind -v ./fuzzcheck$(EXE) $(FUZZDATA)
	OMIT_MISUSE=1 valgrind -v ./testfixture$(EXE) $(TOP)/test/permutations.test valgrind

# A very fast test that checks basic sanity.  The name comes from
# the 60s-era electronics testing:  "Turn it on and see if smoke
# comes out."
#
smoketest:	$(TESTPROGS) fuzzcheck$(EXE)
	./testfixture$(EXE) $(TOP)/test/main.test

# The next two rules are used to support the "threadtest" target. Building
# threadtest runs a few thread-safety tests that are implemented in C. This
# target is invoked by the releasetest.tcl script.
# 
THREADTEST3_SRC = $(TOP)/test/threadtest3.c    \

Changes to src/tokenize.c.

446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
        }
        break;
      }
    }
  }
abort_parse:
  assert( nErr==0 );
  if( zSql[i]==0 && pParse->rc==SQLITE_OK && db->mallocFailed==0 ){
    if( lastTokenParsed!=TK_SEMI ){
      sqlite3Parser(pEngine, TK_SEMI, pParse->sLastToken, pParse);
      pParse->zTail = &zSql[i];
    }
    if( pParse->rc==SQLITE_OK && db->mallocFailed==0 ){
      sqlite3Parser(pEngine, 0, pParse->sLastToken, pParse);
    }







|







446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
        }
        break;
      }
    }
  }
abort_parse:
  assert( nErr==0 );
  if( pParse->rc==SQLITE_OK && db->mallocFailed==0 && zSql[i]==0 ){
    if( lastTokenParsed!=TK_SEMI ){
      sqlite3Parser(pEngine, TK_SEMI, pParse->sLastToken, pParse);
      pParse->zTail = &zSql[i];
    }
    if( pParse->rc==SQLITE_OK && db->mallocFailed==0 ){
      sqlite3Parser(pEngine, 0, pParse->sLastToken, pParse);
    }

Changes to test/fuzzcheck.c.

629
630
631
632
633
634
635

636
637
638

639
640
641
642
643
644
645
  int runFlags = 0;            /* Flags sent to runSql() */
  char *zMsg = 0;              /* Add this message */
  int nSrcDb = 0;              /* Number of source databases */
  char **azSrcDb = 0;          /* Array of source database names */
  int iSrcDb;                  /* Loop over all source databases */
  int nTest = 0;               /* Total number of tests performed */
  char *zDbName = "";          /* Appreviated name of a source database */


  iBegin = timeOfDay();
  g.zArgv0 = argv[0];

  for(i=1; i<argc; i++){
    const char *z = argv[i];
    if( z[0]=='-' ){
      z++;
      if( z[0]=='-' ) z++;
      if( strcmp(z,"dbid")==0 ){
        if( i>=argc-1 ) fatalError("missing arguments on %s", argv[i]);







>



>







629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
  int runFlags = 0;            /* Flags sent to runSql() */
  char *zMsg = 0;              /* Add this message */
  int nSrcDb = 0;              /* Number of source databases */
  char **azSrcDb = 0;          /* Array of source database names */
  int iSrcDb;                  /* Loop over all source databases */
  int nTest = 0;               /* Total number of tests performed */
  char *zDbName = "";          /* Appreviated name of a source database */
  const char *zFailCode = 0;   /* Value of the TEST_FAILURE environment variable */

  iBegin = timeOfDay();
  g.zArgv0 = argv[0];
  zFailCode = getenv("TEST_FAILURE");
  for(i=1; i<argc; i++){
    const char *z = argv[i];
    if( z[0]=='-' ){
      z++;
      if( z[0]=='-' ) z++;
      if( strcmp(z,"dbid")==0 ){
        if( i>=argc-1 ) fatalError("missing arguments on %s", argv[i]);
825
826
827
828
829
830
831
















832
833
834
835
836
837
838
839

840
841
842
843
844
845
846
        if( rc ) fatalError("cannot open inmem database");
        runSql(db, (char*)pSql->a, runFlags);
        sqlite3_close(db);
        if( sqlite3_memory_used()>0 ) fatalError("memory leak");
        reformatVfs();
        nTest++;
        g.zTestName[0] = 0;
















      }
    }
    if( !quietFlag && !verboseFlag ){
      printf(" 100%% - %d tests\n", g.nDb*g.nSql);
    }
  
    /* Clean up at the end of processing a single source database
    */

    blobListFree(g.pFirstSql);
    blobListFree(g.pFirstDb);
    reformatVfs();
 
  } /* End loop over all source databases */

  if( !quietFlag ){







>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>








>







827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
        if( rc ) fatalError("cannot open inmem database");
        runSql(db, (char*)pSql->a, runFlags);
        sqlite3_close(db);
        if( sqlite3_memory_used()>0 ) fatalError("memory leak");
        reformatVfs();
        nTest++;
        g.zTestName[0] = 0;

        /* Simulate an error if the TEST_FAILURE environment variable is "5".
        ** This is used to verify that automated test script really do spot
        ** errors that occur in this test program.
        */
        if( zFailCode ){
          if( zFailCode[0]=='5' && zFailCode[1]==0 ){
            fatalError("simulated failure");
          }else if( zFailCode[0]!=0 ){
            /* If TEST_FAILURE is something other than 5, just exit the test
            ** early */
            printf("\nExit early due to TEST_FAILURE being set\n");
            iSrcDb = nSrcDb-1;
            goto sourcedb_cleanup;
          }
        }
      }
    }
    if( !quietFlag && !verboseFlag ){
      printf(" 100%% - %d tests\n", g.nDb*g.nSql);
    }
  
    /* Clean up at the end of processing a single source database
    */
  sourcedb_cleanup:
    blobListFree(g.pFirstSql);
    blobListFree(g.pFirstDb);
    reformatVfs();
 
  } /* End loop over all source databases */

  if( !quietFlag ){

Added test/fuzzdata1.db.

cannot compute difference between binary files

Deleted test/fuzzdata1.txt.

cannot compute difference between binary files

Added test/fuzzdata2.db.

cannot compute difference between binary files

Deleted test/fuzzdata2.txt.

cannot compute difference between binary files

Added test/fuzzdata3.db.

cannot compute difference between binary files

Changes to test/misc1.test.

689
690
691
692
693
694
695








696
  UPDATE sqlite_master SET sql='CREATE table y(a TEXT, a TEXT)';
  BEGIN;
  CREATE TABLE t2(y);
  ROLLBACK;
  DROP TABLE IF EXISTS t;
} {0 {}}









finish_test







>
>
>
>
>
>
>
>

689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
  UPDATE sqlite_master SET sql='CREATE table y(a TEXT, a TEXT)';
  BEGIN;
  CREATE TABLE t2(y);
  ROLLBACK;
  DROP TABLE IF EXISTS t;
} {0 {}}


# At one point, running this would read one byte passed the end of a 
# buffer, upsetting valgrind.
#
do_test misc1-24.0 {
  list [catch { sqlite3_prepare_v2 db ! -1 dummy } msg] $msg
} {1 {(1) unrecognized token: "!}}

finish_test

Deleted test/mkfuzzdata1.tcl.

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
#!/usr/bin/tclsh
#
# Run this script in order to rebuild the fuzzdata1.txt file containing
# fuzzer data for the fuzzershell utility that is create by afl-fuzz.
#
# This script gathers all of the test cases identified by afl-fuzz and
# runs afl-cmin and afl-tmin over them all to try to generate a mimimum
# set of tests that cover all observed behavior.
# 
# Options:
#
#    --afl-bin DIR1             DIR1 contains the AFL binaries
#    --fuzzershell PATH         Full pathname of instrumented fuzzershell
#    --afl-data DIR3            DIR3 is the "-o" directory from afl-fuzz
#    -o FILE                    Write results into FILE
#
set AFLBIN {}
set FUZZERSHELL {}
set AFLDATA {}
set OUTFILE {}

proc usage {} {
  puts stderr "Usage: $::argv0 --afl-bin DIR --fuzzershell PATH\
                  --afl-data DIR -o FILE"
  exit 1
}
proc cmdlineerr {msg} {
  puts stderr $msg
  usage
}

for {set i 0} {$i<[llength $argv]} {incr i} {
  set x [lindex $argv $i]
  if {[string index $x 0]!="-"} {cmdlineerr "illegal argument: $x"}
  set x [string trimleft $x -]
  incr i
  if {$i>=[llength $argv]} {cmdlineerr "no argument on --$x"}
  set a [lindex $argv $i]
  switch -- $x {
     afl-bin {set AFLBIN $a}
     afl-data {set AFLDATA $a}
     fuzzershell {set FUZZERSHELL $a}
     o {set OUTFILE $a}
     default {cmdlineerr "unknown option: --$x"}
  }
}
proc checkarg {varname option} {
  set val [set ::$varname]
  if {$val==""} {cmdlineerr "required option missing: --$option"}
}
checkarg AFLBIN afl-bin
checkarg AFLDATA afl-data
checkarg FUZZERSHELL fuzzershell
checkarg OUTFILE o
proc checkexec {x} {
  if {![file exec $x]} {cmdlineerr "cannot find $x"}
}
checkexec $AFLBIN/afl-cmin
checkexec $AFLBIN/afl-tmin
checkexec $FUZZERSHELL
proc checkdir {x} {
  if {![file isdir $x]} {cmdlineerr "no such directory: $x"}
}
checkdir $AFLDATA/queue

proc progress {msg} {
  puts "******** $msg"
  flush stdout
}
progress "mkdir tmp1 tmp2"
file mkdir tmp1 tmp2
progress "copying test cases from $AFLDATA into tmp1..."
set n 0
foreach file [glob -nocomplain $AFLDATA/queue/id:*] {
  incr n
  file copy $file tmp1/$n
}
foreach file [glob -nocomplain $AFLDATA/crash*/id:*] {
  incr n
  file copy $file tmp1/$n
}
progress "total $n files copied."
progress "running: $AFLBIN/afl-cmin -i tmp1 -o tmp2 $FUZZERSHELL"
exec $AFLBIN/afl-cmin -i tmp1 -o tmp2 $FUZZERSHELL >&@ stdout
progress "afl-cmin complete."
#
# Experiments show that running afl-tmin is too slow for this application.
# And it doesn't really make the test cases that much smaller.  So let's
# just skip it.
#
# foreach file [glob tmp2/*] {
#   progress "$AFLBIN/afl-tmin -i $file -o tmp3/[file tail $file] $FUZZERSHELL"
#   exec $AFLBIN/afl-tmin -i $file -o tmp3/[file tail $file] \
#       $FUZZERSHELL >&@ stdout
# }
progress "generating final output into $OUTFILE"
set out [open $OUTFILE wb]
puts $out "# Test data for use with fuzzershell.  Automatically
# generated using $argv0.  This file contains binary data
#"
set n 0
foreach file [glob tmp2/*] {
  incr n
  puts -nonewline $out "/****<$n>****/"
  set in [open $file rb]
  puts -nonewline $out [read $in]
  close $in
}
close $out
progress "done.  $n test cases written to $OUTFILE"
progress "clean-up..."
file delete -force tmp1
progress "culled test cases left in the tmp2 directory"
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<
<