DELETED Makefile.arm-wince-mingw32ce-gcc Index: Makefile.arm-wince-mingw32ce-gcc ================================================================== --- Makefile.arm-wince-mingw32ce-gcc +++ /dev/null @@ -1,138 +0,0 @@ -#!/usr/make -# -# Makefile for SQLITE -# -# This is a template makefile for SQLite. Most people prefer to -# use the autoconf generated "configure" script to generate the -# makefile automatically. But that does not work for everybody -# and in every situation. If you are having problems with the -# "configure" script, you might want to try this makefile as an -# alternative. Create a copy of this file, edit the parameters -# below and type "make". -# - -#### The directory where to find the mingw32ce tools -MINGW32CE = /opt/mingw32ce/bin - -#### The target prefix of the mingw32ce tools -TARGET = arm-wince-mingw32ce - -#### The toplevel directory of the source tree. This is the directory -# that contains this "Makefile.in" and the "configure.in" script. -# -TOP = ../sqlite - -#### C Compiler and options for use in building executables that -# will run on the platform that is doing the build. -# -BCC = gcc -g -O2 -#BCC = /opt/ancic/bin/c89 -0 - -#### If the target operating system supports the "usleep()" system -# call, then define the HAVE_USLEEP macro for all C modules. -# -USLEEP = -#USLEEP = -DHAVE_USLEEP=1 - -#### If you want the SQLite library to be safe for use within a -# multi-threaded program, then define the following macro -# appropriately: -# -THREADSAFE = -DTHREADSAFE=1 -#THREADSAFE = -DTHREADSAFE=0 - -#### Specify any extra linker options needed to make the library -# thread safe -# -#THREADLIB = -lpthread -THREADLIB = - -#### Specify any extra libraries needed to access required functions. -# -#TLIBS = -lrt # fdatasync on Solaris 8 -TLIBS = - -#### Leave SQLITE_DEBUG undefined for maximum speed. Use SQLITE_DEBUG=1 -# to check for memory leaks. Use SQLITE_DEBUG=2 to print a log of all -# malloc()s and free()s in order to track down memory leaks. -# -# SQLite uses some expensive assert() statements in the inner loop. -# You can make the library go almost twice as fast if you compile -# with -DNDEBUG=1 -# -#OPTS = -DSQLITE_DEBUG=2 -#OPTS = -DSQLITE_DEBUG=1 -#OPTS = -OPTS = -DNDEBUG=1 -DSQLITE_OS_WIN=1 -D_WIN32_WCE=1 -#OPTS += -DHAVE_FDATASYNC=1 - -#### The suffix to add to executable files. ".exe" for windows. -# Nothing for unix. -# -EXE = .exe -#EXE = - -#### C Compile and options for use in building executables that -# will run on the target platform. This is usually the same -# as BCC, unless you are cross-compiling. -# -#TCC = gcc -O6 -#TCC = gcc -g -O0 -Wall -#TCC = gcc -g -O0 -Wall -fprofile-arcs -ftest-coverage -#TCC = /opt/mingw/bin/i386-mingw32-gcc -O6 -TCC = $(MINGW32CE)/$(TARGET)-gcc -O2 -#TCC = /opt/ansic/bin/c89 -O +z -Wl,-a,archive - -#### Tools used to build a static library. -# -#AR = ar cr -#AR = /opt/mingw/bin/i386-mingw32-ar cr -AR = $(MINGW32CE)/$(TARGET)-ar cr -#RANLIB = ranlib -#RANLIB = /opt/mingw/bin/i386-mingw32-ranlib -RANLIB = $(MINGW32CE)/$(TARGET)-ranlib - -#MKSHLIB = gcc -shared -#SO = so -#SHPREFIX = lib -MKSHLIB = $(MINGW32CE)/$(TARGET)-gcc -shared -SO = dll -SHPREFIX = - -#### Extra compiler options needed for programs that use the TCL library. -# -#TCL_FLAGS = -#TCL_FLAGS = -DSTATIC_BUILD=1 -TCL_FLAGS = -I/home/drh/tcltk/8.5linux -#TCL_FLAGS = -I/home/drh/tcltk/8.5win -DSTATIC_BUILD=1 -#TCL_FLAGS = -I/home/drh/tcltk/8.3hpux - -#### Linker options needed to link against the TCL library. -# -#LIBTCL = -ltcl -lm -ldl -LIBTCL = /home/drh/tcltk/8.5linux/libtcl8.5g.a -lm -ldl -#LIBTCL = /home/drh/tcltk/8.5win/libtcl85s.a -lmsvcrt -#LIBTCL = /home/drh/tcltk/8.3hpux/libtcl8.3.a -ldld -lm -lc - -#### Additional objects for SQLite library when TCL support is enabled. -TCLOBJ = -#TCLOBJ = tclsqlite.o - -#### Compiler options needed for programs that use the readline() library. -# -READLINE_FLAGS = -#READLINE_FLAGS = -DHAVE_READLINE=1 -I/usr/include/readline - -#### Linker options needed by programs using readline() must link against. -# -LIBREADLINE = -#LIBREADLINE = -static -lreadline -ltermcap - -#### Which "awk" program provides nawk compatibilty -# -# NAWK = nawk -NAWK = awk - -# You should not have to change anything below this line -############################################################################### -include $(TOP)/main.mk Index: Makefile.in ================================================================== --- Makefile.in +++ Makefile.in @@ -63,11 +63,11 @@ # TCC += -DSQLITE_THREADSAFE=@SQLITE_THREADSAFE@ # Any target libraries which libsqlite must be linked against # -TLIBS = @LIBS@ +TLIBS = @LIBS@ $(LIBS) # Flags controlling use of the in memory btree implementation # # SQLITE_TEMP_STORE is 0 to force temporary tables to be in a file, 1 to # default to file, 2 to default to memory, and 3 to force temporary @@ -153,13 +153,10 @@ # libtool compile/link/install LTCOMPILE = $(LIBTOOL) --mode=compile --tag=CC $(TCC) $(LTCOMPILE_EXTRAS) LTLINK = $(LIBTOOL) --mode=link $(TCC) $(LTCOMPILE_EXTRAS) @LDFLAGS@ $(LTLINK_EXTRAS) LTINSTALL = $(LIBTOOL) --mode=install $(INSTALL) -# nawk compatible awk. -NAWK = @AWK@ - # You should not have to change anything below this line ############################################################################### USE_AMALGAMATION = @USE_AMALGAMATION@ @@ -173,11 +170,11 @@ fts3_porter.lo fts3_snippet.lo fts3_tokenizer.lo fts3_tokenizer1.lo \ fts3_tokenize_vtab.lo \ fts3_unicode.lo fts3_unicode2.lo fts3_write.lo \ fts5.lo \ func.lo global.lo hash.lo \ - icu.lo insert.lo journal.lo legacy.lo loadext.lo \ + icu.lo insert.lo journal.lo json1.lo legacy.lo loadext.lo \ main.lo malloc.lo mem0.lo mem1.lo mem2.lo mem3.lo mem5.lo \ memjournal.lo \ mutex.lo mutex_noop.lo mutex_unix.lo mutex_w32.lo \ notify.lo opcodes.lo os.lo os_unix.lo os_win.lo \ pager.lo parse.lo pcache.lo pcache1.lo pragma.lo prepare.lo printf.lo \ @@ -270,12 +267,12 @@ $(TOP)/src/sqlite.h.in \ $(TOP)/src/sqlite3ext.h \ $(TOP)/src/sqliteInt.h \ $(TOP)/src/sqliteLimit.h \ $(TOP)/src/table.c \ - $(TOP)/src/threads.c \ $(TOP)/src/tclsqlite.c \ + $(TOP)/src/threads.c \ $(TOP)/src/tokenize.c \ $(TOP)/src/treeview.c \ $(TOP)/src/trigger.c \ $(TOP)/src/utf.c \ $(TOP)/src/update.c \ @@ -345,10 +342,13 @@ $(TOP)/ext/rtree/rtree.h \ $(TOP)/ext/rtree/rtree.c SRC += \ $(TOP)/ext/rbu/sqlite3rbu.h \ $(TOP)/ext/rbu/sqlite3rbu.c +SRC += \ + $(TOP)/ext/misc/json1.c + # Generated source code files # SRC += \ @@ -415,11 +415,10 @@ $(TOP)/ext/misc/fileio.c \ $(TOP)/ext/misc/fuzzer.c \ $(TOP)/ext/fts5/fts5_tcl.c \ $(TOP)/ext/fts5/fts5_test_mi.c \ $(TOP)/ext/misc/ieee754.c \ - $(TOP)/ext/misc/json1.c \ $(TOP)/ext/misc/nextchar.c \ $(TOP)/ext/misc/percentile.c \ $(TOP)/ext/misc/regexp.c \ $(TOP)/ext/misc/series.c \ $(TOP)/ext/misc/spellfix.c \ @@ -539,19 +538,19 @@ $(TOP)/test/fuzzdata1.db \ $(TOP)/test/fuzzdata2.db \ $(TOP)/test/fuzzdata3.db \ $(TOP)/test/fuzzdata4.db -# Extra arguments for including json1 in the build of tools -# -JSON1_DEP = $(TOP)/ext/misc/json1.c sqlite3ext.h -JSON1_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_CORE -JSON1_SRC = $(TOP)/ext/misc/json1.c - # Standard options to testfixture # TESTOPTS = --verbose=file --output=test-out.txt + +# Extra compiler options for various shell tools +# +SHELL_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_FTS5 +FUZZERSHELL_OPT = -DSQLITE_ENABLE_JSON1 +FUZZCHECK_OPT = -DSQLITE_ENABLE_JSON1 # This is the default Makefile target. The objects listed here # are what get build when you type just "make" with no arguments. # all: sqlite3.h libsqlite3.la sqlite3$(TEXE) $(HAVE_TCL:1=libtclsqlite3.la) @@ -571,24 +570,24 @@ libsqlite3.la @TCL_STUB_LIB_SPEC@ $(TLIBS) \ -rpath "$(TCLLIBDIR)" \ -version-info "8:6:8" \ -avoid-version -sqlite3$(TEXE): $(TOP)/src/shell.c libsqlite3.la sqlite3.h $(JSON1_DEP) - $(LTLINK) $(READLINE_FLAGS) $(JSON1_OPT) -o $@ \ - $(TOP)/src/shell.c $(JSON1_SRC) libsqlite3.la \ +sqlite3$(TEXE): $(TOP)/src/shell.c libsqlite3.la sqlite3.h + $(LTLINK) $(READLINE_FLAGS) $(SHELL_OPT) -o $@ \ + $(TOP)/src/shell.c libsqlite3.la \ $(LIBREADLINE) $(TLIBS) -rpath "$(libdir)" sqldiff$(TEXE): $(TOP)/tool/sqldiff.c sqlite3.c sqlite3.h $(LTLINK) -o $@ $(TOP)/tool/sqldiff.c sqlite3.c $(TLIBS) -fuzzershell$(TEXE): $(TOP)/tool/fuzzershell.c sqlite3.c sqlite3.h $(JSON1_DEP) - $(LTLINK) -o $@ $(JSON1_OPT) \ - $(TOP)/tool/fuzzershell.c $(JSON1_SRC) sqlite3.c $(TLIBS) +fuzzershell$(TEXE): $(TOP)/tool/fuzzershell.c sqlite3.c sqlite3.h + $(LTLINK) -o $@ $(FUZZERSHELL_OPT) \ + $(TOP)/tool/fuzzershell.c sqlite3.c $(TLIBS) -fuzzcheck$(TEXE): $(TOP)/test/fuzzcheck.c sqlite3.c sqlite3.h $(JSON1_DEP) - $(LTLINK) -o $@ $(JSON1_OPT) $(TOP)/test/fuzzcheck.c $(JSON1_SRC) sqlite3.c $(TLIBS) +fuzzcheck$(TEXE): $(TOP)/test/fuzzcheck.c sqlite3.c sqlite3.h + $(LTLINK) -o $@ $(FUZZCHECK_OPT) $(TOP)/test/fuzzcheck.c sqlite3.c $(TLIBS) mptester$(TEXE): sqlite3.c $(TOP)/mptest/mptest.c $(LTLINK) -o $@ -I. $(TOP)/mptest/mptest.c sqlite3.c \ $(TLIBS) -rpath "$(libdir)" @@ -610,22 +609,26 @@ # copies of all of the C source code and header files needed to # build on the target system. Some of the C source code and header # files are automatically generated. This target takes care of # all that automatic generation. # -.target_source: $(SRC) $(TOP)/tool/vdbe-compress.tcl +.target_source: $(SRC) $(TOP)/tool/vdbe-compress.tcl fts5.c rm -rf tsrc mkdir tsrc cp -f $(SRC) tsrc rm tsrc/sqlite.h.in tsrc/parse.y $(TCLSH_CMD) $(TOP)/tool/vdbe-compress.tcl $(OPTS) vdbe.new mv vdbe.new tsrc/vdbe.c + cp fts5.c fts5.h tsrc touch .target_source sqlite3.c: .target_source $(TOP)/tool/mksqlite3c.tcl $(TCLSH_CMD) $(TOP)/tool/mksqlite3c.tcl cp tsrc/shell.c tsrc/sqlite3ext.h . + +sqlite3ext.h: .target_source + cp tsrc/sqlite3ext.h . tclsqlite3.c: sqlite3.c echo '#ifndef USE_SYSTEM_SQLITE' >tclsqlite3.c cat sqlite3.c >>tclsqlite3.c echo '#endif /* USE_SYSTEM_SQLITE */' >>tclsqlite3.c @@ -894,26 +897,26 @@ $(LTLINK) -o $@ tclsqlite-shell.lo \ libsqlite3.la $(LIBTCL) # Rules to build opcodes.c and opcodes.h # -opcodes.c: opcodes.h $(TOP)/mkopcodec.awk - $(NAWK) -f $(TOP)/mkopcodec.awk opcodes.h >opcodes.c +opcodes.c: opcodes.h $(TOP)/tool/mkopcodec.tcl + $(TCLSH_CMD) $(TOP)/tool/mkopcodec.tcl opcodes.h >opcodes.c -opcodes.h: parse.h $(TOP)/src/vdbe.c $(TOP)/mkopcodeh.awk - cat parse.h $(TOP)/src/vdbe.c | $(NAWK) -f $(TOP)/mkopcodeh.awk >opcodes.h +opcodes.h: parse.h $(TOP)/src/vdbe.c $(TOP)/tool/mkopcodeh.tcl + cat parse.h $(TOP)/src/vdbe.c | $(TCLSH_CMD) $(TOP)/tool/mkopcodeh.tcl >opcodes.h # Rules to build parse.c and parse.h - the outputs of lemon. # parse.h: parse.c -parse.c: $(TOP)/src/parse.y lemon$(BEXE) $(TOP)/addopcodes.awk +parse.c: $(TOP)/src/parse.y lemon$(BEXE) $(TOP)/tool/addopcodes.tcl cp $(TOP)/src/parse.y . rm -f parse.h ./lemon$(BEXE) $(OPT_FEATURE_FLAGS) $(OPTS) parse.y mv parse.h parse.h.temp - $(NAWK) -f $(TOP)/addopcodes.awk parse.h.temp >parse.h + $(TCLSH_CMD) $(TOP)/tool/addopcodes.tcl parse.h.temp >parse.h sqlite3.h: $(TOP)/src/sqlite.h.in $(TOP)/manifest.uuid $(TOP)/VERSION $(TCLSH_CMD) $(TOP)/tool/mksqlite3h.tcl $(TOP) >sqlite3.h keywordhash.h: $(TOP)/tool/mkkeywordhash.c @@ -985,10 +988,13 @@ $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_write.c rtree.lo: $(TOP)/ext/rtree/rtree.c $(HDR) $(EXTHDR) $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/rtree/rtree.c +json1.lo: $(TOP)/ext/misc/json1.c + $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/misc/json1.c + # FTS5 things # FTS5_SRC = \ $(TOP)/ext/fts5/fts5.h \ $(TOP)/ext/fts5/fts5Int.h \ @@ -1091,11 +1097,11 @@ echo "#define TCLSH 2" > $@ echo "#define SQLITE_ENABLE_DBSTAT_VTAB 1" >> $@ cat sqlite3.c $(TOP)/src/tclsqlite.c >> $@ echo "static const char *tclsh_main_loop(void){" >> $@ echo "static const char *zMainloop = " >> $@ - $(NAWK) -f $(TOP)/tool/tostr.awk $(TOP)/tool/spaceanal.tcl >> $@ + $(TCLSH_CMD) $(TOP)/tool/tostr.tcl $(TOP)/tool/spaceanal.tcl >> $@ echo "; return zMainloop; }" >> $@ sqlite3_analyzer$(TEXE): sqlite3_analyzer.c $(LTLINK) sqlite3_analyzer.c -o $@ $(LIBTCL) $(TLIBS) Index: Makefile.linux-gcc ================================================================== --- Makefile.linux-gcc +++ Makefile.linux-gcc @@ -116,13 +116,8 @@ #### Linker options needed by programs using readline() must link against. # LIBREADLINE = #LIBREADLINE = -static -lreadline -ltermcap -#### Which "awk" program provides nawk compatibilty -# -# NAWK = nawk -NAWK = awk - # You should not have to change anything below this line ############################################################################### include $(TOP)/main.mk Index: Makefile.msc ================================================================== --- Makefile.msc +++ Makefile.msc @@ -388,13 +388,13 @@ # These are additional compiler options used for the shell executable. # !IFNDEF SHELL_COMPILE_OPTS !IF $(DYNAMIC_SHELL)!=0 -SHELL_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1 $(SHELL_CCONV_OPTS) -DSQLITE_API=__declspec(dllimport) +SHELL_COMPILE_OPTS = -DSQLITE_SHELL_JSON1 $(SHELL_CCONV_OPTS) -DSQLITE_API=__declspec(dllimport) !ELSE -SHELL_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1 $(SHELL_CCONV_OPTS) +SHELL_COMPILE_OPTS = -DSQLITE_SHELL_JSON1 $(SHELL_CCONV_OPTS) !ENDIF !ENDIF # This is the core library that the shell executable should depend on. # @@ -566,10 +566,18 @@ !ENDIF !IFNDEF LIBTCL LIBTCL = tcl85.lib !ENDIF + +!IFNDEF LIBTCLSTUB +LIBTCLSTUB = tclstub85.lib +!ENDIF + +!IFNDEF LIBTCLPATH +LIBTCLPATH = c:\tcl\bin +!ENDIF # The locations of the ICU header and library files. These variables # (ICUINCDIR, ICULIBDIR, and LIBICU) may be overridden via the environment # prior to running nmake in order to match the actual installed location on # this machine. @@ -807,16 +815,10 @@ !IF $(USE_ICU)!=0 LTLIBPATHS = $(LTLIBPATHS) /LIBPATH:$(ICULIBDIR) LTLIBS = $(LTLIBS) $(LIBICU) !ENDIF -# nawk compatible awk. -# -!IFNDEF NAWK -NAWK = gawk.exe -!ENDIF - # You should not have to change anything below this line ############################################################################### # Object files for the SQLite library (non-amalgamation). # @@ -1006,11 +1008,12 @@ $(TOP)\ext\icu\sqliteicu.h \ $(TOP)\ext\icu\icu.c \ $(TOP)\ext\rtree\rtree.h \ $(TOP)\ext\rtree\rtree.c \ $(TOP)\ext\rbu\sqlite3rbu.h \ - $(TOP)\ext\rbu\sqlite3rbu.c + $(TOP)\ext\rbu\sqlite3rbu.c \ + $(TOP)\ext\misc\json1.c # Generated source code files # SRC5 = \ @@ -1208,15 +1211,15 @@ $(TOP)\test\fuzzdata1.db \ $(TOP)\test\fuzzdata2.db \ $(TOP)\test\fuzzdata3.db \ $(TOP)\test\fuzzdata4.db -# Extra arguments for including json1 in the build of tools +# Extra compiler options for various shell tools # -JSON1_DEP = sqlite3ext.h $(TOP)\ext\misc\json1.c -JSON1_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_CORE -JSON1_SRC = $(TOP)\ext\misc\json1.c +SHELL_COMPILE_OPTS = -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_FTS5 +FUZZERSHELL_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1 +FUZZCHECK_COMPILE_OPTS = -DSQLITE_ENABLE_JSON1 # Standard options to testfixture # TESTOPTS = --verbose=file --output=test-out.txt @@ -1227,29 +1230,29 @@ libsqlite3.lib: $(LIBOBJ) $(LTLIB) $(LTLIBOPTS) /OUT:$@ $(LIBOBJ) $(TLIBS) libtclsqlite3.lib: tclsqlite.lo libsqlite3.lib - $(LTLIB) $(LTLIBOPTS) $(LTLIBPATHS) /OUT:$@ tclsqlite.lo libsqlite3.lib $(LIBTCL:tcl=tclstub) $(TLIBS) + $(LTLIB) $(LTLIBOPTS) $(LTLIBPATHS) /OUT:$@ tclsqlite.lo libsqlite3.lib $(LIBTCLSTUB) $(TLIBS) -sqlite3.exe: $(TOP)\src\shell.c $(JSON1_DEP) $(SHELL_CORE_DEP) $(LIBRESOBJS) sqlite3.h - $(LTLINK) $(SHELL_COMPILE_OPTS) $(JSON1_OPT) $(READLINE_FLAGS) $(TOP)\src\shell.c $(JSON1_SRC) \ - /link /pdb:sqlite3sh.pdb $(LTLINKOPTS) $(SHELL_LINK_OPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LIBREADLINE) $(LTLIBS) $(TLIBS) +sqlite3.exe: $(TOP)\src\shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) sqlite3.h + $(LTLINK) $(SHELL_COMPILE_OPTS) $(SHELL_COMPILE_OPTS) $(READLINE_FLAGS) $(TOP)\src\shell.c \ + /link /pdb:sqlite3sh.pdb $(LDFLAGS) $(LTLINKOPTS) $(SHELL_LINK_OPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LIBREADLINE) $(LTLIBS) $(TLIBS) sqldiff.exe: $(TOP)\tool\sqldiff.c sqlite3.c sqlite3.h - $(LTLINK) $(NO_WARN) $(TOP)\tool\sqldiff.c sqlite3.c - -fuzzershell.exe: $(TOP)\tool\fuzzershell.c sqlite3.c sqlite3.h $(JSON1_DEP) - $(LTLINK) $(NO_WARN) $(JSON1_OPT) \ - $(TOP)\tool\fuzzershell.c $(JSON1_SRC) sqlite3.c - -fuzzcheck.exe: $(TOP)\test\fuzzcheck.c sqlite3.c sqlite3.h $(JSON1_DEP) - $(LTLINK) $(NO_WARN) $(JSON1_OPT) $(TOP)\test\fuzzcheck.c $(JSON1_SRC) sqlite3.c + $(LTLINK) $(NO_WARN) $(TOP)\tool\sqldiff.c sqlite3.c /link $(LDFLAGS) $(LTLINKOPTS) + +fuzzershell.exe: $(TOP)\tool\fuzzershell.c sqlite3.c sqlite3.h + $(LTLINK) $(NO_WARN) $(FUZZERSHELL_COMPILE_OPTS) \ + $(TOP)\tool\fuzzershell.c sqlite3.c /link $(LDFLAGS) $(LTLINKOPTS) + +fuzzcheck.exe: $(TOP)\test\fuzzcheck.c sqlite3.c sqlite3.h + $(LTLINK) $(NO_WARN) $(FUZZCHECK_COMPILE_OPTS) $(TOP)\test\fuzzcheck.c sqlite3.c /link $(LDFLAGS) $(LTLINKOPTS) mptester.exe: $(TOP)\mptest\mptest.c $(SHELL_CORE_DEP) $(LIBRESOBJS) sqlite3.h $(LTLINK) $(NO_WARN) $(SHELL_COMPILE_OPTS) $(TOP)\mptest\mptest.c \ - /link $(LTLINKOPTS) $(LTLIBPATHS) $(SHELL_LINK_OPTS) $(LIBRESOBJS) $(LIBREADLINE) $(LTLIBS) $(TLIBS) + /link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) $(SHELL_LINK_OPTS) $(LIBRESOBJS) $(LIBREADLINE) $(LTLIBS) $(TLIBS) MPTEST1 = mptester mptest.db $(TOP)\mptest\crash01.test --repeat 20 MPTEST2 = mptester mptest.db $(TOP)\mptest\multiwrite01.test --repeat 20 mptest: mptester.exe @@ -1267,18 +1270,20 @@ # copies of all of the C source code and header files needed to # build on the target system. Some of the C source code and header # files are automatically generated. This target takes care of # all that automatic generation. # -.target_source: $(SRC) $(TOP)\tool\vdbe-compress.tcl +.target_source: $(SRC) $(TOP)\tool\vdbe-compress.tcl fts5.c -rmdir /Q/S tsrc 2>NUL -mkdir tsrc for %i in ($(SRC1)) do copy /Y %i tsrc for %i in ($(SRC2)) do copy /Y %i tsrc for %i in ($(SRC3)) do copy /Y %i tsrc for %i in ($(SRC4)) do copy /Y %i tsrc for %i in ($(SRC5)) do copy /Y %i tsrc + copy /Y fts5.c tsrc + copy /Y fts5.h tsrc del /Q tsrc\sqlite.h.in tsrc\parse.y 2>NUL $(TCLSH_CMD) $(TOP)\tool\vdbe-compress.tcl $(OPTS) < tsrc\vdbe.c > vdbe.new move vdbe.new tsrc\vdbe.c echo > .target_source @@ -1308,11 +1313,11 @@ lempar.c: $(TOP)\src\lempar.c copy $(TOP)\src\lempar.c . lemon.exe: $(TOP)\tool\lemon.c lempar.c $(BCC) $(NO_WARN) -Daccess=_access \ - -Fe$@ $(TOP)\tool\lemon.c /link $(NLTLINKOPTS) $(NLTLIBPATHS) + -Fe$@ $(TOP)\tool\lemon.c /link $(LDFLAGS) $(NLTLINKOPTS) $(NLTLIBPATHS) # Rules to build individual *.lo files from generated *.c files. This # applies to: # # parse.lo @@ -1329,11 +1334,11 @@ !IF $(USE_RC)!=0 $(LIBRESOBJS): $(TOP)\src\sqlite3.rc $(HDR) echo #ifndef SQLITE_RESOURCE_VERSION > sqlite3rc.h for /F %%V in ('type "$(TOP)\VERSION"') do ( \ echo #define SQLITE_RESOURCE_VERSION %%V \ - | $(NAWK) "/.*/ { gsub(/[.]/,\",\");print }" >> sqlite3rc.h \ + | $(TCLSH_CMD) $(TOP)\tool\replace.tcl exact . ^, >> sqlite3rc.h \ ) echo #endif >> sqlite3rc.h $(LTRCOMPILE) -fo $(LIBRESOBJS) $(TOP)\src\sqlite3.rc !ENDIF @@ -1566,40 +1571,40 @@ tclsqlite-shell.lo: $(TOP)\src\tclsqlite.c $(HDR) $(LTCOMPILE) $(NO_WARN) -DTCLSH=1 -DBUILD_sqlite -I$(TCLINCDIR) -c $(TOP)\src\tclsqlite.c tclsqlite3.exe: tclsqlite-shell.lo $(SQLITE3C) $(LIBRESOBJS) - $(LTLINK) $(SQLITE3C) /link $(LTLINKOPTS) $(LTLIBPATHS) /OUT:$@ tclsqlite-shell.lo $(LIBRESOBJS) $(LTLIBS) $(TLIBS) + $(LTLINK) $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /OUT:$@ tclsqlite-shell.lo $(LIBRESOBJS) $(LTLIBS) $(TLIBS) # Rules to build opcodes.c and opcodes.h # -opcodes.c: opcodes.h $(TOP)\mkopcodec.awk - $(NAWK) -f $(TOP)\mkopcodec.awk opcodes.h > opcodes.c +opcodes.c: opcodes.h $(TOP)\tool\mkopcodec.tcl + $(TCLSH_CMD) $(TOP)\tool\mkopcodec.tcl opcodes.h > opcodes.c -opcodes.h: parse.h $(TOP)\src\vdbe.c $(TOP)\mkopcodeh.awk - type parse.h $(TOP)\src\vdbe.c | $(NAWK) -f $(TOP)\mkopcodeh.awk > opcodes.h +opcodes.h: parse.h $(TOP)\src\vdbe.c $(TOP)\tool\mkopcodeh.tcl + type parse.h $(TOP)\src\vdbe.c | $(TCLSH_CMD) $(TOP)\tool\mkopcodeh.tcl > opcodes.h # Rules to build parse.c and parse.h - the outputs of lemon. # parse.h: parse.c -parse.c: $(TOP)\src\parse.y lemon.exe $(TOP)\addopcodes.awk +parse.c: $(TOP)\src\parse.y lemon.exe $(TOP)\tool\addopcodes.tcl del /Q parse.y parse.h parse.h.temp 2>NUL copy $(TOP)\src\parse.y . .\lemon.exe $(REQ_FEATURE_FLAGS) $(OPT_FEATURE_FLAGS) $(OPTS) parse.y move parse.h parse.h.temp - $(NAWK) -f $(TOP)\addopcodes.awk parse.h.temp > parse.h + $(TCLSH_CMD) $(TOP)\tool\addopcodes.tcl parse.h.temp > parse.h sqlite3.h: $(TOP)\src\sqlite.h.in $(TOP)\manifest.uuid $(TOP)\VERSION $(TCLSH_CMD) $(TOP)\tool\mksqlite3h.tcl $(TOP:\=/) > sqlite3.h sqlite3ext.h: .target_source copy tsrc\sqlite3ext.h . mkkeywordhash.exe: $(TOP)\tool\mkkeywordhash.c $(BCC) $(NO_WARN) -Fe$@ $(REQ_FEATURE_FLAGS) $(OPT_FEATURE_FLAGS) $(OPTS) \ - $(TOP)\tool\mkkeywordhash.c /link $(NLTLINKOPTS) $(NLTLIBPATHS) + $(TOP)\tool\mkkeywordhash.c /link $(LDFLAGS) $(NLTLINKOPTS) $(NLTLIBPATHS) keywordhash.h: $(TOP)\tool\mkkeywordhash.c mkkeywordhash.exe .\mkkeywordhash.exe > keywordhash.h @@ -1729,25 +1734,30 @@ testfixture.exe: $(TESTFIXTURE_SRC) $(LIBRESOBJS) $(HDR) $(LTLINK) -DSQLITE_NO_SYNC=1 $(TESTFIXTURE_FLAGS) \ -DBUILD_sqlite -I$(TCLINCDIR) \ $(TESTFIXTURE_SRC) \ - /link $(LTLINKOPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LTLIBS) $(TLIBS) + /link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LTLIBS) $(TLIBS) extensiontest: testfixture.exe testloadext.dll + @set PATH=$(LIBTCLPATH);$(PATH) .\testfixture.exe $(TOP)\test\loadext.test $(TESTOPTS) fulltest: $(TESTPROGS) fuzztest + @set PATH=$(LIBTCLPATH);$(PATH) .\testfixture.exe $(TOP)\test\all.test $(TESTOPTS) soaktest: $(TESTPROGS) + @set PATH=$(LIBTCLPATH);$(PATH) .\testfixture.exe $(TOP)\test\all.test -soak=1 $(TESTOPTS) fulltestonly: $(TESTPROGS) fuzztest + @set PATH=$(LIBTCLPATH);$(PATH) .\testfixture.exe $(TOP)\test\full.test queryplantest: testfixture.exe sqlite3.exe + @set PATH=$(LIBTCLPATH);$(PATH) .\testfixture.exe $(TOP)\test\permutations.test queryplanner $(TESTOPTS) fuzztest: fuzzcheck.exe .\fuzzcheck.exe $(FUZZDATA) @@ -1755,74 +1765,77 @@ .\fuzzcheck.exe --limit-mem 100M $(FUZZDATA) # Minimal testing that runs in less than 3 minutes (on a fast machine) # quicktest: testfixture.exe + @set PATH=$(LIBTCLPATH);$(PATH) .\testfixture.exe $(TOP)\test\extraquick.test $(TESTOPTS) # This is the common case. Run many tests that do not take too long, # including fuzzcheck, sqlite3_analyzer, and sqldiff tests. # test: $(TESTPROGS) fastfuzztest + @set PATH=$(LIBTCLPATH);$(PATH) .\testfixture.exe $(TOP)\test\veryquick.test $(TESTOPTS) smoketest: $(TESTPROGS) + @set PATH=$(LIBTCLPATH);$(PATH) .\testfixture.exe $(TOP)\test\main.test $(TESTOPTS) sqlite3_analyzer.c: $(SQLITE3C) $(TOP)\src\tclsqlite.c $(TOP)\tool\spaceanal.tcl echo #define TCLSH 2 > $@ echo #define SQLITE_ENABLE_DBSTAT_VTAB 1 >> $@ copy $@ + $(SQLITE3C) + $(TOP)\src\tclsqlite.c $@ echo static const char *tclsh_main_loop(void){ >> $@ echo static const char *zMainloop = >> $@ - $(NAWK) -f $(TOP)\tool\tostr.awk $(TOP)\tool\spaceanal.tcl >> $@ + $(TCLSH_CMD) $(TOP)\tool\tostr.tcl $(TOP)\tool\spaceanal.tcl >> $@ echo ; return zMainloop; } >> $@ sqlite3_analyzer.exe: sqlite3_analyzer.c $(LIBRESOBJS) $(LTLINK) $(NO_WARN) -DBUILD_sqlite -I$(TCLINCDIR) sqlite3_analyzer.c \ - /link $(LTLINKOPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LTLIBS) $(TLIBS) + /link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LTLIBS) $(TLIBS) testloadext.lo: $(TOP)\src\test_loadext.c $(LTCOMPILE) $(NO_WARN) -c $(TOP)\src\test_loadext.c testloadext.dll: testloadext.lo $(LD) $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /DLL /OUT:$@ testloadext.lo showdb.exe: $(TOP)\tool\showdb.c $(SQLITE3C) $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ - $(TOP)\tool\showdb.c $(SQLITE3C) + $(TOP)\tool\showdb.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) showstat4.exe: $(TOP)\tool\showstat4.c $(SQLITE3C) $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ - $(TOP)\tool\showstat4.c $(SQLITE3C) + $(TOP)\tool\showstat4.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) showjournal.exe: $(TOP)\tool\showjournal.c $(SQLITE3C) $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ - $(TOP)\tool\showjournal.c $(SQLITE3C) + $(TOP)\tool\showjournal.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) showwal.exe: $(TOP)\tool\showwal.c $(SQLITE3C) $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ - $(TOP)\tool\showwal.c $(SQLITE3C) + $(TOP)\tool\showwal.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) fts3view.exe: $(TOP)\ext\fts3\tool\fts3view.c $(SQLITE3C) $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ - $(TOP)\ext\fts3\tool\fts3view.c $(SQLITE3C) + $(TOP)\ext\fts3\tool\fts3view.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) rollback-test.exe: $(TOP)\tool\rollback-test.c $(SQLITE3C) $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ - $(TOP)\tool\rollback-test.c $(SQLITE3C) + $(TOP)\tool\rollback-test.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) LogEst.exe: $(TOP)\tool\logest.c sqlite3.h - $(LTLINK) $(NO_WARN) -Fe$@ $(TOP)\tool\LogEst.c + $(LTLINK) $(NO_WARN) -Fe$@ $(TOP)\tool\LogEst.c /link $(LDFLAGS) $(LTLINKOPTS) wordcount.exe: $(TOP)\test\wordcount.c $(SQLITE3C) $(LTLINK) $(NO_WARN) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ - $(TOP)\test\wordcount.c $(SQLITE3C) + $(TOP)\test\wordcount.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) speedtest1.exe: $(TOP)\test\speedtest1.c $(SQLITE3C) $(LTLINK) $(NO_WARN) -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ - $(TOP)\test\speedtest1.c $(SQLITE3C) + $(TOP)\test\speedtest1.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) clean: del /Q *.exp *.lo *.ilk *.lib *.obj *.ncb *.pdb *.sdf *.suo 2>NUL del /Q *.bsc *.cod *.da *.bb *.bbg gmon.out 2>NUL del /Q sqlite3.h opcodes.c opcodes.h 2>NUL @@ -1856,10 +1869,10 @@ dll: sqlite3.dll sqlite3.def: libsqlite3.lib echo EXPORTS > sqlite3.def dumpbin /all libsqlite3.lib \ - | $(NAWK) "/ 1 _?sqlite3_/ { sub(/^.* _?/,\"\");print }" \ + | $(TCLSH_CMD) $(TOP)\tool\replace.tcl include "^\s+1 _?(sqlite3_.*)$$" \1 \ | sort >> sqlite3.def sqlite3.dll: $(LIBOBJ) $(LIBRESOBJS) $(CORE_LINK_DEP) $(LD) $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /DLL $(CORE_LINK_OPTS) /OUT:$@ $(LIBOBJ) $(LIBRESOBJS) $(LTLIBS) $(TLIBS) DELETED Makefile.vxworks Index: Makefile.vxworks ================================================================== --- Makefile.vxworks +++ /dev/null @@ -1,673 +0,0 @@ -#!/usr/make -# -# Makefile for SQLITE on VxWorks - -ifeq ($(FORCPU),) - FORCPU = SH32gnule -endif - -TOOL_FAMILY = gnu - -include $(WIND_USR)/tool/gnu/make.$(FORCPU) - -#### The toplevel directory of the source tree. This is the directory -# that contains this "Makefile.in" and the "configure.in" script. -# -TOP = . - -#### C Compiler and options for use in building executables that -# will run on the platform that is doing the build. -# -BCC = gcc -g -O2 -#BCC = /opt/ancic/bin/c89 -0 - -#### If the target operating system supports the "usleep()" system -# call, then define the HAVE_USLEEP macro for all C modules. -# -USLEEP = -#USLEEP = -DHAVE_USLEEP=1 - -#### If you want the SQLite library to be safe for use within a -# multi-threaded program, then define the following macro -# appropriately: -# -THREADSAFE = -DSQLITE_THREADSAFE=1 -#THREADSAFE = -DSQLITE_THREADSAFE=0 - -#### Specify any extra linker options needed to make the library -# thread safe -# -#THREADLIB = -lpthread -THREADLIB = - -#### Specify any extra libraries needed to access required functions. -# -ifeq ($(CPU),SH32) - # for SH4 shared library - TLIBS_SHARED += -L$(WIND_USR)/lib/sh/SH32/commonle/PIC -else - # for all other CPUs shared library - TLIBS_SHARED += $(LD_LINK_PATH_ATEND) $(LD_PARTIAL_LAST_FLAGS) -endif -# for static library -TLIBS += $(LD_LINK_PATH_ATEND) $(LD_PARTIAL_LAST_FLAGS) - -#### Leave SQLITE_DEBUG undefined for maximum speed. Use SQLITE_DEBUG=1 -# to check for memory leaks. Use SQLITE_DEBUG=2 to print a log of all -# malloc()s and free()s in order to track down memory leaks. -# -# SQLite uses some expensive assert() statements in the inner loop. -# You can make the library go almost twice as fast if you compile -# with -DNDEBUG=1 -# -#OPTS = -DSQLITE_DEBUG=2 -#OPTS = -DSQLITE_DEBUG=1 -#OPTS = -OPTS = -DNDEBUG=1 -DSQLITE_OS_UNIX=1 $(THREADSAFE) -OPTS += -DSQLITE_OMIT_LOAD_EXTENSION=1 -OPTS += -DSQLITE_ENABLE_LOCKING_STYLE=1 -OPTS += -DSQLITE_THREAD_OVERRIDE_LOCK=0 -OPTS += -DSQLITE_ENABLE_COLUMN_METADATA=1 -OPTS += -DHAVE_FDATASYNC=1 - -#### The suffix to add to executable files. ".exe" for windows. -# Nothing for unix. -# -EXE = .vxe -#EXE = - -#### C Compile and options for use in building executables that -# will run on the target platform. This is usually the same -# as BCC, unless you are cross-compiling. -# -#TCC = gcc -O6 -#TCC = gcc -g -O0 -Wall -#TCC = gcc -g -O0 -Wall -fprofile-arcs -ftest-coverage -#TCC = /opt/mingw/bin/i386-mingw32-gcc -O6 -TCC = $(CC) $(DEFINE_CC) -O2 -g -mrtp $(CC_ARCH_SPEC) -D_REENTRANT=1 -D_VX_CPU=_VX_$(CPU) -D_VX_TOOL_FAMILY=$(TOOL_FAMILY) -D_VX_TOOL=$(TOOL) -TCC += -I$(WIND_USR)/h -I$(WIND_USR)/h/wrn/coreip -#TCC = /opt/ansic/bin/c89 -O +z -Wl,-a,archive - -#TCC_SHARED = $(TCC) -fPIC -TCC_SHARED = $(TCC) - -#### Tools used to build a static library. -# -#ARX = ar cr -#ARX = /opt/mingw/bin/i386-mingw32-ar cr -AR += cr -#RANLIB = ranlib -#RANLIB = /opt/mingw/bin/i386-mingw32-ranlib - -#MKSHLIB = gcc -shared -#SO = so -#SHPREFIX = lib -MKSHLIB = $(CC) $(DEFINE_CC) -mrtp -shared $(CC_ARCH_SPEC) -D_VX_CPU=_VX_$(CPU) -D_VX_TOOL_FAMILY=$(TOOL_FAMILY) -D_VX_TOOL=$(TOOL) -SO = so -SHPREFIX = lib - -#### Extra compiler options needed for programs that use the TCL library. -# -#TCL_FLAGS = -#TCL_FLAGS = -DSTATIC_BUILD=1 -TCL_FLAGS = -I/home/drh/tcltk/8.5linux -#TCL_FLAGS = -I/home/drh/tcltk/8.5win -DSTATIC_BUILD=1 -#TCL_FLAGS = -I/home/drh/tcltk/8.3hpux - -#### Linker options needed to link against the TCL library. -# -#LIBTCL = -ltcl -lm -ldl -LIBTCL = /home/drh/tcltk/8.5linux/libtcl8.5g.a -lm -ldl -#LIBTCL = /home/drh/tcltk/8.5win/libtcl85s.a -lmsvcrt -#LIBTCL = /home/drh/tcltk/8.3hpux/libtcl8.3.a -ldld -lm -lc - -#### Additional objects for SQLite library when TCL support is enabled. -TCLOBJ = -#TCLOBJ = tclsqlite.o - -#### Compiler options needed for programs that use the readline() library. -# -READLINE_FLAGS = -#READLINE_FLAGS = -DHAVE_READLINE=1 -I/usr/include/readline - -#### Linker options needed by programs using readline() must link against. -# -LIBREADLINE = -#LIBREADLINE = -static -lreadline -ltermcap - -#### Which "awk" program provides nawk compatibilty -# -# NAWK = nawk -NAWK = awk - - -#### Pasted and adapted main.mk file -############################################################################### -# The following macros should be defined before this script is -# invoked: -# -# TOP The toplevel directory of the source tree. This is the -# directory that contains this "Makefile.in" and the -# "configure.in" script. -# -# BCC C Compiler and options for use in building executables that -# will run on the platform that is doing the build. -# -# THREADLIB Specify any extra linker options needed to make the library -# thread safe -# -# OPTS Extra compiler command-line options. -# -# EXE The suffix to add to executable files. ".exe" for windows -# and "" for Unix. -# -# TCC C Compiler and options for use in building executables that -# will run on the target platform. This is usually the same -# as BCC, unless you are cross-compiling. -# -# AR Tools used to build a static library. -# RANLIB -# -# TCL_FLAGS Extra compiler options needed for programs that use the -# TCL library. -# -# LIBTCL Linker options needed to link against the TCL library. -# -# READLINE_FLAGS Compiler options needed for programs that use the -# readline() library. -# -# LIBREADLINE Linker options needed by programs using readline() must -# link against. -# -# NAWK Nawk compatible awk program. Older (obsolete?) solaris -# systems need this to avoid using the original AT&T AWK. -# -# Once the macros above are defined, the rest of this make script will -# build the SQLite library and testing tools. -################################################################################ - -# This is how we compile -# -TCCX = $(TCC) $(OPTS) -I. -I$(TOP)/src -I$(TOP) -TCCX_SHARED = $(TCC_SHARED) $(OPTS) -I. -I$(TOP)/src -I$(TOP) \ - -I$(TOP)/ext/rtree -I$(TOP)/ext/icu -I$(TOP)/ext/fts3 \ - -I$(TOP)/ext/async - -# Object files for the SQLite library. -# -LIBOBJ+= alter.o analyze.o attach.o auth.o \ - backup.o bitvec.o btmutex.o btree.o build.o \ - callback.o complete.o date.o delete.o expr.o fault.o \ - fts3.o fts3_expr.o fts3_hash.o fts3_icu.o fts3_porter.o \ - fts3_tokenizer.o fts3_tokenizer1.o \ - func.o global.o hash.o \ - icu.o insert.o journal.o legacy.o loadext.o \ - main.o malloc.o mem0.o mem1.o mem2.o mem3.o mem5.o \ - memjournal.o \ - mutex.o mutex_noop.o mutex_unix.o mutex_w32.o \ - notify.o opcodes.o os.o os_unix.o os_win.o \ - pager.o parse.o pcache.o pcache1.o pragma.o prepare.o printf.o \ - random.o resolve.o rowset.o rtree.o select.o status.o \ - table.o tokenize.o trigger.o \ - update.o util.o vacuum.o \ - vdbe.o vdbeapi.o vdbeaux.o vdbeblob.o vdbemem.o \ - walker.o where.o utf.o vtab.o - - - -# All of the source code files. -# -SRC = \ - $(TOP)/src/alter.c \ - $(TOP)/src/analyze.c \ - $(TOP)/src/attach.c \ - $(TOP)/src/auth.c \ - $(TOP)/src/backup.c \ - $(TOP)/src/bitvec.c \ - $(TOP)/src/btmutex.c \ - $(TOP)/src/btree.c \ - $(TOP)/src/btree.h \ - $(TOP)/src/btreeInt.h \ - $(TOP)/src/build.c \ - $(TOP)/src/callback.c \ - $(TOP)/src/complete.c \ - $(TOP)/src/ctime.c \ - $(TOP)/src/date.c \ - $(TOP)/src/delete.c \ - $(TOP)/src/expr.c \ - $(TOP)/src/fault.c \ - $(TOP)/src/func.c \ - $(TOP)/src/global.c \ - $(TOP)/src/hash.c \ - $(TOP)/src/hash.h \ - $(TOP)/src/hwtime.h \ - $(TOP)/src/insert.c \ - $(TOP)/src/journal.c \ - $(TOP)/src/legacy.c \ - $(TOP)/src/loadext.c \ - $(TOP)/src/main.c \ - $(TOP)/src/malloc.c \ - $(TOP)/src/mem0.c \ - $(TOP)/src/mem1.c \ - $(TOP)/src/mem2.c \ - $(TOP)/src/mem3.c \ - $(TOP)/src/mem5.c \ - $(TOP)/src/memjournal.c \ - $(TOP)/src/msvc.h \ - $(TOP)/src/mutex.c \ - $(TOP)/src/mutex.h \ - $(TOP)/src/mutex_noop.c \ - $(TOP)/src/mutex_unix.c \ - $(TOP)/src/mutex_w32.c \ - $(TOP)/src/notify.c \ - $(TOP)/src/os.c \ - $(TOP)/src/os.h \ - $(TOP)/src/os_common.h \ - $(TOP)/src/os_setup.h \ - $(TOP)/src/os_unix.c \ - $(TOP)/src/os_win.c \ - $(TOP)/src/os_win.h \ - $(TOP)/src/pager.c \ - $(TOP)/src/pager.h \ - $(TOP)/src/parse.y \ - $(TOP)/src/pcache.c \ - $(TOP)/src/pcache.h \ - $(TOP)/src/pcache1.c \ - $(TOP)/src/pragma.c \ - $(TOP)/src/prepare.c \ - $(TOP)/src/printf.c \ - $(TOP)/src/random.c \ - $(TOP)/src/resolve.c \ - $(TOP)/src/rowset.c \ - $(TOP)/src/select.c \ - $(TOP)/src/status.c \ - $(TOP)/src/shell.c \ - $(TOP)/src/sqlite.h.in \ - $(TOP)/src/sqlite3ext.h \ - $(TOP)/src/sqliteInt.h \ - $(TOP)/src/sqliteLimit.h \ - $(TOP)/src/table.c \ - $(TOP)/src/tclsqlite.c \ - $(TOP)/src/tokenize.c \ - $(TOP)/src/trigger.c \ - $(TOP)/src/utf.c \ - $(TOP)/src/update.c \ - $(TOP)/src/util.c \ - $(TOP)/src/vacuum.c \ - $(TOP)/src/vdbe.c \ - $(TOP)/src/vdbe.h \ - $(TOP)/src/vdbeapi.c \ - $(TOP)/src/vdbeaux.c \ - $(TOP)/src/vdbeblob.c \ - $(TOP)/src/vdbemem.c \ - $(TOP)/src/vdbeInt.h \ - $(TOP)/src/vtab.c \ - $(TOP)/src/walker.c \ - $(TOP)/src/where.c - -# Source code for extensions -# -SRC += \ - $(TOP)/ext/fts1/fts1.c \ - $(TOP)/ext/fts1/fts1.h \ - $(TOP)/ext/fts1/fts1_hash.c \ - $(TOP)/ext/fts1/fts1_hash.h \ - $(TOP)/ext/fts1/fts1_porter.c \ - $(TOP)/ext/fts1/fts1_tokenizer.h \ - $(TOP)/ext/fts1/fts1_tokenizer1.c -SRC += \ - $(TOP)/ext/fts2/fts2.c \ - $(TOP)/ext/fts2/fts2.h \ - $(TOP)/ext/fts2/fts2_hash.c \ - $(TOP)/ext/fts2/fts2_hash.h \ - $(TOP)/ext/fts2/fts2_icu.c \ - $(TOP)/ext/fts2/fts2_porter.c \ - $(TOP)/ext/fts2/fts2_tokenizer.h \ - $(TOP)/ext/fts2/fts2_tokenizer.c \ - $(TOP)/ext/fts2/fts2_tokenizer1.c -SRC += \ - $(TOP)/ext/fts3/fts3.c \ - $(TOP)/ext/fts3/fts3.h \ - $(TOP)/ext/fts3/fts3_expr.c \ - $(TOP)/ext/fts3/fts3_expr.h \ - $(TOP)/ext/fts3/fts3_hash.c \ - $(TOP)/ext/fts3/fts3_hash.h \ - $(TOP)/ext/fts3/fts3_icu.c \ - $(TOP)/ext/fts3/fts3_porter.c \ - $(TOP)/ext/fts3/fts3_tokenizer.h \ - $(TOP)/ext/fts3/fts3_tokenizer.c \ - $(TOP)/ext/fts3/fts3_tokenizer1.c -SRC += \ - $(TOP)/ext/icu/sqliteicu.h \ - $(TOP)/ext/icu/icu.c -SRC += \ - $(TOP)/ext/rtree/rtree.h \ - $(TOP)/ext/rtree/rtree.c - - -# Generated source code files -# -SRC += \ - keywordhash.h \ - opcodes.c \ - opcodes.h \ - parse.c \ - parse.h \ - sqlite3.h - - -# Source code to the test files. -# -TESTSRC = \ - $(TOP)/src/test1.c \ - $(TOP)/src/test2.c \ - $(TOP)/src/test3.c \ - $(TOP)/src/test4.c \ - $(TOP)/src/test5.c \ - $(TOP)/src/test6.c \ - $(TOP)/src/test7.c \ - $(TOP)/src/test8.c \ - $(TOP)/src/test9.c \ - $(TOP)/src/test_autoext.c \ - $(TOP)/src/test_async.c \ - $(TOP)/src/test_backup.c \ - $(TOP)/src/test_btree.c \ - $(TOP)/src/test_config.c \ - $(TOP)/src/test_devsym.c \ - $(TOP)/src/test_func.c \ - $(TOP)/src/test_hexio.c \ - $(TOP)/src/test_journal.c \ - $(TOP)/src/test_malloc.c \ - $(TOP)/src/test_md5.c \ - $(TOP)/src/test_mutex.c \ - $(TOP)/src/test_onefile.c \ - $(TOP)/src/test_osinst.c \ - $(TOP)/src/test_pcache.c \ - $(TOP)/src/test_schema.c \ - $(TOP)/src/test_server.c \ - $(TOP)/src/test_tclvar.c \ - $(TOP)/src/test_thread.c \ - $(TOP)/src/test_vfs.c \ - $(TOP)/src/test_wsd.c \ - -#TESTSRC += $(TOP)/ext/fts2/fts2_tokenizer.c -#TESTSRC += $(TOP)/ext/fts3/fts3_tokenizer.c - -TESTSRC2 = \ - $(TOP)/src/attach.c $(TOP)/src/backup.c $(TOP)/src/btree.c \ - $(TOP)/src/build.c $(TOP)/src/ctime.c $(TOP)/src/date.c \ - $(TOP)/src/expr.c $(TOP)/src/func.c $(TOP)/src/insert.c $(TOP)/src/os.c \ - $(TOP)/src/os_unix.c $(TOP)/src/os_win.c \ - $(TOP)/src/pager.c $(TOP)/src/pragma.c $(TOP)/src/prepare.c \ - $(TOP)/src/printf.c $(TOP)/src/random.c $(TOP)/src/pcache.c \ - $(TOP)/src/pcache1.c $(TOP)/src/select.c $(TOP)/src/tokenize.c \ - $(TOP)/src/utf.c $(TOP)/src/util.c $(TOP)/src/vdbeapi.c $(TOP)/src/vdbeaux.c \ - $(TOP)/src/vdbe.c $(TOP)/src/vdbemem.c $(TOP)/src/where.c parse.c \ - $(TOP)/ext/fts3/fts3.c $(TOP)/ext/fts3/fts3_expr.c \ - $(TOP)/ext/fts3/fts3_tokenizer.c \ - $(TOP)/ext/async/sqlite3async.c - -# Header files used by all library source files. -# -HDR = \ - $(TOP)/src/btree.h \ - $(TOP)/src/btreeInt.h \ - $(TOP)/src/hash.h \ - $(TOP)/src/hwtime.h \ - keywordhash.h \ - $(TOP)/src/msvc.h \ - $(TOP)/src/mutex.h \ - opcodes.h \ - $(TOP)/src/os.h \ - $(TOP)/src/os_common.h \ - $(TOP)/src/os_setup.h \ - $(TOP)/src/os_win.h \ - $(TOP)/src/pager.h \ - $(TOP)/src/pcache.h \ - parse.h \ - sqlite3.h \ - $(TOP)/src/sqlite3ext.h \ - $(TOP)/src/sqliteInt.h \ - $(TOP)/src/sqliteLimit.h \ - $(TOP)/src/vdbe.h \ - $(TOP)/src/vdbeInt.h - -# Header files used by extensions -# -EXTHDR += \ - $(TOP)/ext/fts1/fts1.h \ - $(TOP)/ext/fts1/fts1_hash.h \ - $(TOP)/ext/fts1/fts1_tokenizer.h -EXTHDR += \ - $(TOP)/ext/fts2/fts2.h \ - $(TOP)/ext/fts2/fts2_hash.h \ - $(TOP)/ext/fts2/fts2_tokenizer.h -EXTHDR += \ - $(TOP)/ext/fts3/fts3.h \ - $(TOP)/ext/fts3/fts3_expr.h \ - $(TOP)/ext/fts3/fts3_hash.h \ - $(TOP)/ext/fts3/fts3_tokenizer.h -EXTHDR += \ - $(TOP)/ext/rtree/rtree.h -EXTHDR += \ - $(TOP)/ext/icu/sqliteicu.h - -# This is the default Makefile target. The objects listed here -# are what get build when you type just "make" with no arguments. -# -all: sqlite3.h libsqlite3.a sqlite3$(EXE) - -libsqlite3.a: $(LIBOBJ) - $(AR) libsqlite3.a $(LIBOBJ) - $(RANLIB) libsqlite3.a - -$(SHPREFIX)sqlite3.$(SO): $(LIBOBJ) - $(MKSHLIB) -o $(SHPREFIX)sqlite3.$(SO) $(LIBOBJ) $(TLIBS_SHARED) - -sqlite3$(EXE): $(TOP)/src/shell.c libsqlite3.a sqlite3.h - $(TCCX) $(READLINE_FLAGS) -o sqlite3$(EXE) \ - $(TOP)/src/shell.c \ - $(LIBREADLINE) $(TLIBS) $(THREADLIB) -L. -lsqlite3 - -# This target creates a directory named "tsrc" and fills it with -# copies of all of the C source code and header files needed to -# build on the target system. Some of the C source code and header -# files are automatically generated. This target takes care of -# all that automatic generation. -# -target_source: $(SRC) - rm -rf tsrc - mkdir tsrc - cp -f $(SRC) tsrc - rm tsrc/sqlite.h.in tsrc/parse.y - touch target_source - -sqlite3.c: target_source $(TOP)/tool/mksqlite3c.tcl - tclsh $(TOP)/tool/mksqlite3c.tcl - cp sqlite3.c tclsqlite3.c - cat $(TOP)/src/tclsqlite.c >>tclsqlite3.c - -fts2amal.c: target_source $(TOP)/ext/fts2/mkfts2amal.tcl - tclsh $(TOP)/ext/fts2/mkfts2amal.tcl - -fts3amal.c: target_source $(TOP)/ext/fts3/mkfts3amal.tcl - tclsh $(TOP)/ext/fts3/mkfts3amal.tcl - -# Rules to build the LEMON compiler generator -# -lemon: $(TOP)/tool/lemon.c $(TOP)/src/lempar.c - $(BCC) -o lemon $(TOP)/tool/lemon.c - cp $(TOP)/src/lempar.c . - -# Rules to build individual *.o files from generated *.c files. This -# applies to: -# -# parse.o -# opcodes.o -# -%.o: %.c $(HDR) - $(TCCX_SHARED) -c $< - -# Rules to build individual *.o files from files in the src directory. -# -%.o: $(TOP)/src/%.c $(HDR) - $(TCCX_SHARED) -c $< - -tclsqlite.o: $(TOP)/src/tclsqlite.c $(HDR) - $(TCCX_SHARED) $(TCL_FLAGS) -c $(TOP)/src/tclsqlite.c - - - -# Rules to build opcodes.c and opcodes.h -# -opcodes.c: opcodes.h $(TOP)/mkopcodec.awk - $(NAWK) -f $(TOP)/mkopcodec.awk opcodes.h >opcodes.c - -opcodes.h: parse.h $(TOP)/src/vdbe.c $(TOP)/mkopcodeh.awk - cat parse.h $(TOP)/src/vdbe.c | \ - $(NAWK) -f $(TOP)/mkopcodeh.awk >opcodes.h - -# Rules to build parse.c and parse.h - the outputs of lemon. -# -parse.h: parse.c - -parse.c: $(TOP)/src/parse.y lemon $(TOP)/addopcodes.awk - cp $(TOP)/src/parse.y . - rm -f parse.h - ./lemon $(OPTS) parse.y - mv parse.h parse.h.temp - awk -f $(TOP)/addopcodes.awk parse.h.temp >parse.h - -sqlite3.h: $(TOP)/src/sqlite.h.in - sed -e s/--VERS--/`cat ${TOP}/VERSION`/ \ - -e s/--VERSION-NUMBER--/`cat ${TOP}/VERSION | sed 's/[^0-9]/ /g' | $(NAWK) '{printf "%d%03d%03d",$$1,$$2,$$3}'`/ \ - $(TOP)/src/sqlite.h.in >sqlite3.h - -keywordhash.h: $(TOP)/tool/mkkeywordhash.c - $(BCC) -o mkkeywordhash $(OPTS) $(TOP)/tool/mkkeywordhash.c - ./mkkeywordhash >keywordhash.h - - - -# Rules to build the extension objects. -# -icu.o: $(TOP)/ext/icu/icu.c $(HDR) $(EXTHDR) - $(TCCX_SHARED) -DSQLITE_CORE -c $(TOP)/ext/icu/icu.c - -fts2.o: $(TOP)/ext/fts2/fts2.c $(HDR) $(EXTHDR) - $(TCCX_SHARED) -DSQLITE_CORE -c $(TOP)/ext/fts2/fts2.c - -fts2_hash.o: $(TOP)/ext/fts2/fts2_hash.c $(HDR) $(EXTHDR) - $(TCCX_SHARED) -DSQLITE_CORE -c $(TOP)/ext/fts2/fts2_hash.c - -fts2_icu.o: $(TOP)/ext/fts2/fts2_icu.c $(HDR) $(EXTHDR) - $(TCCX_SHARED) -DSQLITE_CORE -c $(TOP)/ext/fts2/fts2_icu.c - -fts2_porter.o: $(TOP)/ext/fts2/fts2_porter.c $(HDR) $(EXTHDR) - $(TCCX_SHARED) -DSQLITE_CORE -c $(TOP)/ext/fts2/fts2_porter.c - -fts2_tokenizer.o: $(TOP)/ext/fts2/fts2_tokenizer.c $(HDR) $(EXTHDR) - $(TCCX_SHARED) -DSQLITE_CORE -c $(TOP)/ext/fts2/fts2_tokenizer.c - -fts2_tokenizer1.o: $(TOP)/ext/fts2/fts2_tokenizer1.c $(HDR) $(EXTHDR) - $(TCCX_SHARED) -DSQLITE_CORE -c $(TOP)/ext/fts2/fts2_tokenizer1.c - -fts3.o: $(TOP)/ext/fts3/fts3.c $(HDR) $(EXTHDR) - $(TCCX_SHARED) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3.c - -fts3_expr.o: $(TOP)/ext/fts3/fts3_expr.c $(HDR) $(EXTHDR) - $(TCCX_SHARED) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_expr.c - -fts3_hash.o: $(TOP)/ext/fts3/fts3_hash.c $(HDR) $(EXTHDR) - $(TCCX_SHARED) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_hash.c - -fts3_icu.o: $(TOP)/ext/fts3/fts3_icu.c $(HDR) $(EXTHDR) - $(TCCX_SHARED) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_icu.c - -fts3_porter.o: $(TOP)/ext/fts3/fts3_porter.c $(HDR) $(EXTHDR) - $(TCCX_SHARED) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_porter.c - -fts3_tokenizer.o: $(TOP)/ext/fts3/fts3_tokenizer.c $(HDR) $(EXTHDR) - $(TCCX_SHARED) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_tokenizer.c - -fts3_tokenizer1.o: $(TOP)/ext/fts3/fts3_tokenizer1.c $(HDR) $(EXTHDR) - $(TCCX_SHARED) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_tokenizer1.c - -rtree.o: $(TOP)/ext/rtree/rtree.c $(HDR) $(EXTHDR) - $(TCCX_SHARED) -DSQLITE_CORE -c $(TOP)/ext/rtree/rtree.c - - -# Rules for building test programs and for running tests -# -tclsqlite3: $(TOP)/src/tclsqlite.c libsqlite3.a - $(TCCX_SHARED) $(TCL_FLAGS) -DTCLSH=1 -o tclsqlite3 \ - $(TOP)/src/tclsqlite.c libsqlite3.a $(LIBTCL) $(THREADLIB) - - -# Rules to build the 'testfixture' application. -# -TESTFIXTURE_FLAGS = -DTCLSH=1 -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1 -TESTFIXTURE_FLAGS += -DSQLITE_SERVER=1 -DSQLITE_PRIVATE="" -DSQLITE_CORE - -testfixture$(EXE): $(TESTSRC2) libsqlite3.a $(TESTSRC) $(TOP)/src/tclsqlite.c - $(TCCX) $(TCL_FLAGS) $(TESTFIXTURE_FLAGS) \ - $(TESTSRC) $(TESTSRC2) $(TOP)/src/tclsqlite.c \ - -o testfixture$(EXE) $(LIBTCL) $(THREADLIB) libsqlite3.a - -amalgamation-testfixture$(EXE): sqlite3.c $(TESTSRC) $(TOP)/src/tclsqlite.c - $(TCCX) $(TCL_FLAGS) $(TESTFIXTURE_FLAGS) \ - $(TESTSRC) $(TOP)/src/tclsqlite.c sqlite3.c \ - -o testfixture$(EXE) $(LIBTCL) $(THREADLIB) - -fts3-testfixture$(EXE): sqlite3.c fts3amal.c $(TESTSRC) $(TOP)/src/tclsqlite.c - $(TCCX) $(TCL_FLAGS) $(TESTFIXTURE_FLAGS) \ - -DSQLITE_ENABLE_FTS3=1 \ - $(TESTSRC) $(TOP)/src/tclsqlite.c sqlite3.c fts3amal.c \ - -o testfixture$(EXE) $(LIBTCL) $(THREADLIB) - -fulltest: testfixture$(EXE) sqlite3$(EXE) - ./testfixture$(EXE) $(TOP)/test/all.test - -soaktest: testfixture$(EXE) sqlite3$(EXE) - ./testfixture$(EXE) $(TOP)/test/all.test -soak=1 - -fulltestonly: testfixture$(EXE) sqlite3$(EXE) - ./testfixture$(EXE) $(TOP)/test/full.test - -test: testfixture$(EXE) sqlite3$(EXE) - ./testfixture$(EXE) $(TOP)/test/veryquick.test - -sqlite3_analyzer$(EXE): $(TOP)/src/tclsqlite.c sqlite3.c $(TESTSRC) \ - $(TOP)/tool/spaceanal.tcl - sed \ - -e '/^#/d' \ - -e 's,\\,\\\\,g' \ - -e 's,",\\",g' \ - -e 's,^,",' \ - -e 's,$$,\\n",' \ - $(TOP)/tool/spaceanal.tcl >spaceanal_tcl.h - $(TCCX) $(TCL_FLAGS) $(TESTFIXTURE_FLAGS) \ - -DTCLSH=2 -DSQLITE_TEST=1 -DSQLITE_DEBUG=1 -DSQLITE_PRIVATE="" \ - $(TESTSRC) $(TOP)/src/tclsqlite.c sqlite3.c \ - -o sqlite3_analyzer$(EXE) \ - $(LIBTCL) $(THREADLIB) - -TEST_EXTENSION = $(SHPREFIX)testloadext.$(SO) -$(TEST_EXTENSION): $(TOP)/src/test_loadext.c - $(MKSHLIB) $(TOP)/src/test_loadext.c -o $(TEST_EXTENSION) - -extensiontest: testfixture$(EXE) $(TEST_EXTENSION) - ./testfixture$(EXE) $(TOP)/test/loadext.test - -clean: - rm -f *.o sqlite3$(EXE) libsqlite3.a sqlite3.h opcodes.* - rm -f lemon lempar.c parse.* sqlite*.tar.gz mkkeywordhash keywordhash.h - rm -f $(PUBLISH) - rm -f *.da *.bb *.bbg gmon.out - rm -rf quota2a quota2b quota2c - rm -rf tsrc target_source - rm -f testloadext.dll libtestloadext.so - rm -f sqlite3.c fts?amal.c tclsqlite3.c - rm -f sqlite3rc.h - rm -f shell.c sqlite3ext.h - rm -f $(SHPREFIX)sqlite3.$(SO) Index: VERSION ================================================================== --- VERSION +++ VERSION @@ -1,1 +1,1 @@ -3.8.12 +3.9.0 DELETED addopcodes.awk Index: addopcodes.awk ================================================================== --- addopcodes.awk +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/awk -# -# This script appends additional token codes to the end of the -# parse.h file that lemon generates. These extra token codes are -# not used by the parser. But they are used by the tokenizer and/or -# the code generator. -# -# -BEGIN { - max = 0 -} -/^#define TK_/ { - print $0 - if( max<$3 ) max = $3 -} -END { - printf "#define TK_%-29s %4d\n", "TO_TEXT", ++max - printf "#define TK_%-29s %4d\n", "TO_BLOB", ++max - printf "#define TK_%-29s %4d\n", "TO_NUMERIC", ++max - printf "#define TK_%-29s %4d\n", "TO_INT", ++max - printf "#define TK_%-29s %4d\n", "TO_REAL", ++max - printf "#define TK_%-29s %4d\n", "ISNOT", ++max - printf "#define TK_%-29s %4d\n", "END_OF_FILE", ++max - printf "#define TK_%-29s %4d\n", "ILLEGAL", ++max - printf "#define TK_%-29s %4d\n", "SPACE", ++max - printf "#define TK_%-29s %4d\n", "UNCLOSED_STRING", ++max - printf "#define TK_%-29s %4d\n", "FUNCTION", ++max - printf "#define TK_%-29s %4d\n", "COLUMN", ++max - printf "#define TK_%-29s %4d\n", "AGG_FUNCTION", ++max - printf "#define TK_%-29s %4d\n", "AGG_COLUMN", ++max - printf "#define TK_%-29s %4d\n", "UMINUS", ++max - printf "#define TK_%-29s %4d\n", "UPLUS", ++max - printf "#define TK_%-29s %4d\n", "REGISTER", ++max -} Index: autoconf/Makefile.am ================================================================== --- autoconf/Makefile.am +++ autoconf/Makefile.am @@ -1,7 +1,7 @@ -AM_CFLAGS = @THREADSAFE_FLAGS@ @DYNAMIC_EXTENSION_FLAGS@ -DSQLITE_ENABLE_FTS3 -DSQLITE_ENABLE_RTREE +AM_CFLAGS = @THREADSAFE_FLAGS@ @DYNAMIC_EXTENSION_FLAGS@ @FTS5_FLAGS@ @JSON1_FLAGS@ -DSQLITE_ENABLE_FTS3 -DSQLITE_ENABLE_RTREE lib_LTLIBRARIES = libsqlite3.la libsqlite3_la_SOURCES = sqlite3.c libsqlite3_la_LDFLAGS = -no-undefined -version-info 8:6:8 Index: autoconf/configure.ac ================================================================== --- autoconf/configure.ac +++ autoconf/configure.ac @@ -75,10 +75,35 @@ fi AC_MSG_CHECKING([for whether to support dynamic extensions]) AC_MSG_RESULT($enable_dynamic_extensions) AC_SUBST(DYNAMIC_EXTENSION_FLAGS) #----------------------------------------------------------------------- + +#----------------------------------------------------------------------- +# --enable-fts5 +# +AC_ARG_ENABLE(fts5, [AS_HELP_STRING( + [--enable-fts5], [include fts5 support [default=no]])], + [], [enable_fts5=no]) +if test x"$enable_fts5" == "xyes"; then + AC_SEARCH_LIBS(log, m) + FTS5_FLAGS=-DSQLITE_ENABLE_FTS5 +fi +AC_SUBST(FTS5_FLAGS) +#----------------------------------------------------------------------- + +#----------------------------------------------------------------------- +# --enable-json1 +# +AC_ARG_ENABLE(json1, [AS_HELP_STRING( + [--enable-json1], [include json1 support [default=no]])], + [], [enable_json1=no]) +if test x"$enable_json1" == "xyes"; then + JSON1_FLAGS=-DSQLITE_ENABLE_JSON1 +fi +AC_SUBST(JSON1_FLAGS) +#----------------------------------------------------------------------- AC_CHECK_FUNCS(posix_fallocate) #----------------------------------------------------------------------- # UPDATE: Maybe it's better if users just set CFLAGS before invoking Index: configure ================================================================== --- configure +++ configure @@ -1,8 +1,8 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.69 for sqlite 3.8.12. +# Generated by GNU Autoconf 2.69 for sqlite 3.9.0. # # # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc. # # @@ -724,12 +724,12 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='sqlite' PACKAGE_TARNAME='sqlite' -PACKAGE_VERSION='3.8.12' -PACKAGE_STRING='sqlite 3.8.12' +PACKAGE_VERSION='3.9.0' +PACKAGE_STRING='sqlite 3.9.0' PACKAGE_BUGREPORT='' PACKAGE_URL='' # Factoring default headers for most tests. ac_includes_default="\ @@ -801,11 +801,10 @@ RELEASE VERSION program_prefix TCLLIBDIR TCLSH_CMD -AWK INSTALL_DATA INSTALL_SCRIPT INSTALL_PROGRAM CPP OTOOL64 @@ -900,10 +899,15 @@ with_readline_lib with_readline_inc enable_debug enable_amalgamation enable_load_extension +enable_fts3 +enable_fts4 +enable_fts5 +enable_json1 +enable_rtree enable_gcov ' ac_precious_vars='build_alias host_alias target_alias @@ -1452,11 +1456,11 @@ # if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures sqlite 3.8.12 to adapt to many kinds of systems. +\`configure' configures sqlite 3.9.0 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... To assign environment variables (e.g., CC, CFLAGS...), specify them as VAR=VALUE. See below for descriptions of some of the useful variables. @@ -1517,11 +1521,11 @@ _ACEOF fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of sqlite 3.8.12:";; + short | recursive ) echo "Configuration of sqlite 3.9.0:";; esac cat <<\_ACEOF Optional Features: --disable-option-checking ignore unrecognized --enable/--with options @@ -1542,10 +1546,15 @@ --enable-debug enable debugging & verbose explain --disable-amalgamation Disable the amalgamation and instead build all files separately --disable-load-extension Disable loading of external extensions + --enable-fts3 Enable the FTS3 extension + --enable-fts4 Enable the FTS4 extension + --enable-fts5 Enable the FTS5 extension + --enable-json1 Enable the JSON1 extension + --enable-rtree Enable the RTREE extension --enable-gcov Enable coverage testing using gcov Optional Packages: --with-PACKAGE[=ARG] use PACKAGE [ARG=yes] --without-PACKAGE do not use PACKAGE (same as --with-PACKAGE=no) @@ -1632,11 +1641,11 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -sqlite configure 3.8.12 +sqlite configure 3.9.0 generated by GNU Autoconf 2.69 Copyright (C) 2012 Free Software Foundation, Inc. This configure script is free software; the Free Software Foundation gives unlimited permission to copy, distribute and modify it. @@ -2051,11 +2060,11 @@ } # ac_fn_c_check_header_mongrel cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by sqlite $as_me 3.8.12, which was +It was created by sqlite $as_me 3.9.0, which was generated by GNU Autoconf 2.69. Invocation command line was $ $0 $@ _ACEOF @@ -3909,17 +3918,17 @@ if ${lt_cv_nm_interface+:} false; then : $as_echo_n "(cached) " >&6 else lt_cv_nm_interface="BSD nm" echo "int some_variable = 0;" > conftest.$ac_ext - (eval echo "\"\$as_me:3914: $ac_compile\"" >&5) + (eval echo "\"\$as_me:3923: $ac_compile\"" >&5) (eval "$ac_compile" 2>conftest.err) cat conftest.err >&5 - (eval echo "\"\$as_me:3917: $NM \\\"conftest.$ac_objext\\\"\"" >&5) + (eval echo "\"\$as_me:3926: $NM \\\"conftest.$ac_objext\\\"\"" >&5) (eval "$NM \"conftest.$ac_objext\"" 2>conftest.err > conftest.out) cat conftest.err >&5 - (eval echo "\"\$as_me:3920: output\"" >&5) + (eval echo "\"\$as_me:3929: output\"" >&5) cat conftest.out >&5 if $GREP 'External.*some_variable' conftest.out > /dev/null; then lt_cv_nm_interface="MS dumpbin" fi rm -f conftest* @@ -5121,11 +5130,11 @@ fi rm -rf conftest* ;; *-*-irix6*) # Find out which ABI we are using. - echo '#line 5126 "configure"' > conftest.$ac_ext + echo '#line 5135 "configure"' > conftest.$ac_ext if { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$ac_compile\""; } >&5 (eval $ac_compile) 2>&5 ac_status=$? $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 test $ac_status = 0; }; then @@ -6646,15 +6655,15 @@ # The option is referenced via a variable to avoid confusing sed. lt_compile=`echo "$ac_compile" | $SED \ -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:6651: $lt_compile\"" >&5) + (eval echo "\"\$as_me:6660: $lt_compile\"" >&5) (eval "$lt_compile" 2>conftest.err) ac_status=$? cat conftest.err >&5 - echo "$as_me:6655: \$? = $ac_status" >&5 + echo "$as_me:6664: \$? = $ac_status" >&5 if (exit $ac_status) && test -s "$ac_outfile"; then # The compiler can only warn and ignore the option if not recognized # So say no if there are warnings other than the usual output. $ECHO "X$_lt_compiler_boilerplate" | $Xsed -e '/^$/d' >conftest.exp $SED '/^$/d; /^ *+/d' conftest.err >conftest.er2 @@ -6985,15 +6994,15 @@ # The option is referenced via a variable to avoid confusing sed. lt_compile=`echo "$ac_compile" | $SED \ -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:6990: $lt_compile\"" >&5) + (eval echo "\"\$as_me:6999: $lt_compile\"" >&5) (eval "$lt_compile" 2>conftest.err) ac_status=$? cat conftest.err >&5 - echo "$as_me:6994: \$? = $ac_status" >&5 + echo "$as_me:7003: \$? = $ac_status" >&5 if (exit $ac_status) && test -s "$ac_outfile"; then # The compiler can only warn and ignore the option if not recognized # So say no if there are warnings other than the usual output. $ECHO "X$_lt_compiler_boilerplate" | $Xsed -e '/^$/d' >conftest.exp $SED '/^$/d; /^ *+/d' conftest.err >conftest.er2 @@ -7090,15 +7099,15 @@ # with a dollar sign (not a hyphen), so the echo should work correctly. lt_compile=`echo "$ac_compile" | $SED \ -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:7095: $lt_compile\"" >&5) + (eval echo "\"\$as_me:7104: $lt_compile\"" >&5) (eval "$lt_compile" 2>out/conftest.err) ac_status=$? cat out/conftest.err >&5 - echo "$as_me:7099: \$? = $ac_status" >&5 + echo "$as_me:7108: \$? = $ac_status" >&5 if (exit $ac_status) && test -s out/conftest2.$ac_objext then # The compiler can only warn and ignore the option if not recognized # So say no if there are warnings $ECHO "X$_lt_compiler_boilerplate" | $Xsed -e '/^$/d' > out/conftest.exp @@ -7145,15 +7154,15 @@ # with a dollar sign (not a hyphen), so the echo should work correctly. lt_compile=`echo "$ac_compile" | $SED \ -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:7150: $lt_compile\"" >&5) + (eval echo "\"\$as_me:7159: $lt_compile\"" >&5) (eval "$lt_compile" 2>out/conftest.err) ac_status=$? cat out/conftest.err >&5 - echo "$as_me:7154: \$? = $ac_status" >&5 + echo "$as_me:7163: \$? = $ac_status" >&5 if (exit $ac_status) && test -s out/conftest2.$ac_objext then # The compiler can only warn and ignore the option if not recognized # So say no if there are warnings $ECHO "X$_lt_compiler_boilerplate" | $Xsed -e '/^$/d' > out/conftest.exp @@ -9525,11 +9534,11 @@ lt_cv_dlopen_self=cross else lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2 lt_status=$lt_dlunknown cat > conftest.$ac_ext <<_LT_EOF -#line 9530 "configure" +#line 9539 "configure" #include "confdefs.h" #if HAVE_DLFCN_H #include #endif @@ -9621,11 +9630,11 @@ lt_cv_dlopen_self_static=cross else lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2 lt_status=$lt_dlunknown cat > conftest.$ac_ext <<_LT_EOF -#line 9626 "configure" +#line 9635 "configure" #include "confdefs.h" #if HAVE_DLFCN_H #include #endif @@ -9941,52 +9950,10 @@ test -z "$INSTALL_SCRIPT" && INSTALL_SCRIPT='${INSTALL}' test -z "$INSTALL_DATA" && INSTALL_DATA='${INSTALL} -m 644' -for ac_prog in gawk mawk nawk awk -do - # Extract the first word of "$ac_prog", so it can be a program name with args. -set dummy $ac_prog; ac_word=$2 -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 -$as_echo_n "checking for $ac_word... " >&6; } -if ${ac_cv_prog_AWK+:} false; then : - $as_echo_n "(cached) " >&6 -else - if test -n "$AWK"; then - ac_cv_prog_AWK="$AWK" # Let the user override the test. -else -as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in $PATH -do - IFS=$as_save_IFS - test -z "$as_dir" && as_dir=. - for ac_exec_ext in '' $ac_executable_extensions; do - if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then - ac_cv_prog_AWK="$ac_prog" - $as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5 - break 2 - fi -done - done -IFS=$as_save_IFS - -fi -fi -AWK=$ac_cv_prog_AWK -if test -n "$AWK"; then - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $AWK" >&5 -$as_echo "$AWK" >&6; } -else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } -fi - - - test -n "$AWK" && break -done - ######### # Enable large file support (if special flags are necessary) # # Check whether --enable-largefile was given. @@ -11220,10 +11187,181 @@ fi else OPT_FEATURE_FLAGS="-DSQLITE_OMIT_LOAD_EXTENSION=1" fi + +######### +# See whether we should enable Full Text Search extensions +# Check whether --enable-fts3 was given. +if test "${enable_fts3+set}" = set; then : + enableval=$enable_fts3; enable_fts3=yes +else + enable_fts3=no +fi + +if test "${enable_fts3}" = "yes" ; then + OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_FTS3" +fi +# Check whether --enable-fts4 was given. +if test "${enable_fts4+set}" = set; then : + enableval=$enable_fts4; enable_fts4=yes +else + enable_fts4=no +fi + +if test "${enable_fts4}" = "yes" ; then + OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_FTS4" + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing log" >&5 +$as_echo_n "checking for library containing log... " >&6; } +if ${ac_cv_search_log+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_func_search_save_LIBS=$LIBS +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +#ifdef __cplusplus +extern "C" +#endif +char log (); +int +main () +{ +return log (); + ; + return 0; +} +_ACEOF +for ac_lib in '' m; do + if test -z "$ac_lib"; then + ac_res="none required" + else + ac_res=-l$ac_lib + LIBS="-l$ac_lib $ac_func_search_save_LIBS" + fi + if ac_fn_c_try_link "$LINENO"; then : + ac_cv_search_log=$ac_res +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext + if ${ac_cv_search_log+:} false; then : + break +fi +done +if ${ac_cv_search_log+:} false; then : + +else + ac_cv_search_log=no +fi +rm conftest.$ac_ext +LIBS=$ac_func_search_save_LIBS +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_log" >&5 +$as_echo "$ac_cv_search_log" >&6; } +ac_res=$ac_cv_search_log +if test "$ac_res" != no; then : + test "$ac_res" = "none required" || LIBS="$ac_res $LIBS" + +fi + +fi +# Check whether --enable-fts5 was given. +if test "${enable_fts5+set}" = set; then : + enableval=$enable_fts5; enable_fts5=yes +else + enable_fts5=no +fi + +if test "${enable_fts5}" = "yes" ; then + OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_FTS5" + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing log" >&5 +$as_echo_n "checking for library containing log... " >&6; } +if ${ac_cv_search_log+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_func_search_save_LIBS=$LIBS +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +#ifdef __cplusplus +extern "C" +#endif +char log (); +int +main () +{ +return log (); + ; + return 0; +} +_ACEOF +for ac_lib in '' m; do + if test -z "$ac_lib"; then + ac_res="none required" + else + ac_res=-l$ac_lib + LIBS="-l$ac_lib $ac_func_search_save_LIBS" + fi + if ac_fn_c_try_link "$LINENO"; then : + ac_cv_search_log=$ac_res +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext + if ${ac_cv_search_log+:} false; then : + break +fi +done +if ${ac_cv_search_log+:} false; then : + +else + ac_cv_search_log=no +fi +rm conftest.$ac_ext +LIBS=$ac_func_search_save_LIBS +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_log" >&5 +$as_echo "$ac_cv_search_log" >&6; } +ac_res=$ac_cv_search_log +if test "$ac_res" != no; then : + test "$ac_res" = "none required" || LIBS="$ac_res $LIBS" + +fi + +fi + +######### +# See whether we should enable JSON1 +# Check whether --enable-json1 was given. +if test "${enable_json1+set}" = set; then : + enableval=$enable_json1; enable_json1=yes +else + enable_json1=no +fi + +if test "${enable_json1}" = "yes" ; then + OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_JSON1" +fi + +######### +# See whether we should enable RTREE +# Check whether --enable-rtree was given. +if test "${enable_rtree+set}" = set; then : + enableval=$enable_rtree; enable_rtree=yes +else + enable_rtree=no +fi + +if test "${enable_rtree}" = "yes" ; then + OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_RTREE" +fi ######### # attempt to duplicate any OMITS and ENABLES into the $(OPT_FEATURE_FLAGS) parameter for option in $CFLAGS $CPPFLAGS do @@ -11806,11 +11944,11 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # Save the log message, to keep $0 and so on meaningful, and to # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by sqlite $as_me 3.8.12, which was +This file was extended by sqlite $as_me 3.9.0, which was generated by GNU Autoconf 2.69. Invocation command line was CONFIG_FILES = $CONFIG_FILES CONFIG_HEADERS = $CONFIG_HEADERS CONFIG_LINKS = $CONFIG_LINKS @@ -11872,11 +12010,11 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -sqlite config.status 3.8.12 +sqlite config.status 3.9.0 configured by $0, generated by GNU Autoconf 2.69, with options \\"\$ac_cs_config\\" Copyright (C) 2012 Free Software Foundation, Inc. This config.status script is free software; the Free Software Foundation @@ -11883,11 +12021,10 @@ gives unlimited permission to copy, distribute and modify it." ac_pwd='$ac_pwd' srcdir='$srcdir' INSTALL='$INSTALL' -AWK='$AWK' test -n "\$AWK" || AWK=awk _ACEOF cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # The default lists apply if the user does not specify any file. Index: configure.ac ================================================================== --- configure.ac +++ configure.ac @@ -88,11 +88,10 @@ ######### # Programs needed # AC_PROG_LIBTOOL AC_PROG_INSTALL -AC_PROG_AWK ######### # Enable large file support (if special flags are necessary) # AC_SYS_LARGEFILE @@ -557,10 +556,51 @@ OPT_FEATURE_FLAGS="" AC_SEARCH_LIBS(dlopen, dl) else OPT_FEATURE_FLAGS="-DSQLITE_OMIT_LOAD_EXTENSION=1" fi + +######### +# See whether we should enable Full Text Search extensions +AC_ARG_ENABLE(fts3, AC_HELP_STRING([--enable-fts3], + [Enable the FTS3 extension]), + [enable_fts3=yes],[enable_fts3=no]) +if test "${enable_fts3}" = "yes" ; then + OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_FTS3" +fi +AC_ARG_ENABLE(fts4, AC_HELP_STRING([--enable-fts4], + [Enable the FTS4 extension]), + [enable_fts4=yes],[enable_fts4=no]) +if test "${enable_fts4}" = "yes" ; then + OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_FTS4" + AC_SEARCH_LIBS([log],[m]) +fi +AC_ARG_ENABLE(fts5, AC_HELP_STRING([--enable-fts5], + [Enable the FTS5 extension]), + [enable_fts5=yes],[enable_fts5=no]) +if test "${enable_fts5}" = "yes" ; then + OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_FTS5" + AC_SEARCH_LIBS([log],[m]) +fi + +######### +# See whether we should enable JSON1 +AC_ARG_ENABLE(json1, AC_HELP_STRING([--enable-json1], + [Enable the JSON1 extension]), + [enable_json1=yes],[enable_json1=no]) +if test "${enable_json1}" = "yes" ; then + OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_JSON1" +fi + +######### +# See whether we should enable RTREE +AC_ARG_ENABLE(rtree, AC_HELP_STRING([--enable-rtree], + [Enable the RTREE extension]), + [enable_rtree=yes],[enable_rtree=no]) +if test "${enable_rtree}" = "yes" ; then + OPT_FEATURE_FLAGS+=" -DSQLITE_ENABLE_RTREE" +fi ######### # attempt to duplicate any OMITS and ENABLES into the $(OPT_FEATURE_FLAGS) parameter for option in $CFLAGS $CPPFLAGS do Index: ext/fts3/fts3.c ================================================================== --- ext/fts3/fts3.c +++ ext/fts3/fts3.c @@ -1514,10 +1514,23 @@ if( sqlite3_libversion_number()>=3008002 ){ pIdxInfo->estimatedRows = nRow; } #endif } + +/* +** Set the SQLITE_INDEX_SCAN_UNIQUE flag in pIdxInfo->flags. Unless this +** extension is currently being used by a version of SQLite too old to +** support index-info flags. In that case this function is a no-op. +*/ +static void fts3SetUniqueFlag(sqlite3_index_info *pIdxInfo){ +#if SQLITE_VERSION_NUMBER>=3008012 + if( sqlite3_libversion_number()>=3008012 ){ + pIdxInfo->idxFlags |= SQLITE_INDEX_SCAN_UNIQUE; + } +#endif +} /* ** Implementation of the xBestIndex method for FTS3 tables. There ** are three possible strategies, in order of preference: ** @@ -1604,10 +1617,13 @@ iDocidLe = i; break; } } } + + /* If using a docid=? or rowid=? strategy, set the UNIQUE flag. */ + if( pInfo->idxNum==FTS3_DOCID_SEARCH ) fts3SetUniqueFlag(pInfo); iIdx = 1; if( iCons>=0 ){ pInfo->aConstraintUsage[iCons].argvIndex = iIdx++; pInfo->aConstraintUsage[iCons].omit = 1; Index: ext/fts3/fts3Int.h ================================================================== --- ext/fts3/fts3Int.h +++ ext/fts3/fts3Int.h @@ -262,10 +262,11 @@ } *aIndex; int nMaxPendingData; /* Max pending data before flush to disk */ int nPendingData; /* Current bytes of pending data */ sqlite_int64 iPrevDocid; /* Docid of most recently inserted document */ int iPrevLangid; /* Langid of recently inserted document */ + int bPrevDelete; /* True if last operation was a delete */ #if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST) /* State variables used for validating that the transaction control ** methods of the virtual table are called at appropriate times. These ** values do not contribute to FTS functionality; they are used for Index: ext/fts3/fts3_expr.c ================================================================== --- ext/fts3/fts3_expr.c +++ ext/fts3/fts3_expr.c @@ -791,129 +791,155 @@ if( nMaxDepth==0 ){ rc = SQLITE_ERROR; } - if( rc==SQLITE_OK && (eType==FTSQUERY_AND || eType==FTSQUERY_OR) ){ - Fts3Expr **apLeaf; - apLeaf = (Fts3Expr **)sqlite3_malloc(sizeof(Fts3Expr *) * nMaxDepth); - if( 0==apLeaf ){ - rc = SQLITE_NOMEM; - }else{ - memset(apLeaf, 0, sizeof(Fts3Expr *) * nMaxDepth); - } - - if( rc==SQLITE_OK ){ - int i; - Fts3Expr *p; - - /* Set $p to point to the left-most leaf in the tree of eType nodes. */ - for(p=pRoot; p->eType==eType; p=p->pLeft){ - assert( p->pParent==0 || p->pParent->pLeft==p ); - assert( p->pLeft && p->pRight ); - } - - /* This loop runs once for each leaf in the tree of eType nodes. */ - while( 1 ){ - int iLvl; - Fts3Expr *pParent = p->pParent; /* Current parent of p */ - - assert( pParent==0 || pParent->pLeft==p ); - p->pParent = 0; - if( pParent ){ - pParent->pLeft = 0; - }else{ - pRoot = 0; - } - rc = fts3ExprBalance(&p, nMaxDepth-1); - if( rc!=SQLITE_OK ) break; - - for(iLvl=0; p && iLvlpLeft = apLeaf[iLvl]; - pFree->pRight = p; - pFree->pLeft->pParent = pFree; - pFree->pRight->pParent = pFree; - - p = pFree; - pFree = pFree->pParent; - p->pParent = 0; - apLeaf[iLvl] = 0; - } - } - if( p ){ - sqlite3Fts3ExprFree(p); - rc = SQLITE_TOOBIG; - break; - } - - /* If that was the last leaf node, break out of the loop */ - if( pParent==0 ) break; - - /* Set $p to point to the next leaf in the tree of eType nodes */ - for(p=pParent->pRight; p->eType==eType; p=p->pLeft); - - /* Remove pParent from the original tree. */ - assert( pParent->pParent==0 || pParent->pParent->pLeft==pParent ); - pParent->pRight->pParent = pParent->pParent; - if( pParent->pParent ){ - pParent->pParent->pLeft = pParent->pRight; - }else{ - assert( pParent==pRoot ); - pRoot = pParent->pRight; - } - - /* Link pParent into the free node list. It will be used as an - ** internal node of the new tree. */ - pParent->pParent = pFree; - pFree = pParent; - } - - if( rc==SQLITE_OK ){ - p = 0; - for(i=0; ipParent = 0; - }else{ - assert( pFree!=0 ); - pFree->pRight = p; - pFree->pLeft = apLeaf[i]; + if( rc==SQLITE_OK ){ + if( (eType==FTSQUERY_AND || eType==FTSQUERY_OR) ){ + Fts3Expr **apLeaf; + apLeaf = (Fts3Expr **)sqlite3_malloc(sizeof(Fts3Expr *) * nMaxDepth); + if( 0==apLeaf ){ + rc = SQLITE_NOMEM; + }else{ + memset(apLeaf, 0, sizeof(Fts3Expr *) * nMaxDepth); + } + + if( rc==SQLITE_OK ){ + int i; + Fts3Expr *p; + + /* Set $p to point to the left-most leaf in the tree of eType nodes. */ + for(p=pRoot; p->eType==eType; p=p->pLeft){ + assert( p->pParent==0 || p->pParent->pLeft==p ); + assert( p->pLeft && p->pRight ); + } + + /* This loop runs once for each leaf in the tree of eType nodes. */ + while( 1 ){ + int iLvl; + Fts3Expr *pParent = p->pParent; /* Current parent of p */ + + assert( pParent==0 || pParent->pLeft==p ); + p->pParent = 0; + if( pParent ){ + pParent->pLeft = 0; + }else{ + pRoot = 0; + } + rc = fts3ExprBalance(&p, nMaxDepth-1); + if( rc!=SQLITE_OK ) break; + + for(iLvl=0; p && iLvlpLeft = apLeaf[iLvl]; + pFree->pRight = p; pFree->pLeft->pParent = pFree; pFree->pRight->pParent = pFree; p = pFree; pFree = pFree->pParent; p->pParent = 0; - } - } - } - pRoot = p; - }else{ - /* An error occurred. Delete the contents of the apLeaf[] array - ** and pFree list. Everything else is cleaned up by the call to - ** sqlite3Fts3ExprFree(pRoot) below. */ - Fts3Expr *pDel; - for(i=0; ipParent; - sqlite3_free(pDel); - } - } - - assert( pFree==0 ); - sqlite3_free( apLeaf ); - } - } - + apLeaf[iLvl] = 0; + } + } + if( p ){ + sqlite3Fts3ExprFree(p); + rc = SQLITE_TOOBIG; + break; + } + + /* If that was the last leaf node, break out of the loop */ + if( pParent==0 ) break; + + /* Set $p to point to the next leaf in the tree of eType nodes */ + for(p=pParent->pRight; p->eType==eType; p=p->pLeft); + + /* Remove pParent from the original tree. */ + assert( pParent->pParent==0 || pParent->pParent->pLeft==pParent ); + pParent->pRight->pParent = pParent->pParent; + if( pParent->pParent ){ + pParent->pParent->pLeft = pParent->pRight; + }else{ + assert( pParent==pRoot ); + pRoot = pParent->pRight; + } + + /* Link pParent into the free node list. It will be used as an + ** internal node of the new tree. */ + pParent->pParent = pFree; + pFree = pParent; + } + + if( rc==SQLITE_OK ){ + p = 0; + for(i=0; ipParent = 0; + }else{ + assert( pFree!=0 ); + pFree->pRight = p; + pFree->pLeft = apLeaf[i]; + pFree->pLeft->pParent = pFree; + pFree->pRight->pParent = pFree; + + p = pFree; + pFree = pFree->pParent; + p->pParent = 0; + } + } + } + pRoot = p; + }else{ + /* An error occurred. Delete the contents of the apLeaf[] array + ** and pFree list. Everything else is cleaned up by the call to + ** sqlite3Fts3ExprFree(pRoot) below. */ + Fts3Expr *pDel; + for(i=0; ipParent; + sqlite3_free(pDel); + } + } + + assert( pFree==0 ); + sqlite3_free( apLeaf ); + } + }else if( eType==FTSQUERY_NOT ){ + Fts3Expr *pLeft = pRoot->pLeft; + Fts3Expr *pRight = pRoot->pRight; + + pRoot->pLeft = 0; + pRoot->pRight = 0; + pLeft->pParent = 0; + pRight->pParent = 0; + + rc = fts3ExprBalance(&pLeft, nMaxDepth-1); + if( rc==SQLITE_OK ){ + rc = fts3ExprBalance(&pRight, nMaxDepth-1); + } + + if( rc!=SQLITE_OK ){ + sqlite3Fts3ExprFree(pRight); + sqlite3Fts3ExprFree(pLeft); + }else{ + assert( pLeft && pRight ); + pRoot->pLeft = pLeft; + pLeft->pParent = pRoot; + pRoot->pRight = pRight; + pRight->pParent = pRoot; + } + } + } + if( rc!=SQLITE_OK ){ sqlite3Fts3ExprFree(pRoot); pRoot = 0; } *pp = pRoot; Index: ext/fts3/fts3_write.c ================================================================== --- ext/fts3/fts3_write.c +++ ext/fts3/fts3_write.c @@ -858,30 +858,34 @@ ** fts3PendingTermsAdd() are to add term/position-list pairs for the ** contents of the document with docid iDocid. */ static int fts3PendingTermsDocid( Fts3Table *p, /* Full-text table handle */ + int bDelete, /* True if this op is a delete */ int iLangid, /* Language id of row being written */ sqlite_int64 iDocid /* Docid of row being written */ ){ assert( iLangid>=0 ); + assert( bDelete==1 || bDelete==0 ); /* TODO(shess) Explore whether partially flushing the buffer on ** forced-flush would provide better performance. I suspect that if ** we ordered the doclists by size and flushed the largest until the ** buffer was half empty, that would let the less frequent terms ** generate longer doclists. */ - if( iDocid<=p->iPrevDocid + if( iDocidiPrevDocid + || (iDocid==p->iPrevDocid && p->bPrevDelete==0) || p->iPrevLangid!=iLangid || p->nPendingData>p->nMaxPendingData ){ int rc = sqlite3Fts3PendingTermsFlush(p); if( rc!=SQLITE_OK ) return rc; } p->iPrevDocid = iDocid; p->iPrevLangid = iLangid; + p->bPrevDelete = bDelete; return SQLITE_OK; } /* ** Discard the contents of the pending-terms hash tables. @@ -1067,11 +1071,12 @@ rc = fts3SqlStmt(p, SQL_SELECT_CONTENT_BY_ROWID, &pSelect, &pRowid); if( rc==SQLITE_OK ){ if( SQLITE_ROW==sqlite3_step(pSelect) ){ int i; int iLangid = langidFromSelect(p, pSelect); - rc = fts3PendingTermsDocid(p, iLangid, sqlite3_column_int64(pSelect, 0)); + i64 iDocid = sqlite3_column_int64(pSelect, 0); + rc = fts3PendingTermsDocid(p, 1, iLangid, iDocid); for(i=1; rc==SQLITE_OK && i<=p->nColumn; i++){ int iCol = i-1; if( p->abNotindexed[iCol]==0 ){ const char *zText = (const char *)sqlite3_column_text(pSelect, i); rc = fts3PendingTermsAdd(p, iLangid, zText, -1, &aSz[iCol]); @@ -1315,18 +1320,23 @@ if( !pNext || pNext>=&pReader->aNode[pReader->nNode] ){ if( fts3SegReaderIsPending(pReader) ){ Fts3HashElem *pElem = *(pReader->ppNextElem); - if( pElem==0 ){ - pReader->aNode = 0; - }else{ + sqlite3_free(pReader->aNode); + pReader->aNode = 0; + if( pElem ){ + char *aCopy; PendingList *pList = (PendingList *)fts3HashData(pElem); + int nCopy = pList->nData+1; pReader->zTerm = (char *)fts3HashKey(pElem); pReader->nTerm = fts3HashKeysize(pElem); - pReader->nNode = pReader->nDoclist = pList->nData + 1; - pReader->aNode = pReader->aDoclist = pList->aData; + aCopy = (char*)sqlite3_malloc(nCopy); + if( !aCopy ) return SQLITE_NOMEM; + memcpy(aCopy, pList->aData, nCopy); + pReader->nNode = pReader->nDoclist = nCopy; + pReader->aNode = pReader->aDoclist = aCopy; pReader->ppNextElem++; assert( pReader->aNode ); } return SQLITE_OK; } @@ -1562,16 +1572,18 @@ /* ** Free all allocations associated with the iterator passed as the ** second argument. */ void sqlite3Fts3SegReaderFree(Fts3SegReader *pReader){ - if( pReader && !fts3SegReaderIsPending(pReader) ){ - sqlite3_free(pReader->zTerm); + if( pReader ){ + if( !fts3SegReaderIsPending(pReader) ){ + sqlite3_free(pReader->zTerm); + } if( !fts3SegReaderIsRootOnly(pReader) ){ sqlite3_free(pReader->aNode); - sqlite3_blob_close(pReader->pBlob); } + sqlite3_blob_close(pReader->pBlob); } sqlite3_free(pReader); } /* @@ -3510,11 +3522,11 @@ } while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ int iCol; int iLangid = langidFromSelect(p, pStmt); - rc = fts3PendingTermsDocid(p, iLangid, sqlite3_column_int64(pStmt, 0)); + rc = fts3PendingTermsDocid(p, 0, iLangid, sqlite3_column_int64(pStmt, 0)); memset(aSz, 0, sizeof(aSz[0]) * (p->nColumn+1)); for(iCol=0; rc==SQLITE_OK && iColnColumn; iCol++){ if( p->abNotindexed[iCol]==0 ){ const char *z = (const char *) sqlite3_column_text(pStmt, iCol+1); rc = fts3PendingTermsAdd(p, iLangid, z, iCol, &aSz[iCol]); @@ -5615,11 +5627,11 @@ if( rc==SQLITE_CONSTRAINT && p->zContentTbl==0 ){ rc = FTS_CORRUPT_VTAB; } } if( rc==SQLITE_OK && (!isRemove || *pRowid!=p->iPrevDocid ) ){ - rc = fts3PendingTermsDocid(p, iLangid, *pRowid); + rc = fts3PendingTermsDocid(p, 0, iLangid, *pRowid); } if( rc==SQLITE_OK ){ assert( p->iPrevDocid==*pRowid ); rc = fts3InsertTerms(p, iLangid, apVal, aSzIns); } Index: ext/fts5/fts5.h ================================================================== --- ext/fts5/fts5.h +++ ext/fts5/fts5.h @@ -20,10 +20,14 @@ #ifndef _FTS5_H #define _FTS5_H #include "sqlite3.h" + +#ifdef __cplusplus +extern "C" { +#endif /************************************************************************* ** CUSTOM AUXILIARY FUNCTIONS ** ** Virtual table implementations may overload SQL functions by implementing @@ -505,8 +509,12 @@ }; /* ** END OF REGISTRATION API *************************************************************************/ + +#ifdef __cplusplus +} /* end of the 'extern "C"' block */ +#endif #endif /* _FTS5_H */ Index: ext/fts5/fts5Int.h ================================================================== --- ext/fts5/fts5Int.h +++ ext/fts5/fts5Int.h @@ -79,10 +79,24 @@ #else # define assert_nc(x) assert(x) #endif typedef struct Fts5Global Fts5Global; +typedef struct Fts5Colset Fts5Colset; + +/* If a NEAR() clump or phrase may only match a specific set of columns, +** then an object of the following type is used to record the set of columns. +** Each entry in the aiCol[] array is a column that may be matched. +** +** This object is used by fts5_expr.c and fts5_index.c. +*/ +struct Fts5Colset { + int nCol; + int aiCol[1]; +}; + + /************************************************************************** ** Interface to code in fts5_config.c. fts5_config.c contains contains code ** to parse the arguments passed to the CREATE VIRTUAL TABLE statement. */ @@ -238,11 +252,10 @@ #define FTS5_POS2OFFSET(iPos) (int)(iPos & 0xFFFFFFFF) typedef struct Fts5PoslistReader Fts5PoslistReader; struct Fts5PoslistReader { /* Variables used only by sqlite3Fts5PoslistIterXXX() functions. */ - int iCol; /* If (iCol>=0), this column only */ const u8 *a; /* Position list to iterate through */ int n; /* Size of buffer at a[] in bytes */ int i; /* Current offset in a[] */ u8 bFlag; /* For client use (any custom purpose) */ @@ -250,11 +263,10 @@ /* Output variables */ u8 bEof; /* Set to true at EOF */ i64 iPos; /* (iCol<<32) + iPos */ }; int sqlite3Fts5PoslistReaderInit( - int iCol, /* If (iCol>=0), this column only */ const u8 *a, int n, /* Poslist buffer to iterate through */ Fts5PoslistReader *pIter /* Iterator object to initialize */ ); int sqlite3Fts5PoslistReaderNext(Fts5PoslistReader*); @@ -303,11 +315,11 @@ int sqlite3Fts5IndexOpen(Fts5Config *pConfig, int bCreate, Fts5Index**, char**); int sqlite3Fts5IndexClose(Fts5Index *p); /* ** for( -** pIter = sqlite3Fts5IndexQuery(p, "token", 5, 0); +** sqlite3Fts5IndexQuery(p, "token", 5, 0, 0, &pIter); ** 0==sqlite3Fts5IterEof(pIter); ** sqlite3Fts5IterNext(pIter) ** ){ ** i64 iRowid = sqlite3Fts5IterRowid(pIter); ** } @@ -319,11 +331,12 @@ */ int sqlite3Fts5IndexQuery( Fts5Index *p, /* FTS index to query */ const char *pToken, int nToken, /* Token (or prefix) to query for */ int flags, /* Mask of FTS5INDEX_QUERY_X flags */ - Fts5IndexIter **ppIter + Fts5Colset *pColset, /* Match these columns only */ + Fts5IndexIter **ppIter /* OUT: New iterator object */ ); /* ** The various operations on open token or token prefix iterators opened ** using sqlite3Fts5IndexQuery(). @@ -330,11 +343,11 @@ */ int sqlite3Fts5IterEof(Fts5IndexIter*); int sqlite3Fts5IterNext(Fts5IndexIter*); int sqlite3Fts5IterNextFrom(Fts5IndexIter*, i64 iMatch); i64 sqlite3Fts5IterRowid(Fts5IndexIter*); -int sqlite3Fts5IterPoslist(Fts5IndexIter*, const u8 **pp, int *pn, i64 *pi); +int sqlite3Fts5IterPoslist(Fts5IndexIter*,Fts5Colset*, const u8**, int*, i64*); int sqlite3Fts5IterPoslistBuffer(Fts5IndexIter *pIter, Fts5Buffer *pBuf); /* ** Close an iterator opened by sqlite3Fts5IndexQuery(). */ @@ -368,10 +381,11 @@ ** Indicate that subsequent calls to sqlite3Fts5IndexWrite() pertain to ** document iDocid. */ int sqlite3Fts5IndexBeginWrite( Fts5Index *p, /* Index to write to */ + int bDelete, /* True if current operation is a delete */ i64 iDocid /* Docid to add or remove data from */ ); /* ** Flush any data stored in the in-memory hash tables to the database. @@ -431,10 +445,19 @@ u8 sqlite3Fts5GetVarint(const unsigned char*, u64*); int sqlite3Fts5PutVarint(unsigned char *p, u64 v); #define fts5GetVarint32(a,b) sqlite3Fts5GetVarint32(a,(u32*)&b) #define fts5GetVarint sqlite3Fts5GetVarint + +#define fts5FastGetVarint32(a, iOff, nVal) { \ + nVal = (a)[iOff++]; \ + if( nVal & 0x80 ){ \ + iOff--; \ + iOff += fts5GetVarint32(&(a)[iOff], nVal); \ + } \ +} + /* ** End of interface to code in fts5_varint.c. **************************************************************************/ @@ -524,11 +547,12 @@ int sqlite3Fts5DropAll(Fts5Config*); int sqlite3Fts5CreateTable(Fts5Config*, const char*, const char*, int, char **); int sqlite3Fts5StorageDelete(Fts5Storage *p, i64); -int sqlite3Fts5StorageInsert(Fts5Storage *p, sqlite3_value **apVal, int, i64*); +int sqlite3Fts5StorageContentInsert(Fts5Storage *p, sqlite3_value**, i64*); +int sqlite3Fts5StorageIndexInsert(Fts5Storage *p, sqlite3_value**, i64); int sqlite3Fts5StorageIntegrity(Fts5Storage *p); int sqlite3Fts5StorageStmt(Fts5Storage *p, int eStmt, sqlite3_stmt**, char**); void sqlite3Fts5StorageStmtRelease(Fts5Storage *p, int eStmt, sqlite3_stmt*); @@ -563,11 +587,10 @@ typedef struct Fts5ExprNode Fts5ExprNode; typedef struct Fts5Parse Fts5Parse; typedef struct Fts5Token Fts5Token; typedef struct Fts5ExprPhrase Fts5ExprPhrase; typedef struct Fts5ExprNearset Fts5ExprNearset; -typedef struct Fts5ExprColset Fts5ExprColset; struct Fts5Token { const char *p; /* Token text (not NULL terminated) */ int n; /* Size of buffer p in bytes */ }; @@ -631,22 +654,22 @@ Fts5Parse*, Fts5ExprNearset*, Fts5ExprPhrase* ); -Fts5ExprColset *sqlite3Fts5ParseColset( +Fts5Colset *sqlite3Fts5ParseColset( Fts5Parse*, - Fts5ExprColset*, + Fts5Colset*, Fts5Token * ); void sqlite3Fts5ParsePhraseFree(Fts5ExprPhrase*); void sqlite3Fts5ParseNearsetFree(Fts5ExprNearset*); void sqlite3Fts5ParseNodeFree(Fts5ExprNode*); void sqlite3Fts5ParseSetDistance(Fts5Parse*, Fts5ExprNearset*, Fts5Token*); -void sqlite3Fts5ParseSetColset(Fts5Parse*, Fts5ExprNearset*, Fts5ExprColset*); +void sqlite3Fts5ParseSetColset(Fts5Parse*, Fts5ExprNearset*, Fts5Colset*); void sqlite3Fts5ParseFinished(Fts5Parse *pParse, Fts5ExprNode *p); void sqlite3Fts5ParseNear(Fts5Parse *pParse, Fts5Token*); /* ** End of interface to code in fts5_expr.c. Index: ext/fts5/fts5_aux.c ================================================================== --- ext/fts5/fts5_aux.c +++ ext/fts5/fts5_aux.c @@ -11,11 +11,11 @@ ****************************************************************************** */ #include "fts5Int.h" -#include +#include /* amalgamator: keep */ /* ** Object used to iterate through all "coalesced phrase instances" in ** a single column of the current row. If the phrase instances in the ** column being considered do not overlap, this object simply iterates @@ -135,11 +135,11 @@ int *pRc, HighlightContext *p, const char *z, int n ){ if( *pRc==SQLITE_OK ){ - if( n<0 ) n = strlen(z); + if( n<0 ) n = (int)strlen(z); p->zOut = sqlite3_mprintf("%z%.*s", p->zOut, n, z); if( p->zOut==0 ) *pRc = SQLITE_NOMEM; } } Index: ext/fts5/fts5_buffer.c ================================================================== --- ext/fts5/fts5_buffer.c +++ ext/fts5/fts5_buffer.c @@ -90,11 +90,11 @@ void sqlite3Fts5BufferAppendString( int *pRc, Fts5Buffer *pBuf, const char *zStr ){ - int nStr = strlen(zStr); + int nStr = (int)strlen(zStr); sqlite3Fts5BufferAppendBlob(pRc, pBuf, nStr+1, (const u8*)zStr); pBuf->n--; } /* @@ -183,15 +183,15 @@ *piOff = -1; return 1; }else{ i64 iOff = *piOff; int iVal; - i += fts5GetVarint32(&a[i], iVal); + fts5FastGetVarint32(a, i, iVal); if( iVal==1 ){ - i += fts5GetVarint32(&a[i], iVal); + fts5FastGetVarint32(a, i, iVal); iOff = ((i64)iVal) << 32; - i += fts5GetVarint32(&a[i], iVal); + fts5FastGetVarint32(a, i, iVal); } *piOff = iOff + (iVal-2); *pi = i; return 0; } @@ -201,30 +201,24 @@ /* ** Advance the iterator object passed as the only argument. Return true ** if the iterator reaches EOF, or false otherwise. */ int sqlite3Fts5PoslistReaderNext(Fts5PoslistReader *pIter){ - if( sqlite3Fts5PoslistNext64(pIter->a, pIter->n, &pIter->i, &pIter->iPos) - || (pIter->iCol>=0 && (pIter->iPos >> 32) > pIter->iCol) - ){ + if( sqlite3Fts5PoslistNext64(pIter->a, pIter->n, &pIter->i, &pIter->iPos) ){ pIter->bEof = 1; } return pIter->bEof; } int sqlite3Fts5PoslistReaderInit( - int iCol, /* If (iCol>=0), this column only */ const u8 *a, int n, /* Poslist buffer to iterate through */ Fts5PoslistReader *pIter /* Iterator object to initialize */ ){ memset(pIter, 0, sizeof(*pIter)); pIter->a = a; pIter->n = n; - pIter->iCol = iCol; - do { - sqlite3Fts5PoslistReaderNext(pIter); - }while( pIter->bEof==0 && (pIter->iPos >> 32)bEof; } int sqlite3Fts5PoslistWriterAppend( Fts5Buffer *pBuf, @@ -231,17 +225,19 @@ Fts5PoslistWriter *pWriter, i64 iPos ){ static const i64 colmask = ((i64)(0x7FFFFFFF)) << 32; int rc = SQLITE_OK; - if( (iPos & colmask) != (pWriter->iPrev & colmask) ){ - fts5BufferAppendVarint(&rc, pBuf, 1); - fts5BufferAppendVarint(&rc, pBuf, (iPos >> 32)); - pWriter->iPrev = (iPos & colmask); + if( 0==sqlite3Fts5BufferGrow(&rc, pBuf, 5+5+5) ){ + if( (iPos & colmask) != (pWriter->iPrev & colmask) ){ + pBuf->p[pBuf->n++] = 1; + pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], (iPos>>32)); + pWriter->iPrev = (iPos & colmask); + } + pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], (iPos-pWriter->iPrev)+2); + pWriter->iPrev = iPos; } - fts5BufferAppendVarint(&rc, pBuf, (iPos - pWriter->iPrev) + 2); - pWriter->iPrev = iPos; return rc; } void *sqlite3Fts5MallocZero(int *pRc, int nByte){ void *pRet = 0; @@ -266,11 +262,11 @@ */ char *sqlite3Fts5Strndup(int *pRc, const char *pIn, int nIn){ char *zRet = 0; if( *pRc==SQLITE_OK ){ if( nIn<0 ){ - nIn = strlen(pIn); + nIn = (int)strlen(pIn); } zRet = (char*)sqlite3_malloc(nIn+1); if( zRet ){ memcpy(zRet, pIn, nIn); zRet[nIn] = '\0'; @@ -288,15 +284,16 @@ ** ** * All non-ASCII characters, ** * The 52 upper and lower case ASCII characters, and ** * The 10 integer ASCII characters. ** * The underscore character "_" (0x5F). +** * The unicode "subsitute" character (0x1A). */ int sqlite3Fts5IsBareword(char t){ u8 aBareword[128] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x00 .. 0x0F */ - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x10 .. 0x1F */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, /* 0x10 .. 0x1F */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x20 .. 0x2F */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 0x30 .. 0x3F */ 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 0x40 .. 0x4F */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 0x50 .. 0x5F */ 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 0x60 .. 0x6F */ Index: ext/fts5/fts5_config.c ================================================================== --- ext/fts5/fts5_config.c +++ ext/fts5/fts5_config.c @@ -209,11 +209,11 @@ const char *zCmd, /* Special command to parse */ const char *zArg, /* Argument to parse */ char **pzErr /* OUT: Error message */ ){ int rc = SQLITE_OK; - int nCmd = strlen(zCmd); + int nCmd = (int)strlen(zCmd); if( sqlite3_strnicmp("prefix", zCmd, nCmd)==0 ){ const int nByte = sizeof(int) * FTS5_MAX_PREFIX_INDEXES; const char *p; if( pConfig->aPrefix ){ *pzErr = sqlite3_mprintf("multiple prefix=... directives"); @@ -246,11 +246,11 @@ return rc; } if( sqlite3_strnicmp("tokenize", zCmd, nCmd)==0 ){ const char *p = (const char*)zArg; - int nArg = strlen(zArg) + 1; + int nArg = (int)strlen(zArg) + 1; char **azArg = sqlite3Fts5MallocZero(&rc, sizeof(char*) * nArg); char *pDel = sqlite3Fts5MallocZero(&rc, nArg * 2); char *pSpace = pDel; if( azArg && pSpace ){ @@ -362,11 +362,11 @@ char **pzOut, /* OUT: malloc'd buffer containing str/bw */ int *pbQuoted /* OUT: Set to true if dequoting required */ ){ const char *zRet = 0; - int nIn = strlen(zIn); + int nIn = (int)strlen(zIn); char *zOut = sqlite3_malloc(nIn+1); assert( *pRc==SQLITE_OK ); *pbQuoted = 0; *pzOut = 0; Index: ext/fts5/fts5_expr.c ================================================================== --- ext/fts5/fts5_expr.c +++ ext/fts5/fts5_expr.c @@ -30,10 +30,15 @@ ** Functions generated by lemon from fts5parse.y. */ void *sqlite3Fts5ParserAlloc(void *(*mallocProc)(u64)); void sqlite3Fts5ParserFree(void*, void (*freeProc)(void*)); void sqlite3Fts5Parser(void*, int, Fts5Token, Fts5Parse*); +#ifndef NDEBUG +#include +void sqlite3Fts5ParserTrace(FILE*, char*); +#endif + struct Fts5Expr { Fts5Index *pIndex; Fts5ExprNode *pRoot; int bDesc; /* Iterate in descending rowid order */ @@ -87,27 +92,17 @@ Fts5Buffer poslist; /* Current position list */ int nTerm; /* Number of entries in aTerm[] */ Fts5ExprTerm aTerm[1]; /* Terms that make up this phrase */ }; -/* -** If a NEAR() clump may only match a specific set of columns, then -** Fts5ExprNearset.pColset points to an object of the following type. -** Each entry in the aiCol[] array -*/ -struct Fts5ExprColset { - int nCol; - int aiCol[1]; -}; - /* ** One or more phrases that must appear within a certain token distance of ** each other within each matching document. */ struct Fts5ExprNearset { int nNear; /* NEAR parameter */ - Fts5ExprColset *pColset; /* Columns to search (NULL -> all columns) */ + Fts5Colset *pColset; /* Columns to search (NULL -> all columns) */ int nPhrase; /* Number of entries in aPhrase[] array */ Fts5ExprPhrase *apPhrase[1]; /* Array of phrase pointers */ }; @@ -274,18 +269,10 @@ sqlite3_free(p->apExprPhrase); sqlite3_free(p); } } -static int fts5ExprColsetTest(Fts5ExprColset *pColset, int iCol){ - int i; - for(i=0; inCol; i++){ - if( pColset->aiCol[i]==iCol ) return 1; - } - return 0; -} - /* ** Argument pTerm must be a synonym iterator. Return the current rowid ** that it points to. */ static i64 fts5ExprSynonymRowid(Fts5ExprTerm *pTerm, int bDesc, int *pbEof){ @@ -312,10 +299,11 @@ /* ** Argument pTerm must be a synonym iterator. */ static int fts5ExprSynonymPoslist( Fts5ExprTerm *pTerm, + Fts5Colset *pColset, i64 iRowid, int *pbDel, /* OUT: Caller should sqlite3_free(*pa) */ u8 **pa, int *pn ){ Fts5PoslistReader aStatic[4]; @@ -330,11 +318,11 @@ Fts5IndexIter *pIter = p->pIter; if( sqlite3Fts5IterEof(pIter)==0 && sqlite3Fts5IterRowid(pIter)==iRowid ){ const u8 *a; int n; i64 dummy; - rc = sqlite3Fts5IterPoslist(pIter, &a, &n, &dummy); + rc = sqlite3Fts5IterPoslist(pIter, pColset, &a, &n, &dummy); if( rc!=SQLITE_OK ) goto synonym_poslist_out; if( nIter==nAlloc ){ int nByte = sizeof(Fts5PoslistReader) * nAlloc * 2; Fts5PoslistReader *aNew = (Fts5PoslistReader*)sqlite3_malloc(nByte); if( aNew==0 ){ @@ -344,11 +332,11 @@ memcpy(aNew, aIter, sizeof(Fts5PoslistReader) * nIter); nAlloc = nAlloc*2; if( aIter!=aStatic ) sqlite3_free(aIter); aIter = aNew; } - sqlite3Fts5PoslistReaderInit(-1, a, n, &aIter[nIter]); + sqlite3Fts5PoslistReaderInit(a, n, &aIter[nIter]); assert( aIter[nIter].bEof==0 ); nIter++; } } @@ -403,26 +391,20 @@ ** otherwise. It is not considered an error code if the current rowid is ** not a match. */ static int fts5ExprPhraseIsMatch( Fts5ExprNode *pNode, /* Node pPhrase belongs to */ - Fts5ExprColset *pColset, /* Restrict matches to these columns */ + Fts5Colset *pColset, /* Restrict matches to these columns */ Fts5ExprPhrase *pPhrase, /* Phrase object to initialize */ int *pbMatch /* OUT: Set to true if really a match */ ){ Fts5PoslistWriter writer = {0}; Fts5PoslistReader aStatic[4]; Fts5PoslistReader *aIter = aStatic; int i; int rc = SQLITE_OK; - int iCol = -1; - if( pColset && pColset->nCol==1 ){ - iCol = pColset->aiCol[0]; - pColset = 0; - } - fts5BufferZero(&pPhrase->poslist); /* If the aStatic[] array is not large enough, allocate a large array ** using sqlite3_malloc(). This approach could be improved upon. */ if( pPhrase->nTerm>(sizeof(aStatic) / sizeof(aStatic[0])) ){ @@ -438,16 +420,18 @@ i64 dummy; int n = 0; int bFlag = 0; const u8 *a = 0; if( pTerm->pSynonym ){ - rc = fts5ExprSynonymPoslist(pTerm, pNode->iRowid, &bFlag, (u8**)&a, &n); + rc = fts5ExprSynonymPoslist( + pTerm, pColset, pNode->iRowid, &bFlag, (u8**)&a, &n + ); }else{ - rc = sqlite3Fts5IterPoslist(pTerm->pIter, &a, &n, &dummy); + rc = sqlite3Fts5IterPoslist(pTerm->pIter, pColset, &a, &n, &dummy); } if( rc!=SQLITE_OK ) goto ismatch_out; - sqlite3Fts5PoslistReaderInit(iCol, a, n, &aIter[i]); + sqlite3Fts5PoslistReaderInit(a, n, &aIter[i]); aIter[i].bFlag = bFlag; if( aIter[i].bEof ) goto ismatch_out; } while( 1 ){ @@ -466,15 +450,13 @@ if( pPos->iPos>iAdj ) iPos = pPos->iPos-i; } } }while( bMatch==0 ); - if( pColset==0 || fts5ExprColsetTest(pColset, FTS5_POS2COLUMN(iPos)) ){ - /* Append position iPos to the output */ - rc = sqlite3Fts5PoslistWriterAppend(&pPhrase->poslist, &writer, iPos); - if( rc!=SQLITE_OK ) goto ismatch_out; - } + /* Append position iPos to the output */ + rc = sqlite3Fts5PoslistWriterAppend(&pPhrase->poslist, &writer, iPos); + if( rc!=SQLITE_OK ) goto ismatch_out; for(i=0; inTerm; i++){ if( sqlite3Fts5PoslistReaderNext(&aIter[i]) ) goto ismatch_out; } } @@ -765,65 +747,10 @@ *piLast = fts5ExprSynonymRowid(pTerm, bDesc, &bEof); } return bEof; } -/* -** IN/OUT parameter (*pa) points to a position list n bytes in size. If -** the position list contains entries for column iCol, then (*pa) is set -** to point to the sub-position-list for that column and the number of -** bytes in it returned. Or, if the argument position list does not -** contain any entries for column iCol, return 0. -*/ -static int fts5ExprExtractCol( - const u8 **pa, /* IN/OUT: Pointer to poslist */ - int n, /* IN: Size of poslist in bytes */ - int iCol /* Column to extract from poslist */ -){ - int iCurrent = 0; - const u8 *p = *pa; - const u8 *pEnd = &p[n]; /* One byte past end of position list */ - u8 prev = 0; - - while( iCol!=iCurrent ){ - /* Advance pointer p until it points to pEnd or an 0x01 byte that is - ** not part of a varint */ - while( (prev & 0x80) || *p!=0x01 ){ - prev = *p++; - if( p==pEnd ) return 0; - } - *pa = p++; - p += fts5GetVarint32(p, iCurrent); - } - - /* Advance pointer p until it points to pEnd or an 0x01 byte that is - ** not part of a varint */ - assert( (prev & 0x80)==0 ); - while( pnCol; i++){ - const u8 *pSub = pPos; - int nSub = fts5ExprExtractCol(&pSub, nPos, pColset->aiCol[i]); - if( nSub ){ - fts5BufferAppendBlob(&rc, pBuf, nSub, pSub); - } - } - return rc; -} static int fts5ExprNearTest( int *pRc, Fts5Expr *pExpr, /* Expression that pNear is a part of */ Fts5ExprNode *pNode /* The "NEAR" node (FTS5_STRING) */ @@ -866,39 +793,20 @@ ** a new poslist the way we have to for more complicated phrase or NEAR ** expressions. */ Fts5ExprNearset *pNear = pNode->pNear; Fts5ExprPhrase *pPhrase = pNear->apPhrase[0]; Fts5IndexIter *pIter = pPhrase->aTerm[0].pIter; - Fts5ExprColset *pColset = pNear->pColset; - const u8 *pPos; - int nPos; + Fts5Colset *pColset = pNear->pColset; int rc; assert( pNode->eType==FTS5_TERM ); assert( pNear->nPhrase==1 && pPhrase->nTerm==1 ); assert( pPhrase->aTerm[0].pSynonym==0 ); - rc = sqlite3Fts5IterPoslist(pIter, &pPos, &nPos, &pNode->iRowid); - - /* If the term may match any column, then this must be a match. - ** Return immediately in this case. Otherwise, try to find the - ** part of the poslist that corresponds to the required column. - ** If it can be found, return. If it cannot, the next iteration - ** of the loop will test the next rowid in the database for this - ** term. */ - if( pColset==0 ){ - assert( pPhrase->poslist.nSpace==0 ); - pPhrase->poslist.p = (u8*)pPos; - pPhrase->poslist.n = nPos; - }else if( pColset->nCol==1 ){ - assert( pPhrase->poslist.nSpace==0 ); - pPhrase->poslist.n = fts5ExprExtractCol(&pPos, nPos, pColset->aiCol[0]); - pPhrase->poslist.p = (u8*)pPos; - }else if( rc==SQLITE_OK ){ - rc = fts5ExprExtractColset(pColset, pPos, nPos, &pPhrase->poslist); - } - + rc = sqlite3Fts5IterPoslist(pIter, pColset, + (const u8**)&pPhrase->poslist.p, &pPhrase->poslist.n, &pNode->iRowid + ); pNode->bNomatch = (pPhrase->poslist.n==0); return rc; } /* @@ -997,13 +905,14 @@ if( p->pIter ){ sqlite3Fts5IterClose(p->pIter); p->pIter = 0; } rc = sqlite3Fts5IndexQuery( - pExpr->pIndex, p->zTerm, strlen(p->zTerm), + pExpr->pIndex, p->zTerm, (int)strlen(p->zTerm), (pTerm->bPrefix ? FTS5INDEX_QUERY_PREFIX : 0) | (pExpr->bDesc ? FTS5INDEX_QUERY_DESC : 0), + pNear->pColset, &p->pIter ); assert( rc==SQLITE_OK || p->pIter==0 ); if( p->pIter && 0==sqlite3Fts5IterEof(p->pIter) ){ bEof = 0; @@ -1607,11 +1516,11 @@ rc = fts5ParseStringFromToken(pToken, &z); if( rc==SQLITE_OK ){ int flags = FTS5_TOKENIZE_QUERY | (bPrefix ? FTS5_TOKENIZE_QUERY : 0); int n; sqlite3Fts5Dequote(z); - n = strlen(z); + n = (int)strlen(z); rc = sqlite3Fts5Tokenize(pConfig, flags, z, n, &sCtx, fts5ParseTokenize); } sqlite3_free(z); if( rc || (rc = sCtx.rc) ){ pParse->rc = rc; @@ -1680,11 +1589,12 @@ for(i=0; rc==SQLITE_OK && inTerm; i++){ int tflags = 0; Fts5ExprTerm *p; for(p=&pOrig->aTerm[i]; p && rc==SQLITE_OK; p=p->pSynonym){ const char *zTerm = p->zTerm; - rc = fts5ParseTokenize((void*)&sCtx, tflags, zTerm, strlen(zTerm), 0, 0); + rc = fts5ParseTokenize((void*)&sCtx, tflags, zTerm, (int)strlen(zTerm), + 0, 0); tflags = FTS5_TOKEN_COLOCATED; } if( rc==SQLITE_OK ){ sCtx.pPhrase->aTerm[i].bPrefix = pOrig->aTerm[i].bPrefix; } @@ -1752,29 +1662,29 @@ pNear->nNear = nNear; } /* ** The second argument passed to this function may be NULL, or it may be -** an existing Fts5ExprColset object. This function returns a pointer to +** an existing Fts5Colset object. This function returns a pointer to ** a new colset object containing the contents of (p) with new value column ** number iCol appended. ** ** If an OOM error occurs, store an error code in pParse and return NULL. ** The old colset object (if any) is not freed in this case. */ -static Fts5ExprColset *fts5ParseColset( +static Fts5Colset *fts5ParseColset( Fts5Parse *pParse, /* Store SQLITE_NOMEM here if required */ - Fts5ExprColset *p, /* Existing colset object */ + Fts5Colset *p, /* Existing colset object */ int iCol /* New column to add to colset object */ ){ int nCol = p ? p->nCol : 0; /* Num. columns already in colset object */ - Fts5ExprColset *pNew; /* New colset object to return */ + Fts5Colset *pNew; /* New colset object to return */ assert( pParse->rc==SQLITE_OK ); assert( iCol>=0 && iColpConfig->nCol ); - pNew = sqlite3_realloc(p, sizeof(Fts5ExprColset) + sizeof(int)*nCol); + pNew = sqlite3_realloc(p, sizeof(Fts5Colset) + sizeof(int)*nCol); if( pNew==0 ){ pParse->rc = SQLITE_NOMEM; }else{ int *aiCol = pNew->aiCol; int i, j; @@ -1795,16 +1705,16 @@ } return pNew; } -Fts5ExprColset *sqlite3Fts5ParseColset( +Fts5Colset *sqlite3Fts5ParseColset( Fts5Parse *pParse, /* Store SQLITE_NOMEM here if required */ - Fts5ExprColset *pColset, /* Existing colset object */ + Fts5Colset *pColset, /* Existing colset object */ Fts5Token *p ){ - Fts5ExprColset *pRet = 0; + Fts5Colset *pRet = 0; int iCol; char *z; /* Dequoted copy of token p */ z = sqlite3Fts5Strndup(&pParse->rc, p->p, p->n); if( pParse->rc==SQLITE_OK ){ @@ -1830,11 +1740,11 @@ } void sqlite3Fts5ParseSetColset( Fts5Parse *pParse, Fts5ExprNearset *pNear, - Fts5ExprColset *pColset + Fts5Colset *pColset ){ if( pNear ){ pNear->pColset = pColset; }else{ sqlite3_free(pColset); @@ -1922,11 +1832,11 @@ Fts5ExprTerm *p; char *zQuoted; /* Determine the maximum amount of space required. */ for(p=pTerm; p; p=p->pSynonym){ - nByte += strlen(pTerm->zTerm) * 2 + 3 + 2; + nByte += (int)strlen(pTerm->zTerm) * 2 + 3 + 2; } zQuoted = sqlite3_malloc(nByte); if( zQuoted ){ int i = 0; @@ -2282,10 +2192,15 @@ for(i=0; rc==SQLITE_OK && i<(sizeof(aFunc) / sizeof(aFunc[0])); i++){ struct Fts5ExprFunc *p = &aFunc[i]; rc = sqlite3_create_function(db, p->z, -1, SQLITE_UTF8, pCtx, p->x, 0, 0); } + + /* Avoid a warning indicating that sqlite3Fts5ParserTrace() is unused */ +#ifndef NDEBUG + (void)sqlite3Fts5ParserTrace; +#endif return rc; } /* @@ -2318,6 +2233,5 @@ *pa = 0; nRet = 0; } return nRet; } - Index: ext/fts5/fts5_hash.c ================================================================== --- ext/fts5/fts5_hash.c +++ ext/fts5/fts5_hash.c @@ -168,11 +168,11 @@ for(i=0; inSlot; i++){ while( apOld[i] ){ int iHash; Fts5HashEntry *p = apOld[i]; apOld[i] = p->pHashNext; - iHash = fts5HashKey(nNew, (u8*)p->zKey, strlen(p->zKey)); + iHash = fts5HashKey(nNew, (u8*)p->zKey, (int)strlen(p->zKey)); p->pHashNext = apNew[iHash]; apNew[iHash] = p; } } @@ -456,11 +456,11 @@ const u8 **ppDoclist, /* OUT: pointer to doclist */ int *pnDoclist /* OUT: size of doclist in bytes */ ){ Fts5HashEntry *p; if( (p = pHash->pScan) ){ - int nTerm = strlen(p->zKey); + int nTerm = (int)strlen(p->zKey); fts5HashAddPoslistSize(p); *pzTerm = p->zKey; *ppDoclist = (const u8*)&p->zKey[nTerm+1]; *pnDoclist = p->nData - (FTS5_HASHENTRYSIZE + nTerm + 1); }else{ Index: ext/fts5/fts5_index.c ================================================================== --- ext/fts5/fts5_index.c +++ ext/fts5/fts5_index.c @@ -289,11 +289,11 @@ */ Fts5Hash *pHash; /* Hash table for in-memory data */ int nMaxPendingData; /* Max pending data before flush to disk */ int nPendingData; /* Current bytes of pending data */ i64 iWriteRowid; /* Rowid for current doc being written */ - Fts5Buffer scratch; + int bDelete; /* Current write is a delete */ /* Error state. */ int rc; /* Current error code */ /* State used by the fts5DataXXX() functions. */ @@ -305,18 +305,17 @@ sqlite3_stmt *pIdxSelect; int nRead; /* Total number of blocks read */ }; struct Fts5DoclistIter { - u8 *a; - int n; - int i; + u8 *aEof; /* Pointer to 1 byte past end of doclist */ /* Output variables. aPoslist==0 at EOF */ i64 iRowid; u8 *aPoslist; int nPoslist; + int nSize; }; /* ** The contents of the "structure" record for each index are represented ** using an Fts5Structure record in memory. Which uses instances of the @@ -510,12 +509,13 @@ Fts5Structure *pStruct; /* Database structure for this iterator */ Fts5Buffer poslist; /* Buffer containing current poslist */ int nSeg; /* Size of aSeg[] array */ int bRev; /* True to iterate in reverse order */ - int bSkipEmpty; /* True to skip deleted entries */ - int bEof; /* True at EOF */ + u8 bSkipEmpty; /* True to skip deleted entries */ + u8 bEof; /* True at EOF */ + u8 bFiltered; /* True if column-filter already applied */ i64 iSwitchRowid; /* Firstest rowid of other than aFirst[1] */ Fts5CResult *aFirst; /* Current merge state (see above) */ Fts5SegIter aSeg[1]; /* Array of segment iterators */ }; @@ -1456,11 +1456,12 @@ ** read. Before returning, set *pnSz to the number of bytes in the position ** list, and *pbDel to true if the delete flag is set, or false otherwise. */ static int fts5GetPoslistSize(const u8 *p, int *pnSz, int *pbDel){ int nSz; - int n = fts5GetVarint32(p, nSz); + int n = 0; + fts5FastGetVarint32(p, n, nSz); assert_nc( nSz>=0 ); *pnSz = nSz/2; *pbDel = nSz & 0x0001; return n; } @@ -1477,17 +1478,16 @@ ** position list content (if any). */ static void fts5SegIterLoadNPos(Fts5Index *p, Fts5SegIter *pIter){ if( p->rc==SQLITE_OK ){ int iOff = pIter->iLeafOffset; /* Offset to read at */ + int nSz; ASSERT_SZLEAF_OK(pIter->pLeaf); - if( iOff>=pIter->pLeaf->szLeaf ){ - p->rc = FTS5_CORRUPT; - }else{ - const u8 *a = &pIter->pLeaf->p[iOff]; - pIter->iLeafOffset += fts5GetPoslistSize(a, &pIter->nPos, &pIter->bDel); - } + fts5FastGetVarint32(pIter->pLeaf->p, iOff, nSz); + pIter->bDel = (nSz & 0x0001); + pIter->nPos = nSz>>1; + pIter->iLeafOffset = iOff; } } static void fts5SegIterLoadRowid(Fts5Index *p, Fts5SegIter *pIter){ u8 *a = pIter->pLeaf->p; /* Buffer to read data from */ @@ -1764,10 +1764,11 @@ }else if( pIter->pSeg==0 ){ const u8 *pList = 0; const char *zTerm = 0; int nList = 0; + assert( (pIter->flags & FTS5_SEGITER_ONETERM) || pbNewTerm ); if( 0==(pIter->flags & FTS5_SEGITER_ONETERM) ){ sqlite3Fts5HashScanNext(p->pHash); sqlite3Fts5HashScanEntry(p->pHash, &zTerm, &pList, &nList); } if( pList==0 ){ @@ -1776,12 +1777,14 @@ }else{ pIter->pLeaf->p = (u8*)pList; pIter->pLeaf->nn = nList; pIter->pLeaf->szLeaf = nList; pIter->iEndofDoclist = nList+1; - sqlite3Fts5BufferSet(&p->rc, &pIter->term, strlen(zTerm), (u8*)zTerm); + sqlite3Fts5BufferSet(&p->rc, &pIter->term, (int)strlen(zTerm), + (u8*)zTerm); pIter->iLeafOffset = fts5GetVarint(pList, (u64*)&pIter->iRowid); + *pbNewTerm = 1; } }else{ iOff = 0; /* Next entry is not on the current page */ while( iOff==0 ){ @@ -1938,18 +1941,10 @@ } pIter->pDlidx = fts5DlidxIterInit(p, bRev, iSeg, pIter->iTermLeafPgno); } -#define fts5IndexGetVarint32(a, iOff, nVal) { \ - nVal = (a)[iOff++]; \ - if( nVal & 0x80 ){ \ - iOff--; \ - iOff += fts5GetVarint32(&(a)[iOff], nVal); \ - } \ -} - #define fts5IndexSkipVarint(a, iOff) { \ int iEnd = iOff+9; \ while( (a[iOff++] & 0x80) && iOff=nMatch ); @@ -2028,11 +2023,11 @@ iPgidx += fts5GetVarint32(&a[iPgidx], nKeep); iTermOff += nKeep; iOff = iTermOff; /* Read the nKeep field of the next term. */ - fts5IndexGetVarint32(a, iOff, nKeep); + fts5FastGetVarint32(a, iOff, nKeep); } search_failed: if( bGe==0 ){ fts5DataRelease(pIter->pLeaf); @@ -2190,11 +2185,11 @@ assert( p->rc==SQLITE_OK ); if( pTerm==0 || (flags & FTS5INDEX_QUERY_SCAN) ){ p->rc = sqlite3Fts5HashScanInit(p->pHash, (const char*)pTerm, nTerm); sqlite3Fts5HashScanEntry(p->pHash, (const char**)&z, &pList, &nList); - n = (z ? strlen((const char*)z) : 0); + n = (z ? (int)strlen((const char*)z) : 0); }else{ pIter->flags |= FTS5_SEGITER_ONETERM; sqlite3Fts5HashQuery(p->pHash, (const char*)pTerm, nTerm, &pList, &nList); z = pTerm; n = nTerm; @@ -2731,10 +2726,11 @@ Fts5IndexIter *pNew; pNew = fts5MultiIterAlloc(p, 2); if( pNew ){ Fts5SegIter *pIter = &pNew->aSeg[1]; + pNew->bFiltered = 1; pIter->flags = FTS5_SEGITER_ONETERM; if( pData->szLeaf>0 ){ pIter->pLeaf = pData; pIter->iLeafOffset = fts5GetVarint(pData->p, (u64*)&pIter->iRowid); pIter->iEndofDoclist = pData->nn; @@ -3703,14 +3699,19 @@ } } return ret; } -#define fts5BufferSafeAppendBlob(pBuf, pBlob, nBlob) { \ - assert( pBuf->nSpace>=(pBuf->n+nBlob) ); \ - memcpy(&pBuf->p[pBuf->n], pBlob, nBlob); \ - pBuf->n += nBlob; \ +#define fts5BufferSafeAppendBlob(pBuf, pBlob, nBlob) { \ + assert( (pBuf)->nSpace>=((pBuf)->n+nBlob) ); \ + memcpy(&(pBuf)->p[(pBuf)->n], pBlob, nBlob); \ + (pBuf)->n += nBlob; \ +} + +#define fts5BufferSafeAppendVarint(pBuf, iVal) { \ + (pBuf)->n += sqlite3Fts5PutVarint(&(pBuf)->p[(pBuf)->n], (iVal)); \ + assert( (pBuf)->nSpace>=(pBuf)->n ); \ } /* ** Flush the contents of in-memory hash table iHash to a new level-0 ** segment on disk. Also update the corresponding structure record. @@ -3757,11 +3758,11 @@ const u8 *pDoclist; /* Pointer to doclist for this term */ int nDoclist; /* Size of doclist in bytes */ /* Write the term for this entry to disk. */ sqlite3Fts5HashScanEntry(pHash, &zTerm, &pDoclist, &nDoclist); - fts5WriteAppendTerm(p, &writer, strlen(zTerm), (const u8*)zTerm); + fts5WriteAppendTerm(p, &writer, (int)strlen(zTerm), (const u8*)zTerm); assert( writer.bFirstRowidInPage==0 ); if( pgsz>=(pBuf->n + pPgidx->n + nDoclist + 1) ){ /* The entire doclist will fit on the current leaf. */ fts5BufferSafeAppendBlob(pBuf, pDoclist, nDoclist); @@ -3936,16 +3937,85 @@ return fts5IndexReturn(p); } static void fts5PoslistCallback( Fts5Index *p, - void *pCtx, + void *pContext, + const u8 *pChunk, int nChunk +){ + assert_nc( nChunk>=0 ); + if( nChunk>0 ){ + fts5BufferSafeAppendBlob((Fts5Buffer*)pContext, pChunk, nChunk); + } +} + +typedef struct PoslistCallbackCtx PoslistCallbackCtx; +struct PoslistCallbackCtx { + Fts5Buffer *pBuf; /* Append to this buffer */ + Fts5Colset *pColset; /* Restrict matches to this column */ + int eState; /* See above */ +}; + +/* +** TODO: Make this more efficient! +*/ +static int fts5IndexColsetTest(Fts5Colset *pColset, int iCol){ + int i; + for(i=0; inCol; i++){ + if( pColset->aiCol[i]==iCol ) return 1; + } + return 0; +} + +static void fts5PoslistFilterCallback( + Fts5Index *p, + void *pContext, const u8 *pChunk, int nChunk ){ + PoslistCallbackCtx *pCtx = (PoslistCallbackCtx*)pContext; assert_nc( nChunk>=0 ); if( nChunk>0 ){ - fts5BufferAppendBlob(&p->rc, (Fts5Buffer*)pCtx, nChunk, pChunk); + /* Search through to find the first varint with value 1. This is the + ** start of the next columns hits. */ + int i = 0; + int iStart = 0; + + if( pCtx->eState==2 ){ + int iCol; + fts5FastGetVarint32(pChunk, i, iCol); + if( fts5IndexColsetTest(pCtx->pColset, iCol) ){ + pCtx->eState = 1; + fts5BufferSafeAppendVarint(pCtx->pBuf, 1); + }else{ + pCtx->eState = 0; + } + } + + do { + while( ieState ){ + fts5BufferSafeAppendBlob(pCtx->pBuf, &pChunk[iStart], i-iStart); + } + if( i=nChunk ){ + pCtx->eState = 2; + }else{ + fts5FastGetVarint32(pChunk, i, iCol); + pCtx->eState = fts5IndexColsetTest(pCtx->pColset, iCol); + if( pCtx->eState ){ + fts5BufferSafeAppendBlob(pCtx->pBuf, &pChunk[iStart], i-iStart); + iStart = i; + } + } + } + }while( irc, pBuf, pSeg->nPos) ){ + if( pColset==0 ){ + fts5ChunkIterate(p, pSeg, (void*)pBuf, fts5PoslistCallback); + }else{ + PoslistCallbackCtx sCtx; + sCtx.pBuf = pBuf; + sCtx.pColset = pColset; + sCtx.eState = fts5IndexColsetTest(pColset, 0); + assert( sCtx.eState==0 || sCtx.eState==1 ); + fts5ChunkIterate(p, pSeg, (void*)&sCtx, fts5PoslistFilterCallback); + } + } +} + +/* +** IN/OUT parameter (*pa) points to a position list n bytes in size. If +** the position list contains entries for column iCol, then (*pa) is set +** to point to the sub-position-list for that column and the number of +** bytes in it returned. Or, if the argument position list does not +** contain any entries for column iCol, return 0. +*/ +static int fts5IndexExtractCol( + const u8 **pa, /* IN/OUT: Pointer to poslist */ + int n, /* IN: Size of poslist in bytes */ + int iCol /* Column to extract from poslist */ +){ + int iCurrent = 0; /* Anything before the first 0x01 is col 0 */ + const u8 *p = *pa; + const u8 *pEnd = &p[n]; /* One byte past end of position list */ + u8 prev = 0; + + while( iCol!=iCurrent ){ + /* Advance pointer p until it points to pEnd or an 0x01 byte that is + ** not part of a varint */ + while( (prev & 0x80) || *p!=0x01 ){ + prev = *p++; + if( p==pEnd ) return 0; + } + *pa = p++; + p += fts5GetVarint32(p, iCurrent); + } + + /* Advance pointer p until it points to pEnd or an 0x01 byte that is + ** not part of a varint */ + assert( (prev & 0x80)==0 ); + while( prc. It is assumed -** no error has already occurred when this function is called. +** If an error occurs, an error code is left in p->rc. */ -static void fts5MultiIterPoslist( +static int fts5AppendPoslist( Fts5Index *p, + i64 iDelta, Fts5IndexIter *pMulti, - int bSz, /* Append a size field before the data */ + Fts5Colset *pColset, Fts5Buffer *pBuf ){ if( p->rc==SQLITE_OK ){ Fts5SegIter *pSeg = &pMulti->aSeg[ pMulti->aFirst[1].iFirst ]; assert( fts5MultiIterEof(p, pMulti)==0 ); + assert( pSeg->nPos>0 ); + if( 0==fts5BufferGrow(&p->rc, pBuf, pSeg->nPos+9+9) ){ + int iSv1; + int iSv2; + int iData; - if( bSz ){ + /* Append iDelta */ + iSv1 = pBuf->n; + fts5BufferSafeAppendVarint(pBuf, iDelta); + /* WRITEPOSLISTSIZE */ - fts5BufferAppendVarint(&p->rc, pBuf, pSeg->nPos*2); + iSv2 = pBuf->n; + fts5BufferSafeAppendVarint(pBuf, pSeg->nPos*2); + iData = pBuf->n; + + if( pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf + && (pColset==0 || pColset->nCol==1) + ){ + const u8 *pPos = &pSeg->pLeaf->p[pSeg->iLeafOffset]; + int nPos; + if( pColset ){ + nPos = fts5IndexExtractCol(&pPos, pSeg->nPos, pColset->aiCol[0]); + }else{ + nPos = pSeg->nPos; + } + fts5BufferSafeAppendBlob(pBuf, pPos, nPos); + }else{ + fts5SegiterPoslist(p, pSeg, pColset, pBuf); + } + + if( pColset ){ + int nActual = pBuf->n - iData; + if( nActual!=pSeg->nPos ){ + if( nActual==0 ){ + pBuf->n = iSv1; + return 1; + }else{ + int nReq = sqlite3Fts5GetVarintLen((u32)(nActual*2)); + while( iSv2<(iData-nReq) ){ pBuf->p[iSv2++] = 0x80; } + sqlite3Fts5PutVarint(&pBuf->p[iSv2], nActual*2); + } + } + } } - fts5SegiterPoslist(p, pSeg, pBuf); } + + return 0; } static void fts5DoclistIterNext(Fts5DoclistIter *pIter){ - if( pIter->in ){ - int bDummy; - if( pIter->i ){ - i64 iDelta; - pIter->i += fts5GetVarint(&pIter->a[pIter->i], (u64*)&iDelta); - pIter->iRowid += iDelta; - }else{ - pIter->i += fts5GetVarint(&pIter->a[pIter->i], (u64*)&pIter->iRowid); - } - pIter->i += fts5GetPoslistSize( - &pIter->a[pIter->i], &pIter->nPoslist, &bDummy - ); - pIter->aPoslist = &pIter->a[pIter->i]; - pIter->i += pIter->nPoslist; - }else{ - pIter->aPoslist = 0; + u8 *p = pIter->aPoslist + pIter->nSize + pIter->nPoslist; + + assert( pIter->aPoslist ); + if( p>=pIter->aEof ){ + pIter->aPoslist = 0; + }else{ + i64 iDelta; + + p += fts5GetVarint(p, (u64*)&iDelta); + pIter->iRowid += iDelta; + + /* Read position list size */ + if( p[0] & 0x80 ){ + int nPos; + pIter->nSize = fts5GetVarint32(p, nPos); + pIter->nPoslist = (nPos>>1); + }else{ + pIter->nPoslist = ((int)(p[0])) >> 1; + pIter->nSize = 1; + } + + pIter->aPoslist = p; } } static void fts5DoclistIterInit( Fts5Buffer *pBuf, Fts5DoclistIter *pIter ){ memset(pIter, 0, sizeof(*pIter)); - pIter->a = pBuf->p; - pIter->n = pBuf->n; + pIter->aPoslist = pBuf->p; + pIter->aEof = &pBuf->p[pBuf->n]; fts5DoclistIterNext(pIter); } +#if 0 /* ** Append a doclist to buffer pBuf. +** +** This function assumes that space within the buffer has already been +** allocated. */ static void fts5MergeAppendDocid( - int *pRc, /* IN/OUT: Error code */ Fts5Buffer *pBuf, /* Buffer to write to */ i64 *piLastRowid, /* IN/OUT: Previous rowid written (if any) */ i64 iRowid /* Rowid to append */ ){ - if( pBuf->n==0 ){ - fts5BufferAppendVarint(pRc, pBuf, iRowid); - }else{ - fts5BufferAppendVarint(pRc, pBuf, iRowid - *piLastRowid); - } + assert( pBuf->n!=0 || (*piLastRowid)==0 ); + fts5BufferSafeAppendVarint(pBuf, iRowid - *piLastRowid); *piLastRowid = iRowid; } +#endif + +#define fts5MergeAppendDocid(pBuf, iLastRowid, iRowid) { \ + assert( (pBuf)->n!=0 || (iLastRowid)==0 ); \ + fts5BufferSafeAppendVarint((pBuf), (iRowid) - (iLastRowid)); \ + (iLastRowid) = (iRowid); \ +} /* ** Buffers p1 and p2 contain doclists. This function merges the content ** of the two doclists together and sets buffer p1 to the result before ** returning. @@ -4054,57 +4234,62 @@ Fts5Buffer out; Fts5Buffer tmp; memset(&out, 0, sizeof(out)); memset(&tmp, 0, sizeof(tmp)); + sqlite3Fts5BufferGrow(&p->rc, &out, p1->n + p2->n); fts5DoclistIterInit(p1, &i1); fts5DoclistIterInit(p2, &i2); while( p->rc==SQLITE_OK && (i1.aPoslist!=0 || i2.aPoslist!=0) ){ if( i2.aPoslist==0 || (i1.aPoslist && i1.iRowidrc, &out, &iLastRowid, i1.iRowid); - /* WRITEPOSLISTSIZE */ - fts5BufferAppendVarint(&p->rc, &out, i1.nPoslist * 2); - fts5BufferAppendBlob(&p->rc, &out, i1.nPoslist, i1.aPoslist); + fts5MergeAppendDocid(&out, iLastRowid, i1.iRowid); + fts5BufferSafeAppendBlob(&out, i1.aPoslist, i1.nPoslist+i1.nSize); fts5DoclistIterNext(&i1); } else if( i1.aPoslist==0 || i2.iRowid!=i1.iRowid ){ /* Copy entry from i2 */ - fts5MergeAppendDocid(&p->rc, &out, &iLastRowid, i2.iRowid); - /* WRITEPOSLISTSIZE */ - fts5BufferAppendVarint(&p->rc, &out, i2.nPoslist * 2); - fts5BufferAppendBlob(&p->rc, &out, i2.nPoslist, i2.aPoslist); + fts5MergeAppendDocid(&out, iLastRowid, i2.iRowid); + fts5BufferSafeAppendBlob(&out, i2.aPoslist, i2.nPoslist+i2.nSize); fts5DoclistIterNext(&i2); } else{ - Fts5PoslistReader r1; - Fts5PoslistReader r2; + i64 iPos1 = 0; + i64 iPos2 = 0; + int iOff1 = 0; + int iOff2 = 0; + u8 *a1 = &i1.aPoslist[i1.nSize]; + u8 *a2 = &i2.aPoslist[i2.nSize]; + Fts5PoslistWriter writer; - memset(&writer, 0, sizeof(writer)); /* Merge the two position lists. */ - fts5MergeAppendDocid(&p->rc, &out, &iLastRowid, i2.iRowid); + fts5MergeAppendDocid(&out, iLastRowid, i2.iRowid); fts5BufferZero(&tmp); - sqlite3Fts5PoslistReaderInit(-1, i1.aPoslist, i1.nPoslist, &r1); - sqlite3Fts5PoslistReaderInit(-1, i2.aPoslist, i2.nPoslist, &r2); - while( p->rc==SQLITE_OK && (r1.bEof==0 || r2.bEof==0) ){ + + sqlite3Fts5PoslistNext64(a1, i1.nPoslist, &iOff1, &iPos1); + sqlite3Fts5PoslistNext64(a2, i2.nPoslist, &iOff2, &iPos2); + + while( p->rc==SQLITE_OK && (iPos1>=0 || iPos2>=0) ){ i64 iNew; - if( r2.bEof || (r1.bEof==0 && r1.iPos=0 && iPos1rc = sqlite3Fts5PoslistWriterAppend(&tmp, &writer, iNew); } /* WRITEPOSLISTSIZE */ - fts5BufferAppendVarint(&p->rc, &out, tmp.n * 2); - fts5BufferAppendBlob(&p->rc, &out, tmp.n, tmp.p); + fts5BufferSafeAppendVarint(&out, tmp.n * 2); + fts5BufferSafeAppendBlob(&out, tmp.p, tmp.n); fts5DoclistIterNext(&i1); fts5DoclistIterNext(&i2); } } @@ -4123,11 +4308,12 @@ static void fts5SetupPrefixIter( Fts5Index *p, /* Index to read from */ int bDesc, /* True for "ORDER BY rowid DESC" */ const u8 *pToken, /* Buffer containing prefix to match */ int nToken, /* Size of buffer pToken in bytes */ - Fts5IndexIter **ppIter /* OUT: New iterator */ + Fts5Colset *pColset, /* Restrict matches to these columns */ + Fts5IndexIter **ppIter /* OUT: New iterator */ ){ Fts5Structure *pStruct; Fts5Buffer *aBuf; const int nBuf = 32; @@ -4162,18 +4348,22 @@ }else{ fts5MergePrefixLists(p, &doclist, &aBuf[i]); fts5BufferZero(&aBuf[i]); } } + iLastRowid = 0; } - fts5MergeAppendDocid(&p->rc, &doclist, &iLastRowid, iRowid); - fts5MultiIterPoslist(p, p1, 1, &doclist); + if( !fts5AppendPoslist(p, iRowid-iLastRowid, p1, pColset, &doclist) ){ + iLastRowid = iRowid; + } } for(i=0; irc==SQLITE_OK ){ + fts5MergePrefixLists(p, &doclist, &aBuf[i]); + } fts5BufferFree(&aBuf[i]); } fts5MultiIterFree(p, p1); pData = fts5IdxMalloc(p, sizeof(Fts5Data) + doclist.n); @@ -4193,23 +4383,28 @@ /* ** Indicate that all subsequent calls to sqlite3Fts5IndexWrite() pertain ** to the document with rowid iRowid. */ -int sqlite3Fts5IndexBeginWrite(Fts5Index *p, i64 iRowid){ +int sqlite3Fts5IndexBeginWrite(Fts5Index *p, int bDelete, i64 iRowid){ assert( p->rc==SQLITE_OK ); /* Allocate the hash table if it has not already been allocated */ if( p->pHash==0 ){ p->rc = sqlite3Fts5HashNew(&p->pHash, &p->nPendingData); } /* Flush the hash table to disk if required */ - if( iRowid<=p->iWriteRowid || (p->nPendingData > p->nMaxPendingData) ){ + if( iRowidiWriteRowid + || (iRowid==p->iWriteRowid && p->bDelete==0) + || (p->nPendingData > p->nMaxPendingData) + ){ fts5IndexFlush(p); } + p->iWriteRowid = iRowid; + p->bDelete = bDelete; return fts5IndexReturn(p); } /* ** Commit data to disk. @@ -4304,11 +4499,10 @@ sqlite3_finalize(p->pDeleter); sqlite3_finalize(p->pIdxWriter); sqlite3_finalize(p->pIdxDeleter); sqlite3_finalize(p->pIdxSelect); sqlite3Fts5HashFree(p->pHash); - sqlite3Fts5BufferFree(&p->scratch); sqlite3_free(p->zDataTbl); sqlite3_free(p); } return rc; } @@ -4365,10 +4559,11 @@ int i; /* Used to iterate through indexes */ int rc = SQLITE_OK; /* Return code */ Fts5Config *pConfig = p->pConfig; assert( p->rc==SQLITE_OK ); + assert( (iCol<0)==p->bDelete ); /* Add the entry to the main terms index. */ rc = sqlite3Fts5HashWrite( p->pHash, p->iWriteRowid, iCol, iPos, FTS5_MAIN_PREFIX, pToken, nToken ); @@ -4391,10 +4586,11 @@ */ int sqlite3Fts5IndexQuery( Fts5Index *p, /* FTS index to query */ const char *pToken, int nToken, /* Token (or prefix) to query for */ int flags, /* Mask of FTS5INDEX_QUERY_X flags */ + Fts5Colset *pColset, /* Match these columns only */ Fts5IndexIter **ppIter /* OUT: New iterator object */ ){ Fts5Config *pConfig = p->pConfig; Fts5IndexIter *pRet = 0; int iIdx = 0; @@ -4434,11 +4630,11 @@ fts5StructureRelease(pStruct); } }else{ int bDesc = (flags & FTS5INDEX_QUERY_DESC)!=0; buf.p[0] = FTS5_MAIN_PREFIX; - fts5SetupPrefixIter(p, bDesc, buf.p, nToken+1, &pRet); + fts5SetupPrefixIter(p, bDesc, buf.p, nToken+1, pColset, &pRet); } if( p->rc ){ sqlite3Fts5IterClose(pRet); pRet = 0; @@ -4513,10 +4709,30 @@ const char *z = (const char*)fts5MultiIterTerm(pIter, &n); *pn = n-1; return &z[1]; } + +static int fts5IndexExtractColset ( + Fts5Colset *pColset, /* Colset to filter on */ + const u8 *pPos, int nPos, /* Position list */ + Fts5Buffer *pBuf /* Output buffer */ +){ + int rc = SQLITE_OK; + int i; + + fts5BufferZero(pBuf); + for(i=0; inCol; i++){ + const u8 *pSub = pPos; + int nSub = fts5IndexExtractCol(&pSub, nPos, pColset->aiCol[i]); + if( nSub ){ + fts5BufferAppendBlob(&rc, pBuf, nSub, pSub); + } + } + return rc; +} + /* ** Return a pointer to a buffer containing a copy of the position list for ** the current entry. Output variable *pn is set to the size of the buffer ** in bytes before returning. @@ -4524,24 +4740,37 @@ ** The returned position list does not include the "number of bytes" varint ** field that starts the position list on disk. */ int sqlite3Fts5IterPoslist( Fts5IndexIter *pIter, + Fts5Colset *pColset, /* Column filter (or NULL) */ const u8 **pp, /* OUT: Pointer to position-list data */ int *pn, /* OUT: Size of position-list in bytes */ i64 *piRowid /* OUT: Current rowid */ ){ Fts5SegIter *pSeg = &pIter->aSeg[ pIter->aFirst[1].iFirst ]; assert( pIter->pIndex->rc==SQLITE_OK ); *piRowid = pSeg->iRowid; - *pn = pSeg->nPos; - if( pSeg->iLeafOffset+pSeg->nPos <= pSeg->pLeaf->szLeaf ){ - *pp = &pSeg->pLeaf->p[pSeg->iLeafOffset]; + if( pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf ){ + u8 *pPos = &pSeg->pLeaf->p[pSeg->iLeafOffset]; + if( pColset==0 || pIter->bFiltered ){ + *pn = pSeg->nPos; + *pp = pPos; + }else if( pColset->nCol==1 ){ + *pp = pPos; + *pn = fts5IndexExtractCol(pp, pSeg->nPos, pColset->aiCol[0]); + }else{ + fts5BufferZero(&pIter->poslist); + fts5IndexExtractColset(pColset, pPos, pSeg->nPos, &pIter->poslist); + *pp = pIter->poslist.p; + *pn = pIter->poslist.n; + } }else{ fts5BufferZero(&pIter->poslist); - fts5SegiterPoslist(pIter->pIndex, pSeg, &pIter->poslist); + fts5SegiterPoslist(pIter->pIndex, pSeg, pColset, &pIter->poslist); *pp = pIter->poslist.p; + *pn = pIter->poslist.n; } return fts5IndexReturn(pIter->pIndex); } /* @@ -4549,14 +4778,14 @@ ** copies the position list into the buffer supplied as the second ** argument. */ int sqlite3Fts5IterPoslistBuffer(Fts5IndexIter *pIter, Fts5Buffer *pBuf){ Fts5Index *p = pIter->pIndex; - + Fts5SegIter *pSeg = &pIter->aSeg[ pIter->aFirst[1].iFirst ]; assert( p->rc==SQLITE_OK ); fts5BufferZero(pBuf); - fts5MultiIterPoslist(p, pIter, 0, pBuf); + fts5SegiterPoslist(p, pSeg, 0, pBuf); return fts5IndexReturn(p); } /* ** Close an iterator opened by an earlier call to sqlite3Fts5IndexQuery(). @@ -4727,21 +4956,21 @@ int flags, /* Flags for Fts5IndexQuery */ u64 *pCksum /* IN/OUT: Checksum value */ ){ u64 cksum = *pCksum; Fts5IndexIter *pIdxIter = 0; - int rc = sqlite3Fts5IndexQuery(p, z, n, flags, &pIdxIter); + int rc = sqlite3Fts5IndexQuery(p, z, n, flags, 0, &pIdxIter); while( rc==SQLITE_OK && 0==sqlite3Fts5IterEof(pIdxIter) ){ i64 dummy; const u8 *pPos; int nPos; i64 rowid = sqlite3Fts5IterRowid(pIdxIter); - rc = sqlite3Fts5IterPoslist(pIdxIter, &pPos, &nPos, &dummy); + rc = sqlite3Fts5IterPoslist(pIdxIter, 0, &pPos, &nPos, &dummy); if( rc==SQLITE_OK ){ Fts5PoslistReader sReader; - for(sqlite3Fts5PoslistReaderInit(-1, pPos, nPos, &sReader); + for(sqlite3Fts5PoslistReaderInit(pPos, nPos, &sReader); sReader.bEof==0; sqlite3Fts5PoslistReaderNext(&sReader) ){ int iCol = FTS5_POS2COLUMN(sReader.iPos); int iOff = FTS5_POS2OFFSET(sReader.iPos); @@ -4967,11 +5196,10 @@ fts5IntegrityCheckPgidx(p, pLeaf); } fts5DataRelease(pLeaf); if( p->rc ) break; - /* Now check that the iter.nEmpty leaves following the current leaf ** (a) exist and (b) contain no terms. */ fts5IndexIntegrityCheckEmpty( p, pSeg, iIdxPrevLeaf+1, iDlidxPrevLeaf+1, iIdxLeaf-1 ); @@ -5101,11 +5329,11 @@ /* If this is a new term, query for it. Update cksum3 with the results. */ fts5TestTerm(p, &term, z, n, cksum2, &cksum3); poslist.n = 0; - fts5MultiIterPoslist(p, pIter, 0, &poslist); + fts5SegiterPoslist(p, &pIter->aSeg[pIter->aFirst[1].iFirst] , 0, &poslist); while( 0==sqlite3Fts5PoslistNext64(poslist.p, poslist.n, &iOff, &iPos) ){ int iCol = FTS5_POS2COLUMN(iPos); int iTokOff = FTS5_POS2OFFSET(iPos); cksum2 ^= fts5IndexEntryCksum(iRowid, iCol, iTokOff, -1, z, n); } @@ -5240,10 +5468,33 @@ } fts5DebugStructure(pRc, pBuf, p); fts5StructureRelease(p); } + +/* +** This is part of the fts5_decode() debugging aid. +** +** Arguments pBlob/nBlob contain an "averages" record. This function +** appends a human-readable representation of record to the buffer passed +** as the second argument. +*/ +static void fts5DecodeAverages( + int *pRc, /* IN/OUT: error code */ + Fts5Buffer *pBuf, + const u8 *pBlob, int nBlob +){ + int i = 0; + const char *zSpace = ""; + + while( i MATCH ? ORDER BY rank) */ #define FTS5_PLAN_SCAN 5 /* No usable constraint */ #define FTS5_PLAN_ROWID 6 /* (rowid = ?) */ + +/* +** Set the SQLITE_INDEX_SCAN_UNIQUE flag in pIdxInfo->flags. Unless this +** extension is currently being used by a version of SQLite too old to +** support index-info flags. In that case this function is a no-op. +*/ +static void fts5SetUniqueFlag(sqlite3_index_info *pIdxInfo){ +#if SQLITE_VERSION_NUMBER>=3008012 +#ifndef SQLITE_CORE + if( sqlite3_libversion_number()>=3008012 ) +#endif + { + pIdxInfo->idxFlags |= SQLITE_INDEX_SCAN_UNIQUE; + } +#endif +} /* ** Implementation of the xBestIndex method for FTS5 tables. Within the ** WHERE constraint, it searches for the following: ** @@ -490,12 +506,14 @@ int fts5op; /* FTS5 mask for idxFlags */ int iCol; /* 0==rowid, 1==tbl, 2==rank */ int omit; /* True to omit this if found */ int iConsIndex; /* Index in pInfo->aConstraint[] */ } aConstraint[] = { - {SQLITE_INDEX_CONSTRAINT_MATCH, FTS5_BI_MATCH, 1, 1, -1}, - {SQLITE_INDEX_CONSTRAINT_MATCH, FTS5_BI_RANK, 2, 1, -1}, + {SQLITE_INDEX_CONSTRAINT_MATCH|SQLITE_INDEX_CONSTRAINT_EQ, + FTS5_BI_MATCH, 1, 1, -1}, + {SQLITE_INDEX_CONSTRAINT_MATCH|SQLITE_INDEX_CONSTRAINT_EQ, + FTS5_BI_RANK, 2, 1, -1}, {SQLITE_INDEX_CONSTRAINT_EQ, FTS5_BI_ROWID_EQ, 0, 0, -1}, {SQLITE_INDEX_CONSTRAINT_LT|SQLITE_INDEX_CONSTRAINT_LE, FTS5_BI_ROWID_LE, 0, 0, -1}, {SQLITE_INDEX_CONSTRAINT_GT|SQLITE_INDEX_CONSTRAINT_GE, FTS5_BI_ROWID_GE, 0, 0, -1}, @@ -544,10 +562,11 @@ /* Calculate the estimated cost based on the flags set in idxFlags. */ bHasMatch = BitFlagTest(idxFlags, FTS5_BI_MATCH); if( BitFlagTest(idxFlags, FTS5_BI_ROWID_EQ) ){ pInfo->estimatedCost = bHasMatch ? 100.0 : 10.0; + if( bHasMatch==0 ) fts5SetUniqueFlag(pInfo); }else if( BitFlagAllTest(idxFlags, FTS5_BI_ROWID_LE|FTS5_BI_ROWID_GE) ){ pInfo->estimatedCost = bHasMatch ? 500.0 : 250000.0; }else if( BitFlagTest(idxFlags, FTS5_BI_ROWID_LE|FTS5_BI_ROWID_GE) ){ pInfo->estimatedCost = bHasMatch ? 750.0 : 750000.0; }else{ @@ -1282,58 +1301,57 @@ ** INSERT Directives" section of the documentation. It should be updated if ** more commands are added to this function. */ static int fts5SpecialInsert( Fts5Table *pTab, /* Fts5 table object */ - sqlite3_value *pCmd, /* Value inserted into special column */ + const char *zCmd, /* Text inserted into table-name column */ sqlite3_value *pVal /* Value inserted into rank column */ ){ Fts5Config *pConfig = pTab->pConfig; - const char *z = (const char*)sqlite3_value_text(pCmd); int rc = SQLITE_OK; int bError = 0; - if( 0==sqlite3_stricmp("delete-all", z) ){ + if( 0==sqlite3_stricmp("delete-all", zCmd) ){ if( pConfig->eContent==FTS5_CONTENT_NORMAL ){ fts5SetVtabError(pTab, "'delete-all' may only be used with a " "contentless or external content fts5 table" ); rc = SQLITE_ERROR; }else{ rc = sqlite3Fts5StorageDeleteAll(pTab->pStorage); } - }else if( 0==sqlite3_stricmp("rebuild", z) ){ + }else if( 0==sqlite3_stricmp("rebuild", zCmd) ){ if( pConfig->eContent==FTS5_CONTENT_NONE ){ fts5SetVtabError(pTab, "'rebuild' may not be used with a contentless fts5 table" ); rc = SQLITE_ERROR; }else{ rc = sqlite3Fts5StorageRebuild(pTab->pStorage); } - }else if( 0==sqlite3_stricmp("optimize", z) ){ + }else if( 0==sqlite3_stricmp("optimize", zCmd) ){ rc = sqlite3Fts5StorageOptimize(pTab->pStorage); - }else if( 0==sqlite3_stricmp("merge", z) ){ + }else if( 0==sqlite3_stricmp("merge", zCmd) ){ int nMerge = sqlite3_value_int(pVal); rc = sqlite3Fts5StorageMerge(pTab->pStorage, nMerge); - }else if( 0==sqlite3_stricmp("integrity-check", z) ){ + }else if( 0==sqlite3_stricmp("integrity-check", zCmd) ){ rc = sqlite3Fts5StorageIntegrity(pTab->pStorage); #ifdef SQLITE_DEBUG - }else if( 0==sqlite3_stricmp("prefix-index", z) ){ + }else if( 0==sqlite3_stricmp("prefix-index", zCmd) ){ pConfig->bPrefixIndex = sqlite3_value_int(pVal); #endif }else{ rc = sqlite3Fts5IndexLoadConfig(pTab->pIndex); if( rc==SQLITE_OK ){ - rc = sqlite3Fts5ConfigSetValue(pTab->pConfig, z, pVal, &bError); + rc = sqlite3Fts5ConfigSetValue(pTab->pConfig, zCmd, pVal, &bError); } if( rc==SQLITE_OK ){ if( bError ){ rc = SQLITE_ERROR; }else{ - rc = sqlite3Fts5StorageConfigValue(pTab->pStorage, z, pVal, 0); + rc = sqlite3Fts5StorageConfigValue(pTab->pStorage, zCmd, pVal, 0); } } } return rc; } @@ -1349,15 +1367,40 @@ sqlite3_int64 iDel = sqlite3_value_int64(apVal[1]); rc = sqlite3Fts5StorageSpecialDelete(pTab->pStorage, iDel, &apVal[2]); } return rc; } + +static void fts5StorageInsert( + int *pRc, + Fts5Table *pTab, + sqlite3_value **apVal, + i64 *piRowid +){ + int rc = *pRc; + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5StorageContentInsert(pTab->pStorage, apVal, piRowid); + } + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5StorageIndexInsert(pTab->pStorage, apVal, *piRowid); + } + *pRc = rc; +} /* ** This function is the implementation of the xUpdate callback used by ** FTS3 virtual tables. It is invoked by SQLite each time a row is to be ** inserted, updated or deleted. +** +** A delete specifies a single argument - the rowid of the row to remove. +** +** Update and insert operations pass: +** +** 1. The "old" rowid, or NULL. +** 2. The "new" rowid. +** 3. Values for each of the nCol matchable columns. +** 4. Values for the two hidden columns ( and "rank"). */ static int fts5UpdateMethod( sqlite3_vtab *pVtab, /* Virtual table handle */ int nArg, /* Size of argument array */ sqlite3_value **apVal, /* Array of arguments */ @@ -1364,69 +1407,112 @@ sqlite_int64 *pRowid /* OUT: The affected (or effected) rowid */ ){ Fts5Table *pTab = (Fts5Table*)pVtab; Fts5Config *pConfig = pTab->pConfig; int eType0; /* value_type() of apVal[0] */ - int eConflict; /* ON CONFLICT for this DML */ int rc = SQLITE_OK; /* Return code */ /* A transaction must be open when this is called. */ assert( pTab->ts.eState==1 ); + assert( pVtab->zErrMsg==0 ); + assert( nArg==1 || nArg==(2+pConfig->nCol+2) ); + assert( nArg==1 + || sqlite3_value_type(apVal[1])==SQLITE_INTEGER + || sqlite3_value_type(apVal[1])==SQLITE_NULL + ); assert( pTab->pConfig->pzErrmsg==0 ); pTab->pConfig->pzErrmsg = &pTab->base.zErrMsg; - /* A delete specifies a single argument - the rowid of the row to remove. - ** Update and insert operations pass: - ** - ** 1. The "old" rowid, or NULL. - ** 2. The "new" rowid. - ** 3. Values for each of the nCol matchable columns. - ** 4. Values for the two hidden columns ( and "rank"). - */ + /* Put any active cursors into REQUIRE_SEEK state. */ + fts5TripCursors(pTab); eType0 = sqlite3_value_type(apVal[0]); - eConflict = sqlite3_vtab_on_conflict(pConfig->db); - - assert( eType0==SQLITE_INTEGER || eType0==SQLITE_NULL ); - assert( pVtab->zErrMsg==0 ); - assert( (nArg==1 && eType0==SQLITE_INTEGER) || nArg==(2+pConfig->nCol+2) ); - - fts5TripCursors(pTab); - if( eType0==SQLITE_INTEGER ){ - if( fts5IsContentless(pTab) ){ + if( eType0==SQLITE_NULL + && sqlite3_value_type(apVal[2+pConfig->nCol])!=SQLITE_NULL + ){ + /* A "special" INSERT op. These are handled separately. */ + const char *z = (const char*)sqlite3_value_text(apVal[2+pConfig->nCol]); + if( pConfig->eContent!=FTS5_CONTENT_NORMAL + && 0==sqlite3_stricmp("delete", z) + ){ + rc = fts5SpecialDelete(pTab, apVal, pRowid); + }else{ + rc = fts5SpecialInsert(pTab, z, apVal[2 + pConfig->nCol + 1]); + } + }else{ + /* A regular INSERT, UPDATE or DELETE statement. The trick here is that + ** any conflict on the rowid value must be detected before any + ** modifications are made to the database file. There are 4 cases: + ** + ** 1) DELETE + ** 2) UPDATE (rowid not modified) + ** 3) UPDATE (rowid modified) + ** 4) INSERT + ** + ** Cases 3 and 4 may violate the rowid constraint. + */ + int eConflict = sqlite3_vtab_on_conflict(pConfig->db); + + assert( eType0==SQLITE_INTEGER || eType0==SQLITE_NULL ); + assert( nArg!=1 || eType0==SQLITE_INTEGER ); + + /* Filter out attempts to run UPDATE or DELETE on contentless tables. + ** This is not suported. */ + if( eType0==SQLITE_INTEGER && fts5IsContentless(pTab) ){ pTab->base.zErrMsg = sqlite3_mprintf( "cannot %s contentless fts5 table: %s", (nArg>1 ? "UPDATE" : "DELETE from"), pConfig->zName ); rc = SQLITE_ERROR; - }else{ + } + + /* Case 1: DELETE */ + else if( nArg==1 ){ i64 iDel = sqlite3_value_int64(apVal[0]); /* Rowid to delete */ rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel); } - }else{ - sqlite3_value *pCmd = apVal[2 + pConfig->nCol]; - assert( nArg>1 ); - if( SQLITE_NULL!=sqlite3_value_type(pCmd) ){ - const char *z = (const char*)sqlite3_value_text(pCmd); - if( pConfig->eContent!=FTS5_CONTENT_NORMAL - && 0==sqlite3_stricmp("delete", z) + + /* Case 2: INSERT */ + else if( eType0!=SQLITE_INTEGER ){ + /* If this is a REPLACE, first remove the current entry (if any) */ + if( eConflict==SQLITE_REPLACE + && sqlite3_value_type(apVal[1])==SQLITE_INTEGER ){ - rc = fts5SpecialDelete(pTab, apVal, pRowid); + i64 iNew = sqlite3_value_int64(apVal[1]); /* Rowid to delete */ + rc = sqlite3Fts5StorageDelete(pTab->pStorage, iNew); + } + fts5StorageInsert(&rc, pTab, apVal, pRowid); + } + + /* Case 2: UPDATE */ + else{ + i64 iOld = sqlite3_value_int64(apVal[0]); /* Old rowid */ + i64 iNew = sqlite3_value_int64(apVal[1]); /* New rowid */ + if( iOld!=iNew ){ + if( eConflict==SQLITE_REPLACE ){ + rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5StorageDelete(pTab->pStorage, iNew); + } + fts5StorageInsert(&rc, pTab, apVal, pRowid); + }else{ + rc = sqlite3Fts5StorageContentInsert(pTab->pStorage, apVal, pRowid); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld); + } + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5StorageIndexInsert(pTab->pStorage, apVal, *pRowid); + } + } }else{ - rc = fts5SpecialInsert(pTab, pCmd, apVal[2 + pConfig->nCol + 1]); - } - goto update_method_out; - } - } - - - if( rc==SQLITE_OK && nArg>1 ){ - rc = sqlite3Fts5StorageInsert(pTab->pStorage, apVal, eConflict, pRowid); - } - - update_method_out: + rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld); + fts5StorageInsert(&rc, pTab, apVal, pRowid); + } + } + } + pTab->pConfig->pzErrmsg = 0; return rc; } /* @@ -1558,11 +1644,11 @@ /* Initialize all iterators */ for(i=0; idb, zName, -1); if( rc==SQLITE_OK ){ Fts5Auxiliary *pAux; + int nName; /* Size of zName in bytes, including \0 */ int nByte; /* Bytes of space to allocate */ - nByte = sizeof(Fts5Auxiliary) + strlen(zName) + 1; + nName = (int)strlen(zName) + 1; + nByte = sizeof(Fts5Auxiliary) + nName; pAux = (Fts5Auxiliary*)sqlite3_malloc(nByte); if( pAux ){ memset(pAux, 0, nByte); pAux->zFunc = (char*)&pAux[1]; - strcpy(pAux->zFunc, zName); + memcpy(pAux->zFunc, zName, nName); pAux->pGlobal = pGlobal; pAux->pUserData = pUserData; pAux->xFunc = xFunc; pAux->xDestroy = xDestroy; pAux->pNext = pGlobal->pAux; @@ -2165,19 +2253,21 @@ fts5_tokenizer *pTokenizer, /* Tokenizer implementation */ void(*xDestroy)(void*) /* Destructor for pUserData */ ){ Fts5Global *pGlobal = (Fts5Global*)pApi; Fts5TokenizerModule *pNew; + int nName; /* Size of zName and its \0 terminator */ int nByte; /* Bytes of space to allocate */ int rc = SQLITE_OK; - nByte = sizeof(Fts5TokenizerModule) + strlen(zName) + 1; + nName = (int)strlen(zName) + 1; + nByte = sizeof(Fts5TokenizerModule) + nName; pNew = (Fts5TokenizerModule*)sqlite3_malloc(nByte); if( pNew ){ memset(pNew, 0, nByte); pNew->zName = (char*)&pNew[1]; - strcpy(pNew->zName, zName); + memcpy(pNew->zName, zName, nName); pNew->pUserData = pUserData; pNew->x = *pTokenizer; pNew->xDestroy = xDestroy; pNew->pNext = pGlobal->pTok; pGlobal->pTok = pNew; @@ -2308,18 +2398,11 @@ ){ assert( nArg==0 ); sqlite3_result_text(pCtx, "--FTS5-SOURCE-ID--", -1, SQLITE_TRANSIENT); } -#ifdef _WIN32 -__declspec(dllexport) -#endif -int sqlite3_fts5_init( - sqlite3 *db, - char **pzErrMsg, - const sqlite3_api_routines *pApi -){ +static int fts5Init(sqlite3 *db){ static const sqlite3_module fts5Mod = { /* iVersion */ 2, /* xCreate */ fts5CreateMethod, /* xConnect */ fts5ConnectMethod, /* xBestIndex */ fts5BestIndexMethod, @@ -2345,12 +2428,10 @@ }; int rc; Fts5Global *pGlobal = 0; - SQLITE_EXTENSION_INIT2(pApi); - pGlobal = (Fts5Global*)sqlite3_malloc(sizeof(Fts5Global)); if( pGlobal==0 ){ rc = SQLITE_NOMEM; }else{ void *p = (void*)pGlobal; @@ -2378,17 +2459,45 @@ } } return rc; } +/* +** The following functions are used to register the module with SQLite. If +** this module is being built as part of the SQLite core (SQLITE_CORE is +** defined), then sqlite3_open() will call sqlite3Fts5Init() directly. +** +** Or, if this module is being built as a loadable extension, +** sqlite3Fts5Init() is omitted and the two standard entry points +** sqlite3_fts_init() and sqlite3_fts5_init() defined instead. +*/ +#ifndef SQLITE_CORE #ifdef _WIN32 __declspec(dllexport) #endif int sqlite3_fts_init( sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi ){ - return sqlite3_fts5_init(db, pzErrMsg, pApi); + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErrMsg; /* Unused parameter */ + return fts5Init(db); } - +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_fts5_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErrMsg; /* Unused parameter */ + return fts5Init(db); +} +#else +int sqlite3Fts5Init(sqlite3 *db){ + return fts5Init(db); +} +#endif Index: ext/fts5/fts5_storage.c ================================================================== --- ext/fts5/fts5_storage.c +++ ext/fts5/fts5_storage.c @@ -295,14 +295,14 @@ rc = SQLITE_NOMEM; }else{ int i; int iOff; sqlite3_snprintf(nDefn, zDefn, "id INTEGER PRIMARY KEY"); - iOff = strlen(zDefn); + iOff = (int)strlen(zDefn); for(i=0; inCol; i++){ sqlite3_snprintf(nDefn-iOff, &zDefn[iOff], ", c%d", i); - iOff += strlen(&zDefn[iOff]); + iOff += (int)strlen(&zDefn[iOff]); } rc = sqlite3Fts5CreateTable(pConfig, "content", zDefn, 0, pzErr); } sqlite3_free(zDefn); } @@ -390,11 +390,11 @@ if( sqlite3_step(pSeek)==SQLITE_ROW ){ int iCol; Fts5InsertCtx ctx; ctx.pStorage = p; ctx.iCol = -1; - rc = sqlite3Fts5IndexBeginWrite(p->pIndex, iDel); + rc = sqlite3Fts5IndexBeginWrite(p->pIndex, 1, iDel); for(iCol=1; rc==SQLITE_OK && iCol<=pConfig->nCol; iCol++){ if( pConfig->abUnindexed[iCol-1] ) continue; ctx.szCol = 0; rc = sqlite3Fts5Tokenize(pConfig, FTS5_TOKENIZE_DOCUMENT, @@ -547,11 +547,11 @@ int iCol; Fts5InsertCtx ctx; ctx.pStorage = p; ctx.iCol = -1; - rc = sqlite3Fts5IndexBeginWrite(p->pIndex, iDel); + rc = sqlite3Fts5IndexBeginWrite(p->pIndex, 1, iDel); for(iCol=0; rc==SQLITE_OK && iColnCol; iCol++){ if( pConfig->abUnindexed[iCol] ) continue; ctx.szCol = 0; rc = sqlite3Fts5Tokenize(pConfig, FTS5_TOKENIZE_DOCUMENT, @@ -637,11 +637,11 @@ while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pScan) ){ i64 iRowid = sqlite3_column_int64(pScan, 0); sqlite3Fts5BufferZero(&buf); - rc = sqlite3Fts5IndexBeginWrite(p->pIndex, iRowid); + rc = sqlite3Fts5IndexBeginWrite(p->pIndex, 0, iRowid); for(ctx.iCol=0; rc==SQLITE_OK && ctx.iColnCol; ctx.iCol++){ ctx.szCol = 0; if( pConfig->abUnindexed[ctx.iCol]==0 ){ rc = sqlite3Fts5Tokenize(pConfig, FTS5_TOKENIZE_DOCUMENT, @@ -703,62 +703,63 @@ } return rc; } /* -** Insert a new row into the FTS table. +** Insert a new row into the FTS content table. +*/ +int sqlite3Fts5StorageContentInsert( + Fts5Storage *p, + sqlite3_value **apVal, + i64 *piRowid +){ + Fts5Config *pConfig = p->pConfig; + int rc = SQLITE_OK; + + /* Insert the new row into the %_content table. */ + if( pConfig->eContent!=FTS5_CONTENT_NORMAL ){ + if( sqlite3_value_type(apVal[1])==SQLITE_INTEGER ){ + *piRowid = sqlite3_value_int64(apVal[1]); + }else{ + rc = fts5StorageNewRowid(p, piRowid); + } + }else{ + sqlite3_stmt *pInsert = 0; /* Statement to write %_content table */ + int i; /* Counter variable */ + rc = fts5StorageGetStmt(p, FTS5_STMT_INSERT_CONTENT, &pInsert, 0); + for(i=1; rc==SQLITE_OK && i<=pConfig->nCol+1; i++){ + rc = sqlite3_bind_value(pInsert, i, apVal[i]); + } + if( rc==SQLITE_OK ){ + sqlite3_step(pInsert); + rc = sqlite3_reset(pInsert); + } + *piRowid = sqlite3_last_insert_rowid(pConfig->db); + } + + return rc; +} + +/* +** Insert new entries into the FTS index and %_docsize table. */ -int sqlite3Fts5StorageInsert( - Fts5Storage *p, /* Storage module to write to */ - sqlite3_value **apVal, /* Array of values passed to xUpdate() */ - int eConflict, /* on conflict clause */ - i64 *piRowid /* OUT: rowid of new record */ +int sqlite3Fts5StorageIndexInsert( + Fts5Storage *p, + sqlite3_value **apVal, + i64 iRowid ){ Fts5Config *pConfig = p->pConfig; int rc = SQLITE_OK; /* Return code */ - sqlite3_stmt *pInsert = 0; /* Statement used to write %_content table */ - int eStmt = 0; /* Type of statement used on %_content */ - int i; /* Counter variable */ Fts5InsertCtx ctx; /* Tokenization callback context object */ Fts5Buffer buf; /* Buffer used to build up %_docsize blob */ memset(&buf, 0, sizeof(Fts5Buffer)); + ctx.pStorage = p; rc = fts5StorageLoadTotals(p, 1); - /* Insert the new row into the %_content table. */ - if( rc==SQLITE_OK ){ - if( pConfig->eContent!=FTS5_CONTENT_NORMAL ){ - if( sqlite3_value_type(apVal[1])==SQLITE_INTEGER ){ - *piRowid = sqlite3_value_int64(apVal[1]); - }else{ - rc = fts5StorageNewRowid(p, piRowid); - } - }else{ - if( eConflict==SQLITE_REPLACE ){ - eStmt = FTS5_STMT_REPLACE_CONTENT; - rc = fts5StorageDeleteFromIndex(p, sqlite3_value_int64(apVal[1])); - }else{ - eStmt = FTS5_STMT_INSERT_CONTENT; - } - if( rc==SQLITE_OK ){ - rc = fts5StorageGetStmt(p, eStmt, &pInsert, 0); - } - for(i=1; rc==SQLITE_OK && i<=pConfig->nCol+1; i++){ - rc = sqlite3_bind_value(pInsert, i, apVal[i]); - } - if( rc==SQLITE_OK ){ - sqlite3_step(pInsert); - rc = sqlite3_reset(pInsert); - } - *piRowid = sqlite3_last_insert_rowid(pConfig->db); - } - } - - /* Add new entries to the FTS index */ - if( rc==SQLITE_OK ){ - rc = sqlite3Fts5IndexBeginWrite(p->pIndex, *piRowid); - ctx.pStorage = p; + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5IndexBeginWrite(p->pIndex, 0, iRowid); } for(ctx.iCol=0; rc==SQLITE_OK && ctx.iColnCol; ctx.iCol++){ ctx.szCol = 0; if( pConfig->abUnindexed[ctx.iCol]==0 ){ rc = sqlite3Fts5Tokenize(pConfig, @@ -774,11 +775,11 @@ } p->nTotalRow++; /* Write the %_docsize record */ if( rc==SQLITE_OK ){ - rc = fts5StorageInsertDocsize(p, *piRowid, &buf); + rc = fts5StorageInsertDocsize(p, iRowid, &buf); } sqlite3_free(buf.p); /* Write the averages record */ if( rc==SQLITE_OK ){ Index: ext/fts5/fts5_tcl.c ================================================================== --- ext/fts5/fts5_tcl.c +++ ext/fts5/fts5_tcl.c @@ -974,11 +974,10 @@ void * clientData, Tcl_Interp *interp, int objc, Tcl_Obj *CONST objv[] ){ - int bOld = sqlite3_fts5_may_be_corrupt; char *z; int n; unsigned int iVal; int nSlot; Index: ext/fts5/fts5_tokenize.c ================================================================== --- ext/fts5/fts5_tokenize.c +++ ext/fts5/fts5_tokenize.c @@ -240,11 +240,11 @@ Unicode61Tokenizer *p, /* Tokenizer object */ const char *z, /* Characters to treat as exceptions */ int bTokenChars /* 1 for 'tokenchars', 0 for 'separators' */ ){ int rc = SQLITE_OK; - int n = strlen(z); + int n = (int)strlen(z); int *aNew; if( n>0 ){ aNew = (int*)sqlite3_realloc(p->aiException, (n+p->nException)*sizeof(int)); if( aNew ){ Index: ext/fts5/fts5_vocab.c ================================================================== --- ext/fts5/fts5_vocab.c +++ ext/fts5/fts5_vocab.c @@ -166,12 +166,12 @@ }else{ int nByte; /* Bytes of space to allocate */ const char *zDb = bDb ? argv[3] : argv[1]; const char *zTab = bDb ? argv[4] : argv[3]; const char *zType = bDb ? argv[5] : argv[4]; - int nDb = strlen(zDb)+1; - int nTab = strlen(zTab)+1; + int nDb = (int)strlen(zDb)+1; + int nTab = (int)strlen(zTab)+1; int eType; rc = fts5VocabTableType(zType, pzErr, &eType); if( rc==SQLITE_OK ){ assert( eType>=0 && eTypepIter, &pPos, &nPos, &dummy); + rc = sqlite3Fts5IterPoslist(pCsr->pIter, 0, &pPos, &nPos, &dummy); if( rc==SQLITE_OK ){ if( pTab->eType==FTS5_VOCAB_ROW ){ while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){ pCsr->aVal[1]++; } @@ -400,11 +400,11 @@ Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor; int rc; const int flags = FTS5INDEX_QUERY_SCAN; fts5VocabResetCursor(pCsr); - rc = sqlite3Fts5IndexQuery(pCsr->pIndex, 0, 0, flags, &pCsr->pIter); + rc = sqlite3Fts5IndexQuery(pCsr->pIndex, 0, 0, flags, 0, &pCsr->pIter); if( rc==SQLITE_OK ){ rc = fts5VocabNextMethod(pCursor); } return rc; Index: ext/fts5/fts5parse.y ================================================================== --- ext/fts5/fts5parse.y +++ ext/fts5/fts5parse.y @@ -65,10 +65,11 @@ %left NOT. %left TERM. %left COLON. input ::= expr(X). { sqlite3Fts5ParseFinished(pParse, X); } +%destructor input { (void)pParse; } %type cnearset {Fts5ExprNode*} %type expr {Fts5ExprNode*} %type exprlist {Fts5ExprNode*} %destructor cnearset { sqlite3Fts5ParseNodeFree($$); } @@ -99,13 +100,13 @@ cnearset(A) ::= colset(X) COLON nearset(Y). { sqlite3Fts5ParseSetColset(pParse, Y, X); A = sqlite3Fts5ParseNode(pParse, FTS5_STRING, 0, 0, Y); } -%type colset {Fts5ExprColset*} +%type colset {Fts5Colset*} %destructor colset { sqlite3_free($$); } -%type colsetlist {Fts5ExprColset*} +%type colsetlist {Fts5Colset*} %destructor colsetlist { sqlite3_free($$); } colset(A) ::= LCP colsetlist(X) RCP. { A = X; } colset(A) ::= STRING(X). { A = sqlite3Fts5ParseColset(pParse, 0, &X); Index: ext/fts5/test/fts5_common.tcl ================================================================== --- ext/fts5/test/fts5_common.tcl +++ ext/fts5/test/fts5_common.tcl @@ -15,11 +15,10 @@ } source $testdir/tester.tcl catch { sqlite3_fts5_may_be_corrupt 0 - append G(perm:dbconfig) "; load_static_extension \$::dbhandle fts5" reset_db } proc fts5_test_poslist {cmd} { set res [list] Index: ext/fts5/test/fts5aa.test ================================================================== --- ext/fts5/test/fts5aa.test +++ ext/fts5/test/fts5aa.test @@ -157,12 +157,23 @@ do_execsql_test 6.3 { REPLACE INTO t1(rowid, x, y) VALUES('22', 'l l l', 'l l l'); } do_execsql_test 6.4 { + REPLACE INTO t1(x, y) VALUES('x y z', 'x y z'); +} + +do_execsql_test 6.5 { INSERT INTO t1(t1) VALUES('integrity-check') } + +do_execsql_test 6.6 { + SELECT rowid, * FROM t1; +} { + 22 {l l l} {l l l} + 23 {x y z} {x y z} +} #------------------------------------------------------------------------- # reset_db expr srand(0) Index: ext/fts5/test/fts5al.test ================================================================== --- ext/fts5/test/fts5al.test +++ ext/fts5/test/fts5al.test @@ -249,26 +249,38 @@ INSERT INTO t3 VALUES('a three'); INSERT INTO t3 VALUES('a four'); INSERT INTO t3 VALUES('a five'); INSERT INTO t3(t3, rank) VALUES('rank', 'bm25()'); } -breakpoint do_execsql_test 4.3.2 { SELECT * FROM t3 WHERE t3 MATCH 'a' AND rank MATCH 'rowidmod(4)' ORDER BY rank ASC } { {a four} {a one} {a five} {a two} {a three} } + do_execsql_test 4.3.3 { SELECT *, rank FROM t3 WHERE t3 MATCH 'a' AND rank MATCH 'rowidmod(3)' ORDER BY rank ASC } { {a three} 0 {a one} 1 {a four} 1 {a two} 2 {a five} 2 } + +do_execsql_test 4.3.4 { + SELECT * FROM t3('a', 'rowidmod(4)') ORDER BY rank ASC; +} { + {a four} {a one} {a five} {a two} {a three} +} + +do_execsql_test 4.3.5 { + SELECT *, rank FROM t3('a', 'rowidmod(3)') ORDER BY rank ASC +} { + {a three} 0 {a one} 1 {a four} 1 {a two} 2 {a five} 2 +} do_catchsql_test 4.4.3 { SELECT *, rank FROM t3 WHERE t3 MATCH 'a' AND rank MATCH 'xyz(3)' } {1 {no such function: xyz}} do_catchsql_test 4.4.4 { Index: ext/fts5/test/fts5corrupt3.test ================================================================== --- ext/fts5/test/fts5corrupt3.test +++ ext/fts5/test/fts5corrupt3.test @@ -248,12 +248,10 @@ } {} catch { db eval ROLLBACK } } } -} - #------------------------------------------------------------------------ # reset_db do_execsql_test 6.1.0 { CREATE VIRTUAL TABLE t1 USING fts5(a); @@ -333,8 +331,46 @@ } do_catchsql_test 6.3.5 { INSERT INTO t1(t1) VALUES('integrity-check'); } {1 {database disk image is malformed}} + +} + +#------------------------------------------------------------------------ +# +reset_db +reset_db +proc rnddoc {n} { + set map [list a b c d] + set doc [list] + for {set i 0} {$i < $n} {incr i} { + lappend doc "x[lindex $map [expr int(rand()*4)]]" + } + set doc +} + +db func rnddoc rnddoc +do_test 7.0 { + execsql { + CREATE VIRTUAL TABLE t5 USING fts5(x); + INSERT INTO t5 VALUES( rnddoc(10000) ); + INSERT INTO t5 VALUES( rnddoc(10000) ); + INSERT INTO t5 VALUES( rnddoc(10000) ); + INSERT INTO t5 VALUES( rnddoc(10000) ); + INSERT INTO t5(t5) VALUES('optimize'); + } +} {} + +do_test 7.1 { + foreach i [db eval { SELECT rowid FROM t5_data WHERE rowid>100 }] { + db eval BEGIN + db eval {DELETE FROM t5_data WHERE rowid = $i} + set r [catchsql { INSERT INTO t5(t5) VALUES('integrity-check')} ] + if {$r != "1 {database disk image is malformed}"} { error $r } + db eval ROLLBACK + } +} {} + sqlite3_fts5_may_be_corrupt 0 finish_test Index: ext/fts5/test/fts5fault6.test ================================================================== --- ext/fts5/test/fts5fault6.test +++ ext/fts5/test/fts5fault6.test @@ -282,12 +282,10 @@ catch { db close } breakpoint do_faultsim_test 6 -faults oom* -prep { sqlite_orig db test.db sqlite3_db_config_lookaside db 0 0 0 -} -body { - load_static_extension db fts5 } -test { faultsim_test_result {0 {}} {1 {initialization of fts5 failed: }} if {$testrc==0} { db eval { CREATE VIRTUAL TABLE temp.t1 USING fts5(x) } } Index: ext/fts5/test/fts5fault7.test ================================================================== --- ext/fts5/test/fts5fault7.test +++ ext/fts5/test/fts5fault7.test @@ -84,8 +84,29 @@ } -body { db eval COMMIT } -test { faultsim_test_result {0 {}} } + +#------------------------------------------------------------------------- +# Test fault-injection when a segment is promoted. +# +reset_db +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE xy USING fts5(x); + INSERT INTO xy(rowid, x) VALUES(1, '1 2 3'); + INSERT INTO xy(rowid, x) VALUES(2, '2 3 4'); + INSERT INTO xy(rowid, x) VALUES(3, '3 4 5'); +} +faultsim_save_and_close + +do_faultsim_test 2 -faults oom-* -prep { + faultsim_restore_and_reopen +} -body { + db eval { UPDATE OR REPLACE xy SET rowid=3 WHERE rowid = 2 } +} -test { + faultsim_test_result {0 {}} +} + finish_test Index: ext/fts5/test/fts5hash.test ================================================================== --- ext/fts5/test/fts5hash.test +++ ext/fts5/test/fts5hash.test @@ -82,15 +82,17 @@ lappend vocab xyz do_execsql_test 1.1 { CREATE VIRTUAL TABLE vocab USING fts5vocab(eee, 'row'); BEGIN; - WITH ii(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM ii WHERE i<100) - INSERT INTO eee SELECT r($vocab, 5), r($vocab, 7) FROM ii; - INSERT INTO eee(eee) VALUES('integrity-check'); } - +do_test 1.2 { + for {set i 1} {$i <= 100} {incr i} { + execsql { INSERT INTO eee VALUES( r($vocab, 5), r($vocab, 7) ) } + } +} {} + do_test 1.2 { db eval { SELECT term, doc FROM vocab } { set nRow [db one {SELECT count(*) FROM eee WHERE eee MATCH $term}] if {$nRow != $doc} { error "term=$term fts5vocab=$doc cnt=$nRow" ADDED ext/fts5/test/fts5onepass.test Index: ext/fts5/test/fts5onepass.test ================================================================== --- /dev/null +++ ext/fts5/test/fts5onepass.test @@ -0,0 +1,181 @@ +# 2015 Sep 27 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5onepass + +# If SQLITE_ENABLE_FTS3 is defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE ft USING fts5(content); + INSERT INTO ft(rowid, content) VALUES(1, '1 2 3'); + INSERT INTO ft(rowid, content) VALUES(2, '4 5 6'); + INSERT INTO ft(rowid, content) VALUES(3, '7 8 9'); +} + +#------------------------------------------------------------------------- +# Check that UPDATE and DELETE statements that feature "WHERE rowid=?" or +# or "WHERE rowid=?" clauses do not use statement journals. But that other +# DELETE and UPDATE statements do. +# +# Note: "MATCH ? AND rowid=?" does use a statement journal. +# +foreach {tn sql uses} { + 1.1 { DELETE FROM ft } 1 + 1.2 { DELETE FROM ft WHERE rowid=? } 0 + 1.3 { DELETE FROM ft WHERE rowid=? } 0 + 1.4 { DELETE FROM ft WHERE ft MATCH '1' } 1 + 1.5 { DELETE FROM ft WHERE ft MATCH '1' AND rowid=? } 1 + 1.6 { DELETE FROM ft WHERE ft MATCH '1' AND rowid=? } 1 + + 2.1 { UPDATE ft SET content='a b c' } 1 + 2.2 { UPDATE ft SET content='a b c' WHERE rowid=? } 0 + 2.3 { UPDATE ft SET content='a b c' WHERE rowid=? } 0 + 2.4 { UPDATE ft SET content='a b c' WHERE ft MATCH '1' } 1 + 2.5 { UPDATE ft SET content='a b c' WHERE ft MATCH '1' AND rowid=? } 1 + 2.6 { UPDATE ft SET content='a b c' WHERE ft MATCH '1' AND rowid=? } 1 +} { + do_test 1.$tn { sql_uses_stmt db $sql } $uses +} + +#------------------------------------------------------------------------- +# Check that putting a "DELETE/UPDATE ... WHERE rowid=?" statement in a +# trigger program does not prevent the VM from using a statement +# transaction. Even if the calling statement cannot hit a constraint. +# +do_execsql_test 2.0 { + CREATE TABLE t1(x); + + CREATE TRIGGER t1_ai AFTER INSERT ON t1 BEGIN + DELETE FROM ft WHERE rowid=new.x; + END; + + CREATE TRIGGER t1_ad AFTER DELETE ON t1 BEGIN + UPDATE ft SET content = 'a b c' WHERE rowid=old.x; + END; + + CREATE TRIGGER t1_bu BEFORE UPDATE ON t1 BEGIN + DELETE FROM ft WHERE rowid=old.x; + END; +} + +foreach {tn sql uses} { + 1 { INSERT INTO t1 VALUES(1) } 1 + 2 { DELETE FROM t1 WHERE x=4 } 1 + 3 { UPDATE t1 SET x=10 WHERE x=11 } 1 +} { + do_test 2.$tn { sql_uses_stmt db $sql } $uses +} + +#------------------------------------------------------------------------- +# Test that an "UPDATE ... WHERE rowid=?" works and does not corrupt the +# index when it strikes a constraint. Both inside and outside a +# transaction. +# +foreach {tn tcl1 tcl2} { + 1 {} {} + + 2 { + execsql BEGIN + } { + if {[sqlite3_get_autocommit db]==1} { error "transaction rolled back!" } + execsql COMMIT + } +} { + + do_execsql_test 3.$tn.0 { + DROP TABLE IF EXISTS ft2; + CREATE VIRTUAL TABLE ft2 USING fts5(content); + INSERT INTO ft2(rowid, content) VALUES(1, 'a b c'); + INSERT INTO ft2(rowid, content) VALUES(2, 'a b d'); + INSERT INTO ft2(rowid, content) VALUES(3, 'a b e'); + } + + eval $tcl1 + foreach {tn2 sql content} { + 1 { UPDATE ft2 SET rowid=2 WHERE rowid=1 } + { 1 {a b c} 2 {a b d} 3 {a b e} } + + 2 { + INSERT INTO ft2(rowid, content) VALUES(4, 'a b f'); + UPDATE ft2 SET rowid=5 WHERE rowid=4; + UPDATE ft2 SET rowid=3 WHERE rowid=5; + } { 1 {a b c} 2 {a b d} 3 {a b e} 5 {a b f} } + + 3 { + UPDATE ft2 SET rowid=3 WHERE rowid=4; -- matches 0 rows + UPDATE ft2 SET rowid=2 WHERE rowid=3; + } { 1 {a b c} 2 {a b d} 3 {a b e} 5 {a b f} } + + 4 { + INSERT INTO ft2(rowid, content) VALUES(4, 'a b g'); + UPDATE ft2 SET rowid=-1 WHERE rowid=4; + UPDATE ft2 SET rowid=3 WHERE rowid=-1; + } {-1 {a b g} 1 {a b c} 2 {a b d} 3 {a b e} 5 {a b f} } + + 5 { + DELETE FROM ft2 WHERE rowid=451; + DELETE FROM ft2 WHERE rowid=-1; + UPDATE ft2 SET rowid = 2 WHERE rowid = 1; + } {1 {a b c} 2 {a b d} 3 {a b e} 5 {a b f} } + } { + do_catchsql_test 3.$tn.$tn2.a $sql {1 {constraint failed}} + do_execsql_test 3.$tn.$tn2.b { SELECT rowid, content FROM ft2 } $content + + do_execsql_test 3.$tn.$tn2.c { + INSERT INTO ft2(ft2) VALUES('integrity-check'); + } + } + eval $tcl2 +} + +#------------------------------------------------------------------------- +# Check that DELETE and UPDATE operations can be done without flushing +# the in-memory hash table to disk. +# +reset_db +do_execsql_test 4.1.1 { + CREATE VIRTUAL TABLE ttt USING fts5(x); + BEGIN; + INSERT INTO ttt(rowid, x) VALUES(1, 'a b c'); + INSERT INTO ttt(rowid, x) VALUES(2, 'a b c'); + INSERT INTO ttt(rowid, x) VALUES(3, 'a b c'); + COMMIT +} +do_test 4.1.2 { fts5_level_segs ttt } {1} + +do_execsql_test 4.2.1 { + BEGIN; + DELETE FROM ttt WHERE rowid=1; + DELETE FROM ttt WHERE rowid=3; + INSERT INTO ttt(rowid, x) VALUES(4, 'd e f'); + INSERT INTO ttt(rowid, x) VALUES(5, 'd e f'); + COMMIT; +} {} +do_test 4.2.2 { fts5_level_segs ttt } {2} + + +do_execsql_test 4.3.1 { + BEGIN; + UPDATE ttt SET x = 'd e f' WHERE rowid = 2; + UPDATE ttt SET x = 'A B C' WHERE rowid = 4; + INSERT INTO ttt(rowid, x) VALUES(6, 'd e f'); + COMMIT; +} {} +do_test 4.2.2 { fts5_level_segs ttt } {3} + +finish_test + ADDED ext/fts5/test/fts5phrase.test Index: ext/fts5/test/fts5phrase.test ================================================================== --- /dev/null +++ ext/fts5/test/fts5phrase.test @@ -0,0 +1,119 @@ +# 2014 Jan 08 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Tests focused on phrase queries. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5phrase + +# If SQLITE_ENABLE_FTS5 is defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t3 USING fts5(a, b, c); + INSERT INTO t3 VALUES('d e a', 'd i j j f', 'i j i e b f h'); -- 1 + INSERT INTO t3 VALUES('g a e', 'f g i g a', 'h d g i g h c'); -- 2 + INSERT INTO t3 VALUES('e a d', 'e i h a f', 'c e h i f b i'); -- 3 + INSERT INTO t3 VALUES('a g c', 'd j d j c', 'c d f j i g j'); -- 4 + INSERT INTO t3 VALUES('b c b', 'j g c d f', 'j c j d g f b'); -- 5 + INSERT INTO t3 VALUES('j a d', 'e b i h h', 'c c f g d i d'); -- 6 + INSERT INTO t3 VALUES('a d f', 'h g i i i', 'e a g c i f b'); -- 7 + INSERT INTO t3 VALUES('g f d', 'f c g b j', 'b b h h h j j'); -- 8 + INSERT INTO t3 VALUES('f h g', 'c j f g j', 'd h d f e b h'); -- 9 + INSERT INTO t3 VALUES('f h d', 'c i a d b', 'g b j b a d e'); -- 10 + INSERT INTO t3 VALUES('j h h', 'j i h a g', 'd e i e a g j'); -- 11 + INSERT INTO t3 VALUES('a b e', 'h g a g c', 'h c a a d e g'); -- 12 + INSERT INTO t3 VALUES('a j g', 'i h i f i', 'a g h j g i b'); -- 13 + INSERT INTO t3 VALUES('j h e', 'f e d i e', 'i d c f e d c'); -- 14 + INSERT INTO t3 VALUES('d j d', 'd b i a c', 'g d h i d b e'); -- 15 + INSERT INTO t3 VALUES('h j e', 'e b b c f', 'j a f g h d j'); -- 16 + INSERT INTO t3 VALUES('c b j', 'c a b a i', 'h f i d a d c'); -- 17 + INSERT INTO t3 VALUES('e e d', 'i d f c c', 'g i d a f e a'); -- 18 + INSERT INTO t3 VALUES('e i g', 'e a b i h', 'i f d d a d f'); -- 19 + INSERT INTO t3 VALUES('h g f', 'b h h j d', 'i f d e g j a'); -- 20 + INSERT INTO t3 VALUES('e h f', 'j c b c f', 'j a j g h a c'); -- 21 + INSERT INTO t3 VALUES('d c h', 'b g i c e', 'i i c d e h i'); -- 22 + INSERT INTO t3 VALUES('a h i', 'a g d f f', 'e f i i b b h'); -- 23 + INSERT INTO t3 VALUES('d d g', 'c c b c g', 'g c h e b c e'); -- 24 + INSERT INTO t3 VALUES('a b b', 'b f a d i', 'd a h a b c i'); -- 25 + INSERT INTO t3 VALUES('a f d', 'a j e a h', 'j i h j a i f'); -- 26 + INSERT INTO t3 VALUES('d j d', 'h a d i a', 'h h f j h g a'); -- 27 + INSERT INTO t3 VALUES('g a e', 'd g f a g', 'i d b c g g j'); -- 28 + INSERT INTO t3 VALUES('j e h', 'g h j h g', 'd a e j a a h'); -- 29 + INSERT INTO t3 VALUES('e j e', 'g e j g c', 'f c e b e e a'); -- 30 + INSERT INTO t3 VALUES('h f f', 'i j g e c', 'j j f c a i j'); -- 31 + INSERT INTO t3 VALUES('a g c', 'c g d b i', 'g h c b a a f'); -- 32 + INSERT INTO t3 VALUES('c h i', 'j d h e e', 'a h i d c c j'); -- 33 + INSERT INTO t3 VALUES('d a c', 'e d d b j', 'c e b b h i h'); -- 34 + INSERT INTO t3 VALUES('d f h', 'c a f c c', 'j b b c c j f'); -- 35 + INSERT INTO t3 VALUES('b g h', 'g c c c f', 'c g c f h e e'); -- 36 + INSERT INTO t3 VALUES('f e a', 'b h f j h', 'j g h f d g f'); -- 37 + INSERT INTO t3 VALUES('h f a', 'a e i j g', 'f d a f d f c'); -- 38 + INSERT INTO t3 VALUES('f i c', 'f i i i i', 'e c f d h j f'); -- 39 + INSERT INTO t3 VALUES('h h d', 'd i e d i', 'd f e i a h a'); -- 40 + INSERT INTO t3 VALUES('f g c', 'd a f c h', 'b b g j c e g'); -- 41 + INSERT INTO t3 VALUES('h i h', 'h d j d e', 'e d b b i e g'); -- 42 + INSERT INTO t3 VALUES('b h i', 'j e i d a', 'j j h e e c a'); -- 43 + INSERT INTO t3 VALUES('g i g', 'f c c f d', 'a c i c a d a'); -- 44 + INSERT INTO t3 VALUES('c c f', 'a b j d b', 'c a e g f e c'); -- 45 + INSERT INTO t3 VALUES('d h j', 'g c b j d', 'e a h f h j g'); -- 46 + INSERT INTO t3 VALUES('a a d', 'j e j a i', 'i d c f f f b'); -- 47 + INSERT INTO t3 VALUES('b g j', 'e c i h f', 'd d h b g a d'); -- 48 + INSERT INTO t3 VALUES('c i a', 'a c c c c', 'e h i e h i e'); -- 49 + INSERT INTO t3 VALUES('f f c', 'f f b i i', 'f f a j e c i'); -- 50 +} + +proc pmatch {col expr} { + return [expr {[string first $expr $col]>=0}] +} +db func pmatch pmatch + +foreach {tn cols tokens} { + 1 a "c c" + 2 b "c c" + 3 c "c c" + 4 {a b c} "c c" + 5 {a b c} "b h" + 6 {a b} "b h" + 7 {a c} "b h" + 8 {c a} "b h" + 9 {c} "i e" + 10 {b} "i e" + 11 {a} "i e" +} { + set fts "{$cols}:[join $tokens +]" + set where [list] + foreach c $cols { lappend where "pmatch($c, '$tokens')" } + set where [join $where " OR "] + + set res [db eval "SELECT rowid FROM t3 WHERE $where"] + do_execsql_test "1.$tn.$fts->([llength $res] rows)" { + SELECT rowid FROM t3($fts) + } $res +} + +do_execsql_test 2.0 { + SELECT rowid, + highlight(t3, 0, '*', '*'), + highlight(t3, 1, '*', '*'), + highlight(t3, 2, '*', '*') + FROM t3('a:f+f') +} { + 31 {h *f f*} {i j g e c} {j j f c a i j} + 50 {*f f* c} {f f b i i} {f f a j e c i} +} + +finish_test + Index: ext/fts5/test/fts5prefix.test ================================================================== --- ext/fts5/test/fts5prefix.test +++ ext/fts5/test/fts5prefix.test @@ -60,8 +60,244 @@ 2 "SELECT rowid FROM t1 WHERE t1 MATCH '\u1234\u5678*'" 2 } { do_execsql_test 2.3.$tn $q $res } +#------------------------------------------------------------------------- +# Check that prefix queries with: +# +# * a column filter, and +# * no prefix index. +# +# work Ok. +# +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE t3 USING fts5(a, b, c); + INSERT INTO t3(t3, rank) VALUES('pgsz', 32); + BEGIN; + INSERT INTO t3 VALUES('acb ccc bba', 'cca bba bca', 'bbc ccc bca'); -- 1 + INSERT INTO t3 VALUES('cbb cac cab', 'abb aac bba', 'aab ccc cac'); -- 2 + INSERT INTO t3 VALUES('aac bcb aac', 'acb bcb caa', 'aca bab bca'); -- 3 + INSERT INTO t3 VALUES('aab ccb ccc', 'aca cba cca', 'aca aac cbb'); -- 4 + INSERT INTO t3 VALUES('bac aab bab', 'ccb bac cba', 'acb aba abb'); -- 5 + INSERT INTO t3 VALUES('bab abc ccb', 'acb cba abb', 'cbb aaa cab'); -- 6 + INSERT INTO t3 VALUES('cbb bbc baa', 'aab aca baa', 'bcc cca aca'); -- 7 + INSERT INTO t3 VALUES('abc bba abb', 'cac abc cba', 'acc aac cac'); -- 8 + INSERT INTO t3 VALUES('bbc bbc cab', 'bcb ccb cba', 'bcc cac acb'); -- 9 + COMMIT; +} + +foreach {tn match res} { + 1 "a : c*" {1 2 4 6 7 9} + 2 "b : c*" {1 3 4 5 6 8 9} + 3 "c : c*" {1 2 4 6 7 8 9} + 4 "a : b*" {1 3 5 6 7 8 9} + 5 "b : b*" {1 2 3 5 7 9} + 6 "c : b*" {1 3 7 9} + 7 "a : a*" {1 3 4 5 6 8} + 8 "b : a*" {2 3 4 6 7 8} + 9 "c : a*" {2 3 4 5 6 7 8 9} +} { + do_execsql_test 3.1.$tn { + SELECT rowid FROM t3($match) + } $res +} + +do_test 3.2 { + expr srand(0) + execsql { DELETE FROM t3 } + for {set i 0} {$i < 1000} {incr i} { + set a [fts5_rnddoc 3] + set b [fts5_rnddoc 8] + set c [fts5_rnddoc 20] + execsql { INSERT INTO t3 VALUES($a, $b, $c) } + } + execsql { INSERT INTO t3(t3) VALUES('integrity-check') } +} {} + +proc gmatch {col pattern} { + expr {[lsearch -glob $col $pattern]>=0} +} +db func gmatch gmatch + +proc ghl {col pattern} { + foreach t $col { + if {[string match $pattern $t]} { + lappend res "*$t*" + } else { + lappend res $t + } + } + set res +} +db func ghl ghl + +set COLS(a) 0 +set COLS(b) 1 +set COLS(c) 2 + +for {set x 0} {$x<2} {incr x} { + foreach {tn pattern} { + 1 {xa*} + 2 {xb*} + 3 {xc*} + 4 {xd*} + 5 {xe*} + 6 {xf*} + 7 {xg*} + 8 {xh*} + 9 {xi*} + 10 {xj*} + } { + foreach col {a b c} { + + # Check that the list of returned rowids is correct. + # + set res [db eval "SELECT rowid FROM t3 WHERE gmatch($col, '$pattern')"] + set query "$col : $pattern" + do_execsql_test 3.3.$x.$tn.$col.rowid { + SELECT rowid FROM t3($query); + } $res + + # Check that the highlight() function works. + # + set res [db eval \ + "SELECT ghl($col, '$pattern') FROM t3 WHERE gmatch($col, '$pattern')" + ] + set idx $COLS($col) + do_execsql_test 3.3.$x.$tn.$col.highlight { + SELECT highlight(t3, $idx, '*', '*') FROM t3($query); + } $res + } + + foreach colset {{a b} {b c} {c a} {a c} {b a}} { + # Check that the list of returned rowids is correct. + # + foreach {col1 col2} $colset {} + set expr "gmatch($col1, '$pattern') OR gmatch($col2, '$pattern')" + set res [db eval "SELECT rowid FROM t3 WHERE $expr"] + set query "{$colset} : $pattern" + do_execsql_test 3.3.$x.$tn.{$colset}.rowid { + SELECT rowid FROM t3($query); + } $res + + set resq "SELECT ghl($col1, '$pattern'), ghl($col2, '$pattern')" + append resq " FROM t3 WHERE $expr" + set res [db eval $resq] + set idx1 $COLS($col1) + set idx2 $COLS($col2) + do_execsql_test 3.3.$x.$tn.{$colset}.highlight { + SELECT highlight(t3, $idx1, '*', '*'), highlight(t3, $idx2, '*', '*') + FROM t3($query) + } $res + } + } + execsql { INSERT INTO t3(t3) VALUES('optimize') } + execsql { INSERT INTO t3(t3) VALUES('integrity-check') } +} + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 4.0 { + CREATE VIRTUAL TABLE t2 USING fts5(c1, c2); + INSERT INTO t2 VALUES('xa xb', 'xb xa'); + + INSERT INTO t2 SELECT c1||' '||c1, c2||' '||c2 FROM t2; -- 2 + INSERT INTO t2 SELECT c1||' '||c1, c2||' '||c2 FROM t2; -- 4 + INSERT INTO t2 SELECT c1||' '||c1, c2||' '||c2 FROM t2; -- 8 + INSERT INTO t2 SELECT c1||' '||c1, c2||' '||c2 FROM t2; -- 16 + INSERT INTO t2 SELECT c1||' '||c1, c2||' '||c2 FROM t2; -- 32 + INSERT INTO t2 SELECT c1||' '||c1, c2||' '||c2 FROM t2; -- 64 + INSERT INTO t2 SELECT c1||' '||c1, c2||' '||c2 FROM t2; -- 128 + INSERT INTO t2 SELECT c1||' '||c1, c2||' '||c2 FROM t2; -- 256 + INSERT INTO t2 SELECT c1||' '||c1, c2||' '||c2 FROM t2; -- 512 + INSERT INTO t2 SELECT c1||' '||c1, c2||' '||c2 FROM t2; -- 1024 + INSERT INTO t2 SELECT c1||' '||c1, c2||' '||c2 FROM t2; -- 2048 + INSERT INTO t2 SELECT c1||' '||c1, c2||' '||c2 FROM t2; -- 4096 + + SELECT count(*) FROM t2('x*'); +} {4096} + +do_execsql_test 4.1 { + UPDATE t2 SET c2 = 'ya yb'; + SELECT count(*) FROM t2('c1:x*'); + SELECT count(*) FROM t2('c2:x*'); +} {4096 0} + +do_execsql_test 4.2 { + UPDATE t2 SET c2 = 'xa'; + SELECT count(*) FROM t2('c1:x*'); + SELECT count(*) FROM t2('c2:x*'); +} {4096 4096} + +#------------------------------------------------------------------------- +# +reset_db +proc rnddoc {n} { + set map [list a b c d] + set doc [list] + for {set i 0} {$i < $n} {incr i} { + lappend doc "x[lindex $map [expr int(rand()*4)]]" + } + set doc +} +set cols [list] +for {set i 1} {$i<250} {incr i} { + lappend cols "c$i" + lappend vals "'[rnddoc 10]'" +} + +do_test 5.0 { + execsql "CREATE VIRTUAL TABLE t4 USING fts5([join $cols ,])" + execsql {INSERT INTO t4(t4, rank) VALUES('pgsz', 32)} + execsql "INSERT INTO t4 VALUES([join $vals ,])" + execsql "INSERT INTO t4 VALUES([join $vals ,])" + execsql "INSERT INTO t4 VALUES([join $vals ,])" + execsql "INSERT INTO t4 VALUES([join $vals ,])" +} {} + +proc gmatch {col pattern} { + expr {[lsearch -glob $col $pattern]>=0} +} +db func gmatch gmatch +foreach {tn col pattern} { + 1 c100 {xa*} + 2 c200 {xb*} +} { + set res [db eval "SELECT rowid FROM t4 WHERE gmatch($col, \$pattern)"] + set query "$col : $pattern" + do_execsql_test 5.$tn { SELECT rowid FROM t4($query) } $res +} + +reset_db +db func fts5_rnddoc fts5_rnddoc +do_test 6.0 { + execsql { + CREATE VIRTUAL TABLE t5 USING fts5(x, y); + INSERT INTO t5 VALUES( fts5_rnddoc(10000), fts5_rnddoc(10000) ); + INSERT INTO t5 VALUES( fts5_rnddoc(10000), fts5_rnddoc(10000) ); + INSERT INTO t5 VALUES( fts5_rnddoc(10000), fts5_rnddoc(10000) ); + INSERT INTO t5 VALUES( fts5_rnddoc(10000), fts5_rnddoc(10000) ); + } +} {} + +proc gmatch {col pattern} { + expr {[lsearch -glob $col $pattern]>=0} +} +db func gmatch gmatch +foreach {tn col pattern} { + 1 y {xa*} + 2 y {xb*} + 3 y {xc*} + 4 x {xa*} + 5 x {xb*} + 6 x {xc*} +} { + set res [db eval "SELECT rowid FROM t5 WHERE gmatch($col, \$pattern)"] + set query "$col : $pattern" + do_execsql_test 6.$tn { SELECT rowid FROM t5($query) } $res +} finish_test + Index: ext/fts5/test/fts5simple.test ================================================================== --- ext/fts5/test/fts5simple.test +++ ext/fts5/test/fts5simple.test @@ -17,11 +17,10 @@ ifcapable !fts5 { finish_test return } -if 1 { #------------------------------------------------------------------------- # set doc "x x [string repeat {y } 50]z z" do_execsql_test 1.0 { CREATE VIRTUAL TABLE t1 USING fts5(x); @@ -135,12 +134,10 @@ do_execsql_test 5.4 { SELECT rowid FROM tt WHERE tt MATCH 'a*'; } {1 2} -} - do_execsql_test 5.5 { DELETE FROM tt; BEGIN; INSERT INTO tt VALUES('aa'); INSERT INTO tt VALUES('ab'); @@ -182,8 +179,148 @@ } {1 {fts5: syntax error near ""}} do_catchsql_test 6.3 { SELECT * FROM xyz WHERE xyz MATCH NULL } {1 {fts5: syntax error near ""}} +#------------------------------------------------------------------------- + +do_execsql_test 7.1 { + CREATE VIRTUAL TABLE ft2 USING fts5(content); + INSERT INTO ft2(rowid, content) VALUES(1, 'a b c'); + INSERT INTO ft2(rowid, content) VALUES(2, 'a b d'); +} + +do_catchsql_test 7.2 { + BEGIN; + UPDATE ft2 SET rowid=2 WHERE rowid=1; +} {1 {constraint failed}} + +do_execsql_test 7.3 { + COMMIT; + INSERT INTO ft2(ft2) VALUES('integrity-check'); +} {} + +do_execsql_test 7.4 { + SELECT * FROM ft2; +} {{a b c} {a b d}} + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 8.1 { + CREATE VIRTUAL TABLE ft2 USING fts5(content); + INSERT INTO ft2(rowid, content) VALUES(1, 'a b'); +} + +do_execsql_test 8.2 { + BEGIN; + INSERT INTO ft2(rowid, content) VALUES(4, 'a x'); +} + +do_execsql_test 8.3 { + INSERT INTO ft2(ft2) VALUES('integrity-check'); +} + +#------------------------------------------------------------------------- +# Check that the "table function" syntax works. +# +reset_db +do_execsql_test 9.1 { + CREATE VIRTUAL TABLE ft2 USING fts5(content); + INSERT INTO ft2(rowid, content) VALUES(1, 'a b'); + INSERT INTO ft2(rowid, content) VALUES(2, 'a b c d'); + INSERT INTO ft2(rowid, content) VALUES(3, 'c d e f'); +} + +do_execsql_test 9.2 { + SELECT rowid FROM ft2('a'); +} {1 2} + +do_execsql_test 9.3 { + SELECT rowid FROM ft2('b AND c'); +} {2} + +#------------------------------------------------------------------------- +# +do_execsql_test 10.0 { + CREATE VIRTUAL TABLE t3 USING fts5(a, b, c); + INSERT INTO t3 VALUES('bac aab bab', 'c bac c', 'acb aba abb'); -- 1 + INSERT INTO t3 VALUES('bab abc c', 'acb c abb', 'c aaa c'); -- 2 +} + +do_execsql_test 10.1 { + SELECT rowid FROM t3('c: c*'); +} {2} + +do_execsql_test 10.2 { + SELECT rowid FROM t3('b: acb'); +} {2} + +#------------------------------------------------------------------------- +# Test that character 0x1A is allowed in fts5 barewords. +# +do_test 11.0 { + execsql "CREATE VIRTUAL TABLE t4 USING fts5(x, tokenize=\"ascii tokenchars '\x1A'\")" + execsql " + INSERT INTO t4 VALUES('a b c \x1A'); + INSERT INTO t4 VALUES('a b c d\x1A'); + INSERT INTO t4 VALUES('a b c \x1Ad'); + INSERT INTO t4 VALUES('a b c d'); + " +} {} + +do_test 11.1 { + execsql "SELECT rowid FROM t4('\x1A')" +} {1} +do_test 11.2 { + execsql "SELECT rowid FROM t4('\x1A*')" +} {1 3} +do_test 11.3 { + execsql "SELECT rowid FROM t4('d\x1A')" +} {2} + +do_test 11.4 { + catchsql "SELECT rowid FROM t4('d\x1B')" +} {/fts5: syntax error/} +do_test 11.5 { + catchsql "SELECT rowid FROM t4('d\x19')" +} {/fts5: syntax error/} + +#------------------------------------------------------------------------- +# +do_test 12.1 { + execsql { + CREATE VIRTUAL TABLE xx USING fts5(x,y); + BEGIN; + INSERT INTO xx VALUES('1 2 3', 'a b c'); + } +} {} + +do_execsql_test 12.2 { + SELECT rowid FROM xx('x:a'); + COMMIT; +} {} + +#------------------------------------------------------------------------- +# Try an UPDATE OR REPLACE query. +# +do_execsql_test 13.1 { + CREATE VIRTUAL TABLE xy USING fts5(x); + INSERT INTO xy(rowid, x) VALUES(1, '1 2 3'); + INSERT INTO xy(rowid, x) VALUES(2, '2 3 4'); + INSERT INTO xy(rowid, x) VALUES(3, '3 4 5'); +} + +do_execsql_test 13.2 { + UPDATE OR REPLACE xy SET rowid=3 WHERE rowid = 2; + SELECT rowid, x FROM xy; +} { + 1 {1 2 3} + 3 {2 3 4} +} + +do_execsql_test 13.3 { + INSERT INTO xy(xy) VALUES('integrity-check'); +} finish_test ADDED ext/fts5/tool/fts5txt2db.tcl Index: ext/fts5/tool/fts5txt2db.tcl ================================================================== --- /dev/null +++ ext/fts5/tool/fts5txt2db.tcl @@ -0,0 +1,135 @@ + + +proc usage {} { + puts stderr "$::argv0 ?OPTIONS? DATABASE FILE1..." + puts stderr "" + puts stderr "Options are" + puts stderr " -fts5" + puts stderr " -fts4" + puts stderr " -colsize " + puts stderr { +This script is designed to create fts4/5 tables with more than one column. +The -colsize option should be set to a Tcl list of integer values, one for +each column in the table. Each value is the number of tokens that will be +inserted into the column value for each row. For example, setting the -colsize +option to "5 10" creates an FTS table with 2 columns, with roughly 5 and 10 +tokens per row in each, respectively. + +Each "FILE" argument should be a text file. The contents of these text files is +split on whitespace characters to form a list of tokens. The first N1 tokens +are used for the first column of the first row, where N1 is the first element +of the -colsize list. The next N2 are used for the second column of the first +row, and so on. Rows are added to the table until the entire list of tokens +is exhausted. +} + exit -1 +} + +set O(aColsize) [list 10 10 10] +set O(tblname) t1 +set O(fts) fts5 + + +set options_with_values {-colsize} + +for {set i 0} {$i < [llength $argv]} {incr i} { + set opt [lindex $argv $i] + if {[string range $opt 0 0]!="-"} break + + if {[lsearch $options_with_values $opt]>=0} { + incr i + if {$i==[llength $argv]} usage + set val [lindex $argv $i] + } + + switch -- $opt { + -colsize { + set O(aColSize) $val + } + + -fts4 { + set O(fts) fts4 + } + + -fts5 { + set O(fts) fts5 + } + } +} + +if {$i > [llength $argv]-2} usage +set O(db) [lindex $argv $i] +set O(files) [lrange $argv [expr $i+1] end] + +foreach {k v} [lrange $argv 0 end-2] { + switch -- $k { + -colsize { + set O(aColSize) $v + } + + -colsize { + set O(aColSize) $v + } + } + +} + +sqlite3 db $O(db) +load_static_extension db fts5 + + +# Create the FTS table in the db. Return a list of the table columns. +# +proc create_table {} { + global O + set cols [list a b c d e f g h i j k l m n o p q r s t u v w x y z] + + set nCol [llength $O(aColsize)] + set cols [lrange $cols 0 [expr $nCol-1]] + + set sql "CREATE VIRTUAL TABLE IF NOT EXISTS $O(tblname) USING $O(fts) (" + append sql [join $cols ,] + append sql ");" + + db eval $sql + return $cols +} + +# Return a list of tokens from the named file. +# +proc readfile {file} { + set fd [open $file] + set data [read $fd] + close $fd + split $data +} + + +# Load all the data into a big list of tokens. +# +set tokens [list] +foreach f $O(files) { + set tokens [concat $tokens [readfile $f]] +} + +set N [llength $tokens] +set i 0 +set cols [create_table] +set sql "INSERT INTO $O(tblname) VALUES(\$[lindex $cols 0]" +foreach c [lrange $cols 1 end] { + append sql ", \$$c" +} +append sql ")" + +db eval BEGIN + while {$i < $N} { + foreach c $cols s $O(aColsize) { + set $c [lrange $tokens $i [expr $i+$s-1]] + incr i $s + } + db eval $sql + } +db eval COMMIT + + + Index: ext/fts5/tool/mkfts5c.tcl ================================================================== --- ext/fts5/tool/mkfts5c.tcl +++ ext/fts5/tool/mkfts5c.tcl @@ -5,10 +5,11 @@ set srcdir [file dirname [file dirname [info script]]] set G(src) [string map [list %dir% $srcdir] { %dir%/fts5.h %dir%/fts5Int.h fts5parse.h + fts5parse.c %dir%/fts5_aux.c %dir%/fts5_buffer.c %dir%/fts5_config.c %dir%/fts5_expr.c %dir%/fts5_hash.c @@ -17,16 +18,15 @@ %dir%/fts5_storage.c %dir%/fts5_tokenize.c %dir%/fts5_unicode2.c %dir%/fts5_varint.c %dir%/fts5_vocab.c - fts5parse.c }] set G(hdr) { -#if !defined(SQLITE_TEST) || defined(SQLITE_ENABLE_FTS5) +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS5) #if !defined(NDEBUG) && !defined(SQLITE_DEBUG) # define NDEBUG 1 #endif #if defined(NDEBUG) && defined(SQLITE_DEBUG) @@ -35,11 +35,11 @@ } set G(footer) { -#endif /* !defined(SQLITE_TEST) || defined(SQLITE_ENABLE_FTS5) */ +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS5) */ } #------------------------------------------------------------------------- # Read and return the entire contents of text file $zFile from disk. # @@ -85,11 +85,13 @@ lappend sub_map yy fts5yy YY fts5YY TOKEN FTS5TOKEN } foreach line [split $data "\n"] { if {[regexp {^#include.*fts5} $line]} continue - if {[regexp {^(const )?[a-zA-Z][a-zA-Z0-9]* [*]?sqlite3Fts5} $line]} { + if { ![regexp { sqlite3Fts5Init\(} $line] + && [regexp {^(const )?[a-zA-Z][a-zA-Z0-9]* [*]?sqlite3Fts5} $line] + } { set line "static $line" } set line [string map $sub_map $line] puts $G(fd) $line } @@ -105,9 +107,5 @@ fts5c_init fts5.c foreach f $G(src) { fts5c_printfile $f } fts5c_close - - - - Index: ext/misc/json1.c ================================================================== --- ext/misc/json1.c +++ ext/misc/json1.c @@ -19,34 +19,67 @@ ** a JSONB type in the future which stores a binary encoding of JSON in ** a BLOB, but there is no support for JSONB in the current implementation. ** This implementation parses JSON text at 250 MB/s, so it is hard to see ** how JSONB might improve on that.) */ +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_JSON1) #if !defined(_SQLITEINT_H_) #include "sqlite3ext.h" #endif SQLITE_EXTENSION_INIT1 #include #include -#include +#include /* amalgamator: keep */ #include #include #define UNUSED_PARAM(X) (void)(X) +#ifndef LARGEST_INT64 +# define LARGEST_INT64 (0xffffffff|(((sqlite3_int64)0x7fffffff)<<32)) +# define SMALLEST_INT64 (((sqlite3_int64)-1) - LARGEST_INT64) +#endif + /* ** Versions of isspace(), isalnum() and isdigit() to which it is safe ** to pass signed char values. */ -#define safe_isspace(x) isspace((unsigned char)(x)) #define safe_isdigit(x) isdigit((unsigned char)(x)) #define safe_isalnum(x) isalnum((unsigned char)(x)) -/* Unsigned integer types */ -typedef sqlite3_uint64 u64; -typedef unsigned int u32; -typedef unsigned char u8; +/* +** Growing our own isspace() routine this way is twice as fast as +** the library isspace() function, resulting in a 7% overall performance +** increase for the parser. (Ubuntu14.10 gcc 4.8.4 x64 with -Os). +*/ +static const char jsonIsSpace[] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 1, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +}; +#define safe_isspace(x) (jsonIsSpace[(unsigned char)x]) + +#ifndef SQLITE_AMALGAMATION + /* Unsigned integer types. These are already defined in the sqliteInt.h, + ** but the definitions need to be repeated for separate compilation. */ + typedef sqlite3_uint64 u64; + typedef unsigned int u32; + typedef unsigned char u8; +#endif /* Objects */ typedef struct JsonString JsonString; typedef struct JsonNode JsonNode; typedef struct JsonParse JsonParse; @@ -451,22 +484,40 @@ } case JSON_FALSE: { sqlite3_result_int(pCtx, 0); break; } - case JSON_REAL: { - double r = strtod(pNode->u.zJContent, 0); - sqlite3_result_double(pCtx, r); - break; - } case JSON_INT: { sqlite3_int64 i = 0; const char *z = pNode->u.zJContent; if( z[0]=='-' ){ z++; } - while( z[0]>='0' && z[0]<='9' ){ i = i*10 + *(z++) - '0'; } + while( z[0]>='0' && z[0]<='9' ){ + unsigned v = *(z++) - '0'; + if( i>=LARGEST_INT64/10 ){ + if( i>LARGEST_INT64/10 ) goto int_as_real; + if( z[0]>='0' && z[0]<='9' ) goto int_as_real; + if( v==9 ) goto int_as_real; + if( v==8 ){ + if( pNode->u.zJContent[0]=='-' ){ + sqlite3_result_int64(pCtx, SMALLEST_INT64); + goto int_done; + }else{ + goto int_as_real; + } + } + } + i = i*10 + v; + } if( pNode->u.zJContent[0]=='-' ){ i = -i; } sqlite3_result_int64(pCtx, i); + int_done: + break; + int_as_real: /* fall through to real */; + } + case JSON_REAL: { + double r = strtod(pNode->u.zJContent, 0); + sqlite3_result_double(pCtx, r); break; } case JSON_STRING: { #if 0 /* Never happens because JNODE_RAW is only set by json_set(), ** json_insert() and json_replace() and those routines do not @@ -545,10 +596,48 @@ jsonReturnJson(pNode, pCtx, aReplace); break; } } } + +/* Forward reference */ +static int jsonParseAddNode(JsonParse*,u32,u32,const char*); + +/* +** A macro to hint to the compiler that a function should not be +** inlined. +*/ +#if defined(__GNUC__) +# define JSON_NOINLINE __attribute__((noinline)) +#elif defined(_MSC_VER) && _MSC_VER>=1310 +# define JSON_NOINLINE __declspec(noinline) +#else +# define JSON_NOINLINE +#endif + + +static JSON_NOINLINE int jsonParseAddNodeExpand( + JsonParse *pParse, /* Append the node to this object */ + u32 eType, /* Node type */ + u32 n, /* Content size or sub-node count */ + const char *zContent /* Content */ +){ + u32 nNew; + JsonNode *pNew; + assert( pParse->nNode>=pParse->nAlloc ); + if( pParse->oom ) return -1; + nNew = pParse->nAlloc*2 + 10; + pNew = sqlite3_realloc(pParse->aNode, sizeof(JsonNode)*nNew); + if( pNew==0 ){ + pParse->oom = 1; + return -1; + } + pParse->nAlloc = nNew; + pParse->aNode = pNew; + assert( pParse->nNodenAlloc ); + return jsonParseAddNode(pParse, eType, n, zContent); +} /* ** Create a new JsonNode instance based on the arguments and append that ** instance to the JsonParse. Return the index in pParse->aNode[] of the ** new node, or -1 if a memory allocation fails. @@ -559,21 +648,11 @@ u32 n, /* Content size or sub-node count */ const char *zContent /* Content */ ){ JsonNode *p; if( pParse->nNode>=pParse->nAlloc ){ - u32 nNew; - JsonNode *pNew; - if( pParse->oom ) return -1; - nNew = pParse->nAlloc*2 + 10; - pNew = sqlite3_realloc(pParse->aNode, sizeof(JsonNode)*nNew); - if( pNew==0 ){ - pParse->oom = 1; - return -1; - } - pParse->nAlloc = nNew; - pParse->aNode = pNew; + return jsonParseAddNodeExpand(pParse, eType, n, zContent); } p = &pParse->aNode[pParse->nNode]; p->eType = (u8)eType; p->jnFlags = 0; p->iVal = 0; @@ -595,12 +674,11 @@ u32 j; int iThis; int x; JsonNode *pNode; while( safe_isspace(pParse->zJson[i]) ){ i++; } - if( (c = pParse->zJson[i])==0 ) return 0; - if( c=='{' ){ + if( (c = pParse->zJson[i])=='{' ){ /* Parse object */ iThis = jsonParseAddNode(pParse, JSON_OBJECT, 0, 0); if( iThis<0 ) return -1; for(j=i+1;;j++){ while( safe_isspace(pParse->zJson[j]) ){ j++; } @@ -716,10 +794,12 @@ return j; }else if( c=='}' ){ return -2; /* End of {...} */ }else if( c==']' ){ return -3; /* End of [...] */ + }else if( c==0 ){ + return 0; /* End of file */ }else{ return -1; /* Syntax error */ } } @@ -803,11 +883,11 @@ /* ** Compare the OBJECT label at pNode against zKey,nKey. Return true on ** a match. */ -static int jsonLabelCompare(JsonNode *pNode, const char *zKey, int nKey){ +static int jsonLabelCompare(JsonNode *pNode, const char *zKey, u32 nKey){ if( pNode->jnFlags & JNODE_RAW ){ if( pNode->n!=nKey ) return 0; return strncmp(pNode->u.zJContent, zKey, nKey)==0; }else{ if( pNode->n!=nKey+2 ) return 0; @@ -1781,11 +1861,11 @@ return rc; }else if( p->bRecursive && jsonParseFindParents(&p->sParse) ){ jsonEachCursorReset(p); return SQLITE_NOMEM; }else{ - JsonNode *pNode; + JsonNode *pNode = 0; if( idxNum==3 ){ const char *zErr = 0; zRoot = (const char*)sqlite3_value_text(argv[1]); if( zRoot==0 ) return SQLITE_OK; n = sqlite3_value_bytes(argv[1]); @@ -1882,23 +1962,16 @@ 0 /* xRollbackTo */ }; #endif /* SQLITE_OMIT_VIRTUALTABLE */ /**************************************************************************** -** The following routine is the only publically visible identifier in this -** file. Call the following routine in order to register the various SQL +** The following routines are the only publically visible identifiers in this +** file. Call the following routines in order to register the various SQL ** functions and the virtual table implemented by this file. ****************************************************************************/ -#ifdef _WIN32 -__declspec(dllexport) -#endif -int sqlite3_json_init( - sqlite3 *db, - char **pzErrMsg, - const sqlite3_api_routines *pApi -){ +int sqlite3Json1Init(sqlite3 *db){ int rc = SQLITE_OK; unsigned int i; static const struct { const char *zName; int nArg; @@ -1932,12 +2005,10 @@ } aMod[] = { { "json_each", &jsonEachModule }, { "json_tree", &jsonTreeModule }, }; #endif - SQLITE_EXTENSION_INIT2(pApi); - (void)pzErrMsg; /* Unused parameter */ for(i=0; irc==SQLITE_OK ){ p->rc = pReal->pMethods->xLock(pReal, SQLITE_LOCK_EXCLUSIVE); } } +#if defined(_WIN32_WCE) +static LPWSTR rbuWinUtf8ToUnicode(const char *zFilename){ + int nChar; + LPWSTR zWideFilename; + + nChar = MultiByteToWideChar(CP_UTF8, 0, zFilename, -1, NULL, 0); + if( nChar==0 ){ + return 0; + } + zWideFilename = sqlite3_malloc( nChar*sizeof(zWideFilename[0]) ); + if( zWideFilename==0 ){ + return 0; + } + memset(zWideFilename, 0, nChar*sizeof(zWideFilename[0])); + nChar = MultiByteToWideChar(CP_UTF8, 0, zFilename, -1, zWideFilename, + nChar); + if( nChar==0 ){ + sqlite3_free(zWideFilename); + zWideFilename = 0; + } + return zWideFilename; +} +#endif + /* ** The RBU handle is currently in RBU_STAGE_OAL state, with a SHARED lock ** on the database file. This proc moves the *-oal file to the *-wal path, ** then reopens the database file (this time in vanilla, non-oal, WAL mode). ** If an error occurs, leave an error code and error message in the rbu @@ -2414,14 +2442,41 @@ /* Re-open the databases. */ rbuObjIterFinalize(&p->objiter); sqlite3_close(p->dbMain); sqlite3_close(p->dbRbu); + p->dbMain = 0; + p->dbRbu = 0; + +#if defined(_WIN32_WCE) + { + LPWSTR zWideOal; + LPWSTR zWideWal; + + zWideOal = rbuWinUtf8ToUnicode(zOal); + if( zWideOal ){ + zWideWal = rbuWinUtf8ToUnicode(zWal); + if( zWideWal ){ + if( MoveFileW(zWideOal, zWideWal) ){ + p->rc = SQLITE_OK; + }else{ + p->rc = SQLITE_IOERR; + } + sqlite3_free(zWideWal); + }else{ + p->rc = SQLITE_IOERR_NOMEM; + } + sqlite3_free(zWideOal); + }else{ + p->rc = SQLITE_IOERR_NOMEM; + } + } +#else p->rc = rename(zOal, zWal) ? SQLITE_IOERR : SQLITE_OK; +#endif + if( p->rc==SQLITE_OK ){ - p->dbMain = 0; - p->dbRbu = 0; rbuOpenDatabase(p); rbuSetupCheckpoint(p, 0); } } } Index: ext/rbu/sqlite3rbu.h ================================================================== --- ext/rbu/sqlite3rbu.h +++ ext/rbu/sqlite3rbu.h @@ -266,10 +266,14 @@ #ifndef _SQLITE3RBU_H #define _SQLITE3RBU_H #include "sqlite3.h" /* Required for error code definitions */ + +#ifdef __cplusplus +extern "C" { +#endif typedef struct sqlite3rbu sqlite3rbu; /* ** Open an RBU handle. @@ -444,7 +448,11 @@ ** VFS objects are not reference counted. If a VFS object is destroyed ** before all database handles that use it have been closed, the results ** are undefined. */ void sqlite3rbu_destroy_vfs(const char *zName); + +#ifdef __cplusplus +} /* end of the 'extern "C"' block */ +#endif #endif /* _SQLITE3RBU_H */ Index: ext/rtree/rtree1.test ================================================================== --- ext/rtree/rtree1.test +++ ext/rtree/rtree1.test @@ -32,10 +32,15 @@ # rtree-7.*: Test renaming an r-tree table. # rtree-8.*: Test constrained scans of r-tree data. # # rtree-12.*: Test that on-conflict clauses are supported. # rtree-13.*: Test that bug [d2889096e7bdeac6d] has been fixed. +# rtree-14.*: Test if a non-integer is inserted into the PK column of an +# r-tree table, it is converted to an integer before being +# inserted. Also that if a non-numeric is inserted into one +# of the min/max dimension columns, it is converted to the +# required type before being inserted. # ifcapable !rtree { finish_test return @@ -532,7 +537,57 @@ SELECT 2 UNION ALL SELECT 3 ) SELECT * FROM r CROSS JOIN t9 WHERE id=x; } {1 1 0.0 0.0 2 2 0.0 0.0} + +#------------------------------------------------------------------------- +# Test if a non-integer is inserted into the PK column of an r-tree +# table, it is converted to an integer before being inserted. Also +# that if a non-numeric is inserted into one of the min/max dimension +# columns, it is converted to the required type before being inserted. +# +do_execsql_test 14.1 { + CREATE VIRTUAL TABLE t10 USING rtree(ii, x1, x2); +} + +do_execsql_test 14.2 { + INSERT INTO t10 VALUES(NULL, 1, 2); + INSERT INTO t10 VALUES(NULL, 2, 3); + INSERT INTO t10 VALUES('4xxx', 3, 4); + INSERT INTO t10 VALUES(5.0, 4, 5); + INSERT INTO t10 VALUES(6.4, 5, 6); +} +do_execsql_test 14.3 { + SELECT * FROM t10; +} { + 1 1.0 2.0 2 2.0 3.0 4 3.0 4.0 5 4.0 5.0 6 5.0 6.0 +} + +do_execsql_test 14.4 { + DELETE FROM t10; + INSERT INTO t10 VALUES(1, 'one', 'two'); + INSERT INTO t10 VALUES(2, '52xyz', '81...'); +} +do_execsql_test 14.5 { + SELECT * FROM t10; +} { + 1 0.0 0.0 + 2 52.0 81.0 +} + +do_execsql_test 14.4 { + DROP TABLE t10; + CREATE VIRTUAL TABLE t10 USING rtree_i32(ii, x1, x2); + INSERT INTO t10 VALUES(1, 'one', 'two'); + INSERT INTO t10 VALUES(2, '52xyz', '81...'); + INSERT INTO t10 VALUES(3, 42.3, 49.9); +} +do_execsql_test 14.5 { + SELECT * FROM t10; +} { + 1 0 0 + 2 52 81 + 3 42 49 +} finish_test Index: main.mk ================================================================== --- main.mk +++ main.mk @@ -9,10 +9,12 @@ # BCC C Compiler and options for use in building executables that # will run on the platform that is doing the build. # # THREADLIB Specify any extra linker options needed to make the library # thread safe +# +# LIBS Extra libraries options # # OPTS Extra compiler command-line options. # # EXE The suffix to add to executable files. ".exe" for windows # and "" for Unix. @@ -33,13 +35,10 @@ # readline() library. # # LIBREADLINE Linker options needed by programs using readline() must # link against. # -# NAWK Nawk compatible awk program. Older (obsolete?) solaris -# systems need this to avoid using the original AT&T AWK. -# # Once the macros above are defined, the rest of this make script will # build the SQLite library and testing tools. ################################################################################ # This is how we compile @@ -46,23 +45,25 @@ # TCCX = $(TCC) $(OPTS) -I. -I$(TOP)/src -I$(TOP) TCCX += -I$(TOP)/ext/rtree -I$(TOP)/ext/icu -I$(TOP)/ext/fts3 TCCX += -I$(TOP)/ext/async -I$(TOP)/ext/userauth TCCX += -I$(TOP)/ext/fts5 +THREADLIB += $(LIBS) # Object files for the SQLite library. # LIBOBJ+= vdbe.o parse.o \ alter.o analyze.o attach.o auth.o \ backup.o bitvec.o btmutex.o btree.o build.o \ - callback.o complete.o ctime.o date.o dbstat.o delete.o expr.o fault.o fkey.o \ + callback.o complete.o ctime.o date.o dbstat.o delete.o expr.o \ + fault.o fkey.o \ fts3.o fts3_aux.o fts3_expr.o fts3_hash.o fts3_icu.o fts3_porter.o \ fts3_snippet.o fts3_tokenizer.o fts3_tokenizer1.o \ fts3_tokenize_vtab.o \ fts3_unicode.o fts3_unicode2.o \ - fts3_write.o func.o global.o hash.o \ - icu.o insert.o journal.o legacy.o loadext.o \ + fts3_write.o fts5.o func.o global.o hash.o \ + icu.o insert.o journal.o json1.o legacy.o loadext.o \ main.o malloc.o mem0.o mem1.o mem2.o mem3.o mem5.o \ memjournal.o \ mutex.o mutex_noop.o mutex_unix.o mutex_w32.o \ notify.o opcodes.o os.o os_unix.o os_win.o \ pager.o pcache.o pcache1.o pragma.o prepare.o printf.o \ @@ -223,14 +224,15 @@ $(TOP)/ext/rtree/rtree.h \ $(TOP)/ext/rtree/rtree.c SRC += \ $(TOP)/ext/userauth/userauth.c \ $(TOP)/ext/userauth/sqlite3userauth.h - SRC += \ $(TOP)/ext/rbu/sqlite3rbu.c \ $(TOP)/ext/rbu/sqlite3rbu.h +SRC += \ + $(TOP)/ext/misc/json1.c # FTS5 things # FTS5_HDR = \ @@ -319,22 +321,20 @@ $(TOP)/ext/misc/closure.c \ $(TOP)/ext/misc/eval.c \ $(TOP)/ext/misc/fileio.c \ $(TOP)/ext/misc/fuzzer.c \ $(TOP)/ext/misc/ieee754.c \ - $(TOP)/ext/misc/json1.c \ $(TOP)/ext/misc/nextchar.c \ $(TOP)/ext/misc/percentile.c \ $(TOP)/ext/misc/regexp.c \ $(TOP)/ext/misc/series.c \ $(TOP)/ext/misc/spellfix.c \ $(TOP)/ext/misc/totype.c \ $(TOP)/ext/misc/wholenumber.c \ $(TOP)/ext/misc/vfslog.c \ $(TOP)/ext/fts5/fts5_tcl.c \ - $(TOP)/ext/fts5/fts5_test_mi.c \ - fts5.c + $(TOP)/ext/fts5/fts5_test_mi.c #TESTSRC += $(TOP)/ext/fts2/fts2_tokenizer.c #TESTSRC += $(TOP)/ext/fts3/fts3_tokenizer.c @@ -377,11 +377,11 @@ $(TOP)/ext/fts3/fts3.c \ $(TOP)/ext/fts3/fts3_aux.c \ $(TOP)/ext/fts3/fts3_expr.c \ $(TOP)/ext/fts3/fts3_tokenizer.c \ $(TOP)/ext/fts3/fts3_write.c \ - $(TOP)/ext/async/sqlite3async.c + $(TOP)/ext/async/sqlite3async.c # Header files used by all library source files. # HDR = \ $(TOP)/src/btree.h \ @@ -449,19 +449,19 @@ $(TOP)/test/fuzzdata1.db \ $(TOP)/test/fuzzdata2.db \ $(TOP)/test/fuzzdata3.db \ $(TOP)/test/fuzzdata4.db -# Extra arguments for including json1 in the build of tools -# -JSON1_DEP = $(TOP)/ext/misc/json1.c sqlite3ext.h -JSON1_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_CORE -JSON1_SRC = $(TOP)/ext/misc/json1.c - # Standard options to testfixture # TESTOPTS = --verbose=file --output=test-out.txt + +# Extra compiler options for various shell tools +# +SHELL_OPT = -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_FTS5 +FUZZERSHELL_OPT = -DSQLITE_ENABLE_JSON1 +FUZZCHECK_OPT = -DSQLITE_ENABLE_JSON1 # This is the default Makefile target. The objects listed here # are what get build when you type just "make" with no arguments. # all: sqlite3.h libsqlite3.a sqlite3$(EXE) @@ -468,28 +468,27 @@ libsqlite3.a: $(LIBOBJ) $(AR) libsqlite3.a $(LIBOBJ) $(RANLIB) libsqlite3.a -sqlite3$(EXE): $(TOP)/src/shell.c libsqlite3.a sqlite3.h $(JSON1_DEP) - $(TCCX) $(READLINE_FLAGS) $(JSON1_OPT) -o sqlite3$(EXE) \ - $(TOP)/src/shell.c $(JSON1_SRC) \ - libsqlite3.a $(LIBREADLINE) $(TLIBS) $(THREADLIB) +sqlite3$(EXE): $(TOP)/src/shell.c libsqlite3.a sqlite3.h + $(TCCX) $(READLINE_FLAGS) -o sqlite3$(EXE) $(SHELL_OPT) \ + $(TOP)/src/shell.c libsqlite3.a $(LIBREADLINE) $(TLIBS) $(THREADLIB) sqldiff$(EXE): $(TOP)/tool/sqldiff.c sqlite3.c sqlite3.h $(TCCX) -o sqldiff$(EXE) -DSQLITE_THREADSAFE=0 \ $(TOP)/tool/sqldiff.c sqlite3.c $(TLIBS) $(THREADLIB) -fuzzershell$(EXE): $(TOP)/tool/fuzzershell.c sqlite3.c sqlite3.h $(JSON1_DEP) +fuzzershell$(EXE): $(TOP)/tool/fuzzershell.c sqlite3.c sqlite3.h $(TCCX) -o fuzzershell$(EXE) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \ - $(JSON1_OPT) $(TOP)/tool/fuzzershell.c $(JSON1_SRC) sqlite3.c \ + $(FUZZERSHELL_OPT) $(TOP)/tool/fuzzershell.c sqlite3.c \ $(TLIBS) $(THREADLIB) -fuzzcheck$(EXE): $(TOP)/test/fuzzcheck.c sqlite3.c sqlite3.h $(JSON1_DEP) +fuzzcheck$(EXE): $(TOP)/test/fuzzcheck.c sqlite3.c sqlite3.h $(TCCX) -o fuzzcheck$(EXE) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION \ - -DSQLITE_ENABLE_MEMSYS5 $(JSON1_OPT) \ - $(TOP)/test/fuzzcheck.c $(JSON1_SRC) sqlite3.c $(TLIBS) $(THREADLIB) + -DSQLITE_ENABLE_MEMSYS5 $(FUZZCHECK_OPT) \ + $(TOP)/test/fuzzcheck.c sqlite3.c $(TLIBS) $(THREADLIB) mptester$(EXE): sqlite3.c $(TOP)/mptest/mptest.c $(TCCX) -o $@ -I. $(TOP)/mptest/mptest.c sqlite3.c \ $(TLIBS) $(THREADLIB) @@ -513,26 +512,30 @@ # copies of all of the C source code and header files needed to # build on the target system. Some of the C source code and header # files are automatically generated. This target takes care of # all that automatic generation. # -target_source: $(SRC) $(TOP)/tool/vdbe-compress.tcl +target_source: $(SRC) $(TOP)/tool/vdbe-compress.tcl fts5.c rm -rf tsrc mkdir tsrc cp -f $(SRC) tsrc rm tsrc/sqlite.h.in tsrc/parse.y tclsh $(TOP)/tool/vdbe-compress.tcl $(OPTS) vdbe.new mv vdbe.new tsrc/vdbe.c + cp fts5.c fts5.h tsrc touch target_source sqlite3.c: target_source $(TOP)/tool/mksqlite3c.tcl tclsh $(TOP)/tool/mksqlite3c.tcl cp tsrc/shell.c tsrc/sqlite3ext.h . echo '#ifndef USE_SYSTEM_SQLITE' >tclsqlite3.c cat sqlite3.c >>tclsqlite3.c echo '#endif /* USE_SYSTEM_SQLITE */' >>tclsqlite3.c cat $(TOP)/src/tclsqlite.c >>tclsqlite3.c + +sqlite3ext.h: target_source + cp tsrc/sqlite3ext.h . sqlite3.c-debug: target_source $(TOP)/tool/mksqlite3c.tcl tclsh $(TOP)/tool/mksqlite3c.tcl --linemacros echo '#ifndef USE_SYSTEM_SQLITE' >tclsqlite3.c cat sqlite3.c >>tclsqlite3.c @@ -574,27 +577,27 @@ # Rules to build opcodes.c and opcodes.h # -opcodes.c: opcodes.h $(TOP)/mkopcodec.awk - $(NAWK) -f $(TOP)/mkopcodec.awk opcodes.h >opcodes.c +opcodes.c: opcodes.h $(TOP)/tool/mkopcodec.tcl + tclsh $(TOP)/tool/mkopcodec.tcl opcodes.h >opcodes.c -opcodes.h: parse.h $(TOP)/src/vdbe.c $(TOP)/mkopcodeh.awk +opcodes.h: parse.h $(TOP)/src/vdbe.c $(TOP)/tool/mkopcodeh.tcl cat parse.h $(TOP)/src/vdbe.c | \ - $(NAWK) -f $(TOP)/mkopcodeh.awk >opcodes.h + tclsh $(TOP)/tool/mkopcodeh.tcl >opcodes.h # Rules to build parse.c and parse.h - the outputs of lemon. # parse.h: parse.c -parse.c: $(TOP)/src/parse.y lemon $(TOP)/addopcodes.awk +parse.c: $(TOP)/src/parse.y lemon $(TOP)/tool/addopcodes.tcl cp $(TOP)/src/parse.y . rm -f parse.h ./lemon -s $(OPTS) parse.y mv parse.h parse.h.temp - $(NAWK) -f $(TOP)/addopcodes.awk parse.h.temp >parse.h + tclsh $(TOP)/tool/addopcodes.tcl parse.h.temp >parse.h sqlite3.h: $(TOP)/src/sqlite.h.in $(TOP)/manifest.uuid $(TOP)/VERSION $(TOP)/ext/rtree/sqlite3rtree.h tclsh $(TOP)/tool/mksqlite3h.tcl $(TOP) >sqlite3.h keywordhash.h: $(TOP)/tool/mkkeywordhash.c @@ -663,12 +666,20 @@ $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_unicode2.c fts3_write.o: $(TOP)/ext/fts3/fts3_write.c $(HDR) $(EXTHDR) $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_write.c +fts5.o: fts5.c + $(TCCX) -DSQLITE_CORE -c fts5.c + +json1.o: $(TOP)/ext/misc/json1.c + $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/misc/json1.c + rtree.o: $(TOP)/ext/rtree/rtree.c $(HDR) $(EXTHDR) $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/rtree/rtree.c + + fts5parse.c: $(TOP)/ext/fts5/fts5parse.y lemon cp $(TOP)/ext/fts5/fts5parse.y . rm -f fts5parse.h ./lemon $(OPTS) fts5parse.y @@ -696,11 +707,11 @@ echo "#define TCLSH 2" > $@ echo "#define SQLITE_ENABLE_DBSTAT_VTAB 1" >> $@ cat sqlite3.c $(TOP)/src/tclsqlite.c >> $@ echo "static const char *tclsh_main_loop(void){" >> $@ echo "static const char *zMainloop = " >> $@ - $(NAWK) -f $(TOP)/tool/tostr.awk $(TOP)/tool/spaceanal.tcl >> $@ + tclsh $(TOP)/tool/tostr.tcl $(TOP)/tool/spaceanal.tcl >> $@ echo "; return zMainloop; }" >> $@ sqlite3_analyzer$(EXE): sqlite3_analyzer.c $(TCCX) $(TCL_FLAGS) sqlite3_analyzer.c -o $@ $(LIBTCL) $(THREADLIB) @@ -712,13 +723,13 @@ testfixture$(EXE): $(TESTSRC2) libsqlite3.a $(TESTSRC) $(TOP)/src/tclsqlite.c $(TCCX) $(TCL_FLAGS) -DTCLSH=1 $(TESTFIXTURE_FLAGS) \ $(TESTSRC) $(TESTSRC2) $(TOP)/src/tclsqlite.c \ -o testfixture$(EXE) $(LIBTCL) libsqlite3.a $(THREADLIB) -amalgamation-testfixture$(EXE): sqlite3.c fts5.c $(TESTSRC) $(TOP)/src/tclsqlite.c +amalgamation-testfixture$(EXE): sqlite3.c $(TESTSRC) $(TOP)/src/tclsqlite.c $(TCCX) $(TCL_FLAGS) -DTCLSH=1 $(TESTFIXTURE_FLAGS) \ - $(TESTSRC) $(TOP)/src/tclsqlite.c sqlite3.c fts5.c \ + $(TESTSRC) $(TOP)/src/tclsqlite.c sqlite3.c \ -o testfixture$(EXE) $(LIBTCL) $(THREADLIB) fts3-testfixture$(EXE): sqlite3.c fts3amal.c $(TESTSRC) $(TOP)/src/tclsqlite.c $(TCCX) $(TCL_FLAGS) -DTCLSH=1 $(TESTFIXTURE_FLAGS) \ -DSQLITE_ENABLE_FTS3=1 \ DELETED mkopcodec.awk Index: mkopcodec.awk ================================================================== --- mkopcodec.awk +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/awk -f -# -# This AWK script scans the opcodes.h file (which is itself generated by -# another awk script) and uses the information gleaned to create the -# opcodes.c source file. -# -# Opcodes.c contains strings which are the symbolic names for the various -# opcodes used by the VDBE. These strings are used when disassembling a -# VDBE program during tracing or as a result of the EXPLAIN keyword. -# -BEGIN { - print "/* Automatically generated. Do not edit */" - print "/* See the mkopcodec.awk script for details. */" - printf "#if !defined(SQLITE_OMIT_EXPLAIN)" - printf " || defined(VDBE_PROFILE)" - print " || defined(SQLITE_DEBUG)" - print "#if defined(SQLITE_ENABLE_EXPLAIN_COMMENTS) || defined(SQLITE_DEBUG)" - print "# define OpHelp(X) \"\\0\" X" - print "#else" - print "# define OpHelp(X)" - print "#endif" - print "const char *sqlite3OpcodeName(int i){" - print " static const char *const azName[] = { \"?\"," - mx = 0 -} -/^.define OP_/ { - sub("OP_","",$2) - i = $3+0 - label[i] = $2 - if( mx=0 ) continue; - if( name=="OP_Transaction" \ - || name=="OP_AutoCommit" \ - || name=="OP_Savepoint" \ - || name=="OP_Checkpoint" \ - || name=="OP_Vacuum" \ - || name=="OP_JournalMode" \ - || name=="OP_VUpdate" \ - || name=="OP_VFilter" \ - || name=="OP_Next" \ - || name=="OP_NextIfOpen" \ - || name=="OP_SorterNext" \ - || name=="OP_Prev" \ - || name=="OP_PrevIfOpen" \ - ){ - cnt++ - while( used[cnt] ) cnt++ - op[name] = cnt - used[cnt] = 1 - def[cnt] = name - } - } - - # Generate the numeric values for opcodes - for(i=0; idb; b.pSrc = pFrom; b.pDest = pTo; b.iNext = 1; + +#ifdef SQLITE_HAS_CODEC + sqlite3PagerAlignReserve(sqlite3BtreePager(pTo), sqlite3BtreePager(pFrom)); +#endif /* 0x7FFFFFFF is the hard limit for the number of pages in a database ** file. By passing this as the number of pages to copy to ** sqlite3_backup_step(), we can guarantee that the copy finishes ** within a single call (unless an error occurs). The assert() statement Index: src/btree.c ================================================================== --- src/btree.c +++ src/btree.c @@ -659,11 +659,11 @@ if( rc==SQLITE_OK ){ btreeReleaseAllCursorPages(pCur); pCur->eState = CURSOR_REQUIRESEEK; } - invalidateOverflowCache(pCur); + pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl|BTCF_AtLast); return rc; } /* Forward reference */ static int SQLITE_NOINLINE saveCursorsOnList(BtCursor*,Pgno,BtCursor*); @@ -6497,11 +6497,17 @@ if( (aData[1]==0 && aData[2]==0) || (pSlot = pageFindSlot(pPg,sz,&rc))==0 ){ pData -= sz; if( pDataapCell[i], sz); + /* pSlot and pCArray->apCell[i] will never overlap on a well-formed + ** database. But they might for a corrupt database. Hence use memmove() + ** since memcpy() sends SIGABORT with overlapping buffers on OpenBSD */ + assert( (pSlot+sz)<=pCArray->apCell[i] + || pSlot>=(pCArray->apCell[i]+sz) + || CORRUPT_DB ); + memmove(pSlot, pCArray->apCell[i], sz); put2byte(pCellptr, (pSlot - aData)); pCellptr += 2; } *ppData = pData; return 0; @@ -7622,11 +7628,11 @@ ** It is critical that the child page be defragmented before being ** copied into the parent, because if the parent is page 1 then it will ** by smaller than the child due to the database header, and so all the ** free space needs to be up front. */ - assert( nNew==1 ); + assert( nNew==1 || CORRUPT_DB ); rc = defragmentPage(apNew[0]); testcase( rc!=SQLITE_OK ); assert( apNew[0]->nFree == (get2byte(&apNew[0]->aData[5])-apNew[0]->cellOffset-apNew[0]->nCell*2) || rc!=SQLITE_OK Index: src/build.c ================================================================== --- src/build.c +++ src/build.c @@ -190,10 +190,12 @@ DbMaskTest(pParse->writeMask,iDb), /* P2 */ pParse->cookieValue[iDb], /* P3 */ db->aDb[iDb].pSchema->iGeneration /* P4 */ ); if( db->init.busy==0 ) sqlite3VdbeChangeP5(v, 1); + VdbeComment((v, + "usesStmtJournal=%d", pParse->mayAbort && pParse->isMultiWrite)); } #ifndef SQLITE_OMIT_VIRTUALTABLE for(i=0; inVtabLock; i++){ char *vtab = (char *)sqlite3GetVTable(db, pParse->apVtabLock[i]); sqlite3VdbeAddOp4(v, OP_VBegin, 0, 0, 0, vtab, P4_VTAB); @@ -981,11 +983,11 @@ ** indices to be created and the table record must come before the ** indices. Hence, the record number for the table must be allocated ** now. */ if( !db->init.busy && (v = sqlite3GetVdbe(pParse))!=0 ){ - int j1; + int addr1; int fileFormat; int reg1, reg2, reg3; /* nullRow[] is an OP_Record encoding of a row containing 5 NULLs */ static const char nullRow[] = { 6, 0, 0, 0, 0, 0 }; sqlite3BeginWriteOperation(pParse, 1, iDb); @@ -1002,18 +1004,18 @@ reg1 = pParse->regRowid = ++pParse->nMem; reg2 = pParse->regRoot = ++pParse->nMem; reg3 = ++pParse->nMem; sqlite3VdbeAddOp3(v, OP_ReadCookie, iDb, reg3, BTREE_FILE_FORMAT); sqlite3VdbeUsesBtree(v, iDb); - j1 = sqlite3VdbeAddOp1(v, OP_If, reg3); VdbeCoverage(v); + addr1 = sqlite3VdbeAddOp1(v, OP_If, reg3); VdbeCoverage(v); fileFormat = (db->flags & SQLITE_LegacyFileFmt)!=0 ? 1 : SQLITE_MAX_FILE_FORMAT; sqlite3VdbeAddOp2(v, OP_Integer, fileFormat, reg3); sqlite3VdbeAddOp3(v, OP_SetCookie, iDb, BTREE_FILE_FORMAT, reg3); sqlite3VdbeAddOp2(v, OP_Integer, ENC(db), reg3); sqlite3VdbeAddOp3(v, OP_SetCookie, iDb, BTREE_TEXT_ENCODING, reg3); - sqlite3VdbeJumpHere(v, j1); + sqlite3VdbeJumpHere(v, addr1); /* This just creates a place-holder record in the sqlite_master table. ** The record created does not contain anything yet. It will be replaced ** by the real entry in code generated at sqlite3EndTable(). ** @@ -2080,12 +2082,11 @@ int iDb; sqlite3 *db = pParse->db; if( pParse->nVar>0 ){ sqlite3ErrorMsg(pParse, "parameters are not allowed in views"); - sqlite3SelectDelete(db, pSelect); - return; + goto create_view_fail; } sqlite3StartTable(pParse, pName1, pName2, isTemp, 1, 0, noErr); p = pParse->pNewTable; if( p==0 || pParse->nErr ) goto create_view_fail; sqlite3TwoPartName(pParse, pName1, pName2, &pName); @@ -3133,11 +3134,11 @@ } /* Analyze the list of expressions that form the terms of the index and ** report any errors. In the common case where the expression is exactly ** a table column, store that column in aiColumn[]. For general expressions, - ** populate pIndex->aColExpr and store -2 in aiColumn[]. + ** populate pIndex->aColExpr and store XN_EXPR (-2) in aiColumn[]. ** ** TODO: Issue a warning if two or more columns of the index are identical. ** TODO: Issue a warning if the table primary key is used as part of the ** index key. */ @@ -3162,12 +3163,12 @@ if( !db->mallocFailed ){ assert( pCopy!=0 ); pListItem = &pCopy->a[i]; } } - j = -2; - pIndex->aiColumn[i] = -2; + j = XN_EXPR; + pIndex->aiColumn[i] = XN_EXPR; pIndex->uniqNotNull = 0; }else{ j = pCExpr->iColumn; assert( j<=0x7fff ); if( j<0 ){ @@ -3216,11 +3217,11 @@ i++; } } assert( i==pIndex->nColumn ); }else{ - pIndex->aiColumn[i] = -1; + pIndex->aiColumn[i] = XN_ROWID; pIndex->azColl[i] = "BINARY"; } sqlite3DefaultRowEst(pIndex); if( pParse->pNewTable==0 ) estimateIndexWidth(pIndex); Index: src/ctime.c ================================================================== --- src/ctime.c +++ src/ctime.c @@ -93,16 +93,22 @@ #if SQLITE_ENABLE_FTS3_PARENTHESIS "ENABLE_FTS3_PARENTHESIS", #endif #if SQLITE_ENABLE_FTS4 "ENABLE_FTS4", +#endif +#if SQLITE_ENABLE_FTS5 + "ENABLE_FTS5", #endif #if SQLITE_ENABLE_ICU "ENABLE_ICU", #endif #if SQLITE_ENABLE_IOTRACE "ENABLE_IOTRACE", +#endif +#if SQLITE_ENABLE_JSON1 + "ENABLE_JSON1", #endif #if SQLITE_ENABLE_LOAD_EXTENSION "ENABLE_LOAD_EXTENSION", #endif #if SQLITE_ENABLE_LOCKING_STYLE Index: src/delete.c ================================================================== --- src/delete.c +++ src/delete.c @@ -409,11 +409,11 @@ ** ONEPASS_MULTI: One-pass approach - any number of rows may be deleted. */ pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, 0, 0, wcf, iTabCur+1); if( pWInfo==0 ) goto delete_from_cleanup; eOnePass = sqlite3WhereOkOnePass(pWInfo, aiCurOnePass); - assert( IsVirtual(pTab)==0 || eOnePass==ONEPASS_OFF ); + assert( IsVirtual(pTab)==0 || eOnePass!=ONEPASS_MULTI ); assert( IsVirtual(pTab) || bComplex || eOnePass!=ONEPASS_OFF ); /* Keep track of the number of rows to be deleted */ if( db->flags & SQLITE_CountRows ){ sqlite3VdbeAddOp2(v, OP_AddImm, memCnt, 1); @@ -420,11 +420,11 @@ } /* Extract the rowid or primary key for the current row */ if( pPk ){ for(i=0; iaiColumn[i]>=(-1) ); + assert( pPk->aiColumn[i]>=0 ); sqlite3ExprCodeGetColumnOfTable(v, pTab, iTabCur, pPk->aiColumn[i], iPk+i); } iKey = iPk; }else{ @@ -492,11 +492,11 @@ /* Set up a loop over the rowids/primary-keys that were found in the ** where-clause loop above. */ if( eOnePass!=ONEPASS_OFF ){ assert( nKey==nPk ); /* OP_Found will use an unpacked key */ - if( aToOpen[iDataCur-iTabCur] ){ + if( !IsVirtual(pTab) && aToOpen[iDataCur-iTabCur] ){ assert( pPk!=0 || pTab->pSelect!=0 ); sqlite3VdbeAddOp4Int(v, OP_NotFound, iDataCur, addrBypass, iKey, nKey); VdbeCoverage(v); } }else if( pPk ){ @@ -514,11 +514,15 @@ if( IsVirtual(pTab) ){ const char *pVTab = (const char *)sqlite3GetVTable(db, pTab); sqlite3VtabMakeWritable(pParse, pTab); sqlite3VdbeAddOp4(v, OP_VUpdate, 0, 1, iKey, pVTab, P4_VTAB); sqlite3VdbeChangeP5(v, OE_Abort); + assert( eOnePass==ONEPASS_OFF || eOnePass==ONEPASS_SINGLE ); sqlite3MayAbort(pParse); + if( eOnePass==ONEPASS_SINGLE && sqlite3IsToplevel(pParse) ){ + pParse->isMultiWrite = 0; + } }else #endif { int count = (pParse->nested==0); /* True to count changes */ int iIdxNoSeek = -1; @@ -852,11 +856,11 @@ regBase = sqlite3GetTempRange(pParse, nCol); if( pPrior && (regBase!=regPrior || pPrior->pPartIdxWhere) ) pPrior = 0; for(j=0; jaiColumn[j]==pIdx->aiColumn[j] - && pPrior->aiColumn[j]>=(-1) + && pPrior->aiColumn[j]!=XN_EXPR ){ /* This column was already computed by the previous index */ continue; } sqlite3ExprCodeLoadIndexColumn(pParse, pIdx, iDataCur, j, regBase+j); Index: src/expr.c ================================================================== --- src/expr.c +++ src/expr.c @@ -1593,17 +1593,17 @@ ** it contains any NULL entries. Cause the register at regHasNull to be set ** to a non-NULL value if iCur contains no NULLs. Cause register regHasNull ** to be set to NULL if iCur contains one or more NULL values. */ static void sqlite3SetHasNullFlag(Vdbe *v, int iCur, int regHasNull){ - int j1; + int addr1; sqlite3VdbeAddOp2(v, OP_Integer, 0, regHasNull); - j1 = sqlite3VdbeAddOp1(v, OP_Rewind, iCur); VdbeCoverage(v); + addr1 = sqlite3VdbeAddOp1(v, OP_Rewind, iCur); VdbeCoverage(v); sqlite3VdbeAddOp3(v, OP_Column, iCur, 0, regHasNull); sqlite3VdbeChangeP5(v, OPFLAG_TYPEOFARG); VdbeComment((v, "first_entry_in(%d)", iCur)); - sqlite3VdbeJumpHere(v, j1); + sqlite3VdbeJumpHere(v, addr1); } #ifndef SQLITE_OMIT_SUBQUERY /* @@ -2199,24 +2199,24 @@ }else{ /* In this branch, the RHS of the IN might contain a NULL and ** the presence of a NULL on the RHS makes a difference in the ** outcome. */ - int j1; + int addr1; /* First check to see if the LHS is contained in the RHS. If so, ** then the answer is TRUE the presence of NULLs in the RHS does ** not matter. If the LHS is not contained in the RHS, then the ** answer is NULL if the RHS contains NULLs and the answer is ** FALSE if the RHS is NULL-free. */ - j1 = sqlite3VdbeAddOp4Int(v, OP_Found, pExpr->iTable, 0, r1, 1); + addr1 = sqlite3VdbeAddOp4Int(v, OP_Found, pExpr->iTable, 0, r1, 1); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_IsNull, rRhsHasNull, destIfNull); VdbeCoverage(v); sqlite3VdbeGoto(v, destIfFalse); - sqlite3VdbeJumpHere(v, j1); + sqlite3VdbeJumpHere(v, addr1); } } } sqlite3ReleaseTempReg(pParse, r1); sqlite3ExprCachePop(pParse); @@ -2441,19 +2441,19 @@ int iTabCur, /* Cursor pointing to a table row */ int iIdxCol, /* The column of the index to be loaded */ int regOut /* Store the index column value in this register */ ){ i16 iTabCol = pIdx->aiColumn[iIdxCol]; - if( iTabCol>=(-1) ){ + if( iTabCol==XN_EXPR ){ + assert( pIdx->aColExpr ); + assert( pIdx->aColExpr->nExpr>iIdxCol ); + pParse->iSelfTab = iTabCur; + sqlite3ExprCode(pParse, pIdx->aColExpr->a[iIdxCol].pExpr, regOut); + }else{ sqlite3ExprCodeGetColumnOfTable(pParse->pVdbe, pIdx->pTable, iTabCur, iTabCol, regOut); - return; } - assert( pIdx->aColExpr ); - assert( pIdx->aColExpr->nExpr>iIdxCol ); - pParse->iSelfTab = iTabCur; - sqlite3ExprCode(pParse, pIdx->aColExpr->a[iIdxCol].pExpr, regOut); } /* ** Generate code to extract the value of the iCol-th column of a table. */ @@ -2479,13 +2479,16 @@ } } /* ** Generate code that will extract the iColumn-th column from -** table pTab and store the column value in a register. An effort -** is made to store the column value in register iReg, but this is -** not guaranteed. The location of the column value is returned. +** table pTab and store the column value in a register. +** +** An effort is made to store the column value in register iReg. This +** is not garanteeed for GetColumn() - the result can be stored in +** any register. But the result is guaranteed to land in register iReg +** for GetColumnToReg(). ** ** There must be an open cursor to pTab in iTable when this routine ** is called. If iColumn<0 then code is generated that extracts the rowid. */ int sqlite3ExprCodeGetColumn( @@ -2492,11 +2495,11 @@ Parse *pParse, /* Parsing and code generating context */ Table *pTab, /* Description of the table we are reading from */ int iColumn, /* Index of the table column */ int iTable, /* The cursor pointing to the table */ int iReg, /* Store results here */ - u8 p5 /* P5 value for OP_Column */ + u8 p5 /* P5 value for OP_Column + FLAGS */ ){ Vdbe *v = pParse->pVdbe; int i; struct yColCache *p; @@ -2514,10 +2517,21 @@ }else{ sqlite3ExprCacheStore(pParse, iTable, iColumn, iReg); } return iReg; } +void sqlite3ExprCodeGetColumnToReg( + Parse *pParse, /* Parsing and code generating context */ + Table *pTab, /* Description of the table we are reading from */ + int iColumn, /* Index of the table column */ + int iTable, /* The cursor pointing to the table */ + int iReg /* Store results here */ +){ + int r1 = sqlite3ExprCodeGetColumn(pParse, pTab, iColumn, iTable, iReg, 0); + if( r1!=iReg ) sqlite3VdbeAddOp2(pParse->pVdbe, OP_SCopy, r1, iReg); +} + /* ** Clear all column cache entries. */ void sqlite3ExprCacheClear(Parse *pParse){ Index: src/fkey.c ================================================================== --- src/fkey.c +++ src/fkey.c @@ -250,10 +250,12 @@ for(i=0; iaiColumn[i]; /* Index of column in parent tbl */ char *zDfltColl; /* Def. collation for column */ char *zIdxCol; /* Name of indexed column */ + if( iCol<0 ) break; /* No foreign keys against expression indexes */ + /* If the index uses a collation sequence that is different from ** the default collation sequence for the column, this index is ** unusable. Bail out early in this case. */ zDfltColl = pParent->aCol[iCol].zColl; if( !zDfltColl ){ @@ -402,10 +404,11 @@ if( pTab==pFKey->pFrom && nIncr==1 ){ int iJump = sqlite3VdbeCurrentAddr(v) + nCol + 1; for(i=0; iaiColumn[i]+1+regData; + assert( pIdx->aiColumn[i]>=0 ); assert( aiCol[i]!=pTab->iPKey ); if( pIdx->aiColumn[i]==pTab->iPKey ){ /* The parent key is a composite key that includes the IPK column */ iParent = regData; } @@ -610,10 +613,11 @@ Expr *pEq, *pAll = 0; Index *pPk = sqlite3PrimaryKeyIndex(pTab); assert( pIdx!=0 ); for(i=0; inKeyCol; i++){ i16 iCol = pIdx->aiColumn[i]; + assert( iCol>=0 ); pLeft = exprTableRegister(pParse, pTab, regData, iCol); pRight = exprTableColumn(db, pTab, pSrc->a[0].iCursor, iCol); pEq = sqlite3PExpr(pParse, TK_EQ, pLeft, pRight, 0); pAll = sqlite3ExprAnd(db, pAll, pEq); } @@ -929,10 +933,11 @@ } for(i=0; inCol; i++){ if( aiCol[i]==pTab->iPKey ){ aiCol[i] = -1; } + assert( pIdx==0 || pIdx->aiColumn[i]>=0 ); #ifndef SQLITE_OMIT_AUTHORIZATION /* Request permission to read the parent key columns. If the ** authorization callback returns SQLITE_IGNORE, behave as if any ** values read from the parent table are NULL. */ if( db->xAuth ){ @@ -1060,11 +1065,14 @@ } for(p=sqlite3FkReferences(pTab); p; p=p->pNextTo){ Index *pIdx = 0; sqlite3FkLocateIndex(pParse, pTab, p, &pIdx, 0); if( pIdx ){ - for(i=0; inKeyCol; i++) mask |= COLUMN_MASK(pIdx->aiColumn[i]); + for(i=0; inKeyCol; i++){ + assert( pIdx->aiColumn[i]>=0 ); + mask |= COLUMN_MASK(pIdx->aiColumn[i]); + } } } } return mask; } @@ -1183,10 +1191,11 @@ Expr *pEq; /* tFromCol = OLD.tToCol */ iFromCol = aiCol ? aiCol[i] : pFKey->aCol[0].iFrom; assert( iFromCol>=0 ); assert( pIdx!=0 || (pTab->iPKey>=0 && pTab->iPKeynCol) ); + assert( pIdx==0 || pIdx->aiColumn[i]>=0 ); tToCol.z = pTab->aCol[pIdx ? pIdx->aiColumn[i] : pTab->iPKey].zName; tFromCol.z = pFKey->pFrom->aCol[iFromCol].zName; tToCol.n = sqlite3Strlen30(tToCol.z); tFromCol.n = sqlite3Strlen30(tFromCol.z); Index: src/global.c ================================================================== --- src/global.c +++ src/global.c @@ -172,11 +172,10 @@ 0x7ffffffe, /* mxStrlen */ 0, /* neverCorrupt */ 128, /* szLookaside */ 500, /* nLookaside */ {0,0,0,0,0,0,0,0}, /* m */ - (void*)0, /* pMutex */ {0,0,0,0,0,0,0,0,0}, /* mutex */ {0,0,0,0,0,0,0,0,0,0,0,0,0},/* pcache2 */ (void*)0, /* pHeap */ 0, /* nHeap */ 0, 0, /* mnHeap, mxHeap */ Index: src/insert.c ================================================================== --- src/insert.c +++ src/insert.c @@ -88,15 +88,15 @@ } for(n=0; nnColumn; n++){ i16 x = pIdx->aiColumn[n]; if( x>=0 ){ pIdx->zColAff[n] = pTab->aCol[x].affinity; - }else if( x==(-1) ){ + }else if( x==XN_ROWID ){ pIdx->zColAff[n] = SQLITE_AFF_INTEGER; }else{ char aff; - assert( x==(-2) ); + assert( x==XN_EXPR ); assert( pIdx->aColExpr!=0 ); aff = sqlite3ExprAffinity(pIdx->aColExpr->a[n].pExpr); if( aff==0 ) aff = SQLITE_AFF_BLOB; pIdx->zColAff[n] = aff; } @@ -258,11 +258,11 @@ Vdbe *v = pParse->pVdbe; /* VDBE under construction */ /* This routine is never called during trigger-generation. It is ** only called from the top-level */ assert( pParse->pTriggerTab==0 ); - assert( pParse==sqlite3ParseToplevel(pParse) ); + assert( sqlite3IsToplevel(pParse) ); assert( v ); /* We failed long ago if this is not so */ for(p = pParse->pAinc; p; p = p->pNext){ pDb = &db->aDb[p->iDb]; memId = p->regCtr; @@ -311,20 +311,20 @@ sqlite3 *db = pParse->db; assert( v ); for(p = pParse->pAinc; p; p = p->pNext){ Db *pDb = &db->aDb[p->iDb]; - int j1; + int addr1; int iRec; int memId = p->regCtr; iRec = sqlite3GetTempReg(pParse); assert( sqlite3SchemaMutexHeld(db, 0, pDb->pSchema) ); sqlite3OpenTable(pParse, 0, p->iDb, pDb->pSchema->pSeqTab, OP_OpenWrite); - j1 = sqlite3VdbeAddOp1(v, OP_NotNull, memId+1); VdbeCoverage(v); + addr1 = sqlite3VdbeAddOp1(v, OP_NotNull, memId+1); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_NewRowid, 0, memId+1); - sqlite3VdbeJumpHere(v, j1); + sqlite3VdbeJumpHere(v, addr1); sqlite3VdbeAddOp3(v, OP_MakeRecord, memId-1, 2, iRec); sqlite3VdbeAddOp3(v, OP_Insert, 0, iRec, memId+1); sqlite3VdbeChangeP5(v, OPFLAG_APPEND); sqlite3VdbeAddOp0(v, OP_Close); sqlite3ReleaseTempReg(pParse, iRec); @@ -812,21 +812,21 @@ ** not happened yet) so we substitute a rowid of -1 */ if( ipkColumn<0 ){ sqlite3VdbeAddOp2(v, OP_Integer, -1, regCols); }else{ - int j1; + int addr1; assert( !withoutRowid ); if( useTempTable ){ sqlite3VdbeAddOp3(v, OP_Column, srcTab, ipkColumn, regCols); }else{ assert( pSelect==0 ); /* Otherwise useTempTable is true */ sqlite3ExprCode(pParse, pList->a[ipkColumn].pExpr, regCols); } - j1 = sqlite3VdbeAddOp1(v, OP_NotNull, regCols); VdbeCoverage(v); + addr1 = sqlite3VdbeAddOp1(v, OP_NotNull, regCols); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_Integer, -1, regCols); - sqlite3VdbeJumpHere(v, j1); + sqlite3VdbeJumpHere(v, addr1); sqlite3VdbeAddOp1(v, OP_MustBeInt, regCols); VdbeCoverage(v); } /* Cannot have triggers on a virtual table. If it were possible, ** this block would have to account for hidden column. @@ -896,18 +896,18 @@ } /* If the PRIMARY KEY expression is NULL, then use OP_NewRowid ** to generate a unique primary key value. */ if( !appendFlag ){ - int j1; + int addr1; if( !IsVirtual(pTab) ){ - j1 = sqlite3VdbeAddOp1(v, OP_NotNull, regRowid); VdbeCoverage(v); + addr1 = sqlite3VdbeAddOp1(v, OP_NotNull, regRowid); VdbeCoverage(v); sqlite3VdbeAddOp3(v, OP_NewRowid, iDataCur, regRowid, regAutoinc); - sqlite3VdbeJumpHere(v, j1); + sqlite3VdbeJumpHere(v, addr1); }else{ - j1 = sqlite3VdbeCurrentAddr(v); - sqlite3VdbeAddOp2(v, OP_IsNull, regRowid, j1+2); VdbeCoverage(v); + addr1 = sqlite3VdbeCurrentAddr(v); + sqlite3VdbeAddOp2(v, OP_IsNull, regRowid, addr1+2); VdbeCoverage(v); } sqlite3VdbeAddOp1(v, OP_MustBeInt, regRowid); VdbeCoverage(v); } }else if( IsVirtual(pTab) || withoutRowid ){ sqlite3VdbeAddOp2(v, OP_Null, 0, regRowid); @@ -1157,11 +1157,11 @@ sqlite3 *db; /* Database connection */ int i; /* loop counter */ int ix; /* Index loop counter */ int nCol; /* Number of columns */ int onError; /* Conflict resolution strategy */ - int j1; /* Address of jump instruction */ + int addr1; /* Address of jump instruction */ int seenReplace = 0; /* True if REPLACE is used to resolve INT PK conflict */ int nPkField; /* Number of fields in PRIMARY KEY. 1 for ROWID tables */ int ipkTop = 0; /* Top of the rowid change constraint check */ int ipkBottom = 0; /* Bottom of the rowid change constraint check */ u8 isUpdate; /* True if this is an UPDATE operation */ @@ -1228,13 +1228,14 @@ VdbeCoverage(v); break; } default: { assert( onError==OE_Replace ); - j1 = sqlite3VdbeAddOp1(v, OP_NotNull, regNewData+1+i); VdbeCoverage(v); + addr1 = sqlite3VdbeAddOp1(v, OP_NotNull, regNewData+1+i); + VdbeCoverage(v); sqlite3ExprCode(pParse, pTab->aCol[i].pDflt, regNewData+1+i); - sqlite3VdbeJumpHere(v, j1); + sqlite3VdbeJumpHere(v, addr1); break; } } } @@ -1406,24 +1407,24 @@ */ regIdx = sqlite3GetTempRange(pParse, pIdx->nColumn); for(i=0; inColumn; i++){ int iField = pIdx->aiColumn[i]; int x; - if( iField==(-2) ){ + if( iField==XN_EXPR ){ pParse->ckBase = regNewData+1; sqlite3ExprCode(pParse, pIdx->aColExpr->a[i].pExpr, regIdx+i); pParse->ckBase = 0; VdbeComment((v, "%s column %d", pIdx->zName, i)); }else{ - if( iField==(-1) || iField==pTab->iPKey ){ + if( iField==XN_ROWID || iField==pTab->iPKey ){ if( regRowid==regIdx+i ) continue; /* ROWID already in regIdx+i */ x = regNewData; regRowid = pIdx->pPartIdxWhere ? -1 : regIdx+i; }else{ x = iField + regNewData + 1; } - sqlite3VdbeAddOp2(v, OP_SCopy, x, regIdx+i); + sqlite3VdbeAddOp2(v, iField<0 ? OP_IntCopy : OP_SCopy, x, regIdx+i); VdbeComment((v, "%s", iField<0 ? "rowid" : pTab->aCol[iField].zName)); } } sqlite3VdbeAddOp3(v, OP_MakeRecord, regIdx, pIdx->nColumn, aRegIdx[ix]); VdbeComment((v, "for %s", pIdx->zName)); @@ -1471,10 +1472,11 @@ int x; /* Extract the PRIMARY KEY from the end of the index entry and ** store it in registers regR..regR+nPk-1 */ if( pIdx!=pPk ){ for(i=0; inKeyCol; i++){ + assert( pPk->aiColumn[i]>=0 ); x = sqlite3ColumnOfIndex(pIdx, pPk->aiColumn[i]); sqlite3VdbeAddOp3(v, OP_Column, iThisCur, x, regR+i); VdbeComment((v, "%s.%s", pTab->zName, pTab->aCol[pPk->aiColumn[i]].zName)); } @@ -1492,10 +1494,11 @@ int regCmp = (IsPrimaryKeyIndex(pIdx) ? regIdx : regR); for(i=0; inKeyCol; i++){ char *p4 = (char*)sqlite3LocateCollSeq(pParse, pPk->azColl[i]); x = pPk->aiColumn[i]; + assert( x>=0 ); if( i==(pPk->nKeyCol-1) ){ addrJump = addrUniqueOk; op = OP_Eq; } sqlite3VdbeAddOp4(v, op, @@ -1743,11 +1746,11 @@ } for(i=0; inKeyCol; i++){ if( pSrc->aiColumn[i]!=pDest->aiColumn[i] ){ return 0; /* Different columns indexed */ } - if( pSrc->aiColumn[i]==(-2) ){ + if( pSrc->aiColumn[i]==XN_EXPR ){ assert( pSrc->aColExpr!=0 && pDest->aColExpr!=0 ); if( sqlite3ExprCompare(pSrc->aColExpr->a[i].pExpr, pDest->aColExpr->a[i].pExpr, -1)!=0 ){ return 0; /* Different expressions in the index */ } Index: src/loadext.c ================================================================== --- src/loadext.c +++ src/loadext.c @@ -406,11 +406,11 @@ /* Version 3.8.11 and later */ (sqlite3_value*(*)(const sqlite3_value*))sqlite3_value_dup, sqlite3_value_free, sqlite3_result_zeroblob64, sqlite3_bind_zeroblob64, - /* Version 3.8.12 and later */ + /* Version 3.9.0 and later */ sqlite3_value_subtype, sqlite3_result_subtype }; /* Index: src/main.c ================================================================== --- src/main.c +++ src/main.c @@ -23,10 +23,16 @@ # include "rtree.h" #endif #ifdef SQLITE_ENABLE_ICU # include "sqliteicu.h" #endif +#ifdef SQLITE_ENABLE_JSON1 +int sqlite3Json1Init(sqlite3*); +#endif +#ifdef SQLITE_ENABLE_FTS5 +int sqlite3Fts5Init(sqlite3*); +#endif #ifndef SQLITE_AMALGAMATION /* IMPLEMENTATION-OF: R-46656-45156 The sqlite3_version[] string constant ** contains the text of SQLITE_VERSION macro. */ @@ -169,17 +175,11 @@ ** malloc subsystem - this implies that the allocation of a static ** mutex must not require support from the malloc subsystem. */ MUTEX_LOGIC( pMaster = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); ) sqlite3_mutex_enter(pMaster); - if( sqlite3GlobalConfig.isInit ){ - assert( sqlite3GlobalConfig.isMutexInit ); - assert( sqlite3GlobalConfig.isMallocInit ); - sqlite3_mutex_leave(pMaster); - return SQLITE_OK; - } - sqlite3GlobalConfig.isMutexInit = 1; /* possibly redundant */ + sqlite3GlobalConfig.isMutexInit = 1; if( !sqlite3GlobalConfig.isMallocInit ){ rc = sqlite3MallocInit(); } if( rc==SQLITE_OK ){ sqlite3GlobalConfig.isMallocInit = 1; @@ -333,21 +333,10 @@ if( sqlite3GlobalConfig.isMutexInit ){ sqlite3MutexEnd(); sqlite3GlobalConfig.isMutexInit = 0; } - /* - ** Force the state of the mutex subsystem to be completely reset now, even - ** if the configured xMutexEnd(), if any, failed. This is not thread-safe. - ** This is necessary even if the xMutexInit() was never called, due to the - ** possiblity of this state being changed via SQLITE_CONFIG_MUTEX. After - ** this point, the application must enable any custom mutex implementation - ** again via SQLITE_CONFIG_MUTEX, if necessary. - */ - sqlite3GlobalConfig.pMutex = 0; - memset(&sqlite3GlobalConfig.mutex, 0, sizeof(sqlite3_mutex_methods)); - return SQLITE_OK; } /* ** This API allows applications to modify the global configuration of @@ -399,26 +388,23 @@ break; } #endif #if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE>0 /* IMP: R-63666-48755 */ case SQLITE_CONFIG_MUTEX: { - /* Atomically compare-and-swap the mutex implementation pointer to - * help prevent a race condition with sqlite3MutexInit(). */ - if( sqlite3CompareAndSwap((void * volatile *)&sqlite3GlobalConfig.pMutex, - 0, &sqlite3GlobalConfig.mutex)==0 ){ - /* Specify an alternative mutex implementation */ - sqlite3GlobalConfig.mutex = *va_arg(ap, sqlite3_mutex_methods*); - }else{ - rc = SQLITE_ERROR; - } + /* Specify an alternative mutex implementation */ + sqlite3MutexCopy(&sqlite3GlobalConfig.mutex, + va_arg(ap, sqlite3_mutex_methods*)); + sqlite3MemoryBarrier(); break; } #endif #if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE>0 /* IMP: R-14450-37597 */ case SQLITE_CONFIG_GETMUTEX: { /* Retrieve the current mutex implementation */ - *va_arg(ap, sqlite3_mutex_methods*) = sqlite3GlobalConfig.mutex; + sqlite3MemoryBarrier(); + sqlite3MutexCopy(va_arg(ap, sqlite3_mutex_methods*), + &sqlite3GlobalConfig.mutex); break; } #endif case SQLITE_CONFIG_MALLOC: { @@ -2894,15 +2880,21 @@ extern int sqlite3Fts2Init(sqlite3*); rc = sqlite3Fts2Init(db); } #endif -#ifdef SQLITE_ENABLE_FTS3 +#ifdef SQLITE_ENABLE_FTS3 /* automatically defined by SQLITE_ENABLE_FTS4 */ if( !db->mallocFailed && rc==SQLITE_OK ){ rc = sqlite3Fts3Init(db); } #endif + +#ifdef SQLITE_ENABLE_FTS5 + if( !db->mallocFailed && rc==SQLITE_OK ){ + rc = sqlite3Fts5Init(db); + } +#endif #ifdef SQLITE_ENABLE_ICU if( !db->mallocFailed && rc==SQLITE_OK ){ rc = sqlite3IcuInit(db); } @@ -2917,10 +2909,16 @@ #ifdef SQLITE_ENABLE_DBSTAT_VTAB if( !db->mallocFailed && rc==SQLITE_OK){ rc = sqlite3DbstatRegister(db); } #endif + +#ifdef SQLITE_ENABLE_JSON1 + if( !db->mallocFailed && rc==SQLITE_OK){ + rc = sqlite3Json1Init(db); + } +#endif /* -DSQLITE_DEFAULT_LOCKING_MODE=1 makes EXCLUSIVE the default locking ** mode. -DSQLITE_DEFAULT_LOCKING_MODE=0 make NORMAL the default locking ** mode. Doing nothing at all also makes NORMAL the default. */ Index: src/malloc.c ================================================================== --- src/malloc.c +++ src/malloc.c @@ -129,13 +129,11 @@ int rc; if( sqlite3GlobalConfig.m.xMalloc==0 ){ sqlite3MemSetDefault(); } memset(&mem0, 0, sizeof(mem0)); - if( sqlite3GlobalConfig.bCoreMutex ){ - mem0.mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MEM); - } + mem0.mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MEM); if( sqlite3GlobalConfig.pScratch && sqlite3GlobalConfig.szScratch>=100 && sqlite3GlobalConfig.nScratch>0 ){ int i, n, sz; ScratchFreeslot *pSlot; sz = ROUNDDOWN8(sqlite3GlobalConfig.szScratch); @@ -222,11 +220,11 @@ static int mallocWithAlarm(int n, void **pp){ int nFull; void *p; assert( sqlite3_mutex_held(mem0.mutex) ); nFull = sqlite3GlobalConfig.m.xRoundup(n); - sqlite3StatusSet(SQLITE_STATUS_MALLOC_SIZE, n); + sqlite3StatusHighwater(SQLITE_STATUS_MALLOC_SIZE, n); if( mem0.alarmThreshold>0 ){ sqlite3_int64 nUsed = sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED); if( nUsed >= mem0.alarmThreshold - nFull ){ mem0.nearlyFull = 1; sqlite3MallocAlarm(nFull); @@ -314,11 +312,11 @@ void *sqlite3ScratchMalloc(int n){ void *p; assert( n>0 ); sqlite3_mutex_enter(mem0.mutex); - sqlite3StatusSet(SQLITE_STATUS_SCRATCH_SIZE, n); + sqlite3StatusHighwater(SQLITE_STATUS_SCRATCH_SIZE, n); if( mem0.nScratchFree && sqlite3GlobalConfig.szScratch>=n ){ p = mem0.pScratchFree; mem0.pScratchFree = mem0.pScratchFree->pNext; mem0.nScratchFree--; sqlite3StatusUp(SQLITE_STATUS_SCRATCH_USED, 1); @@ -409,10 +407,11 @@ int sqlite3MallocSize(void *p){ assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) ); return sqlite3GlobalConfig.m.xSize(p); } int sqlite3DbMallocSize(sqlite3 *db, void *p){ + assert( p!=0 ); if( db==0 || !isLookaside(db,p) ){ #if SQLITE_DEBUG if( db==0 ){ assert( sqlite3MemdebugNoType(p, (u8)~MEMTYPE_HEAP) ); assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) ); @@ -428,11 +427,11 @@ } } sqlite3_uint64 sqlite3_msize(void *p){ assert( sqlite3MemdebugNoType(p, (u8)~MEMTYPE_HEAP) ); assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) ); - return (sqlite3_uint64)sqlite3GlobalConfig.m.xSize(p); + return p ? sqlite3GlobalConfig.m.xSize(p) : 0; } /* ** Free memory previously obtained from sqlite3Malloc(). */ @@ -454,11 +453,11 @@ /* ** Add the size of memory allocation "p" to the count in ** *db->pnBytesFreed. */ static SQLITE_NOINLINE void measureAllocationSize(sqlite3 *db, void *p){ - *db->pnBytesFreed += sqlite3DbMallocSize(db,p); + if( p ) *db->pnBytesFreed += sqlite3DbMallocSize(db,p); } /* ** Free memory that might be associated with a particular database ** connection. @@ -516,11 +515,11 @@ nNew = sqlite3GlobalConfig.m.xRoundup((int)nBytes); if( nOld==nNew ){ pNew = pOld; }else if( sqlite3GlobalConfig.bMemstat ){ sqlite3_mutex_enter(mem0.mutex); - sqlite3StatusSet(SQLITE_STATUS_MALLOC_SIZE, (int)nBytes); + sqlite3StatusHighwater(SQLITE_STATUS_MALLOC_SIZE, (int)nBytes); nDiff = nNew - nOld; if( sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED) >= mem0.alarmThreshold-nDiff ){ sqlite3MallocAlarm(nDiff); } Index: src/mem1.c ================================================================== --- src/mem1.c +++ src/mem1.c @@ -169,15 +169,15 @@ /* ** Report the allocated size of a prior return from xMalloc() ** or xRealloc(). */ static int sqlite3MemSize(void *pPrior){ + assert( pPrior!=0 ); #ifdef SQLITE_MALLOCSIZE - return pPrior ? (int)SQLITE_MALLOCSIZE(pPrior) : 0; + return (int)SQLITE_MALLOCSIZE(pPrior); #else sqlite3_int64 *p; - if( pPrior==0 ) return 0; p = (sqlite3_int64*)pPrior; p--; return (int)p[0]; #endif } Index: src/mem3.c ================================================================== --- src/mem3.c +++ src/mem3.c @@ -474,11 +474,11 @@ ** size returned omits the 8-byte header overhead. This only ** works for chunks that are currently checked out. */ static int memsys3Size(void *p){ Mem3Block *pBlock; - if( p==0 ) return 0; + assert( p!=0 ); pBlock = (Mem3Block*)p; assert( (pBlock[-1].u.hdr.size4x&1)!=0 ); return (pBlock[-1].u.hdr.size4x&~3)*2 - 4; } Index: src/mem5.c ================================================================== --- src/mem5.c +++ src/mem5.c @@ -198,16 +198,15 @@ ** Return the size of an outstanding allocation, in bytes. The ** size returned omits the 8-byte header overhead. This only ** works for chunks that are currently checked out. */ static int memsys5Size(void *p){ - int iSize = 0; - if( p ){ - int i = (int)(((u8 *)p-mem5.zPool)/mem5.szAtom); - assert( i>=0 && i=0 && ixMutexInit = pFrom->xMutexInit; pTo->xMutexEnd = pFrom->xMutexEnd; @@ -73,35 +47,33 @@ /* ** Initialize the mutex system. */ int sqlite3MutexInit(void){ - static int initPending = 0; int rc; - if( sqlite3CompareAndSwap((void * volatile *)&sqlite3GlobalConfig.pMutex, - 0, &sqlite3GlobalConfig.mutex)==0 || mutexIsInvalid() ){ - /* If the mutex implementation pointer has not been set, then the user - ** did not install a mutex implementation via sqlite3_config() prior to - ** sqlite3_initialize() being called. This block copies the pointers - ** for the default implementation into the sqlite3GlobalConfig structure. + + sqlite3MemoryBarrier(); + if( sqlite3CompareAndSwap( + (void * volatile *)&sqlite3GlobalConfig.mutex.xMutexAlloc, + 0, sqlite3GlobalConfig.mutex.xMutexAlloc)==0 ){ + /* If the xMutexAlloc method has not been set, then the user did not + ** install a mutex implementation via sqlite3_config() prior to + ** sqlite3_initialize() being called. This block copies pointers to + ** the default implementation into the sqlite3GlobalConfig structure. */ sqlite3_mutex_methods const *pFrom; if( sqlite3GlobalConfig.bCoreMutex ){ pFrom = sqlite3DefaultMutex(); }else{ pFrom = sqlite3NoopMutex(); } - mutexCopy(&sqlite3GlobalConfig.mutex, pFrom); + sqlite3MutexCopy(&sqlite3GlobalConfig.mutex, pFrom); sqlite3MemoryBarrier(); } - if( !initPending ){ - assert( sqlite3GlobalConfig.mutex.xMutexInit ); - initPending = 1; - rc = sqlite3GlobalConfig.mutex.xMutexInit(); - initPending = 0; - } + assert( sqlite3GlobalConfig.mutex.xMutexInit ); + rc = sqlite3GlobalConfig.mutex.xMutexInit(); #ifdef SQLITE_DEBUG GLOBAL(int, mutexIsInit) = 1; #endif Index: src/mutex_unix.c ================================================================== --- src/mutex_unix.c +++ src/mutex_unix.c @@ -79,11 +79,13 @@ return p->nRef==0 || pthread_equal(p->owner, pthread_self())==0; } #endif /* -** Try to provide a memory barrier operation, needed for initialization only. +** Try to provide a memory barrier operation, needed for initialization +** and also for the implementation of xShmBarrier in the VFS in cases +** where SQLite is compiled without mutexes. */ void sqlite3MemoryBarrier(void){ #if defined(SQLITE_MEMORY_BARRIER) SQLITE_MEMORY_BARRIER; #elif defined(__GNUC__) && GCC_VERSION>=4001000 Index: src/mutex_w32.c ================================================================== --- src/mutex_w32.c +++ src/mutex_w32.c @@ -76,18 +76,23 @@ return winMutexNotheld2(p, tid); } #endif /* -** Try to provide a memory barrier operation, needed for initialization only. +** Try to provide a memory barrier operation, needed for initialization +** and also for the xShmBarrier method of the VFS in cases when SQLite is +** compiled without mutexes (SQLITE_THREADSAFE=0). */ void sqlite3MemoryBarrier(void){ #if defined(SQLITE_MEMORY_BARRIER) SQLITE_MEMORY_BARRIER; #elif defined(__GNUC__) __sync_synchronize(); -#else +#elif !defined(SQLITE_DISABLE_INTRINSIC) && \ + defined(_MSC_VER) && _MSC_VER>=1300 + _ReadWriteBarrier(); +#elif defined(MemoryBarrier) MemoryBarrier(); #endif } /* Index: src/os_unix.c ================================================================== --- src/os_unix.c +++ src/os_unix.c @@ -4658,11 +4658,12 @@ */ static void unixShmBarrier( sqlite3_file *fd /* Database file holding the shared memory */ ){ UNUSED_PARAMETER(fd); - unixEnterMutex(); + sqlite3MemoryBarrier(); /* compiler-defined memory barrier */ + unixEnterMutex(); /* Also mutex, for redundancy */ unixLeaveMutex(); } /* ** Close a connection to shared-memory. Delete the underlying Index: src/os_win.c ================================================================== --- src/os_win.c +++ src/os_win.c @@ -3848,12 +3848,12 @@ */ static void winShmBarrier( sqlite3_file *fd /* Database holding the shared memory */ ){ UNUSED_PARAMETER(fd); - /* MemoryBarrier(); // does not work -- do not know why not */ - winShmEnterMutex(); + sqlite3MemoryBarrier(); /* compiler-defined memory barrier */ + winShmEnterMutex(); /* Also mutex, for redundancy */ winShmLeaveMutex(); } /* ** This function is called to obtain a pointer to region iRegion of the Index: src/pager.c ================================================================== --- src/pager.c +++ src/pager.c @@ -2113,10 +2113,24 @@ } } #else # define pagerReportSize(X) /* No-op if we do not support a codec */ #endif + +#ifdef SQLITE_HAS_CODEC +/* +** Make sure the number of reserved bits is the same in the destination +** pager as it is in the source. This comes up when a VACUUM changes the +** number of reserved bits to the "optimal" amount. +*/ +void sqlite3PagerAlignReserve(Pager *pDest, Pager *pSrc){ + if( pDest->nReserve!=pSrc->nReserve ){ + pDest->nReserve = pSrc->nReserve; + pagerReportSize(pDest); + } +} +#endif /* ** Read a single page from either the journal file (if isMainJrnl==1) or ** from the sub-journal (if isMainJrnl==0) and playback that page. ** The page begins at offset *pOffset into the file. The *pOffset Index: src/pager.h ================================================================== --- src/pager.h +++ src/pager.h @@ -116,10 +116,13 @@ int sqlite3PagerReadFileheader(Pager*, int, unsigned char*); /* Functions used to configure a Pager object. */ void sqlite3PagerSetBusyhandler(Pager*, int(*)(void *), void *); int sqlite3PagerSetPagesize(Pager*, u32*, int); +#ifdef SQLITE_HAS_CODEC +void sqlite3PagerAlignReserve(Pager*,Pager*); +#endif int sqlite3PagerMaxPageCount(Pager*, int); void sqlite3PagerSetCachesize(Pager*, int); void sqlite3PagerSetMmapLimit(Pager *, sqlite3_int64); void sqlite3PagerShrink(Pager*); void sqlite3PagerSetFlags(Pager*,unsigned); Index: src/pcache1.c ================================================================== --- src/pcache1.c +++ src/pcache1.c @@ -318,11 +318,11 @@ if( p ){ pcache1.pFree = pcache1.pFree->pNext; pcache1.nFreeSlot--; pcache1.bUnderPressure = pcache1.nFreeSlot=0 ); - sqlite3StatusSet(SQLITE_STATUS_PAGECACHE_SIZE, nByte); + sqlite3StatusHighwater(SQLITE_STATUS_PAGECACHE_SIZE, nByte); sqlite3StatusUp(SQLITE_STATUS_PAGECACHE_USED, 1); } sqlite3_mutex_leave(pcache1.mutex); } if( p==0 ){ @@ -332,11 +332,11 @@ p = sqlite3Malloc(nByte); #ifndef SQLITE_DISABLE_PAGECACHE_OVERFLOW_STATS if( p ){ int sz = sqlite3MallocSize(p); sqlite3_mutex_enter(pcache1.mutex); - sqlite3StatusSet(SQLITE_STATUS_PAGECACHE_SIZE, nByte); + sqlite3StatusHighwater(SQLITE_STATUS_PAGECACHE_SIZE, nByte); sqlite3StatusUp(SQLITE_STATUS_PAGECACHE_OVERFLOW, sz); sqlite3_mutex_leave(pcache1.mutex); } #endif sqlite3MemdebugSetType(p, MEMTYPE_PCACHE); Index: src/pragma.c ================================================================== --- src/pragma.c +++ src/pragma.c @@ -1359,12 +1359,13 @@ ** messages have been generated, output OK. Otherwise output the ** error message */ static const int iLn = VDBE_OFFSET_LINENO(2); static const VdbeOpList endCode[] = { - { OP_IfNeg, 1, 0, 0}, /* 0 */ - { OP_String8, 0, 3, 0}, /* 1 */ + { OP_AddImm, 1, 0, 0}, /* 0 */ + { OP_If, 1, 0, 0}, /* 1 */ + { OP_String8, 0, 3, 0}, /* 2 */ { OP_ResultRow, 3, 1, 0}, }; int isQuick = (sqlite3Tolower(zLeft[0])=='q'); @@ -1522,12 +1523,12 @@ int uniqOk = sqlite3VdbeMakeLabel(v); int jmp6; int kk; for(kk=0; kknKeyCol; kk++){ int iCol = pIdx->aiColumn[kk]; - assert( iCol>=0 && iColnCol ); - if( pTab->aCol[iCol].notNull ) continue; + assert( iCol!=XN_ROWID && iColnCol ); + if( iCol>=0 && pTab->aCol[iCol].notNull ) continue; sqlite3VdbeAddOp2(v, OP_IsNull, r1+kk, uniqOk); VdbeCoverage(v); } jmp6 = sqlite3VdbeAddOp1(v, OP_Next, iIdxCur+j); VdbeCoverage(v); sqlite3VdbeGoto(v, uniqOk); @@ -1561,13 +1562,13 @@ } #endif /* SQLITE_OMIT_BTREECOUNT */ } } addr = sqlite3VdbeAddOpList(v, ArraySize(endCode), endCode, iLn); - sqlite3VdbeChangeP3(v, addr, -mxErr); - sqlite3VdbeJumpHere(v, addr); - sqlite3VdbeChangeP4(v, addr+1, "ok", P4_STATIC); + sqlite3VdbeChangeP2(v, addr, -mxErr); + sqlite3VdbeJumpHere(v, addr+1); + sqlite3VdbeChangeP4(v, addr+2, "ok", P4_STATIC); } break; #endif /* SQLITE_OMIT_INTEGRITY_CHECK */ #ifndef SQLITE_OMIT_UTF16 Index: src/select.c ================================================================== --- src/select.c +++ src/select.c @@ -577,11 +577,11 @@ if( pSelect->iOffset ){ iLimit = pSelect->iOffset+1; }else{ iLimit = pSelect->iLimit; } - addr = sqlite3VdbeAddOp3(v, OP_IfNotZero, iLimit, 0, -1); VdbeCoverage(v); + addr = sqlite3VdbeAddOp3(v, OP_IfNotZero, iLimit, 0, 1); VdbeCoverage(v); sqlite3VdbeAddOp1(v, OP_Last, pSort->iECursor); sqlite3VdbeAddOp1(v, OP_Delete, pSort->iECursor); sqlite3VdbeJumpHere(v, addr); } } @@ -593,15 +593,12 @@ Vdbe *v, /* Generate code into this VM */ int iOffset, /* Register holding the offset counter */ int iContinue /* Jump here to skip the current record */ ){ if( iOffset>0 ){ - int addr; - addr = sqlite3VdbeAddOp3(v, OP_IfNeg, iOffset, 0, -1); VdbeCoverage(v); - sqlite3VdbeGoto(v, iContinue); - VdbeComment((v, "skip OFFSET records")); - sqlite3VdbeJumpHere(v, addr); + sqlite3VdbeAddOp3(v, OP_IfPos, iOffset, iContinue, 1); VdbeCoverage(v); + VdbeComment((v, "OFFSET")); } } /* ** Add code that will check to make sure the N registers starting at iMem @@ -1813,11 +1810,11 @@ */ static void computeLimitRegisters(Parse *pParse, Select *p, int iBreak){ Vdbe *v = 0; int iLimit = 0; int iOffset; - int addr1, n; + int n; if( p->iLimit ) return; /* ** "LIMIT -1" always shows all rows. There is some ** controversy about what the correct behavior should be. @@ -1848,18 +1845,14 @@ p->iOffset = iOffset = ++pParse->nMem; pParse->nMem++; /* Allocate an extra register for limit+offset */ sqlite3ExprCode(pParse, p->pOffset, iOffset); sqlite3VdbeAddOp1(v, OP_MustBeInt, iOffset); VdbeCoverage(v); VdbeComment((v, "OFFSET counter")); - addr1 = sqlite3VdbeAddOp1(v, OP_IfPos, iOffset); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Integer, 0, iOffset); - sqlite3VdbeJumpHere(v, addr1); + sqlite3VdbeAddOp3(v, OP_SetIfNotPos, iOffset, iOffset, 0); sqlite3VdbeAddOp3(v, OP_Add, iLimit, iOffset, iOffset+1); VdbeComment((v, "LIMIT+OFFSET")); - addr1 = sqlite3VdbeAddOp1(v, OP_IfPos, iLimit); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Integer, -1, iOffset+1); - sqlite3VdbeJumpHere(v, addr1); + sqlite3VdbeAddOp3(v, OP_SetIfNotPos, iLimit, iOffset+1, -1); } } } #ifndef SQLITE_OMIT_COMPOUND_SELECT @@ -2271,10 +2264,15 @@ p->iLimit = pPrior->iLimit; p->iOffset = pPrior->iOffset; if( p->iLimit ){ addr = sqlite3VdbeAddOp1(v, OP_IfNot, p->iLimit); VdbeCoverage(v); VdbeComment((v, "Jump ahead if LIMIT reached")); + if( p->iOffset ){ + sqlite3VdbeAddOp3(v, OP_SetIfNotPos, p->iOffset, p->iOffset, 0); + sqlite3VdbeAddOp3(v, OP_Add, p->iLimit, p->iOffset, p->iOffset+1); + sqlite3VdbeAddOp3(v, OP_SetIfNotPos, p->iLimit, p->iOffset+1, -1); + } } explainSetInteger(iSub2, pParse->iNextSelectId); rc = sqlite3Select(pParse, p, &dest); testcase( rc!=SQLITE_OK ); pDelete = p->pPrior; @@ -2578,16 +2576,16 @@ iContinue = sqlite3VdbeMakeLabel(v); /* Suppress duplicates for UNION, EXCEPT, and INTERSECT */ if( regPrev ){ - int j1, j2; - j1 = sqlite3VdbeAddOp1(v, OP_IfNot, regPrev); VdbeCoverage(v); - j2 = sqlite3VdbeAddOp4(v, OP_Compare, pIn->iSdst, regPrev+1, pIn->nSdst, + int addr1, addr2; + addr1 = sqlite3VdbeAddOp1(v, OP_IfNot, regPrev); VdbeCoverage(v); + addr2 = sqlite3VdbeAddOp4(v, OP_Compare, pIn->iSdst, regPrev+1, pIn->nSdst, (char*)sqlite3KeyInfoRef(pKeyInfo), P4_KEYINFO); - sqlite3VdbeAddOp3(v, OP_Jump, j2+2, iContinue, j2+2); VdbeCoverage(v); - sqlite3VdbeJumpHere(v, j1); + sqlite3VdbeAddOp3(v, OP_Jump, addr2+2, iContinue, addr2+2); VdbeCoverage(v); + sqlite3VdbeJumpHere(v, addr1); sqlite3VdbeAddOp3(v, OP_Copy, pIn->iSdst, regPrev+1, pIn->nSdst-1); sqlite3VdbeAddOp2(v, OP_Integer, 1, regPrev); } if( pParse->db->mallocFailed ) return 0; @@ -2800,11 +2798,11 @@ int regPrev; /* A range of registers to hold previous output */ int savedLimit; /* Saved value of p->iLimit */ int savedOffset; /* Saved value of p->iOffset */ int labelCmpr; /* Label for the start of the merge algorithm */ int labelEnd; /* Label for the end of the overall SELECT stmt */ - int j1; /* Jump instructions that get retargetted */ + int addr1; /* Jump instructions that get retargetted */ int op; /* One of TK_ALL, TK_UNION, TK_EXCEPT, TK_INTERSECT */ KeyInfo *pKeyDup = 0; /* Comparison information for duplicate removal */ KeyInfo *pKeyMerge; /* Comparison information for merging rows */ sqlite3 *db; /* Database connection */ ExprList *pOrderBy; /* The ORDER BY clause */ @@ -2936,23 +2934,23 @@ /* Generate a coroutine to evaluate the SELECT statement to the ** left of the compound operator - the "A" select. */ addrSelectA = sqlite3VdbeCurrentAddr(v) + 1; - j1 = sqlite3VdbeAddOp3(v, OP_InitCoroutine, regAddrA, 0, addrSelectA); + addr1 = sqlite3VdbeAddOp3(v, OP_InitCoroutine, regAddrA, 0, addrSelectA); VdbeComment((v, "left SELECT")); pPrior->iLimit = regLimitA; explainSetInteger(iSub1, pParse->iNextSelectId); sqlite3Select(pParse, pPrior, &destA); sqlite3VdbeAddOp1(v, OP_EndCoroutine, regAddrA); - sqlite3VdbeJumpHere(v, j1); + sqlite3VdbeJumpHere(v, addr1); /* Generate a coroutine to evaluate the SELECT statement on ** the right - the "B" select */ addrSelectB = sqlite3VdbeCurrentAddr(v) + 1; - j1 = sqlite3VdbeAddOp3(v, OP_InitCoroutine, regAddrB, 0, addrSelectB); + addr1 = sqlite3VdbeAddOp3(v, OP_InitCoroutine, regAddrB, 0, addrSelectB); VdbeComment((v, "right SELECT")); savedLimit = p->iLimit; savedOffset = p->iOffset; p->iLimit = regLimitB; p->iOffset = 0; @@ -3039,11 +3037,11 @@ sqlite3VdbeAddOp2(v, OP_Yield, regAddrB, addrEofB); VdbeCoverage(v); sqlite3VdbeGoto(v, labelCmpr); /* This code runs once to initialize everything. */ - sqlite3VdbeJumpHere(v, j1); + sqlite3VdbeJumpHere(v, addr1); sqlite3VdbeAddOp2(v, OP_Yield, regAddrA, addrEofA_noB); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_Yield, regAddrB, addrEofB); VdbeCoverage(v); /* Implement the main merge loop */ @@ -3082,11 +3080,11 @@ #endif #if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) /* Forward Declarations */ static void substExprList(sqlite3*, ExprList*, int, ExprList*); -static void substSelect(sqlite3*, Select *, int, ExprList *); +static void substSelect(sqlite3*, Select *, int, ExprList*, int); /* ** Scan through the expression pExpr. Replace every reference to ** a column in table number iTable with a copy of the iColumn-th ** entry in pEList. (But leave references to the ROWID column @@ -3119,11 +3117,11 @@ } }else{ pExpr->pLeft = substExpr(db, pExpr->pLeft, iTable, pEList); pExpr->pRight = substExpr(db, pExpr->pRight, iTable, pEList); if( ExprHasProperty(pExpr, EP_xIsSelect) ){ - substSelect(db, pExpr->x.pSelect, iTable, pEList); + substSelect(db, pExpr->x.pSelect, iTable, pEList, 1); }else{ substExprList(db, pExpr->x.pList, iTable, pEList); } } return pExpr; @@ -3142,29 +3140,32 @@ } static void substSelect( sqlite3 *db, /* Report malloc errors here */ Select *p, /* SELECT statement in which to make substitutions */ int iTable, /* Table to be replaced */ - ExprList *pEList /* Substitute values */ + ExprList *pEList, /* Substitute values */ + int doPrior /* Do substitutes on p->pPrior too */ ){ SrcList *pSrc; struct SrcList_item *pItem; int i; if( !p ) return; - substExprList(db, p->pEList, iTable, pEList); - substExprList(db, p->pGroupBy, iTable, pEList); - substExprList(db, p->pOrderBy, iTable, pEList); - p->pHaving = substExpr(db, p->pHaving, iTable, pEList); - p->pWhere = substExpr(db, p->pWhere, iTable, pEList); - substSelect(db, p->pPrior, iTable, pEList); - pSrc = p->pSrc; - assert( pSrc ); /* Even for (SELECT 1) we have: pSrc!=0 but pSrc->nSrc==0 */ - if( ALWAYS(pSrc) ){ + do{ + substExprList(db, p->pEList, iTable, pEList); + substExprList(db, p->pGroupBy, iTable, pEList); + substExprList(db, p->pOrderBy, iTable, pEList); + p->pHaving = substExpr(db, p->pHaving, iTable, pEList); + p->pWhere = substExpr(db, p->pWhere, iTable, pEList); + pSrc = p->pSrc; + assert( pSrc!=0 ); for(i=pSrc->nSrc, pItem=pSrc->a; i>0; i--, pItem++){ - substSelect(db, pItem->pSelect, iTable, pEList); + substSelect(db, pItem->pSelect, iTable, pEList, 1); + if( pItem->fg.isTabFunc ){ + substExprList(db, pItem->u1.pFuncArg, iTable, pEList); + } } - } + }while( doPrior && (p = p->pPrior)!=0 ); } #endif /* !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) */ #if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) /* @@ -3312,11 +3313,11 @@ int iFrom, /* Index in p->pSrc->a[] of the inner subquery */ int isAgg, /* True if outer SELECT uses aggregate functions */ int subqueryIsAgg /* True if the subquery uses aggregate functions */ ){ const char *zSavedAuthContext = pParse->zAuthContext; - Select *pParent; + Select *pParent; /* Current UNION ALL term of the other query */ Select *pSub; /* The inner query or "subquery" */ Select *pSub1; /* Pointer to the rightmost select in sub-query */ SrcList *pSrc; /* The FROM clause of the outer query */ SrcList *pSubSrc; /* The FROM clause of the subquery */ ExprList *pList; /* The result set of the outer query */ @@ -3607,13 +3608,13 @@ ** ** SELECT * FROM tabA, (SELECT * FROM sub1, sub2), tabB; ** ** The outer query has 3 slots in its FROM clause. One slot of the ** outer query (the middle slot) is used by the subquery. The next - ** block of code will expand the out query to 4 slots. The middle - ** slot is expanded to two slots in order to make space for the - ** two elements in the FROM clause of the subquery. + ** block of code will expand the outer query FROM clause to 4 slots. + ** The middle slot is expanded to two slots in order to make space + ** for the two elements in the FROM clause of the subquery. */ if( nSubSrc>1 ){ pParent->pSrc = pSrc = sqlite3SrcListEnlarge(db, pSrc, nSubSrc-1,iFrom+1); if( db->mallocFailed ){ break; @@ -3648,15 +3649,10 @@ char *zName = sqlite3DbStrDup(db, pList->a[i].zSpan); sqlite3Dequote(zName); pList->a[i].zName = zName; } } - substExprList(db, pParent->pEList, iParent, pSub->pEList); - if( isAgg ){ - substExprList(db, pParent->pGroupBy, iParent, pSub->pEList); - pParent->pHaving = substExpr(db, pParent->pHaving, iParent, pSub->pEList); - } if( pSub->pOrderBy ){ /* At this point, any non-zero iOrderByCol values indicate that the ** ORDER BY column expression is identical to the iOrderByCol'th ** expression returned by SELECT statement pSub. Since these values ** do not necessarily correspond to columns in SELECT statement pParent, @@ -3672,31 +3668,24 @@ } assert( pParent->pOrderBy==0 ); assert( pSub->pPrior==0 ); pParent->pOrderBy = pOrderBy; pSub->pOrderBy = 0; - }else if( pParent->pOrderBy ){ - substExprList(db, pParent->pOrderBy, iParent, pSub->pEList); - } - if( pSub->pWhere ){ - pWhere = sqlite3ExprDup(db, pSub->pWhere, 0); - }else{ - pWhere = 0; - } + } + pWhere = sqlite3ExprDup(db, pSub->pWhere, 0); if( subqueryIsAgg ){ assert( pParent->pHaving==0 ); pParent->pHaving = pParent->pWhere; pParent->pWhere = pWhere; - pParent->pHaving = substExpr(db, pParent->pHaving, iParent, pSub->pEList); pParent->pHaving = sqlite3ExprAnd(db, pParent->pHaving, sqlite3ExprDup(db, pSub->pHaving, 0)); assert( pParent->pGroupBy==0 ); pParent->pGroupBy = sqlite3ExprListDup(db, pSub->pGroupBy, 0); }else{ - pParent->pWhere = substExpr(db, pParent->pWhere, iParent, pSub->pEList); pParent->pWhere = sqlite3ExprAnd(db, pParent->pWhere, pWhere); } + substSelect(db, pParent, iParent, pSub->pEList, 0); /* The flattened query is distinct if either the inner or the ** outer query is distinct. */ pParent->selFlags |= pSub->selFlags & SF_Distinct; @@ -4219,21 +4208,13 @@ ** an entry of the FROM clause is a subquery instead of a table or view, ** then create a transient table structure to describe the subquery. */ for(i=0, pFrom=pTabList->a; inSrc; i++, pFrom++){ Table *pTab; - assert( pFrom->fg.isRecursive==0 || pFrom->pTab ); + assert( pFrom->fg.isRecursive==0 || pFrom->pTab!=0 ); if( pFrom->fg.isRecursive ) continue; - if( pFrom->pTab!=0 ){ - /* This statement has already been prepared. There is no need - ** to go further. */ - assert( i==0 ); -#ifndef SQLITE_OMIT_CTE - selectPopWith(pWalker, p); -#endif - return WRC_Prune; - } + assert( pFrom->pTab==0 ); #ifndef SQLITE_OMIT_CTE if( withExpand(pWalker, pFrom) ) return WRC_Abort; if( pFrom->pTab ) {} else #endif if( pFrom->zName==0 ){ @@ -4265,19 +4246,23 @@ return WRC_Abort; } pTab->nRef++; #if !defined(SQLITE_OMIT_VIEW) || !defined (SQLITE_OMIT_VIRTUALTABLE) if( pTab->pSelect || IsVirtual(pTab) ){ + i16 nCol; if( sqlite3ViewGetColumnNames(pParse, pTab) ) return WRC_Abort; assert( pFrom->pSelect==0 ); if( pFrom->fg.isTabFunc && !IsVirtual(pTab) ){ sqlite3ErrorMsg(pParse, "'%s' is not a function", pTab->zName); return WRC_Abort; } pFrom->pSelect = sqlite3SelectDup(db, pTab->pSelect, 0); sqlite3SelectSetName(pFrom->pSelect, pTab->zName); + nCol = pTab->nCol; + pTab->nCol = -1; sqlite3WalkSelect(pWalker, pFrom->pSelect); + pTab->nCol = nCol; } #endif } /* Locate the index named by the INDEXED BY clause, if any. */ @@ -4521,23 +4506,23 @@ int i; SrcList *pTabList; struct SrcList_item *pFrom; assert( p->selFlags & SF_Resolved ); - if( (p->selFlags & SF_HasTypeInfo)==0 ){ - p->selFlags |= SF_HasTypeInfo; - pParse = pWalker->pParse; - pTabList = p->pSrc; - for(i=0, pFrom=pTabList->a; inSrc; i++, pFrom++){ - Table *pTab = pFrom->pTab; - if( ALWAYS(pTab!=0) && (pTab->tabFlags & TF_Ephemeral)!=0 ){ - /* A sub-query in the FROM clause of a SELECT */ - Select *pSel = pFrom->pSelect; - if( pSel ){ - while( pSel->pPrior ) pSel = pSel->pPrior; - selectAddColumnTypeAndCollation(pParse, pTab, pSel); - } + assert( (p->selFlags & SF_HasTypeInfo)==0 ); + p->selFlags |= SF_HasTypeInfo; + pParse = pWalker->pParse; + pTabList = p->pSrc; + for(i=0, pFrom=pTabList->a; inSrc; i++, pFrom++){ + Table *pTab = pFrom->pTab; + assert( pTab!=0 ); + if( (pTab->tabFlags & TF_Ephemeral)!=0 ){ + /* A sub-query in the FROM clause of a SELECT */ + Select *pSel = pFrom->pSelect; + if( pSel ){ + while( pSel->pPrior ) pSel = pSel->pPrior; + selectAddColumnTypeAndCollation(pParse, pTab, pSel); } } } } #endif @@ -4859,11 +4844,21 @@ #if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) for(i=0; !p->pPrior && inSrc; i++){ struct SrcList_item *pItem = &pTabList->a[i]; Select *pSub = pItem->pSelect; int isAggSub; + Table *pTab = pItem->pTab; if( pSub==0 ) continue; + + /* Catch mismatch in the declared columns of a view and the number of + ** columns in the SELECT on the RHS */ + if( pTab->nCol!=pSub->pEList->nExpr ){ + sqlite3ErrorMsg(pParse, "expected %d columns for '%s' but got %d", + pTab->nCol, pTab->zName, pSub->pEList->nExpr); + goto select_end; + } + isAggSub = (pSub->selFlags & SF_Aggregate)!=0; if( flattenSubquery(pParse, p, i, isAgg, isAggSub) ){ /* This subquery can be absorbed into its parent. */ if( isAggSub ){ isAgg = 1; @@ -5211,11 +5206,11 @@ /* Processing for aggregates with GROUP BY is very different and ** much more complex than aggregates without a GROUP BY. */ if( pGroupBy ){ KeyInfo *pKeyInfo; /* Keying information for the group by clause */ - int j1; /* A-vs-B comparision jump */ + int addr1; /* A-vs-B comparision jump */ int addrOutputRow; /* Start of subroutine that outputs a result row */ int regOutputRow; /* Return address register for output subroutine */ int addrSetAbort; /* Set the abort flag and return */ int addrTopOfLoop; /* Top of the input loop */ int addrSortingIdx; /* The OP_OpenEphemeral for the sorting index */ @@ -5298,17 +5293,12 @@ j = nGroupBy; for(i=0; iiSorterColumn>=j ){ int r1 = j + regBase; - int r2; - - r2 = sqlite3ExprCodeGetColumn(pParse, - pCol->pTab, pCol->iColumn, pCol->iTable, r1, 0); - if( r1!=r2 ){ - sqlite3VdbeAddOp2(v, OP_SCopy, r2, r1); - } + sqlite3ExprCodeGetColumnToReg(pParse, + pCol->pTab, pCol->iColumn, pCol->iTable, r1); j++; } } regRecord = sqlite3GetTempReg(pParse); sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase, nCol, regRecord); @@ -5359,12 +5349,12 @@ sqlite3ExprCode(pParse, pGroupBy->a[j].pExpr, iBMem+j); } } sqlite3VdbeAddOp4(v, OP_Compare, iAMem, iBMem, pGroupBy->nExpr, (char*)sqlite3KeyInfoRef(pKeyInfo), P4_KEYINFO); - j1 = sqlite3VdbeCurrentAddr(v); - sqlite3VdbeAddOp3(v, OP_Jump, j1+1, 0, j1+1); VdbeCoverage(v); + addr1 = sqlite3VdbeCurrentAddr(v); + sqlite3VdbeAddOp3(v, OP_Jump, addr1+1, 0, addr1+1); VdbeCoverage(v); /* Generate code that runs whenever the GROUP BY changes. ** Changes in the GROUP BY are detected by the previous code ** block. If there were no changes, this block is skipped. ** @@ -5382,11 +5372,11 @@ VdbeComment((v, "reset accumulator")); /* Update the aggregate accumulators based on the content of ** the current row */ - sqlite3VdbeJumpHere(v, j1); + sqlite3VdbeJumpHere(v, addr1); updateAccumulator(pParse, &sAggInfo); sqlite3VdbeAddOp2(v, OP_Integer, 1, iUseFlag); VdbeComment((v, "indicate data in accumulator")); /* End of the loop Index: src/shell.c ================================================================== --- src/shell.c +++ src/shell.c @@ -2610,10 +2610,26 @@ } sqlite3_free(zSchemaTab); return 0; } +/* +** Print the current sqlite3_errmsg() value to stderr and return 1. +*/ +static int shellDatabaseError(sqlite3 *db){ + const char *zErr = sqlite3_errmsg(db); + fprintf(stderr, "Error: %s\n", zErr); + return 1; +} + +/* +** Print an out-of-memory message to stderr and return 1. +*/ +static int shellNomemError(void){ + fprintf(stderr, "Error: out of memory\n"); + return 1; +} /* ** If an input line begins with "." then invoke this routine to ** process that line. ** @@ -3711,17 +3727,21 @@ int nRow, nAlloc; char *zSql = 0; int ii; open_db(p, 0); rc = sqlite3_prepare_v2(p->db, "PRAGMA database_list", -1, &pStmt, 0); - if( rc ) return rc; + if( rc ) return shellDatabaseError(p->db); + + /* Create an SQL statement to query for the list of tables in the + ** main and all attached databases where the table name matches the + ** LIKE pattern bound to variable "?1". */ zSql = sqlite3_mprintf( "SELECT name FROM sqlite_master" " WHERE type IN ('table','view')" " AND name NOT LIKE 'sqlite_%%'" " AND name LIKE ?1"); - while( sqlite3_step(pStmt)==SQLITE_ROW ){ + while( zSql && sqlite3_step(pStmt)==SQLITE_ROW ){ const char *zDbName = (const char*)sqlite3_column_text(pStmt, 1); if( zDbName==0 || strcmp(zDbName,"main")==0 ) continue; if( strcmp(zDbName,"temp")==0 ){ zSql = sqlite3_mprintf( "%z UNION ALL " @@ -3736,15 +3756,21 @@ " WHERE type IN ('table','view')" " AND name NOT LIKE 'sqlite_%%'" " AND name LIKE ?1", zSql, zDbName, zDbName); } } - sqlite3_finalize(pStmt); - zSql = sqlite3_mprintf("%z ORDER BY 1", zSql); - rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + rc = sqlite3_finalize(pStmt); + if( zSql && rc==SQLITE_OK ){ + zSql = sqlite3_mprintf("%z ORDER BY 1", zSql); + if( zSql ) rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + } sqlite3_free(zSql); - if( rc ) return rc; + if( !zSql ) return shellNomemError(); + if( rc ) return shellDatabaseError(p->db); + + /* Run the SQL statement prepared by the above block. Store the results + ** as an array of nul-terminated strings in azResult[]. */ nRow = nAlloc = 0; azResult = 0; if( nArg>1 ){ sqlite3_bind_text(pStmt, 1, azArg[1], -1, SQLITE_TRANSIENT); }else{ @@ -3754,21 +3780,29 @@ if( nRow>=nAlloc ){ char **azNew; int n2 = nAlloc*2 + 10; azNew = sqlite3_realloc64(azResult, sizeof(azResult[0])*n2); if( azNew==0 ){ - fprintf(stderr, "Error: out of memory\n"); + rc = shellNomemError(); break; } nAlloc = n2; azResult = azNew; } azResult[nRow] = sqlite3_mprintf("%s", sqlite3_column_text(pStmt, 0)); - if( azResult[nRow] ) nRow++; + if( 0==azResult[nRow] ){ + rc = shellNomemError(); + break; + } + nRow++; + } + if( sqlite3_finalize(pStmt)!=SQLITE_OK ){ + rc = shellDatabaseError(p->db); } - sqlite3_finalize(pStmt); - if( nRow>0 ){ + + /* Pretty-print the contents of array azResult[] to the output */ + if( rc==0 && nRow>0 ){ int len, maxlen = 0; int i, j; int nPrintCol, nPrintRow; for(i=0; iout, "%s%-*s", zSp, maxlen, azResult[j] ? azResult[j]:""); } fprintf(p->out, "\n"); } } + for(ii=0; ii=8 && strncmp(azArg[0], "testctrl", n)==0 && nArg>=2 ){ @@ -4617,17 +4652,10 @@ return 1; #endif } data.out = stdout; -#ifdef SQLITE_ENABLE_JSON1 - { - extern int sqlite3_json_init(sqlite3*); - sqlite3_auto_extension((void(*)(void))sqlite3_json_init); - } -#endif - /* Go ahead and open the database file if it already exists. If the ** file does not exist, delay opening it. This prevents empty database ** files from being created if a user mistypes the database name argument ** to the sqlite command-line tool. */ Index: src/sqlite.h.in ================================================================== --- src/sqlite.h.in +++ src/sqlite.h.in @@ -4358,11 +4358,11 @@ const void *sqlite3_value_text16be(sqlite3_value*); int sqlite3_value_type(sqlite3_value*); int sqlite3_value_numeric_type(sqlite3_value*); /* -** CAPI3REF: Obtaining SQL Values +** CAPI3REF: Finding The Subtype Of SQL Values ** METHOD: sqlite3_value ** ** The sqlite3_value_subtype(V) function returns the subtype for ** an [application-defined SQL function] argument V. The subtype ** information can be used to pass a limited amount of context from @@ -5631,18 +5631,36 @@ ** indicates that the expense of the operation is similar to that of a ** binary search on a unique indexed field of an SQLite table with N rows. ** ** ^The estimatedRows value is an estimate of the number of rows that ** will be returned by the strategy. +** +** The xBestIndex method may optionally populate the idxFlags field with a +** mask of SQLITE_INDEX_SCAN_* flags. Currently there is only one such flag - +** SQLITE_INDEX_SCAN_UNIQUE. If the xBestIndex method sets this flag, SQLite +** assumes that the strategy may visit at most one row. +** +** Additionally, if xBestIndex sets the SQLITE_INDEX_SCAN_UNIQUE flag, then +** SQLite also assumes that if a call to the xUpdate() method is made as +** part of the same statement to delete or update a virtual table row and the +** implementation returns SQLITE_CONSTRAINT, then there is no need to rollback +** any database changes. In other words, if the xUpdate() returns +** SQLITE_CONSTRAINT, the database contents must be exactly as they were +** before xUpdate was called. By contrast, if SQLITE_INDEX_SCAN_UNIQUE is not +** set and xUpdate returns SQLITE_CONSTRAINT, any database changes made by +** the xUpdate method are automatically rolled back by SQLite. ** ** IMPORTANT: The estimatedRows field was added to the sqlite3_index_info ** structure for SQLite version 3.8.2. If a virtual table extension is ** used with an SQLite version earlier than 3.8.2, the results of attempting ** to read or write the estimatedRows field are undefined (but are likely ** to included crashing the application). The estimatedRows field should ** therefore only be used if [sqlite3_libversion_number()] returns a -** value greater than or equal to 3008002. +** value greater than or equal to 3008002. Similarly, the idxFlags field +** was added for version 3.9.0. It may therefore only be used if +** sqlite3_libversion_number() returns a value greater than or equal to +** 3009000. */ struct sqlite3_index_info { /* Inputs */ int nConstraint; /* Number of entries in aConstraint */ struct sqlite3_index_constraint { @@ -5666,12 +5684,19 @@ int needToFreeIdxStr; /* Free idxStr using sqlite3_free() if true */ int orderByConsumed; /* True if output is already ordered */ double estimatedCost; /* Estimated cost of using this index */ /* Fields below are only available in SQLite 3.8.2 and later */ sqlite3_int64 estimatedRows; /* Estimated number of rows returned */ + /* Fields below are only available in SQLite 3.9.0 and later */ + int idxFlags; /* Mask of SQLITE_INDEX_SCAN_* flags */ }; +/* +** CAPI3REF: Virtual Table Scan Flags +*/ +#define SQLITE_INDEX_SCAN_UNIQUE 1 /* Scan visits at most 1 row */ + /* ** CAPI3REF: Virtual Table Constraint Operator Codes ** ** These macros defined the allowed values for the ** [sqlite3_index_info].aConstraint[].op field. Each value represents @@ -6545,11 +6570,12 @@ ** handed to [scratch memory allocator]. Only the value returned in the ** *pHighwater parameter to [sqlite3_status()] is of interest. ** The value written into the *pCurrent parameter is undefined.)^ ** ** [[SQLITE_STATUS_PARSER_STACK]] ^(
SQLITE_STATUS_PARSER_STACK
-**
This parameter records the deepest parser stack. It is only +**
The *pHighwater parameter records the deepest parser stack. +** The *pCurrent value is undefined. The *pHighwater value is only ** meaningful if SQLite is compiled with [YYTRACKMAXSTACKDEPTH].
)^ ** ** ** New status parameters may be added from time to time. */ Index: src/sqlite3ext.h ================================================================== --- src/sqlite3ext.h +++ src/sqlite3ext.h @@ -270,11 +270,11 @@ /* Version 3.8.11 and later */ sqlite3_value *(*value_dup)(const sqlite3_value*); void (*value_free)(sqlite3_value*); int (*result_zeroblob64)(sqlite3_context*,sqlite3_uint64); int (*bind_zeroblob64)(sqlite3_stmt*, int, sqlite3_uint64); - /* Version 3.8.12 and later */ + /* Version 3.9.0 and later */ unsigned int (*value_subtype)(sqlite3_value*); void (*result_subtype)(sqlite3_context*,unsigned int); }; /* @@ -509,11 +509,11 @@ /* Version 3.8.11 and later */ #define sqlite3_value_dup sqlite3_api->value_dup #define sqlite3_value_free sqlite3_api->value_free #define sqlite3_result_zeroblob64 sqlite3_api->result_zeroblob64 #define sqlite3_bind_zeroblob64 sqlite3_api->bind_zeroblob64 -/* Version 3.8.12 and later */ +/* Version 3.9.0 and later */ #define sqlite3_value_subtype sqlite3_api->value_subtype #define sqlite3_result_subtype sqlite3_api->result_subtype #endif /* !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) */ #if !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) Index: src/sqliteInt.h ================================================================== --- src/sqliteInt.h +++ src/sqliteInt.h @@ -194,10 +194,11 @@ # if defined(_MSC_VER) && _MSC_VER>=1300 # if !defined(_WIN32_WCE) # include # pragma intrinsic(_byteswap_ushort) # pragma intrinsic(_byteswap_ulong) +# pragma intrinsic(_ReadWriteBarrier) # else # include # endif # endif #endif @@ -1915,10 +1916,16 @@ #define IsPrimaryKeyIndex(X) ((X)->idxType==SQLITE_IDXTYPE_PRIMARYKEY) /* Return true if index X is a UNIQUE index */ #define IsUniqueIndex(X) ((X)->onError!=OE_None) +/* The Index.aiColumn[] values are normally positive integer. But +** there are some negative values that have special meaning: +*/ +#define XN_ROWID (-1) /* Indexed column is the rowid */ +#define XN_EXPR (-2) /* Indexed column is an expression */ + /* ** Each sample stored in the sqlite_stat3 table is represented in memory ** using a structure of this type. See documentation at the top of the ** analyze.c source file for additional information. */ @@ -2936,11 +2943,10 @@ int mxStrlen; /* Maximum string length */ int neverCorrupt; /* Database is always well-formed */ int szLookaside; /* Default lookaside buffer size */ int nLookaside; /* Default lookaside buffer count */ sqlite3_mem_methods m; /* Low-level memory allocation interface */ - sqlite3_mutex_methods *pMutex; /* Address of mutex member or zero. */ sqlite3_mutex_methods mutex; /* Low-level mutex interface */ sqlite3_pcache_methods2 pcache2; /* Low-level page-cache interface */ void *pHeap; /* Heap storage space */ int nHeap; /* Size of pHeap[] */ int mnReq, mxReq; /* Min and max heap requests sizes */ @@ -3194,12 +3200,14 @@ sqlite3_mutex *sqlite3MutexAlloc(int); int sqlite3MutexInit(void); int sqlite3MutexEnd(void); #endif #if !defined(SQLITE_MUTEX_OMIT) && !defined(SQLITE_MUTEX_NOOP) + void sqlite3MutexCopy(sqlite3_mutex_methods *, sqlite3_mutex_methods const *); void sqlite3MemoryBarrier(void); #else +# define sqlite3MutexCopy(x,y) # define sqlite3MemoryBarrier() #endif #if !defined(SQLITE_MUTEX_OMIT) # if !defined(SQLITE_MUTEX_NOOP) void *sqlite3CompareAndSwap(void * volatile *, void *, void *); @@ -3211,11 +3219,11 @@ #endif sqlite3_int64 sqlite3StatusValue(int); void sqlite3StatusUp(int, int); void sqlite3StatusDown(int, int); -void sqlite3StatusSet(int, int); +void sqlite3StatusHighwater(int, int); /* Access to mutexes used by sqlite3_status() */ sqlite3_mutex *sqlite3Pcache1Mutex(void); sqlite3_mutex *sqlite3MallocMutex(void); @@ -3395,10 +3403,11 @@ #define ONEPASS_OFF 0 /* Use of ONEPASS not allowed */ #define ONEPASS_SINGLE 1 /* ONEPASS valid for a single row update */ #define ONEPASS_MULTI 2 /* ONEPASS is valid for multiple rows */ void sqlite3ExprCodeLoadIndexColumn(Parse*, Index*, int, int, int); int sqlite3ExprCodeGetColumn(Parse*, Table*, int, int, int, u8); +void sqlite3ExprCodeGetColumnToReg(Parse*, Table*, int, int, int); void sqlite3ExprCodeGetColumnOfTable(Vdbe*, Table*, int, int, int); void sqlite3ExprCodeMove(Parse*, int, int, int); void sqlite3ExprCacheStore(Parse*, int, int, int); void sqlite3ExprCachePush(Parse*); void sqlite3ExprCachePop(Parse*); @@ -3513,19 +3522,21 @@ TriggerStep *sqlite3TriggerDeleteStep(sqlite3*,Token*, Expr*); void sqlite3DeleteTrigger(sqlite3*, Trigger*); void sqlite3UnlinkAndDeleteTrigger(sqlite3*,int,const char*); u32 sqlite3TriggerColmask(Parse*,Trigger*,ExprList*,int,int,Table*,int); # define sqlite3ParseToplevel(p) ((p)->pToplevel ? (p)->pToplevel : (p)) +# define sqlite3IsToplevel(p) ((p)->pToplevel==0) #else # define sqlite3TriggersExist(B,C,D,E,F) 0 # define sqlite3DeleteTrigger(A,B) # define sqlite3DropTriggerPtr(A,B) # define sqlite3UnlinkAndDeleteTrigger(A,B,C) # define sqlite3CodeRowTrigger(A,B,C,D,E,F,G,H,I) # define sqlite3CodeRowTriggerDirect(A,B,C,D,E,F) # define sqlite3TriggerList(X, Y) 0 # define sqlite3ParseToplevel(p) p +# define sqlite3IsToplevel(p) 1 # define sqlite3TriggerColmask(A,B,C,D,E,F,G) 0 #endif int sqlite3JoinType(Parse*, Token*, Token*, Token*); void sqlite3CreateForeignKey(Parse*, ExprList*, Token*, ExprList*, int); Index: src/status.c ================================================================== --- src/status.c +++ src/status.c @@ -106,22 +106,25 @@ assert( op>=0 && op=0 && op=0 && opwsdStat.mxValue[op] ){ - wsdStat.mxValue[op] = wsdStat.nowValue[op]; + assert( op==SQLITE_STATUS_MALLOC_SIZE + || op==SQLITE_STATUS_PAGECACHE_SIZE + || op==SQLITE_STATUS_SCRATCH_SIZE + || op==SQLITE_STATUS_PARSER_STACK ); + if( X>wsdStat.mxValue[op] ){ + wsdStat.mxValue[op] = X; } } /* ** Query status information. @@ -250,14 +253,14 @@ pSchema->tblHash.count + pSchema->trigHash.count + pSchema->idxHash.count + pSchema->fkeyHash.count ); - nByte += sqlite3MallocSize(pSchema->tblHash.ht); - nByte += sqlite3MallocSize(pSchema->trigHash.ht); - nByte += sqlite3MallocSize(pSchema->idxHash.ht); - nByte += sqlite3MallocSize(pSchema->fkeyHash.ht); + nByte += sqlite3_msize(pSchema->tblHash.ht); + nByte += sqlite3_msize(pSchema->trigHash.ht); + nByte += sqlite3_msize(pSchema->idxHash.ht); + nByte += sqlite3_msize(pSchema->fkeyHash.ht); for(p=sqliteHashFirst(&pSchema->trigHash); p; p=sqliteHashNext(p)){ sqlite3DeleteTrigger(db, (Trigger*)sqliteHashData(p)); } for(p=sqliteHashFirst(&pSchema->tblHash); p; p=sqliteHashNext(p)){ Index: src/test1.c ================================================================== --- src/test1.c +++ src/test1.c @@ -6375,33 +6375,27 @@ extern int sqlite3_closure_init(sqlite3*,char**,const sqlite3_api_routines*); extern int sqlite3_eval_init(sqlite3*,char**,const sqlite3_api_routines*); extern int sqlite3_fileio_init(sqlite3*,char**,const sqlite3_api_routines*); extern int sqlite3_fuzzer_init(sqlite3*,char**,const sqlite3_api_routines*); extern int sqlite3_ieee_init(sqlite3*,char**,const sqlite3_api_routines*); - extern int sqlite3_json_init(sqlite3*,char**,const sqlite3_api_routines*); extern int sqlite3_nextchar_init(sqlite3*,char**,const sqlite3_api_routines*); extern int sqlite3_percentile_init(sqlite3*,char**,const sqlite3_api_routines*); extern int sqlite3_regexp_init(sqlite3*,char**,const sqlite3_api_routines*); extern int sqlite3_series_init(sqlite3*,char**,const sqlite3_api_routines*); extern int sqlite3_spellfix_init(sqlite3*,char**,const sqlite3_api_routines*); extern int sqlite3_totype_init(sqlite3*,char**,const sqlite3_api_routines*); extern int sqlite3_wholenumber_init(sqlite3*,char**,const sqlite3_api_routines*); - extern int sqlite3_fts5_init(sqlite3*,char**,const sqlite3_api_routines*); static const struct { const char *zExtName; int (*pInit)(sqlite3*,char**,const sqlite3_api_routines*); } aExtension[] = { { "amatch", sqlite3_amatch_init }, { "closure", sqlite3_closure_init }, { "eval", sqlite3_eval_init }, -#ifdef SQLITE_ENABLE_FTS5 - { "fts5", sqlite3_fts5_init }, -#endif { "fileio", sqlite3_fileio_init }, { "fuzzer", sqlite3_fuzzer_init }, { "ieee754", sqlite3_ieee_init }, - { "json", sqlite3_json_init }, { "nextchar", sqlite3_nextchar_init }, { "percentile", sqlite3_percentile_init }, { "regexp", sqlite3_regexp_init }, { "series", sqlite3_series_init }, { "spellfix", sqlite3_spellfix_init }, @@ -6424,11 +6418,15 @@ } if( i>=ArraySize(aExtension) ){ Tcl_AppendResult(interp, "no such extension: ", zName, (char*)0); return TCL_ERROR; } - rc = aExtension[i].pInit(db, &zErrMsg, 0); + if( aExtension[i].pInit ){ + rc = aExtension[i].pInit(db, &zErrMsg, 0); + }else{ + rc = SQLITE_OK; + } if( rc!=SQLITE_OK || zErrMsg ){ Tcl_AppendResult(interp, "initialization of ", zName, " failed: ", zErrMsg, (char*)0); sqlite3_free(zErrMsg); return TCL_ERROR; Index: src/test_config.c ================================================================== --- src/test_config.c +++ src/test_config.c @@ -164,10 +164,16 @@ #ifdef SQLITE_ENABLE_ATOMIC_WRITE Tcl_SetVar2(interp, "sqlite_options", "atomicwrite", "1", TCL_GLOBAL_ONLY); #else Tcl_SetVar2(interp, "sqlite_options", "atomicwrite", "0", TCL_GLOBAL_ONLY); #endif + +#ifdef SQLITE_ENABLE_JSON1 + Tcl_SetVar2(interp, "sqlite_options", "json1", "1", TCL_GLOBAL_ONLY); +#else + Tcl_SetVar2(interp, "sqlite_options", "json1", "0", TCL_GLOBAL_ONLY); +#endif #ifdef SQLITE_OMIT_ATTACH Tcl_SetVar2(interp, "sqlite_options", "attach", "0", TCL_GLOBAL_ONLY); #else Tcl_SetVar2(interp, "sqlite_options", "attach", "1", TCL_GLOBAL_ONLY); Index: src/threads.c ================================================================== --- src/threads.c +++ src/threads.c @@ -65,10 +65,14 @@ p = sqlite3Malloc(sizeof(*p)); if( p==0 ) return SQLITE_NOMEM; memset(p, 0, sizeof(*p)); p->xTask = xTask; p->pIn = pIn; + /* If the SQLITE_TESTCTRL_FAULT_INSTALL callback is registered to a + ** function that returns SQLITE_ERROR when passed the argument 200, that + ** forces worker threads to run sequentially and deterministically + ** for testing purposes. */ if( sqlite3FaultSim(200) ){ rc = 1; }else{ rc = pthread_create(&p->tid, 0, xTask, pIn); } @@ -149,11 +153,16 @@ assert( ppThread!=0 ); assert( xTask!=0 ); *ppThread = 0; p = sqlite3Malloc(sizeof(*p)); if( p==0 ) return SQLITE_NOMEM; - if( sqlite3GlobalConfig.bCoreMutex==0 ){ + /* If the SQLITE_TESTCTRL_FAULT_INSTALL callback is registered to a + ** function that returns SQLITE_ERROR when passed the argument 200, that + ** forces worker threads to run sequentially and deterministically + ** (via the sqlite3FaultSim() term of the conditional) for testing + ** purposes. */ + if( sqlite3GlobalConfig.bCoreMutex==0 || sqlite3FaultSim(200) ){ memset(p, 0, sizeof(*p)); }else{ p->xTask = xTask; p->pIn = pIn; p->tid = (void*)_beginthreadex(0, 0, sqlite3ThreadProc, p, 0, &p->id); @@ -177,11 +186,11 @@ BOOL bRc; assert( ppOut!=0 ); if( NEVER(p==0) ) return SQLITE_NOMEM; if( p->xTask==0 ){ - assert( p->id==GetCurrentThreadId() ); + /* assert( p->id==GetCurrentThreadId() ); */ rc = WAIT_OBJECT_0; assert( p->tid==0 ); }else{ assert( p->id!=0 && p->id!=GetCurrentThreadId() ); rc = sqlite3Win32Wait((HANDLE)p->tid); Index: src/tokenize.c ================================================================== --- src/tokenize.c +++ src/tokenize.c @@ -465,11 +465,11 @@ sqlite3Parser(pEngine, 0, pParse->sLastToken, pParse); } } #ifdef YYTRACKMAXSTACKDEPTH sqlite3_mutex_enter(sqlite3MallocMutex()); - sqlite3StatusSet(SQLITE_STATUS_PARSER_STACK, + sqlite3StatusHighwater(SQLITE_STATUS_PARSER_STACK, sqlite3ParserStackPeak(pEngine) ); sqlite3_mutex_leave(sqlite3MallocMutex()); #endif /* YYDEBUG */ sqlite3ParserFree(pEngine, sqlite3_free); Index: src/update.c ================================================================== --- src/update.c +++ src/update.c @@ -132,13 +132,13 @@ int nKey = 0; /* Number of elements in regKey for WITHOUT ROWID */ int aiCurOnePass[2]; /* The write cursors opened by WHERE_ONEPASS */ /* Register Allocations */ int regRowCount = 0; /* A count of rows changed */ - int regOldRowid; /* The old rowid */ - int regNewRowid; /* The new rowid */ - int regNew; /* Content of the NEW.* table in triggers */ + int regOldRowid = 0; /* The old rowid */ + int regNewRowid = 0; /* The new rowid */ + int regNew = 0; /* Content of the NEW.* table in triggers */ int regOld = 0; /* Content of OLD.* table in triggers */ int regRowSet = 0; /* Rowset of rows to be updated */ int regKey = 0; /* composite PRIMARY KEY value */ memset(&sContext, 0, sizeof(sContext)); @@ -298,33 +298,24 @@ v = sqlite3GetVdbe(pParse); if( v==0 ) goto update_cleanup; if( pParse->nested==0 ) sqlite3VdbeCountChanges(v); sqlite3BeginWriteOperation(pParse, 1, iDb); -#ifndef SQLITE_OMIT_VIRTUALTABLE - /* Virtual tables must be handled separately */ - if( IsVirtual(pTab) ){ - updateVirtualTable(pParse, pTabList, pTab, pChanges, pRowidExpr, aXRef, - pWhere, onError); - pWhere = 0; - pTabList = 0; - goto update_cleanup; - } -#endif - - /* Allocate required registers. */ - regRowSet = ++pParse->nMem; - regOldRowid = regNewRowid = ++pParse->nMem; - if( chngPk || pTrigger || hasFK ){ - regOld = pParse->nMem + 1; - pParse->nMem += pTab->nCol; - } - if( chngKey || pTrigger || hasFK ){ - regNewRowid = ++pParse->nMem; - } - regNew = pParse->nMem + 1; - pParse->nMem += pTab->nCol; + /* Allocate required registers. */ + if( !IsVirtual(pTab) ){ + regRowSet = ++pParse->nMem; + regOldRowid = regNewRowid = ++pParse->nMem; + if( chngPk || pTrigger || hasFK ){ + regOld = pParse->nMem + 1; + pParse->nMem += pTab->nCol; + } + if( chngKey || pTrigger || hasFK ){ + regNewRowid = ++pParse->nMem; + } + regNew = pParse->nMem + 1; + pParse->nMem += pTab->nCol; + } /* Start the view context. */ if( isView ){ sqlite3AuthContextPush(pParse, &sContext, pTab->zName); } @@ -342,10 +333,19 @@ ** WHERE clause. */ if( sqlite3ResolveExprNames(&sNC, pWhere) ){ goto update_cleanup; } + +#ifndef SQLITE_OMIT_VIRTUALTABLE + /* Virtual tables must be handled separately */ + if( IsVirtual(pTab) ){ + updateVirtualTable(pParse, pTabList, pTab, pChanges, pRowidExpr, aXRef, + pWhere, onError); + goto update_cleanup; + } +#endif /* Begin the database scan */ if( HasRowid(pTab) ){ sqlite3VdbeAddOp3(v, OP_Null, 0, regRowSet, regOldRowid); @@ -382,11 +382,11 @@ pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, 0, 0, WHERE_ONEPASS_DESIRED, iIdxCur); if( pWInfo==0 ) goto update_cleanup; okOnePass = sqlite3WhereOkOnePass(pWInfo, aiCurOnePass); for(i=0; iaiColumn[i]>=(-1) ); + assert( pPk->aiColumn[i]>=0 ); sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, pPk->aiColumn[i], iPk+i); } if( okOnePass ){ sqlite3VdbeChangeToNoop(v, addrOpen); @@ -505,11 +505,10 @@ ** be used eliminates some redundant opcodes. */ newmask = sqlite3TriggerColmask( pParse, pTrigger, pChanges, 1, TRIGGER_BEFORE, pTab, onError ); - /*sqlite3VdbeAddOp3(v, OP_Null, 0, regNew, regNew+pTab->nCol-1);*/ for(i=0; inCol; i++){ if( i==pTab->iPKey ){ sqlite3VdbeAddOp2(v, OP_Null, 0, regNew+i); }else{ j = aXRef[i]; @@ -521,11 +520,11 @@ ** if there are one or more BEFORE triggers that use this value via ** a new.* reference in a trigger program. */ testcase( i==31 ); testcase( i==32 ); - sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, i, regNew+i); + sqlite3ExprCodeGetColumnToReg(pParse, pTab, i, iDataCur, regNew+i); }else{ sqlite3VdbeAddOp2(v, OP_Null, 0, regNew+i); } } } @@ -563,11 +562,11 @@ } } } if( !isView ){ - int j1 = 0; /* Address of jump instruction */ + int addr1 = 0; /* Address of jump instruction */ int bReplace = 0; /* True if REPLACE conflict resolution might happen */ /* Do constraint checks. */ assert( regOldRowid>0 ); sqlite3GenerateConstraintChecks(pParse, pTab, aRegIdx, iDataCur, iIdxCur, @@ -579,13 +578,13 @@ } /* Delete the index entries associated with the current record. */ if( bReplace || chngKey ){ if( pPk ){ - j1 = sqlite3VdbeAddOp4Int(v, OP_NotFound, iDataCur, 0, regKey, nKey); + addr1 = sqlite3VdbeAddOp4Int(v, OP_NotFound, iDataCur, 0, regKey, nKey); }else{ - j1 = sqlite3VdbeAddOp3(v, OP_NotExists, iDataCur, 0, regOldRowid); + addr1 = sqlite3VdbeAddOp3(v, OP_NotExists, iDataCur, 0, regOldRowid); } VdbeCoverageNeverTaken(v); } sqlite3GenerateRowIndexDelete(pParse, pTab, iDataCur, iIdxCur, aRegIdx, -1); @@ -592,11 +591,11 @@ /* If changing the record number, delete the old record. */ if( hasFK || chngKey || pPk!=0 ){ sqlite3VdbeAddOp2(v, OP_Delete, iDataCur, 0); } if( bReplace || chngKey ){ - sqlite3VdbeJumpHere(v, j1); + sqlite3VdbeJumpHere(v, addr1); } if( hasFK ){ sqlite3FkCheck(pParse, pTab, 0, regNewRowid, aXRef, chngKey); } @@ -683,25 +682,27 @@ #ifndef SQLITE_OMIT_VIRTUALTABLE /* ** Generate code for an UPDATE of a virtual table. ** -** The strategy is that we create an ephemeral table that contains +** There are two possible strategies - the default and the special +** "onepass" strategy. Onepass is only used if the virtual table +** implementation indicates that pWhere may match at most one row. +** +** The default strategy is to create an ephemeral table that contains ** for each row to be changed: ** ** (A) The original rowid of that row. -** (B) The revised rowid for the row. (note1) +** (B) The revised rowid for the row. ** (C) The content of every column in the row. ** -** Then we loop over this ephemeral table and for each row in -** the ephemeral table call VUpdate. -** -** When finished, drop the ephemeral table. -** -** (note1) Actually, if we know in advance that (A) is always the same -** as (B) we only store (A), then duplicate (A) when pulling -** it out of the ephemeral table before calling VUpdate. +** Then loop through the contents of this ephemeral table executing a +** VUpdate for each row. When finished, drop the ephemeral table. +** +** The "onepass" strategy does not use an ephemeral table. Instead, it +** stores the same values (A, B and C above) in a register array and +** makes a single invocation of VUpdate. */ static void updateVirtualTable( Parse *pParse, /* The parsing context */ SrcList *pSrc, /* The virtual table to be modified */ Table *pTab, /* The virtual table */ @@ -710,67 +711,97 @@ int *aXRef, /* Mapping from columns of pTab to entries in pChanges */ Expr *pWhere, /* WHERE clause of the UPDATE statement */ int onError /* ON CONFLICT strategy */ ){ Vdbe *v = pParse->pVdbe; /* Virtual machine under construction */ - ExprList *pEList = 0; /* The result set of the SELECT statement */ - Select *pSelect = 0; /* The SELECT statement */ - Expr *pExpr; /* Temporary expression */ int ephemTab; /* Table holding the result of the SELECT */ int i; /* Loop counter */ - int addr; /* Address of top of loop */ - int iReg; /* First register in set passed to OP_VUpdate */ sqlite3 *db = pParse->db; /* Database connection */ const char *pVTab = (const char*)sqlite3GetVTable(db, pTab); - SelectDest dest; - - /* Construct the SELECT statement that will find the new values for - ** all updated rows. - */ - pEList = sqlite3ExprListAppend(pParse, 0, sqlite3Expr(db, TK_ID, "_rowid_")); - if( pRowid ){ - pEList = sqlite3ExprListAppend(pParse, pEList, - sqlite3ExprDup(db, pRowid, 0)); - } - assert( pTab->iPKey<0 ); - for(i=0; inCol; i++){ - if( aXRef[i]>=0 ){ - pExpr = sqlite3ExprDup(db, pChanges->a[aXRef[i]].pExpr, 0); - }else{ - pExpr = sqlite3Expr(db, TK_ID, pTab->aCol[i].zName); - } - pEList = sqlite3ExprListAppend(pParse, pEList, pExpr); - } - pSelect = sqlite3SelectNew(pParse, pEList, pSrc, pWhere, 0, 0, 0, 0, 0, 0); - - /* Create the ephemeral table into which the update results will - ** be stored. - */ + WhereInfo *pWInfo; + int nArg = 2 + pTab->nCol; /* Number of arguments to VUpdate */ + int regArg; /* First register in VUpdate arg array */ + int regRec; /* Register in which to assemble record */ + int regRowid; /* Register for ephem table rowid */ + int iCsr = pSrc->a[0].iCursor; /* Cursor used for virtual table scan */ + int aDummy[2]; /* Unused arg for sqlite3WhereOkOnePass() */ + int bOnePass; /* True to use onepass strategy */ + int addr; /* Address of OP_OpenEphemeral */ + + /* Allocate nArg registers to martial the arguments to VUpdate. Then + ** create and open the ephemeral table in which the records created from + ** these arguments will be temporarily stored. */ assert( v ); ephemTab = pParse->nTab++; - - /* fill the ephemeral table - */ - sqlite3SelectDestInit(&dest, SRT_EphemTab, ephemTab); - sqlite3Select(pParse, pSelect, &dest); - - /* Generate code to scan the ephemeral table and call VUpdate. */ - iReg = ++pParse->nMem; - pParse->nMem += pTab->nCol+1; - addr = sqlite3VdbeAddOp2(v, OP_Rewind, ephemTab, 0); VdbeCoverage(v); - sqlite3VdbeAddOp3(v, OP_Column, ephemTab, 0, iReg); - sqlite3VdbeAddOp3(v, OP_Column, ephemTab, (pRowid?1:0), iReg+1); + addr= sqlite3VdbeAddOp2(v, OP_OpenEphemeral, ephemTab, nArg); + regArg = pParse->nMem + 1; + pParse->nMem += nArg; + regRec = ++pParse->nMem; + regRowid = ++pParse->nMem; + + /* Start scanning the virtual table */ + pWInfo = sqlite3WhereBegin(pParse, pSrc, pWhere, 0,0,WHERE_ONEPASS_DESIRED,0); + if( pWInfo==0 ) return; + + /* Populate the argument registers. */ + sqlite3VdbeAddOp2(v, OP_Rowid, iCsr, regArg); + if( pRowid ){ + sqlite3ExprCode(pParse, pRowid, regArg+1); + }else{ + sqlite3VdbeAddOp2(v, OP_Rowid, iCsr, regArg+1); + } for(i=0; inCol; i++){ - sqlite3VdbeAddOp3(v, OP_Column, ephemTab, i+1+(pRowid!=0), iReg+2+i); + if( aXRef[i]>=0 ){ + sqlite3ExprCode(pParse, pChanges->a[aXRef[i]].pExpr, regArg+2+i); + }else{ + sqlite3VdbeAddOp3(v, OP_VColumn, iCsr, i, regArg+2+i); + } + } + + bOnePass = sqlite3WhereOkOnePass(pWInfo, aDummy); + + if( bOnePass ){ + /* If using the onepass strategy, no-op out the OP_OpenEphemeral coded + ** above. Also, if this is a top-level parse (not a trigger), clear the + ** multi-write flag so that the VM does not open a statement journal */ + sqlite3VdbeChangeToNoop(v, addr); + if( sqlite3IsToplevel(pParse) ){ + pParse->isMultiWrite = 0; + } + }else{ + /* Create a record from the argument register contents and insert it into + ** the ephemeral table. */ + sqlite3VdbeAddOp3(v, OP_MakeRecord, regArg, nArg, regRec); + sqlite3VdbeAddOp2(v, OP_NewRowid, ephemTab, regRowid); + sqlite3VdbeAddOp3(v, OP_Insert, ephemTab, regRec, regRowid); + } + + + if( bOnePass==0 ){ + /* End the virtual table scan */ + sqlite3WhereEnd(pWInfo); + + /* Begin scannning through the ephemeral table. */ + addr = sqlite3VdbeAddOp1(v, OP_Rewind, ephemTab); VdbeCoverage(v); + + /* Extract arguments from the current row of the ephemeral table and + ** invoke the VUpdate method. */ + for(i=0; inCol+2, iReg, pVTab, P4_VTAB); + sqlite3VdbeAddOp4(v, OP_VUpdate, 0, nArg, regArg, pVTab, P4_VTAB); sqlite3VdbeChangeP5(v, onError==OE_Default ? OE_Abort : onError); sqlite3MayAbort(pParse); - sqlite3VdbeAddOp2(v, OP_Next, ephemTab, addr+1); VdbeCoverage(v); - sqlite3VdbeJumpHere(v, addr); - sqlite3VdbeAddOp2(v, OP_Close, ephemTab, 0); - /* Cleanup */ - sqlite3SelectDelete(db, pSelect); + /* End of the ephemeral table scan. Or, if using the onepass strategy, + ** jump to here if the scan visited zero rows. */ + if( bOnePass==0 ){ + sqlite3VdbeAddOp2(v, OP_Next, ephemTab, addr+1); VdbeCoverage(v); + sqlite3VdbeJumpHere(v, addr); + sqlite3VdbeAddOp2(v, OP_Close, ephemTab, 0); + }else{ + sqlite3WhereEnd(pWInfo); + } } #endif /* SQLITE_OMIT_VIRTUALTABLE */ Index: src/vdbe.c ================================================================== --- src/vdbe.c +++ src/vdbe.c @@ -516,20 +516,28 @@ #endif /* ** Return the register of pOp->p2 after first preparing it to be ** overwritten with an integer value. -*/ +*/ +static SQLITE_NOINLINE Mem *out2PrereleaseWithClear(Mem *pOut){ + sqlite3VdbeMemSetNull(pOut); + pOut->flags = MEM_Int; + return pOut; +} static Mem *out2Prerelease(Vdbe *p, VdbeOp *pOp){ Mem *pOut; assert( pOp->p2>0 ); assert( pOp->p2<=(p->nMem-p->nCursor) ); pOut = &p->aMem[pOp->p2]; memAboutToChange(p, pOut); - if( VdbeMemDynamic(pOut) ) sqlite3VdbeMemSetNull(pOut); - pOut->flags = MEM_Int; - return pOut; + if( VdbeMemDynamic(pOut) ){ + return out2PrereleaseWithClear(pOut); + }else{ + pOut->flags = MEM_Int; + return pOut; + } } /* ** Execute as much of a VDBE program as we can. @@ -1262,10 +1270,26 @@ #ifdef SQLITE_DEBUG if( pOut->pScopyFrom==0 ) pOut->pScopyFrom = pIn1; #endif break; } + +/* Opcode: IntCopy P1 P2 * * * +** Synopsis: r[P2]=r[P1] +** +** Transfer the integer value held in register P1 into register P2. +** +** This is an optimized version of SCopy that works only for integer +** values. +*/ +case OP_IntCopy: { /* out2 */ + pIn1 = &aMem[pOp->p1]; + assert( (pIn1->flags & MEM_Int)!=0 ); + pOut = &aMem[pOp->p2]; + sqlite3VdbeMemSetInt64(pOut, pIn1->u.i); + break; +} /* Opcode: ResultRow P1 P2 * * * ** Synopsis: output=r[P1@P2] ** ** The registers P1 through P1+P2-1 contain a single row of @@ -5669,56 +5693,62 @@ } break; } #endif /* SQLITE_OMIT_AUTOINCREMENT */ -/* Opcode: IfPos P1 P2 * * * -** Synopsis: if r[P1]>0 goto P2 +/* Opcode: IfPos P1 P2 P3 * * +** Synopsis: if r[P1]>0 then r[P1]-=P3, goto P2 ** ** Register P1 must contain an integer. -** If the value of register P1 is 1 or greater, jump to P2 and -** add the literal value P3 to register P1. +** If the value of register P1 is 1 or greater, subtract P3 from the +** value in P1 and jump to P2. ** ** If the initial value of register P1 is less than 1, then the ** value is unchanged and control passes through to the next instruction. */ case OP_IfPos: { /* jump, in1 */ pIn1 = &aMem[pOp->p1]; assert( pIn1->flags&MEM_Int ); VdbeBranchTaken( pIn1->u.i>0, 2); - if( pIn1->u.i>0 ) goto jump_to_p2; + if( pIn1->u.i>0 ){ + pIn1->u.i -= pOp->p3; + goto jump_to_p2; + } break; } -/* Opcode: IfNeg P1 P2 P3 * * -** Synopsis: r[P1]+=P3, if r[P1]<0 goto P2 +/* Opcode: SetIfNotPos P1 P2 P3 * * +** Synopsis: if r[P1]<=0 then r[P2]=P3 ** -** Register P1 must contain an integer. Add literal P3 to the value in -** register P1 then if the value of register P1 is less than zero, jump to P2. +** Register P1 must contain an integer. +** If the value of register P1 is not positive (if it is less than 1) then +** set the value of register P2 to be the integer P3. */ -case OP_IfNeg: { /* jump, in1 */ +case OP_SetIfNotPos: { /* in1, in2 */ pIn1 = &aMem[pOp->p1]; assert( pIn1->flags&MEM_Int ); - pIn1->u.i += pOp->p3; - VdbeBranchTaken(pIn1->u.i<0, 2); - if( pIn1->u.i<0 ) goto jump_to_p2; + if( pIn1->u.i<=0 ){ + pOut = out2Prerelease(p, pOp); + pOut->u.i = pOp->p3; + } break; } /* Opcode: IfNotZero P1 P2 P3 * * -** Synopsis: if r[P1]!=0 then r[P1]+=P3, goto P2 +** Synopsis: if r[P1]!=0 then r[P1]-=P3, goto P2 ** ** Register P1 must contain an integer. If the content of register P1 is -** initially nonzero, then add P3 to P1 and jump to P2. If register P1 is -** initially zero, leave it unchanged and fall through. +** initially nonzero, then subtract P3 from the value in register P1 and +** jump to P2. If register P1 is initially zero, leave it unchanged +** and fall through. */ case OP_IfNotZero: { /* jump, in1 */ pIn1 = &aMem[pOp->p1]; assert( pIn1->flags&MEM_Int ); VdbeBranchTaken(pIn1->u.i<0, 2); if( pIn1->u.i ){ - pIn1->u.i += pOp->p3; + pIn1->u.i -= pOp->p3; goto jump_to_p2; } break; } Index: src/vdbeblob.c ================================================================== --- src/vdbeblob.c +++ src/vdbeblob.c @@ -246,11 +246,11 @@ #endif for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ int j; for(j=0; jnKeyCol; j++){ /* FIXME: Be smarter about indexes that use expressions */ - if( pIdx->aiColumn[j]==iCol || pIdx->aiColumn[j]==(-2) ){ + if( pIdx->aiColumn[j]==iCol || pIdx->aiColumn[j]==XN_EXPR ){ zFault = "indexed"; } } } if( zFault ){ Index: src/vdbesort.c ================================================================== --- src/vdbesort.c +++ src/vdbesort.c @@ -976,11 +976,11 @@ pKeyInfo->nXField += (pKeyInfo->nField - nField); pKeyInfo->nField = nField; } pSorter->pgsz = pgsz = sqlite3BtreeGetPageSize(db->aDb[0].pBt); pSorter->nTask = nWorker + 1; - pSorter->iPrev = nWorker-1; + pSorter->iPrev = (u8)(nWorker - 1); pSorter->bUseThreads = (pSorter->nTask>1); pSorter->db = db; for(i=0; inTask; i++){ SortSubtask *pTask = &pSorter->aTask[i]; pTask->pSorter = pSorter; Index: src/vtab.c ================================================================== --- src/vtab.c +++ src/vtab.c @@ -935,11 +935,13 @@ ** sqlite3.aVTrans[] array. */ rc = growVTrans(db); if( rc==SQLITE_OK ){ rc = pModule->xBegin(pVTab->pVtab); if( rc==SQLITE_OK ){ + int iSvpt = db->nStatement + db->nSavepoint; addToVTrans(db, pVTab); + if( iSvpt ) rc = sqlite3VtabSavepoint(db, SAVEPOINT_BEGIN, iSvpt-1); } } } return rc; } @@ -1141,11 +1143,11 @@ ** Erase the eponymous virtual table instance associated with ** virtual table module pMod, if it exists. */ void sqlite3VtabEponymousTableClear(sqlite3 *db, Module *pMod){ Table *pTab = pMod->pEpoTab; - if( (pTab = pMod->pEpoTab)!=0 ){ + if( pTab!=0 ){ sqlite3DeleteColumnNames(db, pTab); sqlite3VtabClear(db, pTab); sqlite3DbFree(db, pTab); pMod->pEpoTab = 0; } Index: src/where.c ================================================================== --- src/where.c +++ src/where.c @@ -85,10 +85,17 @@ ** aiCur[0] and aiCur[1] both get -1 if the where-clause logic is ** unable to use the ONEPASS optimization. */ int sqlite3WhereOkOnePass(WhereInfo *pWInfo, int *aiCur){ memcpy(aiCur, pWInfo->aiCurOnePass, sizeof(int)*2); +#ifdef WHERETRACE_ENABLED + if( sqlite3WhereTrace && pWInfo->eOnePass!=ONEPASS_OFF ){ + sqlite3DebugPrintf("%s cursors: %d %d\n", + pWInfo->eOnePass==ONEPASS_SINGLE ? "ONEPASS_SINGLE" : "ONEPASS_MULTI", + aiCur[0], aiCur[1]); + } +#endif return pWInfo->eOnePass; } /* ** Move the content of pSrc into pDest @@ -180,25 +187,24 @@ int k = pScan->k; /* Where to start scanning */ while( pScan->iEquiv<=pScan->nEquiv ){ iCur = pScan->aiCur[pScan->iEquiv-1]; iColumn = pScan->aiColumn[pScan->iEquiv-1]; - assert( iColumn!=(-2) || pScan->pIdxExpr!=0 ); + if( iColumn==XN_EXPR && pScan->pIdxExpr==0 ) return 0; while( (pWC = pScan->pWC)!=0 ){ for(pTerm=pWC->a+k; knTerm; k++, pTerm++){ if( pTerm->leftCursor==iCur && pTerm->u.leftColumn==iColumn - && (iColumn!=(-2) - || sqlite3ExprCompare(pTerm->pExpr->pLeft,pScan->pIdxExpr,iCur)==0) + && (iColumn!=XN_EXPR + || sqlite3ExprCompare(pTerm->pExpr->pLeft,pScan->pIdxExpr,iCur)==0) && (pScan->iEquiv<=1 || !ExprHasProperty(pTerm->pExpr, EP_FromJoin)) ){ if( (pTerm->eOperator & WO_EQUIV)!=0 && pScan->nEquivaiCur) + && (pX = sqlite3ExprSkipCollate(pTerm->pExpr->pRight))->op==TK_COLUMN ){ int j; - pX = sqlite3ExprSkipCollate(pTerm->pExpr->pRight); - assert( pX->op==TK_COLUMN ); for(j=0; jnEquiv; j++){ if( pScan->aiCur[j]==pX->iTable && pScan->aiColumn[j]==pX->iColumn ){ break; } @@ -280,11 +286,11 @@ pScan->pWC = pWC; pScan->pIdxExpr = 0; if( pIdx ){ j = iColumn; iColumn = pIdx->aiColumn[j]; - if( iColumn==(-2) ) pScan->pIdxExpr = pIdx->aColExpr->a[j].pExpr; + if( iColumn==XN_EXPR ) pScan->pIdxExpr = pIdx->aColExpr->a[j].pExpr; } if( pIdx && iColumn>=0 ){ pScan->idxaff = pIdx->pTable->aCol[iColumn].affinity; pScan->zCollName = pIdx->azColl[j]; }else{ @@ -719,11 +725,11 @@ pIdx->azColl[n] = "BINARY"; n++; } } assert( n==nKeyCol ); - pIdx->aiColumn[n] = -1; + pIdx->aiColumn[n] = XN_ROWID; pIdx->azColl[n] = "BINARY"; /* Create the automatic index */ assert( pLevel->iIdxCur>=0 ); pLevel->iIdxCur = pParse->nTab++; @@ -1158,10 +1164,11 @@ #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 /* ** Return the affinity for a single column of an index. */ static char sqlite3IndexColumnAffinity(sqlite3 *db, Index *pIdx, int iCol){ + assert( iCol>=0 && iColnColumn ); if( !pIdx->zColAff ){ if( sqlite3IndexAffinityStr(db, pIdx)==0 ) return SQLITE_AFF_BLOB; } return pIdx->zColAff[iCol]; } @@ -1215,12 +1222,11 @@ int nEq = pLoop->u.btree.nEq; sqlite3 *db = pParse->db; int nLower = -1; int nUpper = p->nSample+1; int rc = SQLITE_OK; - int iCol = p->aiColumn[nEq]; - u8 aff = sqlite3IndexColumnAffinity(db, p, iCol); + u8 aff = sqlite3IndexColumnAffinity(db, p, nEq); CollSeq *pColl; sqlite3_value *p1 = 0; /* Value extracted from pLower */ sqlite3_value *p2 = 0; /* Value extracted from pUpper */ sqlite3_value *pVal = 0; /* Value extracted from record */ @@ -2234,11 +2240,13 @@ }else if( eOp & (WO_EQ|WO_IS) ){ int iCol = pProbe->aiColumn[saved_nEq]; pNew->wsFlags |= WHERE_COLUMN_EQ; assert( saved_nEq==pNew->u.btree.nEq ); - if( iCol==(-1) || (iCol>0 && nInMul==0 && saved_nEq==pProbe->nKeyCol-1) ){ + if( iCol==XN_ROWID + || (iCol>0 && nInMul==0 && saved_nEq==pProbe->nKeyCol-1) + ){ if( iCol>=0 && pProbe->uniqNotNull==0 ){ pNew->wsFlags |= WHERE_UNQ_WANTED; }else{ pNew->wsFlags |= WHERE_ONEROW; } @@ -2420,21 +2428,28 @@ WhereLoopBuilder *pBuilder, Index *pIndex, int iCursor ){ ExprList *pOB; + ExprList *aColExpr; int ii, jj; if( pIndex->bUnordered ) return 0; if( (pOB = pBuilder->pWInfo->pOrderBy)==0 ) return 0; for(ii=0; iinExpr; ii++){ Expr *pExpr = sqlite3ExprSkipCollate(pOB->a[ii].pExpr); - if( pExpr->op!=TK_COLUMN ) return 0; - if( pExpr->iTable==iCursor ){ + if( pExpr->op==TK_COLUMN && pExpr->iTable==iCursor ){ if( pExpr->iColumn<0 ) return 1; for(jj=0; jjnKeyCol; jj++){ if( pExpr->iColumn==pIndex->aiColumn[jj] ) return 1; + } + }else if( (aColExpr = pIndex->aColExpr)!=0 ){ + for(jj=0; jjnKeyCol; jj++){ + if( pIndex->aiColumn[jj]!=XN_EXPR ) continue; + if( sqlite3ExprCompare(pExpr,aColExpr->a[jj].pExpr,iCursor)==0 ){ + return 1; + } } } } return 0; } @@ -2579,11 +2594,11 @@ if( !pBuilder->pOrSet /* Not part of an OR optimization */ && (pWInfo->wctrlFlags & WHERE_NO_AUTOINDEX)==0 && (pWInfo->pParse->db->flags & SQLITE_AutoIndex)!=0 && pSrc->pIBIndex==0 /* Has no INDEXED BY clause */ && !pSrc->fg.notIndexed /* Has no NOT INDEXED clause */ - && HasRowid(pTab) /* Is not a WITHOUT ROWID table. (FIXME: Why not?) */ + && HasRowid(pTab) /* Not WITHOUT ROWID table. (FIXME: Why not?) */ && !pSrc->fg.isCorrelated /* Not a correlated subquery */ && !pSrc->fg.isRecursive /* Not a recursive common table expression. */ ){ /* Generate auto-index WhereLoops */ WhereTerm *pTerm; @@ -2824,10 +2839,11 @@ pIdxInfo->idxNum = 0; pIdxInfo->needToFreeIdxStr = 0; pIdxInfo->orderByConsumed = 0; pIdxInfo->estimatedCost = SQLITE_BIG_DBL / (double)2; pIdxInfo->estimatedRows = 25; + pIdxInfo->idxFlags = 0; rc = vtabBestIndex(pParse, pTab, pIdxInfo); if( rc ) goto whereLoopAddVtab_exit; pIdxCons = *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint; pNew->prereq = mExtra; mxTerm = -1; @@ -2869,10 +2885,11 @@ ** consume the ORDER BY clause because (1) the order of IN terms ** is not necessarily related to the order of output terms and ** (2) Multiple outputs from a single IN value will not merge ** together. */ pIdxInfo->orderByConsumed = 0; + pIdxInfo->idxFlags &= ~SQLITE_INDEX_SCAN_UNIQUE; } } } if( i>=nConstraint ){ pNew->nLTerm = mxTerm+1; @@ -2884,10 +2901,18 @@ pNew->u.vtab.isOrdered = (i8)(pIdxInfo->orderByConsumed ? pIdxInfo->nOrderBy : 0); pNew->rSetup = 0; pNew->rRun = sqlite3LogEstFromDouble(pIdxInfo->estimatedCost); pNew->nOut = sqlite3LogEst(pIdxInfo->estimatedRows); + + /* Set the WHERE_ONEROW flag if the xBestIndex() method indicated + ** that the scan will visit at most one row. Clear it otherwise. */ + if( pIdxInfo->idxFlags & SQLITE_INDEX_SCAN_UNIQUE ){ + pNew->wsFlags |= WHERE_ONEROW; + }else{ + pNew->wsFlags &= ~WHERE_ONEROW; + } whereLoopInsert(pBuilder, pNew); if( pNew->u.vtab.needFree ){ sqlite3_free(pNew->u.vtab.idxStr); pNew->u.vtab.needFree = 0; } @@ -3205,11 +3230,12 @@ return 0; }else{ nKeyCol = pIndex->nKeyCol; nColumn = pIndex->nColumn; assert( nColumn==nKeyCol+1 || !HasRowid(pIndex->pTable) ); - assert( pIndex->aiColumn[nColumn-1]==(-1) || !HasRowid(pIndex->pTable)); + assert( pIndex->aiColumn[nColumn-1]==XN_ROWID + || !HasRowid(pIndex->pTable)); isOrderDistinct = IsUniqueIndex(pIndex); } /* Loop through all columns of the index and deal with the ones ** that are not constrained by == or IN. @@ -3237,11 +3263,11 @@ if( pIndex ){ iColumn = pIndex->aiColumn[j]; revIdx = pIndex->aSortOrder[j]; if( iColumn==pIndex->pTable->iPKey ) iColumn = -1; }else{ - iColumn = -1; + iColumn = XN_ROWID; revIdx = 0; } /* An unconstrained column that might be NULL means that this ** WhereLoop is not well-ordered @@ -3263,13 +3289,19 @@ if( MASKBIT(i) & obSat ) continue; pOBExpr = sqlite3ExprSkipCollate(pOrderBy->a[i].pExpr); testcase( wctrlFlags & WHERE_GROUPBY ); testcase( wctrlFlags & WHERE_DISTINCTBY ); if( (wctrlFlags & (WHERE_GROUPBY|WHERE_DISTINCTBY))==0 ) bOnce = 0; - if( pOBExpr->op!=TK_COLUMN ) continue; - if( pOBExpr->iTable!=iCur ) continue; - if( pOBExpr->iColumn!=iColumn ) continue; + if( iColumn>=(-1) ){ + if( pOBExpr->op!=TK_COLUMN ) continue; + if( pOBExpr->iTable!=iCur ) continue; + if( pOBExpr->iColumn!=iColumn ) continue; + }else{ + if( sqlite3ExprCompare(pOBExpr,pIndex->aColExpr->a[j].pExpr,iCur) ){ + continue; + } + } if( iColumn>=0 ){ pColl = sqlite3ExprCollSeq(pWInfo->pParse, pOrderBy->a[i].pExpr); if( !pColl ) pColl = db->pDfltColl; if( sqlite3StrICmp(pColl->zName, pIndex->azColl[j])!=0 ) continue; } @@ -4096,11 +4128,12 @@ pWInfo->pOrderBy = pResultSet; } } /* Construct the WhereLoop objects */ - WHERETRACE(0xffff,("*** Optimizer Start ***\n")); + WHERETRACE(0xffff,("*** Optimizer Start *** (wctrlFlags: 0x%x)\n", + wctrlFlags)); #if defined(WHERETRACE_ENABLED) if( sqlite3WhereTrace & 0x100 ){ /* Display all terms of the WHERE clause */ int i; for(i=0; inTerm; i++){ whereTermPrint(&sWLB.pWC->a[i], i); @@ -4515,11 +4548,14 @@ if( pLoop->wsFlags & (WHERE_INDEXED|WHERE_IDX_ONLY) ){ pIdx = pLoop->u.btree.pIndex; }else if( pLoop->wsFlags & WHERE_MULTI_OR ){ pIdx = pLevel->u.pCovidx; } - if( pIdx && !db->mallocFailed ){ + if( pIdx + && (pWInfo->eOnePass==ONEPASS_OFF || !HasRowid(pIdx->pTable)) + && !db->mallocFailed + ){ last = sqlite3VdbeCurrentAddr(v); k = pLevel->addrBody; pOp = sqlite3VdbeGetOp(v, k); for(; kp1!=pLevel->iTabCur ) continue; @@ -4527,10 +4563,11 @@ int x = pOp->p2; assert( pIdx->pTable==pTab ); if( !HasRowid(pTab) ){ Index *pPk = sqlite3PrimaryKeyIndex(pTab); x = pPk->aiColumn[x]; + assert( x>=0 ); } x = sqlite3ColumnOfIndex(pIdx, x); if( x>=0 ){ pOp->p2 = x; pOp->p1 = pLevel->iIdxCur; Index: src/wherecode.c ================================================================== --- src/wherecode.c +++ src/wherecode.c @@ -44,12 +44,12 @@ /* ** Return the name of the i-th column of the pIdx index. */ static const char *explainIndexColumnName(Index *pIdx, int i){ i = pIdx->aiColumn[i]; - if( i==(-2) ) return ""; - if( i==(-1) ) return "rowid"; + if( i==XN_EXPR ) return ""; + if( i==XN_ROWID ) return "rowid"; return pIdx->pTable->aCol[i].zName; } /* ** Argument pLevel describes a strategy for scanning table pTab. This @@ -512,11 +512,11 @@ VdbeCoverageIf(v, bRev==0); VdbeCoverageIf(v, bRev!=0); sqlite3VdbeJumpHere(v, j); for(j=0; jaiColumn[j]==(-2) ); + testcase( pIdx->aiColumn[j]==XN_EXPR ); VdbeComment((v, "%s", explainIndexColumnName(pIdx, j))); } } /* Evaluate the equality constraints @@ -698,12 +698,12 @@ for(j=0; ju.vtab.omitMask>>j)&1 ){ disableTerm(pLevel, pLoop->aLTerm[j]); } } - pLevel->op = OP_VNext; pLevel->p1 = iCur; + pLevel->op = pWInfo->eOnePass ? OP_Noop : OP_VNext; pLevel->p2 = sqlite3VdbeCurrentAddr(v); sqlite3ReleaseTempRange(pParse, iReg, nConstraint+2); sqlite3ExprCachePop(pParse); }else #endif /* SQLITE_OMIT_VIRTUALTABLE */ @@ -1264,11 +1264,11 @@ for(ii=0; iinTerm; ii++){ WhereTerm *pOrTerm = &pOrWc->a[ii]; if( pOrTerm->leftCursor==iCur || (pOrTerm->eOperator & WO_AND)!=0 ){ WhereInfo *pSubWInfo; /* Info for single OR-term scan */ Expr *pOrExpr = pOrTerm->pExpr; /* Current OR clause term */ - int j1 = 0; /* Address of jump operation */ + int jmp1 = 0; /* Address of jump operation */ if( pAndExpr && !ExprHasProperty(pOrExpr, EP_FromJoin) ){ pAndExpr->pLeft = pOrExpr; pOrExpr = pAndExpr; } /* Loop through table entries that match term pOrTerm. */ @@ -1291,11 +1291,12 @@ if( (pWInfo->wctrlFlags & WHERE_DUPLICATES_OK)==0 ){ int r; int iSet = ((ii==pOrWc->nTerm-1)?-1:ii); if( HasRowid(pTab) ){ r = sqlite3ExprCodeGetColumn(pParse, pTab, -1, iCur, regRowid, 0); - j1 = sqlite3VdbeAddOp4Int(v, OP_RowSetTest, regRowset, 0, r,iSet); + jmp1 = sqlite3VdbeAddOp4Int(v, OP_RowSetTest, regRowset, 0, + r,iSet); VdbeCoverage(v); }else{ Index *pPk = sqlite3PrimaryKeyIndex(pTab); int nPk = pPk->nKeyCol; int iPk; @@ -1302,15 +1303,11 @@ /* Read the PK into an array of temp registers. */ r = sqlite3GetTempRange(pParse, nPk); for(iPk=0; iPkaiColumn[iPk]; - int rx; - rx = sqlite3ExprCodeGetColumn(pParse, pTab, iCol, iCur,r+iPk,0); - if( rx!=r+iPk ){ - sqlite3VdbeAddOp2(v, OP_SCopy, rx, r+iPk); - } + sqlite3ExprCodeGetColumnToReg(pParse, pTab, iCol, iCur, r+iPk); } /* Check if the temp table already contains this key. If so, ** the row has already been included in the result set and ** can be ignored (by jumping past the Gosub below). Otherwise, @@ -1321,11 +1318,11 @@ ** is zero, assume that the key cannot already be present in ** the temp table. And if iSet is -1, assume that there is no ** need to insert the key into the temp table, as it will never ** be tested for. */ if( iSet ){ - j1 = sqlite3VdbeAddOp4Int(v, OP_Found, regRowset, 0, r, nPk); + jmp1 = sqlite3VdbeAddOp4Int(v, OP_Found, regRowset, 0, r, nPk); VdbeCoverage(v); } if( iSet>=0 ){ sqlite3VdbeAddOp3(v, OP_MakeRecord, r, nPk, regRowid); sqlite3VdbeAddOp3(v, OP_IdxInsert, regRowset, regRowid, 0); @@ -1340,11 +1337,11 @@ /* Invoke the main loop body as a subroutine */ sqlite3VdbeAddOp2(v, OP_Gosub, regReturn, iLoopBody); /* Jump here (skipping the main loop body subroutine) if the ** current sub-WHERE row is a duplicate from prior sub-WHEREs. */ - if( j1 ) sqlite3VdbeJumpHere(v, j1); + if( jmp1 ) sqlite3VdbeJumpHere(v, jmp1); /* The pSubWInfo->untestedTerms flag means that this OR term ** contained one or more AND term from a notReady table. The ** terms from the notReady table could not be tested and will ** need to be tested later. Index: src/whereexpr.c ================================================================== --- src/whereexpr.c +++ src/whereexpr.c @@ -948,11 +948,10 @@ }else{ pDup = pExpr; pNew = pTerm; } exprCommute(pParse, pDup); - pLeft = sqlite3ExprSkipCollate(pDup->pLeft); pNew->leftCursor = iCur; pNew->u.leftColumn = iColumn; testcase( (prereqLeft | extraRight) != prereqLeft ); pNew->prereqRight = prereqLeft | extraRight; pNew->prereqAll = prereqAll; Index: test/delete4.test ================================================================== --- test/delete4.test +++ test/delete4.test @@ -95,8 +95,51 @@ } {1 3 5 7} do_execsql_test 3.4 { PRAGMA integrity_check; } {ok} + +# Between 2015-09-14 and 2015-09-28, the following test cases would result +# in corruption (wrong # of entries in index) due to a bug in the ONEPASS +# optimization. +# +do_execsql_test 4.1 { + DROP TABLE IF EXISTS t4; + CREATE TABLE t4(col0, col1); + INSERT INTO "t4" VALUES(14, 'abcde'); + CREATE INDEX idx_t4_0 ON t4 (col1, col0); + CREATE INDEX idx_t4_3 ON t4 (col0); + DELETE FROM t4 WHERE col0=69 OR col0>7; + PRAGMA integrity_check; +} {ok} +do_execsql_test 4.2 { + DROP TABLE IF EXISTS t4; + CREATE TABLE t4(col0, col1); + INSERT INTO "t4" VALUES(14, 'abcde'); + CREATE INDEX idx_t4_3 ON t4 (col0); + CREATE INDEX idx_t4_0 ON t4 (col1, col0); + DELETE FROM t4 WHERE col0=69 OR col0>7; + PRAGMA integrity_check; +} {ok} +do_execsql_test 4.11 { + DROP TABLE IF EXISTS t4; + CREATE TABLE t4(col0, col1, pk PRIMARY KEY) WITHOUT ROWID; + INSERT INTO t4 VALUES(14, 'abcde','xyzzy'); + CREATE INDEX idx_t4_0 ON t4 (col1, col0); + CREATE INDEX idx_t4_3 ON t4 (col0); + DELETE FROM t4 WHERE col0=69 OR col0>7; + PRAGMA integrity_check; +} {ok} +do_execsql_test 4.12 { + DROP TABLE IF EXISTS t4; + CREATE TABLE t4(col0, col1, pk PRIMARY KEY) WITHOUT ROWID; + INSERT INTO t4 VALUES(14, 'abcde','xyzzy'); + CREATE INDEX idx_t4_3 ON t4 (col0); + CREATE INDEX idx_t4_0 ON t4 (col1, col0); + DELETE FROM t4 WHERE col0=69 OR col0>7; + PRAGMA integrity_check; +} {ok} + + finish_test Index: test/fkey1.test ================================================================== --- test/fkey1.test +++ test/fkey1.test @@ -148,7 +148,41 @@ SELECT 2, """5" FROM """4"; } {1 abc} do_execsql_test fkey1-4.2 { PRAGMA table_info="""1"; } {0 {"2} TEXT 0 {} 1 1 {"3} TEXT 0 {} 0} + +#------------------------------------------------------------------------- +# +do_execsql_test fkey1-5.1 { + CREATE TABLE t11( + x INTEGER PRIMARY KEY, + parent REFERENCES t11 ON DELETE CASCADE + ); + INSERT INTO t11 VALUES (1, NULL), (2, 1), (3, 2); +} {} + +# The REPLACE part of this statement deletes the row (2, 1). Then the +# DELETE CASCADE caused by deleting that row removes the (3, 2) row. Which +# would have been the parent of the new row being inserted. Causing an +# FK violation. +# +do_catchsql_test fkey1-5.2 { + INSERT OR REPLACE INTO t11 VALUES (2, 3); +} {1 {FOREIGN KEY constraint failed}} + +# A similar test to the above. +do_execsql_test fkey1-5.3 { + CREATE TABLE Foo ( + Id INTEGER PRIMARY KEY, + ParentId INTEGER REFERENCES Foo(Id) ON DELETE CASCADE, C1 + ); + INSERT OR REPLACE INTO Foo(Id, ParentId, C1) VALUES (1, null, 'A'); + INSERT OR REPLACE INTO Foo(Id, ParentId, C1) VALUES (2, 1, 'A-2-1'); + INSERT OR REPLACE INTO Foo(Id, ParentId, C1) VALUES (3, 2, 'A-3-2'); + INSERT OR REPLACE INTO Foo(Id, ParentId, C1) VALUES (4, 3, 'A-4-3'); +} +do_catchsql_test fkey1-5.4 { + INSERT OR REPLACE INTO Foo(Id, ParentId, C1) VALUES (2, 3, 'A-2-3'); +} {1 {FOREIGN KEY constraint failed}} finish_test Index: test/fts3conf.test ================================================================== --- test/fts3conf.test +++ test/fts3conf.test @@ -84,15 +84,15 @@ 7 "INSERT OR ABORT $T2" 1 1 {{a b c d} {e f g h} {i j k l}} 8 "INSERT OR FAIL $T2" 1 1 {{a b c d} {e f g h} {i j k l} z} 9 "INSERT OR IGNORE $T2" 1 0 {{a b c d} {e f g h} {i j k l} z} 10 "INSERT OR REPLACE $T2" 1 0 {{a b c d} y {i j k l} z} - 11 "UPDATE OR ROLLBACK $T3" 1 1 {{a b c d} {e f g h}} - 12 "UPDATE OR ABORT $T3" 1 1 {{a b c d} {e f g h} {i j k l}} - 13 "UPDATE OR FAIL $T3" 1 1 {{a b c d} {e f g h} {i j k l}} - 14 "UPDATE OR IGNORE $T3" 1 0 {{a b c d} {e f g h} {i j k l}} - 15 "UPDATE OR REPLACE $T3" 1 0 {{a b c d} {i j k l}} + 11 "UPDATE OR ROLLBACK $T3" 0 1 {{a b c d} {e f g h}} + 12 "UPDATE OR ABORT $T3" 0 1 {{a b c d} {e f g h} {i j k l}} + 13 "UPDATE OR FAIL $T3" 0 1 {{a b c d} {e f g h} {i j k l}} + 14 "UPDATE OR IGNORE $T3" 0 0 {{a b c d} {e f g h} {i j k l}} + 15 "UPDATE OR REPLACE $T3" 0 0 {{a b c d} {i j k l}} 16 "UPDATE OR ROLLBACK $T4" 1 1 {{a b c d} {e f g h}} 17 "UPDATE OR ABORT $T4" 1 1 {{a b c d} {e f g h} {i j k l}} 18 "UPDATE OR FAIL $T4" 1 1 {{e f g h} {i j k l} {a b c d}} 19 "UPDATE OR IGNORE $T4" 1 0 {{e f g h} {i j k l} {a b c d}} @@ -175,7 +175,41 @@ do_execsql_test 3.8 { UPDATE OR REPLACE t3 SET docid = 5, content='three four' WHERE docid = 4; SELECT quote(matchinfo(t3, 'na')) FROM t3 WHERE t3 MATCH 'one' } {X'0200000002000000'} + +#------------------------------------------------------------------------- +# Test that the xSavepoint is invoked correctly if the first write +# operation within a transaction is to a virtual table. +# +do_catchsql_test 4.1.1 { + CREATE VIRTUAL TABLE t0 USING fts4; + BEGIN; + INSERT INTO t0(rowid, content) SELECT + 1, 'abc' UNION ALL SELECT + 2, 'def' UNION ALL SELECT + 1, 'ghi'; +} {1 {constraint failed}} +do_execsql_test 4.1.2 { + COMMIT; +} +do_execsql_test 4.1.3 { + SELECT * FROM t0 WHERE t0 MATCH 'abc'; + INSERT INTO t0(t0) VALUES('integrity-check'); +} {} + +do_execsql_test 4.2.1 { + CREATE VIRTUAL TABLE t01 USING fts4; + BEGIN; + SAVEPOINT abc; + INSERT INTO t01 VALUES('a b c'); + ROLLBACK TO abc; + COMMIT; +} +do_execsql_test 4.2.2 { + SELECT * FROM t01 WHERE t01 MATCH 'b'; + INSERT INTO t01(t01) VALUES('integrity-check'); +} {} finish_test + Index: test/fts3expr3.test ================================================================== --- test/fts3expr3.test +++ test/fts3expr3.test @@ -119,10 +119,12 @@ set node "{[balanced_and_tree $nEntry]}" regsub -all AND $node OR node regsub -all xxx $tree $node tree return $tree } + +if 1 { # Test that queries like "1 AND 2 AND 3 AND 4..." are transformed to # balanced trees by FTS. # for {set i 1} {$i < 100} {incr i} { @@ -199,8 +201,38 @@ do_faultsim_test fts3expr3-fault-1 -faults oom-* -body { test_fts3expr2 $::query } -test { faultsim_test_result [list 0 $::result] } + +} + +#------------------------------------------------------------------- + +foreach {tn expr res} { + 1 {1 OR 2 OR 3 OR 4} {OR {OR {P 1} {P 2}} {OR {P 3} {P 4}}} + 2 {1 OR (2 AND 3 AND 4 AND 5)} + {OR {P 1} {AND {AND {P 2} {P 3}} {AND {P 4} {P 5}}}} + 3 {(2 AND 3 AND 4 AND 5) OR 1} + {OR {AND {AND {P 2} {P 3}} {AND {P 4} {P 5}}} {P 1}} + + 4 {1 AND (2 OR 3 OR 4 OR 5)} + {AND {P 1} {OR {OR {P 2} {P 3}} {OR {P 4} {P 5}}}} + 5 {(2 OR 3 OR 4 OR 5) AND 1} + {AND {OR {OR {P 2} {P 3}} {OR {P 4} {P 5}}} {P 1}} + + 6 {(2 OR 3 OR 4 OR 5) NOT 1} + {NOT {OR {OR {P 2} {P 3}} {OR {P 4} {P 5}}} {P 1}} + + 7 {1 NOT (2 OR 3 OR 4 OR 5)} + {NOT {P 1} {OR {OR {P 2} {P 3}} {OR {P 4} {P 5}}}} + + 8 {(1 OR 2 OR 3 OR 4) NOT (5 AND 6 AND 7 AND 8)} + {NOT {OR {OR {P 1} {P 2}} {OR {P 3} {P 4}}} {AND {AND {P 5} {P 6}} {AND {P 7} {P 8}}}} +} { + do_test 5.1.$tn { + test_fts3expr2 $expr + } $res +} set sqlite_fts3_enable_parentheses 0 finish_test ADDED test/fts4onepass.test Index: test/fts4onepass.test ================================================================== --- /dev/null +++ test/fts4onepass.test @@ -0,0 +1,147 @@ +# 2015 Sep 27 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/fts3_common.tcl +set ::testprefix fts4onepass + +# If SQLITE_ENABLE_FTS3 is defined, omit this file. +ifcapable !fts3 { + finish_test + return +} + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE ft USING fts3; + INSERT INTO ft(rowid, content) VALUES(1, '1 2 3'); + INSERT INTO ft(rowid, content) VALUES(2, '4 5 6'); + INSERT INTO ft(rowid, content) VALUES(3, '7 8 9'); +} + +#------------------------------------------------------------------------- +# Check that UPDATE and DELETE statements that feature "WHERE rowid=?" or +# or "WHERE docid=?" clauses do not use statement journals. But that other +# DELETE and UPDATE statements do. +# +# Note: "MATCH ? AND docid=?" does use a statement journal. +# +foreach {tn sql uses} { + 1.1 { DELETE FROM ft } 1 + 1.2 { DELETE FROM ft WHERE docid=? } 0 + 1.3 { DELETE FROM ft WHERE rowid=? } 0 + 1.4 { DELETE FROM ft WHERE ft MATCH '1' } 1 + 1.5 { DELETE FROM ft WHERE ft MATCH '1' AND docid=? } 1 + 1.6 { DELETE FROM ft WHERE ft MATCH '1' AND rowid=? } 1 + + 2.1 { UPDATE ft SET content='a b c' } 1 + 2.2 { UPDATE ft SET content='a b c' WHERE docid=? } 0 + 2.3 { UPDATE ft SET content='a b c' WHERE rowid=? } 0 + 2.4 { UPDATE ft SET content='a b c' WHERE ft MATCH '1' } 1 + 2.5 { UPDATE ft SET content='a b c' WHERE ft MATCH '1' AND docid=? } 1 + 2.6 { UPDATE ft SET content='a b c' WHERE ft MATCH '1' AND rowid=? } 1 +} { + do_test 1.$tn { sql_uses_stmt db $sql } $uses +} + +#------------------------------------------------------------------------- +# Check that putting a "DELETE/UPDATE ... WHERE rowid=?" statement in a +# trigger program does not prevent the VM from using a statement +# transaction. Even if the calling statement cannot hit a constraint. +# +do_execsql_test 2.0 { + CREATE TABLE t1(x); + + CREATE TRIGGER t1_ai AFTER INSERT ON t1 BEGIN + DELETE FROM ft WHERE rowid=new.x; + END; + + CREATE TRIGGER t1_ad AFTER DELETE ON t1 BEGIN + UPDATE ft SET content = 'a b c' WHERE rowid=old.x; + END; + + CREATE TRIGGER t1_bu BEFORE UPDATE ON t1 BEGIN + DELETE FROM ft WHERE rowid=old.x; + END; +} + +foreach {tn sql uses} { + 1 { INSERT INTO t1 VALUES(1) } 1 + 2 { DELETE FROM t1 WHERE x=4 } 1 + 3 { UPDATE t1 SET x=10 WHERE x=11 } 1 +} { + do_test 2.$tn { sql_uses_stmt db $sql } $uses +} + +#------------------------------------------------------------------------- +# Test that an "UPDATE ... WHERE rowid=?" works and does not corrupt the +# index when it strikes a constraint. Both inside and outside a +# transaction. +# +foreach {tn tcl1 tcl2} { + 1 {} {} + + 2 { + execsql BEGIN + } { + if {[sqlite3_get_autocommit db]==1} { error "transaction rolled back!" } + execsql COMMIT + } +} { + + do_execsql_test 3.$tn.0 { + DROP TABLE IF EXISTS ft2; + CREATE VIRTUAL TABLE ft2 USING fts4; + INSERT INTO ft2(rowid, content) VALUES(1, 'a b c'); + INSERT INTO ft2(rowid, content) VALUES(2, 'a b d'); + INSERT INTO ft2(rowid, content) VALUES(3, 'a b e'); + } + + eval $tcl1 + foreach {tn2 sql content} { + 1 { UPDATE ft2 SET docid=2 WHERE docid=1 } + { 1 {a b c} 2 {a b d} 3 {a b e} } + + 2 { + INSERT INTO ft2(rowid, content) VALUES(4, 'a b f'); + UPDATE ft2 SET docid=5 WHERE docid=4; + UPDATE ft2 SET docid=3 WHERE docid=5; + } { 1 {a b c} 2 {a b d} 3 {a b e} 5 {a b f} } + + 3 { + UPDATE ft2 SET docid=3 WHERE docid=4; -- matches 0 rows + UPDATE ft2 SET docid=2 WHERE docid=3; + } { 1 {a b c} 2 {a b d} 3 {a b e} 5 {a b f} } + + 4 { + INSERT INTO ft2(rowid, content) VALUES(4, 'a b g'); + UPDATE ft2 SET docid=-1 WHERE docid=4; + UPDATE ft2 SET docid=3 WHERE docid=-1; + } {-1 {a b g} 1 {a b c} 2 {a b d} 3 {a b e} 5 {a b f} } + + 5 { + DELETE FROM ft2 WHERE rowid=451; + DELETE FROM ft2 WHERE rowid=-1; + UPDATE ft2 SET docid = 2 WHERE docid = 1; + } {1 {a b c} 2 {a b d} 3 {a b e} 5 {a b f} } + } { + do_catchsql_test 3.$tn.$tn2.a $sql {1 {constraint failed}} + do_execsql_test 3.$tn.$tn2.b { SELECT rowid, content FROM ft2 } $content + do_execsql_test 3.$tn.$tn2.c { + INSERT INTO ft2(ft2) VALUES('integrity-check'); + } + } + eval $tcl2 +} + +finish_test + Index: test/fuzzcheck.c ================================================================== --- test/fuzzcheck.c +++ test/fuzzcheck.c @@ -1049,11 +1049,10 @@ sqlFuzz = 1; } /* Print the description, if there is one */ if( !quietFlag ){ - int i; zDbName = azSrcDb[iSrcDb]; i = strlen(zDbName) - 1; while( i>0 && zDbName[i-1]!='/' && zDbName[i-1]!='\\' ){ i--; } zDbName += i; sqlite3_prepare_v2(db, "SELECT msg FROM readme", -1, &pStmt, 0); @@ -1125,16 +1124,10 @@ openFlags |= SQLITE_OPEN_MEMORY; zVfs = 0; } rc = sqlite3_open_v2("main.db", &db, openFlags, zVfs); if( rc ) fatalError("cannot open inmem database"); -#ifdef SQLITE_ENABLE_JSON1 - { - extern int sqlite3_json_init(sqlite3*); - sqlite3_json_init(db); - } -#endif if( cellSzCkFlag ) runSql(db, "PRAGMA cell_size_check=ON", runFlags); setAlarm(iTimeout); #ifndef SQLITE_OMIT_PROGRESS_CALLBACK if( sqlFuzz || vdbeLimitFlag ){ sqlite3_progress_handler(db, 100000, progressHandler, &vdbeLimitFlag); Index: test/fuzzdata3.db ================================================================== --- test/fuzzdata3.db +++ test/fuzzdata3.db cannot compute difference between binary files Index: test/indexexpr1.test ================================================================== --- test/indexexpr1.test +++ test/indexexpr1.test @@ -86,10 +86,27 @@ EXPLAIN QUERY PLAN SELECT rowid, b, c FROM t1 WHERE substr(a,27,3)=='ord' AND d>=29; } {/USING INDEX t1a2/} +# ORDER BY using an indexed expression +# +do_execsql_test indexexpr1-170 { + CREATE INDEX t1alen ON t1(length(a)); + SELECT length(a) FROM t1 ORDER BY length(a); +} {20 25 27 29 38 52} +do_execsql_test indexexpr1-170eqp { + EXPLAIN QUERY PLAN + SELECT length(a) FROM t1 ORDER BY length(a); +} {/SCAN TABLE t1 USING INDEX t1alen/} +do_execsql_test indexexpr1-171 { + SELECT length(a) FROM t1 ORDER BY length(a) DESC; +} {52 38 29 27 25 20} +do_execsql_test indexexpr1-171eqp { + EXPLAIN QUERY PLAN + SELECT length(a) FROM t1 ORDER BY length(a) DESC; +} {/SCAN TABLE t1 USING INDEX t1alen/} do_execsql_test indexexpr1-200 { DROP TABLE t1; CREATE TABLE t1(id ANY PRIMARY KEY, a,b,c) WITHOUT ROWID; INSERT INTO t1(id,a,b,c) @@ -191,11 +208,12 @@ WITH RECURSIVE c(x) AS (VALUES(1) UNION SELECT x+1 FROM c WHERE x<30) INSERT INTO t3(a,b,c) SELECT x, printf('ab%04xyz',x), random() FROM c; CREATE UNIQUE INDEX t3abc ON t3(CAST(a AS text), b, substr(c,1,3)); SELECT a FROM t3 WHERE CAST(a AS text)<='10' ORDER BY +a; -} {1 10} + PRAGMA integrity_check; +} {1 10 ok} do_catchsql_test indexexpr1-410 { INSERT INTO t3 SELECT * FROM t3 WHERE rowid=10; } {1 {UNIQUE constraint failed: index 't3abc'}} do_execsql_test indexexpr1-500 { @@ -231,9 +249,63 @@ VALUES('t4','t4all','600000 160000 40000 10000 2000 600 100 40 10'); ANALYZE sqlite_master; SELECT i FROM t4 WHERE e=5; } {9} +# Indexed expressions on both sides of an == in a WHERE clause. +# +do_execsql_test indexexpr1-700 { + DROP TABLE IF EXISTS t7; + CREATE TABLE t7(a,b,c); + INSERT INTO t7(a,b,c) VALUES(1,2,2),('abc','def','def'),(4,5,6); + CREATE INDEX t7b ON t7(+b); + CREATE INDEX t7c ON t7(+c); + SELECT *, '|' FROM t7 WHERE +b=+c ORDER BY +a; +} {1 2 2 | abc def def |} +do_execsql_test indexexpr1-710 { + CREATE TABLE t71(a,b,c); + CREATE INDEX t71bc ON t71(b+c); + CREATE TABLE t72(x,y,z); + CREATE INDEX t72yz ON t72(y+z); + INSERT INTO t71(a,b,c) VALUES(1,11,2),(2,7,15),(3,5,4); + INSERT INTO t72(x,y,z) VALUES(1,10,3),(2,8,14),(3,9,9); + SELECT a, x, '|' FROM t71, t72 + WHERE b+c=y+z + ORDER BY +a, +x; +} {1 1 | 2 2 |} + +# Collating sequences on indexes of expressions +# +do_execsql_test indexexpr1-800 { + DROP TABLE IF EXISTS t8; + CREATE TABLE t8(a INTEGER PRIMARY KEY, b TEXT); + CREATE UNIQUE INDEX t8bx ON t8(substr(b,2,4) COLLATE nocase); + INSERT INTO t8(a,b) VALUES(1,'Alice'),(2,'Bartholemew'),(3,'Cynthia'); + SELECT * FROM t8 WHERE substr(b,2,4)='ARTH' COLLATE nocase; +} {2 Bartholemew} +do_catchsql_test indexexpr1-810 { + INSERT INTO t8(a,b) VALUES(4,'BARTHMERE'); +} {1 {UNIQUE constraint failed: index 't8bx'}} +do_catchsql_test indexexpr1-820 { + DROP INDEX t8bx; + CREATE UNIQUE INDEX t8bx ON t8(substr(b,2,4) COLLATE rtrim); + INSERT INTO t8(a,b) VALUES(4,'BARTHMERE'); +} {0 {}} +# Check that PRAGMA integrity_check works correctly on a +# UNIQUE index that includes rowid and expression terms. +# +do_execsql_test indexexpr1-900 { + CREATE TABLE t9(a,b,c,d); + CREATE UNIQUE INDEX t9x1 ON t9(c,abs(d),b); + INSERT INTO t9(rowid,a,b,c,d) VALUES(1,2,3,4,5); + INSERT INTO t9(rowid,a,b,c,d) VALUES(2,NULL,NULL,NULL,NULL); + INSERT INTO t9(rowid,a,b,c,d) VALUES(3,NULL,NULL,NULL,NULL); + INSERT INTO t9(rowid,a,b,c,d) VALUES(4,5,6,7,8); + PRAGMA integrity_check; +} {ok} +do_catchsql_test indexexpr1-910 { + INSERT INTO t9(a,b,c,d) VALUES(5,6,7,-8); +} {1 {UNIQUE constraint failed: index 't9x1'}} finish_test Index: test/json101.test ================================================================== --- test/json101.test +++ test/json101.test @@ -13,33 +13,37 @@ # set testdir [file dirname $argv0] source $testdir/tester.tcl -load_static_extension db json -do_execsql_test json1-1.1.00 { +ifcapable !json1 { + finish_test + return +} + +do_execsql_test json101-1.1.00 { SELECT json_array(1,2.5,null,'hello'); } {[1,2.5,null,"hello"]} -do_execsql_test json1-1.1.01 { +do_execsql_test json101-1.1.01 { SELECT json_array(1,'{"abc":2.5,"def":null,"ghi":hello}',99); -- the second term goes in as a string: } {[1,"{\\"abc\\":2.5,\\"def\\":null,\\"ghi\\":hello}",99]} -do_execsql_test json1-1.1.02 { +do_execsql_test json101-1.1.02 { SELECT json_array(1,json('{"abc":2.5,"def":null,"ghi":"hello"}'),99); -- the second term goes in as JSON } {[1,{"abc":2.5,"def":null,"ghi":"hello"},99]} -do_execsql_test json1-1.1.03 { +do_execsql_test json101-1.1.03 { SELECT json_array(1,json_object('abc',2.5,'def',null,'ghi','hello'),99); -- the second term goes in as JSON } {[1,{"abc":2.5,"def":null,"ghi":"hello"},99]} -do_execsql_test json1-1.2 { +do_execsql_test json101-1.2 { SELECT hex(json_array('String "\ Test')); } {5B22537472696E67205C225C5C2054657374225D} -do_catchsql_test json1-1.3 { +do_catchsql_test json101-1.3 { SELECT json_array(1,printf('%.1000c','x'),x'abcd',3); } {1 {JSON cannot hold BLOB values}} -do_execsql_test json1-1.4 { +do_execsql_test json101-1.4 { SELECT json_array(-9223372036854775808,9223372036854775807,0,1,-1, 0.0, 1.0, -1.0, -1e99, +2e100, 'one','two','three', 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, NULL, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, @@ -47,77 +51,79 @@ 'abcdefghijklmnopqrstuvwyxzABCDEFGHIJKLMNOPQRSTUVWXYZ', 'abcdefghijklmnopqrstuvwyxzABCDEFGHIJKLMNOPQRSTUVWXYZ', 99); } {[-9223372036854775808,9223372036854775807,0,1,-1,0.0,1.0,-1.0,-1.0e+99,2.0e+100,"one","two","three",4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,null,21,22,23,24,25,26,27,28,29,30,31,"abcdefghijklmnopqrstuvwyxzABCDEFGHIJKLMNOPQRSTUVWXYZ","abcdefghijklmnopqrstuvwyxzABCDEFGHIJKLMNOPQRSTUVWXYZ","abcdefghijklmnopqrstuvwyxzABCDEFGHIJKLMNOPQRSTUVWXYZ",99]} -do_execsql_test json1-2.1 { +do_execsql_test json101-2.1 { SELECT json_object('a',1,'b',2.5,'c',null,'d','String Test'); } {{{"a":1,"b":2.5,"c":null,"d":"String Test"}}} -do_catchsql_test json1-2.2 { +do_catchsql_test json101-2.2 { SELECT json_object('a',printf('%.1000c','x'),2,2.5); } {1 {json_object() labels must be TEXT}} -do_catchsql_test json1-2.3 { +do_catchsql_test json101-2.3 { SELECT json_object('a',1,'b'); } {1 {json_object() requires an even number of arguments}} -do_catchsql_test json1-2.4 { +do_catchsql_test json101-2.4 { SELECT json_object('a',printf('%.1000c','x'),'b',x'abcd'); } {1 {JSON cannot hold BLOB values}} -do_execsql_test json1-3.1 { +do_execsql_test json101-3.1 { SELECT json_replace('{"a":1,"b":2}','$.a','[3,4,5]'); } {{{"a":"[3,4,5]","b":2}}} -do_execsql_test json1-3.2 { +do_execsql_test json101-3.2 { SELECT json_replace('{"a":1,"b":2}','$.a',json('[3,4,5]')); } {{{"a":[3,4,5],"b":2}}} -do_execsql_test json1-3.3 { +do_execsql_test json101-3.3 { SELECT json_type(json_set('{"a":1,"b":2}','$.b','{"x":3,"y":4}'),'$.b'); } {text} -do_execsql_test json1-3.4 { +do_execsql_test json101-3.4 { SELECT json_type(json_set('{"a":1,"b":2}','$.b',json('{"x":3,"y":4}')),'$.b'); } {object} -do_execsql_test json1-3.5 { +ifcapable vtab { +do_execsql_test json101-3.5 { SELECT fullkey, atom, '|' FROM json_tree(json_set('{}','$.x',123,'$.x',456)); } {{$} {} | {$.x} 456 |} +} # Per rfc7159, any JSON value is allowed at the top level, and whitespace # is permitting before and/or after that value. # -do_execsql_test json1-4.1 { +do_execsql_test json101-4.1 { CREATE TABLE j1(x); INSERT INTO j1(x) VALUES('true'),('false'),('null'),('123'),('-234'),('34.5e+6'), ('""'),('"\""'),('"\\"'),('"abcdefghijlmnopqrstuvwxyz"'), ('[]'),('{}'),('[true,false,null,123,-234,34.5e+6,{},[]]'), ('{"a":true,"b":{"c":false}}'); SELECT * FROM j1 WHERE NOT json_valid(x); } {} -do_execsql_test json1-4.2 { +do_execsql_test json101-4.2 { SELECT * FROM j1 WHERE NOT json_valid(char(0x20,0x09,0x0a,0x0d)||x); } {} -do_execsql_test json1-4.3 { +do_execsql_test json101-4.3 { SELECT * FROM j1 WHERE NOT json_valid(x||char(0x20,0x09,0x0a,0x0d)); } {} # But an empty string, or a string of pure whitespace is not valid JSON. # -do_execsql_test json1-4.4 { +do_execsql_test json101-4.4 { SELECT json_valid(''), json_valid(char(0x20,0x09,0x0a,0x0d)); } {0 0} # json_remove() and similar functions with no edit operations return their # input unchanged. # -do_execsql_test json1-4.5 { +do_execsql_test json101-4.5 { SELECT x FROM j1 WHERE json_remove(x)<>x; } {} -do_execsql_test json1-4.6 { +do_execsql_test json101-4.6 { SELECT x FROM j1 WHERE json_replace(x)<>x; } {} -do_execsql_test json1-4.7 { +do_execsql_test json101-4.7 { SELECT x FROM j1 WHERE json_set(x)<>x; } {} -do_execsql_test json1-4.8 { +do_execsql_test json101-4.8 { SELECT x FROM j1 WHERE json_insert(x)<>x; } {} # json_extract(JSON,'$') will return objects and arrays without change. # Index: test/json102.test ================================================================== --- test/json102.test +++ test/json102.test @@ -16,11 +16,15 @@ # set testdir [file dirname $argv0] source $testdir/tester.tcl -load_static_extension db json +ifcapable !json1 { + finish_test + return +} + do_execsql_test json102-100 { SELECT json_object('ex','[52,3.14159]'); } {{{"ex":"[52,3.14159]"}}} do_execsql_test json102-110 { SELECT json_object('ex',json('[52,3.14159]')); ADDED test/offset1.test Index: test/offset1.test ================================================================== --- /dev/null +++ test/offset1.test @@ -0,0 +1,161 @@ +# 2015-10-06 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file implements test cases for the [b65cb2c8d91f6685841d7d1e13b6] +# bug: Correct handling of LIMIT and OFFSET on a UNION ALL query where +# the right-hand SELECT contains an ORDER BY in a subquery. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +ifcapable !compound { + finish_test + return +} + +do_execsql_test offset1-1.1 { + CREATE TABLE t1(a,b); + INSERT INTO t1 VALUES(1,'a'),(2,'b'),(3,'c'),(4,'d'),(5,'e'); + CREATE TABLE t2(x,y); + INSERT INTO t2 VALUES(8,'y'),(9,'z'),(6,'w'),(7,'x'); + SELECT count(*) FROM t1, t2; +} {20} + +do_execsql_test offset1-1.2.0 { + SELECT a, b FROM t1 + UNION ALL + SELECT * FROM (SELECT x, y FROM t2 ORDER BY y) + LIMIT 3 OFFSET 0; +} {1 a 2 b 3 c} +do_execsql_test offset1-1.2.1 { + SELECT a, b FROM t1 + UNION ALL + SELECT * FROM (SELECT x, y FROM t2 ORDER BY y) + LIMIT 3 OFFSET 1; +} {2 b 3 c 4 d} +do_execsql_test offset1-1.2.2 { + SELECT a, b FROM t1 + UNION ALL + SELECT * FROM (SELECT x, y FROM t2 ORDER BY y) + LIMIT 3 OFFSET 2; +} {3 c 4 d 5 e} +do_execsql_test offset1-1.2.3 { + SELECT a, b FROM t1 + UNION ALL + SELECT * FROM (SELECT x, y FROM t2 ORDER BY y) + LIMIT 3 OFFSET 3; +} {4 d 5 e 6 w} +do_execsql_test offset1-1.2.4 { + SELECT a, b FROM t1 + UNION ALL + SELECT * FROM (SELECT x, y FROM t2 ORDER BY y) + LIMIT 3 OFFSET 4; +} {5 e 6 w 7 x} +do_execsql_test offset1-1.2.5 { + SELECT a, b FROM t1 + UNION ALL + SELECT * FROM (SELECT x, y FROM t2 ORDER BY y) + LIMIT 3 OFFSET 5; +} {6 w 7 x 8 y} +do_execsql_test offset1-1.2.6 { + SELECT a, b FROM t1 + UNION ALL + SELECT * FROM (SELECT x, y FROM t2 ORDER BY y) + LIMIT 3 OFFSET 6; +} {7 x 8 y 9 z} +do_execsql_test offset1-1.2.7 { + SELECT a, b FROM t1 + UNION ALL + SELECT * FROM (SELECT x, y FROM t2 ORDER BY y) + LIMIT 3 OFFSET 7; +} {8 y 9 z} +do_execsql_test offset1-1.2.8 { + SELECT a, b FROM t1 + UNION ALL + SELECT * FROM (SELECT x, y FROM t2 ORDER BY y) + LIMIT 3 OFFSET 8; +} {9 z} +do_execsql_test offset1-1.2.9 { + SELECT a, b FROM t1 + UNION ALL + SELECT * FROM (SELECT x, y FROM t2 ORDER BY y) + LIMIT 3 OFFSET 9; +} {} + +do_execsql_test offset1-1.3.0 { + SELECT * FROM t1 LIMIT 0; +} {} + +do_execsql_test offset1-1.4.0 { + SELECT a, b FROM t1 + UNION ALL + SELECT * FROM (SELECT x, y FROM t2 ORDER BY y) + LIMIT 0 OFFSET 1; +} {} +do_execsql_test offset1-1.4.1 { + SELECT a, b FROM t1 + UNION ALL + SELECT * FROM (SELECT x, y FROM t2 ORDER BY y) + LIMIT 1 OFFSET 1; +} {2 b} +do_execsql_test offset1-1.4.2 { + SELECT a, b FROM t1 + UNION ALL + SELECT * FROM (SELECT x, y FROM t2 ORDER BY y) + LIMIT 2 OFFSET 1; +} {2 b 3 c} +do_execsql_test offset1-1.4.3 { + SELECT a, b FROM t1 + UNION ALL + SELECT * FROM (SELECT x, y FROM t2 ORDER BY y) + LIMIT 3 OFFSET 1; +} {2 b 3 c 4 d} +do_execsql_test offset1-1.4.4 { + SELECT a, b FROM t1 + UNION ALL + SELECT * FROM (SELECT x, y FROM t2 ORDER BY y) + LIMIT 4 OFFSET 1; +} {2 b 3 c 4 d 5 e} +do_execsql_test offset1-1.4.5 { + SELECT a, b FROM t1 + UNION ALL + SELECT * FROM (SELECT x, y FROM t2 ORDER BY y) + LIMIT 5 OFFSET 1; +} {2 b 3 c 4 d 5 e 6 w} +do_execsql_test offset1-1.4.6 { + SELECT a, b FROM t1 + UNION ALL + SELECT * FROM (SELECT x, y FROM t2 ORDER BY y) + LIMIT 6 OFFSET 1; +} {2 b 3 c 4 d 5 e 6 w 7 x} +do_execsql_test offset1-1.4.7 { + SELECT a, b FROM t1 + UNION ALL + SELECT * FROM (SELECT x, y FROM t2 ORDER BY y) + LIMIT 7 OFFSET 1; +} {2 b 3 c 4 d 5 e 6 w 7 x 8 y} +do_execsql_test offset1-1.4.8 { + SELECT a, b FROM t1 + UNION ALL + SELECT * FROM (SELECT x, y FROM t2 ORDER BY y) + LIMIT 8 OFFSET 1; +} {2 b 3 c 4 d 5 e 6 w 7 x 8 y 9 z} +do_execsql_test offset1-1.4.9 { + SELECT a, b FROM t1 + UNION ALL + SELECT * FROM (SELECT x, y FROM t2 ORDER BY y) + LIMIT 9 OFFSET 1; +} {2 b 3 c 4 d 5 e 6 w 7 x 8 y 9 z} + + + +finish_test Index: test/releasetest.tcl ================================================================== --- test/releasetest.tcl +++ test/releasetest.tcl @@ -76,10 +76,11 @@ "Update-Delete-Limit" { -O2 -DSQLITE_DEFAULT_FILE_FORMAT=4 -DSQLITE_ENABLE_UPDATE_DELETE_LIMIT=1 -DSQLITE_ENABLE_STMT_SCANSTATUS + --enable-json1 } "Check-Symbols" { -DSQLITE_MEMDEBUG=1 -DSQLITE_ENABLE_FTS3_PARENTHESIS=1 -DSQLITE_ENABLE_FTS3=1 @@ -93,10 +94,11 @@ -DSQLITE_ENABLE_ATOMIC_WRITE=1 -DSQLITE_ENABLE_MEMORY_MANAGEMENT=1 -DSQLITE_ENABLE_OVERSIZE_CELL_CHECK=1 -DSQLITE_ENABLE_STAT4 -DSQLITE_ENABLE_STMT_SCANSTATUS + --enable-json1 --enable-fts5 } "Debug-One" { --disable-shared -O2 -DSQLITE_DEBUG=1 @@ -133,10 +135,11 @@ -DSQLITE_MAX_PAGE_SIZE=4096 -DSQLITE_OMIT_LOAD_EXTENSION=1 -DSQLITE_OMIT_PROGRESS_CALLBACK=1 -DSQLITE_OMIT_VIRTUALTABLE=1 -DSQLITE_TEMP_STORE=3 + --enable-json1 } "Device-Two" { -DSQLITE_4_BYTE_ALIGNED_MALLOC=1 -DSQLITE_DEFAULT_AUTOVACUUM=1 -DSQLITE_DEFAULT_CACHE_SIZE=1000 @@ -150,10 +153,11 @@ -DSQLITE_MAX_COMPOUND_SELECT=50 -DSQLITE_MAX_PAGE_SIZE=32768 -DSQLITE_OMIT_TRACE=1 -DSQLITE_TEMP_STORE=3 -DSQLITE_THREADSAFE=2 + --enable-json1 --enable-fts5 } "Locking-Style" { -O2 -DSQLITE_ENABLE_LOCKING_STYLE=1 } @@ -161,10 +165,11 @@ -O1 # Avoid a compiler bug in gcc 4.2.1 build 5658 -DSQLITE_OMIT_LOAD_EXTENSION=1 -DSQLITE_DEFAULT_MEMSTATUS=0 -DSQLITE_THREADSAFE=2 -DSQLITE_OS_UNIX=1 + -DSQLITE_ENABLE_JSON1=1 -DSQLITE_ENABLE_LOCKING_STYLE=1 -DUSE_PREAD=1 -DSQLITE_ENABLE_RTREE=1 -DSQLITE_ENABLE_FTS3=1 -DSQLITE_ENABLE_FTS3_PARENTHESIS=1 @@ -172,10 +177,11 @@ -DSQLITE_MAX_LENGTH=2147483645 -DSQLITE_MAX_VARIABLE_NUMBER=500000 -DSQLITE_DEBUG=1 -DSQLITE_PREFER_PROXY_LOCKING=1 -DSQLITE_ENABLE_API_ARMOR=1 + --enable-json1 --enable-fts5 } "Extra-Robustness" { -DSQLITE_ENABLE_OVERSIZE_CELL_CHECK=1 -DSQLITE_MAX_ATTACHED=62 } @@ -185,10 +191,11 @@ -DSQLITE_ENABLE_COLUMN_METADATA -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_FTS4_PARENTHESIS -DSQLITE_DISABLE_FTS4_DEFERRED -DSQLITE_ENABLE_RTREE + --enable-json1 --enable-fts5 } "No-lookaside" { -DSQLITE_TEST_REALLOC_STRESS=1 -DSQLITE_OMIT_LOOKASIDE=1 -DHAVE_USLEEP=1 @@ -195,10 +202,11 @@ } "Valgrind" { -DSQLITE_ENABLE_STAT4 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_RTREE + --enable-json1 } # The next group of configurations are used only by the # Failure-Detection platform. They are all the same, but we need # different names for them all so that they results appear in separate Index: test/spellfix2.test ================================================================== --- test/spellfix2.test +++ test/spellfix2.test @@ -74,43 +74,45 @@ 32 20 } do_execsql_test 1.6 { SELECT word, distance, matchlen FROM demo - WHERE word MATCH 'amstedam*' AND distance <= 100; + WHERE word MATCH 'amstedam*' AND distance <= 100 + ORDER BY distance, word; } { - amsterdam 100 9 amsterdamh 100 9 - amsterdamm 100 9 amsterdamn 100 9 - amsterdama 100 9 amsterdame 100 9 - amsterdami 100 9 amsterdamo 100 9 - amsterdamu 100 9 amsterdamy 100 9 - amsterdammetje 100 9 amsterdamania 100 9 - amsterdamb 100 9 amsterdamf 100 9 - amsterdamp 100 9 amsterdamv 100 9 - amsterdamw 100 9 amsterdamweg 100 9 - amsterdamc 100 9 amsterdamg 100 9 - amsterdamj 100 9 amsterdamk 100 9 - amsterdamq 100 9 amsterdams 100 9 - amsterdamx 100 9 amsterdamz 100 9 - amsterdamsestraat 100 9 amsterdamd 100 9 - amsterdamt 100 9 amsterdaml 100 9 - amsterdamlaan 100 9 amsterdamr 100 9 + amsterdam 100 9 amsterdama 100 9 + amsterdamania 100 9 amsterdamb 100 9 + amsterdamc 100 9 amsterdamd 100 9 + amsterdame 100 9 amsterdamf 100 9 + amsterdamg 100 9 amsterdamh 100 9 + amsterdami 100 9 amsterdamj 100 9 + amsterdamk 100 9 amsterdaml 100 9 + amsterdamlaan 100 9 amsterdamm 100 9 + amsterdammetje 100 9 amsterdamn 100 9 + amsterdamo 100 9 amsterdamp 100 9 + amsterdamq 100 9 amsterdamr 100 9 + amsterdams 100 9 amsterdamsestraat 100 9 + amsterdamt 100 9 amsterdamu 100 9 + amsterdamv 100 9 amsterdamw 100 9 + amsterdamweg 100 9 amsterdamx 100 9 + amsterdamy 100 9 amsterdamz 100 9 } do_execsql_test 1.7 { SELECT word, distance, matchlen FROM demo - WHERE word MATCH 'amstedam*' AND distance <= 100 AND top=20; + WHERE word MATCH 'amstedam*' AND distance <= 100 AND top=20 + ORDER BY distance, word; } { - amsterdam 100 9 amsterdamh 100 9 - amsterdamm 100 9 amsterdamn 100 9 - amsterdama 100 9 amsterdame 100 9 - amsterdami 100 9 amsterdamo 100 9 - amsterdamu 100 9 amsterdamy 100 9 - amsterdammetje 100 9 amsterdamania 100 9 - amsterdamb 100 9 amsterdamf 100 9 - amsterdamp 100 9 amsterdamv 100 9 - amsterdamw 100 9 amsterdamweg 100 9 - amsterdamc 100 9 amsterdamg 100 9 + amsterdam 100 9 amsterdama 100 9 + amsterdamania 100 9 amsterdamb 100 9 + amsterdamc 100 9 amsterdame 100 9 + amsterdamf 100 9 amsterdamg 100 9 + amsterdamh 100 9 amsterdami 100 9 + amsterdamm 100 9 amsterdammetje 100 9 + amsterdamn 100 9 amsterdamo 100 9 + amsterdamp 100 9 amsterdamu 100 9 + amsterdamv 100 9 amsterdamw 100 9 + amsterdamweg 100 9 amsterdamy 100 9 } finish_test Index: test/tabfunc01.test ================================================================== --- test/tabfunc01.test +++ test/tabfunc01.test @@ -73,12 +73,16 @@ do_execsql_test tabfunc01-2.1 { CREATE TABLE t1(x); INSERT INTO t1(x) VALUES(2),(3); SELECT *, '|' FROM t1, generate_series(1,x) ORDER BY 1, 2 } {2 1 | 2 2 | 3 1 | 3 2 | 3 3 |} - do_execsql_test tabfunc01-2.2 { + SELECT *, '|' FROM (SELECT x FROM t1) AS y, generate_series(1,y.x) + ORDER BY 1, 2; +} {2 1 | 2 2 | 3 1 | 3 2 | 3 3 |} + +do_execsql_test tabfunc01-2.50 { SELECT * FROM generate_series() LIMIT 5; } {0 1 2 3 4} do_execsql_test tabfunc01-3.1 { SELECT DISTINCT value FROM generate_series(1,x), t1 ORDER BY 1; Index: test/uri.test ================================================================== --- test/uri.test +++ test/uri.test @@ -61,10 +61,18 @@ # Windows, we must skip the final two tests here (i.e. the # question mark is illegal in a file name on Windows). # if {$tn>14} break + # + # NOTE: When running on Tcl 8.6 (or higher?) on Windows, a colon within + # the file name no longer tries to access an alternate data stream + # (ADS) named "test.db" for the "http" file, causing some spurious + # failures of this test. + # + if {$tn==12 && $::tcl_version>=8.6} continue + # # NOTE: On Windows, we need to account for the fact that the current # directory does not start with a forward slash. # set uri [string map [list PWD/ /[test_pwd /]] $uri] Index: test/view.test ================================================================== --- test/view.test +++ test/view.test @@ -159,10 +159,20 @@ } } {x 2 y 7 z 1} do_catchsql_test view-3.3.4 { CREATE VIEW v1err(x,y DESC,z) AS SELECT a, b+c, c-b FROM t1; } {1 {syntax error after column name "y"}} +do_catchsql_test view-3.3.5 { + DROP VIEW IF EXISTS v1err; + CREATE VIEW v1err(x,y) AS SELECT a, b+c, c-b FROM t1; + SELECT * FROM v1err; +} {1 {expected 2 columns for 'v1err' but got 3}} +do_catchsql_test view-3.3.6 { + DROP VIEW IF EXISTS v1err; + CREATE VIEW v1err(w,x,y,z) AS SELECT a, b+c, c-b FROM t1; + SELECT * FROM v1err; +} {1 {expected 4 columns for 'v1err' but got 3}} ifcapable compound { do_test view-3.4 { execsql2 { CREATE VIEW v3 AS SELECT a FROM t1 UNION SELECT b FROM t1 ORDER BY b; @@ -462,10 +472,15 @@ do_test view-12.1 { catchsql { CREATE VIEW v12 AS SELECT a FROM t1 WHERE b=? } } {1 {parameters are not allowed in views}} +do_test view-12.2 { + catchsql { + CREATE VIEW v12(x) AS SELECT a FROM t1 WHERE b=? + } +} {1 {parameters are not allowed in views}} ifcapable attach { do_test view-13.1 { forcedelete test2.db catchsql { @@ -480,10 +495,17 @@ # do_test view-14.1 { catchsql { CREATE TEMP VIEW t1 AS SELECT a,b FROM t1; SELECT * FROM temp.t1; + } +} {1 {view t1 is circularly defined}} +do_test view-14.2 { + catchsql { + DROP VIEW IF EXISTS temp.t1; + CREATE TEMP VIEW t1(a,b) AS SELECT a,b FROM t1; + SELECT * FROM temp.t1; } } {1 {view t1 is circularly defined}} # Tickets #1688, #1709 # ADDED tool/GetFile.cs Index: tool/GetFile.cs ================================================================== --- /dev/null +++ tool/GetFile.cs @@ -0,0 +1,450 @@ +/* +** 2015 October 7 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains C# code to download a single file based on a URI. +*/ + +using System; +using System.ComponentModel; +using System.Diagnostics; +using System.IO; +using System.Net; +using System.Reflection; +using System.Runtime.InteropServices; +using System.Threading; + +/////////////////////////////////////////////////////////////////////////////// + +#region Assembly Metadata +[assembly: AssemblyTitle("GetFile Tool")] +[assembly: AssemblyDescription("Download a single file based on a URI.")] +[assembly: AssemblyCompany("SQLite Development Team")] +[assembly: AssemblyProduct("SQLite")] +[assembly: AssemblyCopyright("Public Domain")] +[assembly: ComVisible(false)] +[assembly: Guid("5c4b3728-1693-4a33-a218-8e6973ca15a6")] +[assembly: AssemblyVersion("1.0.*")] + +#if DEBUG +[assembly: AssemblyConfiguration("Debug")] +#else +[assembly: AssemblyConfiguration("Release")] +#endif +#endregion + +/////////////////////////////////////////////////////////////////////////////// + +namespace GetFile +{ + /// + /// This enumeration is used to represent all the possible exit codes from + /// this tool. + /// + internal enum ExitCode + { + /// + /// The file download was a success. + /// + Success = 0, + + /// + /// The command line arguments are missing (i.e. null). Generally, + /// this should not happen. + /// + MissingArgs = 1, + + /// + /// The wrong number of command line arguments was supplied. + /// + WrongNumArgs = 2, + + /// + /// The URI specified on the command line could not be parsed as a + /// supported absolute URI. + /// + BadUri = 3, + + /// + /// The file name portion of the URI specified on the command line + /// could not be extracted from it. + /// + BadFileName = 4, + + /// + /// The temporary directory is either invalid (i.e. null) or does not + /// represent an available directory. + /// + BadTempPath = 5, + + /// + /// An exception was caught in . Generally, this + /// should not happen. + /// + Exception = 6, + + /// + /// The file download was canceled. This tool does not make use of + /// the method; therefore, this + /// should not happen. + /// + DownloadCanceled = 7, + + /// + /// The file download encountered an error. Further information about + /// this error should be displayed on the console. + /// + DownloadError = 8 + } + + /////////////////////////////////////////////////////////////////////////// + + internal static class Program + { + #region Private Data + /// + /// This is used to synchronize multithreaded access to the + /// and + /// fields. + /// + private static readonly object syncRoot = new object(); + + /////////////////////////////////////////////////////////////////////// + + /// + /// This event will be signed when the file download has completed, + /// even if the file download itself was canceled or unsuccessful. + /// + private static EventWaitHandle doneEvent; + + /////////////////////////////////////////////////////////////////////// + + /// + /// The previous file download completion percentage seen by the + /// event handler. This value + /// is never decreased, nor is it ever reset to zero. + /// + private static int previousPercent = 0; + + /////////////////////////////////////////////////////////////////////// + + /// + /// This will be the exit code returned by this tool after the file + /// download completes, successfully or otherwise. This value is only + /// changed by the event handler. + /// + private static ExitCode exitCode = ExitCode.Success; + #endregion + + /////////////////////////////////////////////////////////////////////// + + #region Private Support Methods + /// + /// This method displays an error message to the console and/or + /// displays the command line usage information for this tool. + /// + /// + /// The error message to display, if any. + /// + /// + /// Non-zero to display the command line usage information. + /// + private static void Error( + string message, + bool usage + ) + { + if (message != null) + Console.WriteLine(message); + + string fileName = Path.GetFileName( + Process.GetCurrentProcess().MainModule.FileName); + + Console.WriteLine(String.Format("usage: {0} ", fileName)); + } + + /////////////////////////////////////////////////////////////////////// + + /// + /// This method attempts to determine the file name portion of the + /// specified URI. + /// + /// + /// The URI to process. + /// + /// + /// The file name portion of the specified URI -OR- null if it cannot + /// be determined. + /// + private static string GetFileName( + Uri uri + ) + { + if (uri == null) + return null; + + string pathAndQuery = uri.PathAndQuery; + + if (String.IsNullOrEmpty(pathAndQuery)) + return null; + + int index = pathAndQuery.LastIndexOf('/'); + + if ((index < 0) || (index == pathAndQuery.Length)) + return null; + + return pathAndQuery.Substring(index + 1); + } + #endregion + + /////////////////////////////////////////////////////////////////////// + + #region Private Event Handlers + /// + /// This method is an event handler that is called when the file + /// download completion percentage changes. It will display progress + /// on the console. Special care is taken to make sure that progress + /// events are not displayed out-of-order, even if duplicate and/or + /// out-of-order events are received. + /// + /// + /// The source of the event. + /// + /// + /// Information for the event being processed. + /// + private static void DownloadProgressChanged( + object sender, + DownloadProgressChangedEventArgs e + ) + { + if (e != null) + { + int percent = e.ProgressPercentage; + + lock (syncRoot) + { + if (percent > previousPercent) + { + Console.Write('.'); + + if ((percent % 10) == 0) + Console.Write(" {0}% ", percent); + + previousPercent = percent; + } + } + } + } + + /////////////////////////////////////////////////////////////////////// + + /// + /// This method is an event handler that is called when the file + /// download has completed, successfully or otherwise. It will + /// display the overall result of the file download on the console, + /// including any information, if applicable. + /// The field is changed by this method to + /// indicate the overall result of the file download and the event + /// within the field will be signaled. + /// + /// + /// The source of the event. + /// + /// + /// Information for the event being processed. + /// + private static void DownloadFileCompleted( + object sender, + AsyncCompletedEventArgs e + ) + { + if (e != null) + { + lock (syncRoot) + { + if (previousPercent < 100) + Console.Write(' '); + } + + if (e.Cancelled) + { + Console.WriteLine("Canceled"); + + lock (syncRoot) + { + exitCode = ExitCode.DownloadCanceled; + } + } + else + { + Exception error = e.Error; + + if (error != null) + { + Console.WriteLine("Error: {0}", error); + + lock (syncRoot) + { + exitCode = ExitCode.DownloadError; + } + } + else + { + Console.WriteLine("Done"); + } + } + } + + if (doneEvent != null) + doneEvent.Set(); + } + #endregion + + /////////////////////////////////////////////////////////////////////// + + #region Program Entry Point + /// + /// This is the entry-point for this tool. It handles processing the + /// command line arguments, setting up the web client, downloading the + /// file, and saving it to the file system. + /// + /// + /// The command line arguments. + /// + /// + /// Zero upon success; non-zero on failure. This will be one of the + /// values from the enumeration. + /// + private static int Main( + string[] args + ) + { + // + // NOTE: Sanity check the command line arguments. + // + if (args == null) + { + Error(null, true); + return (int)ExitCode.MissingArgs; + } + + if (args.Length != 1) + { + Error(null, true); + return (int)ExitCode.WrongNumArgs; + } + + // + // NOTE: Attempt to convert the first (and only) command line + // argument to an absolute URI. + // + Uri uri; + + if (!Uri.TryCreate(args[0], UriKind.Absolute, out uri)) + { + Error("Could not create absolute URI from argument.", false); + return (int)ExitCode.BadUri; + } + + // + // NOTE: Attempt to extract the file name portion of the URI we + // just created. + // + string fileName = GetFileName(uri); + + if (fileName == null) + { + Error("Could not extract the file name from the URI.", false); + return (int)ExitCode.BadFileName; + } + + // + // NOTE: Grab the temporary path setup for this process. If it is + // unavailable, we will not continue. + // + string directory = Path.GetTempPath(); + + if (String.IsNullOrEmpty(directory) || + !Directory.Exists(directory)) + { + Error("Temporary directory is invalid or unavailable.", false); + return (int)ExitCode.BadTempPath; + } + + try + { + using (WebClient webClient = new WebClient()) + { + // + // NOTE: Create the event used to signal completion of the + // file download. + // + doneEvent = new ManualResetEvent(false); + + // + // NOTE: Hookup the event handlers we care about on the web + // client. These are necessary because the file is + // downloaded asynchronously. + // + webClient.DownloadProgressChanged += + new DownloadProgressChangedEventHandler( + DownloadProgressChanged); + + webClient.DownloadFileCompleted += + new AsyncCompletedEventHandler( + DownloadFileCompleted); + + // + // NOTE: Build the fully qualified path and file name, + // within the temporary directory, where the file to + // be downloaded will be saved. + // + fileName = Path.Combine(directory, fileName); + + // + // NOTE: If the file name already exists (in the temporary) + // directory, delete it. + // + // TODO: Perhaps an error should be raised here instead? + // + if (File.Exists(fileName)) + File.Delete(fileName); + + // + // NOTE: After kicking off the asynchronous file download + // process, wait [forever] until the "done" event is + // signaled. + // + Console.WriteLine( + "Downloading \"{0}\" to \"{1}\"...", uri, fileName); + + webClient.DownloadFileAsync(uri, fileName); + doneEvent.WaitOne(); + } + + lock (syncRoot) + { + return (int)exitCode; + } + } + catch (Exception e) + { + // + // NOTE: An exception was caught. Report it via the console + // and return failure. + // + Error(e.ToString(), false); + return (int)ExitCode.Exception; + } + } + #endregion + } +} ADDED tool/GetTclKit.bat Index: tool/GetTclKit.bat ================================================================== --- /dev/null +++ tool/GetTclKit.bat @@ -0,0 +1,273 @@ +@ECHO OFF + +:: +:: GetTclKit.bat -- +:: +:: TclKit Download Tool +:: + +SETLOCAL + +REM SET __ECHO=ECHO +REM SET __ECHO2=ECHO +REM SET __ECHO3=ECHO +IF NOT DEFINED _AECHO (SET _AECHO=REM) +IF NOT DEFINED _CECHO (SET _CECHO=REM) +IF NOT DEFINED _VECHO (SET _VECHO=REM) + +SET OVERWRITE=^> +IF DEFINED __ECHO SET OVERWRITE=^^^> + +SET APPEND=^>^> +IF DEFINED __ECHO SET APPEND=^^^>^^^> + +SET PROCESSOR=%1 + +IF DEFINED PROCESSOR ( + CALL :fn_UnquoteVariable PROCESSOR +) ELSE ( + GOTO usage +) + +%_VECHO% Processor = '%PROCESSOR%' + +SET DUMMY2=%2 + +IF DEFINED DUMMY2 ( + GOTO usage +) + +SET ROOT=%~dp0\.. +SET ROOT=%ROOT:\\=\% + +%_VECHO% Root = '%ROOT%' + +SET TOOLS=%~dp0 +SET TOOLS=%TOOLS:~0,-1% + +%_VECHO% Tools = '%TOOLS%' + +IF NOT DEFINED windir ( + ECHO The windir environment variable must be set first. + GOTO errors +) + +%_VECHO% WinDir = '%windir%' + +IF NOT DEFINED TEMP ( + ECHO The TEMP environment variable must be set first. + GOTO errors +) + +%_VECHO% Temp = '%TEMP%' + +IF NOT DEFINED TCLKIT_URI ( + SET TCLKIT_URI=http://tclsh.com/ +) + +%_VECHO% TclKitUri = '%TCLKIT_URI%' + +IF /I "%PROCESSOR%" == "x86" ( + CALL :fn_TclKitX86Variables +) ELSE IF /I "%PROCESSOR%" == "x64" ( + CALL :fn_TclKitX64Variables +) ELSE ( + GOTO usage +) + +%_VECHO% TclKitVersion = '%TCLKIT_VERSION%' +%_VECHO% TclKitPatchLevel = '%TCLKIT_PATCHLEVEL%' +%_VECHO% TclKitNoSdk = '%TCLKIT_NOSDK%' +%_VECHO% TclKitExe = '%TCLKIT_EXE%' +%_VECHO% TclKitLib = '%TCLKIT_LIB%' +%_VECHO% TclKitLibStub = '%TCLKIT_LIB_STUB%' +%_VECHO% TclKitSdk = '%TCLKIT_SDK%' +%_VECHO% TclKitSdkZip = '%TCLKIT_SDK_ZIP%' +%_VECHO% TclKitFiles = '%TCLKIT_FILES%' + +CALL :fn_ResetErrorLevel + +FOR %%T IN (csc.exe) DO ( + SET %%T_PATH=%%~dp$PATH:T +) + +%_VECHO% Csc.exe_PATH = '%csc.exe_PATH%' + +IF DEFINED csc.exe_PATH ( + GOTO skip_addToPath +) + +IF DEFINED FRAMEWORKDIR ( + REM Use the existing .NET Framework directory... +) ELSE IF EXIST "%windir%\Microsoft.NET\Framework64\v2.0.50727" ( + SET FRAMEWORKDIR=%windir%\Microsoft.NET\Framework64\v2.0.50727 +) ELSE IF EXIST "%windir%\Microsoft.NET\Framework64\v3.5" ( + SET FRAMEWORKDIR=%windir%\Microsoft.NET\Framework64\v3.5 +) ELSE IF EXIST "%windir%\Microsoft.NET\Framework64\v4.0.30319" ( + SET FRAMEWORKDIR=%windir%\Microsoft.NET\Framework64\v4.0.30319 +) ELSE IF EXIST "%windir%\Microsoft.NET\Framework\v2.0.50727" ( + SET FRAMEWORKDIR=%windir%\Microsoft.NET\Framework\v2.0.50727 +) ELSE IF EXIST "%windir%\Microsoft.NET\Framework\v3.5" ( + SET FRAMEWORKDIR=%windir%\Microsoft.NET\Framework\v3.5 +) ELSE IF EXIST "%windir%\Microsoft.NET\Framework\v4.0.30319" ( + SET FRAMEWORKDIR=%windir%\Microsoft.NET\Framework\v4.0.30319 +) ELSE ( + ECHO No suitable version of the .NET Framework appears to be installed. + GOTO errors +) + +%_VECHO% FrameworkDir = '%FRAMEWORKDIR%' + +IF NOT EXIST "%FRAMEWORKDIR%\csc.exe" ( + ECHO The file "%FRAMEWORKDIR%\csc.exe" is missing. + GOTO errors +) + +SET PATH=%FRAMEWORKDIR%;%PATH% + +:skip_addToPath + +IF NOT EXIST "%TEMP%\GetFile.exe" ( + %__ECHO% csc.exe "/out:%TEMP%\GetFile.exe" /target:exe "%TOOLS%\GetFile.cs" + + IF ERRORLEVEL 1 ( + ECHO Compilation of "%TOOLS%\GetFile.cs" failed. + GOTO errors + ) +) + +FOR %%F IN (%TCLKIT_FILES%) DO ( + IF NOT EXIST "%TEMP%\%%F" ( + %__ECHO% "%TEMP%\GetFile.exe" "%TCLKIT_URI%%%F" + + IF ERRORLEVEL 1 ( + ECHO Download of "%%F" from "%TCLKIT_URI%" failed. + GOTO errors + ) + ) +) + +IF DEFINED TCLKIT_NOSDK GOTO skip_sdkUnZip + +IF NOT EXIST "%TEMP%\%TCLKIT_SDK%" ( + %__ECHO% MKDIR "%TEMP%\%TCLKIT_SDK%" + + IF ERRORLEVEL 1 ( + ECHO Could not create directory "%TEMP%\%TCLKIT_SDK%". + GOTO errors + ) +) + +%__ECHO% "%TEMP%\unzip.exe" -n "%TEMP%\%TCLKIT_SDK_ZIP%" -d "%TEMP%\%TCLKIT_SDK%" + +IF ERRORLEVEL 1 ( + ECHO Could not unzip "%TEMP%\%TCLKIT_SDK_ZIP%" to "%TEMP%\%TCLKIT_SDK%". + GOTO errors +) + +:skip_sdkUnZip + +%__ECHO% ECHO SET TCLSH_CMD=%TEMP%\%TCLKIT_EXE%%OVERWRITE%"%ROOT%\SetTclKitEnv.bat" + +IF DEFINED TCLKIT_NOSDK GOTO skip_sdkVariables + +%__ECHO% ECHO SET TCLINCDIR=%TEMP%\%TCLKIT_SDK%\include%APPEND%"%ROOT%\SetTclKitEnv.bat" +%__ECHO% ECHO SET TCLLIBDIR=%TEMP%\%TCLKIT_SDK%\lib%APPEND%"%ROOT%\SetTclKitEnv.bat" +%__ECHO% ECHO SET LIBTCLPATH=%TEMP%\%TCLKIT_SDK%\lib%APPEND%"%ROOT%\SetTclKitEnv.bat" +%__ECHO% ECHO SET LIBTCL=%TCLKIT_LIB%%APPEND%"%ROOT%\SetTclKitEnv.bat" +%__ECHO% ECHO SET LIBTCLSTUB=%TCLKIT_LIB_STUB%%APPEND%"%ROOT%\SetTclKitEnv.bat" + +:skip_sdkVariables + +ECHO. +ECHO Wrote "%ROOT%\SetTclKitEnv.bat". +ECHO Please run it to set the necessary Tcl environment variables. +ECHO. + +GOTO no_errors + +:fn_TclKitX86Variables + IF NOT DEFINED TCLKIT_PATCHLEVEL ( + SET TCLKIT_PATCHLEVEL=8.6.4 + ) + SET TCLKIT_VERSION=%TCLKIT_PATCHLEVEL:.=% + SET TCLKIT_VERSION=%TCLKIT_VERSION:~0,2% + SET TCLKIT_EXE=tclkit-%TCLKIT_PATCHLEVEL%.exe + SET TCLKIT_LIB=libtclkit%TCLKIT_PATCHLEVEL:.=%.lib + SET TCLKIT_LIB_STUB=libtclstub%TCLKIT_VERSION:.=%.a + SET TCLKIT_SDK=libtclkit-sdk-x86-%TCLKIT_PATCHLEVEL% + SET TCLKIT_SDK_ZIP=%TCLKIT_SDK%.zip + SET TCLKIT_FILES=%TCLKIT_EXE% + IF NOT DEFINED TCLKIT_NOSDK ( + SET TCLKIT_FILES=%TCLKIT_FILES% unzip.exe %TCLKIT_SDK_ZIP% + ) + GOTO :EOF + +:fn_TclKitX64Variables + IF NOT DEFINED TCLKIT_PATCHLEVEL ( + REM + REM NOTE: By default, use latest available version of the TclKit SDK + REM for x64. However, the "default" TclKit executable for x86 + REM is still used here because it is the only one "well-known" + REM to be available for download. + REM + SET TCLKIT_PATCHLEVEL=8.6.3 + SET TCLKIT_EXE=tclkit-8.6.4.exe + ) ELSE ( + SET TCLKIT_EXE=tclkit-%TCLKIT_PATCHLEVEL%.exe + ) + SET TCLKIT_VERSION=%TCLKIT_PATCHLEVEL:.=% + SET TCLKIT_VERSION=%TCLKIT_VERSION:~0,2% + SET TCLKIT_LIB=libtclkit%TCLKIT_PATCHLEVEL:.=%.lib + SET TCLKIT_LIB_STUB=libtclstub%TCLKIT_VERSION:.=%.a + SET TCLKIT_SDK=libtclkit-sdk-x64-%TCLKIT_PATCHLEVEL% + SET TCLKIT_SDK_ZIP=%TCLKIT_SDK%.zip + SET TCLKIT_FILES=%TCLKIT_EXE% + IF NOT DEFINED TCLKIT_NOSDK ( + SET TCLKIT_FILES=%TCLKIT_FILES% unzip.exe %TCLKIT_SDK_ZIP% + ) + GOTO :EOF + +:fn_UnquoteVariable + IF NOT DEFINED %1 GOTO :EOF + SETLOCAL + SET __ECHO_CMD=ECHO %%%1%% + FOR /F "delims=" %%V IN ('%__ECHO_CMD%') DO ( + SET VALUE=%%V + ) + SET VALUE=%VALUE:"=% + REM " + ENDLOCAL && SET %1=%VALUE% + GOTO :EOF + +:fn_ResetErrorLevel + VERIFY > NUL + GOTO :EOF + +:fn_SetErrorLevel + VERIFY MAYBE 2> NUL + GOTO :EOF + +:usage + ECHO. + ECHO Usage: %~nx0 ^ + ECHO. + ECHO The only supported values for processor are "x86" and "x64". + GOTO errors + +:errors + CALL :fn_SetErrorLevel + ENDLOCAL + ECHO. + ECHO Failure, errors were encountered. + GOTO end_of_file + +:no_errors + CALL :fn_ResetErrorLevel + ENDLOCAL + ECHO. + ECHO Success, no errors were encountered. + GOTO end_of_file + +:end_of_file +%__ECHO% EXIT /B %ERRORLEVEL% ADDED tool/addopcodes.tcl Index: tool/addopcodes.tcl ================================================================== --- /dev/null +++ tool/addopcodes.tcl @@ -0,0 +1,45 @@ +#!/usr/bin/tclsh +# +# This script appends additional token codes to the end of the +# parse.h file that lemon generates. These extra token codes are +# not used by the parser. But they are used by the tokenizer and/or +# the code generator. +# +# +set in [open [lindex $argv 0] rb] +set max 0 +while {![eof $in]} { + set line [gets $in] + if {[regexp {^#define TK_} $line]} { + puts $line + set x [lindex $line 2] + if {$x>$max} {set max $x} + } +} +close $in + +# The following are the extra token codes to be added +# +set extras { + TO_TEXT + TO_BLOB + TO_NUMERIC + TO_INT + TO_REAL + ISNOT + END_OF_FILE + ILLEGAL + SPACE + UNCLOSED_STRING + FUNCTION + COLUMN + AGG_FUNCTION + AGG_COLUMN + UMINUS + UPLUS + REGISTER +} +foreach x $extras { + incr max + puts [format "#define TK_%-29s %4d" $x $max] +} Index: tool/build-all-msvc.bat ================================================================== --- tool/build-all-msvc.bat +++ tool/build-all-msvc.bat @@ -27,13 +27,13 @@ REM In the example above, "C:\dev\sqlite\core" represents the root of the REM source tree for SQLite and "C:\Temp" represents the final destination REM directory for the generated output files. REM REM Please note that the SQLite build process performed by the Makefile -REM associated with this batch script requires both Gawk ^(gawk.exe^) and Tcl -REM 8.5 ^(tclsh85.exe^) to be present in a directory contained in the PATH -REM environment variable unless a pre-existing amalgamation file is used. +REM associated with this batch script requires a Tcl shell to be present +REM in a directory contained in the PATH environment variable unless a +REM pre-existing amalgamation file is used. REM REM There are several environment variables that may be set to modify the REM behavior of this batch script and its associated Makefile. The list of REM platforms to build may be overriden by using the PLATFORMS environment REM variable, which should contain a list of platforms ^(e.g. x86 x86_amd64 @@ -230,37 +230,34 @@ REM REM NOTE: Check for the external tools needed during the build process ^(i.e. REM those that do not get compiled as part of the build process itself^) REM along the PATH. REM -FOR %%T IN (gawk.exe tclsh85.exe) DO ( +IF DEFINED TCLSH_CMD ( + SET TCLSH_FILE=%TCLSH_CMD% +) ELSE ( + SET TCLSH_FILE=tclsh85.exe +) + +FOR %%T IN (%TCLSH_FILE%) DO ( SET %%T_PATH=%%~dp$PATH:T ) REM -REM NOTE: The Gawk executable "gawk.exe" is required during the SQLite build -REM process unless a pre-existing amalgamation file is used. -REM -IF NOT DEFINED gawk.exe_PATH ( - ECHO The Gawk executable "gawk.exe" is required to be in the PATH. - GOTO errors -) - -REM -REM NOTE: The Tcl 8.5 executable "tclsh85.exe" is required during the SQLite -REM build process unless a pre-existing amalgamation file is used. -REM -IF NOT DEFINED tclsh85.exe_PATH ( - ECHO The Tcl 8.5 executable "tclsh85.exe" is required to be in the PATH. +REM NOTE: A Tcl shell executable is required during the SQLite build process +REM unless a pre-existing amalgamation file is used. +REM +IF NOT DEFINED %TCLSH_FILE%_PATH ( + ECHO The Tcl shell executable "%TCLSH_FILE%" is required to be in the PATH. GOTO errors ) REM REM NOTE: Set the TOOLPATH variable to contain all the directories where the REM external tools were found in the search above. REM -SET TOOLPATH=%gawk.exe_PATH%;%tclsh85.exe_PATH% +CALL :fn_CopyVariable %TCLSH_FILE%_PATH TOOLPATH %_VECHO% ToolPath = '%TOOLPATH%' REM REM NOTE: Setting the Windows SDK library path is only required for MSVC DELETED tool/diffdb.c Index: tool/diffdb.c ================================================================== --- tool/diffdb.c +++ /dev/null @@ -1,44 +0,0 @@ -/* -** A utility for printing the differences between two SQLite database files. -*/ -#include -#include -#include -#include -#include -#include -#include - - -#define PAGESIZE 1024 -static int db1 = -1; -static int db2 = -1; - -int main(int argc, char **argv){ - int iPg; - unsigned char a1[PAGESIZE], a2[PAGESIZE]; - if( argc!=3 ){ - fprintf(stderr,"Usage: %s FILENAME FILENAME\n", argv[0]); - exit(1); - } - db1 = open(argv[1], O_RDONLY); - if( db1<0 ){ - fprintf(stderr,"%s: can't open %s\n", argv[0], argv[1]); - exit(1); - } - db2 = open(argv[2], O_RDONLY); - if( db2<0 ){ - fprintf(stderr,"%s: can't open %s\n", argv[0], argv[2]); - exit(1); - } - iPg = 1; - while( read(db1, a1, PAGESIZE)==PAGESIZE && read(db2,a2,PAGESIZE)==PAGESIZE ){ - if( memcmp(a1,a2,PAGESIZE) ){ - printf("Page %d\n", iPg); - } - iPg++; - } - printf("%d pages checked\n", iPg-1); - close(db1); - close(db2); -} ADDED tool/mkopcodec.tcl Index: tool/mkopcodec.tcl ================================================================== --- /dev/null +++ tool/mkopcodec.tcl @@ -0,0 +1,50 @@ +#!/usr/bin/tclsh +# +# This TCL script scans the opcodes.h file (which is itself generated by +# another TCL script) and uses the information gleaned to create the +# opcodes.c source file. +# +# Opcodes.c contains strings which are the symbolic names for the various +# opcodes used by the VDBE. These strings are used when disassembling a +# VDBE program during tracing or as a result of the EXPLAIN keyword. +# +puts "/* Automatically generated. Do not edit */" +puts "/* See the tool/mkopcodec.tcl script for details. */" +puts "#if !defined(SQLITE_OMIT_EXPLAIN) \\" +puts " || defined(VDBE_PROFILE) \\" +puts " || defined(SQLITE_DEBUG)" +puts "#if defined(SQLITE_ENABLE_EXPLAIN_COMMENTS) || defined(SQLITE_DEBUG)" +puts "# define OpHelp(X) \"\\0\" X" +puts "#else" +puts "# define OpHelp(X)" +puts "#endif" +puts "const char *sqlite3OpcodeName(int i)\173" +puts " static const char *const azName\[\] = \173 \"?\"," +set mx 0 + +set in [open [lindex $argv 0] rb] +while {![eof $in]} { + set line [gets $in] + if {[regexp {^#define OP_} $line]} { + set name [lindex $line 1] + regsub {^OP_} $name {} name + set i [lindex $line 2] + set label($i) $name + if {$mx<$i} {set mx $i} + if {[regexp {synopsis: (.*) \*/} $line all x]} { + set synopsis($i) [string trim $x] + } else { + set synopsis($i) {} + } + } +} +close $in + +for {set i 1} {$i<=$mx} {incr i} { + puts [format " /* %3d */ %-18s OpHelp(\"%s\")," \ + $i \"$label($i)\" $synopsis($i)] +} +puts " \175;" +puts " return azName\[i\];" +puts "\175" +puts "#endif" ADDED tool/mkopcodeh.tcl Index: tool/mkopcodeh.tcl ================================================================== --- /dev/null +++ tool/mkopcodeh.tcl @@ -0,0 +1,230 @@ +#!/usr/bin/tclsh +# +# Generate the file opcodes.h. +# +# This TCL script scans a concatenation of the parse.h output file from the +# parser and the vdbe.c source file in order to generate the opcodes numbers +# for all opcodes. +# +# The lines of the vdbe.c that we are interested in are of the form: +# +# case OP_aaaa: /* same as TK_bbbbb */ +# +# The TK_ comment is optional. If it is present, then the value assigned to +# the OP_ is the same as the TK_ value. If missing, the OP_ value is assigned +# a small integer that is different from every other OP_ value. +# +# We go to the trouble of making some OP_ values the same as TK_ values +# as an optimization. During parsing, things like expression operators +# are coded with TK_ values such as TK_ADD, TK_DIVIDE, and so forth. Later +# during code generation, we need to generate corresponding opcodes like +# OP_Add and OP_Divide. By making TK_ADD==OP_Add and TK_DIVIDE==OP_Divide, +# code to translate from one to the other is avoided. This makes the +# code generator run (infinitesimally) faster and more importantly it makes +# the library footprint smaller. +# +# This script also scans for lines of the form: +# +# case OP_aaaa: /* jump, in1, in2, in3, out2-prerelease, out3 */ +# +# When such comments are found on an opcode, it means that certain +# properties apply to that opcode. Set corresponding flags using the +# OPFLG_INITIALIZER macro. +# + +set in stdin +set currentOp {} +set nOp 0 +while {![eof $in]} { + set line [gets $in] + + # Remember the TK_ values from the parse.h file. + # NB: The "TK_" prefix stands for "ToKen", not the graphical Tk toolkit + # commonly associated with TCL. + # + if {[regexp {^#define TK_} $line]} { + set tk([lindex $line 1]) [lindex $line 2] + continue + } + + # Find "/* Opcode: " lines in the vdbe.c file. Each one introduces + # a new opcode. Remember which parameters are used. + # + if {[regexp {^.. Opcode: } $line]} { + set currentOp OP_[lindex $line 2] + set m 0 + foreach term $line { + switch $term { + P1 {incr m 1} + P2 {incr m 2} + P3 {incr m 4} + P4 {incr m 8} + P5 {incr m 16} + } + } + set paramused($currentOp) $m + } + + # Find "** Synopsis: " lines that follow Opcode: + # + if {[regexp {^.. Synopsis: (.*)} $line all x] && $currentOp!=""} { + set synopsis($currentOp) [string trim $x] + } + + # Scan for "case OP_aaaa:" lines in the vdbe.c file + # + if {[regexp {^case OP_} $line]} { + set line [split $line] + set name [string trim [lindex $line 1] :] + set op($name) -1 + set jump($name) 0 + set in1($name) 0 + set in2($name) 0 + set in3($name) 0 + set out1($name) 0 + set out2($name) 0 + for {set i 3} {$i<[llength $line]-1} {incr i} { + switch [string trim [lindex $line $i] ,] { + same { + incr i + if {[lindex $line $i]=="as"} { + incr i + set sym [string trim [lindex $line $i] ,] + set val $tk($sym) + set op($name) $val + set used($val) 1 + set sameas($val) $sym + set def($val) $name + } + } + jump {set jump($name) 1} + in1 {set in1($name) 1} + in2 {set in2($name) 1} + in3 {set in3($name) 1} + out2 {set out2($name) 1} + out3 {set out3($name) 1} + } + } + set order($nOp) $name + incr nOp + } +} + +# Assign numbers to all opcodes and output the result. +# +set cnt 0 +set max 0 +puts "/* Automatically generated. Do not edit */" +puts "/* See the tool/mkopcodeh.tcl script for details */" +set op(OP_Noop) -1 +set order($nOp) OP_Noop +incr nOp +set op(OP_Explain) -1 +set order($nOp) OP_Explain +incr nOp + +# The following are the opcodes that are processed by resolveP2Values() +# +set rp2v_ops { + OP_Transaction + OP_AutoCommit + OP_Savepoint + OP_Checkpoint + OP_Vacuum + OP_JournalMode + OP_VUpdate + OP_VFilter + OP_Next + OP_NextIfOpen + OP_SorterNext + OP_Prev + OP_PrevIfOpen +} + +# Assign small values to opcodes that are processed by resolveP2Values() +# to make code generation for the switch() statement smaller and faster. +# +set cnt 0 +for {set i 0} {$i<$nOp} {incr i} { + set name $order($i) + if {[lsearch $rp2v_ops $name]>=0} { + incr cnt + while {[info exists used($cnt)]} {incr cnt} + set op($name) $cnt + set used($cnt) 1 + set def($cnt) $name + } +} + +# Generate the numeric values for remaining opcodes +# +for {set i 0} {$i<$nOp} {incr i} { + set name $order($i) + if {$op($name)<0} { + incr cnt + while {[info exists used($cnt)]} {incr cnt} + set op($name) $cnt + set used($cnt) 1 + set def($cnt) $name + } +} +set max $cnt +for {set i 1} {$i<=$nOp} {incr i} { + if {![info exists used($i)]} { + set def($i) "OP_NotUsed_$i" + } + set name $def($i) + puts -nonewline [format {#define %-16s %3d} $name $i] + set com {} + if {[info exists sameas($i)]} { + set com "same as $sameas($i)" + } + if {[info exists synopsis($name)]} { + set x $synopsis($name) + if {$com==""} { + set com "synopsis: $x" + } else { + append com ", synopsis: $x" + } + } + if {$com!=""} { + puts -nonewline [format " /* %-42s */" $com] + } + puts "" +} + +# Generate the bitvectors: +# +set bv(0) 0 +for {set i 1} {$i<=$max} {incr i} { + set name $def($i) + if {[info exists jump($name)] && $jump($name)} {set a0 1} {set a0 0} + if {[info exists in1($name)] && $in1($name)} {set a1 2} {set a1 0} + if {[info exists in2($name)] && $in2($name)} {set a2 4} {set a2 0} + if {[info exists in3($name)] && $in3($name)} {set a3 8} {set a3 0} + if {[info exists out2($name)] && $out2($name)} {set a4 16} {set a4 0} + if {[info exists out3($name)] && $out3($name)} {set a5 32} {set a5 0} + set bv($i) [expr {$a0+$a1+$a2+$a3+$a4+$a5}] +} +puts "" +puts "/* Properties such as \"out2\" or \"jump\" that are specified in" +puts "** comments following the \"case\" for each opcode in the vdbe.c" +puts "** are encoded into bitvectors as follows:" +puts "*/" +puts "#define OPFLG_JUMP 0x0001 /* jump: P2 holds jmp target */" +puts "#define OPFLG_IN1 0x0002 /* in1: P1 is an input */" +puts "#define OPFLG_IN2 0x0004 /* in2: P2 is an input */" +puts "#define OPFLG_IN3 0x0008 /* in3: P3 is an input */" +puts "#define OPFLG_OUT2 0x0010 /* out2: P2 is an output */" +puts "#define OPFLG_OUT3 0x0020 /* out3: P3 is an output */" +puts "#define OPFLG_INITIALIZER \173\\" +for {set i 0} {$i<=$max} {incr i} { + if {$i%8==0} { + puts -nonewline [format "/* %3d */" $i] + } + puts -nonewline [format " 0x%02x," $bv($i)] + if {$i%8==7} { + puts "\\" + } +} +puts "\175" Index: tool/mksqlite3c.tcl ================================================================== --- tool/mksqlite3c.tcl +++ tool/mksqlite3c.tcl @@ -209,11 +209,12 @@ } } elseif {[regexp {^#ifdef __cplusplus} $line]} { puts $out "#if 0" } elseif {!$linemacros && [regexp {^#line} $line]} { # Skip #line directives. - } elseif {$addstatic && ![regexp {^(static|typedef)} $line]} { + } elseif {$addstatic + && ![regexp {^(static|typedef|SQLITE_PRIVATE)} $line]} { # Skip adding the SQLITE_PRIVATE or SQLITE_API keyword before # functions if this header file does not need it. if {![info exists varonly_hdr($tail)] && [regexp $declpattern $line all rettype funcname rest]} { regsub {^SQLITE_API } $line {} line @@ -376,10 +377,12 @@ rtree.c icu.c fts3_icu.c sqlite3rbu.c dbstat.c + json1.c + fts5.c } { copy_file tsrc/$file } close $out Index: tool/mksqlite3h.tcl ================================================================== --- tool/mksqlite3h.tcl +++ tool/mksqlite3h.tcl @@ -69,10 +69,11 @@ fconfigure stdout -translation lf set filelist [subst { $TOP/src/sqlite.h.in $TOP/ext/rtree/sqlite3rtree.h + $TOP/ext/fts5/fts5.h }] # These are the functions that accept a variable number of arguments. They # always need to use the "cdecl" calling convention even when another calling # convention (e.g. "stcall") is being used for the rest of the library. @@ -95,11 +96,11 @@ set line [gets $in] # File sqlite3rtree.h contains a line "#include ". Omit this # line when copying sqlite3rtree.h into sqlite3.h. # - if {[string match {*#include**} $line]} continue + if {[string match {*#include*[<"]sqlite3.h[>"]*} $line]} continue regsub -- --VERS-- $line $zVersion line regsub -- --VERSION-NUMBER-- $line $nVersion line regsub -- --SOURCE-ID-- $line "$zDate $zUuid" line DELETED tool/opcodeDoc.awk Index: tool/opcodeDoc.awk ================================================================== --- tool/opcodeDoc.awk +++ /dev/null @@ -1,23 +0,0 @@ -# -# Extract opcode documentation for sqliteVdbe.c and generate HTML -# -BEGIN { - print "" - print "

SQLite Virtual Database Engine Opcodes

" - print "" -} -/ Opcode: /,/\*\// { - if( $2=="Opcode:" ){ - printf "\n\n" - }else if( NF>1 ){ - sub(/^ *\*\* /,"") - gsub(/" -} ADDED tool/replace.tcl Index: tool/replace.tcl ================================================================== --- /dev/null +++ tool/replace.tcl @@ -0,0 +1,20 @@ +#!/usr/bin/tcl +# +# Replace string with another string -OR- include +# only lines successfully modified with a regular +# expression. +# +set mode [string tolower [lindex $argv 0]] +set from [lindex $argv 1] +set to [lindex $argv 2] +if {$mode ni [list exact include]} {exit 1} +if {[string length $from]==0} {exit 2} +while {![eof stdin]} { + set line [gets stdin] + if {[eof stdin]} break + switch -exact $mode { + exact {set line [string map [list $from $to] $line]} + include {if {[regsub -all -- $from $line $to line]==0} continue} + } + puts stdout $line +} Index: tool/showjournal.c ================================================================== --- tool/showjournal.c +++ tool/showjournal.c @@ -28,11 +28,11 @@ static unsigned char *read_content(int N, int iOfst){ int got; unsigned char *pBuf = malloc(N); if( pBuf==0 ) out_of_memory(); fseek(db, iOfst, SEEK_SET); - got = fread(pBuf, 1, N, db); + got = (int)fread(pBuf, 1, N, db); if( got<0 ){ fprintf(stderr, "I/O error reading %d bytes from %d\n", N, iOfst); memset(pBuf, 0, N); }else if( got=nByte ){ sprintf(&zBuf[i], " "); }else{ sprintf(&zBuf[i], " %02x", aData[ofst+j]); val = val*256 + aData[ofst+j]; } - i += strlen(&zBuf[i]); + i += (int)strlen(&zBuf[i]); } sprintf(&zBuf[i], " %10u", val); printf("%s %s\n", zBuf, zMsg); return val; } DELETED tool/space_used.tcl Index: tool/space_used.tcl ================================================================== --- tool/space_used.tcl +++ /dev/null @@ -1,111 +0,0 @@ -# Run this TCL script using "testfixture" in order get a report that shows -# how much disk space is used by a particular data to actually store data -# versus how much space is unused. -# - -# Get the name of the database to analyze -# -if {[llength $argv]!=1} { - puts stderr "Usage: $argv0 database-name" - exit 1 -} -set file_to_analyze [lindex $argv 0] - -# Open the database -# -sqlite db [lindex $argv 0] -set DB [btree_open [lindex $argv 0]] - -# Output the schema for the generated report -# -puts \ -{BEGIN; -CREATE TABLE space_used( - name clob, -- Name of a table or index in the database file - is_index boolean, -- TRUE if it is an index, false for a table - payload int, -- Total amount of data stored in this table or index - pri_pages int, -- Number of primary pages used - ovfl_pages int, -- Number of overflow pages used - pri_unused int, -- Number of unused bytes on primary pages - ovfl_unused int -- Number of unused bytes on overflow pages -);} - -# This query will be used to find the root page number for every index and -# table in the database. -# -set sql { - SELECT name, type, rootpage FROM sqlite_master - UNION ALL - SELECT 'sqlite_master', 'table', 2 - ORDER BY 1 -} - -# Initialize variables used for summary statistics. -# -set total_size 0 -set total_primary 0 -set total_overflow 0 -set total_unused_primary 0 -set total_unused_ovfl 0 - -# Analyze every table in the database, one at a time. -# -foreach {name type rootpage} [db eval $sql] { - set cursor [btree_cursor $DB $rootpage 0] - set go [btree_first $cursor] - set size 0 - catch {unset pg_used} - set unused_ovfl 0 - set n_overflow 0 - while {$go==0} { - set payload [btree_payload_size $cursor] - incr size $payload - set stat [btree_cursor_dump $cursor] - set pgno [lindex $stat 0] - set freebytes [lindex $stat 4] - set pg_used($pgno) $freebytes - if {$payload>238} { - set n [expr {($payload-238+1019)/1020}] - incr n_overflow $n - incr unused_ovfl [expr {$n*1020+238-$payload}] - } - set go [btree_next $cursor] - } - btree_close_cursor $cursor - set n_primary [llength [array names pg_used]] - set unused_primary 0 - foreach x [array names pg_used] {incr unused_primary $pg_used($x)} - regsub -all ' $name '' name - puts -nonewline "INSERT INTO space_used VALUES('$name'" - puts -nonewline ",[expr {$type=="index"}]" - puts ",$size,$n_primary,$n_overflow,$unused_primary,$unused_ovfl);" - incr total_size $size - incr total_primary $n_primary - incr total_overflow $n_overflow - incr total_unused_primary $unused_primary - incr total_unused_ovfl $unused_ovfl -} - -# Output summary statistics: -# -puts "-- Total payload size: $total_size" -puts "-- Total pages used: $total_primary primary and $total_overflow overflow" -set file_pgcnt [expr {[file size [lindex $argv 0]]/1024}] -puts -nonewline "-- Total unused bytes on primary pages: $total_unused_primary" -if {$total_primary>0} { - set upp [expr {$total_unused_primary/$total_primary}] - puts " (avg $upp bytes/page)" -} else { - puts "" -} -puts -nonewline "-- Total unused bytes on overflow pages: $total_unused_ovfl" -if {$total_overflow>0} { - set upp [expr {$total_unused_ovfl/$total_overflow}] - puts " (avg $upp bytes/page)" -} else { - puts "" -} -set n_free [expr {$file_pgcnt-$total_primary-$total_overflow}] -if {$n_free>0} {incr n_free -1} -puts "-- Total pages on freelist: $n_free" -puts "COMMIT;" DELETED tool/tostr.awk Index: tool/tostr.awk ================================================================== --- tool/tostr.awk +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/awk -# -# Convert input text into a C string -# -{ - gsub(/\\/,"\\\\"); - gsub(/\"/,"\\\""); - print "\"" $0 "\\n\""; -} ADDED tool/tostr.tcl Index: tool/tostr.tcl ================================================================== --- /dev/null +++ tool/tostr.tcl @@ -0,0 +1,12 @@ +#!/usr/bin/tcl +# +# Convert input text into a C string +# +set in [open [lindex $argv 0] rb] +while {![eof $in]} { + set line [gets $in] + if {[eof $in]} break; + set x [string map "\\\\ \\\\\\\\ \\\" \\\\\"" $line] + puts "\"$x\\n\"" +} +close $in
%s %s %s %s\n", $3, $4, $5, $6 - }else if( $1=="*/" ){ - printf "