Index: ext/fts3/fts3.c ================================================================== --- ext/fts3/fts3.c +++ ext/fts3/fts3.c @@ -3273,16 +3273,15 @@ }else if( aPoslist==0 ){ aPoslist = pList; nPoslist = nList; }else{ - assert( iPrev>=0 ); - char *aOut = pList; char *p1 = aPoslist; char *p2 = aOut; + assert( iPrev>=0 ); fts3PoslistPhraseMerge(&aOut, iToken-iPrev, 0, 1, &p1, &p2); sqlite3_free(aPoslist); aPoslist = pList; nPoslist = aOut - aPoslist; if( nPoslist==0 ){ Index: ext/fts3/fts3_test.c ================================================================== --- ext/fts3/fts3_test.c +++ ext/fts3/fts3_test.c @@ -240,10 +240,81 @@ ckfree((char *)aPhrase); ckfree((char *)doc.aToken); return rc; } -int Sqlitetestfts3_Init(Tcl_Interp *interp){ - Tcl_CreateObjCommand(interp, "fts3_near_match", fts3_near_match_cmd, 0, 0); +/* +** Tclcmd: fts3_configure_incr_load ?CHUNKSIZE THRESHOLD? +** +** Normally, FTS uses hard-coded values to determine the minimum doclist +** size eligible for incremental loading, and the size of the chunks loaded +** when a doclist is incrementally loaded. This command allows the built-in +** values to be overridden for testing purposes. +** +** If present, the first argument is the chunksize in bytes to load doclists +** in. The second argument is the minimum doclist size in bytes to use +** incremental loading with. +** +** Whether or not the arguments are present, this command returns a list of +** two integers - the initial chunksize and threshold when the command is +** invoked. This can be used to restore the default behaviour after running +** tests. For example: +** +** # Override incr-load settings for testing: +** set cfg [fts3_configure_incr_load $new_chunksize $new_threshold] +** +** .... run tests .... +** +** # Restore initial incr-load settings: +** eval fts3_configure_incr_load $cfg +*/ +static int fts3_configure_incr_load_cmd( + ClientData clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ +#ifdef SQLITE_ENABLE_FTS3 + extern int test_fts3_node_chunksize; + extern int test_fts3_node_chunk_threshold; + int iArg1; + int iArg2; + Tcl_Obj *pRet; + + if( objc!=1 && objc!=3 ){ + Tcl_WrongNumArgs(interp, 1, objv, "?CHUNKSIZE THRESHOLD?"); + return TCL_ERROR; + } + + pRet = Tcl_NewObj(); + Tcl_IncrRefCount(pRet); + Tcl_ListObjAppendElement( + interp, pRet, Tcl_NewIntObj(test_fts3_node_chunksize)); + Tcl_ListObjAppendElement( + interp, pRet, Tcl_NewIntObj(test_fts3_node_chunk_threshold)); + + if( objc==3 ){ + int iArg1; + int iArg2; + if( Tcl_GetIntFromObj(interp, objv[1], &iArg1) + || Tcl_GetIntFromObj(interp, objv[2], &iArg2) + ){ + Tcl_DecrRefCount(pRet); + return TCL_ERROR; + } + test_fts3_node_chunksize = iArg1; + test_fts3_node_chunk_threshold = iArg2; + } + + Tcl_SetObjResult(interp, pRet); + Tcl_DecrRefCount(pRet); +#endif return TCL_OK; } +int Sqlitetestfts3_Init(Tcl_Interp *interp){ + Tcl_CreateObjCommand(interp, "fts3_near_match", fts3_near_match_cmd, 0, 0); + Tcl_CreateObjCommand(interp, + "fts3_configure_incr_load", fts3_configure_incr_load_cmd, 0, 0 + ); + return TCL_OK; +} Index: ext/fts3/fts3_write.c ================================================================== --- ext/fts3/fts3_write.c +++ ext/fts3/fts3_write.c @@ -47,17 +47,23 @@ ** bytes and larger. Nodes are loaded in chunks of FTS3_NODE_CHUNKSIZE bytes. ** The code is written so that the hard lower-limit for each of these values ** is 1. Clearly such small values would be inefficient, but can be useful ** for testing purposes. ** -** TODO: Add a test interface to modify these "constants" from a script for -** this purpose. +** If this module is built with SQLITE_TEST defined, these constants may +** be overridden at runtime for testing purposes. File fts3_test.c contains +** a Tcl interface to read and write the values. */ -#define FTS3_NODE_CHUNKSIZE (4*1024) -#define FTS3_NODE_CHUNK_THRESHOLD (FTS3_NODE_CHUNKSIZE*4) -/* #define FTS3_NODE_CHUNKSIZE 1 */ -/* #define FTS3_NODE_CHUNK_THRESHOLD 1 */ +#ifdef SQLITE_TEST +int test_fts3_node_chunksize = (4*1024); +int test_fts3_node_chunk_threshold = (4*1024)*4; +# define FTS3_NODE_CHUNKSIZE test_fts3_node_chunksize +# define FTS3_NODE_CHUNK_THRESHOLD test_fts3_node_chunk_threshold +#else +# define FTS3_NODE_CHUNKSIZE (4*1024) +# define FTS3_NODE_CHUNK_THRESHOLD (FTS3_NODE_CHUNKSIZE*4) +#endif typedef struct PendingList PendingList; typedef struct SegmentNode SegmentNode; typedef struct SegmentWriter SegmentWriter; Index: src/func.c ================================================================== --- src/func.c +++ src/func.c @@ -606,11 +606,11 @@ }else if( !prevEscape && c==matchOne ){ if( sqlite3Utf8Read(zString, &zString)==0 ){ return 0; } }else if( c==matchSet ){ - int prior_c = 0; + u32 prior_c = 0; assert( esc==0 ); /* This only occurs for GLOB, not LIKE */ seen = 0; invert = 0; c = sqlite3Utf8Read(zString, &zString); if( c==0 ) return 0; Index: test/fts3auto.test ================================================================== --- test/fts3auto.test +++ test/fts3auto.test @@ -527,9 +527,56 @@ do_fts3query_test 4.$tn.4.2 -deferred fi* t1 {on* NEAR fi*} do_fts3query_test 4.$tn.4.3 -deferred fi* t1 {on* NEAR/1 fi*} do_fts3query_test 4.$tn.4.4 -deferred fi* t1 {on* NEAR/2 fi*} do_fts3query_test 4.$tn.4.5 -deferred fi* t1 {on* NEAR/3 fi*} } + +#-------------------------------------------------------------------------- +# The following test cases - fts3auto-5.* - focus on using prefix indexes. +# +set chunkconfig [fts3_configure_incr_load 1 1] +foreach {tn create} { + 1 "fts4(a, b)" + 2 "fts4(a, b, order=DESC, prefix=1)" + 3 "fts4(a, b, order=ASC, prefix=1,3)" + 4 "fts4(a, b, order=DESC, prefix=2,4)" +} { + + execsql [subst { + DROP TABLE t1; + CREATE VIRTUAL TABLE t1 USING $create; + }] + + foreach {a b} { + "the song of songs which is solomons" + "let him kiss me with the kisses of his mouth for thy love is better than wine" + "because of the savour of thy good ointments thy name is as ointment poured forth therefore do the virgins love thee" + "draw me we will run after thee the king hath brought me into his chambers we will be glad and rejoice in thee we will remember thy love more than wine the upright love thee" + "i am black but comely o ye daughters of jerusalem as the tents of kedar as the curtains of solomon" + "look not upon me because i am black because the sun hath looked upon me my mothers children were angry with me they made me the keeper of the vineyards but mine own vineyard have i not kept" + "tell me o thou whom my soul loveth where thou feedest where thou makest thy flock to rest at noon for why should i be as one that turneth aside by the flocks of thy companions?" + "if thou know not o thou fairest among women go thy way forth by the footsteps of the flock and feed thy kids beside the shepherds tents" + "i have compared thee o my love to a company of horses in pharaohs chariots" + "thy cheeks are comely with rows of jewels thy neck with chains of gold" + "we will make thee borders of gold with studs of silver" + "while the king sitteth at his table my spikenard sendeth forth the smell thereof" + "a bundle of myrrh is my wellbeloved unto me he shall lie all night betwixt my breasts" + "my beloved is unto me as a cluster of camphire in the vineyards of en gedi" + "behold thou art fair my love behold thou art fair thou hast doves eyes" + "behold thou art fair my beloved yea pleasant also our bed is green" + "the beams of our house are cedar and our rafters of fir" + } { + execsql {INSERT INTO t1(a, b) VALUES($a, $b)} + } + + do_fts3query_test 5.$tn.1.1 t1 {s*} + do_fts3query_test 5.$tn.1.2 t1 {so*} + do_fts3query_test 5.$tn.1.3 t1 {"s* o*"} + do_fts3query_test 5.$tn.1.4 t1 {b* NEAR/3 a*} + do_fts3query_test 5.$tn.1.5 t1 {th* NEAR/5 a* NEAR/5 w*} + do_fts3query_test 5.$tn.1.6 t1 {"b* th* art* fair*"} +} +eval fts3_configure_incr_load $chunkconfig set sqlite_fts3_enable_parentheses $sfep finish_test