/ Changes On Branch fts3-experimental
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Changes In Branch fts3-experimental Excluding Merge-Ins

This is equivalent to a diff from 461f1a010f to ad8df6c5ee

2010-11-26
15:17
Merge fts3 matchinfo() changes with trunk. (check-in: 7d660b91b7 user: dan tags: trunk)
15:13
Add a couple of comments to fts3_snippet.c. (Closed-Leaf check-in: ad8df6c5ee user: dan tags: fts3-experimental)
10:58
Merge with latest trunk changes. (check-in: 515cb3f4e5 user: dan tags: fts3-experimental)
2010-11-24
13:04
Fix a couple of compiler warnings in test_superlock.c. Add superlock.c to the Makefile.in used by the configure script. (check-in: 461f1a010f user: drh tags: trunk)
2010-11-23
20:55
Cherry-pick the lemon.c updates out of the lemon-update-2010 branch into the trunk. (check-in: 1541ae3fbd user: drh tags: trunk)

Changes to ext/fts3/fts3.c.

2251
2252
2253
2254
2255
2256
2257
2258
2259


2260
2261
2262
2263
2264
2265
2266
2251
2252
2253
2254
2255
2256
2257


2258
2259
2260
2261
2262
2263
2264
2265
2266







-
-
+
+







      }
    }
  }

  for(ii=0; ii<pPhrase->nToken; ii++){
    Fts3PhraseToken *pTok;        /* Token to find doclist for */
    int iTok;                     /* The token being queried this iteration */
    char *pList;                  /* Pointer to token doclist */
    int nList;                    /* Size of buffer at pList */
    char *pList = 0;              /* Pointer to token doclist */
    int nList = 0;                /* Size of buffer at pList */

    /* Select a token to process. If this is an xFilter() call, then tokens 
    ** are processed in order from least to most costly. Otherwise, tokens 
    ** are processed in the order in which they occur in the phrase.
    */
    if( pCsr->eEvalmode==FTS3_EVAL_MATCHINFO ){
      assert( isReqPos );
2294
2295
2296
2297
2298
2299
2300
2301
2302



2303
2304
2305
2306
2307
2308
2309
2294
2295
2296
2297
2298
2299
2300


2301
2302
2303
2304
2305
2306
2307
2308
2309
2310







-
-
+
+
+







        break;
      }
    }

    if( pCsr->eEvalmode==FTS3_EVAL_NEXT && pTok->pDeferred ){
      rc = fts3DeferredTermSelect(pTok->pDeferred, isTermPos, &nList, &pList);
    }else{
      assert( pTok->pArray );
      rc = fts3TermSelect(p, pTok, iCol, isTermPos, &nList, &pList);
      if( pTok->pArray ){
        rc = fts3TermSelect(p, pTok, iCol, isTermPos, &nList, &pList);
      }
      pTok->bFulltext = 1;
    }
    assert( rc!=SQLITE_OK || pCsr->eEvalmode || pTok->pArray==0 );
    if( rc!=SQLITE_OK ) break;

    if( isFirst ){
      pOut = pList;
2523
2524
2525
2526
2527
2528
2529


2530


2531
2532
2533
2534
2535
2536
2537
2524
2525
2526
2527
2528
2529
2530
2531
2532

2533
2534
2535
2536
2537
2538
2539
2540
2541







+
+
-
+
+







static int fts3ExprCost(Fts3Expr *pExpr){
  int nCost;                      /* Return value */
  if( pExpr->eType==FTSQUERY_PHRASE ){
    Fts3Phrase *pPhrase = pExpr->pPhrase;
    int ii;
    nCost = 0;
    for(ii=0; ii<pPhrase->nToken; ii++){
      Fts3SegReaderArray *pArray = pPhrase->aToken[ii].pArray;
      if( pArray ){
      nCost += pPhrase->aToken[ii].pArray->nCost;
        nCost += pPhrase->aToken[ii].pArray->nCost;
      }
    }
  }else{
    nCost = fts3ExprCost(pExpr->pLeft) + fts3ExprCost(pExpr->pRight);
  }
  return nCost;
}

2551
2552
2553
2554
2555
2556
2557
2558

2559
2560
2561
2562
2563
2564
2565
2555
2556
2557
2558
2559
2560
2561

2562
2563
2564
2565
2566
2567
2568
2569







-
+







  ExprAndCost **ppExprCost        /* OUT: Write to *ppExprCost */
){
  if( pExpr->eType==FTSQUERY_AND ){
    fts3ExprAssignCosts(pExpr->pLeft, ppExprCost);
    fts3ExprAssignCosts(pExpr->pRight, ppExprCost);
  }else{
    (*ppExprCost)->pExpr = pExpr;
    (*ppExprCost)->nCost = fts3ExprCost(pExpr);;
    (*ppExprCost)->nCost = fts3ExprCost(pExpr);
    (*ppExprCost)++;
  }
}

/*
** Evaluate the full-text expression pExpr against FTS3 table pTab. Store
** the resulting doclist in *paOut and *pnOut. This routine mallocs for
3269
3270
3271
3272
3273
3274
3275
3276

3277




3278

3279
3280
3281
3282
3283
3284
3285
3273
3274
3275
3276
3277
3278
3279

3280
3281
3282
3283
3284
3285

3286
3287
3288
3289
3290
3291
3292
3293







-
+

+
+
+
+
-
+







*/
static void fts3MatchinfoFunc(
  sqlite3_context *pContext,      /* SQLite function call context */
  int nVal,                       /* Size of argument array */
  sqlite3_value **apVal           /* Array of arguments */
){
  Fts3Cursor *pCsr;               /* Cursor handle passed through apVal[0] */
  assert( nVal==1 );
  assert( nVal==1 || nVal==2 );
  if( SQLITE_OK==fts3FunctionArg(pContext, "matchinfo", apVal[0], &pCsr) ){
    const char *zArg = 0;
    if( nVal>1 ){
      zArg = (const char *)sqlite3_value_text(apVal[1]);
    }
    sqlite3Fts3Matchinfo(pContext, pCsr);
    sqlite3Fts3Matchinfo(pContext, pCsr, zArg);
  }
}

/*
** This routine implements the xFindFunction method for the FTS3
** virtual table.
*/
3460
3461
3462
3463
3464
3465
3466

3467
3468
3469
3470
3471
3472
3473
3468
3469
3470
3471
3472
3473
3474
3475
3476
3477
3478
3479
3480
3481
3482







+







  ** module with sqlite.
  */
  if( SQLITE_OK==rc 
   && SQLITE_OK==(rc = sqlite3Fts3InitHashTable(db, pHash, "fts3_tokenizer"))
   && SQLITE_OK==(rc = sqlite3_overload_function(db, "snippet", -1))
   && SQLITE_OK==(rc = sqlite3_overload_function(db, "offsets", 1))
   && SQLITE_OK==(rc = sqlite3_overload_function(db, "matchinfo", 1))
   && SQLITE_OK==(rc = sqlite3_overload_function(db, "matchinfo", 2))
   && SQLITE_OK==(rc = sqlite3_overload_function(db, "optimize", 1))
  ){
    rc = sqlite3_create_module_v2(
        db, "fts3", &fts3Module, (void *)pHash, hashDestroy
    );
    if( rc==SQLITE_OK ){
      rc = sqlite3_create_module_v2(

Changes to ext/fts3/fts3Int.h.

158
159
160
161
162
163
164

165
166
167
168
169



170
171
172
173


174
175
176
177
178
179
180
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175


176
177
178
179
180
181
182
183
184







+





+
+
+


-
-
+
+







struct Fts3Cursor {
  sqlite3_vtab_cursor base;       /* Base class used by SQLite core */
  i16 eSearch;                    /* Search strategy (see below) */
  u8 isEof;                       /* True if at End Of Results */
  u8 isRequireSeek;               /* True if must seek pStmt to %_content row */
  sqlite3_stmt *pStmt;            /* Prepared statement in use by the cursor */
  Fts3Expr *pExpr;                /* Parsed MATCH query string */
  int nPhrase;                    /* Number of matchable phrases in query */
  Fts3DeferredToken *pDeferred;   /* Deferred search tokens, if any */
  sqlite3_int64 iPrevId;          /* Previous id read from aDoclist */
  char *pNextId;                  /* Pointer into the body of aDoclist */
  char *aDoclist;                 /* List of docids for full-text queries */
  int nDoclist;                   /* Size of buffer at aDoclist */
  int eEvalmode;                  /* An FTS3_EVAL_XX constant */
  int nRowAvg;                    /* Average size of database rows, in pages */

  int isMatchinfoNeeded;          /* True when aMatchinfo[] needs filling in */
  u32 *aMatchinfo;                /* Information about most recent match */
  int eEvalmode;                  /* An FTS3_EVAL_XX constant */
  int nRowAvg;                    /* Average size of database rows, in pages */
  int nMatchinfo;                 /* Number of elements in aMatchinfo[] */
  char *zMatchinfo;               /* Matchinfo specification */
};

#define FTS3_EVAL_FILTER    0
#define FTS3_EVAL_NEXT      1
#define FTS3_EVAL_MATCHINFO 2

/*
283
284
285
286
287
288
289
290
291
292
293
294



295
296
297
298
299
300
301
287
288
289
290
291
292
293


294
295
296
297
298
299
300
301
302
303
304
305
306







-
-



+
+
+







void sqlite3Fts3SegReaderFree(Fts3Table *, Fts3SegReader *);
int sqlite3Fts3SegReaderIterate(
  Fts3Table *, Fts3SegReader **, int, Fts3SegFilter *,
  int (*)(Fts3Table *, void *, char *, int, char *, int),  void *
);
int sqlite3Fts3SegReaderCost(Fts3Cursor *, Fts3SegReader *, int *);
int sqlite3Fts3AllSegdirs(Fts3Table*, sqlite3_stmt **);
int sqlite3Fts3MatchinfoDocsizeLocal(Fts3Cursor*, u32*);
int sqlite3Fts3MatchinfoDocsizeGlobal(Fts3Cursor*, u32*);
int sqlite3Fts3ReadLock(Fts3Table *);
int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char **, int*);

int sqlite3Fts3SelectDoctotal(Fts3Table *, sqlite3_stmt **);
int sqlite3Fts3SelectDocsize(Fts3Table *, sqlite3_int64, sqlite3_stmt **);

void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *);
int sqlite3Fts3DeferToken(Fts3Cursor *, Fts3PhraseToken *, int);
int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *);
void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *);
char *sqlite3Fts3DeferredDoclist(Fts3DeferredToken *, int *);

void sqlite3Fts3SegmentsClose(Fts3Table *);
335
336
337
338
339
340
341
342

343
344
345
346
347
348
349
350
351
352
353
340
341
342
343
344
345
346

347
348
349
350
351
352
353
354
355
356
357
358







-
+











int sqlite3Fts3IsIdChar(char);

/* fts3_snippet.c */
void sqlite3Fts3Offsets(sqlite3_context*, Fts3Cursor*);
void sqlite3Fts3Snippet(sqlite3_context *, Fts3Cursor *, const char *,
  const char *, const char *, int, int
);
void sqlite3Fts3Matchinfo(sqlite3_context *, Fts3Cursor *);
void sqlite3Fts3Matchinfo(sqlite3_context *, Fts3Cursor *, const char *);

/* fts3_expr.c */
int sqlite3Fts3ExprParse(sqlite3_tokenizer *, 
  char **, int, int, const char *, int, Fts3Expr **
);
void sqlite3Fts3ExprFree(Fts3Expr *);
#ifdef SQLITE_TEST
int sqlite3Fts3ExprInitTestInterface(sqlite3 *db);
#endif

#endif /* _FTSINT_H */

Changes to ext/fts3/fts3_snippet.c.

13
14
15
16
17
18
19
















20
21
22
23
24
25
26
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42







+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+








#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3)

#include "fts3Int.h"
#include <string.h>
#include <assert.h>

/*
** Characters that may appear in the second argument to matchinfo().
*/
#define FTS3_MATCHINFO_NPHRASE   'p'        /* 1 value */
#define FTS3_MATCHINFO_NCOL      'c'        /* 1 value */
#define FTS3_MATCHINFO_NDOC      'n'        /* 1 value */
#define FTS3_MATCHINFO_AVGLENGTH 'a'        /* nCol values */
#define FTS3_MATCHINFO_LENGTH    'l'        /* nCol values */
#define FTS3_MATCHINFO_LCS       's'        /* nCol values */
#define FTS3_MATCHINFO_HITS      'x'        /* 3*nCol*nPhrase values */

/*
** The default value for the second argument to matchinfo(). 
*/
#define FTS3_MATCHINFO_DEFAULT   "pcx"


/*
** Used as an fts3ExprIterate() context when loading phrase doclists to
** Fts3Expr.aDoclist[]/nDoclist.
*/
typedef struct LoadDoclistCtx LoadDoclistCtx;
struct LoadDoclistCtx {
66
67
68
69
70
71
72


73
74
75
76
77
78
79
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97







+
+







** This type is used as an fts3ExprIterate() context object while 
** accumulating the data returned by the matchinfo() function.
*/
typedef struct MatchInfo MatchInfo;
struct MatchInfo {
  Fts3Cursor *pCursor;            /* FTS3 Cursor */
  int nCol;                       /* Number of columns in table */
  int nPhrase;                    /* Number of matchable phrases in query */
  sqlite3_int64 nDoc;             /* Number of docs in database */
  u32 *aMatchinfo;                /* Pre-allocated buffer */
};



/*
** The snippet() and offsets() functions both return text values. An instance
266
267
268
269
270
271
272










273
274
275
276
277
278
279
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307







+
+
+
+
+
+
+
+
+
+







  if( rc==SQLITE_OK ){
    (void)fts3ExprIterate(pCsr->pExpr, fts3ExprLoadDoclistsCb2, 0);
  }
  if( pnPhrase ) *pnPhrase = sCtx.nPhrase;
  if( pnToken ) *pnToken = sCtx.nToken;
  return rc;
}

static int fts3ExprPhraseCountCb(Fts3Expr *pExpr, int iPhrase, void *ctx){
  (*(int *)ctx)++;
  return SQLITE_OK;
}
static int fts3ExprPhraseCount(Fts3Expr *pExpr){
  int nPhrase = 0;
  (void)fts3ExprIterate(pExpr, fts3ExprPhraseCountCb, (void *)&nPhrase);
  return nPhrase;
}

/*
** Advance the position list iterator specified by the first two 
** arguments so that it points to the first element with a value greater
** than or equal to parameter iNext.
*/
static void fts3SnippetAdvance(char **ppIter, int *piIter, int iNext){
779
780
781
782
783
784
785
786
787
























788
789

790
791
792
793
794
795
796
797
798
799

800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818


819
820


821
822
823
824
825
826
827
828
829
830
831
832

833
834
835
836
837
838
839
840
841


842
843
844

845
846
847
848
849
850
851
852
853

854
855
856
857
858
859
860
861
862
863
864
865





































































































































































































































































































































866
867
868
869
870
871




872
873
874

875

876
877
878















879
880

881
882
883
884
885
886


887
888
889


890
891
892


893
894
895



896
897


898

899
900
901


902
903

904
905
906


907
908
909
910
911
912

913

914
915
916

917

918
919
920
921

922
923
924
925
926

927
928
929
930
931
932
933
807
808
809
810
811
812
813


814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838

839
840
841
842
843
844
845
846
847
848

849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866


867
868


869
870
871
872
873
874
875
876
877
878
879
880
881

882
883
884
885
886
887
888
889


890
891
892
893

894
895
896
897
898
899
900
901
902

903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245

1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274

1275






1276
1277
1278


1279
1280



1281
1282



1283
1284
1285
1286
1287
1288
1289

1290



1291
1292


1293



1294
1295






1296
1297
1298
1299
1300
1301
1302

1303




1304

1305
1306
1307

1308
1309
1310
1311
1312
1313
1314
1315







-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

-
+









-
+

















-
-
+
+
-
-
+
+











-
+







-
-
+
+


-
+








-
+












+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+





-
+
+
+
+



+

+



+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

-
+
-
-
-
-
-
-
+
+

-
-
+
+
-
-
-
+
+
-
-
-
+
+
+


+
+
-
+
-
-
-
+
+
-
-
+
-
-
-
+
+
-
-
-
-
-
-
+

+



+
-
+
-
-
-
-
+
-



-
+







  }
  pCsr++;
  *pp = pCsr;
}

/*
** fts3ExprIterate() callback used to collect the "global" matchinfo stats
** for a single query. The "global" stats are those elements of the matchinfo
** array that are constant for all rows returned by the current query.
** for a single query. 
**
** fts3ExprIterate() callback to load the 'global' elements of a
** FTS3_MATCHINFO_HITS matchinfo array. The global stats are those elements 
** of the matchinfo array that are constant for all rows returned by the 
** current query.
**
** Argument pCtx is actually a pointer to a struct of type MatchInfo. This
** function populates Matchinfo.aMatchinfo[] as follows:
**
**   for(iCol=0; iCol<nCol; iCol++){
**     aMatchinfo[3*iPhrase*nCol + 3*iCol + 1] = X;
**     aMatchinfo[3*iPhrase*nCol + 3*iCol + 2] = Y;
**   }
**
** where X is the number of matches for phrase iPhrase is column iCol of all
** rows of the table. Y is the number of rows for which column iCol contains
** at least one instance of phrase iPhrase.
**
** If the phrase pExpr consists entirely of deferred tokens, then all X and
** Y values are set to nDoc, where nDoc is the number of documents in the 
** file system. This is done because the full-text index doclist is required
** to calculate these values properly, and the full-text index doclist is
** not available for deferred tokens.
*/
static int fts3ExprGlobalMatchinfoCb(
static int fts3ExprGlobalHitsCb(
  Fts3Expr *pExpr,                /* Phrase expression node */
  int iPhrase,                    /* Phrase number (numbered from zero) */
  void *pCtx                      /* Pointer to MatchInfo structure */
){
  MatchInfo *p = (MatchInfo *)pCtx;
  Fts3Cursor *pCsr = p->pCursor;
  char *pIter;
  char *pEnd;
  char *pFree = 0;
  const int iStart = 2 + (iPhrase * p->nCol * 3) + 1;
  u32 *aOut = &p->aMatchinfo[3*iPhrase*p->nCol];

  assert( pExpr->isLoaded );
  assert( pExpr->eType==FTSQUERY_PHRASE );

  if( pCsr->pDeferred ){
    Fts3Phrase *pPhrase = pExpr->pPhrase;
    int ii;
    for(ii=0; ii<pPhrase->nToken; ii++){
      if( pPhrase->aToken[ii].bFulltext ) break;
    }
    if( ii<pPhrase->nToken ){
      int nFree = 0;
      int rc = sqlite3Fts3ExprLoadFtDoclist(pCsr, pExpr, &pFree, &nFree);
      if( rc!=SQLITE_OK ) return rc;
      pIter = pFree;
      pEnd = &pFree[nFree];
    }else{
      int nDoc = p->aMatchinfo[2 + 3*p->nCol*p->aMatchinfo[0]];
      for(ii=0; ii<p->nCol; ii++){
      int iCol;                   /* Column index */
      for(iCol=0; iCol<p->nCol; iCol++){
        p->aMatchinfo[iStart + ii*3] = nDoc;
        p->aMatchinfo[iStart + ii*3 + 1] = nDoc;
        aOut[iCol*3 + 1] = (u32)p->nDoc;
        aOut[iCol*3 + 2] = (u32)p->nDoc;
      }
      return SQLITE_OK;
    }
  }else{
    pIter = pExpr->aDoclist;
    pEnd = &pExpr->aDoclist[pExpr->nDoclist];
  }

  /* Fill in the global hit count matrix row for this phrase. */
  while( pIter<pEnd ){
    while( *pIter++ & 0x80 );      /* Skip past docid. */
    fts3LoadColumnlistCounts(&pIter, &p->aMatchinfo[iStart], 1);
    fts3LoadColumnlistCounts(&pIter, &aOut[1], 1);
  }

  sqlite3_free(pFree);
  return SQLITE_OK;
}

/*
** fts3ExprIterate() callback used to collect the "local" matchinfo stats
** for a single query. The "local" stats are those elements of the matchinfo
** fts3ExprIterate() callback used to collect the "local" part of the
** FTS3_MATCHINFO_HITS array. The local stats are those elements of the 
** array that are different for each row returned by the query.
*/
static int fts3ExprLocalMatchinfoCb(
static int fts3ExprLocalHitsCb(
  Fts3Expr *pExpr,                /* Phrase expression node */
  int iPhrase,                    /* Phrase number */
  void *pCtx                      /* Pointer to MatchInfo structure */
){
  MatchInfo *p = (MatchInfo *)pCtx;

  if( pExpr->aDoclist ){
    char *pCsr;
    int iStart = 2 + (iPhrase * p->nCol * 3);
    int iStart = iPhrase * p->nCol * 3;
    int i;

    for(i=0; i<p->nCol; i++) p->aMatchinfo[iStart+i*3] = 0;

    pCsr = sqlite3Fts3FindPositions(pExpr, p->pCursor->iPrevId, -1);
    if( pCsr ){
      fts3LoadColumnlistCounts(&pCsr, &p->aMatchinfo[iStart], 0);
    }
  }

  return SQLITE_OK;
}

static int fts3MatchinfoCheck(
  Fts3Table *pTab, 
  char cArg,
  char **pzErr
){
  if( (cArg==FTS3_MATCHINFO_NPHRASE)
   || (cArg==FTS3_MATCHINFO_NCOL)
   || (cArg==FTS3_MATCHINFO_NDOC && pTab->bHasStat)
   || (cArg==FTS3_MATCHINFO_AVGLENGTH && pTab->bHasStat)
   || (cArg==FTS3_MATCHINFO_LENGTH && pTab->bHasDocsize)
   || (cArg==FTS3_MATCHINFO_LCS)
   || (cArg==FTS3_MATCHINFO_HITS)
  ){
    return SQLITE_OK;
  }
  *pzErr = sqlite3_mprintf("unrecognized matchinfo request: %c", cArg);
  return SQLITE_ERROR;
}

static int fts3MatchinfoSize(MatchInfo *pInfo, char cArg){
  int nVal;                       /* Number of integers output by cArg */

  switch( cArg ){
    case FTS3_MATCHINFO_NDOC:
    case FTS3_MATCHINFO_NPHRASE: 
    case FTS3_MATCHINFO_NCOL: 
      nVal = 1;
      break;

    case FTS3_MATCHINFO_AVGLENGTH:
    case FTS3_MATCHINFO_LENGTH:
    case FTS3_MATCHINFO_LCS:
      nVal = pInfo->nCol;
      break;

    default:
      assert( cArg==FTS3_MATCHINFO_HITS );
      nVal = pInfo->nCol * pInfo->nPhrase * 3;
      break;
  }

  return nVal;
}

static int fts3MatchinfoSelectDoctotal(
  Fts3Table *pTab,
  sqlite3_stmt **ppStmt,
  sqlite3_int64 *pnDoc,
  const char **paLen
){
  sqlite3_stmt *pStmt;
  const char *a;
  sqlite3_int64 nDoc;

  if( !*ppStmt ){
    int rc = sqlite3Fts3SelectDoctotal(pTab, ppStmt);
    if( rc!=SQLITE_OK ) return rc;
  }
  pStmt = *ppStmt;

  a = sqlite3_column_blob(pStmt, 0);
  a += sqlite3Fts3GetVarint(a, &nDoc);
  *pnDoc = (u32)nDoc;

  if( paLen ) *paLen = a;
  return SQLITE_OK;
}

/*
** An instance of the following structure is used to store state while 
** iterating through a multi-column position-list corresponding to the
** hits for a single phrase on a single row in order to calculate the
** values for a matchinfo() FTS3_MATCHINFO_LCS request.
*/
typedef struct LcsIterator LcsIterator;
struct LcsIterator {
  Fts3Expr *pExpr;                /* Pointer to phrase expression */
  char *pRead;                    /* Cursor used to iterate through aDoclist */
  int iPosOffset;                 /* Tokens count up to end of this phrase */
  int iCol;                       /* Current column number */
  int iPos;                       /* Current position */
};

/* 
** If LcsIterator.iCol is set to the following value, the iterator has
** finished iterating through all offsets for all columns.
*/
#define LCS_ITERATOR_FINISHED 0x7FFFFFFF;

static int fts3MatchinfoLcsCb(
  Fts3Expr *pExpr,                /* Phrase expression node */
  int iPhrase,                    /* Phrase number (numbered from zero) */
  void *pCtx                      /* Pointer to MatchInfo structure */
){
  LcsIterator *aIter = (LcsIterator *)pCtx;
  aIter[iPhrase].pExpr = pExpr;
  return SQLITE_OK;
}

/*
** Advance the iterator passed as an argument to the next position. Return
** 1 if the iterator is at EOF or if it now points to the start of the
** position list for the next column.
*/
static int fts3LcsIteratorAdvance(LcsIterator *pIter){
  char *pRead = pIter->pRead;
  sqlite3_int64 iRead;
  int rc = 0;

  pRead += sqlite3Fts3GetVarint(pRead, &iRead);
  if( iRead==0 ){
    pIter->iCol = LCS_ITERATOR_FINISHED;
    rc = 1;
  }else{
    if( iRead==1 ){
      pRead += sqlite3Fts3GetVarint(pRead, &iRead);
      pIter->iCol = iRead;
      pIter->iPos = pIter->iPosOffset;
      pRead += sqlite3Fts3GetVarint(pRead, &iRead);
      rc = 1;
    }
    pIter->iPos += (iRead-2);
  }

  pIter->pRead = pRead;
  return rc;
}
  
/*
** This function implements the FTS3_MATCHINFO_LCS matchinfo() flag. 
**
** If the call is successful, the longest-common-substring lengths for each
** column are written into the first nCol elements of the pInfo->aMatchinfo[] 
** array before returning. SQLITE_OK is returned in this case.
**
** Otherwise, if an error occurs, an SQLite error code is returned and the
** data written to the first nCol elements of pInfo->aMatchinfo[] is 
** undefined.
*/
static int fts3MatchinfoLcs(Fts3Cursor *pCsr, MatchInfo *pInfo){
  LcsIterator *aIter;
  int i;
  int iCol;
  int nToken = 0;

  /* Allocate and populate the array of LcsIterator objects. The array
  ** contains one element for each matchable phrase in the query.
  **/
  aIter = sqlite3_malloc(sizeof(LcsIterator) * pCsr->nPhrase);
  if( !aIter ) return SQLITE_NOMEM;
  memset(aIter, 0, sizeof(LcsIterator) * pCsr->nPhrase);
  (void)fts3ExprIterate(pCsr->pExpr, fts3MatchinfoLcsCb, (void*)aIter);
  for(i=0; i<pInfo->nPhrase; i++){
    LcsIterator *pIter = &aIter[i];
    nToken -= pIter->pExpr->pPhrase->nToken;
    pIter->iPosOffset = nToken;
    pIter->pRead = sqlite3Fts3FindPositions(pIter->pExpr, pCsr->iPrevId, -1);
    if( pIter->pRead ){
      pIter->iPos = pIter->iPosOffset;
      fts3LcsIteratorAdvance(&aIter[i]);
    }else{
      pIter->iCol = LCS_ITERATOR_FINISHED;
    }
  }

  for(iCol=0; iCol<pInfo->nCol; iCol++){
    int nLcs = 0;                 /* LCS value for this column */
    int nLive = 0;                /* Number of iterators in aIter not at EOF */

    /* Loop through the iterators in aIter[]. Set nLive to the number of
    ** iterators that point to a position-list corresponding to column iCol.
    */
    for(i=0; i<pInfo->nPhrase; i++){
      assert( aIter[i].iCol>=iCol );
      if( aIter[i].iCol==iCol ) nLive++;
    }

    /* The following loop runs until all iterators in aIter[] have finished
    ** iterating through positions in column iCol. Exactly one of the 
    ** iterators is advanced each time the body of the loop is run.
    */
    while( nLive>0 ){
      LcsIterator *pAdv = 0;      /* The iterator to advance by one position */
      int nThisLcs = 0;           /* LCS for the current iterator positions */

      for(i=0; i<pInfo->nPhrase; i++){
        LcsIterator *pIter = &aIter[i];
        if( iCol!=pIter->iCol ){  
          /* This iterator is already at EOF for this column. */
          nThisLcs = 0;
        }else{
          if( pAdv==0 || pIter->iPos<pAdv->iPos ){
            pAdv = pIter;
          }
          if( nThisLcs==0 || pIter->iPos==pIter[-1].iPos ){
            nThisLcs++;
          }else{
            nThisLcs = 1;
          }
          if( nThisLcs>nLcs ) nLcs = nThisLcs;
        }
      }
      if( fts3LcsIteratorAdvance(pAdv) ) nLive--;
    }

    pInfo->aMatchinfo[iCol] = nLcs;
  }

  sqlite3_free(aIter);
  return SQLITE_OK;
}

/*
** Populate the buffer pInfo->aMatchinfo[] with an array of integers to
** be returned by the matchinfo() function. Argument zArg contains the 
** format string passed as the second argument to matchinfo (or the
** default value "pcx" if no second argument was specified). The format
** string has already been validated and the pInfo->aMatchinfo[] array
** is guaranteed to be large enough for the output.
**
** If bGlobal is true, then populate all fields of the matchinfo() output.
** If it is false, then assume that those fields that do not change between
** rows (i.e. FTS3_MATCHINFO_NPHRASE, NCOL, NDOC, AVGLENGTH and part of HITS)
** have already been populated.
**
** Return SQLITE_OK if successful, or an SQLite error code if an error 
** occurs. If a value other than SQLITE_OK is returned, the state the
** pInfo->aMatchinfo[] buffer is left in is undefined.
*/
static int fts3MatchinfoValues(
  Fts3Cursor *pCsr,               /* FTS3 cursor object */
  int bGlobal,                    /* True to grab the global stats */
  MatchInfo *pInfo,               /* Matchinfo context object */
  const char *zArg                /* Matchinfo format string */
){
  int rc = SQLITE_OK;
  int i;
  Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
  sqlite3_stmt *pSelect = 0;

  for(i=0; rc==SQLITE_OK && zArg[i]; i++){

    switch( zArg[i] ){
      case FTS3_MATCHINFO_NPHRASE:
        if( bGlobal ) pInfo->aMatchinfo[0] = pInfo->nPhrase;
        break;

      case FTS3_MATCHINFO_NCOL:
        if( bGlobal ) pInfo->aMatchinfo[0] = pInfo->nCol;
        break;
        
      case FTS3_MATCHINFO_NDOC:
        if( bGlobal ){
          sqlite3_int64 nDoc;
          rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &nDoc, 0);
          pInfo->aMatchinfo[0] = (u32)nDoc;
        }
        break;

      case FTS3_MATCHINFO_AVGLENGTH: 
        if( bGlobal ){
          sqlite3_int64 nDoc;     /* Number of rows in table */
          const char *a;          /* Aggregate column length array */

          rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &nDoc, &a);
          if( rc==SQLITE_OK ){
            int iCol;
            for(iCol=0; iCol<pInfo->nCol; iCol++){
              sqlite3_int64 nToken;
              a += sqlite3Fts3GetVarint(a, &nToken);
              pInfo->aMatchinfo[iCol] = ((u32)(nToken&0xffffffff)+nDoc/2)/nDoc;
            }
          }
        }
        break;

      case FTS3_MATCHINFO_LENGTH: {
        sqlite3_stmt *pSelectDocsize = 0;
        rc = sqlite3Fts3SelectDocsize(pTab, pCsr->iPrevId, &pSelectDocsize);
        if( rc==SQLITE_OK ){
          int iCol;
          const char *a = sqlite3_column_blob(pSelectDocsize, 0);
          for(iCol=0; iCol<pInfo->nCol; iCol++){
            sqlite3_int64 nToken;
            a += sqlite3Fts3GetVarint(a, &nToken);
            pInfo->aMatchinfo[iCol] = (u32)nToken;
          }
        }
        sqlite3_reset(pSelectDocsize);
        break;
      }

      case FTS3_MATCHINFO_LCS:
        rc = fts3ExprLoadDoclists(pCsr, 0, 0);
        if( rc==SQLITE_OK ){
          rc = fts3MatchinfoLcs(pCsr, pInfo);
        }
        break;

      default: {
        assert( zArg[i]==FTS3_MATCHINFO_HITS );
        Fts3Expr *pExpr = pCsr->pExpr;
        rc = fts3ExprLoadDoclists(pCsr, 0, 0);
        if( rc!=SQLITE_OK ) break;
        if( bGlobal ){
          if( pCsr->pDeferred ){
            rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &pInfo->nDoc, 0);
            if( rc!=SQLITE_OK ) break;
          }
          rc = fts3ExprIterate(pExpr, fts3ExprGlobalHitsCb,(void*)pInfo);
          if( rc!=SQLITE_OK ) break;
        }
        (void)fts3ExprIterate(pExpr, fts3ExprLocalHitsCb,(void*)pInfo);
        break;
      }
    }

    pInfo->aMatchinfo += fts3MatchinfoSize(pInfo, zArg[i]);
  }

  sqlite3_reset(pSelect);
  return rc;
}


/*
** Populate pCsr->aMatchinfo[] with data for the current row. The 
** 'matchinfo' data is an array of 32-bit unsigned integers (C type u32).
*/
static int fts3GetMatchinfo(Fts3Cursor *pCsr){
static int fts3GetMatchinfo(
  Fts3Cursor *pCsr,               /* FTS3 Cursor object */
  const char *zArg                /* Second argument to matchinfo() function */
){
  MatchInfo sInfo;
  Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
  int rc = SQLITE_OK;
  int bGlobal = 0;                /* Collect 'global' stats as well as local */

  memset(&sInfo, 0, sizeof(MatchInfo));
  sInfo.pCursor = pCsr;
  sInfo.nCol = pTab->nColumn;

  /* If there is cached matchinfo() data, but the format string for the 
  ** cache does not match the format string for this request, discard 
  ** the cached data. */
  if( pCsr->zMatchinfo && strcmp(pCsr->zMatchinfo, zArg) ){
    assert( pCsr->aMatchinfo );
    sqlite3_free(pCsr->aMatchinfo);
    pCsr->zMatchinfo = 0;
    pCsr->aMatchinfo = 0;
  }

  /* If Fts3Cursor.aMatchinfo[] is NULL, then this is the first time the
  ** matchinfo function has been called for this query. In this case 
  ** allocate the array used to accumulate the matchinfo data and
  ** initialize those elements that are constant for every row.
  */
  if( pCsr->aMatchinfo==0 ){
    /* If Fts3Cursor.aMatchinfo[] is NULL, then this is the first time the
    int nMatchinfo = 0;           /* Number of u32 elements in match-info */
    ** matchinfo function has been called for this query. In this case 
    ** allocate the array used to accumulate the matchinfo data and
    ** initialize those elements that are constant for every row.
    */
    int nPhrase;                  /* Number of phrases */
    int nMatchinfo;               /* Number of u32 elements in match-info */
    int nArg;                     /* Bytes in zArg */
    int i;                        /* Used to iterate through zArg */

    /* Load doclists for each phrase in the query. */
    rc = fts3ExprLoadDoclists(pCsr, &nPhrase, 0);
    /* Determine the number of phrases in the query */
    pCsr->nPhrase = fts3ExprPhraseCount(pCsr->pExpr);
    if( rc!=SQLITE_OK ){
      return rc;
    }
    sInfo.nPhrase = pCsr->nPhrase;

    nMatchinfo = 2 + 3*sInfo.nCol*nPhrase;
    if( pTab->bHasDocsize ){
      nMatchinfo += 1 + 2*pTab->nColumn;
    /* Determine the number of integers in the buffer returned by this call. */
    for(i=0; zArg[i]; i++){
      nMatchinfo += fts3MatchinfoSize(&sInfo, zArg[i]);
    }

    /* Allocate space for Fts3Cursor.aMatchinfo[] and Fts3Cursor.zMatchinfo. */
    nArg = strlen(zArg);
    sInfo.aMatchinfo = (u32 *)sqlite3_malloc(sizeof(u32)*nMatchinfo);
    pCsr->aMatchinfo = (u32 *)sqlite3_malloc(sizeof(u32)*nMatchinfo + nArg + 1);
    if( !sInfo.aMatchinfo ){ 
      return SQLITE_NOMEM;
    }
    if( !pCsr->aMatchinfo ) return SQLITE_NOMEM;

    memset(sInfo.aMatchinfo, 0, sizeof(u32)*nMatchinfo);

    pCsr->zMatchinfo = (char *)&pCsr->aMatchinfo[nMatchinfo];
    /* First element of match-info is the number of phrases in the query */
    sInfo.aMatchinfo[0] = nPhrase;
    sInfo.aMatchinfo[1] = sInfo.nCol;
    pCsr->nMatchinfo = nMatchinfo;
    memcpy(pCsr->zMatchinfo, zArg, nArg+1);
    if( pTab->bHasDocsize ){
      int ofst = 2 + 3*sInfo.aMatchinfo[0]*sInfo.aMatchinfo[1];
      rc = sqlite3Fts3MatchinfoDocsizeGlobal(pCsr, &sInfo.aMatchinfo[ofst]);
    }
    (void)fts3ExprIterate(pCsr->pExpr, fts3ExprGlobalMatchinfoCb,(void*)&sInfo);
    pCsr->aMatchinfo = sInfo.aMatchinfo;
    memset(pCsr->aMatchinfo, 0, sizeof(u32)*nMatchinfo);
    pCsr->isMatchinfoNeeded = 1;
    bGlobal = 1;
  }

  sInfo.aMatchinfo = pCsr->aMatchinfo;
  sInfo.nPhrase = pCsr->nPhrase;
  if( rc==SQLITE_OK && pCsr->isMatchinfoNeeded ){
  if( pCsr->isMatchinfoNeeded ){
    (void)fts3ExprIterate(pCsr->pExpr, fts3ExprLocalMatchinfoCb, (void*)&sInfo);
    if( pTab->bHasDocsize ){
      int ofst = 2 + 3*sInfo.aMatchinfo[0]*sInfo.aMatchinfo[1];
      rc = sqlite3Fts3MatchinfoDocsizeLocal(pCsr, &sInfo.aMatchinfo[ofst]);
    rc = fts3MatchinfoValues(pCsr, bGlobal, &sInfo, zArg);
    }
    pCsr->isMatchinfoNeeded = 0;
  }

  return SQLITE_OK;
  return rc;
}

/*
** Implementation of snippet() function.
*/
void sqlite3Fts3Snippet(
  sqlite3_context *pCtx,          /* SQLite function call context */
980
981
982
983
984
985
986
987

988
989
990
991
992
993
994
1362
1363
1364
1365
1366
1367
1368

1369
1370
1371
1372
1373
1374
1375
1376







-
+







      memset(pFragment, 0, sizeof(*pFragment));

      /* Loop through all columns of the table being considered for snippets.
      ** If the iCol argument to this function was negative, this means all
      ** columns of the FTS3 table. Otherwise, only column iCol is considered.
      */
      for(iRead=0; iRead<pTab->nColumn; iRead++){
        SnippetFragment sF;
        SnippetFragment sF = {0, 0, 0, 0};
        int iS;
        if( iCol>=0 && iRead!=iCol ) continue;

        /* Find the best snippet of nFToken tokens in column iRead. */
        rc = fts3BestSnippet(nFToken, pCsr, iRead, mCovered, &mSeen, &sF, &iS);
        if( rc!=SQLITE_OK ){
          goto snippet_out;
1207
1208
1209
1210
1211
1212
1213
1214






1215

















1216
1217
1218
1219


1220
1221



1222
1223
1224
1225
1226
1227
1228

1229
1230
1231
1232
1233
1234
1589
1590
1591
1592
1593
1594
1595

1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625


1626
1627
1628
1629
1630
1631




1632

1633
1634
1635
1636
1637







-
+
+
+
+
+
+

+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+




+
+
-
-
+
+
+



-
-
-
-
+
-





  }
  return;
}

/*
** Implementation of matchinfo() function.
*/
void sqlite3Fts3Matchinfo(sqlite3_context *pContext, Fts3Cursor *pCsr){
void sqlite3Fts3Matchinfo(
  sqlite3_context *pContext,      /* Function call context */
  Fts3Cursor *pCsr,               /* FTS3 table cursor */
  const char *zArg                /* Second arg to matchinfo() function */
){
  Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
  int rc;
  int i;
  const char *zFormat;

  if( zArg ){
    for(i=0; zArg[i]; i++){
      char *zErr = 0;
      if( fts3MatchinfoCheck(pTab, zArg[i], &zErr) ){
        sqlite3_result_error(pContext, zErr, -1);
        sqlite3_free(zErr);
        return;
      }
    }
    zFormat = zArg;
  }else{
    zFormat = FTS3_MATCHINFO_DEFAULT;
  }

  if( !pCsr->pExpr ){
    sqlite3_result_blob(pContext, "", 0, SQLITE_STATIC);
    return;
  }

  /* Retrieve matchinfo() data. */
  rc = fts3GetMatchinfo(pCsr);
  sqlite3Fts3SegmentsClose((Fts3Table *)pCsr->base.pVtab );
  rc = fts3GetMatchinfo(pCsr, zFormat);
  sqlite3Fts3SegmentsClose(pTab);

  if( rc!=SQLITE_OK ){
    sqlite3_result_error_code(pContext, rc);
  }else{
    Fts3Table *pTab = (Fts3Table*)pCsr->base.pVtab;
    int n = sizeof(u32)*(2+pCsr->aMatchinfo[0]*pCsr->aMatchinfo[1]*3);
    if( pTab->bHasDocsize ){
      n += sizeof(u32)*(1 + 2*pTab->nColumn);
    int n = pCsr->nMatchinfo * sizeof(u32);
    }
    sqlite3_result_blob(pContext, pCsr->aMatchinfo, n, SQLITE_TRANSIENT);
  }
}

#endif

Changes to ext/fts3/fts3_write.c.

278
279
280
281
282
283
284
















































285
286
287
288
289
290
291
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339







+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+







    for(i=0; rc==SQLITE_OK && i<nParam; i++){
      rc = sqlite3_bind_value(pStmt, i+1, apVal[i]);
    }
  }
  *pp = pStmt;
  return rc;
}

static int fts3SelectDocsize(
  Fts3Table *pTab,                /* FTS3 table handle */
  int eStmt,                      /* Either SQL_SELECT_DOCSIZE or DOCTOTAL */
  sqlite3_int64 iDocid,           /* Docid to bind for SQL_SELECT_DOCSIZE */
  sqlite3_stmt **ppStmt           /* OUT: Statement handle */
){
  sqlite3_stmt *pStmt = 0;        /* Statement requested from fts3SqlStmt() */
  int rc;                         /* Return code */

  assert( eStmt==SQL_SELECT_DOCSIZE || eStmt==SQL_SELECT_DOCTOTAL );

  rc = fts3SqlStmt(pTab, eStmt, &pStmt, 0);
  if( rc==SQLITE_OK ){
    if( eStmt==SQL_SELECT_DOCSIZE ){
      sqlite3_bind_int64(pStmt, 1, iDocid);
    }
    rc = sqlite3_step(pStmt);
    if( rc!=SQLITE_ROW ){
      rc = sqlite3_reset(pStmt);
      if( rc==SQLITE_OK ) rc = SQLITE_CORRUPT;
      pStmt = 0;
    }else{
      rc = SQLITE_OK;
    }
  }

  *ppStmt = pStmt;
  return rc;
}

int sqlite3Fts3SelectDoctotal(
  Fts3Table *pTab,                /* Fts3 table handle */
  sqlite3_stmt **ppStmt           /* OUT: Statement handle */
){
  return fts3SelectDocsize(pTab, SQL_SELECT_DOCTOTAL, 0, ppStmt);
}

int sqlite3Fts3SelectDocsize(
  Fts3Table *pTab,                /* Fts3 table handle */
  sqlite3_int64 iDocid,           /* Docid to read size data for */
  sqlite3_stmt **ppStmt           /* OUT: Statement handle */
){
  return fts3SelectDocsize(pTab, SQL_SELECT_DOCSIZE, iDocid, ppStmt);
}

void sqlite3Fts3MatchinfoLcs(Fts3Expr *pExpr, u32 *aOut){
}

/*
** Similar to fts3SqlStmt(). Except, after binding the parameters in
** array apVal[] to the SQL statement identified by eStmt, the statement
** is executed.
**
** Returns SQLITE_OK if the statement is successfully executed, or an
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2487
2488
2489
2490
2491
2492
2493





































































2494
2495
2496
2497
2498
2499
2500







-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-







    sqlite3_int64 x;
    j += sqlite3Fts3GetVarint(&zBuf[j], &x);
    assert(j<=nBuf);
    a[i] = (u32)(x & 0xffffffff);
  }
}

/*
** Fill in the document size auxiliary information for the matchinfo
** structure.  The auxiliary information is:
**
**    N     Total number of documents in the full-text index
**    a0    Average length of column 0 over the whole index
**    n0    Length of column 0 on the matching row
**    ...
**    aM    Average length of column M over the whole index
**    nM    Length of column M on the matching row
**
** The fts3MatchinfoDocsizeLocal() routine fills in the nX values.
** The fts3MatchinfoDocsizeGlobal() routine fills in N and the aX values.
*/
int sqlite3Fts3MatchinfoDocsizeLocal(Fts3Cursor *pCur, u32 *a){
  const char *pBlob;       /* The BLOB holding %_docsize info */
  int nBlob;               /* Size of the BLOB */
  sqlite3_stmt *pStmt;     /* Statement for reading and writing */
  int i, j;                /* Loop counters */
  sqlite3_int64 x;         /* Varint value */
  int rc;                  /* Result code from subfunctions */
  Fts3Table *p;            /* The FTS table */

  p = (Fts3Table*)pCur->base.pVtab;
  rc = fts3SqlStmt(p, SQL_SELECT_DOCSIZE, &pStmt, 0);
  if( rc ){
    return rc;
  }
  sqlite3_bind_int64(pStmt, 1, pCur->iPrevId);
  if( sqlite3_step(pStmt)==SQLITE_ROW ){
    nBlob = sqlite3_column_bytes(pStmt, 0);
    pBlob = (const char*)sqlite3_column_blob(pStmt, 0);
    for(i=j=0; i<p->nColumn && j<nBlob; i++){
      j = sqlite3Fts3GetVarint(&pBlob[j], &x);
      a[2+i*2] = (u32)(x & 0xffffffff);
    }
  }
  sqlite3_reset(pStmt);
  return SQLITE_OK; 
}
int sqlite3Fts3MatchinfoDocsizeGlobal(Fts3Cursor *pCur, u32 *a){
  const char *pBlob;       /* The BLOB holding %_stat info */
  int nBlob;               /* Size of the BLOB */
  sqlite3_stmt *pStmt;     /* Statement for reading and writing */
  int i, j;                /* Loop counters */
  sqlite3_int64 x;         /* Varint value */
  int nDoc;                /* Number of documents */
  int rc;                  /* Result code from subfunctions */
  Fts3Table *p;            /* The FTS table */

  p = (Fts3Table*)pCur->base.pVtab;
  rc = fts3SqlStmt(p, SQL_SELECT_DOCTOTAL, &pStmt, 0);
  if( rc ){
    return rc;
  }
  if( sqlite3_step(pStmt)==SQLITE_ROW ){
    nBlob = sqlite3_column_bytes(pStmt, 0);
    pBlob = (const char*)sqlite3_column_blob(pStmt, 0);
    j = sqlite3Fts3GetVarint(pBlob, &x);
    a[0] = nDoc = (u32)(x & 0xffffffff);
    for(i=0; i<p->nColumn && j<nBlob; i++){
      j = sqlite3Fts3GetVarint(&pBlob[j], &x);
      a[1+i*2] = ((u32)(x & 0xffffffff) + nDoc/2)/nDoc;
    }
  }
  sqlite3_reset(pStmt);
  return SQLITE_OK; 
}

/*
** Insert the sizes (in tokens) for each column of the document
** with docid equal to p->iPrevDocid.  The sizes are encoded as
** a blob of varints.
*/
static void fts3InsertDocsize(
  int *pRC,         /* Result code */

Changes to test/fts3defer2.test.

49
50
51
52
53
54
55
56

57
58
59
60
61
62
63
64
65

66
67
68
69
70
71
72
49
50
51
52
53
54
55

56
57
58
59
60
61
62
63
64

65
66
67
68
69
70
71
72







-
+








-
+







} {2}

do_execsql_test 1.2.1 {
  SELECT content FROM t1 WHERE t1 MATCH 'f (e NEAR/2 a)';
} {{a b c d e f a x y}}

do_execsql_test 1.2.2 {
  SELECT snippet(t1, '[', ']'), offsets(t1), mit(matchinfo(t1))
  SELECT snippet(t1, '[', ']'), offsets(t1), mit(matchinfo(t1, 'pcxnal'))
  FROM t1 WHERE t1 MATCH 'f (e NEAR/2 a)';
} [list                              \
   {a b c d [e] [f] [a] x y}         \
   {0 1 8 1 0 0 10 1 0 2 12 1}       \
   [list 3 1   1 1 1   1 8 8   1 8 8   8 5001 9]
]

do_execsql_test 1.2.3 {
  SELECT snippet(t1, '[', ']'), offsets(t1), mit(matchinfo(t1))
  SELECT snippet(t1, '[', ']'), offsets(t1), mit(matchinfo(t1, 'pcxnal'))
  FROM t1 WHERE t1 MATCH 'f (e NEAR/3 a)';
} [list                                 \
   {[a] b c d [e] [f] [a] x y}          \
   {0 2 0 1 0 1 8 1 0 0 10 1 0 2 12 1}  \
   [list 3 1   1 1 1   1 8 8   2 8 8   8 5001 9]
]

87
88
89
90
91
92
93

94
95

96
97
98
99
100
101
102
87
88
89
90
91
92
93
94
95

96
97
98
99
100
101
102
103







+

-
+







  1 {}
  2 { INSERT INTO t2(t2) VALUES('optimize') }
  3 { UPDATE t2_segments SET block = zeroblob(length(block)) 
      WHERE length(block)>10000;
  }
} {
  execsql $sql

  do_execsql_test 2.2.$tn {
    SELECT mit(matchinfo(t2)) FROM t2 WHERE t2 MATCH 'a b';
    SELECT mit(matchinfo(t2, 'pcxnal')) FROM t2 WHERE t2 MATCH 'a b';
  } [list                                          \
    [list 2 1  1 54 54  1 3 3  54 372 7]        \
    [list 2 1  1 54 54  1 3 3  54 372 7]        \
  ]
}

do_execsql_test 2.3.1 {
120
121
122
123
124
125
126
127

128
129
130
131
132
133
121
122
123
124
125
126
127

128
129
130
131
132
133
134







-
+






  2 { INSERT INTO t3(t3) VALUES('optimize') }
  3 { UPDATE t3_segments SET block = zeroblob(length(block)) 
      WHERE length(block)>10000;
  }
} {
  execsql $sql
  do_execsql_test 2.4.$tn {
    SELECT docid, mit(matchinfo(t3)) FROM t3 WHERE t3 MATCH '"a b c"';
    SELECT docid, mit(matchinfo(t3, 'pcxnal')) FROM t3 WHERE t3 MATCH '"a b c"';
  } {1 {1 1 1 4 4 11 912 6} 3 {1 1 1 4 4 11 912 6}}
}


finish_test

Changes to test/fts3fault.test.

13
14
15
16
17
18
19


20
21
22
23
24
25
26
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28







+
+







set testdir [file dirname $argv0]
source $testdir/tester.tcl

set ::testprefix fts3fault

# If SQLITE_ENABLE_FTS3 is not defined, omit this file.
ifcapable !fts3 { finish_test ; return }

if 1 {

# Test error handling in the sqlite3Fts3Init() function. This is the 
# function that registers the FTS3 module and various support functions
# with SQLite.
#
do_faultsim_test 1 -body { 
  sqlite3 db test.db 
150
151
152
153
154
155
156
157









































































158
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233








+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

  faultsim_delete_and_reopen
} -body {
  execsql { CREATE VIRTUAL TABLE t1 USING fts4(a, b, matchnfo=fts3) }
} -test {
  faultsim_test_result {1 {unrecognized parameter: matchnfo=fts3}} \
                       {1 {vtable constructor failed: t1}}
}

}

proc mit {blob} {
  set scan(littleEndian) i*
  set scan(bigEndian) I*
  binary scan $blob $scan($::tcl_platform(byteOrder)) r
  return $r
}

do_test 8.0 {
  faultsim_delete_and_reopen
  execsql { CREATE VIRTUAL TABLE t8 USING fts4 }
  execsql "INSERT INTO t8 VALUES('a b c')"
  execsql "INSERT INTO t8 VALUES('b b b')"
  execsql "INSERT INTO t8 VALUES('[string repeat {c } 50000]')"
  execsql "INSERT INTO t8 VALUES('d d d')"
  execsql "INSERT INTO t8 VALUES('e e e')"
  execsql "INSERT INTO t8(t8) VALUES('optimize')"
  faultsim_save_and_close
} {}

do_faultsim_test 8.1 -prep { 
  faultsim_restore_and_reopen
  db func mit mit
} -body {
  execsql { SELECT mit(matchinfo(t8, 'x')) FROM t8 WHERE t8 MATCH 'a b c' }
} -test {
  faultsim_test_result {0 {{1 1 1 1 4 2 1 5 5}}}
}
do_faultsim_test 8.2 -faults oom-t* -prep { 
  faultsim_restore_and_reopen
  db func mit mit
} -body {
  execsql { SELECT mit(matchinfo(t8, 's')) FROM t8 WHERE t8 MATCH 'a b c' }
} -test {
  faultsim_test_result {0 3}
}
do_faultsim_test 8.3 -prep { 
  faultsim_restore_and_reopen
  db func mit mit
} -body {
  execsql { SELECT mit(matchinfo(t8, 'a')) FROM t8 WHERE t8 MATCH 'a b c' }
} -test {
  faultsim_test_result {0 10002}
}
do_faultsim_test 8.4 -prep { 
  faultsim_restore_and_reopen
  db func mit mit
} -body {
  execsql { SELECT mit(matchinfo(t8, 'l')) FROM t8 WHERE t8 MATCH 'a b c' }
} -test {
  faultsim_test_result {0 3}
}

do_test 9.0 {
  faultsim_delete_and_reopen
  execsql {
    CREATE VIRTUAL TABLE t9 USING fts4(tokenize=porter);
    INSERT INTO t9 VALUES(
      'this record is used toooooooooooooooooooooooooooooooooooooo try to'
    );
    SELECT offsets(t9) FROM t9 WHERE t9 MATCH 'to*';
  }
  faultsim_save_and_close
} {}
do_faultsim_test 9.1 -prep {
  faultsim_restore_and_reopen
} -body {
  execsql { SELECT offsets(t9) FROM t9 WHERE t9 MATCH 'to*' }
} -test {
  faultsim_test_result {0 {{0 0 20 39 0 0 64 2}}}
}

finish_test

Changes to test/fts3matchinfo.test.

37
38
39
40
41
42
43
44

45
46
47
48
49
50
51
52

53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70



























































































































































































































































71

37
38
39
40
41
42
43

44


45
46
47
48
49

50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321







-
+
-
-





-
+


















+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

+
  INSERT INTO t1(content) VALUES('I wandered lonely as a cloud');
  INSERT INTO t1(content) VALUES('That floats on high o''er vales and hills,');
  INSERT INTO t1(content) VALUES('When all at once I saw a crowd,');
  INSERT INTO t1(content) VALUES('A host, of golden daffodils,');
  SELECT mit(matchinfo(t1)) FROM t1 WHERE t1 MATCH 'I';
} {{1 1 1 2 2} {1 1 1 2 2}}

# Now create an FTS4 table that does not specify matchinfo=fts3. The 
# Now create an FTS4 table that does not specify matchinfo=fts3.
# %_docsize table is created in this case and the array of integers returned
# by matchinfo() includes the extra data.
#
do_execsql_test 1.2 {
  CREATE VIRTUAL TABLE t2 USING fts4;
  INSERT INTO t2 SELECT * FROM t1;
  SELECT mit(matchinfo(t2)) FROM t2 WHERE t2 MATCH 'I';
} {{1 1 1 2 2 4 7 6} {1 1 1 2 2 4 7 8}}
} {{1 1 1 2 2} {1 1 1 2 2}}

# Test some syntax-error handling.
#
do_catchsql_test 2.0 {
  CREATE VIRTUAL TABLE x1 USING fts4(matchinfo=fs3);
} {1 {unrecognized matchinfo: fs3}}
do_catchsql_test 2.1 {
  CREATE VIRTUAL TABLE x2 USING fts4(mtchinfo=fts3);
} {1 {unrecognized parameter: mtchinfo=fts3}}

# Check that with fts3, the "=" character is permitted in column definitions.
#
do_execsql_test 3.1 {
  CREATE VIRTUAL TABLE t3 USING fts3(mtchinfo=fts3);
  INSERT INTO t3(mtchinfo) VALUES('Beside the lake, beneath the trees');
  SELECT mtchinfo FROM t3;
} {{Beside the lake, beneath the trees}}

do_execsql_test 3.2 {
  CREATE VIRTUAL TABLE xx USING FTS4;
  SELECT * FROM xx WHERE xx MATCH 'abc';
  SELECT * FROM xx WHERE xx MATCH 'a b c';
}


#--------------------------------------------------------------------------
# Proc [do_matchinfo_test] is used to test the FTSX matchinfo() function.
#
# The first argument - $tn - is a test identifier. This may be either a
# full identifier (i.e. "fts3matchinfo-1.1") or, if global var $testprefix
# is set, just the numeric component (i.e. "1.1").
#
# The second argument is the name of an FTSX table. The third is the 
# full text of a WHERE/MATCH expression to query the table for 
# (i.e. "t1 MATCH 'abc'"). The final argument - $results - should be a
# key-value list (serialized array) with matchinfo() format specifiers
# as keys, and the results of executing the statement:
#
#   SELECT matchinfo($tbl, '$key') FROM $tbl WHERE $expr
#
# For example:
#
#   CREATE VIRTUAL TABLE t1 USING fts4;
#   INSERT INTO t1 VALUES('abc');
#   INSERT INTO t1 VALUES('def');
#   INSERT INTO t1 VALUES('abc abc');
#
#   do_matchinfo_test 1.1 t1 "t1 MATCH 'abc'" {
#     n {3 3}
#     p {1 1}
#     c {1 1}
#     x {{1 3 2} {2 3 2}}
#   }
#
# If the $results list contains keys mapped to "-" instead of a matchinfo()
# result, then this command computes the expected results based on other
# mappings to test the matchinfo() function. For example, the command above
# could be changed to:
#
#   do_matchinfo_test 1.1 t1 "t1 MATCH 'abc'" {
#     n {3 3} p {1 1} c {1 1} x {{1 3 2} {2 3 2}}
#     pcx -
#   }
#
# And this command would compute the expected results for matchinfo(t1, 'pcx')
# based on the results of matchinfo(t1, 'p'), matchinfo(t1, 'c') and 
# matchinfo(t1, 'x') in order to test 'pcx'.
#
proc do_matchinfo_test {tn tbl expr results} {

  foreach {fmt res} $results {
    if {$res == "-"} continue
    set resarray($fmt) $res
  }

  set nRow 0
  foreach {fmt res} [array get resarray] {
    if {[llength $res]>$nRow} { set nRow [llength $res] }
  }

  # Construct expected results for any formats for which the caller 
  # supplied result is "-".
  #
  foreach {fmt res} $results {
    if {$res == "-"} {
      set res [list]
      for {set iRow 0} {$iRow<$nRow} {incr iRow} {
        set rowres [list]
        foreach c [split $fmt ""] {
          set rowres [concat $rowres [lindex $resarray($c) $iRow]]
        }
        lappend res $rowres
      }
      set resarray($fmt) $res
    }
  }

  # Test each matchinfo() request individually.
  #
  foreach {fmt res} [array get resarray] {
    set sql "SELECT mit(matchinfo($tbl, '$fmt')) FROM $tbl WHERE $expr"
    do_execsql_test $tn.$fmt $sql [normalize2 $res]
  }

  # Test them all executed together (multiple invocations of matchinfo()).
  #
  set exprlist [list]
  foreach {format res} [array get resarray] {
    lappend exprlist "mit(matchinfo($tbl, '$format'))"
  }
  set allres [list]
  for {set iRow 0} {$iRow<$nRow} {incr iRow} {
    foreach {format res} [array get resarray] {
      lappend allres [lindex $res $iRow]
    }
  }
  set sql "SELECT [join $exprlist ,] FROM $tbl WHERE $expr"
  do_execsql_test $tn.multi $sql [normalize2 $allres]
}
proc normalize2 {list_of_lists} {
  set res [list]
  foreach elem $list_of_lists {
    lappend res [list {*}$elem]
  }
  return $res
}


do_execsql_test 4.1.0 {
  CREATE VIRTUAL TABLE t4 USING fts4(x, y);
  INSERT INTO t4 VALUES('a b c d e', 'f g h i j');
  INSERT INTO t4 VALUES('f g h i j', 'a b c d e');
}

do_matchinfo_test 4.1.1 t4 {t4 MATCH 'a b c'} {
  p {3 3}
  c {2 2}
  x {
    {1 1 1   0 1 1   1 1 1   0 1 1   1 1 1   0 1 1}
    {0 1 1   1 1 1   0 1 1   1 1 1   0 1 1   1 1 1}
  }
  n {2 2}
  l {{5 5} {5 5}}
  a {{5 5} {5 5}}

  s {{3 0} {0 3}}

  xxxxxxxxxxxxxxxxxx - pcx - xpc - ccc - pppxpcpcx - laxnpc -
  xpxsscplax -
}

do_matchinfo_test 4.1.2 t4 {t4 MATCH '"g h i"'} {
  p {1 1}
  c {2 2}
  x {
    {0 1 1   1 1 1}
    {1 1 1   0 1 1}
  }
  n {2 2}
  l {{5 5} {5 5}}
  a {{5 5} {5 5}}

  s {{0 1} {1 0}}

  xxxxxxxxxxxxxxxxxx - pcx - xpc - ccc - pppxpcpcx - laxnpc -
  sxsxs -
}

do_matchinfo_test 4.1.3 t4 {t4 MATCH 'a b'}     { s {{2 0} {0 2}} }
do_matchinfo_test 4.1.4 t4 {t4 MATCH '"a b" c'} { s {{2 0} {0 2}} }
do_matchinfo_test 4.1.5 t4 {t4 MATCH 'a "b c"'} { s {{2 0} {0 2}} }
do_matchinfo_test 4.1.6 t4 {t4 MATCH 'd d'}     { s {{1 0} {0 1}} }

do_execsql_test 4.2.0 {
  CREATE VIRTUAL TABLE t5 USING fts4;
  INSERT INTO t5 VALUES('a a a a a');
  INSERT INTO t5 VALUES('a b a b a');
  INSERT INTO t5 VALUES('c b c b c');
  INSERT INTO t5 VALUES('x x x x x');
}
do_matchinfo_test 4.2.1 t5 {t5 MATCH 'a a'}         { 
  x {{5 8 2   5 8 2} {3 8 2   3 8 2}}
  s {2 1} 
}
do_matchinfo_test 4.2.2 t5 {t5 MATCH 'a b'}         { s {2} }
do_matchinfo_test 4.2.3 t5 {t5 MATCH 'a b a'}       { s {3} }
do_matchinfo_test 4.2.4 t5 {t5 MATCH 'a a a'}       { s {3 1} }
do_matchinfo_test 4.2.5 t5 {t5 MATCH '"a b" "a b"'} { s {2} }
do_matchinfo_test 4.2.6 t5 {t5 MATCH 'a OR b'}      { s {1 2 1} }

do_execsql_test 4.3.0 "INSERT INTO t5 VALUES('x y [string repeat {b } 50000]')";

do_matchinfo_test 4.3.1 t5 {t5 MATCH 'a a'} { 
  x {{5 8 2   5 5 5} {3 8 2   3 5 5}}
  s {2 1} 
}

do_matchinfo_test 4.3.2 t5 {t5 MATCH 'a b'}         { s {2} }
do_matchinfo_test 4.3.3 t5 {t5 MATCH 'a b a'}       { s {3} }
do_matchinfo_test 4.3.4 t5 {t5 MATCH 'a a a'}       { s {3 1} }
do_matchinfo_test 4.3.5 t5 {t5 MATCH '"a b" "a b"'} { s {2} }
do_matchinfo_test 4.3.6 t5 {t5 MATCH 'a OR b'}      { s {1 2 1 1} }

do_execsql_test 4.4.0 {
  INSERT INTO t5(t5) VALUES('optimize');
  UPDATE t5_segments 
  SET block = zeroblob(length(block)) 
  WHERE length(block)>10000;
}

do_matchinfo_test 4.4.2 t5 {t5 MATCH 'a b'}         { s {2} }
do_matchinfo_test 4.4.1 t5 {t5 MATCH 'a a'}         { s {2 1} }
do_matchinfo_test 4.4.2 t5 {t5 MATCH 'a b'}         { s {2} }
do_matchinfo_test 4.4.3 t5 {t5 MATCH 'a b a'}       { s {3} }
do_matchinfo_test 4.4.4 t5 {t5 MATCH 'a a a'}       { s {3 1} }
do_matchinfo_test 4.4.5 t5 {t5 MATCH '"a b" "a b"'} { s {2} }

do_execsql_test 4.5.0 {
  CREATE VIRTUAL TABLE t6 USING fts4(a, b, c);
  INSERT INTO t6 VALUES('a', 'b', 'c');
}
do_matchinfo_test 4.5.1 t6 {t6 MATCH 'a b c'}       { s {{1 1 1}} }


#-------------------------------------------------------------------------
# Check the following restrictions:
#
#   + Matchinfo flags 'a', 'l' and 'n' can only be used with fts4, not fts3.
#   + Matchinfo flag 'l' cannot be used with matchinfo=fts3.
#
do_execsql_test 5.1 {
  CREATE VIRTUAL TABLE t7 USING fts3(a, b);
  INSERT INTO t7 VALUES('u v w', 'x y z');

  CREATE VIRTUAL TABLE t8 USING fts4(a, b, matchinfo=fts3);
  INSERT INTO t8 VALUES('u v w', 'x y z');
}

do_catchsql_test 5.2.1 { 
  SELECT matchinfo(t7, 'a') FROM t7 WHERE t7 MATCH 'x y'
} {1 {unrecognized matchinfo request: a}}
do_catchsql_test 5.2.2 { 
  SELECT matchinfo(t7, 'l') FROM t7 WHERE t7 MATCH 'x y'
} {1 {unrecognized matchinfo request: l}}
do_catchsql_test 5.2.3 { 
  SELECT matchinfo(t7, 'n') FROM t7 WHERE t7 MATCH 'x y'
} {1 {unrecognized matchinfo request: n}}

do_catchsql_test 5.3.1 { 
  SELECT matchinfo(t8, 'l') FROM t8 WHERE t8 MATCH 'x y'
} {1 {unrecognized matchinfo request: l}}

#-------------------------------------------------------------------------
# Test that the offsets() function handles corruption in the %_content
# table correctly.
#
do_execsql_test 6.1 {
  CREATE VIRTUAL TABLE t9 USING fts4;
  INSERT INTO t9 VALUES(
    'this record is used to try to dectect corruption'
  );
  SELECT offsets(t9) FROM t9 WHERE t9 MATCH 'to';
} {{0 0 20 2 0 0 27 2}}

do_catchsql_test 6.2 {
  UPDATE t9_content SET c0content = 'this record is used to'; 
  SELECT offsets(t9) FROM t9 WHERE t9 MATCH 'to';
} {1 {database disk image is malformed}}

finish_test

Changes to test/fts3query.test.

148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176




177

178
179
180
181
182
183
184
148
149
150
151
152
153
154


155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187







-
-




















+
+
+
+

+







  CREATE VIRTUAL TABLE t2 USING FTS4;
  INSERT INTO t2 VALUES('it was the first time in history');
}
do_select_tests 5.2 -errorformat {
  wrong number of arguments to function %s()
} {
  1 "SELECT matchinfo() FROM t2 WHERE t2 MATCH 'history'"       matchinfo
  2 "SELECT matchinfo(t2, t2) FROM t2 WHERE t2 MATCH 'history'" matchinfo

  3 "SELECT snippet(t2, 1, 2, 3, 4, 5, 6) FROM t2 WHERE t2 MATCH 'history'" 
    snippet
}
do_select_tests 5.3 -errorformat {
  illegal first argument to %s
} {
  1 "SELECT matchinfo(content) FROM t2 WHERE t2 MATCH 'history'" matchinfo
  2 "SELECT offsets(content) FROM t2 WHERE t2 MATCH 'history'"   offsets
  3 "SELECT snippet(content) FROM t2 WHERE t2 MATCH 'history'"   snippet
  4 "SELECT optimize(content) FROM t2 WHERE t2 MATCH 'history'"  optimize
}
do_execsql_test 5.4.0 { UPDATE t2_content SET c0content = X'1234' }
do_select_tests 5.4 -errorformat {
  illegal first argument to %s
} {
  1 "SELECT matchinfo(content) FROM t2 WHERE t2 MATCH 'history'" matchinfo
  2 "SELECT offsets(content) FROM t2 WHERE t2 MATCH 'history'"   offsets
  3 "SELECT snippet(content) FROM t2 WHERE t2 MATCH 'history'"   snippet
  4 "SELECT optimize(content) FROM t2 WHERE t2 MATCH 'history'"  optimize
}
do_catchsql_test 5.5.1 {
  SELECT matchinfo(t2, 'abc') FROM t2 WHERE t2 MATCH 'history'
} {1 {unrecognized matchinfo request: b}}

do_execsql_test 5.5 { DROP TABLE t2 }


# Test the snippet() function with 1 to 6 arguments.
# 
do_execsql_test 6.1 {
  CREATE VIRTUAL TABLE t3 USING FTS4(a, b);
  INSERT INTO t3 VALUES('no gestures', 'another intriguing discovery by observing the hand gestures (called beats) people make while speaking. Research has shown that such gestures do more than add visual emphasis to our words (many people gesture while they''re on the telephone, for example); it seems they actually help our brains find words');
}