/ Changes On Branch fts-languageid-bits
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Changes In Branch fts-languageid-bits Excluding Merge-Ins

This is equivalent to a diff from 2b2ade9278 to 949425d467

2013-06-20
19:55
Add tests for modifying the docid and languageid fields of an fts table with a non-zero languageid_bits field. (Closed-Leaf check-in: 949425d467 user: dan tags: fts-languageid-bits)
18:32
Fix some issues related to ORDER BY and fts tables with a non-zero languageid_bits setting. (check-in: 81527768ef user: dan tags: fts-languageid-bits)
14:07
Disable posix_fallocate() for all systems, all the time, unless the HAVE_POSIX_FALLOCATE compile-time macro is supplied. (check-in: b674462243 user: drh tags: trunk)
2013-06-19
23:48
Merge in trunk changes to os_unix.c that allow the code to build on unix platforms that lack posix_fallocate(). (check-in: bf5764067a user: drh tags: nextgen-query-plan-exp)
20:13
Add the languageid_bits= option to fts. Still some problems to work out. (check-in: d36d7e6833 user: dan tags: fts-languageid-bits)
14:49
Only default HAVE_POSIX_FALLOCATE on for linux, and then only if it is not previously defined. (check-in: 2b2ade9278 user: drh tags: trunk)
14:28
Only enable posix_fallocate by default on linux and mac. (check-in: b9b30d4f98 user: drh tags: trunk)

Changes to ext/fts3/fts3.c.

1077
1078
1079
1080
1081
1082
1083

1084
1085
1086
1087
1088
1089
1090
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091







+







  int bNoDocsize = 0;             /* True to omit %_docsize table */
  int bDescIdx = 0;               /* True to store descending indexes */
  char *zPrefix = 0;              /* Prefix parameter value (or NULL) */
  char *zCompress = 0;            /* compress=? parameter (or NULL) */
  char *zUncompress = 0;          /* uncompress=? parameter (or NULL) */
  char *zContent = 0;             /* content=? parameter (or NULL) */
  char *zLanguageid = 0;          /* languageid=? parameter (or NULL) */
  char *zLanguageidBits = 0;      /* languageid_bits=? parameter (or NULL) */

  assert( strlen(argv[0])==4 );
  assert( (sqlite3_strnicmp(argv[0], "fts4", 4)==0 && isFts4)
       || (sqlite3_strnicmp(argv[0], "fts3", 4)==0 && !isFts4)
  );

  nDb = (int)strlen(argv[1]) + 1;
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134








1135
1136
1137
1138
1139
1140
1141
1122
1123
1124
1125
1126
1127
1128







1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143







-
-
-
-
-
-
-
+
+
+
+
+
+
+
+








    /* Check if it is an FTS4 special argument. */
    else if( isFts4 && fts3IsSpecialColumn(z, &nKey, &zVal) ){
      struct Fts4Option {
        const char *zOpt;
        int nOpt;
      } aFts4Opt[] = {
        { "matchinfo",   9 },     /* 0 -> MATCHINFO */
        { "prefix",      6 },     /* 1 -> PREFIX */
        { "compress",    8 },     /* 2 -> COMPRESS */
        { "uncompress", 10 },     /* 3 -> UNCOMPRESS */
        { "order",       5 },     /* 4 -> ORDER */
        { "content",     7 },     /* 5 -> CONTENT */
        { "languageid", 10 }      /* 6 -> LANGUAGEID */
        { "matchinfo",        9 },     /* 0 -> MATCHINFO */
        { "prefix",           6 },     /* 1 -> PREFIX */
        { "compress",         8 },     /* 2 -> COMPRESS */
        { "uncompress",      10 },     /* 3 -> UNCOMPRESS */
        { "order",            5 },     /* 4 -> ORDER */
        { "content",          7 },     /* 5 -> CONTENT */
        { "languageid",      10 },     /* 6 -> LANGUAGEID */
        { "languageid_bits", 15 }      /* 7 -> LANGUAGEID_BITS */
      };

      int iOpt;
      if( !zVal ){
        rc = SQLITE_NOMEM;
      }else{
        for(iOpt=0; iOpt<SizeofArray(aFts4Opt); iOpt++){
1193
1194
1195
1196
1197
1198
1199







1200
1201
1202
1203
1204
1205
1206
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215







+
+
+
+
+
+
+








            case 6:              /* LANGUAGEID */
              assert( iOpt==6 );
              sqlite3_free(zLanguageid);
              zLanguageid = zVal;
              zVal = 0;
              break;

            case 7:              /* LANGUAGEID_BITS */
              assert( iOpt==7 );
              sqlite3_free(zLanguageidBits);
              zLanguageidBits = zVal;
              zVal = 0;
              break;
          }
        }
        sqlite3_free(zVal);
      }
    }

    /* Otherwise, the argument is a column name. */
1288
1289
1290
1291
1292
1293
1294









1295
1296
1297
1298
1299
1300
1301
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319







+
+
+
+
+
+
+
+
+







  p->bFts4 = isFts4;
  p->bDescIdx = bDescIdx;
  p->bAutoincrmerge = 0xff;   /* 0xff means setting unknown */
  p->zContentTbl = zContent;
  p->zLanguageid = zLanguageid;
  zContent = 0;
  zLanguageid = 0;
  if( zLanguageidBits && p->zLanguageid && p->zContentTbl==0 ){
    p->nLanguageidBits = atoi(zLanguageidBits);
    if( p->nLanguageidBits>30 || p->nLanguageidBits<0 ){
      rc = SQLITE_ERROR;
      *pzErr = sqlite3_mprintf("languageid_bits parameter out of range");
      goto fts3_init_out;
    }
  }

  TESTONLY( p->inTransaction = -1 );
  TESTONLY( p->mxSavepoint = -1 );

  p->aIndex = (struct Fts3Index *)&p->azColumn[nCol];
  memcpy(p->aIndex, aIndex, sizeof(struct Fts3Index) * nIndex);
  p->nIndex = nIndex;
  for(i=0; i<nIndex; i++){
1361
1362
1363
1364
1365
1366
1367

1368
1369
1370
1371
1372
1373
1374
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393







+







fts3_init_out:
  sqlite3_free(zPrefix);
  sqlite3_free(aIndex);
  sqlite3_free(zCompress);
  sqlite3_free(zUncompress);
  sqlite3_free(zContent);
  sqlite3_free(zLanguageid);
  sqlite3_free(zLanguageidBits);
  sqlite3_free((void *)aCol);
  if( rc!=SQLITE_OK ){
    if( p ){
      fts3DisconnectMethod((sqlite3_vtab *)p);
    }else if( pTokenizer ){
      pTokenizer->pModule->xDestroy(pTokenizer);
    }
1422
1423
1424
1425
1426
1427
1428

1429
1430
1431
1432
1433
1434


1435
1436
1437
1438
1439
1440
1441
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451



1452
1453
1454
1455
1456
1457
1458
1459
1460







+



-
-
-
+
+







  ** so search through the constraints to see if a more efficient 
  ** strategy is possible.
  */
  pInfo->idxNum = FTS3_FULLSCAN_SEARCH;
  pInfo->estimatedCost = 500000;
  for(i=0; i<pInfo->nConstraint; i++){
    struct sqlite3_index_constraint *pCons = &pInfo->aConstraint[i];
    int iCol = pCons->iColumn;
    if( pCons->usable==0 ) continue;

    /* A direct lookup on the rowid or docid column. Assign a cost of 1.0. */
    if( iCons<0 
     && pCons->op==SQLITE_INDEX_CONSTRAINT_EQ 
     && (pCons->iColumn<0 || pCons->iColumn==p->nColumn+1 )
    if( iCons<0 && pCons->op==SQLITE_INDEX_CONSTRAINT_EQ 
     && (iCol<0 || (iCol==p->nColumn+1 && p->nLanguageidBits==0))
    ){
      pInfo->idxNum = FTS3_DOCID_SEARCH;
      pInfo->estimatedCost = 1.0;
      iCons = i;
    }

    /* A MATCH constraint. Use a full-text search.
1470
1471
1472
1473
1474
1475
1476

1477
1478
1479




1480
1481
1482
1483
1484
1485
1486
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498

1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509







+


-
+
+
+
+







  if( iLangidCons>=0 ){
    pInfo->aConstraintUsage[iLangidCons].argvIndex = 2;
  } 

  /* Regardless of the strategy selected, FTS can deliver rows in rowid (or
  ** docid) order. Both ascending and descending are possible. 
  */
  assert( pInfo->orderByConsumed==0 );
  if( pInfo->nOrderBy==1 ){
    struct sqlite3_index_orderby *pOrder = &pInfo->aOrderBy[0];
    if( pOrder->iColumn<0 || pOrder->iColumn==p->nColumn+1 ){
    if( pOrder->iColumn<0 || (
          (pOrder->iColumn==p->nColumn+1)
       && (pInfo->idxNum>=FTS3_FULLTEXT_SEARCH || p->nLanguageidBits==0)
    )){
      if( pOrder->desc ){
        pInfo->idxStr = "DESC";
      }else{
        pInfo->idxStr = "ASC";
      }
      pInfo->orderByConsumed = 1;
    }
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068




3069
3070
3071
3072
3073
3074
3075
3081
3082
3083
3084
3085
3086
3087




3088
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098







-
-
-
-
+
+
+
+







  Fts3Cursor *pCsr = (Fts3Cursor *) pCursor;
  Fts3Table *p = (Fts3Table *)pCursor->pVtab;

  /* The column value supplied by SQLite must be in range. */
  assert( iCol>=0 && iCol<=p->nColumn+2 );

  if( iCol==p->nColumn+1 ){
    /* This call is a request for the "docid" column. Since "docid" is an 
    ** alias for "rowid", use the xRowid() method to obtain the value.
    */
    sqlite3_result_int64(pCtx, pCsr->iPrevId);
    /* This call is a request for the "docid" column. The value currently
    ** stored in pCsr->iPrevId is a rowid. Transform this to a docid and
    ** return it.  */
    sqlite3_result_int64(pCtx, sqlite3Fts3RowidToDocid(p, pCsr->iPrevId));
  }else if( iCol==p->nColumn ){
    /* The extra column whose name is the same as the table.
    ** Return a blob which is a pointer to the cursor.  */
    sqlite3_result_blob(pCtx, &pCsr, sizeof(pCsr), SQLITE_TRANSIENT);
  }else if( iCol==p->nColumn+2 && pCsr->pExpr ){
    sqlite3_result_int64(pCtx, pCsr->iLangid);
  }else{

Changes to ext/fts3/fts3Int.h.

205
206
207
208
209
210
211

212
213
214
215
216
217
218
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219







+







  const char *zDb;                /* logical database name */
  const char *zName;              /* virtual table name */
  int nColumn;                    /* number of named columns in virtual table */
  char **azColumn;                /* column names.  malloced */
  sqlite3_tokenizer *pTokenizer;  /* tokenizer for inserts and queries */
  char *zContentTbl;              /* content=xxx option, or NULL */
  char *zLanguageid;              /* languageid=xxx option, or NULL */
  int nLanguageidBits;            /* languageid_bits=N option, or 0 */
  u8 bAutoincrmerge;              /* True if automerge=1 */
  u32 nLeafAdd;                   /* Number of leaf blocks added this trans */

  /* Precompiled statements used by the implementation. Each of these 
  ** statements is run and reset within a single virtual table API call. 
  */
  sqlite3_stmt *aStmt[37];
417
418
419
420
421
422
423
424
425


426
427
428
429
430
431
432
418
419
420
421
422
423
424

425
426
427
428
429
430
431
432
433
434







-

+
+







*/
#define FTSQUERY_NEAR   1
#define FTSQUERY_NOT    2
#define FTSQUERY_AND    3
#define FTSQUERY_OR     4
#define FTSQUERY_PHRASE 5


/* fts3_write.c */
i64 sqlite3Fts3DocidToRowid(Fts3Table *p, i64 iDocid, int iLangid);
i64 sqlite3Fts3RowidToDocid(Fts3Table *p, i64 iRowid);
int sqlite3Fts3UpdateMethod(sqlite3_vtab*,int,sqlite3_value**,sqlite3_int64*);
int sqlite3Fts3PendingTermsFlush(Fts3Table *);
void sqlite3Fts3PendingTermsClear(Fts3Table *);
int sqlite3Fts3Optimize(Fts3Table *);
int sqlite3Fts3SegReaderNew(int, int, sqlite3_int64,
  sqlite3_int64, sqlite3_int64, const char *, int, Fts3SegReader**);
int sqlite3Fts3SegReaderPending(

Changes to ext/fts3/fts3_write.c.

483
484
485
486
487
488
489


















490
491
492
493
494
495
496
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514







+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+







  if( rc==SQLITE_OK ){
    sqlite3_step(pStmt);
    rc = sqlite3_reset(pStmt);
  }
  *pRC = rc;
}

static void fts3SqlExecI64(
  int *pRC,                /* Result code */
  Fts3Table *p,            /* The FTS3 table */
  int eStmt,               /* Index of statement to evaluate */
  i64 iVal
){
  sqlite3_stmt *pStmt;
  int rc;
  if( *pRC ) return;
  rc = fts3SqlStmt(p, eStmt, &pStmt, 0); 
  if( rc==SQLITE_OK ){
    sqlite3_bind_int64(pStmt, 1, iVal);
    sqlite3_step(pStmt);
    rc = sqlite3_reset(pStmt);
  }
  *pRC = rc;
}


/*
** This function ensures that the caller has obtained an exclusive 
** shared-cache table-lock on the %_segdir table. This is required before 
** writing data to the fts3 table. If this lock is not acquired first, then
** the caller may end up attempting to take this lock as part of committing
** a transaction, causing SQLite to return SQLITE_LOCKED or 
923
924
925
926
927
928
929
930


931
932
933
934
935
936



937
938
939
940
941
942
943

944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960










961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977






978
979
980
981
982
983
984
985

986
987
988
989
990

991


992
993
994
995
996
997
998
941
942
943
944
945
946
947

948
949
950
951
952
953
954

955
956
957
958
959
960
961
962
963

964
965
966
967
968
969
970
971
972
973
974
975
976





977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999




1000
1001
1002
1003
1004
1005



1006
1007
1008
1009

1010
1011
1012
1013
1014
1015
1016

1017
1018
1019
1020
1021
1022
1023
1024
1025







-
+
+





-
+
+
+






-
+












-
-
-
-
-
+
+
+
+
+
+
+
+
+
+













-
-
-
-
+
+
+
+
+
+
-
-
-




-
+





+
-
+
+







**   apVal[p->nColumn+2]     Hidden column with same name as table
**   apVal[p->nColumn+3]     Hidden "docid" column (alias for rowid)
**   apVal[p->nColumn+4]     Hidden languageid column
*/
static int fts3InsertData(
  Fts3Table *p,                   /* Full-text table */
  sqlite3_value **apVal,          /* Array of values to insert */
  sqlite3_int64 *piDocid          /* OUT: Docid for row just inserted */
  sqlite3_int64 *piRowid,         /* OUT: Rowid for row just inserted */
  i64 iRowid                      /* Explicit rowid, if piRowid==NULL */
){
  int rc;                         /* Return code */
  sqlite3_stmt *pContentInsert;   /* INSERT INTO %_content VALUES(...) */

  if( p->zContentTbl ){
    sqlite3_value *pRowid = apVal[p->nColumn+3];
    sqlite3_value *pRowid;
    assert( p->nLanguageidBits==0 && piRowid );
    pRowid = apVal[p->nColumn+3];
    if( sqlite3_value_type(pRowid)==SQLITE_NULL ){
      pRowid = apVal[1];
    }
    if( sqlite3_value_type(pRowid)!=SQLITE_INTEGER ){
      return SQLITE_CONSTRAINT;
    }
    *piDocid = sqlite3_value_int64(pRowid);
    *piRowid = sqlite3_value_int64(pRowid);
    return SQLITE_OK;
  }

  /* Locate the statement handle used to insert data into the %_content
  ** table. The SQL for this statement is:
  **
  **   INSERT INTO %_content VALUES(?, ?, ?, ...)
  **
  ** The statement features N '?' variables, where N is the number of user
  ** defined columns in the FTS3 table, plus one for the docid field.
  */
  rc = fts3SqlStmt(p, SQL_CONTENT_INSERT, &pContentInsert, &apVal[1]);
  if( rc==SQLITE_OK && p->zLanguageid ){
    rc = sqlite3_bind_int(
        pContentInsert, p->nColumn+2, 
        sqlite3_value_int(apVal[p->nColumn+4])
    );
  if( rc==SQLITE_OK ){
    if( piRowid==0 ){
      sqlite3_bind_int64(pContentInsert, 1, iRowid);
    }
    if( p->zLanguageid ){
      rc = sqlite3_bind_int(
          pContentInsert, p->nColumn+2, 
          sqlite3_value_int(apVal[p->nColumn+4])
      );
    }
  }
  if( rc!=SQLITE_OK ) return rc;

  /* There is a quirk here. The users INSERT statement may have specified
  ** a value for the "rowid" field, for the "docid" field, or for both.
  ** Which is a problem, since "rowid" and "docid" are aliases for the
  ** same value. For example:
  **
  **   INSERT INTO fts3tbl(rowid, docid) VALUES(1, 2);
  **
  ** In FTS3, this is an error. It is an error to specify non-NULL values
  ** for both docid and some other rowid alias.
  */
  if( SQLITE_NULL!=sqlite3_value_type(apVal[3+p->nColumn]) ){
    if( SQLITE_NULL==sqlite3_value_type(apVal[0])
     && SQLITE_NULL!=sqlite3_value_type(apVal[1])
    ){
  assert( p->nLanguageidBits==0 || piRowid==0
      || sqlite3_value_type(apVal[1])!=SQLITE_NULL 
  );
  if( piRowid && p->nLanguageidBits==0 
   && SQLITE_NULL!=sqlite3_value_type(apVal[3+p->nColumn]) 
  ){
      /* A rowid/docid conflict. */
      return SQLITE_ERROR;
    }
    rc = sqlite3_bind_value(pContentInsert, 1, apVal[3+p->nColumn]);
    if( rc!=SQLITE_OK ) return rc;
  }

  /* Execute the statement to insert the record. Set *piDocid to the 
  /* Execute the statement to insert the record. Set *pRowid to the 
  ** new docid value. 
  */
  sqlite3_step(pContentInsert);
  rc = sqlite3_reset(pContentInsert);

  if( piRowid ){
  *piDocid = sqlite3_last_insert_rowid(p->db);
    *piRowid = sqlite3_last_insert_rowid(p->db);
  }
  return rc;
}



/*
** Remove all data from the FTS3 table. Clear the hash table containing
1032
1033
1034
1035
1036
1037
1038
1039

1040
1041
1042
1043
1044
1045
1046
1047
1048

1049

1050
1051
1052
1053
1054
1055
1056
1059
1060
1061
1062
1063
1064
1065

1066
1067
1068
1069
1070
1071
1072
1073
1074

1075
1076
1077
1078
1079
1080
1081
1082
1083
1084







-
+








-
+

+







** The first element in the apVal[] array is assumed to contain the docid
** (an integer) of a row about to be deleted. Remove all terms from the
** full-text index.
*/
static void fts3DeleteTerms( 
  int *pRC,               /* Result code */
  Fts3Table *p,           /* The FTS table to delete from */
  sqlite3_value *pRowid,  /* The docid to be deleted */
  i64 iRowid,             /* The rowid to be deleted */
  u32 *aSz,               /* Sizes of deleted document written here */
  int *pbFound            /* OUT: Set to true if row really does exist */
){
  int rc;
  sqlite3_stmt *pSelect;

  assert( *pbFound==0 );
  if( *pRC ) return;
  rc = fts3SqlStmt(p, SQL_SELECT_CONTENT_BY_ROWID, &pSelect, &pRowid);
  rc = fts3SqlStmt(p, SQL_SELECT_CONTENT_BY_ROWID, &pSelect, 0);
  if( rc==SQLITE_OK ){
    sqlite3_bind_int64(pSelect, 1, iRowid);
    if( SQLITE_ROW==sqlite3_step(pSelect) ){
      int i;
      int iLangid = langidFromSelect(p, pSelect);
      rc = fts3PendingTermsDocid(p, iLangid, sqlite3_column_int64(pSelect, 0));
      for(i=1; rc==SQLITE_OK && i<=p->nColumn; i++){
        const char *zText = (const char *)sqlite3_column_text(pSelect, i);
        rc = fts3PendingTermsAdd(p, iLangid, zText, -1, &aSz[i-1]);
2342
2343
2344
2345
2346
2347
2348
2349

2350
2351
2352
2353
2354
2355
2356
2357

2358

2359
2360
2361
2362
2363
2364
2365
2370
2371
2372
2373
2374
2375
2376

2377
2378
2379
2380
2381
2382
2383
2384

2385
2386
2387
2388
2389
2390
2391
2392
2393
2394







-
+







-
+

+







** are different from that integer. i.e. if deleting the document with docid
** pRowid would mean the FTS3 table were empty.
**
** If successful, *pisEmpty is set to true if the table is empty except for
** document pRowid, or false otherwise, and SQLITE_OK is returned. If an
** error occurs, an SQLite error code is returned.
*/
static int fts3IsEmpty(Fts3Table *p, sqlite3_value *pRowid, int *pisEmpty){
static int fts3IsEmpty(Fts3Table *p, i64 iRowid, int *pisEmpty){
  sqlite3_stmt *pStmt;
  int rc;
  if( p->zContentTbl ){
    /* If using the content=xxx option, assume the table is never empty */
    *pisEmpty = 0;
    rc = SQLITE_OK;
  }else{
    rc = fts3SqlStmt(p, SQL_IS_EMPTY, &pStmt, &pRowid);
    rc = fts3SqlStmt(p, SQL_IS_EMPTY, &pStmt, 0);
    if( rc==SQLITE_OK ){
      sqlite3_bind_int64(pStmt, 1, iRowid);
      if( SQLITE_ROW==sqlite3_step(pStmt) ){
        *pisEmpty = sqlite3_column_int(pStmt, 0);
      }
      rc = sqlite3_reset(pStmt);
    }
  }
  return rc;
5193
5194
5195
5196
5197
5198
5199
5200

5201
5202
5203
5204
5205
5206
5207

5208
5209
5210

5211
5212
5213
5214
5215
5216
5217
5218
5219
5220
5221
5222

5223
5224
5225

5226
5227
5228
5229
5230
5231
5232



















































5233
5234
5235
5236
5237
5238
5239
5240
5241
5242
5243
5244
5245
5246
5247
5248
5249
5250
5251
5252
5253
5254
5255
5256
5257
5258
5259

5260
5261
5262
5263
5264
5265
5266
5267
5268
5269
5270
5271
5272
5273
5274
5275
5276
5277
5278









5279
5280
5281
















5282
5283
5284
5285
5286
5287
5288
5222
5223
5224
5225
5226
5227
5228

5229
5230
5231
5232
5233
5234
5235

5236
5237
5238

5239
5240
5241
5242
5243
5244
5245
5246
5247
5248
5249
5250

5251
5252
5253

5254
5255
5256
5257
5258
5259
5260
5261
5262
5263
5264
5265
5266
5267
5268
5269
5270
5271
5272
5273
5274
5275
5276
5277
5278
5279
5280
5281
5282
5283
5284
5285
5286
5287
5288
5289
5290
5291
5292
5293
5294
5295
5296
5297
5298
5299
5300
5301
5302
5303
5304
5305
5306
5307
5308
5309
5310
5311
5312
5313
5314
5315
5316
5317
5318
5319
5320
5321
5322
5323
5324

5325
5326
5327
5328
5329
5330
5331
5332
5333
5334
5335
5336
5337
5338
5339
5340
5341
5342
5343
5344
5345
5346
5347
5348
5349
5350
5351
5352
5353
5354
5355
5356
5357
5358
5359
5360
5361
5362
5363
5364
5365
5366
5367



5368
5369
5370
5371
5372
5373
5374
5375
5376
5377
5378
5379
5380
5381
5382
5383
5384
5385
5386
5387
5388
5389
5390







-
+






-
+


-
+











-
+


-
+







+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+












-














+



















+
+
+
+
+
+
+
+
+
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+







/*
** SQLite value pRowid contains the rowid of a row that may or may not be
** present in the FTS3 table. If it is, delete it and adjust the contents
** of subsiduary data structures accordingly.
*/
static int fts3DeleteByRowid(
  Fts3Table *p, 
  sqlite3_value *pRowid, 
  i64 iRowid,
  int *pnChng,                    /* IN/OUT: Decrement if row is deleted */
  u32 *aSzDel
){
  int rc = SQLITE_OK;             /* Return code */
  int bFound = 0;                 /* True if *pRowid really is in the table */

  fts3DeleteTerms(&rc, p, pRowid, aSzDel, &bFound);
  fts3DeleteTerms(&rc, p, iRowid, aSzDel, &bFound);
  if( bFound && rc==SQLITE_OK ){
    int isEmpty = 0;              /* Deleting *pRowid leaves the table empty */
    rc = fts3IsEmpty(p, pRowid, &isEmpty);
    rc = fts3IsEmpty(p, iRowid, &isEmpty);
    if( rc==SQLITE_OK ){
      if( isEmpty ){
        /* Deleting this row means the whole table is empty. In this case
        ** delete the contents of all three tables and throw away any
        ** data in the pendingTerms hash table.  */
        rc = fts3DeleteAll(p, 1);
        *pnChng = 0;
        memset(aSzDel, 0, sizeof(u32) * (p->nColumn+1) * 2);
      }else{
        *pnChng = *pnChng - 1;
        if( p->zContentTbl==0 ){
          fts3SqlExec(&rc, p, SQL_DELETE_CONTENT, &pRowid);
          fts3SqlExecI64(&rc, p, SQL_DELETE_CONTENT, iRowid);
        }
        if( p->bHasDocsize ){
          fts3SqlExec(&rc, p, SQL_DELETE_DOCSIZE, &pRowid);
          fts3SqlExecI64(&rc, p, SQL_DELETE_DOCSIZE, iRowid);
        }
      }
    }
  }

  return rc;
}

/*
** Convert a docid (iDocid) and a language id (iLangid) to a rowid,
** according to the configured languageid_bits= value belonging to
** FTS table *p.
**
** The conversion is as follows:
**
**   * The sign bit of iDocid becomes the sign bit of the rowid.
**
**   * iLangid is converted to an unsigned integer and stored in
**     the next most significant Fts3Table.nLanguageidBits bits
**     of the returned rowid.
**
**   * The least signficant (63-nLanguageidBits) of iDocid are
**     copied to the (63-nLanguageidBits) least signifcant bits of
**     the returned rowid.
*/
i64 sqlite3Fts3DocidToRowid(Fts3Table *p, i64 iDocid, int iLangid){
  u64 iRet = iDocid;

  if( p->nLanguageidBits ){
    int iShift = (63 - p->nLanguageidBits);
    u64 mask = ((((u64)1 << p->nLanguageidBits) - 1) << iShift);

    iRet &= ~mask;
    iRet |= (u64)iLangid << iShift;
  }

  assert( sqlite3Fts3RowidToDocid(p, (i64)iRet)==iDocid );
  return (i64)iRet;
}

/*
** Convert a rowid (iRowid) to a docid according to the languageid_bits=
** value belonging to FTS table *p.
*/
i64 sqlite3Fts3RowidToDocid(Fts3Table *p, i64 iRowid){
  u64 iRet = iRowid;
  if( p->nLanguageidBits ){
    static const u64 signbit = ((u64)1 << 63);
    u64 mask = ((((u64)1 << p->nLanguageidBits)-1) << (63-p->nLanguageidBits));

    if( iRet & signbit ){
      iRet |= mask;
    }else{
      iRet &= ~mask;
    }
  }
  return (i64)iRet;
}

/*
** This function does the work for the xUpdate method of FTS3 virtual
** tables. The schema of the virtual table being:
**
**     CREATE TABLE <table name>( 
**       <user columns>,
**       <table name> HIDDEN, 
**       docid HIDDEN, 
**       <langid> HIDDEN
**     );
**
** 
*/
int sqlite3Fts3UpdateMethod(
  sqlite3_vtab *pVtab,            /* FTS3 vtab object */
  int nArg,                       /* Size of argument array */
  sqlite3_value **apVal,          /* Array of arguments */
  sqlite_int64 *pRowid            /* OUT: The affected (or effected) rowid */
){
  Fts3Table *p = (Fts3Table *)pVtab;
  int rc = SQLITE_OK;             /* Return Code */
  int isRemove = 0;               /* True for an UPDATE or DELETE */
  u32 *aSzIns = 0;                /* Sizes of inserted documents */
  u32 *aSzDel = 0;                /* Sizes of deleted documents */
  int nChng = 0;                  /* Net change in number of documents */
  int bInsertDone = 0;
  int iLangid = 0;

  assert( p->pSegments==0 );
  assert( 
      nArg==1                     /* DELETE operations */
   || nArg==(2 + p->nColumn + 3)  /* INSERT or UPDATE operations */
  );

  /* Check for a "special" INSERT operation. One of the form:
  **
  **   INSERT INTO xyz(xyz) VALUES('command');
  */
  if( nArg>1 
   && sqlite3_value_type(apVal[0])==SQLITE_NULL 
   && sqlite3_value_type(apVal[p->nColumn+2])!=SQLITE_NULL 
  ){
    rc = fts3SpecialInsert(p, apVal[p->nColumn+2]);
    goto update_out;
  }

  /* If this is an INSERT or UPDATE, check that the new value for the
  ** languageid is within range. A languageid can never be a negative 
  ** value. If the languageid_bits option was specified when this table 
  ** was created, it must also be less than (2 ^ nLanguageidBits).  
  **
  ** Also check that if a non-zero languageid_bits value was configured,
  ** the specified rowid value must be NULL.
  */
  if( nArg>1 ){
  if( nArg>1 && sqlite3_value_int(apVal[2 + p->nColumn + 2])<0 ){
    rc = SQLITE_CONSTRAINT;
    goto update_out;
    i64 iLangid64 = sqlite3_value_int64(apVal[2 + p->nColumn + 2]);
    if( iLangid64<0 
     || (p->nLanguageidBits && iLangid64>=((i64)1<<p->nLanguageidBits)) 
    ){
      rc = SQLITE_CONSTRAINT;
      goto update_out;
    }
    iLangid = (int)iLangid64;

    if( p->nLanguageidBits 
     && sqlite3_value_type(apVal[0])==SQLITE_NULL
     && sqlite3_value_type(apVal[1])!=SQLITE_NULL
    ){
      rc = SQLITE_CONSTRAINT;
      goto update_out;
    }
  }

  /* Allocate space to hold the change in document sizes */
  aSzDel = sqlite3_malloc( sizeof(aSzDel[0])*(p->nColumn+1)*2 );
  if( aSzDel==0 ){
    rc = SQLITE_NOMEM;
    goto update_out;
5300
5301
5302
5303
5304
5305
5306
5307
5308
5309












5310
5311
5312
5313
5314
5315
5316
5317
5318
5319
5320
5321
5322
5323
5324
5325
5326
5327
5328
5329
5330
5331
5332
5333
5334
5335
5336
5337




















































5338
5339
5340
5341
5342
5343
5344
5345
5346
5347
5348

5349
5350
5351
5352
5353
5354
5355
5356

5357
5358
5359
5360
5361
5362
5363
5402
5403
5404
5405
5406
5407
5408



5409
5410
5411
5412
5413
5414
5415
5416
5417
5418
5419
5420
5421
5422


























5423
5424
5425
5426
5427
5428
5429
5430
5431
5432
5433
5434
5435
5436
5437
5438
5439
5440
5441
5442
5443
5444
5445
5446
5447
5448
5449
5450
5451
5452
5453
5454
5455
5456
5457
5458
5459
5460
5461
5462
5463
5464
5465
5466
5467
5468
5469
5470
5471
5472
5473
5474
5475
5476
5477
5478
5479
5480
5481
5482
5483
5484

5485
5486
5487
5488
5489
5490

5491

5492
5493
5494
5495
5496
5497
5498
5499







-
-
-
+
+
+
+
+
+
+
+
+
+
+
+


-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+










-
+





-

-
+







  ** should be deleted from the database before inserting the new row. Or,
  ** if the on-conflict mode is other than REPLACE, then this method must
  ** detect the conflict and return SQLITE_CONSTRAINT before beginning to
  ** modify the database file.
  */
  if( nArg>1 && p->zContentTbl==0 ){
    /* Find the value object that holds the new rowid value. */
    sqlite3_value *pNewRowid = apVal[3+p->nColumn];
    if( sqlite3_value_type(pNewRowid)==SQLITE_NULL ){
      pNewRowid = apVal[1];
    sqlite3_value *pNewDocid = apVal[3+p->nColumn];
    if( sqlite3_value_type(pNewDocid)==SQLITE_NULL ){
      if( p->nLanguageidBits ){
        rc = SQLITE_CONSTRAINT;
        goto update_out;
      }
      pNewDocid = apVal[1];
    }else if( sqlite3_value_type(apVal[0])==SQLITE_NULL 
           && sqlite3_value_type(apVal[1])!=SQLITE_NULL 
    ){
      rc = SQLITE_ERROR;
      goto update_out;
    }

    if( sqlite3_value_type(pNewRowid)!=SQLITE_NULL && ( 
        sqlite3_value_type(apVal[0])==SQLITE_NULL
     || sqlite3_value_int64(apVal[0])!=sqlite3_value_int64(pNewRowid)
    )){
      /* The new rowid is not NULL (in this case the rowid will be
      ** automatically assigned and there is no chance of a conflict), and 
      ** the statement is either an INSERT or an UPDATE that modifies the
      ** rowid column. So if the conflict mode is REPLACE, then delete any
      ** existing row with rowid=pNewRowid. 
      **
      ** Or, if the conflict mode is not REPLACE, insert the new record into 
      ** the %_content table. If we hit the duplicate rowid constraint (or any
      ** other error) while doing so, return immediately.
      **
      ** This branch may also run if pNewRowid contains a value that cannot
      ** be losslessly converted to an integer. In this case, the eventual 
      ** call to fts3InsertData() (either just below or further on in this
      ** function) will return SQLITE_MISMATCH. If fts3DeleteByRowid is 
      ** invoked, it will delete zero rows (since no row will have
      ** docid=$pNewRowid if $pNewRowid is not an integer value).
      */
      if( sqlite3_vtab_on_conflict(p->db)==SQLITE_REPLACE ){
        rc = fts3DeleteByRowid(p, pNewRowid, &nChng, aSzDel);
      }else{
        rc = fts3InsertData(p, apVal, pRowid);
        bInsertDone = 1;
    if( sqlite3_value_type(pNewDocid)!=SQLITE_NULL ){
      int e = sqlite3_value_numeric_type(pNewDocid);
      i64 iRowid = sqlite3_value_int64(pNewDocid);

      /* Check that the value specified by the user may be losslessly
      ** converted to an integer. If not, return a "data mismatch" error.  */
      if( (e!=SQLITE_INTEGER)
       && (e!=SQLITE_FLOAT || (double)iRowid!=sqlite3_value_double(pNewDocid))
      ){
        rc = SQLITE_MISMATCH;
        goto update_out;
      }

      if( p->nLanguageidBits ){
        /* Check for an out-of-range docid value. */
        if( iRowid>=((i64)1 << (63 - p->nLanguageidBits)) 
         || iRowid<-1*((i64)1 << (63 - p->nLanguageidBits)) 
        ){
          rc = SQLITE_CONSTRAINT;
          goto update_out;
        }

        iRowid = sqlite3Fts3DocidToRowid(p, iRowid, iLangid);
      }

      if( sqlite3_value_type(apVal[0])==SQLITE_NULL
       || sqlite3_value_int64(apVal[0])!=iRowid
      ){
        /* The new rowid is not NULL (in this case the rowid will be
        ** automatically assigned and there is no chance of a conflict), and 
        ** the statement is either an INSERT or an UPDATE that modifies the
        ** rowid column. So if the conflict mode is REPLACE, then delete any
        ** existing row with rowid=pNewRowid. 
        **
        ** Or, if the conflict mode is not REPLACE, insert the new record into 
        ** the %_content table. If we hit the duplicate rowid constraint (or 
        ** any other error) while doing so, return immediately.
        **
        ** This branch may also run if pNewRowid contains a value that cannot
        ** be losslessly converted to an integer. In this case, the eventual 
        ** call to fts3InsertData() (either just below or further on in this
        ** function) will return SQLITE_MISMATCH. If fts3DeleteByRowid is 
        ** invoked, it will delete zero rows (since no row will have
        ** docid=$pNewRowid if $pNewRowid is not an integer value).
        */
        if( sqlite3_vtab_on_conflict(p->db)==SQLITE_REPLACE ){
          rc = fts3DeleteByRowid(p, iRowid, &nChng, aSzDel);
        }else{
          rc = fts3InsertData(p, apVal, 0, iRowid);
          bInsertDone = 1;
          *pRowid = iRowid;
        }
      }
    }
  }
  if( rc!=SQLITE_OK ){
    goto update_out;
  }

  /* If this is a DELETE or UPDATE operation, remove the old record. */
  if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){
    assert( sqlite3_value_type(apVal[0])==SQLITE_INTEGER );
    rc = fts3DeleteByRowid(p, apVal[0], &nChng, aSzDel);
    rc = fts3DeleteByRowid(p, sqlite3_value_int64(apVal[0]), &nChng, aSzDel);
    isRemove = 1;
  }
  
  /* If this is an INSERT or UPDATE operation, insert the new record. */
  if( nArg>1 && rc==SQLITE_OK ){
    int iLangid = sqlite3_value_int(apVal[2 + p->nColumn + 2]);
    if( bInsertDone==0 ){
      rc = fts3InsertData(p, apVal, pRowid);
      rc = fts3InsertData(p, apVal, pRowid, 0);
      if( rc==SQLITE_CONSTRAINT && p->zContentTbl==0 ){
        rc = FTS_CORRUPT_VTAB;
      }
    }
    if( rc==SQLITE_OK && (!isRemove || *pRowid!=p->iPrevDocid ) ){
      rc = fts3PendingTermsDocid(p, iLangid, *pRowid);
    }

Added test/fts4langid2.test.













































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
# 2012 March 01
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#*************************************************************************
# This file implements regression tests for SQLite library.  The
# focus of this script is testing the languageid=xxx FTS4 option.
#

set testdir [file dirname $argv0]
source $testdir/tester.tcl
set ::testprefix fts4langid2

# If SQLITE_ENABLE_FTS3 is defined, omit this file.
ifcapable !fts3 {
  finish_test
  return
}

#-------------------------------------------------------------------------
# Test out-of-range values for the languageid_bits= parameter.
#
do_catchsql_test 1.1 {
  CREATE VIRTUAL TABLE t1 USING fts4(languageid=lid, languageid_bits=31);
} {1 {languageid_bits parameter out of range}}

do_catchsql_test 1.2 {
  CREATE VIRTUAL TABLE t1 USING fts4(languageid=lid, languageid_bits=-1);
} {1 {languageid_bits parameter out of range}}

do_catchsql_test 1.3 {
  CREATE VIRTUAL TABLE t1 USING fts4(languageid=lid, languageid_bits=0);
  CREATE VIRTUAL TABLE t2 USING fts4(languageid=lid, languageid_bits=30);
} {0 {}}

do_execsql_test 1.4 {
  DROP TABLE t1;
  DROP TABLE t2;
}

#-------------------------------------------------------------------------
# Test out-of-range values in the languageid column.
#
do_execsql_test 2.1 {
  CREATE VIRTUAL TABLE t1 USING fts4(languageid=lid, languageid_bits=8);
  CREATE VIRTUAL TABLE t2 USING fts4(languageid=lid, languageid_bits=7);
}

do_catchsql_test 2.2 {
  INSERT INTO t1(docid, lid, content) VALUES(1, 256, 'abc def');
} {1 {constraint failed}}

do_catchsql_test 2.3 {
  INSERT INTO t2(docid, lid, content) VALUES(1, 128, 'abc def');
} {1 {constraint failed}}

do_catchsql_test 2.3 {
  INSERT INTO t1(docid, lid, content) VALUES(1, -1, 'abc def');
} {1 {constraint failed}}

do_execsql_test 2.4 {
  DROP TABLE t1;
  DROP TABLE t2;
}

#-------------------------------------------------------------------------
# Test that if languageid_bits is set to a non-zero value it is
# not possible to specify a non-NULL rowid, even if it is the same
# as the docid.
#
do_execsql_test 3.1 {
  CREATE VIRTUAL TABLE t1 USING fts4(languageid=lid, languageid_bits=4);
  CREATE VIRTUAL TABLE t2 USING fts4(languageid=lid, languageid_bits=0);
}
  
do_catchsql_test 3.2.1 {
  INSERT INTO t1(rowid, lid, content) VALUES(1, 0, 'abc def');
} {1 {constraint failed}}

do_catchsql_test 3.2.2 {
  INSERT INTO t2(rowid, lid, content) VALUES(1, 0, 'abc def');
} {0 {}}

do_catchsql_test 3.3 {
  INSERT INTO t1(rowid, docid, lid, content) VALUES(2, 2, 0, 'abc def');
} {1 {constraint failed}}

do_catchsql_test 3.4 {
  INSERT INTO t1(lid, content) VALUES(0, 'one two def');
} {1 {constraint failed}}

do_execsql_test 3.4 {
  DROP TABLE t1;
  DROP TABLE t2;
}

#-------------------------------------------------------------------------
# Simple tests inserting data with multiple languageid values.
#
do_execsql_test 4.1 {
  CREATE VIRTUAL TABLE t1 USING fts4(languageid=lid, languageid_bits=5);
}

do_execsql_test 4.2 {
  INSERT INTO t1 (docid, lid, content) VALUES(1, 0, '1 2 3');
  INSERT INTO t1 (docid, lid, content) VALUES(1, 1, '1 2 3 4');
}

do_execsql_test 4.3 {
  SELECT docid, lid FROM t1;
} {1 0 1 1}

do_execsql_test 4.4 {
  SELECT docid, lid, content FROM t1 WHERE t1 MATCH '2';
} {1 0 {1 2 3}}

do_execsql_test 4.5 {
  SELECT docid, lid, content FROM t1 WHERE t1 MATCH '2' AND lid=1;
} {1 1 {1 2 3 4}}

do_execsql_test 4.6 {
  UPDATE t1 SET content = 'x y z' || lid;
  SELECT docid, lid FROM t1;
} {1 0 1 1}

do_execsql_test 3.4 {
  DROP TABLE t1;
}

#-------------------------------------------------------------------------
# Tests for docid range boundary conditions. 
#
for {set bits 1} {$bits <= 30} {incr bits} {
  do_execsql_test 5.$bits.1 "
    CREATE VIRTUAL TABLE t1 USING fts4(languageid=lid, languageid_bits=$bits);
  "

  set max_docid [expr int(1<<(63-$bits))-1]
  set min_docid [expr -1*int(1<<(63-$bits))]
  set max_langid [expr (1<<$bits)-1]
  set min_langid 0


  do_catchsql_test 5.$bits.2.1 {
    INSERT INTO t1(docid, lid, content) VALUES($max_docid+1, 4, '');
  } {1 {constraint failed}}
  do_catchsql_test 5.$bits.2.2 {
    INSERT INTO t1(docid, lid, content) VALUES($min_docid-1, 4, '');
  } {1 {constraint failed}}

  do_test 5.$bits.3 {
    foreach {a b c} "
      $min_docid  $min_langid  {1 min min x}
      $min_docid  $max_langid  {2 min max x}
      $max_docid  $min_langid  {3 max min x}
      $max_docid  $max_langid  {4 max max x}
    " {
      execsql { INSERT INTO t1(docid, lid, content) VALUES($a, $b, $c) }
    }
  } {}

  do_execsql_test 5.$bits.4.1 {
    SELECT docid, lid, content FROM t1 ORDER BY content
  } "
      $min_docid  $min_langid  {1 min min x}
      $min_docid  $max_langid  {2 min max x}
      $max_docid  $min_langid  {3 max min x}
      $max_docid  $max_langid  {4 max max x}
  "

  do_execsql_test 5.$bits.4.2 {
    SELECT docid, lid, content FROM t1 WHERE lid=$min_langid AND t1 MATCH 'x'
  } "
      $min_docid  $min_langid  {1 min min x}
      $max_docid  $min_langid  {3 max min x}
  "

  do_execsql_test 5.$bits.4.3 {
    SELECT docid, lid, content FROM t1 WHERE lid=$max_langid AND t1 MATCH 'x'
  } "
      $min_docid  $max_langid  {2 min max x}
      $max_docid  $max_langid  {4 max max x}
  "

  do_execsql_test 5.$bits.4.4 {
    SELECT docid, lid, content FROM t1 WHERE t1 MATCH '1'
  } "
      $min_docid  $min_langid  {1 min min x}
  "

  do_execsql_test 5.$bits.5 { DROP TABLE t1 }
}

#-------------------------------------------------------------------------
# Tests for auxilliary functions with langaugeid_bits tables.
#
proc mit {blob} {
  set scan(littleEndian) i*
  set scan(bigEndian) I*
  binary scan $blob $scan($::tcl_platform(byteOrder)) r
  return $r
}
db func mit mit

do_execsql_test 6.1 {
  CREATE VIRTUAL TABLE t1 USING fts4(languageid_bits=4, languageid=lid);
  INSERT INTO t1(docid,lid,content) VALUES(1, 1, 'one two three four');
  INSERT INTO t1(docid,lid,content) VALUES(2, 1, 'two three four five');
  INSERT INTO t1(docid,lid,content) VALUES(3, 1, 'three four five six');
  INSERT INTO t1(docid,lid,content) VALUES(4, 1, 'four five six seven');

  INSERT INTO t1(docid,lid,content) VALUES(1, 2, 'four three two one');
  INSERT INTO t1(docid,lid,content) VALUES(2, 2, 'five four three two');
  INSERT INTO t1(docid,lid,content) VALUES(3, 2, 'six five four three');
  INSERT INTO t1(docid,lid,content) VALUES(4, 2, 'A B C D');
}

do_execsql_test 6.2.1 {
  SELECT docid, snippet(t1) FROM t1 WHERE t1 MATCH 'one' AND lid=1;
} {1 {<b>one</b> two three four}}
do_execsql_test 6.2.2 {
  SELECT docid, snippet(t1) FROM t1 WHERE t1 MATCH 'one' AND lid=2;
} {1 {four three two <b>one</b>}}

do_execsql_test 6.2.1 {
  SELECT docid, offsets(t1) FROM t1 WHERE t1 MATCH 'two' AND lid=1;
} {1 {0 0 4 3} 2 {0 0 0 3}}
do_execsql_test 6.2.2 {
  SELECT docid, offsets(t1) FROM t1 WHERE t1 MATCH 'two' AND lid=2;
} {1 {0 0 11 3} 2 {0 0 16 3}}

do_execsql_test 6.3.1 {
  SELECT docid, mit(matchinfo(t1)) FROM t1 WHERE t1 MATCH 'two' AND lid=1;
} {1 {1 1 1 2 2} 2 {1 1 1 2 2}}
do_execsql_test 6.3.2 {
  SELECT docid, mit(matchinfo(t1)) FROM t1 WHERE t1 MATCH 'two' AND lid=2;
} {1 {1 1 1 2 2} 2 {1 1 1 2 2}}
do_execsql_test 6.3.3 {
  SELECT docid, mit(matchinfo(t1)) FROM t1 WHERE t1 MATCH 'B' AND lid=1;
} {}
do_execsql_test 6.3.4 {
  SELECT docid, mit(matchinfo(t1)) FROM t1 WHERE t1 MATCH 'B' AND lid=2;
} {4 {1 1 1 1 1}}

do_execsql_test 6.4 {
  CREATE VIRTUAL TABLE t2 USING fts4(languageid_bits=8, languageid=lid);
  INSERT INTO t2(docid,lid,content) VALUES(-1, 0, 'A B C D');
  INSERT INTO t2(docid,lid,content) VALUES(-2, 0, 'D C B A');
  INSERT INTO t2(docid,lid,content) VALUES(-3, 0, 'C B D A');
  INSERT INTO t2(docid,lid,content) VALUES(-4, 0, 'A D B C');

  INSERT INTO t2(docid,lid,content) VALUES(-1, 1, 'A A A A');
  INSERT INTO t2(docid,lid,content) VALUES(-2, 1, 'B B B B');
  INSERT INTO t2(docid,lid,content) VALUES(-3, 1, 'C C C C');
  INSERT INTO t2(docid,lid,content) VALUES(-4, 1, 'D D D D');
}

do_execsql_test 6.4.1 {
  SELECT docid, mit(matchinfo(t2)) FROM t2 WHERE t2 MATCH 'B';
} {
  -4 {1 1 1 4 4}
  -3 {1 1 1 4 4}
  -2 {1 1 1 4 4}
  -1 {1 1 1 4 4}
}
do_execsql_test 6.4.2 {
  SELECT docid, mit(matchinfo(t2)) FROM t2 WHERE t2 MATCH 'B' AND lid=1;
} {-2 {1 1 4 4 1}}

do_execsql_test 6.5 {
  DROP TABLE t1;
  DROP TABLE t2;
}

#-------------------------------------------------------------------------
# Tests for querying by docid.
#
do_execsql_test 7.1 {
  CREATE VIRTUAL TABLE t1 USING fts4(languageid_bits=8, languageid=lid);
  INSERT INTO t1(docid,lid,content) VALUES(10, 10, 'abc def');
}

do_execsql_test 7.2 {
  SELECT docid,lid,content FROM t1 WHERE docid=10;
} {10 10 {abc def}}

do_execsql_test 7.3 {
  SELECT docid,lid,content FROM t1 WHERE docid<11;
} {10 10 {abc def}}

do_execsql_test 7.4 {
  DROP TABLE t1;
}

#-------------------------------------------------------------------------
# Tests for sorting by docid.
#
do_execsql_test 8.1 {
  CREATE VIRTUAL TABLE t1 USING fts4(languageid_bits=6, languageid=lid);
  INSERT INTO t1 (docid,lid,content) VALUES(1, 0, 'abc def');
  INSERT INTO t1 (docid,lid,content) VALUES(3, 0, 'abc ghi');
  INSERT INTO t1 (docid,lid,content) VALUES(2, 0, 'def ghi');

  INSERT INTO t1 (docid,lid,content) VALUES(1, 5, 'A B');
  INSERT INTO t1 (docid,lid,content) VALUES(3, 5, 'A C');
  INSERT INTO t1 (docid,lid,content) VALUES(2, 5, 'B C');
}

do_execsql_test 8.2 {
  SELECT docid FROM t1 ORDER BY docid;
} {1 1 2 2 3 3}
do_execsql_test 8.3 {
  SELECT docid FROM t1 WHERE t1 MATCH 'ghi' ORDER BY docid;
} {2 3}
do_execsql_test 8.4 {
  SELECT docid FROM t1 WHERE t1 MATCH 'ghi' ORDER BY docid DESC;
} {3 2}

# Test that the docid and languageid fields may be updated.
#
do_execsql_test 8.5 { UPDATE t1 SET docid=docid+3, lid=0 WHERE lid=5; }
do_execsql_test 8.6 { SELECT docid FROM t1 ORDER BY docid; } {1 2 3 4 5 6}
do_execsql_test 8.7 { SELECT docid FROM t1 WHERE t1 MATCH 'A' } {4 6}
do_execsql_test 8.8 { SELECT docid FROM t1 WHERE t1 MATCH 'A' AND lid=5 } {}

finish_test

Changes to test/permutations.test.

189
190
191
192
193
194
195
196


197
198
199
200
201
202
203
189
190
191
192
193
194
195

196
197
198
199
200
201
202
203
204







-
+
+







  fts3expr3.test
  fts3near.test fts3query.test fts3shared.test fts3snippet.test 
  fts3sort.test
  fts3fault.test fts3malloc.test fts3matchinfo.test
  fts3aux1.test fts3comp1.test fts3auto.test
  fts4aa.test fts4content.test
  fts3conf.test fts3prefix.test fts3fault2.test fts3corrupt.test
  fts3corrupt2.test fts3first.test fts4langid.test fts4merge.test
  fts3corrupt2.test fts3first.test fts4langid.test fts4langid2.test
  fts4merge.test
  fts4check.test fts4unicode.test
}

test_suite "nofaultsim" -prefix "" -description {
  "Very" quick test suite. Runs in less than 5 minutes on a workstation. 
  This test suite is the same as the "quick" tests, except that some files
  that test malloc and IO errors are omitted.