/ Changes On Branch fts-languageid-bits
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Changes In Branch fts-languageid-bits Excluding Merge-Ins

This is equivalent to a diff from 2b2ade9278 to 949425d467

2013-06-20
19:55
Add tests for modifying the docid and languageid fields of an fts table with a non-zero languageid_bits field. (Closed-Leaf check-in: 949425d467 user: dan tags: fts-languageid-bits)
18:32
Fix some issues related to ORDER BY and fts tables with a non-zero languageid_bits setting. (check-in: 81527768ef user: dan tags: fts-languageid-bits)
14:07
Disable posix_fallocate() for all systems, all the time, unless the HAVE_POSIX_FALLOCATE compile-time macro is supplied. (check-in: b674462243 user: drh tags: trunk)
2013-06-19
23:48
Merge in trunk changes to os_unix.c that allow the code to build on unix platforms that lack posix_fallocate(). (check-in: bf5764067a user: drh tags: nextgen-query-plan-exp)
20:13
Add the languageid_bits= option to fts. Still some problems to work out. (check-in: d36d7e6833 user: dan tags: fts-languageid-bits)
14:49
Only default HAVE_POSIX_FALLOCATE on for linux, and then only if it is not previously defined. (check-in: 2b2ade9278 user: drh tags: trunk)
14:28
Only enable posix_fallocate by default on linux and mac. (check-in: b9b30d4f98 user: drh tags: trunk)

Changes to ext/fts3/fts3.c.

  1077   1077     int bNoDocsize = 0;             /* True to omit %_docsize table */
  1078   1078     int bDescIdx = 0;               /* True to store descending indexes */
  1079   1079     char *zPrefix = 0;              /* Prefix parameter value (or NULL) */
  1080   1080     char *zCompress = 0;            /* compress=? parameter (or NULL) */
  1081   1081     char *zUncompress = 0;          /* uncompress=? parameter (or NULL) */
  1082   1082     char *zContent = 0;             /* content=? parameter (or NULL) */
  1083   1083     char *zLanguageid = 0;          /* languageid=? parameter (or NULL) */
         1084  +  char *zLanguageidBits = 0;      /* languageid_bits=? parameter (or NULL) */
  1084   1085   
  1085   1086     assert( strlen(argv[0])==4 );
  1086   1087     assert( (sqlite3_strnicmp(argv[0], "fts4", 4)==0 && isFts4)
  1087   1088          || (sqlite3_strnicmp(argv[0], "fts3", 4)==0 && !isFts4)
  1088   1089     );
  1089   1090   
  1090   1091     nDb = (int)strlen(argv[1]) + 1;
................................................................................
  1121   1122   
  1122   1123       /* Check if it is an FTS4 special argument. */
  1123   1124       else if( isFts4 && fts3IsSpecialColumn(z, &nKey, &zVal) ){
  1124   1125         struct Fts4Option {
  1125   1126           const char *zOpt;
  1126   1127           int nOpt;
  1127   1128         } aFts4Opt[] = {
  1128         -        { "matchinfo",   9 },     /* 0 -> MATCHINFO */
  1129         -        { "prefix",      6 },     /* 1 -> PREFIX */
  1130         -        { "compress",    8 },     /* 2 -> COMPRESS */
  1131         -        { "uncompress", 10 },     /* 3 -> UNCOMPRESS */
  1132         -        { "order",       5 },     /* 4 -> ORDER */
  1133         -        { "content",     7 },     /* 5 -> CONTENT */
  1134         -        { "languageid", 10 }      /* 6 -> LANGUAGEID */
         1129  +        { "matchinfo",        9 },     /* 0 -> MATCHINFO */
         1130  +        { "prefix",           6 },     /* 1 -> PREFIX */
         1131  +        { "compress",         8 },     /* 2 -> COMPRESS */
         1132  +        { "uncompress",      10 },     /* 3 -> UNCOMPRESS */
         1133  +        { "order",            5 },     /* 4 -> ORDER */
         1134  +        { "content",          7 },     /* 5 -> CONTENT */
         1135  +        { "languageid",      10 },     /* 6 -> LANGUAGEID */
         1136  +        { "languageid_bits", 15 }      /* 7 -> LANGUAGEID_BITS */
  1135   1137         };
  1136   1138   
  1137   1139         int iOpt;
  1138   1140         if( !zVal ){
  1139   1141           rc = SQLITE_NOMEM;
  1140   1142         }else{
  1141   1143           for(iOpt=0; iOpt<SizeofArray(aFts4Opt); iOpt++){
................................................................................
  1193   1195   
  1194   1196               case 6:              /* LANGUAGEID */
  1195   1197                 assert( iOpt==6 );
  1196   1198                 sqlite3_free(zLanguageid);
  1197   1199                 zLanguageid = zVal;
  1198   1200                 zVal = 0;
  1199   1201                 break;
         1202  +
         1203  +            case 7:              /* LANGUAGEID_BITS */
         1204  +              assert( iOpt==7 );
         1205  +              sqlite3_free(zLanguageidBits);
         1206  +              zLanguageidBits = zVal;
         1207  +              zVal = 0;
         1208  +              break;
  1200   1209             }
  1201   1210           }
  1202   1211           sqlite3_free(zVal);
  1203   1212         }
  1204   1213       }
  1205   1214   
  1206   1215       /* Otherwise, the argument is a column name. */
................................................................................
  1288   1297     p->bFts4 = isFts4;
  1289   1298     p->bDescIdx = bDescIdx;
  1290   1299     p->bAutoincrmerge = 0xff;   /* 0xff means setting unknown */
  1291   1300     p->zContentTbl = zContent;
  1292   1301     p->zLanguageid = zLanguageid;
  1293   1302     zContent = 0;
  1294   1303     zLanguageid = 0;
         1304  +  if( zLanguageidBits && p->zLanguageid && p->zContentTbl==0 ){
         1305  +    p->nLanguageidBits = atoi(zLanguageidBits);
         1306  +    if( p->nLanguageidBits>30 || p->nLanguageidBits<0 ){
         1307  +      rc = SQLITE_ERROR;
         1308  +      *pzErr = sqlite3_mprintf("languageid_bits parameter out of range");
         1309  +      goto fts3_init_out;
         1310  +    }
         1311  +  }
         1312  +
  1295   1313     TESTONLY( p->inTransaction = -1 );
  1296   1314     TESTONLY( p->mxSavepoint = -1 );
  1297   1315   
  1298   1316     p->aIndex = (struct Fts3Index *)&p->azColumn[nCol];
  1299   1317     memcpy(p->aIndex, aIndex, sizeof(struct Fts3Index) * nIndex);
  1300   1318     p->nIndex = nIndex;
  1301   1319     for(i=0; i<nIndex; i++){
................................................................................
  1361   1379   fts3_init_out:
  1362   1380     sqlite3_free(zPrefix);
  1363   1381     sqlite3_free(aIndex);
  1364   1382     sqlite3_free(zCompress);
  1365   1383     sqlite3_free(zUncompress);
  1366   1384     sqlite3_free(zContent);
  1367   1385     sqlite3_free(zLanguageid);
         1386  +  sqlite3_free(zLanguageidBits);
  1368   1387     sqlite3_free((void *)aCol);
  1369   1388     if( rc!=SQLITE_OK ){
  1370   1389       if( p ){
  1371   1390         fts3DisconnectMethod((sqlite3_vtab *)p);
  1372   1391       }else if( pTokenizer ){
  1373   1392         pTokenizer->pModule->xDestroy(pTokenizer);
  1374   1393       }
................................................................................
  1422   1441     ** so search through the constraints to see if a more efficient 
  1423   1442     ** strategy is possible.
  1424   1443     */
  1425   1444     pInfo->idxNum = FTS3_FULLSCAN_SEARCH;
  1426   1445     pInfo->estimatedCost = 500000;
  1427   1446     for(i=0; i<pInfo->nConstraint; i++){
  1428   1447       struct sqlite3_index_constraint *pCons = &pInfo->aConstraint[i];
         1448  +    int iCol = pCons->iColumn;
  1429   1449       if( pCons->usable==0 ) continue;
  1430   1450   
  1431   1451       /* A direct lookup on the rowid or docid column. Assign a cost of 1.0. */
  1432         -    if( iCons<0 
  1433         -     && pCons->op==SQLITE_INDEX_CONSTRAINT_EQ 
  1434         -     && (pCons->iColumn<0 || pCons->iColumn==p->nColumn+1 )
         1452  +    if( iCons<0 && pCons->op==SQLITE_INDEX_CONSTRAINT_EQ 
         1453  +     && (iCol<0 || (iCol==p->nColumn+1 && p->nLanguageidBits==0))
  1435   1454       ){
  1436   1455         pInfo->idxNum = FTS3_DOCID_SEARCH;
  1437   1456         pInfo->estimatedCost = 1.0;
  1438   1457         iCons = i;
  1439   1458       }
  1440   1459   
  1441   1460       /* A MATCH constraint. Use a full-text search.
................................................................................
  1470   1489     if( iLangidCons>=0 ){
  1471   1490       pInfo->aConstraintUsage[iLangidCons].argvIndex = 2;
  1472   1491     } 
  1473   1492   
  1474   1493     /* Regardless of the strategy selected, FTS can deliver rows in rowid (or
  1475   1494     ** docid) order. Both ascending and descending are possible. 
  1476   1495     */
         1496  +  assert( pInfo->orderByConsumed==0 );
  1477   1497     if( pInfo->nOrderBy==1 ){
  1478   1498       struct sqlite3_index_orderby *pOrder = &pInfo->aOrderBy[0];
  1479         -    if( pOrder->iColumn<0 || pOrder->iColumn==p->nColumn+1 ){
         1499  +    if( pOrder->iColumn<0 || (
         1500  +          (pOrder->iColumn==p->nColumn+1)
         1501  +       && (pInfo->idxNum>=FTS3_FULLTEXT_SEARCH || p->nLanguageidBits==0)
         1502  +    )){
  1480   1503         if( pOrder->desc ){
  1481   1504           pInfo->idxStr = "DESC";
  1482   1505         }else{
  1483   1506           pInfo->idxStr = "ASC";
  1484   1507         }
  1485   1508         pInfo->orderByConsumed = 1;
  1486   1509       }
................................................................................
  3058   3081     Fts3Cursor *pCsr = (Fts3Cursor *) pCursor;
  3059   3082     Fts3Table *p = (Fts3Table *)pCursor->pVtab;
  3060   3083   
  3061   3084     /* The column value supplied by SQLite must be in range. */
  3062   3085     assert( iCol>=0 && iCol<=p->nColumn+2 );
  3063   3086   
  3064   3087     if( iCol==p->nColumn+1 ){
  3065         -    /* This call is a request for the "docid" column. Since "docid" is an 
  3066         -    ** alias for "rowid", use the xRowid() method to obtain the value.
  3067         -    */
  3068         -    sqlite3_result_int64(pCtx, pCsr->iPrevId);
         3088  +    /* This call is a request for the "docid" column. The value currently
         3089  +    ** stored in pCsr->iPrevId is a rowid. Transform this to a docid and
         3090  +    ** return it.  */
         3091  +    sqlite3_result_int64(pCtx, sqlite3Fts3RowidToDocid(p, pCsr->iPrevId));
  3069   3092     }else if( iCol==p->nColumn ){
  3070   3093       /* The extra column whose name is the same as the table.
  3071   3094       ** Return a blob which is a pointer to the cursor.  */
  3072   3095       sqlite3_result_blob(pCtx, &pCsr, sizeof(pCsr), SQLITE_TRANSIENT);
  3073   3096     }else if( iCol==p->nColumn+2 && pCsr->pExpr ){
  3074   3097       sqlite3_result_int64(pCtx, pCsr->iLangid);
  3075   3098     }else{

Changes to ext/fts3/fts3Int.h.

   205    205     const char *zDb;                /* logical database name */
   206    206     const char *zName;              /* virtual table name */
   207    207     int nColumn;                    /* number of named columns in virtual table */
   208    208     char **azColumn;                /* column names.  malloced */
   209    209     sqlite3_tokenizer *pTokenizer;  /* tokenizer for inserts and queries */
   210    210     char *zContentTbl;              /* content=xxx option, or NULL */
   211    211     char *zLanguageid;              /* languageid=xxx option, or NULL */
          212  +  int nLanguageidBits;            /* languageid_bits=N option, or 0 */
   212    213     u8 bAutoincrmerge;              /* True if automerge=1 */
   213    214     u32 nLeafAdd;                   /* Number of leaf blocks added this trans */
   214    215   
   215    216     /* Precompiled statements used by the implementation. Each of these 
   216    217     ** statements is run and reset within a single virtual table API call. 
   217    218     */
   218    219     sqlite3_stmt *aStmt[37];
................................................................................
   417    418   */
   418    419   #define FTSQUERY_NEAR   1
   419    420   #define FTSQUERY_NOT    2
   420    421   #define FTSQUERY_AND    3
   421    422   #define FTSQUERY_OR     4
   422    423   #define FTSQUERY_PHRASE 5
   423    424   
   424         -
   425    425   /* fts3_write.c */
          426  +i64 sqlite3Fts3DocidToRowid(Fts3Table *p, i64 iDocid, int iLangid);
          427  +i64 sqlite3Fts3RowidToDocid(Fts3Table *p, i64 iRowid);
   426    428   int sqlite3Fts3UpdateMethod(sqlite3_vtab*,int,sqlite3_value**,sqlite3_int64*);
   427    429   int sqlite3Fts3PendingTermsFlush(Fts3Table *);
   428    430   void sqlite3Fts3PendingTermsClear(Fts3Table *);
   429    431   int sqlite3Fts3Optimize(Fts3Table *);
   430    432   int sqlite3Fts3SegReaderNew(int, int, sqlite3_int64,
   431    433     sqlite3_int64, sqlite3_int64, const char *, int, Fts3SegReader**);
   432    434   int sqlite3Fts3SegReaderPending(

Changes to ext/fts3/fts3_write.c.

   483    483     if( rc==SQLITE_OK ){
   484    484       sqlite3_step(pStmt);
   485    485       rc = sqlite3_reset(pStmt);
   486    486     }
   487    487     *pRC = rc;
   488    488   }
   489    489   
          490  +static void fts3SqlExecI64(
          491  +  int *pRC,                /* Result code */
          492  +  Fts3Table *p,            /* The FTS3 table */
          493  +  int eStmt,               /* Index of statement to evaluate */
          494  +  i64 iVal
          495  +){
          496  +  sqlite3_stmt *pStmt;
          497  +  int rc;
          498  +  if( *pRC ) return;
          499  +  rc = fts3SqlStmt(p, eStmt, &pStmt, 0); 
          500  +  if( rc==SQLITE_OK ){
          501  +    sqlite3_bind_int64(pStmt, 1, iVal);
          502  +    sqlite3_step(pStmt);
          503  +    rc = sqlite3_reset(pStmt);
          504  +  }
          505  +  *pRC = rc;
          506  +}
          507  +
   490    508   
   491    509   /*
   492    510   ** This function ensures that the caller has obtained an exclusive 
   493    511   ** shared-cache table-lock on the %_segdir table. This is required before 
   494    512   ** writing data to the fts3 table. If this lock is not acquired first, then
   495    513   ** the caller may end up attempting to take this lock as part of committing
   496    514   ** a transaction, causing SQLite to return SQLITE_LOCKED or 
................................................................................
   923    941   **   apVal[p->nColumn+2]     Hidden column with same name as table
   924    942   **   apVal[p->nColumn+3]     Hidden "docid" column (alias for rowid)
   925    943   **   apVal[p->nColumn+4]     Hidden languageid column
   926    944   */
   927    945   static int fts3InsertData(
   928    946     Fts3Table *p,                   /* Full-text table */
   929    947     sqlite3_value **apVal,          /* Array of values to insert */
   930         -  sqlite3_int64 *piDocid          /* OUT: Docid for row just inserted */
          948  +  sqlite3_int64 *piRowid,         /* OUT: Rowid for row just inserted */
          949  +  i64 iRowid                      /* Explicit rowid, if piRowid==NULL */
   931    950   ){
   932    951     int rc;                         /* Return code */
   933    952     sqlite3_stmt *pContentInsert;   /* INSERT INTO %_content VALUES(...) */
   934    953   
   935    954     if( p->zContentTbl ){
   936         -    sqlite3_value *pRowid = apVal[p->nColumn+3];
          955  +    sqlite3_value *pRowid;
          956  +    assert( p->nLanguageidBits==0 && piRowid );
          957  +    pRowid = apVal[p->nColumn+3];
   937    958       if( sqlite3_value_type(pRowid)==SQLITE_NULL ){
   938    959         pRowid = apVal[1];
   939    960       }
   940    961       if( sqlite3_value_type(pRowid)!=SQLITE_INTEGER ){
   941    962         return SQLITE_CONSTRAINT;
   942    963       }
   943         -    *piDocid = sqlite3_value_int64(pRowid);
          964  +    *piRowid = sqlite3_value_int64(pRowid);
   944    965       return SQLITE_OK;
   945    966     }
   946    967   
   947    968     /* Locate the statement handle used to insert data into the %_content
   948    969     ** table. The SQL for this statement is:
   949    970     **
   950    971     **   INSERT INTO %_content VALUES(?, ?, ?, ...)
   951    972     **
   952    973     ** The statement features N '?' variables, where N is the number of user
   953    974     ** defined columns in the FTS3 table, plus one for the docid field.
   954    975     */
   955    976     rc = fts3SqlStmt(p, SQL_CONTENT_INSERT, &pContentInsert, &apVal[1]);
   956         -  if( rc==SQLITE_OK && p->zLanguageid ){
   957         -    rc = sqlite3_bind_int(
   958         -        pContentInsert, p->nColumn+2, 
   959         -        sqlite3_value_int(apVal[p->nColumn+4])
   960         -    );
          977  +  if( rc==SQLITE_OK ){
          978  +    if( piRowid==0 ){
          979  +      sqlite3_bind_int64(pContentInsert, 1, iRowid);
          980  +    }
          981  +    if( p->zLanguageid ){
          982  +      rc = sqlite3_bind_int(
          983  +          pContentInsert, p->nColumn+2, 
          984  +          sqlite3_value_int(apVal[p->nColumn+4])
          985  +      );
          986  +    }
   961    987     }
   962    988     if( rc!=SQLITE_OK ) return rc;
   963    989   
   964    990     /* There is a quirk here. The users INSERT statement may have specified
   965    991     ** a value for the "rowid" field, for the "docid" field, or for both.
   966    992     ** Which is a problem, since "rowid" and "docid" are aliases for the
   967    993     ** same value. For example:
   968    994     **
   969    995     **   INSERT INTO fts3tbl(rowid, docid) VALUES(1, 2);
   970    996     **
   971    997     ** In FTS3, this is an error. It is an error to specify non-NULL values
   972    998     ** for both docid and some other rowid alias.
   973    999     */
   974         -  if( SQLITE_NULL!=sqlite3_value_type(apVal[3+p->nColumn]) ){
   975         -    if( SQLITE_NULL==sqlite3_value_type(apVal[0])
   976         -     && SQLITE_NULL!=sqlite3_value_type(apVal[1])
   977         -    ){
   978         -      /* A rowid/docid conflict. */
   979         -      return SQLITE_ERROR;
   980         -    }
         1000  +  assert( p->nLanguageidBits==0 || piRowid==0
         1001  +      || sqlite3_value_type(apVal[1])!=SQLITE_NULL 
         1002  +  );
         1003  +  if( piRowid && p->nLanguageidBits==0 
         1004  +   && SQLITE_NULL!=sqlite3_value_type(apVal[3+p->nColumn]) 
         1005  +  ){
   981   1006       rc = sqlite3_bind_value(pContentInsert, 1, apVal[3+p->nColumn]);
   982   1007       if( rc!=SQLITE_OK ) return rc;
   983   1008     }
   984   1009   
   985         -  /* Execute the statement to insert the record. Set *piDocid to the 
         1010  +  /* Execute the statement to insert the record. Set *pRowid to the 
   986   1011     ** new docid value. 
   987   1012     */
   988   1013     sqlite3_step(pContentInsert);
   989   1014     rc = sqlite3_reset(pContentInsert);
   990   1015   
   991         -  *piDocid = sqlite3_last_insert_rowid(p->db);
         1016  +  if( piRowid ){
         1017  +    *piRowid = sqlite3_last_insert_rowid(p->db);
         1018  +  }
   992   1019     return rc;
   993   1020   }
   994   1021   
   995   1022   
   996   1023   
   997   1024   /*
   998   1025   ** Remove all data from the FTS3 table. Clear the hash table containing
................................................................................
  1032   1059   ** The first element in the apVal[] array is assumed to contain the docid
  1033   1060   ** (an integer) of a row about to be deleted. Remove all terms from the
  1034   1061   ** full-text index.
  1035   1062   */
  1036   1063   static void fts3DeleteTerms( 
  1037   1064     int *pRC,               /* Result code */
  1038   1065     Fts3Table *p,           /* The FTS table to delete from */
  1039         -  sqlite3_value *pRowid,  /* The docid to be deleted */
         1066  +  i64 iRowid,             /* The rowid to be deleted */
  1040   1067     u32 *aSz,               /* Sizes of deleted document written here */
  1041   1068     int *pbFound            /* OUT: Set to true if row really does exist */
  1042   1069   ){
  1043   1070     int rc;
  1044   1071     sqlite3_stmt *pSelect;
  1045   1072   
  1046   1073     assert( *pbFound==0 );
  1047   1074     if( *pRC ) return;
  1048         -  rc = fts3SqlStmt(p, SQL_SELECT_CONTENT_BY_ROWID, &pSelect, &pRowid);
         1075  +  rc = fts3SqlStmt(p, SQL_SELECT_CONTENT_BY_ROWID, &pSelect, 0);
  1049   1076     if( rc==SQLITE_OK ){
         1077  +    sqlite3_bind_int64(pSelect, 1, iRowid);
  1050   1078       if( SQLITE_ROW==sqlite3_step(pSelect) ){
  1051   1079         int i;
  1052   1080         int iLangid = langidFromSelect(p, pSelect);
  1053   1081         rc = fts3PendingTermsDocid(p, iLangid, sqlite3_column_int64(pSelect, 0));
  1054   1082         for(i=1; rc==SQLITE_OK && i<=p->nColumn; i++){
  1055   1083           const char *zText = (const char *)sqlite3_column_text(pSelect, i);
  1056   1084           rc = fts3PendingTermsAdd(p, iLangid, zText, -1, &aSz[i-1]);
................................................................................
  2342   2370   ** are different from that integer. i.e. if deleting the document with docid
  2343   2371   ** pRowid would mean the FTS3 table were empty.
  2344   2372   **
  2345   2373   ** If successful, *pisEmpty is set to true if the table is empty except for
  2346   2374   ** document pRowid, or false otherwise, and SQLITE_OK is returned. If an
  2347   2375   ** error occurs, an SQLite error code is returned.
  2348   2376   */
  2349         -static int fts3IsEmpty(Fts3Table *p, sqlite3_value *pRowid, int *pisEmpty){
         2377  +static int fts3IsEmpty(Fts3Table *p, i64 iRowid, int *pisEmpty){
  2350   2378     sqlite3_stmt *pStmt;
  2351   2379     int rc;
  2352   2380     if( p->zContentTbl ){
  2353   2381       /* If using the content=xxx option, assume the table is never empty */
  2354   2382       *pisEmpty = 0;
  2355   2383       rc = SQLITE_OK;
  2356   2384     }else{
  2357         -    rc = fts3SqlStmt(p, SQL_IS_EMPTY, &pStmt, &pRowid);
         2385  +    rc = fts3SqlStmt(p, SQL_IS_EMPTY, &pStmt, 0);
  2358   2386       if( rc==SQLITE_OK ){
         2387  +      sqlite3_bind_int64(pStmt, 1, iRowid);
  2359   2388         if( SQLITE_ROW==sqlite3_step(pStmt) ){
  2360   2389           *pisEmpty = sqlite3_column_int(pStmt, 0);
  2361   2390         }
  2362   2391         rc = sqlite3_reset(pStmt);
  2363   2392       }
  2364   2393     }
  2365   2394     return rc;
................................................................................
  5193   5222   /*
  5194   5223   ** SQLite value pRowid contains the rowid of a row that may or may not be
  5195   5224   ** present in the FTS3 table. If it is, delete it and adjust the contents
  5196   5225   ** of subsiduary data structures accordingly.
  5197   5226   */
  5198   5227   static int fts3DeleteByRowid(
  5199   5228     Fts3Table *p, 
  5200         -  sqlite3_value *pRowid, 
         5229  +  i64 iRowid,
  5201   5230     int *pnChng,                    /* IN/OUT: Decrement if row is deleted */
  5202   5231     u32 *aSzDel
  5203   5232   ){
  5204   5233     int rc = SQLITE_OK;             /* Return code */
  5205   5234     int bFound = 0;                 /* True if *pRowid really is in the table */
  5206   5235   
  5207         -  fts3DeleteTerms(&rc, p, pRowid, aSzDel, &bFound);
         5236  +  fts3DeleteTerms(&rc, p, iRowid, aSzDel, &bFound);
  5208   5237     if( bFound && rc==SQLITE_OK ){
  5209   5238       int isEmpty = 0;              /* Deleting *pRowid leaves the table empty */
  5210         -    rc = fts3IsEmpty(p, pRowid, &isEmpty);
         5239  +    rc = fts3IsEmpty(p, iRowid, &isEmpty);
  5211   5240       if( rc==SQLITE_OK ){
  5212   5241         if( isEmpty ){
  5213   5242           /* Deleting this row means the whole table is empty. In this case
  5214   5243           ** delete the contents of all three tables and throw away any
  5215   5244           ** data in the pendingTerms hash table.  */
  5216   5245           rc = fts3DeleteAll(p, 1);
  5217   5246           *pnChng = 0;
  5218   5247           memset(aSzDel, 0, sizeof(u32) * (p->nColumn+1) * 2);
  5219   5248         }else{
  5220   5249           *pnChng = *pnChng - 1;
  5221   5250           if( p->zContentTbl==0 ){
  5222         -          fts3SqlExec(&rc, p, SQL_DELETE_CONTENT, &pRowid);
         5251  +          fts3SqlExecI64(&rc, p, SQL_DELETE_CONTENT, iRowid);
  5223   5252           }
  5224   5253           if( p->bHasDocsize ){
  5225         -          fts3SqlExec(&rc, p, SQL_DELETE_DOCSIZE, &pRowid);
         5254  +          fts3SqlExecI64(&rc, p, SQL_DELETE_DOCSIZE, iRowid);
  5226   5255           }
  5227   5256         }
  5228   5257       }
  5229   5258     }
  5230   5259   
  5231   5260     return rc;
  5232   5261   }
         5262  +
         5263  +/*
         5264  +** Convert a docid (iDocid) and a language id (iLangid) to a rowid,
         5265  +** according to the configured languageid_bits= value belonging to
         5266  +** FTS table *p.
         5267  +**
         5268  +** The conversion is as follows:
         5269  +**
         5270  +**   * The sign bit of iDocid becomes the sign bit of the rowid.
         5271  +**
         5272  +**   * iLangid is converted to an unsigned integer and stored in
         5273  +**     the next most significant Fts3Table.nLanguageidBits bits
         5274  +**     of the returned rowid.
         5275  +**
         5276  +**   * The least signficant (63-nLanguageidBits) of iDocid are
         5277  +**     copied to the (63-nLanguageidBits) least signifcant bits of
         5278  +**     the returned rowid.
         5279  +*/
         5280  +i64 sqlite3Fts3DocidToRowid(Fts3Table *p, i64 iDocid, int iLangid){
         5281  +  u64 iRet = iDocid;
         5282  +
         5283  +  if( p->nLanguageidBits ){
         5284  +    int iShift = (63 - p->nLanguageidBits);
         5285  +    u64 mask = ((((u64)1 << p->nLanguageidBits) - 1) << iShift);
         5286  +
         5287  +    iRet &= ~mask;
         5288  +    iRet |= (u64)iLangid << iShift;
         5289  +  }
         5290  +
         5291  +  assert( sqlite3Fts3RowidToDocid(p, (i64)iRet)==iDocid );
         5292  +  return (i64)iRet;
         5293  +}
         5294  +
         5295  +/*
         5296  +** Convert a rowid (iRowid) to a docid according to the languageid_bits=
         5297  +** value belonging to FTS table *p.
         5298  +*/
         5299  +i64 sqlite3Fts3RowidToDocid(Fts3Table *p, i64 iRowid){
         5300  +  u64 iRet = iRowid;
         5301  +  if( p->nLanguageidBits ){
         5302  +    static const u64 signbit = ((u64)1 << 63);
         5303  +    u64 mask = ((((u64)1 << p->nLanguageidBits)-1) << (63-p->nLanguageidBits));
         5304  +
         5305  +    if( iRet & signbit ){
         5306  +      iRet |= mask;
         5307  +    }else{
         5308  +      iRet &= ~mask;
         5309  +    }
         5310  +  }
         5311  +  return (i64)iRet;
         5312  +}
  5233   5313   
  5234   5314   /*
  5235   5315   ** This function does the work for the xUpdate method of FTS3 virtual
  5236   5316   ** tables. The schema of the virtual table being:
  5237   5317   **
  5238   5318   **     CREATE TABLE <table name>( 
  5239   5319   **       <user columns>,
  5240   5320   **       <table name> HIDDEN, 
  5241   5321   **       docid HIDDEN, 
  5242   5322   **       <langid> HIDDEN
  5243   5323   **     );
  5244   5324   **
  5245         -** 
  5246   5325   */
  5247   5326   int sqlite3Fts3UpdateMethod(
  5248   5327     sqlite3_vtab *pVtab,            /* FTS3 vtab object */
  5249   5328     int nArg,                       /* Size of argument array */
  5250   5329     sqlite3_value **apVal,          /* Array of arguments */
  5251   5330     sqlite_int64 *pRowid            /* OUT: The affected (or effected) rowid */
  5252   5331   ){
................................................................................
  5253   5332     Fts3Table *p = (Fts3Table *)pVtab;
  5254   5333     int rc = SQLITE_OK;             /* Return Code */
  5255   5334     int isRemove = 0;               /* True for an UPDATE or DELETE */
  5256   5335     u32 *aSzIns = 0;                /* Sizes of inserted documents */
  5257   5336     u32 *aSzDel = 0;                /* Sizes of deleted documents */
  5258   5337     int nChng = 0;                  /* Net change in number of documents */
  5259   5338     int bInsertDone = 0;
         5339  +  int iLangid = 0;
  5260   5340   
  5261   5341     assert( p->pSegments==0 );
  5262   5342     assert( 
  5263   5343         nArg==1                     /* DELETE operations */
  5264   5344      || nArg==(2 + p->nColumn + 3)  /* INSERT or UPDATE operations */
  5265   5345     );
  5266   5346   
................................................................................
  5272   5352      && sqlite3_value_type(apVal[0])==SQLITE_NULL 
  5273   5353      && sqlite3_value_type(apVal[p->nColumn+2])!=SQLITE_NULL 
  5274   5354     ){
  5275   5355       rc = fts3SpecialInsert(p, apVal[p->nColumn+2]);
  5276   5356       goto update_out;
  5277   5357     }
  5278   5358   
  5279         -  if( nArg>1 && sqlite3_value_int(apVal[2 + p->nColumn + 2])<0 ){
  5280         -    rc = SQLITE_CONSTRAINT;
  5281         -    goto update_out;
         5359  +  /* If this is an INSERT or UPDATE, check that the new value for the
         5360  +  ** languageid is within range. A languageid can never be a negative 
         5361  +  ** value. If the languageid_bits option was specified when this table 
         5362  +  ** was created, it must also be less than (2 ^ nLanguageidBits).  
         5363  +  **
         5364  +  ** Also check that if a non-zero languageid_bits value was configured,
         5365  +  ** the specified rowid value must be NULL.
         5366  +  */
         5367  +  if( nArg>1 ){
         5368  +    i64 iLangid64 = sqlite3_value_int64(apVal[2 + p->nColumn + 2]);
         5369  +    if( iLangid64<0 
         5370  +     || (p->nLanguageidBits && iLangid64>=((i64)1<<p->nLanguageidBits)) 
         5371  +    ){
         5372  +      rc = SQLITE_CONSTRAINT;
         5373  +      goto update_out;
         5374  +    }
         5375  +    iLangid = (int)iLangid64;
         5376  +
         5377  +    if( p->nLanguageidBits 
         5378  +     && sqlite3_value_type(apVal[0])==SQLITE_NULL
         5379  +     && sqlite3_value_type(apVal[1])!=SQLITE_NULL
         5380  +    ){
         5381  +      rc = SQLITE_CONSTRAINT;
         5382  +      goto update_out;
         5383  +    }
  5282   5384     }
  5283   5385   
  5284   5386     /* Allocate space to hold the change in document sizes */
  5285   5387     aSzDel = sqlite3_malloc( sizeof(aSzDel[0])*(p->nColumn+1)*2 );
  5286   5388     if( aSzDel==0 ){
  5287   5389       rc = SQLITE_NOMEM;
  5288   5390       goto update_out;
................................................................................
  5300   5402     ** should be deleted from the database before inserting the new row. Or,
  5301   5403     ** if the on-conflict mode is other than REPLACE, then this method must
  5302   5404     ** detect the conflict and return SQLITE_CONSTRAINT before beginning to
  5303   5405     ** modify the database file.
  5304   5406     */
  5305   5407     if( nArg>1 && p->zContentTbl==0 ){
  5306   5408       /* Find the value object that holds the new rowid value. */
  5307         -    sqlite3_value *pNewRowid = apVal[3+p->nColumn];
  5308         -    if( sqlite3_value_type(pNewRowid)==SQLITE_NULL ){
  5309         -      pNewRowid = apVal[1];
         5409  +    sqlite3_value *pNewDocid = apVal[3+p->nColumn];
         5410  +    if( sqlite3_value_type(pNewDocid)==SQLITE_NULL ){
         5411  +      if( p->nLanguageidBits ){
         5412  +        rc = SQLITE_CONSTRAINT;
         5413  +        goto update_out;
         5414  +      }
         5415  +      pNewDocid = apVal[1];
         5416  +    }else if( sqlite3_value_type(apVal[0])==SQLITE_NULL 
         5417  +           && sqlite3_value_type(apVal[1])!=SQLITE_NULL 
         5418  +    ){
         5419  +      rc = SQLITE_ERROR;
         5420  +      goto update_out;
  5310   5421       }
  5311   5422   
  5312         -    if( sqlite3_value_type(pNewRowid)!=SQLITE_NULL && ( 
  5313         -        sqlite3_value_type(apVal[0])==SQLITE_NULL
  5314         -     || sqlite3_value_int64(apVal[0])!=sqlite3_value_int64(pNewRowid)
  5315         -    )){
  5316         -      /* The new rowid is not NULL (in this case the rowid will be
  5317         -      ** automatically assigned and there is no chance of a conflict), and 
  5318         -      ** the statement is either an INSERT or an UPDATE that modifies the
  5319         -      ** rowid column. So if the conflict mode is REPLACE, then delete any
  5320         -      ** existing row with rowid=pNewRowid. 
  5321         -      **
  5322         -      ** Or, if the conflict mode is not REPLACE, insert the new record into 
  5323         -      ** the %_content table. If we hit the duplicate rowid constraint (or any
  5324         -      ** other error) while doing so, return immediately.
  5325         -      **
  5326         -      ** This branch may also run if pNewRowid contains a value that cannot
  5327         -      ** be losslessly converted to an integer. In this case, the eventual 
  5328         -      ** call to fts3InsertData() (either just below or further on in this
  5329         -      ** function) will return SQLITE_MISMATCH. If fts3DeleteByRowid is 
  5330         -      ** invoked, it will delete zero rows (since no row will have
  5331         -      ** docid=$pNewRowid if $pNewRowid is not an integer value).
  5332         -      */
  5333         -      if( sqlite3_vtab_on_conflict(p->db)==SQLITE_REPLACE ){
  5334         -        rc = fts3DeleteByRowid(p, pNewRowid, &nChng, aSzDel);
  5335         -      }else{
  5336         -        rc = fts3InsertData(p, apVal, pRowid);
  5337         -        bInsertDone = 1;
         5423  +    if( sqlite3_value_type(pNewDocid)!=SQLITE_NULL ){
         5424  +      int e = sqlite3_value_numeric_type(pNewDocid);
         5425  +      i64 iRowid = sqlite3_value_int64(pNewDocid);
         5426  +
         5427  +      /* Check that the value specified by the user may be losslessly
         5428  +      ** converted to an integer. If not, return a "data mismatch" error.  */
         5429  +      if( (e!=SQLITE_INTEGER)
         5430  +       && (e!=SQLITE_FLOAT || (double)iRowid!=sqlite3_value_double(pNewDocid))
         5431  +      ){
         5432  +        rc = SQLITE_MISMATCH;
         5433  +        goto update_out;
         5434  +      }
         5435  +
         5436  +      if( p->nLanguageidBits ){
         5437  +        /* Check for an out-of-range docid value. */
         5438  +        if( iRowid>=((i64)1 << (63 - p->nLanguageidBits)) 
         5439  +         || iRowid<-1*((i64)1 << (63 - p->nLanguageidBits)) 
         5440  +        ){
         5441  +          rc = SQLITE_CONSTRAINT;
         5442  +          goto update_out;
         5443  +        }
         5444  +
         5445  +        iRowid = sqlite3Fts3DocidToRowid(p, iRowid, iLangid);
         5446  +      }
         5447  +
         5448  +      if( sqlite3_value_type(apVal[0])==SQLITE_NULL
         5449  +       || sqlite3_value_int64(apVal[0])!=iRowid
         5450  +      ){
         5451  +        /* The new rowid is not NULL (in this case the rowid will be
         5452  +        ** automatically assigned and there is no chance of a conflict), and 
         5453  +        ** the statement is either an INSERT or an UPDATE that modifies the
         5454  +        ** rowid column. So if the conflict mode is REPLACE, then delete any
         5455  +        ** existing row with rowid=pNewRowid. 
         5456  +        **
         5457  +        ** Or, if the conflict mode is not REPLACE, insert the new record into 
         5458  +        ** the %_content table. If we hit the duplicate rowid constraint (or 
         5459  +        ** any other error) while doing so, return immediately.
         5460  +        **
         5461  +        ** This branch may also run if pNewRowid contains a value that cannot
         5462  +        ** be losslessly converted to an integer. In this case, the eventual 
         5463  +        ** call to fts3InsertData() (either just below or further on in this
         5464  +        ** function) will return SQLITE_MISMATCH. If fts3DeleteByRowid is 
         5465  +        ** invoked, it will delete zero rows (since no row will have
         5466  +        ** docid=$pNewRowid if $pNewRowid is not an integer value).
         5467  +        */
         5468  +        if( sqlite3_vtab_on_conflict(p->db)==SQLITE_REPLACE ){
         5469  +          rc = fts3DeleteByRowid(p, iRowid, &nChng, aSzDel);
         5470  +        }else{
         5471  +          rc = fts3InsertData(p, apVal, 0, iRowid);
         5472  +          bInsertDone = 1;
         5473  +          *pRowid = iRowid;
         5474  +        }
  5338   5475         }
  5339   5476       }
  5340   5477     }
  5341   5478     if( rc!=SQLITE_OK ){
  5342   5479       goto update_out;
  5343   5480     }
  5344   5481   
  5345   5482     /* If this is a DELETE or UPDATE operation, remove the old record. */
  5346   5483     if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){
  5347   5484       assert( sqlite3_value_type(apVal[0])==SQLITE_INTEGER );
  5348         -    rc = fts3DeleteByRowid(p, apVal[0], &nChng, aSzDel);
         5485  +    rc = fts3DeleteByRowid(p, sqlite3_value_int64(apVal[0]), &nChng, aSzDel);
  5349   5486       isRemove = 1;
  5350   5487     }
  5351   5488     
  5352   5489     /* If this is an INSERT or UPDATE operation, insert the new record. */
  5353   5490     if( nArg>1 && rc==SQLITE_OK ){
  5354         -    int iLangid = sqlite3_value_int(apVal[2 + p->nColumn + 2]);
  5355   5491       if( bInsertDone==0 ){
  5356         -      rc = fts3InsertData(p, apVal, pRowid);
         5492  +      rc = fts3InsertData(p, apVal, pRowid, 0);
  5357   5493         if( rc==SQLITE_CONSTRAINT && p->zContentTbl==0 ){
  5358   5494           rc = FTS_CORRUPT_VTAB;
  5359   5495         }
  5360   5496       }
  5361   5497       if( rc==SQLITE_OK && (!isRemove || *pRowid!=p->iPrevDocid ) ){
  5362   5498         rc = fts3PendingTermsDocid(p, iLangid, *pRowid);
  5363   5499       }

Added test/fts4langid2.test.

            1  +# 2012 March 01
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#*************************************************************************
           11  +# This file implements regression tests for SQLite library.  The
           12  +# focus of this script is testing the languageid=xxx FTS4 option.
           13  +#
           14  +
           15  +set testdir [file dirname $argv0]
           16  +source $testdir/tester.tcl
           17  +set ::testprefix fts4langid2
           18  +
           19  +# If SQLITE_ENABLE_FTS3 is defined, omit this file.
           20  +ifcapable !fts3 {
           21  +  finish_test
           22  +  return
           23  +}
           24  +
           25  +#-------------------------------------------------------------------------
           26  +# Test out-of-range values for the languageid_bits= parameter.
           27  +#
           28  +do_catchsql_test 1.1 {
           29  +  CREATE VIRTUAL TABLE t1 USING fts4(languageid=lid, languageid_bits=31);
           30  +} {1 {languageid_bits parameter out of range}}
           31  +
           32  +do_catchsql_test 1.2 {
           33  +  CREATE VIRTUAL TABLE t1 USING fts4(languageid=lid, languageid_bits=-1);
           34  +} {1 {languageid_bits parameter out of range}}
           35  +
           36  +do_catchsql_test 1.3 {
           37  +  CREATE VIRTUAL TABLE t1 USING fts4(languageid=lid, languageid_bits=0);
           38  +  CREATE VIRTUAL TABLE t2 USING fts4(languageid=lid, languageid_bits=30);
           39  +} {0 {}}
           40  +
           41  +do_execsql_test 1.4 {
           42  +  DROP TABLE t1;
           43  +  DROP TABLE t2;
           44  +}
           45  +
           46  +#-------------------------------------------------------------------------
           47  +# Test out-of-range values in the languageid column.
           48  +#
           49  +do_execsql_test 2.1 {
           50  +  CREATE VIRTUAL TABLE t1 USING fts4(languageid=lid, languageid_bits=8);
           51  +  CREATE VIRTUAL TABLE t2 USING fts4(languageid=lid, languageid_bits=7);
           52  +}
           53  +
           54  +do_catchsql_test 2.2 {
           55  +  INSERT INTO t1(docid, lid, content) VALUES(1, 256, 'abc def');
           56  +} {1 {constraint failed}}
           57  +
           58  +do_catchsql_test 2.3 {
           59  +  INSERT INTO t2(docid, lid, content) VALUES(1, 128, 'abc def');
           60  +} {1 {constraint failed}}
           61  +
           62  +do_catchsql_test 2.3 {
           63  +  INSERT INTO t1(docid, lid, content) VALUES(1, -1, 'abc def');
           64  +} {1 {constraint failed}}
           65  +
           66  +do_execsql_test 2.4 {
           67  +  DROP TABLE t1;
           68  +  DROP TABLE t2;
           69  +}
           70  +
           71  +#-------------------------------------------------------------------------
           72  +# Test that if languageid_bits is set to a non-zero value it is
           73  +# not possible to specify a non-NULL rowid, even if it is the same
           74  +# as the docid.
           75  +#
           76  +do_execsql_test 3.1 {
           77  +  CREATE VIRTUAL TABLE t1 USING fts4(languageid=lid, languageid_bits=4);
           78  +  CREATE VIRTUAL TABLE t2 USING fts4(languageid=lid, languageid_bits=0);
           79  +}
           80  +  
           81  +do_catchsql_test 3.2.1 {
           82  +  INSERT INTO t1(rowid, lid, content) VALUES(1, 0, 'abc def');
           83  +} {1 {constraint failed}}
           84  +
           85  +do_catchsql_test 3.2.2 {
           86  +  INSERT INTO t2(rowid, lid, content) VALUES(1, 0, 'abc def');
           87  +} {0 {}}
           88  +
           89  +do_catchsql_test 3.3 {
           90  +  INSERT INTO t1(rowid, docid, lid, content) VALUES(2, 2, 0, 'abc def');
           91  +} {1 {constraint failed}}
           92  +
           93  +do_catchsql_test 3.4 {
           94  +  INSERT INTO t1(lid, content) VALUES(0, 'one two def');
           95  +} {1 {constraint failed}}
           96  +
           97  +do_execsql_test 3.4 {
           98  +  DROP TABLE t1;
           99  +  DROP TABLE t2;
          100  +}
          101  +
          102  +#-------------------------------------------------------------------------
          103  +# Simple tests inserting data with multiple languageid values.
          104  +#
          105  +do_execsql_test 4.1 {
          106  +  CREATE VIRTUAL TABLE t1 USING fts4(languageid=lid, languageid_bits=5);
          107  +}
          108  +
          109  +do_execsql_test 4.2 {
          110  +  INSERT INTO t1 (docid, lid, content) VALUES(1, 0, '1 2 3');
          111  +  INSERT INTO t1 (docid, lid, content) VALUES(1, 1, '1 2 3 4');
          112  +}
          113  +
          114  +do_execsql_test 4.3 {
          115  +  SELECT docid, lid FROM t1;
          116  +} {1 0 1 1}
          117  +
          118  +do_execsql_test 4.4 {
          119  +  SELECT docid, lid, content FROM t1 WHERE t1 MATCH '2';
          120  +} {1 0 {1 2 3}}
          121  +
          122  +do_execsql_test 4.5 {
          123  +  SELECT docid, lid, content FROM t1 WHERE t1 MATCH '2' AND lid=1;
          124  +} {1 1 {1 2 3 4}}
          125  +
          126  +do_execsql_test 4.6 {
          127  +  UPDATE t1 SET content = 'x y z' || lid;
          128  +  SELECT docid, lid FROM t1;
          129  +} {1 0 1 1}
          130  +
          131  +do_execsql_test 3.4 {
          132  +  DROP TABLE t1;
          133  +}
          134  +
          135  +#-------------------------------------------------------------------------
          136  +# Tests for docid range boundary conditions. 
          137  +#
          138  +for {set bits 1} {$bits <= 30} {incr bits} {
          139  +  do_execsql_test 5.$bits.1 "
          140  +    CREATE VIRTUAL TABLE t1 USING fts4(languageid=lid, languageid_bits=$bits);
          141  +  "
          142  +
          143  +  set max_docid [expr int(1<<(63-$bits))-1]
          144  +  set min_docid [expr -1*int(1<<(63-$bits))]
          145  +  set max_langid [expr (1<<$bits)-1]
          146  +  set min_langid 0
          147  +
          148  +
          149  +  do_catchsql_test 5.$bits.2.1 {
          150  +    INSERT INTO t1(docid, lid, content) VALUES($max_docid+1, 4, '');
          151  +  } {1 {constraint failed}}
          152  +  do_catchsql_test 5.$bits.2.2 {
          153  +    INSERT INTO t1(docid, lid, content) VALUES($min_docid-1, 4, '');
          154  +  } {1 {constraint failed}}
          155  +
          156  +  do_test 5.$bits.3 {
          157  +    foreach {a b c} "
          158  +      $min_docid  $min_langid  {1 min min x}
          159  +      $min_docid  $max_langid  {2 min max x}
          160  +      $max_docid  $min_langid  {3 max min x}
          161  +      $max_docid  $max_langid  {4 max max x}
          162  +    " {
          163  +      execsql { INSERT INTO t1(docid, lid, content) VALUES($a, $b, $c) }
          164  +    }
          165  +  } {}
          166  +
          167  +  do_execsql_test 5.$bits.4.1 {
          168  +    SELECT docid, lid, content FROM t1 ORDER BY content
          169  +  } "
          170  +      $min_docid  $min_langid  {1 min min x}
          171  +      $min_docid  $max_langid  {2 min max x}
          172  +      $max_docid  $min_langid  {3 max min x}
          173  +      $max_docid  $max_langid  {4 max max x}
          174  +  "
          175  +
          176  +  do_execsql_test 5.$bits.4.2 {
          177  +    SELECT docid, lid, content FROM t1 WHERE lid=$min_langid AND t1 MATCH 'x'
          178  +  } "
          179  +      $min_docid  $min_langid  {1 min min x}
          180  +      $max_docid  $min_langid  {3 max min x}
          181  +  "
          182  +
          183  +  do_execsql_test 5.$bits.4.3 {
          184  +    SELECT docid, lid, content FROM t1 WHERE lid=$max_langid AND t1 MATCH 'x'
          185  +  } "
          186  +      $min_docid  $max_langid  {2 min max x}
          187  +      $max_docid  $max_langid  {4 max max x}
          188  +  "
          189  +
          190  +  do_execsql_test 5.$bits.4.4 {
          191  +    SELECT docid, lid, content FROM t1 WHERE t1 MATCH '1'
          192  +  } "
          193  +      $min_docid  $min_langid  {1 min min x}
          194  +  "
          195  +
          196  +  do_execsql_test 5.$bits.5 { DROP TABLE t1 }
          197  +}
          198  +
          199  +#-------------------------------------------------------------------------
          200  +# Tests for auxilliary functions with langaugeid_bits tables.
          201  +#
          202  +proc mit {blob} {
          203  +  set scan(littleEndian) i*
          204  +  set scan(bigEndian) I*
          205  +  binary scan $blob $scan($::tcl_platform(byteOrder)) r
          206  +  return $r
          207  +}
          208  +db func mit mit
          209  +
          210  +do_execsql_test 6.1 {
          211  +  CREATE VIRTUAL TABLE t1 USING fts4(languageid_bits=4, languageid=lid);
          212  +  INSERT INTO t1(docid,lid,content) VALUES(1, 1, 'one two three four');
          213  +  INSERT INTO t1(docid,lid,content) VALUES(2, 1, 'two three four five');
          214  +  INSERT INTO t1(docid,lid,content) VALUES(3, 1, 'three four five six');
          215  +  INSERT INTO t1(docid,lid,content) VALUES(4, 1, 'four five six seven');
          216  +
          217  +  INSERT INTO t1(docid,lid,content) VALUES(1, 2, 'four three two one');
          218  +  INSERT INTO t1(docid,lid,content) VALUES(2, 2, 'five four three two');
          219  +  INSERT INTO t1(docid,lid,content) VALUES(3, 2, 'six five four three');
          220  +  INSERT INTO t1(docid,lid,content) VALUES(4, 2, 'A B C D');
          221  +}
          222  +
          223  +do_execsql_test 6.2.1 {
          224  +  SELECT docid, snippet(t1) FROM t1 WHERE t1 MATCH 'one' AND lid=1;
          225  +} {1 {<b>one</b> two three four}}
          226  +do_execsql_test 6.2.2 {
          227  +  SELECT docid, snippet(t1) FROM t1 WHERE t1 MATCH 'one' AND lid=2;
          228  +} {1 {four three two <b>one</b>}}
          229  +
          230  +do_execsql_test 6.2.1 {
          231  +  SELECT docid, offsets(t1) FROM t1 WHERE t1 MATCH 'two' AND lid=1;
          232  +} {1 {0 0 4 3} 2 {0 0 0 3}}
          233  +do_execsql_test 6.2.2 {
          234  +  SELECT docid, offsets(t1) FROM t1 WHERE t1 MATCH 'two' AND lid=2;
          235  +} {1 {0 0 11 3} 2 {0 0 16 3}}
          236  +
          237  +do_execsql_test 6.3.1 {
          238  +  SELECT docid, mit(matchinfo(t1)) FROM t1 WHERE t1 MATCH 'two' AND lid=1;
          239  +} {1 {1 1 1 2 2} 2 {1 1 1 2 2}}
          240  +do_execsql_test 6.3.2 {
          241  +  SELECT docid, mit(matchinfo(t1)) FROM t1 WHERE t1 MATCH 'two' AND lid=2;
          242  +} {1 {1 1 1 2 2} 2 {1 1 1 2 2}}
          243  +do_execsql_test 6.3.3 {
          244  +  SELECT docid, mit(matchinfo(t1)) FROM t1 WHERE t1 MATCH 'B' AND lid=1;
          245  +} {}
          246  +do_execsql_test 6.3.4 {
          247  +  SELECT docid, mit(matchinfo(t1)) FROM t1 WHERE t1 MATCH 'B' AND lid=2;
          248  +} {4 {1 1 1 1 1}}
          249  +
          250  +do_execsql_test 6.4 {
          251  +  CREATE VIRTUAL TABLE t2 USING fts4(languageid_bits=8, languageid=lid);
          252  +  INSERT INTO t2(docid,lid,content) VALUES(-1, 0, 'A B C D');
          253  +  INSERT INTO t2(docid,lid,content) VALUES(-2, 0, 'D C B A');
          254  +  INSERT INTO t2(docid,lid,content) VALUES(-3, 0, 'C B D A');
          255  +  INSERT INTO t2(docid,lid,content) VALUES(-4, 0, 'A D B C');
          256  +
          257  +  INSERT INTO t2(docid,lid,content) VALUES(-1, 1, 'A A A A');
          258  +  INSERT INTO t2(docid,lid,content) VALUES(-2, 1, 'B B B B');
          259  +  INSERT INTO t2(docid,lid,content) VALUES(-3, 1, 'C C C C');
          260  +  INSERT INTO t2(docid,lid,content) VALUES(-4, 1, 'D D D D');
          261  +}
          262  +
          263  +do_execsql_test 6.4.1 {
          264  +  SELECT docid, mit(matchinfo(t2)) FROM t2 WHERE t2 MATCH 'B';
          265  +} {
          266  +  -4 {1 1 1 4 4}
          267  +  -3 {1 1 1 4 4}
          268  +  -2 {1 1 1 4 4}
          269  +  -1 {1 1 1 4 4}
          270  +}
          271  +do_execsql_test 6.4.2 {
          272  +  SELECT docid, mit(matchinfo(t2)) FROM t2 WHERE t2 MATCH 'B' AND lid=1;
          273  +} {-2 {1 1 4 4 1}}
          274  +
          275  +do_execsql_test 6.5 {
          276  +  DROP TABLE t1;
          277  +  DROP TABLE t2;
          278  +}
          279  +
          280  +#-------------------------------------------------------------------------
          281  +# Tests for querying by docid.
          282  +#
          283  +do_execsql_test 7.1 {
          284  +  CREATE VIRTUAL TABLE t1 USING fts4(languageid_bits=8, languageid=lid);
          285  +  INSERT INTO t1(docid,lid,content) VALUES(10, 10, 'abc def');
          286  +}
          287  +
          288  +do_execsql_test 7.2 {
          289  +  SELECT docid,lid,content FROM t1 WHERE docid=10;
          290  +} {10 10 {abc def}}
          291  +
          292  +do_execsql_test 7.3 {
          293  +  SELECT docid,lid,content FROM t1 WHERE docid<11;
          294  +} {10 10 {abc def}}
          295  +
          296  +do_execsql_test 7.4 {
          297  +  DROP TABLE t1;
          298  +}
          299  +
          300  +#-------------------------------------------------------------------------
          301  +# Tests for sorting by docid.
          302  +#
          303  +do_execsql_test 8.1 {
          304  +  CREATE VIRTUAL TABLE t1 USING fts4(languageid_bits=6, languageid=lid);
          305  +  INSERT INTO t1 (docid,lid,content) VALUES(1, 0, 'abc def');
          306  +  INSERT INTO t1 (docid,lid,content) VALUES(3, 0, 'abc ghi');
          307  +  INSERT INTO t1 (docid,lid,content) VALUES(2, 0, 'def ghi');
          308  +
          309  +  INSERT INTO t1 (docid,lid,content) VALUES(1, 5, 'A B');
          310  +  INSERT INTO t1 (docid,lid,content) VALUES(3, 5, 'A C');
          311  +  INSERT INTO t1 (docid,lid,content) VALUES(2, 5, 'B C');
          312  +}
          313  +
          314  +do_execsql_test 8.2 {
          315  +  SELECT docid FROM t1 ORDER BY docid;
          316  +} {1 1 2 2 3 3}
          317  +do_execsql_test 8.3 {
          318  +  SELECT docid FROM t1 WHERE t1 MATCH 'ghi' ORDER BY docid;
          319  +} {2 3}
          320  +do_execsql_test 8.4 {
          321  +  SELECT docid FROM t1 WHERE t1 MATCH 'ghi' ORDER BY docid DESC;
          322  +} {3 2}
          323  +
          324  +# Test that the docid and languageid fields may be updated.
          325  +#
          326  +do_execsql_test 8.5 { UPDATE t1 SET docid=docid+3, lid=0 WHERE lid=5; }
          327  +do_execsql_test 8.6 { SELECT docid FROM t1 ORDER BY docid; } {1 2 3 4 5 6}
          328  +do_execsql_test 8.7 { SELECT docid FROM t1 WHERE t1 MATCH 'A' } {4 6}
          329  +do_execsql_test 8.8 { SELECT docid FROM t1 WHERE t1 MATCH 'A' AND lid=5 } {}
          330  +
          331  +finish_test
          332  +

Changes to test/permutations.test.

   189    189     fts3expr3.test
   190    190     fts3near.test fts3query.test fts3shared.test fts3snippet.test 
   191    191     fts3sort.test
   192    192     fts3fault.test fts3malloc.test fts3matchinfo.test
   193    193     fts3aux1.test fts3comp1.test fts3auto.test
   194    194     fts4aa.test fts4content.test
   195    195     fts3conf.test fts3prefix.test fts3fault2.test fts3corrupt.test
   196         -  fts3corrupt2.test fts3first.test fts4langid.test fts4merge.test
          196  +  fts3corrupt2.test fts3first.test fts4langid.test fts4langid2.test
          197  +  fts4merge.test
   197    198     fts4check.test fts4unicode.test
   198    199   }
   199    200   
   200    201   test_suite "nofaultsim" -prefix "" -description {
   201    202     "Very" quick test suite. Runs in less than 5 minutes on a workstation. 
   202    203     This test suite is the same as the "quick" tests, except that some files
   203    204     that test malloc and IO errors are omitted.