/ Check-in [9d418a7a49]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Allow fts5 to filter on multiple MATCH clauses in a single scan.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA3-256: 9d418a7a491761eeb38a70898677a493e2631e5d62e75ee88431f52d3dfd2344
User & Date: dan 2019-09-12 19:38:40
References
2019-09-13
13:23
Ensure that the idxStr for FTS5 is always zero-terminated. Fix for check-in [9d418a7a491761ee] check-in: 090cd07d37 user: drh tags: trunk
Context
2019-09-13
12:24
Fix harmless compiler warnings. check-in: a8927d14f8 user: drh tags: trunk
2019-09-12
19:38
Allow fts5 to filter on multiple MATCH clauses in a single scan. check-in: 9d418a7a49 user: dan tags: trunk
2019-09-11
15:25
Fix typo for one instance of line number handling in the Lemon tool. check-in: 980be1730d user: mistachkin tags: trunk
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to ext/fts5/fts5Int.h.

   691    691   */
   692    692   int sqlite3Fts5ExprFirst(Fts5Expr*, Fts5Index *pIdx, i64 iMin, int bDesc);
   693    693   int sqlite3Fts5ExprNext(Fts5Expr*, i64 iMax);
   694    694   int sqlite3Fts5ExprEof(Fts5Expr*);
   695    695   i64 sqlite3Fts5ExprRowid(Fts5Expr*);
   696    696   
   697    697   void sqlite3Fts5ExprFree(Fts5Expr*);
          698  +int sqlite3Fts5ExprAnd(Fts5Expr **pp1, Fts5Expr *p2);
   698    699   
   699    700   /* Called during startup to register a UDF with SQLite */
   700    701   int sqlite3Fts5ExprInit(Fts5Global*, sqlite3*);
   701    702   
   702    703   int sqlite3Fts5ExprPhraseCount(Fts5Expr*);
   703    704   int sqlite3Fts5ExprPhraseSize(Fts5Expr*, int iPhrase);
   704    705   int sqlite3Fts5ExprPoslist(Fts5Expr*, int, const u8 **);

Changes to ext/fts5/fts5_config.c.

   679    679     );
   680    680   
   681    681     assert( zSql || rc==SQLITE_NOMEM );
   682    682     if( zSql ){
   683    683       rc = sqlite3_declare_vtab(pConfig->db, zSql);
   684    684       sqlite3_free(zSql);
   685    685     }
   686         -  
          686  + 
   687    687     return rc;
   688    688   }
   689    689   
   690    690   /*
   691    691   ** Tokenize the text passed via the second and third arguments.
   692    692   **
   693    693   ** The callback is invoked once for each token in the input text. The

Changes to ext/fts5/fts5_expr.c.

   304    304   void sqlite3Fts5ExprFree(Fts5Expr *p){
   305    305     if( p ){
   306    306       sqlite3Fts5ParseNodeFree(p->pRoot);
   307    307       sqlite3_free(p->apExprPhrase);
   308    308       sqlite3_free(p);
   309    309     }
   310    310   }
          311  +
          312  +int sqlite3Fts5ExprAnd(Fts5Expr **pp1, Fts5Expr *p2){
          313  +  Fts5Parse sParse;
          314  +  memset(&sParse, 0, sizeof(sParse));
          315  +
          316  +  if( *pp1 ){
          317  +    Fts5Expr *p1 = *pp1;
          318  +    int nPhrase = p1->nPhrase + p2->nPhrase;
          319  +
          320  +    p1->pRoot = sqlite3Fts5ParseNode(&sParse, FTS5_AND, p1->pRoot, p2->pRoot,0);
          321  +    p2->pRoot = 0;
          322  +
          323  +    if( sParse.rc==SQLITE_OK ){
          324  +      Fts5ExprPhrase **ap = (Fts5ExprPhrase**)sqlite3_realloc(
          325  +          p1->apExprPhrase, nPhrase * sizeof(Fts5ExprPhrase*)
          326  +      );
          327  +      if( ap==0 ){
          328  +        sParse.rc = SQLITE_NOMEM;
          329  +      }else{
          330  +        int i;
          331  +        memmove(&ap[p2->nPhrase], ap, p1->nPhrase*sizeof(Fts5ExprPhrase*));
          332  +        for(i=0; i<p2->nPhrase; i++){
          333  +          ap[i] = p2->apExprPhrase[i];
          334  +        }
          335  +        p1->nPhrase = nPhrase;
          336  +        p1->apExprPhrase = ap;
          337  +      }
          338  +    }
          339  +    sqlite3_free(p2->apExprPhrase);
          340  +    sqlite3_free(p2);
          341  +  }else{
          342  +    *pp1 = p2;
          343  +  }
          344  +
          345  +  return sParse.rc;
          346  +}
   311    347   
   312    348   /*
   313    349   ** Argument pTerm must be a synonym iterator. Return the current rowid
   314    350   ** that it points to.
   315    351   */
   316    352   static i64 fts5ExprSynonymRowid(Fts5ExprTerm *pTerm, int bDesc, int *pbEof){
   317    353     i64 iRet = 0;

Changes to ext/fts5/fts5_main.c.

   461    461   #endif
   462    462   }
   463    463   
   464    464   /*
   465    465   ** Implementation of the xBestIndex method for FTS5 tables. Within the 
   466    466   ** WHERE constraint, it searches for the following:
   467    467   **
   468         -**   1. A MATCH constraint against the special column.
          468  +**   1. A MATCH constraint against the table column.
   469    469   **   2. A MATCH constraint against the "rank" column.
   470         -**   3. An == constraint against the rowid column.
   471         -**   4. A < or <= constraint against the rowid column.
   472         -**   5. A > or >= constraint against the rowid column.
          470  +**   3. A MATCH constraint against some other column.
          471  +**   4. An == constraint against the rowid column.
          472  +**   5. A < or <= constraint against the rowid column.
          473  +**   6. A > or >= constraint against the rowid column.
   473    474   **
   474         -** Within the ORDER BY, either:
          475  +** Within the ORDER BY, the following are supported:
   475    476   **
   476    477   **   5. ORDER BY rank [ASC|DESC]
   477    478   **   6. ORDER BY rowid [ASC|DESC]
          479  +**
          480  +** Information for the xFilter call is passed via both the idxNum and 
          481  +** idxStr variables. Specifically, idxNum is a bitmask of the following
          482  +** flags used to encode the ORDER BY clause:
          483  +**
          484  +**     FTS5_BI_ORDER_RANK
          485  +**     FTS5_BI_ORDER_ROWID
          486  +**     FTS5_BI_ORDER_DESC
          487  +**
          488  +** idxStr is used to encode data from the WHERE clause. For each argument
          489  +** passed to the xFilter method, the following is appended to idxStr:
          490  +**
          491  +**   Match against table column:            "m"
          492  +**   Match against rank column:             "r"
          493  +**   Match against other column:            "<column-number>"
          494  +**   Equality constraint against the rowid: "="
          495  +**   A < or <= against the rowid:           "<"
          496  +**   A > or >= against the rowid:           ">"
          497  +**
          498  +** This function ensures that there is at most one "r" or "=". And that if
          499  +** there exists an "=" then there is no "<" or ">".
   478    500   **
   479    501   ** Costs are assigned as follows:
   480    502   **
   481    503   **  a) If an unusable MATCH operator is present in the WHERE clause, the
   482    504   **     cost is unconditionally set to 1e50 (a really big number).
   483    505   **
   484    506   **  a) If a MATCH operator is present, the cost depends on the other
................................................................................
   499    521   ** Costs are not modified by the ORDER BY clause.
   500    522   */
   501    523   static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){
   502    524     Fts5Table *pTab = (Fts5Table*)pVTab;
   503    525     Fts5Config *pConfig = pTab->pConfig;
   504    526     const int nCol = pConfig->nCol;
   505    527     int idxFlags = 0;               /* Parameter passed through to xFilter() */
   506         -  int bHasMatch;
   507         -  int iNext;
   508    528     int i;
   509    529   
   510         -  struct Constraint {
   511         -    int op;                       /* Mask against sqlite3_index_constraint.op */
   512         -    int fts5op;                   /* FTS5 mask for idxFlags */
   513         -    int iCol;                     /* 0==rowid, 1==tbl, 2==rank */
   514         -    int omit;                     /* True to omit this if found */
   515         -    int iConsIndex;               /* Index in pInfo->aConstraint[] */
   516         -  } aConstraint[] = {
   517         -    {SQLITE_INDEX_CONSTRAINT_MATCH|SQLITE_INDEX_CONSTRAINT_EQ, 
   518         -                                    FTS5_BI_MATCH,    1, 1, -1},
   519         -    {SQLITE_INDEX_CONSTRAINT_MATCH|SQLITE_INDEX_CONSTRAINT_EQ, 
   520         -                                    FTS5_BI_RANK,     2, 1, -1},
   521         -    {SQLITE_INDEX_CONSTRAINT_EQ,    FTS5_BI_ROWID_EQ, 0, 0, -1},
   522         -    {SQLITE_INDEX_CONSTRAINT_LT|SQLITE_INDEX_CONSTRAINT_LE, 
   523         -                                    FTS5_BI_ROWID_LE, 0, 0, -1},
   524         -    {SQLITE_INDEX_CONSTRAINT_GT|SQLITE_INDEX_CONSTRAINT_GE, 
   525         -                                    FTS5_BI_ROWID_GE, 0, 0, -1},
   526         -  };
          530  +  char *idxStr;
          531  +  int iIdxStr = 0;
          532  +  int iCons = 0;
   527    533   
   528         -  int aColMap[3];
   529         -  aColMap[0] = -1;
   530         -  aColMap[1] = nCol;
   531         -  aColMap[2] = nCol+1;
          534  +  int bSeenEq = 0;
          535  +  int bSeenGt = 0;
          536  +  int bSeenLt = 0;
          537  +  int bSeenMatch = 0;
          538  +  int bSeenRank = 0;
          539  +
   532    540   
   533    541     assert( SQLITE_INDEX_CONSTRAINT_EQ<SQLITE_INDEX_CONSTRAINT_MATCH );
   534    542     assert( SQLITE_INDEX_CONSTRAINT_GT<SQLITE_INDEX_CONSTRAINT_MATCH );
   535    543     assert( SQLITE_INDEX_CONSTRAINT_LE<SQLITE_INDEX_CONSTRAINT_MATCH );
   536    544     assert( SQLITE_INDEX_CONSTRAINT_GE<SQLITE_INDEX_CONSTRAINT_MATCH );
   537    545     assert( SQLITE_INDEX_CONSTRAINT_LE<SQLITE_INDEX_CONSTRAINT_MATCH );
   538    546   
................................................................................
   539    547     if( pConfig->bLock ){
   540    548       pTab->base.zErrMsg = sqlite3_mprintf(
   541    549           "recursively defined fts5 content table"
   542    550       );
   543    551       return SQLITE_ERROR;
   544    552     }
   545    553   
   546         -  /* Set idxFlags flags for all WHERE clause terms that will be used. */
          554  +  idxStr = (char*)sqlite3_malloc(pInfo->nConstraint * 6 + 1);
          555  +  if( idxStr==0 ) return SQLITE_NOMEM;
          556  +  pInfo->idxStr = idxStr;
          557  +  pInfo->needToFreeIdxStr = 1;
          558  +
   547    559     for(i=0; i<pInfo->nConstraint; i++){
   548    560       struct sqlite3_index_constraint *p = &pInfo->aConstraint[i];
   549    561       int iCol = p->iColumn;
   550         -
   551         -    if( (p->op==SQLITE_INDEX_CONSTRAINT_MATCH && iCol>=0 && iCol<=nCol)
   552         -     || (p->op==SQLITE_INDEX_CONSTRAINT_EQ && iCol==nCol)
          562  +    if( p->op==SQLITE_INDEX_CONSTRAINT_MATCH
          563  +     || (p->op==SQLITE_INDEX_CONSTRAINT_EQ && iCol>=nCol)
   553    564       ){
   554    565         /* A MATCH operator or equivalent */
   555         -      if( p->usable ){
   556         -        idxFlags = (idxFlags & 0xFFFF) | FTS5_BI_MATCH | (iCol << 16);
   557         -        aConstraint[0].iConsIndex = i;
   558         -      }else{
          566  +      if( p->usable==0 || iCol<0 ){
   559    567           /* As there exists an unusable MATCH constraint this is an 
   560    568           ** unusable plan. Set a prohibitively high cost. */
   561    569           pInfo->estimatedCost = 1e50;
   562    570           return SQLITE_OK;
   563         -      }
   564         -    }else if( p->op<=SQLITE_INDEX_CONSTRAINT_MATCH ){
   565         -      int j;
   566         -      for(j=1; j<ArraySize(aConstraint); j++){
   567         -        struct Constraint *pC = &aConstraint[j];
   568         -        if( iCol==aColMap[pC->iCol] && (p->op & pC->op) && p->usable ){
   569         -          pC->iConsIndex = i;
   570         -          idxFlags |= pC->fts5op;
   571         -        }
   572         -      }
   573         -    }
   574         -  }
          571  +      }else{
          572  +        if( iCol==nCol+1 ){
          573  +          if( bSeenRank ) continue;
          574  +          idxStr[iIdxStr++] = 'r';
          575  +          bSeenRank = 1;
          576  +        }else{
          577  +          bSeenMatch = 1;
          578  +          idxStr[iIdxStr++] = 'm';
          579  +          if( iCol<nCol ){
          580  +            sqlite3_snprintf(6, &idxStr[iIdxStr], "%d", iCol);
          581  +            idxStr += strlen(&idxStr[iIdxStr]);
          582  +            assert( idxStr[iIdxStr]=='\0' );
          583  +          }
          584  +        }
          585  +        pInfo->aConstraintUsage[i].argvIndex = ++iCons;
          586  +        pInfo->aConstraintUsage[i].omit = 1;
          587  +      }
          588  +    }
          589  +    else if( p->usable && bSeenEq==0 
          590  +      && p->op==SQLITE_INDEX_CONSTRAINT_EQ && iCol<0 
          591  +    ){
          592  +      idxStr[iIdxStr++] = '=';
          593  +      bSeenEq = 1;
          594  +      pInfo->aConstraintUsage[i].argvIndex = ++iCons;
          595  +    }
          596  +  }
          597  +
          598  +  if( bSeenEq==0 ){
          599  +    for(i=0; i<pInfo->nConstraint; i++){
          600  +      struct sqlite3_index_constraint *p = &pInfo->aConstraint[i];
          601  +      if( p->iColumn<0 && p->usable ){
          602  +        int op = p->op;
          603  +        if( op==SQLITE_INDEX_CONSTRAINT_LT || op==SQLITE_INDEX_CONSTRAINT_LE ){
          604  +          if( bSeenLt ) continue;
          605  +          idxStr[iIdxStr++] = '<';
          606  +          pInfo->aConstraintUsage[i].argvIndex = ++iCons;
          607  +          bSeenLt = 1;
          608  +        }else
          609  +        if( op==SQLITE_INDEX_CONSTRAINT_GT || op==SQLITE_INDEX_CONSTRAINT_GE ){
          610  +          if( bSeenGt ) continue;
          611  +          idxStr[iIdxStr++] = '>';
          612  +          pInfo->aConstraintUsage[i].argvIndex = ++iCons;
          613  +          bSeenGt = 1;
          614  +        }
          615  +      }
          616  +    }
          617  +  }
          618  +  idxStr[iIdxStr] = '\0';
   575    619   
   576    620     /* Set idxFlags flags for the ORDER BY clause */
   577    621     if( pInfo->nOrderBy==1 ){
   578    622       int iSort = pInfo->aOrderBy[0].iColumn;
   579         -    if( iSort==(pConfig->nCol+1) && BitFlagTest(idxFlags, FTS5_BI_MATCH) ){
          623  +    if( iSort==(pConfig->nCol+1) && bSeenMatch ){
   580    624         idxFlags |= FTS5_BI_ORDER_RANK;
   581    625       }else if( iSort==-1 ){
   582    626         idxFlags |= FTS5_BI_ORDER_ROWID;
   583    627       }
   584    628       if( BitFlagTest(idxFlags, FTS5_BI_ORDER_RANK|FTS5_BI_ORDER_ROWID) ){
   585    629         pInfo->orderByConsumed = 1;
   586    630         if( pInfo->aOrderBy[0].desc ){
   587    631           idxFlags |= FTS5_BI_ORDER_DESC;
   588    632         }
   589    633       }
   590    634     }
   591    635   
   592    636     /* Calculate the estimated cost based on the flags set in idxFlags. */
   593         -  bHasMatch = BitFlagTest(idxFlags, FTS5_BI_MATCH);
   594         -  if( BitFlagTest(idxFlags, FTS5_BI_ROWID_EQ) ){
   595         -    pInfo->estimatedCost = bHasMatch ? 100.0 : 10.0;
   596         -    if( bHasMatch==0 ) fts5SetUniqueFlag(pInfo);
   597         -  }else if( BitFlagAllTest(idxFlags, FTS5_BI_ROWID_LE|FTS5_BI_ROWID_GE) ){
   598         -    pInfo->estimatedCost = bHasMatch ? 500.0 : 250000.0;
   599         -  }else if( BitFlagTest(idxFlags, FTS5_BI_ROWID_LE|FTS5_BI_ROWID_GE) ){
   600         -    pInfo->estimatedCost = bHasMatch ? 750.0 : 750000.0;
          637  +  if( bSeenEq ){
          638  +    pInfo->estimatedCost = bSeenMatch ? 100.0 : 10.0;
          639  +    if( bSeenMatch==0 ) fts5SetUniqueFlag(pInfo);
          640  +  }else if( bSeenLt && bSeenGt ){
          641  +    pInfo->estimatedCost = bSeenMatch ? 500.0 : 250000.0;
          642  +  }else if( bSeenLt || bSeenGt ){
          643  +    pInfo->estimatedCost = bSeenMatch ? 750.0 : 750000.0;
   601    644     }else{
   602         -    pInfo->estimatedCost = bHasMatch ? 1000.0 : 1000000.0;
   603         -  }
   604         -
   605         -  /* Assign argvIndex values to each constraint in use. */
   606         -  iNext = 1;
   607         -  for(i=0; i<ArraySize(aConstraint); i++){
   608         -    struct Constraint *pC = &aConstraint[i];
   609         -    if( pC->iConsIndex>=0 ){
   610         -      pInfo->aConstraintUsage[pC->iConsIndex].argvIndex = iNext++;
   611         -      pInfo->aConstraintUsage[pC->iConsIndex].omit = (unsigned char)pC->omit;
   612         -    }
          645  +    pInfo->estimatedCost = bSeenMatch ? 1000.0 : 1000000.0;
   613    646     }
   614    647   
   615    648     pInfo->idxNum = idxFlags;
   616    649     return SQLITE_OK;
   617    650   }
   618    651   
   619    652   static int fts5NewTransaction(Fts5FullTable *pTab){
................................................................................
  1128   1161   **   1. Full-text search using a MATCH operator.
  1129   1162   **   2. A by-rowid lookup.
  1130   1163   **   3. A full-table scan.
  1131   1164   */
  1132   1165   static int fts5FilterMethod(
  1133   1166     sqlite3_vtab_cursor *pCursor,   /* The cursor used for this query */
  1134   1167     int idxNum,                     /* Strategy index */
  1135         -  const char *zUnused,            /* Unused */
         1168  +  const char *idxStr,             /* Unused */
  1136   1169     int nVal,                       /* Number of elements in apVal */
  1137   1170     sqlite3_value **apVal           /* Arguments for the indexing scheme */
  1138   1171   ){
  1139   1172     Fts5FullTable *pTab = (Fts5FullTable*)(pCursor->pVtab);
  1140   1173     Fts5Config *pConfig = pTab->p.pConfig;
  1141   1174     Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
  1142   1175     int rc = SQLITE_OK;             /* Error code */
  1143         -  int iVal = 0;                   /* Counter for apVal[] */
  1144   1176     int bDesc;                      /* True if ORDER BY [rank|rowid] DESC */
  1145   1177     int bOrderByRank;               /* True if ORDER BY rank */
  1146   1178     sqlite3_value *pMatch = 0;      /* <tbl> MATCH ? expression (or NULL) */
  1147   1179     sqlite3_value *pRank = 0;       /* rank MATCH ? expression (or NULL) */
  1148   1180     sqlite3_value *pRowidEq = 0;    /* rowid = ? expression (or NULL) */
  1149   1181     sqlite3_value *pRowidLe = 0;    /* rowid <= ? expression (or NULL) */
  1150   1182     sqlite3_value *pRowidGe = 0;    /* rowid >= ? expression (or NULL) */
  1151   1183     int iCol;                       /* Column on LHS of MATCH operator */
  1152   1184     char **pzErrmsg = pConfig->pzErrmsg;
  1153         -
  1154         -  UNUSED_PARAM(zUnused);
  1155         -  UNUSED_PARAM(nVal);
         1185  +  int i;
         1186  +  int iIdxStr = 0;
         1187  +  Fts5Expr *pExpr = 0;
  1156   1188   
  1157   1189     if( pCsr->ePlan ){
  1158   1190       fts5FreeCursorComponents(pCsr);
  1159   1191       memset(&pCsr->ePlan, 0, sizeof(Fts5Cursor) - ((u8*)&pCsr->ePlan-(u8*)pCsr));
  1160   1192     }
  1161   1193   
  1162   1194     assert( pCsr->pStmt==0 );
  1163   1195     assert( pCsr->pExpr==0 );
  1164   1196     assert( pCsr->csrflags==0 );
  1165   1197     assert( pCsr->pRank==0 );
  1166   1198     assert( pCsr->zRank==0 );
  1167   1199     assert( pCsr->zRankArgs==0 );
         1200  +  assert( pTab->pSortCsr==0 || nVal==0 );
  1168   1201   
  1169   1202     assert( pzErrmsg==0 || pzErrmsg==&pTab->p.base.zErrMsg );
  1170   1203     pConfig->pzErrmsg = &pTab->p.base.zErrMsg;
  1171   1204   
  1172         -  /* Decode the arguments passed through to this function.
  1173         -  **
  1174         -  ** Note: The following set of if(...) statements must be in the same
  1175         -  ** order as the corresponding entries in the struct at the top of
  1176         -  ** fts5BestIndexMethod().  */
  1177         -  if( BitFlagTest(idxNum, FTS5_BI_MATCH) ) pMatch = apVal[iVal++];
  1178         -  if( BitFlagTest(idxNum, FTS5_BI_RANK) ) pRank = apVal[iVal++];
  1179         -  if( BitFlagTest(idxNum, FTS5_BI_ROWID_EQ) ) pRowidEq = apVal[iVal++];
  1180         -  if( BitFlagTest(idxNum, FTS5_BI_ROWID_LE) ) pRowidLe = apVal[iVal++];
  1181         -  if( BitFlagTest(idxNum, FTS5_BI_ROWID_GE) ) pRowidGe = apVal[iVal++];
  1182         -  iCol = (idxNum>>16);
  1183         -  assert( iCol>=0 && iCol<=pConfig->nCol );
  1184         -  assert( iVal==nVal );
         1205  +  /* Decode the arguments passed through to this function. */
         1206  +  for(i=0; i<nVal; i++){
         1207  +    switch( idxStr[iIdxStr++] ){
         1208  +      case 'r':
         1209  +        pRank = apVal[i];
         1210  +        break;
         1211  +      case 'm': {
         1212  +        char *zText = sqlite3_value_text(apVal[i]);
         1213  +        if( zText==0 ) zText = "";
         1214  +
         1215  +        if( idxStr[iIdxStr]>='0' && idxStr[iIdxStr]<='9' ){
         1216  +          iCol = 0;
         1217  +          do{
         1218  +            iCol = iCol*10 + (idxStr[iIdxStr]-'0');
         1219  +            iIdxStr++;
         1220  +          }while( idxStr[iIdxStr]>='0' && idxStr[iIdxStr]<='9' );
         1221  +        }else{
         1222  +          iCol = pConfig->nCol;
         1223  +        }
         1224  +
         1225  +        if( zText[0]=='*' ){
         1226  +          /* The user has issued a query of the form "MATCH '*...'". This
         1227  +          ** indicates that the MATCH expression is not a full text query,
         1228  +          ** but a request for an internal parameter.  */
         1229  +          rc = fts5SpecialMatch(pTab, pCsr, &zText[1]);
         1230  +          goto filter_out;
         1231  +        }else{
         1232  +          char **pzErr = &pTab->p.base.zErrMsg;
         1233  +          rc = sqlite3Fts5ExprNew(pConfig, iCol, zText, &pExpr, pzErr);
         1234  +          if( rc==SQLITE_OK ){
         1235  +            rc = sqlite3Fts5ExprAnd(&pCsr->pExpr, pExpr);
         1236  +            pExpr = 0;
         1237  +          }
         1238  +          if( rc!=SQLITE_OK ) goto filter_out;
         1239  +        }
         1240  +
         1241  +        break;
         1242  +      }
         1243  +      case '=':
         1244  +        pRowidEq = apVal[i];
         1245  +        break;
         1246  +      case '<':
         1247  +        pRowidLe = apVal[i];
         1248  +        break;
         1249  +      default: assert( idxStr[iIdxStr-1]=='>' );
         1250  +        pRowidGe = apVal[i];
         1251  +        break;
         1252  +    }
         1253  +  }
  1185   1254     bOrderByRank = ((idxNum & FTS5_BI_ORDER_RANK) ? 1 : 0);
  1186   1255     pCsr->bDesc = bDesc = ((idxNum & FTS5_BI_ORDER_DESC) ? 1 : 0);
  1187   1256   
  1188   1257     /* Set the cursor upper and lower rowid limits. Only some strategies 
  1189   1258     ** actually use them. This is ok, as the xBestIndex() method leaves the
  1190   1259     ** sqlite3_index_constraint.omit flag clear for range constraints
  1191   1260     ** on the rowid field.  */
................................................................................
  1217   1286       }else{
  1218   1287         pCsr->iLastRowid = pTab->pSortCsr->iLastRowid;
  1219   1288         pCsr->iFirstRowid = pTab->pSortCsr->iFirstRowid;
  1220   1289       }
  1221   1290       pCsr->ePlan = FTS5_PLAN_SOURCE;
  1222   1291       pCsr->pExpr = pTab->pSortCsr->pExpr;
  1223   1292       rc = fts5CursorFirst(pTab, pCsr, bDesc);
  1224         -  }else if( pMatch ){
  1225         -    const char *zExpr = (const char*)sqlite3_value_text(apVal[0]);
  1226         -    if( zExpr==0 ) zExpr = "";
  1227         -
         1293  +  }else if( pCsr->pExpr ){
  1228   1294       rc = fts5CursorParseRank(pConfig, pCsr, pRank);
  1229   1295       if( rc==SQLITE_OK ){
  1230         -      if( zExpr[0]=='*' ){
  1231         -        /* The user has issued a query of the form "MATCH '*...'". This
  1232         -        ** indicates that the MATCH expression is not a full text query,
  1233         -        ** but a request for an internal parameter.  */
  1234         -        rc = fts5SpecialMatch(pTab, pCsr, &zExpr[1]);
         1296  +      if( bOrderByRank ){
         1297  +        pCsr->ePlan = FTS5_PLAN_SORTED_MATCH;
         1298  +        rc = fts5CursorFirstSorted(pTab, pCsr, bDesc);
  1235   1299         }else{
  1236         -        char **pzErr = &pTab->p.base.zErrMsg;
  1237         -        rc = sqlite3Fts5ExprNew(pConfig, iCol, zExpr, &pCsr->pExpr, pzErr);
  1238         -        if( rc==SQLITE_OK ){
  1239         -          if( bOrderByRank ){
  1240         -            pCsr->ePlan = FTS5_PLAN_SORTED_MATCH;
  1241         -            rc = fts5CursorFirstSorted(pTab, pCsr, bDesc);
  1242         -          }else{
  1243         -            pCsr->ePlan = FTS5_PLAN_MATCH;
  1244         -            rc = fts5CursorFirst(pTab, pCsr, bDesc);
  1245         -          }
  1246         -        }
         1300  +        pCsr->ePlan = FTS5_PLAN_MATCH;
         1301  +        rc = fts5CursorFirst(pTab, pCsr, bDesc);
  1247   1302         }
  1248   1303       }
  1249   1304     }else if( pConfig->zContent==0 ){
  1250   1305       *pConfig->pzErrmsg = sqlite3_mprintf(
  1251   1306           "%s: table does not support scanning", pConfig->zName
  1252   1307       );
  1253   1308       rc = SQLITE_ERROR;
................................................................................
  1256   1311       ** by rowid (ePlan==FTS5_PLAN_ROWID).  */
  1257   1312       pCsr->ePlan = (pRowidEq ? FTS5_PLAN_ROWID : FTS5_PLAN_SCAN);
  1258   1313       rc = sqlite3Fts5StorageStmt(
  1259   1314           pTab->pStorage, fts5StmtType(pCsr), &pCsr->pStmt, &pTab->p.base.zErrMsg
  1260   1315       );
  1261   1316       if( rc==SQLITE_OK ){
  1262   1317         if( pCsr->ePlan==FTS5_PLAN_ROWID ){
  1263         -        sqlite3_bind_value(pCsr->pStmt, 1, apVal[0]);
         1318  +        sqlite3_bind_value(pCsr->pStmt, 1, pRowidEq);
  1264   1319         }else{
  1265   1320           sqlite3_bind_int64(pCsr->pStmt, 1, pCsr->iFirstRowid);
  1266   1321           sqlite3_bind_int64(pCsr->pStmt, 2, pCsr->iLastRowid);
  1267   1322         }
  1268   1323         rc = fts5NextMethod(pCursor);
  1269   1324       }
  1270   1325     }
  1271   1326   
         1327  + filter_out:
         1328  +  sqlite3Fts5ExprFree(pExpr);
  1272   1329     pConfig->pzErrmsg = pzErrmsg;
  1273   1330     return rc;
  1274   1331   }
  1275   1332   
  1276   1333   /* 
  1277   1334   ** This is the xEof method of the virtual table. SQLite calls this 
  1278   1335   ** routine to find out if it has reached the end of a result set.

Changes to ext/fts5/test/fts5faultB.test.

   142    142     INSERT INTO t1 VALUES('b c d a');  -- 4
   143    143   }
   144    144   do_faultsim_test 5.1 -faults oom* -body {
   145    145     execsql { SELECT rowid FROM t1('^a OR ^b') }
   146    146   } -test {
   147    147     faultsim_test_result {0 {1 4}}
   148    148   }
          149  +
          150  +#-------------------------------------------------------------------------
          151  +# Test OOM injection in a query with two MATCH expressions
          152  +#
          153  +reset_db
          154  +do_execsql_test 6.0 {
          155  +  CREATE VIRTUAL TABLE t1 USING fts5(a);
          156  +  INSERT INTO t1 VALUES('a b c d');  -- 1
          157  +  INSERT INTO t1 VALUES('d a b c');  -- 2
          158  +  INSERT INTO t1 VALUES('c d a b');  -- 3
          159  +  INSERT INTO t1 VALUES('b c d a');  -- 4
          160  +}
          161  +do_faultsim_test 6.1 -faults oom* -body {
          162  +  execsql { SELECT rowid FROM t1 WHERE t1 MATCH 'a' AND t1 MATCH 'b' }
          163  +} -test {
          164  +  faultsim_test_result {0 {1 2 3 4}}
          165  +}
          166  +do_faultsim_test 6.2 -faults oom* -body {
          167  +  execsql { SELECT rowid FROM t1 WHERE t1 MATCH 'a OR b' AND t1 MATCH 'c OR d' }
          168  +} -test {
          169  +  faultsim_test_result {0 {1 2 3 4}}
          170  +}
   149    171   
   150    172   
   151    173   finish_test

Added ext/fts5/test/fts5multi.test.

            1  +# 2014 September 13
            2  +#
            3  +# The author disclaims copyright to this source code.  In place of
            4  +# a legal notice, here is a blessing:
            5  +#
            6  +#    May you do good and not evil.
            7  +#    May you find forgiveness for yourself and forgive others.
            8  +#    May you share freely, never taking more than you give.
            9  +#
           10  +#*************************************************************************
           11  +# This file implements regression tests for SQLite library.  The
           12  +# focus of this script is testing the FTS5 module.
           13  +#
           14  +
           15  +source [file join [file dirname [info script]] fts5_common.tcl]
           16  +set testprefix fts5multi
           17  +
           18  +# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
           19  +ifcapable !fts5 {
           20  +  finish_test
           21  +  return
           22  +}
           23  +
           24  +fts5_aux_test_functions db
           25  +
           26  +do_execsql_test 1.0 {
           27  +  CREATE VIRTUAL TABLE t1 USING fts5(a, b, c);
           28  +  INSERT INTO t1 VALUES('gg bb bb'   ,'gg ff gg'   ,'ii ii');
           29  +  INSERT INTO t1 VALUES('dd dd hh kk','jj'         ,'aa');
           30  +  INSERT INTO t1 VALUES('kk gg ee'   ,'hh cc'      ,'hh jj aa cc');
           31  +  INSERT INTO t1 VALUES('hh'         ,'bb jj cc'   ,'kk ii');
           32  +  INSERT INTO t1 VALUES('kk dd kk ii','aa ee aa'   ,'ee');
           33  +  INSERT INTO t1 VALUES('ee'         ,'ff gg kk aa','ee ff ee');
           34  +  INSERT INTO t1 VALUES('ff jj'      ,'gg ee'      ,'kk ee gg kk');
           35  +  INSERT INTO t1 VALUES('ff ee dd hh','kk ee'      ,'gg dd');
           36  +  INSERT INTO t1 VALUES('bb'         ,'aa'         ,'bb aa');
           37  +  INSERT INTO t1 VALUES('hh cc bb'   ,'ff bb'      ,'cc');
           38  +  INSERT INTO t1 VALUES('jj'         ,'ff dd bb aa','dd dd ff ff');
           39  +  INSERT INTO t1 VALUES('ff dd gg dd','gg aa bb ff','cc');
           40  +  INSERT INTO t1 VALUES('ff aa cc jj','kk'         ,'ii dd');
           41  +  INSERT INTO t1 VALUES('jj dd'      ,'cc'         ,'ii hh ee aa');
           42  +  INSERT INTO t1 VALUES('ff ii hh'   ,'dd'         ,'gg');
           43  +  INSERT INTO t1 VALUES('ff dd gg hh','hh'         ,'ff dd');
           44  +  INSERT INTO t1 VALUES('cc cc'      ,'ff dd ff'   ,'bb');
           45  +  INSERT INTO t1 VALUES('ii'         ,'bb ii'      ,'jj kk');
           46  +  INSERT INTO t1 VALUES('ff hh'      ,'hh bb'      ,'bb dd ee');
           47  +  INSERT INTO t1 VALUES('jj kk'      ,'jj'         ,'gg ff cc');
           48  +  INSERT INTO t1 VALUES('dd kk'      ,'ii gg'      ,'dd');
           49  +  INSERT INTO t1 VALUES('cc'         ,'aa ff'      ,'ii');
           50  +  INSERT INTO t1 VALUES('bb ff bb ii','bb kk bb aa','hh ff ii dd');
           51  +  INSERT INTO t1 VALUES('aa'         ,'ee bb jj jj','dd');
           52  +  INSERT INTO t1 VALUES('kk dd cc'   ,'aa jj'      ,'ee aa ff');
           53  +  INSERT INTO t1 VALUES('aa gg aa'   ,'jj'         ,'ii kk hh gg');
           54  +  INSERT INTO t1 VALUES('ff hh aa'   ,'jj ii'      ,'hh dd bb jj');
           55  +  INSERT INTO t1 VALUES('hh'         ,'aa gg kk'   ,'bb ee');
           56  +  INSERT INTO t1 VALUES('bb'         ,'ee'         ,'gg');
           57  +  INSERT INTO t1 VALUES('dd kk'      ,'kk bb aa'   ,'ee');
           58  +}
           59  +
           60  +foreach {tn c1 e1 c2 e2} {
           61  +  1     t1 aa     t1 bb
           62  +  2     a  aa     b  bb
           63  +  3     a  "aa OR bb OR cc"    b  "jj OR ii OR hh"
           64  +  4     t1  "aa AND bb"       t1  "cc"
           65  +  5     c   "kk"               b  "aa OR bb OR cc OR dd OR ee"
           66  +} {
           67  +  if {$c1=="t1"} {
           68  +    set lhs "( $e1 )"
           69  +  } else {
           70  +    set lhs "$c1 : ( $e1 )"
           71  +  }
           72  +  if {$c2=="t1"} {
           73  +    set rhs "( $e2 )"
           74  +  } else {
           75  +    set rhs "$c2 : ( $e2 )"
           76  +  }
           77  +
           78  +  set q1 "t1 MATCH '($lhs) AND ($rhs)'"
           79  +  set q2 "$c1 MATCH '$e1' AND $c2 MATCH '$e2'"
           80  +
           81  +  set ret [execsql "SELECT rowid FROM t1 WHERE $q1"]
           82  +  set N [llength $ret]
           83  +  do_execsql_test 1.$tn.1.($N) "SELECT rowid FROM t1 WHERE $q2" $ret
           84  +
           85  +  set ret [execsql "SELECT fts5_test_poslist(t1) FROM t1 WHERE $q1"]
           86  +  do_execsql_test 1.$tn.2.($N) "
           87  +    SELECT fts5_test_poslist(t1) FROM t1 WHERE $q2
           88  +  " $ret
           89  +}
           90  +
           91  +do_catchsql_test 2.1.1 {
           92  +  SELECT rowid FROM t1 WHERE t1 MATCH '(NOT' AND t1 MATCH 'aa bb';
           93  +} {1 {fts5: syntax error near "NOT"}}
           94  +do_catchsql_test 2.1.2 {
           95  +  SELECT rowid FROM t1 WHERE t1 MATCH 'aa bb' AND t1 MATCH '(NOT';
           96  +} {1 {fts5: syntax error near "NOT"}}
           97  +
           98  +finish_test
           99  +

Changes to ext/fts5/test/fts5plan.test.

    27     27   }
    28     28   
    29     29   do_eqp_test 1.1 {
    30     30     SELECT * FROM t1, f1 WHERE f1 MATCH t1.x
    31     31   } {
    32     32     QUERY PLAN
    33     33     |--SCAN TABLE t1
    34         -  `--SCAN TABLE f1 VIRTUAL TABLE INDEX 65537:
           34  +  `--SCAN TABLE f1 VIRTUAL TABLE INDEX 0:m
    35     35   }
    36     36   
    37     37   do_eqp_test 1.2 {
    38     38     SELECT * FROM t1, f1 WHERE f1 > t1.x
    39     39   } {
    40     40     QUERY PLAN
    41     41     |--SCAN TABLE f1 VIRTUAL TABLE INDEX 0:
................................................................................
    42     42     `--SCAN TABLE t1
    43     43   }
    44     44   
    45     45   do_eqp_test 1.3 {
    46     46     SELECT * FROM f1 WHERE f1 MATCH ? ORDER BY ff
    47     47   } {
    48     48     QUERY PLAN
    49         -  |--SCAN TABLE f1 VIRTUAL TABLE INDEX 65537:
           49  +  |--SCAN TABLE f1 VIRTUAL TABLE INDEX 0:m
    50     50     `--USE TEMP B-TREE FOR ORDER BY
    51     51   }
    52     52   
    53     53   do_eqp_test 1.4 {
    54     54     SELECT * FROM f1 ORDER BY rank
    55     55   } {
    56     56     QUERY PLAN
    57     57     |--SCAN TABLE f1 VIRTUAL TABLE INDEX 0:
    58     58     `--USE TEMP B-TREE FOR ORDER BY
    59     59   }
    60     60   
    61     61   do_eqp_test 1.5 {
    62     62     SELECT * FROM f1 WHERE rank MATCH ?
    63         -} {SCAN TABLE f1 VIRTUAL TABLE INDEX 2:}
           63  +} {SCAN TABLE f1 VIRTUAL TABLE INDEX 0:r}
    64     64   
    65     65   finish_test

Changes to ext/fts5/test/fts5simple.test.

   463    463   } {11111 11112}
   464    464   do_execsql_test 21.3 {
   465    465     DELETE FROM x1 WHERE rowid=11111;
   466    466     INSERT INTO x1(x1) VALUES('integrity-check');
   467    467     SELECT rowid FROM x1($doc);
   468    468   } {11112}
   469    469   
          470  +#-------------------------------------------------------------------------
          471  +reset_db
          472  +do_execsql_test 22.0 {
          473  +  CREATE VIRTUAL TABLE x1 USING fts5(x);
          474  +  INSERT INTO x1(x) VALUES('a b c');
          475  +  INSERT INTO x1(x) VALUES('x y z');
          476  +  INSERT INTO x1(x) VALUES('c b a');
          477  +  INSERT INTO x1(x) VALUES('z y x');
          478  +}
          479  +
          480  +do_catchsql_test 22.1 {SELECT * FROM x1('')}   {1 {fts5: syntax error near ""}}
          481  +do_catchsql_test 22.2 {SELECT * FROM x1(NULL)} {1 {fts5: syntax error near ""}}
          482  +
   470    483   finish_test