Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Overview
| Comment: | Allow fts5 to filter on multiple MATCH clauses in a single scan. |
|---|---|
| Downloads: | Tarball | ZIP archive | SQL archive |
| Timelines: | family | ancestors | descendants | both | trunk |
| Files: | files | file ages | folders |
| SHA3-256: |
9d418a7a491761eeb38a70898677a493 |
| User & Date: | dan 2019-09-12 19:38:40 |
References
|
2019-09-13
| ||
| 13:23 | Ensure that the idxStr for FTS5 is always zero-terminated. Fix for check-in [9d418a7a491761ee] check-in: 090cd07d37 user: drh tags: trunk | |
Context
|
2019-09-13
| ||
| 12:24 | Fix harmless compiler warnings. check-in: a8927d14f8 user: drh tags: trunk | |
|
2019-09-12
| ||
| 19:38 | Allow fts5 to filter on multiple MATCH clauses in a single scan. check-in: 9d418a7a49 user: dan tags: trunk | |
|
2019-09-11
| ||
| 15:25 | Fix typo for one instance of line number handling in the Lemon tool. check-in: 980be1730d user: mistachkin tags: trunk | |
Changes
Changes to ext/fts5/fts5Int.h.
691 691 */ 692 692 int sqlite3Fts5ExprFirst(Fts5Expr*, Fts5Index *pIdx, i64 iMin, int bDesc); 693 693 int sqlite3Fts5ExprNext(Fts5Expr*, i64 iMax); 694 694 int sqlite3Fts5ExprEof(Fts5Expr*); 695 695 i64 sqlite3Fts5ExprRowid(Fts5Expr*); 696 696 697 697 void sqlite3Fts5ExprFree(Fts5Expr*); 698 +int sqlite3Fts5ExprAnd(Fts5Expr **pp1, Fts5Expr *p2); 698 699 699 700 /* Called during startup to register a UDF with SQLite */ 700 701 int sqlite3Fts5ExprInit(Fts5Global*, sqlite3*); 701 702 702 703 int sqlite3Fts5ExprPhraseCount(Fts5Expr*); 703 704 int sqlite3Fts5ExprPhraseSize(Fts5Expr*, int iPhrase); 704 705 int sqlite3Fts5ExprPoslist(Fts5Expr*, int, const u8 **);
Changes to ext/fts5/fts5_config.c.
679 679 ); 680 680 681 681 assert( zSql || rc==SQLITE_NOMEM ); 682 682 if( zSql ){ 683 683 rc = sqlite3_declare_vtab(pConfig->db, zSql); 684 684 sqlite3_free(zSql); 685 685 } 686 - 686 + 687 687 return rc; 688 688 } 689 689 690 690 /* 691 691 ** Tokenize the text passed via the second and third arguments. 692 692 ** 693 693 ** The callback is invoked once for each token in the input text. The
Changes to ext/fts5/fts5_expr.c.
304 304 void sqlite3Fts5ExprFree(Fts5Expr *p){ 305 305 if( p ){ 306 306 sqlite3Fts5ParseNodeFree(p->pRoot); 307 307 sqlite3_free(p->apExprPhrase); 308 308 sqlite3_free(p); 309 309 } 310 310 } 311 + 312 +int sqlite3Fts5ExprAnd(Fts5Expr **pp1, Fts5Expr *p2){ 313 + Fts5Parse sParse; 314 + memset(&sParse, 0, sizeof(sParse)); 315 + 316 + if( *pp1 ){ 317 + Fts5Expr *p1 = *pp1; 318 + int nPhrase = p1->nPhrase + p2->nPhrase; 319 + 320 + p1->pRoot = sqlite3Fts5ParseNode(&sParse, FTS5_AND, p1->pRoot, p2->pRoot,0); 321 + p2->pRoot = 0; 322 + 323 + if( sParse.rc==SQLITE_OK ){ 324 + Fts5ExprPhrase **ap = (Fts5ExprPhrase**)sqlite3_realloc( 325 + p1->apExprPhrase, nPhrase * sizeof(Fts5ExprPhrase*) 326 + ); 327 + if( ap==0 ){ 328 + sParse.rc = SQLITE_NOMEM; 329 + }else{ 330 + int i; 331 + memmove(&ap[p2->nPhrase], ap, p1->nPhrase*sizeof(Fts5ExprPhrase*)); 332 + for(i=0; i<p2->nPhrase; i++){ 333 + ap[i] = p2->apExprPhrase[i]; 334 + } 335 + p1->nPhrase = nPhrase; 336 + p1->apExprPhrase = ap; 337 + } 338 + } 339 + sqlite3_free(p2->apExprPhrase); 340 + sqlite3_free(p2); 341 + }else{ 342 + *pp1 = p2; 343 + } 344 + 345 + return sParse.rc; 346 +} 311 347 312 348 /* 313 349 ** Argument pTerm must be a synonym iterator. Return the current rowid 314 350 ** that it points to. 315 351 */ 316 352 static i64 fts5ExprSynonymRowid(Fts5ExprTerm *pTerm, int bDesc, int *pbEof){ 317 353 i64 iRet = 0;
Changes to ext/fts5/fts5_main.c.
461 461 #endif 462 462 } 463 463 464 464 /* 465 465 ** Implementation of the xBestIndex method for FTS5 tables. Within the 466 466 ** WHERE constraint, it searches for the following: 467 467 ** 468 -** 1. A MATCH constraint against the special column. 468 +** 1. A MATCH constraint against the table column. 469 469 ** 2. A MATCH constraint against the "rank" column. 470 -** 3. An == constraint against the rowid column. 471 -** 4. A < or <= constraint against the rowid column. 472 -** 5. A > or >= constraint against the rowid column. 470 +** 3. A MATCH constraint against some other column. 471 +** 4. An == constraint against the rowid column. 472 +** 5. A < or <= constraint against the rowid column. 473 +** 6. A > or >= constraint against the rowid column. 473 474 ** 474 -** Within the ORDER BY, either: 475 +** Within the ORDER BY, the following are supported: 475 476 ** 476 477 ** 5. ORDER BY rank [ASC|DESC] 477 478 ** 6. ORDER BY rowid [ASC|DESC] 479 +** 480 +** Information for the xFilter call is passed via both the idxNum and 481 +** idxStr variables. Specifically, idxNum is a bitmask of the following 482 +** flags used to encode the ORDER BY clause: 483 +** 484 +** FTS5_BI_ORDER_RANK 485 +** FTS5_BI_ORDER_ROWID 486 +** FTS5_BI_ORDER_DESC 487 +** 488 +** idxStr is used to encode data from the WHERE clause. For each argument 489 +** passed to the xFilter method, the following is appended to idxStr: 490 +** 491 +** Match against table column: "m" 492 +** Match against rank column: "r" 493 +** Match against other column: "<column-number>" 494 +** Equality constraint against the rowid: "=" 495 +** A < or <= against the rowid: "<" 496 +** A > or >= against the rowid: ">" 497 +** 498 +** This function ensures that there is at most one "r" or "=". And that if 499 +** there exists an "=" then there is no "<" or ">". 478 500 ** 479 501 ** Costs are assigned as follows: 480 502 ** 481 503 ** a) If an unusable MATCH operator is present in the WHERE clause, the 482 504 ** cost is unconditionally set to 1e50 (a really big number). 483 505 ** 484 506 ** a) If a MATCH operator is present, the cost depends on the other ................................................................................ 499 521 ** Costs are not modified by the ORDER BY clause. 500 522 */ 501 523 static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ 502 524 Fts5Table *pTab = (Fts5Table*)pVTab; 503 525 Fts5Config *pConfig = pTab->pConfig; 504 526 const int nCol = pConfig->nCol; 505 527 int idxFlags = 0; /* Parameter passed through to xFilter() */ 506 - int bHasMatch; 507 - int iNext; 508 528 int i; 509 529 510 - struct Constraint { 511 - int op; /* Mask against sqlite3_index_constraint.op */ 512 - int fts5op; /* FTS5 mask for idxFlags */ 513 - int iCol; /* 0==rowid, 1==tbl, 2==rank */ 514 - int omit; /* True to omit this if found */ 515 - int iConsIndex; /* Index in pInfo->aConstraint[] */ 516 - } aConstraint[] = { 517 - {SQLITE_INDEX_CONSTRAINT_MATCH|SQLITE_INDEX_CONSTRAINT_EQ, 518 - FTS5_BI_MATCH, 1, 1, -1}, 519 - {SQLITE_INDEX_CONSTRAINT_MATCH|SQLITE_INDEX_CONSTRAINT_EQ, 520 - FTS5_BI_RANK, 2, 1, -1}, 521 - {SQLITE_INDEX_CONSTRAINT_EQ, FTS5_BI_ROWID_EQ, 0, 0, -1}, 522 - {SQLITE_INDEX_CONSTRAINT_LT|SQLITE_INDEX_CONSTRAINT_LE, 523 - FTS5_BI_ROWID_LE, 0, 0, -1}, 524 - {SQLITE_INDEX_CONSTRAINT_GT|SQLITE_INDEX_CONSTRAINT_GE, 525 - FTS5_BI_ROWID_GE, 0, 0, -1}, 526 - }; 530 + char *idxStr; 531 + int iIdxStr = 0; 532 + int iCons = 0; 527 533 528 - int aColMap[3]; 529 - aColMap[0] = -1; 530 - aColMap[1] = nCol; 531 - aColMap[2] = nCol+1; 534 + int bSeenEq = 0; 535 + int bSeenGt = 0; 536 + int bSeenLt = 0; 537 + int bSeenMatch = 0; 538 + int bSeenRank = 0; 539 + 532 540 533 541 assert( SQLITE_INDEX_CONSTRAINT_EQ<SQLITE_INDEX_CONSTRAINT_MATCH ); 534 542 assert( SQLITE_INDEX_CONSTRAINT_GT<SQLITE_INDEX_CONSTRAINT_MATCH ); 535 543 assert( SQLITE_INDEX_CONSTRAINT_LE<SQLITE_INDEX_CONSTRAINT_MATCH ); 536 544 assert( SQLITE_INDEX_CONSTRAINT_GE<SQLITE_INDEX_CONSTRAINT_MATCH ); 537 545 assert( SQLITE_INDEX_CONSTRAINT_LE<SQLITE_INDEX_CONSTRAINT_MATCH ); 538 546 ................................................................................ 539 547 if( pConfig->bLock ){ 540 548 pTab->base.zErrMsg = sqlite3_mprintf( 541 549 "recursively defined fts5 content table" 542 550 ); 543 551 return SQLITE_ERROR; 544 552 } 545 553 546 - /* Set idxFlags flags for all WHERE clause terms that will be used. */ 554 + idxStr = (char*)sqlite3_malloc(pInfo->nConstraint * 6 + 1); 555 + if( idxStr==0 ) return SQLITE_NOMEM; 556 + pInfo->idxStr = idxStr; 557 + pInfo->needToFreeIdxStr = 1; 558 + 547 559 for(i=0; i<pInfo->nConstraint; i++){ 548 560 struct sqlite3_index_constraint *p = &pInfo->aConstraint[i]; 549 561 int iCol = p->iColumn; 550 - 551 - if( (p->op==SQLITE_INDEX_CONSTRAINT_MATCH && iCol>=0 && iCol<=nCol) 552 - || (p->op==SQLITE_INDEX_CONSTRAINT_EQ && iCol==nCol) 562 + if( p->op==SQLITE_INDEX_CONSTRAINT_MATCH 563 + || (p->op==SQLITE_INDEX_CONSTRAINT_EQ && iCol>=nCol) 553 564 ){ 554 565 /* A MATCH operator or equivalent */ 555 - if( p->usable ){ 556 - idxFlags = (idxFlags & 0xFFFF) | FTS5_BI_MATCH | (iCol << 16); 557 - aConstraint[0].iConsIndex = i; 558 - }else{ 566 + if( p->usable==0 || iCol<0 ){ 559 567 /* As there exists an unusable MATCH constraint this is an 560 568 ** unusable plan. Set a prohibitively high cost. */ 561 569 pInfo->estimatedCost = 1e50; 562 570 return SQLITE_OK; 563 - } 564 - }else if( p->op<=SQLITE_INDEX_CONSTRAINT_MATCH ){ 565 - int j; 566 - for(j=1; j<ArraySize(aConstraint); j++){ 567 - struct Constraint *pC = &aConstraint[j]; 568 - if( iCol==aColMap[pC->iCol] && (p->op & pC->op) && p->usable ){ 569 - pC->iConsIndex = i; 570 - idxFlags |= pC->fts5op; 571 - } 572 - } 573 - } 574 - } 571 + }else{ 572 + if( iCol==nCol+1 ){ 573 + if( bSeenRank ) continue; 574 + idxStr[iIdxStr++] = 'r'; 575 + bSeenRank = 1; 576 + }else{ 577 + bSeenMatch = 1; 578 + idxStr[iIdxStr++] = 'm'; 579 + if( iCol<nCol ){ 580 + sqlite3_snprintf(6, &idxStr[iIdxStr], "%d", iCol); 581 + idxStr += strlen(&idxStr[iIdxStr]); 582 + assert( idxStr[iIdxStr]=='\0' ); 583 + } 584 + } 585 + pInfo->aConstraintUsage[i].argvIndex = ++iCons; 586 + pInfo->aConstraintUsage[i].omit = 1; 587 + } 588 + } 589 + else if( p->usable && bSeenEq==0 590 + && p->op==SQLITE_INDEX_CONSTRAINT_EQ && iCol<0 591 + ){ 592 + idxStr[iIdxStr++] = '='; 593 + bSeenEq = 1; 594 + pInfo->aConstraintUsage[i].argvIndex = ++iCons; 595 + } 596 + } 597 + 598 + if( bSeenEq==0 ){ 599 + for(i=0; i<pInfo->nConstraint; i++){ 600 + struct sqlite3_index_constraint *p = &pInfo->aConstraint[i]; 601 + if( p->iColumn<0 && p->usable ){ 602 + int op = p->op; 603 + if( op==SQLITE_INDEX_CONSTRAINT_LT || op==SQLITE_INDEX_CONSTRAINT_LE ){ 604 + if( bSeenLt ) continue; 605 + idxStr[iIdxStr++] = '<'; 606 + pInfo->aConstraintUsage[i].argvIndex = ++iCons; 607 + bSeenLt = 1; 608 + }else 609 + if( op==SQLITE_INDEX_CONSTRAINT_GT || op==SQLITE_INDEX_CONSTRAINT_GE ){ 610 + if( bSeenGt ) continue; 611 + idxStr[iIdxStr++] = '>'; 612 + pInfo->aConstraintUsage[i].argvIndex = ++iCons; 613 + bSeenGt = 1; 614 + } 615 + } 616 + } 617 + } 618 + idxStr[iIdxStr] = '\0'; 575 619 576 620 /* Set idxFlags flags for the ORDER BY clause */ 577 621 if( pInfo->nOrderBy==1 ){ 578 622 int iSort = pInfo->aOrderBy[0].iColumn; 579 - if( iSort==(pConfig->nCol+1) && BitFlagTest(idxFlags, FTS5_BI_MATCH) ){ 623 + if( iSort==(pConfig->nCol+1) && bSeenMatch ){ 580 624 idxFlags |= FTS5_BI_ORDER_RANK; 581 625 }else if( iSort==-1 ){ 582 626 idxFlags |= FTS5_BI_ORDER_ROWID; 583 627 } 584 628 if( BitFlagTest(idxFlags, FTS5_BI_ORDER_RANK|FTS5_BI_ORDER_ROWID) ){ 585 629 pInfo->orderByConsumed = 1; 586 630 if( pInfo->aOrderBy[0].desc ){ 587 631 idxFlags |= FTS5_BI_ORDER_DESC; 588 632 } 589 633 } 590 634 } 591 635 592 636 /* Calculate the estimated cost based on the flags set in idxFlags. */ 593 - bHasMatch = BitFlagTest(idxFlags, FTS5_BI_MATCH); 594 - if( BitFlagTest(idxFlags, FTS5_BI_ROWID_EQ) ){ 595 - pInfo->estimatedCost = bHasMatch ? 100.0 : 10.0; 596 - if( bHasMatch==0 ) fts5SetUniqueFlag(pInfo); 597 - }else if( BitFlagAllTest(idxFlags, FTS5_BI_ROWID_LE|FTS5_BI_ROWID_GE) ){ 598 - pInfo->estimatedCost = bHasMatch ? 500.0 : 250000.0; 599 - }else if( BitFlagTest(idxFlags, FTS5_BI_ROWID_LE|FTS5_BI_ROWID_GE) ){ 600 - pInfo->estimatedCost = bHasMatch ? 750.0 : 750000.0; 637 + if( bSeenEq ){ 638 + pInfo->estimatedCost = bSeenMatch ? 100.0 : 10.0; 639 + if( bSeenMatch==0 ) fts5SetUniqueFlag(pInfo); 640 + }else if( bSeenLt && bSeenGt ){ 641 + pInfo->estimatedCost = bSeenMatch ? 500.0 : 250000.0; 642 + }else if( bSeenLt || bSeenGt ){ 643 + pInfo->estimatedCost = bSeenMatch ? 750.0 : 750000.0; 601 644 }else{ 602 - pInfo->estimatedCost = bHasMatch ? 1000.0 : 1000000.0; 603 - } 604 - 605 - /* Assign argvIndex values to each constraint in use. */ 606 - iNext = 1; 607 - for(i=0; i<ArraySize(aConstraint); i++){ 608 - struct Constraint *pC = &aConstraint[i]; 609 - if( pC->iConsIndex>=0 ){ 610 - pInfo->aConstraintUsage[pC->iConsIndex].argvIndex = iNext++; 611 - pInfo->aConstraintUsage[pC->iConsIndex].omit = (unsigned char)pC->omit; 612 - } 645 + pInfo->estimatedCost = bSeenMatch ? 1000.0 : 1000000.0; 613 646 } 614 647 615 648 pInfo->idxNum = idxFlags; 616 649 return SQLITE_OK; 617 650 } 618 651 619 652 static int fts5NewTransaction(Fts5FullTable *pTab){ ................................................................................ 1128 1161 ** 1. Full-text search using a MATCH operator. 1129 1162 ** 2. A by-rowid lookup. 1130 1163 ** 3. A full-table scan. 1131 1164 */ 1132 1165 static int fts5FilterMethod( 1133 1166 sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */ 1134 1167 int idxNum, /* Strategy index */ 1135 - const char *zUnused, /* Unused */ 1168 + const char *idxStr, /* Unused */ 1136 1169 int nVal, /* Number of elements in apVal */ 1137 1170 sqlite3_value **apVal /* Arguments for the indexing scheme */ 1138 1171 ){ 1139 1172 Fts5FullTable *pTab = (Fts5FullTable*)(pCursor->pVtab); 1140 1173 Fts5Config *pConfig = pTab->p.pConfig; 1141 1174 Fts5Cursor *pCsr = (Fts5Cursor*)pCursor; 1142 1175 int rc = SQLITE_OK; /* Error code */ 1143 - int iVal = 0; /* Counter for apVal[] */ 1144 1176 int bDesc; /* True if ORDER BY [rank|rowid] DESC */ 1145 1177 int bOrderByRank; /* True if ORDER BY rank */ 1146 1178 sqlite3_value *pMatch = 0; /* <tbl> MATCH ? expression (or NULL) */ 1147 1179 sqlite3_value *pRank = 0; /* rank MATCH ? expression (or NULL) */ 1148 1180 sqlite3_value *pRowidEq = 0; /* rowid = ? expression (or NULL) */ 1149 1181 sqlite3_value *pRowidLe = 0; /* rowid <= ? expression (or NULL) */ 1150 1182 sqlite3_value *pRowidGe = 0; /* rowid >= ? expression (or NULL) */ 1151 1183 int iCol; /* Column on LHS of MATCH operator */ 1152 1184 char **pzErrmsg = pConfig->pzErrmsg; 1153 - 1154 - UNUSED_PARAM(zUnused); 1155 - UNUSED_PARAM(nVal); 1185 + int i; 1186 + int iIdxStr = 0; 1187 + Fts5Expr *pExpr = 0; 1156 1188 1157 1189 if( pCsr->ePlan ){ 1158 1190 fts5FreeCursorComponents(pCsr); 1159 1191 memset(&pCsr->ePlan, 0, sizeof(Fts5Cursor) - ((u8*)&pCsr->ePlan-(u8*)pCsr)); 1160 1192 } 1161 1193 1162 1194 assert( pCsr->pStmt==0 ); 1163 1195 assert( pCsr->pExpr==0 ); 1164 1196 assert( pCsr->csrflags==0 ); 1165 1197 assert( pCsr->pRank==0 ); 1166 1198 assert( pCsr->zRank==0 ); 1167 1199 assert( pCsr->zRankArgs==0 ); 1200 + assert( pTab->pSortCsr==0 || nVal==0 ); 1168 1201 1169 1202 assert( pzErrmsg==0 || pzErrmsg==&pTab->p.base.zErrMsg ); 1170 1203 pConfig->pzErrmsg = &pTab->p.base.zErrMsg; 1171 1204 1172 - /* Decode the arguments passed through to this function. 1173 - ** 1174 - ** Note: The following set of if(...) statements must be in the same 1175 - ** order as the corresponding entries in the struct at the top of 1176 - ** fts5BestIndexMethod(). */ 1177 - if( BitFlagTest(idxNum, FTS5_BI_MATCH) ) pMatch = apVal[iVal++]; 1178 - if( BitFlagTest(idxNum, FTS5_BI_RANK) ) pRank = apVal[iVal++]; 1179 - if( BitFlagTest(idxNum, FTS5_BI_ROWID_EQ) ) pRowidEq = apVal[iVal++]; 1180 - if( BitFlagTest(idxNum, FTS5_BI_ROWID_LE) ) pRowidLe = apVal[iVal++]; 1181 - if( BitFlagTest(idxNum, FTS5_BI_ROWID_GE) ) pRowidGe = apVal[iVal++]; 1182 - iCol = (idxNum>>16); 1183 - assert( iCol>=0 && iCol<=pConfig->nCol ); 1184 - assert( iVal==nVal ); 1205 + /* Decode the arguments passed through to this function. */ 1206 + for(i=0; i<nVal; i++){ 1207 + switch( idxStr[iIdxStr++] ){ 1208 + case 'r': 1209 + pRank = apVal[i]; 1210 + break; 1211 + case 'm': { 1212 + char *zText = sqlite3_value_text(apVal[i]); 1213 + if( zText==0 ) zText = ""; 1214 + 1215 + if( idxStr[iIdxStr]>='0' && idxStr[iIdxStr]<='9' ){ 1216 + iCol = 0; 1217 + do{ 1218 + iCol = iCol*10 + (idxStr[iIdxStr]-'0'); 1219 + iIdxStr++; 1220 + }while( idxStr[iIdxStr]>='0' && idxStr[iIdxStr]<='9' ); 1221 + }else{ 1222 + iCol = pConfig->nCol; 1223 + } 1224 + 1225 + if( zText[0]=='*' ){ 1226 + /* The user has issued a query of the form "MATCH '*...'". This 1227 + ** indicates that the MATCH expression is not a full text query, 1228 + ** but a request for an internal parameter. */ 1229 + rc = fts5SpecialMatch(pTab, pCsr, &zText[1]); 1230 + goto filter_out; 1231 + }else{ 1232 + char **pzErr = &pTab->p.base.zErrMsg; 1233 + rc = sqlite3Fts5ExprNew(pConfig, iCol, zText, &pExpr, pzErr); 1234 + if( rc==SQLITE_OK ){ 1235 + rc = sqlite3Fts5ExprAnd(&pCsr->pExpr, pExpr); 1236 + pExpr = 0; 1237 + } 1238 + if( rc!=SQLITE_OK ) goto filter_out; 1239 + } 1240 + 1241 + break; 1242 + } 1243 + case '=': 1244 + pRowidEq = apVal[i]; 1245 + break; 1246 + case '<': 1247 + pRowidLe = apVal[i]; 1248 + break; 1249 + default: assert( idxStr[iIdxStr-1]=='>' ); 1250 + pRowidGe = apVal[i]; 1251 + break; 1252 + } 1253 + } 1185 1254 bOrderByRank = ((idxNum & FTS5_BI_ORDER_RANK) ? 1 : 0); 1186 1255 pCsr->bDesc = bDesc = ((idxNum & FTS5_BI_ORDER_DESC) ? 1 : 0); 1187 1256 1188 1257 /* Set the cursor upper and lower rowid limits. Only some strategies 1189 1258 ** actually use them. This is ok, as the xBestIndex() method leaves the 1190 1259 ** sqlite3_index_constraint.omit flag clear for range constraints 1191 1260 ** on the rowid field. */ ................................................................................ 1217 1286 }else{ 1218 1287 pCsr->iLastRowid = pTab->pSortCsr->iLastRowid; 1219 1288 pCsr->iFirstRowid = pTab->pSortCsr->iFirstRowid; 1220 1289 } 1221 1290 pCsr->ePlan = FTS5_PLAN_SOURCE; 1222 1291 pCsr->pExpr = pTab->pSortCsr->pExpr; 1223 1292 rc = fts5CursorFirst(pTab, pCsr, bDesc); 1224 - }else if( pMatch ){ 1225 - const char *zExpr = (const char*)sqlite3_value_text(apVal[0]); 1226 - if( zExpr==0 ) zExpr = ""; 1227 - 1293 + }else if( pCsr->pExpr ){ 1228 1294 rc = fts5CursorParseRank(pConfig, pCsr, pRank); 1229 1295 if( rc==SQLITE_OK ){ 1230 - if( zExpr[0]=='*' ){ 1231 - /* The user has issued a query of the form "MATCH '*...'". This 1232 - ** indicates that the MATCH expression is not a full text query, 1233 - ** but a request for an internal parameter. */ 1234 - rc = fts5SpecialMatch(pTab, pCsr, &zExpr[1]); 1296 + if( bOrderByRank ){ 1297 + pCsr->ePlan = FTS5_PLAN_SORTED_MATCH; 1298 + rc = fts5CursorFirstSorted(pTab, pCsr, bDesc); 1235 1299 }else{ 1236 - char **pzErr = &pTab->p.base.zErrMsg; 1237 - rc = sqlite3Fts5ExprNew(pConfig, iCol, zExpr, &pCsr->pExpr, pzErr); 1238 - if( rc==SQLITE_OK ){ 1239 - if( bOrderByRank ){ 1240 - pCsr->ePlan = FTS5_PLAN_SORTED_MATCH; 1241 - rc = fts5CursorFirstSorted(pTab, pCsr, bDesc); 1242 - }else{ 1243 - pCsr->ePlan = FTS5_PLAN_MATCH; 1244 - rc = fts5CursorFirst(pTab, pCsr, bDesc); 1245 - } 1246 - } 1300 + pCsr->ePlan = FTS5_PLAN_MATCH; 1301 + rc = fts5CursorFirst(pTab, pCsr, bDesc); 1247 1302 } 1248 1303 } 1249 1304 }else if( pConfig->zContent==0 ){ 1250 1305 *pConfig->pzErrmsg = sqlite3_mprintf( 1251 1306 "%s: table does not support scanning", pConfig->zName 1252 1307 ); 1253 1308 rc = SQLITE_ERROR; ................................................................................ 1256 1311 ** by rowid (ePlan==FTS5_PLAN_ROWID). */ 1257 1312 pCsr->ePlan = (pRowidEq ? FTS5_PLAN_ROWID : FTS5_PLAN_SCAN); 1258 1313 rc = sqlite3Fts5StorageStmt( 1259 1314 pTab->pStorage, fts5StmtType(pCsr), &pCsr->pStmt, &pTab->p.base.zErrMsg 1260 1315 ); 1261 1316 if( rc==SQLITE_OK ){ 1262 1317 if( pCsr->ePlan==FTS5_PLAN_ROWID ){ 1263 - sqlite3_bind_value(pCsr->pStmt, 1, apVal[0]); 1318 + sqlite3_bind_value(pCsr->pStmt, 1, pRowidEq); 1264 1319 }else{ 1265 1320 sqlite3_bind_int64(pCsr->pStmt, 1, pCsr->iFirstRowid); 1266 1321 sqlite3_bind_int64(pCsr->pStmt, 2, pCsr->iLastRowid); 1267 1322 } 1268 1323 rc = fts5NextMethod(pCursor); 1269 1324 } 1270 1325 } 1271 1326 1327 + filter_out: 1328 + sqlite3Fts5ExprFree(pExpr); 1272 1329 pConfig->pzErrmsg = pzErrmsg; 1273 1330 return rc; 1274 1331 } 1275 1332 1276 1333 /* 1277 1334 ** This is the xEof method of the virtual table. SQLite calls this 1278 1335 ** routine to find out if it has reached the end of a result set.
Changes to ext/fts5/test/fts5faultB.test.
142 142 INSERT INTO t1 VALUES('b c d a'); -- 4 143 143 } 144 144 do_faultsim_test 5.1 -faults oom* -body { 145 145 execsql { SELECT rowid FROM t1('^a OR ^b') } 146 146 } -test { 147 147 faultsim_test_result {0 {1 4}} 148 148 } 149 + 150 +#------------------------------------------------------------------------- 151 +# Test OOM injection in a query with two MATCH expressions 152 +# 153 +reset_db 154 +do_execsql_test 6.0 { 155 + CREATE VIRTUAL TABLE t1 USING fts5(a); 156 + INSERT INTO t1 VALUES('a b c d'); -- 1 157 + INSERT INTO t1 VALUES('d a b c'); -- 2 158 + INSERT INTO t1 VALUES('c d a b'); -- 3 159 + INSERT INTO t1 VALUES('b c d a'); -- 4 160 +} 161 +do_faultsim_test 6.1 -faults oom* -body { 162 + execsql { SELECT rowid FROM t1 WHERE t1 MATCH 'a' AND t1 MATCH 'b' } 163 +} -test { 164 + faultsim_test_result {0 {1 2 3 4}} 165 +} 166 +do_faultsim_test 6.2 -faults oom* -body { 167 + execsql { SELECT rowid FROM t1 WHERE t1 MATCH 'a OR b' AND t1 MATCH 'c OR d' } 168 +} -test { 169 + faultsim_test_result {0 {1 2 3 4}} 170 +} 149 171 150 172 151 173 finish_test
Added ext/fts5/test/fts5multi.test.
1 +# 2014 September 13 2 +# 3 +# The author disclaims copyright to this source code. In place of 4 +# a legal notice, here is a blessing: 5 +# 6 +# May you do good and not evil. 7 +# May you find forgiveness for yourself and forgive others. 8 +# May you share freely, never taking more than you give. 9 +# 10 +#************************************************************************* 11 +# This file implements regression tests for SQLite library. The 12 +# focus of this script is testing the FTS5 module. 13 +# 14 + 15 +source [file join [file dirname [info script]] fts5_common.tcl] 16 +set testprefix fts5multi 17 + 18 +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. 19 +ifcapable !fts5 { 20 + finish_test 21 + return 22 +} 23 + 24 +fts5_aux_test_functions db 25 + 26 +do_execsql_test 1.0 { 27 + CREATE VIRTUAL TABLE t1 USING fts5(a, b, c); 28 + INSERT INTO t1 VALUES('gg bb bb' ,'gg ff gg' ,'ii ii'); 29 + INSERT INTO t1 VALUES('dd dd hh kk','jj' ,'aa'); 30 + INSERT INTO t1 VALUES('kk gg ee' ,'hh cc' ,'hh jj aa cc'); 31 + INSERT INTO t1 VALUES('hh' ,'bb jj cc' ,'kk ii'); 32 + INSERT INTO t1 VALUES('kk dd kk ii','aa ee aa' ,'ee'); 33 + INSERT INTO t1 VALUES('ee' ,'ff gg kk aa','ee ff ee'); 34 + INSERT INTO t1 VALUES('ff jj' ,'gg ee' ,'kk ee gg kk'); 35 + INSERT INTO t1 VALUES('ff ee dd hh','kk ee' ,'gg dd'); 36 + INSERT INTO t1 VALUES('bb' ,'aa' ,'bb aa'); 37 + INSERT INTO t1 VALUES('hh cc bb' ,'ff bb' ,'cc'); 38 + INSERT INTO t1 VALUES('jj' ,'ff dd bb aa','dd dd ff ff'); 39 + INSERT INTO t1 VALUES('ff dd gg dd','gg aa bb ff','cc'); 40 + INSERT INTO t1 VALUES('ff aa cc jj','kk' ,'ii dd'); 41 + INSERT INTO t1 VALUES('jj dd' ,'cc' ,'ii hh ee aa'); 42 + INSERT INTO t1 VALUES('ff ii hh' ,'dd' ,'gg'); 43 + INSERT INTO t1 VALUES('ff dd gg hh','hh' ,'ff dd'); 44 + INSERT INTO t1 VALUES('cc cc' ,'ff dd ff' ,'bb'); 45 + INSERT INTO t1 VALUES('ii' ,'bb ii' ,'jj kk'); 46 + INSERT INTO t1 VALUES('ff hh' ,'hh bb' ,'bb dd ee'); 47 + INSERT INTO t1 VALUES('jj kk' ,'jj' ,'gg ff cc'); 48 + INSERT INTO t1 VALUES('dd kk' ,'ii gg' ,'dd'); 49 + INSERT INTO t1 VALUES('cc' ,'aa ff' ,'ii'); 50 + INSERT INTO t1 VALUES('bb ff bb ii','bb kk bb aa','hh ff ii dd'); 51 + INSERT INTO t1 VALUES('aa' ,'ee bb jj jj','dd'); 52 + INSERT INTO t1 VALUES('kk dd cc' ,'aa jj' ,'ee aa ff'); 53 + INSERT INTO t1 VALUES('aa gg aa' ,'jj' ,'ii kk hh gg'); 54 + INSERT INTO t1 VALUES('ff hh aa' ,'jj ii' ,'hh dd bb jj'); 55 + INSERT INTO t1 VALUES('hh' ,'aa gg kk' ,'bb ee'); 56 + INSERT INTO t1 VALUES('bb' ,'ee' ,'gg'); 57 + INSERT INTO t1 VALUES('dd kk' ,'kk bb aa' ,'ee'); 58 +} 59 + 60 +foreach {tn c1 e1 c2 e2} { 61 + 1 t1 aa t1 bb 62 + 2 a aa b bb 63 + 3 a "aa OR bb OR cc" b "jj OR ii OR hh" 64 + 4 t1 "aa AND bb" t1 "cc" 65 + 5 c "kk" b "aa OR bb OR cc OR dd OR ee" 66 +} { 67 + if {$c1=="t1"} { 68 + set lhs "( $e1 )" 69 + } else { 70 + set lhs "$c1 : ( $e1 )" 71 + } 72 + if {$c2=="t1"} { 73 + set rhs "( $e2 )" 74 + } else { 75 + set rhs "$c2 : ( $e2 )" 76 + } 77 + 78 + set q1 "t1 MATCH '($lhs) AND ($rhs)'" 79 + set q2 "$c1 MATCH '$e1' AND $c2 MATCH '$e2'" 80 + 81 + set ret [execsql "SELECT rowid FROM t1 WHERE $q1"] 82 + set N [llength $ret] 83 + do_execsql_test 1.$tn.1.($N) "SELECT rowid FROM t1 WHERE $q2" $ret 84 + 85 + set ret [execsql "SELECT fts5_test_poslist(t1) FROM t1 WHERE $q1"] 86 + do_execsql_test 1.$tn.2.($N) " 87 + SELECT fts5_test_poslist(t1) FROM t1 WHERE $q2 88 + " $ret 89 +} 90 + 91 +do_catchsql_test 2.1.1 { 92 + SELECT rowid FROM t1 WHERE t1 MATCH '(NOT' AND t1 MATCH 'aa bb'; 93 +} {1 {fts5: syntax error near "NOT"}} 94 +do_catchsql_test 2.1.2 { 95 + SELECT rowid FROM t1 WHERE t1 MATCH 'aa bb' AND t1 MATCH '(NOT'; 96 +} {1 {fts5: syntax error near "NOT"}} 97 + 98 +finish_test 99 +
Changes to ext/fts5/test/fts5plan.test.
27 27 } 28 28 29 29 do_eqp_test 1.1 { 30 30 SELECT * FROM t1, f1 WHERE f1 MATCH t1.x 31 31 } { 32 32 QUERY PLAN 33 33 |--SCAN TABLE t1 34 - `--SCAN TABLE f1 VIRTUAL TABLE INDEX 65537: 34 + `--SCAN TABLE f1 VIRTUAL TABLE INDEX 0:m 35 35 } 36 36 37 37 do_eqp_test 1.2 { 38 38 SELECT * FROM t1, f1 WHERE f1 > t1.x 39 39 } { 40 40 QUERY PLAN 41 41 |--SCAN TABLE f1 VIRTUAL TABLE INDEX 0: ................................................................................ 42 42 `--SCAN TABLE t1 43 43 } 44 44 45 45 do_eqp_test 1.3 { 46 46 SELECT * FROM f1 WHERE f1 MATCH ? ORDER BY ff 47 47 } { 48 48 QUERY PLAN 49 - |--SCAN TABLE f1 VIRTUAL TABLE INDEX 65537: 49 + |--SCAN TABLE f1 VIRTUAL TABLE INDEX 0:m 50 50 `--USE TEMP B-TREE FOR ORDER BY 51 51 } 52 52 53 53 do_eqp_test 1.4 { 54 54 SELECT * FROM f1 ORDER BY rank 55 55 } { 56 56 QUERY PLAN 57 57 |--SCAN TABLE f1 VIRTUAL TABLE INDEX 0: 58 58 `--USE TEMP B-TREE FOR ORDER BY 59 59 } 60 60 61 61 do_eqp_test 1.5 { 62 62 SELECT * FROM f1 WHERE rank MATCH ? 63 -} {SCAN TABLE f1 VIRTUAL TABLE INDEX 2:} 63 +} {SCAN TABLE f1 VIRTUAL TABLE INDEX 0:r} 64 64 65 65 finish_test
Changes to ext/fts5/test/fts5simple.test.
463 463 } {11111 11112} 464 464 do_execsql_test 21.3 { 465 465 DELETE FROM x1 WHERE rowid=11111; 466 466 INSERT INTO x1(x1) VALUES('integrity-check'); 467 467 SELECT rowid FROM x1($doc); 468 468 } {11112} 469 469 470 +#------------------------------------------------------------------------- 471 +reset_db 472 +do_execsql_test 22.0 { 473 + CREATE VIRTUAL TABLE x1 USING fts5(x); 474 + INSERT INTO x1(x) VALUES('a b c'); 475 + INSERT INTO x1(x) VALUES('x y z'); 476 + INSERT INTO x1(x) VALUES('c b a'); 477 + INSERT INTO x1(x) VALUES('z y x'); 478 +} 479 + 480 +do_catchsql_test 22.1 {SELECT * FROM x1('')} {1 {fts5: syntax error near ""}} 481 +do_catchsql_test 22.2 {SELECT * FROM x1(NULL)} {1 {fts5: syntax error near ""}} 482 + 470 483 finish_test