/ Check-in [fbd038bb57]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Further optimizations for the 'y' and 'b' matchinfo operators.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | fts3-matchinfo-y
Files: files | file ages | folders
SHA1: fbd038bb57322e1ed2e1ee52f3d134594b6bfcc0
User & Date: dan 2015-05-06 08:43:26
Context
2015-05-06
17:41
More optimization for the 'y' and 'b' matchinfo directives. check-in: 8c5b9fedfc user: dan tags: fts3-matchinfo-y
08:43
Further optimizations for the 'y' and 'b' matchinfo operators. check-in: fbd038bb57 user: dan tags: fts3-matchinfo-y
2015-05-05
20:39
Add the fts3 matchinfo 'b' flag. check-in: b9b77972d8 user: dan tags: fts3-matchinfo-y
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to ext/fts3/fts3_snippet.c.

   234    234   */
   235    235   static void fts3GetDeltaPosition(char **pp, int *piPos){
   236    236     int iVal;
   237    237     *pp += fts3GetVarint32(*pp, &iVal);
   238    238     *piPos += (iVal-2);
   239    239   }
   240    240   
          241  +static int fts3ExprLHitsCb(Fts3Expr*, int, void*);
          242  +
   241    243   /*
   242    244   ** Helper function for fts3ExprIterate() (see below).
   243    245   */
   244    246   static int fts3ExprIterate2(
   245    247     Fts3Expr *pExpr,                /* Expression to iterate phrases of */
   246         -  int bExcludeEof,
   247    248     int *piPhrase,                  /* Pointer to phrase counter */
   248    249     int (*x)(Fts3Expr*,int,void*),  /* Callback function to invoke for phrases */
   249    250     void *pCtx                      /* Second argument to pass to callback */
   250    251   ){
   251    252     int rc;                         /* Return code */
   252    253   
   253         -  if( bExcludeEof && pExpr->bEof ){
          254  +  if( x==fts3ExprLHitsCb && pExpr->bEof ){
   254    255       rc = SQLITE_OK;
   255    256     }else{
   256    257       int eType = pExpr->eType;     /* Type of expression node pExpr */
   257    258       if( eType!=FTSQUERY_PHRASE ){
   258    259         assert( pExpr->pLeft && pExpr->pRight );
   259         -      rc = fts3ExprIterate2(pExpr->pLeft, bExcludeEof, piPhrase, x, pCtx);
          260  +      rc = fts3ExprIterate2(pExpr->pLeft, piPhrase, x, pCtx);
   260    261         if( rc==SQLITE_OK && eType!=FTSQUERY_NOT ){
   261         -        rc = fts3ExprIterate2(pExpr->pRight, bExcludeEof, piPhrase, x, pCtx);
          262  +        rc = fts3ExprIterate2(pExpr->pRight, piPhrase, x, pCtx);
   262    263         }
   263    264       }else{
   264    265         rc = x(pExpr, *piPhrase, pCtx);
   265    266         (*piPhrase)++;
   266    267       }
   267    268     }
   268    269     return rc;
................................................................................
   276    277   ** If the callback function returns anything other than SQLITE_OK, 
   277    278   ** the iteration is abandoned and the error code returned immediately.
   278    279   ** Otherwise, SQLITE_OK is returned after a callback has been made for
   279    280   ** all eligible phrase nodes.
   280    281   */
   281    282   static int fts3ExprIterate(
   282    283     Fts3Expr *pExpr,                /* Expression to iterate phrases of */
   283         -  int bExcludeEof,                /* Include nodes already at EOF */
   284    284     int (*x)(Fts3Expr*,int,void*),  /* Callback function to invoke for phrases */
   285    285     void *pCtx                      /* Second argument to pass to callback */
   286    286   ){
   287    287     int iPhrase = 0;                /* Variable used as the phrase counter */
   288         -  return fts3ExprIterate2(pExpr, bExcludeEof, &iPhrase, x, pCtx);
          288  +  return fts3ExprIterate2(pExpr, &iPhrase, x, pCtx);
   289    289   }
   290    290   
   291    291   /*
   292    292   ** This is an fts3ExprIterate() callback used while loading the doclists
   293    293   ** for each phrase into Fts3Expr.aDoclist[]/nDoclist. See also
   294    294   ** fts3ExprLoadDoclists().
   295    295   */
................................................................................
   320    320     Fts3Cursor *pCsr,               /* Fts3 cursor for current query */
   321    321     int *pnPhrase,                  /* OUT: Number of phrases in query */
   322    322     int *pnToken                    /* OUT: Number of tokens in query */
   323    323   ){
   324    324     int rc;                         /* Return Code */
   325    325     LoadDoclistCtx sCtx = {0,0,0};  /* Context for fts3ExprIterate() */
   326    326     sCtx.pCsr = pCsr;
   327         -  rc = fts3ExprIterate(pCsr->pExpr, 0, fts3ExprLoadDoclistsCb, (void *)&sCtx);
          327  +  rc = fts3ExprIterate(pCsr->pExpr, fts3ExprLoadDoclistsCb, (void *)&sCtx);
   328    328     if( pnPhrase ) *pnPhrase = sCtx.nPhrase;
   329    329     if( pnToken ) *pnToken = sCtx.nToken;
   330    330     return rc;
   331    331   }
   332    332   
   333    333   static int fts3ExprPhraseCountCb(Fts3Expr *pExpr, int iPhrase, void *ctx){
   334    334     (*(int *)ctx)++;
   335    335     UNUSED_PARAMETER(pExpr);
   336    336     UNUSED_PARAMETER(iPhrase);
   337    337     return SQLITE_OK;
   338    338   }
   339    339   static int fts3ExprPhraseCount(Fts3Expr *pExpr){
   340    340     int nPhrase = 0;
   341         -  (void)fts3ExprIterate(pExpr, 0, fts3ExprPhraseCountCb, (void *)&nPhrase);
          341  +  (void)fts3ExprIterate(pExpr, fts3ExprPhraseCountCb, (void *)&nPhrase);
   342    342     return nPhrase;
   343    343   }
   344    344   
   345    345   /*
   346    346   ** Advance the position list iterator specified by the first two 
   347    347   ** arguments so that it points to the first element with a value greater
   348    348   ** than or equal to parameter iNext.
................................................................................
   550    550     ** the set of phrases in the expression to populate the aPhrase[] array.
   551    551     */
   552    552     sIter.pCsr = pCsr;
   553    553     sIter.iCol = iCol;
   554    554     sIter.nSnippet = nSnippet;
   555    555     sIter.nPhrase = nList;
   556    556     sIter.iCurrent = -1;
   557         -  rc = fts3ExprIterate(pCsr->pExpr, 0, fts3SnippetFindPositions, (void*)&sIter);
          557  +  rc = fts3ExprIterate(pCsr->pExpr, fts3SnippetFindPositions, (void*)&sIter);
   558    558     if( rc==SQLITE_OK ){
   559    559   
   560    560       /* Set the *pmSeen output variable. */
   561    561       for(i=0; i<nList; i++){
   562    562         if( sIter.aPhrase[i].pHead ){
   563    563           *pmSeen |= (u64)1 << i;
   564    564         }
................................................................................
   919    919     }
   920    920   
   921    921     return rc;
   922    922   }
   923    923   
   924    924   /*
   925    925   ** fts3ExprIterate() callback used to gather information for the matchinfo
   926         -** directive 'y'.
          926  +** directives 'y' and 'b'.
   927    927   */
   928    928   static int fts3ExprLHitsCb(
   929    929     Fts3Expr *pExpr,                /* Phrase expression node */
   930    930     int iPhrase,                    /* Phrase number */
   931    931     void *pCtx                      /* Pointer to MatchInfo structure */
   932    932   ){
   933    933     int rc = SQLITE_OK;
................................................................................
  1120   1120   
  1121   1121     /* Allocate and populate the array of LcsIterator objects. The array
  1122   1122     ** contains one element for each matchable phrase in the query.
  1123   1123     **/
  1124   1124     aIter = sqlite3_malloc(sizeof(LcsIterator) * pCsr->nPhrase);
  1125   1125     if( !aIter ) return SQLITE_NOMEM;
  1126   1126     memset(aIter, 0, sizeof(LcsIterator) * pCsr->nPhrase);
  1127         -  (void)fts3ExprIterate(pCsr->pExpr, 0, fts3MatchinfoLcsCb, (void*)aIter);
         1127  +  (void)fts3ExprIterate(pCsr->pExpr, fts3MatchinfoLcsCb, (void*)aIter);
  1128   1128   
  1129   1129     for(i=0; i<pInfo->nPhrase; i++){
  1130   1130       LcsIterator *pIter = &aIter[i];
  1131   1131       nToken -= pIter->pExpr->pPhrase->nToken;
  1132   1132       pIter->iPosOffset = nToken;
  1133   1133     }
  1134   1134   
................................................................................
  1268   1268           }
  1269   1269           break;
  1270   1270   
  1271   1271         case FTS3_MATCHINFO_LHITS_BM:
  1272   1272         case FTS3_MATCHINFO_LHITS: {
  1273   1273           int nZero = fts3MatchinfoSize(pInfo, zArg[i]) * sizeof(u32);
  1274   1274           memset(pInfo->aMatchinfo, 0, nZero);
  1275         -        (void)fts3ExprIterate(pCsr->pExpr, 1, fts3ExprLHitsCb, (void*)pInfo);
         1275  +        (void)fts3ExprIterate(pCsr->pExpr, fts3ExprLHitsCb, (void*)pInfo);
  1276   1276           break;
  1277   1277         }
  1278   1278   
  1279   1279         default: {
  1280   1280           Fts3Expr *pExpr;
  1281   1281           assert( zArg[i]==FTS3_MATCHINFO_HITS );
  1282   1282           pExpr = pCsr->pExpr;
................................................................................
  1283   1283           rc = fts3ExprLoadDoclists(pCsr, 0, 0);
  1284   1284           if( rc!=SQLITE_OK ) break;
  1285   1285           if( bGlobal ){
  1286   1286             if( pCsr->pDeferred ){
  1287   1287               rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &pInfo->nDoc, 0);
  1288   1288               if( rc!=SQLITE_OK ) break;
  1289   1289             }
  1290         -          rc = fts3ExprIterate(pExpr, 0, fts3ExprGlobalHitsCb,(void*)pInfo);
         1290  +          rc = fts3ExprIterate(pExpr, fts3ExprGlobalHitsCb,(void*)pInfo);
  1291   1291             if( rc!=SQLITE_OK ) break;
  1292   1292           }
  1293         -        (void)fts3ExprIterate(pExpr, 0, fts3ExprLocalHitsCb,(void*)pInfo);
         1293  +        (void)fts3ExprIterate(pExpr, fts3ExprLocalHitsCb,(void*)pInfo);
  1294   1294           break;
  1295   1295         }
  1296   1296       }
  1297   1297   
  1298   1298       pInfo->aMatchinfo += fts3MatchinfoSize(pInfo, zArg[i]);
  1299   1299     }
  1300   1300   
................................................................................
  1344   1344   
  1345   1345       /* Determine the number of phrases in the query */
  1346   1346       pCsr->nPhrase = fts3ExprPhraseCount(pCsr->pExpr);
  1347   1347       sInfo.nPhrase = pCsr->nPhrase;
  1348   1348   
  1349   1349       /* Determine the number of integers in the buffer returned by this call. */
  1350   1350       for(i=0; zArg[i]; i++){
         1351  +      char *zErr = 0;
         1352  +      if( fts3MatchinfoCheck(pTab, zArg[i], &zErr) ){
         1353  +        sqlite3_result_error(pCtx, zErr, -1);
         1354  +        sqlite3_free(zErr);
         1355  +        return;
         1356  +      }
  1351   1357         nMatchinfo += fts3MatchinfoSize(&sInfo, zArg[i]);
  1352   1358       }
  1353   1359   
  1354   1360       /* Allocate space for Fts3Cursor.aMatchinfo[] and Fts3Cursor.zMatchinfo. */
  1355   1361       pCsr->pMIBuffer = fts3MIBufferNew(nMatchinfo, zArg);
  1356   1362       if( !pCsr->pMIBuffer ) rc = SQLITE_NOMEM;
  1357   1363   
................................................................................
  1585   1591   
  1586   1592       /* Initialize the contents of sCtx.aTerm[] for column iCol. There is 
  1587   1593       ** no way that this operation can fail, so the return code from
  1588   1594       ** fts3ExprIterate() can be discarded.
  1589   1595       */
  1590   1596       sCtx.iCol = iCol;
  1591   1597       sCtx.iTerm = 0;
  1592         -    (void)fts3ExprIterate(pCsr->pExpr, 0, fts3ExprTermOffsetInit, (void*)&sCtx);
         1598  +    (void)fts3ExprIterate(pCsr->pExpr, fts3ExprTermOffsetInit, (void*)&sCtx);
  1593   1599   
  1594   1600       /* Retreive the text stored in column iCol. If an SQL NULL is stored 
  1595   1601       ** in column iCol, jump immediately to the next iteration of the loop.
  1596   1602       ** If an OOM occurs while retrieving the data (this can happen if SQLite
  1597   1603       ** needs to transform the data from utf-16 to utf-8), return SQLITE_NOMEM 
  1598   1604       ** to the caller. 
  1599   1605       */
................................................................................
  1677   1683   */
  1678   1684   void sqlite3Fts3Matchinfo(
  1679   1685     sqlite3_context *pContext,      /* Function call context */
  1680   1686     Fts3Cursor *pCsr,               /* FTS3 table cursor */
  1681   1687     const char *zArg                /* Second arg to matchinfo() function */
  1682   1688   ){
  1683   1689     Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
  1684         -  int rc;
  1685         -  int i;
  1686   1690     const char *zFormat;
         1691  +  int rc;
  1687   1692   
  1688   1693     if( zArg ){
  1689         -    for(i=0; zArg[i]; i++){
  1690         -      char *zErr = 0;
  1691         -      if( fts3MatchinfoCheck(pTab, zArg[i], &zErr) ){
  1692         -        sqlite3_result_error(pContext, zErr, -1);
  1693         -        sqlite3_free(zErr);
  1694         -        return;
  1695         -      }
  1696         -    }
  1697   1694       zFormat = zArg;
  1698   1695     }else{
  1699   1696       zFormat = FTS3_MATCHINFO_DEFAULT;
  1700   1697     }
  1701   1698   
  1702   1699     if( !pCsr->pExpr ){
  1703   1700       sqlite3_result_blob(pContext, "", 0, SQLITE_STATIC);
  1704   1701       return;
         1702  +  }else{
         1703  +    /* Retrieve matchinfo() data. */
         1704  +    rc = fts3GetMatchinfo(pContext, pCsr, zFormat);
         1705  +    sqlite3Fts3SegmentsClose(pTab);
  1705   1706     }
  1706         -
  1707         -  /* Retrieve matchinfo() data. */
  1708         -  rc = fts3GetMatchinfo(pContext, pCsr, zFormat);
  1709         -  sqlite3Fts3SegmentsClose(pTab);
  1710   1707   }
  1711   1708   
  1712   1709   #endif