/ Check-in [8c5b9fedfc]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:More optimization for the 'y' and 'b' matchinfo directives.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | fts3-matchinfo-y
Files: files | file ages | folders
SHA1: 8c5b9fedfcee3ac22a222819dceb981ad94a9903
User & Date: dan 2015-05-06 17:41:19
Context
2015-05-06
17:51
Remove some dead code from fts3_snippet.c. check-in: 46b2d3cef5 user: dan tags: fts3-matchinfo-y
17:41
More optimization for the 'y' and 'b' matchinfo directives. check-in: 8c5b9fedfc user: dan tags: fts3-matchinfo-y
08:43
Further optimizations for the 'y' and 'b' matchinfo operators. check-in: fbd038bb57 user: dan tags: fts3-matchinfo-y
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to ext/fts3/fts3Int.h.

   424    424   
   425    425     /* The following are used by the fts3_eval.c module. */
   426    426     sqlite3_int64 iDocid;      /* Current docid */
   427    427     u8 bEof;                   /* True this expression is at EOF already */
   428    428     u8 bStart;                 /* True if iDocid is valid */
   429    429     u8 bDeferred;              /* True if this expression is entirely deferred */
   430    430   
   431         -  u32 *aMI;
          431  +  /* The following are used by the fts3_snippet.c module. */
          432  +  int iPhrase;               /* Index of this phrase in matchinfo() results */
          433  +  u32 *aMI;                  /* See above */
   432    434   };
   433    435   
   434    436   /*
   435    437   ** Candidate values for Fts3Query.eType. Note that the order of the first
   436    438   ** four values is in order of precedence when parsing expressions. For 
   437    439   ** example, the following:
   438    440   **

Changes to ext/fts3/fts3_snippet.c.

   283    283     Fts3Expr *pExpr,                /* Expression to iterate phrases of */
   284    284     int (*x)(Fts3Expr*,int,void*),  /* Callback function to invoke for phrases */
   285    285     void *pCtx                      /* Second argument to pass to callback */
   286    286   ){
   287    287     int iPhrase = 0;                /* Variable used as the phrase counter */
   288    288     return fts3ExprIterate2(pExpr, &iPhrase, x, pCtx);
   289    289   }
          290  +
   290    291   
   291    292   /*
   292    293   ** This is an fts3ExprIterate() callback used while loading the doclists
   293    294   ** for each phrase into Fts3Expr.aDoclist[]/nDoclist. See also
   294    295   ** fts3ExprLoadDoclists().
   295    296   */
   296    297   static int fts3ExprLoadDoclistsCb(Fts3Expr *pExpr, int iPhrase, void *ctx){
................................................................................
   328    329     if( pnPhrase ) *pnPhrase = sCtx.nPhrase;
   329    330     if( pnToken ) *pnToken = sCtx.nToken;
   330    331     return rc;
   331    332   }
   332    333   
   333    334   static int fts3ExprPhraseCountCb(Fts3Expr *pExpr, int iPhrase, void *ctx){
   334    335     (*(int *)ctx)++;
   335         -  UNUSED_PARAMETER(pExpr);
   336         -  UNUSED_PARAMETER(iPhrase);
          336  +  pExpr->iPhrase = iPhrase;
   337    337     return SQLITE_OK;
   338    338   }
   339    339   static int fts3ExprPhraseCount(Fts3Expr *pExpr){
   340    340     int nPhrase = 0;
   341    341     (void)fts3ExprIterate(pExpr, fts3ExprPhraseCountCb, (void *)&nPhrase);
   342    342     return nPhrase;
   343    343   }
................................................................................
   850    850       c = *pEnd++ & 0x80;
   851    851       if( !c ) nEntry++;
   852    852     }
   853    853   
   854    854     *ppCollist = pEnd;
   855    855     return nEntry;
   856    856   }
          857  +
          858  +/*
          859  +** This function gathers 'y' or 'b' data for a single phrase.
          860  +*/
          861  +static void fts3ExprLHits(
          862  +  Fts3Expr *pExpr,                /* Phrase expression node */
          863  +  MatchInfo *p                    /* Matchinfo context */
          864  +){
          865  +  Fts3Table *pTab = (Fts3Table *)p->pCursor->base.pVtab;
          866  +  int iStart;
          867  +  Fts3Phrase *pPhrase = pExpr->pPhrase;
          868  +  char *pIter = pPhrase->doclist.pList;
          869  +  int iCol = 0;
          870  +
          871  +  assert( p->flag==FTS3_MATCHINFO_LHITS_BM || p->flag==FTS3_MATCHINFO_LHITS );
          872  +  if( p->flag==FTS3_MATCHINFO_LHITS ){
          873  +    iStart = pExpr->iPhrase * p->nCol;
          874  +  }else{
          875  +    iStart = pExpr->iPhrase * ((p->nCol + 31) / 32);
          876  +  }
          877  +
          878  +  while( 1 ){
          879  +    int nHit = fts3ColumnlistCount(&pIter);
          880  +    if( (pPhrase->iColumn>=pTab->nColumn || pPhrase->iColumn==iCol) ){
          881  +      if( p->flag==FTS3_MATCHINFO_LHITS ){
          882  +        p->aMatchinfo[iStart + iCol] = (u32)nHit;
          883  +      }else if( nHit ){
          884  +        p->aMatchinfo[iStart + (iCol+1)/32] |= (1 << (iCol&0x1F));
          885  +      }
          886  +    }
          887  +    assert( *pIter==0x00 || *pIter==0x01 );
          888  +    if( *pIter!=0x01 ) break;
          889  +    pIter++;
          890  +    pIter += fts3GetVarint32(pIter, &iCol);
          891  +  }
          892  +}
          893  +
          894  +/*
          895  +** Gather the results for matchinfo directives 'y' and 'b'.
          896  +*/
          897  +static void fts3ExprLHitGather(
          898  +  Fts3Expr *pExpr,
          899  +  MatchInfo *p
          900  +){
          901  +  assert( (pExpr->pLeft==0)==(pExpr->pRight==0) );
          902  +  if( pExpr->bEof==0 && pExpr->iDocid==p->pCursor->iPrevId ){
          903  +    if( pExpr->pLeft ){
          904  +      fts3ExprLHitGather(pExpr->pLeft, p);
          905  +      fts3ExprLHitGather(pExpr->pRight, p);
          906  +    }else{
          907  +      fts3ExprLHits(pExpr, p);
          908  +    }
          909  +  }
          910  +}
   857    911   
   858    912   /*
   859    913   ** fts3ExprIterate() callback used to collect the "global" matchinfo stats
   860    914   ** for a single query. 
   861    915   **
   862    916   ** fts3ExprIterate() callback to load the 'global' elements of a
   863    917   ** FTS3_MATCHINFO_HITS matchinfo array. The global stats are those elements 
................................................................................
  1268   1322           }
  1269   1323           break;
  1270   1324   
  1271   1325         case FTS3_MATCHINFO_LHITS_BM:
  1272   1326         case FTS3_MATCHINFO_LHITS: {
  1273   1327           int nZero = fts3MatchinfoSize(pInfo, zArg[i]) * sizeof(u32);
  1274   1328           memset(pInfo->aMatchinfo, 0, nZero);
  1275         -        (void)fts3ExprIterate(pCsr->pExpr, fts3ExprLHitsCb, (void*)pInfo);
         1329  +        fts3ExprLHitGather(pCsr->pExpr, pInfo);
  1276   1330           break;
  1277   1331         }
  1278   1332   
  1279   1333         default: {
  1280   1334           Fts3Expr *pExpr;
  1281   1335           assert( zArg[i]==FTS3_MATCHINFO_HITS );
  1282   1336           pExpr = pCsr->pExpr;
................................................................................
  1303   1357   }
  1304   1358   
  1305   1359   
  1306   1360   /*
  1307   1361   ** Populate pCsr->aMatchinfo[] with data for the current row. The 
  1308   1362   ** 'matchinfo' data is an array of 32-bit unsigned integers (C type u32).
  1309   1363   */
  1310         -static int fts3GetMatchinfo(
         1364  +static void fts3GetMatchinfo(
  1311   1365     sqlite3_context *pCtx,        /* Return results here */
  1312   1366     Fts3Cursor *pCsr,               /* FTS3 Cursor object */
  1313   1367     const char *zArg                /* Second argument to matchinfo() function */
  1314   1368   ){
  1315   1369     MatchInfo sInfo;
  1316   1370     Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
  1317   1371     int rc = SQLITE_OK;
................................................................................
  1335   1389     /* If Fts3Cursor.pMIBuffer is NULL, then this is the first time the
  1336   1390     ** matchinfo function has been called for this query. In this case 
  1337   1391     ** allocate the array used to accumulate the matchinfo data and
  1338   1392     ** initialize those elements that are constant for every row.
  1339   1393     */
  1340   1394     if( pCsr->pMIBuffer==0 ){
  1341   1395       int nMatchinfo = 0;           /* Number of u32 elements in match-info */
  1342         -    int nArg;                     /* Bytes in zArg */
  1343   1396       int i;                        /* Used to iterate through zArg */
  1344   1397   
  1345   1398       /* Determine the number of phrases in the query */
  1346   1399       pCsr->nPhrase = fts3ExprPhraseCount(pCsr->pExpr);
  1347   1400       sInfo.nPhrase = pCsr->nPhrase;
  1348   1401   
  1349   1402       /* Determine the number of integers in the buffer returned by this call. */
................................................................................
  1384   1437     if( rc!=SQLITE_OK ){
  1385   1438       sqlite3_result_error_code(pCtx, rc);
  1386   1439       if( xDestroyOut ) xDestroyOut(aOut);
  1387   1440     }else{
  1388   1441       int n = pCsr->pMIBuffer->nElem * sizeof(u32);
  1389   1442       sqlite3_result_blob(pCtx, aOut, n, xDestroyOut);
  1390   1443     }
  1391         -
  1392         -  return rc;
  1393   1444   }
  1394   1445   
  1395   1446   /*
  1396   1447   ** Implementation of snippet() function.
  1397   1448   */
  1398   1449   void sqlite3Fts3Snippet(
  1399   1450     sqlite3_context *pCtx,          /* SQLite function call context */
................................................................................
  1684   1735   void sqlite3Fts3Matchinfo(
  1685   1736     sqlite3_context *pContext,      /* Function call context */
  1686   1737     Fts3Cursor *pCsr,               /* FTS3 table cursor */
  1687   1738     const char *zArg                /* Second arg to matchinfo() function */
  1688   1739   ){
  1689   1740     Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
  1690   1741     const char *zFormat;
  1691         -  int rc;
  1692   1742   
  1693   1743     if( zArg ){
  1694   1744       zFormat = zArg;
  1695   1745     }else{
  1696   1746       zFormat = FTS3_MATCHINFO_DEFAULT;
  1697   1747     }
  1698   1748   
  1699   1749     if( !pCsr->pExpr ){
  1700   1750       sqlite3_result_blob(pContext, "", 0, SQLITE_STATIC);
  1701   1751       return;
  1702   1752     }else{
  1703   1753       /* Retrieve matchinfo() data. */
  1704         -    rc = fts3GetMatchinfo(pContext, pCsr, zFormat);
         1754  +    fts3GetMatchinfo(pContext, pCsr, zFormat);
  1705   1755       sqlite3Fts3SegmentsClose(pTab);
  1706   1756     }
  1707   1757   }
  1708   1758   
  1709   1759   #endif