/ Check-in [2e7679a1df]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Add new fts3 matchinfo option 'b'. Also optimize existing option 'y'.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: 2e7679a1df4020dc0166f5de8ffd664df18a3002
User & Date: dan 2015-05-11 19:01:18
Context
2015-05-12
00:46
Fix sqldiff.exe so that it always runs in single-thread mode. check-in: c223910e72 user: drh tags: trunk
2015-05-11
19:01
Add new fts3 matchinfo option 'b'. Also optimize existing option 'y'. check-in: 2e7679a1df user: dan tags: trunk
18:48
Add missing "finish_test" commands to the end of the two new test scripts for sqlite3_analyzer and sqldiff. check-in: 1d5e72b1c4 user: drh tags: trunk
18:46
Merge latest trunk changes into this branch. Closed-Leaf check-in: 82e5a6e088 user: dan tags: fts3-matchinfo-y
Changes
Hide Diffs Side-by-Side Diffs Ignore Whitespace Patch

Changes to ext/fts3/fts3.c.

  1671   1671   static int fts3CloseMethod(sqlite3_vtab_cursor *pCursor){
  1672   1672     Fts3Cursor *pCsr = (Fts3Cursor *)pCursor;
  1673   1673     assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 );
  1674   1674     sqlite3_finalize(pCsr->pStmt);
  1675   1675     sqlite3Fts3ExprFree(pCsr->pExpr);
  1676   1676     sqlite3Fts3FreeDeferredTokens(pCsr);
  1677   1677     sqlite3_free(pCsr->aDoclist);
  1678         -  sqlite3_free(pCsr->aMatchinfo);
         1678  +  sqlite3Fts3MIBufferFree(pCsr->pMIBuffer);
  1679   1679     assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 );
  1680   1680     sqlite3_free(pCsr);
  1681   1681     return SQLITE_OK;
  1682   1682   }
  1683   1683   
  1684   1684   /*
  1685   1685   ** If pCsr->pStmt has not been prepared (i.e. if pCsr->pStmt==0), then
................................................................................
  3172   3172     if( idxNum & FTS3_HAVE_DOCID_GE ) pDocidGe = apVal[iIdx++];
  3173   3173     if( idxNum & FTS3_HAVE_DOCID_LE ) pDocidLe = apVal[iIdx++];
  3174   3174     assert( iIdx==nVal );
  3175   3175   
  3176   3176     /* In case the cursor has been used before, clear it now. */
  3177   3177     sqlite3_finalize(pCsr->pStmt);
  3178   3178     sqlite3_free(pCsr->aDoclist);
  3179         -  sqlite3_free(pCsr->aMatchinfo);
         3179  +  sqlite3Fts3MIBufferFree(pCsr->pMIBuffer);
  3180   3180     sqlite3Fts3ExprFree(pCsr->pExpr);
  3181   3181     memset(&pCursor[1], 0, sizeof(Fts3Cursor)-sizeof(sqlite3_vtab_cursor));
  3182   3182   
  3183   3183     /* Set the lower and upper bounds on docids to return */
  3184   3184     pCsr->iMinDocid = fts3DocidRange(pDocidGe, SMALLEST_INT64);
  3185   3185     pCsr->iMaxDocid = fts3DocidRange(pDocidLe, LARGEST_INT64);
  3186   3186   

Changes to ext/fts3/fts3Int.h.

   193    193   
   194    194   typedef struct Fts3Doclist Fts3Doclist;
   195    195   typedef struct Fts3SegFilter Fts3SegFilter;
   196    196   typedef struct Fts3DeferredToken Fts3DeferredToken;
   197    197   typedef struct Fts3SegReader Fts3SegReader;
   198    198   typedef struct Fts3MultiSegReader Fts3MultiSegReader;
   199    199   
          200  +typedef struct MatchinfoBuffer MatchinfoBuffer;
          201  +
   200    202   /*
   201    203   ** A connection to a fulltext index is an instance of the following
   202    204   ** structure. The xCreate and xConnect methods create an instance
   203    205   ** of this structure and xDestroy and xDisconnect free that instance.
   204    206   ** All other methods receive a pointer to the structure as one of their
   205    207   ** arguments.
   206    208   */
................................................................................
   302    304     u8 bDesc;                       /* True to sort in descending order */
   303    305     int eEvalmode;                  /* An FTS3_EVAL_XX constant */
   304    306     int nRowAvg;                    /* Average size of database rows, in pages */
   305    307     sqlite3_int64 nDoc;             /* Documents in table */
   306    308     i64 iMinDocid;                  /* Minimum docid to return */
   307    309     i64 iMaxDocid;                  /* Maximum docid to return */
   308    310     int isMatchinfoNeeded;          /* True when aMatchinfo[] needs filling in */
   309         -  u32 *aMatchinfo;                /* Information about most recent match */
   310         -  int nMatchinfo;                 /* Number of elements in aMatchinfo[] */
   311         -  char *zMatchinfo;               /* Matchinfo specification */
          311  +  MatchinfoBuffer *pMIBuffer;     /* Buffer for matchinfo data */
   312    312   };
   313    313   
   314    314   #define FTS3_EVAL_FILTER    0
   315    315   #define FTS3_EVAL_NEXT      1
   316    316   #define FTS3_EVAL_MATCHINFO 2
   317    317   
   318    318   /*
................................................................................
   424    424   
   425    425     /* The following are used by the fts3_eval.c module. */
   426    426     sqlite3_int64 iDocid;      /* Current docid */
   427    427     u8 bEof;                   /* True this expression is at EOF already */
   428    428     u8 bStart;                 /* True if iDocid is valid */
   429    429     u8 bDeferred;              /* True if this expression is entirely deferred */
   430    430   
   431         -  u32 *aMI;
          431  +  /* The following are used by the fts3_snippet.c module. */
          432  +  int iPhrase;               /* Index of this phrase in matchinfo() results */
          433  +  u32 *aMI;                  /* See above */
   432    434   };
   433    435   
   434    436   /*
   435    437   ** Candidate values for Fts3Query.eType. Note that the order of the first
   436    438   ** four values is in order of precedence when parsing expressions. For 
   437    439   ** example, the following:
   438    440   **
................................................................................
   560    562   
   561    563   /* fts3_snippet.c */
   562    564   void sqlite3Fts3Offsets(sqlite3_context*, Fts3Cursor*);
   563    565   void sqlite3Fts3Snippet(sqlite3_context *, Fts3Cursor *, const char *,
   564    566     const char *, const char *, int, int
   565    567   );
   566    568   void sqlite3Fts3Matchinfo(sqlite3_context *, Fts3Cursor *, const char *);
          569  +void sqlite3Fts3MIBufferFree(MatchinfoBuffer *p);
   567    570   
   568    571   /* fts3_expr.c */
   569    572   int sqlite3Fts3ExprParse(sqlite3_tokenizer *, int,
   570    573     char **, int, int, int, const char *, int, Fts3Expr **, char **
   571    574   );
   572    575   void sqlite3Fts3ExprFree(Fts3Expr *);
   573    576   #ifdef SQLITE_TEST

Changes to ext/fts3/fts3_snippet.c.

    24     24   #define FTS3_MATCHINFO_NCOL      'c'        /* 1 value */
    25     25   #define FTS3_MATCHINFO_NDOC      'n'        /* 1 value */
    26     26   #define FTS3_MATCHINFO_AVGLENGTH 'a'        /* nCol values */
    27     27   #define FTS3_MATCHINFO_LENGTH    'l'        /* nCol values */
    28     28   #define FTS3_MATCHINFO_LCS       's'        /* nCol values */
    29     29   #define FTS3_MATCHINFO_HITS      'x'        /* 3*nCol*nPhrase values */
    30     30   #define FTS3_MATCHINFO_LHITS     'y'        /* nCol*nPhrase values */
           31  +#define FTS3_MATCHINFO_LHITS_BM  'b'        /* nCol*nPhrase values */
    31     32   
    32     33   /*
    33     34   ** The default value for the second argument to matchinfo(). 
    34     35   */
    35     36   #define FTS3_MATCHINFO_DEFAULT   "pcx"
    36     37   
    37     38   
................................................................................
    85     86   */
    86     87   typedef struct MatchInfo MatchInfo;
    87     88   struct MatchInfo {
    88     89     Fts3Cursor *pCursor;            /* FTS3 Cursor */
    89     90     int nCol;                       /* Number of columns in table */
    90     91     int nPhrase;                    /* Number of matchable phrases in query */
    91     92     sqlite3_int64 nDoc;             /* Number of docs in database */
           93  +  char flag;
    92     94     u32 *aMatchinfo;                /* Pre-allocated buffer */
    93     95   };
    94     96   
           97  +/*
           98  +** An instance of this structure is used to manage a pair of buffers, each
           99  +** (nElem * sizeof(u32)) bytes in size. See the MatchinfoBuffer code below
          100  +** for details.
          101  +*/
          102  +struct MatchinfoBuffer {
          103  +  u8 aRef[3];
          104  +  int nElem;
          105  +  int bGlobal;                    /* Set if global data is loaded */
          106  +  char *zMatchinfo;
          107  +  u32 aMatchinfo[0];
          108  +};
    95    109   
    96    110   
    97    111   /*
    98    112   ** The snippet() and offsets() functions both return text values. An instance
    99    113   ** of the following structure is used to accumulate those values while the
   100    114   ** functions are running. See fts3StringAppend() for details.
   101    115   */
................................................................................
   102    116   typedef struct StrBuffer StrBuffer;
   103    117   struct StrBuffer {
   104    118     char *z;                        /* Pointer to buffer containing string */
   105    119     int n;                          /* Length of z in bytes (excl. nul-term) */
   106    120     int nAlloc;                     /* Allocated size of buffer z in bytes */
   107    121   };
   108    122   
          123  +
          124  +/*************************************************************************
          125  +** Start of MatchinfoBuffer code.
          126  +*/
          127  +
          128  +/*
          129  +** Allocate a two-slot MatchinfoBuffer object.
          130  +*/
          131  +static MatchinfoBuffer *fts3MIBufferNew(int nElem, const char *zMatchinfo){
          132  +  MatchinfoBuffer *pRet;
          133  +  int nByte = sizeof(u32) * (2*nElem + 2) + sizeof(MatchinfoBuffer);
          134  +  int nStr = strlen(zMatchinfo);
          135  +
          136  +  pRet = sqlite3_malloc(nByte + nStr+1);
          137  +  if( pRet ){
          138  +    memset(pRet, 0, nByte);
          139  +    pRet->aMatchinfo[0] = (u8*)(&pRet->aMatchinfo[1]) - (u8*)pRet;
          140  +    pRet->aMatchinfo[1+nElem] = pRet->aMatchinfo[0] + sizeof(u32)*(nElem+1);
          141  +    pRet->nElem = nElem;
          142  +    pRet->zMatchinfo = ((char*)pRet) + nByte;
          143  +    memcpy(pRet->zMatchinfo, zMatchinfo, nStr+1);
          144  +    pRet->aRef[0] = 1;
          145  +  }
          146  +
          147  +  return pRet;
          148  +}
          149  +
          150  +static void fts3MIBufferFree(void *p){
          151  +  MatchinfoBuffer *pBuf = (MatchinfoBuffer*)((u8*)p - ((u32*)p)[-1]);
          152  +
          153  +  assert( (u32*)p==&pBuf->aMatchinfo[1] 
          154  +       || (u32*)p==&pBuf->aMatchinfo[pBuf->nElem+2] 
          155  +  );
          156  +  if( (u32*)p==&pBuf->aMatchinfo[1] ){
          157  +    pBuf->aRef[1] = 0;
          158  +  }else{
          159  +    pBuf->aRef[2] = 0;
          160  +  }
          161  +
          162  +  if( pBuf->aRef[0]==0 && pBuf->aRef[1]==0 && pBuf->aRef[2]==0 ){
          163  +    sqlite3_free(pBuf);
          164  +  }
          165  +}
          166  +
          167  +static void (*fts3MIBufferAlloc(MatchinfoBuffer *p, u32 **paOut))(void*){
          168  +  void (*xRet)(void*) = 0;
          169  +  u32 *aOut = 0;
          170  +
          171  +  if( p->aRef[1]==0 ){
          172  +    p->aRef[1] = 1;
          173  +    aOut = &p->aMatchinfo[1];
          174  +    xRet = fts3MIBufferFree;
          175  +  }
          176  +  else if( p->aRef[2]==0 ){
          177  +    p->aRef[2] = 1;
          178  +    aOut = &p->aMatchinfo[p->nElem+2];
          179  +    xRet = fts3MIBufferFree;
          180  +  }else{
          181  +    aOut = (u32*)sqlite3_malloc(p->nElem * sizeof(u32));
          182  +    if( aOut ){
          183  +      xRet = sqlite3_free;
          184  +      if( p->bGlobal ) memcpy(aOut, &p->aMatchinfo[1], p->nElem*sizeof(u32));
          185  +    }
          186  +  }
          187  +
          188  +  *paOut = aOut;
          189  +  return xRet;
          190  +}
          191  +
          192  +static void fts3MIBufferSetGlobal(MatchinfoBuffer *p){
          193  +  p->bGlobal = 1;
          194  +  memcpy(&p->aMatchinfo[2+p->nElem], &p->aMatchinfo[1], p->nElem*sizeof(u32));
          195  +}
          196  +
          197  +/*
          198  +** Free a MatchinfoBuffer object allocated using fts3MIBufferNew()
          199  +*/
          200  +void sqlite3Fts3MIBufferFree(MatchinfoBuffer *p){
          201  +  if( p ){
          202  +    assert( p->aRef[0]==1 );
          203  +    p->aRef[0] = 0;
          204  +    if( p->aRef[0]==0 && p->aRef[1]==0 && p->aRef[2]==0 ){
          205  +      sqlite3_free(p);
          206  +    }
          207  +  }
          208  +}
          209  +
          210  +/* 
          211  +** End of MatchinfoBuffer code.
          212  +*************************************************************************/
          213  +
   109    214   
   110    215   /*
   111    216   ** This function is used to help iterate through a position-list. A position
   112    217   ** list is a list of unique integers, sorted from smallest to largest. Each
   113    218   ** element of the list is represented by an FTS3 varint that takes the value
   114    219   ** of the difference between the current element and the previous one plus
   115    220   ** two. For example, to store the position-list:
................................................................................
   139    244   static int fts3ExprIterate2(
   140    245     Fts3Expr *pExpr,                /* Expression to iterate phrases of */
   141    246     int *piPhrase,                  /* Pointer to phrase counter */
   142    247     int (*x)(Fts3Expr*,int,void*),  /* Callback function to invoke for phrases */
   143    248     void *pCtx                      /* Second argument to pass to callback */
   144    249   ){
   145    250     int rc;                         /* Return code */
   146         -  int eType = pExpr->eType;       /* Type of expression node pExpr */
          251  +  int eType = pExpr->eType;     /* Type of expression node pExpr */
   147    252   
   148    253     if( eType!=FTSQUERY_PHRASE ){
   149    254       assert( pExpr->pLeft && pExpr->pRight );
   150    255       rc = fts3ExprIterate2(pExpr->pLeft, piPhrase, x, pCtx);
   151    256       if( rc==SQLITE_OK && eType!=FTSQUERY_NOT ){
   152    257         rc = fts3ExprIterate2(pExpr->pRight, piPhrase, x, pCtx);
   153    258       }
................................................................................
   172    277     Fts3Expr *pExpr,                /* Expression to iterate phrases of */
   173    278     int (*x)(Fts3Expr*,int,void*),  /* Callback function to invoke for phrases */
   174    279     void *pCtx                      /* Second argument to pass to callback */
   175    280   ){
   176    281     int iPhrase = 0;                /* Variable used as the phrase counter */
   177    282     return fts3ExprIterate2(pExpr, &iPhrase, x, pCtx);
   178    283   }
          284  +
   179    285   
   180    286   /*
   181    287   ** This is an fts3ExprIterate() callback used while loading the doclists
   182    288   ** for each phrase into Fts3Expr.aDoclist[]/nDoclist. See also
   183    289   ** fts3ExprLoadDoclists().
   184    290   */
   185    291   static int fts3ExprLoadDoclistsCb(Fts3Expr *pExpr, int iPhrase, void *ctx){
................................................................................
   217    323     if( pnPhrase ) *pnPhrase = sCtx.nPhrase;
   218    324     if( pnToken ) *pnToken = sCtx.nToken;
   219    325     return rc;
   220    326   }
   221    327   
   222    328   static int fts3ExprPhraseCountCb(Fts3Expr *pExpr, int iPhrase, void *ctx){
   223    329     (*(int *)ctx)++;
   224         -  UNUSED_PARAMETER(pExpr);
   225         -  UNUSED_PARAMETER(iPhrase);
          330  +  pExpr->iPhrase = iPhrase;
   226    331     return SQLITE_OK;
   227    332   }
   228    333   static int fts3ExprPhraseCount(Fts3Expr *pExpr){
   229    334     int nPhrase = 0;
   230    335     (void)fts3ExprIterate(pExpr, fts3ExprPhraseCountCb, (void *)&nPhrase);
   231    336     return nPhrase;
   232    337   }
................................................................................
   439    544     ** the set of phrases in the expression to populate the aPhrase[] array.
   440    545     */
   441    546     sIter.pCsr = pCsr;
   442    547     sIter.iCol = iCol;
   443    548     sIter.nSnippet = nSnippet;
   444    549     sIter.nPhrase = nList;
   445    550     sIter.iCurrent = -1;
   446         -  rc = fts3ExprIterate(pCsr->pExpr, fts3SnippetFindPositions, (void *)&sIter);
          551  +  rc = fts3ExprIterate(pCsr->pExpr, fts3SnippetFindPositions, (void*)&sIter);
   447    552     if( rc==SQLITE_OK ){
   448    553   
   449    554       /* Set the *pmSeen output variable. */
   450    555       for(i=0; i<nList; i++){
   451    556         if( sIter.aPhrase[i].pHead ){
   452    557           *pmSeen |= (u64)1 << i;
   453    558         }
................................................................................
   739    844       c = *pEnd++ & 0x80;
   740    845       if( !c ) nEntry++;
   741    846     }
   742    847   
   743    848     *ppCollist = pEnd;
   744    849     return nEntry;
   745    850   }
          851  +
          852  +/*
          853  +** This function gathers 'y' or 'b' data for a single phrase.
          854  +*/
          855  +static void fts3ExprLHits(
          856  +  Fts3Expr *pExpr,                /* Phrase expression node */
          857  +  MatchInfo *p                    /* Matchinfo context */
          858  +){
          859  +  Fts3Table *pTab = (Fts3Table *)p->pCursor->base.pVtab;
          860  +  int iStart;
          861  +  Fts3Phrase *pPhrase = pExpr->pPhrase;
          862  +  char *pIter = pPhrase->doclist.pList;
          863  +  int iCol = 0;
          864  +
          865  +  assert( p->flag==FTS3_MATCHINFO_LHITS_BM || p->flag==FTS3_MATCHINFO_LHITS );
          866  +  if( p->flag==FTS3_MATCHINFO_LHITS ){
          867  +    iStart = pExpr->iPhrase * p->nCol;
          868  +  }else{
          869  +    iStart = pExpr->iPhrase * ((p->nCol + 31) / 32);
          870  +  }
          871  +
          872  +  while( 1 ){
          873  +    int nHit = fts3ColumnlistCount(&pIter);
          874  +    if( (pPhrase->iColumn>=pTab->nColumn || pPhrase->iColumn==iCol) ){
          875  +      if( p->flag==FTS3_MATCHINFO_LHITS ){
          876  +        p->aMatchinfo[iStart + iCol] = (u32)nHit;
          877  +      }else if( nHit ){
          878  +        p->aMatchinfo[iStart + (iCol+1)/32] |= (1 << (iCol&0x1F));
          879  +      }
          880  +    }
          881  +    assert( *pIter==0x00 || *pIter==0x01 );
          882  +    if( *pIter!=0x01 ) break;
          883  +    pIter++;
          884  +    pIter += fts3GetVarint32(pIter, &iCol);
          885  +  }
          886  +}
          887  +
          888  +/*
          889  +** Gather the results for matchinfo directives 'y' and 'b'.
          890  +*/
          891  +static void fts3ExprLHitGather(
          892  +  Fts3Expr *pExpr,
          893  +  MatchInfo *p
          894  +){
          895  +  assert( (pExpr->pLeft==0)==(pExpr->pRight==0) );
          896  +  if( pExpr->bEof==0 && pExpr->iDocid==p->pCursor->iPrevId ){
          897  +    if( pExpr->pLeft ){
          898  +      fts3ExprLHitGather(pExpr->pLeft, p);
          899  +      fts3ExprLHitGather(pExpr->pRight, p);
          900  +    }else{
          901  +      fts3ExprLHits(pExpr, p);
          902  +    }
          903  +  }
          904  +}
   746    905   
   747    906   /*
   748    907   ** fts3ExprIterate() callback used to collect the "global" matchinfo stats
   749    908   ** for a single query. 
   750    909   **
   751    910   ** fts3ExprIterate() callback to load the 'global' elements of a
   752    911   ** FTS3_MATCHINFO_HITS matchinfo array. The global stats are those elements 
................................................................................
   806    965         p->aMatchinfo[iStart+i*3] = 0;
   807    966       }
   808    967     }
   809    968   
   810    969     return rc;
   811    970   }
   812    971   
   813         -/*
   814         -** fts3ExprIterate() callback used to gather information for the matchinfo
   815         -** directive 'y'.
   816         -*/
   817         -static int fts3ExprLHitsCb(
   818         -  Fts3Expr *pExpr,                /* Phrase expression node */
   819         -  int iPhrase,                    /* Phrase number */
   820         -  void *pCtx                      /* Pointer to MatchInfo structure */
   821         -){
   822         -  MatchInfo *p = (MatchInfo *)pCtx;
   823         -  Fts3Table *pTab = (Fts3Table *)p->pCursor->base.pVtab;
   824         -  int rc = SQLITE_OK;
   825         -  int iStart = iPhrase * p->nCol;
   826         -  Fts3Expr *pEof;                 /* Ancestor node already at EOF */
   827         -  
   828         -  /* This must be a phrase */
   829         -  assert( pExpr->pPhrase );
   830         -
   831         -  /* Initialize all output integers to zero. */
   832         -  memset(&p->aMatchinfo[iStart], 0, sizeof(u32) * p->nCol);
   833         -
   834         -  /* Check if this or any parent node is at EOF. If so, then all output
   835         -  ** values are zero.  */
   836         -  for(pEof=pExpr; pEof && pEof->bEof==0; pEof=pEof->pParent);
   837         -
   838         -  if( pEof==0 && pExpr->iDocid==p->pCursor->iPrevId ){
   839         -    Fts3Phrase *pPhrase = pExpr->pPhrase;
   840         -    char *pIter = pPhrase->doclist.pList;
   841         -    int iCol = 0;
   842         -
   843         -    while( 1 ){
   844         -      int nHit = fts3ColumnlistCount(&pIter);
   845         -      if( (pPhrase->iColumn>=pTab->nColumn || pPhrase->iColumn==iCol) ){
   846         -        p->aMatchinfo[iStart + iCol] = (u32)nHit;
   847         -      }
   848         -      assert( *pIter==0x00 || *pIter==0x01 );
   849         -      if( *pIter!=0x01 ) break;
   850         -      pIter++;
   851         -      pIter += fts3GetVarint32(pIter, &iCol);
   852         -    }
   853         -  }
   854         -
   855         -  return rc;
   856         -}
   857         -
   858    972   static int fts3MatchinfoCheck(
   859    973     Fts3Table *pTab, 
   860    974     char cArg,
   861    975     char **pzErr
   862    976   ){
   863    977     if( (cArg==FTS3_MATCHINFO_NPHRASE)
   864    978      || (cArg==FTS3_MATCHINFO_NCOL)
   865    979      || (cArg==FTS3_MATCHINFO_NDOC && pTab->bFts4)
   866    980      || (cArg==FTS3_MATCHINFO_AVGLENGTH && pTab->bFts4)
   867    981      || (cArg==FTS3_MATCHINFO_LENGTH && pTab->bHasDocsize)
   868    982      || (cArg==FTS3_MATCHINFO_LCS)
   869    983      || (cArg==FTS3_MATCHINFO_HITS)
   870    984      || (cArg==FTS3_MATCHINFO_LHITS)
          985  +   || (cArg==FTS3_MATCHINFO_LHITS_BM)
   871    986     ){
   872    987       return SQLITE_OK;
   873    988     }
   874    989     sqlite3Fts3ErrMsg(pzErr, "unrecognized matchinfo request: %c", cArg);
   875    990     return SQLITE_ERROR;
   876    991   }
   877    992   
................................................................................
   890   1005       case FTS3_MATCHINFO_LCS:
   891   1006         nVal = pInfo->nCol;
   892   1007         break;
   893   1008   
   894   1009       case FTS3_MATCHINFO_LHITS:
   895   1010         nVal = pInfo->nCol * pInfo->nPhrase;
   896   1011         break;
         1012  +
         1013  +    case FTS3_MATCHINFO_LHITS_BM:
         1014  +      nVal = pInfo->nPhrase * ((pInfo->nCol + 31) / 32);
         1015  +      break;
   897   1016   
   898   1017       default:
   899   1018         assert( cArg==FTS3_MATCHINFO_HITS );
   900   1019         nVal = pInfo->nCol * pInfo->nPhrase * 3;
   901   1020         break;
   902   1021     }
   903   1022   
................................................................................
  1085   1204   ){
  1086   1205     int rc = SQLITE_OK;
  1087   1206     int i;
  1088   1207     Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
  1089   1208     sqlite3_stmt *pSelect = 0;
  1090   1209   
  1091   1210     for(i=0; rc==SQLITE_OK && zArg[i]; i++){
  1092         -
         1211  +    pInfo->flag = zArg[i];
  1093   1212       switch( zArg[i] ){
  1094   1213         case FTS3_MATCHINFO_NPHRASE:
  1095   1214           if( bGlobal ) pInfo->aMatchinfo[0] = pInfo->nPhrase;
  1096   1215           break;
  1097   1216   
  1098   1217         case FTS3_MATCHINFO_NCOL:
  1099   1218           if( bGlobal ) pInfo->aMatchinfo[0] = pInfo->nCol;
................................................................................
  1145   1264         case FTS3_MATCHINFO_LCS:
  1146   1265           rc = fts3ExprLoadDoclists(pCsr, 0, 0);
  1147   1266           if( rc==SQLITE_OK ){
  1148   1267             rc = fts3MatchinfoLcs(pCsr, pInfo);
  1149   1268           }
  1150   1269           break;
  1151   1270   
  1152         -      case FTS3_MATCHINFO_LHITS:
  1153         -        (void)fts3ExprIterate(pCsr->pExpr, fts3ExprLHitsCb, (void*)pInfo);
         1271  +      case FTS3_MATCHINFO_LHITS_BM:
         1272  +      case FTS3_MATCHINFO_LHITS: {
         1273  +        int nZero = fts3MatchinfoSize(pInfo, zArg[i]) * sizeof(u32);
         1274  +        memset(pInfo->aMatchinfo, 0, nZero);
         1275  +        fts3ExprLHitGather(pCsr->pExpr, pInfo);
  1154   1276           break;
         1277  +      }
  1155   1278   
  1156   1279         default: {
  1157   1280           Fts3Expr *pExpr;
  1158   1281           assert( zArg[i]==FTS3_MATCHINFO_HITS );
  1159   1282           pExpr = pCsr->pExpr;
  1160   1283           rc = fts3ExprLoadDoclists(pCsr, 0, 0);
  1161   1284           if( rc!=SQLITE_OK ) break;
................................................................................
  1180   1303   }
  1181   1304   
  1182   1305   
  1183   1306   /*
  1184   1307   ** Populate pCsr->aMatchinfo[] with data for the current row. The 
  1185   1308   ** 'matchinfo' data is an array of 32-bit unsigned integers (C type u32).
  1186   1309   */
  1187         -static int fts3GetMatchinfo(
         1310  +static void fts3GetMatchinfo(
         1311  +  sqlite3_context *pCtx,        /* Return results here */
  1188   1312     Fts3Cursor *pCsr,               /* FTS3 Cursor object */
  1189   1313     const char *zArg                /* Second argument to matchinfo() function */
  1190   1314   ){
  1191   1315     MatchInfo sInfo;
  1192   1316     Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
  1193   1317     int rc = SQLITE_OK;
  1194   1318     int bGlobal = 0;                /* Collect 'global' stats as well as local */
  1195   1319   
         1320  +  u32 *aOut = 0;
         1321  +  void (*xDestroyOut)(void*) = 0;
         1322  +
  1196   1323     memset(&sInfo, 0, sizeof(MatchInfo));
  1197   1324     sInfo.pCursor = pCsr;
  1198   1325     sInfo.nCol = pTab->nColumn;
  1199   1326   
  1200   1327     /* If there is cached matchinfo() data, but the format string for the 
  1201   1328     ** cache does not match the format string for this request, discard 
  1202   1329     ** the cached data. */
  1203         -  if( pCsr->zMatchinfo && strcmp(pCsr->zMatchinfo, zArg) ){
  1204         -    assert( pCsr->aMatchinfo );
  1205         -    sqlite3_free(pCsr->aMatchinfo);
  1206         -    pCsr->zMatchinfo = 0;
  1207         -    pCsr->aMatchinfo = 0;
         1330  +  if( pCsr->pMIBuffer && strcmp(pCsr->pMIBuffer->zMatchinfo, zArg) ){
         1331  +    sqlite3Fts3MIBufferFree(pCsr->pMIBuffer);
         1332  +    pCsr->pMIBuffer = 0;
  1208   1333     }
  1209   1334   
  1210         -  /* If Fts3Cursor.aMatchinfo[] is NULL, then this is the first time the
         1335  +  /* If Fts3Cursor.pMIBuffer is NULL, then this is the first time the
  1211   1336     ** matchinfo function has been called for this query. In this case 
  1212   1337     ** allocate the array used to accumulate the matchinfo data and
  1213   1338     ** initialize those elements that are constant for every row.
  1214   1339     */
  1215         -  if( pCsr->aMatchinfo==0 ){
         1340  +  if( pCsr->pMIBuffer==0 ){
  1216   1341       int nMatchinfo = 0;           /* Number of u32 elements in match-info */
  1217         -    int nArg;                     /* Bytes in zArg */
  1218   1342       int i;                        /* Used to iterate through zArg */
  1219   1343   
  1220   1344       /* Determine the number of phrases in the query */
  1221   1345       pCsr->nPhrase = fts3ExprPhraseCount(pCsr->pExpr);
  1222   1346       sInfo.nPhrase = pCsr->nPhrase;
  1223   1347   
  1224   1348       /* Determine the number of integers in the buffer returned by this call. */
  1225   1349       for(i=0; zArg[i]; i++){
         1350  +      char *zErr = 0;
         1351  +      if( fts3MatchinfoCheck(pTab, zArg[i], &zErr) ){
         1352  +        sqlite3_result_error(pCtx, zErr, -1);
         1353  +        sqlite3_free(zErr);
         1354  +        return;
         1355  +      }
  1226   1356         nMatchinfo += fts3MatchinfoSize(&sInfo, zArg[i]);
  1227   1357       }
  1228   1358   
  1229   1359       /* Allocate space for Fts3Cursor.aMatchinfo[] and Fts3Cursor.zMatchinfo. */
  1230         -    nArg = (int)strlen(zArg);
  1231         -    pCsr->aMatchinfo = (u32 *)sqlite3_malloc(sizeof(u32)*nMatchinfo + nArg + 1);
  1232         -    if( !pCsr->aMatchinfo ) return SQLITE_NOMEM;
         1360  +    pCsr->pMIBuffer = fts3MIBufferNew(nMatchinfo, zArg);
         1361  +    if( !pCsr->pMIBuffer ) rc = SQLITE_NOMEM;
  1233   1362   
  1234         -    pCsr->zMatchinfo = (char *)&pCsr->aMatchinfo[nMatchinfo];
  1235         -    pCsr->nMatchinfo = nMatchinfo;
  1236         -    memcpy(pCsr->zMatchinfo, zArg, nArg+1);
  1237         -    memset(pCsr->aMatchinfo, 0, sizeof(u32)*nMatchinfo);
  1238   1363       pCsr->isMatchinfoNeeded = 1;
  1239   1364       bGlobal = 1;
  1240   1365     }
  1241   1366   
  1242         -  sInfo.aMatchinfo = pCsr->aMatchinfo;
  1243         -  sInfo.nPhrase = pCsr->nPhrase;
  1244         -  if( pCsr->isMatchinfoNeeded ){
         1367  +  if( rc==SQLITE_OK ){
         1368  +    xDestroyOut = fts3MIBufferAlloc(pCsr->pMIBuffer, &aOut);
         1369  +    if( xDestroyOut==0 ){
         1370  +      rc = SQLITE_NOMEM;
         1371  +    }
         1372  +  }
         1373  +
         1374  +  if( rc==SQLITE_OK ){
         1375  +    sInfo.aMatchinfo = aOut;
         1376  +    sInfo.nPhrase = pCsr->nPhrase;
  1245   1377       rc = fts3MatchinfoValues(pCsr, bGlobal, &sInfo, zArg);
  1246         -    pCsr->isMatchinfoNeeded = 0;
         1378  +    if( bGlobal ){
         1379  +      fts3MIBufferSetGlobal(pCsr->pMIBuffer);
         1380  +    }
  1247   1381     }
  1248   1382   
  1249         -  return rc;
         1383  +  if( rc!=SQLITE_OK ){
         1384  +    sqlite3_result_error_code(pCtx, rc);
         1385  +    if( xDestroyOut ) xDestroyOut(aOut);
         1386  +  }else{
         1387  +    int n = pCsr->pMIBuffer->nElem * sizeof(u32);
         1388  +    sqlite3_result_blob(pCtx, aOut, n, xDestroyOut);
         1389  +  }
  1250   1390   }
  1251   1391   
  1252   1392   /*
  1253   1393   ** Implementation of snippet() function.
  1254   1394   */
  1255   1395   void sqlite3Fts3Snippet(
  1256   1396     sqlite3_context *pCtx,          /* SQLite function call context */
................................................................................
  1448   1588   
  1449   1589       /* Initialize the contents of sCtx.aTerm[] for column iCol. There is 
  1450   1590       ** no way that this operation can fail, so the return code from
  1451   1591       ** fts3ExprIterate() can be discarded.
  1452   1592       */
  1453   1593       sCtx.iCol = iCol;
  1454   1594       sCtx.iTerm = 0;
  1455         -    (void)fts3ExprIterate(pCsr->pExpr, fts3ExprTermOffsetInit, (void *)&sCtx);
         1595  +    (void)fts3ExprIterate(pCsr->pExpr, fts3ExprTermOffsetInit, (void*)&sCtx);
  1456   1596   
  1457   1597       /* Retreive the text stored in column iCol. If an SQL NULL is stored 
  1458   1598       ** in column iCol, jump immediately to the next iteration of the loop.
  1459   1599       ** If an OOM occurs while retrieving the data (this can happen if SQLite
  1460   1600       ** needs to transform the data from utf-16 to utf-8), return SQLITE_NOMEM 
  1461   1601       ** to the caller. 
  1462   1602       */
................................................................................
  1540   1680   */
  1541   1681   void sqlite3Fts3Matchinfo(
  1542   1682     sqlite3_context *pContext,      /* Function call context */
  1543   1683     Fts3Cursor *pCsr,               /* FTS3 table cursor */
  1544   1684     const char *zArg                /* Second arg to matchinfo() function */
  1545   1685   ){
  1546   1686     Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
  1547         -  int rc;
  1548         -  int i;
  1549   1687     const char *zFormat;
  1550   1688   
  1551   1689     if( zArg ){
  1552         -    for(i=0; zArg[i]; i++){
  1553         -      char *zErr = 0;
  1554         -      if( fts3MatchinfoCheck(pTab, zArg[i], &zErr) ){
  1555         -        sqlite3_result_error(pContext, zErr, -1);
  1556         -        sqlite3_free(zErr);
  1557         -        return;
  1558         -      }
  1559         -    }
  1560   1690       zFormat = zArg;
  1561   1691     }else{
  1562   1692       zFormat = FTS3_MATCHINFO_DEFAULT;
  1563   1693     }
  1564   1694   
  1565   1695     if( !pCsr->pExpr ){
  1566   1696       sqlite3_result_blob(pContext, "", 0, SQLITE_STATIC);
  1567   1697       return;
  1568         -  }
  1569         -
  1570         -  /* Retrieve matchinfo() data. */
  1571         -  rc = fts3GetMatchinfo(pCsr, zFormat);
  1572         -  sqlite3Fts3SegmentsClose(pTab);
  1573         -
  1574         -  if( rc!=SQLITE_OK ){
  1575         -    sqlite3_result_error_code(pContext, rc);
  1576   1698     }else{
  1577         -    int n = pCsr->nMatchinfo * sizeof(u32);
  1578         -    sqlite3_result_blob(pContext, pCsr->aMatchinfo, n, SQLITE_TRANSIENT);
         1699  +    /* Retrieve matchinfo() data. */
         1700  +    fts3GetMatchinfo(pContext, pCsr, zFormat);
         1701  +    sqlite3Fts3SegmentsClose(pTab);
  1579   1702     }
  1580   1703   }
  1581   1704   
  1582   1705   #endif

Changes to test/fts3matchinfo.test.

   503    503     7 "a OR (a AND b)" {
   504    504         1 {1 2 1 2 0 1}   2 {1 0 1 0 1 0}   3 {0 1 0 1 1 2}   4 {1 0 1 0 0 1}   
   505    505         5 {1 0 1 0 0 1}   6 {1 0 1 0 2 2}   7 {2 1 0 0 0 0}   8 {1 2 1 2 2 1}   
   506    506         9 {1 1 1 1 1 3}  10 {1 3 0 0 0 0}
   507    507     }
   508    508   
   509    509   } {
   510         -  do_execsql_test 11.1.$tn  {
          510  +  do_execsql_test 11.1.$tn.1  {
   511    511       SELECT rowid, mit(matchinfo(tt, 'y')) FROM tt WHERE tt MATCH $expr
   512    512     } $res
          513  +
          514  +  set r2 [list]
          515  +  foreach {rowid L} $res {
          516  +    lappend r2 $rowid
          517  +    set M [list]
          518  +    foreach {a b} $L {
          519  +      lappend M [expr ($a ? 1 : 0) + ($b ? 2 : 0)]
          520  +    }
          521  +    lappend r2 $M
          522  +  }
          523  +
          524  +  do_execsql_test 11.1.$tn.2  {
          525  +    SELECT rowid, mit(matchinfo(tt, 'b')) FROM tt WHERE tt MATCH $expr
          526  +  } $r2
          527  +  breakpoint
          528  +
          529  +  do_execsql_test 11.1.$tn.2  {
          530  +    SELECT rowid, mit(matchinfo(tt, 'b')) FROM tt WHERE tt MATCH $expr
          531  +  } $r2
   513    532   }
   514    533   set sqlite_fts3_enable_parentheses 0
   515    534   
          535  +#---------------------------------------------------------------------------
          536  +# Test the 'b' matchinfo flag
          537  +#
          538  +set sqlite_fts3_enable_parentheses 1
          539  +reset_db
          540  +db func mit mit
          541  +
          542  +do_test 12.0 {
          543  +  set cols [list]
          544  +  for {set i 0} {$i < 50} {incr i} { lappend cols "c$i" }
          545  +  execsql "CREATE VIRTUAL TABLE tt USING fts3([join $cols ,])"
          546  +} {}
          547  +
          548  +do_execsql_test 12.1 {
          549  +  INSERT INTO tt (rowid, c4, c45) VALUES(1, 'abc', 'abc');
          550  +  SELECT mit(matchinfo(tt, 'b')) FROM tt WHERE tt MATCH 'abc';
          551  +} [list [list [expr 1<<4] [expr 1<<(45-32)]]]
          552  +
          553  +set sqlite_fts3_enable_parentheses 0
   516    554   finish_test
          555  +

Changes to test/fts3query.test.

   169    169   } {
   170    170     1 "SELECT matchinfo(content) FROM t2 WHERE t2 MATCH 'history'" matchinfo
   171    171     2 "SELECT offsets(content) FROM t2 WHERE t2 MATCH 'history'"   offsets
   172    172     3 "SELECT snippet(content) FROM t2 WHERE t2 MATCH 'history'"   snippet
   173    173     4 "SELECT optimize(content) FROM t2 WHERE t2 MATCH 'history'"  optimize
   174    174   }
   175    175   do_catchsql_test 5.5.1 {
   176         -  SELECT matchinfo(t2, 'abc') FROM t2 WHERE t2 MATCH 'history'
   177         -} {1 {unrecognized matchinfo request: b}}
          176  +  SELECT matchinfo(t2, 'abcd') FROM t2 WHERE t2 MATCH 'history'
          177  +} {1 {unrecognized matchinfo request: d}}
   178    178   
   179    179   do_execsql_test 5.5 { DROP TABLE t2 }
   180    180   
   181    181   
   182    182   # Test the snippet() function with 1 to 6 arguments.
   183    183   # 
   184    184   do_execsql_test 6.1 {