Many hyperlinks are disabled.
Use anonymous login
to enable hyperlinks.
Changes In Branch fts3-changes Excluding Merge-Ins
This is equivalent to a diff from 29e69f389c to 91daea7d2e
2011-06-28
| ||
14:16 | Merge the fts3-changes branch back into the trunk. (check-in: b9477eb056 user: dan tags: trunk) | |
11:58 | Add a fix and tests for the FTS deferred token logic. (Closed-Leaf check-in: 91daea7d2e user: dan tags: fts3-changes) | |
09:51 | Merge latest trunk changes with fts3-changes branch. (check-in: 226686475c user: dan tags: fts3-changes) | |
07:15 | Changes to allow FTS to be compiled as a loadable module again. (check-in: 29e69f389c user: dan tags: trunk) | |
2011-06-27
| ||
19:37 | Remove an unnecessary assignment from vdbeapi.c. (check-in: 6c871ac1fa user: dan tags: trunk) | |
Changes to ext/fts3/fts3.c.
︙ | ︙ | |||
308 309 310 311 312 313 314 315 316 317 318 319 320 321 | #include "fts3.h" #ifndef SQLITE_CORE # include "sqlite3ext.h" SQLITE_EXTENSION_INIT1 #endif /* ** Write a 64-bit variable-length integer to memory starting at p[0]. ** The length of data written will be between 1 and FTS3_VARINT_MAX bytes. ** The number of bytes written is returned. */ int sqlite3Fts3PutVarint(char *p, sqlite_int64 v){ unsigned char *q = (unsigned char *) p; | > > > > > | 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 | #include "fts3.h" #ifndef SQLITE_CORE # include "sqlite3ext.h" SQLITE_EXTENSION_INIT1 #endif static int fts3EvalNext(Fts3Cursor *pCsr); static int fts3EvalStart(Fts3Cursor *pCsr); static int fts3TermSegReaderCursor( Fts3Cursor *, const char *, int, int, Fts3MultiSegReader **); /* ** Write a 64-bit variable-length integer to memory starting at p[0]. ** The length of data written will be between 1 and FTS3_VARINT_MAX bytes. ** The number of bytes written is returned. */ int sqlite3Fts3PutVarint(char *p, sqlite_int64 v){ unsigned char *q = (unsigned char *) p; |
︙ | ︙ | |||
816 817 818 819 820 821 822 823 | for(i=0; i<p->nColumn; i++){ fts3Appendf(pRc, &zRet, ",%s(?)", zFunction); } sqlite3_free(zFree); return zRet; } static int fts3GobbleInt(const char **pp, int *pnOut){ | > > > > > > > > > > > > > | | > | > > > > > > > > > > > > > > > > | < | | | | 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 | for(i=0; i<p->nColumn; i++){ fts3Appendf(pRc, &zRet, ",%s(?)", zFunction); } sqlite3_free(zFree); return zRet; } /* ** This function interprets the string at (*pp) as a non-negative integer ** value. It reads the integer and sets *pnOut to the value read, then ** sets *pp to point to the byte immediately following the last byte of ** the integer value. ** ** Only decimal digits ('0'..'9') may be part of an integer value. ** ** If *pp does not being with a decimal digit SQLITE_ERROR is returned and ** the output value undefined. Otherwise SQLITE_OK is returned. ** ** This function is used when parsing the "prefix=" FTS4 parameter. */ static int fts3GobbleInt(const char **pp, int *pnOut){ const char *p = *pp; /* Iterator pointer */ int nInt = 0; /* Output value */ for(p=*pp; p[0]>='0' && p[0]<='9'; p++){ nInt = nInt * 10 + (p[0] - '0'); } if( p==*pp ) return SQLITE_ERROR; *pnOut = nInt; *pp = p; return SQLITE_OK; } /* ** This function is called to allocate an array of Fts3Index structures ** representing the indexes maintained by the current FTS table. FTS tables ** always maintain the main "terms" index, but may also maintain one or ** more "prefix" indexes, depending on the value of the "prefix=" parameter ** (if any) specified as part of the CREATE VIRTUAL TABLE statement. ** ** Argument zParam is passed the value of the "prefix=" option if one was ** specified, or NULL otherwise. ** ** If no error occurs, SQLITE_OK is returned and *apIndex set to point to ** the allocated array. *pnIndex is set to the number of elements in the ** array. If an error does occur, an SQLite error code is returned. ** ** Regardless of whether or not an error is returned, it is the responsibility ** of the caller to call sqlite3_free() on the output array to free it. */ static int fts3PrefixParameter( const char *zParam, /* ABC in prefix=ABC parameter to parse */ int *pnIndex, /* OUT: size of *apIndex[] array */ struct Fts3Index **apIndex /* OUT: Array of indexes for this table */ ){ struct Fts3Index *aIndex; /* Allocated array */ int nIndex = 1; /* Number of entries in array */ if( zParam && zParam[0] ){ const char *p; nIndex++; for(p=zParam; *p; p++){ if( *p==',' ) nIndex++; } } aIndex = sqlite3_malloc(sizeof(struct Fts3Index) * nIndex); *apIndex = aIndex; *pnIndex = nIndex; if( !aIndex ){ return SQLITE_NOMEM; } memset(aIndex, 0, sizeof(struct Fts3Index) * nIndex); if( zParam ){ |
︙ | ︙ | |||
904 905 906 907 908 909 910 | int nDb; /* Bytes required to hold database name */ int nName; /* Bytes required to hold table name */ int isFts4 = (argv[0][3]=='4'); /* True for FTS4, false for FTS3 */ const char **aCol; /* Array of column names */ sqlite3_tokenizer *pTokenizer = 0; /* Tokenizer for this table */ int nIndex; /* Size of aIndex[] array */ | | < | 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 | int nDb; /* Bytes required to hold database name */ int nName; /* Bytes required to hold table name */ int isFts4 = (argv[0][3]=='4'); /* True for FTS4, false for FTS3 */ const char **aCol; /* Array of column names */ sqlite3_tokenizer *pTokenizer = 0; /* Tokenizer for this table */ int nIndex; /* Size of aIndex[] array */ struct Fts3Index *aIndex = 0; /* Array of indexes for this table */ /* The results of parsing supported FTS4 key=value options: */ int bNoDocsize = 0; /* True to omit %_docsize table */ int bDescIdx = 0; /* True to store descending indexes */ char *zPrefix = 0; /* Prefix parameter value (or NULL) */ char *zCompress = 0; /* compress=? parameter (or NULL) */ char *zUncompress = 0; /* uncompress=? parameter (or NULL) */ |
︙ | ︙ | |||
1042 1043 1044 1045 1046 1047 1048 | if( pTokenizer==0 ){ rc = sqlite3Fts3InitTokenizer(pHash, "simple", &pTokenizer, pzErr); if( rc!=SQLITE_OK ) goto fts3_init_out; } assert( pTokenizer ); | | | 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 | if( pTokenizer==0 ){ rc = sqlite3Fts3InitTokenizer(pHash, "simple", &pTokenizer, pzErr); if( rc!=SQLITE_OK ) goto fts3_init_out; } assert( pTokenizer ); rc = fts3PrefixParameter(zPrefix, &nIndex, &aIndex); if( rc==SQLITE_ERROR ){ assert( zPrefix ); *pzErr = sqlite3_mprintf("error parsing prefix parameter: %s", zPrefix); } if( rc!=SQLITE_OK ) goto fts3_init_out; /* Allocate and populate the Fts3Table structure. */ |
︙ | ︙ | |||
1129 1130 1131 1132 1133 1134 1135 | p->nNodeSize = p->nPgsz-35; /* Declare the table schema to SQLite. */ fts3DeclareVtab(&rc, p); fts3_init_out: sqlite3_free(zPrefix); | | | 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 | p->nNodeSize = p->nPgsz-35; /* Declare the table schema to SQLite. */ fts3DeclareVtab(&rc, p); fts3_init_out: sqlite3_free(zPrefix); sqlite3_free(aIndex); sqlite3_free(zCompress); sqlite3_free(zUncompress); sqlite3_free((void *)aCol); if( rc!=SQLITE_OK ){ if( p ){ fts3DisconnectMethod((sqlite3_vtab *)p); }else if( pTokenizer ){ |
︙ | ︙ | |||
1720 1721 1722 1723 1724 1725 1726 | *p++ = POS_END; *pp = p; *pp1 = p1 + 1; *pp2 = p2 + 1; } /* | < < > > | 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 | *p++ = POS_END; *pp = p; *pp1 = p1 + 1; *pp2 = p2 + 1; } /* ** This function is used to merge two position lists into one. When it is ** called, *pp1 and *pp2 must both point to position lists. A position-list is ** the part of a doclist that follows each document id. For example, if a row ** contains: ** ** 'a b c'|'x y z'|'a b b a' ** ** Then the position list for this row for token 'b' would consist of: ** ** 0x02 0x01 0x02 0x03 0x03 0x00 ** ** When this function returns, both *pp1 and *pp2 are left pointing to the ** byte following the 0x00 terminator of their respective position lists. ** ** If isSaveLeft is 0, an entry is added to the output position list for ** each position in *pp2 for which there exists one or more positions in ** *pp1 so that (pos(*pp2)>pos(*pp1) && pos(*pp2)-pos(*pp1)<=nToken). i.e. ** when the *pp1 token appears before the *pp2 token, but not more than nToken ** slots before it. ** ** e.g. nToken==1 searches for adjacent positions. */ static int fts3PoslistPhraseMerge( char **pp, /* IN/OUT: Preallocated output buffer */ int nToken, /* Maximum difference in token positions */ int isSaveLeft, /* Save the left position */ int isExact, /* If *pp1 is exactly nTokens before *pp2 */ char **pp1, /* IN/OUT: Left input list */ |
︙ | ︙ | |||
1907 1908 1909 1910 1911 1912 1913 | res = 0; } return res; } /* | | | > < | | | > > > > > > > > > > > > | | | | > > > > > > > > > > > > > > > | > > > > > > > > > > > > > > > > > > > > > > > > | | 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 | res = 0; } return res; } /* ** An instance of this function is used to merge together the (potentially ** large number of) doclists for each term that matches a prefix query. ** See function fts3TermSelectMerge() for details. */ typedef struct TermSelect TermSelect; struct TermSelect { char *aaOutput[16]; /* Malloc'd output buffers */ int anOutput[16]; /* Size each output buffer in bytes */ }; /* ** This function is used to read a single varint from a buffer. Parameter ** pEnd points 1 byte past the end of the buffer. When this function is ** called, if *pp points to pEnd or greater, then the end of the buffer ** has been reached. In this case *pp is set to 0 and the function returns. ** ** If *pp does not point to or past pEnd, then a single varint is read ** from *pp. *pp is then set to point 1 byte past the end of the read varint. ** ** If bDescIdx is false, the value read is added to *pVal before returning. ** If it is true, the value read is subtracted from *pVal before this ** function returns. */ static void fts3GetDeltaVarint3( char **pp, /* IN/OUT: Point to read varint from */ char *pEnd, /* End of buffer */ int bDescIdx, /* True if docids are descending */ sqlite3_int64 *pVal /* IN/OUT: Integer value */ ){ if( *pp>=pEnd ){ *pp = 0; }else{ sqlite3_int64 iVal; *pp += sqlite3Fts3GetVarint(*pp, &iVal); if( bDescIdx ){ *pVal -= iVal; }else{ *pVal += iVal; } } } /* ** This function is used to write a single varint to a buffer. The varint ** is written to *pp. Before returning, *pp is set to point 1 byte past the ** end of the value written. ** ** If *pbFirst is zero when this function is called, the value written to ** the buffer is that of parameter iVal. ** ** If *pbFirst is non-zero when this function is called, then the value ** written is either (iVal-*piPrev) (if bDescIdx is zero) or (*piPrev-iVal) ** (if bDescIdx is non-zero). ** ** Before returning, this function always sets *pbFirst to 1 and *piPrev ** to the value of parameter iVal. */ static void fts3PutDeltaVarint3( char **pp, /* IN/OUT: Output pointer */ int bDescIdx, /* True for descending docids */ sqlite3_int64 *piPrev, /* IN/OUT: Previous value written to list */ int *pbFirst, /* IN/OUT: True after first int written */ sqlite3_int64 iVal /* Write this value to the list */ ){ sqlite3_int64 iWrite; if( bDescIdx==0 || *pbFirst==0 ){ iWrite = iVal - *piPrev; }else{ iWrite = *piPrev - iVal; } assert( *pbFirst || *piPrev==0 ); assert( *pbFirst==0 || iWrite>0 ); *pp += sqlite3Fts3PutVarint(*pp, iWrite); *piPrev = iVal; *pbFirst = 1; } /* ** This macro is used by various functions that merge doclists. The two ** arguments are 64-bit docid values. If the value of the stack variable ** bDescDoclist is 0 when this macro is invoked, then it returns (i1-i2). ** Otherwise, (i2-i1). ** ** Using this makes it easier to write code that can merge doclists that are ** sorted in either ascending or descending order. */ #define DOCID_CMP(i1, i2) ((bDescDoclist?-1:1) * (i1-i2)) /* ** This function does an "OR" merge of two doclists (output contains all ** positions contained in either argument doclist). If the docids in the ** input doclists are sorted in ascending order, parameter bDescDoclist ** should be false. If they are sorted in ascending order, it should be ** passed a non-zero value. ** ** If no error occurs, *paOut is set to point at an sqlite3_malloc'd buffer ** containing the output doclist and SQLITE_OK is returned. In this case ** *pnOut is set to the number of bytes in the output doclist. ** ** If an error occurs, an SQLite error code is returned. The output values ** are undefined in this case. */ static int fts3DoclistOrMerge( int bDescDoclist, /* True if arguments are desc */ char *a1, int n1, /* First doclist */ char *a2, int n2, /* Second doclist */ char **paOut, int *pnOut /* OUT: Malloc'd doclist */ ){ sqlite3_int64 i1 = 0; sqlite3_int64 i2 = 0; sqlite3_int64 iPrev = 0; |
︙ | ︙ | |||
1985 1986 1987 1988 1989 1990 1991 | aOut = sqlite3_malloc(n1+n2); if( !aOut ) return SQLITE_NOMEM; p = aOut; fts3GetDeltaVarint3(&p1, pEnd1, 0, &i1); fts3GetDeltaVarint3(&p2, pEnd2, 0, &i2); while( p1 || p2 ){ | | | | | | | | | > > > > > > > > > > > > | | 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 | aOut = sqlite3_malloc(n1+n2); if( !aOut ) return SQLITE_NOMEM; p = aOut; fts3GetDeltaVarint3(&p1, pEnd1, 0, &i1); fts3GetDeltaVarint3(&p2, pEnd2, 0, &i2); while( p1 || p2 ){ sqlite3_int64 iDiff = DOCID_CMP(i1, i2); if( p2 && p1 && iDiff==0 ){ fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i1); fts3PoslistMerge(&p, &p1, &p2); fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1); fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2); }else if( !p2 || (p1 && iDiff<0) ){ fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i1); fts3PoslistCopy(&p, &p1); fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1); }else{ fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i2); fts3PoslistCopy(&p, &p2); fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2); } } *paOut = aOut; *pnOut = (p-aOut); return SQLITE_OK; } /* ** This function does a "phrase" merge of two doclists. In a phrase merge, ** the output contains a copy of each position from the right-hand input ** doclist for which there is a position in the left-hand input doclist ** exactly nDist tokens before it. ** ** If the docids in the input doclists are sorted in ascending order, ** parameter bDescDoclist should be false. If they are sorted in ascending ** order, it should be passed a non-zero value. ** ** The right-hand input doclist is overwritten by this function. */ static void fts3DoclistPhraseMerge( int bDescDoclist, /* True if arguments are desc */ int nDist, /* Distance from left to right (1=adjacent) */ char *aLeft, int nLeft, /* Left doclist */ char *aRight, int *pnRight /* IN/OUT: Right/output doclist */ ){ sqlite3_int64 i1 = 0; sqlite3_int64 i2 = 0; sqlite3_int64 iPrev = 0; |
︙ | ︙ | |||
2032 2033 2034 2035 2036 2037 2038 | assert( nDist>0 ); p = aOut; fts3GetDeltaVarint3(&p1, pEnd1, 0, &i1); fts3GetDeltaVarint3(&p2, pEnd2, 0, &i2); while( p1 && p2 ){ | | | | | | | | | 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 | assert( nDist>0 ); p = aOut; fts3GetDeltaVarint3(&p1, pEnd1, 0, &i1); fts3GetDeltaVarint3(&p2, pEnd2, 0, &i2); while( p1 && p2 ){ sqlite3_int64 iDiff = DOCID_CMP(i1, i2); if( iDiff==0 ){ char *pSave = p; sqlite3_int64 iPrevSave = iPrev; int bFirstOutSave = bFirstOut; fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i1); if( 0==fts3PoslistPhraseMerge(&p, nDist, 0, 1, &p1, &p2) ){ p = pSave; iPrev = iPrevSave; bFirstOut = bFirstOutSave; } fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1); fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2); }else if( iDiff<0 ){ fts3PoslistCopy(0, &p1); fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1); }else{ fts3PoslistCopy(0, &p2); fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2); } } *pnRight = p - aOut; } /* ** Merge all doclists in the TermSelect.aaOutput[] array into a single ** doclist stored in TermSelect.aaOutput[0]. If successful, delete all ** other doclists (except the aaOutput[0] one) and return SQLITE_OK. ** ** If an OOM error occurs, return SQLITE_NOMEM. In this case it is ** the responsibility of the caller to free any doclists left in the ** TermSelect.aaOutput[] array. */ static int fts3TermSelectFinishMerge(Fts3Table *p, TermSelect *pTS){ char *aOut = 0; int nOut = 0; int i; /* Loop through the doclists in the aaOutput[] array. Merge them all ** into a single doclist. */ |
︙ | ︙ | |||
2109 2110 2111 2112 2113 2114 2115 | pTS->aaOutput[0] = aOut; pTS->anOutput[0] = nOut; return SQLITE_OK; } /* | > > > > | > > | | > > > | | | < < | | < < < < < < | 2205 2206 2207 2208 2209 2210 2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 2235 2236 2237 | pTS->aaOutput[0] = aOut; pTS->anOutput[0] = nOut; return SQLITE_OK; } /* ** Merge the doclist aDoclist/nDoclist into the TermSelect object passed ** as the first argument. The merge is an "OR" merge (see function ** fts3DoclistOrMerge() for details). ** ** This function is called with the doclist for each term that matches ** a queried prefix. It merges all these doclists into one, the doclist ** for the specified prefix. Since there can be a very large number of ** doclists to merge, the merging is done pair-wise using the TermSelect ** object. ** ** This function returns SQLITE_OK if the merge is successful, or an ** SQLite error code (SQLITE_NOMEM) if an error occurs. */ static int fts3TermSelectMerge( Fts3Table *p, /* FTS table handle */ TermSelect *pTS, /* TermSelect object to merge into */ char *aDoclist, /* Pointer to doclist */ int nDoclist /* Size of aDoclist in bytes */ ){ if( pTS->aaOutput[0]==0 ){ /* If this is the first term selected, copy the doclist to the output ** buffer using memcpy(). */ pTS->aaOutput[0] = sqlite3_malloc(nDoclist); pTS->anOutput[0] = nDoclist; if( pTS->aaOutput[0] ){ memcpy(pTS->aaOutput[0], aDoclist, nDoclist); |
︙ | ︙ | |||
2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211 | } pCsr->apSegment = apNew; } pCsr->apSegment[pCsr->nSegment++] = pNew; return SQLITE_OK; } static int fts3SegReaderCursor( Fts3Table *p, /* FTS3 table handle */ int iIndex, /* Index to search (from 0 to p->nIndex-1) */ int iLevel, /* Level of segments to scan */ const char *zTerm, /* Term to query for */ int nTerm, /* Size of zTerm in bytes */ int isPrefix, /* True for a prefix search */ int isScan, /* True to scan from zTerm to EOF */ | > > > > > > > | | < | > | 2294 2295 2296 2297 2298 2299 2300 2301 2302 2303 2304 2305 2306 2307 2308 2309 2310 2311 2312 2313 2314 2315 2316 2317 2318 2319 2320 2321 2322 2323 2324 2325 2326 2327 | } pCsr->apSegment = apNew; } pCsr->apSegment[pCsr->nSegment++] = pNew; return SQLITE_OK; } /* ** Add seg-reader objects to the Fts3MultiSegReader object passed as the ** 8th argument. ** ** This function returns SQLITE_OK if successful, or an SQLite error code ** otherwise. */ static int fts3SegReaderCursor( Fts3Table *p, /* FTS3 table handle */ int iIndex, /* Index to search (from 0 to p->nIndex-1) */ int iLevel, /* Level of segments to scan */ const char *zTerm, /* Term to query for */ int nTerm, /* Size of zTerm in bytes */ int isPrefix, /* True for a prefix search */ int isScan, /* True to scan from zTerm to EOF */ Fts3MultiSegReader *pCsr /* Cursor object to populate */ ){ int rc = SQLITE_OK; /* Error code */ sqlite3_stmt *pStmt = 0; /* Statement to iterate through segments */ int rc2; /* Result of sqlite3_reset() */ /* If iLevel is less than 0 and this is not a scan, include a seg-reader ** for the pending-terms. If this is a scan, then this call must be being ** made by an fts4aux module, not an FTS table. In this case calling ** Fts3SegReaderPending might segfault, as the data structures used by ** fts4aux are not completely populated. So it's easiest to filter these ** calls out here. */ |
︙ | ︙ | |||
2298 2299 2300 2301 2302 2303 2304 2305 | memset(pCsr, 0, sizeof(Fts3MultiSegReader)); return fts3SegReaderCursor( p, iIndex, iLevel, zTerm, nTerm, isPrefix, isScan, pCsr ); } static int fts3SegReaderCursorAddZero( | > > > > > > | | | | | > > > > > > > > > > > > | | | 2402 2403 2404 2405 2406 2407 2408 2409 2410 2411 2412 2413 2414 2415 2416 2417 2418 2419 2420 2421 2422 2423 2424 2425 2426 2427 2428 2429 2430 2431 2432 2433 2434 2435 2436 2437 2438 2439 2440 2441 2442 2443 2444 2445 2446 2447 2448 2449 2450 2451 | memset(pCsr, 0, sizeof(Fts3MultiSegReader)); return fts3SegReaderCursor( p, iIndex, iLevel, zTerm, nTerm, isPrefix, isScan, pCsr ); } /* ** In addition to its current configuration, have the Fts3MultiSegReader ** passed as the 4th argument also scan the doclist for term zTerm/nTerm. ** ** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code. */ static int fts3SegReaderCursorAddZero( Fts3Table *p, /* FTS virtual table handle */ const char *zTerm, /* Term to scan doclist of */ int nTerm, /* Number of bytes in zTerm */ Fts3MultiSegReader *pCsr /* Fts3MultiSegReader to modify */ ){ return fts3SegReaderCursor(p, 0, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 0, 0,pCsr); } /* ** Open an Fts3MultiSegReader to scan the doclist for term zTerm/nTerm. Or, ** if isPrefix is true, to scan the doclist for all terms for which ** zTerm/nTerm is a prefix. If successful, return SQLITE_OK and write ** a pointer to the new Fts3MultiSegReader to *ppSegcsr. Otherwise, return ** an SQLite error code. ** ** It is the responsibility of the caller to free this object by eventually ** passing it to fts3SegReaderCursorFree() ** ** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code. ** Output parameter *ppSegcsr is set to 0 if an error occurs. */ static int fts3TermSegReaderCursor( Fts3Cursor *pCsr, /* Virtual table cursor handle */ const char *zTerm, /* Term to query for */ int nTerm, /* Size of zTerm in bytes */ int isPrefix, /* True for a prefix search */ Fts3MultiSegReader **ppSegcsr /* OUT: Allocated seg-reader cursor */ ){ Fts3MultiSegReader *pSegcsr; /* Object to allocate and return */ int rc = SQLITE_NOMEM; /* Return code */ pSegcsr = sqlite3_malloc(sizeof(Fts3MultiSegReader)); if( pSegcsr ){ int i; int bFound = 0; /* True once an index has been found */ Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; |
︙ | ︙ | |||
2359 2360 2361 2362 2363 2364 2365 2366 2367 2368 2369 2370 2371 2372 | } } *ppSegcsr = pSegcsr; return rc; } static void fts3SegReaderCursorFree(Fts3MultiSegReader *pSegcsr){ sqlite3Fts3SegReaderFinish(pSegcsr); sqlite3_free(pSegcsr); } /* ** This function retreives the doclist for the specified term (or term | > > > | < < < < < < < < | | < | < < | < | | 2481 2482 2483 2484 2485 2486 2487 2488 2489 2490 2491 2492 2493 2494 2495 2496 2497 2498 2499 2500 2501 2502 2503 2504 2505 2506 2507 2508 2509 2510 2511 2512 2513 2514 2515 2516 2517 2518 2519 2520 2521 2522 2523 2524 2525 2526 2527 2528 2529 2530 2531 2532 2533 2534 2535 2536 2537 | } } *ppSegcsr = pSegcsr; return rc; } /* ** Free an Fts3MultiSegReader allocated by fts3TermSegReaderCursor(). */ static void fts3SegReaderCursorFree(Fts3MultiSegReader *pSegcsr){ sqlite3Fts3SegReaderFinish(pSegcsr); sqlite3_free(pSegcsr); } /* ** This function retreives the doclist for the specified term (or term ** prefix) from the database. */ static int fts3TermSelect( Fts3Table *p, /* Virtual table handle */ Fts3PhraseToken *pTok, /* Token to query for */ int iColumn, /* Column to query (or -ve for all columns) */ int *pnOut, /* OUT: Size of buffer at *ppOut */ char **ppOut /* OUT: Malloced result buffer */ ){ int rc; /* Return code */ Fts3MultiSegReader *pSegcsr; /* Seg-reader cursor for this term */ TermSelect tsc; /* Object for pair-wise doclist merging */ Fts3SegFilter filter; /* Segment term filter configuration */ pSegcsr = pTok->pSegcsr; memset(&tsc, 0, sizeof(TermSelect)); filter.flags = FTS3_SEGMENT_IGNORE_EMPTY | FTS3_SEGMENT_REQUIRE_POS | (pTok->isPrefix ? FTS3_SEGMENT_PREFIX : 0) | (iColumn<p->nColumn ? FTS3_SEGMENT_COLUMN_FILTER : 0); filter.iCol = iColumn; filter.zTerm = pTok->z; filter.nTerm = pTok->n; rc = sqlite3Fts3SegReaderStart(p, pSegcsr, &filter); while( SQLITE_OK==rc && SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, pSegcsr)) ){ rc = fts3TermSelectMerge(p, &tsc, pSegcsr->aDoclist, pSegcsr->nDoclist); } if( rc==SQLITE_OK ){ rc = fts3TermSelectFinishMerge(p, &tsc); } if( rc==SQLITE_OK ){ *ppOut = tsc.aaOutput[0]; *pnOut = tsc.anOutput[0]; }else{ int i; for(i=0; i<SizeofArray(tsc.aaOutput); i++){ |
︙ | ︙ | |||
2436 2437 2438 2439 2440 2441 2442 | ** in buffer aList[], size nList bytes. ** ** If the isPoslist argument is true, then it is assumed that the doclist ** contains a position-list following each docid. Otherwise, it is assumed ** that the doclist is simply a list of docids stored as delta encoded ** varints. */ | | < < < < < < < < | | | | < | 2549 2550 2551 2552 2553 2554 2555 2556 2557 2558 2559 2560 2561 2562 2563 2564 2565 2566 2567 2568 2569 2570 2571 | ** in buffer aList[], size nList bytes. ** ** If the isPoslist argument is true, then it is assumed that the doclist ** contains a position-list following each docid. Otherwise, it is assumed ** that the doclist is simply a list of docids stored as delta encoded ** varints. */ static int fts3DoclistCountDocids(char *aList, int nList){ int nDoc = 0; /* Return value */ if( aList ){ char *aEnd = &aList[nList]; /* Pointer to one byte after EOF */ char *p = aList; /* Cursor */ while( p<aEnd ){ nDoc++; while( (*p++)&0x80 ); /* Skip docid varint */ fts3PoslistCopy(0, &p); /* Skip over position list */ } } return nDoc; } /* |
︙ | ︙ | |||
2483 2484 2485 2486 2487 2488 2489 | pCsr->isEof = 1; rc = sqlite3_reset(pCsr->pStmt); }else{ pCsr->iPrevId = sqlite3_column_int64(pCsr->pStmt, 0); rc = SQLITE_OK; } }else{ | | | 2587 2588 2589 2590 2591 2592 2593 2594 2595 2596 2597 2598 2599 2600 2601 | pCsr->isEof = 1; rc = sqlite3_reset(pCsr->pStmt); }else{ pCsr->iPrevId = sqlite3_column_int64(pCsr->pStmt, 0); rc = SQLITE_OK; } }else{ rc = fts3EvalNext((Fts3Cursor *)pCursor); } assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); return rc; } /* ** This is the xFilter interface for the virtual table. See |
︙ | ︙ | |||
2560 2561 2562 2563 2564 2565 2566 | } return rc; } rc = sqlite3Fts3ReadLock(p); if( rc!=SQLITE_OK ) return rc; | | | 2664 2665 2666 2667 2668 2669 2670 2671 2672 2673 2674 2675 2676 2677 2678 | } return rc; } rc = sqlite3Fts3ReadLock(p); if( rc!=SQLITE_OK ) return rc; rc = fts3EvalStart(pCsr); sqlite3Fts3SegmentsClose(p); if( rc!=SQLITE_OK ) return rc; pCsr->pNextId = pCsr->aDoclist; pCsr->iPrevId = 0; } |
︙ | ︙ | |||
2967 2968 2969 2970 2971 2972 2973 2974 2975 2976 2977 2978 2979 2980 2981 2982 2983 2984 2985 2986 2987 2988 2989 2990 2991 2992 2993 2994 2995 2996 | fts3DbExec(&rc, db, "ALTER TABLE %Q.'%q_segdir' RENAME TO '%q_segdir';", p->zDb, p->zName, zName ); return rc; } static int fts3SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){ UNUSED_PARAMETER(iSavepoint); assert( ((Fts3Table *)pVtab)->inTransaction ); assert( ((Fts3Table *)pVtab)->mxSavepoint < iSavepoint ); TESTONLY( ((Fts3Table *)pVtab)->mxSavepoint = iSavepoint ); return fts3SyncMethod(pVtab); } static int fts3ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){ TESTONLY( Fts3Table *p = (Fts3Table*)pVtab ); UNUSED_PARAMETER(iSavepoint); UNUSED_PARAMETER(pVtab); assert( p->inTransaction ); assert( p->mxSavepoint >= iSavepoint ); TESTONLY( p->mxSavepoint = iSavepoint-1 ); return SQLITE_OK; } static int fts3RollbackToMethod(sqlite3_vtab *pVtab, int iSavepoint){ Fts3Table *p = (Fts3Table*)pVtab; UNUSED_PARAMETER(iSavepoint); assert( p->inTransaction ); assert( p->mxSavepoint >= iSavepoint ); TESTONLY( p->mxSavepoint = iSavepoint ); sqlite3Fts3PendingTermsClear(p); | > > > > > > > > > > > > > > > > > | 3071 3072 3073 3074 3075 3076 3077 3078 3079 3080 3081 3082 3083 3084 3085 3086 3087 3088 3089 3090 3091 3092 3093 3094 3095 3096 3097 3098 3099 3100 3101 3102 3103 3104 3105 3106 3107 3108 3109 3110 3111 3112 3113 3114 3115 3116 3117 | fts3DbExec(&rc, db, "ALTER TABLE %Q.'%q_segdir' RENAME TO '%q_segdir';", p->zDb, p->zName, zName ); return rc; } /* ** The xSavepoint() method. ** ** Flush the contents of the pending-terms table to disk. */ static int fts3SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){ UNUSED_PARAMETER(iSavepoint); assert( ((Fts3Table *)pVtab)->inTransaction ); assert( ((Fts3Table *)pVtab)->mxSavepoint < iSavepoint ); TESTONLY( ((Fts3Table *)pVtab)->mxSavepoint = iSavepoint ); return fts3SyncMethod(pVtab); } /* ** The xRelease() method. ** ** This is a no-op. */ static int fts3ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){ TESTONLY( Fts3Table *p = (Fts3Table*)pVtab ); UNUSED_PARAMETER(iSavepoint); UNUSED_PARAMETER(pVtab); assert( p->inTransaction ); assert( p->mxSavepoint >= iSavepoint ); TESTONLY( p->mxSavepoint = iSavepoint-1 ); return SQLITE_OK; } /* ** The xRollbackTo() method. ** ** Discard the contents of the pending terms table. */ static int fts3RollbackToMethod(sqlite3_vtab *pVtab, int iSavepoint){ Fts3Table *p = (Fts3Table*)pVtab; UNUSED_PARAMETER(iSavepoint); assert( p->inTransaction ); assert( p->mxSavepoint >= iSavepoint ); TESTONLY( p->mxSavepoint = iSavepoint ); sqlite3Fts3PendingTermsClear(p); |
︙ | ︙ | |||
3131 3132 3133 3134 3135 3136 3137 | assert( rc!=SQLITE_OK ); if( pHash ){ sqlite3Fts3HashClear(pHash); sqlite3_free(pHash); } return rc; } | < < < < < < < < < < < < | | | | > > > > > > > > | | | | | | 3252 3253 3254 3255 3256 3257 3258 3259 3260 3261 3262 3263 3264 3265 3266 3267 3268 3269 3270 3271 3272 3273 3274 3275 3276 3277 3278 3279 3280 3281 3282 3283 3284 3285 3286 3287 3288 3289 3290 3291 3292 3293 3294 3295 3296 3297 3298 3299 3300 3301 3302 3303 3304 3305 3306 3307 3308 3309 3310 3311 3312 3313 3314 3315 3316 3317 3318 3319 3320 3321 3322 3323 3324 3325 3326 3327 | assert( rc!=SQLITE_OK ); if( pHash ){ sqlite3Fts3HashClear(pHash); sqlite3_free(pHash); } return rc; } /* ** Allocate an Fts3MultiSegReader for each token in the expression headed ** by pExpr. ** ** An Fts3SegReader object is a cursor that can seek or scan a range of ** entries within a single segment b-tree. An Fts3MultiSegReader uses multiple ** Fts3SegReader objects internally to provide an interface to seek or scan ** within the union of all segments of a b-tree. Hence the name. ** ** If the allocated Fts3MultiSegReader just seeks to a single entry in a ** segment b-tree (if the term is not a prefix or it is a prefix for which ** there exists prefix b-tree of the right length) then it may be traversed ** and merged incrementally. Otherwise, it has to be merged into an in-memory ** doclist and then traversed. */ static void fts3EvalAllocateReaders( Fts3Cursor *pCsr, /* FTS cursor handle */ Fts3Expr *pExpr, /* Allocate readers for this expression */ int *pnToken, /* OUT: Total number of tokens in phrase. */ int *pnOr, /* OUT: Total number of OR nodes in expr. */ int *pRc /* IN/OUT: Error code */ ){ if( pExpr && SQLITE_OK==*pRc ){ if( pExpr->eType==FTSQUERY_PHRASE ){ int i; int nToken = pExpr->pPhrase->nToken; *pnToken += nToken; for(i=0; i<nToken; i++){ Fts3PhraseToken *pToken = &pExpr->pPhrase->aToken[i]; int rc = fts3TermSegReaderCursor(pCsr, pToken->z, pToken->n, pToken->isPrefix, &pToken->pSegcsr ); if( rc!=SQLITE_OK ){ *pRc = rc; return; } } assert( pExpr->pPhrase->iDoclistToken==0 ); pExpr->pPhrase->iDoclistToken = -1; }else{ *pnOr += (pExpr->eType==FTSQUERY_OR); fts3EvalAllocateReaders(pCsr, pExpr->pLeft, pnToken, pnOr, pRc); fts3EvalAllocateReaders(pCsr, pExpr->pRight, pnToken, pnOr, pRc); } } } /* ** Arguments pList/nList contain the doclist for token iToken of phrase p. ** It is merged into the main doclist stored in p->doclist.aAll/nAll. ** ** This function assumes that pList points to a buffer allocated using ** sqlite3_malloc(). This function takes responsibility for eventually ** freeing the buffer. */ static void fts3EvalPhraseMergeToken( Fts3Table *pTab, /* FTS Table pointer */ Fts3Phrase *p, /* Phrase to merge pList/nList into */ int iToken, /* Token pList/nList corresponds to */ char *pList, /* Pointer to doclist */ int nList /* Number of bytes in pList */ ){ assert( iToken!=p->iDoclistToken ); if( pList==0 ){ sqlite3_free(p->doclist.aAll); p->doclist.aAll = 0; p->doclist.nAll = 0; |
︙ | ︙ | |||
3245 3246 3247 3248 3249 3250 3251 3252 | p->doclist.aAll = pRight; p->doclist.nAll = nRight; } if( iToken>p->iDoclistToken ) p->iDoclistToken = iToken; } static int fts3EvalPhraseLoad( | > > > > > > | | | > > > > > > > > > > | | < < | | | | 3362 3363 3364 3365 3366 3367 3368 3369 3370 3371 3372 3373 3374 3375 3376 3377 3378 3379 3380 3381 3382 3383 3384 3385 3386 3387 3388 3389 3390 3391 3392 3393 3394 3395 3396 3397 3398 3399 3400 3401 3402 3403 3404 3405 3406 3407 3408 3409 3410 3411 3412 3413 3414 3415 3416 3417 3418 3419 3420 3421 3422 3423 | p->doclist.aAll = pRight; p->doclist.nAll = nRight; } if( iToken>p->iDoclistToken ) p->iDoclistToken = iToken; } /* ** Load the doclist for phrase p into p->doclist.aAll/nAll. The loaded doclist ** does not take deferred tokens into account. ** ** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code. */ static int fts3EvalPhraseLoad( Fts3Cursor *pCsr, /* FTS Cursor handle */ Fts3Phrase *p /* Phrase object */ ){ Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; int iToken; int rc = SQLITE_OK; for(iToken=0; rc==SQLITE_OK && iToken<p->nToken; iToken++){ Fts3PhraseToken *pToken = &p->aToken[iToken]; assert( pToken->pDeferred==0 || pToken->pSegcsr==0 ); if( pToken->pSegcsr ){ int nThis = 0; char *pThis = 0; rc = fts3TermSelect(pTab, pToken, p->iColumn, &nThis, &pThis); if( rc==SQLITE_OK ){ fts3EvalPhraseMergeToken(pTab, p, iToken, pThis, nThis); } } assert( pToken->pSegcsr==0 ); } return rc; } /* ** This function is called on each phrase after the position lists for ** any deferred tokens have been loaded into memory. It updates the phrases ** current position list to include only those positions that are really ** instances of the phrase (after considering deferred tokens). If this ** means that the phrase does not appear in the current row, doclist.pList ** and doclist.nList are both zeroed. ** ** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code. */ static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){ int iToken; /* Used to iterate through phrase tokens */ int rc = SQLITE_OK; /* Return code */ char *aPoslist = 0; /* Position list for deferred tokens */ int nPoslist = 0; /* Number of bytes in aPoslist */ int iPrev = -1; /* Token number of previous deferred token */ assert( pPhrase->doclist.bFreeList==0 ); for(iToken=0; rc==SQLITE_OK && iToken<pPhrase->nToken; iToken++){ Fts3PhraseToken *pToken = &pPhrase->aToken[iToken]; Fts3DeferredToken *pDeferred = pToken->pDeferred; |
︙ | ︙ | |||
3324 3325 3326 3327 3328 3329 3330 3331 3332 3333 3334 3335 3336 3337 | } } iPrev = iToken; } } if( iPrev>=0 ){ if( nMaxUndeferred<0 ){ pPhrase->doclist.pList = aPoslist; pPhrase->doclist.nList = nPoslist; pPhrase->doclist.iDocid = pCsr->iPrevId; pPhrase->doclist.bFreeList = 1; }else{ int nDistance; | > | 3455 3456 3457 3458 3459 3460 3461 3462 3463 3464 3465 3466 3467 3468 3469 | } } iPrev = iToken; } } if( iPrev>=0 ){ int nMaxUndeferred = pPhrase->iDoclistToken; if( nMaxUndeferred<0 ){ pPhrase->doclist.pList = aPoslist; pPhrase->doclist.nList = nPoslist; pPhrase->doclist.iDocid = pCsr->iPrevId; pPhrase->doclist.bFreeList = 1; }else{ int nDistance; |
︙ | ︙ | |||
3372 3373 3374 3375 3376 3377 3378 3379 3380 | } /* ** This function is called for each Fts3Phrase in a full-text query ** expression to initialize the mechanism for returning rows. Once this ** function has been called successfully on an Fts3Phrase, it may be ** used with fts3EvalPhraseNext() to iterate through the matching docids. */ static int fts3EvalPhraseStart(Fts3Cursor *pCsr, int bOptOk, Fts3Phrase *p){ | > > > > > > | | 3504 3505 3506 3507 3508 3509 3510 3511 3512 3513 3514 3515 3516 3517 3518 3519 3520 3521 3522 3523 3524 3525 3526 | } /* ** This function is called for each Fts3Phrase in a full-text query ** expression to initialize the mechanism for returning rows. Once this ** function has been called successfully on an Fts3Phrase, it may be ** used with fts3EvalPhraseNext() to iterate through the matching docids. ** ** If parameter bOptOk is true, then the phrase may (or may not) use the ** incremental loading strategy. Otherwise, the entire doclist is loaded into ** memory within this call. ** ** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code. */ static int fts3EvalPhraseStart(Fts3Cursor *pCsr, int bOptOk, Fts3Phrase *p){ int rc; /* Error code */ Fts3PhraseToken *pFirst = &p->aToken[0]; Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; if( pCsr->bDesc==pTab->bDescIdx && bOptOk==1 && p->nToken==1 && pFirst->pSegcsr |
︙ | ︙ | |||
3402 3403 3404 3405 3406 3407 3408 | assert( rc!=SQLITE_OK || p->nToken<1 || p->aToken[0].pSegcsr==0 || p->bIncr ); return rc; } /* ** This function is used to iterate backwards (from the end to start) | | > > > > > > | 3540 3541 3542 3543 3544 3545 3546 3547 3548 3549 3550 3551 3552 3553 3554 3555 3556 3557 3558 3559 3560 | assert( rc!=SQLITE_OK || p->nToken<1 || p->aToken[0].pSegcsr==0 || p->bIncr ); return rc; } /* ** This function is used to iterate backwards (from the end to start) ** through doclists. It is used by this module to iterate through phrase ** doclists in reverse and by the fts3_write.c module to iterate through ** pending-terms lists when writing to databases with "order=desc". ** ** The doclist may be sorted in ascending (parameter bDescIdx==0) or ** descending (parameter bDescIdx==1) order of docid. Regardless, this ** function iterates from the end of the doclist to the beginning. */ void sqlite3Fts3DoclistPrev( int bDescIdx, /* True if the doclist is desc */ char *aDoclist, /* Pointer to entire doclist */ int nDoclist, /* Length of aDoclist in bytes */ char **ppIter, /* IN/OUT: Iterator pointer */ sqlite3_int64 *piDocid, /* IN/OUT: Docid pointer */ |
︙ | ︙ | |||
3467 3468 3469 3470 3471 3472 3473 | ** SQLITE_OK. ** ** If there is no "next" entry and no error occurs, then *pbEof is set to ** 1 before returning. Otherwise, if no error occurs and the iterator is ** successfully advanced, *pbEof is set to 0. */ static int fts3EvalPhraseNext( | | | | | 3611 3612 3613 3614 3615 3616 3617 3618 3619 3620 3621 3622 3623 3624 3625 3626 3627 | ** SQLITE_OK. ** ** If there is no "next" entry and no error occurs, then *pbEof is set to ** 1 before returning. Otherwise, if no error occurs and the iterator is ** successfully advanced, *pbEof is set to 0. */ static int fts3EvalPhraseNext( Fts3Cursor *pCsr, /* FTS Cursor handle */ Fts3Phrase *p, /* Phrase object to advance to next docid */ u8 *pbEof /* OUT: Set to 1 if EOF */ ){ int rc = SQLITE_OK; Fts3Doclist *pDL = &p->doclist; Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; if( p->bIncr ){ assert( p->nToken==1 ); |
︙ | ︙ | |||
3515 3516 3517 3518 3519 3520 3521 | } pDL->pList = pIter; fts3PoslistCopy(0, &pIter); pDL->nList = (pIter - pDL->pList); /* pIter now points just past the 0x00 that terminates the position- ** list for document pDL->iDocid. However, if this position-list was | | | > > > > > > > > > > > > > > > > | | | | > > > > > > > > > > > > | | > > > > > > > | | | | | | | 3659 3660 3661 3662 3663 3664 3665 3666 3667 3668 3669 3670 3671 3672 3673 3674 3675 3676 3677 3678 3679 3680 3681 3682 3683 3684 3685 3686 3687 3688 3689 3690 3691 3692 3693 3694 3695 3696 3697 3698 3699 3700 3701 3702 3703 3704 3705 3706 3707 3708 3709 3710 3711 3712 3713 3714 3715 3716 3717 3718 3719 3720 3721 3722 3723 3724 3725 3726 3727 3728 3729 3730 3731 3732 3733 3734 3735 3736 3737 3738 3739 3740 3741 3742 3743 3744 3745 3746 3747 3748 3749 3750 3751 3752 3753 3754 3755 3756 3757 3758 3759 3760 3761 3762 | } pDL->pList = pIter; fts3PoslistCopy(0, &pIter); pDL->nList = (pIter - pDL->pList); /* pIter now points just past the 0x00 that terminates the position- ** list for document pDL->iDocid. However, if this position-list was ** edited in place by fts3EvalNearTrim(), then pIter may not actually ** point to the start of the next docid value. The following line deals ** with this case by advancing pIter past the zero-padding added by ** fts3EvalNearTrim(). */ while( pIter<pEnd && *pIter==0 ) pIter++; pDL->pNextDocid = pIter; assert( pIter>=&pDL->aAll[pDL->nAll] || *pIter ); *pbEof = 0; } } return rc; } /* ** ** If *pRc is not SQLITE_OK when this function is called, it is a no-op. ** Otherwise, fts3EvalPhraseStart() is called on all phrases within the ** expression. Also the Fts3Expr.bDeferred variable is set to true for any ** expressions for which all descendent tokens are deferred. ** ** If parameter bOptOk is zero, then it is guaranteed that the ** Fts3Phrase.doclist.aAll/nAll variables contain the entire doclist for ** each phrase in the expression (subject to deferred token processing). ** Or, if bOptOk is non-zero, then one or more tokens within the expression ** may be loaded incrementally, meaning doclist.aAll/nAll is not available. ** ** If an error occurs within this function, *pRc is set to an SQLite error ** code before returning. */ static void fts3EvalStartReaders( Fts3Cursor *pCsr, /* FTS Cursor handle */ Fts3Expr *pExpr, /* Expression to initialize phrases in */ int bOptOk, /* True to enable incremental loading */ int *pRc /* IN/OUT: Error code */ ){ if( pExpr && SQLITE_OK==*pRc ){ if( pExpr->eType==FTSQUERY_PHRASE ){ int i; int nToken = pExpr->pPhrase->nToken; for(i=0; i<nToken; i++){ if( pExpr->pPhrase->aToken[i].pDeferred==0 ) break; } pExpr->bDeferred = (i==nToken); *pRc = fts3EvalPhraseStart(pCsr, bOptOk, pExpr->pPhrase); }else{ fts3EvalStartReaders(pCsr, pExpr->pLeft, bOptOk, pRc); fts3EvalStartReaders(pCsr, pExpr->pRight, bOptOk, pRc); pExpr->bDeferred = (pExpr->pLeft->bDeferred && pExpr->pRight->bDeferred); } } } /* ** An array of the following structures is assembled as part of the process ** of selecting tokens to defer before the query starts executing (as part ** of the xFilter() method). There is one element in the array for each ** token in the FTS expression. ** ** Tokens are divided into AND/NEAR clusters. All tokens in a cluster belong ** to phrases that are connected only by AND and NEAR operators (not OR or ** NOT). When determining tokens to defer, each AND/NEAR cluster is considered ** separately. The root of a tokens AND/NEAR cluster is stored in ** Fts3TokenAndCost.pRoot. */ typedef struct Fts3TokenAndCost Fts3TokenAndCost; struct Fts3TokenAndCost { Fts3Phrase *pPhrase; /* The phrase the token belongs to */ int iToken; /* Position of token in phrase */ Fts3PhraseToken *pToken; /* The token itself */ Fts3Expr *pRoot; /* Root of NEAR/AND cluster */ int nOvfl; /* Number of overflow pages to load doclist */ int iCol; /* The column the token must match */ }; /* ** This function is used to populate an allocated Fts3TokenAndCost array. ** ** If *pRc is not SQLITE_OK when this function is called, it is a no-op. ** Otherwise, if an error occurs during execution, *pRc is set to an ** SQLite error code. */ static void fts3EvalTokenCosts( Fts3Cursor *pCsr, /* FTS Cursor handle */ Fts3Expr *pRoot, /* Root of current AND/NEAR cluster */ Fts3Expr *pExpr, /* Expression to consider */ Fts3TokenAndCost **ppTC, /* Write new entries to *(*ppTC)++ */ Fts3Expr ***ppOr, /* Write new OR root to *(*ppOr)++ */ int *pRc /* IN/OUT: Error code */ ){ if( *pRc==SQLITE_OK && pExpr ){ if( pExpr->eType==FTSQUERY_PHRASE ){ Fts3Phrase *pPhrase = pExpr->pPhrase; int i; for(i=0; *pRc==SQLITE_OK && i<pPhrase->nToken; i++){ Fts3TokenAndCost *pTC = (*ppTC)++; |
︙ | ︙ | |||
3601 3602 3603 3604 3605 3606 3607 3608 3609 3610 | (*ppOr)++; } fts3EvalTokenCosts(pCsr, pRoot, pExpr->pRight, ppTC, ppOr, pRc); } } } static int fts3EvalAverageDocsize(Fts3Cursor *pCsr, int *pnPage){ if( pCsr->nRowAvg==0 ){ /* The average document size, which is required to calculate the cost | > > > > > > > > > > > | | | | | | | | | | | 3780 3781 3782 3783 3784 3785 3786 3787 3788 3789 3790 3791 3792 3793 3794 3795 3796 3797 3798 3799 3800 3801 3802 3803 3804 3805 3806 3807 3808 3809 3810 3811 3812 3813 3814 3815 3816 3817 | (*ppOr)++; } fts3EvalTokenCosts(pCsr, pRoot, pExpr->pRight, ppTC, ppOr, pRc); } } } /* ** Determine the average document (row) size in pages. If successful, ** write this value to *pnPage and return SQLITE_OK. Otherwise, return ** an SQLite error code. ** ** The average document size in pages is calculated by first calculating ** determining the average size in bytes, B. If B is less than the amount ** of data that will fit on a single leaf page of an intkey table in ** this database, then the average docsize is 1. Otherwise, it is 1 plus ** the number of overflow pages consumed by a record B bytes in size. */ static int fts3EvalAverageDocsize(Fts3Cursor *pCsr, int *pnPage){ if( pCsr->nRowAvg==0 ){ /* The average document size, which is required to calculate the cost ** of each doclist, has not yet been determined. Read the required ** data from the %_stat table to calculate it. ** ** Entry 0 of the %_stat table is a blob containing (nCol+1) FTS3 ** varints, where nCol is the number of columns in the FTS3 table. ** The first varint is the number of documents currently stored in ** the table. The following nCol varints contain the total amount of ** data stored in all rows of each column of the table, from left ** to right. */ int rc; Fts3Table *p = (Fts3Table*)pCsr->base.pVtab; sqlite3_stmt *pStmt; sqlite3_int64 nDoc = 0; sqlite3_int64 nByte = 0; const char *pEnd; const char *a; |
︙ | ︙ | |||
3648 3649 3650 3651 3652 3653 3654 3655 | if( rc!=SQLITE_OK ) return rc; } *pnPage = pCsr->nRowAvg; return SQLITE_OK; } static int fts3EvalSelectDeferred( | > > > > > > > > > > > > > > | | | | > | < | < | > > | | > > > | | > > > > > > > > > > > > > > > > > > > > > > > > > | | | > | | | | < | > > > > | < > > > | > > > > > | < | > > > > < < < < < < < < > > > > > > > > > > > > | | < < < < < < < < < < < < < < < < < | < < | | | > > > | > > > > > > > > > > > > > > > > > > > > > > | | | 3838 3839 3840 3841 3842 3843 3844 3845 3846 3847 3848 3849 3850 3851 3852 3853 3854 3855 3856 3857 3858 3859 3860 3861 3862 3863 3864 3865 3866 3867 3868 3869 3870 3871 3872 3873 3874 3875 3876 3877 3878 3879 3880 3881 3882 3883 3884 3885 3886 3887 3888 3889 3890 3891 3892 3893 3894 3895 3896 3897 3898 3899 3900 3901 3902 3903 3904 3905 3906 3907 3908 3909 3910 3911 3912 3913 3914 3915 3916 3917 3918 3919 3920 3921 3922 3923 3924 3925 3926 3927 3928 3929 3930 3931 3932 3933 3934 3935 3936 3937 3938 3939 3940 3941 3942 3943 3944 3945 3946 3947 3948 3949 3950 3951 3952 3953 3954 3955 3956 3957 3958 3959 3960 3961 3962 3963 3964 3965 3966 3967 3968 3969 3970 3971 3972 3973 3974 3975 3976 3977 3978 3979 3980 3981 3982 3983 3984 3985 3986 3987 3988 3989 3990 3991 3992 3993 3994 3995 3996 3997 3998 3999 4000 4001 4002 4003 4004 4005 4006 4007 4008 4009 4010 4011 4012 4013 4014 4015 4016 4017 4018 4019 4020 4021 4022 4023 4024 4025 4026 4027 4028 4029 4030 4031 4032 4033 4034 4035 4036 4037 4038 4039 4040 4041 4042 4043 4044 4045 4046 4047 4048 4049 4050 4051 4052 4053 4054 4055 4056 4057 4058 4059 4060 4061 4062 | if( rc!=SQLITE_OK ) return rc; } *pnPage = pCsr->nRowAvg; return SQLITE_OK; } /* ** This function is called to select the tokens (if any) that will be ** deferred. The array aTC[] has already been populated when this is ** called. ** ** This function is called once for each AND/NEAR cluster in the ** expression. Each invocation determines which tokens to defer within ** the cluster with root node pRoot. See comments above the definition ** of struct Fts3TokenAndCost for more details. ** ** If no error occurs, SQLITE_OK is returned and sqlite3Fts3DeferToken() ** called on each token to defer. Otherwise, an SQLite error code is ** returned. */ static int fts3EvalSelectDeferred( Fts3Cursor *pCsr, /* FTS Cursor handle */ Fts3Expr *pRoot, /* Consider tokens with this root node */ Fts3TokenAndCost *aTC, /* Array of expression tokens and costs */ int nTC /* Number of entries in aTC[] */ ){ Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; int nDocSize = 0; /* Number of pages per doc loaded */ int rc = SQLITE_OK; /* Return code */ int ii; /* Iterator variable for various purposes */ int nOvfl = 0; /* Total overflow pages used by doclists */ int nToken = 0; /* Total number of tokens in cluster */ int nMinEst = 0; /* The minimum count for any phrase so far. */ int nLoad4 = 1; /* (Phrases that will be loaded)^4. */ /* Count the tokens in this AND/NEAR cluster. If none of the doclists ** associated with the tokens spill onto overflow pages, or if there is ** only 1 token, exit early. No tokens to defer in this case. */ for(ii=0; ii<nTC; ii++){ if( aTC[ii].pRoot==pRoot ){ nOvfl += aTC[ii].nOvfl; nToken++; } } if( nOvfl==0 || nToken<2 ) return SQLITE_OK; /* Obtain the average docsize (in pages). */ rc = fts3EvalAverageDocsize(pCsr, &nDocSize); assert( rc!=SQLITE_OK || nDocSize>0 ); /* Iterate through all tokens in this AND/NEAR cluster, in ascending order ** of the number of overflow pages that will be loaded by the pager layer ** to retrieve the entire doclist for the token from the full-text index. ** Load the doclists for tokens that are either: ** ** a. The cheapest token in the entire query (i.e. the one visited by the ** first iteration of this loop), or ** ** b. Part of a multi-token phrase. ** ** After each token doclist is loaded, merge it with the others from the ** same phrase and count the number of documents that the merged doclist ** contains. Set variable "nMinEst" to the smallest number of documents in ** any phrase doclist for which 1 or more token doclists have been loaded. ** Let nOther be the number of other phrases for which it is certain that ** one or more tokens will not be deferred. ** ** Then, for each token, defer it if loading the doclist would result in ** loading N or more overflow pages into memory, where N is computed as: ** ** (nMinEst + 4^nOther - 1) / (4^nOther) */ for(ii=0; ii<nToken && rc==SQLITE_OK; ii++){ int iTC; /* Used to iterate through aTC[] array. */ Fts3TokenAndCost *pTC = 0; /* Set to cheapest remaining token. */ /* Set pTC to point to the cheapest remaining token. */ for(iTC=0; iTC<nTC; iTC++){ if( aTC[iTC].pToken && aTC[iTC].pRoot==pRoot && (!pTC || aTC[iTC].nOvfl<pTC->nOvfl) ){ pTC = &aTC[iTC]; } } assert( pTC ); if( ii && pTC->nOvfl>=((nMinEst+(nLoad4/4)-1)/(nLoad4/4))*nDocSize ){ /* The number of overflow pages to load for this (and therefore all ** subsequent) tokens is greater than the estimated number of pages ** that will be loaded if all subsequent tokens are deferred. */ Fts3PhraseToken *pToken = pTC->pToken; rc = sqlite3Fts3DeferToken(pCsr, pToken, pTC->iCol); fts3SegReaderCursorFree(pToken->pSegcsr); pToken->pSegcsr = 0; }else{ nLoad4 = nLoad4*4; if( ii==0 || pTC->pPhrase->nToken>1 ){ /* Either this is the cheapest token in the entire query, or it is ** part of a multi-token phrase. Either way, the entire doclist will ** (eventually) be loaded into memory. It may as well be now. */ Fts3PhraseToken *pToken = pTC->pToken; int nList = 0; char *pList = 0; rc = fts3TermSelect(pTab, pToken, pTC->iCol, &nList, &pList); assert( rc==SQLITE_OK || pList==0 ); if( rc==SQLITE_OK ){ int nCount; fts3EvalPhraseMergeToken(pTab, pTC->pPhrase, pTC->iToken,pList,nList); nCount = fts3DoclistCountDocids( pTC->pPhrase->doclist.aAll, pTC->pPhrase->doclist.nAll ); if( ii==0 || nCount<nMinEst ) nMinEst = nCount; } } } pTC->pToken = 0; } return rc; } /* ** This function is called from within the xFilter method. It initializes ** the full-text query currently stored in pCsr->pExpr. To iterate through ** the results of a query, the caller does: ** ** fts3EvalStart(pCsr); ** while( 1 ){ ** fts3EvalNext(pCsr); ** if( pCsr->bEof ) break; ** ... return row pCsr->iPrevId to the caller ... ** } */ static int fts3EvalStart(Fts3Cursor *pCsr){ Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; int rc = SQLITE_OK; int nToken = 0; int nOr = 0; /* Allocate a MultiSegReader for each token in the expression. */ fts3EvalAllocateReaders(pCsr, pCsr->pExpr, &nToken, &nOr, &rc); /* Determine which, if any, tokens in the expression should be deferred. */ if( rc==SQLITE_OK && nToken>1 && pTab->bHasStat ){ Fts3TokenAndCost *aTC; Fts3Expr **apOr; aTC = (Fts3TokenAndCost *)sqlite3_malloc( sizeof(Fts3TokenAndCost) * nToken + sizeof(Fts3Expr *) * nOr * 2 ); apOr = (Fts3Expr **)&aTC[nToken]; if( !aTC ){ rc = SQLITE_NOMEM; }else{ int ii; Fts3TokenAndCost *pTC = aTC; Fts3Expr **ppOr = apOr; fts3EvalTokenCosts(pCsr, 0, pCsr->pExpr, &pTC, &ppOr, &rc); nToken = pTC-aTC; nOr = ppOr-apOr; if( rc==SQLITE_OK ){ rc = fts3EvalSelectDeferred(pCsr, 0, aTC, nToken); for(ii=0; rc==SQLITE_OK && ii<nOr; ii++){ rc = fts3EvalSelectDeferred(pCsr, apOr[ii], aTC, nToken); } } sqlite3_free(aTC); } } fts3EvalStartReaders(pCsr, pCsr->pExpr, 1, &rc); return rc; } /* ** Invalidate the current position list for phrase pPhrase. */ static void fts3EvalInvalidatePoslist(Fts3Phrase *pPhrase){ if( pPhrase->doclist.bFreeList ){ sqlite3_free(pPhrase->doclist.pList); } pPhrase->doclist.pList = 0; pPhrase->doclist.nList = 0; pPhrase->doclist.bFreeList = 0; } /* ** This function is called to edit the position list associated with ** the phrase object passed as the fifth argument according to a NEAR ** condition. For example: ** ** abc NEAR/5 "def ghi" ** ** Parameter nNear is passed the NEAR distance of the expression (5 in ** the example above). When this function is called, *paPoslist points to ** the position list, and *pnToken is the number of phrase tokens in, the ** phrase on the other side of the NEAR operator to pPhrase. For example, ** if pPhrase refers to the "def ghi" phrase, then *paPoslist points to ** the position list associated with phrase "abc". ** ** All positions in the pPhrase position list that are not sufficiently ** close to a position in the *paPoslist position list are removed. If this ** leaves 0 positions, zero is returned. Otherwise, non-zero. ** ** Before returning, *paPoslist is set to point to the position lsit ** associated with pPhrase. And *pnToken is set to the number of tokens in ** pPhrase. */ static int fts3EvalNearTrim( int nNear, /* NEAR distance. As in "NEAR/nNear". */ char *aTmp, /* Temporary space to use */ char **paPoslist, /* IN/OUT: Position list */ int *pnToken, /* IN/OUT: Tokens in phrase of *paPoslist */ Fts3Phrase *pPhrase /* The phrase object to trim the doclist of */ ){ int nParam1 = nNear + pPhrase->nToken; int nParam2 = nNear + *pnToken; |
︙ | ︙ | |||
3823 3824 3825 3826 3827 3828 3829 3830 3831 3832 3833 3834 3835 3836 | *paPoslist = pPhrase->doclist.pList; *pnToken = pPhrase->nToken; } return res; } static int fts3EvalNearTest(Fts3Expr *pExpr, int *pRc){ int res = 1; /* The following block runs if pExpr is the root of a NEAR query. ** For example, the query: ** ** "w" NEAR "x" NEAR "y" NEAR "z" | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 4080 4081 4082 4083 4084 4085 4086 4087 4088 4089 4090 4091 4092 4093 4094 4095 4096 4097 4098 4099 4100 4101 4102 4103 4104 4105 4106 4107 4108 4109 4110 4111 4112 4113 4114 4115 4116 4117 4118 4119 4120 4121 4122 4123 4124 4125 4126 4127 4128 4129 4130 4131 4132 4133 4134 4135 4136 4137 4138 4139 4140 4141 4142 4143 4144 4145 4146 4147 4148 4149 4150 4151 4152 4153 4154 4155 4156 4157 4158 4159 4160 4161 4162 4163 4164 4165 4166 4167 4168 4169 4170 4171 4172 4173 4174 4175 4176 4177 4178 4179 4180 4181 4182 4183 4184 4185 4186 4187 4188 4189 4190 4191 4192 4193 4194 4195 4196 4197 4198 4199 4200 4201 4202 4203 4204 4205 4206 4207 4208 4209 4210 4211 4212 4213 4214 4215 4216 4217 4218 4219 4220 4221 4222 4223 4224 4225 4226 4227 4228 4229 4230 4231 4232 4233 4234 4235 4236 4237 4238 4239 4240 4241 4242 4243 4244 4245 4246 4247 4248 4249 4250 4251 4252 4253 4254 4255 4256 4257 4258 4259 | *paPoslist = pPhrase->doclist.pList; *pnToken = pPhrase->nToken; } return res; } /* ** This function is a no-op if *pRc is other than SQLITE_OK when it is called. ** Otherwise, it advances the expression passed as the second argument to ** point to the next matching row in the database. Expressions iterate through ** matching rows in docid order. Ascending order if Fts3Cursor.bDesc is zero, ** or descending if it is non-zero. ** ** If an error occurs, *pRc is set to an SQLite error code. Otherwise, if ** successful, the following variables in pExpr are set: ** ** Fts3Expr.bEof (non-zero if EOF - there is no next row) ** Fts3Expr.iDocid (valid if bEof==0. The docid of the next row) ** ** If the expression is of type FTSQUERY_PHRASE, and the expression is not ** at EOF, then the following variables are populated with the position list ** for the phrase for the visited row: ** ** FTs3Expr.pPhrase->doclist.nList (length of pList in bytes) ** FTs3Expr.pPhrase->doclist.pList (pointer to position list) ** ** It says above that this function advances the expression to the next ** matching row. This is usually true, but there are the following exceptions: ** ** 1. Deferred tokens are not taken into account. If a phrase consists ** entirely of deferred tokens, it is assumed to match every row in ** the db. In this case the position-list is not populated at all. ** ** Or, if a phrase contains one or more deferred tokens and one or ** more non-deferred tokens, then the expression is advanced to the ** next possible match, considering only non-deferred tokens. In other ** words, if the phrase is "A B C", and "B" is deferred, the expression ** is advanced to the next row that contains an instance of "A * C", ** where "*" may match any single token. The position list in this case ** is populated as for "A * C" before returning. ** ** 2. NEAR is treated as AND. If the expression is "x NEAR y", it is ** advanced to point to the next row that matches "x AND y". ** ** See fts3EvalTestDeferredAndNear() for details on testing if a row is ** really a match, taking into account deferred tokens and NEAR operators. */ static void fts3EvalNextRow( Fts3Cursor *pCsr, /* FTS Cursor handle */ Fts3Expr *pExpr, /* Expr. to advance to next matching row */ int *pRc /* IN/OUT: Error code */ ){ if( *pRc==SQLITE_OK ){ int bDescDoclist = pCsr->bDesc; /* Used by DOCID_CMP() macro */ assert( pExpr->bEof==0 ); pExpr->bStart = 1; switch( pExpr->eType ){ case FTSQUERY_NEAR: case FTSQUERY_AND: { Fts3Expr *pLeft = pExpr->pLeft; Fts3Expr *pRight = pExpr->pRight; assert( !pLeft->bDeferred || !pRight->bDeferred ); if( pLeft->bDeferred ){ /* LHS is entirely deferred. So we assume it matches every row. ** Advance the RHS iterator to find the next row visited. */ fts3EvalNextRow(pCsr, pRight, pRc); pExpr->iDocid = pRight->iDocid; pExpr->bEof = pRight->bEof; }else if( pRight->bDeferred ){ /* RHS is entirely deferred. So we assume it matches every row. ** Advance the LHS iterator to find the next row visited. */ fts3EvalNextRow(pCsr, pLeft, pRc); pExpr->iDocid = pLeft->iDocid; pExpr->bEof = pLeft->bEof; }else{ /* Neither the RHS or LHS are deferred. */ fts3EvalNextRow(pCsr, pLeft, pRc); fts3EvalNextRow(pCsr, pRight, pRc); while( !pLeft->bEof && !pRight->bEof && *pRc==SQLITE_OK ){ sqlite3_int64 iDiff = DOCID_CMP(pLeft->iDocid, pRight->iDocid); if( iDiff==0 ) break; if( iDiff<0 ){ fts3EvalNextRow(pCsr, pLeft, pRc); }else{ fts3EvalNextRow(pCsr, pRight, pRc); } } pExpr->iDocid = pLeft->iDocid; pExpr->bEof = (pLeft->bEof || pRight->bEof); } break; } case FTSQUERY_OR: { Fts3Expr *pLeft = pExpr->pLeft; Fts3Expr *pRight = pExpr->pRight; sqlite3_int64 iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid); assert( pLeft->bStart || pLeft->iDocid==pRight->iDocid ); assert( pRight->bStart || pLeft->iDocid==pRight->iDocid ); if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){ fts3EvalNextRow(pCsr, pLeft, pRc); }else if( pLeft->bEof || (pRight->bEof==0 && iCmp>0) ){ fts3EvalNextRow(pCsr, pRight, pRc); }else{ fts3EvalNextRow(pCsr, pLeft, pRc); fts3EvalNextRow(pCsr, pRight, pRc); } pExpr->bEof = (pLeft->bEof && pRight->bEof); iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid); if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){ pExpr->iDocid = pLeft->iDocid; }else{ pExpr->iDocid = pRight->iDocid; } break; } case FTSQUERY_NOT: { Fts3Expr *pLeft = pExpr->pLeft; Fts3Expr *pRight = pExpr->pRight; if( pRight->bStart==0 ){ fts3EvalNextRow(pCsr, pRight, pRc); assert( *pRc!=SQLITE_OK || pRight->bStart ); } fts3EvalNextRow(pCsr, pLeft, pRc); if( pLeft->bEof==0 ){ while( !*pRc && !pRight->bEof && DOCID_CMP(pLeft->iDocid, pRight->iDocid)>0 ){ fts3EvalNextRow(pCsr, pRight, pRc); } } pExpr->iDocid = pLeft->iDocid; pExpr->bEof = pLeft->bEof; break; } default: { Fts3Phrase *pPhrase = pExpr->pPhrase; fts3EvalInvalidatePoslist(pPhrase); *pRc = fts3EvalPhraseNext(pCsr, pPhrase, &pExpr->bEof); pExpr->iDocid = pPhrase->doclist.iDocid; break; } } } } /* ** If *pRc is not SQLITE_OK, or if pExpr is not the root node of a NEAR ** cluster, then this function returns 1 immediately. ** ** Otherwise, it checks if the current row really does match the NEAR ** expression, using the data currently stored in the position lists ** (Fts3Expr->pPhrase.doclist.pList/nList) for each phrase in the expression. ** ** If the current row is a match, the position list associated with each ** phrase in the NEAR expression is edited in place to contain only those ** phrase instances sufficiently close to their peers to satisfy all NEAR ** constraints. In this case it returns 1. If the NEAR expression does not ** match the current row, 0 is returned. The position lists may or may not ** be edited if 0 is returned. */ static int fts3EvalNearTest(Fts3Expr *pExpr, int *pRc){ int res = 1; /* The following block runs if pExpr is the root of a NEAR query. ** For example, the query: ** ** "w" NEAR "x" NEAR "y" NEAR "z" |
︙ | ︙ | |||
3844 3845 3846 3847 3848 3849 3850 | ** | | ** +--NEAR--+ "y" ** | | ** "w" "x" ** ** The right-hand child of a NEAR node is always a phrase. The ** left-hand child may be either a phrase or a NEAR node. There are | | | 4267 4268 4269 4270 4271 4272 4273 4274 4275 4276 4277 4278 4279 4280 4281 | ** | | ** +--NEAR--+ "y" ** | | ** "w" "x" ** ** The right-hand child of a NEAR node is always a phrase. The ** left-hand child may be either a phrase or a NEAR node. There are ** no exceptions to this - it's the way the parser in fts3_expr.c works. */ if( *pRc==SQLITE_OK && pExpr->eType==FTSQUERY_NEAR && pExpr->bEof==0 && (pExpr->pParent==0 || pExpr->pParent->eType!=FTSQUERY_NEAR) ){ Fts3Expr *p; |
︙ | ︙ | |||
3871 3872 3873 3874 3875 3876 3877 | }else{ char *aPoslist = p->pPhrase->doclist.pList; int nToken = p->pPhrase->nToken; for(p=p->pParent;res && p && p->eType==FTSQUERY_NEAR; p=p->pParent){ Fts3Phrase *pPhrase = p->pRight->pPhrase; int nNear = p->nNear; | | | | < < | < < < | < < < < < < < < | < < < < < < < < < < < < < < < < < | < < < < < < < < < | < < < < < | < < < < | < < | < < < < < < < < | < < < < < < < | < < | < < < | < < < < | < < < < < | < < < < < < < < < < < < < < < < < < < < | | | | 4294 4295 4296 4297 4298 4299 4300 4301 4302 4303 4304 4305 4306 4307 4308 4309 4310 4311 4312 4313 4314 4315 4316 4317 4318 4319 4320 4321 4322 4323 4324 4325 4326 4327 4328 4329 4330 4331 4332 4333 4334 4335 4336 4337 4338 4339 4340 4341 4342 4343 4344 4345 4346 4347 4348 4349 4350 4351 | }else{ char *aPoslist = p->pPhrase->doclist.pList; int nToken = p->pPhrase->nToken; for(p=p->pParent;res && p && p->eType==FTSQUERY_NEAR; p=p->pParent){ Fts3Phrase *pPhrase = p->pRight->pPhrase; int nNear = p->nNear; res = fts3EvalNearTrim(nNear, aTmp, &aPoslist, &nToken, pPhrase); } aPoslist = pExpr->pRight->pPhrase->doclist.pList; nToken = pExpr->pRight->pPhrase->nToken; for(p=pExpr->pLeft; p && res; p=p->pLeft){ int nNear = p->pParent->nNear; Fts3Phrase *pPhrase = ( p->eType==FTSQUERY_NEAR ? p->pRight->pPhrase : p->pPhrase ); res = fts3EvalNearTrim(nNear, aTmp, &aPoslist, &nToken, pPhrase); } } sqlite3_free(aTmp); } return res; } /* ** This function is a helper function for fts3EvalTestDeferredAndNear(). ** Assuming no error occurs or has occurred, It returns non-zero if the ** expression passed as the second argument matches the row that pCsr ** currently points to, or zero if it does not. ** ** If *pRc is not SQLITE_OK when this function is called, it is a no-op. ** If an error occurs during execution of this function, *pRc is set to ** the appropriate SQLite error code. In this case the returned value is ** undefined. */ static int fts3EvalTestExpr( Fts3Cursor *pCsr, /* FTS cursor handle */ Fts3Expr *pExpr, /* Expr to test. May or may not be root. */ int *pRc /* IN/OUT: Error code */ ){ int bHit = 1; /* Return value */ if( *pRc==SQLITE_OK ){ switch( pExpr->eType ){ case FTSQUERY_NEAR: case FTSQUERY_AND: bHit = ( fts3EvalTestExpr(pCsr, pExpr->pLeft, pRc) && fts3EvalTestExpr(pCsr, pExpr->pRight, pRc) && fts3EvalNearTest(pExpr, pRc) ); /* If the NEAR expression does not match any rows, zero the doclist for ** all phrases involved in the NEAR. This is because the snippet(), ** offsets() and matchinfo() functions are not supposed to recognize ** any instances of phrases that are part of unmatched NEAR queries. |
︙ | ︙ | |||
4039 4040 4041 4042 4043 4044 4045 | if( bHit==0 && pExpr->eType==FTSQUERY_NEAR && (pExpr->pParent==0 || pExpr->pParent->eType!=FTSQUERY_NEAR) ){ Fts3Expr *p; for(p=pExpr; p->pPhrase==0; p=p->pLeft){ if( p->pRight->iDocid==pCsr->iPrevId ){ | | | | | | | | > > > > > > > > > > > > > | | | > > > > > > > | > > | | | | | | 4363 4364 4365 4366 4367 4368 4369 4370 4371 4372 4373 4374 4375 4376 4377 4378 4379 4380 4381 4382 4383 4384 4385 4386 4387 4388 4389 4390 4391 4392 4393 4394 4395 4396 4397 4398 4399 4400 4401 4402 4403 4404 4405 4406 4407 4408 4409 4410 4411 4412 4413 4414 4415 4416 4417 4418 4419 4420 4421 4422 4423 4424 4425 4426 4427 4428 4429 4430 4431 4432 4433 4434 4435 4436 4437 4438 4439 4440 4441 4442 4443 4444 4445 4446 4447 4448 4449 4450 4451 4452 4453 4454 4455 4456 4457 4458 4459 4460 4461 4462 4463 4464 4465 4466 4467 4468 4469 4470 4471 4472 4473 4474 4475 4476 4477 4478 4479 4480 4481 4482 4483 4484 4485 4486 4487 4488 4489 4490 4491 4492 4493 4494 4495 4496 4497 4498 4499 4500 4501 4502 4503 4504 4505 4506 4507 4508 4509 4510 4511 4512 4513 4514 4515 4516 4517 | if( bHit==0 && pExpr->eType==FTSQUERY_NEAR && (pExpr->pParent==0 || pExpr->pParent->eType!=FTSQUERY_NEAR) ){ Fts3Expr *p; for(p=pExpr; p->pPhrase==0; p=p->pLeft){ if( p->pRight->iDocid==pCsr->iPrevId ){ fts3EvalInvalidatePoslist(p->pRight->pPhrase); } } if( p->iDocid==pCsr->iPrevId ){ fts3EvalInvalidatePoslist(p->pPhrase); } } break; case FTSQUERY_OR: { int bHit1 = fts3EvalTestExpr(pCsr, pExpr->pLeft, pRc); int bHit2 = fts3EvalTestExpr(pCsr, pExpr->pRight, pRc); bHit = bHit1 || bHit2; break; } case FTSQUERY_NOT: bHit = ( fts3EvalTestExpr(pCsr, pExpr->pLeft, pRc) && !fts3EvalTestExpr(pCsr, pExpr->pRight, pRc) ); break; default: { if( pCsr->pDeferred && (pExpr->iDocid==pCsr->iPrevId || pExpr->bDeferred) ){ Fts3Phrase *pPhrase = pExpr->pPhrase; assert( pExpr->bDeferred || pPhrase->doclist.bFreeList==0 ); if( pExpr->bDeferred ){ fts3EvalInvalidatePoslist(pPhrase); } *pRc = fts3EvalDeferredPhrase(pCsr, pPhrase); bHit = (pPhrase->doclist.pList!=0); pExpr->iDocid = pCsr->iPrevId; }else{ bHit = (pExpr->bEof==0 && pExpr->iDocid==pCsr->iPrevId); } break; } } } return bHit; } /* ** This function is called as the second part of each xNext operation when ** iterating through the results of a full-text query. At this point the ** cursor points to a row that matches the query expression, with the ** following caveats: ** ** * Up until this point, "NEAR" operators in the expression have been ** treated as "AND". ** ** * Deferred tokens have not yet been considered. ** ** If *pRc is not SQLITE_OK when this function is called, it immediately ** returns 0. Otherwise, it tests whether or not after considering NEAR ** operators and deferred tokens the current row is still a match for the ** expression. It returns 1 if both of the following are true: ** ** 1. *pRc is SQLITE_OK when this function returns, and ** ** 2. After scanning the current FTS table row for the deferred tokens, ** it is determined that the row does *not* match the query. ** ** Or, if no error occurs and it seems the current row does match the FTS ** query, return 0. */ static int fts3EvalTestDeferredAndNear(Fts3Cursor *pCsr, int *pRc){ int rc = *pRc; int bMiss = 0; if( rc==SQLITE_OK ){ /* If there are one or more deferred tokens, load the current row into ** memory and scan it to determine the position list for each deferred ** token. Then, see if this row is really a match, considering deferred ** tokens and NEAR operators (neither of which were taken into account ** earlier, by fts3EvalNextRow()). */ if( pCsr->pDeferred ){ rc = fts3CursorSeek(0, pCsr); if( rc==SQLITE_OK ){ rc = sqlite3Fts3CacheDeferredDoclists(pCsr); } } bMiss = (0==fts3EvalTestExpr(pCsr, pCsr->pExpr, &rc)); /* Free the position-lists accumulated for each deferred token above. */ sqlite3Fts3FreeDeferredDoclists(pCsr); *pRc = rc; } return (rc==SQLITE_OK && bMiss); } /* ** Advance to the next document that matches the FTS expression in ** Fts3Cursor.pExpr. */ static int fts3EvalNext(Fts3Cursor *pCsr){ int rc = SQLITE_OK; /* Return Code */ Fts3Expr *pExpr = pCsr->pExpr; assert( pCsr->isEof==0 ); if( pExpr==0 ){ pCsr->isEof = 1; }else{ do { if( pCsr->isRequireSeek==0 ){ sqlite3_reset(pCsr->pStmt); } assert( sqlite3_data_count(pCsr->pStmt)==0 ); fts3EvalNextRow(pCsr, pExpr, &rc); pCsr->isEof = pExpr->bEof; pCsr->isRequireSeek = 1; pCsr->isMatchinfoNeeded = 1; pCsr->iPrevId = pExpr->iDocid; }while( pCsr->isEof==0 && fts3EvalTestDeferredAndNear(pCsr, &rc) ); } return rc; } /* ** Restart interation for expression pExpr so that the next call to ** fts3EvalNext() visits the first row. Do not allow incremental ** loading or merging of phrase doclists for this iteration. ** ** If *pRc is other than SQLITE_OK when this function is called, it is ** a no-op. If an error occurs within this function, *pRc is set to an ** SQLite error code before returning. */ static void fts3EvalRestart( Fts3Cursor *pCsr, Fts3Expr *pExpr, int *pRc ){ if( pExpr && *pRc==SQLITE_OK ){ Fts3Phrase *pPhrase = pExpr->pPhrase; if( pPhrase ){ fts3EvalInvalidatePoslist(pPhrase); if( pPhrase->bIncr ){ assert( pPhrase->nToken==1 ); assert( pPhrase->aToken[0].pSegcsr ); sqlite3Fts3MsrIncrRestart(pPhrase->aToken[0].pSegcsr); *pRc = fts3EvalPhraseStart(pCsr, 0, pPhrase); } |
︙ | ︙ | |||
4273 4274 4275 4276 4277 4278 4279 | do { /* Ensure the %_content statement is reset. */ if( pCsr->isRequireSeek==0 ) sqlite3_reset(pCsr->pStmt); assert( sqlite3_data_count(pCsr->pStmt)==0 ); /* Advance to the next document */ | | | | 4619 4620 4621 4622 4623 4624 4625 4626 4627 4628 4629 4630 4631 4632 4633 4634 4635 4636 4637 4638 4639 4640 | do { /* Ensure the %_content statement is reset. */ if( pCsr->isRequireSeek==0 ) sqlite3_reset(pCsr->pStmt); assert( sqlite3_data_count(pCsr->pStmt)==0 ); /* Advance to the next document */ fts3EvalNextRow(pCsr, pRoot, &rc); pCsr->isEof = pRoot->bEof; pCsr->isRequireSeek = 1; pCsr->isMatchinfoNeeded = 1; pCsr->iPrevId = pRoot->iDocid; }while( pCsr->isEof==0 && pRoot->eType==FTSQUERY_NEAR && fts3EvalTestDeferredAndNear(pCsr, &rc) ); if( rc==SQLITE_OK && pCsr->isEof==0 ){ fts3EvalUpdateCounts(pRoot); } } |
︙ | ︙ | |||
4302 4303 4304 4305 4306 4307 4308 | ** order. For this reason, even though it seems more defensive, the ** do loop can not be written: ** ** do {...} while( pRoot->iDocid<iDocid && rc==SQLITE_OK ); */ fts3EvalRestart(pCsr, pRoot, &rc); do { | | | | 4648 4649 4650 4651 4652 4653 4654 4655 4656 4657 4658 4659 4660 4661 4662 4663 4664 4665 | ** order. For this reason, even though it seems more defensive, the ** do loop can not be written: ** ** do {...} while( pRoot->iDocid<iDocid && rc==SQLITE_OK ); */ fts3EvalRestart(pCsr, pRoot, &rc); do { fts3EvalNextRow(pCsr, pRoot, &rc); assert( pRoot->bEof==0 ); }while( pRoot->iDocid!=iDocid && rc==SQLITE_OK ); fts3EvalTestDeferredAndNear(pCsr, &rc); } } return rc; } /* ** This function is used by the matchinfo() module to query a phrase |
︙ | ︙ | |||
4436 4437 4438 4439 4440 4441 4442 | ** * the contents of pPhrase->doclist, and ** * any Fts3MultiSegReader objects held by phrase tokens. */ void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *pPhrase){ if( pPhrase ){ int i; sqlite3_free(pPhrase->doclist.aAll); | | > > > > > > > > > > > > > > | 4782 4783 4784 4785 4786 4787 4788 4789 4790 4791 4792 4793 4794 4795 4796 4797 4798 4799 4800 4801 4802 4803 4804 4805 4806 4807 4808 4809 4810 4811 4812 | ** * the contents of pPhrase->doclist, and ** * any Fts3MultiSegReader objects held by phrase tokens. */ void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *pPhrase){ if( pPhrase ){ int i; sqlite3_free(pPhrase->doclist.aAll); fts3EvalInvalidatePoslist(pPhrase); memset(&pPhrase->doclist, 0, sizeof(Fts3Doclist)); for(i=0; i<pPhrase->nToken; i++){ fts3SegReaderCursorFree(pPhrase->aToken[i].pSegcsr); pPhrase->aToken[i].pSegcsr = 0; } } } #if !SQLITE_CORE /* ** Initialize API pointer table, if required. */ int sqlite3_extension_init( sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi ){ SQLITE_EXTENSION_INIT2(pApi) return sqlite3Fts3Init(db); } #endif #endif |
Changes to ext/fts3/fts3Int.h.
︙ | ︙ | |||
503 504 505 506 507 508 509 | int nTerm, /* Size of zTerm in bytes */ int isPrefix, /* True for a prefix search */ Fts3MultiSegReader **ppSegcsr /* OUT: Allocated seg-reader cursor */ ); void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *); | < < < | 503 504 505 506 507 508 509 510 511 512 513 514 515 516 | int nTerm, /* Size of zTerm in bytes */ int isPrefix, /* True for a prefix search */ Fts3MultiSegReader **ppSegcsr /* OUT: Allocated seg-reader cursor */ ); void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *); int sqlite3Fts3MsrIncrStart( Fts3Table*, Fts3MultiSegReader*, int, const char*, int); int sqlite3Fts3MsrIncrNext( Fts3Table *, Fts3MultiSegReader *, sqlite3_int64 *, char **, int *); char *sqlite3Fts3EvalPhrasePoslist(Fts3Cursor *, Fts3Expr *, int iCol); int sqlite3Fts3MsrOvfl(Fts3Cursor *, Fts3MultiSegReader *, int *); int sqlite3Fts3MsrIncrRestart(Fts3MultiSegReader *pCsr); |
︙ | ︙ |
Changes to test/fts3auto.test.
︙ | ︙ | |||
103 104 105 106 107 108 109 | do_execsql_test $tn$title.4 " SELECT docid, mit(matchinfo($tbl, 'x')) FROM $tbl WHERE $tbl MATCH '$match' ORDER BY docid ASC " $matchinfo_asc } | | | > | > | 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 | do_execsql_test $tn$title.4 " SELECT docid, mit(matchinfo($tbl, 'x')) FROM $tbl WHERE $tbl MATCH '$match' ORDER BY docid ASC " $matchinfo_asc } # fts3_make_deferrable TABLE TOKEN ?NROW? # proc fts3_make_deferrable {tbl token {nRow 0}} { set stmt [sqlite3_prepare db "SELECT * FROM $tbl" -1 dummy] set name [sqlite3_column_name $stmt 0] sqlite3_finalize $stmt if {$nRow==0} { set nRow [db one "SELECT count(*) FROM $tbl"] } set pgsz [db one "PRAGMA page_size"] execsql BEGIN for {set i 0} {$i < ($nRow * $pgsz * 1.2)/100} {incr i} { set doc [string repeat "$token " 100] execsql "INSERT INTO $tbl ($name) VALUES(\$doc)" } execsql "INSERT INTO $tbl ($name) VALUES('aaaaaaa ${token}aaaaa')" |
︙ | ︙ | |||
648 649 650 651 652 653 654 655 656 657 658 | do_fts3query_test 6.$tn.2 t1 {b:G AND c:I} do_fts3query_test 6.$tn.3 t1 {b:G NEAR c:I} do_fts3query_test 6.$tn.4 t1 {a:C OR b:G OR c:K OR d:C} do_fts3query_test 6.$tn.5 t1 {a:G OR b:G} catchsql { COMMIT } } set sqlite_fts3_enable_parentheses $sfep finish_test | > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > > | 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 | do_fts3query_test 6.$tn.2 t1 {b:G AND c:I} do_fts3query_test 6.$tn.3 t1 {b:G NEAR c:I} do_fts3query_test 6.$tn.4 t1 {a:C OR b:G OR c:K OR d:C} do_fts3query_test 6.$tn.5 t1 {a:G OR b:G} catchsql { COMMIT } } foreach {tn create} { 1 "fts4(x)" 2 "fts4(x, order=DESC)" } { execsql [subst { DROP TABLE IF EXISTS t1; CREATE VIRTUAL TABLE t1 USING $create; }] foreach {x} { "F E N O T K X V A X I E X A P G Q V H U" "R V A E T C V Q N I E L O N U G J K L U" "U Y I G W M V F J L X I D C H F P J Q B" "S G D Z X R P G S S Y B K A S G A I L L" "L S I C H T Z S R Q P R N K J X L F M J" "C C C D P X B Z C M A D A C X S B T X V" "W Y J M D R G V R K B X S A W R I T N C" "P K L W T M S P O Y Y V V O E H Q A I R" "C D Y I C Z F H J C O Y A Q F L S B D K" "P G S C Y C Y V I M B D S Z D D Y W I E" "Z K Z U E E S F Y X T U A L W O U J C Q" "P A T Z S W L P L Q V Y Y I P W U X S S" "I U I H U O F Z F R H R F T N D X A G M" "N A B M S H K X S O Y D T X S B R Y H Z" "L U D A S K I L S V Z J P U B E B Y H M" } { execsql { INSERT INTO t1 VALUES($x) } } # Add extra documents to the database such that token "B" will be considered # deferrable if considering the other tokens means that 2 or fewer documents # will be loaded into memory. # fts3_make_deferrable t1 B 2 # B is not deferred in either of the first two tests below, since filtering # on "M" or "D" returns 10 documents or so. But filtering on "M * D" only # returns 2, so B is deferred in this case. # do_fts3query_test 7.$tn.1 t1 {"M B"} do_fts3query_test 7.$tn.2 t1 {"B D"} do_fts3query_test 7.$tn.3 -deferred B t1 {"M B D"} } set sqlite_fts3_enable_parentheses $sfep finish_test |