Index: src/delete.c ================================================================== --- src/delete.c +++ src/delete.c @@ -395,11 +395,11 @@ } /* Construct a query to find the rowid or primary key for every row ** to be deleted, based on the WHERE clause. */ - pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, 0, 0, + pWInfo = sqlite3WhereBegin(pParse, 0, pTabList, pWhere, 0, 0, WHERE_ONEPASS_DESIRED|WHERE_DUPLICATES_OK, iTabCur+1); if( pWInfo==0 ) goto delete_from_cleanup; okOnePass = sqlite3WhereOkOnePass(pWInfo, aiCurOnePass); Index: src/fkey.c ================================================================== --- src/fkey.c +++ src/fkey.c @@ -629,11 +629,11 @@ sqlite3ResolveExprNames(&sNameContext, pWhere); /* Create VDBE to loop through the entries in pSrc that match the WHERE ** clause. For each row found, increment either the deferred or immediate ** foreign key constraint counter. */ - pWInfo = sqlite3WhereBegin(pParse, pSrc, pWhere, 0, 0, 0, 0); + pWInfo = sqlite3WhereBegin(pParse, 0, pSrc, pWhere, 0, 0, 0, 0); sqlite3VdbeAddOp2(v, OP_FkCounter, pFKey->isDeferred, nIncr); if( pWInfo ){ sqlite3WhereEnd(pWInfo); } Index: src/select.c ================================================================== --- src/select.c +++ src/select.c @@ -3786,10 +3786,125 @@ } return nChng; } #endif /* !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) */ +/* +** Generate code for all sub-queries in the FROM clause +*/ +#if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) +void sqlite3ManifestSubqueries( + Parse *pParse, /* Parsing context */ + Select *p, /* SELECT statement being generated */ + SrcList *pTabList /* The FROM clause */ +){ + int i; + sqlite3 *db = pParse->db; + Vdbe *v = pParse->pVdbe; + for(i=0; inSrc; i++){ + struct SrcList_item *pItem = &pTabList->a[i]; + SelectDest dest; + Select *pSub = pItem->pSelect; + if( pSub==0 ) continue; + + /* Sometimes the code for a subquery will be generated more than + ** once, if the subquery is part of the WHERE clause in a LEFT JOIN, + ** for example. In that case, do not regenerate the code to manifest + ** a view or the co-routine to implement a view. The first instance + ** is sufficient, though the subroutine to manifest the view does need + ** to be invoked again. */ + if( pItem->addrFillSub ){ + if( pItem->viaCoroutine==0 ){ + sqlite3VdbeAddOp2(v, OP_Gosub, pItem->regReturn, pItem->addrFillSub); + } + continue; + } + + /* Increment Parse.nHeight by the height of the largest expression + ** tree referred to by this, the parent select. The child select + ** may contain expression trees of at most + ** (SQLITE_MAX_EXPR_DEPTH-Parse.nHeight) height. This is a bit + ** more conservative than necessary, but much easier than enforcing + ** an exact limit. + */ + pParse->nHeight += sqlite3SelectExprHeight(p); + + /* Make copies of constant WHERE-clause terms in the outer query down + ** inside the subquery. This can help the subquery to run more efficiently. + */ + if( (pItem->jointype & JT_OUTER)==0 + && pushDownWhereTerms(db, pSub, p->pWhere, pItem->iCursor) + ){ +#if SELECTTRACE_ENABLED + if( sqlite3SelectTrace & 0x100 ){ + SELECTTRACE(0x100,pParse,p,("After WHERE-clause push-down:\n")); + sqlite3TreeViewSelect(0, p, 0); + } +#endif + } + + /* Generate code to implement the subquery + */ + if( pTabList->nSrc==1 + && (p->selFlags & SF_All)==0 + && OptimizationEnabled(db, SQLITE_SubqCoroutine) + ){ + /* Implement a co-routine that will return a single row of the result + ** set on each invocation. + */ + int addrTop = sqlite3VdbeCurrentAddr(v)+1; + pItem->regReturn = ++pParse->nMem; + sqlite3VdbeAddOp3(v, OP_InitCoroutine, pItem->regReturn, 0, addrTop); + VdbeComment((v, "%s", pItem->pTab->zName)); + pItem->addrFillSub = addrTop; + sqlite3SelectDestInit(&dest, SRT_Coroutine, pItem->regReturn); + explainSetInteger(pItem->iSelectId, (u8)pParse->iNextSelectId); + sqlite3Select(pParse, pSub, &dest); + pItem->pTab->nRowLogEst = sqlite3LogEst(pSub->nSelectRow); + pItem->viaCoroutine = 1; + pItem->regResult = dest.iSdst; + sqlite3VdbeAddOp1(v, OP_EndCoroutine, pItem->regReturn); + sqlite3VdbeJumpHere(v, addrTop-1); + sqlite3ClearTempRegCache(pParse); + }else{ + /* Generate a subroutine that will fill an ephemeral table with + ** the content of this subquery. pItem->addrFillSub will point + ** to the address of the generated subroutine. pItem->regReturn + ** is a register allocated to hold the subroutine return address + */ + int topAddr; + int onceAddr = 0; + int retAddr; + assert( pItem->addrFillSub==0 ); + pItem->regReturn = ++pParse->nMem; + topAddr = sqlite3VdbeAddOp2(v, OP_Integer, 0, pItem->regReturn); + pItem->addrFillSub = topAddr+1; + if( pItem->isCorrelated==0 ){ + /* If the subquery is not correlated and if we are not inside of + ** a trigger, then we only need to compute the value of the subquery + ** once. */ + onceAddr = sqlite3CodeOnce(pParse); VdbeCoverage(v); + VdbeComment((v, "materialize \"%s\"", pItem->pTab->zName)); + }else{ + VdbeNoopComment((v, "materialize \"%s\"", pItem->pTab->zName)); + } + sqlite3SelectDestInit(&dest, SRT_EphemTab, pItem->iCursor); + explainSetInteger(pItem->iSelectId, (u8)pParse->iNextSelectId); + sqlite3Select(pParse, pSub, &dest); + pItem->pTab->nRowLogEst = sqlite3LogEst(pSub->nSelectRow); + if( onceAddr ) sqlite3VdbeJumpHere(v, onceAddr); + retAddr = sqlite3VdbeAddOp1(v, OP_Return, pItem->regReturn); + VdbeComment((v, "end %s", pItem->pTab->zName)); + sqlite3VdbeChangeP1(v, topAddr, retAddr); + sqlite3ClearTempRegCache(pParse); + } + if( db->mallocFailed ) break; + pParse->nHeight -= sqlite3SelectExprHeight(p); + } +} +#endif + /* ** Based on the contents of the AggInfo structure indicated by the first ** argument, this function checks if the following are true: ** ** * the query contains just a single aggregate function, @@ -4882,114 +4997,17 @@ #endif return rc; } #endif - /* Generate code for all sub-queries in the FROM clause - */ -#if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) - for(i=0; inSrc; i++){ - struct SrcList_item *pItem = &pTabList->a[i]; - SelectDest dest; - Select *pSub = pItem->pSelect; - if( pSub==0 ) continue; - - /* Sometimes the code for a subquery will be generated more than - ** once, if the subquery is part of the WHERE clause in a LEFT JOIN, - ** for example. In that case, do not regenerate the code to manifest - ** a view or the co-routine to implement a view. The first instance - ** is sufficient, though the subroutine to manifest the view does need - ** to be invoked again. */ - if( pItem->addrFillSub ){ - if( pItem->viaCoroutine==0 ){ - sqlite3VdbeAddOp2(v, OP_Gosub, pItem->regReturn, pItem->addrFillSub); - } - continue; - } - - /* Increment Parse.nHeight by the height of the largest expression - ** tree referred to by this, the parent select. The child select - ** may contain expression trees of at most - ** (SQLITE_MAX_EXPR_DEPTH-Parse.nHeight) height. This is a bit - ** more conservative than necessary, but much easier than enforcing - ** an exact limit. - */ - pParse->nHeight += sqlite3SelectExprHeight(p); - - /* Make copies of constant WHERE-clause terms in the outer query down - ** inside the subquery. This can help the subquery to run more efficiently. - */ - if( (pItem->jointype & JT_OUTER)==0 - && pushDownWhereTerms(db, pSub, p->pWhere, pItem->iCursor) - ){ -#if SELECTTRACE_ENABLED - if( sqlite3SelectTrace & 0x100 ){ - SELECTTRACE(0x100,pParse,p,("After WHERE-clause push-down:\n")); - sqlite3TreeViewSelect(0, p, 0); - } -#endif - } - - /* Generate code to implement the subquery - */ - if( pTabList->nSrc==1 - && (p->selFlags & SF_All)==0 - && OptimizationEnabled(db, SQLITE_SubqCoroutine) - ){ - /* Implement a co-routine that will return a single row of the result - ** set on each invocation. - */ - int addrTop = sqlite3VdbeCurrentAddr(v)+1; - pItem->regReturn = ++pParse->nMem; - sqlite3VdbeAddOp3(v, OP_InitCoroutine, pItem->regReturn, 0, addrTop); - VdbeComment((v, "%s", pItem->pTab->zName)); - pItem->addrFillSub = addrTop; - sqlite3SelectDestInit(&dest, SRT_Coroutine, pItem->regReturn); - explainSetInteger(pItem->iSelectId, (u8)pParse->iNextSelectId); - sqlite3Select(pParse, pSub, &dest); - pItem->pTab->nRowLogEst = sqlite3LogEst(pSub->nSelectRow); - pItem->viaCoroutine = 1; - pItem->regResult = dest.iSdst; - sqlite3VdbeAddOp1(v, OP_EndCoroutine, pItem->regReturn); - sqlite3VdbeJumpHere(v, addrTop-1); - sqlite3ClearTempRegCache(pParse); - }else{ - /* Generate a subroutine that will fill an ephemeral table with - ** the content of this subquery. pItem->addrFillSub will point - ** to the address of the generated subroutine. pItem->regReturn - ** is a register allocated to hold the subroutine return address - */ - int topAddr; - int onceAddr = 0; - int retAddr; - assert( pItem->addrFillSub==0 ); - pItem->regReturn = ++pParse->nMem; - topAddr = sqlite3VdbeAddOp2(v, OP_Integer, 0, pItem->regReturn); - pItem->addrFillSub = topAddr+1; - if( pItem->isCorrelated==0 ){ - /* If the subquery is not correlated and if we are not inside of - ** a trigger, then we only need to compute the value of the subquery - ** once. */ - onceAddr = sqlite3CodeOnce(pParse); VdbeCoverage(v); - VdbeComment((v, "materialize \"%s\"", pItem->pTab->zName)); - }else{ - VdbeNoopComment((v, "materialize \"%s\"", pItem->pTab->zName)); - } - sqlite3SelectDestInit(&dest, SRT_EphemTab, pItem->iCursor); - explainSetInteger(pItem->iSelectId, (u8)pParse->iNextSelectId); - sqlite3Select(pParse, pSub, &dest); - pItem->pTab->nRowLogEst = sqlite3LogEst(pSub->nSelectRow); - if( onceAddr ) sqlite3VdbeJumpHere(v, onceAddr); - retAddr = sqlite3VdbeAddOp1(v, OP_Return, pItem->regReturn); - VdbeComment((v, "end %s", pItem->pTab->zName)); - sqlite3VdbeChangeP1(v, topAddr, retAddr); - sqlite3ClearTempRegCache(pParse); - } + if( !OptimizationEnabled(db, SQLITE_LateSubquery) ){ + /* Manifest the subqueries. This needs to be done before calling + ** sqlite3WhereBegin() so that the Table.nRowLogEst value can be set + ** correctly for the subqueries. */ + sqlite3ManifestSubqueries(pParse, p, pTabList); if( db->mallocFailed ) goto select_end; - pParse->nHeight -= sqlite3SelectExprHeight(p); } -#endif /* Various elements of the SELECT copied into local variables for ** convenience */ pEList = p->pEList; pWhere = p->pWhere; @@ -5084,11 +5102,11 @@ if( !isAgg && pGroupBy==0 ){ /* No aggregate functions and no GROUP BY clause */ u16 wctrlFlags = (sDistinct.isTnct ? WHERE_WANT_DISTINCT : 0); /* Begin the database scan. */ - pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, sSort.pOrderBy, + pWInfo = sqlite3WhereBegin(pParse, p, pTabList, pWhere, sSort.pOrderBy, p->pEList, wctrlFlags, 0); if( pWInfo==0 ) goto select_end; if( sqlite3WhereOutputRowCount(pWInfo) < p->nSelectRow ){ p->nSelectRow = sqlite3WhereOutputRowCount(pWInfo); } @@ -5240,11 +5258,11 @@ ** This might involve two separate loops with an OP_Sort in between, or ** it might be a single loop that uses an index to extract information ** in the right order to begin with. */ sqlite3VdbeAddOp2(v, OP_Gosub, regReset, addrReset); - pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pGroupBy, 0, + pWInfo = sqlite3WhereBegin(pParse, p, pTabList, pWhere, pGroupBy, 0, WHERE_GROUPBY | (orderByGrp ? WHERE_SORTBYGROUP : 0), 0 ); if( pWInfo==0 ) goto select_end; if( sqlite3WhereIsOrdered(pWInfo)==pGroupBy->nExpr ){ /* The optimizer is able to deliver rows in group by order so @@ -5535,11 +5553,11 @@ /* This case runs if the aggregate has no GROUP BY clause. The ** processing is much simpler since there is only a single row ** of output. */ resetAccumulator(pParse, &sAggInfo); - pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pMinMax,0,flag,0); + pWInfo = sqlite3WhereBegin(pParse,p,pTabList, pWhere, pMinMax,0,flag,0); if( pWInfo==0 ){ sqlite3ExprListDelete(db, pDel); goto select_end; } updateAccumulator(pParse, &sAggInfo); Index: src/sqliteInt.h ================================================================== --- src/sqliteInt.h +++ src/sqliteInt.h @@ -1271,10 +1271,11 @@ #define SQLITE_OrderByIdxJoin 0x0080 /* ORDER BY of joins via index */ #define SQLITE_SubqCoroutine 0x0100 /* Evaluate subqueries as coroutines */ #define SQLITE_Transitive 0x0200 /* Transitive constraints */ #define SQLITE_OmitNoopJoin 0x0400 /* Omit unused tables in joins */ #define SQLITE_Stat34 0x0800 /* Use STAT3 or STAT4 data */ +#define SQLITE_LateSubquery 0x1000 /* Plan main Q before rendering subQ */ #define SQLITE_AllOpts 0xffff /* All optimizations */ /* ** Macros for testing whether or not optimizations are enabled or disabled. */ @@ -3283,10 +3284,11 @@ void sqlite3SrcListDelete(sqlite3*, SrcList*); Index *sqlite3AllocateIndexObject(sqlite3*,i16,int,char**); Index *sqlite3CreateIndex(Parse*,Token*,Token*,SrcList*,ExprList*,int,Token*, Expr*, int, int); void sqlite3DropIndex(Parse*, SrcList*, int); +void sqlite3ManifestSubqueries(Parse*, Select*, SrcList*); int sqlite3Select(Parse*, Select*, SelectDest*); Select *sqlite3SelectNew(Parse*,ExprList*,SrcList*,Expr*,ExprList*, Expr*,ExprList*,u16,Expr*,Expr*); void sqlite3SelectDelete(sqlite3*, Select*); Table *sqlite3SrcListLookup(Parse*, SrcList*); @@ -3295,11 +3297,12 @@ #if defined(SQLITE_ENABLE_UPDATE_DELETE_LIMIT) && !defined(SQLITE_OMIT_SUBQUERY) Expr *sqlite3LimitWhere(Parse*,SrcList*,Expr*,ExprList*,Expr*,Expr*,char*); #endif void sqlite3DeleteFrom(Parse*, SrcList*, Expr*); void sqlite3Update(Parse*, SrcList*, ExprList*, Expr*, int); -WhereInfo *sqlite3WhereBegin(Parse*,SrcList*,Expr*,ExprList*,ExprList*,u16,int); +WhereInfo *sqlite3WhereBegin(Parse*,Select*,SrcList*,Expr*,ExprList*, + ExprList*,u16,int); void sqlite3WhereEnd(WhereInfo*); u64 sqlite3WhereOutputRowCount(WhereInfo*); int sqlite3WhereIsDistinct(WhereInfo*); int sqlite3WhereIsOrdered(WhereInfo*); int sqlite3WhereIsSorted(WhereInfo*); Index: src/update.c ================================================================== --- src/update.c +++ src/update.c @@ -345,11 +345,11 @@ /* Begin the database scan */ if( HasRowid(pTab) ){ sqlite3VdbeAddOp3(v, OP_Null, 0, regRowSet, regOldRowid); pWInfo = sqlite3WhereBegin( - pParse, pTabList, pWhere, 0, 0, WHERE_ONEPASS_DESIRED, iIdxCur + pParse, 0, pTabList, pWhere, 0, 0, WHERE_ONEPASS_DESIRED, iIdxCur ); if( pWInfo==0 ) goto update_cleanup; okOnePass = sqlite3WhereOkOnePass(pWInfo, aiCurOnePass); /* Remember the rowid of every item to be updated. @@ -374,11 +374,11 @@ regKey = ++pParse->nMem; iEph = pParse->nTab++; sqlite3VdbeAddOp2(v, OP_Null, 0, iPk); addrOpen = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, iEph, nPk); sqlite3VdbeSetP4KeyInfo(pParse, pPk); - pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, 0, 0, + pWInfo = sqlite3WhereBegin(pParse, 0, pTabList, pWhere, 0, 0, WHERE_ONEPASS_DESIRED, iIdxCur); if( pWInfo==0 ) goto update_cleanup; okOnePass = sqlite3WhereOkOnePass(pWInfo, aiCurOnePass); for(i=0; iaiColumn[i], Index: src/where.c ================================================================== --- src/where.c +++ src/where.c @@ -2553,19 +2553,20 @@ pNew->nLTerm = 1; pNew->aLTerm[0] = pTerm; /* TUNING: One-time cost for computing the automatic index is ** estimated to be X*N*log2(N) where N is the number of rows in ** the table being indexed and where X is 7 (LogEst=28) for normal - ** tables or 1.375 (LogEst=4) for views and subqueries. The value + ** tables or 0.3333 (LogEst=-16) for views and subqueries. The value ** of X is smaller for views and subqueries so that the query planner ** will be more aggressive about generating automatic indexes for ** those objects, since there is no opportunity to add schema ** indexes on subqueries and views. */ - pNew->rSetup = rLogSize + rSize + 4; + pNew->rSetup = rLogSize + rSize - 16; if( pTab->pSelect==0 && (pTab->tabFlags & TF_Ephemeral)==0 ){ - pNew->rSetup += 24; + pNew->rSetup += 44; } + if( pNew->rSetup<1 ) pNew->rSetup = 1; ApplyCostMultiplier(pNew->rSetup, pTab->costMult); /* TUNING: Each index lookup yields 20 rows in the table. This ** is more than the usual guess of 10 rows, since we have no way ** of knowing how selective the index will ultimately be. It would ** not be unreasonable to make this value much larger. */ @@ -3894,10 +3895,11 @@ ** be used to compute the appropriate cursor depending on which index is ** used. */ WhereInfo *sqlite3WhereBegin( Parse *pParse, /* The parser context */ + Select *pSelect, /* SELECT stmt that owns this WHERE. Might be NULL */ SrcList *pTabList, /* FROM clause: A list of all tables to be scanned */ Expr *pWhere, /* The WHERE clause */ ExprList *pOrderBy, /* An ORDER BY (or GROUP BY) clause, or NULL */ ExprList *pResultSet, /* Result set of the query */ u16 wctrlFlags, /* One of the WHERE_* flags defined in sqliteInt.h */ @@ -4167,10 +4169,20 @@ pWInfo->okOnePass = 1; if( HasRowid(pTabList->a[0].pTab) ){ pWInfo->a[0].pWLoop->wsFlags &= ~WHERE_IDX_ONLY; } } + + /* If this WHERE clause is part of a SELECT statement, then there + ** might be subqueries in the FROM clause that need to be manifested. + ** This works mostly - except the Table.nRowLogEst value is not set + ** correctly for the subquery, resulting in a bad plan in some cases. + */ + if( OptimizationEnabled(db, SQLITE_LateSubquery) && pSelect!=0 ){ + sqlite3ManifestSubqueries(pParse, pSelect, pTabList); + if( db->mallocFailed ) goto whereBeginError; + } /* Open all tables in the pTabList and any indices selected for ** searching those tables. */ for(ii=0, pLevel=pWInfo->a; iipLeft = pOrExpr; pOrExpr = pAndExpr; } /* Loop through table entries that match term pOrTerm. */ WHERETRACE(0xffff, ("Subplan for OR-clause:\n")); - pSubWInfo = sqlite3WhereBegin(pParse, pOrTab, pOrExpr, 0, 0, + pSubWInfo = sqlite3WhereBegin(pParse, 0, pOrTab, pOrExpr, 0, 0, wctrlFlags, iCovCur); assert( pSubWInfo || pParse->nErr || db->mallocFailed ); if( pSubWInfo ){ WhereLoop *pSubLoop; int addrExplain = sqlite3WhereExplainOneScan( Index: test/eqp.test ================================================================== --- test/eqp.test +++ test/eqp.test @@ -79,44 +79,44 @@ } do_eqp_test 1.7 { SELECT * FROM t3 JOIN (SELECT 1) } { - 0 0 1 {SCAN SUBQUERY 1} - 0 1 0 {SCAN TABLE t3} + 0 0 0 {SCAN TABLE t3} + 0 1 1 {SCAN SUBQUERY 1} } do_eqp_test 1.8 { SELECT * FROM t3 JOIN (SELECT 1 UNION SELECT 2) } { 1 0 0 {COMPOUND SUBQUERIES 2 AND 3 USING TEMP B-TREE (UNION)} - 0 0 1 {SCAN SUBQUERY 1} - 0 1 0 {SCAN TABLE t3} + 0 0 0 {SCAN TABLE t3} + 0 1 1 {SCAN SUBQUERY 1} } do_eqp_test 1.9 { SELECT * FROM t3 JOIN (SELECT 1 EXCEPT SELECT a FROM t3 LIMIT 17) } { 3 0 0 {SCAN TABLE t3} 1 0 0 {COMPOUND SUBQUERIES 2 AND 3 USING TEMP B-TREE (EXCEPT)} - 0 0 1 {SCAN SUBQUERY 1} - 0 1 0 {SCAN TABLE t3} + 0 0 0 {SCAN TABLE t3} + 0 1 1 {SCAN SUBQUERY 1} } do_eqp_test 1.10 { SELECT * FROM t3 JOIN (SELECT 1 INTERSECT SELECT a FROM t3 LIMIT 17) } { 3 0 0 {SCAN TABLE t3} 1 0 0 {COMPOUND SUBQUERIES 2 AND 3 USING TEMP B-TREE (INTERSECT)} - 0 0 1 {SCAN SUBQUERY 1} - 0 1 0 {SCAN TABLE t3} + 0 0 0 {SCAN TABLE t3} + 0 1 1 {SCAN SUBQUERY 1} } do_eqp_test 1.11 { SELECT * FROM t3 JOIN (SELECT 1 UNION ALL SELECT a FROM t3 LIMIT 17) } { 3 0 0 {SCAN TABLE t3} 1 0 0 {COMPOUND SUBQUERIES 2 AND 3 (UNION ALL)} - 0 0 1 {SCAN SUBQUERY 1} - 0 1 0 {SCAN TABLE t3} + 0 0 0 {SCAN TABLE t3} + 0 1 1 {SCAN SUBQUERY 1} } #------------------------------------------------------------------------- # Test cases eqp-2.* - tests for single select statements. #