Index: src/btree.c ================================================================== --- src/btree.c +++ src/btree.c @@ -7348,18 +7348,20 @@ ){ MemPage *pPage; int rc; unsigned char *pCell; int i; + int hdr; assert( sqlite3_mutex_held(pBt->mutex) ); if( pgno>btreePagecount(pBt) ){ return SQLITE_CORRUPT_BKPT; } rc = getAndInitPage(pBt, pgno, &pPage, 0); if( rc ) return rc; + hdr = pPage->hdrOffset; for(i=0; inCell; i++){ pCell = findCell(pPage, i); if( !pPage->leaf ){ rc = clearDatabasePage(pBt, get4byte(pCell), 1, pnChange); if( rc ) goto cleardatabasepage_out; @@ -7366,20 +7368,20 @@ } rc = clearCell(pPage, pCell); if( rc ) goto cleardatabasepage_out; } if( !pPage->leaf ){ - rc = clearDatabasePage(pBt, get4byte(&pPage->aData[8]), 1, pnChange); + rc = clearDatabasePage(pBt, get4byte(&pPage->aData[hdr+8]), 1, pnChange); if( rc ) goto cleardatabasepage_out; }else if( pnChange ){ assert( pPage->intKey ); *pnChange += pPage->nCell; } if( freePageFlag ){ freePage(pPage, &rc); }else if( (rc = sqlite3PagerWrite(pPage->pDbPage))==0 ){ - zeroPage(pPage, pPage->aData[0] | PTF_LEAF); + zeroPage(pPage, pPage->aData[hdr] | PTF_LEAF); } cleardatabasepage_out: releasePage(pPage); return rc; Index: src/build.c ================================================================== --- src/build.c +++ src/build.c @@ -4196,5 +4196,75 @@ } } } return sqlite3KeyInfoRef(pIdx->pKeyInfo); } + +#ifndef SQLITE_OMIT_CTE +/* +** This routine is invoked once per CTE by the parser while parsing a +** WITH clause. +*/ +With *sqlite3WithAdd( + Parse *pParse, /* Parsing context */ + With *pWith, /* Existing WITH clause, or NULL */ + Token *pName, /* Name of the common-table */ + ExprList *pArglist, /* Optional column name list for the table */ + Select *pQuery /* Query used to initialize the table */ +){ + sqlite3 *db = pParse->db; + With *pNew; + char *zName; + + /* Check that the CTE name is unique within this WITH clause. If + ** not, store an error in the Parse structure. */ + zName = sqlite3NameFromToken(pParse->db, pName); + if( zName && pWith ){ + int i; + for(i=0; inCte; i++){ + if( sqlite3StrICmp(zName, pWith->a[i].zName)==0 ){ + sqlite3ErrorMsg(pParse, "duplicate WITH table name: %s", zName); + } + } + } + + if( pWith ){ + int nByte = sizeof(*pWith) + (sizeof(pWith->a[1]) * pWith->nCte); + pNew = sqlite3DbRealloc(db, pWith, nByte); + }else{ + pNew = sqlite3DbMallocZero(db, sizeof(*pWith)); + } + assert( zName!=0 || pNew==0 ); + assert( db->mallocFailed==0 || pNew==0 ); + + if( pNew==0 ){ + sqlite3ExprListDelete(db, pArglist); + sqlite3SelectDelete(db, pQuery); + sqlite3DbFree(db, zName); + pNew = pWith; + }else{ + pNew->a[pNew->nCte].pSelect = pQuery; + pNew->a[pNew->nCte].pCols = pArglist; + pNew->a[pNew->nCte].zName = zName; + pNew->a[pNew->nCte].zErr = 0; + pNew->nCte++; + } + + return pNew; +} + +/* +** Free the contents of the With object passed as the second argument. +*/ +void sqlite3WithDelete(sqlite3 *db, With *pWith){ + if( pWith ){ + int i; + for(i=0; inCte; i++){ + struct Cte *pCte = &pWith->a[i]; + sqlite3ExprListDelete(db, pCte->pCols); + sqlite3SelectDelete(db, pCte->pSelect); + sqlite3DbFree(db, pCte->zName); + } + sqlite3DbFree(db, pWith); + } +} +#endif /* !defined(SQLITE_OMIT_CTE) */ Index: src/expr.c ================================================================== --- src/expr.c +++ src/expr.c @@ -893,10 +893,37 @@ } } return pNew; } +/* +** Create and return a deep copy of the object passed as the second +** argument. If an OOM condition is encountered, NULL is returned +** and the db->mallocFailed flag set. +*/ +#ifndef SQLITE_OMIT_CTE +static With *withDup(sqlite3 *db, With *p){ + With *pRet = 0; + if( p ){ + int nByte = sizeof(*p) + sizeof(p->a[0]) * (p->nCte-1); + pRet = sqlite3DbMallocZero(db, nByte); + if( pRet ){ + int i; + pRet->nCte = p->nCte; + for(i=0; inCte; i++){ + pRet->a[i].pSelect = sqlite3SelectDup(db, p->a[i].pSelect, 0); + pRet->a[i].pCols = sqlite3ExprListDup(db, p->a[i].pCols, 0); + pRet->a[i].zName = sqlite3DbStrDup(db, p->a[i].zName); + } + } + } + return pRet; +} +#else +# define withDup(x,y) 0 +#endif + /* ** The following group of routines make deep copies of expressions, ** expression lists, ID lists, and select statements. The copies can ** be deleted (by being passed to their respective ...Delete() routines) ** without effecting the originals. @@ -973,10 +1000,11 @@ pNewItem->iCursor = pOldItem->iCursor; pNewItem->addrFillSub = pOldItem->addrFillSub; pNewItem->regReturn = pOldItem->regReturn; pNewItem->isCorrelated = pOldItem->isCorrelated; pNewItem->viaCoroutine = pOldItem->viaCoroutine; + pNewItem->isRecursive = pOldItem->isRecursive; pNewItem->zIndex = sqlite3DbStrDup(db, pOldItem->zIndex); pNewItem->notIndexed = pOldItem->notIndexed; pNewItem->pIndex = pOldItem->pIndex; pTab = pNewItem->pTab = pOldItem->pTab; if( pTab ){ @@ -1034,10 +1062,11 @@ pNew->selFlags = p->selFlags & ~SF_UsesEphemeral; pNew->pRightmost = 0; pNew->addrOpenEphm[0] = -1; pNew->addrOpenEphm[1] = -1; pNew->addrOpenEphm[2] = -1; + pNew->pWith = withDup(db, p->pWith); return pNew; } #else Select *sqlite3SelectDup(sqlite3 *db, Select *p, int flags){ assert( p==0 ); @@ -1553,13 +1582,15 @@ pTab = p->pSrc->a[0].pTab; pExpr = p->pEList->a[0].pExpr; iCol = (i16)pExpr->iColumn; /* Code an OP_VerifyCookie and OP_TableLock for . */ - iDb = sqlite3SchemaToIndex(db, pTab->pSchema); - sqlite3CodeVerifySchema(pParse, iDb); - sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName); + if( pTab->pSchema ){ + iDb = sqlite3SchemaToIndex(db, pTab->pSchema); + sqlite3CodeVerifySchema(pParse, iDb); + sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName); + } /* This function is only called from two places. In both cases the vdbe ** has already been allocated. So assume sqlite3GetVdbe() is always ** successful here. */ Index: src/insert.c ================================================================== --- src/insert.c +++ src/insert.c @@ -665,11 +665,12 @@ ** Then special optimizations can be applied that make the transfer ** very fast and which reduce fragmentation of indices. ** ** This is the 2nd template. */ - if( pColumn==0 && xferOptimization(pParse, pTab, pSelect, onError, iDb) ){ + if( pColumn==0 && pParse->pWith==0 + && xferOptimization(pParse, pTab, pSelect, onError, iDb) ){ assert( !pTrigger ); assert( pList==0 ); goto insert_end; } #endif /* SQLITE_OMIT_XFER_OPT */ Index: src/parse.y ================================================================== --- src/parse.y +++ src/parse.y @@ -202,12 +202,12 @@ // %fallback ID ABORT ACTION AFTER ANALYZE ASC ATTACH BEFORE BEGIN BY CASCADE CAST COLUMNKW CONFLICT DATABASE DEFERRED DESC DETACH EACH END EXCLUSIVE EXPLAIN FAIL FOR IGNORE IMMEDIATE INITIALLY INSTEAD LIKE_KW MATCH NO PLAN - QUERY KEY OF OFFSET PRAGMA RAISE RELEASE REPLACE RESTRICT ROW ROLLBACK - SAVEPOINT TEMP TRIGGER VACUUM VIEW VIRTUAL WITHOUT + QUERY KEY OF OFFSET PRAGMA RAISE RECURSIVE RELEASE REPLACE RESTRICT ROW + ROLLBACK SAVEPOINT TEMP TRIGGER VACUUM VIEW VIRTUAL WITH WITHOUT %ifdef SQLITE_OMIT_COMPOUND_SELECT EXCEPT INTERSECT UNION %endif SQLITE_OMIT_COMPOUND_SELECT REINDEX RENAME CTIME_KW IF . @@ -405,16 +405,27 @@ sqlite3SelectDelete(pParse->db, X); } %type select {Select*} %destructor select {sqlite3SelectDelete(pParse->db, $$);} +%type selectnowith {Select*} +%destructor selectnowith {sqlite3SelectDelete(pParse->db, $$);} %type oneselect {Select*} %destructor oneselect {sqlite3SelectDelete(pParse->db, $$);} -select(A) ::= oneselect(X). {A = X;} +select(A) ::= with(W) selectnowith(X). { + if( X ){ + X->pWith = W; + }else{ + sqlite3WithDelete(pParse->db, W); + } + A = X; +} + +selectnowith(A) ::= oneselect(X). {A = X;} %ifndef SQLITE_OMIT_COMPOUND_SELECT -select(A) ::= select(X) multiselect_op(Y) oneselect(Z). { +selectnowith(A) ::= selectnowith(X) multiselect_op(Y) oneselect(Z). { if( Z ){ Z->op = (u8)Y; Z->pPrior = X; if( Y!=TK_ALL ) pParse->hasCompound = 1; }else{ @@ -646,19 +657,21 @@ {A.pOffset = X.pExpr; A.pLimit = Y.pExpr;} /////////////////////////// The DELETE statement ///////////////////////////// // %ifdef SQLITE_ENABLE_UPDATE_DELETE_LIMIT -cmd ::= DELETE FROM fullname(X) indexed_opt(I) where_opt(W) +cmd ::= with(C) DELETE FROM fullname(X) indexed_opt(I) where_opt(W) orderby_opt(O) limit_opt(L). { + sqlite3WithPush(pParse, C, 1); sqlite3SrcListIndexedBy(pParse, X, &I); W = sqlite3LimitWhere(pParse, X, W, O, L.pLimit, L.pOffset, "DELETE"); sqlite3DeleteFrom(pParse,X,W); } %endif %ifndef SQLITE_ENABLE_UPDATE_DELETE_LIMIT -cmd ::= DELETE FROM fullname(X) indexed_opt(I) where_opt(W). { +cmd ::= with(C) DELETE FROM fullname(X) indexed_opt(I) where_opt(W). { + sqlite3WithPush(pParse, C, 1); sqlite3SrcListIndexedBy(pParse, X, &I); sqlite3DeleteFrom(pParse,X,W); } %endif @@ -669,21 +682,23 @@ where_opt(A) ::= WHERE expr(X). {A = X.pExpr;} ////////////////////////// The UPDATE command //////////////////////////////// // %ifdef SQLITE_ENABLE_UPDATE_DELETE_LIMIT -cmd ::= UPDATE orconf(R) fullname(X) indexed_opt(I) SET setlist(Y) where_opt(W) - orderby_opt(O) limit_opt(L). { +cmd ::= with(C) UPDATE orconf(R) fullname(X) indexed_opt(I) SET setlist(Y) + where_opt(W) orderby_opt(O) limit_opt(L). { + sqlite3WithPush(pParse, C, 1); sqlite3SrcListIndexedBy(pParse, X, &I); sqlite3ExprListCheckLength(pParse,Y,"set list"); W = sqlite3LimitWhere(pParse, X, W, O, L.pLimit, L.pOffset, "UPDATE"); sqlite3Update(pParse,X,Y,W,R); } %endif %ifndef SQLITE_ENABLE_UPDATE_DELETE_LIMIT -cmd ::= UPDATE orconf(R) fullname(X) indexed_opt(I) SET setlist(Y) +cmd ::= with(C) UPDATE orconf(R) fullname(X) indexed_opt(I) SET setlist(Y) where_opt(W). { + sqlite3WithPush(pParse, C, 1); sqlite3SrcListIndexedBy(pParse, X, &I); sqlite3ExprListCheckLength(pParse,Y,"set list"); sqlite3Update(pParse,X,Y,W,R); } %endif @@ -700,14 +715,19 @@ sqlite3ExprListSetName(pParse, A, &X, 1); } ////////////////////////// The INSERT command ///////////////////////////////// // -cmd ::= insert_cmd(R) INTO fullname(X) inscollist_opt(F) select(S). - {sqlite3Insert(pParse, X, S, F, R);} -cmd ::= insert_cmd(R) INTO fullname(X) inscollist_opt(F) DEFAULT VALUES. - {sqlite3Insert(pParse, X, 0, F, R);} +cmd ::= with(W) insert_cmd(R) INTO fullname(X) inscollist_opt(F) select(S). { + sqlite3WithPush(pParse, W, 1); + sqlite3Insert(pParse, X, S, F, R); +} +cmd ::= with(W) insert_cmd(R) INTO fullname(X) inscollist_opt(F) DEFAULT VALUES. +{ + sqlite3WithPush(pParse, W, 1); + sqlite3Insert(pParse, X, 0, F, R); +} %type insert_cmd {u8} insert_cmd(A) ::= INSERT orconf(R). {A = R;} insert_cmd(A) ::= REPLACE. {A = OE_Replace;} @@ -849,14 +869,12 @@ {spanBinaryExpr(&A,pParse,@OP,&X,&Y);} expr(A) ::= expr(X) STAR|SLASH|REM(OP) expr(Y). {spanBinaryExpr(&A,pParse,@OP,&X,&Y);} expr(A) ::= expr(X) CONCAT(OP) expr(Y). {spanBinaryExpr(&A,pParse,@OP,&X,&Y);} %type likeop {struct LikeOp} -likeop(A) ::= LIKE_KW(X). {A.eOperator = X; A.bNot = 0;} -likeop(A) ::= NOT LIKE_KW(X). {A.eOperator = X; A.bNot = 1;} -likeop(A) ::= MATCH(X). {A.eOperator = X; A.bNot = 0;} -likeop(A) ::= NOT MATCH(X). {A.eOperator = X; A.bNot = 1;} +likeop(A) ::= LIKE_KW|MATCH(X). {A.eOperator = X; A.bNot = 0;} +likeop(A) ::= NOT LIKE_KW|MATCH(X). {A.eOperator = X; A.bNot = 1;} expr(A) ::= expr(X) likeop(OP) expr(Y). [LIKE_KW] { ExprList *pList; pList = sqlite3ExprListAppend(pParse,0, Y.pExpr); pList = sqlite3ExprListAppend(pParse,pList, X.pExpr); A.pExpr = sqlite3ExprFunction(pParse, pList, &OP.eOperator); @@ -1362,5 +1380,25 @@ lp ::= LP(X). {sqlite3VtabArgExtend(pParse,&X);} anylist ::= . anylist ::= anylist LP anylist RP. anylist ::= anylist ANY. %endif SQLITE_OMIT_VIRTUALTABLE + + +//////////////////////// COMMON TABLE EXPRESSIONS //////////////////////////// +%type with {With*} +%type wqlist {With*} +%destructor with {sqlite3WithDelete(pParse->db, $$);} +%destructor wqlist {sqlite3WithDelete(pParse->db, $$);} + +with(A) ::= . {A = 0;} +%ifndef SQLITE_OMIT_CTE +with(A) ::= WITH wqlist(W). { A = W; } +with(A) ::= WITH RECURSIVE wqlist(W). { A = W; } + +wqlist(A) ::= nm(X) idxlist_opt(Y) AS LP select(Z) RP. { + A = sqlite3WithAdd(pParse, 0, &X, Y, Z); +} +wqlist(A) ::= wqlist(W) COMMA nm(X) idxlist_opt(Y) AS LP select(Z) RP. { + A = sqlite3WithAdd(pParse, W, &X, Y, Z); +} +%endif SQLITE_OMIT_CTE Index: src/select.c ================================================================== --- src/select.c +++ src/select.c @@ -27,10 +27,11 @@ sqlite3ExprDelete(db, p->pHaving); sqlite3ExprListDelete(db, p->pOrderBy); sqlite3SelectDelete(db, p->pPrior); sqlite3ExprDelete(db, p->pLimit); sqlite3ExprDelete(db, p->pOffset); + sqlite3WithDelete(db, p->pWith); } /* ** Initialize a SelectDest structure. */ @@ -688,16 +689,30 @@ } #endif /* Store the result as data using a unique key. */ + case SRT_DistTable: case SRT_Table: case SRT_EphemTab: { int r1 = sqlite3GetTempReg(pParse); testcase( eDest==SRT_Table ); testcase( eDest==SRT_EphemTab ); sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nColumn, r1); +#ifndef SQLITE_OMIT_CTE + if( eDest==SRT_DistTable ){ + /* If the destination is DistTable, then cursor (iParm+1) is open + ** on an ephemeral index. If the current row is already present + ** in the index, do not write it to the output. If not, add the + ** current row to the index and proceed with writing it to the + ** output table as well. */ + int addr = sqlite3VdbeCurrentAddr(v) + 4; + sqlite3VdbeAddOp4Int(v, OP_Found, iParm+1, addr, r1, 0); + sqlite3VdbeAddOp2(v, OP_IdxInsert, iParm+1, r1); + assert( pOrderBy==0 ); + } +#endif if( pOrderBy ){ pushOntoSorter(pParse, pOrderBy, p, r1); }else{ int r2 = sqlite3GetTempReg(pParse); sqlite3VdbeAddOp2(v, OP_NewRowid, iParm, r2); @@ -1200,11 +1215,11 @@ sNC.pSrcList = pS->pSrc; sNC.pNext = pNC; sNC.pParse = pNC->pParse; zType = columnType(&sNC, p,&zOrigDb,&zOrigTab,&zOrigCol, &estWidth); } - }else if( ALWAYS(pTab->pSchema) ){ + }else if( pTab->pSchema ){ /* A real table */ assert( !pS ); if( iCol<0 ) iCol = pTab->iPKey; assert( iCol==-1 || (iCol>=0 && iColnCol) ); #ifdef SQLITE_ENABLE_COLUMN_METADATA @@ -1727,10 +1742,11 @@ /* Make sure there is no ORDER BY or LIMIT clause on prior SELECTs. Only ** the last (right-most) SELECT in the series may have an ORDER BY or LIMIT. */ assert( p && p->pPrior ); /* Calling function guarantees this much */ + assert( (p->selFlags & SF_Recursive)==0 || p->op==TK_ALL || p->op==TK_UNION ); db = pParse->db; pPrior = p->pPrior; assert( pPrior->pRightmost!=pPrior ); assert( pPrior->pRightmost==p->pRightmost ); dest = *pDest; @@ -1771,16 +1787,96 @@ " do not have the same number of result columns", selectOpName(p->op)); } rc = 1; goto multi_select_end; } + +#ifndef SQLITE_OMIT_CTE + if( p->selFlags & SF_Recursive ){ + SrcList *pSrc = p->pSrc; + int nCol = p->pEList->nExpr; + int addrNext; + int addrSwap; + int iCont, iBreak; + int tmp1; /* Intermediate table */ + int tmp2; /* Next intermediate table */ + int tmp3 = 0; /* To ensure unique results if UNION */ + int eDest = SRT_Table; + SelectDest tmp2dest; + int i; + + /* Check that there is no ORDER BY or LIMIT clause. Neither of these + ** are supported on recursive queries. */ + assert( p->pOffset==0 || p->pLimit ); + if( p->pOrderBy || p->pLimit ){ + sqlite3ErrorMsg(pParse, "%s in a recursive query is not allowed", + p->pOrderBy ? "ORDER BY" : "LIMIT" + ); + goto multi_select_end; + } + + if( sqlite3AuthCheck(pParse, SQLITE_RECURSIVE, 0, 0, 0) ){ + goto multi_select_end; + } + iBreak = sqlite3VdbeMakeLabel(v); + iCont = sqlite3VdbeMakeLabel(v); + + for(i=0; ALWAYS(inSrc); i++){ + if( pSrc->a[i].isRecursive ){ + tmp1 = pSrc->a[i].iCursor; + break; + } + } + + tmp2 = pParse->nTab++; + if( p->op==TK_UNION ){ + eDest = SRT_DistTable; + tmp3 = pParse->nTab++; + } + sqlite3SelectDestInit(&tmp2dest, eDest, tmp2); + + sqlite3VdbeAddOp2(v, OP_OpenEphemeral, tmp1, nCol); + sqlite3VdbeAddOp2(v, OP_OpenEphemeral, tmp2, nCol); + if( tmp3 ){ + p->addrOpenEphm[0] = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, tmp3, 0); + p->selFlags |= SF_UsesEphemeral; + } + + /* Store the results of the initial SELECT in tmp2. */ + rc = sqlite3Select(pParse, pPrior, &tmp2dest); + if( rc ) goto multi_select_end; + + /* Clear tmp1. Then switch the contents of tmp1 and tmp2. Then return + ** the contents of tmp1 to the caller. Or, if tmp1 is empty at this + ** point, the recursive query has finished - jump to address iBreak. */ + addrSwap = sqlite3VdbeAddOp2(v, OP_SwapCursors, tmp1, tmp2); + sqlite3VdbeAddOp2(v, OP_Rewind, tmp1, iBreak); + addrNext = sqlite3VdbeCurrentAddr(v); + selectInnerLoop(pParse, p, p->pEList, tmp1, p->pEList->nExpr, + 0, 0, &dest, iCont, iBreak); + sqlite3VdbeResolveLabel(v, iCont); + sqlite3VdbeAddOp2(v, OP_Next, tmp1, addrNext); + + /* Execute the recursive SELECT. Store the results in tmp2. While this + ** SELECT is running, the contents of tmp1 are read by recursive + ** references to the current CTE. */ + p->pPrior = 0; + rc = sqlite3Select(pParse, p, &tmp2dest); + assert( p->pPrior==0 ); + p->pPrior = pPrior; + if( rc ) goto multi_select_end; + + sqlite3VdbeAddOp2(v, OP_Goto, 0, addrSwap); + sqlite3VdbeResolveLabel(v, iBreak); + }else +#endif /* Compound SELECTs that have an ORDER BY clause are handled separately. */ if( p->pOrderBy ){ return multiSelectOrderBy(pParse, p, pDest); - } + }else /* Generate code for the left and right SELECT statements. */ switch( p->op ){ case TK_ALL: { @@ -2839,10 +2935,18 @@ ** appear as unmodified result columns in the outer query. But we ** have other optimizations in mind to deal with that case. ** ** (21) The subquery does not use LIMIT or the outer query is not ** DISTINCT. (See ticket [752e1646fc]). +** +** (22) The subquery is not a recursive CTE. +** +** (23) The parent is not a recursive CTE, or the sub-query is not a +** compound query. This restriction is because transforming the +** parent to a compound query confuses the code that handles +** recursive queries in multiSelect(). +** ** ** In this routine, the "p" parameter is a pointer to the outer query. ** The subquery is p->pSrc->a[iFrom]. isAgg is true if the outer query ** uses aggregates and subqueryIsAgg is true if the subquery uses aggregates. ** @@ -2911,10 +3015,12 @@ if( isAgg && pSub->pOrderBy ) return 0; /* Restriction (16) */ if( pSub->pLimit && p->pWhere ) return 0; /* Restriction (19) */ if( pSub->pLimit && (p->selFlags & SF_Distinct)!=0 ){ return 0; /* Restriction (21) */ } + if( pSub->selFlags & SF_Recursive ) return 0; /* Restriction (22) */ + if( (p->selFlags & SF_Recursive) && pSub->pPrior ) return 0; /* (23) */ /* OBSOLETE COMMENT 1: ** Restriction 3: If the subquery is a join, make sure the subquery is ** not used as the right operand of an outer join. Examples of why this ** is not allowed: @@ -3392,10 +3498,179 @@ pNew->pLimit = 0; pNew->pOffset = 0; return WRC_Continue; } +#ifndef SQLITE_OMIT_CTE +/* +** Argument pWith (which may be NULL) points to a linked list of nested +** WITH contexts, from inner to outermost. If the table identified by +** FROM clause element pItem is really a common-table-expression (CTE) +** then return a pointer to the CTE definition for that table. Otherwise +** return NULL. +*/ +static struct Cte *searchWith(With *pWith, struct SrcList_item *pItem){ + const char *zName; + if( pItem->zDatabase==0 && (zName = pItem->zName)!=0 ){ + With *p; + for(p=pWith; p; p=p->pOuter){ + int i; + for(i=0; inCte; i++){ + if( sqlite3StrICmp(zName, p->a[i].zName)==0 ){ + return &p->a[i]; + } + } + } + } + return 0; +} + +/* The code generator maintains a stack of active WITH clauses +** with the inner-most WITH clause being at the top of the stack. +** +** This routine pushes the WITH clause passed as the second argument +** onto the top of the stack. If argument bFree is true, then this +** WITH clause will never be popped from the stack. In this case it +** should be freed along with the Parse object. In other cases, when +** bFree==0, the With object will be freed along with the SELECT +** statement with which it is associated. +*/ +void sqlite3WithPush(Parse *pParse, With *pWith, u8 bFree){ + assert( bFree==0 || pParse->pWith==0 ); + if( pWith ){ + pWith->pOuter = pParse->pWith; + pParse->pWith = pWith; + pParse->bFreeWith = bFree; + } +} + +/* +** This function checks if argument pFrom refers to a CTE declared by +** a WITH clause on the stack currently maintained by the parser. And, +** if currently processing a CTE expression, if it is a recursive +** reference to the current CTE. +** +** If pFrom falls into either of the two categories above, pFrom->pTab +** and other fields are populated accordingly. The caller should check +** (pFrom->pTab!=0) to determine whether or not a successful match +** was found. +** +** Whether or not a match is found, SQLITE_OK is returned if no error +** occurs. If an error does occur, an error message is stored in the +** parser and some error code other than SQLITE_OK returned. +*/ +static int withExpand( + Walker *pWalker, + struct SrcList_item *pFrom +){ + Table *pTab; + Parse *pParse = pWalker->pParse; + sqlite3 *db = pParse->db; + struct Cte *pCte; + + assert( pFrom->pTab==0 ); + + pCte = searchWith(pParse->pWith, pFrom); + if( pCte ){ + ExprList *pEList; + Select *pSel; + Select *pLeft; /* Left-most SELECT statement */ + int bMayRecursive; /* True if compound joined by UNION [ALL] */ + + /* If pCte->zErr is non-NULL at this point, then this is an illegal + ** recursive reference to CTE pCte. Leave an error in pParse and return + ** early. If pCte->zErr is NULL, then this is not a recursive reference. + ** In this case, proceed. */ + if( pCte->zErr ){ + sqlite3ErrorMsg(pParse, pCte->zErr, pCte->zName); + return WRC_Abort; + } + + pFrom->pTab = pTab = sqlite3DbMallocZero(db, sizeof(Table)); + if( pTab==0 ) return WRC_Abort; + pTab->nRef = 1; + pTab->zName = sqlite3DbStrDup(db, pCte->zName); + pTab->iPKey = -1; + pTab->nRowEst = 1048576; + pTab->tabFlags |= TF_Ephemeral; + pFrom->pSelect = sqlite3SelectDup(db, pCte->pSelect, 0); + if( db->mallocFailed ) return SQLITE_NOMEM; + assert( pFrom->pSelect ); + + /* Check if this is a recursive CTE. */ + pSel = pFrom->pSelect; + bMayRecursive = ( pSel->op==TK_ALL || pSel->op==TK_UNION ); + if( bMayRecursive ){ + int i; + SrcList *pSrc = pFrom->pSelect->pSrc; + for(i=0; inSrc; i++){ + struct SrcList_item *pItem = &pSrc->a[i]; + if( pItem->zDatabase==0 + && pItem->zName!=0 + && 0==sqlite3StrICmp(pItem->zName, pCte->zName) + ){ + pItem->pTab = pTab; + pItem->isRecursive = 1; + pTab->nRef++; + pSel->selFlags |= SF_Recursive; + } + } + } + + /* Only one recursive reference is permitted. */ + if( pTab->nRef>2 ){ + sqlite3ErrorMsg( + pParse, "multiple references to recursive table: %s", pCte->zName + ); + return WRC_Abort; + } + assert( pTab->nRef==1 || ((pSel->selFlags&SF_Recursive) && pTab->nRef==2 )); + + pCte->zErr = "circular reference: %s"; + sqlite3WalkSelect(pWalker, bMayRecursive ? pSel->pPrior : pSel); + + for(pLeft=pSel; pLeft->pPrior; pLeft=pLeft->pPrior); + pEList = pLeft->pEList; + if( pCte->pCols ){ + if( pEList->nExpr!=pCte->pCols->nExpr ){ + sqlite3ErrorMsg(pParse, "table %s has %d values for %d columns", + pCte->zName, pEList->nExpr, pCte->pCols->nExpr + ); + return WRC_Abort; + } + pEList = pCte->pCols; + } + selectColumnsFromExprList(pParse, pEList, &pTab->nCol, &pTab->aCol); + + if( bMayRecursive ){ + if( pSel->selFlags & SF_Recursive ){ + pCte->zErr = "multiple recursive references: %s"; + }else{ + pCte->zErr = "recursive reference in a subquery: %s"; + } + sqlite3WalkSelect(pWalker, pSel); + } + pCte->zErr = 0; + } + + return SQLITE_OK; +} +#endif + +#ifndef SQLITE_OMIT_CTE +static void selectPopWith(Walker *pWalker, Select *p){ + Parse *pParse = pWalker->pParse; + if( p->pWith ){ + assert( pParse->pWith==p->pWith ); + pParse->pWith = p->pWith->pOuter; + } + return WRC_Continue; +} +#else +#define selectPopWith 0 +#endif + /* ** This routine is a Walker callback for "expanding" a SELECT statement. ** "Expanding" means to do the following: ** ** (1) Make sure VDBE cursor numbers have been assigned to every @@ -3435,10 +3710,11 @@ if( NEVER(p->pSrc==0) || (selFlags & SF_Expanded)!=0 ){ return WRC_Prune; } pTabList = p->pSrc; pEList = p->pEList; + sqlite3WithPush(pParse, p->pWith, 0); /* Make sure cursor numbers have been assigned to all entries in ** the FROM clause of the SELECT statement. */ sqlite3SrcListAssignCursors(pParse, pTabList); @@ -3447,16 +3723,25 @@ ** an entry of the FROM clause is a subquery instead of a table or view, ** then create a transient table structure to describe the subquery. */ for(i=0, pFrom=pTabList->a; inSrc; i++, pFrom++){ Table *pTab; + assert( pFrom->isRecursive==0 || pFrom->pTab ); + if( pFrom->isRecursive ) continue; if( pFrom->pTab!=0 ){ /* This statement has already been prepared. There is no need ** to go further. */ assert( i==0 ); +#ifndef SQLITE_OMIT_CTE + selectPopWith(pWalker, p); +#endif return WRC_Prune; } +#ifndef SQLITE_OMIT_CTE + if( withExpand(pWalker, pFrom) ) return WRC_Abort; + if( pFrom->pTab ) {} else +#endif if( pFrom->zName==0 ){ #ifndef SQLITE_OMIT_SUBQUERY Select *pSel = pFrom->pSelect; /* A sub-query in the FROM clause of a SELECT */ assert( pSel!=0 ); @@ -3715,10 +4000,11 @@ if( pParse->hasCompound ){ w.xSelectCallback = convertCompoundSelectToSubquery; sqlite3WalkSelect(&w, pSelect); } w.xSelectCallback = selectExpander; + w.xSelectCallback2 = selectPopWith; sqlite3WalkSelect(&w, pSelect); } #ifndef SQLITE_OMIT_SUBQUERY @@ -3733,11 +4019,11 @@ ** The Table structure that represents the result set was constructed ** by selectExpander() but the type and collation information was omitted ** at that point because identifiers had not yet been resolved. This ** routine is called after identifier resolution. */ -static int selectAddSubqueryTypeInfo(Walker *pWalker, Select *p){ +static void selectAddSubqueryTypeInfo(Walker *pWalker, Select *p){ Parse *pParse; int i; SrcList *pTabList; struct SrcList_item *pFrom; @@ -3749,17 +4035,17 @@ for(i=0, pFrom=pTabList->a; inSrc; i++, pFrom++){ Table *pTab = pFrom->pTab; if( ALWAYS(pTab!=0) && (pTab->tabFlags & TF_Ephemeral)!=0 ){ /* A sub-query in the FROM clause of a SELECT */ Select *pSel = pFrom->pSelect; - assert( pSel ); - while( pSel->pPrior ) pSel = pSel->pPrior; - selectAddColumnTypeAndCollation(pParse, pTab, pSel); + if( pSel ){ + while( pSel->pPrior ) pSel = pSel->pPrior; + selectAddColumnTypeAndCollation(pParse, pTab, pSel); + } } } } - return WRC_Continue; } #endif /* @@ -3771,14 +4057,13 @@ */ static void sqlite3SelectAddTypeInfo(Parse *pParse, Select *pSelect){ #ifndef SQLITE_OMIT_SUBQUERY Walker w; memset(&w, 0, sizeof(w)); - w.xSelectCallback = selectAddSubqueryTypeInfo; + w.xSelectCallback2 = selectAddSubqueryTypeInfo; w.xExprCallback = exprWalkNoop; w.pParse = pParse; - w.bSelectDepthFirst = 1; sqlite3WalkSelect(&w, pSelect); #endif } Index: src/sqlite.h.in ================================================================== --- src/sqlite.h.in +++ src/sqlite.h.in @@ -2559,10 +2559,11 @@ #define SQLITE_CREATE_VTABLE 29 /* Table Name Module Name */ #define SQLITE_DROP_VTABLE 30 /* Table Name Module Name */ #define SQLITE_FUNCTION 31 /* NULL Function Name */ #define SQLITE_SAVEPOINT 32 /* Operation Savepoint Name */ #define SQLITE_COPY 0 /* No longer used */ +#define SQLITE_RECURSIVE 33 /* NULL NULL */ /* ** CAPI3REF: Tracing And Profiling Functions ** ** These routines register callback functions that can be used for Index: src/sqliteInt.h ================================================================== --- src/sqliteInt.h +++ src/sqliteInt.h @@ -758,10 +758,11 @@ typedef struct UnpackedRecord UnpackedRecord; typedef struct VTable VTable; typedef struct VtabCtx VtabCtx; typedef struct Walker Walker; typedef struct WhereInfo WhereInfo; +typedef struct With With; /* ** Defer sourcing vdbe.h and btree.h until after the "u8" and ** "BusyHandler" typedefs. vdbe.h also requires a few of the opaque ** pointer types (i.e. FuncDef) defined above. @@ -1426,18 +1427,19 @@ Schema *pSchema; /* Schema that contains this table */ Table *pNextZombie; /* Next on the Parse.pZombieTab list */ }; /* -** Allowed values for Tabe.tabFlags. +** Allowed values for Table.tabFlags. */ #define TF_Readonly 0x01 /* Read-only system table */ #define TF_Ephemeral 0x02 /* An ephemeral table */ #define TF_HasPrimaryKey 0x04 /* Table has a primary key */ #define TF_Autoincrement 0x08 /* Integer primary key is autoincrement */ #define TF_Virtual 0x10 /* Is a virtual table */ #define TF_WithoutRowid 0x20 /* No rowid used. PRIMARY KEY is the key */ +#define TF_Recursive 0x40 /* Recursive reference within CTE */ /* ** Test to see whether or not a table is a virtual table. This is ** done as a macro so that it will be optimized out when virtual @@ -2015,10 +2017,11 @@ int regReturn; /* Register holding return address of addrFillSub */ u8 jointype; /* Type of join between this able and the previous */ unsigned notIndexed :1; /* True if there is a NOT INDEXED clause */ unsigned isCorrelated :1; /* True if sub-query is correlated */ unsigned viaCoroutine :1; /* Implemented as a co-routine */ + unsigned isRecursive :1; /* True for recursive reference in WITH */ #ifndef SQLITE_OMIT_EXPLAIN u8 iSelectId; /* If pSelect!=0, the id of the sub-select in EQP */ #endif int iCursor; /* The VDBE cursor number used to access this table */ Expr *pOn; /* The ON clause of a join */ @@ -2141,10 +2144,11 @@ Select *pPrior; /* Prior select in a compound select statement */ Select *pNext; /* Next select to the left in a compound */ Select *pRightmost; /* Right-most select in a compound select statement */ Expr *pLimit; /* LIMIT expression. NULL means not used. */ Expr *pOffset; /* OFFSET expression. NULL means not used. */ + With *pWith; /* WITH clause attached to this select. Or NULL. */ }; /* ** Allowed values for Select.selFlags. The "SF" prefix stands for ** "Select Flag". @@ -2158,10 +2162,11 @@ #define SF_UseSorter 0x0040 /* Sort using a sorter */ #define SF_Values 0x0080 /* Synthesized from VALUES clause */ #define SF_Materialize 0x0100 /* Force materialization of views */ #define SF_NestedFrom 0x0200 /* Part of a parenthesized FROM clause */ #define SF_MaybeConvert 0x0400 /* Need convertCompoundSelectToSubquery() */ +#define SF_Recursive 0x0800 /* The recursive part of a recursive CTE */ /* ** The results of a select can be distributed in several ways. The ** "SRT" prefix means "SELECT Result Type". @@ -2178,10 +2183,11 @@ #define SRT_Mem 6 /* Store result in a memory cell */ #define SRT_Set 7 /* Store results as keys in an index */ #define SRT_Table 8 /* Store result as data with an automatic rowid */ #define SRT_EphemTab 9 /* Create transient tab and store like SRT_Table */ #define SRT_Coroutine 10 /* Generate a single row of result */ +#define SRT_DistTable 11 /* Like SRT_TABLE, but unique results only */ /* ** An instance of this object describes where to put of the results of ** a SELECT statement. */ @@ -2362,10 +2368,12 @@ Token sArg; /* Complete text of a module argument */ Table **apVtabLock; /* Pointer to virtual tables needing locking */ #endif Table *pZombieTab; /* List of Table objects to delete after code gen */ TriggerPrg *pTriggerPrg; /* Linked list of coded triggers */ + With *pWith; /* Current WITH clause, or NULL */ + u8 bFreeWith; /* True if pWith should be freed with parser */ }; /* ** Return true if currently inside an sqlite3_declare_vtab() call. */ @@ -2603,13 +2611,13 @@ ** Context pointer passed down through the tree-walk. */ struct Walker { int (*xExprCallback)(Walker*, Expr*); /* Callback for expressions */ int (*xSelectCallback)(Walker*,Select*); /* Callback for SELECTs */ + void (*xSelectCallback2)(Walker*,Select*);/* Second callback for SELECTs */ Parse *pParse; /* Parser context. */ int walkerDepth; /* Number of subqueries */ - u8 bSelectDepthFirst; /* Do subqueries first */ union { /* Extra data for callback */ NameContext *pNC; /* Naming context */ int i; /* Integer value */ SrcList *pSrcList; /* FROM clause */ struct SrcCount *pSrcCount; /* Counting column references */ @@ -2628,10 +2636,25 @@ ** callbacks. */ #define WRC_Continue 0 /* Continue down into children */ #define WRC_Prune 1 /* Omit children but continue walking siblings */ #define WRC_Abort 2 /* Abandon the tree walk */ + +/* +** An instance of this structure represents a set of one or more CTEs +** (common table expressions) created by a single WITH clause. +*/ +struct With { + int nCte; /* Number of CTEs in the WITH clause */ + With *pOuter; /* Containing WITH clause, or NULL */ + struct Cte { /* For each CTE in the WITH clause.... */ + char *zName; /* Name of this CTE */ + ExprList *pCols; /* List of explicit column names, or NULL */ + Select *pSelect; /* The definition of this CTE */ + const char *zErr; /* Error message for circular references */ + } a[1]; +}; /* ** Assuming zIn points to the first byte of a UTF-8 character, ** advance zIn to point to the first byte of the next UTF-8 character. */ @@ -3327,10 +3350,18 @@ const char *sqlite3JournalModename(int); #ifndef SQLITE_OMIT_WAL int sqlite3Checkpoint(sqlite3*, int, int, int*, int*); int sqlite3WalDefaultHook(void*,sqlite3*,const char*,int); #endif +#ifndef SQLITE_OMIT_CTE + With *sqlite3WithAdd(Parse*,With*,Token*,ExprList*,Select*); + void sqlite3WithDelete(sqlite3*,With*); + void sqlite3WithPush(Parse*, With*, u8); +#else +#define sqlite3WithPush(x,y,z) +#define sqlite3WithDelete(x,y) +#endif /* Declarations for functions in fkey.c. All of these are replaced by ** no-op macros if OMIT_FOREIGN_KEY is defined. In this case no foreign ** key functionality is available. If OMIT_TRIGGER is defined but ** OMIT_FOREIGN_KEY is not, only some of the functions are no-oped. In Index: src/tclsqlite.c ================================================================== --- src/tclsqlite.c +++ src/tclsqlite.c @@ -912,10 +912,11 @@ case SQLITE_ANALYZE : zCode="SQLITE_ANALYZE"; break; case SQLITE_CREATE_VTABLE : zCode="SQLITE_CREATE_VTABLE"; break; case SQLITE_DROP_VTABLE : zCode="SQLITE_DROP_VTABLE"; break; case SQLITE_FUNCTION : zCode="SQLITE_FUNCTION"; break; case SQLITE_SAVEPOINT : zCode="SQLITE_SAVEPOINT"; break; + case SQLITE_RECURSIVE : zCode="SQLITE_RECURSIVE"; break; default : zCode="????"; break; } Tcl_DStringInit(&str); Tcl_DStringAppend(&str, pDb->zAuth, -1); Tcl_DStringAppendElement(&str, zCode); Index: src/test_config.c ================================================================== --- src/test_config.c +++ src/test_config.c @@ -222,10 +222,16 @@ #ifdef SQLITE_OMIT_CHECK Tcl_SetVar2(interp, "sqlite_options", "check", "0", TCL_GLOBAL_ONLY); #else Tcl_SetVar2(interp, "sqlite_options", "check", "1", TCL_GLOBAL_ONLY); #endif + +#ifdef SQLITE_OMIT_CTE + Tcl_SetVar2(interp, "sqlite_options", "cte", "0", TCL_GLOBAL_ONLY); +#else + Tcl_SetVar2(interp, "sqlite_options", "cte", "1", TCL_GLOBAL_ONLY); +#endif #ifdef SQLITE_ENABLE_COLUMN_METADATA Tcl_SetVar2(interp, "sqlite_options", "columnmetadata", "1", TCL_GLOBAL_ONLY); #else Tcl_SetVar2(interp, "sqlite_options", "columnmetadata", "0", TCL_GLOBAL_ONLY); Index: src/tokenize.c ================================================================== --- src/tokenize.c +++ src/tokenize.c @@ -492,10 +492,11 @@ ** will take responsibility for freeing the Table structure. */ sqlite3DeleteTable(db, pParse->pNewTable); } + if( pParse->bFreeWith ) sqlite3WithDelete(db, pParse->pWith); sqlite3DeleteTrigger(db, pParse->pNewTrigger); for(i=pParse->nzVar-1; i>=0; i--) sqlite3DbFree(db, pParse->azVar[i]); sqlite3DbFree(db, pParse->azVar); while( pParse->pAinc ){ AutoincInfo *p = pParse->pAinc; Index: src/vdbe.c ================================================================== --- src/vdbe.c +++ src/vdbe.c @@ -3367,10 +3367,36 @@ } pCx->isOrdered = (pOp->p5!=BTREE_UNORDERED); break; } +#ifndef SQLITE_OMIT_CTE +/* Opcode: SwapCursors P1 P2 * * * +** +** Parameters P1 and P2 are both cursors opened by the OpenEphemeral +** opcode. This opcode deletes the contents of epheremal table P1, +** then renames P2 to P1 and P1 to P2. In other words, following this +** opcode cursor P2 is open on an empty table and P1 is open on the +** table that was initially accessed by P2. +*/ +case OP_SwapCursors: { + Mem tmp; + VdbeCursor *pTmp; + + tmp = p->aMem[p->nMem - pOp->p1]; + p->aMem[p->nMem - pOp->p1] = p->aMem[p->nMem - pOp->p2]; + p->aMem[p->nMem - pOp->p2] = tmp; + + pTmp = p->apCsr[pOp->p1]; + p->apCsr[pOp->p1] = p->apCsr[pOp->p2]; + p->apCsr[pOp->p2] = pTmp; + + rc = sqlite3BtreeClearTable(pTmp->pBt, MASTER_ROOT + !pTmp->isTable, 0); + break; +} +#endif /* ifndef SQLITE_OMIT_CTE */ + /* Opcode: SorterOpen P1 * * P4 * ** ** This opcode works like OP_OpenEphemeral except that it opens ** a transient index that is specifically designed to sort large ** tables using an external merge-sort algorithm. Index: src/walker.c ================================================================== --- src/walker.c +++ src/walker.c @@ -111,42 +111,43 @@ } /* ** Call sqlite3WalkExpr() for every expression in Select statement p. ** Invoke sqlite3WalkSelect() for subqueries in the FROM clause and -** on the compound select chain, p->pPrior. Invoke the xSelectCallback() -** either before or after the walk of expressions and FROM clause, depending -** on whether pWalker->bSelectDepthFirst is false or true, respectively. +** on the compound select chain, p->pPrior. +** +** If it is not NULL, the xSelectCallback() callback is invoked before +** the walk of the expressions and FROM clause. The xSelectCallback2() +** method, if it is not NULL, is invoked following the walk of the +** expressions and FROM clause. ** ** Return WRC_Continue under normal conditions. Return WRC_Abort if ** there is an abort request. ** ** If the Walker does not have an xSelectCallback() then this routine ** is a no-op returning WRC_Continue. */ int sqlite3WalkSelect(Walker *pWalker, Select *p){ int rc; - if( p==0 || pWalker->xSelectCallback==0 ) return WRC_Continue; + if( p==0 || (pWalker->xSelectCallback==0 && pWalker->xSelectCallback2==0) ){ + return WRC_Continue; + } rc = WRC_Continue; pWalker->walkerDepth++; while( p ){ - if( !pWalker->bSelectDepthFirst ){ + if( pWalker->xSelectCallback ){ rc = pWalker->xSelectCallback(pWalker, p); if( rc ) break; } if( sqlite3WalkSelectExpr(pWalker, p) || sqlite3WalkSelectFrom(pWalker, p) ){ pWalker->walkerDepth--; return WRC_Abort; } - if( pWalker->bSelectDepthFirst ){ - rc = pWalker->xSelectCallback(pWalker, p); - /* Depth-first search is currently only used for - ** selectAddSubqueryTypeInfo() and that routine always returns - ** WRC_Continue (0). So the following branch is never taken. */ - if( NEVER(rc) ) break; + if( pWalker->xSelectCallback2 ){ + pWalker->xSelectCallback2(pWalker, p); } p = p->pPrior; } pWalker->walkerDepth--; return rc & WRC_Abort; Index: src/where.c ================================================================== --- src/where.c +++ src/where.c @@ -4194,10 +4194,11 @@ && pSrc->pIndex==0 && !pSrc->viaCoroutine && !pSrc->notIndexed && HasRowid(pTab) && !pSrc->isCorrelated + && !pSrc->isRecursive ){ /* Generate auto-index WhereLoops */ WhereTerm *pTerm; WhereTerm *pWCEnd = pWC->a + pWC->nTerm; for(pTerm=pWC->a; rc==SQLITE_OK && pTerm