Index: src/select.c ================================================================== --- src/select.c +++ src/select.c @@ -460,17 +460,17 @@ /* ** Add code to implement the OFFSET */ static void codeOffset( Vdbe *v, /* Generate code into this VM */ - Select *p, /* The SELECT statement being coded */ + int iOffset, /* Register holding the offset counter */ int iContinue /* Jump here to skip the current record */ ){ - if( p->iOffset && iContinue!=0 ){ + if( iOffset>0 && iContinue!=0 ){ int addr; - sqlite3VdbeAddOp2(v, OP_AddImm, p->iOffset, -1); - addr = sqlite3VdbeAddOp1(v, OP_IfNeg, p->iOffset); + sqlite3VdbeAddOp2(v, OP_AddImm, iOffset, -1); + addr = sqlite3VdbeAddOp1(v, OP_IfNeg, iOffset); sqlite3VdbeAddOp2(v, OP_Goto, 0, iContinue); VdbeComment((v, "skip OFFSET records")); sqlite3VdbeJumpHere(v, addr); } } @@ -541,21 +541,20 @@ /* ** This routine generates the code for the inside of the inner loop ** of a SELECT. ** -** If srcTab and nColumn are both zero, then the pEList expressions -** are evaluated in order to get the data for this row. If nColumn>0 -** then data is pulled from srcTab and pEList is used only to get the -** datatypes for each column. +** If srcTab is negative, then the pEList expressions +** are evaluated in order to get the data for this row. If srcTab is +** zero or more, then data is pulled from srcTab and pEList is used only +** to get number columns and the datatype for each column. */ static void selectInnerLoop( Parse *pParse, /* The parser context */ Select *p, /* The complete select statement being coded */ ExprList *pEList, /* List of values being extracted */ int srcTab, /* Pull data from this table */ - int nColumn, /* Number of columns in the source table */ ExprList *pOrderBy, /* If not NULL, sort results using this key */ DistinctCtx *pDistinct, /* If not NULL, info on how to process DISTINCT */ SelectDest *pDest, /* How to dispose of the results */ int iContinue, /* Jump here to continue with next row */ int iBreak /* Jump here to break out of the inner loop */ @@ -567,61 +566,54 @@ int eDest = pDest->eDest; /* How to dispose of results */ int iParm = pDest->iSDParm; /* First argument to disposal method */ int nResultCol; /* Number of result columns */ assert( v ); - if( NEVER(v==0) ) return; assert( pEList!=0 ); hasDistinct = pDistinct ? pDistinct->eTnctType : WHERE_DISTINCT_NOOP; if( pOrderBy==0 && !hasDistinct ){ - codeOffset(v, p, iContinue); + codeOffset(v, p->iOffset, iContinue); } /* Pull the requested columns. */ - if( nColumn>0 ){ - nResultCol = nColumn; - }else{ - nResultCol = pEList->nExpr; - } + nResultCol = pEList->nExpr; if( pDest->iSdst==0 ){ pDest->iSdst = pParse->nMem+1; pDest->nSdst = nResultCol; pParse->nMem += nResultCol; }else{ assert( pDest->nSdst==nResultCol ); } regResult = pDest->iSdst; - if( nColumn>0 ){ - for(i=0; i=0 ){ + for(i=0; ia[i].zName)); } }else if( eDest!=SRT_Exists ){ /* If the destination is an EXISTS(...) expression, the actual ** values returned by the SELECT are not required. */ sqlite3ExprCodeExprList(pParse, pEList, regResult, (eDest==SRT_Output)?SQLITE_ECEL_DUP:0); } - nColumn = nResultCol; /* If the DISTINCT keyword was present on the SELECT statement ** and this row has been seen before, then do not make this row ** part of the result. */ if( hasDistinct ){ - assert( pEList!=0 ); - assert( pEList->nExpr==nColumn ); switch( pDistinct->eTnctType ){ case WHERE_DISTINCT_ORDERED: { VdbeOp *pOp; /* No longer required OpenEphemeral instr. */ int iJump; /* Jump destination */ int regPrev; /* Previous row content */ /* Allocate space for the previous row */ regPrev = pParse->nMem+1; - pParse->nMem += nColumn; + pParse->nMem += nResultCol; /* Change the OP_OpenEphemeral coded earlier to an OP_Null ** sets the MEM_Cleared bit on the first register of the ** previous value. This will cause the OP_Ne below to always ** fail on the first iteration of the loop even if the first @@ -631,23 +623,23 @@ pOp = sqlite3VdbeGetOp(v, pDistinct->addrTnct); pOp->opcode = OP_Null; pOp->p1 = 1; pOp->p2 = regPrev; - iJump = sqlite3VdbeCurrentAddr(v) + nColumn; - for(i=0; ia[i].pExpr); - if( iaddrTnct); @@ -654,16 +646,16 @@ break; } default: { assert( pDistinct->eTnctType==WHERE_DISTINCT_UNORDERED ); - codeDistinct(pParse, pDistinct->tabTnct, iContinue, nColumn, regResult); + codeDistinct(pParse, pDistinct->tabTnct, iContinue, nResultCol, regResult); break; } } if( pOrderBy==0 ){ - codeOffset(v, p, iContinue); + codeOffset(v, p->iOffset, iContinue); } } switch( eDest ){ /* In this mode, write each query result to the key of the temporary @@ -671,11 +663,11 @@ */ #ifndef SQLITE_OMIT_COMPOUND_SELECT case SRT_Union: { int r1; r1 = sqlite3GetTempReg(pParse); - sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nColumn, r1); + sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nResultCol, r1); sqlite3VdbeAddOp2(v, OP_IdxInsert, iParm, r1); sqlite3ReleaseTempReg(pParse, r1); break; } @@ -682,24 +674,24 @@ /* Construct a record from the query result, but instead of ** saving that record, use it as a key to delete elements from ** the temporary table iParm. */ case SRT_Except: { - sqlite3VdbeAddOp3(v, OP_IdxDelete, iParm, regResult, nColumn); + sqlite3VdbeAddOp3(v, OP_IdxDelete, iParm, regResult, nResultCol); break; } -#endif +#endif /* SQLITE_OMIT_COMPOUND_SELECT */ /* Store the result as data using a unique key. */ case SRT_DistTable: case SRT_Table: case SRT_EphemTab: { int r1 = sqlite3GetTempReg(pParse); testcase( eDest==SRT_Table ); testcase( eDest==SRT_EphemTab ); - sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nColumn, r1); + sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nResultCol, r1); #ifndef SQLITE_OMIT_CTE if( eDest==SRT_DistTable ){ /* If the destination is DistTable, then cursor (iParm+1) is open ** on an ephemeral index. If the current row is already present ** in the index, do not write it to the output. If not, add the @@ -728,11 +720,11 @@ /* If we are creating a set for an "expr IN (SELECT ...)" construct, ** then there should be a single item on the stack. Write this ** item into the set table with bogus data. */ case SRT_Set: { - assert( nColumn==1 ); + assert( nResultCol==1 ); pDest->affSdst = sqlite3CompareAffinity(pEList->a[0].pExpr, pDest->affSdst); if( pOrderBy ){ /* At first glance you would think we could optimize out the ** ORDER BY in this case since the order of entries in the set @@ -760,11 +752,11 @@ /* If this is a scalar select that is part of an expression, then ** store the results in the appropriate memory cell and break out ** of the scan loop. */ case SRT_Mem: { - assert( nColumn==1 ); + assert( nResultCol==1 ); if( pOrderBy ){ pushOntoSorter(pParse, pOrderBy, p, regResult); }else{ sqlite3ExprCodeMove(pParse, regResult, iParm, 1); /* The LIMIT clause will jump out of the loop for us */ @@ -781,21 +773,66 @@ case SRT_Output: { testcase( eDest==SRT_Coroutine ); testcase( eDest==SRT_Output ); if( pOrderBy ){ int r1 = sqlite3GetTempReg(pParse); - sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nColumn, r1); + sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nResultCol, r1); pushOntoSorter(pParse, pOrderBy, p, r1); sqlite3ReleaseTempReg(pParse, r1); }else if( eDest==SRT_Coroutine ){ sqlite3VdbeAddOp1(v, OP_Yield, pDest->iSDParm); }else{ - sqlite3VdbeAddOp2(v, OP_ResultRow, regResult, nColumn); - sqlite3ExprCacheAffinityChange(pParse, regResult, nColumn); + sqlite3VdbeAddOp2(v, OP_ResultRow, regResult, nResultCol); + sqlite3ExprCacheAffinityChange(pParse, regResult, nResultCol); + } + break; + } + +#ifndef SQLITE_OMIT_CTE + /* Write the results into a priority queue that is order according to + ** pDest->pOrderBy (in pSO). pDest->iSDParm (in iParm) is the cursor for an + ** index with pSO->nExpr+2 columns. Build a key using pSO for the first + ** pSO->nExpr columns, then make sure all keys are unique by adding a + ** final OP_Sequence column. The last column is the record as a blob. + */ + case SRT_DistQueue: + case SRT_Queue: { + int nKey; + int r1, r2, r3; + int addrTest = 0; + ExprList *pSO; + pSO = pDest->pOrderBy; + assert( pSO ); + nKey = pSO->nExpr; + r1 = sqlite3GetTempReg(pParse); + r2 = sqlite3GetTempRange(pParse, nKey+2); + r3 = r2+nKey+1; + sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nResultCol, r3); + if( eDest==SRT_DistQueue ){ + /* If the destination is DistQueue, then cursor (iParm+1) is open + ** on a second ephemeral index that holds all values every previously + ** added to the queue. Only add this new value if it has never before + ** been added */ + addrTest = sqlite3VdbeAddOp4Int(v, OP_Found, iParm+1, 0, r3, 0); + sqlite3VdbeAddOp2(v, OP_IdxInsert, iParm+1, r3); + } + for(i=0; ia[i].u.x.iOrderByCol - 1, + r2+i); } + sqlite3VdbeAddOp2(v, OP_Sequence, iParm, r2+nKey); + sqlite3VdbeAddOp3(v, OP_MakeRecord, r2, nKey+2, r1); + sqlite3VdbeAddOp2(v, OP_IdxInsert, iParm, r1); + if( addrTest ) sqlite3VdbeJumpHere(v, addrTest); + sqlite3ReleaseTempReg(pParse, r1); + sqlite3ReleaseTempRange(pParse, r2, nKey+2); break; } +#endif /* SQLITE_OMIT_CTE */ + + #if !defined(SQLITE_OMIT_TRIGGER) /* Discard the results. This is used for SELECT statements inside ** the body of a TRIGGER. The purpose of such selects is to call ** user-defined functions that have side effects. We do not care @@ -881,19 +918,19 @@ ** ** Space to hold the KeyInfo structure is obtain from malloc. The calling ** function is responsible for seeing that this structure is eventually ** freed. */ -static KeyInfo *keyInfoFromExprList(Parse *pParse, ExprList *pList){ +static KeyInfo *keyInfoFromExprList(Parse *pParse, ExprList *pList, int nExtra){ int nExpr; KeyInfo *pInfo; struct ExprList_item *pItem; sqlite3 *db = pParse->db; int i; nExpr = pList->nExpr; - pInfo = sqlite3KeyInfoAlloc(db, nExpr, 1); + pInfo = sqlite3KeyInfoAlloc(db, nExpr+nExtra, 1); if( pInfo ){ assert( sqlite3KeyInfoIsWriteable(pInfo) ); for(i=0, pItem=pList->a; ipExpr); @@ -1030,17 +1067,17 @@ if( p->selFlags & SF_UseSorter ){ int regSortOut = ++pParse->nMem; int ptab2 = pParse->nTab++; sqlite3VdbeAddOp3(v, OP_OpenPseudo, ptab2, regSortOut, pOrderBy->nExpr+2); addr = 1 + sqlite3VdbeAddOp2(v, OP_SorterSort, iTab, addrBreak); - codeOffset(v, p, addrContinue); + codeOffset(v, p->iOffset, addrContinue); sqlite3VdbeAddOp2(v, OP_SorterData, iTab, regSortOut); sqlite3VdbeAddOp3(v, OP_Column, ptab2, pOrderBy->nExpr+1, regRow); sqlite3VdbeChangeP5(v, OPFLAG_CLEARCACHE); }else{ addr = 1 + sqlite3VdbeAddOp2(v, OP_Sort, iTab, addrBreak); - codeOffset(v, p, addrContinue); + codeOffset(v, p->iOffset, addrContinue); sqlite3VdbeAddOp3(v, OP_Column, iTab, pOrderBy->nExpr+1, regRow); } switch( eDest ){ case SRT_Table: case SRT_EphemTab: { @@ -1600,12 +1637,17 @@ ** the limit and offset. If there is no limit and/or offset, then ** iLimit and iOffset are negative. ** ** This routine changes the values of iLimit and iOffset only if ** a limit or offset is defined by pLimit and pOffset. iLimit and -** iOffset should have been preset to appropriate default values -** (usually but not always -1) prior to calling this routine. +** iOffset should have been preset to appropriate default values (zero) +** prior to calling this routine. +** +** The iOffset register (if it exists) is initialized to the value +** of the OFFSET. The iLimit register is initialized to LIMIT. Register +** iOffset+1 is initialized to LIMIT+OFFSET. +** ** Only if pLimit!=0 or pOffset!=0 do the limit registers get ** redefined. The UNION ALL operator uses this property to force ** the reuse of the same limit and offset registers across multiple ** SELECT statements. */ @@ -1625,11 +1667,11 @@ sqlite3ExprCacheClear(pParse); assert( p->pOffset==0 || p->pLimit!=0 ); if( p->pLimit ){ p->iLimit = iLimit = ++pParse->nMem; v = sqlite3GetVdbe(pParse); - if( NEVER(v==0) ) return; /* VDBE should have already been allocated */ + assert( v!=0 ); if( sqlite3ExprIsInteger(p->pLimit, &n) ){ sqlite3VdbeAddOp2(v, OP_Integer, n, iLimit); VdbeComment((v, "LIMIT counter")); if( n==0 ){ sqlite3VdbeAddOp2(v, OP_Goto, 0, iBreak); @@ -1682,11 +1724,169 @@ } return pRet; } #endif /* SQLITE_OMIT_COMPOUND_SELECT */ -/* Forward reference */ +#ifndef SQLITE_OMIT_CTE +/* +** This routine generates VDBE code to compute the content of a WITH RECURSIVE +** query of the form: +** +** AS ( UNION [ALL] ) +** \___________/ \_______________/ +** p->pPrior p +** +** +** There is exactly one reference to the recursive-table in the FROM clause +** of recursive-query, marked with the SrcList->a[].isRecursive flag. +** +** The setup-query runs once to generate an initial set of rows that go +** into a Queue table. Rows are extracted from the Queue table one by +** one. Each row extracted from Queue is output to pDest. Then the single +** extracted row (now in the iCurrent table) becomes the content of the +** recursive-table for a recursive-query run. The output of the recursive-query +** is added back into the Queue table. Then another row is extracted from Queue +** and the iteration continues until the Queue table is empty. +** +** If the compound query operator is UNION then no duplicate rows are ever +** inserted into the Queue table. The iDistinct table keeps a copy of all rows +** that have ever been inserted into Queue and causes duplicates to be +** discarded. If the operator is UNION ALL, then duplicates are allowed. +** +** If the query has an ORDER BY, then entries in the Queue table are kept in +** ORDER BY order and the first entry is extracted for each cycle. Without +** an ORDER BY, the Queue table is just a FIFO. +** +** If a LIMIT clause is provided, then the iteration stops after LIMIT rows +** have been output to pDest. A LIMIT of zero means to output no rows and a +** negative LIMIT means to output all rows. If there is also an OFFSET clause +** with a positive value, then the first OFFSET outputs are discarded rather +** than being sent to pDest. The LIMIT count does not begin until after OFFSET +** rows have been skipped. +*/ +static void generateWithRecursiveQuery( + Parse *pParse, /* Parsing context */ + Select *p, /* The recursive SELECT to be coded */ + SelectDest *pDest /* What to do with query results */ +){ + SrcList *pSrc = p->pSrc; /* The FROM clause of the recursive query */ + int nCol = p->pEList->nExpr; /* Number of columns in the recursive table */ + Vdbe *v = pParse->pVdbe; /* The prepared statement under construction */ + Select *pSetup = p->pPrior; /* The setup query */ + int addrTop; /* Top of the loop */ + int addrCont, addrBreak; /* CONTINUE and BREAK addresses */ + int iCurrent; /* The Current table */ + int regCurrent; /* Register holding Current table */ + int iQueue; /* The Queue table */ + int iDistinct = 0; /* To ensure unique results if UNION */ + int eDest = SRT_Table; /* How to write to Queue */ + SelectDest destQueue; /* SelectDest targetting the Queue table */ + int i; /* Loop counter */ + int rc; /* Result code */ + ExprList *pOrderBy; /* The ORDER BY clause */ + Expr *pLimit, *pOffset; /* Saved LIMIT and OFFSET */ + int regLimit, regOffset; /* Registers used by LIMIT and OFFSET */ + + /* Obtain authorization to do a recursive query */ + if( sqlite3AuthCheck(pParse, SQLITE_RECURSIVE, 0, 0, 0) ) return; + + /* Process the LIMIT and OFFSET clauses, if they exist */ + addrBreak = sqlite3VdbeMakeLabel(v); + computeLimitRegisters(pParse, p, addrBreak); + pLimit = p->pLimit; + pOffset = p->pOffset; + regLimit = p->iLimit; + regOffset = p->iOffset; + p->pLimit = p->pOffset = 0; + p->iLimit = p->iOffset = 0; + + /* Locate the cursor number of the Current table */ + for(i=0; ALWAYS(inSrc); i++){ + if( pSrc->a[i].isRecursive ){ + iCurrent = pSrc->a[i].iCursor; + break; + } + } + + /* Detach the ORDER BY clause from the compound SELECT */ + pOrderBy = p->pOrderBy; + p->pOrderBy = 0; + + /* Allocate cursors numbers for Queue and Distinct. The cursor number for + ** the Distinct table must be exactly one greater than Queue in order + ** for the SRT_DistTable and SRT_DistQueue destinations to work. */ + iQueue = pParse->nTab++; + if( p->op==TK_UNION ){ + eDest = pOrderBy ? SRT_DistQueue : SRT_DistTable; + iDistinct = pParse->nTab++; + }else{ + eDest = pOrderBy ? SRT_Queue : SRT_Table; + } + sqlite3SelectDestInit(&destQueue, eDest, iQueue); + + /* Allocate cursors for Current, Queue, and Distinct. */ + regCurrent = ++pParse->nMem; + sqlite3VdbeAddOp3(v, OP_OpenPseudo, iCurrent, regCurrent, nCol); + if( pOrderBy ){ + KeyInfo *pKeyInfo = keyInfoFromExprList(pParse, pOrderBy, 1); + sqlite3VdbeAddOp4(v, OP_OpenEphemeral, iQueue, pOrderBy->nExpr+2, 0, + (char*)pKeyInfo, P4_KEYINFO); + destQueue.pOrderBy = pOrderBy; + }else{ + sqlite3VdbeAddOp2(v, OP_OpenEphemeral, iQueue, nCol); + } + VdbeComment((v, "Queue table")); + if( iDistinct ){ + p->addrOpenEphm[0] = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, iDistinct, 0); + p->selFlags |= SF_UsesEphemeral; + } + + /* Store the results of the setup-query in Queue. */ + rc = sqlite3Select(pParse, pSetup, &destQueue); + if( rc ) goto end_of_recursive_query; + + /* Find the next row in the Queue and output that row */ + addrTop = sqlite3VdbeAddOp2(v, OP_Rewind, iQueue, addrBreak); + + /* Transfer the next row in Queue over to Current */ + sqlite3VdbeAddOp1(v, OP_NullRow, iCurrent); /* To reset column cache */ + if( pOrderBy ){ + sqlite3VdbeAddOp3(v, OP_Column, iQueue, pOrderBy->nExpr+1, regCurrent); + }else{ + sqlite3VdbeAddOp2(v, OP_RowData, iQueue, regCurrent); + } + sqlite3VdbeAddOp1(v, OP_Delete, iQueue); + + /* Output the single row in Current */ + addrCont = sqlite3VdbeMakeLabel(v); + codeOffset(v, regOffset, addrCont); + selectInnerLoop(pParse, p, p->pEList, iCurrent, + 0, 0, pDest, addrCont, addrBreak); + if( regLimit ) sqlite3VdbeAddOp3(v, OP_IfZero, regLimit, addrBreak, -1); + sqlite3VdbeResolveLabel(v, addrCont); + + /* Execute the recursive SELECT taking the single row in Current as + ** the value for the recursive-table. Store the results in the Queue. + */ + p->pPrior = 0; + sqlite3Select(pParse, p, &destQueue); + assert( p->pPrior==0 ); + p->pPrior = pSetup; + + /* Keep running the loop until the Queue is empty */ + sqlite3VdbeAddOp2(v, OP_Goto, 0, addrTop); + sqlite3VdbeResolveLabel(v, addrBreak); + +end_of_recursive_query: + p->pOrderBy = pOrderBy; + p->pLimit = pLimit; + p->pOffset = pOffset; + return; +} +#endif + +/* Forward references */ static int multiSelectOrderBy( Parse *pParse, /* Parsing context */ Select *p, /* The right-most of SELECTs to be coded */ SelectDest *pDest /* What to do with query results */ ); @@ -1790,85 +1990,11 @@ goto multi_select_end; } #ifndef SQLITE_OMIT_CTE if( p->selFlags & SF_Recursive ){ - SrcList *pSrc = p->pSrc; - int nCol = p->pEList->nExpr; - int addrNext; - int addrSwap; - int iCont, iBreak; - int tmp1; /* Intermediate table */ - int tmp2; /* Next intermediate table */ - int tmp3 = 0; /* To ensure unique results if UNION */ - int eDest = SRT_Table; - SelectDest tmp2dest; - int i; - - /* Check that there is no ORDER BY or LIMIT clause. Neither of these - ** are supported on recursive queries. */ - assert( p->pOffset==0 || p->pLimit ); - if( p->pOrderBy || p->pLimit ){ - sqlite3ErrorMsg(pParse, "%s in a recursive query", - p->pOrderBy ? "ORDER BY" : "LIMIT" - ); - goto multi_select_end; - } - - if( sqlite3AuthCheck(pParse, SQLITE_RECURSIVE, 0, 0, 0) ){ - goto multi_select_end; - } - iBreak = sqlite3VdbeMakeLabel(v); - iCont = sqlite3VdbeMakeLabel(v); - - for(i=0; ALWAYS(inSrc); i++){ - if( pSrc->a[i].isRecursive ){ - tmp1 = pSrc->a[i].iCursor; - break; - } - } - - tmp2 = pParse->nTab++; - if( p->op==TK_UNION ){ - eDest = SRT_DistTable; - tmp3 = pParse->nTab++; - } - sqlite3SelectDestInit(&tmp2dest, eDest, tmp2); - - sqlite3VdbeAddOp2(v, OP_OpenEphemeral, tmp1, nCol); - sqlite3VdbeAddOp2(v, OP_OpenEphemeral, tmp2, nCol); - if( tmp3 ){ - p->addrOpenEphm[0] = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, tmp3, 0); - p->selFlags |= SF_UsesEphemeral; - } - - /* Store the results of the initial SELECT in tmp2. */ - rc = sqlite3Select(pParse, pPrior, &tmp2dest); - if( rc ) goto multi_select_end; - - /* Clear tmp1. Then switch the contents of tmp1 and tmp2. Then return - ** the contents of tmp1 to the caller. Or, if tmp1 is empty at this - ** point, the recursive query has finished - jump to address iBreak. */ - addrSwap = sqlite3VdbeAddOp2(v, OP_SwapCursors, tmp1, tmp2); - sqlite3VdbeAddOp2(v, OP_Rewind, tmp1, iBreak); - addrNext = sqlite3VdbeCurrentAddr(v); - selectInnerLoop(pParse, p, p->pEList, tmp1, p->pEList->nExpr, - 0, 0, &dest, iCont, iBreak); - sqlite3VdbeResolveLabel(v, iCont); - sqlite3VdbeAddOp2(v, OP_Next, tmp1, addrNext); - - /* Execute the recursive SELECT. Store the results in tmp2. While this - ** SELECT is running, the contents of tmp1 are read by recursive - ** references to the current CTE. */ - p->pPrior = 0; - rc = sqlite3Select(pParse, p, &tmp2dest); - assert( p->pPrior==0 ); - p->pPrior = pPrior; - if( rc ) goto multi_select_end; - - sqlite3VdbeAddOp2(v, OP_Goto, 0, addrSwap); - sqlite3VdbeResolveLabel(v, iBreak); + generateWithRecursiveQuery(pParse, p, &dest); }else #endif /* Compound SELECTs that have an ORDER BY clause are handled separately. */ @@ -2007,11 +2133,11 @@ iBreak = sqlite3VdbeMakeLabel(v); iCont = sqlite3VdbeMakeLabel(v); computeLimitRegisters(pParse, p, iBreak); sqlite3VdbeAddOp2(v, OP_Rewind, unionTab, iBreak); iStart = sqlite3VdbeCurrentAddr(v); - selectInnerLoop(pParse, p, p->pEList, unionTab, p->pEList->nExpr, + selectInnerLoop(pParse, p, p->pEList, unionTab, 0, 0, &dest, iCont, iBreak); sqlite3VdbeResolveLabel(v, iCont); sqlite3VdbeAddOp2(v, OP_Next, unionTab, iStart); sqlite3VdbeResolveLabel(v, iBreak); sqlite3VdbeAddOp2(v, OP_Close, unionTab, 0); @@ -2085,11 +2211,11 @@ sqlite3VdbeAddOp2(v, OP_Rewind, tab1, iBreak); r1 = sqlite3GetTempReg(pParse); iStart = sqlite3VdbeAddOp2(v, OP_RowKey, tab1, r1); sqlite3VdbeAddOp4Int(v, OP_NotFound, tab2, iCont, r1, 0); sqlite3ReleaseTempReg(pParse, r1); - selectInnerLoop(pParse, p, p->pEList, tab1, p->pEList->nExpr, + selectInnerLoop(pParse, p, p->pEList, tab1, 0, 0, &dest, iCont, iBreak); sqlite3VdbeResolveLabel(v, iCont); sqlite3VdbeAddOp2(v, OP_Next, tab1, iStart); sqlite3VdbeResolveLabel(v, iBreak); sqlite3VdbeAddOp2(v, OP_Close, tab2, 0); @@ -2207,11 +2333,11 @@ } if( pParse->db->mallocFailed ) return 0; /* Suppress the first OFFSET entries if there is an OFFSET clause */ - codeOffset(v, p, iContinue); + codeOffset(v, p->iOffset, iContinue); switch( pDest->eDest ){ /* Store the result as data using a unique key. */ case SRT_Table: @@ -4153,11 +4279,11 @@ if( pE->x.pList==0 || pE->x.pList->nExpr!=1 ){ sqlite3ErrorMsg(pParse, "DISTINCT aggregates must have exactly one " "argument"); pFunc->iDistinct = -1; }else{ - KeyInfo *pKeyInfo = keyInfoFromExprList(pParse, pE->x.pList); + KeyInfo *pKeyInfo = keyInfoFromExprList(pParse, pE->x.pList, 0); sqlite3VdbeAddOp4(v, OP_OpenEphemeral, pFunc->iDistinct, 0, 0, (char*)pKeyInfo, P4_KEYINFO); } } } @@ -4286,54 +4412,12 @@ #endif /* ** Generate code for the SELECT statement given in the p argument. ** -** The results are distributed in various ways depending on the -** contents of the SelectDest structure pointed to by argument pDest -** as follows: -** -** pDest->eDest Result -** ------------ ------------------------------------------- -** SRT_Output Generate a row of output (using the OP_ResultRow -** opcode) for each row in the result set. -** -** SRT_Mem Only valid if the result is a single column. -** Store the first column of the first result row -** in register pDest->iSDParm then abandon the rest -** of the query. This destination implies "LIMIT 1". -** -** SRT_Set The result must be a single column. Store each -** row of result as the key in table pDest->iSDParm. -** Apply the affinity pDest->affSdst before storing -** results. Used to implement "IN (SELECT ...)". -** -** SRT_Union Store results as a key in a temporary table -** identified by pDest->iSDParm. -** -** SRT_Except Remove results from the temporary table pDest->iSDParm. -** -** SRT_Table Store results in temporary table pDest->iSDParm. -** This is like SRT_EphemTab except that the table -** is assumed to already be open. -** -** SRT_EphemTab Create an temporary table pDest->iSDParm and store -** the result there. The cursor is left open after -** returning. This is like SRT_Table except that -** this destination uses OP_OpenEphemeral to create -** the table first. -** -** SRT_Coroutine Generate a co-routine that returns a new row of -** results each time it is invoked. The entry point -** of the co-routine is stored in register pDest->iSDParm. -** -** SRT_Exists Store a 1 in memory cell pDest->iSDParm if the result -** set is not empty. -** -** SRT_Discard Throw the results away. This is used by SELECT -** statements within triggers whose only purpose is -** the side-effects of functions. +** The results are returned according to the SelectDest structure. +** See comments in sqliteInt.h for further information. ** ** This routine returns the number of errors. If any errors are ** encountered, then an appropriate error message is left in ** pParse->zErrMsg. ** @@ -4604,11 +4688,11 @@ ** we figure out that the sorting index is not needed. The addrSortIndex ** variable is used to facilitate that change. */ if( pOrderBy ){ KeyInfo *pKeyInfo; - pKeyInfo = keyInfoFromExprList(pParse, pOrderBy); + pKeyInfo = keyInfoFromExprList(pParse, pOrderBy, 0); pOrderBy->iECursor = pParse->nTab++; p->addrOpenEphm[2] = addrSortIndex = sqlite3VdbeAddOp4(v, OP_OpenEphemeral, pOrderBy->iECursor, pOrderBy->nExpr+2, 0, (char*)pKeyInfo, P4_KEYINFO); @@ -4636,11 +4720,11 @@ */ if( p->selFlags & SF_Distinct ){ sDistinct.tabTnct = pParse->nTab++; sDistinct.addrTnct = sqlite3VdbeAddOp4(v, OP_OpenEphemeral, sDistinct.tabTnct, 0, 0, - (char*)keyInfoFromExprList(pParse, p->pEList), + (char*)keyInfoFromExprList(pParse, p->pEList, 0), P4_KEYINFO); sqlite3VdbeChangeP5(v, BTREE_UNORDERED); sDistinct.eTnctType = WHERE_DISTINCT_UNORDERED; }else{ sDistinct.eTnctType = WHERE_DISTINCT_NOOP; @@ -4670,11 +4754,11 @@ sqlite3VdbeChangeToNoop(v, addrSortIndex); p->addrOpenEphm[2] = -1; } /* Use the standard inner loop. */ - selectInnerLoop(pParse, p, pEList, 0, 0, pOrderBy, &sDistinct, pDest, + selectInnerLoop(pParse, p, pEList, -1, pOrderBy, &sDistinct, pDest, sqlite3WhereContinueLabel(pWInfo), sqlite3WhereBreakLabel(pWInfo)); /* End the database scan loop. */ @@ -4760,11 +4844,11 @@ ** implement it. Allocate that sorting index now. If it turns out ** that we do not need it after all, the OP_SorterOpen instruction ** will be converted into a Noop. */ sAggInfo.sortingIdx = pParse->nTab++; - pKeyInfo = keyInfoFromExprList(pParse, pGroupBy); + pKeyInfo = keyInfoFromExprList(pParse, pGroupBy, 0); addrSortingIdx = sqlite3VdbeAddOp4(v, OP_SorterOpen, sAggInfo.sortingIdx, sAggInfo.nSortingColumn, 0, (char*)pKeyInfo, P4_KEYINFO); /* Initialize memory locations used by GROUP BY aggregate processing @@ -4942,11 +5026,11 @@ sqlite3VdbeAddOp2(v, OP_IfPos, iUseFlag, addrOutputRow+2); VdbeComment((v, "Groupby result generator entry point")); sqlite3VdbeAddOp1(v, OP_Return, regOutputRow); finalizeAggFunctions(pParse, &sAggInfo); sqlite3ExprIfFalse(pParse, pHaving, addrOutputRow+1, SQLITE_JUMPIFNULL); - selectInnerLoop(pParse, p, p->pEList, 0, 0, pOrderBy, + selectInnerLoop(pParse, p, p->pEList, -1, pOrderBy, &sDistinct, pDest, addrOutputRow+1, addrSetAbort); sqlite3VdbeAddOp1(v, OP_Return, regOutputRow); VdbeComment((v, "end groupby result generator")); @@ -5085,11 +5169,11 @@ finalizeAggFunctions(pParse, &sAggInfo); } pOrderBy = 0; sqlite3ExprIfFalse(pParse, pHaving, addrEnd, SQLITE_JUMPIFNULL); - selectInnerLoop(pParse, p, p->pEList, 0, 0, 0, 0, + selectInnerLoop(pParse, p, p->pEList, -1, 0, 0, pDest, addrEnd, addrEnd); sqlite3ExprListDelete(db, pDel); } sqlite3VdbeResolveLabel(v, addrEnd); Index: src/shell.c ================================================================== --- src/shell.c +++ src/shell.c @@ -1175,11 +1175,11 @@ ** all opcodes that occur between the p2 jump destination and the opcode ** itself by 2 spaces. ** ** * For each "Goto", if the jump destination is earlier in the program ** and ends on one of: -** Yield SeekGt SeekLt RowSetRead +** Yield SeekGt SeekLt RowSetRead Rewind ** then indent all opcodes between the earlier instruction ** and "Goto" by 2 spaces. */ static void explain_data_prepare(struct callback_data *p, sqlite3_stmt *pSql){ const char *zSql; /* The text of the SQL statement */ @@ -1187,11 +1187,11 @@ int *abYield = 0; /* True if op is an OP_Yield */ int nAlloc = 0; /* Allocated size of p->aiIndent[], abYield */ int iOp; /* Index of operation in p->aiIndent[] */ const char *azNext[] = { "Next", "Prev", "VPrev", "VNext", "SorterNext", 0 }; - const char *azYield[] = { "Yield", "SeekLt", "SeekGt", "RowSetRead", 0 }; + const char *azYield[] = { "Yield", "SeekLt", "SeekGt", "RowSetRead", "Rewind", 0 }; const char *azGoto[] = { "Goto", 0 }; /* Try to figure out if this is really an EXPLAIN statement. If this ** cannot be verified, return early. */ zSql = sqlite3_sql(pSql); @@ -1224,11 +1224,11 @@ if( str_in_array(zOp, azNext) ){ for(i=p2op; iaiIndent[i] += 2; } if( str_in_array(zOp, azGoto) && p2opnIndent && abYield[p2op] ){ - for(i=p2op; iaiIndent[i] += 2; + for(i=p2op+1; iaiIndent[i] += 2; } } p->iIndent = 0; sqlite3_free(abYield); Index: src/sqliteInt.h ================================================================== --- src/sqliteInt.h +++ src/sqliteInt.h @@ -2165,12 +2165,70 @@ #define SF_MaybeConvert 0x0400 /* Need convertCompoundSelectToSubquery() */ #define SF_Recursive 0x0800 /* The recursive part of a recursive CTE */ /* -** The results of a select can be distributed in several ways. The -** "SRT" prefix means "SELECT Result Type". +** The results of a SELECT can be distributed in several ways, as defined +** by one of the following macros. The "SRT" prefix means "SELECT Result +** Type". +** +** SRT_Union Store results as a key in a temporary index +** identified by pDest->iSDParm. +** +** SRT_Except Remove results from the temporary index pDest->iSDParm. +** +** SRT_Exists Store a 1 in memory cell pDest->iSDParm if the result +** set is not empty. +** +** SRT_Discard Throw the results away. This is used by SELECT +** statements within triggers whose only purpose is +** the side-effects of functions. +** +** All of the above are free to ignore their ORDER BY clause. Those that +** follow must honor the ORDER BY clause. +** +** SRT_Output Generate a row of output (using the OP_ResultRow +** opcode) for each row in the result set. +** +** SRT_Mem Only valid if the result is a single column. +** Store the first column of the first result row +** in register pDest->iSDParm then abandon the rest +** of the query. This destination implies "LIMIT 1". +** +** SRT_Set The result must be a single column. Store each +** row of result as the key in table pDest->iSDParm. +** Apply the affinity pDest->affSdst before storing +** results. Used to implement "IN (SELECT ...)". +** +** SRT_EphemTab Create an temporary table pDest->iSDParm and store +** the result there. The cursor is left open after +** returning. This is like SRT_Table except that +** this destination uses OP_OpenEphemeral to create +** the table first. +** +** SRT_Coroutine Generate a co-routine that returns a new row of +** results each time it is invoked. The entry point +** of the co-routine is stored in register pDest->iSDParm +** and the result row is stored in pDest->nDest registers +** starting with pDest->iSdst. +** +** SRT_Table Store results in temporary table pDest->iSDParm. +** This is like SRT_EphemTab except that the table +** is assumed to already be open. +** +** SRT_DistTable Store results in a temporary table pDest->iSDParm. +** But also use temporary table pDest->iSDParm+1 as +** a record of all prior results and ignore any duplicate +** rows. Name means: "Distinct Table". +** +** SRT_Queue Store results in priority queue pDest->iSDParm (really +** an index). Append a sequence number so that all entries +** are distinct. +** +** SRT_DistQueue Store results in priority queue pDest->iSDParm only if +** the same record has never been stored before. The +** index at pDest->iSDParm+1 hold all prior stores. */ #define SRT_Union 1 /* Store result as keys in an index */ #define SRT_Except 2 /* Remove result from a UNION index */ #define SRT_Exists 3 /* Store 1 if the result is not empty */ #define SRT_Discard 4 /* Do not save the results anywhere */ @@ -2179,25 +2237,28 @@ #define IgnorableOrderby(X) ((X->eDest)<=SRT_Discard) #define SRT_Output 5 /* Output each row of result */ #define SRT_Mem 6 /* Store result in a memory cell */ #define SRT_Set 7 /* Store results as keys in an index */ -#define SRT_Table 8 /* Store result as data with an automatic rowid */ -#define SRT_EphemTab 9 /* Create transient tab and store like SRT_Table */ -#define SRT_Coroutine 10 /* Generate a single row of result */ -#define SRT_DistTable 11 /* Like SRT_TABLE, but unique results only */ +#define SRT_EphemTab 8 /* Create transient tab and store like SRT_Table */ +#define SRT_Coroutine 9 /* Generate a single row of result */ +#define SRT_Table 10 /* Store result as data with an automatic rowid */ +#define SRT_DistTable 11 /* Like SRT_Table, but unique results only */ +#define SRT_Queue 12 /* Store result in an queue */ +#define SRT_DistQueue 13 /* Like SRT_Queue, but unique results only */ /* ** An instance of this object describes where to put of the results of ** a SELECT statement. */ struct SelectDest { - u8 eDest; /* How to dispose of the results. On of SRT_* above. */ - char affSdst; /* Affinity used when eDest==SRT_Set */ - int iSDParm; /* A parameter used by the eDest disposal method */ - int iSdst; /* Base register where results are written */ - int nSdst; /* Number of registers allocated */ + u8 eDest; /* How to dispose of the results. On of SRT_* above. */ + char affSdst; /* Affinity used when eDest==SRT_Set */ + int iSDParm; /* A parameter used by the eDest disposal method */ + int iSdst; /* Base register where results are written */ + int nSdst; /* Number of registers allocated */ + ExprList *pOrderBy; /* Key columns for SRT_Queue and SRT_DistQueue */ }; /* ** During code generation of statements that do inserts into AUTOINCREMENT ** tables, the following information is attached to the Table.u.autoInc.p Index: src/vdbe.c ================================================================== --- src/vdbe.c +++ src/vdbe.c @@ -3367,37 +3367,10 @@ } pCx->isOrdered = (pOp->p5!=BTREE_UNORDERED); break; } -#ifndef SQLITE_OMIT_CTE -/* Opcode: SwapCursors P1 P2 * * * -** -** Parameters P1 and P2 are both cursors opened by the OpenEphemeral -** opcode. This opcode deletes the contents of epheremal table P1, -** then renames P2 to P1 and P1 to P2. In other words, following this -** opcode cursor P2 is open on an empty table and P1 is open on the -** table that was initially accessed by P2. -*/ -case OP_SwapCursors: { - Mem tmp; - VdbeCursor *pTmp; - - tmp = p->aMem[p->nMem - pOp->p1]; - p->aMem[p->nMem - pOp->p1] = p->aMem[p->nMem - pOp->p2]; - p->aMem[p->nMem - pOp->p2] = tmp; - - pTmp = p->apCsr[pOp->p1]; - p->apCsr[pOp->p1] = p->apCsr[pOp->p2]; - p->apCsr[pOp->p2] = pTmp; - - assert( pTmp->isTable ); - rc = sqlite3BtreeClearTable(pTmp->pBt, MASTER_ROOT, 0); - break; -} -#endif /* ifndef SQLITE_OMIT_CTE */ - /* Opcode: SorterOpen P1 * * P4 * ** ** This opcode works like OP_OpenEphemeral except that it opens ** a transient index that is specifically designed to sort large ** tables using an external merge-sort algorithm. @@ -4391,11 +4364,10 @@ pC = p->apCsr[pOp->p1]; assert( pC!=0 ); pC->nullRow = 1; pC->rowidIsValid = 0; pC->cacheStatus = CACHE_STALE; - assert( pC->pCursor || pC->pVtabCursor ); if( pC->pCursor ){ sqlite3BtreeClearCursor(pC->pCursor); } break; } Index: src/where.c ================================================================== --- src/where.c +++ src/where.c @@ -3408,14 +3408,20 @@ ** scan of the entire table. */ static const u8 aStep[] = { OP_Next, OP_Prev }; static const u8 aStart[] = { OP_Rewind, OP_Last }; assert( bRev==0 || bRev==1 ); - pLevel->op = aStep[bRev]; - pLevel->p1 = iCur; - pLevel->p2 = 1 + sqlite3VdbeAddOp2(v, aStart[bRev], iCur, addrBrk); - pLevel->p5 = SQLITE_STMTSTATUS_FULLSCAN_STEP; + if( pTabItem->isRecursive ){ + /* Tables marked isRecursive have only a single row that is stored in + ** a pseudo-cursor. No need to Rewind or Next such cursors. */ + pLevel->op = OP_Noop; + }else{ + pLevel->op = aStep[bRev]; + pLevel->p1 = iCur; + pLevel->p2 = 1 + sqlite3VdbeAddOp2(v, aStart[bRev], iCur, addrBrk); + pLevel->p5 = SQLITE_STMTSTATUS_FULLSCAN_STEP; + } } /* Insert code to test every subexpression that can be completely ** computed using the current set of tables. */ Index: test/with1.test ================================================================== --- test/with1.test +++ test/with1.test @@ -150,16 +150,64 @@ } {1 2 3 4 5 6 7 8 9 10} do_catchsql_test 5.2 { WITH i(x) AS ( VALUES(1) UNION ALL SELECT x+1 FROM i ORDER BY 1) SELECT x FROM i LIMIT 10; -} {1 {ORDER BY in a recursive query}} +} {0 {1 2 3 4 5 6 7 8 9 10}} + +do_execsql_test 5.2.1 { + CREATE TABLE edge(xfrom, xto, seq, PRIMARY KEY(xfrom, xto)) WITHOUT ROWID; + INSERT INTO edge VALUES(0, 1, 10); + INSERT INTO edge VALUES(1, 2, 20); + INSERT INTO edge VALUES(0, 3, 30); + INSERT INTO edge VALUES(2, 4, 40); + INSERT INTO edge VALUES(3, 4, 40); + INSERT INTO edge VALUES(2, 5, 50); + INSERT INTO edge VALUES(3, 6, 60); + INSERT INTO edge VALUES(5, 7, 70); + INSERT INTO edge VALUES(3, 7, 70); + INSERT INTO edge VALUES(4, 8, 80); + INSERT INTO edge VALUES(7, 8, 80); + INSERT INTO edge VALUES(8, 9, 90); + + WITH RECURSIVE + ancest(id, mtime) AS + (VALUES(0, 0) + UNION + SELECT edge.xto, edge.seq FROM edge, ancest + WHERE edge.xfrom=ancest.id + ORDER BY 2 + ) + SELECT * FROM ancest; +} {0 0 1 10 2 20 3 30 4 40 5 50 6 60 7 70 8 80 9 90} +do_execsql_test 5.2.2 { + WITH RECURSIVE + ancest(id, mtime) AS + (VALUES(0, 0) + UNION ALL + SELECT edge.xto, edge.seq FROM edge, ancest + WHERE edge.xfrom=ancest.id + ORDER BY 2 + ) + SELECT * FROM ancest; +} {0 0 1 10 2 20 3 30 4 40 4 40 5 50 6 60 7 70 7 70 8 80 8 80 8 80 8 80 9 90 9 90 9 90 9 90} +do_execsql_test 5.2.3 { + WITH RECURSIVE + ancest(id, mtime) AS + (VALUES(0, 0) + UNION ALL + SELECT edge.xto, edge.seq FROM edge, ancest + WHERE edge.xfrom=ancest.id + ORDER BY 2 LIMIT 4 OFFSET 2 + ) + SELECT * FROM ancest; +} {2 20 3 30 4 40 4 40} do_catchsql_test 5.3 { - WITH i(x) AS ( VALUES(1) UNION ALL SELECT x+1 FROM i LIMIT 10 ) - SELECT x FROM i LIMIT 10; -} {1 {LIMIT in a recursive query}} + WITH i(x) AS ( VALUES(1) UNION ALL SELECT x+1 FROM i LIMIT 5) + SELECT x FROM i; +} {0 {1 2 3 4 5}} do_execsql_test 5.4 { WITH i(x) AS ( VALUES(1) UNION ALL SELECT (x+1)%10 FROM i) SELECT x FROM i LIMIT 20; } {1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0}