Index: src/parse.y ================================================================== --- src/parse.y +++ src/parse.y @@ -214,10 +214,13 @@ QUERY KEY OF OFFSET PRAGMA RAISE RECURSIVE RELEASE REPLACE RESTRICT ROW ROWS ROLLBACK SAVEPOINT TEMP TRIGGER VACUUM VIEW VIRTUAL WITH WITHOUT %ifdef SQLITE_OMIT_COMPOUND_SELECT EXCEPT INTERSECT UNION %endif SQLITE_OMIT_COMPOUND_SELECT +%ifndef SQLITE_OMIT_WINDOWFUNC + CURRENT FOLLOWING PARTITION PRECEDING RANGE UNBOUNDED +%endif SQLITE_OMIT_WINDOWFUNC REINDEX RENAME CTIME_KW IF . %wildcard ANY. // Define operator precedence early so that this is the first occurrence @@ -1040,106 +1043,10 @@ } term(A) ::= CTIME_KW(OP). { A = sqlite3ExprFunction(pParse, 0, &OP); } -%ifndef SQLITE_OMIT_WINDOWFUNC - -%type windowdefn_opt {Window*} -%destructor windowdefn_opt {sqlite3WindowDelete(pParse->db, $$);} -windowdefn_opt(A) ::= . { A = 0; } -windowdefn_opt(A) ::= WINDOW windowdefn_list(B). { A = B; } - -%type windowdefn_list {Window*} -%destructor windowdefn_list {sqlite3WindowDelete(pParse->db, $$);} -windowdefn_list(A) ::= windowdefn(Z). { A = Z; } -windowdefn_list(A) ::= windowdefn_list(Y) COMMA windowdefn(Z). { - if( Z ) Z->pNextWin = Y; - A = Z; -} - -%type windowdefn {Window*} -%destructor windowdefn {sqlite3WindowDelete(pParse->db, $$);} -windowdefn(A) ::= nm(X) AS window(Y). { - if( Y ){ - Y->zName = sqlite3DbStrNDup(pParse->db, X.z, X.n); - } - A = Y; -} - -%type over_opt {Window*} -%destructor over_opt {sqlite3WindowDelete(pParse->db, $$);} - -%type window {Window*} -%destructor window {sqlite3WindowDelete(pParse->db, $$);} - -%type frame_opt {Window*} -%destructor frame_opt {sqlite3WindowDelete(pParse->db, $$);} - -%type window_or_nm {Window*} -%destructor window_or_nm { -sqlite3WindowDelete(pParse->db, $$);} - -%type part_opt {ExprList*} -%destructor part_opt {sqlite3ExprListDelete(pParse->db, $$);} - -%type filter_opt {Expr*} -%destructor filter_opt {sqlite3ExprDelete(pParse->db, $$);} - -%type range_or_rows {int} - -%type frame_bound {struct FrameBound} -%destructor frame_bound {sqlite3ExprDelete(pParse->db, $$.pExpr);} - -over_opt(A) ::= . { A = 0; } -over_opt(A) ::= filter_opt(W) OVER window_or_nm(Z). { - A = Z; - if( A ) A->pFilter = W; -} - -window_or_nm(A) ::= window(Z). {A = Z;} -window_or_nm(A) ::= nm(Z). { - A = (Window*)sqlite3DbMallocZero(pParse->db, sizeof(Window)); - if( A ){ - A->zName = sqlite3DbStrNDup(pParse->db, Z.z, Z.n); - } -} - -window(A) ::= LP part_opt(X) orderby_opt(Y) frame_opt(Z) RP. { - A = Z; - if( A ){ - A->pPartition = X; - A->pOrderBy = Y; - } -} - -part_opt(A) ::= PARTITION BY exprlist(X). { A = X; } -part_opt(A) ::= . { A = 0; } -filter_opt(A) ::= . { A = 0; } -filter_opt(A) ::= FILTER LP WHERE expr(X) RP. { A = X; } - -frame_opt(A) ::= . { - A = sqlite3WindowAlloc(pParse, TK_RANGE, TK_UNBOUNDED, 0, TK_CURRENT, 0); -} -frame_opt(A) ::= range_or_rows(X) frame_bound(Y). { - A = sqlite3WindowAlloc(pParse, X, Y.eType, Y.pExpr, TK_CURRENT, 0); -} -frame_opt(A) ::= range_or_rows(X) BETWEEN frame_bound(Y) AND frame_bound(Z). { - A = sqlite3WindowAlloc(pParse, X, Y.eType, Y.pExpr, Z.eType, Z.pExpr); -} - -range_or_rows(A) ::= RANGE. { A = TK_RANGE; } -range_or_rows(A) ::= ROWS. { A = TK_ROWS; } - -frame_bound(A) ::= UNBOUNDED PRECEDING. { A.eType = TK_UNBOUNDED; A.pExpr = 0; } -frame_bound(A) ::= expr(X) PRECEDING. { A.eType = TK_PRECEDING; A.pExpr = X; } -frame_bound(A) ::= CURRENT ROW. { A.eType = TK_CURRENT ; A.pExpr = 0; } -frame_bound(A) ::= expr(X) FOLLOWING. { A.eType = TK_FOLLOWING; A.pExpr = X; } -frame_bound(A) ::= UNBOUNDED FOLLOWING. { A.eType = TK_UNBOUNDED; A.pExpr = 0; } - -%endif // SQLITE_OMIT_WINDOWFUNC - expr(A) ::= LP nexprlist(X) COMMA expr(Y) RP. { ExprList *pList = sqlite3ExprListAppend(pParse, X, Y); A = sqlite3PExpr(pParse, TK_VECTOR, 0, 0); if( A ){ A->x.pList = pList; @@ -1687,5 +1594,106 @@ } wqlist(A) ::= wqlist(A) COMMA nm(X) eidlist_opt(Y) AS LP select(Z) RP. { A = sqlite3WithAdd(pParse, A, &X, Y, Z); } %endif SQLITE_OMIT_CTE + +//////////////////////// WINDOW FUNCTION EXPRESSIONS ///////////////////////// +// These must be at the end of this file. Specifically, the rules that +// introduce tokens WINDOW, OVER and FILTER must appear last. This causes +// the integer values assigned to these tokens to be larger than all other +// tokens that may be output by the tokenizer except TK_SPACE and TK_ILLEGAL. +// +%ifndef SQLITE_OMIT_WINDOWFUNC +%type windowdefn_list {Window*} +%destructor windowdefn_list {sqlite3WindowDelete(pParse->db, $$);} +windowdefn_list(A) ::= windowdefn(Z). { A = Z; } +windowdefn_list(A) ::= windowdefn_list(Y) COMMA windowdefn(Z). { + if( Z ) Z->pNextWin = Y; + A = Z; +} + +%type windowdefn {Window*} +%destructor windowdefn {sqlite3WindowDelete(pParse->db, $$);} +windowdefn(A) ::= nm(X) AS window(Y). { + if( Y ){ + Y->zName = sqlite3DbStrNDup(pParse->db, X.z, X.n); + } + A = Y; +} + +%type window {Window*} +%destructor window {sqlite3WindowDelete(pParse->db, $$);} + +%type frame_opt {Window*} +%destructor frame_opt {sqlite3WindowDelete(pParse->db, $$);} + +%type window_or_nm {Window*} +%destructor window_or_nm { +sqlite3WindowDelete(pParse->db, $$);} + +%type part_opt {ExprList*} +%destructor part_opt {sqlite3ExprListDelete(pParse->db, $$);} + +%type filter_opt {Expr*} +%destructor filter_opt {sqlite3ExprDelete(pParse->db, $$);} + +%type range_or_rows {int} + +%type frame_bound {struct FrameBound} +%destructor frame_bound {sqlite3ExprDelete(pParse->db, $$.pExpr);} + +window_or_nm(A) ::= window(Z). {A = Z;} +window_or_nm(A) ::= nm(Z). { + A = (Window*)sqlite3DbMallocZero(pParse->db, sizeof(Window)); + if( A ){ + A->zName = sqlite3DbStrNDup(pParse->db, Z.z, Z.n); + } +} + +window(A) ::= LP part_opt(X) orderby_opt(Y) frame_opt(Z) RP. { + A = Z; + if( A ){ + A->pPartition = X; + A->pOrderBy = Y; + } +} + +part_opt(A) ::= PARTITION BY exprlist(X). { A = X; } +part_opt(A) ::= . { A = 0; } + +frame_opt(A) ::= . { + A = sqlite3WindowAlloc(pParse, TK_RANGE, TK_UNBOUNDED, 0, TK_CURRENT, 0); +} +frame_opt(A) ::= range_or_rows(X) frame_bound(Y). { + A = sqlite3WindowAlloc(pParse, X, Y.eType, Y.pExpr, TK_CURRENT, 0); +} +frame_opt(A) ::= range_or_rows(X) BETWEEN frame_bound(Y) AND frame_bound(Z). { + A = sqlite3WindowAlloc(pParse, X, Y.eType, Y.pExpr, Z.eType, Z.pExpr); +} + +range_or_rows(A) ::= RANGE. { A = TK_RANGE; } +range_or_rows(A) ::= ROWS. { A = TK_ROWS; } + +frame_bound(A) ::= UNBOUNDED PRECEDING. { A.eType = TK_UNBOUNDED; A.pExpr = 0; } +frame_bound(A) ::= expr(X) PRECEDING. { A.eType = TK_PRECEDING; A.pExpr = X; } +frame_bound(A) ::= CURRENT ROW. { A.eType = TK_CURRENT ; A.pExpr = 0; } +frame_bound(A) ::= expr(X) FOLLOWING. { A.eType = TK_FOLLOWING; A.pExpr = X; } +frame_bound(A) ::= UNBOUNDED FOLLOWING. { A.eType = TK_UNBOUNDED; A.pExpr = 0; } + +%type windowdefn_opt {Window*} +%destructor windowdefn_opt {sqlite3WindowDelete(pParse->db, $$);} +windowdefn_opt(A) ::= . { A = 0; } +windowdefn_opt(A) ::= WINDOW windowdefn_list(B). { A = B; } + +%type over_opt {Window*} +%destructor over_opt {sqlite3WindowDelete(pParse->db, $$);} +over_opt(A) ::= . { A = 0; } +over_opt(A) ::= filter_opt(W) OVER window_or_nm(Z). { + A = Z; + if( A ) A->pFilter = W; +} + +filter_opt(A) ::= . { A = 0; } +filter_opt(A) ::= FILTER LP WHERE expr(X) RP. { A = X; } +%endif // SQLITE_OMIT_WINDOWFUNC + Index: src/resolve.c ================================================================== --- src/resolve.c +++ src/resolve.c @@ -796,11 +796,15 @@ sqlite3ErrorMsg(pParse,"wrong number of arguments to function %.*s()", nId, zId); pNC->nErr++; } if( is_agg ){ +#ifndef SQLITE_OMIT_WINDOWFUNC pNC->ncFlags &= ~(pExpr->pWin ? NC_AllowWin : NC_AllowAgg); +#else + pNC->ncFlags &= ~NC_AllowAgg; +#endif } sqlite3WalkExprList(pWalker, pList); if( is_agg ){ #ifndef SQLITE_OMIT_WINDOWFUNC if( pExpr->pWin ){ Index: src/sqliteInt.h ================================================================== --- src/sqliteInt.h +++ src/sqliteInt.h @@ -4297,10 +4297,11 @@ #ifndef SQLITE_AMALGAMATION void *sqlite3ParserAlloc(void*(*)(u64), Parse*); void sqlite3ParserFree(void*, void(*)(void*)); #endif void sqlite3Parser(void*, int, Token); +int sqlite3ParserFallback(int); #ifdef YYTRACKMAXSTACKDEPTH int sqlite3ParserStackPeak(void*); #endif void sqlite3AutoLoadExtensions(sqlite3*); Index: src/tokenize.c ================================================================== --- src/tokenize.c +++ src/tokenize.c @@ -186,10 +186,86 @@ /* Make the IdChar function accessible from ctime.c */ #ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS int sqlite3IsIdChar(u8 c){ return IdChar(c); } #endif +#ifndef SQLITE_OMIT_WINDOWFUNC +/* +** Return the id of the next token in string (*pz). Before returning, set +** (*pz) to point to the byte following the parsed token. +*/ +static int getToken(const unsigned char **pz){ + const unsigned char *z = *pz; + int t; /* Token type to return */ + do { + z += sqlite3GetToken(z, &t); + }while( t==TK_SPACE ); + if( t==TK_ID + || t==TK_STRING + || t==TK_JOIN_KW + || t==TK_WINDOW + || t==TK_OVER + || sqlite3ParserFallback(t)==TK_ID + ){ + t = TK_ID; + } + *pz = z; + return t; +} + +/* +** The following three functions are called immediately after the tokenizer +** reads the keywords WINDOW, OVER and FILTER, respectively, to determine +** whether the token should be treated as a keyword or an SQL identifier. +** This cannot be handled by the usual lemon %fallback method, due to +** the ambiguity in some constructions. e.g. +** +** SELECT sum(x) OVER ... +** +** In the above, "OVER" might be a keyword, or it might be an alias for the +** sum(x) expression. If a "%fallback ID OVER" directive were added to +** grammar, then SQLite would always treat "OVER" as an alias, making it +** impossible to call a window-function without a FILTER clause. +** +** WINDOW is treated as a keyword if: +** +** * the following token is an identifier, or a keyword that can fallback +** to being an identifier, and +** * the token after than one is TK_AS. +** +** OVER is a keyword if: +** +** * the previous token was TK_RP, and +** * the next token is either TK_LP or an identifier. +** +** FILTER is a keyword if: +** +** * the previous token was TK_RP, and +** * the next token is TK_LP. +*/ +static int analyzeWindowKeyword(const unsigned char *z){ + int t; + t = getToken(&z); + if( t!=TK_ID ) return TK_ID; + t = getToken(&z); + if( t!=TK_AS ) return TK_ID; + return TK_WINDOW; +} +static int analyzeOverKeyword(const unsigned char *z, int lastToken){ + if( lastToken==TK_RP ){ + int t = getToken(&z); + if( t==TK_LP || t==TK_ID ) return TK_OVER; + } + return TK_ID; +} +static int analyzeFilterKeyword(const unsigned char *z, int lastToken){ + if( lastToken==TK_RP && getToken(&z)==TK_LP ){ + return TK_FILTER; + } + return TK_ID; +} +#endif // SQLITE_OMIT_WINDOWFUNC /* ** Return the length (in bytes) of the token that begins at z[0]. ** Store the token type in *tokenType before returning. */ @@ -507,48 +583,66 @@ assert( pParse->pNewTable==0 ); assert( pParse->pNewTrigger==0 ); assert( pParse->nVar==0 ); assert( pParse->pVList==0 ); while( 1 ){ - if( zSql[0]!=0 ){ - n = sqlite3GetToken((u8*)zSql, &tokenType); - mxSqlLen -= n; - if( mxSqlLen<0 ){ - pParse->rc = SQLITE_TOOBIG; - break; - } - }else{ - /* Upon reaching the end of input, call the parser two more times - ** with tokens TK_SEMI and 0, in that order. */ - if( lastTokenParsed==TK_SEMI ){ - tokenType = 0; - }else if( lastTokenParsed==0 ){ - break; - }else{ - tokenType = TK_SEMI; - } - n = 0; - } + n = sqlite3GetToken((u8*)zSql, &tokenType); + mxSqlLen -= n; + if( mxSqlLen<0 ){ + pParse->rc = SQLITE_TOOBIG; + break; + } +#ifndef SQLITE_OMIT_WINDOWFUNC + if( tokenType>=TK_WINDOW ){ + assert( tokenType==TK_SPACE || tokenType==TK_OVER || tokenType==TK_FILTER + || tokenType==TK_ILLEGAL || tokenType==TK_WINDOW + ); +#else if( tokenType>=TK_SPACE ){ assert( tokenType==TK_SPACE || tokenType==TK_ILLEGAL ); +#endif // SQLITE_OMIT_WINDOWFUNC if( db->u1.isInterrupted ){ pParse->rc = SQLITE_INTERRUPT; break; } - if( tokenType==TK_ILLEGAL ){ + if( tokenType==TK_SPACE ){ + zSql += n; + continue; + } + if( zSql[0]==0 ){ + /* Upon reaching the end of input, call the parser two more times + ** with tokens TK_SEMI and 0, in that order. */ + if( lastTokenParsed==TK_SEMI ){ + tokenType = 0; + }else if( lastTokenParsed==0 ){ + break; + }else{ + tokenType = TK_SEMI; + } + n = 0; +#ifndef SQLITE_OMIT_WINDOWFUNC + }else if( tokenType==TK_WINDOW ){ + assert( n==6 ); + tokenType = analyzeWindowKeyword((const u8*)&zSql[6]); + }else if( tokenType==TK_OVER ){ + assert( n==4 ); + tokenType = analyzeOverKeyword((const u8*)&zSql[4], lastTokenParsed); + }else if( tokenType==TK_FILTER ){ + assert( n==6 ); + tokenType = analyzeFilterKeyword((const u8*)&zSql[6], lastTokenParsed); +#endif // SQLITE_OMIT_WINDOWFUNC + }else{ sqlite3ErrorMsg(pParse, "unrecognized token: \"%.*s\"", n, zSql); break; } - zSql += n; - }else{ - pParse->sLastToken.z = zSql; - pParse->sLastToken.n = n; - sqlite3Parser(pEngine, tokenType, pParse->sLastToken); - lastTokenParsed = tokenType; - zSql += n; - if( pParse->rc!=SQLITE_OK || db->mallocFailed ) break; - } + } + pParse->sLastToken.z = zSql; + pParse->sLastToken.n = n; + sqlite3Parser(pEngine, tokenType, pParse->sLastToken); + lastTokenParsed = tokenType; + zSql += n; + if( pParse->rc!=SQLITE_OK || db->mallocFailed ) break; } assert( nErr==0 ); #ifdef YYTRACKMAXSTACKDEPTH sqlite3_mutex_enter(sqlite3MallocMutex()); sqlite3StatusHighwater(SQLITE_STATUS_PARSER_STACK, ADDED test/window6.test Index: test/window6.test ================================================================== --- /dev/null +++ test/window6.test @@ -0,0 +1,143 @@ +# 2018 May 8 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. Specifically, +# it tests the sqlite3_create_window_function() API. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix window6 + +ifcapable !windowfunc { + finish_test + return +} + +set setup { + CREATE TABLE %t1(%x, %y %typename); + INSERT INTO %t1 VALUES(1, 'a'); + INSERT INTO %t1 VALUES(2, 'b'); + INSERT INTO %t1 VALUES(3, 'c'); + INSERT INTO %t1 VALUES(4, 'd'); + INSERT INTO %t1 VALUES(5, 'e'); +} + +foreach {tn vars} { + 1 {} + 2 { set A(%t1) over } + 3 { set A(%x) over } + 4 { + set A(%alias) over + set A(%x) following + set A(%y) over + } + 5 { + set A(%t1) over + set A(%x) following + set A(%y) preceding + set A(%w) current + set A(%alias) filter + set A(%typename) window + } + + 6 { + set A(%x) window + } +} { + set A(%t1) t1 + set A(%x) x + set A(%y) y + set A(%w) w + set A(%alias) alias + set A(%typename) integer + eval $vars + + set MAP [array get A] + set setup_sql [string map $MAP $setup] + reset_db + execsql $setup_sql + + do_execsql_test 1.$tn.1 [string map $MAP { + SELECT group_concat(%x, '.') OVER (ORDER BY %y) FROM %t1 + }] {1 1.2 1.2.3 1.2.3.4 1.2.3.4.5} + + do_execsql_test 1.$tn.2 [string map $MAP { + SELECT sum(%x) OVER %w FROM %t1 WINDOW %w AS (ORDER BY %y) + }] {1 3 6 10 15} + + do_execsql_test 1.$tn.3 [string map $MAP { + SELECT sum(%alias.%x) OVER %w FROM %t1 %alias WINDOW %w AS (ORDER BY %y) + }] {1 3 6 10 15} + + do_execsql_test 1.$tn.4 [string map $MAP { + SELECT sum(%x) %alias FROM %t1 + }] {15} +} + + +proc winproc {args} { return "window: $args" } +db func window winproc +do_execsql_test 2.0 { + SELECT window('hello world'); +} {{window: {hello world}}} + +proc wincmp {a b} { string compare $b $a } +db collate window wincmp +do_execsql_test 3.0 { + CREATE TABLE window(x COLLATE window); + INSERT INTO window VALUES('bob'), ('alice'), ('cate'); + SELECT * FROM window ORDER BY x COLLATE window; +} {cate bob alice} +do_execsql_test 3.1 { + DROP TABLE window; + CREATE TABLE x1(x); + INSERT INTO x1 VALUES('bob'), ('alice'), ('cate'); + CREATE INDEX window ON x1(x COLLATE window); + SELECT * FROM x1 ORDER BY x COLLATE window; +} {cate bob alice} + + +do_execsql_test 4.0 { CREATE TABLE t4(x, y); } + +# do_execsql_test 4.1 { PRAGMA parser_trace = 1 } +do_execsql_test 4.1 { + SELECT * FROM t4 window, t4; +} + +#------------------------------------------------------------------------- +reset_db + +do_execsql_test 5.0 { + CREATE TABLE over(x, over); + CREATE TABLE window(x, window); + INSERT INTO over VALUES(1, 2), (3, 4), (5, 6); + INSERT INTO window VALUES(1, 2), (3, 4), (5, 6); + SELECT sum(x) over FROM over +} {9} + +do_execsql_test 5.1 { + SELECT sum(x) over over FROM over WINDOW over AS () +} {9 9 9} + +do_execsql_test 5.2 { + SELECT sum(over) over over over FROM over over WINDOW over AS (ORDER BY over) +} {2 6 12} + +do_execsql_test 5.3 { + SELECT sum(over) over over over FROM over over WINDOW over AS (ORDER BY over); +} {2 6 12} + +do_execsql_test 5.4 { + SELECT sum(window) OVER window window FROM window window window window AS (ORDER BY window); +} {2 6 12} + +finish_test + Index: tool/lemon.c ================================================================== --- tool/lemon.c +++ tool/lemon.c @@ -270,10 +270,11 @@ int dtnum; /* The data type number. In the parser, the value ** stack is a union. The .yy%d element of this ** union is the correct data type for this object */ int bContent; /* True if this symbol ever carries content - if ** it is ever more than just syntax */ + int bWeakFallback; /* True for a weak fallback */ /* The following fields are used by MULTITERMINALs only */ int nsubsym; /* Number of constituent symbols in the MULTI */ struct symbol **subsym; /* Array of constituent symbols */ }; @@ -2207,10 +2208,11 @@ RESYNC_AFTER_RULE_ERROR, RESYNC_AFTER_DECL_ERROR, WAITING_FOR_DESTRUCTOR_SYMBOL, WAITING_FOR_DATATYPE_SYMBOL, WAITING_FOR_FALLBACK_ID, + WAITING_FOR_WEAK_FALLBACK_ID, WAITING_FOR_WILDCARD_ID, WAITING_FOR_CLASS_ID, WAITING_FOR_CLASS_TOKEN, WAITING_FOR_TOKEN_NAME }; @@ -2665,10 +2667,16 @@ psp->errorcnt++; psp->state = RESYNC_AFTER_DECL_ERROR; } break; case WAITING_FOR_FALLBACK_ID: + if( x[0]=='?' ){ + psp->state = WAITING_FOR_WEAK_FALLBACK_ID; + break; + } + /* Fall through */ + case WAITING_FOR_WEAK_FALLBACK_ID: if( x[0]=='.' ){ psp->state = WAITING_FOR_DECL_OR_RULE; }else if( !ISUPPER(x[0]) ){ ErrorMsg(psp->filename, psp->tokenlineno, "%%fallback argument \"%s\" should be a token", x); @@ -2681,10 +2689,12 @@ ErrorMsg(psp->filename, psp->tokenlineno, "More than one fallback assigned to token %s", x); psp->errorcnt++; }else{ sp->fallback = psp->fallback; + sp->bWeakFallback = (psp->state==WAITING_FOR_WEAK_FALLBACK_ID); + psp->state = WAITING_FOR_FALLBACK_ID; psp->gp->has_fallback = 1; } } break; case WAITING_FOR_TOKEN_NAME: @@ -4491,10 +4501,13 @@ lemp->tablesize += (mx+1)*szCodeType; for(i=0; i<=mx; i++){ struct symbol *p = lemp->symbols[i]; if( p->fallback==0 ){ fprintf(out, " 0, /* %10s => nothing */\n", p->name); + }else if( p->bWeakFallback ){ + fprintf(out, " %4d, /* %10s => %s (weak) */\n", + -p->fallback->index, p->name, p->fallback->name); }else{ fprintf(out, " %3d, /* %10s => %s */\n", p->fallback->index, p->name, p->fallback->name); } lineno++; @@ -5110,22 +5123,16 @@ sp = Symbol_find(x); if( sp==0 ){ sp = (struct symbol *)calloc(1, sizeof(struct symbol) ); MemoryCheck(sp); + memset(sp, 0, sizeof(*sp)); sp->name = Strsafe(x); sp->type = ISUPPER(*x) ? TERMINAL : NONTERMINAL; - sp->rule = 0; - sp->fallback = 0; sp->prec = -1; sp->assoc = UNK; - sp->firstset = 0; sp->lambda = LEMON_FALSE; - sp->destructor = 0; - sp->destLineno = 0; - sp->datatype = 0; - sp->useCnt = 0; Symbol_insert(sp,sp->name); } sp->useCnt++; return sp; } Index: tool/lempar.c ================================================================== --- tool/lempar.c +++ tool/lempar.c @@ -88,10 +88,11 @@ # define INTERFACE 1 #endif /************* Begin control #defines *****************************************/ %% /************* End control #defines *******************************************/ +#define YY_NLOOKAHEAD ((int)(sizeof(yy_lookahead)/sizeof(yy_lookahead[0]))) /* Define the yytestcase() macro to be a no-op if is not already defined ** otherwise. ** ** Applications can choose to define yytestcase() in the %include section @@ -170,11 +171,11 @@ ** This feature can be used, for example, to cause some keywords in a language ** to revert to identifiers if they keyword does not apply in the context where ** it appears. */ #ifdef YYFALLBACK -static const YYCODETYPE yyFallback[] = { +static const int yyFallback[] = { %% }; #endif /* YYFALLBACK */ /* The following structure represents a single element of the @@ -517,28 +518,38 @@ yycoverage[stateno][iLookAhead] = 1; #endif do{ i = yy_shift_ofst[stateno]; assert( i>=0 ); - assert( i+YYNTOKEN<=(int)sizeof(yy_lookahead)/sizeof(yy_lookahead[0]) ); + /*assert( i+YYNTOKEN<=YY_NLOOKAHEAD );*/ assert( iLookAhead!=YYNOCODE ); assert( iLookAhead < YYNTOKEN ); i += iLookAhead; - if( yy_lookahead[i]!=iLookAhead ){ -#ifdef YYFALLBACK - YYCODETYPE iFallback; /* Fallback token */ - if( iLookAhead %s\n", - yyTracePrompt, yyTokenName[iLookAhead], yyTokenName[iFallback]); - } -#endif - assert( yyFallback[iFallback]==0 ); /* Fallback loop must terminate */ - iLookAhead = iFallback; - continue; + if( i>=YY_NLOOKAHEAD || yy_lookahead[i]!=iLookAhead + ){ +#ifdef YYFALLBACK + int iFallback; /* Fallback token */ + if( iLookAhead0 ){ + /* A strong fallback happens regardless */ +#ifndef NDEBUG + if( yyTraceFILE ){ + fprintf(yyTraceFILE, "%sFALLBACK %s => %s\n", + yyTracePrompt, yyTokenName[iLookAhead], yyTokenName[iFallback]); + } +#endif + assert( yyFallback[iFallback]==0 ); /* Fallback loop must terminate */ + iLookAhead = iFallback; + continue; + } } #endif #ifdef YYWILDCARD { int j = i - iLookAhead + YYWILDCARD; @@ -1055,5 +1066,19 @@ fprintf(yyTraceFILE,"]\n"); } #endif return; } + +/* +** Return the fallback token corresponding to canonical token iToken, or +** 0 if iToken has no fallback. +*/ +int ParseFallback(int iToken){ +#ifdef YYFALLBACK + if( iToken