From 03efe0c43855ff6af7eb2194f6d4a5f022bc415d Mon Sep 17 00:00:00 2001 From: ken Date: Fri, 7 Oct 2016 20:25:54 -0700 Subject: [PATCH] fixes --- src/apc/lexer.c | 65 ++++++++++++++++++++++---------------------- src/apc/lexer_lex.rl | 37 +++++++++++-------------- 2 files changed, 49 insertions(+), 53 deletions(-) diff --git a/src/apc/lexer.c b/src/apc/lexer.c index 4d3ab0a..6ef1b4c 100644 --- a/src/apc/lexer.c +++ b/src/apc/lexer.c @@ -31,9 +31,9 @@ int lexer_init(void); int lexer(void); void lexer_pushtok(int, YYSTYPE); -extern //ragel +extern //lexer_lex.rl int lexer_lex(const char*); -struct dirent* lexer_direntpa[DE_STACKSIZE]; +struct dirent* lexer_direntpa[DE_STACKSIZE], **lexer_direntpp; /* Private */ extern //scanner.c int scanner_init(void); @@ -45,17 +45,15 @@ extern //bison YYSTYPE yylval; static struct tok -{ union YYSTYPE val; //token val - int tt; //token type +{ YYSTYPE lval; //token val + int tok_t; //token type } token_stack[TK_STACKSIZE]; static union tokp { int* tpt; //token pointer type struct tok* tok; - union YYSTYPE* tvp; //token value pointer + YYSTYPE* tvp; //token value pointer } tks, tkx; -static -struct dirent** dps; /* Directory Entity Array/Stack Simple array for keeping track of dirents yet to be processed by the scanner. @@ -64,7 +62,7 @@ struct dirent** dps; lexer as a stack. */ #define DE_STACK (lexer_direntpa) -#define DE_STACKP (dps) +#define DE_STACKP (lexer_direntpp) #define DE_LEN() (DE_STACKP - DE_STACK) #define DE_INIT() (DE_STACKP = DE_STACK) #define DE_POP() (*--DE_STACKP) @@ -83,7 +81,7 @@ struct dirent** dps; #define TK_STACKPL (tks.tvp) #define TK_STACKX (tkx.tok) #define TK_STACKXI (tkx.tpt) -#define TK_LEN() (TK_STACKP - TK_STACKX) +#define TK_LEN() (TK_STACKX - TK_STACKP) #define TK_INIT() (TK_STACKP = TK_STACKX = TK_STACK) #define TK_POP() (*TK_STACKP++) #define TK_POPI() (*TK_STACKPI++); @@ -102,24 +100,41 @@ int lexer_init /* Lexer If the token buffer is empty, 'lexer' will initialize the token buffer and - call 'lexer_scandir'. If #SCANDIR_ERROR is returned, an error is printed + call 'lexer_scandir'. If SCAN_ERROR is returned, an error is printed before sending a null return to bison. If 0 tokens are generated, the error printing is skipped. In all other cases, 'yylval' is set, and the token's integer representation is returned. */ int lexer +#define $($)#$ #define SCAN_ERROR -1 #define TK_EMPTY (TK_STACKP == TK_STACKX) +#define FAIL(...) \ + do { \ + fprintf(stderr,__VA_ARGS__); \ + goto done; \ + } while (0) () -{ if (TK_EMPTY) - { TK_INIT(); - if (scanner() == 0) - { yylval.val = 0; - return 0; +{start: + while (DE_LEN() > 0) //lex any directory entries in our stack + if (lexer_lex(DE_POP()->d_name) == 0) //fail if it generates no tokens + FAIL("Lexer failed to tokenize [%s]\n",(*DE_STACKP)->d_name); + if (TK_EMPTY) //if there are no tokens, + { TK_INIT(); //initialize the token stack back to 0 + switch (scanner()) + { case SCAN_ERROR: //if an error occurred, + FAIL("Scanner error\n"); + case 0: //if the the scanner finds no dirents, + goto done; //then we are done + default: //if we found some elements to scan, + goto start; //start over and lex them } } yylval = TK_POPL(); return TK_POPI(); + done: + yylval.val = 0; + return 0; } @@ -127,27 +142,13 @@ int lexer This receiver takes a struct tok and pushes it to the FIFO stack. */ void lexer_pushtok -#define S(S)#S //stringifier -#define ERR_TK "Fatal: Generated over " S(TK_STACKSIZE) " tokens in one pass." +#define $($)#$ //stringifier +#define ERR_TK "Fatal: Generated over " $(TK_STACKSIZE) " tokens in one pass." ( int tok, YYSTYPE lval ) { if (TK_LEN() >= TK_STACKSIZE) { fprintf(stderr, ERR_TK); exit(EXIT_FAILURE); } TK_PUSH(tok, lval); + printf("Pushed Token %i | %i\n", TK_STACK[TK_LEN() - 1].tok_t, TK_STACK[TK_LEN() - 1].lval.val); } -/* init_file: - if (lsp != NULL) - while ((c = *lsp++) == *csp) - { switch (c) - { case DELIM: - delimeters_skipped++; - default: - csp++; //delayed to ensure csp is the start of scannable text - break; - } - } - last_string = string; - scan_text: - return scanner_tokenize(csp); -*/ diff --git a/src/apc/lexer_lex.rl b/src/apc/lexer_lex.rl index d27c40b..4c9aeea 100644 --- a/src/apc/lexer_lex.rl +++ b/src/apc/lexer_lex.rl @@ -15,25 +15,23 @@ char* ttos(const char* str, int); #define MAX_TOK_LEN 64 #define MAX_TOKENS 16 #define MAX_STR_SIZE (MAX_TOK_LEN * MAX_TOKENS) - +#define $($)#$ +#define PUSHTOK(TOK,LFUNC,UTYPE) \ + do { \ + printf("PUSHTOK(" $(TOK) $(LFUNC) $(UTYPE) ")\n"); \ + tok_t = TOK; \ + yylval.UTYPE = LFUNC(ts, p-ts); \ + lexer_pushtok(tok_t, yylval); \ + ++ntok; \ + } while (0) %%{ machine token_matcher; # set up yylval and tok_t to be pushed to stack - action set_ref { - tok_t = REF; \ - yylval.ref = ttor(ts, p-ts); \ - lexer_pushtok(tok_t, yylval); } - - action set_val { tok_t = NUM; \ - yylval.val = ttov(ts, p-ts); \ - lexer_pushtok(tok_t, yylval); } - - action set_name { tok_t = NAME; \ - yylval.str = ttos(ts, p-ts); \ - lexer_pushtok(tok_t, yylval); } - + action set_ref { PUSHTOK(REF, ttor, ref); } + action set_val { PUSHTOK(NUM, ttov, val); } + action set_name { PUSHTOK(NAME, ttos, str); } action set_ts { ts = p; } # instantiate machines for each possible token @@ -54,17 +52,14 @@ char* ttos(const char* str, int); int lexer_lex (const char* str) { const char *p, *pe, *ts, *eof; - int cs, tok_t ; //tok_t == token type - + int cs, tok_t, ntok = 0; + printf ("Lexing: %s\n",str); p = ts = str; pe = p + strlen(str) + 1; %%write init; %%write exec; - - lexer_pushtok(tok_t, yylval); - - printf (str); - return 1; + printf ("Lexed %i tokens\n",ntok); + return ntok; } int ipow(int base, int exp) -- 2.18.0