From: ken Date: Sat, 8 Oct 2016 03:28:55 +0000 (-0700) Subject: merge X-Git-Url: https://www.kengrimes.com/gitweb/?p=henge%2Fwebcc.git;a=commitdiff_plain;h=a91ae3357781f8723999dd498bae19d46a4281ac;hp=5119a6c1f3d7a6db4bcb4f60b76bab5a11deab7c merge --- diff --git a/src/apc/lexer.c b/src/apc/lexer.c index 4d3ab0a..6ef1b4c 100644 --- a/src/apc/lexer.c +++ b/src/apc/lexer.c @@ -31,9 +31,9 @@ int lexer_init(void); int lexer(void); void lexer_pushtok(int, YYSTYPE); -extern //ragel +extern //lexer_lex.rl int lexer_lex(const char*); -struct dirent* lexer_direntpa[DE_STACKSIZE]; +struct dirent* lexer_direntpa[DE_STACKSIZE], **lexer_direntpp; /* Private */ extern //scanner.c int scanner_init(void); @@ -45,17 +45,15 @@ extern //bison YYSTYPE yylval; static struct tok -{ union YYSTYPE val; //token val - int tt; //token type +{ YYSTYPE lval; //token val + int tok_t; //token type } token_stack[TK_STACKSIZE]; static union tokp { int* tpt; //token pointer type struct tok* tok; - union YYSTYPE* tvp; //token value pointer + YYSTYPE* tvp; //token value pointer } tks, tkx; -static -struct dirent** dps; /* Directory Entity Array/Stack Simple array for keeping track of dirents yet to be processed by the scanner. @@ -64,7 +62,7 @@ struct dirent** dps; lexer as a stack. */ #define DE_STACK (lexer_direntpa) -#define DE_STACKP (dps) +#define DE_STACKP (lexer_direntpp) #define DE_LEN() (DE_STACKP - DE_STACK) #define DE_INIT() (DE_STACKP = DE_STACK) #define DE_POP() (*--DE_STACKP) @@ -83,7 +81,7 @@ struct dirent** dps; #define TK_STACKPL (tks.tvp) #define TK_STACKX (tkx.tok) #define TK_STACKXI (tkx.tpt) -#define TK_LEN() (TK_STACKP - TK_STACKX) +#define TK_LEN() (TK_STACKX - TK_STACKP) #define TK_INIT() (TK_STACKP = TK_STACKX = TK_STACK) #define TK_POP() (*TK_STACKP++) #define TK_POPI() (*TK_STACKPI++); @@ -102,24 +100,41 @@ int lexer_init /* Lexer If the token buffer is empty, 'lexer' will initialize the token buffer and - call 'lexer_scandir'. If #SCANDIR_ERROR is returned, an error is printed + call 'lexer_scandir'. If SCAN_ERROR is returned, an error is printed before sending a null return to bison. If 0 tokens are generated, the error printing is skipped. In all other cases, 'yylval' is set, and the token's integer representation is returned. */ int lexer +#define $($)#$ #define SCAN_ERROR -1 #define TK_EMPTY (TK_STACKP == TK_STACKX) +#define FAIL(...) \ + do { \ + fprintf(stderr,__VA_ARGS__); \ + goto done; \ + } while (0) () -{ if (TK_EMPTY) - { TK_INIT(); - if (scanner() == 0) - { yylval.val = 0; - return 0; +{start: + while (DE_LEN() > 0) //lex any directory entries in our stack + if (lexer_lex(DE_POP()->d_name) == 0) //fail if it generates no tokens + FAIL("Lexer failed to tokenize [%s]\n",(*DE_STACKP)->d_name); + if (TK_EMPTY) //if there are no tokens, + { TK_INIT(); //initialize the token stack back to 0 + switch (scanner()) + { case SCAN_ERROR: //if an error occurred, + FAIL("Scanner error\n"); + case 0: //if the the scanner finds no dirents, + goto done; //then we are done + default: //if we found some elements to scan, + goto start; //start over and lex them } } yylval = TK_POPL(); return TK_POPI(); + done: + yylval.val = 0; + return 0; } @@ -127,27 +142,13 @@ int lexer This receiver takes a struct tok and pushes it to the FIFO stack. */ void lexer_pushtok -#define S(S)#S //stringifier -#define ERR_TK "Fatal: Generated over " S(TK_STACKSIZE) " tokens in one pass." +#define $($)#$ //stringifier +#define ERR_TK "Fatal: Generated over " $(TK_STACKSIZE) " tokens in one pass." ( int tok, YYSTYPE lval ) { if (TK_LEN() >= TK_STACKSIZE) { fprintf(stderr, ERR_TK); exit(EXIT_FAILURE); } TK_PUSH(tok, lval); + printf("Pushed Token %i | %i\n", TK_STACK[TK_LEN() - 1].tok_t, TK_STACK[TK_LEN() - 1].lval.val); } -/* init_file: - if (lsp != NULL) - while ((c = *lsp++) == *csp) - { switch (c) - { case DELIM: - delimeters_skipped++; - default: - csp++; //delayed to ensure csp is the start of scannable text - break; - } - } - last_string = string; - scan_text: - return scanner_tokenize(csp); -*/ diff --git a/src/apc/lexer_lex.rl b/src/apc/lexer_lex.rl deleted file mode 100644 index d87f47d..0000000 --- a/src/apc/lexer_lex.rl +++ /dev/null @@ -1,123 +0,0 @@ -/* Ragel State Machine for tokenizing text */ -#include -#include -#include - -extern void lexer_pushtok(int, YYSTYPE); - -int lexer_lex(const char*); -int ipow(int, int); -int ttov(const char* str, int); -uint64_t ttor(const char* str, int); -char* ttos(const char* str, int); - - -#define MAX_TOK_LEN 64 -#define MAX_TOKENS 16 -#define MAX_STR_SIZE (MAX_TOK_LEN * MAX_TOKENS) - - -%%{ - machine token_matcher; - - # set up yylval and tok_t to be pushed to stack - action set_ref { - tok_t = REF; \ - yylval.ref = ttor(ts, p-ts); \ - lexer_pushtok(tok_t, yylval); \ - num_tokens++; } - - action set_val { tok_t = NUM; \ - yylval.val = ttov(ts, p-ts); \ - lexer_pushtok(tok_t, yylval); \ - num_tokens++; } - - action set_name { tok_t = NAME; \ - yylval.str = ttos(ts, p-ts); \ - lexer_pushtok(tok_t, yylval); \ - num_tokens++; } - - action set_ts { ts = p; } - - # instantiate machines for each possible token - ref = '0x' xdigit+ %set_ref; - val = digit+ %set_val; - name = alpha+ %set_name; - tok = ref | val | name; - segment = (tok . '_') %set_ts; - - main := segment* . tok; -}%% - - -%%write data; - -/* Scan filename and push the its tokens - onto the stack */ -int lexer_lex (const char* str) -{ - const char *p, *pe, *ts, *eof; - int cs, tok_t, num_tokens; //tok_t == token type - - num_tokens = 0; - - p = ts = str; - pe = p + strlen(str) + 1; - %%write init; - %%write exec; - - - printf (str); - return num_tokens; -} - -int ipow(int base, int exp) -{ - int result = 1; - while (exp) - { - if (exp & 1) - result = result * base; - exp = exp >> 1; - base *= base; - } - - return result; -} - -/* Token to Value */ -int ttov(const char* str, int len) -{ - int i, val = 0; - - for (i = 0; i < len; i++) - { - val += ((str[len - (i + 1)] - '0') * ipow(10,i)); - } - - return val; -} - -uint64_t ttor(const char* str, int len) -{ - int i; - uint64_t num = 0; - - for (i = 0; i < len; i++) - { - num += ((str[len - (i + 1)] - '0') * ipow(10,i)); - } - - return num; -} - -char* ttos(const char* str, int len) -{ - int i; - char token_buf[MAX_TOK_LEN]; - - memmove(token_buf, str, len); - token_buf[len+1] = '\0'; - - return strdup(token_buf); -} diff --git a/src/apc/scanner.c b/src/apc/scanner.c index 580ad12..c669b10 100644 --- a/src/apc/scanner.c +++ b/src/apc/scanner.c @@ -22,8 +22,9 @@ #include "parser.tab.h" /* Public */ -int scanner_init(void); -int scanner(void); +int scanner_init(void); +void scanner_quit(void); +int scanner(void); /* Private */ #ifndef DL_STACKSIZE #define DL_STACKSIZE 64 @@ -38,7 +39,7 @@ void lexer_pushtok(int, int); static int dredge_current_depth(void); extern //lexer.c -struct dirent* lexer_direntpa[]; +struct dirent* lexer_direntpa[], **lexer_direntpp; extern //SRC_DIR/bin/tools/apc.c const char* cargs['Z']; @@ -87,7 +88,15 @@ int scanner_init () { DL_INIT(); DL_STACK[0].dirp = opendir(ROOTDIR); - return !chdir(ROOTDIR) && (DL_STACK[0].dirp == NULL || dredge_current_depth() == 0); + printf("Root dir %s\n",ROOTDIR); + return !chdir(ROOTDIR) && (DL_STACK[0].dirp == NULL || dredge_current_depth() == -1); +} + +/* Quit */ +void scanner_quit +() +{ if (DL_CURDIR()) + closedir(DL_CURDIR()); } /* Scanner @@ -99,9 +108,10 @@ int scanner_init after pushing a new directory to the directory list, the scanner will dredge the directory and alphabetically sort all file entries into the lexer's file array, while placing all subdirectory entries in the current depth's child - directory stack to the scanned later. + directory stack to be scanned later. - Returns the number of elements added to the lexer's file array. + Returns the number of elements added to the lexer's file array, or -1 on + error */ int scanner #define $($)#$ //stringifier @@ -111,8 +121,8 @@ int scanner #define ERR_DEPTH "Fatal: Maximum directory depth of " $(DL_STACKSIZE) \ " exceeded during directory scan\n" #define ERR_DL "Fatal: Directory List Stack Corruption %x\n", DL_LEN() -#define TOK_CLOPEN 0x55, 0 //TODO -#define TOK_CLCLOSE 0x56, 0 //TODO +#define TOK_CLOPEN 0x55, 1 //TODO +#define TOK_CLCLOSE 0x56, 1 //TODO () { struct dirent* direntp; struct DIR* DIRp; @@ -121,30 +131,31 @@ int scanner { fprintf(stderr, ERR_CHILD); goto fail; } - if (DL_CD_LEN() > 0) //There are entities to process at this depth - { if ((direntp = DL_CD_POP()) == NULL) //If the dirent is null, the library - goto libfail; //function in dirent has failed - lexer_lex(direntp->d_name); //lex the directory name - if (DL_LEN() >= DL_STACKSIZE) //fail if maxdepth exceeded + if (DL_CD_LEN() > 0) //There are entities to process + { if ((direntp = DL_CD_POP()) == NULL)//If the dirent is null, the library + goto libfail; //function in dirent has failed + printf("Lexdir %s\n",direntp->d_name); + lexer_lex(direntp->d_name); //lex the directory name + if (DL_LEN() >= DL_STACKSIZE) //fail if maxdepth exceeded { fprintf(stderr, ERR_DEPTH); goto fail; } - if (chdir(direntp->d_name)) //move into the new directory + if (chdir(direntp->d_name)) //move into the new directory goto libfail; DL_PUSH(opendir(CWDSTR)); - if (DL_CURDIR() == NULL) //open the cwd + if (DL_CURDIR() == NULL) //open the cwd goto libfail; - lexer_pushtok(TOK_CLOPEN); //Push "Open Directory" token - return dredge_current_depth(); //Filter and sort the current depth + lexer_pushtok(TOK_CLOPEN); //Push "Open Directory" token + return dredge_current_depth(); //Filter and sort the current depth } - else if (DL_LEN() >= 0) //Any dirs left? (Including root) - { if (closedir(DL_POP())) //close the directory we just left + else if (DL_LEN() >= 0) //Any dirs left? (Including root) + { if (closedir(DL_POP())) //close the directory we just left goto libfail; - lexer_pushtok(TOK_CLCLOSE); //Push "Close Directory" token - if (DL_LEN() == -1) //If we just popped root, we're done - return 0; - if (!chdir("..")) //Move up a directory and start over - goto parse; + if (DL_LEN() == -1) //If we just popped root, + return 0; //we're done + lexer_pushtok(TOK_CLCLOSE); //Else push "Close Directory" token, + if (!chdir("..")) //move up a directory and + goto parse; //start over } fprintf(stderr, ERR_DL); libfail: @@ -168,8 +179,8 @@ static inline int dredge_current_depth #define READDIR_ERROR (-1) #define READDIR_DONE (0) -#define DPS_LEN() (direntpp - lexer_direntpa) -#define DPS_PUSH(E) (*direntpp++ = E) +#define DPS_LEN() (lexer_direntpp - lexer_direntpa) +#define DPS_PUSH(E) (*lexer_direntpp++ = E) () { struct dirent** direntpp = lexer_direntpa; DIR* cwd = DL_CURDIR(); @@ -179,13 +190,11 @@ int dredge_current_depth if ((direntp = readdir(cwd)) != NULL) { switch (direntp->d_type) { case DT_REG: - printf("String to tokenize %s\n", direntp->d_name); DPS_PUSH(direntp); goto scan_next; case DT_DIR: if (*(direntp->d_name) == '.') //skip hidden files and relative dirs goto scan_next; - printf("Pushing child directory %s\n", direntp->d_name); DL_CD_PUSH(direntp); goto scan_next; case DT_UNKNOWN: diff --git a/src/bin/tools/testscanner.c b/src/bin/tools/testscanner.c index 270f1ff..c217faa 100644 --- a/src/bin/tools/testscanner.c +++ b/src/bin/tools/testscanner.c @@ -9,10 +9,20 @@ #include #include //lib errors /* Internal */ +#include + +extern //scanner.c +int scanner(void); extern //scanner.c int scanner_init(void); -extern +extern //scanner.c void scanner_quit(void); +extern //parser.tab.h +YYSTYPE yylval; +extern //lexer.c +int lexer(void); +extern //lexer.c +int lexer_init(void); /* Ansi Term Colors */ #define RED "\x1b[31m" @@ -25,6 +35,10 @@ void scanner_quit(void); int main(void); int test_init(void); +int test_scan(void); +int test_lex(void); + +const char* cargs['Z'] = {0}; int main #define $($)#$ @@ -36,23 +50,46 @@ int main PRINTINFO(T); \ if (U()) \ PRINTFAIL(U); \ - PRINTPASS(U); \ + else \ + PRINTPASS(U); \ } while (0) () -{ RUN_UNIT(test_init,"Initializing\n"); +{ cargs['d'] = "../../src"; + RUN_UNIT(test_scan,"Scanning\n"); return 0; } -int test_init -#define TESTS 50 +int test_scan () -{ static int n = 0; - printf("Init Run %-2i\n",n+1); - if (scanner_init()) - { perror("scanner init"); +{ static int scanned_total = 0; + static int n = 0; + int scanned, lexed; + int i; + if (lexer_init()) + { perror("LIB:"); return -1; } - scanner_quit(); - return (++n < TESTS) ? test_init() : scanner_init(); + loop: + if ((lexed = test_lex()) == 0) + { printf("No tokens to parse\n"); + return 0; + } + else + { printf("Parsed %i Token%c\n", lexed, lexed > 1 ? 's' : ' '); + goto loop; + } + return 0; } +int test_lex +() +{ int tok_val, n = 0; + while ((tok_val = lexer()) || yylval.val) + { if (++n % 9 == 0) + printf("\n"); + printf("[T:%4i|Y:%4i]",tok_val,yylval.val); + yylval.val = 0; + } + printf(";\n"); + return n; +}