int lexer_init(void);
int lexer(void);
void lexer_pushtok(int, YYSTYPE);
-extern //ragel
+extern //lexer_lex.rl
int lexer_lex(const char*);
-struct dirent* lexer_direntpa[DE_STACKSIZE];
+struct dirent* lexer_direntpa[DE_STACKSIZE], **lexer_direntpp;
/* Private */
extern //scanner.c
int scanner_init(void);
YYSTYPE yylval;
static
struct tok
-{ union YYSTYPE val; //token val
- int tt; //token type
+{ YYSTYPE lval; //token val
+ int tok_t; //token type
} token_stack[TK_STACKSIZE];
static
union tokp
{ int* tpt; //token pointer type
struct tok* tok;
- union YYSTYPE* tvp; //token value pointer
+ YYSTYPE* tvp; //token value pointer
} tks, tkx;
-static
-struct dirent** dps;
/* Directory Entity Array/Stack
Simple array for keeping track of dirents yet to be processed by the scanner.
lexer as a stack.
*/
#define DE_STACK (lexer_direntpa)
-#define DE_STACKP (dps)
+#define DE_STACKP (lexer_direntpp)
#define DE_LEN() (DE_STACKP - DE_STACK)
#define DE_INIT() (DE_STACKP = DE_STACK)
#define DE_POP() (*--DE_STACKP)
#define TK_STACKPL (tks.tvp)
#define TK_STACKX (tkx.tok)
#define TK_STACKXI (tkx.tpt)
-#define TK_LEN() (TK_STACKP - TK_STACKX)
+#define TK_LEN() (TK_STACKX - TK_STACKP)
#define TK_INIT() (TK_STACKP = TK_STACKX = TK_STACK)
#define TK_POP() (*TK_STACKP++)
#define TK_POPI() (*TK_STACKPI++);
/* Lexer
If the token buffer is empty, 'lexer' will initialize the token buffer and
- call 'lexer_scandir'. If #SCANDIR_ERROR is returned, an error is printed
+ call 'lexer_scandir'. If SCAN_ERROR is returned, an error is printed
before sending a null return to bison. If 0 tokens are generated, the error
printing is skipped. In all other cases, 'yylval' is set, and the token's
integer representation is returned.
*/
int lexer
+#define $($)#$
#define SCAN_ERROR -1
#define TK_EMPTY (TK_STACKP == TK_STACKX)
+#define FAIL(...) \
+ do { \
+ fprintf(stderr,__VA_ARGS__); \
+ goto done; \
+ } while (0)
()
-{ if (TK_EMPTY)
- { TK_INIT();
- if (scanner() == 0)
- { yylval.val = 0;
- return 0;
+{start:
+ while (DE_LEN() > 0) //lex any directory entries in our stack
+ if (lexer_lex(DE_POP()->d_name) == 0) //fail if it generates no tokens
+ FAIL("Lexer failed to tokenize [%s]\n",(*DE_STACKP)->d_name);
+ if (TK_EMPTY) //if there are no tokens,
+ { TK_INIT(); //initialize the token stack back to 0
+ switch (scanner())
+ { case SCAN_ERROR: //if an error occurred,
+ FAIL("Scanner error\n");
+ case 0: //if the the scanner finds no dirents,
+ goto done; //then we are done
+ default: //if we found some elements to scan,
+ goto start; //start over and lex them
}
}
yylval = TK_POPL();
return TK_POPI();
+ done:
+ yylval.val = 0;
+ return 0;
}
This receiver takes a struct tok and pushes it to the FIFO stack.
*/
void lexer_pushtok
-#define S(S)#S //stringifier
-#define ERR_TK "Fatal: Generated over " S(TK_STACKSIZE) " tokens in one pass."
+#define $($)#$ //stringifier
+#define ERR_TK "Fatal: Generated over " $(TK_STACKSIZE) " tokens in one pass."
( int tok, YYSTYPE lval )
{ if (TK_LEN() >= TK_STACKSIZE)
{ fprintf(stderr, ERR_TK);
exit(EXIT_FAILURE);
}
TK_PUSH(tok, lval);
+ printf("Pushed Token %i | %i\n", TK_STACK[TK_LEN() - 1].tok_t, TK_STACK[TK_LEN() - 1].lval.val);
}
-/* init_file:
- if (lsp != NULL)
- while ((c = *lsp++) == *csp)
- { switch (c)
- { case DELIM:
- delimeters_skipped++;
- default:
- csp++; //delayed to ensure csp is the start of scannable text
- break;
- }
- }
- last_string = string;
- scan_text:
- return scanner_tokenize(csp);
-*/
+++ /dev/null
-/* Ragel State Machine for tokenizing text */
-#include <stdio.h>
-#include <string.h>
-#include <apc/parser.tab.h>
-
-extern void lexer_pushtok(int, YYSTYPE);
-
-int lexer_lex(const char*);
-int ipow(int, int);
-int ttov(const char* str, int);
-uint64_t ttor(const char* str, int);
-char* ttos(const char* str, int);
-
-
-#define MAX_TOK_LEN 64
-#define MAX_TOKENS 16
-#define MAX_STR_SIZE (MAX_TOK_LEN * MAX_TOKENS)
-
-
-%%{
- machine token_matcher;
-
- # set up yylval and tok_t to be pushed to stack
- action set_ref {
- tok_t = REF; \
- yylval.ref = ttor(ts, p-ts); \
- lexer_pushtok(tok_t, yylval); \
- num_tokens++; }
-
- action set_val { tok_t = NUM; \
- yylval.val = ttov(ts, p-ts); \
- lexer_pushtok(tok_t, yylval); \
- num_tokens++; }
-
- action set_name { tok_t = NAME; \
- yylval.str = ttos(ts, p-ts); \
- lexer_pushtok(tok_t, yylval); \
- num_tokens++; }
-
- action set_ts { ts = p; }
-
- # instantiate machines for each possible token
- ref = '0x' xdigit+ %set_ref;
- val = digit+ %set_val;
- name = alpha+ %set_name;
- tok = ref | val | name;
- segment = (tok . '_') %set_ts;
-
- main := segment* . tok;
-}%%
-
-
-%%write data;
-
-/* Scan filename and push the its tokens
- onto the stack */
-int lexer_lex (const char* str)
-{
- const char *p, *pe, *ts, *eof;
- int cs, tok_t, num_tokens; //tok_t == token type
-
- num_tokens = 0;
-
- p = ts = str;
- pe = p + strlen(str) + 1;
- %%write init;
- %%write exec;
-
-
- printf (str);
- return num_tokens;
-}
-
-int ipow(int base, int exp)
-{
- int result = 1;
- while (exp)
- {
- if (exp & 1)
- result = result * base;
- exp = exp >> 1;
- base *= base;
- }
-
- return result;
-}
-
-/* Token to Value */
-int ttov(const char* str, int len)
-{
- int i, val = 0;
-
- for (i = 0; i < len; i++)
- {
- val += ((str[len - (i + 1)] - '0') * ipow(10,i));
- }
-
- return val;
-}
-
-uint64_t ttor(const char* str, int len)
-{
- int i;
- uint64_t num = 0;
-
- for (i = 0; i < len; i++)
- {
- num += ((str[len - (i + 1)] - '0') * ipow(10,i));
- }
-
- return num;
-}
-
-char* ttos(const char* str, int len)
-{
- int i;
- char token_buf[MAX_TOK_LEN];
-
- memmove(token_buf, str, len);
- token_buf[len+1] = '\0';
-
- return strdup(token_buf);
-}
#include "parser.tab.h"
/* Public */
-int scanner_init(void);
-int scanner(void);
+int scanner_init(void);
+void scanner_quit(void);
+int scanner(void);
/* Private */
#ifndef DL_STACKSIZE
#define DL_STACKSIZE 64
static
int dredge_current_depth(void);
extern //lexer.c
-struct dirent* lexer_direntpa[];
+struct dirent* lexer_direntpa[], **lexer_direntpp;
extern //SRC_DIR/bin/tools/apc.c
const char* cargs['Z'];
()
{ DL_INIT();
DL_STACK[0].dirp = opendir(ROOTDIR);
- return !chdir(ROOTDIR) && (DL_STACK[0].dirp == NULL || dredge_current_depth() == 0);
+ printf("Root dir %s\n",ROOTDIR);
+ return !chdir(ROOTDIR) && (DL_STACK[0].dirp == NULL || dredge_current_depth() == -1);
+}
+
+/* Quit */
+void scanner_quit
+()
+{ if (DL_CURDIR())
+ closedir(DL_CURDIR());
}
/* Scanner
after pushing a new directory to the directory list, the scanner will dredge
the directory and alphabetically sort all file entries into the lexer's file
array, while placing all subdirectory entries in the current depth's child
- directory stack to the scanned later.
+ directory stack to be scanned later.
- Returns the number of elements added to the lexer's file array.
+ Returns the number of elements added to the lexer's file array, or -1 on
+ error
*/
int scanner
#define $($)#$ //stringifier
#define ERR_DEPTH "Fatal: Maximum directory depth of " $(DL_STACKSIZE) \
" exceeded during directory scan\n"
#define ERR_DL "Fatal: Directory List Stack Corruption %x\n", DL_LEN()
-#define TOK_CLOPEN 0x55, 0 //TODO
-#define TOK_CLCLOSE 0x56, 0 //TODO
+#define TOK_CLOPEN 0x55, 1 //TODO
+#define TOK_CLCLOSE 0x56, 1 //TODO
()
{ struct dirent* direntp;
struct DIR* DIRp;
{ fprintf(stderr, ERR_CHILD);
goto fail;
}
- if (DL_CD_LEN() > 0) //There are entities to process at this depth
- { if ((direntp = DL_CD_POP()) == NULL) //If the dirent is null, the library
- goto libfail; //function in dirent has failed
- lexer_lex(direntp->d_name); //lex the directory name
- if (DL_LEN() >= DL_STACKSIZE) //fail if maxdepth exceeded
+ if (DL_CD_LEN() > 0) //There are entities to process
+ { if ((direntp = DL_CD_POP()) == NULL)//If the dirent is null, the library
+ goto libfail; //function in dirent has failed
+ printf("Lexdir %s\n",direntp->d_name);
+ lexer_lex(direntp->d_name); //lex the directory name
+ if (DL_LEN() >= DL_STACKSIZE) //fail if maxdepth exceeded
{ fprintf(stderr, ERR_DEPTH);
goto fail;
}
- if (chdir(direntp->d_name)) //move into the new directory
+ if (chdir(direntp->d_name)) //move into the new directory
goto libfail;
DL_PUSH(opendir(CWDSTR));
- if (DL_CURDIR() == NULL) //open the cwd
+ if (DL_CURDIR() == NULL) //open the cwd
goto libfail;
- lexer_pushtok(TOK_CLOPEN); //Push "Open Directory" token
- return dredge_current_depth(); //Filter and sort the current depth
+ lexer_pushtok(TOK_CLOPEN); //Push "Open Directory" token
+ return dredge_current_depth(); //Filter and sort the current depth
}
- else if (DL_LEN() >= 0) //Any dirs left? (Including root)
- { if (closedir(DL_POP())) //close the directory we just left
+ else if (DL_LEN() >= 0) //Any dirs left? (Including root)
+ { if (closedir(DL_POP())) //close the directory we just left
goto libfail;
- lexer_pushtok(TOK_CLCLOSE); //Push "Close Directory" token
- if (DL_LEN() == -1) //If we just popped root, we're done
- return 0;
- if (!chdir("..")) //Move up a directory and start over
- goto parse;
+ if (DL_LEN() == -1) //If we just popped root,
+ return 0; //we're done
+ lexer_pushtok(TOK_CLCLOSE); //Else push "Close Directory" token,
+ if (!chdir("..")) //move up a directory and
+ goto parse; //start over
}
fprintf(stderr, ERR_DL);
libfail:
int dredge_current_depth
#define READDIR_ERROR (-1)
#define READDIR_DONE (0)
-#define DPS_LEN() (direntpp - lexer_direntpa)
-#define DPS_PUSH(E) (*direntpp++ = E)
+#define DPS_LEN() (lexer_direntpp - lexer_direntpa)
+#define DPS_PUSH(E) (*lexer_direntpp++ = E)
()
{ struct dirent** direntpp = lexer_direntpa;
DIR* cwd = DL_CURDIR();
if ((direntp = readdir(cwd)) != NULL)
{ switch (direntp->d_type)
{ case DT_REG:
- printf("String to tokenize %s\n", direntp->d_name);
DPS_PUSH(direntp);
goto scan_next;
case DT_DIR:
if (*(direntp->d_name) == '.') //skip hidden files and relative dirs
goto scan_next;
- printf("Pushing child directory %s\n", direntp->d_name);
DL_CD_PUSH(direntp);
goto scan_next;
case DT_UNKNOWN:
#include <stdlib.h>
#include <errno.h> //lib errors
/* Internal */
+#include <apc/parser.tab.h>
+
+extern //scanner.c
+int scanner(void);
extern //scanner.c
int scanner_init(void);
-extern
+extern //scanner.c
void scanner_quit(void);
+extern //parser.tab.h
+YYSTYPE yylval;
+extern //lexer.c
+int lexer(void);
+extern //lexer.c
+int lexer_init(void);
/* Ansi Term Colors */
#define RED "\x1b[31m"
int main(void);
int test_init(void);
+int test_scan(void);
+int test_lex(void);
+
+const char* cargs['Z'] = {0};
int main
#define $($)#$
PRINTINFO(T); \
if (U()) \
PRINTFAIL(U); \
- PRINTPASS(U); \
+ else \
+ PRINTPASS(U); \
} while (0)
()
-{ RUN_UNIT(test_init,"Initializing\n");
+{ cargs['d'] = "../../src";
+ RUN_UNIT(test_scan,"Scanning\n");
return 0;
}
-int test_init
-#define TESTS 50
+int test_scan
()
-{ static int n = 0;
- printf("Init Run %-2i\n",n+1);
- if (scanner_init())
- { perror("scanner init");
+{ static int scanned_total = 0;
+ static int n = 0;
+ int scanned, lexed;
+ int i;
+ if (lexer_init())
+ { perror("LIB:");
return -1;
}
- scanner_quit();
- return (++n < TESTS) ? test_init() : scanner_init();
+ loop:
+ if ((lexed = test_lex()) == 0)
+ { printf("No tokens to parse\n");
+ return 0;
+ }
+ else
+ { printf("Parsed %i Token%c\n", lexed, lexed > 1 ? 's' : ' ');
+ goto loop;
+ }
+ return 0;
}
+int test_lex
+()
+{ int tok_val, n = 0;
+ while ((tok_val = lexer()) || yylval.val)
+ { if (++n % 9 == 0)
+ printf("\n");
+ printf("[T:%4i|Y:%4i]",tok_val,yylval.val);
+ yylval.val = 0;
+ }
+ printf(";\n");
+ return n;
+}
projects / henge / webcc.git / commitdiff