2 \brief lexical analyzer implementation for APC
3 \details The lexer manages two FIFO stacks. One for maintaining tokens, the
4 other for maintaining a list of files to be scanned. During
5 execution, the lexer will return a token from its token queue if any
6 are present. If not, the lexer will will pop an element from its
7 file queue to 'scanner' to be tokenized. If the file queue is empty,
8 the lexer will instead call 'parsedir' to traverse the directory tree
9 and tokenize the results. If 'parsedir' does not generate any new
11 \author Jordan Lavatai
13 ----------------------------------------------------------------------------*/
27 #include <limits.h> //realpath, NAME_MAX, PATH_MAX
29 /* Redefinitions of NAME_MAX and PATH_MAX */
30 //#define NAME_MAX NAME_MAX/4
31 //#define PATH_MAX PATH_MAX/4
34 #include "parser.tab.h"
36 #define DE_STACKSIZE 1024
39 #define TK_STACKSIZE 1024
41 #ifndef MAX_SETNAME_LEN //max setname length
42 #define MAX_SETNAME_LEN 32
48 int lexer_lexfile(const uint8_t*);
49 void lexer_pushtok(int, YYSTYPE
);
50 uint8_t const* lexer_get_current_filepath(void);
51 int lexer_lexfilename(uint8_t*);
52 struct dirent
* lexer_direntpa
[DE_STACKSIZE
],** lexer_direntpp
,** lexer_direntpb
;
55 int lexer_lexstring(uint8_t*, int);
57 int scanner_init(void);
61 int dredge_current_depth(void);
65 uint8_t const* current_filename
;
68 { YYSTYPE lval
; //token val
69 int tok_t
; //token type
70 } token_stack
[TK_STACKSIZE
], *tsp
, *tsx
;
72 /* Directory Entity Array/Stack
73 Simple array for keeping track of dirents yet to be processed by the scanner.
74 If this list is empty and there are no tokens, the lexer is done.
75 This array is populated by the scanner as an array, and popped locally by the
76 lexer as a stack, and is popped as a FIFO stack.
78 #define DE_STACK (lexer_direntpa)
79 #define DE_STACKP (lexer_direntpp)
80 #define DE_STACKB (lexer_direntpb)
81 #define DE_LEN() (DE_STACKP - DE_STACKB)
82 #define DE_INIT() (DE_STACKP = DE_STACKB = DE_STACK)
83 #define DE_POP() (*DE_STACKB++)
86 This is a FIFO stack whose pointers are a union of either a pointer to an
87 integer, or a pointer to two integers (a struct tok). This way, integers may
88 be added or removed from the stack either singularly (IPUSH/IPOP), or as a
89 full token of two integers (PUSH/POP).
90 An alignment error will occur if IPOP or IPUSH are used a non-even number of
93 #define TK_STACK (token_stack)
94 #define TK_STACKP (tsp)
95 #define TK_STACKX (tsx)
96 #define TK_LEN() (TK_STACKX - TK_STACKP)
97 #define TK_INIT() (TK_STACKP = TK_STACKX = TK_STACK)
98 #define TK_POP() (*TK_STACKP++)
99 #define TK_PUSH(T,L) (*TK_STACKX++ = (struct tok){L,T})
102 The initializer returns boolean true if an error occurs, which may be handled
109 return scanner_init();
113 If the token buffer is empty, 'lexer' will initialize the token buffer and
114 call 'lexer_scandir'. If SCAN_ERROR is returned, an error is printed
115 before sending a null return to bison. If 0 tokens are generated, the error
116 printing is skipped. In all other cases, 'yylval' is set, and the token's
117 integer representation is returned.
121 #define SCAN_ERROR -1
122 #define TK_EMPTY (TK_STACKP == TK_STACKX)
125 fprintf(stderr,__VA_ARGS__); \
131 while (DE_LEN() > 0)//lex any directory entries in our stack
133 if (lexer_lexfile(DE_POP()->d_name
) == 0)
134 FAIL("Lexer failed to tokenize [%s]\n",(*DE_STACKB
)->d_name
);
136 if (TK_EMPTY
) //if there are no tokens,
137 { TK_INIT(); //initialize the token stack back to 0
139 { case SCAN_ERROR
: //if an error occurred,
140 FAIL("Scanner error\n");
141 case 0: //if the the scanner finds no dirents,
142 goto done
; //then we are done
143 default: //if we found some elements to scan,
144 goto start
; //start over and lex them
157 This receiver takes a struct tok and pushes it to the FIFO stack.
160 #define $($)#$ //stringifier
161 #define ERR_TK "Fatal: Generated over " $(TK_STACKSIZE) " tokens in one pass."
162 ( int tok
, YYSTYPE lval
)
163 { if (TK_LEN() >= TK_STACKSIZE
)
164 { fprintf(stderr
, ERR_TK
);
170 /* Lexical analysis of a file
171 Strips a filename to its base name, then sends it to lexer_lex
174 #define HIDDEN_WARNING "%s is hidden and will not be parsed!\n", filename
175 ( const uint8_t *filename
177 { static uint8_t fname
[NAME_MAX
];
178 uint8_t *last_period
= NULL
, *iter
;
180 if (*filename
== '.')
181 { fprintf (stderr
, HIDDEN_WARNING
);
184 /* Copy the filename and remove its suffix */
185 u8_strncpy(fname
,filename
,NAME_MAX
);
187 for (iter
= fname
; *iter
; iter
++) //find the last '.' char
190 if (last_period
) //if we found one,
191 *last_period
= 0; //truncate the string there
192 /* Register the current_filename */
193 current_filename
= filename
;
195 return lexer_lexfilename(fname
);
198 uint8_t const* lexer_get_current_filepath
200 { static uint8_t current_path
[PATH_MAX
];
201 static uint8_t const* last_filename
;
202 if ((!last_filename
|| last_filename
!= current_filename
) &&
203 (realpath(current_filename
, current_path
) != (char*) current_path
))
204 { perror("realpath: ");
207 return (const char*)current_path
;
210 /* Scan filename and push the its tokens
212 int lexer_lexfilename
215 int ntok
, i
, cmp
, len
, set_len
, height
, width
;
216 char map_key
[] = "_m_";
217 static uint8_t set_name
[MAX_SETNAME_LEN
] = {0};
218 uint8_t *first
, *map_begin
;
220 printf("Starting lexerfilename on %s\n", str
);
224 printf("Lexfilename:: str is NULL so fail\n");
225 printf("setname is %s\n", set_name
);
227 /* If last file was a mapfile, then its 5th to last token should
228 be a MOPEN. If this is the case, then we only pass MOPEN, height,
229 weight and name of the current file. */
230 if( (TK_STACKX
- 5)->tok_t
== MOPEN
)
231 { printf("The last file was a mapfile\n");
232 if( (map_begin
= strstr(map_key
, str
)) ) //if the current file is a mapfile
233 { printf("The current file is a variant of the last mapfile\n");
234 printf("Start lexing mapfile %s\n", str
);
235 ntok
+= lexer_lexstring(map_begin
, strlen(map_begin
));
237 printf("Current file is not a variant of the last mapfile\n");
239 else //last file was not a mapfile
240 { printf("Last file was not a mapfile\n");
242 first
= (uint8_t*) u8_strchr(str
, '_'); //find the first '_' to find where str set_name ends
244 if(set_name
[0] != 0) //if there is a set_name from last str
245 { printf("There is a set_name (%s) present\n", set_name
);
246 set_len
= first
- str
;
248 if(u8_strncmp(str
, set_name
, set_len
) == 0) //check if it matches the current set_name
249 { str
= str
+ set_len
+ 1; //if so, remove it from str
250 printf("str set_name matched last set_name, set str to %s\n", str
);
252 else //update set_name to be str set_name
253 { u8_cpy(set_name
, str
, set_len
);
254 set_name
[set_len
] = 0;
259 { u8_cpy(set_name
, str
, first
-str
);
261 /* Call lexer_lexstring to tokenize the string */
262 printf("calling lexstring to tokenize str (%s) of len %d\n", str
, u8_strlen(str
));
263 ntok
+= lexer_lexstring(str
, u8_strlen(str
));
266 /*TODO: if regfile, store full path for later */
268 printf("Ending lexer_lex on %s, %d tokens were lexed\n", str
, ntok
);
272 /* int lexer_lexmapfile */
273 /* #define INC_X() */
274 /* (int height, int width) */
278 /* /\* Give scanner_scanpixels a buffer and a len. Iterate through */
279 /* buf with buf[n]. If n == 0, do nothing. if n has a value, push x, */
280 /* push y, push (z = n << 24), push (ref_id = n >> 8) *\/ */
281 /* //scanner_scanpixels() */
283 /* for(i = 0; i < len; i++) */
284 /* if(buf[i] == 0) */
293 /* fname_bytes = (uint8_t*)(DE_POP()->d_name); */
294 /* printf("d_name is %s\n", fname_bytes); */
295 /* for (fnp = filename, i = 0; i < NAME_MAX; i += unit_size, fnp++) */
296 /* { unit_size = u8_mblen(fname_bytes + i, min(4, NAME_MAX - i)); */
297 /* if (u8_mbtouc(fnp, fname_bytes + i, unit_size) == -1) //add ucs4 char to the filename */
298 /* FAIL("Lexer failed to convert ^%s to unicode\n", (fname_bytes + i)); */
299 /* if (*fnp == 0) //added a terminating char */
302 /* if(u8_mbtouc(filename, DE_POP()->d_name, NAME_MAX) == -1) */
303 /* FAIL("Lexer failed to convert d_name into uint8_t\n"); */
304 /* ulc_fprintf(stdout, "filename is %11U\n c", filename); */