lexer.rkt

#lang racket

(require parser-tools/lex
         (prefix-in : parser-tools/lex-sre)
         parser-tools/yacc)

(define full-lexer (lexer
   (whitespace (full-lexer input-port))
   ((:or (:: (:+ (char-range #\0 #\9))) (:: (:: (:+ (char-range #\0 #\9)) #\. (:+ (char-range #\0 #\9))))) (token-NUM (string->number lexeme)))
   ((eof) (token-EOF))
   (";" (token-SEMICOLON))
   ("pass" (token-PASS))
   ("break" (token-BREAK))
   ("continue" (token-CONTINUE))
   ("=" (token-ASSIGNMENT))
   ("return" (token-RETURN))
   ("global" (token-GLOBAL))
   ("def" (token-DEF))
   ("(" (token-OP))
   (")" (token-CP))
   (":" (token-COLON))
   ("," (token-COMMA))
   ("if" (token-IF))
   ("else" (token-ELSE))
   ("for" (token-FOR))
   ("in" (token-IN))
   ("or" (token-OR))
   ("and" (token-AND))
   ("not" (token-NOT))
   ("==" (token-EQUALS))
   ("<" (token-LT))
   ("<=" (token-LET))
   (">" (token-GT))
   (">=" (token-GET))
   ("+" (token-PLUS))
   ("-" (token-MINUS))
   ("*" (token-TIMES))
   ("/" (token-DIVIDES))
   ("**" (token-POWER))
   ("[" (token-OB))
   ("]" (token-CB))
   ("True" (token-TRUE))
   ("False" (token-FALSE))
   ("None" (token-NONE))
   ((:+ (:or (char-range #\0 #\9) (char-range #\a #\z) (char-range #\A #\Z) #\_)) (token-ID lexeme))))

(define-tokens a (NUM ID))
(define-empty-tokens b (EOF SEMICOLON PASS BREAK CONTINUE ASSIGNMENT RETURN GLOBAL DEF OP CP COLON COMMA TRUE FALSE
                        IF ELSE FOR IN OR AND NOT EQUALS LT LET GT GET PLUS MINUS TIMES DIVIDES POWER OB CB NONE))

(provide (all-defined-out))