add initial lexer

lexer-test.rkt

@@ -0,0 +1,8 @@
+#lang br
+(require "lexer.rkt" brag/support rackunit)
+
+(define (lex str)
+  (apply-port-proc basic-lexer str))
+
+(lex "@a123")
+(lex "{@_a12|foo|123.12}123abc\\@\\{mn̂g他@@愛@@貓，也愛這塊土地%註解%。這也是%%。")

lexer.rkt

@@ -0,0 +1,26 @@
+#lang br
+(require brag/support)
+
+(define-lex-abbrev digits (:+ (char-set "0123456789")))
+(define-lex-abbrev raw-id (:: (:or "_" (:/ "a" "z") (:/ "A" "Z")) (:* (:or "_" (:/ "0" "9") (:/ "a" "z") (:/ "A" "Z")))))
+(define-lex-abbrev any-other-char (:~ (char-set "0123456789@\\{}%|")))
+
+(define basic-lexer
+  (lexer-srcloc
+   [(:or "\r\n" "\n") (token 'NEWLINE lexeme)] ; newline
+   [whitespace (token lexeme #:skip? #t)] ; whitespace
+   [(:: "\\"(char-set "@\\{}%|")) (token 'CHAR (substring lexeme
+                      1 (string-length lexeme)))]
+  [(:: "@" raw-id) (token 'ID (substring lexeme
+                      1 (string-length lexeme)))] ; @id 
+  ["{" (token 'L_PAREN lexeme)] ; {
+  ["}" (token 'R_PAREN lexeme)] ; }
+  ["|" (token 'SEPERATOR lexeme)] ; | (general) seperator
+  ["@@" (token 'G_SEPERATOR lexeme)] ; @@ great seperator
+   [digits (token 'INTEGER (string->number lexeme))] ;123
+   [(:: (:? (char-set "-")) digits "." digits) ; 123.456
+    (token 'DECIMAL (string->number lexeme))]
+   [ (from/to "%" "%") ; %COMMENT%
+    (token 'COMMENT #:skip? #t)]
+  [any-other-char (token 'CHAR lexeme)])) ; any char into char
+(provide basic-lexer)

main.rkt

@@ -0,0 +1,11 @@
+#lang br/quicklang
+(require "parser.rkt" "tokenizer.rkt")
+
+(define (read-syntax path port)
+  (define parse-tree (parse path (make-tokenizer port path)))
+  (strip-bindings
+   #`(module uahgi2-mod uahgi2/expander
+       #,parse-tree)))
+
+(module+ reader
+  (provide read-syntax))

sample.rkt

@@ -0,0 +1,7 @@
+#lang uahgi2
+{@set|@paperwidth|2100}
+{@set|@paperheight|2970}
+{@set|@textsize|12}
+{@maintext@@
+他對這塊土地有著愛惜的情感。今天是@today星期@dayofweek。
+@@}