yoxem/uann
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

add tokenize's function, add interface Token

Browse source
This commit is contained in:
Tan, Kian-ting 2023-09-07 00:01:15 +08:00
parent e2668789e2
commit 5fbb467bdf
5 changed files with 250 additions and 21 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
README.md
View file

@ -5,3 +5,4 @@ another personal draught of a typesetting language and engine.
- 20230904 建立 thenDo、matchRange的函數、refactor harfbuzzjs 以及libpdf 等測試界面
- 20230905-06: 建立 : `toSome`, initial of basic tokenizer (`tokenize`),
`matchAny`, `notDo`, `orDo`, `zeroOrMoreDo`, `zeroOrOnceDo`
- 20230905-07:強化`tokenize`, 加強功能，加`Token`界面。

96
src/index.js
View file

@ -1,6 +1,6 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.tokenize = exports.zeroOrOnceDo = exports.notDo = exports.zeroOrMoreDo = exports.orDo = exports.thenDo = exports.charToCodepoint = exports.matchRange = exports.matchAny = exports.match1Char = void 0;
exports.tokenize = exports.zeroOrOnceDo = exports.notDo = exports.zeroOrMoreDo = exports.orDo = exports.thenDo = exports.charToCodepoint = exports.matchRange = exports.matchAny = exports.match1Char = exports.TokenType = void 0;
var fs = require('fs');
/**
* wrap a x in a `Some(T)`
@ -10,6 +10,32 @@ var fs = require('fs');
function toSome(x) {
return { _tag: "Some", value: x };
}
/**
* The types of Token
* NL, // newline
*
* SP, // half-width space and tab
*
* ID, // identifier
*
* STR, // string
*
* OP, // operator or something like it
*
* FLO, // float num
*
* INT, // Integer
*/
var TokenType;
(function (TokenType) {
TokenType[TokenType["NL"] = 0] = "NL";
TokenType[TokenType["SP"] = 1] = "SP";
TokenType[TokenType["ID"] = 2] = "ID";
TokenType[TokenType["STR"] = 3] = "STR";
TokenType[TokenType["OP"] = 4] = "OP";
TokenType[TokenType["FLO"] = 5] = "FLO";
TokenType[TokenType["INT"] = 6] = "INT";
})(TokenType || (exports.TokenType = TokenType = {}));
/**
* @description
* it returns a function which test if the first char of the `remained` part of
@ -211,20 +237,70 @@ exports.zeroOrOnceDo = zeroOrOnceDo;
function tokenize(input) {
var input_matchee_pair = toSome({ matched: "",
remained: input });
// integer = ([+]|[-])\d\d?
// integer = ([+]|[-])?\d\d*
let integer = (x) => {
let wrapped_x = toSome(x);
let plusMinus = orDo(match1Char('+'), match1Char('-')); // ([+]|[-])
let d = matchRange('0', '9'); // \d
return thenDo(thenDo(thenDo(wrapped_x, zeroOrOnceDo(plusMinus)), d), zeroOrMoreDo(d));
var result = thenDo(thenDo(thenDo(wrapped_x, zeroOrOnceDo(plusMinus)), d), zeroOrMoreDo(d));
if (result._tag == "Some") {
result.value.matched_type = TokenType.INT;
}
return result;
};
console.log(input + ", result: ");
console.log(thenDo(input_matchee_pair, integer));
let space = (x) => {
let wrapped_x = toSome(x);
let s_aux = orDo(match1Char(' '), match1Char('\t')); // (" " | "\t")
var result = thenDo(thenDo(wrapped_x, s_aux), zeroOrMoreDo(s_aux));
if (result._tag == "Some") {
result.value.matched_type = TokenType.SP;
}
return result;
};
let newline = (x) => {
let wrapped_x = toSome(x);
// nl = \r?\n
let result = thenDo(thenDo(wrapped_x, zeroOrOnceDo(match1Char('\r'))), match1Char('\n'));
if (result._tag == "Some") {
result.value.matched_type = TokenType.NL;
}
return result;
};
let term = (token_list, x) => {
var ln = 1;
var col = 0;
var old_x = x;
let term_list = [newline, space, integer];
let term_aux = term_list.reduce((x, y) => orDo(x, y));
var new_x = thenDo(old_x, term_aux);
while (new_x._tag != "None") {
if (new_x.value.matched_type != TokenType.NL) {
col += new_x.value.matched.length;
token_list.push({ text: new_x.value.matched,
type: new_x.value.matched_type,
ln: ln,
col: col });
}
else {
col = 0;
ln += 1;
token_list.push({ text: new_x.value.matched,
type: new_x.value.matched_type,
ln: ln,
col: col });
}
old_x = toSome({ matched: "",
remained: new_x.value.remained });
new_x = thenDo(old_x, term_aux);
}
if (old_x.value.remained.length) {
console.log(token_list);
throw new Error("the code can't be tokenized is near Ln. " + ln + ", Col." + col
+ ", starting with " + old_x.value.remained.substring(0, 10));
}
return token_list;
};
console.log(term([], input_matchee_pair));
// TODO: id, string, space, basic operator, 3 marks: @, {, }.
}
exports.tokenize = tokenize;
tokenize("+123");
tokenize("123");
tokenize("-123");
tokenize(" 123");
tokenize("c123");

126
src/index.ts
View file

@ -27,12 +27,54 @@ export type Maybe<T> = Some<T> | None;
/**
* @description
* the pair of the string to be matched later and the string that have been matched
* @var matched : have been matched
* @var remained : will be tested whether it'll be matched.
* @var matched_type (optional): the type of the matched string
*/
export interface MatcheePair {
/** have been matched */
matched : string
/** will be tested whether it'll be matched. */
remained : string
matched_type?: TokenType
}
/**
* The types of Token
* NL, // newline
*
* SP, // half-width space and tab
*
* ID, // identifier
*
* STR, // string
*
* OP, // operator or something like it
*
* FLO, // float num
*
* INT, // Integer
*/
export enum TokenType{
NL, // newlinw
SP, // half-width space and tab
ID, // identifier
STR, // string
OP, // operator
FLO, // float num
INT, // integer
}
/**
* tokenized token.
* @var text : the content text
* @var type (optional): the type of the token
* @var col : the column number
* @var ln : the line number
*/
export interface Token{
text: string,
type?: TokenType,
col: number,
ln: number,
}
/**
@ -239,18 +281,82 @@ export function tokenize(input : string){
{ let wrapped_x = toSome(x);
let plusMinus = orDo(match1Char('+'), match1Char('-')); // ([+]|[-])
let d = matchRange('0','9'); // \d
return thenDo(thenDo(thenDo(wrapped_x,
var result = thenDo(thenDo(thenDo(wrapped_x,
zeroOrOnceDo(plusMinus)),d),
zeroOrMoreDo(d));
if (result._tag == "Some"){
result.value.matched_type = TokenType.INT;
}
return result;
}
console.log(input+", result: ");
console.log(thenDo(input_matchee_pair, integer));
let space = (x : MatcheePair) =>{
let wrapped_x = toSome(x);
let s_aux = orDo(match1Char(' '), match1Char('\t')); // (" " | "\t")
var result = thenDo(thenDo(wrapped_x, s_aux), zeroOrMoreDo(s_aux));
if (result._tag == "Some"){
result.value.matched_type = TokenType.SP;
}
return result;
}
let newline = (x : MatcheePair) =>{
let wrapped_x = toSome(x);
// nl = \r?\n
let result = thenDo(thenDo(wrapped_x,
zeroOrOnceDo(match1Char('\r'))), match1Char('\n'));
if (result._tag == "Some"){
result.value.matched_type = TokenType.NL;
}
return result;
}
let term = (token_list : Array<Token>, x : Some<MatcheePair>)=>{
var ln = 1;
var col = 0;
var old_x = x;
let term_list = [newline, space, integer];
let term_aux = term_list.reduce((x,y)=> orDo(x,y));
var new_x : Maybe<MatcheePair> = thenDo(old_x, term_aux);
while (new_x._tag != "None"){
if (new_x.value.matched_type != TokenType.NL){
col += new_x.value.matched.length;
token_list.push({text : new_x.value.matched,
type: new_x.value.matched_type,
ln : ln,
col : col});
}
else{
col = 0;
ln += 1;
token_list.push({text : new_x.value.matched,
type: new_x.value.matched_type,
ln : ln,
col : col});
}
old_x = toSome({matched : "",
remained : new_x.value.remained});
new_x = thenDo(old_x, term_aux);
}
if (old_x.value.remained.length){
console.log(token_list);
throw new Error("the code can't be tokenized is near Ln. "+ln+", Col."+col
+", starting with "+ old_x.value.remained.substring(0,10));
}
return token_list;
}
console.log(term([], input_matchee_pair));
// TODO: id, string, space, basic operator, 3 marks: @, {, }.
}
tokenize("+123");
tokenize("123");
tokenize("-123");
tokenize(" 123");
tokenize("c123");

21
tests/index.js
View file

@ -57,6 +57,27 @@ let doTestRes9 = thenDo(doThenTestee9, src_1.matchAny);
assert(doTestRes9._tag == "Some");
assert(doTestRes9.value.matched == "妳");
assert(doTestRes9.value.remained == "的");
(0, src_1.tokenize)("+123");
(0, src_1.tokenize)("123");
(0, src_1.tokenize)("-123");
(0, src_1.tokenize)(" 123");
try {
(0, src_1.tokenize)("c123");
}
catch (error) {
console.log(error);
}
(0, src_1.tokenize)(" ");
(0, src_1.tokenize)(" ");
(0, src_1.tokenize)(" \t");
(0, src_1.tokenize)(" \t123");
try {
(0, src_1.tokenize)(" \t123aaa456");
}
catch (error) {
console.log(error);
}
(0, src_1.tokenize)(" \t123\n456");
// harfbuzz test
let harfbuzz = require("../src/harfbuzz.js");
harfbuzz.harfbuzzTest("123.abc");

27
tests/index.ts
View file

@ -1,4 +1,4 @@
import { matchAny } from "../src";
import { matchAny, tokenize } from "../src";
let assert = require("assert");
let cloMain = require("../src");
@ -74,6 +74,31 @@ assert(doTestRes9._tag == "Some");
assert(doTestRes9.value.matched == "妳");
assert(doTestRes9.value.remained == "的");
tokenize("+123");
tokenize("123");
tokenize("-123");
tokenize(" 123");
try {
tokenize("c123");
} catch (error) {
console.log(error);
}
tokenize(" ");
tokenize(" ");
tokenize(" \t");
tokenize(" \t123");
try {
tokenize(" \t123aaa456");
} catch (error) {
console.log(error);
}
tokenize(" \t123\n456");
// harfbuzz test
let harfbuzz = require("../src/harfbuzz.js");