code转tokens

/*
* code 转tokens
* */
//判断字符
function isEqual(test,code,pos,end) {
    if(pos<end&&code.charCodeAt(pos-1)!==92&&test===code.substr(pos,test.length)){
        return true;
    }
    return false;
}
//找到字符
function find(test,code,pos,end){
    while (pos<end&&!isEqual(test,code,pos,end)) {pos++}
    return pos;
}

function getToken(obj,code) {
    const token={};
    Object.assign(token,obj)
    token.value=code.substr(token.start,token.end-token.start)
    return token;
}
//分隔符
const Punctuation=[
    '...',
    '{|',
    '|}',
    '::',
    '=>',
    '${',
    '(',
    ')',
    ',',
    '@',
    ':',
    '[',
    '.',
    '?',
    ']',
    '{',
    '`',
    '}',
    ';'
];
//运算符
const Operators=[
    '!==',
    '===',
    '!=',
    '**',
    '-=',
    '++',
    '--',
    '||',
    '&&',
    '>>',
    '<<',
    '+=',
    '==',
    '+',
    '&',
    '>',
    '^',
    '|',
    '<',
    '-',
    '%',
    '*',
    '/',
    '='
];
//空白
const Emptys=[
    ' ',
    '
',
    '
',
]
//注释
const Comments=[
    ['//','
'],
    ['/*','*/'],
];
//文本
const notes=[
    ['"','"'],
    ["'","'"],
    ["`","`"],
];
const rockArr=[].concat(Emptys,Punctuation,Operators)
rockArr.sort(function (p1,p2) {
    return p2.length>p1.length;
})
//获取'无子'元素
function getRock(code,start,end,preChild) {
    if(start>=end){return }
    //碰撞空白
    while (code[start]===' '||code[start]==='
'||code[start]==='
'||code[start]==='	'){
        start++;
    }
    let pos=start;
    //数字和变量
    if( (/[_$a-z0-9]/i.test(code[pos])||/.d/.test(code.substr(pos,2)))&&code.charCodeAt(pos-1)!==92 ){
        pos++;
        while (pos<end&&(/[_$a-z0-9]/i.test(code[pos])||/.d/.test(code.substr(pos,2)))){
            pos++;
        }
        return getToken({start:start,end:pos,type:'variable'},code)
    }
    //注释
    for(let i=0;i<Comments.length;i++){
        const arr=Comments[i];
        if(isEqual(arr[0],code,pos,end)){
            pos=find(arr[1],code,pos+arr[0].length,end)+arr[1].length;
            break;
        }
    }
    if(pos>start){
        return getToken({start:start,end:pos,type:'comment'},code)
    }
    if(!preChild||(preChild.type==='symbol'&&code.substring(preChild.start,preChild.end)!==')')){
        //文本
        for(let i=0;i<notes.length;i++){
            const arr=notes[i];
            if(isEqual(arr[0],code,pos,end)){
                pos=find(arr[1],code,pos+arr[0].length,end)+arr[1].length;
                break;
            }
        }

        if(pos>start&&pos<end){
            return getToken({start:start,end:pos,type:'string'},code)
        }


    }

    //symbol
    for(let i=0;i<rockArr.length;i++){
        if(isEqual(rockArr[i],code,pos,end)){
            pos=pos+rockArr[i].length;
            break;
        }
    }
    if(pos>start){
        return getToken({start:start,end:pos,type:'symbol'},code)
    }
    throw code[pos];
}
//获取子元素
function getChild(code,pos,end,preChild) {
    const cTok=getRock(code,pos,end,preChild);
    //template
    if(isEqual('`',code,pos,end)){
        const templateTree={start:pos,end:0,type:'Template',child:[]}
        let isEnd=false;
        let pos3=pos+1;

        while (pos3<end&&!isEnd) {
            if(isEqual('${',code,pos3,end)){
                const start=pos3;
                pos3=find('}',code,pos3+2,end);
                let token={
                    start:start,
                    end:pos3+1,
                    type:'TemplateExpression',
                    child:getAllChild(code,start+2,pos3)
                };
                templateTree.child.push(token)
            }else if(isEqual("`",code,pos3,end)){
                isEnd=true;
            }
            pos3++
        }

        templateTree.end=pos3;
        if(templateTree.child.length>0){
            templateTree.quais=[]
            for(let i=0;i<templateTree.child.length;i++){
                const tok=templateTree.child[i];
                if(i===0){
                    templateTree.quais.push({start:templateTree.start+1,end:tok.start})
                }else{
                    templateTree.quais.push({start:templateTree.child[i-1].end,end:tok.start})
                }
                if(i===templateTree.child.length-1){
                    templateTree.quais.push({start:tok.end,end:templateTree.end-1})
                }

            }
        }
        return templateTree
    }

    if(!preChild||(preChild.type==='symbol'&&code.substring(preChild.start,preChild.end)!==')')){
        //jsx
        if(cTok.value==='<'){
            const tagName=getRock(code,cTok.end,end).value;
            if(/^[a-z][a-z0-9]*$/.test(tagName)){
                let isEnd=false;
                let isCloseI=1;
                let pos3=pos+1;
                while (pos3<end&&!isEnd) {
                    if(isEqual('"',code,pos3,end)){
                        pos3=find('"',code,pos3+1,end)
                    }
                    if(isEqual("'",code,pos3,end)){
                        pos3=find("'",code,pos3+1,end)
                    }
                    if(isEqual("{",code,pos3,end)){
                        pos3=find("}",code,pos3+1,end)
                    }
                    if(isEqual(">",code,pos3,end)){
                        if(isCloseI===0){
                            isEnd=true
                        }
                    }else if(isEqual("<",code,pos3,end)){
                        const tok=getRock(code,pos3+1,end);
                        if(code.substring(tok.start,tok.end)==='/'){
                            isCloseI--;
                            pos3=tok.end-1;
                        }else{
                            isCloseI++;

                        }
                    }else if(isEqual("/",code,pos3,end)){
                        const tok=getRock(code,pos3+1,end);
                        if(code[tok.start]==='>'){
                            isCloseI--;
                            pos3=tok.end-1;
                            if(isCloseI===0){
                                isEnd=true
                            }
                        }
                    }
                    pos3++
                }
                if(code[pos3-1]==='>'){
                    return {start:pos,end:pos3,type:'JSXElement'}
                }
            }

        }


        //正则
        if(isEqual('/',code,pos,end)){
            let pos3=pos+2;
            while (pos3<end&&!isEqual('/',code,pos3,end)) {
                if(isEqual('[',code,pos3,end)){
                    pos3=find(']',code,pos3+1,end)
                }
                pos3++
            }
            if(pos3<end){
                pos3++;
                while (pos3<end&&/[igmsUxaDe]/.test(code[pos3])) {pos3++;}
                return getToken({start:pos,end:pos3},code)
            }
        }
    }

    return cTok

}
function transform(token,code) {
    if(token.type==='JSXElement'){
        return getTagChild(code,token.start,token.end)
    }
    return token;
}
//获取所有子元素
function getAllChild(code,start,end) {
    const childs=[]
    let token=getChild(code,start,end)
    while (token){
        childs.push(transform(token,code))
        token=getChild(code,token.end,end,token)
    }
    return childs;
}
function CodeToTokens(code){
    const token={
        start:0,
        end:code.length,
        child:getAllChild(code,0,code.length)
    };
    return token;
}
module.exports=CodeToTokens;

//解析jsx
function getTagChild(code,start,end) {
    if(start>=end){return }
    let pos=start;
    //碰撞空白
    while (!(isEqual('<',code,pos,end)&&!isEqual('<',code,pos-1,end))&&pos<end){
        pos++;
    }
    if(pos>start){
        return {start:start,end:pos}
    }
    //jsx
    const json={
        start:pos,
        end:0,
        openingElement:null,
        closingElement:null,
    }
    let child=null;
    let isEnd=false;
    let isCloseI=1;
    let pos3=pos+1;
    while (pos3<end&&!isEnd) {
        if(isEqual('"',code,pos3,end)){
            pos3=find('"',code,pos3+1,end)
        }
        if(isEqual("'",code,pos3,end)){
            pos3=find("'",code,pos3+1,end)
        }
        if(isEqual("{",code,pos3,end)){
            pos3=find("}",code,pos3+1,end)
        }
        if(isEqual(">",code,pos3,end)){
            if(isCloseI===0){
                isEnd=true
                json.closingElement={
                    type:'JSXClosingElement',
                    start:child.end,
                    end:pos3+1,
                }
            }
            if(isCloseI===1){
                if(!json.openingElement){
                    json.openingElement={
                        type:'JSXOpeningElement',
                        start:pos,
                        end:pos3+1,
                    }
                }

            }
        }else if(isEqual("<",code,pos3,end)){
            const tok=getRock(code,pos3+1,end);
            if(code.substring(tok.start,tok.end)==='/'){
                isCloseI--;
                if(isCloseI===0){
                    console.log(pos3,json.openingElement.end)
                    if(pos3>json.openingElement.end){
                        child={
                            start:json.openingElement.end,
                            end:pos3,
                        }
                    }

                }
                pos3=tok.start;
            }else{
                isCloseI++;

            }
        }else if(isEqual("/",code,pos3,end)){
            const tok=getRock(code,pos3+1,end);

            if(code[tok.start]==='>'){
                isCloseI--;
                pos3=tok.end-1;
                if(isCloseI===0){
                    isEnd=true
                    json.openingElement={
                        type:'JSXOpeningElement',
                        start:pos,
                        end:pos3+1,
                    }
                }
            }
        }
        pos3++
    }
    if(code[pos3-1]==='>'){
        json.end=pos3;
        if(child){
            json.child=getAllTagChild(code,child.start,child.end)
        }

        return getToken(json,code)
    }
}
//获取所有子元素
function getAllTagChild(code,start,end) {
    const childs=[]
    let token=getTagChild(code,start,end)

    while (token){
        childs.push(token)
        token=getTagChild(code,token.end,end,token)
    }
    return childs;
}

//test
const fs=require('fs');
const code=fs.readFileSync('./code.js').toString()
const tree=CodeToTokens(code)
console.log(tree)