Table of Contents

Class TSourceToken

Namespace
gudusoft.gsqlparser
Assembly
gudusoft.gsqlparser.dll

Represents a source token which is the basic syntactical unit of SQL. All parse tree nodes including one or more tokens.
A list of source token will be available after parse or tokenize the input SQL.

  TGSqlParser sqlparser = new TGSqlParser(EDbVendor.dbvoracle);
  sqlparser.sqltext = "select col from t";
  int ret = sqlparser.parse();
  // this will geneate a list of tokens only, no parse tree node is available.
  // int ret = sqlparser.tokenizeSqltext();
  if (ret == 0){
   for(int i=0;i<sqlparser.sourcetokenlist.size();i++){
     TSourceToken st =  sqlparser.sourcetokenlist.get(i);
     System.out.println(st.tokentype.toString()+" "+st.toString());
   }
  }else{
    System.out.println(sqlparser.getErrormessage());
  }

}

Get a list of source tokens after call parse() or just call tokenizeSqltext() if you only need to access tokens of input SQL without generating full parse tree nodes.

tokencode is the unique id represents one kind of token. The most important type of whitespace, return, and tokentype

public class TSourceToken
Inheritance
TSourceToken
Inherited Members

Constructors

TSourceToken() 

public TSourceToken()

TSourceToken(string) 

public TSourceToken(string s)

Parameters

s string

Fields

astext

Text representation for current token.

public string astext

Field Value

string

columnNo

the column number of the first character for current token.

public long columnNo

Field Value

long

container

Container for current token, this is the reference to TSourceTokenList

public TSourceTokenList container

Field Value

TSourceTokenList

dolqstart 

public string dolqstart

Field Value

string

lineNo

the line number of the first character for current token.

public long lineNo

Field Value

long

offset

Token's offset from the beginning of the input query.

<pre><code class="lang-csharp">public void testOffset(){
    TGSqlParser sqlparser = new TGSqlParser(EDbVendor.dbvoracle);
    sqlparser.sqltext = "select f from t\n" +
            "where f>1\n";
    assertTrue(sqlparser.parse() == 0);
    for (int i=0;i<sqlparser.sourcetokenlist.size();i++){
        TSourceToken st = sqlparser.sourcetokenlist.get(i);
        String textFromOffset = sqlparser.sqltext.toString().substring((int)st.offset,(int)st.offset+st.toString().length());
        assertTrue(st.toString().equalsIgnoreCase(textFromOffset));
    }
}
 }</code></pre>

public long offset

Field Value

long

posinlist

Index of this token in the container, start from 0

public void testPosinList(){
    TGSqlParser sqlparser = new TGSqlParser(EDbVendor.dbvoracle);
    sqlparser.sqltext = "select f from t\n" +
            "where f>1\n";
    assertTrue(sqlparser.parse() == 0);
    for (int i=0;i<sqlparser.sourcetokenlist.size();i++){
        assertTrue(i == sqlparser.sourcetokenlist.get(i).posinlist);
    }
}
 }
public int posinlist

Field Value

int

stmt

position in activeTokenList, start from 0

public TCustomSqlStatement stmt

Field Value

TCustomSqlStatement

tag 

public int tag

Field Value

int

tokencode

Unique id of this token used by parser internally. check available value start from cmtslashstar

public int tokencode

Field Value

int

tokenstatus 

public ETokenStatus tokenstatus

Field Value

ETokenStatus

tokentype

check available value in ETokenType

public ETokenType tokentype

Field Value

ETokenType

Properties

AllowRewrite

During the process of building AST by the parser, this value is false This value turns to true after user call TCustomSqlStatement.ReWriteInit() manually.

public bool AllowRewrite { get; set; }

Property Value

bool

AlternativeToken 

public virtual TSourceToken AlternativeToken { get; set; }

Property Value

TSourceToken

Changed

Node include this token was modified if it's true. Only allowed to set to true when AllowRewrite is true

public bool Changed { get; set; }

Property Value

bool

DbObjType 

public virtual EDbObjectType DbObjType { get; set; }

Property Value

EDbObjectType

Dbvendor 

public virtual EDbVendor Dbvendor { get; set; }

Property Value

EDbVendor

Gsqlparser 

public virtual TGSqlParser Gsqlparser { get; set; }

Property Value

TGSqlParser

LinkToken 

public virtual TSourceToken LinkToken { get; set; }

Property Value

TSourceToken

NextTokenInChain 

public TSourceToken NextTokenInChain { get; set; }

Property Value

TSourceToken

PrevTokenInChain 

public TSourceToken PrevTokenInChain { get; set; }

Property Value

TSourceToken

ReplaceToken 

public virtual TSourceToken ReplaceToken { get; set; }

Property Value

TSourceToken

String

set new string of this token

public virtual string String { get; set; }

Property Value

string

TextWithoutQuoted 

public virtual string TextWithoutQuoted { get; }

Property Value

string

TokensAfter 

public virtual TSourceTokenList TokensAfter { get; }

Property Value

TSourceTokenList

TokensBefore 

public virtual TSourceTokenList TokensBefore { get; }

Property Value

TSourceTokenList

nodesEndWithThisToken

source token can be end token of one or more parse tree nodes NodesEndWithThisToken includes all those parse tree nodes

public List<TParseTreeNode> nodesEndWithThisToken { get; }

Property Value

List<TParseTreeNode>

nodesStartFromThisToken

source token can be start token of one or more parse tree nodes, NodesStartFromThisToken includes those parse tree nodes.

public List<TParseTreeNode> nodesStartFromThisToken { get; }

Property Value

List<TParseTreeNode>

Methods

ToString()

Original text for current token.

public override string ToString()

Returns

string

Original token text.

firstTokenOfLine() 

public virtual bool firstTokenOfLine()

Returns

bool

isnonsolidtoken()

Is current token a solid token or not.

public virtual bool isnonsolidtoken()

Returns

bool

true if it's a non-solid token.

isnonsolidtoken(ETokenType)

Token except ttwhitespace,ttreturn,ttsimplecomment,ttbracketedcomment is solid token.

public static bool isnonsolidtoken(ETokenType tokentype)

Parameters

tokentype ETokenType

Returns

bool

True if token type is not one of ttwhitespace,ttreturn,ttsimplecomment,ttbracketedcomment

issolidtoken()

Is current token a non-solid token or not.

public virtual bool issolidtoken()

Returns

bool

true if it's a solid token.

lastTokenOfLine() 

public virtual bool lastTokenOfLine()

Returns

bool

nextSolidToken() 

public virtual TSourceToken nextSolidToken()

Returns

TSourceToken

prevSolidToken() 

public virtual TSourceToken prevSolidToken()

Returns

TSourceToken

removeMyFromTokenList() 

public virtual int removeMyFromTokenList()

Returns

int

searchToken(int, int) 

public virtual TSourceToken searchToken(int targetTokenCode, int range)

Parameters

targetTokenCode int
range int

Returns

TSourceToken

searchToken(string, int) 

public virtual TSourceToken searchToken(string targetTokenText, int range)

Parameters

targetTokenText string
range int

Returns

TSourceToken

searchTokenAfterObjectName()

Search token after an object.

return new scott.func(x1); current token is new, then call searchTokenAfterObjectName will return ( token.

public virtual TSourceToken searchTokenAfterObjectName()

Returns

TSourceToken

solid token after an object name

searchTokenAtTheEndOfSameLine() 

public TSourceToken searchTokenAtTheEndOfSameLine()

Returns

TSourceToken

setLinkToken(TSourceToken) 

public void setLinkToken(TSourceToken st)

Parameters

st TSourceToken

toScript() 

public string toScript()

Returns

string

toStringDebug() 

public virtual string toStringDebug()

Returns

string

toUnQuotedString() 

public virtual string toUnQuotedString()

Returns

string
Edit this page