Class Token

Inheritance
System.Object
SegmentElement
Token
DateTimeToken
NumberToken
SimpleToken
TagToken
TokenBundle
Inherited Members
System.Object.ToString()
System.Object.Equals(System.Object, System.Object)
System.Object.ReferenceEquals(System.Object, System.Object)
System.Object.GetType()
System.Object.MemberwiseClone()
Namespace: Sdl.LanguagePlatform.Core.Tokenization
Assembly: Sdl.LanguagePlatform.Core.dll
Syntax
public abstract class Token : SegmentElement

Constructors

Token()

Declaration
public Token()

Token(Token)

Declaration
public Token(Token other)
Parameters
Type Name Description
Token other

Token(String)

Declaration
public Token(string text)
Parameters
Type Name Description
System.String text

Properties

Culture

Declaration
public CultureInfo Culture { get; set; }
Property Value
Type Description
System.Globalization.CultureInfo

IsPlaceable

Declaration
public virtual bool IsPlaceable { get; }
Property Value
Type Description
System.Boolean

IsPunctuation

Declaration
public bool IsPunctuation { get; }
Property Value
Type Description
System.Boolean

IsSubstitutable

Declaration
public virtual bool IsSubstitutable { get; }
Property Value
Type Description
System.Boolean

IsWhitespace

Declaration
public bool IsWhitespace { get; }
Property Value
Type Description
System.Boolean

IsWord

Declaration
public bool IsWord { get; }
Property Value
Type Description
System.Boolean

Span

Declaration
public SegmentRange Span { get; set; }
Property Value
Type Description
SegmentRange

Text

Declaration
public virtual string Text { get; set; }
Property Value
Type Description
System.String

TokenizationContext

Declaration
public TokenizationContext TokenizationContext { get; set; }
Property Value
Type Description
TokenizationContext

Type

Declaration
public TokenType Type { get; set; }
Property Value
Type Description
TokenType

Methods

Equals(Object)

Declaration
public override bool Equals(object obj)
Parameters
Type Name Description
System.Object obj
Returns
Type Description
System.Boolean
Overrides
System.Object.Equals(System.Object)

GetBundleSimilarity(SegmentElement)

Declaration
protected SegmentElement.Similarity GetBundleSimilarity(SegmentElement other)
Parameters
Type Name Description
SegmentElement other
Returns
Type Description
SegmentElement.Similarity

GetHashCode()

Declaration
public override int GetHashCode()
Returns
Type Description
System.Int32
Overrides
System.Object.GetHashCode()

GetTokenType()

Declaration
protected abstract TokenType GetTokenType()
Returns
Type Description
TokenType

GetWeakHashCode()

Declaration
public override int GetWeakHashCode()
Returns
Type Description
System.Int32
Overrides

SetTokenType(TokenType)

Declaration
protected virtual void SetTokenType(TokenType tokenType)
Parameters
Type Name Description
TokenType tokenType