トークン読み込み用クラス (各言語共通)

これまでC# で作成してきた「SourceToHTML」を、VB.NET に焼きなおしてみます。

TokenReaderCommon.vb

'*******************************************************************************************************

'  トークンごとに その種類と、内容を返す ( 共通 )

'*******************************************************************************************************

Public MustInherit Class TokenReaderCommon

    Inherits TokenReader
    '予約語 コレクション

    Protected _keyWords As System.Collections.Specialized.StringCollection
    '演算子 コレクション

    Protected _operators As String

    '---------------------------------------------------------------------------------------------------

    '   初期化

    '---------------------------------------------------------------------------------------------------

    Friend Sub New(ByVal aReader As Reader, ByVal langType As String)

        MyBase.New(aReader, langType)
        '予約語 コレクションを設定

        initKeyWord(langType)
        '演算子 コレクションを設定

        initOperator(langType)

    End Sub

    '---------------------------------------------------------------------------------------------------

    '   終了

    '---------------------------------------------------------------------------------------------------

    Protected Overrides Sub Finalize()

        _keyWords   = Nothing

    End Sub

    '---------------------------------------------------------------------------------------------------

    '  予約語 コレクションを設定

    '---------------------------------------------------------------------------------------------------

    Private Sub initKeyWord(ByVal langType As String)

        '予約語 コレクションを設定 (".\cs7\key.txt")

        _keyWords = New System.Collections.Specialized.StringCollection()
        '予約語ファイルがなければ、予約語コレクションは空っぽ

        If (System.IO.File.Exists(System.Windows.Forms.Application.StartupPath + "\" + langType + "\key.txt")) Then

            Dim keyReader As System.IO.StreamReader = New System.IO.StreamReader(System.Windows.Forms.Application.StartupPath + "\" + langType + "\key.txt", System.Text.Encoding.GetEncoding("Shift_JIS"))

            Dim s As String

            Do

                s = keyReader.ReadLine()

                If (s Is Nothing) Then Exit Do

                _keyWords.Add(s)

            Loop

            keyReader.Close()

            keyReader = Nothing

        End If

    End Sub

    '---------------------------------------------------------------------------------------------------

    '  演算子 コレクションを設定

    '---------------------------------------------------------------------------------------------------

    Private Sub initOperator(ByVal langType As String)

        _operators = ""
        '演算子ファイルがなければ、演算子コレクションは空っぽ

        If (System.IO.File.Exists(System.Windows.Forms.Application.StartupPath + "\" + langType + "\ope.txt")) Then

            Dim opeReader As System.IO.StreamReader = New System.IO.StreamReader(System.Windows.Forms.Application.StartupPath + "\" + langType + "\ope.txt", System.Text.Encoding.GetEncoding("Shift_JIS"))

            Dim s As String

            Do

                s = opeReader.ReadLine()

                If (s Is Nothing) Then Exit Do

                _operators += s

            Loop

            opeReader.Close()

            opeReader = Nothing

        End If

    End Sub

    '---------------------------------------------------------------------------------------------------

    '   状態を更新

    '---------------------------------------------------------------------------------------------------

    Protected Overrides Sub getNextToken()

        MyBase.getNextToken()
        Do

            Select Case _context.State

                Case "その他" :                         getNextStateOther()

                Case "演算子" :                         getNextStateOther()

                Case "区切り記号" :                     getNextStateOther()

                Case "不明" :                           getNextStateOther()

                Case "単一行コメント" :                 getNextStateComSin()

                Case "複数行コメント" :                 getNextStateComMul()

                Case "文字列" :                         getNextStateStr()

                Case "文字列中のエスケープシーケンス" : getNextStateStrEsc()

                Case "逐語的文字列" :                   getNextStateStrLit()

                Case "文字" :                           getNextStateCha()

                Case "文字中のエスケープシーケンス" :   getNextStateChaEsc()

                Case "識別子" :                         getNextStateIdw()

                Case "エスケープされた識別子" :         getNextStateIdwEsc()

                Case "数字" :                           getNextStateNum()

                Case "ディレクティブ" :                 getNextStateDir()

                Case "日付" :                           getNextStateDtm()

            End Select
        Loop While(_context.State = "不明")

    End Sub

    '---------------------------------------------------------------------------------------------------

    '   状態を更新 (その他・区切り記号・不明)

    '---------------------------------------------------------------------------------------------------

    Private Sub getNextStateOther()

        _context.State                                  =   "その他"

        _context.nextToken.tokenKind                    =   "その他"
        ' 状態を更新

        If      (IsIdent())     Then:   _context.State      =   "識別子"

        ElseIf  (IsEscape())    Then:   _context.State      =   "エスケープされた識別子"

        ElseIf  (IsNumber())    Then:   _context.State      =   "数字"

        ElseIf  (IsDirective()) Then:   _context.State      =   "ディレクティブ"

        ElseIf  (IsComSingle()) Then:   _context.State      =   "単一行コメント"

        ElseIf  (IsComMulti())  Then:   _context.State      =   "複数行コメント"

        ElseIf  (IsString())    Then:   _context.State      =   "文字列"

        ElseIf  (IsChar())      Then:   _context.State      =   "文字"

        ElseIf  (IsLiteral())   Then:   _context.State      =   "逐語的文字列"

        ElseIf  (IsDate())      Then:   _context.State      =   "日付"

        ElseIf  (IsDelim())     Then:   _context.State      =   "区切り記号"

        ElseIf  (IsOperator())  Then:   _context.State      =   "演算子"

        Else:                           _context.State      =   "その他"

        End If
        ' トークンの種類を取得

        Select Case _context.State

            Case    "逐語的文字列":             _context.nextToken.tokenKind    =   "文字列"

            Case    "エスケープされた識別子":   _context.nextToken.tokenKind    =   "識別子"

            Case    "単一行コメント":           _context.nextToken.tokenKind    =   "コメント"

            Case    "複数行コメント":           _context.nextToken.tokenKind    =   "コメント"

            Case    "日付":                     _context.nextToken.tokenKind    =   "文字列"

            Case    Else:                       _context.nextToken.tokenKind    =   _context.State

        End Select

    End Sub

    '---------------------------------------------------------------------------------------------------

    '   単一行コメントか？

    '---------------------------------------------------------------------------------------------------

    Protected Overridable Function IsComSingle() As Boolean

        Return ((_context.currChar = "/"c) AndAlso (_context.nextChar = "/"c))

    End Function

    '---------------------------------------------------------------------------------------------------

    '   複数行コメントか？

    '---------------------------------------------------------------------------------------------------

    Protected Overridable Function IsComMulti() As Boolean

        Return ((_context.currChar = "/"c) AndAlso (_context.nextChar = "*"c))

    End Function

    '---------------------------------------------------------------------------------------------------

    '   文字列か？

    '---------------------------------------------------------------------------------------------------

    Protected Overridable Function IsString() As Boolean

        Return False

    End Function

    '---------------------------------------------------------------------------------------------------

    '   文字か？

    '---------------------------------------------------------------------------------------------------

    Protected Overridable Function IsChar() As Boolean

        Return False

    End Function

    '---------------------------------------------------------------------------------------------------

    '   逐語的文字列か？

    '---------------------------------------------------------------------------------------------------

    Protected Overridable Function IsLiteral() As Boolean

        Return False

    End Function

    '---------------------------------------------------------------------------------------------------

    '   ディレクティブか？

    '---------------------------------------------------------------------------------------------------

    Protected Overridable Function IsDirective() As Boolean

        Return False

    End Function

    '---------------------------------------------------------------------------------------------------

    '   識別子か？

    '---------------------------------------------------------------------------------------------------

    Protected Overridable Function IsIdent() As Boolean

        Return (Char.IsLetter(_context.currChar) OrElse (_context.currChar = "_"c))

    End Function

    '---------------------------------------------------------------------------------------------------

    '   エスケープされた識別子か？

    '---------------------------------------------------------------------------------------------------

    Protected Overridable Function IsEscape() As Boolean

        Return False

    End Function

    '---------------------------------------------------------------------------------------------------

    '   数字か？

    '---------------------------------------------------------------------------------------------------

    Protected Overridable Function IsNumber() As Boolean

        Return (Char.IsDigit(_context.currChar))

    End Function

    '---------------------------------------------------------------------------------------------------

    '   演算子か？

    '---------------------------------------------------------------------------------------------------

    Protected Overridable Function IsOperator() As Boolean

        If (Char.IsPunctuation(_context.currChar) OrElse Char.IsSymbol(_context.currChar)) Then

            If (_operators.IndexOf(_context.currChar) >= 0) Then Return True

        End If

        Return False

    End Function

    '---------------------------------------------------------------------------------------------------

    '   区切り記号か？

    '---------------------------------------------------------------------------------------------------

    Protected Overridable Function IsDelim() As Boolean

        If (Char.IsPunctuation(_context.currChar) OrElse Char.IsSymbol(_context.currChar)) Then

            If (_operators.IndexOf(_context.currChar) < 0) Then Return True

        End If
        Return False

    End Function

    '---------------------------------------------------------------------------------------------------

    '   日付か？

    '---------------------------------------------------------------------------------------------------

    Protected Overridable Function IsDate() As Boolean

        Return False

    End Function

    '---------------------------------------------------------------------------------------------------

    '   状態を更新 (単一行コメント)

    '---------------------------------------------------------------------------------------------------

    Protected Overridable Sub getNextStateComSin()

        _context.State                      =   "単一行コメント"

        _context.nextToken.tokenKind        =   "コメント"
        If (_context.currChar               =   NEWLINE) Then

            _context.State                  =   "その他"

            _context.nextToken.tokenKind    =   "その他"

        End If

    End Sub

    '---------------------------------------------------------------------------------------------------

    '   状態を更新 (複数行コメント)

    '---------------------------------------------------------------------------------------------------

    Protected Overridable Sub getNextStateComMul()

        _context.State                      =   "複数行コメント"

        _context.nextToken.tokenKind        =   "コメント"
        If ((_context.prevChar              =   "*"c) _

            AndAlso (_context.currChar      =   "/"c)) Then

            _context.State                  =   "その他"

        End If

    End Sub

    '---------------------------------------------------------------------------------------------------

    '   状態を更新 (文字列)

    '---------------------------------------------------------------------------------------------------

    Protected Overridable Sub getNextStateStr()

        _context.State                      =   "文字列"

        _context.nextToken.tokenKind        =   "文字列"
        ' 各派生クラスで処理する

    End Sub

    '---------------------------------------------------------------------------------------------------

    '   状態を更新 (文字列中のエスケープシーケンス)

    '---------------------------------------------------------------------------------------------------

    Protected Overridable Sub getNextStateStrEsc()

        _context.State                      =   "文字列"

        _context.nextToken.tokenKind        =   "文字列"

    End Sub

    '---------------------------------------------------------------------------------------------------

    '   状態を更新 (逐語的文字列)

    '---------------------------------------------------------------------------------------------------

    Protected Overridable Sub getNextStateStrLit()

        _context.State                      =   "逐語的文字列"

        _context.nextToken.tokenKind        =   "文字列"
        ' 各派生クラスで処理する

    End Sub

    '---------------------------------------------------------------------------------------------------

    '   状態を更新 (文字)

    '---------------------------------------------------------------------------------------------------

    Protected Overridable Sub getNextStateCha()

        _context.State                      =   "文字"

        _context.nextToken.tokenKind        =   "文字"
        ' 各派生クラスで処理する

    End Sub

    '---------------------------------------------------------------------------------------------------

    '   状態を更新 (文字中のエスケープシーケンス)

    '---------------------------------------------------------------------------------------------------

    Protected Overridable Sub getNextStateChaEsc()

        _context.State                      =   "文字"

        _context.nextToken.tokenKind        =   "文字"

    End Sub

    '---------------------------------------------------------------------------------------------------

    '   状態を更新 (識別子)

    '---------------------------------------------------------------------------------------------------

    Protected Overridable Sub getNextStateIdw()

        _context.State                      =   "識別子"

        _context.nextToken.tokenKind        =   "識別子"
        If (Not Char.IsLetter(_context.currChar) AndAlso Not Char.IsDigit(_context.currChar) AndAlso (_context.currChar <> "_"c)) Then

            _context.State                  =   "不明"
            '文字種が変わったら、識別子が 予約語ではないか 確認する

            Dim keyWord As String = getKeyWord()

            If (keyWord <> "") Then

                _context.currToken.tokenKind    =   "予約語"

                _context.currToken.tokenString  =   keyWord

            End If

        End If

    End Sub

    '---------------------------------------------------------------------------------------------------

    ' 予約語ではないか？

    '---------------------------------------------------------------------------------------------------

    Protected Overridable Function getKeyWord() As String

        Return ""

    End Function

    '---------------------------------------------------------------------------------------------------

    '   状態を更新 (エスケープされた識別子)

    '---------------------------------------------------------------------------------------------------

    Protected Overridable Sub getNextStateIdwEsc()

        _context.State                      =   "エスケープされた識別子"

        _context.nextToken.tokenKind        =   "識別子"
        ' 各派生クラスで処理する

    End Sub

    '---------------------------------------------------------------------------------------------------

    '   状態を更新 (数字)

    '---------------------------------------------------------------------------------------------------

    Protected Overridable Sub getNextStateNum()

        _context.State                      =   "数字"

        _context.nextToken.tokenKind        =   "数字"
        If ((_context.currChar = "."c) AndAlso (_context.nextChar = "."c)) Then

            _context.State                  =   "不明"
        ElseIf (Not Char.IsLetter(_context.currChar) AndAlso Not Char.IsDigit(_context.currChar) AndAlso (_context.currChar <> "."c)) Then

            _context.State                  =   "不明"
            ' "+" か "-" で

            If ((_context.currChar = "+"c) OrElse (_context.currChar = "-"c)) Then

                '１つ前の文字が "e" か "E" で

                If ((_context.prevChar = "e"c) OrElse (_context.prevChar = "E"c)) Then

                    '16進表記でなければ、数字

                    If (IsNotHex()) Then

                        _context.State      =   "数字"

                    End If

                End If

            End If

        End If

    End Sub

    '---------------------------------------------------------------------------------------------------

    ' 16進表記か？

    '---------------------------------------------------------------------------------------------------

    Protected Overridable Function IsNotHex() As Boolean

        ' 0x で始まっていたら、16進

        If (_context.currToken.tokenString.Length > 1) Then

            If (_context.currToken.tokenString.Chars(0) = "0") Then

                If ((_context.currToken.tokenString.Chars(1) = "x"c) OrElse (_context.currToken.tokenString.Chars(1) = "X"c)) Then

                    Return False

                End If

            End If

        End If
        Return True

    End Function

    '---------------------------------------------------------------------------------------------------

    '   状態を更新 (ディレクティブ)

    '---------------------------------------------------------------------------------------------------

    Protected Overridable Sub getNextStateDir()

        _context.State                      =   "ディレクティブ"

        _context.nextToken.tokenKind        =   "ディレクティブ"
        ' 各派生クラスで処理する

    End Sub

    '---------------------------------------------------------------------------------------------------

    '   状態を更新 (日付)

    '---------------------------------------------------------------------------------------------------

    Protected Overridable Sub getNextStateDtm()

        _context.State                      =   "日付"

        _context.nextToken.tokenKind        =   "文字列"
        ' 各派生クラスで処理する (VBだけ)

    End Sub

End Class