lexer.h

Go to the documentation of this file.

 
#include "elementa/license.inc"
#include "elementa/checks.inc"
 
#ifndef ELEMENTA_PARSING_LEXER_H
#define ELEMENTA_PARSING_LEXER_H
 
#include <string>
#include <memory>
#include <utility>
#include <vector>
#include <list>
#include <functional>
#include "elementa/base/strings.h"
#include "elementa/base/exceptions.h"
#include "elementa/base/serial_channels.h"
#include "elementa/adts/fsms.h"
 
namespace elementa
{
 
namespace parsing
{
 
/* **********************************************************************
 
    Errors for lexers
 
*************************************************************************/
 
class LexerError: public elementa::base::Exc
{
 public:
 
    LexerError(const std::string & expl): 
                    elementa::base::Exc{std::string{"LEXICAL ERROR. "} + expl}{}
                    
    ELE_CLASS_EXCOVERRIDE(LexerError)                   
};
 
 
class UnexpSymbol: public LexerError
{
 public:
 
    UnexpSymbol(const std::string & details):
                                LexerError{
                                    elementa::base::concatWithMiddle(
                                        "Unexpected lexical element",details)}
        {}
        
    ELE_CLASS_EXCOVERRIDE(UnexpSymbol)      
};
 
 
class ExpSymbol: public LexerError
{
 public:
 
    ExpSymbol(const std::string & details):
                                LexerError{
                                    elementa::base::concatWithMiddle(
                                        "Expected some lexical element",
                                        details)}
        {}
        
    ELE_CLASS_EXCOVERRIDE(ExpSymbol)        
};
 
class UnexpEnd: public LexerError
{
 public:
 
    UnexpEnd(const std::string & details):
                                LexerError{
                                    elementa::base::concatWithMiddle(
                                        "Unexpected end of channel",details)}
        {}
        
    ELE_CLASS_EXCOVERRIDE(UnexpEnd)     
};
 
 
class InvLexElem: public LexerError
{
 public:
 
    InvLexElem(const std::string & details):
                                LexerError{
                                    elementa::base::concatWithMiddle(
                                        "Invalid lexical element",details)}
        {}
        
    ELE_CLASS_EXCOVERRIDE(InvLexElem)       
};
 
 
/* **********************************************************************
 
    Alias: TokenId
    Abstract Base class: Token
    Class: TextToken
 
*************************************************************************/
 
 
using TokenId = int;
 
 
class Token
{
 public:
 
    using Ptr = std::shared_ptr<Token>; 
    
    using PtrVector = std::vector<Ptr>; 
 
    TokenId c; 
 
    Token(void) = default;
    Token(TokenId tid):c{tid} {}
    Token(const Token &) = default;
    Token(Token &&) = default;
    Token & operator=(const Token &) = default;
    Token & operator=(Token &&) = default;
    virtual ~Token(void) = default;
 
    virtual bool operator==(const Token &) const = 0;
    virtual bool operator!=(const Token &o) const
        { return(!operator==(o)); }
 
    virtual const std::string & value(void) const = 0;
 
    virtual void setValue(const std::string & v) = 0;
 
    virtual std::string to_string(void) const;
};
 
 
class TextToken: public Token
{
 public:
 
    class List: public std::list<TextToken>
    {
     public:
 
        using Base = std::list<TextToken>;
        using Base::Base;
 
 
        std::string to_string(void) const;
    };
 
 
    TextToken(void) = default;
    TextToken(TokenId tid, const std::string & val):Token{tid},v_{val} {}
    TextToken(const TextToken & oth):Token{oth}
        { v_ = oth.v_; }
    TextToken(TextToken && oth):Token{std::move(oth)}
        { v_ = std::move(oth.v_); }
    TextToken & operator=(const TextToken & oth)
        { if (this != &oth) 
          { Token::operator=(oth); v_ = oth.v_; }
          return(*this); }
    TextToken & operator=(TextToken && oth)
        { if (this != &oth) 
          { Token::operator=(std::move(oth)); v_ = std::move(oth.v_); }
          return(*this); }
 
    void setValue(const std::string & v)
        { v_ = v; }
        
    bool operator==(const Token & o) const
        { return(v_ == dynamic_cast<const TextToken &>(o).v_); }
 
    const std::string & value(void) const { return(v_); }
 
 private:
 
    std::string v_;
};
 
 
/* **********************************************************************
 
    Pure abstract base class: Lexer
 
*************************************************************************/
 
 
class Lexer
{
 public:
 
 
    Lexer(elementa::base::SerChIFilter<elementa::base::SerChFilt_Loc> & inputch)
                                                            : inputch_{inputch} 
        {}
 
    virtual ~Lexer(void) = default;
    
    Lexer(const Lexer &) = delete;
    Lexer(Lexer &&) = delete;
    Lexer & operator=(const Lexer &) = delete;
    Lexer & operator=(Lexer &&) = delete;
 
    elementa::base::InSerCh & inputChannel(void) const noexcept 
        { return(inputch_.channel()); }
        
    elementa::base::SerChIFilter<elementa::base::SerChFilt_Loc> & 
        locInputChannel(void) const noexcept
            { return(inputch_); }
        
    elementa::base::SerChLoc & location(void) const noexcept
        { return(*(inputch_.filter().ploc)); }
 
 
    virtual Token::Ptr getToken(void) = 0;
 
 protected:
 
    elementa::base::SerChIFilter<elementa::base::SerChFilt_Loc> & inputch_;
};
 
 
/* **********************************************************************
 
    Class: FSMLexer
 
*************************************************************************/
 
 
class FSMLexer: public Lexer
{
 public:
 
    using FSMClass = elementa::adts::BaseTextFSM;
 
    using TermRecog = std::pair< TokenId , FSMClass::Ptr >;
 
    class Recognizers: public std::vector<TermRecog>
    {
     public:
        
        using BaseVector = std::vector<TermRecog>;
        
        using BaseVector::BaseVector;
        
        using ListOfInds = std::list<size_type>;
        
        Recognizers(const Recognizers &) = delete;
        Recognizers(Recognizers &&) = delete;
        Recognizers & operator=(const Recognizers &) = delete;
        Recognizers & operator=(Recognizers &&) = delete;
        
 
        FSMClass::Ptr find(TokenId tid) const
        {
            for (const auto & rp: *this) if (rp.first == tid) return(rp.second);
            return(FSMClass::Ptr{});
        }
        
        const ListOfInds & active(void) const noexcept { return(active_); }
        
 
        void reset(void);
        
 
        void feed(char o);
        
 
        size_type whoWins(void) const;
        
        void debug(void) const;
 
     private:
 
        Recognizers::size_type firstrecogfinished_;     
        ListOfInds active_; // list, for efficient dels 
    };
    
 
    class LexErr_Unrecog: public LexerError
    {
     public:
 
        LexErr_Unrecog(const std::string & unrecogtxt): 
            LexerError{kUnrecogPreff + unrecogtxt},
            offending_{unrecogtxt}
            {}
            
        ELE_CLASS_EXCOVERRIDE(LexErr_Unrecog)           
            
        
        const std::string & getOffendingText(void) const noexcept 
            { return(offending_); }
                        
     private:
     
        static const std::string kUnrecogPreff;
        
        std::string offending_;
    };
 
 
    FSMLexer(elementa::base::SerChIFilter<elementa::base::SerChFilt_Loc> 
                & inputch, 
             Recognizers & definition):     Lexer{inputch},
                                            recognizers_{definition}
        {}
 
    FSMLexer(const FSMLexer &) = delete;
    FSMLexer(FSMLexer &&) = delete;
    FSMLexer & operator=(const FSMLexer &) = delete;
    FSMLexer & operator=(FSMLexer &&) = delete;
 
 
    Token::Ptr getToken(void) final;
 
 private:
 
    Recognizers & recognizers_;
};
 
 
/* **********************************************************************
 
    Class: LexerCollector
 
*************************************************************************/
 
class LexerCollector
{
 public:
 
 
    using Observer = std::function<bool(Lexer &, 
                                        const LexerError *, Token::Ptr &)>;
 
 
    LexerCollector(Lexer & lexer, Observer & obs):lexer_{lexer},
                                                  obs_{obs} {}
                                            
    LexerCollector(const LexerCollector &) = delete;
    LexerCollector(LexerCollector &&) = delete;
    LexerCollector & operator=(const LexerCollector &) = delete;
    LexerCollector & operator=(LexerCollector &&) = delete;
                                                      
    virtual ~LexerCollector(void) = default;
 
 
    void collect(bool log = false);
    
    const Token::PtrVector & log(void) const noexcept { return(toks_); }
    
 private:
 
    Lexer & lexer_;
    Observer & obs_;
    Token::PtrVector toks_;
};
 
 
// Static asserts at the end because a bug in doxygen that stops generating
// code when this is found.
 
static_assert(std::is_integral<TokenId>::value &&
              std::is_signed<TokenId>::value,
              "TokenId must be an integral, signed type");
 // Lexer
 
} // End parsing namespace
 
} // End elementa namespace 
 
 
#endif