AndroidCamera/FaceBlit/dlib-19.21/dlib/tokenizer/tokenizer_kernel_1.cpp

// Copyright (C) 2005  Davis E. King (davis@dlib.net)
// License: Boost Software License   See LICENSE.txt for the full license.
#ifndef DLIB_TOKENIZER_KERNEL_1_CPp_
#define DLIB_TOKENIZER_KERNEL_1_CPp_
#include "tokenizer_kernel_1.h"

#include <iostream>
#include <cstdio>

namespace dlib
{

// ----------------------------------------------------------------------------------------

    tokenizer_kernel_1::
    tokenizer_kernel_1 (        
    ) :
        headset(0),
        bodyset(0),
        have_peeked(false)
    {
        try
        {
            headset = new bool[UCHAR_MAX];
            bodyset = new bool[UCHAR_MAX];

            clear();
        }
        catch (...)
        {
            if (headset) delete [] headset;
            if (bodyset) delete [] bodyset;
            throw;
        }
    }

// ----------------------------------------------------------------------------------------

    tokenizer_kernel_1::
    ~tokenizer_kernel_1 (
    )
    {
        delete [] bodyset;
        delete [] headset;
    }

// ----------------------------------------------------------------------------------------

    void tokenizer_kernel_1::
    clear(
    )
    {
        using namespace std;

        in = 0;
        streambuf = 0;
        have_peeked = false;

        head = "_" + lowercase_letters() + uppercase_letters();
        body = "_" + lowercase_letters() + uppercase_letters() + numbers();

        for (unsigned long i = 0; i < UCHAR_MAX; ++i)
        {
            headset[i] = false;
            bodyset[i] = false;
        }

        for (string::size_type i = 0; i < head.size(); ++i)
            headset[static_cast<unsigned char>(head[i])] = true;
        for (string::size_type i = 0; i < body.size(); ++i)
            bodyset[static_cast<unsigned char>(body[i])] = true;
    }

// ----------------------------------------------------------------------------------------

    void tokenizer_kernel_1::
    set_stream (
        std::istream& in_
    )
    {
        in = &in_;
        streambuf = in_.rdbuf();
        have_peeked = false;
    }

// ----------------------------------------------------------------------------------------

    bool tokenizer_kernel_1::
    stream_is_set (
    ) const
    {
        return (in != 0);
    }

// ----------------------------------------------------------------------------------------

    std::istream& tokenizer_kernel_1::
    get_stream (
    ) const
    {
        return *in;
    }

// ----------------------------------------------------------------------------------------

    void tokenizer_kernel_1::
    get_token (
        int& type,
        std::string& token
    )
    {
        if (!have_peeked)
        {
            std::streambuf::int_type ch;
            ch = streambuf->sbumpc();

            switch (ch)
            {
            case EOF:
                type = END_OF_FILE;
                token.clear();
                return;

            case '\n':
                type = END_OF_LINE;
                token = "\n";
                return;

            case '\r':
            case ' ':
            case '\t':
                type = WHITE_SPACE;
                token = static_cast<char>(ch);
                ch = streambuf->sgetc();
                while ((ch == ' ' || ch == '\t' || ch == '\r') && ch != EOF)
                {
                    token += static_cast<char>(ch);
                    ch = streambuf->snextc();
                }
                return;

            default:
                if (headset[static_cast<unsigned char>(ch)])
                {
                    type = IDENTIFIER;
                    token = static_cast<char>(ch);
                    ch = streambuf->sgetc();
                    while ( bodyset[static_cast<unsigned char>(ch)] && ch != EOF )
                    {
                        token += static_cast<char>(ch);
                        ch = streambuf->snextc();
                    }
                }
                else if ('0' <= ch && ch <= '9')
                {
                    type = NUMBER;
                    token = static_cast<char>(ch);
                    ch = streambuf->sgetc();
                    while (('0' <= ch && ch <= '9') && ch != EOF)
                    {
                        token += static_cast<char>(ch);
                        ch = streambuf->snextc();
                    }
                }
                else
                {
                    type = CHAR;
                    token = static_cast<char>(ch);
                }
                return;
            } // switch (ch)
        }
        
        // if we get this far it means we have peeked so we should 
        // return the peek data.
        type = next_type;
        token = next_token;
        have_peeked = false;
    }

// ----------------------------------------------------------------------------------------

    int tokenizer_kernel_1::
    peek_type (
    ) const
    {
        const_cast<tokenizer_kernel_1*>(this)->get_token(next_type,next_token);
        have_peeked = true;
        return next_type;
    }

// ----------------------------------------------------------------------------------------

    const std::string& tokenizer_kernel_1::
    peek_token (
    ) const
    {
        const_cast<tokenizer_kernel_1*>(this)->get_token(next_type,next_token);
        have_peeked = true;
        return next_token;
    }

// ----------------------------------------------------------------------------------------

    void tokenizer_kernel_1::
    swap (
        tokenizer_kernel_1& item
    )
    {
        exchange(in,item.in);
        exchange(streambuf,item.streambuf);
        exchange(head,item.head);
        exchange(body,item.body);
        exchange(bodyset,item.bodyset);
        exchange(headset,item.headset);
        exchange(have_peeked,item.have_peeked);
        exchange(next_type,item.next_type);
        exchange(next_token,item.next_token);
    }

// ----------------------------------------------------------------------------------------
    
    void tokenizer_kernel_1::
    set_identifier_token (
        const std::string& head_,
        const std::string& body_
    )
    {
        using namespace std;

        head = head_;
        body = body_;

        for (unsigned long i = 0; i < UCHAR_MAX; ++i)
        {
            headset[i] = false;
            bodyset[i] = false;
        }

        for (string::size_type i = 0; i < head.size(); ++i)
            headset[static_cast<unsigned char>(head[i])] = true;
        for (string::size_type i = 0; i < body.size(); ++i)
            bodyset[static_cast<unsigned char>(body[i])] = true;
    }

// ----------------------------------------------------------------------------------------
    
    const std::string tokenizer_kernel_1::
    get_identifier_head (
    ) const
    {
        return head;
    }

// ----------------------------------------------------------------------------------------
    
    const std::string tokenizer_kernel_1::
    get_identifier_body (
    ) const
    {
        return body;
    }

// ----------------------------------------------------------------------------------------
    
    const std::string tokenizer_kernel_1::
    lowercase_letters (
    ) const
    {
        return std::string("abcdefghijklmnopqrstuvwxyz");
    }

// ----------------------------------------------------------------------------------------
    
    const std::string tokenizer_kernel_1::
    uppercase_letters (
    ) const
    {
        return std::string("ABCDEFGHIJKLMNOPQRSTUVWXYZ");
    }

// ----------------------------------------------------------------------------------------
    
    const std::string tokenizer_kernel_1::
    numbers (
    ) const
    {
        return std::string("0123456789");
    }
    
// ----------------------------------------------------------------------------------------
    
}
#endif // DLIB_TOKENIZER_KERNEL_1_CPp_