// Author: Steve Lombardi
// Date: May 3, 2010
// Originally posted on www.stephenlombardi.com
// Purpose: make Boost.Tokenizer useful

#include <iostream>
#include <fstream>
#include <iterator>
#include <list>
#include <numeric>
#include <sstream>
#include <string>

#include <boost/tokenizer.hpp>

// helper functions for creating a hetergenous container of tfuncs

template< class IteratorTT >
class tfunc_polym {
public:
	virtual bool operator( )( IteratorTT & next, IteratorTT end, std::string & tok ) const = 0;
	virtual void reset( ) = 0;
};

template< class TFunc, class IteratorT >
class tfunc_polym_derived : public tfunc_polym< IteratorT > {
public:
	tfunc_polym_derived( TFunc tfunc ) : tfunc_( tfunc ) { }
	bool operator( )( IteratorT & next, IteratorT end, std::string & tok ) const {
		return tfunc_( next, end, tok );
	}
	void reset( ) {
		tfunc_.reset( );
	}
private:
	TFunc tfunc_;
};

template< class IteratorT >
class tfunc_polym_adapter {
public:
	tfunc_polym_adapter( tfunc_polym< IteratorT > * tfunc ) : tfunc_( tfunc ) { }
	bool operator( )( IteratorT & next, IteratorT end, std::string & tok ) const {
		return (*tfunc_)( next, end, tok );
	}
	void reset( ) {
		tfunc_->reset( );
	}
private:
	tfunc_polym< IteratorT > * tfunc_;
};

// basic tfunc types

class find_str_tfunc {
public:
	find_str_tfunc( const std::string & str ) : str_( str ) { }
	template< class Iterator >
	bool operator( )( Iterator & next, Iterator end, std::string & tok ) const {
		std::string::const_iterator iter = str_.begin( );
		for( ; *iter == *next && iter != str_.end( ); ++iter, ++next );
		if( iter == str_.end( ) ) {
			tok = str_;
			return true;
		} else {
			return false;
		}
	}
	void reset( ) {
	}
private:
	std::string str_;
};

class find_ident_tfunc {
public:
	template< class Iterator >
	bool operator( )( Iterator & next, Iterator end, std::string & tok ) const {
		Iterator begin = next;
		if( *next >= 'A' && *next <= 'Z' || *next >= 'a' && *next <= 'z' ) {
			++next;
			for( ; ( *next >= 'A' && *next <= 'Z' || *next >= 'a' && *next <= 'z' || *next >= '0' && *next <= '9' ) && next != end; ++next );
			tok = std::string( begin, next );
			return true;
		} else {
			return false;
		}
	}
	void reset( ) {
	}
private:
};

class find_integer_tfunc {
public:
	template< class Iterator >
	bool operator( )( Iterator & next, Iterator end, std::string & tok ) const {
		Iterator begin = next;
		if( *next >= '0' && *next <= '9' ) {
			++next;
			for( ; ( *next >= '0' && *next <= '9' ) && next != end; ++next );
			//tok = std::string( begin, next );
			tok = std::accumulate( begin, next, std::string( ) );
			return true;
		} else {
			return false;
		}
	}
	void reset( ) {
	}
private:
};

// tfunc type to combine other tfuncs

template< class TFunc >
class multi_tfunc {
public:
	template< class X >
	multi_tfunc( X begin, X end ) : tfuncs_( begin, end ) { }

	template< class Iterator >
	bool operator( )( Iterator & next, Iterator end, std::string & tok ) const {
		for( ; *next == 9 || *next == 10 || *next == 13 || *next == 32; ++next );

		std::string longest;
		for( typename std::list< TFunc >::const_iterator iter = tfuncs_.begin( ); iter != tfuncs_.end( ); ++iter ) {
			Iterator nextcopy = next;
			std::string result;
			if( (*iter)( nextcopy, end, result ) && result.size( ) > longest.size( ) ) {
				longest = result;
			}
		}

		if( longest.empty( ) ) {
			return false;
		} else {
			std::advance( next, longest.size( ) );
			tok = longest;
			return true;
		}
	}
	void reset( ) {
	}
private:
	std::list< TFunc > tfuncs_;
};

// buffered_iterator

template< class IterT, class IterDerefT >
class buffered_iterator : public std::iterator< std::forward_iterator_tag, IterDerefT, ptrdiff_t, IterDerefT*, IterDerefT& > {
public:
	struct refcounter {
		refcounter( IterT iter ) : iter_( iter ), buffer_( std::list< IterDerefT >( 1 ) ), refs_( std::list< int >( 1, 0 ) ) { }
		IterT iter_;
		std::list< IterDerefT > buffer_;
		std::list< int > refs_;
	};

	explicit buffered_iterator( IterT iter ) 
		: refcounter_( new refcounter( iter ) ), bufferiter_( refcounter_->buffer_.begin( ) ), refiter_( refcounter_->refs_.begin( ) ) { alloc( ); ++*this; }
	buffered_iterator( const buffered_iterator< IterT, IterDerefT > & x ) 
		: refcounter_( x.refcounter_ ), bufferiter_( x.bufferiter_ ), refiter_( x.refiter_ ) { alloc( ); }
	~buffered_iterator( ) { dealloc( ); }

	buffered_iterator< IterT, IterDerefT > & operator=( const buffered_iterator< IterT, IterDerefT > & x ) {
		if( this != &x ) {
			dealloc( );
			refcounter_ = x.refcounter_;
			bufferiter_ = x.bufferiter_;
			refiter_ = x.refiter_;
			alloc( );
		}
	}

	buffered_iterator< IterT, IterDerefT > & operator++( ) {
		// increment the buffer iterators
		++bufferiter_;
		--(*refiter_);
		++refiter_;

		if( bufferiter_ == refcounter_->buffer_.end( ) && refiter_ == refcounter_->refs_.end( ) ) {
			// get input from iterator if we've hit the end of the buffer
			refcounter_->buffer_.push_back( *refcounter_->iter_ );
			refcounter_->refs_.push_back( 1 );
			++refcounter_->iter_;

			// adjust the buffer iterators so they point to a valid location
			--bufferiter_;
			--refiter_;
		} else {
			++(*refiter_);
		}

		minimize( );

		return *this;
	}

	buffered_iterator< IterT, IterDerefT > operator++( int ) {
		buffered_iterator< IterT, IterDerefT > tmp = *this;
		++*this;
		return tmp;
	}

	IterDerefT & operator*( ) {
		return *bufferiter_;
	}

	bool operator==( const buffered_iterator< IterT, IterDerefT > & x ) const {
		if( refcounter_ == x.refcounter_ ) {
			return bufferiter_ == x.bufferiter_;
		} else {
			return refcounter_->iter_ == x.refcounter_->iter_;
		}
	}

	bool operator!=( const buffered_iterator< IterT, IterDerefT > & x ) const {
		return !( *this == x );
	}
private:
	void alloc( ) {
		++(*refiter_);
	}

	void dealloc( ) {
		--(*refiter_);

		minimize( );

		if( refcounter_->refs_.empty( ) ) {
			delete refcounter_;
		}
	}

	void minimize( ) {
		// remove any portions of the beginning of the buffer that aren't referenced
		for( std::list< int >::iterator iter = refcounter_->refs_.begin( ); iter != refcounter_->refs_.end( ) && *iter == 0; ) {
			++iter;
			refcounter_->buffer_.pop_front( );
			refcounter_->refs_.pop_front( );
		}
	}

	refcounter * refcounter_;
	typename std::list< IterDerefT >::iterator bufferiter_;
	std::list< int >::iterator refiter_;
};

// a basic example
typedef buffered_iterator< std::istreambuf_iterator< char >, char > fiter_t;
typedef multi_tfunc< tfunc_polym_adapter< fiter_t > > multi_tfunc_t;
typedef boost::token_iterator_generator< multi_tfunc_t, fiter_t >::type titer_t;

int main( ) {
	std::list< tfunc_polym_adapter< fiter_t > > tfuncs;
	tfuncs.push_back( new tfunc_polym_derived< find_str_tfunc, fiter_t >( find_str_tfunc( "while" ) ) );
	tfuncs.push_back( new tfunc_polym_derived< find_str_tfunc, fiter_t >( find_str_tfunc( "(" ) ) );
	tfuncs.push_back( new tfunc_polym_derived< find_str_tfunc, fiter_t >( find_str_tfunc( ")" ) ) );
	tfuncs.push_back( new tfunc_polym_derived< find_str_tfunc, fiter_t >( find_str_tfunc( "{" ) ) );
	tfuncs.push_back( new tfunc_polym_derived< find_str_tfunc, fiter_t >( find_str_tfunc( "}" ) ) );
	tfuncs.push_back( new tfunc_polym_derived< find_str_tfunc, fiter_t >( find_str_tfunc( ">" ) ) );
	tfuncs.push_back( new tfunc_polym_derived< find_ident_tfunc, fiter_t >( find_ident_tfunc( ) ) );
	tfuncs.push_back( new tfunc_polym_derived< find_integer_tfunc, fiter_t >( find_integer_tfunc( ) ) );

	std::ifstream in( "input.txt" );
	fiter_t last = fiter_t( std::istreambuf_iterator< char >( ) );

	multi_tfunc_t tfunc = multi_tfunc_t( tfuncs.begin( ), tfuncs.end( ) );
	titer_t beg = boost::make_token_iterator< std::string >( fiter_t( std::istreambuf_iterator< char >( in ) ), last, tfunc );
	titer_t end = boost::make_token_iterator< std::string >( last, last, tfunc );

	for( ; beg != end; ++beg ) {
		std::cout << *beg << std::endl;
	}


	return 0;
}