Simply add tokenizer.hpp to your c++ project !
// tokenizer initialization
tokenizer tok;
tok.set_dropped_delimiters(" \t\r\n");
tok.add_kept_delimiter("=");
tok.add_kept_delimiter("*");
tok.add_kept_delimiter("{");
tok.add_kept_delimiter("}");
tok.add_kept_delimiter("(");
tok.add_kept_delimiter(")");
tok.add_kept_delimiter(";");
tok.add_kept_delimiter("&");
tok.add_kept_delimiter("&&");
tok.add_kept_delimiter(":");
tok.add_kept_delimiter("::");
tok.add_kept_delimiter("\"");
// tokenize
tok.start(str);
while (!tok.finished())
{
const std::string& token = tok.next();
std::cout << token << "\n";
}So the following input string :
class my_class
{
private:
int* iptr = nullptr;
std::string str = "";
my_class(std::string&& _str);
}will output this tokens :
class
my_class
{
private
:
int
*
iptr
=
nullptr
;
std
::
string
str
=
"
"
;
my_class
(
std
::
string
&&
_str
)
;
}