Something went wrong. Try again.
Reactos
Something went wrong. Try again.
C++
at listview
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313// tokenize.cpp
#ifdef _MSC_VER#pragma warning ( disable : 4786 )#endif//_MSC_VER
#include <string>#include <vector>#include <conio.h>
#include "assert.h"#include "tokenize.h"#include "skip_ws.h"
using std::string;using std::vector;
void tokenize ( const string& text, vector<string>& tokens ){ tokens.resize ( 0 ); string s ( text ); char* p = &s[0]; while ( *p ) { // skip whitespace p = skip_ws ( p ); // check for literal string if ( *p == '\"' ) { // skip initial quote char* end = p + 1; for ( ;; ) { if ( *end == '\\' ) { end++; switch ( *end ) { case 'x': case 'X': ASSERT(0); // come back to this.... break; case '0': ASSERT(0); break; default: end++; break; } } else if ( *end == '\"' ) { end++; break; } else end++; } tokens.push_back ( string ( p, end-p ) ); p = end; } else if ( __iscsymf(*p) ) { char* end = p + 1; while ( __iscsym ( *end ) ) end++; tokens.push_back ( string ( p, end-p ) ); p = end; } else if ( isdigit(*p) || *p == '.' ) { char* end = p; while ( isdigit(*end) ) end++; bool f = false; if ( *end == '.' ) { end++; while ( isdigit(*end) ) end++; f = true; } if ( *end == 'f' || *end == 'F' ) end++; else if ( !f && ( *end == 'l' || *end == 'L' ) ) end++; tokens.push_back ( string ( p, end-p ) ); p = end; } else switch ( *p ) { case '.': tokens.push_back ( "." ); p++; break; case ',': tokens.push_back ( "," ); p++; break; case '(': tokens.push_back ( "(" ); p++; break; case ')': tokens.push_back ( ")" ); p++; break; case '{': tokens.push_back ( "{" ); p++; break; case '}': tokens.push_back ( "}" ); p++; break; case '[': tokens.push_back ( "[" ); p++; break; case ']': tokens.push_back ( "]" ); p++; break; case ';': tokens.push_back ( ";" ); p++; break; case '\\': switch ( p[1] ) { case '\n': tokens.push_back ( string ( p, 2 ) ); p += 2; break; default: ASSERT(0); // shouldn't hit here, I think tokens.push_back ( "\\" ); p++; break; } break; case '|': switch ( p[1] ) { case '|': tokens.push_back ( string ( p, 2 ) ); p += 2; break; default: tokens.push_back ( "|" ); p++; break; } break; case '&': switch ( p[1] ) { case '&': tokens.push_back ( string ( p, 2 ) ); p += 2; break; default: tokens.push_back ( "&" ); p++; break; } break; case '<': switch ( p[1] ) { case '<': if ( p[2] == '=' ) tokens.push_back ( string ( p, 3 ) ), p += 3; else tokens.push_back ( string ( p, 2 ) ), p += 2; break; case '=': tokens.push_back ( string ( p, 2 ) ); p += 2; break; default: tokens.push_back ( "<" ); p++; break; } break; case '>': switch ( p[1] ) { case '>': if ( p[2] == '=' ) tokens.push_back ( string ( p, 3 ) ), p += 3; else tokens.push_back ( string ( p, 2 ) ), p += 2; break; case '=': tokens.push_back ( string ( p, 2 ) ); p += 2; break; default: tokens.push_back ( ">" ); p++; break; } break; case '!': switch ( p[1] ) { case '=': tokens.push_back ( string ( p, 2 ) ); p += 2; break; default: tokens.push_back ( "!" ); p++; break; } break; case '=': switch ( p[1] ) { case '=': tokens.push_back ( string ( p, 2 ) ); p += 2; break; default: tokens.push_back ( "=" ); p++; break; } break; case ':': switch ( p[1] ) { case ':': tokens.push_back ( string ( p, 2 ) ); p += 2; break; default: tokens.push_back ( ":" ); p++; break; } break; case '*': switch ( p[1] ) { case '=': tokens.push_back ( string ( p, 2 ) ); p += 2; break; default: tokens.push_back ( "*" ); p++; break; } break; case '/': switch ( p[1] ) { case '=': tokens.push_back ( string ( p, 2 ) ); p += 2; break; default: tokens.push_back ( "/" ); p++; break; } break; case '+': switch ( p[1] ) { case '+': case '=': tokens.push_back ( string ( p, 2 ) ); p += 2; break; default: tokens.push_back ( "+" ); p++; break; } break; case '-': switch ( p[1] ) { case '-': case '=': tokens.push_back ( string ( p, 2 ) ); p += 2; break; default: tokens.push_back ( "-" ); p++; break; } break; case '#': while ( *p && *p != '\n' ) p++; break; case 0: break; default: printf ( "choked on '%c' in tokenize() - press any key to continue\n", *p ); getch(); p++; break; } }}