C++/std:string 토크나이즈

vector<string> tokenize(const string& str, const string& delimiters)
{
	vector<string> tokens;

string::size_type lastPos = 0, pos = 0;  
	int count = 0;

if(str.length()<1)  return tokens;

// skip delimiters at beginning.  
	lastPos = str.find_first_not_of(delimiters, 0);
	  
	if((str.substr(0, lastPos-pos).length()) > 0)
	{  	
		count = str.substr(0, lastPos-pos).length();

for(int i=0; i < count; i++)  	
 			tokens.push_back("");

if(string::npos == lastPos)
			tokens.push_back("");
	}

// find first "non-delimiter".
	pos = str.find_first_of(delimiters, lastPos);

while (string::npos != pos || string::npos != lastPos)
	{  	      	    
 		// found a token, add it to the vector.
 		tokens.push_back( str.substr(lastPos, pos - lastPos));
				
		// skip delimiters.  Note the "not_of"
 		lastPos = str.find_first_not_of(delimiters, pos);   	   	    
		
		if((string::npos != pos) && (str.substr(pos, lastPos-pos).length() > 1))  		
		{
			count = str.substr(pos, lastPos-pos).length();

for(int i=0; i < count; i++)
 				tokens.push_back("");
		}
		
		pos = str.find_first_of(delimiters, lastPos);
	}

return tokens;
}

...

string delimiters(" ");

// 공백 구분자로 토크나이즈			
vector<string> tokens = tokenize(src, delimiters);

명차 한자리에 랄프 로렌 자동차 컬렉션
↓
vector<string>
"명차" "한자리에" "랄프" "자동차" "컬렉션"

이 글에는 0 개의 댓글이 있습니다.

HyacinthWiki

In Search of the Miraculous

C++/std:string 토크나이즈