How do I tokenize a string in C++?

Question

Java has a convenient split method: String str = "The quick brown fox"; String[] results = str.split(" "); Is there an easy way to do this in C++?

Answer 1

You can use strtok() which is in string.h

Be careful though, last time I saw it, it chokes on empty tokens where there is a double delimiter.

[ Or if the delimiter just happens to appear in the data. For example, if the delimiter is the comma,

then strtok() will choke on a field containing the name James Smith, Jr. ]

Otherwise, you have to parse it yourself...

Here's some code:

=============================================================

using namespace std;

#include <iostream>

#include <string.h>

#include <stdlib.h>

#define MAX_BUFF_SIZE (1000)

/******************************************************************

inbuff: input buffer is NULL terminated

tokens: array of strings is sufficient to store all of the tokens; buffers MUST already be allocated

delim: delimiter character

returns # of parsed tokens on success, or -1 on failure

tokens array is populated with parsed tokens

********************************************************************/

int MyStringTokenizer ( char * inbuff, char delim, char * tokens[] )

{

char * cur;

char * delimPtr;

int iReturn = -1;

char tempBuff[MAX_BUFF_SIZE];

//starts at the beginning

delimPtr = cur = &inbuff[0] ;

bool done_flag = false;

while (!done_flag)

{

//walks the delimiter pointer until either the delimiter or NULL terminator is found

while (

((*delimPtr) != delim) && ((*delimPtr) != 0)

)

{

delimPtr++;

}

// NULL terminator found: but still must parse last token, so sets the flag

if ((*delimPtr)==0)

{

done_flag=true;

}

//parses the token

memset(tempBuff,0,MAX_BUFF_SIZE);

memcpy(tempBuff,cur,delimPtr-cur);

strcpy(tokens[++iReturn],tempBuff);

//advances the pointers to next token

cur = delimPtr+1;

delimPtr = cur;

} //main while

//iReturn contains the array index, must increment

if (iReturn>0) { iReturn++; }

return(iReturn);

} //MyStringTokenizer

int main()

{

char inbuff[MAX_BUFF_SIZE];

char inBuff[MAX_BUFF_SIZE];

memset(inbuff,0,MAX_BUFF_SIZE);

strcpy(inbuff,"apples,oranges,grapes,cherries,lemon,peach,watermelon,pineapple,cantelope,strawberry,kiwi,banana");

strcpy(inBuff,"apples,oranges,grapes,cherries,lemon,peach,watermelon,pineapple,cantelope,strawberry,kiwi,banana");

char * tokens[12];

//first call to strtok requires the input buffer to be parsed

tokens[0] = strtok(inbuff,",");

cout << tokens[0]<< endl;

//subsequent calls require NULL

for (int iLoop=1; iLoop<12; iLoop++)

{

tokens[iLoop]= strtok(NULL,","); //returns NULL if there are no more tokens

cout << tokens[iLoop] << endl;

}

char *Tokens[12]; //allocates the memory

for (int iLoop=0; iLoop<12; iLoop++)

{

Tokens[iLoop]=new char[25];

}

int numTokens = MyStringTokenizer(inBuff,',',Tokens);

cout << "# of tokens: " << numTokens << endl;

//outputs the token and frees the memory

for (int iLoop=0; iLoop<numTokens; iLoop++)

{

cout << Tokens[iLoop] << endl;

delete(Tokens[iLoop]);

}

}