libsent/src/util/mystrtok.c

Go to the documentation of this file.

/*
 * Copyright (c) 1991-2006 Kawahara Lab., Kyoto University
 * Copyright (c) 2000-2005 Shikano Lab., Nara Institute of Science and Technology
 * Copyright (c) 2005-2006 Julius project team, Nagoya Institute of Technology
 * All rights reserved
 */

#include <sent/stddefs.h>

/* strtok with quotation handling */

#define ISTOKEN(A) strchr(delim, A)

char *
mystrtok_quotation(char *str, char *delim, int left_paren, int right_paren, int mode)
{
  static char *buf;             /* target string buffer */
  static char *pos;             /* current pointer position */
  char *p;
  char *from;
  int c;

  if (str != NULL) {
    pos = buf = str;
  }

  /* find start point */
  p = pos;
  while ((c = *p) != '\0' && ISTOKEN(c)) *p++;
  if (*p == '\0') return NULL;  /* no token left */

  /* if mode == 1, exit here */
  if (mode == 1) {
    pos = p;
    return p;
  }
  
  /* copy to ret_buf until end point is found */
  c = *p;
  if (c == left_paren) {
    p++;
    if (*p == '\0') return NULL;
    from = p;
    while ((c = *p) != '\0' && (c != right_paren)) p++;
    /* if quotation not terminated, allow the rest as one token */
    /* if (*p == '\0') return NULL; */
  } else {
    from = p;
    while ((c = *p) != '\0' && (!ISTOKEN(c))) p++;
  }
  if (*p != '\0') {
    *p = '\0';
    p++;
  }
  pos = p;
  return from;
}


char  *
mystrtok_quote(char *str, char *delim)
{
  return(mystrtok_quotation(str, delim, 34, 34, 0)); /* "\"" == 34 */
}

char  *
mystrtok(char *str, char *delim)
{
  return(mystrtok_quotation(str, delim, -1, -1, 0));
}

char *
mystrtok_movetonext(char *str, char *delim)
{
  return(mystrtok_quotation(str, delim, -1, -1, 1));
}

---