libsent/src/voca/voca_load_wordlist.c

Go to the documentation of this file.

/*
 * Copyright (c) 1991-2007 Kawahara Lab., Kyoto University
 * Copyright (c) 2000-2005 Shikano Lab., Nara Institute of Science and Technology
 * Copyright (c) 2005-2007 Julius project team, Nagoya Institute of Technology
 * All rights reserved
 */

#include <sent/stddefs.h>
#include <sent/vocabulary.h>
#include <sent/htk_hmm.h>

/* 
 * dictinary format:
 * 
 * 1 words per line.
 * 
 * fields: OutputString phone1 phone2 ....
 * 
 *     OutputString
 *                 String to output when the word is recognized.
 *
 *     phone1 phone2 ....
 *                 sequence of logical HMM name (normally phoneme)
 *                 to express the pronunciation
 */

#define PHONEMELEN_STEP  30     
static char buf[MAXLINELEN];    
static char bufbak[MAXLINELEN]; 


static void
add_to_error(WORD_INFO *winfo, char *name)
{
  char *buf;
  char *match;

  buf = (char *)mymalloc(strlen(name) + 1);
  strcpy(buf, name);
  if (winfo->errph_root == NULL) {
    winfo->errph_root = aptree_make_root_node(buf);
  } else {
    match = aptree_search_data(buf, winfo->errph_root);
    if (match == NULL || !strmatch(match, buf)) {
      aptree_add_entry(buf, buf, match, &(winfo->errph_root));
    }
  }
}

static void
callback_list_error(void *x)
{
  char *name;
  name = x;
  jlog("Error: voca_load_wordlist: %s\n", name);
}
static void
list_error(WORD_INFO *winfo)
{
  jlog("Error: voca_load_wordlist: begin missing phones\n");
  aptree_traverse_and_do(winfo->errph_root, callback_list_error);
  jlog("Error: voca_load_wordlist: end missing phones\n");
}

boolean
voca_load_word_line(char *buf, WORD_INFO *winfo, HTK_HMM_INFO *hmminfo, char *headphone, char *tailphone, char *contextphone)
{
  WORD_ID vnum;

  winfo->linenum++;
  vnum = winfo->num;
  if (vnum >= winfo->maxnum) {
    if (winfo_expand(winfo) == FALSE) return FALSE;
  }
  if (voca_load_wordlist_line(buf, &vnum, winfo->linenum, winfo, hmminfo, winfo->do_conv, &(winfo->ok_flag), headphone, tailphone, contextphone) == FALSE) {
    return FALSE;
  }
  winfo->num = vnum;
  return TRUE;
}
boolean
voca_load_wordlist(FILE *fp, WORD_INFO *winfo, HTK_HMM_INFO *hmminfo, char *headphone, char *tailphone, char *contextphone)
{
  boolean ret;

  voca_load_start(winfo, hmminfo, FALSE);
  while (getl(buf, sizeof(buf), fp) != NULL) {
    if (voca_load_word_line(buf, winfo, hmminfo, headphone, tailphone, contextphone) == FALSE) break;
  }
  ret = voca_load_end(winfo);

  return(ret);
}

boolean
voca_load_wordlist_line(char *buf, WORD_ID *vnum_p, int linenum, WORD_INFO *winfo, HTK_HMM_INFO *hmminfo, boolean do_conv, boolean *ok_flag, char *headphone, char *tailphone, char *contextphone)
{
  char *ptmp, *lp = NULL, *p;
  static char cbuf[MAX_HMMNAME_LEN];
  static HMM_Logical **tmpwseq = NULL;
  static int tmpmaxlen;
  int len;
  HMM_Logical *tmplg;
  boolean pok, first;
  int vnum;

  vnum = *vnum_p;

  if (strmatch(buf, "DICEND")) return FALSE;

  /* allocate temporal work area for the first call */
  if (tmpwseq == NULL) {
    tmpmaxlen = PHONEMELEN_STEP;
    tmpwseq = (HMM_Logical **)mymalloc(sizeof(HMM_Logical *) * tmpmaxlen);
  }

  /* backup whole line for debug output */
  strcpy(bufbak, buf);
  
  /* Output string */
  if ((ptmp = mystrtok_quote(buf, " \t\n")) == NULL) {
    jlog("Error: voca_load_wordlist: line %d: corrupted data:\n> %s\n", linenum, bufbak);
    winfo->errnum++;
    *ok_flag = FALSE;
    return TRUE;
  }
  winfo->wname[vnum] = strcpy((char *)mybmalloc2(strlen(ptmp)+1, &(winfo->mroot)), ptmp);

  /* reset transparent flag */
  winfo->is_transparent[vnum] = FALSE;

  /* just move pointer to next token */
  if ((ptmp = mystrtok_movetonext(NULL, " \t\n")) == NULL) {
    jlog("Error: voca_load_wordlist: line %d: corrupted data:\n> %s\n", linenum, bufbak);
    winfo->errnum++;
    *ok_flag = FALSE;
    return TRUE;
  }
#ifdef CLASS_NGRAM
  winfo->cprob[vnum] = 0.0;     /* prob = 1.0, logprob = 0.0 */
#endif
  
  if (ptmp[0] == '@') {         /* class N-gram prob */
#ifdef CLASS_NGRAM
    /* word probability within the class (for class N-gram) */
    /* format: classname @classprob wordname [output] phoneseq */
    /* classname equals to wname, and wordname will be omitted */
    /* format: @%f (log scale) */
    /* if "@" not found or "@0", it means class == word */
    if ((ptmp = mystrtok(NULL, " \t\n")) == NULL) {
      jlog("Error: voca_load_wordlist: line %d: corrupted data:\n> %s\n", linenum, bufbak);
      winfo->errnum++;
      *ok_flag = FALSE;
      return TRUE;
    }
    if (ptmp[1] == '\0') {      /* space between '@' and figures */
      jlog("Error: voca_load_wordlist: line %d: value after '@' missing, maybe wrong space?\n> %s\n", linenum, bufbak);
      winfo->errnum++;
      *ok_flag = FALSE;
      return TRUE;
    }
    winfo->cprob[vnum] = atof(&(ptmp[1]));
    if (winfo->cprob[vnum] != 0.0) winfo->cwnum++;
    /* read next word entry (just skip them) */
    if ((ptmp = mystrtok(NULL, " \t\n")) == NULL) {
      jlog("Error: voca_load_wordlist: line %d: corrupted data:\n> %s\n", linenum,bufbak);
      winfo->errnum++;
      *ok_flag = FALSE;
      return TRUE;
    }
    /* move to the next word entry */
    if ((ptmp = mystrtok_movetonext(NULL, " \t\n")) == NULL) {
      jlog("Error: voca_load_wordlist: line %d: corrupted data:\n> %s\n", linenum, bufbak);
      winfo->errnum++;
      *ok_flag = FALSE;
      return TRUE;
    }
#else  /* ~CLASS_NGRAM */
    jlog("Error: voca_load_wordlist: line %d: cannot handle in-class word probability\n> %s\n", linenum, ptmp, bufbak);
    winfo->errnum++;
    *ok_flag = FALSE;
    return TRUE;
#endif /* CLASS_NGRAM */
  }

  /* OutputString */
  switch(ptmp[0]) {
  case '[':                     /* ignore transparency */
    ptmp = mystrtok_quotation(NULL, " \t\n", '[', ']', 0);
    break;
  case '{':                     /* ignore transparency */
    ptmp = mystrtok_quotation(NULL, " \t\n", '{', '}', 0);
    break;
  default:
    /* ALLOW no entry for output */
    /* same as wname is used */
    ptmp = winfo->wname[vnum];
  }
  if (ptmp == NULL) {
    jlog("Error: voca_load_htkdict: line %d: corrupted data:\n> %s\n", linenum, bufbak);
    winfo->errnum++;
    *ok_flag = FALSE;
    return TRUE;
  }
  winfo->woutput[vnum] = strcpy((char *)mybmalloc2(strlen(ptmp)+1, &(winfo->mroot)), ptmp);
    
  /* phoneme sequence */
  if (hmminfo == NULL) {
    /* don't read */
    winfo->wseq[vnum] = NULL;
    winfo->wlen[vnum] = 0;
  } else {

    len = 0;
    first = TRUE;
    pok = TRUE;

    for (;;) {
      if (do_conv) {
        if (first) {
          /* init phone cycler */
          cycle_triphone(NULL);
          /* insert head phone at beginning of word */
          if (contextphone) {
            cycle_triphone(contextphone);
          } else {
            cycle_triphone("NULL_C");
          }
          if ((lp = mystrtok(NULL, " \t\n")) == NULL) {
            jlog("Error: voca_load_wordlist: line %d: word %s has no phoneme:\n> %s\n", linenum, winfo->wname[vnum], bufbak);
            winfo->errnum++;
            *ok_flag = FALSE;
            return TRUE;
          }
          p = cycle_triphone(lp);
          first = FALSE;
        } else {                /* do_conv, not first */
          if (lp != NULL) {     /* some token processed at last loop */
            lp = mystrtok(NULL, " \t\n");
            if (lp != NULL) {
              /* token exist */
              p = cycle_triphone(lp);
            } else {
              /* no more token, insert tail phone at end of word */
              if (contextphone) {
                p = cycle_triphone(contextphone);
              } else {
                p = cycle_triphone("NULL_C");
              }
            }
          } else {              /* no more token at last input  */
            /* flush tone cycler */
            p = cycle_triphone_flush();
          }
        }
      } else {                  /* not do_conv */
        if (first) {
          p = lp = headphone;
          first = FALSE;
        } else {
          if (lp != NULL) {     /* some token processed at last loop */
            p = lp = mystrtok(NULL, " \t\n");
            /* if no more token, use tailphone */
            if (lp == NULL) p = tailphone;
          } else {
            /* no more token at last input, exit loop */
            p = NULL;
          }
        }
      }
      if (p == NULL) break;
      /* for headphone and tailphone, their context should not be handled */
      /* and when they appear as context they should be replaced by contextphone */
      if (do_conv) {
        center_name(p, cbuf);
        if (contextphone) {
          if (strmatch(cbuf, contextphone)) {
            if (len == 0) {
              p = headphone;
            } else if (lp == NULL) {
              p = tailphone;
            }
          }
        } else {
          if (strmatch(cbuf, "NULL_C")) {
            if (len == 0) {
              p = headphone;
            } else if (lp == NULL) {
              p = tailphone;
            }
          } else {
            if (strnmatch(p, "NULL_C", 6)) {
              if (strnmatch(&(p[strlen(p)-6]), "NULL_C", 6)) {
                p = cbuf;
              } else {
                p = rightcenter_name(p, cbuf);
              }
            } else if (strnmatch(&(p[strlen(p)-6]), "NULL_C", 6)) {
              p = leftcenter_name(p, cbuf);
            }
          }
        }
      }
      //printf("[[%s]]\n", p);

      /* both defined/pseudo phone is allowed */
      tmplg = htk_hmmdata_lookup_logical(hmminfo, p);
      if (tmplg == NULL) {
        /* not found */
        if (do_conv) {
          /* logical phone was not found */
          jlog("Error: voca_load_wordlist: line %d: logical phone \"%s\" not found\n", linenum, p);
          snprintf(cbuf,MAX_HMMNAME_LEN,"%s", p);
        } else {
          jlog("Error: voca_load_wordlist: line %d: phone \"%s\" not found\n", linenum, p);
          snprintf(cbuf, MAX_HMMNAME_LEN, "%s", p);
        }
        add_to_error(winfo, cbuf);
        pok = FALSE;
      } else {
        /* found */
        if (len >= tmpmaxlen) {
          /* expand wseq area by PHONEMELEN_STEP */
          tmpmaxlen += PHONEMELEN_STEP;
          tmpwseq = (HMM_Logical **)myrealloc(tmpwseq, sizeof(HMM_Logical *) * tmpmaxlen);
        }
        /* store to temporal buffer */
        tmpwseq[len] = tmplg;
      }
      len++;
    }
    if (!pok) {                 /* error in phoneme */
      jlog("Error: voca_load_wordlist: the line content was: %s\n", bufbak);
      winfo->errnum++;
      *ok_flag = FALSE;
      return TRUE;
    }
    if (len == 0) {
      jlog("Error: voca_load_wordlist: line %d: no phone specified:\n> %s\n", linenum, bufbak);
      winfo->errnum++;
      *ok_flag = FALSE;
      return TRUE;
    }
    /* store to winfo */
    winfo->wseq[vnum] = (HMM_Logical **)mybmalloc2(sizeof(HMM_Logical *) * len, &(winfo->mroot));
    memcpy(winfo->wseq[vnum], tmpwseq, sizeof(HMM_Logical *) * len);
    winfo->wlen[vnum] = len;
  }

  vnum++;
  *vnum_p = vnum;
  
  return(TRUE);
}

---