libjulius/src/gmm.c

Go to the documentation of this file.

/*
 * Copyright (c) 2003-2005 Shikano Lab., Nara Institute of Science and Technology
 * Copyright (c) 2005-2007 Julius project team, Nagoya Institute of Technology
 * All rights reserved
 */

#include <julius/julius.h>

#undef MES

static int
gmm_find_insert_point(GMMCalc *gc, LOGPROB score, int len)
{
  /* binary search on score */
  int left = 0;
  int right = len - 1;
  int mid;

  while (left < right) {
    mid = (left + right) / 2;
    if (gc->OP_calced_score[mid] > score) {
      left = mid + 1;
    } else {
      right = mid;
    }
  }
  return(left);
}

static int
gmm_cache_push(GMMCalc *gc, int id, LOGPROB score, int len)
{
  int insertp;

  if (len == 0) {               /* first one */
    gc->OP_calced_score[0] = score;
    gc->OP_calced_id[0] = id;
    return(1);
  }
  if (gc->OP_calced_score[len-1] >= score) { /* bottom */
    if (len < gc->OP_gprune_num) {          /* append to bottom */
      gc->OP_calced_score[len] = score;
      gc->OP_calced_id[len] = id;
      len++;
    }
    return len;
  }
  if (gc->OP_calced_score[0] < score) {
    insertp = 0;
  } else {
    insertp = gmm_find_insert_point(gc, score, len);
  }
  if (len < gc->OP_gprune_num) {
    memmove(&(gc->OP_calced_score[insertp+1]), &(gc->OP_calced_score[insertp]), sizeof(LOGPROB)*(len - insertp));
    memmove(&(gc->OP_calced_id[insertp+1]), &(gc->OP_calced_id[insertp]), sizeof(int)*(len - insertp));    
  } else if (insertp < len - 1) {
    memmove(&(gc->OP_calced_score[insertp+1]), &(gc->OP_calced_score[insertp]), sizeof(LOGPROB)*(len - insertp - 1));
    memmove(&(gc->OP_calced_id[insertp+1]), &(gc->OP_calced_id[insertp]), sizeof(int)*(len - insertp - 1));
  }
  gc->OP_calced_score[insertp] = score;
  gc->OP_calced_id[insertp] = id;
  if (len < gc->OP_gprune_num) len++;
  return(len);
}

static LOGPROB
gmm_compute_g_base(GMMCalc *gc, HTK_HMM_Dens *binfo)
{
  VECT tmp, x;
  VECT *mean;
  VECT *var;
  VECT *vec = gc->OP_vec;
  short veclen = gc->OP_veclen;

  if (binfo == NULL) return(LOG_ZERO);
  mean = binfo->mean;
  var = binfo->var->vec;
  tmp = 0.0;
  for (; veclen > 0; veclen--) {
    x = *(vec++) - *(mean++);
    tmp += x * x * *(var++);
  }
  return((tmp + binfo->gconst) * -0.5);
}

static LOGPROB
gmm_compute_g_safe(GMMCalc *gc, HTK_HMM_Dens *binfo, LOGPROB thres)
{
  VECT tmp, x;
  VECT *mean;
  VECT *var;
  VECT *vec = gc->OP_vec;
  short veclen = gc->OP_veclen;
  VECT fthres = thres * (-2.0);

  if (binfo == NULL) return(LOG_ZERO);
  mean = binfo->mean;
  var = binfo->var->vec;
  tmp = binfo->gconst;
  for (; veclen > 0; veclen--) {
    x = *(vec++) - *(mean++);
    tmp += x * x * *(var++);
    if (tmp > fthres)  return LOG_ZERO;
  }
  return(tmp * -0.5);
}

static void
gmm_gprune_safe_init(GMMCalc *gc, HTK_HMM_INFO *hmminfo, int prune_num)
{
  /* store the pruning num to local area */
  gc->OP_gprune_num = prune_num;
  /* maximum Gaussian set size = maximum mixture size */
  gc->OP_calced_maxnum = hmminfo->maxmixturenum;
  /* allocate memory for storing list of currently computed Gaussian in a frame */
  gc->OP_calced_score = (LOGPROB *)mymalloc(sizeof(LOGPROB) * gc->OP_gprune_num);
  gc->OP_calced_id = (int *)mymalloc(sizeof(int) * gc->OP_gprune_num);
}

static void
gmm_gprune_safe(GMMCalc *gc, HTK_HMM_Dens **g, int gnum)
{
  int i, num = 0;
  LOGPROB score, thres;

  thres = LOG_ZERO;
  for (i = 0; i < gnum; i++) {
    if (num < gc->OP_gprune_num) {
      score = gmm_compute_g_base(gc, g[i]);
    } else {
      score = gmm_compute_g_safe(gc, g[i], thres);
      if (score <= thres) continue;
    }
    num = gmm_cache_push(gc, i, score, num);
    thres = gc->OP_calced_score[num-1];
  }
  gc->OP_calced_num = num;
}

static LOGPROB
gmm_calc_mix(GMMCalc *gc, HTK_HMM_State *s)
{
  int i;
  LOGPROB logprob = LOG_ZERO;

  /* compute Gaussian set */
  gmm_gprune_safe(gc, s->b, s->mix_num);
  /* computed Gaussians will be set in:
     score ... OP_calced_score[0..OP_calced_num]
     id    ... OP_calced_id[0..OP_calced_num] */
  
  /* sum */
  for(i=0;i<gc->OP_calced_num;i++) {
    gc->OP_calced_score[i] += s->bweight[gc->OP_calced_id[i]];
  }
  logprob = addlog_array(gc->OP_calced_score, gc->OP_calced_num);
  if (logprob <= LOG_ZERO) return LOG_ZERO;
  return (logprob * INV_LOG_TEN);
}

static LOGPROB
outprob_state_nocache(GMMCalc *gc, int t, HTK_HMM_State *stateinfo, HTK_Param *param)
{
  /* set global values for outprob functions to access them */
  gc->OP_vec = param->parvec[t];
  gc->OP_veclen = param->veclen;
  return(gmm_calc_mix(gc, stateinfo));
}

/************************************************************************/
/* global functions */

boolean
gmm_init(Recog *recog)
{
  HTK_HMM_INFO *gmm;
  HTK_HMM_Data *d;
  GMMCalc *gc;
  int i;

  gmm = recog->gmm;

  /* check GMM format */
  /* tied-mixture GMM is not supported */
  if (gmm->is_tied_mixture) {
    jlog("ERROR: gmm_init: tied-mixture GMM is not supported\n");
    return FALSE;
  }
  /* assume 3 state GMM (only one output state) */
  for(d=gmm->start;d;d=d->next) {
    if (d->state_num > 3) {
      jlog("ERROR: gmm_init: more than three states (one output state) defined in GMM [%s]\n", d->name);
      return FALSE;
    }
  }

  /* check if CMN needed */

  /* allocate work area */
  if (recog->gc == NULL) {
    gc = (GMMCalc *)mymalloc(sizeof(GMMCalc));
    recog->gc = gc;
  }
  
  /* allocate buffers */
  gc->gmm_score = (LOGPROB *)mymalloc(sizeof(LOGPROB) * gmm->totalhmmnum);

#ifdef GMM_VAD
  gc->nframe = recog->jconf->detect.gmm_margin;
  gc->rates = (LOGPROB *)mymalloc(sizeof(LOGPROB) * gc->nframe);
#endif

  gc->is_voice = (boolean *)mymalloc(sizeof(boolean) * gmm->totalhmmnum);
  i = 0;
  if (recog->jconf->reject.gmm_reject_cmn_string) {
    for(d=recog->gmm->start;d;d=d->next) {
      if (strstr(recog->jconf->reject.gmm_reject_cmn_string, d->name)) {
        gc->is_voice[i] = FALSE;
      } else {
        gc->is_voice[i] = TRUE;
      }
      i++;
    }
  } else {
    for(d=recog->gmm->start;d;d=d->next) {
      gc->is_voice[i] = TRUE;
      i++;
    }
  }

  /* initialize work area */
  gmm_gprune_safe_init(gc, gmm, recog->jconf->reject.gmm_gprune_num);

  /* check if variances are inversed */
  if (!gmm->variance_inversed) {
    /* here, inverse all variance values for faster computation */
    htk_hmm_inverse_variances(gmm);
    gmm->variance_inversed = TRUE;
  }

  return TRUE;
}

void
gmm_prepare(Recog *recog)
{
  HTK_HMM_Data *d;
  int i;

  /* initialize score buffer and frame count */
  i = 0;
  for(d=recog->gmm->start;d;d=d->next) {
    recog->gc->gmm_score[i] = 0.0;
    i++;
  }
#ifdef GMM_VAD
  for(i=0;i<recog->gc->nframe;i++) recog->gc->rates[i] = 0.0;
  recog->gc->framep = 0;
  recog->gc->filled = FALSE;
  recog->gc->in_voice = FALSE;
#endif

  recog->gc->framecount = 0;

#ifdef GMM_VAD_DEBUG
  printf("GMM_VAD: init\n");
#endif
}

void
gmm_proceed(Recog *recog)
{
  HTK_HMM_Data *d;
  GMMCalc *gc;
  int i;
  MFCCCalc *mfcc;
  LOGPROB score;
#ifdef GMM_VAD
  LOGPROB max_n;
  LOGPROB max_v;
#endif

  mfcc = recog->gmmmfcc;
  gc = recog->gc;

  if (!mfcc->valid) return;

  gc->framecount++;

#ifdef GMM_VAD
  max_n = max_v = LOG_ZERO;
#endif

  i = 0;
  for(d=recog->gmm->start;d;d=d->next) {
    score = outprob_state_nocache(gc, mfcc->f, d->s[1], mfcc->param);
    gc->gmm_score[i] += score;
#ifdef GMM_VAD
    if (gc->is_voice[i]) {
      if (max_v < score) max_v = score;
    } else {
      if (max_n < score) max_n = score;
    }
#endif
#ifdef MES
    jlog("DEBUG: [%s: total=%f avg=%f]\n", d->name, gc->gmm_score[i], gc->gmm_score[i] / (float)gc->framecount);
#endif
    i++;
  }
#ifdef GMM_VAD
#ifdef GMM_VAD_DEBUG
  //printf("GMM_VAD: max_v = %f, max_n = %f, rate = %f\n", max_v, max_n, max_v - max_n, gc->framep);
#endif
  /* set rate of this frame */
  gc->rates[gc->framep] = max_v - max_n;
#ifdef GMM_VAD_DEBUG
  printf("GMM_VAD: %f\n", max_v - max_n);
#endif
  /* increment current frame pointer */
  gc->framep++;
  /* if reached end, go to start point */
  if (gc->framep >= gc->nframe) {
    gc->filled = TRUE;
    gc->framep = 0;
  }
#endif
}

void
gmm_end(Recog *recog)
{
  HTK_HMM_INFO *gmm;
  LOGPROB *score;
  HTK_HMM_Data *d;
  LOGPROB maxprob;
  HTK_HMM_Data *dmax;
#ifdef CONFIDENCE_MEASURE
  LOGPROB sum;
#endif
  int i;
  int maxid;

  if (recog->gc->framecount == 0) return;

  gmm = recog->gmm;
  score = recog->gc->gmm_score;

  /* get max score */
  i = 0;
  maxprob = LOG_ZERO;
  dmax = NULL;
  for(d=gmm->start;d;d=d->next) {
    if (maxprob < score[i]) {
      dmax = d;
      maxprob = score[i];
      maxid = i;
    }
    i++;
  }
  recog->gc->max_d = dmax;
  recog->gc->max_i = maxid;

#ifdef CONFIDENCE_MEASURE
  /* compute CM */
  sum = 0.0;
  i = 0;
  for(d=gmm->start;d;d=d->next) {
    //sum += pow(10, recog->jconf->annotate.cm_alpha * (score[i] - maxprob));
    sum += pow(10, 0.05 * (score[i] - maxprob));
    i++;
  }
  recog->gc->gmm_max_cm = 1.0 / sum;
#endif
  
  /* output result */
  callback_exec(CALLBACK_RESULT_GMM, recog);

}


boolean
gmm_valid_input(Recog *recog)
{
  if (recog->gc->max_d == NULL) return FALSE;
  if (recog->gc->is_voice[recog->gc->max_i]) {
    return TRUE;
  }
  return FALSE;
}

void
gmm_free(Recog *recog)
{
  if (recog->gc) {
    free(recog->gc->OP_calced_score);
    free(recog->gc->OP_calced_id);
    free(recog->gc->is_voice);
#ifdef GMM_VAD
    free(recog->gc->rates);
#endif
    free(recog->gc->gmm_score);
    free(recog->gc);
    recog->gc = NULL;
  }
}

#ifdef GMM_VAD

static void
voice_activity_score(GMMCalc *gc, float *mean_ret, float *var_ret, int *count_ret)
{
  int i, len;
  LOGPROB mean;
  LOGPROB var;
  LOGPROB x;
  int count;

  if (!gc->filled) {
    /* cycle buffer not filled yet */
    *mean_ret = 0.0;
    *var_ret = 0.0;
    *count_ret = 0;
    return;
  }

  if (gc->filled) {
    len = gc->nframe;
  } else {
    len = gc->framep;
  }

  mean = 0;
  count = 0;
  for(i=0;i<len;i++) {
    mean += gc->rates[i];
    if (gc->rates[i] > 0.0) count++;
  }
  mean /= (float)len;
  var = 0.0;
  for(i=0;i<len;i++) {
    x = mean - gc->rates[i];
    var += x * x;
  }
  var /= (float)len;

  *mean_ret = mean;
  *var_ret = var;
  *count_ret = count;
}

void
gmm_check_trigger(Recog *recog)
{
  GMMCalc *gc;
  gc = recog->gc;
  float mean;
  float var;
  int count;

  gc->up_trigger = gc->down_trigger = FALSE;

  voice_activity_score(gc, &mean, &var, &count);

  if (gc->in_voice) {
    if (mean < -0.2) {
      gc->down_trigger = TRUE;
      gc->in_voice = FALSE;
    }
  } else {
    if (mean > 0.7) {
      gc->up_trigger = TRUE;
      gc->in_voice = TRUE;
    }
  }

#ifdef GMM_VAD_DEBUG
  printf("GMM_VAD: %s: %f %f %d", gc->in_voice ? "VOICE" : "NOISE", mean, var, count);
  if (gc->up_trigger) printf(": BEGIN");
  if (gc->down_trigger) printf(": END");
  printf("\n");
#endif

}
#endif /* GMM_VAD */

/* end of file */

---