libjulius/src/recogmain.c

Go to the documentation of this file.

/*
 * Copyright (c) 1991-2007 Kawahara Lab., Kyoto University
 * Copyright (c) 1997-2000 Information-technology Promotion Agency, Japan
 * Copyright (c) 2000-2005 Shikano Lab., Nara Institute of Science and Technology
 * Copyright (c) 2005-2007 Julius project team, Nagoya Institute of Technology
 * All rights reserved
 */
#define GLOBAL_VARIABLE_DEFINE  
#include <julius/julius.h>
#include <signal.h>
#if defined(_WIN32) && !defined(__CYGWIN32__)
#include <mbctype.h>
#include <mbstring.h>
#endif

/* ---------- utility functions -----------------------------------------*/
#ifdef REPORT_MEMORY_USAGE

static void
print_mem()
{
  char buf[200];
  sprintf(buf,"ps -o vsz,rss -p %d",getpid());
  system(buf);
  fflush(stdout);
  fflush(stderr);
}
#endif
          

void
result_sentence_malloc(RecogProcess *r, int num)
{
  int i;
  r->result.sent = (Sentence *)mymalloc(sizeof(Sentence) * num);
  for(i=0;i<num;i++) {
    r->result.sent[i].align.filled = FALSE;
    r->result.sent[i].align.w = NULL;
    r->result.sent[i].align.ph = NULL;
    r->result.sent[i].align.loc = NULL;
    r->result.sent[i].align.begin_frame = NULL;
    r->result.sent[i].align.end_frame = NULL;
    r->result.sent[i].align.avgscore = NULL;
    r->result.sent[i].align.is_iwsp = NULL;
  }
  r->result.sentnum = 0;
}

void
result_sentence_free(RecogProcess *r)
{  
  int i;
  if (r->result.sent) {
    for(i=0;i<r->result.sentnum;i++) {
      if (r->result.sent[i].align.w) free(r->result.sent[i].align.w);
      if (r->result.sent[i].align.ph) free(r->result.sent[i].align.ph);
      if (r->result.sent[i].align.loc) free(r->result.sent[i].align.loc);
      if (r->result.sent[i].align.begin_frame) free(r->result.sent[i].align.begin_frame);
      if (r->result.sent[i].align.end_frame) free(r->result.sent[i].align.end_frame);
      if (r->result.sent[i].align.avgscore) free(r->result.sent[i].align.avgscore);
      if (r->result.sent[i].align.is_iwsp) free(r->result.sent[i].align.is_iwsp);
    }
    free(r->result.sent);
    r->result.sent = NULL;
  }
}

void
clear_result(RecogProcess *r)
{
#ifdef WORD_GRAPH
  /* clear 1st pass word graph output */
  wordgraph_clean(&(r->result.wg1));
#endif

  if (r->lmvar == LM_DFA_WORD) {
    if (r->result.status == J_RESULT_STATUS_SUCCESS) {
      /* clear word recog result of first pass as in final result */
      free(r->result.sent);
    }
  } else {
    if (r->graphout) {
      if (r->config->graph.confnet) {
        /* free confusion network clusters */
        cn_free_all(&(r->result.confnet));
      } else if (r->config->graph.lattice) {
      }
      /* clear all wordgraph */
      wordgraph_clean(&(r->result.wg));
    }
    result_sentence_free(r);
  }
}

/* --------------------- speech buffering ------------------ */

static int
adin_cut_callback_store_buffer(SP16 *now, int len, Recog *recog)
{
  if (recog->speechlen == 0) {          /* first part of a segment */
    if (!recog->process_active) {
      return(1);
    }
  }

  if (recog->speechlen + len > recog->speechalloclen) {
    while (recog->speechlen + len > recog->speechalloclen) {
      recog->speechalloclen += MAX_SPEECH_ALLOC_STEP;
    }
    if (recog->speech == NULL) {
      recog->speech = (SP16 *)mymalloc(sizeof(SP16) * recog->speechalloclen);
    } else {
      if (debug2_flag) {
        jlog("STAT: expanding recog->speech to %d samples\n", recog->speechalloclen);
      }
      recog->speech = (SP16 *)myrealloc(recog->speech, sizeof(SP16) * recog->speechalloclen);
    }
  }

  /* store now[0..len] to recog->speech[recog->speechlen] */
  memcpy(&(recog->speech[recog->speechlen]), now, len * sizeof(SP16));
  recog->speechlen += len;
  return(0);                    /* tell adin_go to continue reading */
}


/* --------------------- adin check callback --------------- */
static int
callback_check_in_adin(Recog *recog)
{
  /* module: check command and terminate recording when requested */
  callback_exec(CALLBACK_POLL, recog);
  /* With audio input via adinnet, TERMINATE command will issue terminate
     command to the adinnet client.  The client then stops recording
     immediately and return end-of-segment ack.  Then it will cause this
     process to stop recognition as normal.  So we need not to
     perform immediate termination at this callback, but just ignore the
     results in the main.c.  */
#if 1
/* 
 *   if (recog->jconf->input.speech_input != SP_ADINNET) {
 *     if (recog->process_want_terminate) {
 *       return(-2);
 *     }
 *     if (recog->process_want_reload) {
 *       return(-1);
 *     }
 *   }
 */
  if (recog->process_want_terminate) { /* TERMINATE ... force termination */
    return(-2);
  }
  if (recog->process_want_reload) {
    return(-1);
  }
#else
  if (recog->process_want_terminate /* TERMINATE ... force termination */
      && recog->jconf->input.speech_input != SP_ADINNET) {
    return(-2);
  }
  if (recog->process_want_reload) {
    return(-1);
  }
#endif
  return(0);
}

/*********************/
/* open input stream */
/*********************/
int
j_open_stream(Recog *recog, char *file_or_dev_name)
{
  Jconf *jconf;

  jconf = recog->jconf;

  if (jconf->input.speech_input == SP_MFCFILE) {
    /* read parameter file */
    param_init_content(recog->mfcclist->param);
    if (rdparam(file_or_dev_name, recog->mfcclist->param) == FALSE) {
      jlog("ERROR: error in reading parameter file: %s\n", file_or_dev_name);
      return -1;
    }
    /* check and strip invalid frames */
    if (jconf->preprocess.strip_zero_sample) {
      param_strip_zero(recog->mfcclist->param);
    }

    /* output frame length */
    callback_exec(CALLBACK_STATUS_PARAM, recog);
  } else {                      /* raw speech input */
    /* begin A/D input */
    if (adin_begin(recog->adin) == FALSE) {
      return -2;
    }
  }
    
#if 0
    /* if not module mode, process becomes online after all initialize done */
    process_online = TRUE;
    callback_exec(CALLBACK_EVENT_PROCESS_ONLINE, recog);
#endif

  return 0;

}

/**********************************************************************/
/**********************************************************************/
/**********************************************************************/

static void
result_error(Recog *recog, int status)
{
  MFCCCalc *mfcc;
  RecogProcess *r;
  boolean ok_p;

  for(r=recog->process_list;r;r=r->next) r->result.status = status;

  ok_p = FALSE;
  for(mfcc=recog->mfcclist;mfcc;mfcc=mfcc->next) {
    if (mfcc->f > 0) {
      ok_p = TRUE;
      break;
    }
  }
  if (ok_p) {                   /* had some input */
    /* output as rejected */
    callback_exec(CALLBACK_RESULT, recog);
  }
}

static int
j_recognize_stream_core(Recog *recog)
{
  Jconf *jconf;
  int ret;
  float seclen, mseclen;
  RecogProcess *r;
  MFCCCalc *mfcc;
  PROCESS_AM *am;
  PROCESS_LM *lm;
  boolean ok_p;
  boolean process_segment_last;

  jconf = recog->jconf;

  if (jconf->input.speech_input != SP_MFCFILE) {
    for(mfcc=recog->mfcclist;mfcc;mfcc=mfcc->next) {
      param_init_content(mfcc->param);
    }
  }

  /* if no process instance exist, start with terminated */
  if (recog->process_list == NULL) {
    jlog("STAT: no recog process, engine inactive\n");
    j_request_pause(recog);
  }

  /* update initial recognition process status */
  for(r=recog->process_list;r;r=r->next) {
    if (r->active > 0) {
      r->live = TRUE;
    } else if (r->active < 0) {
      r->live = FALSE;
    }
    r->active = 0;
  }

  /******************************************************************/
  /* do recognition for each incoming segment from the input stream */
  /******************************************************************/
  while (1) {
    
  start_recog:

    /*************************************/
    /* Update recognition process status */
    /*************************************/
    for(r=recog->process_list;r;r=r->next) {
      if (r->active > 0) {
        r->live = TRUE;
        jlog("STAT: SR%02d %s now active\n", r->config->id, r->config->name);
      } else if (r->active < 0) {
        r->live = FALSE;
        jlog("STAT: SR%02d %s now inactive\n", r->config->id, r->config->name);
      }
      r->active = 0;
    }
    if (debug2_flag) {
      for(r=recog->process_list;r;r=r->next) {
        jlog("DEBUG: %s: SR%02d %s\n", r->live ? "live" : "dead", r->config->id, r->config->name);
      }
    }
    /* check if any process is live */
    if (recog->process_active) {
      ok_p = FALSE;
      for(r=recog->process_list;r;r=r->next) {
        if (r->live) ok_p = TRUE;
      }
      if (!ok_p) {              /* no process is alive */
        /* make whole process as inactive */
        jlog("STAT: all recog process inactive, pause engine now\n");
        j_request_pause(recog);
      }
    }

    /* Check whether process status was changed while in the last run */
    if (recog->process_online != recog->process_active) {
      recog->process_online = recog->process_active;
      if (recog->process_online) callback_exec(CALLBACK_EVENT_PROCESS_ONLINE, recog);
      else callback_exec(CALLBACK_EVENT_PROCESS_OFFLINE, recog);
    }
    /* execute poll callback */
    if (recog->process_active) {
      callback_exec(CALLBACK_POLL, recog);
    }
    /* reset reload flag here */
    j_reset_reload(recog);

    if (!recog->process_active) {
      /* now sleeping, return */
      /* in the next call, we will resume from here */
      return 1;
    }
    /* update process status */
    if (recog->process_online != recog->process_active) {
      recog->process_online = recog->process_active;
      if (recog->process_online) callback_exec(CALLBACK_EVENT_PROCESS_ONLINE, recog);
      else callback_exec(CALLBACK_EVENT_PROCESS_OFFLINE, recog);
    }

    /*********************************************************/
    /* check for grammar to change, and rebuild if necessary */
    /*********************************************************/
    for(lm=recog->lmlist;lm;lm=lm->next) {
      if (lm->lmtype == LM_DFA) {
        multigram_update(lm); /* some modification occured if return TRUE*/
      }
    }
    for(r=recog->process_list;r;r=r->next) {
      if (!r->live) continue;
      if (r->lmtype == LM_DFA && r->lm->global_modified) {
        multigram_build(r);
      }
    }
    for(lm=recog->lmlist;lm;lm=lm->next) {
      if (lm->lmtype == LM_DFA) lm->global_modified = FALSE;
    }

    ok_p = FALSE;
    for(r=recog->process_list;r;r=r->next) {
      if (!r->live) continue;
      if (r->lmtype == LM_DFA) {
        if (r->lm->winfo == NULL ||
            (r->lmvar == LM_DFA_GRAMMAR && r->lm->dfa == NULL)) {
          /* make this instance inactive */
          r->active = -1;
          ok_p = TRUE;
        }
      }
    }
    if (ok_p) {                 /* at least one instance has no grammar */
      goto start_recog;
    }

    /**************************************/
    /* getting input and perform 1st pass */
    /**************************************/
    if (jconf->input.speech_input == SP_MFCFILE) {
      /************************/
      /* parameter file input */
      /************************/
      /********************************/
      /* check the analized parameter */
      /********************************/
      /* parameter type check --- compare the type to that of HMM,
         and adjust them if necessary */
      if (jconf->input.paramtype_check_flag) {
        for(am=recog->amlist;am;am=am->next) {
          /* return param itself or new malloced param */
          if (param_check_and_adjust(am->hmminfo, am->mfcc->param, verbose_flag) == -1) {       /* failed */
            
            for(mfcc=recog->mfcclist;mfcc;mfcc=mfcc->next) {
              param_init_content(mfcc->param);
            }
            /* tell failure */
            result_error(recog, J_RESULT_STATUS_FAIL);
            goto end_recog;
          }
        }
      }
      /* whole input is already read, so set input status to end of stream */
      /* and jump to the start point of 1st pass */
      ret = 0;
    } else {
      /****************************************************/
      /* raw wave data input (mic, file, adinnet, etc...) */
      /****************************************************/
      if (jconf->decodeopt.realtime_flag) {
        /********************************************/
        /* REALTIME ON-THE-FLY DECODING OF 1ST-PASS */
        /********************************************/
        /* store, analysis and search in a pipeline  */
        /* main function is RealTimePipeLine() at realtime-1stpass.c, and
           it will be periodically called for each incoming input segment
           from the AD-in function adin_go().  RealTimePipeLine() will be
           called as a callback function from adin_go() */
        /* after this part, directly jump to the beginning of the 2nd pass */

        if (recog->process_segment) {
          /*****************************************************************/
          /* short-pause segmentation: process last remaining frames first */
          /*****************************************************************/
          /* last was segmented by short pause */
          /* the margin segment in the last input will be re-processed first,
             and then the speech input will be processed */
          /* output listening start message */
          callback_exec(CALLBACK_EVENT_SPEECH_READY, recog);
          /* process the last remaining parameters */
          ret = RealTimeResume(recog);
          if (ret < 0) {                /* error end in the margin */
            jlog("ERROR: failed to process last remaining samples on RealTimeResume\n"); /* exit now! */
            return -1;
          }
          if (ret != 1) {       /* if segmented again in the margin, not process the rest */
            /* last parameters has been processed, so continue with the
               current input as normal */
            /* process the incoming input */
            ret = adin_go(RealTimePipeLine, callback_check_in_adin, recog);
            if (ret < 0) {              /* error end in adin_go */
              if (ret == -2 || recog->process_want_terminate) {
                /* terminated by callback */
                RealTimeTerminate(recog);
                /* reset param */
                for(mfcc=recog->mfcclist;mfcc;mfcc=mfcc->next) {
                  param_init_content(mfcc->param);
                }
                /* execute callback at end of pass1 */
                if (recog->triggered) {
                  callback_exec(CALLBACK_EVENT_PASS1_END, recog);
                  /* output result terminate */
                  result_error(recog, J_RESULT_STATUS_TERMINATE);
                }
                goto end_recog; /* cancel this recognition */
              }
              jlog("ERROR: an error occured at on-the-fly 1st pass decoding\n");          /* exit now! */
              return(-1);
            }
          }
          
        } else {

          /***********************************************************/
          /* last was not segmented, process the new incoming input  */
          /***********************************************************/
          /* end of this input will be determined by either end of stream
             (in case of file input), or silence detection by adin_go(), or
             'TERMINATE' command from module (if module mode) */
          /* prepare work area for on-the-fly processing */
          if (RealTimePipeLinePrepare(recog) == FALSE) {
            jlog("ERROR: failed to prepare for on-the-fly 1st pass decoding");
            return (-1);
          }
          /* output 'listening start' message */
          callback_exec(CALLBACK_EVENT_SPEECH_READY, recog);
          /* process the incoming input */
          ret = adin_go(RealTimePipeLine, callback_check_in_adin, recog);
          if (ret < 0) {                /* error end in adin_go */
            if (ret == -2 || recog->process_want_terminate) {   
              /* terminated by callback */
              RealTimeTerminate(recog);
              /* reset param */
              for(mfcc=recog->mfcclist;mfcc;mfcc=mfcc->next) {
                param_init_content(mfcc->param);
              }
              /* execute callback at end of pass1 */
              if (recog->triggered) {
                callback_exec(CALLBACK_EVENT_PASS1_END, recog);
                /* output result terminate */
                result_error(recog, J_RESULT_STATUS_TERMINATE);
              }
              goto end_recog;
            }
            jlog("ERROR: an error occured at on-the-fly 1st pass decoding\n");          /* exit now! */
            return(-1);
          }
        }
        /******************************************************************/
        /* speech stream has been processed on-the-fly, and 1st pass ends */
        /******************************************************************/
        /* last procedure of 1st-pass */
        if (RealTimeParam(recog) == FALSE) {
          jlog("ERROR: fatal error occured, program terminates now\n");
          return -1;
        }

#ifdef BACKEND_VAD
        /* if not triggered, skip this segment */
        if (recog->jconf->decodeopt.segment && ! recog->triggered) {
          goto end_recog;
        }
#endif

        /* execute callback for 1st pass result */
        /* result.status <0 must be skipped inside callback */
        callback_exec(CALLBACK_RESULT_PASS1, recog);
#ifdef WORD_GRAPH
        /* result.wg1 == NULL should be skipped inside callback */
        callback_exec(CALLBACK_RESULT_PASS1_GRAPH, recog);
#endif
        /* execute callback at end of pass1 */
        callback_exec(CALLBACK_EVENT_PASS1_END, recog);
        /* output frame length */
        callback_exec(CALLBACK_STATUS_PARAM, recog);
        /* if terminate signal has been received, discard this input */
        if (recog->process_want_terminate) {
          result_error(recog, J_RESULT_STATUS_TERMINATE);
          goto end_recog;
        }

        /* end of 1st pass, jump to 2nd pass */
        goto end_1pass;
        
      } /* end of realtime_flag && speech stream input */
      
      /******************************************/
      /* buffered speech input (not on-the-fly) */
      /******************************************/
      if (!recog->process_segment) { /* no segment left */

        /****************************************/
        /* store raw speech samples to speech[] */
        /****************************************/
        recog->speechlen = 0;
        for(mfcc=recog->mfcclist;mfcc;mfcc=mfcc->next) {
          param_init_content(mfcc->param);
        }
        /* output 'listening start' message */
        callback_exec(CALLBACK_EVENT_SPEECH_READY, recog);
        /* tell module to start recording */
        /* the "adin_cut_callback_store_buffer" simply stores
           the input speech to a buffer "speech[]" */
        /* end of this input will be determined by either end of stream
           (in case of file input), or silence detection by adin_go(), or
           'TERMINATE' command from module (if module mode) */
        ret = adin_go(adin_cut_callback_store_buffer, callback_check_in_adin, recog);
        if (ret < 0) {          /* error end in adin_go */
          if (ret == -2 || recog->process_want_terminate) {
            /* terminated by module */
            /* output fail */
            result_error(recog, J_RESULT_STATUS_TERMINATE);
            goto end_recog;
          }
          jlog("ERROR: an error occured while recording input\n");
          return -1;
        }
        
        /* output recorded length */
        seclen = (float)recog->speechlen / (float)jconf->input.sfreq;
        jlog("STAT: %d samples (%.2f sec.)\n", recog->speechlen, seclen);
        
        /* -rejectshort »ØÄê»þ, ÆþÎϤ¬»ØÄê»þ´Ö°Ê²¼¤Ç¤¢¤ì¤Ð
           ¤³¤³¤ÇÆþÎϤò´þµÑ¤¹¤ë */
        /* when using "-rejectshort", and input was shorter than
           specified, reject the input here */
        if (jconf->reject.rejectshortlen > 0) {
          if (seclen * 1000.0 < jconf->reject.rejectshortlen) {
            result_error(recog, J_RESULT_STATUS_REJECT_SHORT);
            goto end_recog;
          }
        }
        
        /**********************************************/
        /* acoustic analysis and encoding of speech[] */
        /**********************************************/
        jlog("STAT: ### speech analysis (waveform -> MFCC)\n");
        /* CMN will be computed for the whole buffered input */
        if (wav2mfcc(recog->speech, recog->speechlen, recog) == FALSE) {
          /* error end, end stream */
          ret = -1;
          /* tell failure */
          result_error(recog, J_RESULT_STATUS_FAIL);
          goto end_recog;
        }
        
        /* if terminate signal has been received, cancel this input */
        if (recog->process_want_terminate) {
          result_error(recog, J_RESULT_STATUS_TERMINATE);
          goto end_recog;
        }
        
        /* output frame length */
        callback_exec(CALLBACK_STATUS_PARAM, recog);
        
      }
    }   /* end of data input */
    /* parameter has been got in 'param' */
    
    /******************************************************/
    /* 1st-pass --- backward search to compute heuristics */
    /******************************************************/
    /* (for buffered speech input and HTK parameter file input) */
    if (!jconf->decodeopt.realtime_flag) {
      /* prepare for outprob cache for each HMM state and time frame */
      /* assume all MFCCCalc has params of the same sample num */
      for(am=recog->amlist;am;am=am->next) {
        outprob_prepare(&(am->hmmwrk), am->mfcc->param->samplenum);
      }
    }

    /* if terminate signal has been received, cancel this input */
    if (recog->process_want_terminate) {
      result_error(recog, J_RESULT_STATUS_TERMINATE);
      goto end_recog;
    }
    

    /****************************************************/
    /* execute computation of left-to-right backtrellis */
    /****************************************************/
    if (get_back_trellis(recog) == FALSE) {
      jlog("ERROR: fatal error occured, program terminates now\n");
      return -1;
    }
#ifdef BACKEND_VAD
    /* if not triggered, skip this segment */
    if (recog->jconf->decodeopt.segment && ! recog->triggered) {
      goto end_recog;
    }
#endif

    /* execute callback for 1st pass result */
    /* result.status <0 must be skipped inside callback */
    callback_exec(CALLBACK_RESULT_PASS1, recog);
#ifdef WORD_GRAPH
    /* result.wg1 == NULL should be skipped inside callback */
    callback_exec(CALLBACK_RESULT_PASS1_GRAPH, recog);
#endif

    /* execute callback at end of pass1 */
    if (recog->triggered) {
      callback_exec(CALLBACK_EVENT_PASS1_END, recog);
    }

  end_1pass:

    /**********************************/
    /* end processing of the 1st-pass */
    /**********************************/
    /* on-the-fly 1st pass processing will join here */
    
    /* -rejectshort »ØÄê»þ, ÆþÎϤ¬»ØÄê»þ´Ö°Ê²¼¤Ç¤¢¤ì¤Ðõº÷¼ºÇԤȤ·¤Æ */
    /* Â裲¥Ñ¥¹¤ò¼Â¹Ô¤»¤º¤Ë¤³¤³¤Ç½ªÎ»¤¹¤ë */
    /* when using "-rejectshort", and input was shorter than the specified
       length, terminate search here and output recognition failure */
    if (jconf->reject.rejectshortlen > 0) {
      mseclen = (float)recog->mfcclist->param->samplenum * (float)jconf->input.period * (float)jconf->input.frameshift / 10000.0;
      if (mseclen < jconf->reject.rejectshortlen) {
        result_error(recog, J_RESULT_STATUS_REJECT_SHORT);
        goto end_recog;
      }
    }
#ifdef POWER_REJECT
    if (power_reject(recog)) {
      result_error(recog, J_RESULT_STATUS_REJECT_POWER);
      goto end_recog;
    }
#endif
    
    /* if terminate signal has been received, cancel this input */
    if (recog->process_want_terminate) {
      result_error(recog, J_RESULT_STATUS_TERMINATE);
      goto end_recog;
    }
    
    /* if GMM is specified and result are to be rejected, terminate search here */
    if (jconf->reject.gmm_reject_cmn_string != NULL) {
      if (! gmm_valid_input(recog)) {
        result_error(recog, J_RESULT_STATUS_REJECT_GMM);
        goto end_recog;
      }
    }

    /***********************************************/
    /* 2nd-pass --- forward search with heuristics */
    /***********************************************/
#if !defined(PASS2_STRICT_IWCD) || defined(FIX_35_PASS2_STRICT_SCORE)    
    /* adjust trellis score not to contain outprob of the last frames */
    for(r=recog->process_list;r;r=r->next) {
      if (!r->live) continue;
      /* if [-1pass] is specified, skip 2nd pass */
      if (r->config->compute_only_1pass) continue;
      /* if search already failed on 1st pass, skip 2nd pass */
      if (r->result.status < 0) continue;
      if (! r->am->hmminfo->multipath) {
        bt_discount_pescore(r->wchmm, r->backtrellis, r->am->mfcc->param);
      }
#ifdef LM_FIX_DOUBLE_SCORING
      if (r->lmtype == LM_PROB) {
        bt_discount_lm(r->backtrellis);
      }
#endif
    }
#endif
    
    /* execute stack-decoding search */
    callback_exec(CALLBACK_EVENT_PASS2_BEGIN, recog);

    for(r=recog->process_list;r;r=r->next) {
      if (!r->live) continue;
      /* if [-1pass] is specified, skip 2nd pass */
      if (r->config->compute_only_1pass) continue;
      /* if search already failed on 1st pass, skip 2nd pass */
      if (r->result.status < 0) continue;
      if (r->lmtype == LM_PROB) {
        wchmm_fbs(r->am->mfcc->param, r, 0, 0);
      } else if (r->lmtype == LM_DFA) {
        if (r->config->output.multigramout_flag) {
          /* execute 2nd pass multiple times for each grammar sequencially */
          /* to output result for each grammar */
          MULTIGRAM *m;
          for(m = r->lm->grammars; m; m = m->next) {
            if (m->active) {
              jlog("STAT: execute 2nd pass limiting words for gram #%d\n", m->id);
              wchmm_fbs(r->am->mfcc->param, r, m->cate_begin, m->dfa->term_num);
            }
          }
        } else {
          /* only the best among all grammar will be output */
          wchmm_fbs(r->am->mfcc->param, r, 0, r->lm->dfa->term_num);
        }
      }
    }

    /* output result */
    callback_exec(CALLBACK_RESULT, recog);
    /* output graph */
    /* r->result.wg == NULL should be skipped inside the callback */
    ok_p = FALSE;
    for(r=recog->process_list;r;r=r->next) {
      if (!r->live) continue;
      if (r->config->graph.lattice) ok_p = TRUE;
    }
    if (ok_p) callback_exec(CALLBACK_RESULT_GRAPH, recog);
    /* output confnet */
    /* r->result.confnet == NULL should be skipped inside the callback */
    ok_p = FALSE;
    for(r=recog->process_list;r;r=r->next) {
      if (!r->live) continue;
      if (r->config->graph.confnet) ok_p = TRUE;
    }
    if (ok_p) callback_exec(CALLBACK_RESULT_CONFNET, recog);

    /* clear work area for output */
    for(r=recog->process_list;r;r=r->next) {
      if (!r->live) continue;
      clear_result(r);
    }
    
    /* output end of 2nd pass */
    callback_exec(CALLBACK_EVENT_PASS2_END, recog);

  end_recog:
    /**********************/
    /* end of recognition */
    /**********************/

    process_segment_last = recog->process_segment;
    if (jconf->decodeopt.segment) { /* sp-segment mode */
      /* param is now shrinked to hold only the processed input, and */
      /* the rests are holded in (newly allocated) "rest_param" */
      /* if this is the last segment, rest_param is NULL */
      /* assume all segmentation are synchronized */
      recog->process_segment = FALSE;
      for(mfcc=recog->mfcclist;mfcc;mfcc=mfcc->next) {
        if (mfcc->rest_param != NULL) {
          /* process the rest parameters in the next loop */
          recog->process_segment = TRUE;
          free_param(mfcc->param);
          mfcc->param = mfcc->rest_param;
          mfcc->rest_param = NULL;
        }
      }
    }

    /* callback of recognition end */
    if (jconf->decodeopt.segment) {
#ifdef BACKEND_VAD
      if (recog->triggered) callback_exec(CALLBACK_EVENT_SEGMENT_END, recog);
      if (process_segment_last && !recog->process_segment) callback_exec(CALLBACK_EVENT_RECOGNITION_END, recog);
#else
      callback_exec(CALLBACK_EVENT_SEGMENT_END, recog);
      if (!recog->process_segment) callback_exec(CALLBACK_EVENT_RECOGNITION_END, recog);
#endif
    } else {
      callback_exec(CALLBACK_EVENT_RECOGNITION_END, recog);
    }


    /* update CMN info for next input (in case of realtime wave input) */
    if (jconf->input.speech_input != SP_MFCFILE && jconf->decodeopt.realtime_flag) {
      for(mfcc=recog->mfcclist;mfcc;mfcc=mfcc->next) {
        if (mfcc->param->samplenum > 0) {
          RealTimeCMNUpdate(mfcc, recog);
        }
      }
    }
    
    if (verbose_flag) jlog("\n");
    jlog_flush();

    if (jconf->decodeopt.segment) { /* sp-segment mode */
      if (recog->process_segment == TRUE) {
        if (verbose_flag) jlog("STAT: <<<restart the rest>>>\n\n");
      } else {
        /* input has reached end of stream, terminate program */
        if (ret <= 0 && ret != -2) break;
      }
    } else {                    /* not sp-segment mode */
      /* input has reached end of stream, terminate program */
      if (ret <= 0 && ret != -2) break;
    }

    /* recognition continues for next (silence-aparted) segment */
      
  } /* END OF STREAM LOOP */
    
    /* input stream ended. it will happen when
       - input speech file has reached the end of file, 
       - adinnet input has received end of segment mark from client,
       - adinnet input has received end of input from client,
       - adinnet client disconnected.
    */

  if (jconf->input.speech_input != SP_MFCFILE) {
    /* close the stream */
    adin_end(recog->adin);
  }

  /* return to the opening of input stream */

  return(0);

}

int
j_recognize_stream(Recog *recog)
{
  int ret;

  do {
    
    ret = j_recognize_stream_core(recog);

    switch(ret) {
    case 1:           /* paused by a callback (stream will continue) */
      /* call pause event callbacks */
      callback_exec(CALLBACK_EVENT_PAUSE, recog);
      /* call pause functions */
      /* block until all pause functions exits */
      if (! callback_exist(recog, CALLBACK_PAUSE_FUNCTION)) {
        jlog("WARNING: pause requested but no pause function specified\n");
        jlog("WARNING: engine will resume now immediately\n");
      }
      callback_exec(CALLBACK_PAUSE_FUNCTION, recog);
      /* after here, recognition will restart for the rest input */
      /* call resume event callbacks */
      callback_exec(CALLBACK_EVENT_RESUME, recog);
      break;
    case 0:                     /* end of stream */
      /* go on to the next input */
      break;
    case -1:            /* error */
      jlog("ERROR: an error occured while recognition, terminate stream\n");
      return -1;
    }
  } while (ret == 1);           /* loop when paused by callback */

  return 0;
}

/* end of file */

---