libjulius/include/julius/recog.h

Go to the documentation of this file.

/*
 * Copyright (c) 1991-2007 Kawahara Lab., Kyoto University
 * Copyright (c) 2000-2005 Shikano Lab., Nara Institute of Science and Technology
 * Copyright (c) 2005-2007 Julius project team, Nagoya Institute of Technology
 * All rights reserved
 */

/*
*/

#ifndef __J_RECOG_H__
#define __J_RECOG_H__

#include <sent/stddefs.h>
#include <sent/hmm.h>
#include <sent/vocabulary.h>
#include <sent/ngram2.h>
#include <sent/dfa.h>
#include <julius/wchmm.h>
#include <julius/search.h>
#include <julius/callback.h>
#include <julius/jconf.h>

/*
  How tokens are managed:
   o  tlist[][] is a token stocker.  It holds all tokens in sequencial
      buffer.  They are malloced first on startup, and refered by ID while
      Viterbi procedure.  In word-pair mode, each token also has a link to
      another token to allow a node to have more than 1 token.
      
   o  token[n] holds the current ID number of a token associated to a
      lexicon tree node 'n'.

  */
typedef struct __FSBeam__ {
  /* token stocker */
  TOKEN2 *tlist[2];     
  TOKENID *tindex[2];   
  int maxtnum;          
  int expand_step;      
  boolean expanded;     
  int tnum[2];          
  int n_start;          
  int n_end;            
  int tl;               
  int tn;               
    
  /* Active token list */
  TOKENID *token;       
#ifdef UNIGRAM_FACTORING
  /* for wordend processing with 1-gram factoring */
  LOGPROB wordend_best_score; 
  int wordend_best_node;        
  TRELLIS_ATOM *wordend_best_tre; 
  WORD_ID wordend_best_last_cword;      
#endif

  int totalnodenum;     
  TRELLIS_ATOM bos;     
  boolean nodes_malloced; 
  LOGPROB lm_weight;           
  LOGPROB lm_penalty;          
  LOGPROB lm_penalty_trans; 
  LOGPROB penalty1; 
#if defined(WPAIR) && defined(WPAIR_KEEP_NLIMIT)
  boolean wpair_keep_nlimit; 
#endif
  /* for short-pause segmentation */
  boolean in_sparea;         
  int tmp_sparea_start;         
#ifdef SP_BREAK_RESUME_WORD_BEGIN
  WORD_ID tmp_sp_break_last_word; 
#else
  WORD_ID last_tre_word;        
#endif
  boolean first_sparea;  
  int sp_duration;   
#ifdef SPSEGMENT_NAIST
  boolean after_trigger;        
  int trigger_duration;         
  boolean want_rewind;          
  int rewind_frame;             
  boolean want_rewind_reprocess; 
#endif
  char *pausemodelnames;        
  char **pausemodel;            
  int pausemodelnum;            
} FSBeam;


typedef struct __RealBeam__ {
  /* input parameter */
  int maxframelen;              

  SP16 *window;         
  int windowlen;                
  int windownum;                

  /* for short-pause segmentation */
  boolean last_is_segmented; 
  SP16 *rest_Speech; 
  int rest_alloc_len;   
  int rest_len;         

} RealBeam;

typedef struct __StackDecode__ {
  int hypo_len_count[MAXSEQNUM+1];      
  int maximum_filled_length; 
#ifdef SCAN_BEAM
  LOGPROB *framemaxscore; 
#endif
  NODE *stocker_root; 
  int popctr;           
  int genectr;          
  int pushctr;          
  int finishnum;        
  NODE *current;                

#ifdef CONFIDENCE_MEASURE
  LOGPROB cm_alpha;             
# ifdef CM_MULTIPLE_ALPHA
  LOGPROB *cmsumlist;        
  int cmsumlistlen;             
# endif
# ifdef CM_SEARCH
  LOGPROB cm_tmpbestscore; 
#  ifndef CM_MULTIPLE_ALPHA
  LOGPROB cm_tmpsum;            
#  endif
  int l_stacksize;              
  int l_stacknum;               
  NODE *l_start;        
  NODE *l_bottom;       
# endif
# ifdef CM_NBEST
  LOGPROB *sentcm = NULL;       
  LOGPROB *wordcm = NULL;       
  int sentnum;          
# endif
#endif /* CONFIDENCE_MEASURE */

} StackDecode;

typedef struct {
  LOGPROB (*uniprob)(WORD_INFO *, WORD_ID, LOGPROB); 
  LOGPROB (*biprob)(WORD_INFO *, WORD_ID, WORD_ID, LOGPROB); 
  LOGPROB (*lmprob)(WORD_INFO *, WORD_ID *, int, WORD_ID, LOGPROB); 
} LMFunc;

typedef struct __gmm_calc__{
  LOGPROB *gmm_score;   
  boolean *is_voice;            
  int framecount;               
  LOGPROB *OP_calced_score; 
  int *OP_calced_id; 
  int OP_calced_num; 
  int OP_calced_maxnum; 
  int OP_gprune_num; 
  VECT *OP_vec;         
  short OP_veclen;              
  HTK_HMM_Data *max_d;  
  int max_i;                    
#ifdef CONFIDENCE_MEASURE
  LOGPROB gmm_max_cm;   
#endif
#ifdef GMM_VAD
  LOGPROB *rates;   
  int nframe;                   
  boolean filled;
  int framep;                   

  boolean in_voice;             
  boolean up_trigger;           
  boolean down_trigger;         
  boolean after_trigger;        
  boolean want_rewind;          
  boolean want_rewind_reprocess; 
  int rewind_frame;             
  int duration;                 
#endif
} GMMCalc;

typedef struct __sentence__ {
  WORD_ID word[MAXSEQNUM];      
  int word_num;                 
  LOGPROB score;                
  LOGPROB confidence[MAXSEQNUM]; 
  LOGPROB score_lm;             
  LOGPROB score_am;             
  int gram_id;                  

  struct {
    boolean filled;             
    int num;                    
    short unittype;             

    WORD_ID *w;                 
    HMM_Logical **ph;     
    short *loc; 
    boolean *is_iwsp;           

    int *begin_frame;           
    int *end_frame;             
    LOGPROB *avgscore;          
   
    LOGPROB allscore;           
  } align;

} Sentence;

typedef struct __adin__ {
  /* functions */
  boolean (*ad_standby)(int, void *);
  boolean (*ad_begin)();
  boolean (*ad_end)();
  boolean (*ad_resume)();
  boolean (*ad_pause)();
  int (*ad_read)(SP16 *, int);

  /* configuration parameters */
  int thres;            
  int noise_zerocross;  
  int nc_max;           
  boolean adin_cut_on;  
  boolean silence_cut_default; 
  boolean strip_flag;   
  boolean enable_thread;        
  boolean need_zmean;   

  /* work area */
  int c_length; 
  int c_offset; 
  SP16 *swapbuf;                
  int sbsize;    
  int sblen;    
  int rest_tail;                

  ZEROCROSS zc;                 

#ifdef HAVE_PTHREAD
  /* Variables related to POSIX threading */
  pthread_mutex_t mutex;        
  SP16 *speech;         
  int speechlen;                
/*
 * Semaphore to start/stop recognition.
 * 
 * If TRUE, A/D-in thread will store incoming samples to @a speech and
 * main thread will detect and process them.
 * If FALSE, A/D-in thread will still get input and check trigger as the same
 * as TRUE case, but does not store them to @a speech.
 * 
 */
  boolean transfer_online;
  boolean adinthread_buffer_overflowed;

  boolean ignore_speech_while_recog; 

#endif

  /* Input data buffer */
  SP16 *buffer; 
  int bpmax;            
  int bp;                       
  int current_len;              
  SP16 *cbuf;           
  boolean down_sample; 
  SP16 *buffer48; 
  int io_rate; 

  boolean is_valid_data;        
  int nc;               
  boolean end_of_stream;        
  boolean need_init;    

  DS_BUFFER *ds;           

  boolean rehash; 
} ADIn;

typedef struct __Output__ {
  int status;

  int num_frame;                
  int length_msec;              

  Sentence *sent;               
  int sentnum;                  

  WordGraph *wg1;               
  int wg1_num;                  

  WordGraph *wg;                

  CN_CLUSTER *confnet;          

  Sentence pass1;               

} Output;  


/**********************************************************************/
/**********************************************************************/
/**********************************************************************/

typedef struct __mfcc_calc__ {

  short id;

  Value *para;

  boolean htk_loaded;
  boolean hmm_loaded;

  boolean paramtype_check_flag;

  MFCCWork *wrk;

  HTK_Param *param;

  HTK_Param *rest_param;

  struct {
    char *load_filename;
    boolean update;
    char *save_filename;     
    float map_weight;

    boolean loaded;

    CMNWork *wrk;

  } cmn;

  struct {
    float *ssbuf;
    
    int sslen;
    
    float ss_alpha;

    float ss_floor;

    boolean sscalc;

    int sscalc_len;

    char *ssload_filename;

    MFCCWork *mfccwrk_ss;
    
  } frontend;

  ENERGYWork ewrk;

  DeltaBuf *db;
  DeltaBuf *ab;
  VECT *tmpmfcc;

  boolean valid;

  int f;

  int last_time;

  int sparea_start;

  boolean segmented;

#ifdef POWER_REJECT
  float avg_power;
#endif

  struct __mfcc_calc__ *next;

} MFCCCalc;

typedef struct __process_am__ {

  JCONF_AM *config;

  MFCCCalc *mfcc;

  HTK_HMM_INFO *hmminfo;

  HTK_HMM_INFO *hmm_gs;

  HMMWork hmmwrk;

  struct __process_am__ *next;
  
} PROCESS_AM;

typedef struct __process_lm__ {

  JCONF_LM *config;

  PROCESS_AM *am;


  int lmtype;

  int lmvar;

  WORD_INFO *winfo;

  NGRAM_INFO *ngram;

  MULTIGRAM *grammars;

  int gram_maxid;

  DFA_INFO *dfa;

  boolean global_modified;

  LMFunc lmfunc;

  struct __process_lm__ *next;

} PROCESS_LM;

typedef struct __recogprocess__ {

  boolean live;

  short active;

  JCONF_SEARCH *config;

  PROCESS_AM *am;

  PROCESS_LM *lm;

  int lmtype;

  int lmvar;

  boolean ccd_flag;

  WCHMM_INFO *wchmm;

  int trellis_beam_width;

  BACKTRELLIS *backtrellis;

  FSBeam pass1;

  StackDecode pass2;

  WORD_ID pass1_wseq[MAXSEQNUM];

  int pass1_wnum;

  LOGPROB pass1_score;

  WORD_ID sp_break_last_word;
  WORD_ID sp_break_last_nword;
  boolean sp_break_last_nword_allow_override;
  WORD_ID sp_break_2_begin_word;
  WORD_ID sp_break_2_end_word;

  int peseqlen;         

  int graph_totalwordnum;

  Output result;

  boolean graphout;

#ifdef DETERMINE
  int determine_count;
  LOGPROB determine_maxnodescore;
  boolean determined;
  LOGPROB determine_last_wid;
  boolean have_determine;
#endif

  boolean have_interim;

  void *hook;

  struct __recogprocess__ *next;

} RecogProcess;

typedef struct __Recog__ {

  /*******************************************/
  Jconf *jconf;

  /*******************************************/
  ADIn *adin;

  RealBeam real;

  MFCCCalc *mfcclist;

  PROCESS_AM *amlist;

  PROCESS_LM *lmlist;

  RecogProcess *process_list;


  boolean process_segment;

  /*******************************************/
  /* inputs */

  SP16 *speech;

  int speechalloclen;

  int speechlen;                

  int peseqlen;         

  /*******************************************/

  HTK_HMM_INFO *gmm;

  MFCCCalc *gmmmfcc;

  GMMCalc *gc;

  /*******************************************/
  /* misc. */

  boolean process_active;

  boolean process_want_terminate;

  boolean process_want_reload;

  short gram_switch_input_method;

  boolean process_online;

  boolean (*calc_vector)(MFCCCalc *, SP16 *, int);

  boolean triggered;

  void (*callback_function[SIZEOF_CALLBACK_ID][MAX_CALLBACK_HOOK])();
  void *callback_user_data[SIZEOF_CALLBACK_ID][MAX_CALLBACK_HOOK];
  int callback_function_num[SIZEOF_CALLBACK_ID];
  int callback_list_code[MAX_CALLBACK_HOOK*SIZEOF_CALLBACK_ID];
  int callback_list_loc[MAX_CALLBACK_HOOK*SIZEOF_CALLBACK_ID];
  int callback_num;

  /*******************************************/

  void *hook;

} Recog;

#endif /* __J_RECOG_H__ */

---