libsent/include/sent/ngram2.h

Go to the documentation of this file.

/*
 * Copyright (c) 1991-2006 Kawahara Lab., Kyoto University
 * Copyright (c) 2000-2005 Shikano Lab., Nara Institute of Science and Technology
 * Copyright (c) 2005-2006 Julius project team, Nagoya Institute of Technology
 * All rights reserved
 */

/***** revision 3 *****/
/* ngram2.h .. n-gram language model for speech recognition */
/* Third revision: sequencial allocation */
/* for disk-LM and rapid read-in */

#ifndef __SENT_NGRAM2_H__
#define __SENT_NGRAM2_H__

#include <sent/stddefs.h>
#include <sent/ptree.h>


#define MAX_N 3

typedef unsigned char NNID_UPPER; 
typedef unsigned short NNID_LOWER; 
typedef int NNID;              
#define NNID_INVALID -1         
#define NNID_INVALID_UPPER 255  
#define NNIDMAX 16711680        


typedef struct {
  int version;                  
  boolean from_bin;             
  WORD_ID max_word_num;         
  NNID ngram_num[MAX_N];        
  NNID bigram_bo_num;           

  WORD_ID unk_id;
  int unk_num;                  
  LOGPROB unk_num_log;          
  boolean isopen;               

  /* basic data (nid: 0 - max_word_num-1) */
  char **wname;                 
  PATNODE *root;                
  
  /* 1-gram ( nid: 0 - ngram_num[0]-1 ) */
  LOGPROB *p;                   
  LOGPROB *bo_wt_lr;            
  LOGPROB *bo_wt_rl;            
  NNID *n2_bgn; 
  WORD_ID *n2_num;              
  
  /* 2-gram (n2: 0 - ngram_num[1] - 1) */
  WORD_ID *n2tonid;             
  LOGPROB *p_lr;                
  LOGPROB *p_rl;                
  NNID_UPPER *n2bo_upper; 
  NNID_LOWER *n2bo_lower; 
  
  /* 2-gram back-off values (separated from  rev.3.5) */
  LOGPROB *bo_wt_rrl;           
  NNID *n3_bgn;                 
  NNID_UPPER *n3_bgn_upper; 
  NNID_LOWER *n3_bgn_lower; 
  WORD_ID *n3_num;              
  
  /* 3-gram (n3: 0 - ngram_num[2] - 1) */
  WORD_ID *n3tonid;             
  LOGPROB *p_rrl;               
} NGRAM_INFO;


/* Definitions for binary N-gram */

#define BINGRAM_IDSTR "julius_bingram_v3"
#define BINGRAM_IDSTR_V4 "julius_bingram_v4"
#define BINGRAM_HDSIZE 512
#define BINGRAM_SIZESTR_HEAD "word="
#define BINGRAM_SIZESTR_BODY_4BYTE "4byte(int)"
#define BINGRAM_SIZESTR_BODY_2BYTE "2byte(unsigned short)"
#ifdef WORDS_INT
#define BINGRAM_SIZESTR_BODY BINGRAM_SIZESTR_BODY_4BYTE
#else
#define BINGRAM_SIZESTR_BODY BINGRAM_SIZESTR_BODY_2BYTE
#endif
#define BINGRAM_BYTEORDER_HEAD "byteorder="
#ifdef WORDS_BIGENDIAN
#define BINGRAM_NATURAL_BYTEORDER "BE"
#else
#define BINGRAM_NATURAL_BYTEORDER "LE"
#endif


/* function declaration */
NNID search_bigram(NGRAM_INFO *ndata, WORD_ID w_l, WORD_ID w_r);
// NNID search_trigram(NGRAM_INFO *ndata,  NNID n2, WORD_ID wkey);
LOGPROB uni_prob(NGRAM_INFO *ndata, WORD_ID w);
LOGPROB bi_prob_lr(NGRAM_INFO *ndata, WORD_ID w1, WORD_ID w2);
LOGPROB bi_prob_rl(NGRAM_INFO *ndata, WORD_ID w1, WORD_ID w2);
LOGPROB tri_prob_rl(NGRAM_INFO *ndata, WORD_ID w1, WORD_ID w2, WORD_ID w3);

boolean ngram_read_arpa(FILE *fp, NGRAM_INFO *ndata, int direction);
void set_unknown_id(NGRAM_INFO *ndata);
boolean ngram_read_bin(FILE *fp, NGRAM_INFO *ndata);
boolean ngram_write_bin(FILE *fp, NGRAM_INFO *ndata, char *header_str);

void ngram_make_lookup_tree(NGRAM_INFO *ndata);
WORD_ID ngram_lookup_word(NGRAM_INFO *ndata, char *wordstr);
WORD_ID make_ngram_ref(NGRAM_INFO *, char *);

NGRAM_INFO *ngram_info_new();
void ngram_info_free(NGRAM_INFO *ngram);
void init_ngram_bin(NGRAM_INFO *ndata, char *ngram_file);
void init_ngram_arpa(NGRAM_INFO *ndata, char *lrfile, char *rlfile);

void ngram_compact_bigram_context(NGRAM_INFO *ndata);

void print_ngram_info(NGRAM_INFO *ndata);

#include <sent/vocabulary.h>
void make_voca_ref(NGRAM_INFO *ndata, WORD_INFO *winfo);

#endif /* __SENT_NGRAM2_H__ */

---