julius/output_module.c

Go to the documentation of this file.

/*
 * Copyright (c) 1991-2007 Kawahara Lab., Kyoto University
 * Copyright (c) 2000-2005 Shikano Lab., Nara Institute of Science and Technology
 * Copyright (c) 2005-2007 Julius project team, Nagoya Institute of Technology
 * All rights reserved
 */

#include "app.h"

#include <time.h>

extern int module_sd;
extern boolean separate_score_flag;


static char *hookstr[] = {"", "delete", "activate", "deactivate"};

/**********************************************************************/
/* process online/offline status  */

static void
status_process_online(Recog *recog, void *dummy)
{
  module_send(module_sd, "<STARTPROC/>\n.\n");
}
static void
status_process_offline(Recog *recog, void *dummy)
{
  module_send(module_sd, "<STOPPROC/>\n.\n");
}

/**********************************************************************/
/* decode outcode "WLPSwlps" to each boolean value */
/* default: "WLPS" */
static boolean out1_word = FALSE, out1_lm = FALSE, out1_phone = FALSE, out1_score = FALSE;
static boolean out2_word = TRUE, out2_lm = TRUE, out2_phone = TRUE, out2_score = TRUE;
static boolean out1_never = TRUE, out2_never = FALSE;
#ifdef CONFIDENCE_MEASURE
static boolean out2_cm = TRUE;
#endif

void
decode_output_selection(char *str)
{
  int i;
  out1_word = out1_lm = out1_phone = out1_score = FALSE;
  out2_word = out2_lm = out2_phone = out2_score = FALSE;
#ifdef CONFIDENCE_MEASURE
  out2_cm = FALSE;
#endif
  for(i = strlen(str) - 1; i >= 0; i--) {
    switch(str[i]) {
    case 'W': out2_word  = TRUE; break;
    case 'L': out2_lm    = TRUE; break;
    case 'P': out2_phone = TRUE; break;
    case 'S': out2_score = TRUE; break;
    case 'w': out1_word  = TRUE; break;
    case 'l': out1_lm    = TRUE; break;
    case 'p': out1_phone = TRUE; break;
    case 's': out1_score = TRUE; break;
#ifdef CONFIDENCE_MEASURE
    case 'C': out2_cm    = TRUE; break;
#endif
    default:
      fprintf(stderr, "Error: unknown outcode `%c', ignored\n", str[i]);
      break;
    }
  }
  out1_never = ! (out1_word | out1_lm | out1_phone | out1_score);
  out2_never = ! (out2_word | out2_lm | out2_phone | out2_score
#ifdef CONFIDENCE_MEASURE
                  | out2_cm
#endif
                  );

}

static void
msock_word_out1(WORD_ID w, RecogProcess *r)
{
  int j;
  static char buf[MAX_HMMNAME_LEN];
  WORD_INFO *winfo;

  winfo = r->lm->winfo;

  if (out1_word) {
    module_send(module_sd, " WORD=\"%s\"", winfo->woutput[w]);
  }
  if (out1_lm) {
    module_send(module_sd, " CLASSID=\"%s\"", winfo->wname[w]);
  }
  if (out1_phone) {
    module_send(module_sd, " PHONE=\"");
    for(j=0;j<winfo->wlen[w];j++) {
      center_name(winfo->wseq[w][j]->name, buf);
      if (j == 0) module_send(module_sd, "%s", buf);
      else module_send(module_sd, " %s", buf);
    }
    module_send(module_sd, "\"");
  }
}

static void
msock_word_out2(WORD_ID w, RecogProcess *r)
{
  int j;
  static char buf[MAX_HMMNAME_LEN];
  WORD_INFO *winfo;

  winfo = r->lm->winfo;

  if (out2_word) {
    module_send(module_sd, " WORD=\"%s\"", winfo->woutput[w]);
  }
  if (out2_lm) {
    module_send(module_sd, " CLASSID=\"%s\"", winfo->wname[w]);
  }
  if (out2_phone) {
    module_send(module_sd, " PHONE=\"");
    for(j=0;j<winfo->wlen[w];j++) {
      center_name(winfo->wseq[w][j]->name, buf);
      if (j == 0) module_send(module_sd, "%s", buf);
      else module_send(module_sd, " %s", buf);
    }
    module_send(module_sd, "\"");
  }
}


/**********************************************************************/
/* 1st pass output */

static void
status_pass1_begin(Recog *recog, void *dummy)
{
  module_send(module_sd, "<STARTRECOG/>\n.\n");
}

static void
result_pass1_current(Recog *recog, void *dummy)
{
  int i;
  WORD_INFO *winfo;
  WORD_ID *seq;
  int num;
  RecogProcess *r;
  boolean multi;

  if (out1_never) return;       /* no output specified */

  if (recog->process_list->next != NULL) multi = TRUE;
  else multi = FALSE;

  for(r=recog->process_list;r;r=r->next) {
    if (! r->live) continue;
    if (! r->have_interim) continue;

    winfo = r->lm->winfo;
    seq = r->result.pass1.word;
    num = r->result.pass1.word_num;

    if (multi) {
      module_send(module_sd, "<RECOGOUT ID=\"SR%02d\" NAME=\"%s\">\n", r->config->id, r->config->name);
    } else {
      module_send(module_sd, "<RECOGOUT>\n");
    }
    if (out1_score) {
      module_send(module_sd, "  <PHYPO PASS=\"1\" SCORE=\"%f\" FRAME=\"%d\" TIME=\"%ld\"/>\n", r->result.pass1.score, r->result.num_frame, time(NULL));
    } else {
      module_send(module_sd, "  <PHYPO PASS=\"1\" FRAME=\"%d\" TIME=\"%ld\"/>\n", r->result.num_frame, time(NULL));
    }
    for (i=0;i<num;i++) {
      module_send(module_sd, "    <WHYPO");
      msock_word_out1(seq[i], r);
      module_send(module_sd, "/>\n");
    }
    module_send(module_sd, "  </PHYPO>\n</RECOGOUT>\n.\n");
  }
}

static void
result_pass1_final(Recog *recog, void *dummy)
{
  int i;
  RecogProcess *r;
  boolean multi;

  if (out1_never) return;       /* no output specified */

  if (recog->process_list->next != NULL) multi = TRUE;
  else multi = FALSE;

  for(r=recog->process_list;r;r=r->next) {
    if (! r->live) continue;
    if (r->result.status < 0) continue; /* search already failed  */

    if (multi) {
      module_send(module_sd, "<RECOGOUT ID=\"SR%02d\" NAME=\"%s\">\n", r->config->id, r->config->name);
    } else {
      module_send(module_sd, "<RECOGOUT>\n");
    }
    if (out1_score) {
      module_send(module_sd, "  <SHYPO PASS=\"1\" SCORE=\"%f\">\n", r->result.pass1.score);
    } else {
      module_send(module_sd, "  <SHYPO PASS=\"1\">\n", r->result.pass1.score);
    }
    for (i=0;i<r->result.pass1.word_num;i++) {
      module_send(module_sd, "    <WHYPO");
      msock_word_out1(r->result.pass1.word[i], r);
      module_send(module_sd, "/>\n");
    }
    module_send(module_sd, "  </SHYPO>\n</RECOGOUT>\n.\n");
  }
}

static void
status_pass1_end(Recog *recog, void *dummy)
{
  module_send(module_sd, "<ENDRECOG/>\n.\n");
}

/**********************************************************************/
/* 2nd pass output */

static void
result_pass2(Recog *recog, void *dummy)
{
  int i, n, num;
  WORD_INFO *winfo;
  WORD_ID *seq;
  int seqnum;
  Sentence *s;
  RecogProcess *r;
  boolean multi;

  if (recog->process_list->next != NULL) multi = TRUE;
  else multi = FALSE;

  for(r=recog->process_list;r;r=r->next) {
    if (! r->live) continue;

    if (r->result.status < 0) {
      switch(r->result.status) {
      case J_RESULT_STATUS_REJECT_POWER:
        module_send(module_sd, "<REJECTED REASON=\"by power\"");
        break;
      case J_RESULT_STATUS_TERMINATE:
        module_send(module_sd, "<REJECTED REASON=\"input terminated by request\"");
        break;
      case J_RESULT_STATUS_ONLY_SILENCE:
        module_send(module_sd, "<REJECTED REASON=\"result has pause words only\"");
        break;
      case J_RESULT_STATUS_REJECT_GMM:
        module_send(module_sd, "<REJECTED REASON=\"by GMM\"");
        break;
      case J_RESULT_STATUS_REJECT_SHORT:
        module_send(module_sd, "<REJECTED REASON=\"too short input\"");
        break;
      case J_RESULT_STATUS_FAIL:
        module_send(module_sd, "<RECOGFAIL");
        break;
      }
      if (multi) {
        module_send(module_sd, " ID=\"SR%02d\" NAME=\"%s\"", r->config->id, r->config->name);
      }
      module_send(module_sd, "/>\n.\n");
      continue;
    }

    if (out2_never) continue;   /* no output specified */

    winfo = r->lm->winfo;
    num = r->result.sentnum;

    if (multi) {
      module_send(module_sd, "<RECOGOUT ID=\"SR%02d\" NAME=\"%s\">\n", r->config->id, r->config->name);
    } else {
      module_send(module_sd, "<RECOGOUT>\n");
    }
    for(n=0;n<num;n++) {
      s = &(r->result.sent[n]);
      seq = s->word;
      seqnum = s->word_num;
      
      module_send(module_sd, "  <SHYPO RANK=\"%d\"", n+1);
      if (out2_score) {
        module_send(module_sd, " SCORE=\"%f\"", s->score);
        if (r->lmtype == LM_PROB) {
          if (separate_score_flag) {
            module_send(module_sd, " AMSCORE=\"%f\" LMSCORE=\"%f\"", s->score_am, s->score_lm);
          }
        }
      }
      if (r->lmtype == LM_DFA) {
        /* output which grammar the best hypothesis belongs to */
        module_send(module_sd, " GRAM=\"%d\"", s->gram_id);
      }
      
      module_send(module_sd, ">\n");
      for (i=0;i<seqnum;i++) {
        module_send(module_sd, "    <WHYPO");
        msock_word_out2(seq[i], r);
#ifdef CONFIDENCE_MEASURE
#ifdef CM_MULTIPLE_ALPHA
        /* currently not handle multiple alpha output */
#else
        if (out2_cm) {
          module_send(module_sd, " CM=\"%5.3f\"", s->confidence[i]);
        }
#endif
#endif /* CONFIDENCE_MEASURE */
        module_send(module_sd, "/>\n");
      }
      module_send(module_sd, "  </SHYPO>\n");
    }
    module_send(module_sd, "</RECOGOUT>\n.\n");

  }

}


/**********************************************************************/
/* word graph output */

static void
result_graph(Recog *recog, void *dummy)
{
  WordGraph *wg;
  int i;
  int nodenum, arcnum;
  WORD_INFO *winfo;
  WordGraph *root;
  RecogProcess *r;
  boolean multi;

  if (recog->process_list->next != NULL) multi = TRUE;
  else multi = FALSE;

  for(r=recog->process_list;r;r=r->next) {
    if (! r->live) continue;
    if (r->result.wg == NULL) continue; /* no graph obtained */

    winfo = r->lm->winfo;
    root = r->result.wg;
    nodenum = r->graph_totalwordnum;
    arcnum = 0;
    for(wg=root;wg;wg=wg->next) {
      arcnum += wg->rightwordnum;
    }

    module_send(module_sd, "<GRAPHOUT");
    if (multi) module_send(module_sd, " ID=\"SR%02d\" NAME=\"%s\"", r->config->id, r->config->name);
    module_send(module_sd, " NODENUM=\"%d\" ARCNUM=\"%d\">\n", nodenum, arcnum);
    for(wg=root;wg;wg=wg->next) {
      module_send(module_sd, "    <NODE GID=\"%d\"", wg->id);
      msock_word_out2(wg->wid, r);
      module_send(module_sd, " BEGIN=\"%d\"", wg->lefttime);
      module_send(module_sd, " END=\"%d\"", wg->righttime);
      module_send(module_sd, "/>\n");
    }
    for(wg=root;wg;wg=wg->next) {
      for(i=0;i<wg->rightwordnum;i++) {
        module_send(module_sd, "    <ARC FROM=\"%d\" TO=\"%d\"/>\n", wg->id, wg->rightword[i]->id);
      }
    }
    module_send(module_sd, "</GRAPHOUT>\n.\n");
  }
}

static void
status_recready(Recog *recog, void *dummy)
{
  module_send(module_sd, "<INPUT STATUS=\"LISTEN\" TIME=\"%ld\"/>\n.\n", time(NULL));
}

static void
status_recstart(Recog *recog, void *dummy)
{
  module_send(module_sd, "<INPUT STATUS=\"STARTREC\" TIME=\"%ld\"/>\n.\n", time(NULL));
}
static void
status_recend(Recog *recog, void *dummy)
{
  module_send(module_sd, "<INPUT STATUS=\"ENDREC\" TIME=\"%ld\"/>\n.\n", time(NULL));
}
static void
status_param(Recog *recog, void *dummy)
{
  MFCCCalc *mfcc;
  boolean multi;
  int frames;
  int msec;

  if (recog->mfcclist->next != NULL) multi = TRUE;
  else multi = FALSE;

  for(mfcc=recog->mfcclist;mfcc;mfcc=mfcc->next) {
    frames = mfcc->param->samplenum;
    msec = (float)mfcc->param->samplenum * (float)recog->jconf->input.period * (float)recog->jconf->input.frameshift / 10000.0;
    if (multi) {
      module_send(module_sd, "<INPUTPARAM MFCCID=\"%02d\" FRAMES=\"%d\" MSEC=\"%d\"/>\n.\n", mfcc->id, frames, msec);
    } else {
      module_send(module_sd, "<INPUTPARAM FRAMES=\"%d\" MSEC=\"%d\"/>\n.\n", frames, msec);
    }
  }
}

/********************* RESULT OUTPUT FOR GMM *************************/
static void
result_gmm(Recog *recog, void *dummy)
{
  module_send(module_sd, "<GMM RESULT=\"%s\"", recog->gc->max_d->name);
#ifdef CONFIDENCE_MEASURE
  module_send(module_sd, " CMSCORE=\"%f\"", recog->gc->gmm_max_cm);
#endif
  module_send(module_sd, "/>\n.\n");
}

void
send_gram_info(RecogProcess *r)
{
  MULTIGRAM *m;

  module_send(module_sd, "<GRAMINFO>\n");
  for(m=r->lm->grammars;m;m=m->next) {
    module_send(module_sd, "  #%2d: [%-11s] %4d words, %3d categories, %4d nodes",
                m->id,
                m->active ? "active" : "inactive",
                m->winfo->num, m->dfa->term_num, m->dfa->state_num);
    if (m->newbie) module_send(module_sd, " (new)");
    if (m->hook != MULTIGRAM_DEFAULT) {
      module_send(module_sd, " (next: %s)", hookstr[m->hook]);
    }
    module_send(module_sd, " \"%s\"\n", m->name);
  }
  if (r->lm->dfa != NULL) {
    module_send(module_sd, "  Global:            %4d words, %3d categories, %4d nodes\n", r->lm->winfo->num, r->lm->dfa->term_num, r->lm->dfa->state_num);
  }
  module_send(module_sd, "</GRAMINFO>\n.\n");
}

/**********************************************************************/
/* register functions for module output */
void
setup_output_msock(Recog *recog, void *data)
{
  callback_add(recog, CALLBACK_EVENT_PROCESS_ONLINE, status_process_online, data);
  callback_add(recog, CALLBACK_EVENT_PROCESS_OFFLINE, status_process_offline, data);
  //callback_add(recog, CALLBACK_EVENT_STREAM_BEGIN,     , data);
  //callback_add(recog, CALLBACK_EVENT_STREAM_END,        , data);
  callback_add(recog, CALLBACK_EVENT_SPEECH_READY, status_recready, data);
  callback_add(recog, CALLBACK_EVENT_SPEECH_START, status_recstart, data);
  callback_add(recog, CALLBACK_EVENT_SPEECH_STOP, status_recend, data);
  callback_add(recog, CALLBACK_EVENT_PASS1_BEGIN, status_pass1_begin, data);
  callback_add(recog, CALLBACK_EVENT_PASS1_END, status_pass1_end, data);
  callback_add(recog, CALLBACK_RESULT_PASS1_INTERIM, result_pass1_current, data);
  callback_add(recog, CALLBACK_RESULT_PASS1, result_pass1_final, data);

  callback_add(recog, CALLBACK_STATUS_PARAM, status_param, data);

  callback_add(recog, CALLBACK_RESULT, result_pass2, data); // rejected, failed
  callback_add(recog, CALLBACK_RESULT_GMM, result_gmm, data);
  /* below will not be called if "-graphout" not specified */
  callback_add(recog, CALLBACK_RESULT_GRAPH, result_graph, data);

  //callback_add(recog, CALLBACK_EVENT_PAUSE, status_pause, data);
  //callback_add(recog, CALLBACK_EVENT_RESUME, status_resume, data);

}

---