PocketSphinx  0.6
bin_mdef.c
1 /* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */
2 /* ====================================================================
3  * Copyright (c) 2005 Carnegie Mellon University. All rights
4  * reserved.
5  *
6  * Redistribution and use in source and binary forms, with or without
7  * modification, are permitted provided that the following conditions
8  * are met:
9  *
10  * 1. Redistributions of source code must retain the above copyright
11  * notice, this list of conditions and the following disclaimer.
12  *
13  * 2. Redistributions in binary form must reproduce the above copyright
14  * notice, this list of conditions and the following disclaimer in
15  * the documentation and/or other materials provided with the
16  * distribution.
17  *
18  * This work was supported in part by funding from the Defense Advanced
19  * Research Projects Agency and the National Science Foundation of the
20  * United States of America, and the CMU Sphinx Speech Consortium.
21  *
22  * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND
23  * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
24  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
25  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY
26  * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
27  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
28  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
29  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
30  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
31  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
32  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
33  *
34  * ====================================================================
35  *
36  */
37 /*********************************************************************
38  *
39  * File: bin_mdef.c
40  *
41  * Description:
42  * Binary format model definition files, with support for
43  * heterogeneous topologies and variable-size N-phones
44  *
45  * Author:
46  * David Huggins-Daines <dhuggins@cs.cmu.edu>
47  *********************************************************************/
48 
49 /* System headers. */
50 #include <stdio.h>
51 #include <string.h>
52 #include <assert.h>
53 
54 /* SphinxBase headers. */
55 #include <sphinxbase/prim_type.h>
56 #include <sphinxbase/ckd_alloc.h>
57 #include <sphinxbase/byteorder.h>
58 #include <sphinxbase/case.h>
59 #include <sphinxbase/err.h>
60 
61 /* Local headers. */
62 #include "mdef.h"
63 #include "bin_mdef.h"
64 
65 bin_mdef_t *
66 bin_mdef_read_text(cmd_ln_t *config, const char *filename)
67 {
68  bin_mdef_t *bmdef;
69  mdef_t *mdef;
70  int i, nodes, ci_idx, lc_idx, rc_idx;
71  int nchars;
72 
73  if ((mdef = mdef_init((char *) filename, TRUE)) == NULL)
74  return NULL;
75 
76  /* Enforce some limits. */
77  if (mdef->n_sen > BAD_SENID) {
78  E_ERROR("Number of senones exceeds limit: %d > %d\n",
79  mdef->n_sen, BAD_SENID);
80  mdef_free(mdef);
81  return NULL;
82  }
83  if (mdef->n_sseq > BAD_SSID) {
84  E_ERROR("Number of senone sequences exceeds limit: %d > %d\n",
85  mdef->n_sseq, BAD_SSID);
86  mdef_free(mdef);
87  return NULL;
88  }
89 
90 
91  bmdef = ckd_calloc(1, sizeof(*bmdef));
92  bmdef->refcnt = 1;
93 
94  /* Easy stuff. The mdef.c code has done the heavy lifting for us. */
95  bmdef->n_ciphone = mdef->n_ciphone;
96  bmdef->n_phone = mdef->n_phone;
97  bmdef->n_emit_state = mdef->n_emit_state;
98  bmdef->n_ci_sen = mdef->n_ci_sen;
99  bmdef->n_sen = mdef->n_sen;
100  bmdef->n_tmat = mdef->n_tmat;
101  bmdef->n_sseq = mdef->n_sseq;
102  bmdef->sseq = mdef->sseq;
103  bmdef->cd2cisen = mdef->cd2cisen;
104  bmdef->sen2cimap = mdef->sen2cimap;
105  bmdef->n_ctx = 3; /* Triphones only. */
106  bmdef->sil = mdef->sil;
107  mdef->sseq = NULL; /* We are taking over this one. */
108  mdef->cd2cisen = NULL; /* And this one. */
109  mdef->sen2cimap = NULL; /* And this one. */
110 
111  /* Get the phone names. If they are not sorted
112  * ASCII-betically then we are in a world of hurt and
113  * therefore will simply refuse to continue. */
114  bmdef->ciname = ckd_calloc(bmdef->n_ciphone, sizeof(*bmdef->ciname));
115  nchars = 0;
116  for (i = 0; i < bmdef->n_ciphone; ++i)
117  nchars += strlen(mdef->ciphone[i].name) + 1;
118  bmdef->ciname[0] = ckd_calloc(nchars, 1);
119  strcpy(bmdef->ciname[0], mdef->ciphone[0].name);
120  for (i = 1; i < bmdef->n_ciphone; ++i) {
121  bmdef->ciname[i] =
122  bmdef->ciname[i - 1] + strlen(bmdef->ciname[i - 1]) + 1;
123  strcpy(bmdef->ciname[i], mdef->ciphone[i].name);
124  if (i > 0 && strcmp(bmdef->ciname[i - 1], bmdef->ciname[i]) > 0) {
125  /* FIXME: there should be a solution to this, actually. */
126  E_ERROR("Phone names are not in sorted order, sorry.");
127  bin_mdef_free(bmdef);
128  return NULL;
129  }
130  }
131 
132  /* Copy over phone information. */
133  bmdef->phone = ckd_calloc(bmdef->n_phone, sizeof(*bmdef->phone));
134  for (i = 0; i < mdef->n_phone; ++i) {
135  bmdef->phone[i].ssid = mdef->phone[i].ssid;
136  bmdef->phone[i].tmat = mdef->phone[i].tmat;
137  if (i < bmdef->n_ciphone) {
138  bmdef->phone[i].info.ci.filler = mdef->ciphone[i].filler;
139  }
140  else {
141  bmdef->phone[i].info.cd.wpos = mdef->phone[i].wpos;
142  bmdef->phone[i].info.cd.ctx[0] = mdef->phone[i].ci;
143  bmdef->phone[i].info.cd.ctx[1] = mdef->phone[i].lc;
144  bmdef->phone[i].info.cd.ctx[2] = mdef->phone[i].rc;
145  }
146  }
147 
148  /* Walk the wpos_ci_lclist once to find the total number of
149  * nodes and the starting locations for each level. */
150  nodes = lc_idx = ci_idx = rc_idx = 0;
151  for (i = 0; i < N_WORD_POSN; ++i) {
152  int j;
153  for (j = 0; j < mdef->n_ciphone; ++j) {
154  ph_lc_t *lc;
155 
156  for (lc = mdef->wpos_ci_lclist[i][j]; lc; lc = lc->next) {
157  ph_rc_t *rc;
158  for (rc = lc->rclist; rc; rc = rc->next) {
159  ++nodes; /* RC node */
160  }
161  ++nodes; /* LC node */
162  ++rc_idx; /* Start of RC nodes (after LC nodes) */
163  }
164  ++nodes; /* CI node */
165  ++lc_idx; /* Start of LC nodes (after CI nodes) */
166  ++rc_idx; /* Start of RC nodes (after CI and LC nodes) */
167  }
168  ++nodes; /* wpos node */
169  ++ci_idx; /* Start of CI nodes (after wpos nodes) */
170  ++lc_idx; /* Start of LC nodes (after CI nodes) */
171  ++rc_idx; /* STart of RC nodes (after wpos, CI, and LC nodes) */
172  }
173  E_INFO("Allocating %d * %d bytes (%d KiB) for CD tree\n",
174  nodes, sizeof(*bmdef->cd_tree),
175  nodes * sizeof(*bmdef->cd_tree) / 1024);
176  bmdef->n_cd_tree = nodes;
177  bmdef->cd_tree = ckd_calloc(nodes, sizeof(*bmdef->cd_tree));
178  for (i = 0; i < N_WORD_POSN; ++i) {
179  int j;
180 
181  bmdef->cd_tree[i].ctx = i;
182  bmdef->cd_tree[i].n_down = mdef->n_ciphone;
183  bmdef->cd_tree[i].c.down = ci_idx;
184 #if 0
185  E_INFO("%d => %c (%d@%d)\n",
186  i, (WPOS_NAME)[i],
187  bmdef->cd_tree[i].n_down, bmdef->cd_tree[i].c.down);
188 #endif
189 
190  /* Now we can build the rest of the tree. */
191  for (j = 0; j < mdef->n_ciphone; ++j) {
192  ph_lc_t *lc;
193 
194  bmdef->cd_tree[ci_idx].ctx = j;
195  bmdef->cd_tree[ci_idx].c.down = lc_idx;
196  for (lc = mdef->wpos_ci_lclist[i][j]; lc; lc = lc->next) {
197  ph_rc_t *rc;
198 
199  bmdef->cd_tree[lc_idx].ctx = lc->lc;
200  bmdef->cd_tree[lc_idx].c.down = rc_idx;
201  for (rc = lc->rclist; rc; rc = rc->next) {
202  bmdef->cd_tree[rc_idx].ctx = rc->rc;
203  bmdef->cd_tree[rc_idx].n_down = 0;
204  bmdef->cd_tree[rc_idx].c.pid = rc->pid;
205 #if 0
206  E_INFO("%d => %s %s %s %c (%d@%d)\n",
207  rc_idx,
208  bmdef->ciname[j],
209  bmdef->ciname[lc->lc],
210  bmdef->ciname[rc->rc],
211  (WPOS_NAME)[i],
212  bmdef->cd_tree[rc_idx].n_down,
213  bmdef->cd_tree[rc_idx].c.down);
214 #endif
215 
216  ++bmdef->cd_tree[lc_idx].n_down;
217  ++rc_idx;
218  }
219  /* If there are no triphones here,
220  * this is considered a leafnode, so
221  * set the pid to -1. */
222  if (bmdef->cd_tree[lc_idx].n_down == 0)
223  bmdef->cd_tree[lc_idx].c.pid = -1;
224 #if 0
225  E_INFO("%d => %s %s %c (%d@%d)\n",
226  lc_idx,
227  bmdef->ciname[j],
228  bmdef->ciname[lc->lc],
229  (WPOS_NAME)[i],
230  bmdef->cd_tree[lc_idx].n_down,
231  bmdef->cd_tree[lc_idx].c.down);
232 #endif
233 
234  ++bmdef->cd_tree[ci_idx].n_down;
235  ++lc_idx;
236  }
237 
238  /* As above, so below. */
239  if (bmdef->cd_tree[ci_idx].n_down == 0)
240  bmdef->cd_tree[ci_idx].c.pid = -1;
241 #if 0
242  E_INFO("%d => %d=%s (%d@%d)\n",
243  ci_idx, j, bmdef->ciname[j],
244  bmdef->cd_tree[ci_idx].n_down,
245  bmdef->cd_tree[ci_idx].c.down);
246 #endif
247 
248  ++ci_idx;
249  }
250  }
251 
252  mdef_free(mdef);
253 
254  bmdef->alloc_mode = BIN_MDEF_FROM_TEXT;
255  return bmdef;
256 }
257 
258 bin_mdef_t *
259 bin_mdef_retain(bin_mdef_t *m)
260 {
261  ++m->refcnt;
262  return m;
263 }
264 
265 int
266 bin_mdef_free(bin_mdef_t * m)
267 {
268  if (m == NULL)
269  return 0;
270  if (--m->refcnt > 0)
271  return m->refcnt;
272 
273  switch (m->alloc_mode) {
274  case BIN_MDEF_FROM_TEXT:
275  ckd_free(m->ciname[0]);
276  ckd_free(m->sseq[0]);
277  ckd_free(m->phone);
278  ckd_free(m->cd_tree);
279  break;
280  case BIN_MDEF_IN_MEMORY:
281  ckd_free(m->ciname[0]);
282  break;
283  case BIN_MDEF_ON_DISK:
284  break;
285  }
286  if (m->filemap)
287  mmio_file_unmap(m->filemap);
288  ckd_free(m->cd2cisen);
289  ckd_free(m->sen2cimap);
290  ckd_free(m->ciname);
291  ckd_free(m->sseq);
292  ckd_free(m);
293  return 0;
294 }
295 
296 static const char format_desc[] =
297  "BEGIN FILE FORMAT DESCRIPTION\n"
298  "int32 n_ciphone; /**< Number of base (CI) phones */\n"
299  "int32 n_phone; /**< Number of base (CI) phones + (CD) triphones */\n"
300  "int32 n_emit_state; /**< Number of emitting states per phone (0 if heterogeneous) */\n"
301  "int32 n_ci_sen; /**< Number of CI senones; these are the first */\n"
302  "int32 n_sen; /**< Number of senones (CI+CD) */\n"
303  "int32 n_tmat; /**< Number of transition matrices */\n"
304  "int32 n_sseq; /**< Number of unique senone sequences */\n"
305  "int32 n_ctx; /**< Number of phones of context */\n"
306  "int32 n_cd_tree; /**< Number of nodes in CD tree structure */\n"
307  "int32 sil; /**< CI phone ID for silence */\n"
308  "char ciphones[][]; /**< CI phone strings (null-terminated) */\n"
309  "char padding[]; /**< Padding to a 4-bytes boundary */\n"
310  "struct { int16 ctx; int16 n_down; int32 pid/down } cd_tree[];\n"
311  "struct { int32 ssid; int32 tmat; int8 attr[4] } phones[];\n"
312  "int16 sseq[]; /**< Unique senone sequences */\n"
313  "int8 sseq_len[]; /**< Number of states in each sseq (none if homogeneous) */\n"
314  "END FILE FORMAT DESCRIPTION\n";
315 
316 bin_mdef_t *
317 bin_mdef_read(cmd_ln_t *config, const char *filename)
318 {
319  bin_mdef_t *m;
320  FILE *fh;
321  size_t tree_start;
322  int32 val, i, swap, pos, end;
323  int32 *sseq_size;
324  int do_mmap;
325 
326  /* Try to read it as text first. */
327  if ((m = bin_mdef_read_text(config, filename)) != NULL)
328  return m;
329 
330  E_INFO("Reading binary model definition: %s\n", filename);
331  if ((fh = fopen(filename, "rb")) == NULL)
332  return NULL;
333 
334  if (fread(&val, 4, 1, fh) != 1) {
335  fclose(fh);
336  E_ERROR_SYSTEM("Failed to read byte-order marker from %s\n",
337  filename);
338  return NULL;
339  }
340  swap = 0;
341  if (val == BIN_MDEF_OTHER_ENDIAN) {
342  swap = 1;
343  E_INFO("Must byte-swap %s\n", filename);
344  }
345  if (fread(&val, 4, 1, fh) != 1) {
346  fclose(fh);
347  E_ERROR_SYSTEM("Failed to read version from %s\n", filename);
348  return NULL;
349  }
350  if (swap)
351  SWAP_INT32(&val);
352  if (val > BIN_MDEF_FORMAT_VERSION) {
353  E_ERROR("File format version %d for %s is newer than library\n",
354  val, filename);
355  fclose(fh);
356  return NULL;
357  }
358  if (fread(&val, 4, 1, fh) != 1) {
359  fclose(fh);
360  E_ERROR_SYSTEM("Failed to read header length from %s\n", filename);
361  return NULL;
362  }
363  if (swap)
364  SWAP_INT32(&val);
365  /* Skip format descriptor. */
366  fseek(fh, val, SEEK_CUR);
367 
368  /* Finally allocate it. */
369  m = ckd_calloc(1, sizeof(*m));
370  m->refcnt = 1;
371 
372  /* Check these, to make gcc/glibc shut up. */
373 #define FREAD_SWAP32_CHK(dest) \
374  if (fread((dest), 4, 1, fh) != 1) { \
375  fclose(fh); \
376  ckd_free(m); \
377  E_ERROR_SYSTEM("Failed to read %s from %s\n", #dest, filename); \
378  return NULL; \
379  } \
380  if (swap) SWAP_INT32(dest);
381 
382  FREAD_SWAP32_CHK(&m->n_ciphone);
383  FREAD_SWAP32_CHK(&m->n_phone);
384  FREAD_SWAP32_CHK(&m->n_emit_state);
385  FREAD_SWAP32_CHK(&m->n_ci_sen);
386  FREAD_SWAP32_CHK(&m->n_sen);
387  FREAD_SWAP32_CHK(&m->n_tmat);
388  FREAD_SWAP32_CHK(&m->n_sseq);
389  FREAD_SWAP32_CHK(&m->n_ctx);
390  FREAD_SWAP32_CHK(&m->n_cd_tree);
391  FREAD_SWAP32_CHK(&m->sil);
392 
393  /* CI names are first in the file. */
394  m->ciname = ckd_calloc(m->n_ciphone, sizeof(*m->ciname));
395 
396  /* Decide whether to read in the whole file or mmap it. */
397  do_mmap = config ? cmd_ln_boolean_r(config, "-mmap") : TRUE;
398  if (swap) {
399  E_WARN("-mmap specified, but mdef is other-endian. Will not memory-map.\n");
400  do_mmap = FALSE;
401  }
402  /* Actually try to mmap it. */
403  if (do_mmap) {
404  m->filemap = mmio_file_read(filename);
405  if (m->filemap == NULL)
406  do_mmap = FALSE;
407  }
408  pos = ftell(fh);
409  if (do_mmap) {
410  /* Get the base pointer from the memory map. */
411  m->ciname[0] = (char *)mmio_file_ptr(m->filemap) + pos;
412  /* Success! */
413  m->alloc_mode = BIN_MDEF_ON_DISK;
414  }
415  else {
416  /* Read everything into memory. */
417  m->alloc_mode = BIN_MDEF_IN_MEMORY;
418  fseek(fh, 0, SEEK_END);
419  end = ftell(fh);
420  fseek(fh, pos, SEEK_SET);
421  m->ciname[0] = ckd_malloc(end - pos);
422  if (fread(m->ciname[0], 1, end - pos, fh) != end - pos)
423  E_FATAL("Failed to read %d bytes of data from %s\n", end - pos, filename);
424  }
425 
426  for (i = 1; i < m->n_ciphone; ++i)
427  m->ciname[i] = m->ciname[i - 1] + strlen(m->ciname[i - 1]) + 1;
428 
429  /* Skip past the padding. */
430  tree_start =
431  m->ciname[i - 1] + strlen(m->ciname[i - 1]) + 1 - m->ciname[0];
432  tree_start = (tree_start + 3) & ~3;
433  m->cd_tree = (cd_tree_t *) (m->ciname[0] + tree_start);
434  if (swap) {
435  for (i = 0; i < m->n_cd_tree; ++i) {
436  SWAP_INT16(&m->cd_tree[i].ctx);
437  SWAP_INT16(&m->cd_tree[i].n_down);
438  SWAP_INT32(&m->cd_tree[i].c.down);
439  }
440  }
441  m->phone = (mdef_entry_t *) (m->cd_tree + m->n_cd_tree);
442  if (swap) {
443  for (i = 0; i < m->n_phone; ++i) {
444  SWAP_INT32(&m->phone[i].ssid);
445  SWAP_INT32(&m->phone[i].tmat);
446  }
447  }
448  sseq_size = (int32 *) (m->phone + m->n_phone);
449  if (swap)
450  SWAP_INT32(sseq_size);
451  m->sseq = ckd_calloc(m->n_sseq, sizeof(*m->sseq));
452  m->sseq[0] = (uint16 *) (sseq_size + 1);
453  if (swap) {
454  for (i = 0; i < *sseq_size; ++i)
455  SWAP_INT16(m->sseq[0] + i);
456  }
457  if (m->n_emit_state) {
458  for (i = 1; i < m->n_sseq; ++i)
459  m->sseq[i] = m->sseq[0] + i * m->n_emit_state;
460  }
461  else {
462  m->sseq_len = (uint8 *) (m->sseq[0] + *sseq_size);
463  for (i = 1; i < m->n_sseq; ++i)
464  m->sseq[i] = m->sseq[i - 1] + m->sseq_len[i - 1];
465  }
466 
467  /* Now build the CD-to-CI mappings using the senone sequences.
468  * This is the only really accurate way to do it, though it is
469  * still inaccurate in the case of heterogeneous topologies or
470  * cross-state tying. */
471  m->cd2cisen = (int16 *) ckd_malloc(m->n_sen * sizeof(*m->cd2cisen));
472  m->sen2cimap = (int16 *) ckd_malloc(m->n_sen * sizeof(*m->sen2cimap));
473 
474  /* Default mappings (identity, none) */
475  for (i = 0; i < m->n_ci_sen; ++i)
476  m->cd2cisen[i] = i;
477  for (; i < m->n_sen; ++i)
478  m->cd2cisen[i] = -1;
479  for (i = 0; i < m->n_sen; ++i)
480  m->sen2cimap[i] = -1;
481  for (i = 0; i < m->n_phone; ++i) {
482  int32 j, ssid = m->phone[i].ssid;
483 
484  for (j = 0; j < bin_mdef_n_emit_state_phone(m, i); ++j) {
485  int s = bin_mdef_sseq2sen(m, ssid, j);
486  int ci = bin_mdef_pid2ci(m, i);
487  /* Take the first one and warn if we have cross-state tying. */
488  if (m->sen2cimap[s] == -1)
489  m->sen2cimap[s] = ci;
490  if (m->sen2cimap[s] != ci)
491  E_WARN
492  ("Senone %d is shared between multiple base phones\n",
493  s);
494 
495  if (j > bin_mdef_n_emit_state_phone(m, ci))
496  E_WARN("CD phone %d has fewer states than CI phone %d\n",
497  i, ci);
498  else
499  m->cd2cisen[s] =
500  bin_mdef_sseq2sen(m, m->phone[ci].ssid, j);
501  }
502  }
503 
504  /* Set the silence phone. */
505  m->sil = bin_mdef_ciphone_id(m, S3_SILENCE_CIPHONE);
506 
507  E_INFO
508  ("%d CI-phone, %d CD-phone, %d emitstate/phone, %d CI-sen, %d Sen, %d Sen-Seq\n",
509  m->n_ciphone, m->n_phone - m->n_ciphone, m->n_emit_state,
510  m->n_ci_sen, m->n_sen, m->n_sseq);
511  fclose(fh);
512  return m;
513 }
514 
515 int
516 bin_mdef_write(bin_mdef_t * m, const char *filename)
517 {
518  FILE *fh;
519  int32 val, i;
520 
521  if ((fh = fopen(filename, "wb")) == NULL)
522  return -1;
523 
524  /* Byteorder marker. */
525  val = BIN_MDEF_NATIVE_ENDIAN;
526  fwrite(&val, 1, 4, fh);
527  /* Version. */
528  val = BIN_MDEF_FORMAT_VERSION;
529  fwrite(&val, 1, sizeof(val), fh);
530 
531  /* Round the format descriptor size up to a 4-byte boundary. */
532  val = ((sizeof(format_desc) + 3) & ~3);
533  fwrite(&val, 1, sizeof(val), fh);
534  fwrite(format_desc, 1, sizeof(format_desc), fh);
535  /* Pad it with zeros. */
536  i = 0;
537  fwrite(&i, 1, val - sizeof(format_desc), fh);
538 
539  /* Binary header things. */
540  fwrite(&m->n_ciphone, 4, 1, fh);
541  fwrite(&m->n_phone, 4, 1, fh);
542  fwrite(&m->n_emit_state, 4, 1, fh);
543  fwrite(&m->n_ci_sen, 4, 1, fh);
544  fwrite(&m->n_sen, 4, 1, fh);
545  fwrite(&m->n_tmat, 4, 1, fh);
546  fwrite(&m->n_sseq, 4, 1, fh);
547  fwrite(&m->n_ctx, 4, 1, fh);
548  fwrite(&m->n_cd_tree, 4, 1, fh);
549  /* Write this as a 32-bit value to preserve alignment for the
550  * non-mmap case (we want things aligned both from the
551  * beginning of the file and the beginning of the phone
552  * strings). */
553  val = m->sil;
554  fwrite(&val, 4, 1, fh);
555 
556  /* Phone strings. */
557  for (i = 0; i < m->n_ciphone; ++i)
558  fwrite(m->ciname[i], 1, strlen(m->ciname[i]) + 1, fh);
559  /* Pad with zeros. */
560  val = (ftell(fh) + 3) & ~3;
561  i = 0;
562  fwrite(&i, 1, val - ftell(fh), fh);
563 
564  /* Write CD-tree */
565  fwrite(m->cd_tree, sizeof(*m->cd_tree), m->n_cd_tree, fh);
566  /* Write phones */
567  fwrite(m->phone, sizeof(*m->phone), m->n_phone, fh);
568  if (m->n_emit_state) {
569  /* Write size of sseq */
570  val = m->n_sseq * m->n_emit_state;
571  fwrite(&val, 4, 1, fh);
572 
573  /* Write sseq */
574  fwrite(m->sseq[0], sizeof(**m->sseq),
575  m->n_sseq * m->n_emit_state, fh);
576  }
577  else {
578  int32 n;
579 
580  /* Calcluate size of sseq */
581  n = 0;
582  for (i = 0; i < m->n_sseq; ++i)
583  n += m->sseq_len[i];
584 
585  /* Write size of sseq */
586  fwrite(&n, 4, 1, fh);
587 
588  /* Write sseq */
589  fwrite(m->sseq[0], sizeof(**m->sseq), n, fh);
590 
591  /* Write sseq_len */
592  fwrite(m->sseq_len, 1, m->n_sseq, fh);
593  }
594  fclose(fh);
595 
596  return 0;
597 }
598 
599 int
600 bin_mdef_write_text(bin_mdef_t * m, const char *filename)
601 {
602  FILE *fh;
603  int p, i, n_total_state;
604 
605  if (strcmp(filename, "-") == 0)
606  fh = stdout;
607  else {
608  if ((fh = fopen(filename, "w")) == NULL)
609  return -1;
610  }
611 
612  fprintf(fh, "0.3\n");
613  fprintf(fh, "%d n_base\n", m->n_ciphone);
614  fprintf(fh, "%d n_tri\n", m->n_phone - m->n_ciphone);
615  if (m->n_emit_state)
616  n_total_state = m->n_phone * (m->n_emit_state + 1);
617  else {
618  n_total_state = 0;
619  for (i = 0; i < m->n_phone; ++i)
620  n_total_state += m->sseq_len[m->phone[i].ssid] + 1;
621  }
622  fprintf(fh, "%d n_state_map\n", n_total_state);
623  fprintf(fh, "%d n_tied_state\n", m->n_sen);
624  fprintf(fh, "%d n_tied_ci_state\n", m->n_ci_sen);
625  fprintf(fh, "%d n_tied_tmat\n", m->n_tmat);
626  fprintf(fh, "#\n# Columns definitions\n");
627  fprintf(fh, "#%4s %3s %3s %1s %6s %4s %s\n",
628  "base", "lft", "rt", "p", "attrib", "tmat",
629  " ... state id's ...");
630 
631  for (p = 0; p < m->n_ciphone; p++) {
632  int n_state;
633 
634  fprintf(fh, "%5s %3s %3s %1s", m->ciname[p], "-", "-", "-");
635 
636  if (bin_mdef_is_fillerphone(m, p))
637  fprintf(fh, " %6s", "filler");
638  else
639  fprintf(fh, " %6s", "n/a");
640  fprintf(fh, " %4d", m->phone[p].tmat);
641 
642  if (m->n_emit_state)
643  n_state = m->n_emit_state;
644  else
645  n_state = m->sseq_len[m->phone[p].ssid];
646  for (i = 0; i < n_state; i++) {
647  fprintf(fh, " %6u", m->sseq[m->phone[p].ssid][i]);
648  }
649  fprintf(fh, " N\n");
650  }
651 
652 
653  for (; p < m->n_phone; p++) {
654  int n_state;
655 
656  fprintf(fh, "%5s %3s %3s %c",
657  m->ciname[m->phone[p].info.cd.ctx[0]],
658  m->ciname[m->phone[p].info.cd.ctx[1]],
659  m->ciname[m->phone[p].info.cd.ctx[2]],
660  (WPOS_NAME)[m->phone[p].info.cd.wpos]);
661 
662  if (bin_mdef_is_fillerphone(m, p))
663  fprintf(fh, " %6s", "filler");
664  else
665  fprintf(fh, " %6s", "n/a");
666  fprintf(fh, " %4d", m->phone[p].tmat);
667 
668 
669  if (m->n_emit_state)
670  n_state = m->n_emit_state;
671  else
672  n_state = m->sseq_len[m->phone[p].ssid];
673  for (i = 0; i < n_state; i++) {
674  fprintf(fh, " %6u", m->sseq[m->phone[p].ssid][i]);
675  }
676  fprintf(fh, " N\n");
677  }
678 
679  if (strcmp(filename, "-") != 0)
680  fclose(fh);
681  return 0;
682 }
683 
684 int
685 bin_mdef_ciphone_id(bin_mdef_t * m, const char *ciphone)
686 {
687  int low, mid, high;
688 
689  /* Exact binary search on m->ciphone */
690  low = 0;
691  high = m->n_ciphone;
692  while (low < high) {
693  int c;
694 
695  mid = (low + high) / 2;
696  c = strcmp(ciphone, m->ciname[mid]);
697  if (c == 0)
698  return mid;
699  else if (c > 0)
700  low = mid + 1;
701  else if (c < 0)
702  high = mid;
703  }
704  return -1;
705 }
706 
707 int
708 bin_mdef_ciphone_id_nocase(bin_mdef_t * m, const char *ciphone)
709 {
710  int low, mid, high;
711 
712  /* Exact binary search on m->ciphone */
713  low = 0;
714  high = m->n_ciphone;
715  while (low < high) {
716  int c;
717 
718  mid = (low + high) / 2;
719  c = strcmp_nocase(ciphone, m->ciname[mid]);
720  if (c == 0)
721  return mid;
722  else if (c > 0)
723  low = mid + 1;
724  else if (c < 0)
725  high = mid;
726  }
727  return -1;
728 }
729 
730 const char *
731 bin_mdef_ciphone_str(bin_mdef_t * m, int32 ci)
732 {
733  assert(m != NULL);
734  assert(ci < m->n_ciphone);
735  return m->ciname[ci];
736 }
737 
738 int
739 bin_mdef_phone_id(bin_mdef_t * m, int32 ci, int32 lc, int32 rc, int32 wpos)
740 {
741  cd_tree_t *cd_tree;
742  int level, max;
743  int16 ctx[4];
744 
745  assert(m);
746 
747  /* In the future, we might back off when context is not available,
748  * but for now we'll just return the CI phone. */
749  if (lc < 0 || rc < 0)
750  return ci;
751 
752  assert((ci >= 0) && (ci < m->n_ciphone));
753  assert((lc >= 0) && (lc < m->n_ciphone));
754  assert((rc >= 0) && (rc < m->n_ciphone));
755  assert((wpos >= 0) && (wpos < N_WORD_POSN));
756 
757  /* Create a context list, mapping fillers to silence. */
758  ctx[0] = wpos;
759  ctx[1] = ci;
760  ctx[2] = (m->sil >= 0
761  && m->phone[lc].info.ci.filler) ? m->sil : lc;
762  ctx[3] = (m->sil >= 0
763  && m->phone[rc].info.ci.filler) ? m->sil : rc;
764 
765  /* Walk down the cd_tree. */
766  cd_tree = m->cd_tree;
767  level = 0; /* What level we are on. */
768  max = N_WORD_POSN; /* Number of nodes on this level. */
769  while (level < 4) {
770  int i;
771 
772 #if 0
773  E_INFO("Looking for context %d=%s in %d at %d\n",
774  ctx[level], m->ciname[ctx[level]],
775  max, cd_tree - m->cd_tree);
776 #endif
777  for (i = 0; i < max; ++i) {
778 #if 0
779  E_INFO("Look at context %d=%s at %d\n",
780  cd_tree[i].ctx,
781  m->ciname[cd_tree[i].ctx], cd_tree + i - m->cd_tree);
782 #endif
783  if (cd_tree[i].ctx == ctx[level])
784  break;
785  }
786  if (i == max)
787  return -1;
788 #if 0
789  E_INFO("Found context %d=%s at %d, n_down=%d, down=%d\n",
790  ctx[level], m->ciname[ctx[level]],
791  cd_tree + i - m->cd_tree,
792  cd_tree[i].n_down, cd_tree[i].c.down);
793 #endif
794  /* Leaf node, stop here. */
795  if (cd_tree[i].n_down == 0)
796  return cd_tree[i].c.pid;
797 
798  /* Go down one level. */
799  max = cd_tree[i].n_down;
800  cd_tree = m->cd_tree + cd_tree[i].c.down;
801  ++level;
802  }
803  /* We probably shouldn't get here. */
804  return -1;
805 }
806 
807 int
808 bin_mdef_phone_id_nearest(bin_mdef_t * m, int32 b, int32 l, int32 r, int32 pos)
809 {
810  int p, tmppos;
811 
812 
813 
814  /* In the future, we might back off when context is not available,
815  * but for now we'll just return the CI phone. */
816  if (l < 0 || r < 0)
817  return b;
818 
819  p = bin_mdef_phone_id(m, b, l, r, pos);
820  if (p >= 0)
821  return p;
822 
823  /* Exact triphone not found; backoff to other word positions */
824  for (tmppos = 0; tmppos < N_WORD_POSN; tmppos++) {
825  if (tmppos != pos) {
826  p = bin_mdef_phone_id(m, b, l, r, tmppos);
827  if (p >= 0)
828  return p;
829  }
830  }
831 
832  /* Nothing yet; backoff to silence phone if non-silence filler context */
833  /* In addition, backoff to silence phone on left/right if in beginning/end position */
834  if (m->sil >= 0) {
835  int newl = l, newr = r;
836  if (m->phone[(int)l].info.ci.filler
837  || pos == WORD_POSN_BEGIN || pos == WORD_POSN_SINGLE)
838  newl = m->sil;
839  if (m->phone[(int)r].info.ci.filler
840  || pos == WORD_POSN_END || pos == WORD_POSN_SINGLE)
841  newr = m->sil;
842  if ((newl != l) || (newr != r)) {
843  p = bin_mdef_phone_id(m, b, newl, newr, pos);
844  if (p >= 0)
845  return p;
846 
847  for (tmppos = 0; tmppos < N_WORD_POSN; tmppos++) {
848  if (tmppos != pos) {
849  p = bin_mdef_phone_id(m, b, newl, newr, tmppos);
850  if (p >= 0)
851  return p;
852  }
853  }
854  }
855  }
856 
857  /* Nothing yet; backoff to base phone */
858  return b;
859 }
860 
861 int
862 bin_mdef_phone_str(bin_mdef_t * m, int pid, char *buf)
863 {
864  char *wpos_name;
865 
866  assert(m);
867  assert((pid >= 0) && (pid < m->n_phone));
868  wpos_name = WPOS_NAME;
869 
870  buf[0] = '\0';
871  if (pid < m->n_ciphone)
872  sprintf(buf, "%s", bin_mdef_ciphone_str(m, pid));
873  else {
874  sprintf(buf, "%s %s %s %c",
875  bin_mdef_ciphone_str(m, m->phone[pid].info.cd.ctx[0]),
876  bin_mdef_ciphone_str(m, m->phone[pid].info.cd.ctx[1]),
877  bin_mdef_ciphone_str(m, m->phone[pid].info.cd.ctx[2]),
878  wpos_name[m->phone[pid].info.cd.wpos]);
879  }
880  return 0;
881 }