55 #include <sphinxbase/prim_type.h>
56 #include <sphinxbase/ckd_alloc.h>
57 #include <sphinxbase/byteorder.h>
58 #include <sphinxbase/case.h>
59 #include <sphinxbase/err.h>
66 bin_mdef_read_text(cmd_ln_t *
config,
const char *filename)
70 int i, nodes, ci_idx, lc_idx, rc_idx;
73 if ((mdef =
mdef_init((
char *) filename, TRUE)) == NULL)
78 E_ERROR(
"Number of senones exceeds limit: %d > %d\n",
84 E_ERROR(
"Number of senone sequences exceeds limit: %d > %d\n",
91 E_ERROR(
"Number of phones exceeds limit: %d > %d\n",
97 bmdef = ckd_calloc(1,
sizeof(*bmdef));
124 bmdef->
ciname[0] = ckd_calloc(nchars, 1);
128 bmdef->
ciname[i - 1] + strlen(bmdef->
ciname[i - 1]) + 1;
130 if (i > 0 && strcmp(bmdef->
ciname[i - 1], bmdef->
ciname[i]) > 0) {
132 E_ERROR(
"Phone names are not in sorted order, sorry.");
133 bin_mdef_free(bmdef);
140 for (i = 0; i < mdef->
n_phone; ++i) {
143 if (i < bmdef->n_ciphone) {
148 bmdef->
phone[i].info.cd.ctx[0] = mdef->
phone[i].ci;
149 bmdef->
phone[i].info.cd.ctx[1] = mdef->
phone[i].lc;
156 nodes = lc_idx = ci_idx = rc_idx = 0;
164 for (rc = lc->rclist; rc; rc = rc->next) {
179 E_INFO(
"Allocating %d * %d bytes (%d KiB) for CD tree\n",
180 nodes,
sizeof(*bmdef->
cd_tree),
181 nodes *
sizeof(*bmdef->
cd_tree) / 1024);
191 E_INFO(
"%d => %c (%d@%d)\n",
207 for (rc = lc->rclist; rc; rc = rc->next) {
212 E_INFO(
"%d => %s %s %s %c (%d@%d)\n",
231 E_INFO(
"%d => %s %s %c (%d@%d)\n",
248 E_INFO(
"%d => %d=%s (%d@%d)\n",
249 ci_idx, j, bmdef->
ciname[j],
260 bmdef->alloc_mode = BIN_MDEF_FROM_TEXT;
279 switch (m->alloc_mode) {
280 case BIN_MDEF_FROM_TEXT:
282 ckd_free(m->
sseq[0]);
286 case BIN_MDEF_IN_MEMORY:
289 case BIN_MDEF_ON_DISK:
302 static const char format_desc[] =
303 "BEGIN FILE FORMAT DESCRIPTION\n"
304 "int32 n_ciphone; /**< Number of base (CI) phones */\n"
305 "int32 n_phone; /**< Number of base (CI) phones + (CD) triphones */\n"
306 "int32 n_emit_state; /**< Number of emitting states per phone (0 if heterogeneous) */\n"
307 "int32 n_ci_sen; /**< Number of CI senones; these are the first */\n"
308 "int32 n_sen; /**< Number of senones (CI+CD) */\n"
309 "int32 n_tmat; /**< Number of transition matrices */\n"
310 "int32 n_sseq; /**< Number of unique senone sequences */\n"
311 "int32 n_ctx; /**< Number of phones of context */\n"
312 "int32 n_cd_tree; /**< Number of nodes in CD tree structure */\n"
313 "int32 sil; /**< CI phone ID for silence */\n"
314 "char ciphones[][]; /**< CI phone strings (null-terminated) */\n"
315 "char padding[]; /**< Padding to a 4-bytes boundary */\n"
316 "struct { int16 ctx; int16 n_down; int32 pid/down } cd_tree[];\n"
317 "struct { int32 ssid; int32 tmat; int8 attr[4] } phones[];\n"
318 "int16 sseq[]; /**< Unique senone sequences */\n"
319 "int8 sseq_len[]; /**< Number of states in each sseq (none if homogeneous) */\n"
320 "END FILE FORMAT DESCRIPTION\n";
323 bin_mdef_read(cmd_ln_t *
config,
const char *filename)
334 if ((m = bin_mdef_read_text(config, filename)) != NULL)
337 E_INFO(
"Reading binary model definition: %s\n", filename);
338 if ((fh = fopen(filename,
"rb")) == NULL)
341 if (fread(&val, 4, 1, fh) != 1) {
343 E_ERROR_SYSTEM(
"Failed to read byte-order marker from %s\n",
348 if (val == BIN_MDEF_OTHER_ENDIAN) {
350 E_INFO(
"Must byte-swap %s\n", filename);
352 if (fread(&val, 4, 1, fh) != 1) {
354 E_ERROR_SYSTEM(
"Failed to read version from %s\n", filename);
359 if (val > BIN_MDEF_FORMAT_VERSION) {
360 E_ERROR(
"File format version %d for %s is newer than library\n",
365 if (fread(&val, 4, 1, fh) != 1) {
367 E_ERROR_SYSTEM(
"Failed to read header length from %s\n", filename);
373 FSEEK(fh, val, SEEK_CUR);
376 m = ckd_calloc(1,
sizeof(*m));
380 #define FREAD_SWAP32_CHK(dest) \
381 if (fread((dest), 4, 1, fh) != 1) { \
384 E_ERROR_SYSTEM("Failed to read %s from %s\n", #dest, filename); \
387 if (swap) SWAP_INT32(dest);
393 FREAD_SWAP32_CHK(&m->
n_sen);
394 FREAD_SWAP32_CHK(&m->
n_tmat);
395 FREAD_SWAP32_CHK(&m->
n_sseq);
396 FREAD_SWAP32_CHK(&m->
n_ctx);
398 FREAD_SWAP32_CHK(&m->
sil);
404 do_mmap = config ? cmd_ln_boolean_r(config,
"-mmap") : TRUE;
406 E_WARN(
"-mmap specified, but mdef is other-endian. Will not memory-map.\n");
411 m->
filemap = mmio_file_read(filename);
420 m->alloc_mode = BIN_MDEF_ON_DISK;
424 m->alloc_mode = BIN_MDEF_IN_MEMORY;
425 FSEEK(fh, 0, SEEK_END);
427 FSEEK(fh, pos, SEEK_SET);
428 m->
ciname[0] = ckd_malloc(end - pos);
429 if (fread(m->
ciname[0], 1, end - pos, fh) != end - pos)
430 E_FATAL(
"Failed to read %" PRIdOFF_T
" bytes of data from %s\n", end - pos, filename);
439 tree_start = (tree_start + 3) & ~3;
450 for (i = 0; i < m->
n_phone; ++i) {
457 SWAP_INT32(sseq_size);
459 m->
sseq[0] = (uint16 *) (sseq_size + 1);
461 for (i = 0; i < *sseq_size; ++i)
462 SWAP_INT16(m->
sseq[0] + i);
465 for (i = 1; i < m->
n_sseq; ++i)
470 for (i = 1; i < m->
n_sseq; ++i)
484 for (; i < m->
n_sen; ++i)
486 for (i = 0; i < m->
n_sen; ++i)
488 for (i = 0; i < m->
n_phone; ++i) {
491 for (j = 0; j < bin_mdef_n_emit_state_phone(m, i); ++j) {
492 int s = bin_mdef_sseq2sen(m, ssid, j);
493 int ci = bin_mdef_pid2ci(m, i);
499 (
"Senone %d is shared between multiple base phones\n",
502 if (j > bin_mdef_n_emit_state_phone(m, ci))
503 E_WARN(
"CD phone %d has fewer states than CI phone %d\n",
507 bin_mdef_sseq2sen(m, m->
phone[ci].
ssid, j);
515 (
"%d CI-phone, %d CD-phone, %d emitstate/phone, %d CI-sen, %d Sen, %d Sen-Seq\n",
528 if ((fh = fopen(filename,
"wb")) == NULL)
532 val = BIN_MDEF_NATIVE_ENDIAN;
533 fwrite(&val, 1, 4, fh);
535 val = BIN_MDEF_FORMAT_VERSION;
536 fwrite(&val, 1,
sizeof(val), fh);
539 val = ((
sizeof(format_desc) + 3) & ~3);
540 fwrite(&val, 1,
sizeof(val), fh);
541 fwrite(format_desc, 1,
sizeof(format_desc), fh);
544 fwrite(&i, 1, val -
sizeof(format_desc), fh);
551 fwrite(&m->
n_sen, 4, 1, fh);
552 fwrite(&m->
n_tmat, 4, 1, fh);
553 fwrite(&m->
n_sseq, 4, 1, fh);
554 fwrite(&m->
n_ctx, 4, 1, fh);
561 fwrite(&val, 4, 1, fh);
567 val = (FTELL(fh) + 3) & ~3;
569 fwrite(&i, 1, val - FTELL(fh), fh);
578 fwrite(&val, 4, 1, fh);
581 fwrite(m->
sseq[0],
sizeof(**m->
sseq),
589 for (i = 0; i < m->
n_sseq; ++i)
593 fwrite(&n, 4, 1, fh);
596 fwrite(m->
sseq[0],
sizeof(**m->
sseq), n, fh);
610 int p, i, n_total_state;
612 if (strcmp(filename,
"-") == 0)
615 if ((fh = fopen(filename,
"w")) == NULL)
619 fprintf(fh,
"0.3\n");
620 fprintf(fh,
"%d n_base\n", m->
n_ciphone);
626 for (i = 0; i < m->
n_phone; ++i)
629 fprintf(fh,
"%d n_state_map\n", n_total_state);
630 fprintf(fh,
"%d n_tied_state\n", m->
n_sen);
631 fprintf(fh,
"%d n_tied_ci_state\n", m->
n_ci_sen);
632 fprintf(fh,
"%d n_tied_tmat\n", m->
n_tmat);
633 fprintf(fh,
"#\n# Columns definitions\n");
634 fprintf(fh,
"#%4s %3s %3s %1s %6s %4s %s\n",
635 "base",
"lft",
"rt",
"p",
"attrib",
"tmat",
636 " ... state id's ...");
641 fprintf(fh,
"%5s %3s %3s %1s", m->
ciname[p],
"-",
"-",
"-");
643 if (bin_mdef_is_fillerphone(m, p))
644 fprintf(fh,
" %6s",
"filler");
646 fprintf(fh,
" %6s",
"n/a");
653 for (i = 0; i < n_state; i++) {
663 fprintf(fh,
"%5s %3s %3s %c",
669 if (bin_mdef_is_fillerphone(m, p))
670 fprintf(fh,
" %6s",
"filler");
672 fprintf(fh,
" %6s",
"n/a");
680 for (i = 0; i < n_state; i++) {
686 if (strcmp(filename,
"-") != 0)
702 mid = (low + high) / 2;
703 c = strcmp(ciphone, m->
ciname[mid]);
715 bin_mdef_ciphone_id_nocase(
bin_mdef_t * m,
const char *ciphone)
725 mid = (low + high) / 2;
726 c = strcmp_nocase(ciphone, m->
ciname[mid]);
741 assert(ci < m->n_ciphone);
746 bin_mdef_phone_id(
bin_mdef_t * m, int32 ci, int32 lc, int32 rc, int32 wpos)
756 if (lc < 0 || rc < 0)
759 assert((ci >= 0) && (ci < m->n_ciphone));
760 assert((lc >= 0) && (lc < m->n_ciphone));
761 assert((rc >= 0) && (rc < m->n_ciphone));
767 ctx[2] = (m->
sil >= 0
768 && m->
phone[lc].info.
ci.filler) ? m->
sil : lc;
769 ctx[3] = (m->
sil >= 0
770 && m->
phone[rc].info.
ci.filler) ? m->
sil : rc;
780 E_INFO(
"Looking for context %d=%s in %d at %d\n",
781 ctx[level], m->
ciname[ctx[level]],
784 for (i = 0; i < max; ++i) {
786 E_INFO(
"Look at context %d=%s at %d\n",
790 if (cd_tree[i].ctx == ctx[level])
796 E_INFO(
"Found context %d=%s at %d, n_down=%d, down=%d\n",
797 ctx[level], m->
ciname[ctx[level]],
802 if (cd_tree[i].n_down == 0)
803 return cd_tree[i].c.
pid;
815 bin_mdef_phone_id_nearest(
bin_mdef_t * m, int32 b, int32 l, int32 r, int32 pos)
826 p = bin_mdef_phone_id(m, b, l, r, pos);
833 p = bin_mdef_phone_id(m, b, l, r, tmppos);
842 int newl = l, newr = r;
843 if (m->
phone[(
int)l].info.
ci.filler
846 if (m->
phone[(
int)r].info.
ci.filler
849 if ((newl != l) || (newr != r)) {
850 p = bin_mdef_phone_id(m, b, newl, newr, pos);
856 p = bin_mdef_phone_id(m, b, newl, newr, tmppos);
874 assert((pid >= 0) && (pid < m->n_phone));
878 if (pid < m->n_ciphone)
879 sprintf(buf,
"%s", bin_mdef_ciphone_str(m, pid));
881 sprintf(buf,
"%s %s %s %c",
882 bin_mdef_ciphone_str(m, m->
phone[pid].info.cd.ctx[0]),
883 bin_mdef_ciphone_str(m, m->
phone[pid].info.cd.ctx[1]),
884 bin_mdef_ciphone_str(m, m->
phone[pid].info.cd.ctx[2]),
885 wpos_name[m->
phone[pid].info.cd.wpos]);