PocketSphinx  0.6
dict.c
1 /* -*- c-basic-offset: 4; indent-tabs-mode: nil -*- */
2 /* ====================================================================
3  * Copyright (c) 1999-2004 Carnegie Mellon University. All rights
4  * reserved.
5  *
6  * Redistribution and use in source and binary forms, with or without
7  * modification, are permitted provided that the following conditions
8  * are met:
9  *
10  * 1. Redistributions of source code must retain the above copyright
11  * notice, this list of conditions and the following disclaimer.
12  *
13  * 2. Redistributions in binary form must reproduce the above copyright
14  * notice, this list of conditions and the following disclaimer in
15  * the documentation and/or other materials provided with the
16  * distribution.
17  *
18  * This work was supported in part by funding from the Defense Advanced
19  * Research Projects Agency and the National Science Foundation of the
20  * United States of America, and the CMU Sphinx Speech Consortium.
21  *
22  * THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND
23  * ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
24  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
25  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY
26  * NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
27  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
28  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
29  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
30  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
31  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
32  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
33  *
34  * ====================================================================
35  *
36  */
37 
38 /* System headers. */
39 #include <string.h>
40 
41 /* SphinxBase headers. */
42 #include <sphinxbase/pio.h>
43 #include <sphinxbase/strfuncs.h>
44 
45 /* Local headers. */
46 #include "dict.h"
47 
48 
49 #define DELIM " \t\n" /* Set of field separator characters */
50 #define DEFAULT_NUM_PHONE (MAX_S3CIPID+1)
51 
52 #if WIN32
53 #define snprintf sprintf_s
54 #endif
55 
56 extern const char *const cmu6_lts_phone_table[];
57 
58 static s3cipid_t
59 dict_ciphone_id(dict_t * d, const char *str)
60 {
61  if (d->nocase)
62  return bin_mdef_ciphone_id_nocase(d->mdef, str);
63  else
64  return bin_mdef_ciphone_id(d->mdef, str);
65 }
66 
67 
68 const char *
69 dict_ciphone_str(dict_t * d, s3wid_t wid, int32 pos)
70 {
71  assert(d != NULL);
72  assert((wid >= 0) && (wid < d->n_word));
73  assert((pos >= 0) && (pos < d->word[wid].pronlen));
74 
75  return bin_mdef_ciphone_str(d->mdef, d->word[wid].ciphone[pos]);
76 }
77 
78 
79 s3wid_t
80 dict_add_word(dict_t * d, char const *word, s3cipid_t const * p, int32 np)
81 {
82  int32 len;
83  dictword_t *wordp;
84  s3wid_t newwid;
85  char *wword;
86 
87  if (d->n_word >= d->max_words) {
88  E_INFO("Reallocating to %d KiB for word entries\n",
89  (d->max_words + S3DICT_INC_SZ) * sizeof(dictword_t) / 1024);
90  d->word =
91  (dictword_t *) ckd_realloc(d->word,
92  (d->max_words +
93  S3DICT_INC_SZ) * sizeof(dictword_t));
94  d->max_words = d->max_words + S3DICT_INC_SZ;
95  return BAD_S3WID;
96  }
97 
98  wordp = d->word + d->n_word;
99  wordp->word = (char *) ckd_salloc(word); /* Freed in dict_free */
100 
101  /* Associate word string with d->n_word in hash table */
102  if (hash_table_enter_int32(d->ht, wordp->word, d->n_word) != d->n_word) {
103  ckd_free(wordp->word);
104  wordp->word = NULL;
105  return BAD_S3WID;
106  }
107 
108  /* Fill in word entry, and set defaults */
109  if (p && (np > 0)) {
110  wordp->ciphone = (s3cipid_t *) ckd_malloc(np * sizeof(s3cipid_t)); /* Freed in dict_free */
111  memcpy(wordp->ciphone, p, np * sizeof(s3cipid_t));
112  wordp->pronlen = np;
113  }
114  else {
115  wordp->ciphone = NULL;
116  wordp->pronlen = 0;
117  }
118  wordp->alt = BAD_S3WID;
119  wordp->basewid = d->n_word;
120 
121  /* Determine base/alt wids */
122  wword = ckd_salloc(word);
123  if ((len = dict_word2basestr(wword)) > 0) {
124  int32 w;
125 
126  /* Truncated to a baseword string; find its ID */
127  if (hash_table_lookup_int32(d->ht, wword, &w) < 0) {
128  E_ERROR("Missing base word for: %s\n", word);
129  ckd_free(wword);
130  ckd_free(wordp->word);
131  wordp->word = NULL;
132  return BAD_S3WID;
133  }
134 
135  /* Link into alt list */
136  wordp->basewid = w;
137  wordp->alt = d->word[w].alt;
138  d->word[w].alt = d->n_word;
139  }
140  ckd_free(wword);
141 
142  newwid = d->n_word++;
143 
144  return newwid;
145 }
146 
147 
148 static int32
149 dict_read(FILE * fp, dict_t * d)
150 {
151  lineiter_t *li;
152  char **wptr;
153  s3cipid_t *p;
154  int32 lineno, nwd;
155  s3wid_t w;
156  int32 i, maxwd;
157  size_t stralloc, phnalloc;
158 
159  maxwd = 512;
160  p = (s3cipid_t *) ckd_calloc(maxwd + 4, sizeof(*p));
161  wptr = (char **) ckd_calloc(maxwd, sizeof(char *)); /* Freed below */
162 
163  lineno = 0;
164  stralloc = phnalloc = 0;
165  for (li = lineiter_start(fp); li; li = lineiter_next(li)) {
166  lineno++;
167  if (0 == strncmp(li->buf, "##", 2)
168  || 0 == strncmp(li->buf, ";;", 2))
169  continue;
170 
171  if ((nwd = str2words(li->buf, wptr, maxwd)) < 0) {
172  /* Increase size of p, wptr. */
173  nwd = str2words(li->buf, NULL, 0);
174  assert(nwd > maxwd); /* why else would it fail? */
175  maxwd = nwd;
176  p = (s3cipid_t *) ckd_realloc(p, (maxwd + 4) * sizeof(*p));
177  wptr = (char **) ckd_realloc(wptr, maxwd * sizeof(*wptr));
178  }
179 
180  if (nwd == 0) /* Empty line */
181  continue;
182  /* wptr[0] is the word-string and wptr[1..nwd-1] the pronunciation sequence */
183  if (nwd == 1) {
184  E_ERROR("Line %d: No pronunciation for word '%s'; ignored\n",
185  lineno, wptr[0]);
186  continue;
187  }
188 
189 
190  /* Convert pronunciation string to CI-phone-ids */
191  for (i = 1; i < nwd; i++) {
192  p[i - 1] = dict_ciphone_id(d, wptr[i]);
193  if (NOT_S3CIPID(p[i - 1])) {
194  E_ERROR("Line %d: Phone '%s' is mising in the acoustic model; word '%s' ignored\n",
195  lineno, wptr[i], wptr[0]);
196  break;
197  }
198  }
199 
200  if (i == nwd) { /* All CI-phones successfully converted to IDs */
201  w = dict_add_word(d, wptr[0], p, nwd - 1);
202  if (NOT_S3WID(w))
203  E_ERROR
204  ("Line %d: Failed to add the word '%s' (duplicate?); ignored\n",
205  lineno, wptr[0]);
206  else {
207  stralloc += strlen(d->word[w].word);
208  phnalloc += d->word[w].pronlen * sizeof(s3cipid_t);
209  }
210  }
211  }
212  E_INFO("Allocated %d KiB for strings, %d KiB for phones\n",
213  (int)stralloc / 1024, (int)phnalloc / 1024);
214  ckd_free(p);
215  ckd_free(wptr);
216 
217  return 0;
218 }
219 
220 int
221 dict_write(dict_t *dict, char const *filename, char const *format)
222 {
223  FILE *fh;
224  int i;
225 
226  if ((fh = fopen(filename, "w")) == NULL) {
227  E_ERROR_SYSTEM("Failed to open '%s'", filename);
228  return -1;
229  }
230  for (i = 0; i < dict->n_word; ++i) {
231  char *phones;
232  int j, phlen;
233  if (!dict_real_word(dict, i))
234  continue;
235  for (phlen = j = 0; j < dict_pronlen(dict, i); ++j)
236  phlen += strlen(dict_ciphone_str(dict, i, j)) + 1;
237  phones = ckd_calloc(1, phlen);
238  for (j = 0; j < dict_pronlen(dict, i); ++j) {
239  strcat(phones, dict_ciphone_str(dict, i, j));
240  if (j != dict_pronlen(dict, i) - 1)
241  strcat(phones, " ");
242  }
243  fprintf(fh, "%-30s %s\n", dict_wordstr(dict, i), phones);
244  ckd_free(phones);
245  }
246  fclose(fh);
247  return 0;
248 }
249 
250 
251 dict_t *
252 dict_init(cmd_ln_t *config, bin_mdef_t * mdef)
253 {
254  FILE *fp, *fp2;
255  int32 n;
256  lineiter_t *li;
257  dict_t *d;
258  s3cipid_t sil;
259  char const *dictfile = NULL, *fillerfile = NULL;
260 
261  if (config) {
262  dictfile = cmd_ln_str_r(config, "-dict");
263  fillerfile = cmd_ln_str_r(config, "-fdict");
264  }
265 
266  /*
267  * First obtain #words in dictionary (for hash table allocation).
268  * Reason: The PC NT system doesn't like to grow memory gradually. Better to allocate
269  * all the required memory in one go.
270  */
271  fp = NULL;
272  n = 0;
273  if (dictfile) {
274  if ((fp = fopen(dictfile, "r")) == NULL)
275  E_FATAL_SYSTEM("Failed to open dictionary file '%s' for reading", dictfile);
276  for (li = lineiter_start(fp); li; li = lineiter_next(li)) {
277  if (li->buf[0] != '#')
278  n++;
279  }
280  rewind(fp);
281  }
282 
283  fp2 = NULL;
284  if (fillerfile) {
285  if ((fp2 = fopen(fillerfile, "r")) == NULL)
286  E_FATAL_SYSTEM("Failed to open filler dictionary file '%s' for reading", fillerfile);
287  for (li = lineiter_start(fp2); li; li = lineiter_next(li)) {
288  if (li->buf[0] != '#')
289  n++;
290  }
291  rewind(fp2);
292  }
293 
294  /*
295  * Allocate dict entries. HACK!! Allow some extra entries for words not in file.
296  * Also check for type size restrictions.
297  */
298  d = (dict_t *) ckd_calloc(1, sizeof(dict_t)); /* freed in dict_free() */
299  d->refcnt = 1;
300  d->max_words =
301  (n + S3DICT_INC_SZ < MAX_S3WID) ? n + S3DICT_INC_SZ : MAX_S3WID;
302  if (n >= MAX_S3WID)
303  E_FATAL("Number of words in dictionaries (%d) exceeds limit (%d)\n", n,
304  MAX_S3WID);
305 
306  E_INFO("Allocating %d * %d bytes (%d KiB) for word entries\n",
307  d->max_words, sizeof(dictword_t),
308  d->max_words * sizeof(dictword_t) / 1024);
309  d->word = (dictword_t *) ckd_calloc(d->max_words, sizeof(dictword_t)); /* freed in dict_free() */
310  d->n_word = 0;
311  if (mdef)
312  d->mdef = bin_mdef_retain(mdef);
313 
314  /* Create new hash table for word strings; case-insensitive word strings */
315  if (config && cmd_ln_exists_r(config, "-dictcase"))
316  d->nocase = cmd_ln_boolean_r(config, "-dictcase");
317  d->ht = hash_table_new(d->max_words, d->nocase);
318 
319  /* Digest main dictionary file */
320  if (fp) {
321  E_INFO("Reading main dictionary: %s\n", dictfile);
322  dict_read(fp, d);
323  fclose(fp);
324  E_INFO("%d words read\n", d->n_word);
325  }
326 
327  /* Now the filler dictionary file, if it exists */
328  d->filler_start = d->n_word;
329  if (fillerfile) {
330  E_INFO("Reading filler dictionary: %s\n", fillerfile);
331  dict_read(fp2, d);
332  fclose(fp2);
333  E_INFO("%d words read\n", d->n_word - d->filler_start);
334  }
335  if (mdef)
336  sil = bin_mdef_silphone(mdef);
337  else
338  sil = 0;
339  if (dict_wordid(d, S3_START_WORD) == BAD_S3WID) {
340  dict_add_word(d, S3_START_WORD, &sil, 1);
341  }
342  if (dict_wordid(d, S3_FINISH_WORD) == BAD_S3WID) {
343  dict_add_word(d, S3_FINISH_WORD, &sil, 1);
344  }
345  if (dict_wordid(d, S3_SILENCE_WORD) == BAD_S3WID) {
346  dict_add_word(d, S3_SILENCE_WORD, &sil, 1);
347  }
348 
349  d->filler_end = d->n_word - 1;
350 
351  /* Initialize distinguished word-ids */
352  d->startwid = dict_wordid(d, S3_START_WORD);
353  d->finishwid = dict_wordid(d, S3_FINISH_WORD);
354  d->silwid = dict_wordid(d, S3_SILENCE_WORD);
355 
356  if ((d->filler_start > d->filler_end)
357  || (!dict_filler_word(d, d->silwid)))
358  E_FATAL("Word '%s' must occur (only) in filler dictionary\n",
359  S3_SILENCE_WORD);
360 
361  /* No check that alternative pronunciations for filler words are in filler range!! */
362 
363  return d;
364 }
365 
366 
367 s3wid_t
368 dict_wordid(dict_t *d, const char *word)
369 {
370  int32 w;
371 
372  assert(d);
373  assert(word);
374 
375  if (hash_table_lookup_int32(d->ht, word, &w) < 0)
376  return (BAD_S3WID);
377  return w;
378 }
379 
380 
381 int
382 dict_filler_word(dict_t *d, s3wid_t w)
383 {
384  assert(d);
385  assert((w >= 0) && (w < d->n_word));
386 
387  w = dict_basewid(d, w);
388  if ((w == d->startwid) || (w == d->finishwid))
389  return 0;
390  if ((w >= d->filler_start) && (w <= d->filler_end))
391  return 1;
392  return 0;
393 }
394 
395 int
396 dict_real_word(dict_t *d, s3wid_t w)
397 {
398  assert(d);
399  assert((w >= 0) && (w < d->n_word));
400 
401  w = dict_basewid(d, w);
402  if ((w == d->startwid) || (w == d->finishwid))
403  return 0;
404  if ((w >= d->filler_start) && (w <= d->filler_end))
405  return 0;
406  return 1;
407 }
408 
409 
410 int32
411 dict_word2basestr(char *word)
412 {
413  int32 i, len;
414 
415  len = strlen(word);
416  if (word[len - 1] == ')') {
417  for (i = len - 2; (i > 0) && (word[i] != '('); --i);
418 
419  if (i > 0) {
420  /* The word is of the form <baseword>(...); strip from left-paren */
421  word[i] = '\0';
422  return i;
423  }
424  }
425 
426  return -1;
427 }
428 
429 dict_t *
430 dict_retain(dict_t *d)
431 {
432  ++d->refcnt;
433  return d;
434 }
435 
436 int
437 dict_free(dict_t * d)
438 {
439  int i;
440  dictword_t *word;
441 
442  if (d == NULL)
443  return 0;
444  if (--d->refcnt > 0)
445  return d->refcnt;
446 
447  /* First Step, free all memory allocated for each word */
448  for (i = 0; i < d->n_word; i++) {
449  word = (dictword_t *) & (d->word[i]);
450  if (word->word)
451  ckd_free((void *) word->word);
452  if (word->ciphone)
453  ckd_free((void *) word->ciphone);
454  }
455 
456  if (d->word)
457  ckd_free((void *) d->word);
458  if (d->ht)
459  hash_table_free(d->ht);
460  if (d->mdef)
461  bin_mdef_free(d->mdef);
462  ckd_free((void *) d);
463 
464  return 0;
465 }
466 
467 void
468 dict_report(dict_t * d)
469 {
470  E_INFO_NOFN("Initialization of dict_t, report:\n");
471  E_INFO_NOFN("Max word: %d\n", d->max_words);
472  E_INFO_NOFN("No of word: %d\n", d->n_word);
473  E_INFO_NOFN("\n");
474 }