bes  Updated for version 3.20.6
util_ff.cc
1 
2 // -*- mode: c++; c-basic-offset:4 -*-
3 
4 // This file is part of ff_handler a FreeForm API handler for the OPeNDAP
5 // DAP2 data server.
6 
7 // Copyright (c) 2005 OPeNDAP, Inc.
8 // Author: James Gallagher <jgallagher@opendap.org>
9 //
10 // This is free software; you can redistribute it and/or modify it under the
11 // terms of the GNU Lesser General Public License as published by the Free
12 // Software Foundation; either version 2.1 of the License, or (at your
13 // option) any later version.
14 //
15 // This software is distributed in the hope that it will be useful, but
16 // WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
17 // or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
18 // License for more details.
19 //
20 // You should have received a copy of the GNU Lesser General Public
21 // License along with this library; if not, write to the Free Software
22 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
23 //
24 // You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
25 
26 // (c) COPYRIGHT URI/MIT 1997-99
27 // Please read the full copyright statement in the file COPYRIGHT.
28 //
29 // Authors: reza (Reza Nekovei)
30 
31 // Utility functions for the FreeForm data server.
32 //
33 // jhrg 3/29/96
34 
35 #include "config_ff.h"
36 
37 static char rcsid[] not_used =
38  { "$Id$" };
39 
40 #ifndef WIN32
41 #include <unistd.h> // for access
42 #else
43 #define F_OK 0
44 #endif
45 
46 #include <iostream>
47 #include <sstream>
48 #include <fstream>
49 #include <string>
50 #include <vector>
51 #include <cstdlib>
52 
53 #include <BESRegex.h>
54 #include <BESDebug.h>
55 
56 #include <BaseType.h>
57 #include <Byte.h>
58 #include <Int16.h>
59 #include <Int32.h>
60 #include <UInt16.h>
61 #include <UInt32.h>
62 #include <Float32.h>
63 #include <Float64.h>
64 #include <InternalErr.h>
65 #include <dods-limits.h>
66 #include <util.h>
67 #include <debug.h>
68 
69 #include "FFRequestHandler.h"
70 #include "util_ff.h"
71 
72 using namespace std;
73 
87 static string &remove_paths(string &src)
88 {
89  size_t p1 = src.find_first_of('/');
90  if (p1 == string::npos)
91  return src;
92  size_t p2 = src.find_last_of('/');
93  // The string has one '/', not a big deal
94  if (p2 == p1)
95  return src;
96 
97  src.erase(p1, p2-p1+1);
98  return src;
99 }
100 
101 // These two functions are defined in FFND/error.c. They were originally
102 // static functions. I used them to read error strings since FreeForm
103 // did not have a good way to get the error text. jhrg 9/11/12
104 extern "C" FF_ERROR_PTR pull_error(void);
105 extern "C" BOOLEAN is_a_warning(FF_ERROR_PTR error);
106 
120 static string freeform_error_message()
121 {
122  FF_ERROR_PTR error = pull_error();
123  if (!error)
124  throw BESInternalError("Called the FreeForm error message code, but there was no error.", __FILE__, __LINE__);
125 
126  ostringstream oss;
127  do {
128  if (is_a_warning(error))
129  oss << "Warning: ";
130  else
131  oss << "Error: ";
132 
133  // if either of these contain a pathname, remove it
134  string problem = error->problem;
135  string message = error->message;
136  oss << remove_paths(problem) << ": " << remove_paths(message) << endl;
137 
138  ff_destroy_error (error);
139  error = pull_error();
140  } while (error);
141 
142  return oss.str();
143 }
144 
157 long read_ff(const char *dataset, const char *if_file, const char *o_format, char *o_buffer, unsigned long bsize)
158 {
159  FF_BUFSIZE_PTR newform_log = NULL;
160  FF_STD_ARGS_PTR std_args = NULL;
161 
162  try {
163  std_args = ff_create_std_args();
164  if (!std_args)
165  throw BESInternalError("FreeForm could not allocate a 'stdargs' object.", __FILE__, __LINE__);
166 
167  // set the std_arg structure values - cast away const for dataset, if_file,
168  // and o_format.
169  std_args->error_prompt = FALSE;
170  std_args->user.is_stdin_redirected = 0;
171  std_args->input_file = (char*) (dataset);
172  std_args->input_format_file = (char*) (if_file);
173  std_args->output_file = NULL;
174  std_args->output_format_buffer = (char*) (o_format);
175  std_args->log_file = (char *) "/dev/null";
176 #if 0
177  // This just doesn't seem to work within the BES framework. jhrg 9/11/12
178  std_args->log_file = (char *)"/tmp/ffdods.log";
179 #endif
180 
181  // memory freed automatically on exit
182  vector<char> l_bufsz(sizeof(FF_BUFSIZE));
183  //bufsz = (FF_BUFSIZE *)&l_bufsz[0];
184  FF_BUFSIZE_PTR bufsz = reinterpret_cast<FF_BUFSIZE_PTR>(&l_bufsz[0]);
185 
186  bufsz->usage = 1;
187  bufsz->buffer = o_buffer;
188  bufsz->total_bytes = (FF_BSS_t) bsize;
189  bufsz->bytes_used = (FF_BSS_t) 0;
190 
191  std_args->output_bufsize = bufsz;
192 
193  newform_log = ff_create_bufsize(SCRATCH_QUANTA);
194  if (!newform_log)
195  throw BESInternalError("FreeForm could not allocate a 'newform_log' object.", __FILE__, __LINE__);
196 
197  // passing 0 for the FILE* param is a wash since a non-null
198  // value for newform_log selects that as the 'logging' sink.
199  // jhrg 9/11/12
200  int status = newform(std_args, newform_log, 0 /*stderr*/);
201 
202  BESDEBUG("ff", "FreeForm: newform returns " << status << endl);
203 
204  if (err_count()) {
205  string message = freeform_error_message();
206  BESDEBUG("ff", "FreeForm: error message " << message << endl);
207  throw BESError(message, BES_SYNTAX_USER_ERROR, __FILE__, __LINE__);
208  }
209 
210  ff_destroy_bufsize(newform_log);
211  ff_destroy_std_args(std_args);
212 
213  return bufsz->bytes_used;
214  }
215  catch (...) {
216  if (newform_log)
217  ff_destroy_bufsize(newform_log);
218  if (std_args)
219  ff_destroy_std_args(std_args);
220 
221  throw;
222  }
223 
224  return 0;
225 }
226 
233 void free_ff_char_vector(char **v, int len)
234 {
235  for (int i = 0; i < len; ++i)
236  if (v && v[i])
237  free (v[i]);
238  if (v && len > 0)
239  free (v);
240 }
241 
242 // Given the name of a DODS data type, return the name of the corresponding
243 // FreeForm data type.
244 //
245 // Returns: a const char * if the DODS type maps into a FreeForm type,
246 // otherwise NULL.
247 
248 const string ff_types(Type dods_type)
249 {
250  switch (dods_type) {
251  case dods_byte_c:
252  return "uint8";
253  case dods_int16_c:
254  return "int16";
255  case dods_uint16_c:
256  return "uint16";
257  case dods_int32_c:
258  return "int32";
259  case dods_uint32_c:
260  return "uint32";
261  case dods_float32_c:
262  return "float32";
263  case dods_float64_c:
264  return "float64";
265  case dods_str_c:
266  return "text";
267  case dods_url_c:
268  return "text";
269  default:
270  throw Error("ff_types: DODS type " + D2type_name(dods_type) + " does not map to a FreeForm type.");
271  }
272 }
273 
274 // Given the name of a DODS data type, return the precision of the
275 // corresponding FreeForm data type.
276 //
277 // Returns: a positive integer if the DODS type maps into a FreeForm type,
278 // otherwise -1.
279 
280 int ff_prec(Type dods_type)
281 {
282  switch (dods_type) {
283  case dods_byte_c:
284  case dods_int16_c:
285  case dods_uint16_c:
286  case dods_int32_c:
287  case dods_uint32_c:
288  return 0;
289  case dods_float32_c:
290  return DODS_FLT_DIG;
291  case dods_float64_c:
292  return DODS_DBL_DIG;
293  case dods_str_c:
294  case dods_url_c:
295  return 0;
296  default:
297  throw Error("ff_prec: DODS type " + D2type_name(dods_type) + " does not map to a FreeForm type.");
298  }
299 }
300 
306 const string
307 make_output_format(const string & name, Type type, const int width)
308 {
309  ostringstream str;
310 
311  str << "binary_output_data \"DODS binary output data\"" << endl;
312  str << name << " 1 " << width << " " << ff_types(type)
313  << " " << ff_prec(type) << endl;
314 
315  return str.str();
316 }
317 
318 // format for multi-dimension array
319 const string
320 makeND_output_format(const string & name, Type type, const int width,
321  int ndim, const long *start, const long *edge, const
322  long *stride, string * dname)
323 {
324  ostringstream str;
325  str << "binary_output_data \"DODS binary output data\"" << endl;
326  str << name << " 1 " << width << " ARRAY";
327 
328  for (int i = 0; i < ndim; i++)
329  str << "[" << "\"" << dname[i] << "\" " << start[i] + 1 << " to "
330  << start[i] + (edge[i] - 1) * stride[i] +
331  1 << " by " << stride[i] << " ]";
332 
333  str << " of " << ff_types(type) << " " << ff_prec(type) << endl;
334 
335  DBG(cerr << "ND output format: " << str.str() << endl);
336 
337  return str.str();
338 }
339 
345 const string & format_delimiter(const string & new_delimiter)
346 {
347  static string delimiter = ".";
348 
349  if (new_delimiter != "")
350  delimiter = new_delimiter;
351 
352  return delimiter;
353 }
354 
361 const string & format_extension(const string & new_extension)
362 {
363  static string extension = ".fmt";
364 
365  if (new_extension != "")
366  extension = new_extension;
367 
368  return extension;
369 }
370 
373 static bool
374 cmp_array_conduit(FF_ARRAY_CONDUIT_PTR src_conduit,
375  FF_ARRAY_CONDUIT_PTR trg_conduit)
376 {
377  if (src_conduit->input && trg_conduit->input)
378  return (bool) ff_format_comp(src_conduit->input->fd->format,
379  trg_conduit->input->fd->format);
380  else if (src_conduit->output && trg_conduit->output)
381  return (bool) ff_format_comp(src_conduit->output->fd->format,
382  trg_conduit->output->fd->format);
383  else
384  return false;
385 }
386 
387 static int merge_redundant_conduits(FF_ARRAY_CONDUIT_LIST conduit_list)
388 {
389  int error = 0;
390  error = list_replace_items((pgenobj_cmp_t) cmp_array_conduit,
391  conduit_list);
392  return (error);
393 }
394 
403 int SetDodsDB(FF_STD_ARGS_PTR std_args, DATA_BIN_HANDLE dbin_h, char *Msgt)
404 {
405  FORMAT_DATA_LIST format_data_list = NULL;
406  int error = 0;
407 
408  assert(dbin_h);
409 
410  if (!dbin_h) {
411  snprintf(Msgt, Msgt_size, "Error: NULL DATA_BIN_HANDLE in %s", ROUTINE_NAME);
412  return (ERR_API);
413  }
414 
415  if (!*dbin_h) {
416  *dbin_h = db_make(std_args->input_file);
417 
418  if (!*dbin_h) {
419  snprintf(Msgt, Msgt_size, "Error in Standard Data Bin");
420  return (ERR_MEM_LACK);
421  }
422  }
423 
424  /* Now set the formats and the auxiliary files */
425 
426  if (db_set(*dbin_h, DBSET_READ_EQV, std_args->input_file)) {
427  snprintf(Msgt, Msgt_size, "Error making name table for %s",
428  std_args->input_file);
429  return (DBSET_READ_EQV);
430  }
431 
432  if (db_set(*dbin_h,
433  DBSET_INPUT_FORMATS,
434  std_args->input_file,
435  std_args->output_file,
436  std_args->input_format_file,
437  std_args->input_format_buffer,
438  std_args->input_format_title, &format_data_list)) {
439  if (format_data_list)
440  dll_free_holdings(format_data_list);
441 
442  snprintf(Msgt, Msgt_size, "Error setting an input format for %s",
443  std_args->input_file);
444  return (DBSET_INPUT_FORMATS);
445  }
446 
447  error =
448  db_set(*dbin_h, DBSET_CREATE_CONDUITS, std_args, format_data_list);
449  dll_free_holdings(format_data_list);
450  if (error) {
451  snprintf(Msgt, Msgt_size, "Error creating array information for %s",
452  std_args->input_file);
453  return (DBSET_CREATE_CONDUITS);
454  }
455 
456  if (db_set(*dbin_h, DBSET_HEADER_FILE_NAMES, FFF_INPUT,
457  std_args->input_file)) {
458  snprintf(Msgt, Msgt_size, "Error determining input header file names for %s",
459  std_args->input_file);
460  return (DBSET_HEADER_FILE_NAMES);
461  }
462 
463  if (db_set(*dbin_h, DBSET_HEADERS)) {
464  snprintf(Msgt, Msgt_size, "getting header file for %s", std_args->input_file);
465  return (DBSET_HEADERS);
466  }
467 
468 
469  if (db_set(*dbin_h, DBSET_INIT_CONDUITS, FFF_DATA,
470  std_args->records_to_read)) {
471  snprintf(Msgt, Msgt_size, "Error creating array information for %s",
472  std_args->input_file);
473  return (DBSET_INIT_CONDUITS);
474  }
475 
476  error = merge_redundant_conduits((*dbin_h)->array_conduit_list);
477  if (error)
478  snprintf(Msgt, Msgt_size, "Error merging redundent conduits");
479 
480  return (error);
481 }
482 
488 long Records(const string &filename)
489 {
490  int error = 0;
491  DATA_BIN_PTR dbin = NULL;
492  FF_STD_ARGS_PTR SetUps = NULL;
493  PROCESS_INFO_LIST pinfo_list = NULL;
494  PROCESS_INFO_PTR pinfo = NULL;
495  static char Msgt[255];
496 
497  SetUps = ff_create_std_args();
498  if (!SetUps) {
499  return -1;
500  }
501 
503  SetUps->user.is_stdin_redirected = 0;
504  SetUps->input_file = const_cast<char*>(filename.c_str());
505 
506  SetUps->output_file = NULL;
507 
508  error = SetDodsDB(SetUps, &dbin, Msgt);
509  if (error && error < ERR_WARNING_ONLY) {
510  ff_destroy_std_args(SetUps);
511  db_destroy(dbin);
512  return -1;
513  }
514 
515  ff_destroy_std_args(SetUps);
516 
517  error = db_ask(dbin, DBASK_PROCESS_INFO, FFF_INPUT | FFF_DATA, &pinfo_list);
518  if (error)
519  return (-1);
520 
521  pinfo_list = dll_first(pinfo_list);
522 
523  pinfo = ((PROCESS_INFO_PTR) (pinfo_list)->data.u.pi);
524 
525  long num_records = PINFO_SUPER_ARRAY_ELS(pinfo);
526 
527  ff_destroy_process_info_list(pinfo_list);
528  db_destroy(dbin);
529 
530  return num_records;
531 }
532 
533 
534 bool file_exist(const char *filename)
535 {
536  return access(filename, F_OK) == 0;
537 }
538 
551 const string
552 find_ancillary_rss_formats(const string & dataset, const string & /* delimiter */,
553  const string & /* extension */)
554 {
555  string FormatFile;
556  string FormatPath = FFRequestHandler::get_RSS_format_files();
557  string BaseName;
558  string FileName;
559 
560  // Separate the filename from the pathname, for both plain files
561  // and cached decompressed files (ones with '#' in their names).
562  size_t delim = dataset.rfind("#");
563  if (delim != string::npos)
564  FileName = dataset.substr(delim + 1, dataset.length() - delim + 1);
565  else {
566  delim = dataset.rfind("/");
567  if (delim != string::npos)
568  FileName = dataset.substr(delim + 1, dataset.length() - delim + 1);
569  else
570  FileName = dataset;
571  }
572 
573  // The file/dataset name has to have an underscore...
574  delim = FileName.find("_");
575  if ( delim != string::npos ) {
576  BaseName = FileName.substr(0,delim+1);
577  }
578  else {
579  throw Error("Could not find input format for: " + dataset);
580  }
581 
582  // Now determine if this is files holds averaged or daily data.
583  string DatePart = FileName.substr(delim+1, FileName.length()-delim+1);
584 
585  if (FormatPath[FormatPath.length()-1] != '/')
586  FormatPath.append("/");
587 
588  if ( (DatePart.find("_") != string::npos) || (DatePart.length() < 10) )
589  FormatFile = FormatPath + BaseName + "averaged.fmt";
590  else
591  FormatFile = FormatPath + BaseName + "daily.fmt";
592 
593  return string(FormatFile);
594 }
595 
608 const string
609 find_ancillary_rss_das(const string & dataset, const string & /* delimiter */,
610  const string & /* extension */)
611 {
612  string FormatFile;
613  string FormatPath = FFRequestHandler::get_RSS_format_files();
614  string BaseName;
615  string FileName;
616 
617  size_t delim = dataset.rfind("#");
618  if (delim != string::npos)
619  FileName = dataset.substr(delim + 1, dataset.length() - delim + 1);
620  else {
621  delim = dataset.rfind("/");
622  if (delim != string::npos)
623  FileName = dataset.substr(delim + 1, dataset.length() - delim + 1);
624  else
625  FileName = dataset;
626  }
627 
628  delim = FileName.find("_");
629  if ( delim != string::npos ) {
630  BaseName = FileName.substr(0,delim+1);
631  }
632  else {
633  string msg = "Could not find input format for: ";
634  msg += dataset;
635  throw InternalErr(msg);
636  }
637 
638  string DatePart = FileName.substr(delim+1, FileName.length()-delim+1);
639 
640  if (FormatPath[FormatPath.length()-1] != '/')
641  FormatPath.append("/");
642 
643  if ( (DatePart.find("_") != string::npos) || (DatePart.length() < 10) )
644  FormatFile = FormatPath + BaseName + "averaged.das";
645  else
646  FormatFile = FormatPath + BaseName + "daily.das";
647 
648  return string(FormatFile);
649 }
650 
651 // These functions are used by the Date/Time Factory classes but they might
652 // be generally useful in writing server-side functions. 1/21/2002 jhrg
653 
654 bool is_integer_type(BaseType * btp)
655 {
656  switch (btp->type()) {
657  case dods_null_c:
658  return false;
659 
660  case dods_byte_c:
661  case dods_int16_c:
662  case dods_uint16_c:
663  case dods_int32_c:
664  case dods_uint32_c:
665  return true;
666 
667  case dods_float32_c:
668  case dods_float64_c:
669  case dods_str_c:
670  case dods_url_c:
671  case dods_array_c:
672  case dods_structure_c:
673  case dods_sequence_c:
674  case dods_grid_c:
675  default:
676  return false;
677  }
678 }
679 
680 bool is_float_type(BaseType * btp)
681 {
682  switch (btp->type()) {
683  case dods_null_c:
684  case dods_byte_c:
685  case dods_int16_c:
686  case dods_uint16_c:
687  case dods_int32_c:
688  case dods_uint32_c:
689  return false;
690 
691  case dods_float32_c:
692  case dods_float64_c:
693  return true;
694 
695  case dods_str_c:
696  case dods_url_c:
697  case dods_array_c:
698  case dods_structure_c:
699  case dods_sequence_c:
700  case dods_grid_c:
701  default:
702  return false;
703  }
704 }
705 
710 dods_uint32 get_integer_value(BaseType * var) throw(InternalErr)
711 {
712  if (!var)
713  return 0;
714 
715  switch (var->type()) {
716  case dods_byte_c:
717  return static_cast<Byte*>(var)->value();
718 
719  case dods_int16_c:
720  return static_cast<Int16*>(var)->value();
721 
722  case dods_int32_c:
723  return static_cast<Int32*>(var)->value();
724 
725  case dods_uint16_c:
726  return static_cast<UInt16*>(var)->value();
727 
728  case dods_uint32_c:
729  return static_cast<UInt32*>(var)->value();
730 
731  default:
732  throw InternalErr(__FILE__, __LINE__,
733  "Tried to get an integer value for a non-integer datatype!");
734  }
735 }
736 
737 dods_float64 get_float_value(BaseType * var) throw(InternalErr)
738 {
739  if (!var)
740  return 0.0;
741 
742  switch (var->type()) {
743  case dods_int16_c:
744  case dods_uint16_c:
745  case dods_int32_c:
746  case dods_uint32_c:
747  return get_integer_value(var);
748 
749  case dods_float32_c:
750  return static_cast<Float32*>(var)->value();
751 
752  case dods_float64_c:
753  return static_cast<Float64*>(var)->value();
754 
755  default:
756  throw InternalErr(__FILE__, __LINE__,
757  "Tried to get an float value for a non-numeric datatype!");
758  }
759 }
760 
761 string get_Regex_format_file(const string & filename)
762 {
763  string::size_type found = filename.find_last_of("/\\");
764  string base_name = filename.substr(found+1);
765  string retVal = "";
766  std::map<string,string> mapFF = FFRequestHandler::get_fmt_regex_map();
767  for (auto rgx = mapFF.begin(); rgx != mapFF.end(); ++ rgx) {
768  BESDEBUG("ff", "get_Regex_format_file() - filename: '" << filename << "'" <<
769  " regex: '" << (*rgx).first << "'" <<
770  " format: '" << (*rgx).second << "'" << endl);
771  BESRegex regex(((*rgx).first).c_str());
772  if ( (unsigned long) regex.match(base_name.c_str(), base_name.length()) == base_name.length() ){
773  retVal = string((*rgx).second);
774  break;
775  }
776  }
777  BESDEBUG("ff", "get_Regex_format_file() - returning format filename: '"<< retVal << "'" << endl);
778  return retVal;
779 }
struct_ff_std_args
Definition: freeform.h:867
struct_ff_dll_node
Definition: freeform.h:1278
struct_ff_error
Definition: freeform.h:943
struct_ff_process_info
Definition: freeform.h:1193
struct_ff_bufsize
Definition: freeform.h:845
struct_ff_array_conduit
Definition: freeform.h:1178
Type
Type
Type of JSON value.
Definition: cmr_module/rapidjson/rapidjson.h:603
struct_databin
Definition: freeform.h:1294
BESInternalError
exception thrown if internal error encountered
Definition: BESInternalError.h:43
Error
BESRegex
Definition: BESRegex.h:41
BESError
Abstract exception class for the BES with basic string message.
Definition: BESError.h:58