bes  Updated for version 3.20.6
CmrApi.cc
1 // -*- mode: c++; c-basic-offset:4 -*-
2 
3 // This file is part of cmr_module, A C++ MODULE that can be loaded in to
4 // the OPeNDAP Back-End Server (BES) and is able to handle remote requests.
5 
6 // Copyright (c) 2015 OPeNDAP, Inc.
7 // Author: Nathan Potter <ndp@opendap.org>
8 //
9 // This library is free software; you can redistribute it and/or
10 // modify it under the terms of the GNU Lesser General Public
11 // License as published by the Free Software Foundation; either
12 // version 2.1 of the License, or (at your option) any later version.
13 //
14 // This library is distributed in the hope that it will be useful,
15 // but WITHOUT ANY WARRANTY; without even the implied warranty of
16 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 // Lesser General Public License for more details.
18 //
19 // You should have received a copy of the GNU Lesser General Public
20 // License along with this library; if not, write to the Free Software
21 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
22 //
23 // You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
24 
25 /*
26  * CmrApi.cc
27  *
28  * Created on: July, 13 2018
29  * Author: ndp
30  */
31 #include <memory>
32 #include "rapidjson/document.h"
33 #include "rapidjson/writer.h"
34 #include "rapidjson/prettywriter.h"
35 #include "rapidjson/stringbuffer.h"
36 #include "rapidjson/filereadstream.h"
37 #include <cstdio>
38 #include <cstring>
39 #include <iostream>
40 #include <sstream>
41 
42 
43 #include <util.h>
44 #include <debug.h>
45 
46 #include <BESError.h>
47 #include <BESSyntaxUserError.h>
48 #include <BESDebug.h>
49 #include <BESUtil.h>
50 #include <TheBESKeys.h>
51 
52 #include "CmrApi.h"
53 #include "CmrNames.h"
54 #include "RemoteHttpResource.h"
55 #include "CmrError.h"
56 #include "rjson_utils.h"
57 
58 using namespace std;
59 
60 #define prolog string("CmrApi::").append(__func__).append("() - ")
61 
62 namespace cmr {
63 
67 const rapidjson::Value&
68 CmrApi::get_children(const rapidjson::Value& obj) {
70 
71  itr = obj.FindMember("children");
72  bool result = itr != obj.MemberEnd();
73  string msg = prolog + (result?"Located":"FAILED to locate") + " the value 'children' in the object.";
74  BESDEBUG(MODULE, msg << endl);
75  if(!result){
76  throw CmrError(msg,__FILE__,__LINE__);
77  }
78 
79  const rapidjson::Value& children = itr->value;
80  result = children.IsArray();
81  msg = prolog + "The value 'children' is" + (result?"":" NOT") + " an array.";
82  BESDEBUG(MODULE, msg << endl);
83  if(!result){
84  throw CmrError(msg,__FILE__,__LINE__);
85  }
86  return children;
87 }
88 
92 const rapidjson::Value&
93 CmrApi::get_feed(const rapidjson::Document &cmr_doc){
94 
95  bool result = cmr_doc.IsObject();
96  string msg = prolog + "Json document is" + (result?"":" NOT") + " an object.";
97  BESDEBUG(MODULE, msg << endl);
98  if(!result){
99  throw CmrError(msg,__FILE__,__LINE__);
100  }
101 
102  //################### feed
103  rapidjson::Value::ConstMemberIterator itr = cmr_doc.FindMember("feed");
104  result = itr != cmr_doc.MemberEnd();
105  msg = prolog + (result?"Located":"FAILED to locate") + " the value 'feed'.";
106  BESDEBUG(MODULE, msg << endl);
107  if(!result){
108  throw CmrError(msg,__FILE__,__LINE__);
109  }
110 
111  const rapidjson::Value& feed = itr->value;
112  result = feed.IsObject();
113  msg = prolog + "The value 'feed' is" + (result?"":" NOT") + " an object.";
114  BESDEBUG(MODULE, msg << endl);
115  if(!result){
116  throw CmrError(msg,__FILE__,__LINE__);
117  }
118  return feed;
119 }
120 
124 const rapidjson::Value&
125 CmrApi::get_entries(const rapidjson::Document &cmr_doc){
126  bool result;
127  string msg;
128 
129  const rapidjson::Value& feed = get_feed(cmr_doc);
130 
131  rapidjson::Value::ConstMemberIterator itr = feed.FindMember("entry");
132  result = itr != feed.MemberEnd();
133  msg = prolog + (result?"Located":"FAILED to locate") + " the value 'entry'.";
134  BESDEBUG(MODULE, msg << endl);
135  if(!result){
136  throw CmrError(msg,__FILE__,__LINE__);
137  }
138 
139  const rapidjson::Value& entry = itr->value;
140  result = entry.IsArray();
141  msg = prolog + "The value 'entry' is" + (result?"":" NOT") + " an Array.";
142  BESDEBUG(MODULE, msg << endl);
143  if(!result){
144  throw CmrError(msg,__FILE__,__LINE__);
145  }
146  return entry;
147 }
148 
152 const rapidjson::Value&
153 CmrApi::get_temporal_group(const rapidjson::Document &cmr_doc){
154  rjson_utils ru;
155 
156  bool result;
157  string msg;
158  const rapidjson::Value& feed = get_feed(cmr_doc);
159 
160  //################### facets
161  rapidjson::Value::ConstMemberIterator itr = feed.FindMember("facets");
162  result = itr != feed.MemberEnd();
163  msg = prolog + (result?"Located":"FAILED to locate") + " the value 'facets'." ;
164  BESDEBUG(MODULE, msg << endl);
165  if(!result){
166  throw CmrError(msg,__FILE__,__LINE__);
167  }
168 
169  const rapidjson::Value& facets_obj = itr->value;
170  result = facets_obj.IsObject();
171  msg = prolog + "The value 'facets' is" + (result?"":" NOT") + " an object.";
172  BESDEBUG(MODULE, msg << endl);
173  if(!result){
174  throw CmrError(msg,__FILE__,__LINE__);
175  }
176 
177  const rapidjson::Value& facets = get_children(facets_obj);
178  for (rapidjson::SizeType i = 0; i < facets.Size(); i++) { // Uses SizeType instead of size_t
179  const rapidjson::Value& facet = facets[i];
180 
181  string facet_title = ru.getStringValue(facet,"title");
182  string temporal_title("Temporal");
183  if(facet_title == temporal_title){
184  msg = prolog + "Found Temporal object.";
185  BESDEBUG(MODULE, msg << endl);
186  return facet;
187  }
188  else {
189  msg = prolog + "The child of 'facets' with title '"+facet_title+"' does not match 'Temporal'";
190  BESDEBUG(MODULE, msg << endl);
191  }
192  }
193  msg = prolog + "Failed to locate the Temporal facet.";
194  BESDEBUG(MODULE, msg << endl);
195  throw CmrError(msg,__FILE__,__LINE__);
196 
197 } // CmrApi::get_temporal_group()
198 
202 const rapidjson::Value&
203 CmrApi::get_year_group(const rapidjson::Document &cmr_doc){
204  rjson_utils rju;
205  string msg;
206 
207  const rapidjson::Value& temporal_group = get_temporal_group(cmr_doc);
208  const rapidjson::Value& temporal_children = get_children(temporal_group);
209  for (rapidjson::SizeType j = 0; j < temporal_children.Size(); j++) { // Uses SizeType instead of size_t
210  const rapidjson::Value& temporal_child = temporal_children[j];
211 
212  string temporal_child_title = rju.getStringValue(temporal_child,"title");
213  string year_title("Year");
214  if(temporal_child_title == year_title){
215  msg = prolog + "Found Year object.";
216  BESDEBUG(MODULE, msg << endl);
217  return temporal_child;
218  }
219  else {
220  msg = prolog + "The child of 'Temporal' with title '"+temporal_child_title+"' does not match 'Year'";
221  BESDEBUG(MODULE, msg << endl);
222  }
223  }
224  msg = prolog + "Failed to locate the Year group.";
225  BESDEBUG(MODULE, msg << endl);
226  throw CmrError(msg,__FILE__,__LINE__);
227 }
228 
232 const rapidjson::Value&
233 CmrApi::get_month_group(const string r_year, const rapidjson::Document &cmr_doc){
234  rjson_utils rju;
235  string msg;
236 
237  const rapidjson::Value& year_group = get_year_group(cmr_doc);
238  const rapidjson::Value& years = get_children(year_group);
239  for (rapidjson::SizeType i = 0; i < years.Size(); i++) { // Uses SizeType instead of size_t
240  const rapidjson::Value& year_obj = years[i];
241 
242  string year_title = rju.getStringValue(year_obj,"title");
243  if(r_year == year_title){
244  msg = prolog + "Found Year object.";
245  BESDEBUG(MODULE, msg << endl);
246 
247  const rapidjson::Value& year_children = get_children(year_obj);
248  for (rapidjson::SizeType j = 0; j < year_children.Size(); j++) { // Uses SizeType instead of size_t
249  const rapidjson::Value& child = year_children[i];
250  string title = rju.getStringValue(child,"title");
251  string month_title("Month");
252  if(title == month_title){
253  msg = prolog + "Found Month object.";
254  BESDEBUG(MODULE, msg << endl);
255  return child;
256  }
257  else {
258  msg = prolog + "The child of 'Year' with title '"+title+"' does not match 'Month'";
259  BESDEBUG(MODULE, msg << endl);
260  }
261  }
262  }
263  else {
264  msg = prolog + "The child of 'Year' group with title '"+year_title+"' does not match the requested year ("+r_year+")";
265  BESDEBUG(MODULE, msg << endl);
266  }
267  }
268  msg = prolog + "Failed to locate the Year group.";
269  BESDEBUG(MODULE, msg << endl);
270  throw CmrError(msg,__FILE__,__LINE__);
271 }
272 
273 const rapidjson::Value&
274 CmrApi::get_month(const string r_month, const string r_year, const rapidjson::Document &cmr_doc){
275  rjson_utils rju;
276  stringstream msg;
277 
278  const rapidjson::Value& month_group = get_month_group(r_year,cmr_doc);
279  const rapidjson::Value& months = get_children(month_group);
280  for (rapidjson::SizeType i = 0; i < months.Size(); i++) { // Uses SizeType instead of size_t
281  const rapidjson::Value& month = months[i];
282  string month_id = rju.getStringValue(month,"title");
283  if(month_id == r_month){
284  msg.str("");
285  msg << prolog << "Located requested month ("<<r_month << ")";
286  BESDEBUG(MODULE, msg.str() << endl);
287  return month;
288  }
289  else {
290  msg.str("");
291  msg << prolog << "The month titled '"<<month_id << "' does not match the requested month ("<< r_month <<")";
292  BESDEBUG(MODULE, msg.str() << endl);
293  }
294  }
295  msg.str("");
296  msg << prolog << "Failed to locate request Year/Month.";
297  BESDEBUG(MODULE, msg.str() << endl);
298  throw CmrError(msg.str(),__FILE__,__LINE__);
299 }
300 
301 const rapidjson::Value&
302 CmrApi::get_day_group(const string r_month, const string r_year, const rapidjson::Document &cmr_doc){
303  rjson_utils rju;
304  stringstream msg;
305 
306  const rapidjson::Value& month = get_month(r_month, r_year, cmr_doc);
307  const rapidjson::Value& month_children = get_children(month);
308 
309  for (rapidjson::SizeType k = 0; k < month_children.Size(); k++) { // Uses SizeType instead of size_t
310  const rapidjson::Value& object = month_children[k];
311  string title = rju.getStringValue(object,"title");
312  string day_group_title = "Day";
313  if(title == day_group_title){
314  msg.str("");
315  msg << prolog << "Located Day group for year: " << r_year << " month: "<< r_month;
316  BESDEBUG(MODULE, msg.str() << endl);
317  return object;
318  }
319  }
320  msg.str("");
321  msg << prolog << "Failed to locate requested Day year: " << r_year << " month: "<< r_month;
322  BESDEBUG(MODULE, msg.str() << endl);
323  throw CmrError(msg.str(),__FILE__,__LINE__);
324 }
325 
326 
333 void
334 CmrApi::get_years(string collection_name, vector<string> &years_result){
335  rjson_utils rju;
336  // bool result;
337  string msg;
338 
339  string url = BESUtil::assemblePath(cmr_search_endpoint_url,"granules.json") + "?concept_id="+collection_name +"&include_facets=v2";
341  rju.getJsonDoc(url,doc);
342 
343  const rapidjson::Value& year_group = get_year_group(doc);
344  const rapidjson::Value& years = get_children(year_group);
345  for (rapidjson::SizeType k = 0; k < years.Size(); k++) { // Uses SizeType instead of size_t
346  const rapidjson::Value& year_obj = years[k];
347  string year = rju.getStringValue(year_obj,"title");
348  years_result.push_back(year);
349  }
350 } // CmrApi::get_years()
351 
352 
361 void
362 CmrApi::get_months(string collection_name, string r_year, vector<string> &months_result){
363  rjson_utils rju;
364 
365  stringstream msg;
366 
367  string url = BESUtil::assemblePath(cmr_search_endpoint_url,"granules.json")
368  + "?concept_id="+collection_name
369  +"&include_facets=v2"
370  +"&temporal_facet[0][year]="+r_year;
371 
373  rju.getJsonDoc(url,doc);
374  BESDEBUG(MODULE, prolog << "Got JSON Document: "<< endl << rju.jsonDocToString(doc) << endl);
375 
376  const rapidjson::Value& year_group = get_year_group(doc);
377  const rapidjson::Value& years = get_children(year_group);
378  if(years.Size() != 1){
379  msg.str("");
380  msg << prolog << "We expected to get back one year (" << r_year << ") but we got back " << years.Size();
381  BESDEBUG(MODULE, msg.str() << endl);
382  throw CmrError(msg.str(),__FILE__,__LINE__);
383  }
384 
385  const rapidjson::Value& year = years[0];
386  string year_title = rju.getStringValue(year,"title");
387  if(year_title != r_year){
388  msg.str("");
389  msg << prolog << "The returned year (" << year_title << ") does not match the requested year ("<< r_year << ")";
390  BESDEBUG(MODULE, msg.str() << endl);
391  throw CmrError(msg.str(),__FILE__,__LINE__);
392  }
393 
394  const rapidjson::Value& year_children = get_children(year);
395  if(year_children.Size() != 1){
396  msg.str("");
397  msg << prolog << "We expected to get back one child for the year (" << r_year << ") but we got back " << years.Size();
398  BESDEBUG(MODULE, msg.str() << endl);
399  throw CmrError(msg.str(),__FILE__,__LINE__);
400  }
401 
402  const rapidjson::Value& month_group = year_children[0];
403  string title = rju.getStringValue(month_group,"title");
404  if(title != string("Month")){
405  msg.str("");
406  msg << prolog << "We expected to get back a Month object, but we did not.";
407  BESDEBUG(MODULE, msg.str() << endl);
408  throw CmrError(msg.str(),__FILE__,__LINE__);
409  }
410 
411  const rapidjson::Value& months = get_children(month_group);
412  for (rapidjson::SizeType i = 0; i < months.Size(); i++) { // Uses SizeType instead of size_t
413  const rapidjson::Value& month = months[i];
414  string month_id = rju.getStringValue(month,"title");
415  months_result.push_back(month_id);
416  }
417  return;
418 
419 } // CmrApi::get_months()
420 
424 void
425 CmrApi::get_days(string collection_name, string r_year, string r_month, vector<string> &days_result){
426  rjson_utils rju;
427  stringstream msg;
428 
429  string url = BESUtil::assemblePath(cmr_search_endpoint_url,"granules.json")
430  + "?concept_id="+collection_name
431  +"&include_facets=v2"
432  +"&temporal_facet[0][year]="+r_year
433  +"&temporal_facet[0][month]="+r_month;
434 
435  rapidjson::Document cmr_doc;
436  rju.getJsonDoc(url,cmr_doc);
437  BESDEBUG(MODULE, prolog << "Got JSON Document: "<< endl << rju.jsonDocToString(cmr_doc) << endl);
438 
439  const rapidjson::Value& day_group = get_day_group(r_month, r_year, cmr_doc);
440  const rapidjson::Value& days = get_children(day_group);
441  for (rapidjson::SizeType i = 0; i < days.Size(); i++) { // Uses SizeType instead of size_t
442  const rapidjson::Value& day = days[i];
443  string day_id = rju.getStringValue(day,"title");
444  days_result.push_back(day_id);
445  }
446 }
447 
448 
449 
453 void
454 CmrApi::get_granule_ids(string collection_name, string r_year, string r_month, string r_day, vector<string> &granules_ids){
455  rjson_utils rju;
456  stringstream msg;
457  rapidjson::Document cmr_doc;
458 
459  granule_search(collection_name, r_year, r_month, r_day, cmr_doc);
460 
461  const rapidjson::Value& entries = get_entries(cmr_doc);
462  for (rapidjson::SizeType i = 0; i < entries.Size(); i++) { // Uses SizeType instead of size_t
463  const rapidjson::Value& granule = entries[i];
464  string day_id = rju.getStringValue(granule,"id");
465  granules_ids.push_back(day_id);
466  }
467 
468 }
469 
470 
474 unsigned long
475 CmrApi::granule_count(string collection_name, string r_year, string r_month, string r_day){
476  stringstream msg;
477  rapidjson::Document cmr_doc;
478  granule_search(collection_name, r_year, r_month, r_day, cmr_doc);
479  const rapidjson::Value& entries = get_entries(cmr_doc);
480  return entries.Size();
481 }
482 
487 void
488 CmrApi::granule_search(string collection_name, string r_year, string r_month, string r_day, rapidjson::Document &result_doc){
489  rjson_utils rju;
490 
491  string url = BESUtil::assemblePath(cmr_search_endpoint_url,"granules.json")
492  + "?concept_id="+collection_name
493  + "&include_facets=v2"
494  + "&page_size=2000";
495 
496  if(!r_year.empty())
497  url += "&temporal_facet[0][year]="+r_year;
498 
499  if(!r_month.empty())
500  url += "&temporal_facet[0][month]="+r_month;
501 
502  if(!r_day.empty())
503  url += "&temporal_facet[0][day]="+r_day;
504 
505  BESDEBUG(MODULE, prolog << "CMR Granule Search Request Url: : " << url << endl);
506  rju.getJsonDoc(url,result_doc);
507  BESDEBUG(MODULE, prolog << "Got JSON Document: "<< endl << rju.jsonDocToString(result_doc) << endl);
508 }
509 
510 
511 
515 void
516 CmrApi::get_granules(string collection_name, string r_year, string r_month, string r_day, vector<Granule *> &granules){
517  stringstream msg;
518  rapidjson::Document cmr_doc;
519 
520  granule_search(collection_name, r_year, r_month, r_day, cmr_doc);
521 
522  const rapidjson::Value& entries = get_entries(cmr_doc);
523  for (rapidjson::SizeType i = 0; i < entries.Size(); i++) { // Uses SizeType instead of size_t
524  const rapidjson::Value& granule_obj = entries[i];
525  // rapidjson::Value grnl(granule_obj, cmr_doc.GetAllocator());
526  Granule *g = new Granule(granule_obj);
527  granules.push_back(g);
528  }
529 
530 }
531 
532 
533 
534 void
535 CmrApi::get_collection_ids(std::vector<std::string> &collection_ids){
536  bool found = false;
537  string key = CMR_COLLECTIONS;
538  TheBESKeys::TheKeys()->get_values(CMR_COLLECTIONS, collection_ids, found);
539  if(!found){
540  throw BESInternalError(string("The '") +CMR_COLLECTIONS
541  + "' field has not been configured.", __FILE__, __LINE__);
542  }
543 }
544 
545 
549 cmr::Granule* CmrApi::get_granule(string collection_name, string r_year, string r_month, string r_day, string granule_id)
550 {
551  vector<Granule *> granules;
552  Granule *result = 0;
553 
554  get_granules(collection_name, r_year, r_month, r_day, granules);
555  for(size_t i=0; i<granules.size() ;i++){
556  string id = granules[i]->getName();
557  BESDEBUG(MODULE, prolog << "Comparing granule id: " << granule_id << " to collection member id: " << id << endl);
558  if( id == granule_id){
559  result = granules[i];
560  }
561  else {
562  delete granules[i];
563  granules[i] = 0;
564  }
565  }
566  return result;
567 }
568 
569 
570 
571 
572 
573 
574 } // namespace cmr
575 
GenericMemberIterator
(Constant) member iterator for a JSON object value
Definition: cmr_module/rapidjson/document.h:101
cmr::rjson_utils::jsonDocToString
std::string jsonDocToString(rapidjson::Document &d)
Definition: rjson_utils.cc:110
Document
GenericDocument< UTF8<> > Document
GenericDocument with UTF8 encoding.
Definition: cmr_module/rapidjson/document.h:2402
cmr::rjson_utils::getStringValue
std::string getStringValue(const rapidjson::Value &object, const std::string &name)
Definition: rjson_utils.cc:84
BESUtil::assemblePath
static std::string assemblePath(const std::string &firstPart, const std::string &secondPart, bool leadingSlash=false, bool trailingSlash=false)
Assemble path fragments making sure that they are separated by a single '/' character.
Definition: BESUtil.cc:821
cmr::CmrError
Definition: CmrError.h:34
TheBESKeys::TheKeys
static TheBESKeys * TheKeys()
Definition: TheBESKeys.cc:62
cmr::Granule
Definition: Granule.h:43
cmr::rjson_utils::getJsonDoc
void getJsonDoc(const std::string &url, rapidjson::Document &d)
Definition: rjson_utils.cc:57
SizeType
RAPIDJSON_NAMESPACE_BEGIN typedef unsigned SizeType
Size type (for string lengths, array sizes, etc.)
Definition: cmr_module/rapidjson/rapidjson.h:380
Value
GenericValue< UTF8<> > Value
GenericValue with UTF8 encoding.
Definition: cmr_module/rapidjson/document.h:2010
BESInternalError
exception thrown if internal error encountered
Definition: BESInternalError.h:43
TheBESKeys::get_values
void get_values(const std::string &s, std::vector< std::string > &vals, bool &found)
Retrieve the values of a given key, if set.
Definition: TheBESKeys.cc:303
cmr::rjson_utils
Definition: rjson_utils.h:42