30 #include <sys/types.h>
44 #include "BESCatalogUtils.h"
45 #include "BESCatalogEntry.h"
47 #include "CatalogNode.h"
48 #include "CatalogItem.h"
51 #include "BESContainerStorageList.h"
52 #include "BESFileContainerStorage.h"
55 #include "BESInternalError.h"
56 #include "BESForbiddenError.h"
57 #include "BESNotFoundError.h"
58 #include "BESSyntaxUserError.h"
60 #include "TheBESKeys.h"
65 #include "CmrCatalog.h"
70 #define prolog std::string("CmrCatalog::").append(__func__).append("() - ")
85 CmrCatalog::CmrCatalog(
const std::string &name ) :
BESCatalog(name) {
89 throw BESInternalError(
string(
"The CMR module must define at least one collection name using the key; '")+CMR_COLLECTIONS
90 +
"'", __FILE__, __LINE__);
96 throw BESInternalError(
string(
"The CMR module must define at least one facet name using the key; '")+CMR_COLLECTIONS
97 +
"'", __FILE__, __LINE__);
101 CmrCatalog::~CmrCatalog()
105 CmrCatalog::get_node(
const string &path)
const
107 return get_node_NEW(path);
112 CmrCatalog::get_node_NEW(
const string &ppath)
const
116 BESDEBUG(MODULE, prolog <<
"path: '" << path <<
"' path_elements.size(): " << path_elements.size() << endl);
123 if(path_elements.empty()){
125 node->set_lmt(epoch_time);
126 node->set_catalog_name(CMR_CATALOG_NAME);
127 for(
size_t i=0; i<d_collections.size() ; i++){
129 collection->
set_name(d_collections[i]);
130 collection->
set_type(CatalogItem::node);
131 node->add_node(collection);
135 for(
size_t i=0; i< path_elements.size() ;i++){
136 if(path_elements[i]==
"-")
137 path_elements[i] =
"";
140 string collection = path_elements[0];
141 BESDEBUG(MODULE, prolog <<
"Checking for collection: " << collection <<
" d_collections.size(): " << d_collections.size() << endl);
142 bool valid_collection =
false;
143 for(
size_t i=0; i<d_collections.size() && !valid_collection ; i++){
144 if(collection == d_collections[i])
145 valid_collection =
true;
147 if(!valid_collection){
148 throw BESNotFoundError(
"The CMR catalog does not contain a collection named '"+collection+
"'",__FILE__,__LINE__);
150 BESDEBUG(MODULE, prolog <<
"Collection " << collection <<
" is valid." << endl);
151 if(path_elements.size() >1){
152 string facet = path_elements[1];
153 bool valid_facet =
false;
154 for(
size_t i=0; i<d_facets.size() && !valid_facet ; i++){
155 if(facet == d_facets[i])
159 throw BESNotFoundError(
"The CMR collection '"+collection+
"' does not contain a facet named '"+facet+
"'",__FILE__,__LINE__);
162 if(facet==
"temporal"){
163 BESDEBUG(MODULE, prolog <<
"Found Temporal Facet"<< endl);
165 node->set_lmt(epoch_time);
166 node->set_catalog_name(CMR_CATALOG_NAME);
169 switch( path_elements.size()){
173 vector<string> years;
175 BESDEBUG(MODULE, prolog <<
"Getting year nodes for collection: " << collection<< endl);
176 cmrApi.get_years(collection, years);
177 for(
size_t i=0; i<years.size() ; i++){
179 collection->
set_type(CatalogItem::node);
182 collection->
set_lmt(epoch_time);
184 node->add_node(collection);
191 string year = path_elements[2];
194 vector<string> months;
196 BESDEBUG(MODULE, prolog <<
"Getting month nodes for collection: " << collection <<
" year: " << year << endl);
197 cmrApi.get_months(collection, year, months);
198 for(
size_t i=0; i<months.size() ; i++){
200 collection->
set_type(CatalogItem::node);
203 collection->
set_lmt(epoch_time);
205 node->add_node(collection);
212 string year = path_elements[2];
213 string month = path_elements[3];
217 BESDEBUG(MODULE, prolog <<
"Getting day nodes for collection: " << collection <<
" year: " << year <<
" month: " << month << endl);
218 cmrApi.get_days(collection, year, month, days);
219 for(
size_t i=0; i<days.size() ; i++){
221 collection->
set_type(CatalogItem::node);
224 collection->
set_lmt(epoch_time);
226 node->add_node(collection);
233 string year = path_elements[2];
234 string month = path_elements[3];
235 string day = path_elements[4];
236 BESDEBUG(MODULE, prolog <<
"Getting granule leaves for collection: " << collection <<
" year: " << year <<
" month: " << month <<
" day: " << day << endl);
237 vector<Granule *> granules;
238 cmrApi.get_granules(collection, year, month, day, granules);
239 for(
size_t i=0; i<granules.size() ; i++){
247 string year = path_elements[2];
248 string month = path_elements[3];
249 string day = path_elements[4];
250 string granule_id = path_elements[5];
251 BESDEBUG(MODULE, prolog <<
"Request resolved to leaf granule/dataset name, collection: " << collection <<
" year: " << year
252 <<
" month: " << month <<
" day: " << day <<
" granule: " << granule_id << endl);
253 Granule *granule = cmrApi.get_granule(collection,year,month,day,granule_id);
256 granuleItem->
set_type(CatalogItem::leaf);
257 granuleItem->
set_name(granule->getName());
259 granuleItem->
set_lmt(granule->getLastModifiedStr());
260 granuleItem->
set_size(granule->getSize());
261 node->set_leaf(granuleItem);
271 throw BESSyntaxUserError(
"CmrCatalog: The path '"+path+
"' does not describe a valid temporal facet search.",__FILE__,__LINE__);
278 throw BESNotFoundError(
"The CMR catalog only supports temporal faceting.",__FILE__,__LINE__);
282 BESDEBUG(MODULE, prolog <<
"Building facet list for collection: " << collection << endl);
284 node->set_lmt(epoch_time);
285 node->set_catalog_name(CMR_CATALOG_NAME);
286 for(
size_t i=0; i<d_facets.size() ; i++){
289 collection->
set_type(CatalogItem::node);
290 collection->
set_lmt(epoch_time);
291 BESDEBUG(MODULE, prolog <<
"Adding facet: " << d_facets[i] << endl);
292 node->add_node(collection);
326 BESDEBUG(MODULE, prolog <<
"path: '" << path <<
"' path_elements.size(): " << path_elements.size() << endl);
333 if(path_elements.empty()){
335 node->set_lmt(epoch_time);
336 node->set_catalog_name(CMR_CATALOG_NAME);
337 for(
size_t i=0; i<d_collections.size() ; i++){
339 collection->
set_name(d_collections[i]);
340 collection->
set_type(CatalogItem::node);
341 node->add_node(collection);
345 string collection = path_elements[0];
346 BESDEBUG(MODULE, prolog <<
"Checking for collection: " << collection <<
" d_collections.size(): " << d_collections.size() << endl);
347 bool valid_collection =
false;
348 for(
size_t i=0; i<d_collections.size() && !valid_collection ; i++){
349 if(collection == d_collections[i])
350 valid_collection =
true;
352 if(!valid_collection){
353 throw BESNotFoundError(
"The CMR catalog does not contain a collection named '"+collection+
"'",__FILE__,__LINE__);
355 BESDEBUG(MODULE, prolog <<
"Collection " << collection <<
" is valid." << endl);
356 if(path_elements.size() >1){
357 string facet = path_elements[1];
358 bool valid_facet =
false;
359 for(
size_t i=0; i<d_facets.size() && !valid_facet ; i++){
360 if(facet == d_facets[i])
364 throw BESNotFoundError(
"The CMR collection '"+collection+
"' does not contain a facet named '"+facet+
"'",__FILE__,__LINE__);
367 if(facet==
"temporal"){
368 BESDEBUG(MODULE, prolog <<
"Found Temporal Facet"<< endl);
370 node->set_lmt(epoch_time);
371 node->set_catalog_name(CMR_CATALOG_NAME);
374 switch( path_elements.size()){
377 vector<string> years;
379 BESDEBUG(MODULE, prolog <<
"Getting year nodes for collection: " << collection<< endl);
381 for(
size_t i=0; i<years.size() ; i++){
383 collection->
set_type(CatalogItem::node);
386 collection->
set_lmt(epoch_time);
388 node->add_node(collection);
394 string year = path_elements[2];
397 vector<string> months;
399 BESDEBUG(MODULE, prolog <<
"Getting month nodes for collection: " << collection <<
" year: " << year << endl);
401 for(
size_t i=0; i<months.size() ; i++){
403 collection->
set_type(CatalogItem::node);
406 collection->
set_lmt(epoch_time);
408 node->add_node(collection);
414 string year = path_elements[2];
415 string month = path_elements[3];
419 BESDEBUG(MODULE, prolog <<
"Getting day nodes for collection: " << collection <<
" year: " << year <<
" month: " << month << endl);
420 cmrApi.
get_days(collection, year, month, days);
421 for(
size_t i=0; i<days.size() ; i++){
423 collection->
set_type(CatalogItem::node);
426 collection->
set_lmt(epoch_time);
428 node->add_node(collection);
434 string year = path_elements[2];
435 string month = path_elements[3];
436 string day = path_elements[4];
437 BESDEBUG(MODULE, prolog <<
"Getting granule leaves for collection: " << collection <<
" year: " << year <<
" month: " << month <<
" day: " << day << endl);
438 vector<Granule *> granules;
439 cmrApi.
get_granules(collection, year, month, day, granules);
440 for(
size_t i=0; i<granules.size() ; i++){
446 throw BESSyntaxUserError(
"CmrCatalog: The path '"+path+
"' does not describe a valid temporal facet search.",__FILE__,__LINE__);
451 throw BESNotFoundError(
"The CMR catalog only supports temporal faceting.",__FILE__,__LINE__);
455 BESDEBUG(MODULE, prolog <<
"Building facet list for collection: " << collection << endl);
457 node->set_lmt(epoch_time);
458 node->set_catalog_name(CMR_CATALOG_NAME);
459 for(
size_t i=0; i<d_facets.size() ; i++){
462 collection->
set_type(CatalogItem::node);
463 collection->
set_lmt(epoch_time);
464 BESDEBUG(MODULE, prolog <<
"Adding facet: " << d_facets[i] << endl);
465 node->add_node(collection);
474 CmrCatalog::get_node(
const string &path)
const
477 string rootdir = d_utils->get_root_dir();
485 string fullpath = rootdir + path;
487 DIR *dip = opendir(fullpath.c_str());
490 "A CMRCatalog can only return nodes for directory. The path '" + path
491 +
"' is not a directory for BESCatalog '" +
get_catalog_name() +
"'.", __FILE__, __LINE__);
498 if (d_utils->exclude(path))
500 string(
"The path '") + path +
"' is not included in the catalog '" +
get_catalog_name() +
"'.",
507 int statret = stat(fullpath.c_str(), &buf);
509 node->set_lmt(get_time(buf.st_mtime));
512 while ((dit = readdir(dip)) != NULL) {
513 string item = dit->d_name;
514 if (item ==
"." || item ==
"..")
continue;
516 string item_path = fullpath +
"/" + item;
526 if (d_utils->follow_sym_links() ==
false) {
528 (void) lstat(item_path.c_str(), &lbuf);
529 if (S_ISLNK(lbuf.st_mode))
continue;
533 statret = stat(item_path.c_str(), &buf);
534 if (statret == 0 && S_ISDIR(buf.st_mode) && !d_utils->exclude(item)) {
537 node->add_item(
new CatalogItem(item, 0, get_time(buf.st_mtime), CatalogItem::node));
539 node->add_node(
new CatalogItem(item, 0, get_time(buf.st_mtime), CatalogItem::node));
541 else if (statret == 0 && S_ISREG(buf.st_mode) && d_utils->include(item)) {
544 node->add_item(
new CatalogItem(item, buf.st_size, get_time(buf.st_mtime),
545 d_utils->is_data(item), CatalogItem::leaf));
547 node->add_leaf(
new CatalogItem(item, buf.st_size, get_time(buf.st_mtime),
548 d_utils->is_data(item), CatalogItem::leaf));
551 VERBOSE(
"Excluded the item '" << item_path <<
"' from the catalog '" <<
get_catalog_name() <<
"' node listing.");
559 sort(node->nodes_begin(), node->nodes_end(), ordering);
560 sort(node->leaves_begin(), node->leaves_end(), ordering);
581 strm << BESIndent::LMarg << prolog <<
"(" << (
void *)
this <<
")" << endl;
584 strm << BESIndent::LMarg <<
"catalog utilities: " << endl;
587 BESIndent::UnIndent();
588 BESIndent::UnIndent();