30 #include <sys/types.h> 44 #include "BESCatalogUtils.h" 45 #include "BESCatalogEntry.h" 47 #include "CatalogNode.h" 48 #include "CatalogItem.h" 51 #include "BESContainerStorageList.h" 52 #include "BESFileContainerStorage.h" 55 #include "BESInternalError.h" 56 #include "BESForbiddenError.h" 57 #include "BESNotFoundError.h" 58 #include "BESSyntaxUserError.h" 60 #include "TheBESKeys.h" 65 #include "CmrCatalog.h" 70 #define prolog std::string("CmrCatalog::").append(__func__).append("() - ") 85 CmrCatalog::CmrCatalog(
const std::string &name ) :
BESCatalog(name) {
89 throw BESInternalError(
string(
"The CMR module must define at least one collection name using the key; '")+CMR_COLLECTIONS
90 +
"'", __FILE__, __LINE__);
96 throw BESInternalError(
string(
"The CMR module must define at least one facet name using the key; '")+CMR_COLLECTIONS
97 +
"'", __FILE__, __LINE__);
101 CmrCatalog::~CmrCatalog()
105 CmrCatalog::get_node(
const string &path)
const 107 return get_node_NEW(path);
112 CmrCatalog::get_node_NEW(
const string &ppath)
const 116 BESDEBUG(MODULE, prolog <<
"path: '" << path <<
"' path_elements.size(): " << path_elements.size() << endl);
123 if(path_elements.empty()){
125 node->set_lmt(epoch_time);
126 node->set_catalog_name(CMR_CATALOG_NAME);
127 for(
size_t i=0; i<d_collections.size() ; i++){
129 collection->
set_name(d_collections[i]);
130 collection->
set_type(CatalogItem::node);
131 node->add_node(collection);
135 for(
size_t i=0; i< path_elements.size() ;i++){
136 if(path_elements[i]==
"-")
137 path_elements[i] =
"";
140 string collection = path_elements[0];
141 BESDEBUG(MODULE, prolog <<
"Checking for collection: " << collection <<
" d_collections.size(): " << d_collections.size() << endl);
142 bool valid_collection =
false;
143 for(
size_t i=0; i<d_collections.size() && !valid_collection ; i++){
144 if(collection == d_collections[i])
145 valid_collection =
true;
147 if(!valid_collection){
148 throw BESNotFoundError(
"The CMR catalog does not contain a collection named '"+collection+
"'",__FILE__,__LINE__);
150 BESDEBUG(MODULE, prolog <<
"Collection " << collection <<
" is valid." << endl);
151 if(path_elements.size() >1){
152 string facet = path_elements[1];
153 bool valid_facet =
false;
154 for(
size_t i=0; i<d_facets.size() && !valid_facet ; i++){
155 if(facet == d_facets[i])
159 throw BESNotFoundError(
"The CMR collection '"+collection+
"' does not contain a facet named '"+facet+
"'",__FILE__,__LINE__);
162 if(facet==
"temporal"){
163 BESDEBUG(MODULE, prolog <<
"Found Temporal Facet"<< endl);
165 node->set_lmt(epoch_time);
166 node->set_catalog_name(CMR_CATALOG_NAME);
169 switch( path_elements.size()){
173 vector<string> years;
175 BESDEBUG(MODULE, prolog <<
"Getting year nodes for collection: " << collection<< endl);
176 cmrApi.get_years(collection, years);
177 for(
size_t i=0; i<years.size() ; i++){
179 collection->
set_type(CatalogItem::node);
182 collection->
set_lmt(epoch_time);
184 node->add_node(collection);
191 string year = path_elements[2];
194 vector<string> months;
196 BESDEBUG(MODULE, prolog <<
"Getting month nodes for collection: " << collection <<
" year: " << year << endl);
197 cmrApi.get_months(collection, year, months);
198 for(
size_t i=0; i<months.size() ; i++){
200 collection->
set_type(CatalogItem::node);
203 collection->
set_lmt(epoch_time);
205 node->add_node(collection);
212 string year = path_elements[2];
213 string month = path_elements[3];
217 BESDEBUG(MODULE, prolog <<
"Getting day nodes for collection: " << collection <<
" year: " << year <<
" month: " << month << endl);
218 cmrApi.get_days(collection, year, month, days);
219 for(
size_t i=0; i<days.size() ; i++){
221 collection->
set_type(CatalogItem::node);
224 collection->
set_lmt(epoch_time);
226 node->add_node(collection);
233 string year = path_elements[2];
234 string month = path_elements[3];
235 string day = path_elements[4];
236 BESDEBUG(MODULE, prolog <<
"Getting granule leaves for collection: " << collection <<
" year: " << year <<
" month: " << month <<
" day: " << day << endl);
237 vector<Granule *> granules;
238 cmrApi.get_granules(collection, year, month, day, granules);
239 for(
size_t i=0; i<granules.size() ; i++){
247 string year = path_elements[2];
248 string month = path_elements[3];
249 string day = path_elements[4];
250 string granule_id = path_elements[5];
251 BESDEBUG(MODULE, prolog <<
"Request resolved to leaf granule/dataset name, collection: " << collection <<
" year: " << year
252 <<
" month: " << month <<
" day: " << day <<
" granule: " << granule_id << endl);
253 Granule *granule = cmrApi.get_granule(collection,year,month,day,granule_id);
256 granuleItem->
set_type(CatalogItem::leaf);
257 granuleItem->
set_name(granule->getName());
259 granuleItem->
set_lmt(granule->getLastModifiedStr());
260 granuleItem->
set_size(granule->getSize());
261 node->set_leaf(granuleItem);
271 throw BESSyntaxUserError(
"CmrCatalog: The path '"+path+
"' does not describe a valid temporal facet search.",__FILE__,__LINE__);
278 throw BESNotFoundError(
"The CMR catalog only supports temporal faceting.",__FILE__,__LINE__);
282 BESDEBUG(MODULE, prolog <<
"Building facet list for collection: " << collection << endl);
284 node->set_lmt(epoch_time);
285 node->set_catalog_name(CMR_CATALOG_NAME);
286 for(
size_t i=0; i<d_facets.size() ; i++){
289 collection->
set_type(CatalogItem::node);
290 collection->
set_lmt(epoch_time);
291 BESDEBUG(MODULE, prolog <<
"Adding facet: " << d_facets[i] << endl);
292 node->add_node(collection);
326 BESDEBUG(MODULE, prolog <<
"path: '" << path <<
"' path_elements.size(): " << path_elements.size() << endl);
333 if(path_elements.empty()){
335 node->set_lmt(epoch_time);
336 node->set_catalog_name(CMR_CATALOG_NAME);
337 for(
size_t i=0; i<d_collections.size() ; i++){
339 collection->
set_name(d_collections[i]);
340 collection->
set_type(CatalogItem::node);
341 node->add_node(collection);
345 string collection = path_elements[0];
346 BESDEBUG(MODULE, prolog <<
"Checking for collection: " << collection <<
" d_collections.size(): " << d_collections.size() << endl);
347 bool valid_collection =
false;
348 for(
size_t i=0; i<d_collections.size() && !valid_collection ; i++){
349 if(collection == d_collections[i])
350 valid_collection =
true;
352 if(!valid_collection){
353 throw BESNotFoundError(
"The CMR catalog does not contain a collection named '"+collection+
"'",__FILE__,__LINE__);
355 BESDEBUG(MODULE, prolog <<
"Collection " << collection <<
" is valid." << endl);
356 if(path_elements.size() >1){
357 string facet = path_elements[1];
358 bool valid_facet =
false;
359 for(
size_t i=0; i<d_facets.size() && !valid_facet ; i++){
360 if(facet == d_facets[i])
364 throw BESNotFoundError(
"The CMR collection '"+collection+
"' does not contain a facet named '"+facet+
"'",__FILE__,__LINE__);
367 if(facet==
"temporal"){
368 BESDEBUG(MODULE, prolog <<
"Found Temporal Facet"<< endl);
370 node->set_lmt(epoch_time);
371 node->set_catalog_name(CMR_CATALOG_NAME);
374 switch( path_elements.size()){
377 vector<string> years;
379 BESDEBUG(MODULE, prolog <<
"Getting year nodes for collection: " << collection<< endl);
381 for(
size_t i=0; i<years.size() ; i++){
383 collection->
set_type(CatalogItem::node);
386 collection->
set_lmt(epoch_time);
388 node->add_node(collection);
394 string year = path_elements[2];
397 vector<string> months;
399 BESDEBUG(MODULE, prolog <<
"Getting month nodes for collection: " << collection <<
" year: " << year << endl);
401 for(
size_t i=0; i<months.size() ; i++){
403 collection->
set_type(CatalogItem::node);
406 collection->
set_lmt(epoch_time);
408 node->add_node(collection);
414 string year = path_elements[2];
415 string month = path_elements[3];
419 BESDEBUG(MODULE, prolog <<
"Getting day nodes for collection: " << collection <<
" year: " << year <<
" month: " << month << endl);
420 cmrApi.
get_days(collection, year, month, days);
421 for(
size_t i=0; i<days.size() ; i++){
423 collection->
set_type(CatalogItem::node);
426 collection->
set_lmt(epoch_time);
428 node->add_node(collection);
434 string year = path_elements[2];
435 string month = path_elements[3];
436 string day = path_elements[4];
437 BESDEBUG(MODULE, prolog <<
"Getting granule leaves for collection: " << collection <<
" year: " << year <<
" month: " << month <<
" day: " << day << endl);
438 vector<Granule *> granules;
439 cmrApi.
get_granules(collection, year, month, day, granules);
440 for(
size_t i=0; i<granules.size() ; i++){
446 throw BESSyntaxUserError(
"CmrCatalog: The path '"+path+
"' does not describe a valid temporal facet search.",__FILE__,__LINE__);
451 throw BESNotFoundError(
"The CMR catalog only supports temporal faceting.",__FILE__,__LINE__);
455 BESDEBUG(MODULE, prolog <<
"Building facet list for collection: " << collection << endl);
457 node->set_lmt(epoch_time);
458 node->set_catalog_name(CMR_CATALOG_NAME);
459 for(
size_t i=0; i<d_facets.size() ; i++){
462 collection->
set_type(CatalogItem::node);
463 collection->
set_lmt(epoch_time);
464 BESDEBUG(MODULE, prolog <<
"Adding facet: " << d_facets[i] << endl);
465 node->add_node(collection);
474 CmrCatalog::get_node(
const string &path)
const 477 string rootdir = d_utils->get_root_dir();
485 string fullpath = rootdir + path;
487 DIR *dip = opendir(fullpath.c_str());
490 "A CMRCatalog can only return nodes for directory. The path '" + path
491 +
"' is not a directory for BESCatalog '" +
get_catalog_name() +
"'.", __FILE__, __LINE__);
498 if (d_utils->exclude(path))
500 string(
"The path '") + path +
"' is not included in the catalog '" +
get_catalog_name() +
"'.",
507 int statret = stat(fullpath.c_str(), &buf);
509 node->set_lmt(get_time(buf.st_mtime));
512 while ((dit = readdir(dip)) != NULL) {
513 string item = dit->d_name;
514 if (item ==
"." || item ==
"..")
continue;
516 string item_path = fullpath +
"/" + item;
526 if (d_utils->follow_sym_links() ==
false) {
528 (void) lstat(item_path.c_str(), &lbuf);
529 if (S_ISLNK(lbuf.st_mode))
continue;
533 statret = stat(item_path.c_str(), &buf);
534 if (statret == 0 && S_ISDIR(buf.st_mode) && !d_utils->exclude(item)) {
537 node->add_item(
new CatalogItem(item, 0, get_time(buf.st_mtime), CatalogItem::node));
539 node->add_node(
new CatalogItem(item, 0, get_time(buf.st_mtime), CatalogItem::node));
541 else if (statret == 0 && S_ISREG(buf.st_mode) && d_utils->include(item)) {
544 node->add_item(
new CatalogItem(item, buf.st_size, get_time(buf.st_mtime),
545 d_utils->is_data(item), CatalogItem::leaf));
547 node->add_leaf(
new CatalogItem(item, buf.st_size, get_time(buf.st_mtime),
548 d_utils->is_data(item), CatalogItem::leaf));
551 VERBOSE(
"Excluded the item '" << item_path <<
"' from the catalog '" <<
get_catalog_name() <<
"' node listing.");
559 sort(node->nodes_begin(), node->nodes_end(), ordering);
560 sort(node->leaves_begin(), node->leaves_end(), ordering);
581 strm << BESIndent::LMarg << prolog <<
"(" << (
void *)
this <<
")" << endl;
584 strm << BESIndent::LMarg <<
"catalog utilities: " << endl;
587 BESIndent::UnIndent();
588 BESIndent::UnIndent();
void set_name(std::string n)
Set the name of the item.
error thrown if the resource requested cannot be found
virtual std::string get_catalog_name() const
Get the name for this catalog.
static string get_time(bool use_local_time=false)
exception thrown if inernal error encountered
void get_months(std::string collection_name, std::string year, std::vector< std::string > &months_result)
void get_years(std::string collection_name, std::vector< std::string > &years_result)
void set_type(item_type t)
Set the type for this item.
static std::string normalize_path(const std::string &path, bool leading_separator, bool trailing_separator, const string separator="/")
Removes duplicate separators and provides leading and trailing separators as directed.
void set_is_data(bool id)
Is this item data that the BES should interpret?
error thrown if there is a user syntax error in the request or any other user error
void set_lmt(std::string lmt)
Set the LMT for this item.
static TheBESKeys * TheKeys()
void get_values(const std::string &s, std::vector< std::string > &vals, bool &found)
Retrieve the values of a given key, if set.
static std::vector< std::string > split(const string &s, char delim='/', bool skip_empty=true)
Splits the string s into the return vector of tokens using the delimiter delim and skipping empty val...
virtual void dump(ostream &strm) const
dump the contents of this object to the specified ostream
virtual void dump(ostream &strm) const
dumps information about this object
void get_granules(std::string collection_name, string r_year, string r_month, string r_day, std::vector< cmr::Granule * > &granules)
Catalogs provide a hierarchical organization for data.
void get_days(std::string collection_name, std::string r_year, std::string r_month, std::vector< std::string > &days_result)
error thrown if the BES is not allowed to access the resource requested
void set_size(size_t s)
Set the size of the item.
virtual BESCatalogUtils * get_catalog_utils() const
Get a pointer to the utilities, customized for this catalog.
static void check_path(const string &path, const string &root, bool follow_sym_links)
Check if the specified path is valid.
virtual bes::CatalogNode * get_node_OLD(const std::string &path) const
Get a CatalogNode for the given path in the current catalog.