35 #include <sys/types.h>
45 #include "BESCatalogUtils.h"
46 #include "BESCatalogList.h"
47 #include "TheBESKeys.h"
48 #include "BESInternalError.h"
49 #include "BESSyntaxUserError.h"
50 #include "BESNotFoundError.h"
54 #include "BESContainerStorageList.h"
55 #include "BESContainerStorage.h"
56 #include "BESCatalogEntry.h"
61 map<string, BESCatalogUtils *> BESCatalogUtils::_instances;
88 BESCatalogUtils::BESCatalogUtils(
const string &n,
bool strict) :
89 d_name(n), d_follow_syms(false)
91 string key =
"BES.Catalog." + n +
".RootDirectory";
94 if (strict && (!found || d_root_dir ==
"")) {
95 string s = key +
" not defined in BES configuration file";
99 if(d_root_dir !=
"UNUSED"){
101 DIR *dip = opendir(d_root_dir.c_str());
103 string serr =
"BESCatalogDirectory - root directory " + d_root_dir +
" does not exist";
110 key = (string)
"BES.Catalog." + n +
".Exclude";
113 vector<string>::iterator ei = vals.begin();
114 vector<string>::iterator ee = vals.end();
115 for (; ei != ee; ei++) {
116 string e_str = (*ei);
117 if (!e_str.empty() && e_str !=
";")
BESUtil::explode(
';', e_str, d_exclude);
120 key = (string)
"BES.Catalog." + n +
".Include";
123 vector<string>::iterator ii = vals.begin();
124 vector<string>::iterator ie = vals.end();
125 for (; ii != ie; ii++) {
126 string i_str = (*ii);
127 if (!i_str.empty() && i_str !=
";")
BESUtil::explode(
';', i_str, d_include);
130 key =
"BES.Catalog." + n +
".TypeMatch";
131 list<string> match_list;
134 if (strict && (!found || vals.size() == 0)) {
135 string s = key +
" not defined in key file";
138 vector<string>::iterator vi = vals.begin();
139 vector<string>::iterator ve = vals.end();
140 for (; vi != ve; vi++) {
144 list<string>::iterator mli = match_list.begin();
145 list<string>::iterator mle = match_list.end();
146 for (; mli != mle; mli++) {
147 if (!((*mli).empty()) && *(mli) !=
";") {
150 if (amatch.size() != 2) {
151 string s = (string)
"Catalog type match malformed, " +
"looking for type:regexp;[type:regexp;]";
154 list<string>::iterator ami = amatch.begin();
155 handler_regex newval;
156 newval.handler = (*ami);
158 newval.regex = (*ami);
159 d_match_list.push_back(newval);
163 key = (string)
"BES.Catalog." + n +
".FollowSymLinks";
167 if (s_str ==
"yes" || s_str ==
"on" || s_str ==
"true") {
168 d_follow_syms =
true;
185 bool toInclude =
false;
190 if (d_include.size() == 0) {
194 list<string>::const_iterator i_iter = d_include.begin();
195 list<string>::const_iterator i_end = d_include.end();
196 for (; i_iter != i_end; i_iter++) {
197 string reg = *i_iter;
203 if (reg_expr.
match(inQuestion.c_str(), inQuestion.length())
204 ==
static_cast<int>(inQuestion.length())) {
209 string serr = (string)
"Unable to get catalog information, "
210 +
"malformed Catalog Include parameter " +
"in bes configuration file around " + reg +
": "
218 if (toInclude ==
true) {
236 list<string>::const_iterator e_iter = d_exclude.begin();
237 list<string>::const_iterator e_end = d_exclude.end();
238 for (; e_iter != e_end; e_iter++) {
239 string reg = *e_iter;
243 if (reg_expr.
match(inQuestion.c_str(), inQuestion.length()) ==
static_cast<int>(inQuestion.length())) {
248 string serr = (string)
"Unable to get catalog information, " +
"malformed Catalog Exclude parameter "
249 +
"in bes configuration file around " + reg +
": " + e.
get_message();
262 BESCatalogUtils::match_citer BESCatalogUtils::match_list_begin()
const
264 return d_match_list.begin();
272 BESCatalogUtils::match_citer BESCatalogUtils::match_list_end()
const
274 return d_match_list.end();
290 unsigned int cnt = 0;
293 int statret = stat(fullnode.c_str(), &cbuf);
295 if (errno == ENOENT) {
296 char *s_err = strerror(errno);
297 throw BESNotFoundError((s_err) ?
string(s_err) :
string(
"Node ") + use_node +
" does not exist", __FILE__,
302 char *s_err = strerror(errno);
303 throw BESNotFoundError((s_err) ?
string(s_err) :
string(
"Access denied for node ") + use_node, __FILE__,
309 while ((dit = readdir(dip)) != NULL) {
310 string dirEntry = dit->d_name;
311 if (dirEntry ==
"." || dirEntry ==
"..") {
315 string fullPath = fullnode +
"/" + dirEntry;
318 if (follow_sym_links() ==
false) {
320 (void) lstat(fullPath.c_str(), &lbuf);
321 if (S_ISLNK(lbuf.st_mode))
330 statret = stat(fullPath.c_str(), &buf);
331 if (statret == 0 && S_ISDIR(buf.st_mode)) {
332 if (
exclude(dirEntry) ==
false) {
335 bes_add_stat_info(curr_entry, buf);
337 entry->add_entry(curr_entry);
343 curr_entry->add_entry(blank_entry);
346 else if (statret == 0 && S_ISREG(buf.st_mode)) {
347 if (!dirs_only &&
include(dirEntry)) {
349 bes_add_stat_info(curr_entry, buf);
351 list<string> services;
353 isData(fullPath, d_name, services);
354 curr_entry->set_service_list(services);
356 bes_add_stat_info(curr_entry, buf);
358 entry->add_entry(curr_entry);
372 map<string, string> props;
373 if (entry->get_catalog() == defcatname) {
374 props[
"name"] = entry->get_name();
377 string name = entry->get_catalog() +
"/";
378 if (entry->get_name() !=
"/") {
379 name = name + entry->get_name();
381 props[
"name"] = name;
383 props[
"catalog"] = entry->get_catalog();
384 props[
"size"] = entry->get_size();
385 props[
"lastModified"] = entry->get_mod_date() +
"T" + entry->get_mod_time();
386 if (entry->is_collection()) {
387 props[
"node"] =
"true";
389 strm << entry->get_count();
390 props[
"count"] = strm.str();
393 props[
"node"] =
"false";
395 info->begin_tag(
"dataset", &props);
397 list<string> services = entry->get_service_list();
398 if (services.size()) {
399 list<string>::const_iterator si = services.begin();
400 list<string>::const_iterator se = services.end();
401 for (; si != se; si++) {
402 info->add_tag(
"serviceRef", (*si));
424 for (BESCatalogUtils::match_citer i = match_list_begin(), e = match_list_end(); i != e; ++i) {
426 if (expr.
match(item.c_str(), item.length()) == (
int)item.length()) {
449 for (BESCatalogUtils::match_citer i = match_list_begin(), e = match_list_end(); i != e; ++i) {
451 if (expr.
match(item.c_str(), item.length()) == (
int)item.length()) {
466 void BESCatalogUtils::bes_add_stat_info(
BESCatalogEntry *entry,
const string &fullnode)
469 int statret = stat(fullnode.c_str(), &cbuf);
471 bes_add_stat_info(entry, cbuf);
475 void BESCatalogUtils::bes_add_stat_info(
BESCatalogEntry *entry,
struct stat &buf)
477 off_t sz = buf.st_size;
482 time_t mod = buf.st_mtime;
483 struct tm *stm = gmtime(&mod);
485 strftime(mdate, 64,
"%Y-%m-%d", stm);
487 strftime(mtime, 64,
"%T", stm);
491 entry->set_mod_date(sdt.str());
495 entry->set_mod_time(stt.str());
498 bool BESCatalogUtils::isData(
const string &inQuestion,
const string &catalog, list<string> &services)
501 if (!store)
return false;
503 return store->
isData(inQuestion, services);
508 strm << BESIndent::LMarg <<
"BESCatalogUtils::dump - (" << (
void *)
this <<
")" << endl;
511 strm << BESIndent::LMarg <<
"root directory: " << d_root_dir << endl;
513 if (d_include.size()) {
514 strm << BESIndent::LMarg <<
"include list:" << endl;
516 list<string>::const_iterator i_iter = d_include.begin();
517 list<string>::const_iterator i_end = d_include.end();
518 for (; i_iter != i_end; i_iter++) {
519 if (!(*i_iter).empty()) {
520 strm << BESIndent::LMarg << *i_iter << endl;
523 BESIndent::UnIndent();
526 strm << BESIndent::LMarg <<
"include list: empty" << endl;
529 if (d_exclude.size()) {
530 strm << BESIndent::LMarg <<
"exclude list:" << endl;
532 list<string>::const_iterator e_iter = d_exclude.begin();
533 list<string>::const_iterator e_end = d_exclude.end();
534 for (; e_iter != e_end; e_iter++) {
535 if (!(*e_iter).empty()) {
536 strm << BESIndent::LMarg << *e_iter << endl;
539 BESIndent::UnIndent();
542 strm << BESIndent::LMarg <<
"exclude list: empty" << endl;
545 if (d_match_list.size()) {
546 strm << BESIndent::LMarg <<
"type matches:" << endl;
548 BESCatalogUtils::match_citer i = d_match_list.begin();
549 BESCatalogUtils::match_citer ie = d_match_list.end();
550 for (; i != ie; i++) {
551 handler_regex match = (*i);
552 strm << BESIndent::LMarg << match.handler <<
" : " << match.regex << endl;
554 BESIndent::UnIndent();
557 strm << BESIndent::LMarg <<
" type matches: empty" << endl;
561 strm << BESIndent::LMarg <<
" follow symbolic links: on" << endl;
564 strm << BESIndent::LMarg <<
" follow symbolic links: off" << endl;
567 BESIndent::UnIndent();
572 BESCatalogUtils::Utils(
const string &cat_name)
577 BESCatalogUtils::_instances[cat_name] = utils;
586 void BESCatalogUtils::delete_all_catalogs()
588 map<string, BESCatalogUtils*>::iterator i = BESCatalogUtils::_instances.begin();
589 map<string, BESCatalogUtils*>::iterator e = BESCatalogUtils::_instances.end();
591 delete (*i++).second;