OPeNDAP Hyrax Back End Server (BES)  Updated for version 3.8.3
BESCatalogUtils.cc
Go to the documentation of this file.
1 // BESCatalogUtils.cc
2 
3 // This file is part of bes, A C++ back-end server implementation framework
4 // for the OPeNDAP Data Access Protocol.
5 
6 // Copyright (c) 2004-2009 University Corporation for Atmospheric Research
7 // Author: Patrick West <pwest@ucar.edu> and Jose Garcia <jgarcia@ucar.edu>
8 //
9 // This library is free software; you can redistribute it and/or
10 // modify it under the terms of the GNU Lesser General Public
11 // License as published by the Free Software Foundation; either
12 // version 2.1 of the License, or (at your option) any later version.
13 //
14 // This library is distributed in the hope that it will be useful,
15 // but WITHOUT ANY WARRANTY; without even the implied warranty of
16 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 // Lesser General Public License for more details.
18 //
19 // You should have received a copy of the GNU Lesser General Public
20 // License along with this library; if not, write to the Free Software
21 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
22 //
23 // You can contact University Corporation for Atmospheric Research at
24 // 3080 Center Green Drive, Boulder, CO 80301
25 
26 // (c) COPYRIGHT University Corporation for Atmospheric Research 2004-2005
27 // Please read the full copyright statement in the file COPYRIGHT_UCAR.
28 //
29 // Authors:
30 // pwest Patrick West <pwest@ucar.edu>
31 // jgarcia Jose Garcia <jgarcia@ucar.edu>
32 
33 #include "config.h"
34 
35 #include <sys/types.h>
36 #include <sys/stat.h>
37 #include <dirent.h>
38 
39 #include <cerrno>
40 #include <iostream>
41 #include <sstream>
42 #include <list>
43 #include <cstring>
44 
45 using std::cout;
46 using std::endl;
47 using std::ostringstream;
48 using std::list;
49 
50 #include "BESCatalogUtils.h"
51 #include "BESCatalogList.h"
52 #include "TheBESKeys.h"
53 #include "BESInternalError.h"
54 #include "BESSyntaxUserError.h"
55 #include "BESNotFoundError.h"
56 #include "BESRegex.h"
57 #include "BESUtil.h"
58 //#include "BESDapNames.h"
59 #include "BESInfo.h"
61 #include "BESContainerStorage.h"
62 #include "BESCatalogEntry.h"
63 
64 map<string, BESCatalogUtils *> BESCatalogUtils::_instances;
65 
66 BESCatalogUtils::BESCatalogUtils(const string &n) :
67  _name(n), _follow_syms(false) {
68  string key = "BES.Catalog." + n + ".RootDirectory";
69  bool found = false;
70  TheBESKeys::TheKeys()->get_value(key, _root_dir, found);
71  if (!found || _root_dir == "") {
72  string s = key + " not defined in BES configuration file";
73  throw BESSyntaxUserError(s, __FILE__, __LINE__);
74  }
75  DIR *dip = opendir(_root_dir.c_str());
76  if (dip == NULL) {
77  string serr = "BESCatalogDirectory - root directory " + _root_dir
78  + " does not exist";
79  throw BESNotFoundError(serr, __FILE__, __LINE__);
80  }
81  closedir(dip);
82 
83  found = false;
84  key = (string) "BES.Catalog." + n + ".Exclude";
85  vector<string> vals;
86  TheBESKeys::TheKeys()->get_values(key, vals, found);
87  vector<string>::iterator ei = vals.begin();
88  vector<string>::iterator ee = vals.end();
89  for (; ei != ee; ei++) {
90  string e_str = (*ei);
91  if (!e_str.empty() && e_str != ";")
92  BESUtil::explode(';', e_str, _exclude);
93  }
94 
95  key = (string) "BES.Catalog." + n + ".Include";
96  vals.clear();
97  TheBESKeys::TheKeys()->get_values(key, vals, found);
98  vector<string>::iterator ii = vals.begin();
99  vector<string>::iterator ie = vals.end();
100  for (; ii != ie; ii++) {
101  string i_str = (*ii);
102  if (!i_str.empty() && i_str != ";")
103  BESUtil::explode(';', i_str, _include);
104  }
105 
106  key = "BES.Catalog." + n + ".TypeMatch";
107  list<string> match_list;
108  vals.clear();
109  TheBESKeys::TheKeys()->get_values(key, vals, found);
110  if (!found || vals.size() == 0) {
111  string s = key + " not defined in key file";
112  throw BESInternalError(s, __FILE__, __LINE__);
113  }
114  vector<string>::iterator vi = vals.begin();
115  vector<string>::iterator ve = vals.end();
116  for (; vi != ve; vi++) {
117  BESUtil::explode(';', (*vi), match_list);
118  }
119 
120  list<string>::iterator mli = match_list.begin();
121  list<string>::iterator mle = match_list.end();
122  for (; mli != mle; mli++) {
123  if (!((*mli).empty()) && *(mli) != ";") {
124  list<string> amatch;
125  BESUtil::explode(':', (*mli), amatch);
126  if (amatch.size() != 2) {
127  string s = (string) "Catalog type match malformed, "
128  + "looking for type:regexp;[type:regexp;]";
129  throw BESInternalError(s, __FILE__, __LINE__);
130  }
131  list<string>::iterator ami = amatch.begin();
132  type_reg newval;
133  newval.type = (*ami);
134  ami++;
135  newval.reg = (*ami);
136  _match_list.push_back(newval);
137  }
138  }
139 
140  key = (string) "BES.Catalog." + n + ".FollowSymLinks";
141  string s_str;
142  TheBESKeys::TheKeys()->get_value(key, s_str, found);
143  s_str = BESUtil::lowercase(s_str);
144  if (s_str == "yes" || s_str == "on" || s_str == "true") {
145  _follow_syms = true;
146  }
147 }
148 
149 bool BESCatalogUtils::include(const string &inQuestion) const {
150  bool toInclude = false;
151 
152  // First check the file against the include list. If the file should be
153  // included then check the exclude list to see if there are exceptions
154  // to the include list.
155  if (_include.size() == 0) {
156  toInclude = true;
157  } else {
158  list<string>::const_iterator i_iter = _include.begin();
159  list<string>::const_iterator i_end = _include.end();
160  for (; i_iter != i_end; i_iter++) {
161  string reg = *i_iter;
162  if (!reg.empty()) {
163  try {
164  // must match exactly, meaing result is = to length of string
165  // in question
166  BESRegex reg_expr(reg.c_str());
167  if (reg_expr.match(inQuestion.c_str(), inQuestion.length())
168  == static_cast<int> (inQuestion.length())) {
169  toInclude = true;
170  }
171  } catch (BESError &e) {
172  string serr =
173  (string) "Unable to get catalog information, "
174  + "malformed Catalog Include parameter "
175  + "in bes configuration file around " + reg
176  + ": " + e.get_message();
177  throw BESInternalError(serr, __FILE__, __LINE__);
178  }
179  }
180  }
181  }
182 
183  if (toInclude == true) {
184  if (exclude(inQuestion)) {
185  toInclude = false;
186  }
187  }
188 
189  return toInclude;
190 }
191 
192 bool BESCatalogUtils::exclude(const string &inQuestion) const {
193  list<string>::const_iterator e_iter = _exclude.begin();
194  list<string>::const_iterator e_end = _exclude.end();
195  for (; e_iter != e_end; e_iter++) {
196  string reg = *e_iter;
197  if (!reg.empty()) {
198  try {
199  BESRegex reg_expr(reg.c_str());
200  if (reg_expr.match(inQuestion.c_str(), inQuestion.length())
201  == static_cast<int> (inQuestion.length())) {
202  return true;
203  }
204  } catch (BESError &e) {
205  string serr = (string) "Unable to get catalog information, "
206  + "malformed Catalog Exclude parameter "
207  + "in bes configuration file around " + reg + ": "
208  + e.get_message();
209  throw BESInternalError(serr, __FILE__, __LINE__);
210  }
211  }
212  }
213  return false;
214 }
215 
217  return _match_list.begin();
218 }
219 
221  return _match_list.end();
222 }
223 
224 unsigned int BESCatalogUtils::get_entries(DIR *dip, const string &fullnode,
225  const string &use_node, const string &/*coi*/, BESCatalogEntry *entry,
226  bool dirs_only) {
227  unsigned int cnt = 0;
228  struct stat cbuf;
229  int statret = stat(fullnode.c_str(), &cbuf);
230  int my_errno = errno;
231  if (statret == 0) {
232  struct dirent *dit;
233  struct stat buf;
234  struct stat lbuf;
235 
236  while ((dit = readdir(dip)) != NULL) {
237  string dirEntry = dit->d_name;
238  if (dirEntry != "." && dirEntry != "..") {
239  string fullPath = fullnode + "/" + dirEntry;
240 
241  // if follow_sym_links is true then continue with
242  // the checking. If false, first see if the entry is
243  // a symbolic link. If it is, do not include in the
244  // listing for this node. If not, then continue
245  // checking the entry.
246  bool continue_checking = true;
247  if (follow_sym_links() == false) {
248 #if 0
249  int lstatret = lstat( fullPath.c_str(), &lbuf );
250 #endif
251  (void) lstat(fullPath.c_str(), &lbuf);
252  if (S_ISLNK( lbuf.st_mode )) {
253  continue_checking = false;
254  }
255  }
256 
257  if (continue_checking) {
258  // look at the mode and determine if this is a
259  // directory or a regular file. If it is not
260  // accessible, the stat fails, is not a directory
261  // or regular file, then simply do not include it.
262  statret = stat(fullPath.c_str(), &buf);
263  if (statret == 0 && S_ISDIR( buf.st_mode )) {
264  if (exclude(dirEntry) == false) {
265  BESCatalogEntry *curr_entry = new BESCatalogEntry(
266  dirEntry, entry->get_catalog());
267 
268  bes_get_stat_info(curr_entry, buf);
269 
270  entry->add_entry(curr_entry);
271 
272  // we don't go further then this, so we need
273  // to add a blank node here so that we know
274  // it's a node (collection)
275  BESCatalogEntry *blank_entry = new BESCatalogEntry(
276  ".blank", entry->get_catalog());
277  curr_entry->add_entry(blank_entry);
278  }
279  } else if (statret == 0 && S_ISREG( buf.st_mode )) {
280  if (!dirs_only && include(dirEntry)) {
281  BESCatalogEntry *curr_entry = new BESCatalogEntry(
282  dirEntry, entry->get_catalog());
283  bes_get_stat_info(curr_entry, buf);
284 
285  list<string> services;
286  isData(fullPath, _name, services);
287  curr_entry->set_service_list(services);
288 
289  bes_get_stat_info(curr_entry, buf);
290 
291  entry->add_entry(curr_entry);
292  }
293  }
294  }
295  }
296  }
297  } else {
298  // ENOENT means that the path or part of the path does not exist
299  if (my_errno == ENOENT) {
300  string error = "Node " + use_node + " does not exist";
301  char *s_err = strerror(my_errno);
302  if (s_err) {
303  error = s_err;
304  }
305  throw BESNotFoundError(error, __FILE__, __LINE__);
306  }
307  // any other error means that access is denied for some reason
308  else {
309  string error = "Access denied for node " + use_node;
310  char *s_err = strerror(my_errno);
311  if (s_err) {
312  error = error + s_err;
313  }
314  throw BESNotFoundError(error, __FILE__, __LINE__);
315  }
316  }
317  return cnt;
318 }
319 
321  string defcatname = BESCatalogList::TheCatalogList()->default_catalog();
322 
323  // start with the external entry
324  map<string, string> props;
325  if (entry->get_catalog() == defcatname) {
326  props["name"] = entry->get_name();
327  } else {
328  string name = entry->get_catalog() + "/";
329  if (entry->get_name() != "/") {
330  name = name + entry->get_name();
331  }
332  props["name"] = name;
333  }
334  props["catalog"] = entry->get_catalog();
335  props["size"] = entry->get_size();
336  props["lastModified"] = entry->get_mod_date() + "T" + entry->get_mod_time();
337  if (entry->is_collection()) {
338  props["node"] = "true";
339  ostringstream strm;
340  strm << entry->get_count();
341  props["count"] = strm.str();
342  } else {
343  props["node"] = "false";
344  }
345  info->begin_tag("dataset", &props);
346 
347  list<string> services = entry->get_service_list();
348  if (services.size()) {
349  list<string>::const_iterator si = services.begin();
350  list<string>::const_iterator se = services.end();
351  for (; si != se; si++) {
352  info->add_tag("serviceRef", (*si));
353  }
354  }
355 }
356 
358  const string &fullnode) {
359  struct stat cbuf;
360  int statret = stat(fullnode.c_str(), &cbuf);
361  if (statret == 0) {
362  bes_get_stat_info(entry, cbuf);
363  }
364 }
365 
366 void BESCatalogUtils::bes_get_stat_info(BESCatalogEntry *entry,
367  struct stat &buf) {
368  off_t sz = buf.st_size;
369  entry->set_size(sz);
370 
371  // %T = %H:%M:%S
372  // %F = %Y-%m-%d
373  time_t mod = buf.st_mtime;
374  struct tm *stm = gmtime(&mod);
375  char mdate[64];
376  strftime(mdate, 64, "%Y-%m-%d", stm);
377  char mtime[64];
378  strftime(mtime, 64, "%T", stm);
379 
380  ostringstream sdt;
381  sdt << mdate;
382  entry->set_mod_date(sdt.str());
383 
384  ostringstream stt;
385  stt << mtime;
386  entry->set_mod_time(stt.str());
387 }
388 
389 bool BESCatalogUtils::isData(const string &inQuestion, const string &catalog,
390  list<string> &services) {
391  BESContainerStorage *store =
393  if (!store)
394  return false;
395 
396  return store->isData(inQuestion, services);
397 }
398 
399 void BESCatalogUtils::dump(ostream &strm) const {
400  strm << BESIndent::LMarg << "BESCatalogUtils::dump - (" << (void *) this
401  << ")" << endl;
403 
404  strm << BESIndent::LMarg << "root directory: " << _root_dir << endl;
405 
406  if (_include.size()) {
407  strm << BESIndent::LMarg << "include list:" << endl;
409  list<string>::const_iterator i_iter = _include.begin();
410  list<string>::const_iterator i_end = _include.end();
411  for (; i_iter != i_end; i_iter++) {
412  if (!(*i_iter).empty()) {
413  strm << BESIndent::LMarg << *i_iter << endl;
414  }
415  }
417  } else {
418  strm << BESIndent::LMarg << "include list: empty" << endl;
419  }
420 
421  if (_exclude.size()) {
422  strm << BESIndent::LMarg << "exclude list:" << endl;
424  list<string>::const_iterator e_iter = _exclude.begin();
425  list<string>::const_iterator e_end = _exclude.end();
426  for (; e_iter != e_end; e_iter++) {
427  if (!(*e_iter).empty()) {
428  strm << BESIndent::LMarg << *e_iter << endl;
429  }
430  }
432  } else {
433  strm << BESIndent::LMarg << "exclude list: empty" << endl;
434  }
435 
436  if (_match_list.size()) {
437  strm << BESIndent::LMarg << "type matches:" << endl;
439  BESCatalogUtils::match_citer i = _match_list.begin();
440  BESCatalogUtils::match_citer ie = _match_list.end();
441  for (; i != ie; i++) {
442  type_reg match = (*i);
443  strm << BESIndent::LMarg << match.type << " : " << match.reg
444  << endl;
445  }
447  } else {
448  strm << BESIndent::LMarg << " type matches: empty" << endl;
449  }
450 
451  if (_follow_syms) {
452  strm << BESIndent::LMarg << " follow symbolic links: on" << endl;
453  } else {
454  strm << BESIndent::LMarg << " follow symbolic links: off" << endl;
455  }
456 
458 }
459 
461 BESCatalogUtils::Utils(const string &cat_name) {
462  BESCatalogUtils *utils = BESCatalogUtils::_instances[cat_name];
463  if (!utils) {
464  utils = new BESCatalogUtils(cat_name);
465  BESCatalogUtils::_instances[cat_name] = utils;
466  }
467  return utils;
468 }
469 
470 // Added 12/24/12
472 {
473  map<string, BESCatalogUtils*>::iterator i = BESCatalogUtils::_instances.begin();
474  map<string, BESCatalogUtils*>::iterator e = BESCatalogUtils::_instances.end();
475  while (i != e) {
476  delete (*i++).second;
477  }
478 }
479 
480 
virtual list< string > get_service_list()
error thrown if the resource requested cannot be found
provides persistent storage for data storage information represented by a container.
BESCatalogUtils::match_citer match_list_begin() const
BESCatalogUtils::match_citer match_list_end() const
virtual string get_catalog()
exception thrown if inernal error encountered
virtual unsigned int get_count()
virtual BESContainerStorage * find_persistence(const string &persist_name)
find the persistence store with the given name
static string lowercase(const string &s)
Convert a string to all lower case.
Definition: BESUtil.cc:179
virtual void add_tag(const string &tag_name, const string &tag_data, map< string, string > *attrs=0)=0
a C++ interface to POSIX regular expression functions.
Definition: BESRegex.h:41
virtual void set_mod_date(const string &mod_date)
static void Indent()
Definition: BESIndent.cc:38
error thrown if there is a user syntax error in the request or any other user error ...
virtual void set_service_list(list< string > &slist)
vector< type_reg >::const_iterator match_citer
virtual void set_mod_time(const string &mod_time)
virtual string get_message()
get the error message for this exception
Definition: BESError.h:94
virtual bool isData(const string &inQuestion, list< string > &provides)=0
determine if the given container is data and what services are available for it
virtual void add_entry(BESCatalogEntry *entry)
informational response object
Definition: BESInfo.h:68
Abstract exception class for the BES with basic string message.
Definition: BESError.h:51
static void bes_add_stat_info(BESCatalogEntry *entry, const string &fullnode)
virtual string default_catalog()
virtual string get_name()
virtual void begin_tag(const string &tag_name, map< string, string > *attrs=0)
Definition: BESInfo.cc:142
static ostream & LMarg(ostream &strm)
Definition: BESIndent.cc:73
bool follow_sym_links() const
static void explode(char delim, const string &str, list< string > &values)
explode a string into an array given a delimiter
Definition: BESUtil.cc:457
virtual void dump(ostream &strm) const
dump the contents of this object to the specified ostream
static bool isData(const string &inQuestion, const string &catalog, list< string > &services)
virtual string get_mod_time()
static void display_entry(BESCatalogEntry *entry, BESInfo *info)
void get_value(const string &s, string &val, bool &found)
Retrieve the value of a given key, if set.
Definition: BESKeys.cc:453
static BESContainerStorageList * TheList()
virtual void set_size(off_t size)
static void delete_all_catalogs()
virtual bool include(const string &inQuestion) const
void get_values(const string &s, vector< string > &vals, bool &found)
Retrieve the values of a given key, if set.
Definition: BESKeys.cc:488
virtual string get_size()
virtual bool exclude(const string &inQuestion) const
static void UnIndent()
Definition: BESIndent.cc:44
static BESCatalogUtils * Utils(const string &name)
virtual unsigned int get_entries(DIR *dip, const string &fullnode, const string &use_node, const string &coi, BESCatalogEntry *entry, bool dirs_only)
virtual bool is_collection()
static BESCatalogList * TheCatalogList()
returns the singleton BESCatalogList instance.
static BESKeys * TheKeys()
Definition: TheBESKeys.cc:48
virtual string get_mod_date()