bes  Updated for version 3.17.4
BESFileLockingCache.cc
1 // This file was originally part of bes, A C++ back-end server
2 // implementation framework for the OPeNDAP Data Access Protocol.
3 // Copied to libdap. This is used to cache responses built from
4 // functional CE expressions.
5 
6 // Moved back to the BES. 6/11/13 jhrg
7 
8 // Copyright (c) 2012 OPeNDAP, Inc
9 // Author: James Gallagher <jgallagher@opendap.org>
10 // Patrick West <pwest@ucar.edu> and Jose Garcia <jgarcia@ucar.edu>
11 //
12 // This library is free software; you can redistribute it and/or
13 // modify it under the terms of the GNU Lesser General Public
14 // License as published by the Free Software Foundation; either
15 // version 2.1 of the License, or (at your option) any later version.
16 //
17 // This library is distributed in the hope that it will be useful,
18 // but WITHOUT ANY WARRANTY; without even the implied warranty of
19 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20 // Lesser General Public License for more details.
21 //
22 // You should have received a copy of the GNU Lesser General Public
23 // License along with this library; if not, write to the Free Software
24 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
25 //
26 // You can contact University Corporation for Atmospheric Research at
27 // 3080 Center Green Drive, Boulder, CO 80301
28 
29 #include "config.h"
30 
31 #include <sys/file.h>
32 #include <sys/stat.h>
33 #include <unistd.h>
34 #include <dirent.h>
35 #include <fcntl.h>
36 
37 #ifdef HAVE_STDLIB_H
38 #include <stdlib.h>
39 #endif
40 
41 #include <string>
42 #include <sstream>
43 #include <vector>
44 #include <cstring>
45 #include <cerrno>
46 
47 #include "BESInternalError.h"
48 
49 #include "BESUtil.h"
50 #include "BESDebug.h"
51 #include "BESLog.h"
52 
53 #include "BESFileLockingCache.h"
54 
55 using namespace std;
56 
57 // conversion factor
58 static const unsigned long long BYTES_PER_MEG = 1048576ULL;
59 
60 // Max cache size in megs, so we can check the user input and warn.
61 // 2^64 / 2^20 == 2^44
62 static const unsigned long long MAX_CACHE_SIZE_IN_MEGABYTES = (1ULL << 44);
63 
77 BESFileLockingCache::BESFileLockingCache(const string &cache_dir, const string &prefix, unsigned long long size) :
78  d_cache_dir(cache_dir), d_prefix(prefix), d_max_cache_size_in_bytes(size), d_target_size(0), d_cache_info(""),
79  d_cache_info_fd(-1)
80 {
81  m_initialize_cache_info();
82 }
83 
84 void BESFileLockingCache::initialize(const string &cache_dir, const string &prefix, unsigned long long size)
85 {
86  d_cache_dir = cache_dir;
87  d_prefix = prefix;
89 
90  m_initialize_cache_info();
91 }
92 
93 static inline string get_errno()
94 {
95  char *s_err = strerror(errno);
96  if (s_err)
97  return s_err;
98  else
99  return "Unknown error.";
100 }
101 
102 // Build a lock of a certain type.
103 static inline struct flock *lock(int type)
104 {
105  static struct flock lock;
106  lock.l_type = type;
107  lock.l_whence = SEEK_SET;
108  lock.l_start = 0;
109  lock.l_len = 0;
110  lock.l_pid = getpid();
111 
112  return &lock;
113 }
114 
115 inline void BESFileLockingCache::m_record_descriptor(const string &file, int fd)
116 {
117  BESDEBUG("cache",
118  "BESFileLockingCache::m_record_descriptor() - Recording descriptor: " << file << ", " << fd << endl);
119  d_locks.insert(std::pair<string, int>(file, fd));
120 }
121 
122 inline int BESFileLockingCache::m_get_descriptor(const string &file)
123 {
124  BESDEBUG("cache", "BESFileLockingCache::m_get_descriptor(): d_locks size: " << d_locks.size() << endl);
125  FilesAndLockDescriptors::iterator i = d_locks.find(file);
126  if (i == d_locks.end()) return -1;
127 
128  int fd = i->second;
129  BESDEBUG("cache",
130  "BESFileLockingCache::m_get_descriptor(): Found file descriptor [" << fd << "] for file: " << file << endl);
131  d_locks.erase(i);
132  return fd;
133 }
134 
135 string lockStatus(const int fd)
136 {
137  struct flock isLocked, lock_query;
138 
139  isLocked.l_type = F_WRLCK; /* Test for any lock on any part of a file. */
140  isLocked.l_start = 0;
141  isLocked.l_whence = SEEK_SET;
142  isLocked.l_len = 0;
143  lock_query = isLocked;
144 
145  int ret = fcntl(fd, F_GETLK, &lock_query);
146 
147  stringstream ss;
148  ss << endl;
149  if (ret == -1) {
150  ss << "ERROR! fnctl(" << fd << ",F_GETLK, &lock) returned: " << ret << " errno[" << errno << "]: "
151  << strerror(errno) << endl;
152  }
153  else {
154  ss << "SUCCESS. fnctl(" << fd << ",F_GETLK, &lock) returned: " << ret << endl;
155  }
156 
157  ss << "lock_info.l_len: " << lock_query.l_len << endl;
158  ss << "lock_info.l_pid: " << lock_query.l_pid << endl;
159  ss << "lock_info.l_start: " << lock_query.l_start << endl;
160 
161  string type;
162  switch (lock_query.l_type) {
163  case F_RDLCK:
164  type = "F_RDLCK";
165  break;
166  case F_WRLCK:
167  type = "F_WRLCK";
168  break;
169  case F_UNLCK:
170  type = "F_UNLCK";
171  break;
172 
173  }
174 
175  ss << "lock_info.l_type: " << type << endl;
176  ss << "lock_info.l_whence: " << lock_query.l_whence << endl;
177 
178  return ss.str();
179 }
180 
186 static void unlock(int fd)
187 {
188  if (fcntl(fd, F_SETLK, lock(F_UNLCK)) == -1) {
189  throw BESInternalError("An error occurred trying to unlock the file: " + get_errno(), __FILE__, __LINE__);
190  }
191 
192  BESDEBUG("cache", "BESFileLockingCache::unlock() - lock status: " << lockStatus(fd) << endl);
193 
194  if (close(fd) == -1) throw BESInternalError("Could not close the (just) unlocked file.", __FILE__, __LINE__);
195 
196  BESDEBUG("cache", "BESFileLockingCache::unlock() - File Closed. fd: " << fd << endl);
197 }
198 
211 static bool getSharedLock(const string &file_name, int &ref_fd)
212 {
213  BESDEBUG("cache", "getSharedLock(): Acquiring cache read lock for " << file_name <<endl);
214 
215  int fd;
216  if ((fd = open(file_name.c_str(), O_RDONLY)) < 0) {
217  switch (errno) {
218  case ENOENT:
219  return false;
220 
221  default:
222  throw BESInternalError(get_errno(), __FILE__, __LINE__);
223  }
224  }
225 
226  struct flock *l = lock(F_RDLCK);
227  if (fcntl(fd, F_SETLKW, l) == -1) {
228  close(fd);
229  ostringstream oss;
230  oss << "cache process: " << l->l_pid << " triggered a locking error: " << get_errno();
231  throw BESInternalError(oss.str(), __FILE__, __LINE__);
232  }
233 
234  BESDEBUG("cache", "getSharedLock(): SUCCESS Read Lock Acquired For " << file_name <<endl);
235 
236  // Success
237  ref_fd = fd;
238  return true;
239 }
240 
253 bool BESFileLockingCache::getExclusiveLock(string file_name, int &ref_fd)
254 {
255  BESDEBUG("cache", "BESFileLockingCache::getExclusiveLock() - " << file_name <<endl);
256 
257  int fd;
258  if ((fd = open(file_name.c_str(), O_RDWR)) < 0) {
259  switch (errno) {
260  case ENOENT:
261  return false;
262 
263  default:
264  BESDEBUG("cache",
265  "BESFileLockingCache::getExclusiveLock() - FAILED to open file. name: " << file_name << " name_length: " << file_name.length( )<<endl);
266  throw BESInternalError(get_errno(), __FILE__, __LINE__);
267  }
268  }
269 
270  struct flock *l = lock(F_WRLCK);
271  if (fcntl(fd, F_SETLKW, l) == -1) {
272  close(fd);
273  ostringstream oss;
274  oss << "cache process: " << l->l_pid << " triggered a locking error: " << get_errno();
275  throw BESInternalError(oss.str(), __FILE__, __LINE__);
276  }
277 
278  BESDEBUG("cache", "BESFileLockingCache::getExclusiveLock() - exit: " << file_name <<endl);
279 
280  // Success
281  ref_fd = fd;
282  return true;
283 }
284 
296 static bool getExclusiveLockNB(string file_name, int &ref_fd)
297 {
298  BESDEBUG("cache", "getExclusiveLock_nonblocking: " << file_name <<endl);
299 
300  int fd;
301  if ((fd = open(file_name.c_str(), O_RDWR)) < 0) {
302  switch (errno) {
303  case ENOENT:
304  return false;
305 
306  default:
307  throw BESInternalError(get_errno(), __FILE__, __LINE__);
308  }
309  }
310 
311  struct flock *l = lock(F_WRLCK);
312  if (fcntl(fd, F_SETLK, l) == -1) {
313  switch (errno) {
314  case EAGAIN:
315  BESDEBUG("cache",
316  "getExclusiveLock_nonblocking exit (false): " << file_name << " by: " << l->l_pid << endl);
317  close(fd);
318  return false;
319 
320  default: {
321  close(fd);
322  ostringstream oss;
323  oss << "cache process: " << l->l_pid << " triggered a locking error: " << get_errno();
324  throw BESInternalError(oss.str(), __FILE__, __LINE__);
325  }
326  }
327  }
328 
329  BESDEBUG("cache", "getExclusiveLock_nonblocking exit (true): " << file_name <<endl);
330 
331  // Success
332  ref_fd = fd;
333  return true;
334 }
335 
349 static bool createLockedFile(string file_name, int &ref_fd)
350 {
351  BESDEBUG("cache", "createLockedFile() - filename: " << file_name <<endl);
352 
353  int fd;
354  if ((fd = open(file_name.c_str(), O_CREAT | O_EXCL | O_RDWR, 0666)) < 0) {
355  switch (errno) {
356  case EEXIST:
357  return false;
358 
359  default:
360  throw BESInternalError(file_name + ": " + get_errno(), __FILE__, __LINE__);
361  }
362  }
363 
364  struct flock *l = lock(F_WRLCK);
365  if (fcntl(fd, F_SETLKW, l) == -1) {
366  close(fd);
367  ostringstream oss;
368  oss << "cache process: " << l->l_pid << " triggered a locking error: " << get_errno();
369  throw BESInternalError(oss.str(), __FILE__, __LINE__);
370  }
371 
372  BESDEBUG("cache", "createLockedFile exit: " << file_name <<endl);
373 
374  // Success
375  ref_fd = fd;
376  return true;
377 }
378 
380 void BESFileLockingCache::m_check_ctor_params()
381 {
382  // TODO Should this really be a fatal error? What about just not
383  // using the cache in this case or writing out a warning message
384  // to the log. jhrg 10/23/15
385  //
386  // Yes, leave this as a fatal error and handle the case when cache_dir is
387  // empty in code that specializes this class. Those child classes are
388  // all singletons and their get_instance() methods need to return null
389  // when caching is turned off. You cannot do that here without throwing
390  // and we don't want to throw an exception for every call to a child's
391  // get_instance() method just because someone doesn't want to use a cache.
392  // jhrg 9/27/16
393  if (d_cache_dir.empty()) {
394  string err = "BESFileLockingCache::m_check_ctor_params() - The cache directory was not specified";
395  throw BESInternalError(err, __FILE__, __LINE__);
396  }
397 
398  int status = mkdir(d_cache_dir.c_str(), 0775);
399  // If there is an error and it's not that the dir already exists,
400  // throw an exception.
401  if (status == -1 && errno != EEXIST) {
402  string err = "The cache directory " + d_cache_dir + " could not be created: " + strerror(errno);
403  throw BESError(err, BES_SYNTAX_USER_ERROR, __FILE__, __LINE__);
404  }
405 
406  if (d_prefix.empty()) {
407  string err = "The cache file prefix was not specified, must not be empty";
408  throw BESError(err, BES_SYNTAX_USER_ERROR, __FILE__, __LINE__);
409  }
410 
411  if (d_max_cache_size_in_bytes <= 0) {
412  string err = "The cache size was not specified, must be greater than zero";
413  throw BESError(err, BES_SYNTAX_USER_ERROR, __FILE__, __LINE__);
414  }
415 
416  BESDEBUG("cache",
417  "BESFileLockingCache::m_check_ctor_params() - directory " << d_cache_dir << ", prefix " << d_prefix << ", max size " << d_max_cache_size_in_bytes << endl);
418 }
419 
421 void BESFileLockingCache::m_initialize_cache_info()
422 {
423  BESDEBUG("cache", "BESFileLockingCache::m_initialize_cache_info() - BEGIN" << endl);
424 
425  // The value set in configuration files, etc., is the size in megabytes. The private
426  // variable holds the size in bytes (converted below).
427  d_max_cache_size_in_bytes = min(d_max_cache_size_in_bytes, MAX_CACHE_SIZE_IN_MEGABYTES);
428  d_max_cache_size_in_bytes *= BYTES_PER_MEG;
429  d_target_size = d_max_cache_size_in_bytes * 0.8;
430  BESDEBUG("cache",
431  "BESFileLockingCache::m_initialize_cache_info() - d_max_cache_size_in_bytes: " << d_max_cache_size_in_bytes << " d_target_size: "<<d_target_size<< endl);
432 
433  m_check_ctor_params(); // Throws BESInternalError on error.
434 
435  d_cache_info = BESUtil::assemblePath(d_cache_dir, d_prefix + ".cache_control", true);
436 
437  BESDEBUG("cache", "BESFileLockingCache::m_initialize_cache_info() - d_cache_info: " << d_cache_info << endl);
438 
439  // See if we can create it. If so, that means it doesn't exist. So make it and
440  // set the cache initial size to zero.
441  if (createLockedFile(d_cache_info, d_cache_info_fd)) {
442  // initialize the cache size to zero
443  unsigned long long size = 0;
444  if (write(d_cache_info_fd, &size, sizeof(unsigned long long)) != sizeof(unsigned long long))
445  throw BESInternalError("Could not write size info to the cache info file `" + d_cache_info + "`", __FILE__,
446  __LINE__);
447 
448  // This leaves the d_cache_info_fd file descriptor open
449  unlock_cache();
450  }
451  else {
452  if ((d_cache_info_fd = open(d_cache_info.c_str(), O_RDWR)) == -1) {
453  throw BESInternalError(get_errno(), __FILE__, __LINE__);
454  }
455  }
456 
457  BESDEBUG("cache", "BESFileLockingCache::m_initialize_cache_info() - d_cache_info_fd: " << d_cache_info_fd << endl);
458  BESDEBUG("cache", "BESFileLockingCache::m_initialize_cache_info() - END" << endl);
459 }
460 
461 const string chars_excluded_from_filenames = "<>=,/()\\\"\':? []()$";
462 
477 string BESFileLockingCache::get_cache_file_name(const string &src, bool mangle)
478 {
479  // Old way of building String, retired 10/02/2015 - ndp
480  // Return d_cache_dir + "/" + d_prefix + BESFileLockingCache::DAP_CACHE_CHAR + target;
481  BESDEBUG("cache", __FUNCTION__ << " - src: '" << src << "' mangle: "<< mangle << endl);
482 
483  string target = getCacheFilePrefix() + src;
484 #if 0
485  // BUG This was causing the prefix to be treated as a path component.
486  // jhrg 9/27/16
487  target = BESUtil::assemblePath(getCacheFilePrefix(), src);
488 #endif
489 
490  if (mangle) {
491 #if 0
492  // This is not needed. jhrg 9/27/16
493  if (target.at(0) == '/') {
494  target = src.substr(1, target.length() - 1);
495  }
496 #endif
497  string::size_type pos = target.find_first_of(chars_excluded_from_filenames);
498  while (pos != string::npos) {
499  target.replace(pos, 1, "#", 1);
500  pos = target.find_first_of(chars_excluded_from_filenames);
501  }
502  }
503 
504  if (target.length() > 254) {
505  ostringstream msg;
506  msg << "Cache filename is longer than 254 characters (name length: ";
507  msg << target.length() << ", name: " << target;
508  throw BESInternalError(msg.str(), __FILE__, __LINE__);
509  }
510 
511  target = BESUtil::assemblePath(getCacheDirectory(), target, true);
512 
513  BESDEBUG("cache", __FUNCTION__ << " - target: '" << target << "'" << endl);
514 
515  return target;
516 }
517 
535 bool BESFileLockingCache::get_read_lock(const string &target, int &fd)
536 {
537  lock_cache_read();
538 
539  bool status = getSharedLock(target, fd);
540 
541  BESDEBUG("cache2",
542  "BESFileLockingCache::get_read_lock() - " << target << " (status: " << status << ", fd: " << fd << ")" << endl);
543 
544  if (status) m_record_descriptor(target, fd);
545 
546  unlock_cache();
547 
548  return status;
549 }
550 
563 bool BESFileLockingCache::create_and_lock(const string &target, int &fd)
564 {
566 
567  bool status = createLockedFile(target, fd);
568 
569  BESDEBUG("cache2",
570  "BESFileLockingCache::create_and_lock() - " << target << " (status: " << status << ", fd: " << fd << ")" << endl);
571 
572  if (status) m_record_descriptor(target, fd);
573 
574  unlock_cache();
575 
576  return status;
577 }
578 
593 {
594  struct flock lock;
595  lock.l_type = F_RDLCK;
596  lock.l_whence = SEEK_SET;
597  lock.l_start = 0;
598  lock.l_len = 0;
599  lock.l_pid = getpid();
600 
601  if (fcntl(fd, F_SETLKW, &lock) == -1) {
602  throw BESInternalError(get_errno(), __FILE__, __LINE__);
603  }
604 
605  BESDEBUG("cache", "BESFileLockingCache::exclusive_to_shared_lock() - lock status: " << lockStatus(fd) << endl);
606 }
607 
617 {
618  BESDEBUG("cache", "BESFileLockingCache::lock_cache_write() - d_cache_info_fd: " << d_cache_info_fd << endl);
619 
620  if (fcntl(d_cache_info_fd, F_SETLKW, lock(F_WRLCK)) == -1) {
621  throw BESInternalError("An error occurred trying to lock the cache-control file" + get_errno(), __FILE__,
622  __LINE__);
623  }
624 
625  BESDEBUG("cache", "BESFileLockingCache::lock_cache_write() - lock status: " << lockStatus(d_cache_info_fd) << endl);
626 }
627 
632 {
633  BESDEBUG("cache", "BESFileLockingCache::lock_cache_read() - d_cache_info_fd: " << d_cache_info_fd << endl);
634 
635  if (fcntl(d_cache_info_fd, F_SETLKW, lock(F_RDLCK)) == -1) {
636  throw BESInternalError("An error occurred trying to lock the cache-control file" + get_errno(), __FILE__,
637  __LINE__);
638  }
639 
640  BESDEBUG("cache", "BESFileLockingCache::lock_cache_read() - lock status: " << lockStatus(d_cache_info_fd) << endl);
641 }
642 
649 {
650  BESDEBUG("cache", "BESFileLockingCache::unlock_cache() - d_cache_info_fd: " << d_cache_info_fd << endl);
651 
652  if (fcntl(d_cache_info_fd, F_SETLK, lock(F_UNLCK)) == -1) {
653  throw BESInternalError("An error occurred trying to unlock the cache-control file" + get_errno(), __FILE__,
654  __LINE__);
655  }
656 
657  BESDEBUG("cache", "BESFileLockingCache::unlock_cache() - lock status: " << lockStatus(d_cache_info_fd) << endl);
658 }
659 
675 void BESFileLockingCache::unlock_and_close(const string &file_name)
676 {
677  BESDEBUG("cache2", "BESFileLockingCache::unlock_and_close() - BEGIN file: " << file_name << endl);
678 
679  int fd = m_get_descriptor(file_name); // returns -1 when no more files desp. remain
680  while (fd != -1) {
681  unlock(fd);
682  fd = m_get_descriptor(file_name);
683  }
684 
685  BESDEBUG("cache", "BESFileLockingCache::unlock_and_close() - lock status: " << lockStatus(d_cache_info_fd) << endl);
686  BESDEBUG("cache2", "BESFileLockingCache::unlock_and_close() - END"<< endl);
687 }
688 
699 unsigned long long BESFileLockingCache::update_cache_info(const string &target)
700 {
701  unsigned long long current_size;
702  try {
704 
705  if (lseek(d_cache_info_fd, 0, SEEK_SET) == -1)
706  throw BESInternalError("Could not rewind to front of cache info file.", __FILE__, __LINE__);
707 
708  // read the size from the cache info file
709  if (read(d_cache_info_fd, &current_size, sizeof(unsigned long long)) != sizeof(unsigned long long))
710  throw BESInternalError("Could not get read size info from the cache info file!", __FILE__, __LINE__);
711 
712  struct stat buf;
713  int statret = stat(target.c_str(), &buf);
714  if (statret == 0)
715  current_size += buf.st_size;
716  else
717  throw BESInternalError("Could not read the size of the new file: " + target + " : " + get_errno(), __FILE__,
718  __LINE__);
719 
720  BESDEBUG("cache", "BESFileLockingCache::update_cache_info() - cache size updated to: " << current_size << endl);
721 
722  if (lseek(d_cache_info_fd, 0, SEEK_SET) == -1)
723  throw BESInternalError("Could not rewind to front of cache info file.", __FILE__, __LINE__);
724 
725  if (write(d_cache_info_fd, &current_size, sizeof(unsigned long long)) != sizeof(unsigned long long))
726  throw BESInternalError("Could not write size info from the cache info file!", __FILE__, __LINE__);
727 
728  unlock_cache();
729  }
730  catch (...) {
731  unlock_cache();
732  throw;
733  }
734 
735  return current_size;
736 }
737 
742 bool BESFileLockingCache::cache_too_big(unsigned long long current_size) const
743 {
744  return current_size > d_max_cache_size_in_bytes;
745 }
746 
755 {
756  unsigned long long current_size;
757  try {
758  lock_cache_read();
759 
760  if (lseek(d_cache_info_fd, 0, SEEK_SET) == -1)
761  throw BESInternalError("Could not rewind to front of cache info file.", __FILE__, __LINE__);
762  // read the size from the cache info file
763  if (read(d_cache_info_fd, &current_size, sizeof(unsigned long long)) != sizeof(unsigned long long))
764  throw BESInternalError("Could not get read size info from the cache info file!", __FILE__, __LINE__);
765 
766  unlock_cache();
767  }
768  catch (...) {
769  unlock_cache();
770  throw;
771  }
772 
773  return current_size;
774 }
775 
776 static bool entry_op(cache_entry &e1, cache_entry &e2)
777 {
778  return e1.time < e2.time;
779 }
780 
782 unsigned long long BESFileLockingCache::m_collect_cache_dir_info(CacheFiles &contents)
783 {
784  DIR *dip = opendir(d_cache_dir.c_str());
785  if (!dip) throw BESInternalError("Unable to open cache directory " + d_cache_dir, __FILE__, __LINE__);
786 
787  struct dirent *dit;
788  vector<string> files;
789  // go through the cache directory and collect all of the files that
790  // start with the matching prefix
791  while ((dit = readdir(dip)) != NULL) {
792  string dirEntry = dit->d_name;
793  if (dirEntry.compare(0, d_prefix.length(), d_prefix) == 0 && dirEntry != d_cache_info) {
794  files.push_back(d_cache_dir + "/" + dirEntry);
795  }
796  }
797 
798  closedir(dip);
799 
800  unsigned long long current_size = 0;
801  struct stat buf;
802  for (vector<string>::iterator file = files.begin(); file != files.end(); ++file) {
803  if (stat(file->c_str(), &buf) == 0) {
804  current_size += buf.st_size;
805  cache_entry entry;
806  entry.name = *file;
807  entry.size = buf.st_size;
808  entry.time = buf.st_atime;
809  // Sanity check; Removed after initial testing since some files might be zero bytes
810 #if 0
811  if (entry.size == 0)
812  throw BESInternalError("Zero-byte file found in cache. " + *file, __FILE__, __LINE__);
813 #endif
814  contents.push_back(entry);
815  }
816  }
817 
818  // Sort so smaller (older) times are first.
819  contents.sort(entry_op);
820 
821  return current_size;
822 }
823 
835 void BESFileLockingCache::update_and_purge(const string &new_file)
836 {
837  BESDEBUG("cache", "purge - starting the purge" << endl);
838 
839  try {
841 
842  CacheFiles contents;
843  unsigned long long computed_size = m_collect_cache_dir_info(contents);
844 #if 0
845  if (BESISDEBUG( "cache_contents" )) {
846  BESDEBUG("cache", "BEFORE Purge " << computed_size/BYTES_PER_MEG << endl );
847  CacheFiles::iterator ti = contents.begin();
848  CacheFiles::iterator te = contents.end();
849  for (; ti != te; ti++) {
850  BESDEBUG("cache", (*ti).time << ": " << (*ti).name << ": size " << (*ti).size/BYTES_PER_MEG << endl );
851  }
852  }
853 #endif
854  BESDEBUG("cache",
855  "BESFileLockingCache::update_and_purge() - current and target size (in MB) " << computed_size/BYTES_PER_MEG << ", " << d_target_size/BYTES_PER_MEG << endl);
856 
857  // This deletes files and updates computed_size
858  if (cache_too_big(computed_size)) {
859 
860  // d_target_size is 80% of the maximum cache size.
861  // Grab the first which is the oldest in terms of access time.
862  CacheFiles::iterator i = contents.begin();
863  while (i != contents.end() && computed_size > d_target_size) {
864  // Grab an exclusive lock but do not block - if another process has the file locked
865  // just move on to the next file. Also test to see if the current file is the file
866  // this process just added to the cache - don't purge that!
867  int cfile_fd;
868  if (i->name != new_file && getExclusiveLockNB(i->name, cfile_fd)) {
869  BESDEBUG("cache", "purge: " << i->name << " removed." << endl);
870 
871  if (unlink(i->name.c_str()) != 0)
872  throw BESInternalError(
873  "Unable to purge the file " + i->name + " from the cache: " + get_errno(), __FILE__,
874  __LINE__);
875 
876  unlock(cfile_fd);
877  computed_size -= i->size;
878  }
879  ++i;
880 
881  BESDEBUG("cache",
882  "BESFileLockingCache::update_and_purge() - current and target size (in MB) " << computed_size/BYTES_PER_MEG << ", " << d_target_size/BYTES_PER_MEG << endl);
883  }
884  }
885 
886  if (lseek(d_cache_info_fd, 0, SEEK_SET) == -1)
887  throw BESInternalError("Could not rewind to front of cache info file.", __FILE__, __LINE__);
888 
889  if (write(d_cache_info_fd, &computed_size, sizeof(unsigned long long)) != sizeof(unsigned long long))
890  throw BESInternalError("Could not write size info to the cache info file!", __FILE__, __LINE__);
891 #if 0
892  if (BESISDEBUG( "cache_contents" )) {
893  contents.clear();
894  computed_size = m_collect_cache_dir_info(contents);
895  BESDEBUG("cache", "AFTER Purge " << computed_size/BYTES_PER_MEG << endl );
896  CacheFiles::iterator ti = contents.begin();
897  CacheFiles::iterator te = contents.end();
898  for (; ti != te; ti++) {
899  BESDEBUG("cache", (*ti).time << ": " << (*ti).name << ": size " << (*ti).size/BYTES_PER_MEG << endl );
900  }
901  }
902 #endif
903  unlock_cache();
904  }
905  catch (...) {
906  unlock_cache();
907  throw;
908  }
909 }
910 
922 void BESFileLockingCache::purge_file(const string &file)
923 {
924  BESDEBUG("cache", "BESFileLockingCache::purge_file() - starting the purge" << endl);
925 
926  try {
928 
929  // Grab an exclusive lock on the file
930  int cfile_fd;
931  if (getExclusiveLock(file, cfile_fd)) {
932  // Get the file's size
933  unsigned long long size = 0;
934  struct stat buf;
935  if (stat(file.c_str(), &buf) == 0) {
936  size = buf.st_size;
937  }
938 
939  BESDEBUG("cache", "BESFileLockingCache::purge_file() - " << file << " removed." << endl);
940 
941  if (unlink(file.c_str()) != 0)
942  throw BESInternalError("Unable to purge the file " + file + " from the cache: " + get_errno(), __FILE__,
943  __LINE__);
944 
945  unlock(cfile_fd);
946 
947  unsigned long long cache_size = get_cache_size() - size;
948 
949  if (lseek(d_cache_info_fd, 0, SEEK_SET) == -1)
950  throw BESInternalError("Could not rewind to front of cache info file.", __FILE__, __LINE__);
951 
952  if (write(d_cache_info_fd, &cache_size, sizeof(unsigned long long)) != sizeof(unsigned long long))
953  throw BESInternalError("Could not write size info to the cache info file!", __FILE__, __LINE__);
954  }
955 
956  unlock_cache();
957  }
958  catch (...) {
959  unlock_cache();
960  throw;
961  }
962 }
963 
964 const string BESFileLockingCache::getCacheFilePrefix()
965 {
966  return d_prefix;
967 }
968 
969 const string BESFileLockingCache::getCacheDirectory()
970 {
971  return d_cache_dir;
972 }
973 
980 bool BESFileLockingCache::dir_exists(const string &dir)
981 {
982  struct stat buf;
983 
984  return (stat(dir.c_str(), &buf) == 0) && (buf.st_mode & S_IFDIR);
985 }
986 
994 void BESFileLockingCache::dump(ostream &strm) const
995 {
996  strm << BESIndent::LMarg << "BESFileLockingCache::dump - (" << (void *) this << ")" << endl;
997  BESIndent::Indent();
998  strm << BESIndent::LMarg << "cache dir: " << d_cache_dir << endl;
999  strm << BESIndent::LMarg << "prefix: " << d_prefix << endl;
1000  strm << BESIndent::LMarg << "size (bytes): " << d_max_cache_size_in_bytes << endl;
1001  BESIndent::UnIndent();
1002 }
virtual bool cache_too_big(unsigned long long current_size) const
look at the cache size; is it too large? Look at the cache size and see if it is too big...
exception thrown if inernal error encountered
virtual bool create_and_lock(const string &target, int &fd)
Create a file in the cache and lock it for write access. If the file does not exist, make it, open it for read-write access and get an exclusive lock on it. The locking operation blocks, although that should never happen.
STL namespace.
virtual unsigned long long get_cache_size()
Get the cache size. Read the size information from the cache info file and return it...
static string assemblePath(const string &firstPart, const string &secondPart, bool addLeadingSlash=false)
Assemble path fragments making sure that they are separated by a single &#39;/&#39; character.
Definition: BESUtil.cc:754
static bool dir_exists(const string &dir)
Abstract exception class for the BES with basic string message.
Definition: BESError.h:56
virtual void purge_file(const string &file)
Purge a single file from the cache.
virtual void lock_cache_write()
virtual void dump(ostream &strm) const
dumps information about this object
virtual string get_cache_file_name(const string &src, bool mangle=true)
virtual bool get_read_lock(const string &target, int &fd)
Get a read-only lock on the file if it exists.
virtual void update_and_purge(const string &new_file)
Purge files from the cache.
virtual unsigned long long update_cache_info(const string &target)
Update the cache info file to include &#39;target&#39;.
unsigned long long d_max_cache_size_in_bytes
How many bytes can the cache hold before we have to purge.
virtual void lock_cache_read()
virtual void exclusive_to_shared_lock(int fd)
Transfer from an exclusive lock to a shared lock. If the file has an exclusive write lock on it...
virtual bool getExclusiveLock(string file_name, int &ref_fd)
virtual void unlock_and_close(const string &target)