bes  Updated for version 3.17.4
BESStoredDapResultCache.cc
1 // -*- mode: c++; c-basic-offset:4 -*-
2 
3 // This file is part of libdap, A C++ implementation of the OPeNDAP Data
4 // Access Protocol.
5 
6 // Copyright (c) 2011 OPeNDAP, Inc.
7 // Author: James Gallagher <jgallagher@opendap.org>
8 //
9 // This library is free software; you can redistribute it and/or
10 // modify it under the terms of the GNU Lesser General Public
11 // License as published by the Free Software Foundation; either
12 // version 2.1 of the License, or (at your option) any later version.
13 //
14 // This library is distributed in the hope that it will be useful,
15 // but WITHOUT ANY WARRANTY; without even the implied warranty of
16 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 // Lesser General Public License for more details.
18 //
19 // You should have received a copy of the GNU Lesser General Public
20 // License along with this library; if not, write to the Free Software
21 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
22 //
23 // You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
24 
25 #include "config.h"
26 
27 //#define DODS_DEBUG
28 
29 #include <sys/stat.h>
30 
31 #include <iostream>
32 #ifdef HAVE_TR1_FUNCTIONAL
33 #include <tr1/functional>
34 #endif
35 #include <string>
36 #include <fstream>
37 #include <sstream>
38 
39 #include <DDS.h>
40 #include <DMR.h>
41 #include <DapXmlNamespaces.h>
42 #include <ConstraintEvaluator.h>
43 #include <DDXParserSAX2.h>
44 
45 // These are needed because D4ParserSax2.h does not properly declare
46 // the classes. I think. Check on that... jhrg 3/28/14
47 #include <D4EnumDefs.h>
48 #include <D4Dimensions.h>
49 #include <D4Group.h>
50 
51 #include <D4ParserSax2.h>
52 
53 // DAP2 Stored results are not supported by default. If we do start using this.
54 // It would be better to use the CacheMarshaller and CacheUnMarshaller code
55 // since that does not translate data into network byte order. Also, there
56 // may be a bug in the XDRStreamUnMarshaller code - in/with get_opaque() - that
57 // breaks Sequence::deserialize(). jhrg 5/25/16
58 #ifdef DAP2_STORED_RESULTS
59 #include <XDRStreamMarshaller.h>
60 #include <XDRStreamUnMarshaller.h>
61 #endif
62 
63 #include <chunked_istream.h>
64 #include <D4StreamUnMarshaller.h>
65 
66 #include <debug.h>
67 #include <mime_util.h> // for last_modified_time() and rfc_822_date()
68 #include <util.h>
69 
70 #include "BESStoredDapResultCache.h"
71 #include "BESDapResponseBuilder.h"
72 #include "BESInternalError.h"
73 
74 #include "BESUtil.h"
75 #include "TheBESKeys.h"
76 #include "BESDebug.h"
77 
78 #ifdef HAVE_TR1_FUNCTIONAL
79 #define HASH_OBJ std::tr1::hash
80 #else
81 #define HASH_OBJ std::hash
82 #endif
83 
84 #define CRLF "\r\n"
85 #define BES_DATA_ROOT "BES.Data.RootDirectory"
86 #define BES_CATALOG_ROOT "BES.Catalog.catalog.RootDirectory"
87 
88 using namespace std;
89 using namespace libdap;
90 
91 BESStoredDapResultCache *BESStoredDapResultCache::d_instance = 0;
92 const string BESStoredDapResultCache::SUBDIR_KEY = "DAP.StoredResultsCache.subdir";
93 const string BESStoredDapResultCache::PREFIX_KEY = "DAP.StoredResultsCache.prefix";
94 const string BESStoredDapResultCache::SIZE_KEY = "DAP.StoredResultsCache.size";
95 
96 unsigned long BESStoredDapResultCache::getCacheSizeFromConfig()
97 {
98  bool found;
99  string size;
100  unsigned long size_in_megabytes = 0;
101  TheBESKeys::TheKeys()->get_value(SIZE_KEY, size, found);
102  if (found) {
103  istringstream iss(size);
104  iss >> size_in_megabytes;
105  }
106  else {
107  string msg = "[ERROR] BESStoredDapResultCache::getCacheSize() - The BES Key " + SIZE_KEY
108  + " is not set! It MUST be set to utilize the Stored Result Caching system. ";
109  BESDEBUG("cache", msg << endl);
110  throw BESInternalError(msg, __FILE__, __LINE__);
111  }
112  return size_in_megabytes;
113 }
114 
115 string BESStoredDapResultCache::getSubDirFromConfig()
116 {
117  bool found;
118  string subdir = "";
119  TheBESKeys::TheKeys()->get_value(SUBDIR_KEY, subdir, found);
120 
121  if (!found) {
122  string msg = "[ERROR] BESStoredDapResultCache::getSubDirFromConfig() - The BES Key " + SUBDIR_KEY
123  + " is not set! It MUST be set to utilize the Stored Result Caching system. ";
124  BESDEBUG("cache", msg << endl);
125  throw BESInternalError(msg, __FILE__, __LINE__);
126  }
127  else {
128  while (*subdir.begin() == '/' && subdir.length() > 0) {
129  subdir = subdir.substr(1);
130  }
131  // So if it's value is "/" or the empty string then the subdir will default to the root
132  // directory of the BES data system.
133  }
134 
135  return subdir;
136 }
137 
138 string BESStoredDapResultCache::getResultPrefixFromConfig()
139 {
140  bool found;
141  string prefix = "";
142  TheBESKeys::TheKeys()->get_value(PREFIX_KEY, prefix, found);
143  if (found) {
144  prefix = BESUtil::lowercase(prefix);
145  }
146  else {
147  string msg = "[ERROR] BESStoredDapResultCache::getResultPrefix() - The BES Key " + PREFIX_KEY
148  + " is not set! It MUST be set to utilize the Stored Result Caching system. ";
149  BESDEBUG("cache", msg << endl);
150  throw BESInternalError(msg, __FILE__, __LINE__);
151  }
152 
153  return prefix;
154 }
155 
156 string BESStoredDapResultCache::getBesDataRootDirFromConfig()
157 {
158  bool found;
159  string cacheDir = "";
160  TheBESKeys::TheKeys()->get_value( BES_CATALOG_ROOT, cacheDir, found);
161  if (!found) {
162  TheBESKeys::TheKeys()->get_value( BES_DATA_ROOT, cacheDir, found);
163  if (!found) {
164  string msg = ((string) "[ERROR] BESStoredDapResultCache::getStoredResultsDir() - Neither the BES Key ")
165  + BES_CATALOG_ROOT + "or the BES key " + BES_DATA_ROOT
166  + " have been set! One MUST be set to utilize the Stored Result Caching system. ";
167  BESDEBUG("cache", msg << endl);
168  throw BESInternalError(msg, __FILE__, __LINE__);
169  }
170  }
171  return cacheDir;
172 
173 }
174 
175 BESStoredDapResultCache::BESStoredDapResultCache()
176 {
177  BESDEBUG("cache", "BESStoredDapResultCache::BESStoredDapResultCache() - BEGIN" << endl);
178 
179  d_storedResultsSubdir = getSubDirFromConfig();
180  d_dataRootDir = getBesDataRootDirFromConfig();
181  string resultsDir = BESUtil::assemblePath(d_dataRootDir, d_storedResultsSubdir);
182 
183  d_resultFilePrefix = getResultPrefixFromConfig();
184  d_maxCacheSize = getCacheSizeFromConfig();
185 
186  BESDEBUG("cache",
187  "BESStoredDapResultCache() - Stored results cache configuration params: " << resultsDir << ", " << d_resultFilePrefix << ", " << d_maxCacheSize << endl);
188 
189  initialize(resultsDir, d_resultFilePrefix, d_maxCacheSize);
190 
191  BESDEBUG("cache", "BESStoredDapResultCache::BESStoredDapResultCache() - END" << endl);
192 }
193 
197 BESStoredDapResultCache::BESStoredDapResultCache(const string &data_root_dir, const string &stored_results_subdir,
198  const string &result_file_prefix, unsigned long long max_cache_size)
199 {
200 
201  d_storedResultsSubdir = stored_results_subdir;
202  d_dataRootDir = data_root_dir;
203  d_resultFilePrefix = result_file_prefix;
204  d_maxCacheSize = max_cache_size;
205  initialize(BESUtil::assemblePath(d_dataRootDir, stored_results_subdir), d_resultFilePrefix, d_maxCacheSize);
206 }
207 
209 BESStoredDapResultCache::get_instance(const string &data_root_dir, const string &stored_results_subdir,
210  const string &result_file_prefix, unsigned long long max_cache_size)
211 {
212  if (d_instance == 0) {
213  if (dir_exists(data_root_dir)) {
214  try {
215  d_instance = new BESStoredDapResultCache(data_root_dir, stored_results_subdir, result_file_prefix,
216  max_cache_size);
217 #ifdef HAVE_ATEXIT
218  atexit(delete_instance);
219 #endif
220  }
221  catch (BESInternalError &bie) {
222  BESDEBUG("cache",
223  "[ERROR] BESStoredDapResultCache::get_instance(): Failed to obtain cache! msg: " << bie.get_message() << endl);
224  }
225  }
226  }
227  return d_instance;
228 }
229 
235 {
236  if (d_instance == 0) {
237  try {
238  d_instance = new BESStoredDapResultCache();
239 #ifdef HAVE_ATEXIT
240  atexit(delete_instance);
241 #endif
242  }
243  catch (BESInternalError &bie) {
244  BESDEBUG("cache",
245  "[ERROR] BESStoredDapResultCache::get_instance(): Failed to obtain cache! msg: " << bie.get_message() << endl);
246  }
247  }
248 
249  return d_instance;
250 }
251 
261 bool BESStoredDapResultCache::is_valid(const string &cache_file_name, const string &dataset)
262 {
263  // If the cached response is zero bytes in size, it's not valid.
264  // (hmmm...)
265 
266  off_t entry_size = 0;
267  time_t entry_time = 0;
268  struct stat buf;
269  if (stat(cache_file_name.c_str(), &buf) == 0) {
270  entry_size = buf.st_size;
271  entry_time = buf.st_mtime;
272  }
273  else {
274  return false;
275  }
276 
277  if (entry_size == 0) return false;
278 
279  time_t dataset_time = entry_time;
280  if (stat(dataset.c_str(), &buf) == 0) {
281  dataset_time = buf.st_mtime;
282  }
283 
284  // Trick: if the d_dataset is not a file, stat() returns error and
285  // the times stay equal and the code uses the cache entry.
286 
287  // TODO Fix this so that the code can get a LMT from the correct
288  // handler.
289  if (dataset_time > entry_time) return false;
290 
291  return true;
292 }
293 
294 #ifdef DAP2_STORED_RESULTS
295 
306 bool BESStoredDapResultCache::read_dap2_data_from_cache(const string &cache_file_name, DDS *fdds)
307 {
308  BESDEBUG("cache",
309  "BESStoredDapResultCache::read_dap2_data_from_cache() - Opening cache file: " << cache_file_name << endl);
310 
311  int fd = 1;
312 
313  try {
314  if (get_read_lock(cache_file_name, fd)) {
315 
316  ifstream data(cache_file_name.c_str());
317 
318  // Rip off the MIME headers from the response if they are present
319  string mime = get_next_mime_header(data);
320  while (!mime.empty()) {
321  mime = get_next_mime_header(data);
322  }
323 
324  // Parse the DDX; throw an exception on error.
325  DDXParser ddx_parser(fdds->get_factory());
326 
327  // Read the MPM boundary and then read the subsequent headers
328  string boundary = read_multipart_boundary(data);
329  BESDEBUG("cache",
330  "BESStoredDapResultCache::read_dap2_data_from_cache() - MPM Boundary: " << boundary << endl);
331 
332  read_multipart_headers(data, "text/xml", dods_ddx);
333 
334  BESDEBUG("cache",
335  "BESStoredDapResultCache::read_dap2_data_from_cache() - Read the multipart haeaders" << endl);
336 
337  // Parse the DDX, reading up to and including the next boundary.
338  // Return the CID for the matching data part
339  string data_cid;
340  try {
341  ddx_parser.intern_stream(data, fdds, data_cid, boundary);
342  BESDEBUG("cache",
343  "BESStoredDapResultCache::read_dap2_data_from_cache() - Dataset name: " << fdds->get_dataset_name() << endl);
344  }
345  catch (Error &e) {
346  BESDEBUG("cache",
347  "BESStoredDapResultCache::read_dap2_data_from_cache() - DDX Parser Error: " << e.get_error_message() << endl);
348  throw;
349  }
350 
351  // Munge the CID into something we can work with
352  BESDEBUG("cache",
353  "BESStoredDapResultCache::read_dap2_data_from_cache() - Data CID (before): " << data_cid << endl);
354  data_cid = cid_to_header_value(data_cid);
355  BESDEBUG("cache",
356  "BESStoredDapResultCache::read_dap2_data_from_cache() - Data CID (after): " << data_cid << endl);
357 
358  // Read the data part's MPM part headers (boundary was read by
359  // DDXParse::intern)
360  read_multipart_headers(data, "application/octet-stream", dods_data_ddx, data_cid);
361 
362  // Now read the data
363 
364  // XDRFileUnMarshaller um(data);
365  XDRStreamUnMarshaller um(data);
366  for (DDS::Vars_iter i = fdds->var_begin(); i != fdds->var_end(); i++) {
367  (*i)->deserialize(um, fdds);
368  }
369 
370  data.close();
371  unlock_and_close(cache_file_name /* was fd */);
372  return true;
373  }
374  else {
375  BESDEBUG("cache", "BESStoredDapResultCache - The requested file does not exist. File: " + cache_file_name);
376 
377  return false;
378  }
379  }
380  catch (...) {
381  BESDEBUG("cache",
382  "BESStoredDapResultCache::read_dap4_data_from_cache() - caught exception, unlocking cache and re-throw." << endl);
383  // I think this call is not needed. jhrg 10/23/12
384  if (fd != -1) unlock_and_close(cache_file_name /* was fd */);
385  throw;
386  }
387 }
388 #endif
389 
401 bool BESStoredDapResultCache::read_dap4_data_from_cache(const string &cache_file_name, libdap::DMR *dmr)
402 {
403  BESDEBUG("cache", "BESStoredDapResultCache::read_dap4_data_from_cache() - BEGIN" << endl);
404 
405  int fd = 1;
406 
407  try {
408  if (get_read_lock(cache_file_name, fd)) {
409  BESDEBUG("cache",
410  "BESStoredDapResultCache::read_dap4_data_from_cache() - Opening cache file: " << cache_file_name << endl);
411  fstream in(cache_file_name.c_str(), ios::in | ios::binary);
412 
413  // Gobble up the response's initial set of MIME headers. Normally
414  // a client would extract information from these headers.
415  // NOTE - I am dumping this call because it basically just
416  // slurps up lines until it finds a blank line, regardless of what the
417  // lines actually have in the. So basically if the stream DOESN't have
418  // a mime header then this call will read (and ignore) the entire
419  // XML encoding of the DMR. doh.
420  // remove_mime_header(in);
421 
422  chunked_istream cis(in, CHUNK_SIZE);
423 
424  bool debug = BESDebug::IsSet("parser");
425 
426  // parse the DMR, stopping when the boundary is found.
427  // force chunk read
428  // get chunk size
429  int chunk_size = cis.read_next_chunk();
430 
431  BESDEBUG("cache",
432  "BESStoredDapResultCache::read_dap4_data_from_cache() - First chunk_size: " << chunk_size << endl);
433 
434  if (chunk_size == EOF) {
435  throw InternalErr(__FILE__, __LINE__,
436  "BESStoredDapResultCache::read_dap4_data_from_cache() - Failed to read first chunk from file. Chunk size = EOF (aka "
437  + libdap::long_to_string(EOF) + ")");
438  }
439 
440  // get chunk
441  char chunk[chunk_size];
442  cis.read(chunk, chunk_size);
443  BESDEBUG("cache", "BESStoredDapResultCache::read_dap4_data_from_cache() - Read first chunk." << endl);
444 
445  // parse char * with given size
446  D4ParserSax2 parser;
447  // '-2' to discard the CRLF pair
448  parser.intern(chunk, chunk_size - 2, dmr, debug);
449  BESDEBUG("cache", "BESStoredDapResultCache::read_dap4_data_from_cache() - Parsed first chunk." << endl);
450 
451  D4StreamUnMarshaller um(cis, cis.twiddle_bytes());
452 
453  dmr->root()->deserialize(um, *dmr);
454  BESDEBUG("cache", "BESStoredDapResultCache::read_dap4_data_from_cache() - Deserialized data." << endl);
455 
456  BESDEBUG("cache", "BESStoredDapResultCache::read_dap4_data_from_cache() - END" << endl);
457 
458  in.close();
459  unlock_and_close(cache_file_name /* was fd */);
460 
461  return true;
462 
463  }
464  else {
465  BESDEBUG("cache", "BESStoredDapResultCache - The requested file does not exist. File: " + cache_file_name);
466 
467  return false;
468 
469  }
470  }
471  catch (...) {
472  BESDEBUG("cache",
473  "BESStoredDapResultCache::read_dap4_data_from_cache() - caught exception, unlocking cache and re-throw." << endl);
474  // I think this call is not needed. jhrg 10/23/12
475  if (fd != -1) unlock_and_close(cache_file_name /* was fd */);
476  throw;
477  }
478 }
479 
480 #ifdef DAP2_STORED_RESULTS
481 
485 DDS *
486 BESStoredDapResultCache::get_cached_dap2_data_ddx(const string &cache_file_name, BaseTypeFactory *factory,
487  const string &filename)
488 {
489  BESDEBUG("cache",
490  "BESStoredDapResultCache::get_cached_dap2_data_ddx() - Reading cache for " << cache_file_name << endl);
491 
492  DDS *fdds = new DDS(factory);
493 
494  if (read_dap2_data_from_cache(cache_file_name, fdds)) {
495 
496  fdds->filename(filename);
497  //fdds->set_dataset_name( "function_result_" + name_path(filename) ) ;
498 
499  BESDEBUG("cache", "DDS Filename: " << fdds->filename() << endl);
500  BESDEBUG("cache", "DDS Dataset name: " << fdds->get_dataset_name() << endl);
501 
502  fdds->set_factory(0);
503 
504  // mark everything as read. and send. That is, make sure that when a response
505  // is retrieved from the cache, all of the variables are marked as to be sent
506  DDS::Vars_iter i = fdds->var_begin();
507  while (i != fdds->var_end()) {
508  (*i)->set_read_p(true);
509  (*i++)->set_send_p(true);
510  }
511 
512  return fdds;
513  }
514  else {
515  delete fdds;
516  return 0;
517  }
518 
519 }
520 #endif
521 
526 DMR *
527 BESStoredDapResultCache::get_cached_dap4_data(const string &cache_file_name, libdap::D4BaseTypeFactory *factory,
528  const string &filename)
529 {
530  BESDEBUG("cache",
531  "BESStoredDapResultCache::get_cached_dap4_data() - Reading cache for " << cache_file_name << endl);
532 
533  DMR *fdmr = new DMR(factory);
534 
535  BESDEBUG("cache", "BESStoredDapResultCache::get_cached_dap4_data() - DMR Filename: " << fdmr->filename() << endl);
536  fdmr->set_filename(filename);
537 
538  if (read_dap4_data_from_cache(cache_file_name, fdmr)) {
539  BESDEBUG("cache",
540  "BESStoredDapResultCache::get_cached_dap4_data() - DMR Dataset name: " << fdmr->name() << endl);
541 
542  fdmr->set_factory(0);
543 
544  // mark everything as read. and send. That is, make sure that when a response
545  // is retrieved from the cache, all of the variables are marked as to be sent
546  fdmr->root()->set_send_p(true);
547  fdmr->root()->set_read_p(true);
548 
549  return fdmr;
550  }
551 
552  return 0;
553 }
554 
555 #ifdef DAP2_STORED_RESULTS
556 
560 string BESStoredDapResultCache::store_dap2_result(DDS &dds, const string &constraint, BESDapResponseBuilder *rb,
561  ConstraintEvaluator *eval)
562 {
563  BESDEBUG("cache", "BESStoredDapResultCache::store_dap2_result() - BEGIN" << endl);
564  // These are used for the cached or newly created DDS object
565  BaseTypeFactory factory;
566 
567  // Get the cache filename for this thing. Do not use the default
568  // name mangling; instead use what build_cache_file_name() does.
569  string local_id = get_stored_result_local_id(dds.filename(), constraint, DAP_3_2);
570  BESDEBUG("cache", "BESStoredDapResultCache::store_dap2_result() - local_id: "<< local_id << endl);
571  string cache_file_name = get_cache_file_name(local_id, /*mangle*/false);
572  BESDEBUG("cache", "BESStoredDapResultCache::store_dap2_result() - cache_file_name: "<< cache_file_name << endl);
573  int fd;
574  try {
575  // If the object in the cache is not valid, remove it. The read_lock will
576  // then fail and the code will drop down to the create_and_lock() call.
577  // is_valid() tests for a non-zero object and for d_dateset newer than
578  // the cached object.
579  if (!is_valid(cache_file_name, dds.filename())) purge_file(cache_file_name);
580 
581  if (get_read_lock(cache_file_name, fd)) {
582  BESDEBUG("cache",
583  "BESStoredDapResultCache::store_dap2_result() - Stored Result already exists. Not rewriting file: " << cache_file_name << endl);
584  }
585  else if (create_and_lock(cache_file_name, fd)) {
586  // If here, the cache_file_name could not be locked for read access;
587  // try to build it. First make an empty file and get an exclusive lock on it.
588  BESDEBUG("cache",
589  "BESStoredDapResultCache::store_dap2_result() - cache_file_name " << cache_file_name << ", constraint: " << constraint << endl);
590 
591 #if 0 // I shut this off because we know that the constraint and functions have already been evaluated - ndp
592  DDS *fdds;
593 
594  fdds = new DDS(dds);
595  eval->parse_constraint(constraint, *fdds);
596 
597  if (eval->function_clauses()) {
598  DDS *temp_fdds = eval->eval_function_clauses(*fdds);
599  delete fdds;
600  fdds = temp_fdds;
601  }
602 #endif
603 
604  ofstream data_stream(cache_file_name.c_str());
605  if (!data_stream)
606  throw InternalErr(__FILE__, __LINE__,
607  "Could not open '" + cache_file_name + "' to write cached response.");
608 
609  string start = "dataddx_cache_start", boundary = "dataddx_cache_boundary";
610 
611  // Use a ConstraintEvaluator that has not parsed a CE so the code can use
612  // the send method(s)
613  ConstraintEvaluator eval;
614 
615  // Setting the version to 3.2 causes send_data_ddx to write the MIME headers that
616  // the cache expects.
617  dds.set_dap_version("3.2");
618 
619  // This is a bit of a hack, but it effectively uses ResponseBuilder to write the
620  // cached object/response without calling the machinery in one of the send_*()
621  // methods. Those methods assume they need to evaluate the BESDapResponseBuilder's
622  // CE, which is not necessary and will alter the values of the send_p property
623  // of the DDS's variables.
624  set_mime_multipart(data_stream, boundary, start, dods_data_ddx, x_plain,
625  last_modified_time(rb->get_dataset_name()));
626  //data_stream << flush;
627  rb->serialize_dap2_data_ddx(data_stream, (DDS**) &dds, eval, boundary, start);
628  //data_stream << flush;
629 
630  data_stream << CRLF << "--" << boundary << "--" << CRLF;
631 
632  data_stream.close();
633 
634  // Change the exclusive lock on the new file to a shared lock. This keeps
635  // other processes from purging the new file and ensures that the reading
636  // process can use it.
637  exclusive_to_shared_lock(fd);
638 
639  // Now update the total cache size info and purge if needed. The new file's
640  // name is passed into the purge method because this process cannot detect its
641  // own lock on the file.
642  unsigned long long size = update_cache_info(cache_file_name);
643  if (cache_too_big(size)) update_and_purge(cache_file_name);
644 
645  }
646  // get_read_lock() returns immediately if the file does not exist,
647  // but blocks waiting to get a shared lock if the file does exist.
648  else if (get_read_lock(cache_file_name, fd)) {
649  BESDEBUG("cache",
650  "BESStoredDapResultCache::store_dap2_result() - Stored Result already exists. Not rewriting file: " << cache_file_name << endl);
651  }
652  else {
653  throw InternalErr(__FILE__, __LINE__,
654  "BESStoredDapResultCache::store_dap2_result() - Cache error during function invocation.");
655  }
656 
657  BESDEBUG("cache",
658  "BESStoredDapResultCache::store_dap2_result() - unlocking and closing cache file "<< cache_file_name << endl);
659  unlock_and_close(cache_file_name);
660  }
661  catch (...) {
662  BESDEBUG("cache",
663  "BESStoredDapResultCache::store_dap2_result() - caught exception, unlocking cache and re-throw." << endl);
664  // I think this call is not needed. jhrg 10/23/12
665  unlock_cache();
666  throw;
667  }
668 
669  BESDEBUG("cache", "BESStoredDapResultCache::store_dap2_result() - END (local_id=`"<< local_id << "')" << endl);
670  return local_id;
671 }
672 #endif
673 
681 string BESStoredDapResultCache::get_stored_result_local_id(const string &dataset, const string &ce,
682  libdap::DAPVersion version)
683 {
684  BESDEBUG("cache", "get_stored_result_local_id() - BEGIN. dataset: " << dataset << ", ce: " << ce << endl);
685  std::ostringstream ostr;
686  HASH_OBJ<std::string> str_hash;
687  string name = dataset + "#" + ce;
688  ostr << str_hash(name);
689  string hashed_name = ostr.str();
690  BESDEBUG("cache", "get_stored_result_local_id() - hashed_name: " << hashed_name << endl);
691 
692  string suffix = "";
693  switch (version) {
694 #ifdef DAP2_STORED_RESULTS
695  case DAP_2_0:
696  suffix = ".dods";
697  break;
698 
699  case DAP_3_2:
700  suffix = ".data_ddx";
701  break;
702 #endif
703  case DAP_4_0:
704  suffix = ".dap";
705  break;
706 
707  default:
708  throw BESInternalError("BESStoredDapResultCache::get_stored_result_local_id() - Unrecognized DAP version!!",
709  __FILE__, __LINE__);
710  break;
711  }
712 
713  BESDEBUG("cache", "get_stored_result_local_id() - Data file suffix: " << suffix << endl);
714 
715  string local_id = d_resultFilePrefix + hashed_name + suffix;
716  BESDEBUG("cache", "get_stored_result_local_id() - file: " << local_id << endl);
717 
718  local_id = BESUtil::assemblePath(d_storedResultsSubdir, local_id);
719 
720  BESDEBUG("cache", "get_stored_result_local_id() - END. local_id: " << local_id << endl);
721  return local_id;
722 }
723 
728 string BESStoredDapResultCache::store_dap4_result(DMR &dmr, const string &constraint, BESDapResponseBuilder *rb)
729 {
730  BESDEBUG("cache", "BESStoredDapResultCache::store_dap4_result() - BEGIN" << endl);
731  // These are used for the cached or newly created DDS object
732  BaseTypeFactory factory;
733 
734  // Get the cache filename for this thing. Do not use the default
735  // name mangling; instead use what build_cache_file_name() does.
736  string local_id = get_stored_result_local_id(dmr.filename(), constraint, DAP_4_0);
737  BESDEBUG("cache", "BESStoredDapResultCache::store_dap4_result() - local_id: "<< local_id << endl);
738  string cache_file_name = get_cache_file_name(local_id, /*mangle*/false);
739  BESDEBUG("cache", "BESStoredDapResultCache::store_dap4_result() - cache_file_name: "<< cache_file_name << endl);
740  int fd;
741  try {
742  // If the object in the cache is not valid, remove it. The read_lock will
743  // then fail and the code will drop down to the create_and_lock() call.
744  // is_valid() tests for a non-zero object and for d_dateset newer than
745  // the cached object.
746  if (!is_valid(cache_file_name, dmr.filename())) {
747  BESDEBUG("cache",
748  "BESStoredDapResultCache::store_dap4_result() - File is not valid. Purging file from cache. filename: " << cache_file_name << endl);
749  purge_file(cache_file_name);
750  }
751 
752  if (get_read_lock(cache_file_name, fd)) {
753  BESDEBUG("cache",
754  "BESStoredDapResultCache::store_dap4_result() - Stored Result already exists. Not rewriting file: " << cache_file_name << endl);
755  }
756  else if (create_and_lock(cache_file_name, fd)) {
757  // If here, the cache_file_name could not be locked for read access;
758  // try to build it. First make an empty file and get an exclusive lock on it.
759  BESDEBUG("cache",
760  "BESStoredDapResultCache::store_dap4_result() - cache_file_name: " << cache_file_name << ", constraint: " << constraint << endl);
761 
762  ofstream data_stream(cache_file_name.c_str());
763  if (!data_stream)
764  throw InternalErr(__FILE__, __LINE__,
765  "Could not open '" + cache_file_name + "' to write cached response.");
766 
767  //data_stream << flush;
768  rb->serialize_dap4_data(data_stream, dmr, false);
769  //data_stream << flush;
770 
771  data_stream.close();
772 
773  // Change the exclusive lock on the new file to a shared lock. This keeps
774  // other processes from purging the new file and ensures that the reading
775  // process can use it.
776  exclusive_to_shared_lock(fd);
777 
778  // Now update the total cache size info and purge if needed. The new file's
779  // name is passed into the purge method because this process cannot detect its
780  // own lock on the file.
781  unsigned long long size = update_cache_info(cache_file_name);
782  if (cache_too_big(size)) update_and_purge(cache_file_name);
783  }
784  // get_read_lock() returns immediately if the file does not exist,
785  // but blocks waiting to get a shared lock if the file does exist.
786  else if (get_read_lock(cache_file_name, fd)) {
787  BESDEBUG("cache",
788  "BESStoredDapResultCache::store_dap4_result() - Couldn't create and lock file, But I got a read lock. " "Result may have been created by another process. " "Not rewriting file: " << cache_file_name << endl);
789  }
790  else {
791  throw InternalErr(__FILE__, __LINE__,
792  "BESStoredDapResultCache::store_dap4_result() - Cache error during function invocation.");
793  }
794 
795  BESDEBUG("cache",
796  "BESStoredDapResultCache::store_dap4_result() - unlocking and closing cache file "<< cache_file_name << endl);
797  unlock_and_close(cache_file_name);
798  }
799  catch (...) {
800  BESDEBUG("cache",
801  "BESStoredDapResultCache::store_dap4_result() - caught exception, unlocking cache and re-throw." << endl);
802  // I think this call is not needed. jhrg 10/23/12
803  unlock_cache();
804  throw;
805  }
806 
807  BESDEBUG("cache", "BESStoredDapResultCache::store_dap4_result() - END (local_id=`"<< local_id << "')" << endl);
808  return local_id;
809 
810 }
811 
exception thrown if inernal error encountered
static string lowercase(const string &s)
Definition: BESUtil.cc:186
virtual std::string get_message()
get the error message for this exception
Definition: BESError.h:97
STL namespace.
virtual string store_dap4_result(libdap::DMR &dmr, const string &constraint, BESDapResponseBuilder *rb)
libdap::DMR * get_cached_dap4_data(const string &cache_file_name, libdap::D4BaseTypeFactory *factory, const string &filename)
virtual std::string get_dataset_name() const
Get the dataset name.
static string assemblePath(const string &firstPart, const string &secondPart, bool addLeadingSlash=false)
Assemble path fragments making sure that they are separated by a single &#39;/&#39; character.
Definition: BESUtil.cc:754
static BESStoredDapResultCache * get_instance()
static bool IsSet(const string &flagName)
see if the debug context flagName is set to true
Definition: BESDebug.h:167
void get_value(const string &s, string &val, bool &found)
Retrieve the value of a given key, if set.
Definition: BESKeys.cc:483
static BESKeys * TheKeys()
Definition: TheBESKeys.cc:43
virtual void serialize_dap4_data(std::ostream &out, libdap::DMR &dmr, bool with_mime_headers=true)