Bitcoin Core  0.18.99
P2P Digital Currency
db.cpp
Go to the documentation of this file.
1 // Copyright (c) 2009-2010 Satoshi Nakamoto
2 // Copyright (c) 2009-2019 The Bitcoin Core developers
3 // Distributed under the MIT software license, see the accompanying
4 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
5 
6 #include <wallet/db.h>
7 
8 #include <util/strencodings.h>
9 
10 #include <stdint.h>
11 
12 #ifndef WIN32
13 #include <sys/stat.h>
14 #endif
15 
16 #include <boost/thread.hpp>
17 
18 namespace {
19 
29 void CheckUniqueFileid(const BerkeleyEnvironment& env, const std::string& filename, Db& db, WalletDatabaseFileId& fileid)
30 {
31  if (env.IsMock()) return;
32 
33  int ret = db.get_mpf()->get_fileid(fileid.value);
34  if (ret != 0) {
35  throw std::runtime_error(strprintf("BerkeleyBatch: Can't open database %s (get_fileid failed with %d)", filename, ret));
36  }
37 
38  for (const auto& item : env.m_fileids) {
39  if (fileid == item.second && &fileid != &item.second) {
40  throw std::runtime_error(strprintf("BerkeleyBatch: Can't open database %s (duplicates fileid %s from %s)", filename,
41  HexStr(std::begin(item.second.value), std::end(item.second.value)), item.first));
42  }
43  }
44 }
45 
46 CCriticalSection cs_db;
47 std::map<std::string, std::weak_ptr<BerkeleyEnvironment>> g_dbenvs GUARDED_BY(cs_db);
48 } // namespace
49 
51 {
52  return memcmp(value, &rhs.value, sizeof(value)) == 0;
53 }
54 
55 static void SplitWalletPath(const fs::path& wallet_path, fs::path& env_directory, std::string& database_filename)
56 {
57  if (fs::is_regular_file(wallet_path)) {
58  // Special case for backwards compatibility: if wallet path points to an
59  // existing file, treat it as the path to a BDB data file in a parent
60  // directory that also contains BDB log files.
61  env_directory = wallet_path.parent_path();
62  database_filename = wallet_path.filename().string();
63  } else {
64  // Normal case: Interpret wallet path as a directory path containing
65  // data and log files.
66  env_directory = wallet_path;
67  database_filename = "wallet.dat";
68  }
69 }
70 
71 bool IsWalletLoaded(const fs::path& wallet_path)
72 {
73  fs::path env_directory;
74  std::string database_filename;
75  SplitWalletPath(wallet_path, env_directory, database_filename);
76  LOCK(cs_db);
77  auto env = g_dbenvs.find(env_directory.string());
78  if (env == g_dbenvs.end()) return false;
79  auto database = env->second.lock();
80  return database && database->IsDatabaseLoaded(database_filename);
81 }
82 
83 fs::path WalletDataFilePath(const fs::path& wallet_path)
84 {
85  fs::path env_directory;
86  std::string database_filename;
87  SplitWalletPath(wallet_path, env_directory, database_filename);
88  return env_directory / database_filename;
89 }
90 
98 std::shared_ptr<BerkeleyEnvironment> GetWalletEnv(const fs::path& wallet_path, std::string& database_filename)
99 {
100  fs::path env_directory;
101  SplitWalletPath(wallet_path, env_directory, database_filename);
102  LOCK(cs_db);
103  auto inserted = g_dbenvs.emplace(env_directory.string(), std::weak_ptr<BerkeleyEnvironment>());
104  if (inserted.second) {
105  auto env = std::make_shared<BerkeleyEnvironment>(env_directory.string());
106  inserted.first->second = env;
107  return env;
108  }
109  return inserted.first->second.lock();
110 }
111 
112 //
113 // BerkeleyBatch
114 //
115 
117 {
118  if (!fDbEnvInit)
119  return;
120 
121  fDbEnvInit = false;
122 
123  for (auto& db : m_databases) {
124  auto count = mapFileUseCount.find(db.first);
125  assert(count == mapFileUseCount.end() || count->second == 0);
126  BerkeleyDatabase& database = db.second.get();
127  if (database.m_db) {
128  database.m_db->close(0);
129  database.m_db.reset();
130  }
131  }
132 
133  FILE* error_file = nullptr;
134  dbenv->get_errfile(&error_file);
135 
136  int ret = dbenv->close(0);
137  if (ret != 0)
138  LogPrintf("BerkeleyEnvironment::Close: Error %d closing database environment: %s\n", ret, DbEnv::strerror(ret));
139  if (!fMockDb)
140  DbEnv((u_int32_t)0).remove(strPath.c_str(), 0);
141 
142  if (error_file) fclose(error_file);
143 
144  UnlockDirectory(strPath, ".walletlock");
145 }
146 
148 {
149  dbenv.reset(new DbEnv(DB_CXX_NO_EXCEPTIONS));
150  fDbEnvInit = false;
151  fMockDb = false;
152 }
153 
154 BerkeleyEnvironment::BerkeleyEnvironment(const fs::path& dir_path) : strPath(dir_path.string())
155 {
156  Reset();
157 }
158 
160 {
161  LOCK(cs_db);
162  g_dbenvs.erase(strPath);
163  Close();
164 }
165 
167 {
168  if (fDbEnvInit)
169  return true;
170 
171  boost::this_thread::interruption_point();
172 
173  fs::path pathIn = strPath;
174  TryCreateDirectories(pathIn);
175  if (!LockDirectory(pathIn, ".walletlock")) {
176  LogPrintf("Cannot obtain a lock on wallet directory %s. Another instance of bitcoin may be using it.\n", strPath);
177  return false;
178  }
179 
180  fs::path pathLogDir = pathIn / "database";
181  TryCreateDirectories(pathLogDir);
182  fs::path pathErrorFile = pathIn / "db.log";
183  LogPrintf("BerkeleyEnvironment::Open: LogDir=%s ErrorFile=%s\n", pathLogDir.string(), pathErrorFile.string());
184 
185  unsigned int nEnvFlags = 0;
186  if (gArgs.GetBoolArg("-privdb", DEFAULT_WALLET_PRIVDB))
187  nEnvFlags |= DB_PRIVATE;
188 
189  dbenv->set_lg_dir(pathLogDir.string().c_str());
190  dbenv->set_cachesize(0, 0x100000, 1); // 1 MiB should be enough for just the wallet
191  dbenv->set_lg_bsize(0x10000);
192  dbenv->set_lg_max(1048576);
193  dbenv->set_lk_max_locks(40000);
194  dbenv->set_lk_max_objects(40000);
195  dbenv->set_errfile(fsbridge::fopen(pathErrorFile, "a"));
196  dbenv->set_flags(DB_AUTO_COMMIT, 1);
197  dbenv->set_flags(DB_TXN_WRITE_NOSYNC, 1);
198  dbenv->log_set_config(DB_LOG_AUTO_REMOVE, 1);
199  int ret = dbenv->open(strPath.c_str(),
200  DB_CREATE |
201  DB_INIT_LOCK |
202  DB_INIT_LOG |
203  DB_INIT_MPOOL |
204  DB_INIT_TXN |
205  DB_THREAD |
206  DB_RECOVER |
207  nEnvFlags,
208  S_IRUSR | S_IWUSR);
209  if (ret != 0) {
210  LogPrintf("BerkeleyEnvironment::Open: Error %d opening database environment: %s\n", ret, DbEnv::strerror(ret));
211  int ret2 = dbenv->close(0);
212  if (ret2 != 0) {
213  LogPrintf("BerkeleyEnvironment::Open: Error %d closing failed database environment: %s\n", ret2, DbEnv::strerror(ret2));
214  }
215  Reset();
216  if (retry) {
217  // try moving the database env out of the way
218  fs::path pathDatabaseBak = pathIn / strprintf("database.%d.bak", GetTime());
219  try {
220  fs::rename(pathLogDir, pathDatabaseBak);
221  LogPrintf("Moved old %s to %s. Retrying.\n", pathLogDir.string(), pathDatabaseBak.string());
222  } catch (const fs::filesystem_error&) {
223  // failure is ok (well, not really, but it's not worse than what we started with)
224  }
225  // try opening it again one more time
226  if (!Open(false /* retry */)) {
227  // if it still fails, it probably means we can't even create the database env
228  return false;
229  }
230  } else {
231  return false;
232  }
233  }
234 
235  fDbEnvInit = true;
236  fMockDb = false;
237  return true;
238 }
239 
242 {
243  Reset();
244 
245  boost::this_thread::interruption_point();
246 
247  LogPrint(BCLog::DB, "BerkeleyEnvironment::MakeMock\n");
248 
249  dbenv->set_cachesize(1, 0, 1);
250  dbenv->set_lg_bsize(10485760 * 4);
251  dbenv->set_lg_max(10485760);
252  dbenv->set_lk_max_locks(10000);
253  dbenv->set_lk_max_objects(10000);
254  dbenv->set_flags(DB_AUTO_COMMIT, 1);
255  dbenv->log_set_config(DB_LOG_IN_MEMORY, 1);
256  int ret = dbenv->open(nullptr,
257  DB_CREATE |
258  DB_INIT_LOCK |
259  DB_INIT_LOG |
260  DB_INIT_MPOOL |
261  DB_INIT_TXN |
262  DB_THREAD |
263  DB_PRIVATE,
264  S_IRUSR | S_IWUSR);
265  if (ret > 0)
266  throw std::runtime_error(strprintf("BerkeleyEnvironment::MakeMock: Error %d opening database environment.", ret));
267 
268  fDbEnvInit = true;
269  fMockDb = true;
270 }
271 
272 BerkeleyEnvironment::VerifyResult BerkeleyEnvironment::Verify(const std::string& strFile, recoverFunc_type recoverFunc, std::string& out_backup_filename)
273 {
274  LOCK(cs_db);
275  assert(mapFileUseCount.count(strFile) == 0);
276 
277  Db db(dbenv.get(), 0);
278  int result = db.verify(strFile.c_str(), nullptr, nullptr, 0);
279  if (result == 0)
281  else if (recoverFunc == nullptr)
283 
284  // Try to recover:
285  bool fRecovered = (*recoverFunc)(fs::path(strPath) / strFile, out_backup_filename);
287 }
288 
290 {
291  m_dbt.set_flags(DB_DBT_MALLOC);
292 }
293 
294 BerkeleyBatch::SafeDbt::SafeDbt(void* data, size_t size)
295  : m_dbt(data, size)
296 {
297 }
298 
300 {
301  if (m_dbt.get_data() != nullptr) {
302  // Clear memory, e.g. in case it was a private key
303  memory_cleanse(m_dbt.get_data(), m_dbt.get_size());
304  // under DB_DBT_MALLOC, data is malloced by the Dbt, but must be
305  // freed by the caller.
306  // https://docs.oracle.com/cd/E17275_01/html/api_reference/C/dbt.html
307  if (m_dbt.get_flags() & DB_DBT_MALLOC) {
308  free(m_dbt.get_data());
309  }
310  }
311 }
312 
314 {
315  return m_dbt.get_data();
316 }
317 
319 {
320  return m_dbt.get_size();
321 }
322 
323 BerkeleyBatch::SafeDbt::operator Dbt*()
324 {
325  return &m_dbt;
326 }
327 
328 bool BerkeleyBatch::Recover(const fs::path& file_path, void *callbackDataIn, bool (*recoverKVcallback)(void* callbackData, CDataStream ssKey, CDataStream ssValue), std::string& newFilename)
329 {
330  std::string filename;
331  std::shared_ptr<BerkeleyEnvironment> env = GetWalletEnv(file_path, filename);
332 
333  // Recovery procedure:
334  // move wallet file to walletfilename.timestamp.bak
335  // Call Salvage with fAggressive=true to
336  // get as much data as possible.
337  // Rewrite salvaged data to fresh wallet file
338  // Set -rescan so any missing transactions will be
339  // found.
340  int64_t now = GetTime();
341  newFilename = strprintf("%s.%d.bak", filename, now);
342 
343  int result = env->dbenv->dbrename(nullptr, filename.c_str(), nullptr,
344  newFilename.c_str(), DB_AUTO_COMMIT);
345  if (result == 0)
346  LogPrintf("Renamed %s to %s\n", filename, newFilename);
347  else
348  {
349  LogPrintf("Failed to rename %s to %s\n", filename, newFilename);
350  return false;
351  }
352 
353  std::vector<BerkeleyEnvironment::KeyValPair> salvagedData;
354  bool fSuccess = env->Salvage(newFilename, true, salvagedData);
355  if (salvagedData.empty())
356  {
357  LogPrintf("Salvage(aggressive) found no records in %s.\n", newFilename);
358  return false;
359  }
360  LogPrintf("Salvage(aggressive) found %u records\n", salvagedData.size());
361 
362  std::unique_ptr<Db> pdbCopy = MakeUnique<Db>(env->dbenv.get(), 0);
363  int ret = pdbCopy->open(nullptr, // Txn pointer
364  filename.c_str(), // Filename
365  "main", // Logical db name
366  DB_BTREE, // Database type
367  DB_CREATE, // Flags
368  0);
369  if (ret > 0) {
370  LogPrintf("Cannot create database file %s\n", filename);
371  pdbCopy->close(0);
372  return false;
373  }
374 
375  DbTxn* ptxn = env->TxnBegin();
376  for (BerkeleyEnvironment::KeyValPair& row : salvagedData)
377  {
378  if (recoverKVcallback)
379  {
380  CDataStream ssKey(row.first, SER_DISK, CLIENT_VERSION);
381  CDataStream ssValue(row.second, SER_DISK, CLIENT_VERSION);
382  if (!(*recoverKVcallback)(callbackDataIn, ssKey, ssValue))
383  continue;
384  }
385  Dbt datKey(&row.first[0], row.first.size());
386  Dbt datValue(&row.second[0], row.second.size());
387  int ret2 = pdbCopy->put(ptxn, &datKey, &datValue, DB_NOOVERWRITE);
388  if (ret2 > 0)
389  fSuccess = false;
390  }
391  ptxn->commit(0);
392  pdbCopy->close(0);
393 
394  return fSuccess;
395 }
396 
397 bool BerkeleyBatch::VerifyEnvironment(const fs::path& file_path, std::string& errorStr)
398 {
399  std::string walletFile;
400  std::shared_ptr<BerkeleyEnvironment> env = GetWalletEnv(file_path, walletFile);
401  fs::path walletDir = env->Directory();
402 
403  LogPrintf("Using BerkeleyDB version %s\n", DbEnv::version(nullptr, nullptr, nullptr));
404  LogPrintf("Using wallet %s\n", file_path.string());
405 
406  if (!env->Open(true /* retry */)) {
407  errorStr = strprintf(_("Error initializing wallet database environment %s!"), walletDir);
408  return false;
409  }
410 
411  return true;
412 }
413 
414 bool BerkeleyBatch::VerifyDatabaseFile(const fs::path& file_path, std::string& warningStr, std::string& errorStr, BerkeleyEnvironment::recoverFunc_type recoverFunc)
415 {
416  std::string walletFile;
417  std::shared_ptr<BerkeleyEnvironment> env = GetWalletEnv(file_path, walletFile);
418  fs::path walletDir = env->Directory();
419 
420  if (fs::exists(walletDir / walletFile))
421  {
422  std::string backup_filename;
423  BerkeleyEnvironment::VerifyResult r = env->Verify(walletFile, recoverFunc, backup_filename);
425  {
426  warningStr = strprintf(_("Warning: Wallet file corrupt, data salvaged!"
427  " Original %s saved as %s in %s; if"
428  " your balance or transactions are incorrect you should"
429  " restore from a backup."),
430  walletFile, backup_filename, walletDir);
431  }
433  {
434  errorStr = strprintf(_("%s corrupt, salvage failed"), walletFile);
435  return false;
436  }
437  }
438  // also return true if files does not exists
439  return true;
440 }
441 
442 /* End of headers, beginning of key/value data */
443 static const char *HEADER_END = "HEADER=END";
444 /* End of key/value data */
445 static const char *DATA_END = "DATA=END";
446 
447 bool BerkeleyEnvironment::Salvage(const std::string& strFile, bool fAggressive, std::vector<BerkeleyEnvironment::KeyValPair>& vResult)
448 {
449  LOCK(cs_db);
450  assert(mapFileUseCount.count(strFile) == 0);
451 
452  u_int32_t flags = DB_SALVAGE;
453  if (fAggressive)
454  flags |= DB_AGGRESSIVE;
455 
456  std::stringstream strDump;
457 
458  Db db(dbenv.get(), 0);
459  int result = db.verify(strFile.c_str(), nullptr, &strDump, flags);
460  if (result == DB_VERIFY_BAD) {
461  LogPrintf("BerkeleyEnvironment::Salvage: Database salvage found errors, all data may not be recoverable.\n");
462  if (!fAggressive) {
463  LogPrintf("BerkeleyEnvironment::Salvage: Rerun with aggressive mode to ignore errors and continue.\n");
464  return false;
465  }
466  }
467  if (result != 0 && result != DB_VERIFY_BAD) {
468  LogPrintf("BerkeleyEnvironment::Salvage: Database salvage failed with result %d.\n", result);
469  return false;
470  }
471 
472  // Format of bdb dump is ascii lines:
473  // header lines...
474  // HEADER=END
475  // hexadecimal key
476  // hexadecimal value
477  // ... repeated
478  // DATA=END
479 
480  std::string strLine;
481  while (!strDump.eof() && strLine != HEADER_END)
482  getline(strDump, strLine); // Skip past header
483 
484  std::string keyHex, valueHex;
485  while (!strDump.eof() && keyHex != DATA_END) {
486  getline(strDump, keyHex);
487  if (keyHex != DATA_END) {
488  if (strDump.eof())
489  break;
490  getline(strDump, valueHex);
491  if (valueHex == DATA_END) {
492  LogPrintf("BerkeleyEnvironment::Salvage: WARNING: Number of keys in data does not match number of values.\n");
493  break;
494  }
495  vResult.push_back(make_pair(ParseHex(keyHex), ParseHex(valueHex)));
496  }
497  }
498 
499  if (keyHex != DATA_END) {
500  LogPrintf("BerkeleyEnvironment::Salvage: WARNING: Unexpected end of file while reading salvage output.\n");
501  return false;
502  }
503 
504  return (result == 0);
505 }
506 
507 
509 {
510  dbenv->txn_checkpoint(0, 0, 0);
511  if (fMockDb)
512  return;
513  dbenv->lsn_reset(strFile.c_str(), 0);
514 }
515 
516 
517 BerkeleyBatch::BerkeleyBatch(BerkeleyDatabase& database, const char* pszMode, bool fFlushOnCloseIn) : pdb(nullptr), activeTxn(nullptr)
518 {
519  fReadOnly = (!strchr(pszMode, '+') && !strchr(pszMode, 'w'));
520  fFlushOnClose = fFlushOnCloseIn;
521  env = database.env.get();
522  if (database.IsDummy()) {
523  return;
524  }
525  const std::string &strFilename = database.strFile;
526 
527  bool fCreate = strchr(pszMode, 'c') != nullptr;
528  unsigned int nFlags = DB_THREAD;
529  if (fCreate)
530  nFlags |= DB_CREATE;
531 
532  {
533  LOCK(cs_db);
534  if (!env->Open(false /* retry */))
535  throw std::runtime_error("BerkeleyBatch: Failed to open database environment.");
536 
537  pdb = database.m_db.get();
538  if (pdb == nullptr) {
539  int ret;
540  std::unique_ptr<Db> pdb_temp = MakeUnique<Db>(env->dbenv.get(), 0);
541 
542  bool fMockDb = env->IsMock();
543  if (fMockDb) {
544  DbMpoolFile* mpf = pdb_temp->get_mpf();
545  ret = mpf->set_flags(DB_MPOOL_NOFILE, 1);
546  if (ret != 0) {
547  throw std::runtime_error(strprintf("BerkeleyBatch: Failed to configure for no temp file backing for database %s", strFilename));
548  }
549  }
550 
551  ret = pdb_temp->open(nullptr, // Txn pointer
552  fMockDb ? nullptr : strFilename.c_str(), // Filename
553  fMockDb ? strFilename.c_str() : "main", // Logical db name
554  DB_BTREE, // Database type
555  nFlags, // Flags
556  0);
557 
558  if (ret != 0) {
559  throw std::runtime_error(strprintf("BerkeleyBatch: Error %d, can't open database %s", ret, strFilename));
560  }
561 
562  // Call CheckUniqueFileid on the containing BDB environment to
563  // avoid BDB data consistency bugs that happen when different data
564  // files in the same environment have the same fileid.
565  //
566  // Also call CheckUniqueFileid on all the other g_dbenvs to prevent
567  // bitcoin from opening the same data file through another
568  // environment when the file is referenced through equivalent but
569  // not obviously identical symlinked or hard linked or bind mounted
570  // paths. In the future a more relaxed check for equal inode and
571  // device ids could be done instead, which would allow opening
572  // different backup copies of a wallet at the same time. Maybe even
573  // more ideally, an exclusive lock for accessing the database could
574  // be implemented, so no equality checks are needed at all. (Newer
575  // versions of BDB have an set_lk_exclusive method for this
576  // purpose, but the older version we use does not.)
577  for (const auto& env : g_dbenvs) {
578  CheckUniqueFileid(*env.second.lock().get(), strFilename, *pdb_temp, this->env->m_fileids[strFilename]);
579  }
580 
581  pdb = pdb_temp.release();
582  database.m_db.reset(pdb);
583 
584  if (fCreate && !Exists(std::string("version"))) {
585  bool fTmp = fReadOnly;
586  fReadOnly = false;
588  fReadOnly = fTmp;
589  }
590  }
591  ++env->mapFileUseCount[strFilename];
592  strFile = strFilename;
593  }
594 }
595 
597 {
598  if (activeTxn)
599  return;
600 
601  // Flush database activity from memory pool to disk log
602  unsigned int nMinutes = 0;
603  if (fReadOnly)
604  nMinutes = 1;
605 
606  if (env) { // env is nullptr for dummy databases (i.e. in tests). Don't actually flush if env is nullptr so we don't segfault
607  env->dbenv->txn_checkpoint(nMinutes ? gArgs.GetArg("-dblogsize", DEFAULT_WALLET_DBLOGSIZE) * 1024 : 0, nMinutes, 0);
608  }
609 }
610 
612 {
613  ++nUpdateCounter;
614 }
615 
617 {
618  if (!pdb)
619  return;
620  if (activeTxn)
621  activeTxn->abort();
622  activeTxn = nullptr;
623  pdb = nullptr;
624 
625  if (fFlushOnClose)
626  Flush();
627 
628  {
629  LOCK(cs_db);
630  --env->mapFileUseCount[strFile];
631  }
632  env->m_db_in_use.notify_all();
633 }
634 
635 void BerkeleyEnvironment::CloseDb(const std::string& strFile)
636 {
637  {
638  LOCK(cs_db);
639  auto it = m_databases.find(strFile);
640  assert(it != m_databases.end());
641  BerkeleyDatabase& database = it->second.get();
642  if (database.m_db) {
643  // Close the database handle
644  database.m_db->close(0);
645  database.m_db.reset();
646  }
647  }
648 }
649 
651 {
652  // Make sure that no Db's are in use
653  AssertLockNotHeld(cs_db);
654  std::unique_lock<CCriticalSection> lock(cs_db);
655  m_db_in_use.wait(lock, [this](){
656  for (auto& count : mapFileUseCount) {
657  if (count.second > 0) return false;
658  }
659  return true;
660  });
661 
662  std::vector<std::string> filenames;
663  for (auto it : m_databases) {
664  filenames.push_back(it.first);
665  }
666  // Close the individual Db's
667  for (const std::string& filename : filenames) {
668  CloseDb(filename);
669  }
670  // Reset the environment
671  Flush(true); // This will flush and close the environment
672  Reset();
673  Open(true);
674 }
675 
676 bool BerkeleyBatch::Rewrite(BerkeleyDatabase& database, const char* pszSkip)
677 {
678  if (database.IsDummy()) {
679  return true;
680  }
681  BerkeleyEnvironment *env = database.env.get();
682  const std::string& strFile = database.strFile;
683  while (true) {
684  {
685  LOCK(cs_db);
686  if (!env->mapFileUseCount.count(strFile) || env->mapFileUseCount[strFile] == 0) {
687  // Flush log data to the dat file
688  env->CloseDb(strFile);
689  env->CheckpointLSN(strFile);
690  env->mapFileUseCount.erase(strFile);
691 
692  bool fSuccess = true;
693  LogPrintf("BerkeleyBatch::Rewrite: Rewriting %s...\n", strFile);
694  std::string strFileRes = strFile + ".rewrite";
695  { // surround usage of db with extra {}
696  BerkeleyBatch db(database, "r");
697  std::unique_ptr<Db> pdbCopy = MakeUnique<Db>(env->dbenv.get(), 0);
698 
699  int ret = pdbCopy->open(nullptr, // Txn pointer
700  strFileRes.c_str(), // Filename
701  "main", // Logical db name
702  DB_BTREE, // Database type
703  DB_CREATE, // Flags
704  0);
705  if (ret > 0) {
706  LogPrintf("BerkeleyBatch::Rewrite: Can't create database file %s\n", strFileRes);
707  fSuccess = false;
708  }
709 
710  Dbc* pcursor = db.GetCursor();
711  if (pcursor)
712  while (fSuccess) {
715  int ret1 = db.ReadAtCursor(pcursor, ssKey, ssValue);
716  if (ret1 == DB_NOTFOUND) {
717  pcursor->close();
718  break;
719  } else if (ret1 != 0) {
720  pcursor->close();
721  fSuccess = false;
722  break;
723  }
724  if (pszSkip &&
725  strncmp(ssKey.data(), pszSkip, std::min(ssKey.size(), strlen(pszSkip))) == 0)
726  continue;
727  if (strncmp(ssKey.data(), "\x07version", 8) == 0) {
728  // Update version:
729  ssValue.clear();
730  ssValue << CLIENT_VERSION;
731  }
732  Dbt datKey(ssKey.data(), ssKey.size());
733  Dbt datValue(ssValue.data(), ssValue.size());
734  int ret2 = pdbCopy->put(nullptr, &datKey, &datValue, DB_NOOVERWRITE);
735  if (ret2 > 0)
736  fSuccess = false;
737  }
738  if (fSuccess) {
739  db.Close();
740  env->CloseDb(strFile);
741  if (pdbCopy->close(0))
742  fSuccess = false;
743  } else {
744  pdbCopy->close(0);
745  }
746  }
747  if (fSuccess) {
748  Db dbA(env->dbenv.get(), 0);
749  if (dbA.remove(strFile.c_str(), nullptr, 0))
750  fSuccess = false;
751  Db dbB(env->dbenv.get(), 0);
752  if (dbB.rename(strFileRes.c_str(), nullptr, strFile.c_str(), 0))
753  fSuccess = false;
754  }
755  if (!fSuccess)
756  LogPrintf("BerkeleyBatch::Rewrite: Failed to rewrite database file %s\n", strFileRes);
757  return fSuccess;
758  }
759  }
760  MilliSleep(100);
761  }
762 }
763 
764 
765 void BerkeleyEnvironment::Flush(bool fShutdown)
766 {
767  int64_t nStart = GetTimeMillis();
768  // Flush log data to the actual data file on all files that are not in use
769  LogPrint(BCLog::DB, "BerkeleyEnvironment::Flush: [%s] Flush(%s)%s\n", strPath, fShutdown ? "true" : "false", fDbEnvInit ? "" : " database not started");
770  if (!fDbEnvInit)
771  return;
772  {
773  LOCK(cs_db);
774  std::map<std::string, int>::iterator mi = mapFileUseCount.begin();
775  while (mi != mapFileUseCount.end()) {
776  std::string strFile = (*mi).first;
777  int nRefCount = (*mi).second;
778  LogPrint(BCLog::DB, "BerkeleyEnvironment::Flush: Flushing %s (refcount = %d)...\n", strFile, nRefCount);
779  if (nRefCount == 0) {
780  // Move log data to the dat file
781  CloseDb(strFile);
782  LogPrint(BCLog::DB, "BerkeleyEnvironment::Flush: %s checkpoint\n", strFile);
783  dbenv->txn_checkpoint(0, 0, 0);
784  LogPrint(BCLog::DB, "BerkeleyEnvironment::Flush: %s detach\n", strFile);
785  if (!fMockDb)
786  dbenv->lsn_reset(strFile.c_str(), 0);
787  LogPrint(BCLog::DB, "BerkeleyEnvironment::Flush: %s closed\n", strFile);
788  mapFileUseCount.erase(mi++);
789  } else
790  mi++;
791  }
792  LogPrint(BCLog::DB, "BerkeleyEnvironment::Flush: Flush(%s)%s took %15dms\n", fShutdown ? "true" : "false", fDbEnvInit ? "" : " database not started", GetTimeMillis() - nStart);
793  if (fShutdown) {
794  char** listp;
795  if (mapFileUseCount.empty()) {
796  dbenv->log_archive(&listp, DB_ARCH_REMOVE);
797  Close();
798  if (!fMockDb) {
799  fs::remove_all(fs::path(strPath) / "database");
800  }
801  }
802  }
803  }
804 }
805 
807 {
808  if (database.IsDummy()) {
809  return true;
810  }
811  bool ret = false;
812  BerkeleyEnvironment *env = database.env.get();
813  const std::string& strFile = database.strFile;
814  TRY_LOCK(cs_db, lockDb);
815  if (lockDb)
816  {
817  // Don't do this if any databases are in use
818  int nRefCount = 0;
819  std::map<std::string, int>::iterator mit = env->mapFileUseCount.begin();
820  while (mit != env->mapFileUseCount.end())
821  {
822  nRefCount += (*mit).second;
823  mit++;
824  }
825 
826  if (nRefCount == 0)
827  {
828  boost::this_thread::interruption_point();
829  std::map<std::string, int>::iterator mi = env->mapFileUseCount.find(strFile);
830  if (mi != env->mapFileUseCount.end())
831  {
832  LogPrint(BCLog::DB, "Flushing %s\n", strFile);
833  int64_t nStart = GetTimeMillis();
834 
835  // Flush wallet file so it's self contained
836  env->CloseDb(strFile);
837  env->CheckpointLSN(strFile);
838 
839  env->mapFileUseCount.erase(mi++);
840  LogPrint(BCLog::DB, "Flushed %s %dms\n", strFile, GetTimeMillis() - nStart);
841  ret = true;
842  }
843  }
844  }
845 
846  return ret;
847 }
848 
849 bool BerkeleyDatabase::Rewrite(const char* pszSkip)
850 {
851  return BerkeleyBatch::Rewrite(*this, pszSkip);
852 }
853 
854 bool BerkeleyDatabase::Backup(const std::string& strDest)
855 {
856  if (IsDummy()) {
857  return false;
858  }
859  while (true)
860  {
861  {
862  LOCK(cs_db);
863  if (!env->mapFileUseCount.count(strFile) || env->mapFileUseCount[strFile] == 0)
864  {
865  // Flush log data to the dat file
866  env->CloseDb(strFile);
867  env->CheckpointLSN(strFile);
868  env->mapFileUseCount.erase(strFile);
869 
870  // Copy wallet file
871  fs::path pathSrc = env->Directory() / strFile;
872  fs::path pathDest(strDest);
873  if (fs::is_directory(pathDest))
874  pathDest /= strFile;
875 
876  try {
877  if (fs::equivalent(pathSrc, pathDest)) {
878  LogPrintf("cannot backup to wallet source file %s\n", pathDest.string());
879  return false;
880  }
881 
882  fs::copy_file(pathSrc, pathDest, fs::copy_option::overwrite_if_exists);
883  LogPrintf("copied %s to %s\n", strFile, pathDest.string());
884  return true;
885  } catch (const fs::filesystem_error& e) {
886  LogPrintf("error copying %s to %s - %s\n", strFile, pathDest.string(), fsbridge::get_filesystem_error_message(e));
887  return false;
888  }
889  }
890  }
891  MilliSleep(100);
892  }
893 }
894 
895 void BerkeleyDatabase::Flush(bool shutdown)
896 {
897  if (!IsDummy()) {
898  env->Flush(shutdown);
899  if (shutdown) {
900  LOCK(cs_db);
901  g_dbenvs.erase(env->Directory().string());
902  env = nullptr;
903  } else {
904  // TODO: To avoid g_dbenvs.erase erasing the environment prematurely after the
905  // first database shutdown when multiple databases are open in the same
906  // environment, should replace raw database `env` pointers with shared or weak
907  // pointers, or else separate the database and environment shutdowns so
908  // environments can be shut down after databases.
909  env->m_fileids.erase(strFile);
910  }
911  }
912 }
913 
915 {
916  if (!IsDummy()) {
917  env->ReloadDbEnv();
918  }
919 }
void Flush(bool shutdown)
Make sure all changes are flushed to disk.
Definition: db.cpp:895
bool fReadOnly
Definition: db.h:228
BerkeleyEnvironment()
Construct an in-memory mock Berkeley environment for testing and as a place-holder for g_dbenvs empla...
Definition: db.cpp:241
bool fDbEnvInit
Definition: db.h:39
FILE * fopen(const fs::path &p, const char *mode)
Definition: fs.cpp:15
void Close()
Definition: db.cpp:116
#define TRY_LOCK(cs, name)
Definition: sync.h:186
VerifyResult
Verify that database file strFile is OK.
Definition: db.h:68
std::unordered_map< std::string, WalletDatabaseFileId > m_fileids
Definition: db.h:49
int64_t GetTimeMillis()
Returns the system time (not mockable)
Definition: time.cpp:54
void Close()
Definition: db.cpp:616
#define strprintf
Format arguments and return the string or write to given std::ostream (see tinyformat::format doc for...
Definition: tinyformat.h:1067
bool fFlushOnClose
Definition: db.h:229
bool TryCreateDirectories(const fs::path &p)
Ignores exceptions thrown by Boost&#39;s create_directories if the requested directory exists...
Definition: system.cpp:975
std::vector< unsigned char > ParseHex(const char *psz)
bool Exists(const K &key)
Definition: db.h:325
void MilliSleep(int64_t n)
Definition: time.cpp:75
value_type * data()
Definition: streams.h:303
An instance of this class represents one database.
Definition: db.h:113
static void LogPrintf(const char *fmt, const Args &... args)
Definition: logging.h:144
bool IsDatabaseLoaded(const std::string &db_filename) const
Definition: db.h:59
Dbc * GetCursor()
Definition: db.h:341
Double ended buffer combining vector and stream-like interfaces.
Definition: streams.h:203
void ReloadDbEnv()
Definition: db.cpp:914
bool Backup(const std::string &strDest)
Back up the entire database to a file.
Definition: db.cpp:854
bool GetBoolArg(const std::string &strArg, bool fDefault) const
Return boolean argument or default value.
Definition: system.cpp:510
static const char * filenames[]
Definition: unitester.cpp:80
fs::path Directory() const
Definition: db.h:60
bool Open(bool retry)
Definition: db.cpp:166
u_int32_t get_size() const
Definition: db.cpp:318
bool IsWalletLoaded(const fs::path &wallet_path)
Return whether a wallet database is currently loaded.
Definition: db.cpp:71
static bool PeriodicFlush(BerkeleyDatabase &database)
Definition: db.cpp:806
int ReadAtCursor(Dbc *pcursor, CDataStream &ssKey, CDataStream &ssValue)
Definition: db.h:352
std::shared_ptr< BerkeleyEnvironment > env
Pointer to shared database environment.
Definition: db.h:186
std::map< std::string, int > mapFileUseCount
Definition: db.h:47
std::unique_ptr< DbEnv > dbenv
Definition: db.h:46
void CheckpointLSN(const std::string &strFile)
Definition: db.cpp:508
static const char * DATA_END
Definition: db.cpp:445
std::pair< std::vector< unsigned char >, std::vector< unsigned char > > KeyValPair
Salvage data from a file that Verify says is bad.
Definition: db.h:80
bool LockDirectory(const fs::path &directory, const std::string lockfile_name, bool probe_only)
Definition: system.cpp:83
void memory_cleanse(void *ptr, size_t len)
Secure overwrite a buffer (possibly containing secret data) with zero-bytes.
Definition: cleanse.cpp:14
size_type size() const
Definition: streams.h:294
std::string strFile
Definition: db.h:226
static void SplitWalletPath(const fs::path &wallet_path, fs::path &env_directory, std::string &database_filename)
Definition: db.cpp:55
bool Rewrite(const char *pszSkip=nullptr)
Rewrite the entire database on disk, with the exception of key pszSkip if non-zero.
Definition: db.cpp:849
#define LOCK(cs)
Definition: sync.h:182
static const char * HEADER_END
Definition: db.cpp:443
bool WriteVersion(int nVersion)
Definition: db.h:408
fs::path WalletDataFilePath(const fs::path &wallet_path)
Given a wallet directory path or legacy file path, return path to main data file in the wallet databa...
Definition: db.cpp:83
DbTxn * activeTxn
Definition: db.h:227
static void LogPrint(const BCLog::LogFlags &category, const Args &... args)
Definition: logging.h:159
RAII class that provides access to a Berkeley database.
Definition: db.h:202
void Flush()
Definition: db.cpp:596
std::string strFile
Definition: db.h:192
static const unsigned int DEFAULT_WALLET_DBLOGSIZE
Definition: db.h:26
bool operator==(const WalletDatabaseFileId &rhs) const
Definition: db.cpp:50
static bool VerifyEnvironment(const fs::path &file_path, std::string &errorStr)
Definition: db.cpp:397
static bool Recover(const fs::path &file_path, void *callbackDataIn, bool(*recoverKVcallback)(void *callbackData, CDataStream ssKey, CDataStream ssValue), std::string &out_backup_filename)
Definition: db.cpp:328
bool Salvage(const std::string &strFile, bool fAggressive, std::vector< KeyValPair > &vResult)
Definition: db.cpp:447
std::string _(const char *psz)
Translation function.
Definition: system.h:50
int flags
Definition: bitcoin-tx.cpp:507
void ReloadDbEnv()
Definition: db.cpp:650
static bool Rewrite(BerkeleyDatabase &database, const char *pszSkip=nullptr)
Definition: db.cpp:676
std::string get_filesystem_error_message(const fs::filesystem_error &e)
Definition: fs.cpp:103
void IncrementUpdateCounter()
Definition: db.cpp:611
std::shared_ptr< BerkeleyEnvironment > GetWalletEnv(const fs::path &wallet_path, std::string &database_filename)
Get BerkeleyEnvironment and database filename given a wallet path.
Definition: db.cpp:98
std::string strPath
Definition: db.h:43
BerkeleyBatch(BerkeleyDatabase &database, const char *pszMode="r+", bool fFlushOnCloseIn=true)
Definition: db.cpp:517
void UnlockDirectory(const fs::path &directory, const std::string &lockfile_name)
Definition: system.cpp:107
std::string GetArg(const std::string &strArg, const std::string &strDefault) const
Return string argument or default value.
Definition: system.cpp:494
u_int8_t value[DB_FILE_ID_LEN]
Definition: db.h:30
std::string HexStr(const T itbegin, const T itend)
Definition: strencodings.h:125
ArgsManager gArgs
Definition: system.cpp:72
void Flush(bool fShutdown)
Definition: db.cpp:765
Db * pdb
Definition: db.h:225
#define AssertLockNotHeld(cs)
Definition: sync.h:71
static int count
Definition: tests.c:45
#define GUARDED_BY(x)
Definition: threadsafety.h:38
bool(* recoverFunc_type)(const fs::path &file_path, std::string &out_backup_filename)
Definition: db.h:71
VerifyResult Verify(const std::string &strFile, recoverFunc_type recoverFunc, std::string &out_backup_filename)
Definition: db.cpp:272
static const bool DEFAULT_WALLET_PRIVDB
Definition: db.h:27
void clear()
Definition: streams.h:300
static bool VerifyDatabaseFile(const fs::path &file_path, std::string &warningStr, std::string &errorStr, BerkeleyEnvironment::recoverFunc_type recoverFunc)
Definition: db.cpp:414
void Reset()
Definition: db.cpp:147
void CloseDb(const std::string &strFile)
Definition: db.cpp:635
bool IsMock() const
Definition: db.h:57
~BerkeleyEnvironment()
Definition: db.cpp:159
int64_t GetTime()
Return system time (or mocked time, if set)
Definition: time.cpp:20
auto it
Definition: validation.cpp:360
bool fMockDb
Definition: db.h:40
static const int CLIENT_VERSION
bitcoind-res.rc includes this file, but it cannot cope with real c++ code.
Definition: clientversion.h:38
std::unique_ptr< Db > m_db
Database pointer.
Definition: db.h:189
const void * get_data() const
Definition: db.cpp:313
std::condition_variable_any m_db_in_use
Definition: db.h:50
bool IsDummy()
Return whether this database handle is a dummy for testing.
Definition: db.h:198