Bitcoin Core  0.18.99
P2P Digital Currency
db.cpp
Go to the documentation of this file.
1 // Copyright (c) 2009-2010 Satoshi Nakamoto
2 // Copyright (c) 2009-2019 The Bitcoin Core developers
3 // Distributed under the MIT software license, see the accompanying
4 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
5 
6 #include <wallet/db.h>
7 
8 #include <addrman.h>
9 #include <hash.h>
10 #include <protocol.h>
11 #include <util/strencodings.h>
12 #include <wallet/walletutil.h>
13 
14 #include <stdint.h>
15 
16 #ifndef WIN32
17 #include <sys/stat.h>
18 #endif
19 
20 #include <boost/thread.hpp>
21 
22 namespace {
23 
33 void CheckUniqueFileid(const BerkeleyEnvironment& env, const std::string& filename, Db& db, WalletDatabaseFileId& fileid)
34 {
35  if (env.IsMock()) return;
36 
37  int ret = db.get_mpf()->get_fileid(fileid.value);
38  if (ret != 0) {
39  throw std::runtime_error(strprintf("BerkeleyBatch: Can't open database %s (get_fileid failed with %d)", filename, ret));
40  }
41 
42  for (const auto& item : env.m_fileids) {
43  if (fileid == item.second && &fileid != &item.second) {
44  throw std::runtime_error(strprintf("BerkeleyBatch: Can't open database %s (duplicates fileid %s from %s)", filename,
45  HexStr(std::begin(item.second.value), std::end(item.second.value)), item.first));
46  }
47  }
48 }
49 
50 CCriticalSection cs_db;
51 std::map<std::string, std::weak_ptr<BerkeleyEnvironment>> g_dbenvs GUARDED_BY(cs_db);
52 } // namespace
53 
55 {
56  return memcmp(value, &rhs.value, sizeof(value)) == 0;
57 }
58 
59 static void SplitWalletPath(const fs::path& wallet_path, fs::path& env_directory, std::string& database_filename)
60 {
61  if (fs::is_regular_file(wallet_path)) {
62  // Special case for backwards compatibility: if wallet path points to an
63  // existing file, treat it as the path to a BDB data file in a parent
64  // directory that also contains BDB log files.
65  env_directory = wallet_path.parent_path();
66  database_filename = wallet_path.filename().string();
67  } else {
68  // Normal case: Interpret wallet path as a directory path containing
69  // data and log files.
70  env_directory = wallet_path;
71  database_filename = "wallet.dat";
72  }
73 }
74 
75 bool IsWalletLoaded(const fs::path& wallet_path)
76 {
77  fs::path env_directory;
78  std::string database_filename;
79  SplitWalletPath(wallet_path, env_directory, database_filename);
80  LOCK(cs_db);
81  auto env = g_dbenvs.find(env_directory.string());
82  if (env == g_dbenvs.end()) return false;
83  auto database = env->second.lock();
84  return database && database->IsDatabaseLoaded(database_filename);
85 }
86 
87 fs::path WalletDataFilePath(const fs::path& wallet_path)
88 {
89  fs::path env_directory;
90  std::string database_filename;
91  SplitWalletPath(wallet_path, env_directory, database_filename);
92  return env_directory / database_filename;
93 }
94 
102 std::shared_ptr<BerkeleyEnvironment> GetWalletEnv(const fs::path& wallet_path, std::string& database_filename)
103 {
104  fs::path env_directory;
105  SplitWalletPath(wallet_path, env_directory, database_filename);
106  LOCK(cs_db);
107  auto inserted = g_dbenvs.emplace(env_directory.string(), std::weak_ptr<BerkeleyEnvironment>());
108  if (inserted.second) {
109  auto env = std::make_shared<BerkeleyEnvironment>(env_directory.string());
110  inserted.first->second = env;
111  return env;
112  }
113  return inserted.first->second.lock();
114 }
115 
116 //
117 // BerkeleyBatch
118 //
119 
121 {
122  if (!fDbEnvInit)
123  return;
124 
125  fDbEnvInit = false;
126 
127  for (auto& db : m_databases) {
128  auto count = mapFileUseCount.find(db.first);
129  assert(count == mapFileUseCount.end() || count->second == 0);
130  BerkeleyDatabase& database = db.second.get();
131  if (database.m_db) {
132  database.m_db->close(0);
133  database.m_db.reset();
134  }
135  }
136 
137  FILE* error_file = nullptr;
138  dbenv->get_errfile(&error_file);
139 
140  int ret = dbenv->close(0);
141  if (ret != 0)
142  LogPrintf("BerkeleyEnvironment::Close: Error %d closing database environment: %s\n", ret, DbEnv::strerror(ret));
143  if (!fMockDb)
144  DbEnv((u_int32_t)0).remove(strPath.c_str(), 0);
145 
146  if (error_file) fclose(error_file);
147 
148  UnlockDirectory(strPath, ".walletlock");
149 }
150 
152 {
153  dbenv.reset(new DbEnv(DB_CXX_NO_EXCEPTIONS));
154  fDbEnvInit = false;
155  fMockDb = false;
156 }
157 
158 BerkeleyEnvironment::BerkeleyEnvironment(const fs::path& dir_path) : strPath(dir_path.string())
159 {
160  Reset();
161 }
162 
164 {
165  LOCK(cs_db);
166  g_dbenvs.erase(strPath);
167  Close();
168 }
169 
171 {
172  if (fDbEnvInit)
173  return true;
174 
175  boost::this_thread::interruption_point();
176 
177  fs::path pathIn = strPath;
178  TryCreateDirectories(pathIn);
179  if (!LockDirectory(pathIn, ".walletlock")) {
180  LogPrintf("Cannot obtain a lock on wallet directory %s. Another instance of bitcoin may be using it.\n", strPath);
181  return false;
182  }
183 
184  fs::path pathLogDir = pathIn / "database";
185  TryCreateDirectories(pathLogDir);
186  fs::path pathErrorFile = pathIn / "db.log";
187  LogPrintf("BerkeleyEnvironment::Open: LogDir=%s ErrorFile=%s\n", pathLogDir.string(), pathErrorFile.string());
188 
189  unsigned int nEnvFlags = 0;
190  if (gArgs.GetBoolArg("-privdb", DEFAULT_WALLET_PRIVDB))
191  nEnvFlags |= DB_PRIVATE;
192 
193  dbenv->set_lg_dir(pathLogDir.string().c_str());
194  dbenv->set_cachesize(0, 0x100000, 1); // 1 MiB should be enough for just the wallet
195  dbenv->set_lg_bsize(0x10000);
196  dbenv->set_lg_max(1048576);
197  dbenv->set_lk_max_locks(40000);
198  dbenv->set_lk_max_objects(40000);
199  dbenv->set_errfile(fsbridge::fopen(pathErrorFile, "a"));
200  dbenv->set_flags(DB_AUTO_COMMIT, 1);
201  dbenv->set_flags(DB_TXN_WRITE_NOSYNC, 1);
202  dbenv->log_set_config(DB_LOG_AUTO_REMOVE, 1);
203  int ret = dbenv->open(strPath.c_str(),
204  DB_CREATE |
205  DB_INIT_LOCK |
206  DB_INIT_LOG |
207  DB_INIT_MPOOL |
208  DB_INIT_TXN |
209  DB_THREAD |
210  DB_RECOVER |
211  nEnvFlags,
212  S_IRUSR | S_IWUSR);
213  if (ret != 0) {
214  LogPrintf("BerkeleyEnvironment::Open: Error %d opening database environment: %s\n", ret, DbEnv::strerror(ret));
215  int ret2 = dbenv->close(0);
216  if (ret2 != 0) {
217  LogPrintf("BerkeleyEnvironment::Open: Error %d closing failed database environment: %s\n", ret2, DbEnv::strerror(ret2));
218  }
219  Reset();
220  if (retry) {
221  // try moving the database env out of the way
222  fs::path pathDatabaseBak = pathIn / strprintf("database.%d.bak", GetTime());
223  try {
224  fs::rename(pathLogDir, pathDatabaseBak);
225  LogPrintf("Moved old %s to %s. Retrying.\n", pathLogDir.string(), pathDatabaseBak.string());
226  } catch (const fs::filesystem_error&) {
227  // failure is ok (well, not really, but it's not worse than what we started with)
228  }
229  // try opening it again one more time
230  if (!Open(false /* retry */)) {
231  // if it still fails, it probably means we can't even create the database env
232  return false;
233  }
234  } else {
235  return false;
236  }
237  }
238 
239  fDbEnvInit = true;
240  fMockDb = false;
241  return true;
242 }
243 
246 {
247  Reset();
248 
249  boost::this_thread::interruption_point();
250 
251  LogPrint(BCLog::DB, "BerkeleyEnvironment::MakeMock\n");
252 
253  dbenv->set_cachesize(1, 0, 1);
254  dbenv->set_lg_bsize(10485760 * 4);
255  dbenv->set_lg_max(10485760);
256  dbenv->set_lk_max_locks(10000);
257  dbenv->set_lk_max_objects(10000);
258  dbenv->set_flags(DB_AUTO_COMMIT, 1);
259  dbenv->log_set_config(DB_LOG_IN_MEMORY, 1);
260  int ret = dbenv->open(nullptr,
261  DB_CREATE |
262  DB_INIT_LOCK |
263  DB_INIT_LOG |
264  DB_INIT_MPOOL |
265  DB_INIT_TXN |
266  DB_THREAD |
267  DB_PRIVATE,
268  S_IRUSR | S_IWUSR);
269  if (ret > 0)
270  throw std::runtime_error(strprintf("BerkeleyEnvironment::MakeMock: Error %d opening database environment.", ret));
271 
272  fDbEnvInit = true;
273  fMockDb = true;
274 }
275 
276 BerkeleyEnvironment::VerifyResult BerkeleyEnvironment::Verify(const std::string& strFile, recoverFunc_type recoverFunc, std::string& out_backup_filename)
277 {
278  LOCK(cs_db);
279  assert(mapFileUseCount.count(strFile) == 0);
280 
281  Db db(dbenv.get(), 0);
282  int result = db.verify(strFile.c_str(), nullptr, nullptr, 0);
283  if (result == 0)
285  else if (recoverFunc == nullptr)
287 
288  // Try to recover:
289  bool fRecovered = (*recoverFunc)(fs::path(strPath) / strFile, out_backup_filename);
291 }
292 
294 {
295  m_dbt.set_flags(DB_DBT_MALLOC);
296 }
297 
298 BerkeleyBatch::SafeDbt::SafeDbt(void* data, size_t size)
299  : m_dbt(data, size)
300 {
301 }
302 
304 {
305  if (m_dbt.get_data() != nullptr) {
306  // Clear memory, e.g. in case it was a private key
307  memory_cleanse(m_dbt.get_data(), m_dbt.get_size());
308  // under DB_DBT_MALLOC, data is malloced by the Dbt, but must be
309  // freed by the caller.
310  // https://docs.oracle.com/cd/E17275_01/html/api_reference/C/dbt.html
311  if (m_dbt.get_flags() & DB_DBT_MALLOC) {
312  free(m_dbt.get_data());
313  }
314  }
315 }
316 
318 {
319  return m_dbt.get_data();
320 }
321 
323 {
324  return m_dbt.get_size();
325 }
326 
327 BerkeleyBatch::SafeDbt::operator Dbt*()
328 {
329  return &m_dbt;
330 }
331 
332 bool BerkeleyBatch::Recover(const fs::path& file_path, void *callbackDataIn, bool (*recoverKVcallback)(void* callbackData, CDataStream ssKey, CDataStream ssValue), std::string& newFilename)
333 {
334  std::string filename;
335  std::shared_ptr<BerkeleyEnvironment> env = GetWalletEnv(file_path, filename);
336 
337  // Recovery procedure:
338  // move wallet file to walletfilename.timestamp.bak
339  // Call Salvage with fAggressive=true to
340  // get as much data as possible.
341  // Rewrite salvaged data to fresh wallet file
342  // Set -rescan so any missing transactions will be
343  // found.
344  int64_t now = GetTime();
345  newFilename = strprintf("%s.%d.bak", filename, now);
346 
347  int result = env->dbenv->dbrename(nullptr, filename.c_str(), nullptr,
348  newFilename.c_str(), DB_AUTO_COMMIT);
349  if (result == 0)
350  LogPrintf("Renamed %s to %s\n", filename, newFilename);
351  else
352  {
353  LogPrintf("Failed to rename %s to %s\n", filename, newFilename);
354  return false;
355  }
356 
357  std::vector<BerkeleyEnvironment::KeyValPair> salvagedData;
358  bool fSuccess = env->Salvage(newFilename, true, salvagedData);
359  if (salvagedData.empty())
360  {
361  LogPrintf("Salvage(aggressive) found no records in %s.\n", newFilename);
362  return false;
363  }
364  LogPrintf("Salvage(aggressive) found %u records\n", salvagedData.size());
365 
366  std::unique_ptr<Db> pdbCopy = MakeUnique<Db>(env->dbenv.get(), 0);
367  int ret = pdbCopy->open(nullptr, // Txn pointer
368  filename.c_str(), // Filename
369  "main", // Logical db name
370  DB_BTREE, // Database type
371  DB_CREATE, // Flags
372  0);
373  if (ret > 0) {
374  LogPrintf("Cannot create database file %s\n", filename);
375  pdbCopy->close(0);
376  return false;
377  }
378 
379  DbTxn* ptxn = env->TxnBegin();
380  for (BerkeleyEnvironment::KeyValPair& row : salvagedData)
381  {
382  if (recoverKVcallback)
383  {
384  CDataStream ssKey(row.first, SER_DISK, CLIENT_VERSION);
385  CDataStream ssValue(row.second, SER_DISK, CLIENT_VERSION);
386  if (!(*recoverKVcallback)(callbackDataIn, ssKey, ssValue))
387  continue;
388  }
389  Dbt datKey(&row.first[0], row.first.size());
390  Dbt datValue(&row.second[0], row.second.size());
391  int ret2 = pdbCopy->put(ptxn, &datKey, &datValue, DB_NOOVERWRITE);
392  if (ret2 > 0)
393  fSuccess = false;
394  }
395  ptxn->commit(0);
396  pdbCopy->close(0);
397 
398  return fSuccess;
399 }
400 
401 bool BerkeleyBatch::VerifyEnvironment(const fs::path& file_path, std::string& errorStr)
402 {
403  std::string walletFile;
404  std::shared_ptr<BerkeleyEnvironment> env = GetWalletEnv(file_path, walletFile);
405  fs::path walletDir = env->Directory();
406 
407  LogPrintf("Using BerkeleyDB version %s\n", DbEnv::version(nullptr, nullptr, nullptr));
408  LogPrintf("Using wallet %s\n", file_path.string());
409 
410  if (!env->Open(true /* retry */)) {
411  errorStr = strprintf(_("Error initializing wallet database environment %s!"), walletDir);
412  return false;
413  }
414 
415  return true;
416 }
417 
418 bool BerkeleyBatch::VerifyDatabaseFile(const fs::path& file_path, std::string& warningStr, std::string& errorStr, BerkeleyEnvironment::recoverFunc_type recoverFunc)
419 {
420  std::string walletFile;
421  std::shared_ptr<BerkeleyEnvironment> env = GetWalletEnv(file_path, walletFile);
422  fs::path walletDir = env->Directory();
423 
424  if (fs::exists(walletDir / walletFile))
425  {
426  std::string backup_filename;
427  BerkeleyEnvironment::VerifyResult r = env->Verify(walletFile, recoverFunc, backup_filename);
429  {
430  warningStr = strprintf(_("Warning: Wallet file corrupt, data salvaged!"
431  " Original %s saved as %s in %s; if"
432  " your balance or transactions are incorrect you should"
433  " restore from a backup."),
434  walletFile, backup_filename, walletDir);
435  }
437  {
438  errorStr = strprintf(_("%s corrupt, salvage failed"), walletFile);
439  return false;
440  }
441  }
442  // also return true if files does not exists
443  return true;
444 }
445 
446 /* End of headers, beginning of key/value data */
447 static const char *HEADER_END = "HEADER=END";
448 /* End of key/value data */
449 static const char *DATA_END = "DATA=END";
450 
451 bool BerkeleyEnvironment::Salvage(const std::string& strFile, bool fAggressive, std::vector<BerkeleyEnvironment::KeyValPair>& vResult)
452 {
453  LOCK(cs_db);
454  assert(mapFileUseCount.count(strFile) == 0);
455 
456  u_int32_t flags = DB_SALVAGE;
457  if (fAggressive)
458  flags |= DB_AGGRESSIVE;
459 
460  std::stringstream strDump;
461 
462  Db db(dbenv.get(), 0);
463  int result = db.verify(strFile.c_str(), nullptr, &strDump, flags);
464  if (result == DB_VERIFY_BAD) {
465  LogPrintf("BerkeleyEnvironment::Salvage: Database salvage found errors, all data may not be recoverable.\n");
466  if (!fAggressive) {
467  LogPrintf("BerkeleyEnvironment::Salvage: Rerun with aggressive mode to ignore errors and continue.\n");
468  return false;
469  }
470  }
471  if (result != 0 && result != DB_VERIFY_BAD) {
472  LogPrintf("BerkeleyEnvironment::Salvage: Database salvage failed with result %d.\n", result);
473  return false;
474  }
475 
476  // Format of bdb dump is ascii lines:
477  // header lines...
478  // HEADER=END
479  // hexadecimal key
480  // hexadecimal value
481  // ... repeated
482  // DATA=END
483 
484  std::string strLine;
485  while (!strDump.eof() && strLine != HEADER_END)
486  getline(strDump, strLine); // Skip past header
487 
488  std::string keyHex, valueHex;
489  while (!strDump.eof() && keyHex != DATA_END) {
490  getline(strDump, keyHex);
491  if (keyHex != DATA_END) {
492  if (strDump.eof())
493  break;
494  getline(strDump, valueHex);
495  if (valueHex == DATA_END) {
496  LogPrintf("BerkeleyEnvironment::Salvage: WARNING: Number of keys in data does not match number of values.\n");
497  break;
498  }
499  vResult.push_back(make_pair(ParseHex(keyHex), ParseHex(valueHex)));
500  }
501  }
502 
503  if (keyHex != DATA_END) {
504  LogPrintf("BerkeleyEnvironment::Salvage: WARNING: Unexpected end of file while reading salvage output.\n");
505  return false;
506  }
507 
508  return (result == 0);
509 }
510 
511 
513 {
514  dbenv->txn_checkpoint(0, 0, 0);
515  if (fMockDb)
516  return;
517  dbenv->lsn_reset(strFile.c_str(), 0);
518 }
519 
520 
521 BerkeleyBatch::BerkeleyBatch(BerkeleyDatabase& database, const char* pszMode, bool fFlushOnCloseIn) : pdb(nullptr), activeTxn(nullptr)
522 {
523  fReadOnly = (!strchr(pszMode, '+') && !strchr(pszMode, 'w'));
524  fFlushOnClose = fFlushOnCloseIn;
525  env = database.env.get();
526  if (database.IsDummy()) {
527  return;
528  }
529  const std::string &strFilename = database.strFile;
530 
531  bool fCreate = strchr(pszMode, 'c') != nullptr;
532  unsigned int nFlags = DB_THREAD;
533  if (fCreate)
534  nFlags |= DB_CREATE;
535 
536  {
537  LOCK(cs_db);
538  if (!env->Open(false /* retry */))
539  throw std::runtime_error("BerkeleyBatch: Failed to open database environment.");
540 
541  pdb = database.m_db.get();
542  if (pdb == nullptr) {
543  int ret;
544  std::unique_ptr<Db> pdb_temp = MakeUnique<Db>(env->dbenv.get(), 0);
545 
546  bool fMockDb = env->IsMock();
547  if (fMockDb) {
548  DbMpoolFile* mpf = pdb_temp->get_mpf();
549  ret = mpf->set_flags(DB_MPOOL_NOFILE, 1);
550  if (ret != 0) {
551  throw std::runtime_error(strprintf("BerkeleyBatch: Failed to configure for no temp file backing for database %s", strFilename));
552  }
553  }
554 
555  ret = pdb_temp->open(nullptr, // Txn pointer
556  fMockDb ? nullptr : strFilename.c_str(), // Filename
557  fMockDb ? strFilename.c_str() : "main", // Logical db name
558  DB_BTREE, // Database type
559  nFlags, // Flags
560  0);
561 
562  if (ret != 0) {
563  throw std::runtime_error(strprintf("BerkeleyBatch: Error %d, can't open database %s", ret, strFilename));
564  }
565 
566  // Call CheckUniqueFileid on the containing BDB environment to
567  // avoid BDB data consistency bugs that happen when different data
568  // files in the same environment have the same fileid.
569  //
570  // Also call CheckUniqueFileid on all the other g_dbenvs to prevent
571  // bitcoin from opening the same data file through another
572  // environment when the file is referenced through equivalent but
573  // not obviously identical symlinked or hard linked or bind mounted
574  // paths. In the future a more relaxed check for equal inode and
575  // device ids could be done instead, which would allow opening
576  // different backup copies of a wallet at the same time. Maybe even
577  // more ideally, an exclusive lock for accessing the database could
578  // be implemented, so no equality checks are needed at all. (Newer
579  // versions of BDB have an set_lk_exclusive method for this
580  // purpose, but the older version we use does not.)
581  for (const auto& env : g_dbenvs) {
582  CheckUniqueFileid(*env.second.lock().get(), strFilename, *pdb_temp, this->env->m_fileids[strFilename]);
583  }
584 
585  pdb = pdb_temp.release();
586  database.m_db.reset(pdb);
587 
588  if (fCreate && !Exists(std::string("version"))) {
589  bool fTmp = fReadOnly;
590  fReadOnly = false;
591  WriteVersion(CLIENT_VERSION);
592  fReadOnly = fTmp;
593  }
594  }
595  ++env->mapFileUseCount[strFilename];
596  strFile = strFilename;
597  }
598 }
599 
601 {
602  if (activeTxn)
603  return;
604 
605  // Flush database activity from memory pool to disk log
606  unsigned int nMinutes = 0;
607  if (fReadOnly)
608  nMinutes = 1;
609 
610  env->dbenv->txn_checkpoint(nMinutes ? gArgs.GetArg("-dblogsize", DEFAULT_WALLET_DBLOGSIZE) * 1024 : 0, nMinutes, 0);
611 }
612 
614 {
615  ++nUpdateCounter;
616 }
617 
619 {
620  if (!pdb)
621  return;
622  if (activeTxn)
623  activeTxn->abort();
624  activeTxn = nullptr;
625  pdb = nullptr;
626 
627  if (fFlushOnClose)
628  Flush();
629 
630  {
631  LOCK(cs_db);
632  --env->mapFileUseCount[strFile];
633  }
634  env->m_db_in_use.notify_all();
635 }
636 
637 void BerkeleyEnvironment::CloseDb(const std::string& strFile)
638 {
639  {
640  LOCK(cs_db);
641  auto it = m_databases.find(strFile);
642  assert(it != m_databases.end());
643  BerkeleyDatabase& database = it->second.get();
644  if (database.m_db) {
645  // Close the database handle
646  database.m_db->close(0);
647  database.m_db.reset();
648  }
649  }
650 }
651 
653 {
654  // Make sure that no Db's are in use
655  AssertLockNotHeld(cs_db);
656  std::unique_lock<CCriticalSection> lock(cs_db);
657  m_db_in_use.wait(lock, [this](){
658  for (auto& count : mapFileUseCount) {
659  if (count.second > 0) return false;
660  }
661  return true;
662  });
663 
664  std::vector<std::string> filenames;
665  for (auto it : m_databases) {
666  filenames.push_back(it.first);
667  }
668  // Close the individual Db's
669  for (const std::string& filename : filenames) {
670  CloseDb(filename);
671  }
672  // Reset the environment
673  Flush(true); // This will flush and close the environment
674  Reset();
675  Open(true);
676 }
677 
678 bool BerkeleyBatch::Rewrite(BerkeleyDatabase& database, const char* pszSkip)
679 {
680  if (database.IsDummy()) {
681  return true;
682  }
683  BerkeleyEnvironment *env = database.env.get();
684  const std::string& strFile = database.strFile;
685  while (true) {
686  {
687  LOCK(cs_db);
688  if (!env->mapFileUseCount.count(strFile) || env->mapFileUseCount[strFile] == 0) {
689  // Flush log data to the dat file
690  env->CloseDb(strFile);
691  env->CheckpointLSN(strFile);
692  env->mapFileUseCount.erase(strFile);
693 
694  bool fSuccess = true;
695  LogPrintf("BerkeleyBatch::Rewrite: Rewriting %s...\n", strFile);
696  std::string strFileRes = strFile + ".rewrite";
697  { // surround usage of db with extra {}
698  BerkeleyBatch db(database, "r");
699  std::unique_ptr<Db> pdbCopy = MakeUnique<Db>(env->dbenv.get(), 0);
700 
701  int ret = pdbCopy->open(nullptr, // Txn pointer
702  strFileRes.c_str(), // Filename
703  "main", // Logical db name
704  DB_BTREE, // Database type
705  DB_CREATE, // Flags
706  0);
707  if (ret > 0) {
708  LogPrintf("BerkeleyBatch::Rewrite: Can't create database file %s\n", strFileRes);
709  fSuccess = false;
710  }
711 
712  Dbc* pcursor = db.GetCursor();
713  if (pcursor)
714  while (fSuccess) {
715  CDataStream ssKey(SER_DISK, CLIENT_VERSION);
716  CDataStream ssValue(SER_DISK, CLIENT_VERSION);
717  int ret1 = db.ReadAtCursor(pcursor, ssKey, ssValue);
718  if (ret1 == DB_NOTFOUND) {
719  pcursor->close();
720  break;
721  } else if (ret1 != 0) {
722  pcursor->close();
723  fSuccess = false;
724  break;
725  }
726  if (pszSkip &&
727  strncmp(ssKey.data(), pszSkip, std::min(ssKey.size(), strlen(pszSkip))) == 0)
728  continue;
729  if (strncmp(ssKey.data(), "\x07version", 8) == 0) {
730  // Update version:
731  ssValue.clear();
732  ssValue << CLIENT_VERSION;
733  }
734  Dbt datKey(ssKey.data(), ssKey.size());
735  Dbt datValue(ssValue.data(), ssValue.size());
736  int ret2 = pdbCopy->put(nullptr, &datKey, &datValue, DB_NOOVERWRITE);
737  if (ret2 > 0)
738  fSuccess = false;
739  }
740  if (fSuccess) {
741  db.Close();
742  env->CloseDb(strFile);
743  if (pdbCopy->close(0))
744  fSuccess = false;
745  } else {
746  pdbCopy->close(0);
747  }
748  }
749  if (fSuccess) {
750  Db dbA(env->dbenv.get(), 0);
751  if (dbA.remove(strFile.c_str(), nullptr, 0))
752  fSuccess = false;
753  Db dbB(env->dbenv.get(), 0);
754  if (dbB.rename(strFileRes.c_str(), nullptr, strFile.c_str(), 0))
755  fSuccess = false;
756  }
757  if (!fSuccess)
758  LogPrintf("BerkeleyBatch::Rewrite: Failed to rewrite database file %s\n", strFileRes);
759  return fSuccess;
760  }
761  }
762  MilliSleep(100);
763  }
764 }
765 
766 
767 void BerkeleyEnvironment::Flush(bool fShutdown)
768 {
769  int64_t nStart = GetTimeMillis();
770  // Flush log data to the actual data file on all files that are not in use
771  LogPrint(BCLog::DB, "BerkeleyEnvironment::Flush: [%s] Flush(%s)%s\n", strPath, fShutdown ? "true" : "false", fDbEnvInit ? "" : " database not started");
772  if (!fDbEnvInit)
773  return;
774  {
775  LOCK(cs_db);
776  std::map<std::string, int>::iterator mi = mapFileUseCount.begin();
777  while (mi != mapFileUseCount.end()) {
778  std::string strFile = (*mi).first;
779  int nRefCount = (*mi).second;
780  LogPrint(BCLog::DB, "BerkeleyEnvironment::Flush: Flushing %s (refcount = %d)...\n", strFile, nRefCount);
781  if (nRefCount == 0) {
782  // Move log data to the dat file
783  CloseDb(strFile);
784  LogPrint(BCLog::DB, "BerkeleyEnvironment::Flush: %s checkpoint\n", strFile);
785  dbenv->txn_checkpoint(0, 0, 0);
786  LogPrint(BCLog::DB, "BerkeleyEnvironment::Flush: %s detach\n", strFile);
787  if (!fMockDb)
788  dbenv->lsn_reset(strFile.c_str(), 0);
789  LogPrint(BCLog::DB, "BerkeleyEnvironment::Flush: %s closed\n", strFile);
790  mapFileUseCount.erase(mi++);
791  } else
792  mi++;
793  }
794  LogPrint(BCLog::DB, "BerkeleyEnvironment::Flush: Flush(%s)%s took %15dms\n", fShutdown ? "true" : "false", fDbEnvInit ? "" : " database not started", GetTimeMillis() - nStart);
795  if (fShutdown) {
796  char** listp;
797  if (mapFileUseCount.empty()) {
798  dbenv->log_archive(&listp, DB_ARCH_REMOVE);
799  Close();
800  if (!fMockDb) {
801  fs::remove_all(fs::path(strPath) / "database");
802  }
803  }
804  }
805  }
806 }
807 
809 {
810  if (database.IsDummy()) {
811  return true;
812  }
813  bool ret = false;
814  BerkeleyEnvironment *env = database.env.get();
815  const std::string& strFile = database.strFile;
816  TRY_LOCK(cs_db, lockDb);
817  if (lockDb)
818  {
819  // Don't do this if any databases are in use
820  int nRefCount = 0;
821  std::map<std::string, int>::iterator mit = env->mapFileUseCount.begin();
822  while (mit != env->mapFileUseCount.end())
823  {
824  nRefCount += (*mit).second;
825  mit++;
826  }
827 
828  if (nRefCount == 0)
829  {
830  boost::this_thread::interruption_point();
831  std::map<std::string, int>::iterator mi = env->mapFileUseCount.find(strFile);
832  if (mi != env->mapFileUseCount.end())
833  {
834  LogPrint(BCLog::DB, "Flushing %s\n", strFile);
835  int64_t nStart = GetTimeMillis();
836 
837  // Flush wallet file so it's self contained
838  env->CloseDb(strFile);
839  env->CheckpointLSN(strFile);
840 
841  env->mapFileUseCount.erase(mi++);
842  LogPrint(BCLog::DB, "Flushed %s %dms\n", strFile, GetTimeMillis() - nStart);
843  ret = true;
844  }
845  }
846  }
847 
848  return ret;
849 }
850 
851 bool BerkeleyDatabase::Rewrite(const char* pszSkip)
852 {
853  return BerkeleyBatch::Rewrite(*this, pszSkip);
854 }
855 
856 bool BerkeleyDatabase::Backup(const std::string& strDest)
857 {
858  if (IsDummy()) {
859  return false;
860  }
861  while (true)
862  {
863  {
864  LOCK(cs_db);
865  if (!env->mapFileUseCount.count(strFile) || env->mapFileUseCount[strFile] == 0)
866  {
867  // Flush log data to the dat file
868  env->CloseDb(strFile);
869  env->CheckpointLSN(strFile);
870  env->mapFileUseCount.erase(strFile);
871 
872  // Copy wallet file
873  fs::path pathSrc = env->Directory() / strFile;
874  fs::path pathDest(strDest);
875  if (fs::is_directory(pathDest))
876  pathDest /= strFile;
877 
878  try {
879  if (fs::equivalent(pathSrc, pathDest)) {
880  LogPrintf("cannot backup to wallet source file %s\n", pathDest.string());
881  return false;
882  }
883 
884  fs::copy_file(pathSrc, pathDest, fs::copy_option::overwrite_if_exists);
885  LogPrintf("copied %s to %s\n", strFile, pathDest.string());
886  return true;
887  } catch (const fs::filesystem_error& e) {
888  LogPrintf("error copying %s to %s - %s\n", strFile, pathDest.string(), fsbridge::get_filesystem_error_message(e));
889  return false;
890  }
891  }
892  }
893  MilliSleep(100);
894  }
895 }
896 
897 void BerkeleyDatabase::Flush(bool shutdown)
898 {
899  if (!IsDummy()) {
900  env->Flush(shutdown);
901  if (shutdown) {
902  LOCK(cs_db);
903  g_dbenvs.erase(env->Directory().string());
904  env = nullptr;
905  } else {
906  // TODO: To avoid g_dbenvs.erase erasing the environment prematurely after the
907  // first database shutdown when multiple databases are open in the same
908  // environment, should replace raw database `env` pointers with shared or weak
909  // pointers, or else separate the database and environment shutdowns so
910  // environments can be shut down after databases.
911  env->m_fileids.erase(strFile);
912  }
913  }
914 }
915 
917 {
918  if (!IsDummy()) {
919  env->ReloadDbEnv();
920  }
921 }
void Flush(bool shutdown)
Make sure all changes are flushed to disk.
Definition: db.cpp:897
bool fReadOnly
Definition: db.h:228
BerkeleyEnvironment()
Construct an in-memory mock Berkeley environment for testing and as a place-holder for g_dbenvs empla...
Definition: db.cpp:245
bool fDbEnvInit
Definition: db.h:39
FILE * fopen(const fs::path &p, const char *mode)
Definition: fs.cpp:15
void Close()
Definition: db.cpp:120
#define TRY_LOCK(cs, name)
Definition: sync.h:186
VerifyResult
Verify that database file strFile is OK.
Definition: db.h:68
std::unordered_map< std::string, WalletDatabaseFileId > m_fileids
Definition: db.h:49
int64_t GetTimeMillis()
Definition: time.cpp:40
void Close()
Definition: db.cpp:618
#define strprintf
Definition: tinyformat.h:1066
bool fFlushOnClose
Definition: db.h:229
bool TryCreateDirectories(const fs::path &p)
Ignores exceptions thrown by Boost&#39;s create_directories if the requested directory exists...
Definition: system.cpp:983
std::vector< unsigned char > ParseHex(const char *psz)
bool Exists(const K &key)
Definition: db.h:325
void MilliSleep(int64_t n)
Definition: time.cpp:61
value_type * data()
Definition: streams.h:303
An instance of this class represents one database.
Definition: db.h:113
bool IsDatabaseLoaded(const std::string &db_filename) const
Definition: db.h:59
Dbc * GetCursor()
Definition: db.h:341
Double ended buffer combining vector and stream-like interfaces.
Definition: streams.h:203
void ReloadDbEnv()
Definition: db.cpp:916
bool Backup(const std::string &strDest)
Back up the entire database to a file.
Definition: db.cpp:856
bool GetBoolArg(const std::string &strArg, bool fDefault) const
Return boolean argument or default value.
Definition: system.cpp:512
fs::path Directory() const
Definition: db.h:60
bool Open(bool retry)
Definition: db.cpp:170
u_int32_t get_size() const
Definition: db.cpp:322
bool IsWalletLoaded(const fs::path &wallet_path)
Return whether a wallet database is currently loaded.
Definition: db.cpp:75
static bool PeriodicFlush(BerkeleyDatabase &database)
Definition: db.cpp:808
int ReadAtCursor(Dbc *pcursor, CDataStream &ssKey, CDataStream &ssValue)
Definition: db.h:352
std::shared_ptr< BerkeleyEnvironment > env
Pointer to shared database environment.
Definition: db.h:186
std::map< std::string, int > mapFileUseCount
Definition: db.h:47
std::unique_ptr< DbEnv > dbenv
Definition: db.h:46
void CheckpointLSN(const std::string &strFile)
Definition: db.cpp:512
std::pair< std::vector< unsigned char >, std::vector< unsigned char > > KeyValPair
Salvage data from a file that Verify says is bad.
Definition: db.h:80
bool LockDirectory(const fs::path &directory, const std::string lockfile_name, bool probe_only)
Definition: system.cpp:85
void memory_cleanse(void *ptr, size_t len)
Definition: cleanse.cpp:31
size_type size() const
Definition: streams.h:294
std::string strFile
Definition: db.h:226
bool Rewrite(const char *pszSkip=nullptr)
Rewrite the entire database on disk, with the exception of key pszSkip if non-zero.
Definition: db.cpp:851
#define LOCK(cs)
Definition: sync.h:182
bool WriteVersion(int nVersion)
Definition: db.h:408
fs::path WalletDataFilePath(const fs::path &wallet_path)
Given a wallet directory path or legacy file path, return path to main data file in the wallet databa...
Definition: db.cpp:87
DbTxn * activeTxn
Definition: db.h:227
RAII class that provides access to a Berkeley database.
Definition: db.h:202
void Flush()
Definition: db.cpp:600
std::string strFile
Definition: db.h:192
bool operator==(const WalletDatabaseFileId &rhs) const
Definition: db.cpp:54
static bool VerifyEnvironment(const fs::path &file_path, std::string &errorStr)
Definition: db.cpp:401
static bool Recover(const fs::path &file_path, void *callbackDataIn, bool(*recoverKVcallback)(void *callbackData, CDataStream ssKey, CDataStream ssValue), std::string &out_backup_filename)
Definition: db.cpp:332
bool Salvage(const std::string &strFile, bool fAggressive, std::vector< KeyValPair > &vResult)
Definition: db.cpp:451
std::string _(const char *psz)
Translation function.
Definition: system.h:52
int flags
Definition: bitcoin-tx.cpp:507
void ReloadDbEnv()
Definition: db.cpp:652
static bool Rewrite(BerkeleyDatabase &database, const char *pszSkip=nullptr)
Definition: db.cpp:678
std::string get_filesystem_error_message(const fs::filesystem_error &e)
Definition: fs.cpp:103
void IncrementUpdateCounter()
Definition: db.cpp:613
std::shared_ptr< BerkeleyEnvironment > GetWalletEnv(const fs::path &wallet_path, std::string &database_filename)
Get BerkeleyEnvironment and database filename given a wallet path.
Definition: db.cpp:102
std::string strPath
Definition: db.h:43
BerkeleyBatch(BerkeleyDatabase &database, const char *pszMode="r+", bool fFlushOnCloseIn=true)
Definition: db.cpp:521
void UnlockDirectory(const fs::path &directory, const std::string &lockfile_name)
Definition: system.cpp:109
std::string GetArg(const std::string &strArg, const std::string &strDefault) const
Return string argument or default value.
Definition: system.cpp:496
u_int8_t value[DB_FILE_ID_LEN]
Definition: db.h:30
std::string HexStr(const T itbegin, const T itend)
Definition: strencodings.h:125
ArgsManager gArgs
Definition: system.cpp:74
void Flush(bool fShutdown)
Definition: db.cpp:767
Db * pdb
Definition: db.h:225
#define AssertLockNotHeld(cs)
Definition: sync.h:71
#define GUARDED_BY(x)
Definition: threadsafety.h:38
bool(* recoverFunc_type)(const fs::path &file_path, std::string &out_backup_filename)
Definition: db.h:71
VerifyResult Verify(const std::string &strFile, recoverFunc_type recoverFunc, std::string &out_backup_filename)
Definition: db.cpp:276
void clear()
Definition: streams.h:300
static bool VerifyDatabaseFile(const fs::path &file_path, std::string &warningStr, std::string &errorStr, BerkeleyEnvironment::recoverFunc_type recoverFunc)
Definition: db.cpp:418
void Reset()
Definition: db.cpp:151
int64_t GetTime()
GetTimeMicros() and GetTimeMillis() both return the system time, but in different units...
Definition: time.cpp:20
void CloseDb(const std::string &strFile)
Definition: db.cpp:637
bool IsMock() const
Definition: db.h:57
~BerkeleyEnvironment()
Definition: db.cpp:163
bool fMockDb
Definition: db.h:40
std::unique_ptr< Db > m_db
Database pointer.
Definition: db.h:189
const void * get_data() const
Definition: db.cpp:317
std::condition_variable_any m_db_in_use
Definition: db.h:50
bool IsDummy()
Return whether this database handle is a dummy for testing.
Definition: db.h:198