Bitcoin Core  0.19.99
P2P Digital Currency
txdb.cpp
Go to the documentation of this file.
1 // Copyright (c) 2009-2010 Satoshi Nakamoto
2 // Copyright (c) 2009-2018 The Bitcoin Core developers
3 // Distributed under the MIT software license, see the accompanying
4 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
5 
6 #include <txdb.h>
7 
8 #include <pow.h>
9 #include <random.h>
10 #include <shutdown.h>
11 #include <ui_interface.h>
12 #include <uint256.h>
13 #include <util/system.h>
14 #include <util/translation.h>
15 
16 #include <stdint.h>
17 
18 #include <boost/thread.hpp>
19 
20 static const char DB_COIN = 'C';
21 static const char DB_COINS = 'c';
22 static const char DB_BLOCK_FILES = 'f';
23 static const char DB_BLOCK_INDEX = 'b';
24 
25 static const char DB_BEST_BLOCK = 'B';
26 static const char DB_HEAD_BLOCKS = 'H';
27 static const char DB_FLAG = 'F';
28 static const char DB_REINDEX_FLAG = 'R';
29 static const char DB_LAST_BLOCK = 'l';
30 
31 namespace {
32 
33 struct CoinEntry {
34  COutPoint* outpoint;
35  char key;
36  explicit CoinEntry(const COutPoint* ptr) : outpoint(const_cast<COutPoint*>(ptr)), key(DB_COIN) {}
37 
38  template<typename Stream>
39  void Serialize(Stream &s) const {
40  s << key;
41  s << outpoint->hash;
42  s << VARINT(outpoint->n);
43  }
44 
45  template<typename Stream>
46  void Unserialize(Stream& s) {
47  s >> key;
48  s >> outpoint->hash;
49  s >> VARINT(outpoint->n);
50  }
51 };
52 
53 }
54 
55 CCoinsViewDB::CCoinsViewDB(fs::path ldb_path, size_t nCacheSize, bool fMemory, bool fWipe) : db(ldb_path, nCacheSize, fMemory, fWipe, true)
56 {
57 }
58 
59 bool CCoinsViewDB::GetCoin(const COutPoint &outpoint, Coin &coin) const {
60  return db.Read(CoinEntry(&outpoint), coin);
61 }
62 
63 bool CCoinsViewDB::HaveCoin(const COutPoint &outpoint) const {
64  return db.Exists(CoinEntry(&outpoint));
65 }
66 
68  uint256 hashBestChain;
69  if (!db.Read(DB_BEST_BLOCK, hashBestChain))
70  return uint256();
71  return hashBestChain;
72 }
73 
74 std::vector<uint256> CCoinsViewDB::GetHeadBlocks() const {
75  std::vector<uint256> vhashHeadBlocks;
76  if (!db.Read(DB_HEAD_BLOCKS, vhashHeadBlocks)) {
77  return std::vector<uint256>();
78  }
79  return vhashHeadBlocks;
80 }
81 
82 bool CCoinsViewDB::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) {
83  CDBBatch batch(db);
84  size_t count = 0;
85  size_t changed = 0;
86  size_t batch_size = (size_t)gArgs.GetArg("-dbbatchsize", nDefaultDbBatchSize);
87  int crash_simulate = gArgs.GetArg("-dbcrashratio", 0);
88  assert(!hashBlock.IsNull());
89 
90  uint256 old_tip = GetBestBlock();
91  if (old_tip.IsNull()) {
92  // We may be in the middle of replaying.
93  std::vector<uint256> old_heads = GetHeadBlocks();
94  if (old_heads.size() == 2) {
95  assert(old_heads[0] == hashBlock);
96  old_tip = old_heads[1];
97  }
98  }
99 
100  // In the first batch, mark the database as being in the middle of a
101  // transition from old_tip to hashBlock.
102  // A vector is used for future extensibility, as we may want to support
103  // interrupting after partial writes from multiple independent reorgs.
104  batch.Erase(DB_BEST_BLOCK);
105  batch.Write(DB_HEAD_BLOCKS, std::vector<uint256>{hashBlock, old_tip});
106 
107  for (CCoinsMap::iterator it = mapCoins.begin(); it != mapCoins.end();) {
108  if (it->second.flags & CCoinsCacheEntry::DIRTY) {
109  CoinEntry entry(&it->first);
110  if (it->second.coin.IsSpent())
111  batch.Erase(entry);
112  else
113  batch.Write(entry, it->second.coin);
114  changed++;
115  }
116  count++;
117  CCoinsMap::iterator itOld = it++;
118  mapCoins.erase(itOld);
119  if (batch.SizeEstimate() > batch_size) {
120  LogPrint(BCLog::COINDB, "Writing partial batch of %.2f MiB\n", batch.SizeEstimate() * (1.0 / 1048576.0));
121  db.WriteBatch(batch);
122  batch.Clear();
123  if (crash_simulate) {
124  static FastRandomContext rng;
125  if (rng.randrange(crash_simulate) == 0) {
126  LogPrintf("Simulating a crash. Goodbye.\n");
127  _Exit(0);
128  }
129  }
130  }
131  }
132 
133  // In the last batch, mark the database as consistent with hashBlock again.
134  batch.Erase(DB_HEAD_BLOCKS);
135  batch.Write(DB_BEST_BLOCK, hashBlock);
136 
137  LogPrint(BCLog::COINDB, "Writing final batch of %.2f MiB\n", batch.SizeEstimate() * (1.0 / 1048576.0));
138  bool ret = db.WriteBatch(batch);
139  LogPrint(BCLog::COINDB, "Committed %u changed transaction outputs (out of %u) to coin database...\n", (unsigned int)changed, (unsigned int)count);
140  return ret;
141 }
142 
144 {
145  return db.EstimateSize(DB_COIN, (char)(DB_COIN+1));
146 }
147 
148 CBlockTreeDB::CBlockTreeDB(size_t nCacheSize, bool fMemory, bool fWipe) : CDBWrapper(GetDataDir() / "blocks" / "index", nCacheSize, fMemory, fWipe) {
149 }
150 
152  return Read(std::make_pair(DB_BLOCK_FILES, nFile), info);
153 }
154 
155 bool CBlockTreeDB::WriteReindexing(bool fReindexing) {
156  if (fReindexing)
157  return Write(DB_REINDEX_FLAG, '1');
158  else
159  return Erase(DB_REINDEX_FLAG);
160 }
161 
162 void CBlockTreeDB::ReadReindexing(bool &fReindexing) {
163  fReindexing = Exists(DB_REINDEX_FLAG);
164 }
165 
167  return Read(DB_LAST_BLOCK, nFile);
168 }
169 
171 {
172  CCoinsViewDBCursor *i = new CCoinsViewDBCursor(const_cast<CDBWrapper&>(db).NewIterator(), GetBestBlock());
173  /* It seems that there are no "const iterators" for LevelDB. Since we
174  only need read operations on it, use a const-cast to get around
175  that restriction. */
176  i->pcursor->Seek(DB_COIN);
177  // Cache key of first record
178  if (i->pcursor->Valid()) {
179  CoinEntry entry(&i->keyTmp.second);
180  i->pcursor->GetKey(entry);
181  i->keyTmp.first = entry.key;
182  } else {
183  i->keyTmp.first = 0; // Make sure Valid() and GetKey() return false
184  }
185  return i;
186 }
187 
189 {
190  // Return cached key
191  if (keyTmp.first == DB_COIN) {
192  key = keyTmp.second;
193  return true;
194  }
195  return false;
196 }
197 
199 {
200  return pcursor->GetValue(coin);
201 }
202 
204 {
205  return pcursor->GetValueSize();
206 }
207 
209 {
210  return keyTmp.first == DB_COIN;
211 }
212 
214 {
215  pcursor->Next();
216  CoinEntry entry(&keyTmp.second);
217  if (!pcursor->Valid() || !pcursor->GetKey(entry)) {
218  keyTmp.first = 0; // Invalidate cached key after last record so that Valid() and GetKey() return false
219  } else {
220  keyTmp.first = entry.key;
221  }
222 }
223 
224 bool CBlockTreeDB::WriteBatchSync(const std::vector<std::pair<int, const CBlockFileInfo*> >& fileInfo, int nLastFile, const std::vector<const CBlockIndex*>& blockinfo) {
225  CDBBatch batch(*this);
226  for (std::vector<std::pair<int, const CBlockFileInfo*> >::const_iterator it=fileInfo.begin(); it != fileInfo.end(); it++) {
227  batch.Write(std::make_pair(DB_BLOCK_FILES, it->first), *it->second);
228  }
229  batch.Write(DB_LAST_BLOCK, nLastFile);
230  for (std::vector<const CBlockIndex*>::const_iterator it=blockinfo.begin(); it != blockinfo.end(); it++) {
231  batch.Write(std::make_pair(DB_BLOCK_INDEX, (*it)->GetBlockHash()), CDiskBlockIndex(*it));
232  }
233  return WriteBatch(batch, true);
234 }
235 
236 bool CBlockTreeDB::WriteFlag(const std::string &name, bool fValue) {
237  return Write(std::make_pair(DB_FLAG, name), fValue ? '1' : '0');
238 }
239 
240 bool CBlockTreeDB::ReadFlag(const std::string &name, bool &fValue) {
241  char ch;
242  if (!Read(std::make_pair(DB_FLAG, name), ch))
243  return false;
244  fValue = ch == '1';
245  return true;
246 }
247 
248 bool CBlockTreeDB::LoadBlockIndexGuts(const Consensus::Params& consensusParams, std::function<CBlockIndex*(const uint256&)> insertBlockIndex)
249 {
250  std::unique_ptr<CDBIterator> pcursor(NewIterator());
251 
252  pcursor->Seek(std::make_pair(DB_BLOCK_INDEX, uint256()));
253 
254  // Load m_block_index
255  while (pcursor->Valid()) {
256  boost::this_thread::interruption_point();
257  if (ShutdownRequested()) return false;
258  std::pair<char, uint256> key;
259  if (pcursor->GetKey(key) && key.first == DB_BLOCK_INDEX) {
260  CDiskBlockIndex diskindex;
261  if (pcursor->GetValue(diskindex)) {
262  // Construct block index object
263  CBlockIndex* pindexNew = insertBlockIndex(diskindex.GetBlockHash());
264  pindexNew->pprev = insertBlockIndex(diskindex.hashPrev);
265  pindexNew->nHeight = diskindex.nHeight;
266  pindexNew->nFile = diskindex.nFile;
267  pindexNew->nDataPos = diskindex.nDataPos;
268  pindexNew->nUndoPos = diskindex.nUndoPos;
269  pindexNew->nVersion = diskindex.nVersion;
270  pindexNew->hashMerkleRoot = diskindex.hashMerkleRoot;
271  pindexNew->nTime = diskindex.nTime;
272  pindexNew->nBits = diskindex.nBits;
273  pindexNew->nNonce = diskindex.nNonce;
274  pindexNew->nStatus = diskindex.nStatus;
275  pindexNew->nTx = diskindex.nTx;
276 
277  if (!CheckProofOfWork(pindexNew->GetBlockHash(), pindexNew->nBits, consensusParams))
278  return error("%s: CheckProofOfWork failed: %s", __func__, pindexNew->ToString());
279 
280  pcursor->Next();
281  } else {
282  return error("%s: failed to read value", __func__);
283  }
284  } else {
285  break;
286  }
287  }
288 
289  return true;
290 }
291 
292 namespace {
293 
295 class CCoins
296 {
297 public:
299  bool fCoinBase;
300 
302  std::vector<CTxOut> vout;
303 
305  int nHeight;
306 
308  CCoins() : fCoinBase(false), vout(0), nHeight(0) { }
309 
310  template<typename Stream>
311  void Unserialize(Stream &s) {
312  unsigned int nCode = 0;
313  // version
314  unsigned int nVersionDummy;
315  ::Unserialize(s, VARINT(nVersionDummy));
316  // header code
317  ::Unserialize(s, VARINT(nCode));
318  fCoinBase = nCode & 1;
319  std::vector<bool> vAvail(2, false);
320  vAvail[0] = (nCode & 2) != 0;
321  vAvail[1] = (nCode & 4) != 0;
322  unsigned int nMaskCode = (nCode / 8) + ((nCode & 6) != 0 ? 0 : 1);
323  // spentness bitmask
324  while (nMaskCode > 0) {
325  unsigned char chAvail = 0;
326  ::Unserialize(s, chAvail);
327  for (unsigned int p = 0; p < 8; p++) {
328  bool f = (chAvail & (1 << p)) != 0;
329  vAvail.push_back(f);
330  }
331  if (chAvail != 0)
332  nMaskCode--;
333  }
334  // txouts themself
335  vout.assign(vAvail.size(), CTxOut());
336  for (unsigned int i = 0; i < vAvail.size(); i++) {
337  if (vAvail[i])
338  ::Unserialize(s, CTxOutCompressor(vout[i]));
339  }
340  // coinbase height
342  }
343 };
344 
345 }
346 
352  std::unique_ptr<CDBIterator> pcursor(db.NewIterator());
353  pcursor->Seek(std::make_pair(DB_COINS, uint256()));
354  if (!pcursor->Valid()) {
355  return true;
356  }
357 
358  int64_t count = 0;
359  LogPrintf("Upgrading utxo-set database...\n");
360  LogPrintf("[0%%]..."); /* Continued */
361  uiInterface.ShowProgress(_("Upgrading UTXO database").translated, 0, true);
362  size_t batch_size = 1 << 24;
363  CDBBatch batch(db);
364  int reportDone = 0;
365  std::pair<unsigned char, uint256> key;
366  std::pair<unsigned char, uint256> prev_key = {DB_COINS, uint256()};
367  while (pcursor->Valid()) {
368  boost::this_thread::interruption_point();
369  if (ShutdownRequested()) {
370  break;
371  }
372  if (pcursor->GetKey(key) && key.first == DB_COINS) {
373  if (count++ % 256 == 0) {
374  uint32_t high = 0x100 * *key.second.begin() + *(key.second.begin() + 1);
375  int percentageDone = (int)(high * 100.0 / 65536.0 + 0.5);
376  uiInterface.ShowProgress(_("Upgrading UTXO database").translated, percentageDone, true);
377  if (reportDone < percentageDone/10) {
378  // report max. every 10% step
379  LogPrintf("[%d%%]...", percentageDone); /* Continued */
380  reportDone = percentageDone/10;
381  }
382  }
383  CCoins old_coins;
384  if (!pcursor->GetValue(old_coins)) {
385  return error("%s: cannot parse CCoins record", __func__);
386  }
387  COutPoint outpoint(key.second, 0);
388  for (size_t i = 0; i < old_coins.vout.size(); ++i) {
389  if (!old_coins.vout[i].IsNull() && !old_coins.vout[i].scriptPubKey.IsUnspendable()) {
390  Coin newcoin(std::move(old_coins.vout[i]), old_coins.nHeight, old_coins.fCoinBase);
391  outpoint.n = i;
392  CoinEntry entry(&outpoint);
393  batch.Write(entry, newcoin);
394  }
395  }
396  batch.Erase(key);
397  if (batch.SizeEstimate() > batch_size) {
398  db.WriteBatch(batch);
399  batch.Clear();
400  db.CompactRange(prev_key, key);
401  prev_key = key;
402  }
403  pcursor->Next();
404  } else {
405  break;
406  }
407  }
408  db.WriteBatch(batch);
409  db.CompactRange({DB_COINS, uint256()}, key);
410  uiInterface.ShowProgress("", 100, false);
411  LogPrintf("[%s].\n", ShutdownRequested() ? "CANCELLED" : "DONE");
412  return !ShutdownRequested();
413 }
bool Exists(const K &key) const
Definition: dbwrapper.h:265
bool GetValue(Coin &coin) const override
Definition: txdb.cpp:198
static const char DB_LAST_BLOCK
Definition: txdb.cpp:29
bool GetCoin(const COutPoint &outpoint, Coin &coin) const override
Retrieve the Coin (unspent transaction output) for a given outpoint.
Definition: txdb.cpp:59
std::string ToString() const
Definition: chain.h:301
void Clear()
Definition: dbwrapper.h:66
bool ShutdownRequested()
Definition: shutdown.cpp:20
bool Upgrade()
Attempt to update from an older database format. Returns whether an error occurred.
Definition: txdb.cpp:351
Specialization of CCoinsViewCursor to iterate over a CCoinsViewDB.
Definition: txdb.h:69
CBlockIndex * pprev
pointer to the index of the predecessor of this block
Definition: chain.h:146
Batch of changes queued to be written to a CDBWrapper.
Definition: dbwrapper.h:47
uint32_t nStatus
Verification status of this block. See enum BlockStatus.
Definition: chain.h:176
A UTXO entry.
Definition: coins.h:30
static const char DB_BEST_BLOCK
Definition: txdb.cpp:25
bool ReadLastBlockFile(int &nFile)
Definition: txdb.cpp:166
wrapper for CTxOut that provides a more compact serialization
Definition: compressor.h:86
CCoinsViewDB(fs::path ldb_path, size_t nCacheSize, bool fMemory, bool fWipe)
Definition: txdb.cpp:55
unsigned int nHeight
void Erase(const K &key)
Definition: dbwrapper.h:98
std::unique_ptr< CDBIterator > pcursor
Definition: txdb.h:84
void ReadReindexing(bool &fReindexing)
Definition: txdb.cpp:162
static const char DB_COIN
Definition: txdb.cpp:20
static void LogPrintf(const char *fmt, const Args &... args)
Definition: logging.h:144
uint32_t nTime
Definition: chain.h:181
int nFile
Which # file this block is stored in (blk?????.dat)
Definition: chain.h:155
bool GetKey(COutPoint &key) const override
Definition: txdb.cpp:188
bool IsNull() const
Definition: uint256.h:31
bool WriteReindexing(bool fReindexing)
Definition: txdb.cpp:155
CDBIterator * NewIterator()
Definition: dbwrapper.h:308
Definition: coins.h:118
void Serialize(Stream &s, char a)
Definition: serialize.h:204
uint256 GetBlockHash() const
Definition: chain.h:261
static const char DB_BLOCK_FILES
Definition: txdb.cpp:22
bool Erase(const K &key, bool fSync=false)
Definition: dbwrapper.h:284
uint32_t nNonce
Definition: chain.h:183
static const char DB_FLAG
Definition: txdb.cpp:27
unsigned int nDataPos
Byte offset within blk?????.dat where this block&#39;s data is stored.
Definition: chain.h:158
const char * name
Definition: rest.cpp:39
bilingual_str _(const char *psz)
Translation function.
Definition: translation.h:37
Fast randomness source.
Definition: random.h:100
const fs::path & GetDataDir(bool fNetSpecific)
Definition: system.cpp:742
uint32_t n
Definition: transaction.h:22
CDBWrapper db
Definition: txdb.h:49
std::unordered_map< COutPoint, CCoinsCacheEntry, SaltedOutpointHasher > CCoinsMap
Definition: coins.h:131
uint256 hashMerkleRoot
Definition: chain.h:180
static void LogPrint(const BCLog::LogFlags &category, const Args &... args)
Definition: logging.h:159
void Write(const K &key, const V &value)
Definition: dbwrapper.h:73
size_t SizeEstimate() const
Definition: dbwrapper.h:114
An output of a transaction.
Definition: transaction.h:133
Used to marshal pointers into hashes for db storage.
Definition: chain.h:348
static const int64_t nDefaultDbBatchSize
-dbbatchsize default (bytes)
Definition: txdb.h:29
size_t EstimateSize() const override
Estimate database size (0 if not implemented)
Definition: txdb.cpp:143
Parameters that influence chain consensus.
Definition: params.h:47
bool CheckProofOfWork(uint256 hash, unsigned int nBits, const Consensus::Params &params)
Check whether a block hash satisfies the proof-of-work requirement specified by nBits.
Definition: pow.cpp:74
An outpoint - a combination of a transaction hash and an index n into its vout.
Definition: transaction.h:18
std::pair< char, COutPoint > keyTmp
Definition: txdb.h:85
bool Read(const K &key, V &value) const
Definition: dbwrapper.h:231
bool HaveCoin(const COutPoint &outpoint) const override
Just check whether a given outpoint is unspent.
Definition: txdb.cpp:63
bool ReadFlag(const std::string &name, bool &fValue)
Definition: txdb.cpp:240
CBlockTreeDB(size_t nCacheSize, bool fMemory=false, bool fWipe=false)
Definition: txdb.cpp:148
bool WriteBatchSync(const std::vector< std::pair< int, const CBlockFileInfo *> > &fileInfo, int nLastFile, const std::vector< const CBlockIndex *> &blockinfo)
Definition: txdb.cpp:224
bool ReadBlockFileInfo(int nFile, CBlockFileInfo &info)
Definition: txdb.cpp:151
unsigned int nUndoPos
Byte offset within rev?????.dat where this block&#39;s undo data is stored.
Definition: chain.h:161
uint256 GetBestBlock() const override
Retrieve the block hash whose state this CCoinsView currently represents.
Definition: txdb.cpp:67
int32_t nVersion
block header
Definition: chain.h:179
256-bit opaque blob.
Definition: uint256.h:121
uint256 hashPrev
Definition: chain.h:351
static const char DB_BLOCK_INDEX
Definition: txdb.cpp:23
bool Write(const K &key, const V &value, bool fSync=false)
Definition: dbwrapper.h:257
The block chain is a tree shaped structure starting with the genesis block at the root...
Definition: chain.h:139
std::string GetArg(const std::string &strArg, const std::string &strDefault) const
Return string argument or default value.
Definition: system.cpp:496
ArgsManager gArgs
Definition: system.cpp:73
static int count
Definition: tests.c:45
void Unserialize(Stream &s, char &a)
Definition: serialize.h:222
bool LoadBlockIndexGuts(const Consensus::Params &consensusParams, std::function< CBlockIndex *(const uint256 &)> insertBlockIndex)
Definition: txdb.cpp:248
bool WriteFlag(const std::string &name, bool fValue)
Definition: txdb.cpp:236
bool BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) override
Do a bulk modification (multiple Coin changes + BestBlock change).
Definition: txdb.cpp:82
void Next() override
Definition: txdb.cpp:213
CCoinsViewCursor * Cursor() const override
Get a cursor to iterate over the whole state.
Definition: txdb.cpp:170
bool WriteBatch(CDBBatch &batch, bool fSync=false)
Definition: dbwrapper.cpp:183
bool Valid() const override
Definition: txdb.cpp:208
CClientUIInterface uiInterface
int nHeight
height of the entry in the chain. The genesis block has height 0
Definition: chain.h:152
uint256 GetBlockHash() const
Definition: chain.h:388
static const char DB_REINDEX_FLAG
Definition: txdb.cpp:28
unsigned int GetValueSize() const override
Definition: txdb.cpp:203
auto it
Definition: validation.cpp:366
static const char DB_COINS
Definition: txdb.cpp:21
size_t EstimateSize(const K &key_begin, const K &key_end) const
Definition: dbwrapper.h:319
static const char DB_HEAD_BLOCKS
Definition: txdb.cpp:26
bool error(const char *fmt, const Args &... args)
Definition: system.h:47
std::vector< uint256 > GetHeadBlocks() const override
Retrieve the range of blocks that may have been only partially written.
Definition: txdb.cpp:74
uint32_t nBits
Definition: chain.h:182
#define VARINT(obj,...)
Definition: serialize.h:422
uint64_t randrange(uint64_t range) noexcept
Generate a random integer in the range [0..range).
Definition: random.h:167
unsigned int nTx
Number of transactions in this block.
Definition: chain.h:168
uint256 hash
Definition: transaction.h:21
Cursor for iterating over CoinsView state.
Definition: coins.h:134