Bitcoin Core  0.18.99
P2P Digital Currency
txdb.cpp
Go to the documentation of this file.
1 // Copyright (c) 2009-2010 Satoshi Nakamoto
2 // Copyright (c) 2009-2018 The Bitcoin Core developers
3 // Distributed under the MIT software license, see the accompanying
4 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
5 
6 #include <txdb.h>
7 
8 #include <chainparams.h>
9 #include <hash.h>
10 #include <random.h>
11 #include <pow.h>
12 #include <shutdown.h>
13 #include <uint256.h>
14 #include <util/system.h>
15 #include <ui_interface.h>
16 
17 #include <stdint.h>
18 
19 #include <boost/thread.hpp>
20 
21 static const char DB_COIN = 'C';
22 static const char DB_COINS = 'c';
23 static const char DB_BLOCK_FILES = 'f';
24 static const char DB_BLOCK_INDEX = 'b';
25 
26 static const char DB_BEST_BLOCK = 'B';
27 static const char DB_HEAD_BLOCKS = 'H';
28 static const char DB_FLAG = 'F';
29 static const char DB_REINDEX_FLAG = 'R';
30 static const char DB_LAST_BLOCK = 'l';
31 
32 namespace {
33 
34 struct CoinEntry {
35  COutPoint* outpoint;
36  char key;
37  explicit CoinEntry(const COutPoint* ptr) : outpoint(const_cast<COutPoint*>(ptr)), key(DB_COIN) {}
38 
39  template<typename Stream>
40  void Serialize(Stream &s) const {
41  s << key;
42  s << outpoint->hash;
43  s << VARINT(outpoint->n);
44  }
45 
46  template<typename Stream>
47  void Unserialize(Stream& s) {
48  s >> key;
49  s >> outpoint->hash;
50  s >> VARINT(outpoint->n);
51  }
52 };
53 
54 }
55 
56 CCoinsViewDB::CCoinsViewDB(size_t nCacheSize, bool fMemory, bool fWipe) : db(GetDataDir() / "chainstate", nCacheSize, fMemory, fWipe, true)
57 {
58 }
59 
60 bool CCoinsViewDB::GetCoin(const COutPoint &outpoint, Coin &coin) const {
61  return db.Read(CoinEntry(&outpoint), coin);
62 }
63 
64 bool CCoinsViewDB::HaveCoin(const COutPoint &outpoint) const {
65  return db.Exists(CoinEntry(&outpoint));
66 }
67 
69  uint256 hashBestChain;
70  if (!db.Read(DB_BEST_BLOCK, hashBestChain))
71  return uint256();
72  return hashBestChain;
73 }
74 
75 std::vector<uint256> CCoinsViewDB::GetHeadBlocks() const {
76  std::vector<uint256> vhashHeadBlocks;
77  if (!db.Read(DB_HEAD_BLOCKS, vhashHeadBlocks)) {
78  return std::vector<uint256>();
79  }
80  return vhashHeadBlocks;
81 }
82 
83 bool CCoinsViewDB::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) {
84  CDBBatch batch(db);
85  size_t count = 0;
86  size_t changed = 0;
87  size_t batch_size = (size_t)gArgs.GetArg("-dbbatchsize", nDefaultDbBatchSize);
88  int crash_simulate = gArgs.GetArg("-dbcrashratio", 0);
89  assert(!hashBlock.IsNull());
90 
91  uint256 old_tip = GetBestBlock();
92  if (old_tip.IsNull()) {
93  // We may be in the middle of replaying.
94  std::vector<uint256> old_heads = GetHeadBlocks();
95  if (old_heads.size() == 2) {
96  assert(old_heads[0] == hashBlock);
97  old_tip = old_heads[1];
98  }
99  }
100 
101  // In the first batch, mark the database as being in the middle of a
102  // transition from old_tip to hashBlock.
103  // A vector is used for future extensibility, as we may want to support
104  // interrupting after partial writes from multiple independent reorgs.
105  batch.Erase(DB_BEST_BLOCK);
106  batch.Write(DB_HEAD_BLOCKS, std::vector<uint256>{hashBlock, old_tip});
107 
108  for (CCoinsMap::iterator it = mapCoins.begin(); it != mapCoins.end();) {
109  if (it->second.flags & CCoinsCacheEntry::DIRTY) {
110  CoinEntry entry(&it->first);
111  if (it->second.coin.IsSpent())
112  batch.Erase(entry);
113  else
114  batch.Write(entry, it->second.coin);
115  changed++;
116  }
117  count++;
118  CCoinsMap::iterator itOld = it++;
119  mapCoins.erase(itOld);
120  if (batch.SizeEstimate() > batch_size) {
121  LogPrint(BCLog::COINDB, "Writing partial batch of %.2f MiB\n", batch.SizeEstimate() * (1.0 / 1048576.0));
122  db.WriteBatch(batch);
123  batch.Clear();
124  if (crash_simulate) {
125  static FastRandomContext rng;
126  if (rng.randrange(crash_simulate) == 0) {
127  LogPrintf("Simulating a crash. Goodbye.\n");
128  _Exit(0);
129  }
130  }
131  }
132  }
133 
134  // In the last batch, mark the database as consistent with hashBlock again.
135  batch.Erase(DB_HEAD_BLOCKS);
136  batch.Write(DB_BEST_BLOCK, hashBlock);
137 
138  LogPrint(BCLog::COINDB, "Writing final batch of %.2f MiB\n", batch.SizeEstimate() * (1.0 / 1048576.0));
139  bool ret = db.WriteBatch(batch);
140  LogPrint(BCLog::COINDB, "Committed %u changed transaction outputs (out of %u) to coin database...\n", (unsigned int)changed, (unsigned int)count);
141  return ret;
142 }
143 
145 {
146  return db.EstimateSize(DB_COIN, (char)(DB_COIN+1));
147 }
148 
149 CBlockTreeDB::CBlockTreeDB(size_t nCacheSize, bool fMemory, bool fWipe) : CDBWrapper(gArgs.IsArgSet("-blocksdir") ? GetDataDir() / "blocks" / "index" : GetBlocksDir() / "index", nCacheSize, fMemory, fWipe) {
150 }
151 
153  return Read(std::make_pair(DB_BLOCK_FILES, nFile), info);
154 }
155 
156 bool CBlockTreeDB::WriteReindexing(bool fReindexing) {
157  if (fReindexing)
158  return Write(DB_REINDEX_FLAG, '1');
159  else
160  return Erase(DB_REINDEX_FLAG);
161 }
162 
163 void CBlockTreeDB::ReadReindexing(bool &fReindexing) {
164  fReindexing = Exists(DB_REINDEX_FLAG);
165 }
166 
168  return Read(DB_LAST_BLOCK, nFile);
169 }
170 
172 {
173  CCoinsViewDBCursor *i = new CCoinsViewDBCursor(const_cast<CDBWrapper&>(db).NewIterator(), GetBestBlock());
174  /* It seems that there are no "const iterators" for LevelDB. Since we
175  only need read operations on it, use a const-cast to get around
176  that restriction. */
177  i->pcursor->Seek(DB_COIN);
178  // Cache key of first record
179  if (i->pcursor->Valid()) {
180  CoinEntry entry(&i->keyTmp.second);
181  i->pcursor->GetKey(entry);
182  i->keyTmp.first = entry.key;
183  } else {
184  i->keyTmp.first = 0; // Make sure Valid() and GetKey() return false
185  }
186  return i;
187 }
188 
190 {
191  // Return cached key
192  if (keyTmp.first == DB_COIN) {
193  key = keyTmp.second;
194  return true;
195  }
196  return false;
197 }
198 
200 {
201  return pcursor->GetValue(coin);
202 }
203 
205 {
206  return pcursor->GetValueSize();
207 }
208 
210 {
211  return keyTmp.first == DB_COIN;
212 }
213 
215 {
216  pcursor->Next();
217  CoinEntry entry(&keyTmp.second);
218  if (!pcursor->Valid() || !pcursor->GetKey(entry)) {
219  keyTmp.first = 0; // Invalidate cached key after last record so that Valid() and GetKey() return false
220  } else {
221  keyTmp.first = entry.key;
222  }
223 }
224 
225 bool CBlockTreeDB::WriteBatchSync(const std::vector<std::pair<int, const CBlockFileInfo*> >& fileInfo, int nLastFile, const std::vector<const CBlockIndex*>& blockinfo) {
226  CDBBatch batch(*this);
227  for (std::vector<std::pair<int, const CBlockFileInfo*> >::const_iterator it=fileInfo.begin(); it != fileInfo.end(); it++) {
228  batch.Write(std::make_pair(DB_BLOCK_FILES, it->first), *it->second);
229  }
230  batch.Write(DB_LAST_BLOCK, nLastFile);
231  for (std::vector<const CBlockIndex*>::const_iterator it=blockinfo.begin(); it != blockinfo.end(); it++) {
232  batch.Write(std::make_pair(DB_BLOCK_INDEX, (*it)->GetBlockHash()), CDiskBlockIndex(*it));
233  }
234  return WriteBatch(batch, true);
235 }
236 
237 bool CBlockTreeDB::WriteFlag(const std::string &name, bool fValue) {
238  return Write(std::make_pair(DB_FLAG, name), fValue ? '1' : '0');
239 }
240 
241 bool CBlockTreeDB::ReadFlag(const std::string &name, bool &fValue) {
242  char ch;
243  if (!Read(std::make_pair(DB_FLAG, name), ch))
244  return false;
245  fValue = ch == '1';
246  return true;
247 }
248 
249 bool CBlockTreeDB::LoadBlockIndexGuts(const Consensus::Params& consensusParams, std::function<CBlockIndex*(const uint256&)> insertBlockIndex)
250 {
251  std::unique_ptr<CDBIterator> pcursor(NewIterator());
252 
253  pcursor->Seek(std::make_pair(DB_BLOCK_INDEX, uint256()));
254 
255  // Load mapBlockIndex
256  while (pcursor->Valid()) {
257  boost::this_thread::interruption_point();
258  if (ShutdownRequested()) return false;
259  std::pair<char, uint256> key;
260  if (pcursor->GetKey(key) && key.first == DB_BLOCK_INDEX) {
261  CDiskBlockIndex diskindex;
262  if (pcursor->GetValue(diskindex)) {
263  // Construct block index object
264  CBlockIndex* pindexNew = insertBlockIndex(diskindex.GetBlockHash());
265  pindexNew->pprev = insertBlockIndex(diskindex.hashPrev);
266  pindexNew->nHeight = diskindex.nHeight;
267  pindexNew->nFile = diskindex.nFile;
268  pindexNew->nDataPos = diskindex.nDataPos;
269  pindexNew->nUndoPos = diskindex.nUndoPos;
270  pindexNew->nVersion = diskindex.nVersion;
271  pindexNew->hashMerkleRoot = diskindex.hashMerkleRoot;
272  pindexNew->nTime = diskindex.nTime;
273  pindexNew->nBits = diskindex.nBits;
274  pindexNew->nNonce = diskindex.nNonce;
275  pindexNew->nStatus = diskindex.nStatus;
276  pindexNew->nTx = diskindex.nTx;
277 
278  if (!CheckProofOfWork(pindexNew->GetBlockHash(), pindexNew->nBits, consensusParams))
279  return error("%s: CheckProofOfWork failed: %s", __func__, pindexNew->ToString());
280 
281  pcursor->Next();
282  } else {
283  return error("%s: failed to read value", __func__);
284  }
285  } else {
286  break;
287  }
288  }
289 
290  return true;
291 }
292 
293 namespace {
294 
296 class CCoins
297 {
298 public:
300  bool fCoinBase;
301 
303  std::vector<CTxOut> vout;
304 
306  int nHeight;
307 
309  CCoins() : fCoinBase(false), vout(0), nHeight(0) { }
310 
311  template<typename Stream>
312  void Unserialize(Stream &s) {
313  unsigned int nCode = 0;
314  // version
315  unsigned int nVersionDummy;
316  ::Unserialize(s, VARINT(nVersionDummy));
317  // header code
318  ::Unserialize(s, VARINT(nCode));
319  fCoinBase = nCode & 1;
320  std::vector<bool> vAvail(2, false);
321  vAvail[0] = (nCode & 2) != 0;
322  vAvail[1] = (nCode & 4) != 0;
323  unsigned int nMaskCode = (nCode / 8) + ((nCode & 6) != 0 ? 0 : 1);
324  // spentness bitmask
325  while (nMaskCode > 0) {
326  unsigned char chAvail = 0;
327  ::Unserialize(s, chAvail);
328  for (unsigned int p = 0; p < 8; p++) {
329  bool f = (chAvail & (1 << p)) != 0;
330  vAvail.push_back(f);
331  }
332  if (chAvail != 0)
333  nMaskCode--;
334  }
335  // txouts themself
336  vout.assign(vAvail.size(), CTxOut());
337  for (unsigned int i = 0; i < vAvail.size(); i++) {
338  if (vAvail[i])
339  ::Unserialize(s, CTxOutCompressor(vout[i]));
340  }
341  // coinbase height
343  }
344 };
345 
346 }
347 
353  std::unique_ptr<CDBIterator> pcursor(db.NewIterator());
354  pcursor->Seek(std::make_pair(DB_COINS, uint256()));
355  if (!pcursor->Valid()) {
356  return true;
357  }
358 
359  int64_t count = 0;
360  LogPrintf("Upgrading utxo-set database...\n");
361  LogPrintf("[0%%]..."); /* Continued */
362  uiInterface.ShowProgress(_("Upgrading UTXO database"), 0, true);
363  size_t batch_size = 1 << 24;
364  CDBBatch batch(db);
365  int reportDone = 0;
366  std::pair<unsigned char, uint256> key;
367  std::pair<unsigned char, uint256> prev_key = {DB_COINS, uint256()};
368  while (pcursor->Valid()) {
369  boost::this_thread::interruption_point();
370  if (ShutdownRequested()) {
371  break;
372  }
373  if (pcursor->GetKey(key) && key.first == DB_COINS) {
374  if (count++ % 256 == 0) {
375  uint32_t high = 0x100 * *key.second.begin() + *(key.second.begin() + 1);
376  int percentageDone = (int)(high * 100.0 / 65536.0 + 0.5);
377  uiInterface.ShowProgress(_("Upgrading UTXO database"), percentageDone, true);
378  if (reportDone < percentageDone/10) {
379  // report max. every 10% step
380  LogPrintf("[%d%%]...", percentageDone); /* Continued */
381  reportDone = percentageDone/10;
382  }
383  }
384  CCoins old_coins;
385  if (!pcursor->GetValue(old_coins)) {
386  return error("%s: cannot parse CCoins record", __func__);
387  }
388  COutPoint outpoint(key.second, 0);
389  for (size_t i = 0; i < old_coins.vout.size(); ++i) {
390  if (!old_coins.vout[i].IsNull() && !old_coins.vout[i].scriptPubKey.IsUnspendable()) {
391  Coin newcoin(std::move(old_coins.vout[i]), old_coins.nHeight, old_coins.fCoinBase);
392  outpoint.n = i;
393  CoinEntry entry(&outpoint);
394  batch.Write(entry, newcoin);
395  }
396  }
397  batch.Erase(key);
398  if (batch.SizeEstimate() > batch_size) {
399  db.WriteBatch(batch);
400  batch.Clear();
401  db.CompactRange(prev_key, key);
402  prev_key = key;
403  }
404  pcursor->Next();
405  } else {
406  break;
407  }
408  }
409  db.WriteBatch(batch);
410  db.CompactRange({DB_COINS, uint256()}, key);
411  uiInterface.ShowProgress("", 100, false);
412  LogPrintf("[%s].\n", ShutdownRequested() ? "CANCELLED" : "DONE");
413  return !ShutdownRequested();
414 }
bool Exists(const K &key) const
Definition: dbwrapper.h:265
bool GetValue(Coin &coin) const override
Definition: txdb.cpp:199
bool GetCoin(const COutPoint &outpoint, Coin &coin) const override
Retrieve the Coin (unspent transaction output) for a given outpoint.
Definition: txdb.cpp:60
std::string ToString() const
Definition: chain.h:301
void Clear()
Definition: dbwrapper.h:66
bool ShutdownRequested()
Definition: shutdown.cpp:20
bool Upgrade()
Attempt to update from an older database format. Returns whether an error occurred.
Definition: txdb.cpp:352
Specialization of CCoinsViewCursor to iterate over a CCoinsViewDB.
Definition: txdb.h:66
CBlockIndex * pprev
pointer to the index of the predecessor of this block
Definition: chain.h:146
Batch of changes queued to be written to a CDBWrapper.
Definition: dbwrapper.h:47
uint32_t nStatus
Verification status of this block. See enum BlockStatus.
Definition: chain.h:176
A UTXO entry.
Definition: coins.h:29
bool ReadLastBlockFile(int &nFile)
Definition: txdb.cpp:167
wrapper for CTxOut that provides a more compact serialization
Definition: compressor.h:86
unsigned int nHeight
void Erase(const K &key)
Definition: dbwrapper.h:98
std::unique_ptr< CDBIterator > pcursor
Definition: txdb.h:81
void ReadReindexing(bool &fReindexing)
Definition: txdb.cpp:163
const fs::path & GetBlocksDir()
Definition: system.cpp:712
uint32_t nTime
Definition: chain.h:181
int nFile
Which # file this block is stored in (blk?????.dat)
Definition: chain.h:155
bool GetKey(COutPoint &key) const override
Definition: txdb.cpp:189
bool IsNull() const
Definition: uint256.h:31
bool WriteReindexing(bool fReindexing)
Definition: txdb.cpp:156
CDBIterator * NewIterator()
Definition: dbwrapper.h:308
Definition: coins.h:109
void Serialize(Stream &s, char a)
Definition: serialize.h:204
uint256 GetBlockHash() const
Definition: chain.h:261
bool Erase(const K &key, bool fSync=false)
Definition: dbwrapper.h:284
uint32_t nNonce
Definition: chain.h:183
unsigned int nDataPos
Byte offset within blk?????.dat where this block&#39;s data is stored.
Definition: chain.h:158
const char * name
Definition: rest.cpp:38
Fast randomness source.
Definition: random.h:98
const fs::path & GetDataDir(bool fNetSpecific)
Definition: system.cpp:740
uint32_t n
Definition: transaction.h:22
CDBWrapper db
Definition: txdb.h:49
std::unordered_map< COutPoint, CCoinsCacheEntry, SaltedOutpointHasher > CCoinsMap
Definition: coins.h:122
uint256 hashMerkleRoot
Definition: chain.h:180
void Write(const K &key, const V &value)
Definition: dbwrapper.h:73
size_t SizeEstimate() const
Definition: dbwrapper.h:114
An output of a transaction.
Definition: transaction.h:133
Used to marshal pointers into hashes for db storage.
Definition: chain.h:348
size_t EstimateSize() const override
Estimate database size (0 if not implemented)
Definition: txdb.cpp:144
Parameters that influence chain consensus.
Definition: params.h:49
bool CheckProofOfWork(uint256 hash, unsigned int nBits, const Consensus::Params &params)
Check whether a block hash satisfies the proof-of-work requirement specified by nBits.
Definition: pow.cpp:74
An outpoint - a combination of a transaction hash and an index n into its vout.
Definition: transaction.h:18
std::pair< char, COutPoint > keyTmp
Definition: txdb.h:82
bool Read(const K &key, V &value) const
Definition: dbwrapper.h:231
bool HaveCoin(const COutPoint &outpoint) const override
Just check whether a given outpoint is unspent.
Definition: txdb.cpp:64
bool ReadFlag(const std::string &name, bool &fValue)
Definition: txdb.cpp:241
CBlockTreeDB(size_t nCacheSize, bool fMemory=false, bool fWipe=false)
Definition: txdb.cpp:149
bool WriteBatchSync(const std::vector< std::pair< int, const CBlockFileInfo *> > &fileInfo, int nLastFile, const std::vector< const CBlockIndex *> &blockinfo)
Definition: txdb.cpp:225
bool ReadBlockFileInfo(int nFile, CBlockFileInfo &info)
Definition: txdb.cpp:152
CCoinsViewDB(size_t nCacheSize, bool fMemory=false, bool fWipe=false)
Definition: txdb.cpp:56
unsigned int nUndoPos
Byte offset within rev?????.dat where this block&#39;s undo data is stored.
Definition: chain.h:161
uint256 GetBestBlock() const override
Retrieve the block hash whose state this CCoinsView currently represents.
Definition: txdb.cpp:68
std::string _(const char *psz)
Translation function.
Definition: system.h:52
int32_t nVersion
block header
Definition: chain.h:179
256-bit opaque blob.
Definition: uint256.h:121
uint256 hashPrev
Definition: chain.h:351
bool Write(const K &key, const V &value, bool fSync=false)
Definition: dbwrapper.h:257
The block chain is a tree shaped structure starting with the genesis block at the root...
Definition: chain.h:139
std::string GetArg(const std::string &strArg, const std::string &strDefault) const
Return string argument or default value.
Definition: system.cpp:496
constexpr char DB_BEST_BLOCK
Definition: base.cpp:14
ArgsManager gArgs
Definition: system.cpp:74
void Unserialize(Stream &s, char &a)
Definition: serialize.h:222
bool LoadBlockIndexGuts(const Consensus::Params &consensusParams, std::function< CBlockIndex *(const uint256 &)> insertBlockIndex)
Definition: txdb.cpp:249
bool WriteFlag(const std::string &name, bool fValue)
Definition: txdb.cpp:237
bool BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) override
Do a bulk modification (multiple Coin changes + BestBlock change).
Definition: txdb.cpp:83
void Next() override
Definition: txdb.cpp:214
CCoinsViewCursor * Cursor() const override
Get a cursor to iterate over the whole state.
Definition: txdb.cpp:171
bool WriteBatch(CDBBatch &batch, bool fSync=false)
Definition: dbwrapper.cpp:183
bool Valid() const override
Definition: txdb.cpp:209
CClientUIInterface uiInterface
int nHeight
height of the entry in the chain. The genesis block has height 0
Definition: chain.h:152
uint256 GetBlockHash() const
Definition: chain.h:388
unsigned int GetValueSize() const override
Definition: txdb.cpp:204
size_t EstimateSize(const K &key_begin, const K &key_end) const
Definition: dbwrapper.h:319
bool error(const char *fmt, const Args &... args)
Definition: system.h:61
std::vector< uint256 > GetHeadBlocks() const override
Retrieve the range of blocks that may have been only partially written.
Definition: txdb.cpp:75
uint32_t nBits
Definition: chain.h:182
#define VARINT(obj,...)
Definition: serialize.h:422
uint64_t randrange(uint64_t range) noexcept
Generate a random integer in the range [0..range).
Definition: random.h:165
unsigned int nTx
Number of transactions in this block.
Definition: chain.h:168
uint256 hash
Definition: transaction.h:21
Cursor for iterating over CoinsView state.
Definition: coins.h:125