Bitcoin Core  0.18.99
P2P Digital Currency
txindex.cpp
Go to the documentation of this file.
1 // Copyright (c) 2017-2018 The Bitcoin Core developers
2 // Distributed under the MIT software license, see the accompanying
3 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
4 
5 #include <index/txindex.h>
6 #include <shutdown.h>
7 #include <ui_interface.h>
8 #include <util/system.h>
9 #include <validation.h>
10 
11 #include <boost/thread.hpp>
12 
13 constexpr char DB_BEST_BLOCK = 'B';
14 constexpr char DB_TXINDEX = 't';
15 constexpr char DB_TXINDEX_BLOCK = 'T';
16 
17 std::unique_ptr<TxIndex> g_txindex;
18 
19 struct CDiskTxPos : public FlatFilePos
20 {
21  unsigned int nTxOffset; // after header
22 
24 
25  template <typename Stream, typename Operation>
26  inline void SerializationOp(Stream& s, Operation ser_action) {
27  READWRITEAS(FlatFilePos, *this);
28  READWRITE(VARINT(nTxOffset));
29  }
30 
31  CDiskTxPos(const FlatFilePos &blockIn, unsigned int nTxOffsetIn) : FlatFilePos(blockIn.nFile, blockIn.nPos), nTxOffset(nTxOffsetIn) {
32  }
33 
35  SetNull();
36  }
37 
38  void SetNull() {
40  nTxOffset = 0;
41  }
42 };
43 
53 class TxIndex::DB : public BaseIndex::DB
54 {
55 public:
56  explicit DB(size_t n_cache_size, bool f_memory = false, bool f_wipe = false);
57 
60  bool ReadTxPos(const uint256& txid, CDiskTxPos& pos) const;
61 
63  bool WriteTxs(const std::vector<std::pair<uint256, CDiskTxPos>>& v_pos);
64 
67  bool MigrateData(CBlockTreeDB& block_tree_db, const CBlockLocator& best_locator);
68 };
69 
70 TxIndex::DB::DB(size_t n_cache_size, bool f_memory, bool f_wipe) :
71  BaseIndex::DB(GetDataDir() / "indexes" / "txindex", n_cache_size, f_memory, f_wipe)
72 {}
73 
74 bool TxIndex::DB::ReadTxPos(const uint256 &txid, CDiskTxPos& pos) const
75 {
76  return Read(std::make_pair(DB_TXINDEX, txid), pos);
77 }
78 
79 bool TxIndex::DB::WriteTxs(const std::vector<std::pair<uint256, CDiskTxPos>>& v_pos)
80 {
81  CDBBatch batch(*this);
82  for (const auto& tuple : v_pos) {
83  batch.Write(std::make_pair(DB_TXINDEX, tuple.first), tuple.second);
84  }
85  return WriteBatch(batch);
86 }
87 
88 /*
89  * Safely persist a transfer of data from the old txindex database to the new one, and compact the
90  * range of keys updated. This is used internally by MigrateData.
91  */
92 static void WriteTxIndexMigrationBatches(CDBWrapper& newdb, CDBWrapper& olddb,
93  CDBBatch& batch_newdb, CDBBatch& batch_olddb,
94  const std::pair<unsigned char, uint256>& begin_key,
95  const std::pair<unsigned char, uint256>& end_key)
96 {
97  // Sync new DB changes to disk before deleting from old DB.
98  newdb.WriteBatch(batch_newdb, /*fSync=*/ true);
99  olddb.WriteBatch(batch_olddb);
100  olddb.CompactRange(begin_key, end_key);
101 
102  batch_newdb.Clear();
103  batch_olddb.Clear();
104 }
105 
106 bool TxIndex::DB::MigrateData(CBlockTreeDB& block_tree_db, const CBlockLocator& best_locator)
107 {
108  // The prior implementation of txindex was always in sync with block index
109  // and presence was indicated with a boolean DB flag. If the flag is set,
110  // this means the txindex from a previous version is valid and in sync with
111  // the chain tip. The first step of the migration is to unset the flag and
112  // write the chain hash to a separate key, DB_TXINDEX_BLOCK. After that, the
113  // index entries are copied over in batches to the new database. Finally,
114  // DB_TXINDEX_BLOCK is erased from the old database and the block hash is
115  // written to the new database.
116  //
117  // Unsetting the boolean flag ensures that if the node is downgraded to a
118  // previous version, it will not see a corrupted, partially migrated index
119  // -- it will see that the txindex is disabled. When the node is upgraded
120  // again, the migration will pick up where it left off and sync to the block
121  // with hash DB_TXINDEX_BLOCK.
122  bool f_legacy_flag = false;
123  block_tree_db.ReadFlag("txindex", f_legacy_flag);
124  if (f_legacy_flag) {
125  if (!block_tree_db.Write(DB_TXINDEX_BLOCK, best_locator)) {
126  return error("%s: cannot write block indicator", __func__);
127  }
128  if (!block_tree_db.WriteFlag("txindex", false)) {
129  return error("%s: cannot write block index db flag", __func__);
130  }
131  }
132 
133  CBlockLocator locator;
134  if (!block_tree_db.Read(DB_TXINDEX_BLOCK, locator)) {
135  return true;
136  }
137 
138  int64_t count = 0;
139  LogPrintf("Upgrading txindex database... [0%%]\n");
140  uiInterface.ShowProgress(_("Upgrading txindex database"), 0, true);
141  int report_done = 0;
142  const size_t batch_size = 1 << 24; // 16 MiB
143 
144  CDBBatch batch_newdb(*this);
145  CDBBatch batch_olddb(block_tree_db);
146 
147  std::pair<unsigned char, uint256> key;
148  std::pair<unsigned char, uint256> begin_key{DB_TXINDEX, uint256()};
149  std::pair<unsigned char, uint256> prev_key = begin_key;
150 
151  bool interrupted = false;
152  std::unique_ptr<CDBIterator> cursor(block_tree_db.NewIterator());
153  for (cursor->Seek(begin_key); cursor->Valid(); cursor->Next()) {
154  boost::this_thread::interruption_point();
155  if (ShutdownRequested()) {
156  interrupted = true;
157  break;
158  }
159 
160  if (!cursor->GetKey(key)) {
161  return error("%s: cannot get key from valid cursor", __func__);
162  }
163  if (key.first != DB_TXINDEX) {
164  break;
165  }
166 
167  // Log progress every 10%.
168  if (++count % 256 == 0) {
169  // Since txids are uniformly random and traversed in increasing order, the high 16 bits
170  // of the hash can be used to estimate the current progress.
171  const uint256& txid = key.second;
172  uint32_t high_nibble =
173  (static_cast<uint32_t>(*(txid.begin() + 0)) << 8) +
174  (static_cast<uint32_t>(*(txid.begin() + 1)) << 0);
175  int percentage_done = (int)(high_nibble * 100.0 / 65536.0 + 0.5);
176 
177  uiInterface.ShowProgress(_("Upgrading txindex database"), percentage_done, true);
178  if (report_done < percentage_done/10) {
179  LogPrintf("Upgrading txindex database... [%d%%]\n", percentage_done);
180  report_done = percentage_done/10;
181  }
182  }
183 
184  CDiskTxPos value;
185  if (!cursor->GetValue(value)) {
186  return error("%s: cannot parse txindex record", __func__);
187  }
188  batch_newdb.Write(key, value);
189  batch_olddb.Erase(key);
190 
191  if (batch_newdb.SizeEstimate() > batch_size || batch_olddb.SizeEstimate() > batch_size) {
192  // NOTE: it's OK to delete the key pointed at by the current DB cursor while iterating
193  // because LevelDB iterators are guaranteed to provide a consistent view of the
194  // underlying data, like a lightweight snapshot.
195  WriteTxIndexMigrationBatches(*this, block_tree_db,
196  batch_newdb, batch_olddb,
197  prev_key, key);
198  prev_key = key;
199  }
200  }
201 
202  // If these final DB batches complete the migration, write the best block
203  // hash marker to the new database and delete from the old one. This signals
204  // that the former is fully caught up to that point in the blockchain and
205  // that all txindex entries have been removed from the latter.
206  if (!interrupted) {
207  batch_olddb.Erase(DB_TXINDEX_BLOCK);
208  batch_newdb.Write(DB_BEST_BLOCK, locator);
209  }
210 
211  WriteTxIndexMigrationBatches(*this, block_tree_db,
212  batch_newdb, batch_olddb,
213  begin_key, key);
214 
215  if (interrupted) {
216  LogPrintf("[CANCELLED].\n");
217  return false;
218  }
219 
220  uiInterface.ShowProgress("", 100, false);
221 
222  LogPrintf("[DONE].\n");
223  return true;
224 }
225 
226 TxIndex::TxIndex(size_t n_cache_size, bool f_memory, bool f_wipe)
227  : m_db(MakeUnique<TxIndex::DB>(n_cache_size, f_memory, f_wipe))
228 {}
229 
231 
233 {
234  LOCK(cs_main);
235 
236  // Attempt to migrate txindex from the old database to the new one. Even if
237  // chain_tip is null, the node could be reindexing and we still want to
238  // delete txindex records in the old database.
239  if (!m_db->MigrateData(*pblocktree, ::ChainActive().GetLocator())) {
240  return false;
241  }
242 
243  return BaseIndex::Init();
244 }
245 
246 bool TxIndex::WriteBlock(const CBlock& block, const CBlockIndex* pindex)
247 {
248  // Exclude genesis block transaction because outputs are not spendable.
249  if (pindex->nHeight == 0) return true;
250 
251  CDiskTxPos pos(pindex->GetBlockPos(), GetSizeOfCompactSize(block.vtx.size()));
252  std::vector<std::pair<uint256, CDiskTxPos>> vPos;
253  vPos.reserve(block.vtx.size());
254  for (const auto& tx : block.vtx) {
255  vPos.emplace_back(tx->GetHash(), pos);
256  pos.nTxOffset += ::GetSerializeSize(*tx, CLIENT_VERSION);
257  }
258  return m_db->WriteTxs(vPos);
259 }
260 
261 BaseIndex::DB& TxIndex::GetDB() const { return *m_db; }
262 
263 bool TxIndex::FindTx(const uint256& tx_hash, uint256& block_hash, CTransactionRef& tx) const
264 {
265  CDiskTxPos postx;
266  if (!m_db->ReadTxPos(tx_hash, postx)) {
267  return false;
268  }
269 
270  CAutoFile file(OpenBlockFile(postx, true), SER_DISK, CLIENT_VERSION);
271  if (file.IsNull()) {
272  return error("%s: OpenBlockFile failed", __func__);
273  }
274  CBlockHeader header;
275  try {
276  file >> header;
277  if (fseek(file.Get(), postx.nTxOffset, SEEK_CUR)) {
278  return error("%s: fseek(...) failed", __func__);
279  }
280  file >> tx;
281  } catch (const std::exception& e) {
282  return error("%s: Deserialize or I/O error - %s", __func__, e.what());
283  }
284  if (tx->GetHash() != tx_hash) {
285  return error("%s: txid mismatch", __func__);
286  }
287  block_hash = header.GetHash();
288  return true;
289 }
std::shared_ptr< const CTransaction > CTransactionRef
Definition: transaction.h:408
void SerializationOp(Stream &s, Operation ser_action)
Definition: txindex.cpp:26
bool WriteBlock(const CBlock &block, const CBlockIndex *pindex) override
Write update index entries for a newly connected block.
Definition: txindex.cpp:246
void Clear()
Definition: dbwrapper.h:66
bool ShutdownRequested()
Definition: shutdown.cpp:20
unsigned int nTxOffset
Definition: txindex.cpp:21
virtual bool Init()
Initialize internal state from the database and block index.
Definition: base.cpp:55
Describes a place in the block chain to another node such that if the other node doesn&#39;t have the sam...
Definition: block.h:126
constexpr char DB_TXINDEX_BLOCK
Definition: txindex.cpp:15
Batch of changes queued to be written to a CDBWrapper.
Definition: dbwrapper.h:47
Definition: block.h:72
CChain & ChainActive()
Definition: validation.cpp:84
void Erase(const K &key)
Definition: dbwrapper.h:98
unsigned int GetSizeOfCompactSize(uint64_t nSize)
Compact Size size < 253 – 1 byte size <= USHRT_MAX – 3 bytes (253 + 2 bytes) size <= UINT_MAX – 5 ...
Definition: serialize.h:253
constexpr char DB_TXINDEX
Definition: txindex.cpp:14
FlatFilePos GetBlockPos() const
Definition: chain.h:230
std::unique_ptr< T > MakeUnique(Args &&... args)
Substitute for C++14 std::make_unique.
Definition: memory.h:14
int nFile
Definition: flatfile.h:16
const std::unique_ptr< DB > m_db
Definition: txindex.h:20
#define READWRITEAS(type, obj)
Definition: serialize.h:185
unsigned char * begin()
Definition: uint256.h:55
bool Init() override
Override base class init to migrate from old database.
Definition: txindex.cpp:232
CDBIterator * NewIterator()
Definition: dbwrapper.h:308
bool IsNull() const
Return true if the wrapped FILE* is nullptr, false otherwise.
Definition: streams.h:648
Access to the txindex database (indexes/txindex/)
Definition: txindex.cpp:53
bool ReadTxPos(const uint256 &txid, CDiskTxPos &pos) const
Read the disk location of the transaction data with the given hash.
Definition: txindex.cpp:74
size_t GetSerializeSize(const T &t, int nVersion=0)
Definition: serialize.h:992
Base class for indices of blockchain data.
Definition: base.h:22
CCriticalSection cs_main
Mutex to guard access to validation specific variables, such as reading or changing the chainstate...
Definition: validation.cpp:96
bool FindTx(const uint256 &tx_hash, uint256 &block_hash, CTransactionRef &tx) const
Look up a transaction by hash.
Definition: txindex.cpp:263
Access to the block database (blocks/index/)
Definition: txdb.h:88
CDiskTxPos(const FlatFilePos &blockIn, unsigned int nTxOffsetIn)
Definition: txindex.cpp:31
bool MigrateData(CBlockTreeDB &block_tree_db, const CBlockLocator &best_locator)
Migrate txindex data from the block tree DB, where it may be for older nodes that have not been upgra...
Definition: txindex.cpp:106
#define LOCK(cs)
Definition: sync.h:182
std::unique_ptr< TxIndex > g_txindex
The global transaction index, used in GetTransaction. May be null.
Definition: txindex.cpp:17
const fs::path & GetDataDir(bool fNetSpecific)
Definition: system.cpp:738
ADD_SERIALIZE_METHODS
Definition: txindex.cpp:23
void SetNull()
Definition: flatfile.h:42
void Write(const K &key, const V &value)
Definition: dbwrapper.h:73
BaseIndex::DB & GetDB() const override
Definition: txindex.cpp:261
size_t SizeEstimate() const
Definition: dbwrapper.h:114
bool Read(const K &key, V &value) const
Definition: dbwrapper.h:231
bool ReadFlag(const std::string &name, bool &fValue)
Definition: txdb.cpp:239
std::string _(const char *psz)
Translation function.
Definition: system.h:52
TxIndex(size_t n_cache_size, bool f_memory=false, bool f_wipe=false)
Constructs the index, which becomes available to be queried.
Definition: txindex.cpp:226
uint256 GetHash() const
Definition: block.cpp:12
virtual ~TxIndex() override
Definition: txindex.cpp:230
256-bit opaque blob.
Definition: uint256.h:121
bool Write(const K &key, const V &value, bool fSync=false)
Definition: dbwrapper.h:257
void CompactRange(const K &key_begin, const K &key_end) const
Compact a certain range of keys in the database.
Definition: dbwrapper.h:338
std::vector< CTransactionRef > vtx
Definition: block.h:76
FILE * Get() const
Get wrapped FILE* without transfer of ownership.
Definition: streams.h:644
The block chain is a tree shaped structure starting with the genesis block at the root...
Definition: chain.h:139
bool WriteTxs(const std::vector< std::pair< uint256, CDiskTxPos >> &v_pos)
Write a batch of transaction positions to the DB.
Definition: txindex.cpp:79
FILE * OpenBlockFile(const FlatFilePos &pos, bool fReadOnly)
Open a block file (blk?????.dat)
DB(size_t n_cache_size, bool f_memory=false, bool f_wipe=false)
Definition: txindex.cpp:70
constexpr char DB_BEST_BLOCK
Definition: txindex.cpp:13
std::unique_ptr< CBlockTreeDB > pblocktree
Global variable that points to the active block tree (protected by cs_main)
Definition: validation.cpp:173
bool WriteFlag(const std::string &name, bool fValue)
Definition: txdb.cpp:235
bool WriteBatch(CDBBatch &batch, bool fSync=false)
Definition: dbwrapper.cpp:183
CClientUIInterface uiInterface
int nHeight
height of the entry in the chain. The genesis block has height 0
Definition: chain.h:152
void SetNull()
Definition: txindex.cpp:38
#define READWRITE(...)
Definition: serialize.h:184
CDiskTxPos()
Definition: txindex.cpp:34
unsigned int nPos
Definition: flatfile.h:17
bool error(const char *fmt, const Args &... args)
Definition: system.h:61
#define VARINT(obj,...)
Definition: serialize.h:422
TxIndex is used to look up transactions included in the blockchain by hash.
Definition: txindex.h:17
Nodes collect new transactions into a block, hash them into a hash tree, and scan through nonce value...
Definition: block.h:20
Non-refcounted RAII wrapper for FILE*.
Definition: streams.h:603