Bitcoin Core  0.18.99
P2P Digital Currency
txindex.cpp
Go to the documentation of this file.
1 // Copyright (c) 2017-2018 The Bitcoin Core developers
2 // Distributed under the MIT software license, see the accompanying
3 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
4 
5 #include <index/txindex.h>
6 #include <shutdown.h>
7 #include <ui_interface.h>
8 #include <util/system.h>
9 #include <util/translation.h>
10 #include <validation.h>
11 
12 #include <boost/thread.hpp>
13 
14 constexpr char DB_BEST_BLOCK = 'B';
15 constexpr char DB_TXINDEX = 't';
16 constexpr char DB_TXINDEX_BLOCK = 'T';
17 
18 std::unique_ptr<TxIndex> g_txindex;
19 
20 struct CDiskTxPos : public FlatFilePos
21 {
22  unsigned int nTxOffset; // after header
23 
25 
26  template <typename Stream, typename Operation>
27  inline void SerializationOp(Stream& s, Operation ser_action) {
28  READWRITEAS(FlatFilePos, *this);
29  READWRITE(VARINT(nTxOffset));
30  }
31 
32  CDiskTxPos(const FlatFilePos &blockIn, unsigned int nTxOffsetIn) : FlatFilePos(blockIn.nFile, blockIn.nPos), nTxOffset(nTxOffsetIn) {
33  }
34 
36  SetNull();
37  }
38 
39  void SetNull() {
41  nTxOffset = 0;
42  }
43 };
44 
54 class TxIndex::DB : public BaseIndex::DB
55 {
56 public:
57  explicit DB(size_t n_cache_size, bool f_memory = false, bool f_wipe = false);
58 
61  bool ReadTxPos(const uint256& txid, CDiskTxPos& pos) const;
62 
64  bool WriteTxs(const std::vector<std::pair<uint256, CDiskTxPos>>& v_pos);
65 
68  bool MigrateData(CBlockTreeDB& block_tree_db, const CBlockLocator& best_locator);
69 };
70 
71 TxIndex::DB::DB(size_t n_cache_size, bool f_memory, bool f_wipe) :
72  BaseIndex::DB(GetDataDir() / "indexes" / "txindex", n_cache_size, f_memory, f_wipe)
73 {}
74 
75 bool TxIndex::DB::ReadTxPos(const uint256 &txid, CDiskTxPos& pos) const
76 {
77  return Read(std::make_pair(DB_TXINDEX, txid), pos);
78 }
79 
80 bool TxIndex::DB::WriteTxs(const std::vector<std::pair<uint256, CDiskTxPos>>& v_pos)
81 {
82  CDBBatch batch(*this);
83  for (const auto& tuple : v_pos) {
84  batch.Write(std::make_pair(DB_TXINDEX, tuple.first), tuple.second);
85  }
86  return WriteBatch(batch);
87 }
88 
89 /*
90  * Safely persist a transfer of data from the old txindex database to the new one, and compact the
91  * range of keys updated. This is used internally by MigrateData.
92  */
94  CDBBatch& batch_newdb, CDBBatch& batch_olddb,
95  const std::pair<unsigned char, uint256>& begin_key,
96  const std::pair<unsigned char, uint256>& end_key)
97 {
98  // Sync new DB changes to disk before deleting from old DB.
99  newdb.WriteBatch(batch_newdb, /*fSync=*/ true);
100  olddb.WriteBatch(batch_olddb);
101  olddb.CompactRange(begin_key, end_key);
102 
103  batch_newdb.Clear();
104  batch_olddb.Clear();
105 }
106 
107 bool TxIndex::DB::MigrateData(CBlockTreeDB& block_tree_db, const CBlockLocator& best_locator)
108 {
109  // The prior implementation of txindex was always in sync with block index
110  // and presence was indicated with a boolean DB flag. If the flag is set,
111  // this means the txindex from a previous version is valid and in sync with
112  // the chain tip. The first step of the migration is to unset the flag and
113  // write the chain hash to a separate key, DB_TXINDEX_BLOCK. After that, the
114  // index entries are copied over in batches to the new database. Finally,
115  // DB_TXINDEX_BLOCK is erased from the old database and the block hash is
116  // written to the new database.
117  //
118  // Unsetting the boolean flag ensures that if the node is downgraded to a
119  // previous version, it will not see a corrupted, partially migrated index
120  // -- it will see that the txindex is disabled. When the node is upgraded
121  // again, the migration will pick up where it left off and sync to the block
122  // with hash DB_TXINDEX_BLOCK.
123  bool f_legacy_flag = false;
124  block_tree_db.ReadFlag("txindex", f_legacy_flag);
125  if (f_legacy_flag) {
126  if (!block_tree_db.Write(DB_TXINDEX_BLOCK, best_locator)) {
127  return error("%s: cannot write block indicator", __func__);
128  }
129  if (!block_tree_db.WriteFlag("txindex", false)) {
130  return error("%s: cannot write block index db flag", __func__);
131  }
132  }
133 
134  CBlockLocator locator;
135  if (!block_tree_db.Read(DB_TXINDEX_BLOCK, locator)) {
136  return true;
137  }
138 
139  int64_t count = 0;
140  LogPrintf("Upgrading txindex database... [0%%]\n");
141  uiInterface.ShowProgress(_("Upgrading txindex database").translated, 0, true);
142  int report_done = 0;
143  const size_t batch_size = 1 << 24; // 16 MiB
144 
145  CDBBatch batch_newdb(*this);
146  CDBBatch batch_olddb(block_tree_db);
147 
148  std::pair<unsigned char, uint256> key;
149  std::pair<unsigned char, uint256> begin_key{DB_TXINDEX, uint256()};
150  std::pair<unsigned char, uint256> prev_key = begin_key;
151 
152  bool interrupted = false;
153  std::unique_ptr<CDBIterator> cursor(block_tree_db.NewIterator());
154  for (cursor->Seek(begin_key); cursor->Valid(); cursor->Next()) {
155  boost::this_thread::interruption_point();
156  if (ShutdownRequested()) {
157  interrupted = true;
158  break;
159  }
160 
161  if (!cursor->GetKey(key)) {
162  return error("%s: cannot get key from valid cursor", __func__);
163  }
164  if (key.first != DB_TXINDEX) {
165  break;
166  }
167 
168  // Log progress every 10%.
169  if (++count % 256 == 0) {
170  // Since txids are uniformly random and traversed in increasing order, the high 16 bits
171  // of the hash can be used to estimate the current progress.
172  const uint256& txid = key.second;
173  uint32_t high_nibble =
174  (static_cast<uint32_t>(*(txid.begin() + 0)) << 8) +
175  (static_cast<uint32_t>(*(txid.begin() + 1)) << 0);
176  int percentage_done = (int)(high_nibble * 100.0 / 65536.0 + 0.5);
177 
178  uiInterface.ShowProgress(_("Upgrading txindex database").translated, percentage_done, true);
179  if (report_done < percentage_done/10) {
180  LogPrintf("Upgrading txindex database... [%d%%]\n", percentage_done);
181  report_done = percentage_done/10;
182  }
183  }
184 
185  CDiskTxPos value;
186  if (!cursor->GetValue(value)) {
187  return error("%s: cannot parse txindex record", __func__);
188  }
189  batch_newdb.Write(key, value);
190  batch_olddb.Erase(key);
191 
192  if (batch_newdb.SizeEstimate() > batch_size || batch_olddb.SizeEstimate() > batch_size) {
193  // NOTE: it's OK to delete the key pointed at by the current DB cursor while iterating
194  // because LevelDB iterators are guaranteed to provide a consistent view of the
195  // underlying data, like a lightweight snapshot.
196  WriteTxIndexMigrationBatches(*this, block_tree_db,
197  batch_newdb, batch_olddb,
198  prev_key, key);
199  prev_key = key;
200  }
201  }
202 
203  // If these final DB batches complete the migration, write the best block
204  // hash marker to the new database and delete from the old one. This signals
205  // that the former is fully caught up to that point in the blockchain and
206  // that all txindex entries have been removed from the latter.
207  if (!interrupted) {
208  batch_olddb.Erase(DB_TXINDEX_BLOCK);
209  batch_newdb.Write(DB_BEST_BLOCK, locator);
210  }
211 
212  WriteTxIndexMigrationBatches(*this, block_tree_db,
213  batch_newdb, batch_olddb,
214  begin_key, key);
215 
216  if (interrupted) {
217  LogPrintf("[CANCELLED].\n");
218  return false;
219  }
220 
221  uiInterface.ShowProgress("", 100, false);
222 
223  LogPrintf("[DONE].\n");
224  return true;
225 }
226 
227 TxIndex::TxIndex(size_t n_cache_size, bool f_memory, bool f_wipe)
228  : m_db(MakeUnique<TxIndex::DB>(n_cache_size, f_memory, f_wipe))
229 {}
230 
232 
234 {
235  LOCK(cs_main);
236 
237  // Attempt to migrate txindex from the old database to the new one. Even if
238  // chain_tip is null, the node could be reindexing and we still want to
239  // delete txindex records in the old database.
240  if (!m_db->MigrateData(*pblocktree, ::ChainActive().GetLocator())) {
241  return false;
242  }
243 
244  return BaseIndex::Init();
245 }
246 
247 bool TxIndex::WriteBlock(const CBlock& block, const CBlockIndex* pindex)
248 {
249  // Exclude genesis block transaction because outputs are not spendable.
250  if (pindex->nHeight == 0) return true;
251 
252  CDiskTxPos pos(pindex->GetBlockPos(), GetSizeOfCompactSize(block.vtx.size()));
253  std::vector<std::pair<uint256, CDiskTxPos>> vPos;
254  vPos.reserve(block.vtx.size());
255  for (const auto& tx : block.vtx) {
256  vPos.emplace_back(tx->GetHash(), pos);
258  }
259  return m_db->WriteTxs(vPos);
260 }
261 
262 BaseIndex::DB& TxIndex::GetDB() const { return *m_db; }
263 
264 bool TxIndex::FindTx(const uint256& tx_hash, uint256& block_hash, CTransactionRef& tx) const
265 {
266  CDiskTxPos postx;
267  if (!m_db->ReadTxPos(tx_hash, postx)) {
268  return false;
269  }
270 
271  CAutoFile file(OpenBlockFile(postx, true), SER_DISK, CLIENT_VERSION);
272  if (file.IsNull()) {
273  return error("%s: OpenBlockFile failed", __func__);
274  }
275  CBlockHeader header;
276  try {
277  file >> header;
278  if (fseek(file.Get(), postx.nTxOffset, SEEK_CUR)) {
279  return error("%s: fseek(...) failed", __func__);
280  }
281  file >> tx;
282  } catch (const std::exception& e) {
283  return error("%s: Deserialize or I/O error - %s", __func__, e.what());
284  }
285  if (tx->GetHash() != tx_hash) {
286  return error("%s: txid mismatch", __func__);
287  }
288  block_hash = header.GetHash();
289  return true;
290 }
std::shared_ptr< const CTransaction > CTransactionRef
Definition: transaction.h:408
void SerializationOp(Stream &s, Operation ser_action)
Definition: txindex.cpp:27
static void WriteTxIndexMigrationBatches(CDBWrapper &newdb, CDBWrapper &olddb, CDBBatch &batch_newdb, CDBBatch &batch_olddb, const std::pair< unsigned char, uint256 > &begin_key, const std::pair< unsigned char, uint256 > &end_key)
Definition: txindex.cpp:93
bool WriteBlock(const CBlock &block, const CBlockIndex *pindex) override
Write update index entries for a newly connected block.
Definition: txindex.cpp:247
void Clear()
Definition: dbwrapper.h:66
bool ShutdownRequested()
Definition: shutdown.cpp:20
unsigned int nTxOffset
Definition: txindex.cpp:22
virtual bool Init()
Initialize internal state from the database and block index.
Definition: base.cpp:55
Describes a place in the block chain to another node such that if the other node doesn&#39;t have the sam...
Definition: block.h:126
constexpr char DB_TXINDEX_BLOCK
Definition: txindex.cpp:16
Batch of changes queued to be written to a CDBWrapper.
Definition: dbwrapper.h:47
Definition: block.h:72
CChain & ChainActive()
Definition: validation.cpp:92
void Erase(const K &key)
Definition: dbwrapper.h:98
unsigned int GetSizeOfCompactSize(uint64_t nSize)
Compact Size size < 253 – 1 byte size <= USHRT_MAX – 3 bytes (253 + 2 bytes) size <= UINT_MAX – 5 ...
Definition: serialize.h:253
constexpr char DB_TXINDEX
Definition: txindex.cpp:15
static void LogPrintf(const char *fmt, const Args &... args)
Definition: logging.h:144
FlatFilePos GetBlockPos() const
Definition: chain.h:230
std::unique_ptr< T > MakeUnique(Args &&... args)
Substitute for C++14 std::make_unique.
Definition: memory.h:14
int nFile
Definition: flatfile.h:16
const std::unique_ptr< DB > m_db
Definition: txindex.h:20
#define READWRITEAS(type, obj)
Definition: serialize.h:185
unsigned char * begin()
Definition: uint256.h:55
bool Init() override
Override base class init to migrate from old database.
Definition: txindex.cpp:233
CDBIterator * NewIterator()
Definition: dbwrapper.h:308
bool IsNull() const
Return true if the wrapped FILE* is nullptr, false otherwise.
Definition: streams.h:648
Access to the txindex database (indexes/txindex/)
Definition: txindex.cpp:54
bool ReadTxPos(const uint256 &txid, CDiskTxPos &pos) const
Read the disk location of the transaction data with the given hash.
Definition: txindex.cpp:75
size_t GetSerializeSize(const T &t, int nVersion=0)
Definition: serialize.h:992
Base class for indices of blockchain data.
Definition: base.h:22
CCriticalSection cs_main
Mutex to guard access to validation specific variables, such as reading or changing the chainstate...
Definition: validation.cpp:107
bool FindTx(const uint256 &tx_hash, uint256 &block_hash, CTransactionRef &tx) const
Look up a transaction by hash.
Definition: txindex.cpp:264
Access to the block database (blocks/index/)
Definition: txdb.h:91
CDiskTxPos(const FlatFilePos &blockIn, unsigned int nTxOffsetIn)
Definition: txindex.cpp:32
bool MigrateData(CBlockTreeDB &block_tree_db, const CBlockLocator &best_locator)
Migrate txindex data from the block tree DB, where it may be for older nodes that have not been upgra...
Definition: txindex.cpp:107
#define LOCK(cs)
Definition: sync.h:182
std::unique_ptr< TxIndex > g_txindex
The global transaction index, used in GetTransaction. May be null.
Definition: txindex.cpp:18
bilingual_str _(const char *psz)
Translation function.
Definition: translation.h:37
const fs::path & GetDataDir(bool fNetSpecific)
Definition: system.cpp:742
ADD_SERIALIZE_METHODS
Definition: txindex.cpp:24
void SetNull()
Definition: flatfile.h:42
void Write(const K &key, const V &value)
Definition: dbwrapper.h:73
BaseIndex::DB & GetDB() const override
Definition: txindex.cpp:262
size_t SizeEstimate() const
Definition: dbwrapper.h:114
bool Read(const K &key, V &value) const
Definition: dbwrapper.h:231
bool ReadFlag(const std::string &name, bool &fValue)
Definition: txdb.cpp:240
TxIndex(size_t n_cache_size, bool f_memory=false, bool f_wipe=false)
Constructs the index, which becomes available to be queried.
Definition: txindex.cpp:227
uint256 GetHash() const
Definition: block.cpp:12
virtual ~TxIndex() override
Definition: txindex.cpp:231
256-bit opaque blob.
Definition: uint256.h:121
bool Write(const K &key, const V &value, bool fSync=false)
Definition: dbwrapper.h:257
void CompactRange(const K &key_begin, const K &key_end) const
Compact a certain range of keys in the database.
Definition: dbwrapper.h:338
std::vector< CTransactionRef > vtx
Definition: block.h:76
FILE * Get() const
Get wrapped FILE* without transfer of ownership.
Definition: streams.h:644
The block chain is a tree shaped structure starting with the genesis block at the root...
Definition: chain.h:139
bool WriteTxs(const std::vector< std::pair< uint256, CDiskTxPos >> &v_pos)
Write a batch of transaction positions to the DB.
Definition: txindex.cpp:80
FILE * OpenBlockFile(const FlatFilePos &pos, bool fReadOnly)
Open a block file (blk?????.dat)
DB(size_t n_cache_size, bool f_memory=false, bool f_wipe=false)
Definition: txindex.cpp:71
constexpr char DB_BEST_BLOCK
Definition: txindex.cpp:14
std::unique_ptr< CBlockTreeDB > pblocktree
Global variable that points to the active block tree (protected by cs_main)
Definition: validation.cpp:182
static int count
Definition: tests.c:45
bool WriteFlag(const std::string &name, bool fValue)
Definition: txdb.cpp:236
bool WriteBatch(CDBBatch &batch, bool fSync=false)
Definition: dbwrapper.cpp:183
CClientUIInterface uiInterface
int nHeight
height of the entry in the chain. The genesis block has height 0
Definition: chain.h:152
void SetNull()
Definition: txindex.cpp:39
#define READWRITE(...)
Definition: serialize.h:184
CDiskTxPos()
Definition: txindex.cpp:35
static const int CLIENT_VERSION
bitcoind-res.rc includes this file, but it cannot cope with real c++ code.
Definition: clientversion.h:38
unsigned int nPos
Definition: flatfile.h:17
bool error(const char *fmt, const Args &... args)
Definition: system.h:47
#define VARINT(obj,...)
Definition: serialize.h:422
TxIndex is used to look up transactions included in the blockchain by hash.
Definition: txindex.h:17
Nodes collect new transactions into a block, hash them into a hash tree, and scan through nonce value...
Definition: block.h:20
Non-refcounted RAII wrapper for FILE*.
Definition: streams.h:603