Bitcoin Core  0.18.99
P2P Digital Currency
blockfilterindex.cpp
Go to the documentation of this file.
1 // Copyright (c) 2018 The Bitcoin Core developers
2 // Distributed under the MIT software license, see the accompanying
3 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
4 
5 #include <map>
6 
7 #include <dbwrapper.h>
9 #include <util/system.h>
10 #include <validation.h>
11 
12 /* The index database stores three items for each block: the disk location of the encoded filter,
13  * its dSHA256 hash, and the header. Those belonging to blocks on the active chain are indexed by
14  * height, and those belonging to blocks that have been reorganized out of the active chain are
15  * indexed by block hash. This ensures that filter data for any block that becomes part of the
16  * active chain can always be retrieved, alleviating timing concerns.
17  *
18  * The filters themselves are stored in flat files and referenced by the LevelDB entries. This
19  * minimizes the amount of data written to LevelDB and keeps the database values constant size. The
20  * disk location of the next block filter to be written (represented as a FlatFilePos) is stored
21  * under the DB_FILTER_POS key.
22  *
23  * Keys for the height index have the type [DB_BLOCK_HEIGHT, uint32 (BE)]. The height is represented
24  * as big-endian so that sequential reads of filters by height are fast.
25  * Keys for the hash index have the type [DB_BLOCK_HASH, uint256].
26  */
27 constexpr char DB_BLOCK_HASH = 's';
28 constexpr char DB_BLOCK_HEIGHT = 't';
29 constexpr char DB_FILTER_POS = 'P';
30 
31 constexpr unsigned int MAX_FLTR_FILE_SIZE = 0x1000000; // 16 MiB
33 constexpr unsigned int FLTR_FILE_CHUNK_SIZE = 0x100000; // 1 MiB
34 
35 namespace {
36 
37 struct DBVal {
38  uint256 hash;
39  uint256 header;
40  FlatFilePos pos;
41 
43 
44  template <typename Stream, typename Operation>
45  inline void SerializationOp(Stream& s, Operation ser_action) {
46  READWRITE(hash);
47  READWRITE(header);
48  READWRITE(pos);
49  }
50 };
51 
52 struct DBHeightKey {
53  int height;
54 
55  DBHeightKey() : height(0) {}
56  explicit DBHeightKey(int height_in) : height(height_in) {}
57 
58  template<typename Stream>
59  void Serialize(Stream& s) const
60  {
62  ser_writedata32be(s, height);
63  }
64 
65  template<typename Stream>
66  void Unserialize(Stream& s)
67  {
68  char prefix = ser_readdata8(s);
69  if (prefix != DB_BLOCK_HEIGHT) {
70  throw std::ios_base::failure("Invalid format for block filter index DB height key");
71  }
72  height = ser_readdata32be(s);
73  }
74 };
75 
76 struct DBHashKey {
77  uint256 hash;
78 
79  explicit DBHashKey(const uint256& hash_in) : hash(hash_in) {}
80 
82 
83  template <typename Stream, typename Operation>
84  inline void SerializationOp(Stream& s, Operation ser_action) {
85  char prefix = DB_BLOCK_HASH;
86  READWRITE(prefix);
87  if (prefix != DB_BLOCK_HASH) {
88  throw std::ios_base::failure("Invalid format for block filter index DB hash key");
89  }
90 
91  READWRITE(hash);
92  }
93 };
94 
95 }; // namespace
96 
97 static std::map<BlockFilterType, BlockFilterIndex> g_filter_indexes;
98 
100  size_t n_cache_size, bool f_memory, bool f_wipe)
101  : m_filter_type(filter_type)
102 {
103  const std::string& filter_name = BlockFilterTypeName(filter_type);
104  if (filter_name.empty()) throw std::invalid_argument("unknown filter_type");
105 
106  fs::path path = GetDataDir() / "indexes" / "blockfilter" / filter_name;
107  fs::create_directories(path);
108 
109  m_name = filter_name + " block filter index";
110  m_db = MakeUnique<BaseIndex::DB>(path / "db", n_cache_size, f_memory, f_wipe);
111  m_filter_fileseq = MakeUnique<FlatFileSeq>(std::move(path), "fltr", FLTR_FILE_CHUNK_SIZE);
112 }
113 
115 {
116  if (!m_db->Read(DB_FILTER_POS, m_next_filter_pos)) {
117  // Check that the cause of the read failure is that the key does not exist. Any other errors
118  // indicate database corruption or a disk failure, and starting the index would cause
119  // further corruption.
120  if (m_db->Exists(DB_FILTER_POS)) {
121  return error("%s: Cannot read current %s state; index may be corrupted",
122  __func__, GetName());
123  }
124 
125  // If the DB_FILTER_POS is not set, then initialize to the first location.
128  }
129  return BaseIndex::Init();
130 }
131 
133 {
134  const FlatFilePos& pos = m_next_filter_pos;
135 
136  // Flush current filter file to disk.
138  if (file.IsNull()) {
139  return error("%s: Failed to open filter file %d", __func__, pos.nFile);
140  }
141  if (!FileCommit(file.Get())) {
142  return error("%s: Failed to commit filter file %d", __func__, pos.nFile);
143  }
144 
145  batch.Write(DB_FILTER_POS, pos);
146  return BaseIndex::CommitInternal(batch);
147 }
148 
150 {
151  CAutoFile filein(m_filter_fileseq->Open(pos, true), SER_DISK, CLIENT_VERSION);
152  if (filein.IsNull()) {
153  return false;
154  }
155 
156  uint256 block_hash;
157  std::vector<unsigned char> encoded_filter;
158  try {
159  filein >> block_hash >> encoded_filter;
160  filter = BlockFilter(GetFilterType(), block_hash, std::move(encoded_filter));
161  }
162  catch (const std::exception& e) {
163  return error("%s: Failed to deserialize block filter from disk: %s", __func__, e.what());
164  }
165 
166  return true;
167 }
168 
170 {
171  assert(filter.GetFilterType() == GetFilterType());
172 
173  size_t data_size =
176 
177  // If writing the filter would overflow the file, flush and move to the next one.
178  if (pos.nPos + data_size > MAX_FLTR_FILE_SIZE) {
179  CAutoFile last_file(m_filter_fileseq->Open(pos), SER_DISK, CLIENT_VERSION);
180  if (last_file.IsNull()) {
181  LogPrintf("%s: Failed to open filter file %d\n", __func__, pos.nFile);
182  return 0;
183  }
184  if (!TruncateFile(last_file.Get(), pos.nPos)) {
185  LogPrintf("%s: Failed to truncate filter file %d\n", __func__, pos.nFile);
186  return 0;
187  }
188  if (!FileCommit(last_file.Get())) {
189  LogPrintf("%s: Failed to commit filter file %d\n", __func__, pos.nFile);
190  return 0;
191  }
192 
193  pos.nFile++;
194  pos.nPos = 0;
195  }
196 
197  // Pre-allocate sufficient space for filter data.
198  bool out_of_space;
199  m_filter_fileseq->Allocate(pos, data_size, out_of_space);
200  if (out_of_space) {
201  LogPrintf("%s: out of disk space\n", __func__);
202  return 0;
203  }
204 
205  CAutoFile fileout(m_filter_fileseq->Open(pos), SER_DISK, CLIENT_VERSION);
206  if (fileout.IsNull()) {
207  LogPrintf("%s: Failed to open filter file %d\n", __func__, pos.nFile);
208  return 0;
209  }
210 
211  fileout << filter.GetBlockHash() << filter.GetEncodedFilter();
212  return data_size;
213 }
214 
215 bool BlockFilterIndex::WriteBlock(const CBlock& block, const CBlockIndex* pindex)
216 {
217  CBlockUndo block_undo;
218  uint256 prev_header;
219 
220  if (pindex->nHeight > 0) {
221  if (!UndoReadFromDisk(block_undo, pindex)) {
222  return false;
223  }
224 
225  std::pair<uint256, DBVal> read_out;
226  if (!m_db->Read(DBHeightKey(pindex->nHeight - 1), read_out)) {
227  return false;
228  }
229 
230  uint256 expected_block_hash = pindex->pprev->GetBlockHash();
231  if (read_out.first != expected_block_hash) {
232  return error("%s: previous block header belongs to unexpected block %s; expected %s",
233  __func__, read_out.first.ToString(), expected_block_hash.ToString());
234  }
235 
236  prev_header = read_out.second.header;
237  }
238 
239  BlockFilter filter(m_filter_type, block, block_undo);
240 
241  size_t bytes_written = WriteFilterToDisk(m_next_filter_pos, filter);
242  if (bytes_written == 0) return false;
243 
244  std::pair<uint256, DBVal> value;
245  value.first = pindex->GetBlockHash();
246  value.second.hash = filter.GetHash();
247  value.second.header = filter.ComputeHeader(prev_header);
248  value.second.pos = m_next_filter_pos;
249 
250  if (!m_db->Write(DBHeightKey(pindex->nHeight), value)) {
251  return false;
252  }
253 
254  m_next_filter_pos.nPos += bytes_written;
255  return true;
256 }
257 
259  const std::string& index_name,
260  int start_height, int stop_height)
261 {
262  DBHeightKey key(start_height);
263  db_it.Seek(key);
264 
265  for (int height = start_height; height <= stop_height; ++height) {
266  if (!db_it.GetKey(key) || key.height != height) {
267  return error("%s: unexpected key in %s: expected (%c, %d)",
268  __func__, index_name, DB_BLOCK_HEIGHT, height);
269  }
270 
271  std::pair<uint256, DBVal> value;
272  if (!db_it.GetValue(value)) {
273  return error("%s: unable to read value in %s at key (%c, %d)",
274  __func__, index_name, DB_BLOCK_HEIGHT, height);
275  }
276 
277  batch.Write(DBHashKey(value.first), std::move(value.second));
278 
279  db_it.Next();
280  }
281  return true;
282 }
283 
284 bool BlockFilterIndex::Rewind(const CBlockIndex* current_tip, const CBlockIndex* new_tip)
285 {
286  assert(current_tip->GetAncestor(new_tip->nHeight) == new_tip);
287 
288  CDBBatch batch(*m_db);
289  std::unique_ptr<CDBIterator> db_it(m_db->NewIterator());
290 
291  // During a reorg, we need to copy all filters for blocks that are getting disconnected from the
292  // height index to the hash index so we can still find them when the height index entries are
293  // overwritten.
294  if (!CopyHeightIndexToHashIndex(*db_it, batch, m_name, new_tip->nHeight, current_tip->nHeight)) {
295  return false;
296  }
297 
298  // The latest filter position gets written in Commit by the call to the BaseIndex::Rewind.
299  // But since this creates new references to the filter, the position should get updated here
300  // atomically as well in case Commit fails.
301  batch.Write(DB_FILTER_POS, m_next_filter_pos);
302  if (!m_db->WriteBatch(batch)) return false;
303 
304  return BaseIndex::Rewind(current_tip, new_tip);
305 }
306 
307 static bool LookupOne(const CDBWrapper& db, const CBlockIndex* block_index, DBVal& result)
308 {
309  // First check if the result is stored under the height index and the value there matches the
310  // block hash. This should be the case if the block is on the active chain.
311  std::pair<uint256, DBVal> read_out;
312  if (!db.Read(DBHeightKey(block_index->nHeight), read_out)) {
313  return false;
314  }
315  if (read_out.first == block_index->GetBlockHash()) {
316  result = std::move(read_out.second);
317  return true;
318  }
319 
320  // If value at the height index corresponds to an different block, the result will be stored in
321  // the hash index.
322  return db.Read(DBHashKey(block_index->GetBlockHash()), result);
323 }
324 
325 static bool LookupRange(CDBWrapper& db, const std::string& index_name, int start_height,
326  const CBlockIndex* stop_index, std::vector<DBVal>& results)
327 {
328  if (start_height < 0) {
329  return error("%s: start height (%d) is negative", __func__, start_height);
330  }
331  if (start_height > stop_index->nHeight) {
332  return error("%s: start height (%d) is greater than stop height (%d)",
333  __func__, start_height, stop_index->nHeight);
334  }
335 
336  size_t results_size = static_cast<size_t>(stop_index->nHeight - start_height + 1);
337  std::vector<std::pair<uint256, DBVal>> values(results_size);
338 
339  DBHeightKey key(start_height);
340  std::unique_ptr<CDBIterator> db_it(db.NewIterator());
341  db_it->Seek(DBHeightKey(start_height));
342  for (int height = start_height; height <= stop_index->nHeight; ++height) {
343  if (!db_it->Valid() || !db_it->GetKey(key) || key.height != height) {
344  return false;
345  }
346 
347  size_t i = static_cast<size_t>(height - start_height);
348  if (!db_it->GetValue(values[i])) {
349  return error("%s: unable to read value in %s at key (%c, %d)",
350  __func__, index_name, DB_BLOCK_HEIGHT, height);
351  }
352 
353  db_it->Next();
354  }
355 
356  results.resize(results_size);
357 
358  // Iterate backwards through block indexes collecting results in order to access the block hash
359  // of each entry in case we need to look it up in the hash index.
360  for (const CBlockIndex* block_index = stop_index;
361  block_index && block_index->nHeight >= start_height;
362  block_index = block_index->pprev) {
363  uint256 block_hash = block_index->GetBlockHash();
364 
365  size_t i = static_cast<size_t>(block_index->nHeight - start_height);
366  if (block_hash == values[i].first) {
367  results[i] = std::move(values[i].second);
368  continue;
369  }
370 
371  if (!db.Read(DBHashKey(block_hash), results[i])) {
372  return error("%s: unable to read value in %s at key (%c, %s)",
373  __func__, index_name, DB_BLOCK_HASH, block_hash.ToString());
374  }
375  }
376 
377  return true;
378 }
379 
380 bool BlockFilterIndex::LookupFilter(const CBlockIndex* block_index, BlockFilter& filter_out) const
381 {
382  DBVal entry;
383  if (!LookupOne(*m_db, block_index, entry)) {
384  return false;
385  }
386 
387  return ReadFilterFromDisk(entry.pos, filter_out);
388 }
389 
390 bool BlockFilterIndex::LookupFilterHeader(const CBlockIndex* block_index, uint256& header_out) const
391 {
392  DBVal entry;
393  if (!LookupOne(*m_db, block_index, entry)) {
394  return false;
395  }
396 
397  header_out = entry.header;
398  return true;
399 }
400 
401 bool BlockFilterIndex::LookupFilterRange(int start_height, const CBlockIndex* stop_index,
402  std::vector<BlockFilter>& filters_out) const
403 {
404  std::vector<DBVal> entries;
405  if (!LookupRange(*m_db, m_name, start_height, stop_index, entries)) {
406  return false;
407  }
408 
409  filters_out.resize(entries.size());
410  auto filter_pos_it = filters_out.begin();
411  for (const auto& entry : entries) {
412  if (!ReadFilterFromDisk(entry.pos, *filter_pos_it)) {
413  return false;
414  }
415  ++filter_pos_it;
416  }
417 
418  return true;
419 }
420 
421 bool BlockFilterIndex::LookupFilterHashRange(int start_height, const CBlockIndex* stop_index,
422  std::vector<uint256>& hashes_out) const
423 
424 {
425  std::vector<DBVal> entries;
426  if (!LookupRange(*m_db, m_name, start_height, stop_index, entries)) {
427  return false;
428  }
429 
430  hashes_out.clear();
431  hashes_out.reserve(entries.size());
432  for (const auto& entry : entries) {
433  hashes_out.push_back(entry.hash);
434  }
435  return true;
436 }
437 
439 {
440  auto it = g_filter_indexes.find(filter_type);
441  return it != g_filter_indexes.end() ? &it->second : nullptr;
442 }
443 
444 void ForEachBlockFilterIndex(std::function<void (BlockFilterIndex&)> fn)
445 {
446  for (auto& entry : g_filter_indexes) fn(entry.second);
447 }
448 
450  size_t n_cache_size, bool f_memory, bool f_wipe)
451 {
452  auto result = g_filter_indexes.emplace(std::piecewise_construct,
453  std::forward_as_tuple(filter_type),
454  std::forward_as_tuple(filter_type,
455  n_cache_size, f_memory, f_wipe));
456  return result.second;
457 }
458 
460 {
461  return g_filter_indexes.erase(filter_type);
462 }
463 
465 {
466  g_filter_indexes.clear();
467 }
bool GetKey(K &key)
Definition: dbwrapper.h:147
bool LookupFilter(const CBlockIndex *block_index, BlockFilter &filter_out) const
Get a single filter by block.
BlockFilterIndex is used to store and retrieve block filters, hashes, and headers for a range of bloc...
constexpr char DB_BLOCK_HEIGHT
bool FileCommit(FILE *file)
Definition: system.cpp:999
constexpr char DB_BLOCK_HASH
virtual bool Init()
Initialize internal state from the database and block index.
Definition: base.cpp:55
uint8_t ser_readdata8(Stream &s)
Definition: serialize.h:102
CBlockIndex * pprev
pointer to the index of the predecessor of this block
Definition: chain.h:146
Batch of changes queued to be written to a CDBWrapper.
Definition: dbwrapper.h:47
Definition: block.h:72
static bool LookupRange(CDBWrapper &db, const std::string &index_name, int start_height, const CBlockIndex *stop_index, std::vector< DBVal > &results)
BlockFilterIndex * GetBlockFilterIndex(BlockFilterType filter_type)
Get a block filter index by type.
const char * prefix
Definition: rest.cpp:626
void ForEachBlockFilterIndex(std::function< void(BlockFilterIndex &)> fn)
Iterate over all running block filter indexes, invoking fn on each.
constexpr unsigned int MAX_FLTR_FILE_SIZE
bool Rewind(const CBlockIndex *current_tip, const CBlockIndex *new_tip) override
Rewind index to an earlier chain tip during a chain reorg.
static void LogPrintf(const char *fmt, const Args &... args)
Definition: logging.h:144
bool WriteBlock(const CBlock &block, const CBlockIndex *pindex) override
Write update index entries for a newly connected block.
bool InitBlockFilterIndex(BlockFilterType filter_type, size_t n_cache_size, bool f_memory, bool f_wipe)
Initialize a block filter index for the given type if one does not already exist. ...
virtual bool Rewind(const CBlockIndex *current_tip, const CBlockIndex *new_tip)
Rewind index to an earlier chain tip during a chain reorg.
Definition: base.cpp:175
int nFile
Definition: flatfile.h:16
std::unique_ptr< BaseIndex::DB > m_db
CDBIterator * NewIterator()
Definition: dbwrapper.h:308
uint256 GetHash() const
Compute the filter hash.
size_t GetSerializeSize(const T &t, int nVersion=0)
Definition: serialize.h:992
void Serialize(Stream &s, char a)
Definition: serialize.h:204
constexpr char DB_FILTER_POS
bool GetValue(V &value)
Definition: dbwrapper.h:158
uint256 GetBlockHash() const
Definition: chain.h:261
BlockFilterType
Definition: blockfilter.h:87
static bool CopyHeightIndexToHashIndex(CDBIterator &db_it, CDBBatch &batch, const std::string &index_name, int start_height, int stop_height)
const std::vector< unsigned char > & GetEncodedFilter() const
Definition: blockfilter.h:133
bool ReadFilterFromDisk(const FlatFilePos &pos, BlockFilter &filter) const
size_t WriteFilterToDisk(FlatFilePos &pos, const BlockFilter &filter)
Complete block filter struct as defined in BIP 157.
Definition: blockfilter.h:109
#define ADD_SERIALIZE_METHODS
Implement three methods for serializable objects.
Definition: serialize.h:193
bool UndoReadFromDisk(CBlockUndo &blockundo, const CBlockIndex *pindex)
std::string m_name
BlockFilterIndex(BlockFilterType filter_type, size_t n_cache_size, bool f_memory=false, bool f_wipe=false)
Constructs the index, which becomes available to be queried.
void DestroyAllBlockFilterIndexes()
Destroy all open block filter indexes.
const fs::path & GetDataDir(bool fNetSpecific)
Definition: system.cpp:742
constexpr unsigned int FLTR_FILE_CHUNK_SIZE
The pre-allocation chunk size for fltr?????.dat files.
void Write(const K &key, const V &value)
Definition: dbwrapper.h:73
const uint256 & GetBlockHash() const
Definition: blockfilter.h:130
bool LookupFilterHashRange(int start_height, const CBlockIndex *stop_index, std::vector< uint256 > &hashes_out) const
Get a range of filter hashes between two heights on a chain.
virtual bool CommitInternal(CDBBatch &batch)
Virtual method called internally by Commit that can be overridden to atomically commit more index sta...
Definition: base.cpp:168
std::string ToString() const
Definition: uint256.cpp:61
void ser_writedata32be(Stream &s, uint32_t obj)
Definition: serialize.h:92
bool Read(const K &key, V &value) const
Definition: dbwrapper.h:231
BlockFilterType m_filter_type
bool LookupFilterHeader(const CBlockIndex *block_index, uint256 &header_out) const
Get a single filter header by block.
uint256 ComputeHeader(const uint256 &prev_header) const
Compute the filter header given the previous one.
void Next()
Definition: dbwrapper.cpp:239
256-bit opaque blob.
Definition: uint256.h:121
bool TruncateFile(FILE *file, unsigned int length)
Definition: system.cpp:1032
The block chain is a tree shaped structure starting with the genesis block at the root...
Definition: chain.h:139
bool DestroyBlockFilterIndex(BlockFilterType filter_type)
Destroy the block filter index with the given type.
Undo information for a CBlock.
Definition: undo.h:101
FlatFilePos m_next_filter_pos
BlockFilterType GetFilterType() const
static std::map< BlockFilterType, BlockFilterIndex > g_filter_indexes
uint32_t ser_readdata32be(Stream &s)
Definition: serialize.h:126
void Seek(const K &key)
Definition: dbwrapper.h:137
void Unserialize(Stream &s, char &a)
Definition: serialize.h:222
bool CommitInternal(CDBBatch &batch) override
Virtual method called internally by Commit that can be overridden to atomically commit more index sta...
std::unique_ptr< FlatFileSeq > m_filter_fileseq
int nHeight
height of the entry in the chain. The genesis block has height 0
Definition: chain.h:152
CBlockIndex * GetAncestor(int height)
Efficiently find an ancestor of this block.
Definition: chain.cpp:111
#define READWRITE(...)
Definition: serialize.h:184
bool Init() override
Initialize internal state from the database and block index.
BlockFilterType GetFilterType() const
Definition: blockfilter.h:129
auto it
Definition: validation.cpp:366
static const int CLIENT_VERSION
bitcoind-res.rc includes this file, but it cannot cope with real c++ code.
Definition: clientversion.h:38
unsigned int nPos
Definition: flatfile.h:17
bool LookupFilterRange(int start_height, const CBlockIndex *stop_index, std::vector< BlockFilter > &filters_out) const
Get a range of filters between two heights on a chain.
bool error(const char *fmt, const Args &... args)
Definition: system.h:47
static bool LookupOne(const CDBWrapper &db, const CBlockIndex *block_index, DBVal &result)
Non-refcounted RAII wrapper for FILE*.
Definition: streams.h:603
void ser_writedata8(Stream &s, uint8_t obj)
Definition: serialize.h:73
const char * GetName() const override
Get the name of the index for display in logs.
const std::string & BlockFilterTypeName(BlockFilterType filter_type)
Get the human-readable name for a filter type.