Bitcoin Core  27.99.0
P2P Digital Currency
fees.cpp
Go to the documentation of this file.
1 // Copyright (c) 2009-2010 Satoshi Nakamoto
2 // Copyright (c) 2009-2022 The Bitcoin Core developers
3 // Distributed under the MIT software license, see the accompanying
4 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
5 
6 #include <policy/fees.h>
7 
8 #include <clientversion.h>
9 #include <common/system.h>
10 #include <consensus/amount.h>
11 #include <kernel/mempool_entry.h>
12 #include <logging.h>
13 #include <policy/feerate.h>
14 #include <primitives/transaction.h>
15 #include <random.h>
16 #include <serialize.h>
17 #include <streams.h>
18 #include <sync.h>
19 #include <tinyformat.h>
20 #include <uint256.h>
21 #include <util/fs.h>
22 #include <util/serfloat.h>
23 #include <util/time.h>
24 
25 #include <algorithm>
26 #include <cassert>
27 #include <chrono>
28 #include <cmath>
29 #include <cstddef>
30 #include <cstdint>
31 #include <exception>
32 #include <stdexcept>
33 #include <utility>
34 
35 static constexpr double INF_FEERATE = 1e99;
36 
38 {
39  switch (horizon) {
40  case FeeEstimateHorizon::SHORT_HALFLIFE: return "short";
41  case FeeEstimateHorizon::MED_HALFLIFE: return "medium";
42  case FeeEstimateHorizon::LONG_HALFLIFE: return "long";
43  } // no default case, so the compiler can warn about missing cases
44  assert(false);
45 }
46 
47 namespace {
48 
49 struct EncodedDoubleFormatter
50 {
51  template<typename Stream> void Ser(Stream &s, double v)
52  {
53  s << EncodeDouble(v);
54  }
55 
56  template<typename Stream> void Unser(Stream& s, double& v)
57  {
58  uint64_t encoded;
59  s >> encoded;
60  v = DecodeDouble(encoded);
61  }
62 };
63 
64 } // namespace
65 
75 {
76 private:
77  //Define the buckets we will group transactions into
78  const std::vector<double>& buckets; // The upper-bound of the range for the bucket (inclusive)
79  const std::map<double, unsigned int>& bucketMap; // Map of bucket upper-bound to index into all vectors by bucket
80 
81  // For each bucket X:
82  // Count the total # of txs in each bucket
83  // Track the historical moving average of this total over blocks
84  std::vector<double> txCtAvg;
85 
86  // Count the total # of txs confirmed within Y blocks in each bucket
87  // Track the historical moving average of these totals over blocks
88  std::vector<std::vector<double>> confAvg; // confAvg[Y][X]
89 
90  // Track moving avg of txs which have been evicted from the mempool
91  // after failing to be confirmed within Y blocks
92  std::vector<std::vector<double>> failAvg; // failAvg[Y][X]
93 
94  // Sum the total feerate of all tx's in each bucket
95  // Track the historical moving average of this total over blocks
96  std::vector<double> m_feerate_avg;
97 
98  // Combine the conf counts with tx counts to calculate the confirmation % for each Y,X
99  // Combine the total value with the tx counts to calculate the avg feerate per bucket
100 
101  double decay;
102 
103  // Resolution (# of blocks) with which confirmations are tracked
104  unsigned int scale;
105 
106  // Mempool counts of outstanding transactions
107  // For each bucket X, track the number of transactions in the mempool
108  // that are unconfirmed for each possible confirmation value Y
109  std::vector<std::vector<int> > unconfTxs; //unconfTxs[Y][X]
110  // transactions still unconfirmed after GetMaxConfirms for each bucket
111  std::vector<int> oldUnconfTxs;
112 
113  void resizeInMemoryCounters(size_t newbuckets);
114 
115 public:
123  TxConfirmStats(const std::vector<double>& defaultBuckets, const std::map<double, unsigned int>& defaultBucketMap,
124  unsigned int maxPeriods, double decay, unsigned int scale);
125 
127  void ClearCurrent(unsigned int nBlockHeight);
128 
135  void Record(int blocksToConfirm, double val);
136 
138  unsigned int NewTx(unsigned int nBlockHeight, double val);
139 
141  void removeTx(unsigned int entryHeight, unsigned int nBestSeenHeight,
142  unsigned int bucketIndex, bool inBlock);
143 
146  void UpdateMovingAverages();
147 
157  double EstimateMedianVal(int confTarget, double sufficientTxVal,
158  double minSuccess, unsigned int nBlockHeight,
159  EstimationResult *result = nullptr) const;
160 
162  unsigned int GetMaxConfirms() const { return scale * confAvg.size(); }
163 
165  void Write(AutoFile& fileout) const;
166 
171  void Read(AutoFile& filein, int nFileVersion, size_t numBuckets);
172 };
173 
174 
175 TxConfirmStats::TxConfirmStats(const std::vector<double>& defaultBuckets,
176  const std::map<double, unsigned int>& defaultBucketMap,
177  unsigned int maxPeriods, double _decay, unsigned int _scale)
178  : buckets(defaultBuckets), bucketMap(defaultBucketMap), decay(_decay), scale(_scale)
179 {
180  assert(_scale != 0 && "_scale must be non-zero");
181  confAvg.resize(maxPeriods);
182  failAvg.resize(maxPeriods);
183  for (unsigned int i = 0; i < maxPeriods; i++) {
184  confAvg[i].resize(buckets.size());
185  failAvg[i].resize(buckets.size());
186  }
187 
188  txCtAvg.resize(buckets.size());
189  m_feerate_avg.resize(buckets.size());
190 
192 }
193 
194 void TxConfirmStats::resizeInMemoryCounters(size_t newbuckets) {
195  // newbuckets must be passed in because the buckets referred to during Read have not been updated yet.
196  unconfTxs.resize(GetMaxConfirms());
197  for (unsigned int i = 0; i < unconfTxs.size(); i++) {
198  unconfTxs[i].resize(newbuckets);
199  }
200  oldUnconfTxs.resize(newbuckets);
201 }
202 
203 // Roll the unconfirmed txs circular buffer
204 void TxConfirmStats::ClearCurrent(unsigned int nBlockHeight)
205 {
206  for (unsigned int j = 0; j < buckets.size(); j++) {
207  oldUnconfTxs[j] += unconfTxs[nBlockHeight % unconfTxs.size()][j];
208  unconfTxs[nBlockHeight%unconfTxs.size()][j] = 0;
209  }
210 }
211 
212 
213 void TxConfirmStats::Record(int blocksToConfirm, double feerate)
214 {
215  // blocksToConfirm is 1-based
216  if (blocksToConfirm < 1)
217  return;
218  int periodsToConfirm = (blocksToConfirm + scale - 1) / scale;
219  unsigned int bucketindex = bucketMap.lower_bound(feerate)->second;
220  for (size_t i = periodsToConfirm; i <= confAvg.size(); i++) {
221  confAvg[i - 1][bucketindex]++;
222  }
223  txCtAvg[bucketindex]++;
224  m_feerate_avg[bucketindex] += feerate;
225 }
226 
228 {
229  assert(confAvg.size() == failAvg.size());
230  for (unsigned int j = 0; j < buckets.size(); j++) {
231  for (unsigned int i = 0; i < confAvg.size(); i++) {
232  confAvg[i][j] *= decay;
233  failAvg[i][j] *= decay;
234  }
235  m_feerate_avg[j] *= decay;
236  txCtAvg[j] *= decay;
237  }
238 }
239 
240 // returns -1 on error conditions
241 double TxConfirmStats::EstimateMedianVal(int confTarget, double sufficientTxVal,
242  double successBreakPoint, unsigned int nBlockHeight,
243  EstimationResult *result) const
244 {
245  // Counters for a bucket (or range of buckets)
246  double nConf = 0; // Number of tx's confirmed within the confTarget
247  double totalNum = 0; // Total number of tx's that were ever confirmed
248  int extraNum = 0; // Number of tx's still in mempool for confTarget or longer
249  double failNum = 0; // Number of tx's that were never confirmed but removed from the mempool after confTarget
250  const int periodTarget = (confTarget + scale - 1) / scale;
251  const int maxbucketindex = buckets.size() - 1;
252 
253  // We'll combine buckets until we have enough samples.
254  // The near and far variables will define the range we've combined
255  // The best variables are the last range we saw which still had a high
256  // enough confirmation rate to count as success.
257  // The cur variables are the current range we're counting.
258  unsigned int curNearBucket = maxbucketindex;
259  unsigned int bestNearBucket = maxbucketindex;
260  unsigned int curFarBucket = maxbucketindex;
261  unsigned int bestFarBucket = maxbucketindex;
262 
263  // We'll always group buckets into sets that meet sufficientTxVal --
264  // this ensures that we're using consistent groups between different
265  // confirmation targets.
266  double partialNum = 0;
267 
268  bool foundAnswer = false;
269  unsigned int bins = unconfTxs.size();
270  bool newBucketRange = true;
271  bool passing = true;
272  EstimatorBucket passBucket;
273  EstimatorBucket failBucket;
274 
275  // Start counting from highest feerate transactions
276  for (int bucket = maxbucketindex; bucket >= 0; --bucket) {
277  if (newBucketRange) {
278  curNearBucket = bucket;
279  newBucketRange = false;
280  }
281  curFarBucket = bucket;
282  nConf += confAvg[periodTarget - 1][bucket];
283  partialNum += txCtAvg[bucket];
284  totalNum += txCtAvg[bucket];
285  failNum += failAvg[periodTarget - 1][bucket];
286  for (unsigned int confct = confTarget; confct < GetMaxConfirms(); confct++)
287  extraNum += unconfTxs[(nBlockHeight - confct) % bins][bucket];
288  extraNum += oldUnconfTxs[bucket];
289  // If we have enough transaction data points in this range of buckets,
290  // we can test for success
291  // (Only count the confirmed data points, so that each confirmation count
292  // will be looking at the same amount of data and same bucket breaks)
293 
294  if (partialNum < sufficientTxVal / (1 - decay)) {
295  // the buckets we've added in this round aren't sufficient
296  // so keep adding
297  continue;
298  } else {
299  partialNum = 0; // reset for the next range we'll add
300 
301  double curPct = nConf / (totalNum + failNum + extraNum);
302 
303  // Check to see if we are no longer getting confirmed at the success rate
304  if (curPct < successBreakPoint) {
305  if (passing == true) {
306  // First time we hit a failure record the failed bucket
307  unsigned int failMinBucket = std::min(curNearBucket, curFarBucket);
308  unsigned int failMaxBucket = std::max(curNearBucket, curFarBucket);
309  failBucket.start = failMinBucket ? buckets[failMinBucket - 1] : 0;
310  failBucket.end = buckets[failMaxBucket];
311  failBucket.withinTarget = nConf;
312  failBucket.totalConfirmed = totalNum;
313  failBucket.inMempool = extraNum;
314  failBucket.leftMempool = failNum;
315  passing = false;
316  }
317  continue;
318  }
319  // Otherwise update the cumulative stats, and the bucket variables
320  // and reset the counters
321  else {
322  failBucket = EstimatorBucket(); // Reset any failed bucket, currently passing
323  foundAnswer = true;
324  passing = true;
325  passBucket.withinTarget = nConf;
326  nConf = 0;
327  passBucket.totalConfirmed = totalNum;
328  totalNum = 0;
329  passBucket.inMempool = extraNum;
330  passBucket.leftMempool = failNum;
331  failNum = 0;
332  extraNum = 0;
333  bestNearBucket = curNearBucket;
334  bestFarBucket = curFarBucket;
335  newBucketRange = true;
336  }
337  }
338  }
339 
340  double median = -1;
341  double txSum = 0;
342 
343  // Calculate the "average" feerate of the best bucket range that met success conditions
344  // Find the bucket with the median transaction and then report the average feerate from that bucket
345  // This is a compromise between finding the median which we can't since we don't save all tx's
346  // and reporting the average which is less accurate
347  unsigned int minBucket = std::min(bestNearBucket, bestFarBucket);
348  unsigned int maxBucket = std::max(bestNearBucket, bestFarBucket);
349  for (unsigned int j = minBucket; j <= maxBucket; j++) {
350  txSum += txCtAvg[j];
351  }
352  if (foundAnswer && txSum != 0) {
353  txSum = txSum / 2;
354  for (unsigned int j = minBucket; j <= maxBucket; j++) {
355  if (txCtAvg[j] < txSum)
356  txSum -= txCtAvg[j];
357  else { // we're in the right bucket
358  median = m_feerate_avg[j] / txCtAvg[j];
359  break;
360  }
361  }
362 
363  passBucket.start = minBucket ? buckets[minBucket-1] : 0;
364  passBucket.end = buckets[maxBucket];
365  }
366 
367  // If we were passing until we reached last few buckets with insufficient data, then report those as failed
368  if (passing && !newBucketRange) {
369  unsigned int failMinBucket = std::min(curNearBucket, curFarBucket);
370  unsigned int failMaxBucket = std::max(curNearBucket, curFarBucket);
371  failBucket.start = failMinBucket ? buckets[failMinBucket - 1] : 0;
372  failBucket.end = buckets[failMaxBucket];
373  failBucket.withinTarget = nConf;
374  failBucket.totalConfirmed = totalNum;
375  failBucket.inMempool = extraNum;
376  failBucket.leftMempool = failNum;
377  }
378 
379  float passed_within_target_perc = 0.0;
380  float failed_within_target_perc = 0.0;
381  if ((passBucket.totalConfirmed + passBucket.inMempool + passBucket.leftMempool)) {
382  passed_within_target_perc = 100 * passBucket.withinTarget / (passBucket.totalConfirmed + passBucket.inMempool + passBucket.leftMempool);
383  }
384  if ((failBucket.totalConfirmed + failBucket.inMempool + failBucket.leftMempool)) {
385  failed_within_target_perc = 100 * failBucket.withinTarget / (failBucket.totalConfirmed + failBucket.inMempool + failBucket.leftMempool);
386  }
387 
388  LogPrint(BCLog::ESTIMATEFEE, "FeeEst: %d > %.0f%% decay %.5f: feerate: %g from (%g - %g) %.2f%% %.1f/(%.1f %d mem %.1f out) Fail: (%g - %g) %.2f%% %.1f/(%.1f %d mem %.1f out)\n",
389  confTarget, 100.0 * successBreakPoint, decay,
390  median, passBucket.start, passBucket.end,
391  passed_within_target_perc,
392  passBucket.withinTarget, passBucket.totalConfirmed, passBucket.inMempool, passBucket.leftMempool,
393  failBucket.start, failBucket.end,
394  failed_within_target_perc,
395  failBucket.withinTarget, failBucket.totalConfirmed, failBucket.inMempool, failBucket.leftMempool);
396 
397 
398  if (result) {
399  result->pass = passBucket;
400  result->fail = failBucket;
401  result->decay = decay;
402  result->scale = scale;
403  }
404  return median;
405 }
406 
407 void TxConfirmStats::Write(AutoFile& fileout) const
408 {
409  fileout << Using<EncodedDoubleFormatter>(decay);
410  fileout << scale;
411  fileout << Using<VectorFormatter<EncodedDoubleFormatter>>(m_feerate_avg);
412  fileout << Using<VectorFormatter<EncodedDoubleFormatter>>(txCtAvg);
413  fileout << Using<VectorFormatter<VectorFormatter<EncodedDoubleFormatter>>>(confAvg);
414  fileout << Using<VectorFormatter<VectorFormatter<EncodedDoubleFormatter>>>(failAvg);
415 }
416 
417 void TxConfirmStats::Read(AutoFile& filein, int nFileVersion, size_t numBuckets)
418 {
419  // Read data file and do some very basic sanity checking
420  // buckets and bucketMap are not updated yet, so don't access them
421  // If there is a read failure, we'll just discard this entire object anyway
422  size_t maxConfirms, maxPeriods;
423 
424  // The current version will store the decay with each individual TxConfirmStats and also keep a scale factor
425  filein >> Using<EncodedDoubleFormatter>(decay);
426  if (decay <= 0 || decay >= 1) {
427  throw std::runtime_error("Corrupt estimates file. Decay must be between 0 and 1 (non-inclusive)");
428  }
429  filein >> scale;
430  if (scale == 0) {
431  throw std::runtime_error("Corrupt estimates file. Scale must be non-zero");
432  }
433 
434  filein >> Using<VectorFormatter<EncodedDoubleFormatter>>(m_feerate_avg);
435  if (m_feerate_avg.size() != numBuckets) {
436  throw std::runtime_error("Corrupt estimates file. Mismatch in feerate average bucket count");
437  }
438  filein >> Using<VectorFormatter<EncodedDoubleFormatter>>(txCtAvg);
439  if (txCtAvg.size() != numBuckets) {
440  throw std::runtime_error("Corrupt estimates file. Mismatch in tx count bucket count");
441  }
442  filein >> Using<VectorFormatter<VectorFormatter<EncodedDoubleFormatter>>>(confAvg);
443  maxPeriods = confAvg.size();
444  maxConfirms = scale * maxPeriods;
445 
446  if (maxConfirms <= 0 || maxConfirms > 6 * 24 * 7) { // one week
447  throw std::runtime_error("Corrupt estimates file. Must maintain estimates for between 1 and 1008 (one week) confirms");
448  }
449  for (unsigned int i = 0; i < maxPeriods; i++) {
450  if (confAvg[i].size() != numBuckets) {
451  throw std::runtime_error("Corrupt estimates file. Mismatch in feerate conf average bucket count");
452  }
453  }
454 
455  filein >> Using<VectorFormatter<VectorFormatter<EncodedDoubleFormatter>>>(failAvg);
456  if (maxPeriods != failAvg.size()) {
457  throw std::runtime_error("Corrupt estimates file. Mismatch in confirms tracked for failures");
458  }
459  for (unsigned int i = 0; i < maxPeriods; i++) {
460  if (failAvg[i].size() != numBuckets) {
461  throw std::runtime_error("Corrupt estimates file. Mismatch in one of failure average bucket counts");
462  }
463  }
464 
465  // Resize the current block variables which aren't stored in the data file
466  // to match the number of confirms and buckets
467  resizeInMemoryCounters(numBuckets);
468 
469  LogPrint(BCLog::ESTIMATEFEE, "Reading estimates: %u buckets counting confirms up to %u blocks\n",
470  numBuckets, maxConfirms);
471 }
472 
473 unsigned int TxConfirmStats::NewTx(unsigned int nBlockHeight, double val)
474 {
475  unsigned int bucketindex = bucketMap.lower_bound(val)->second;
476  unsigned int blockIndex = nBlockHeight % unconfTxs.size();
477  unconfTxs[blockIndex][bucketindex]++;
478  return bucketindex;
479 }
480 
481 void TxConfirmStats::removeTx(unsigned int entryHeight, unsigned int nBestSeenHeight, unsigned int bucketindex, bool inBlock)
482 {
483  //nBestSeenHeight is not updated yet for the new block
484  int blocksAgo = nBestSeenHeight - entryHeight;
485  if (nBestSeenHeight == 0) // the BlockPolicyEstimator hasn't seen any blocks yet
486  blocksAgo = 0;
487  if (blocksAgo < 0) {
488  LogPrint(BCLog::ESTIMATEFEE, "Blockpolicy error, blocks ago is negative for mempool tx\n");
489  return; //This can't happen because we call this with our best seen height, no entries can have higher
490  }
491 
492  if (blocksAgo >= (int)unconfTxs.size()) {
493  if (oldUnconfTxs[bucketindex] > 0) {
494  oldUnconfTxs[bucketindex]--;
495  } else {
496  LogPrint(BCLog::ESTIMATEFEE, "Blockpolicy error, mempool tx removed from >25 blocks,bucketIndex=%u already\n",
497  bucketindex);
498  }
499  }
500  else {
501  unsigned int blockIndex = entryHeight % unconfTxs.size();
502  if (unconfTxs[blockIndex][bucketindex] > 0) {
503  unconfTxs[blockIndex][bucketindex]--;
504  } else {
505  LogPrint(BCLog::ESTIMATEFEE, "Blockpolicy error, mempool tx removed from blockIndex=%u,bucketIndex=%u already\n",
506  blockIndex, bucketindex);
507  }
508  }
509  if (!inBlock && (unsigned int)blocksAgo >= scale) { // Only counts as a failure if not confirmed for entire period
510  assert(scale != 0);
511  unsigned int periodsAgo = blocksAgo / scale;
512  for (size_t i = 0; i < periodsAgo && i < failAvg.size(); i++) {
513  failAvg[i][bucketindex]++;
514  }
515  }
516 }
517 
519 {
521  return _removeTx(hash, /*inBlock=*/false);
522 }
523 
524 bool CBlockPolicyEstimator::_removeTx(const uint256& hash, bool inBlock)
525 {
527  std::map<uint256, TxStatsInfo>::iterator pos = mapMemPoolTxs.find(hash);
528  if (pos != mapMemPoolTxs.end()) {
529  feeStats->removeTx(pos->second.blockHeight, nBestSeenHeight, pos->second.bucketIndex, inBlock);
530  shortStats->removeTx(pos->second.blockHeight, nBestSeenHeight, pos->second.bucketIndex, inBlock);
531  longStats->removeTx(pos->second.blockHeight, nBestSeenHeight, pos->second.bucketIndex, inBlock);
532  mapMemPoolTxs.erase(hash);
533  return true;
534  } else {
535  return false;
536  }
537 }
538 
539 CBlockPolicyEstimator::CBlockPolicyEstimator(const fs::path& estimation_filepath, const bool read_stale_estimates)
540  : m_estimation_filepath{estimation_filepath}
541 {
542  static_assert(MIN_BUCKET_FEERATE > 0, "Min feerate must be nonzero");
543  size_t bucketIndex = 0;
544 
545  for (double bucketBoundary = MIN_BUCKET_FEERATE; bucketBoundary <= MAX_BUCKET_FEERATE; bucketBoundary *= FEE_SPACING, bucketIndex++) {
546  buckets.push_back(bucketBoundary);
547  bucketMap[bucketBoundary] = bucketIndex;
548  }
549  buckets.push_back(INF_FEERATE);
550  bucketMap[INF_FEERATE] = bucketIndex;
551  assert(bucketMap.size() == buckets.size());
552 
553  feeStats = std::unique_ptr<TxConfirmStats>(new TxConfirmStats(buckets, bucketMap, MED_BLOCK_PERIODS, MED_DECAY, MED_SCALE));
554  shortStats = std::unique_ptr<TxConfirmStats>(new TxConfirmStats(buckets, bucketMap, SHORT_BLOCK_PERIODS, SHORT_DECAY, SHORT_SCALE));
555  longStats = std::unique_ptr<TxConfirmStats>(new TxConfirmStats(buckets, bucketMap, LONG_BLOCK_PERIODS, LONG_DECAY, LONG_SCALE));
556 
558 
559  if (est_file.IsNull()) {
560  LogPrintf("%s is not found. Continue anyway.\n", fs::PathToString(m_estimation_filepath));
561  return;
562  }
563 
564  std::chrono::hours file_age = GetFeeEstimatorFileAge();
565  if (file_age > MAX_FILE_AGE && !read_stale_estimates) {
566  LogPrintf("Fee estimation file %s too old (age=%lld > %lld hours) and will not be used to avoid serving stale estimates.\n", fs::PathToString(m_estimation_filepath), Ticks<std::chrono::hours>(file_age), Ticks<std::chrono::hours>(MAX_FILE_AGE));
567  return;
568  }
569 
570  if (!Read(est_file)) {
571  LogPrintf("Failed to read fee estimates from %s. Continue anyway.\n", fs::PathToString(m_estimation_filepath));
572  }
573 }
574 
576 
578 {
579  processTransaction(tx);
580 }
581 
583 {
584  removeTx(tx->GetHash());
585 }
586 
587 void CBlockPolicyEstimator::MempoolTransactionsRemovedForBlock(const std::vector<RemovedMempoolTransactionInfo>& txs_removed_for_block, unsigned int nBlockHeight)
588 {
589  processBlock(txs_removed_for_block, nBlockHeight);
590 }
591 
593 {
595  const unsigned int txHeight = tx.info.txHeight;
596  const auto& hash = tx.info.m_tx->GetHash();
597  if (mapMemPoolTxs.count(hash)) {
598  LogPrint(BCLog::ESTIMATEFEE, "Blockpolicy error mempool tx %s already being tracked\n",
599  hash.ToString());
600  return;
601  }
602 
603  if (txHeight != nBestSeenHeight) {
604  // Ignore side chains and re-orgs; assuming they are random they don't
605  // affect the estimate. We'll potentially double count transactions in 1-block reorgs.
606  // Ignore txs if BlockPolicyEstimator is not in sync with ActiveChain().Tip().
607  // It will be synced next time a block is processed.
608  return;
609  }
610  // This transaction should only count for fee estimation if:
611  // - it's not being re-added during a reorg which bypasses typical mempool fee limits
612  // - the node is not behind
613  // - the transaction is not dependent on any other transactions in the mempool
614  // - it's not part of a package.
615  const bool validForFeeEstimation = !tx.m_mempool_limit_bypassed && !tx.m_submitted_in_package && tx.m_chainstate_is_current && tx.m_has_no_mempool_parents;
616 
617  // Only want to be updating estimates when our blockchain is synced,
618  // otherwise we'll miscalculate how many blocks its taking to get included.
619  if (!validForFeeEstimation) {
620  untrackedTxs++;
621  return;
622  }
623  trackedTxs++;
624 
625  // Feerates are stored and reported as BTC-per-kb:
626  const CFeeRate feeRate(tx.info.m_fee, tx.info.m_virtual_transaction_size);
627 
628  mapMemPoolTxs[hash].blockHeight = txHeight;
629  unsigned int bucketIndex = feeStats->NewTx(txHeight, static_cast<double>(feeRate.GetFeePerK()));
630  mapMemPoolTxs[hash].bucketIndex = bucketIndex;
631  unsigned int bucketIndex2 = shortStats->NewTx(txHeight, static_cast<double>(feeRate.GetFeePerK()));
632  assert(bucketIndex == bucketIndex2);
633  unsigned int bucketIndex3 = longStats->NewTx(txHeight, static_cast<double>(feeRate.GetFeePerK()));
634  assert(bucketIndex == bucketIndex3);
635 }
636 
638 {
640  if (!_removeTx(tx.info.m_tx->GetHash(), true)) {
641  // This transaction wasn't being tracked for fee estimation
642  return false;
643  }
644 
645  // How many blocks did it take for miners to include this transaction?
646  // blocksToConfirm is 1-based, so a transaction included in the earliest
647  // possible block has confirmation count of 1
648  int blocksToConfirm = nBlockHeight - tx.info.txHeight;
649  if (blocksToConfirm <= 0) {
650  // This can't happen because we don't process transactions from a block with a height
651  // lower than our greatest seen height
652  LogPrint(BCLog::ESTIMATEFEE, "Blockpolicy error Transaction had negative blocksToConfirm\n");
653  return false;
654  }
655 
656  // Feerates are stored and reported as BTC-per-kb:
658 
659  feeStats->Record(blocksToConfirm, static_cast<double>(feeRate.GetFeePerK()));
660  shortStats->Record(blocksToConfirm, static_cast<double>(feeRate.GetFeePerK()));
661  longStats->Record(blocksToConfirm, static_cast<double>(feeRate.GetFeePerK()));
662  return true;
663 }
664 
665 void CBlockPolicyEstimator::processBlock(const std::vector<RemovedMempoolTransactionInfo>& txs_removed_for_block,
666  unsigned int nBlockHeight)
667 {
669  if (nBlockHeight <= nBestSeenHeight) {
670  // Ignore side chains and re-orgs; assuming they are random
671  // they don't affect the estimate.
672  // And if an attacker can re-org the chain at will, then
673  // you've got much bigger problems than "attacker can influence
674  // transaction fees."
675  return;
676  }
677 
678  // Must update nBestSeenHeight in sync with ClearCurrent so that
679  // calls to removeTx (via processBlockTx) correctly calculate age
680  // of unconfirmed txs to remove from tracking.
681  nBestSeenHeight = nBlockHeight;
682 
683  // Update unconfirmed circular buffer
684  feeStats->ClearCurrent(nBlockHeight);
685  shortStats->ClearCurrent(nBlockHeight);
686  longStats->ClearCurrent(nBlockHeight);
687 
688  // Decay all exponential averages
689  feeStats->UpdateMovingAverages();
690  shortStats->UpdateMovingAverages();
691  longStats->UpdateMovingAverages();
692 
693  unsigned int countedTxs = 0;
694  // Update averages with data points from current block
695  for (const auto& tx : txs_removed_for_block) {
696  if (processBlockTx(nBlockHeight, tx))
697  countedTxs++;
698  }
699 
700  if (firstRecordedHeight == 0 && countedTxs > 0) {
701  firstRecordedHeight = nBestSeenHeight;
702  LogPrint(BCLog::ESTIMATEFEE, "Blockpolicy first recorded height %u\n", firstRecordedHeight);
703  }
704 
705 
706  LogPrint(BCLog::ESTIMATEFEE, "Blockpolicy estimates updated by %u of %u block txs, since last block %u of %u tracked, mempool map size %u, max target %u from %s\n",
707  countedTxs, txs_removed_for_block.size(), trackedTxs, trackedTxs + untrackedTxs, mapMemPoolTxs.size(),
708  MaxUsableEstimate(), HistoricalBlockSpan() > BlockSpan() ? "historical" : "current");
709 
710  trackedTxs = 0;
711  untrackedTxs = 0;
712 }
713 
715 {
716  // It's not possible to get reasonable estimates for confTarget of 1
717  if (confTarget <= 1)
718  return CFeeRate(0);
719 
721 }
722 
723 CFeeRate CBlockPolicyEstimator::estimateRawFee(int confTarget, double successThreshold, FeeEstimateHorizon horizon, EstimationResult* result) const
724 {
725  TxConfirmStats* stats = nullptr;
726  double sufficientTxs = SUFFICIENT_FEETXS;
727  switch (horizon) {
729  stats = shortStats.get();
730  sufficientTxs = SUFFICIENT_TXS_SHORT;
731  break;
732  }
734  stats = feeStats.get();
735  break;
736  }
738  stats = longStats.get();
739  break;
740  }
741  } // no default case, so the compiler can warn about missing cases
742  assert(stats);
743 
745  // Return failure if trying to analyze a target we're not tracking
746  if (confTarget <= 0 || (unsigned int)confTarget > stats->GetMaxConfirms())
747  return CFeeRate(0);
748  if (successThreshold > 1)
749  return CFeeRate(0);
750 
751  double median = stats->EstimateMedianVal(confTarget, sufficientTxs, successThreshold, nBestSeenHeight, result);
752 
753  if (median < 0)
754  return CFeeRate(0);
755 
756  return CFeeRate(llround(median));
757 }
758 
760 {
762  switch (horizon) {
764  return shortStats->GetMaxConfirms();
765  }
767  return feeStats->GetMaxConfirms();
768  }
770  return longStats->GetMaxConfirms();
771  }
772  } // no default case, so the compiler can warn about missing cases
773  assert(false);
774 }
775 
777 {
778  if (firstRecordedHeight == 0) return 0;
779  assert(nBestSeenHeight >= firstRecordedHeight);
780 
781  return nBestSeenHeight - firstRecordedHeight;
782 }
783 
785 {
786  if (historicalFirst == 0) return 0;
787  assert(historicalBest >= historicalFirst);
788 
789  if (nBestSeenHeight - historicalBest > OLDEST_ESTIMATE_HISTORY) return 0;
790 
791  return historicalBest - historicalFirst;
792 }
793 
795 {
796  // Block spans are divided by 2 to make sure there are enough potential failing data points for the estimate
797  return std::min(longStats->GetMaxConfirms(), std::max(BlockSpan(), HistoricalBlockSpan()) / 2);
798 }
799 
804 double CBlockPolicyEstimator::estimateCombinedFee(unsigned int confTarget, double successThreshold, bool checkShorterHorizon, EstimationResult *result) const
805 {
806  double estimate = -1;
807  if (confTarget >= 1 && confTarget <= longStats->GetMaxConfirms()) {
808  // Find estimate from shortest time horizon possible
809  if (confTarget <= shortStats->GetMaxConfirms()) { // short horizon
810  estimate = shortStats->EstimateMedianVal(confTarget, SUFFICIENT_TXS_SHORT, successThreshold, nBestSeenHeight, result);
811  }
812  else if (confTarget <= feeStats->GetMaxConfirms()) { // medium horizon
813  estimate = feeStats->EstimateMedianVal(confTarget, SUFFICIENT_FEETXS, successThreshold, nBestSeenHeight, result);
814  }
815  else { // long horizon
816  estimate = longStats->EstimateMedianVal(confTarget, SUFFICIENT_FEETXS, successThreshold, nBestSeenHeight, result);
817  }
818  if (checkShorterHorizon) {
819  EstimationResult tempResult;
820  // If a lower confTarget from a more recent horizon returns a lower answer use it.
821  if (confTarget > feeStats->GetMaxConfirms()) {
822  double medMax = feeStats->EstimateMedianVal(feeStats->GetMaxConfirms(), SUFFICIENT_FEETXS, successThreshold, nBestSeenHeight, &tempResult);
823  if (medMax > 0 && (estimate == -1 || medMax < estimate)) {
824  estimate = medMax;
825  if (result) *result = tempResult;
826  }
827  }
828  if (confTarget > shortStats->GetMaxConfirms()) {
829  double shortMax = shortStats->EstimateMedianVal(shortStats->GetMaxConfirms(), SUFFICIENT_TXS_SHORT, successThreshold, nBestSeenHeight, &tempResult);
830  if (shortMax > 0 && (estimate == -1 || shortMax < estimate)) {
831  estimate = shortMax;
832  if (result) *result = tempResult;
833  }
834  }
835  }
836  }
837  return estimate;
838 }
839 
843 double CBlockPolicyEstimator::estimateConservativeFee(unsigned int doubleTarget, EstimationResult *result) const
844 {
845  double estimate = -1;
846  EstimationResult tempResult;
847  if (doubleTarget <= shortStats->GetMaxConfirms()) {
848  estimate = feeStats->EstimateMedianVal(doubleTarget, SUFFICIENT_FEETXS, DOUBLE_SUCCESS_PCT, nBestSeenHeight, result);
849  }
850  if (doubleTarget <= feeStats->GetMaxConfirms()) {
851  double longEstimate = longStats->EstimateMedianVal(doubleTarget, SUFFICIENT_FEETXS, DOUBLE_SUCCESS_PCT, nBestSeenHeight, &tempResult);
852  if (longEstimate > estimate) {
853  estimate = longEstimate;
854  if (result) *result = tempResult;
855  }
856  }
857  return estimate;
858 }
859 
867 CFeeRate CBlockPolicyEstimator::estimateSmartFee(int confTarget, FeeCalculation *feeCalc, bool conservative) const
868 {
870 
871  if (feeCalc) {
872  feeCalc->desiredTarget = confTarget;
873  feeCalc->returnedTarget = confTarget;
874  }
875 
876  double median = -1;
877  EstimationResult tempResult;
878 
879  // Return failure if trying to analyze a target we're not tracking
880  if (confTarget <= 0 || (unsigned int)confTarget > longStats->GetMaxConfirms()) {
881  return CFeeRate(0); // error condition
882  }
883 
884  // It's not possible to get reasonable estimates for confTarget of 1
885  if (confTarget == 1) confTarget = 2;
886 
887  unsigned int maxUsableEstimate = MaxUsableEstimate();
888  if ((unsigned int)confTarget > maxUsableEstimate) {
889  confTarget = maxUsableEstimate;
890  }
891  if (feeCalc) feeCalc->returnedTarget = confTarget;
892 
893  if (confTarget <= 1) return CFeeRate(0); // error condition
894 
895  assert(confTarget > 0); //estimateCombinedFee and estimateConservativeFee take unsigned ints
906  double halfEst = estimateCombinedFee(confTarget/2, HALF_SUCCESS_PCT, true, &tempResult);
907  if (feeCalc) {
908  feeCalc->est = tempResult;
909  feeCalc->reason = FeeReason::HALF_ESTIMATE;
910  }
911  median = halfEst;
912  double actualEst = estimateCombinedFee(confTarget, SUCCESS_PCT, true, &tempResult);
913  if (actualEst > median) {
914  median = actualEst;
915  if (feeCalc) {
916  feeCalc->est = tempResult;
917  feeCalc->reason = FeeReason::FULL_ESTIMATE;
918  }
919  }
920  double doubleEst = estimateCombinedFee(2 * confTarget, DOUBLE_SUCCESS_PCT, !conservative, &tempResult);
921  if (doubleEst > median) {
922  median = doubleEst;
923  if (feeCalc) {
924  feeCalc->est = tempResult;
926  }
927  }
928 
929  if (conservative || median == -1) {
930  double consEst = estimateConservativeFee(2 * confTarget, &tempResult);
931  if (consEst > median) {
932  median = consEst;
933  if (feeCalc) {
934  feeCalc->est = tempResult;
935  feeCalc->reason = FeeReason::CONSERVATIVE;
936  }
937  }
938  }
939 
940  if (median < 0) return CFeeRate(0); // error condition
941 
942  return CFeeRate(llround(median));
943 }
944 
948 }
949 
951 {
953  if (est_file.IsNull() || !Write(est_file)) {
954  LogPrintf("Failed to write fee estimates to %s. Continue anyway.\n", fs::PathToString(m_estimation_filepath));
955  } else {
956  LogPrintf("Flushed fee estimates to %s.\n", fs::PathToString(m_estimation_filepath.filename()));
957  }
958 }
959 
961 {
962  try {
964  fileout << 149900; // version required to read: 0.14.99 or later
965  fileout << CLIENT_VERSION; // version that wrote the file
966  fileout << nBestSeenHeight;
967  if (BlockSpan() > HistoricalBlockSpan()/2) {
968  fileout << firstRecordedHeight << nBestSeenHeight;
969  }
970  else {
971  fileout << historicalFirst << historicalBest;
972  }
973  fileout << Using<VectorFormatter<EncodedDoubleFormatter>>(buckets);
974  feeStats->Write(fileout);
975  shortStats->Write(fileout);
976  longStats->Write(fileout);
977  }
978  catch (const std::exception&) {
979  LogPrintf("CBlockPolicyEstimator::Write(): unable to write policy estimator data (non-fatal)\n");
980  return false;
981  }
982  return true;
983 }
984 
986 {
987  try {
989  int nVersionRequired, nVersionThatWrote;
990  filein >> nVersionRequired >> nVersionThatWrote;
991  if (nVersionRequired > CLIENT_VERSION) {
992  throw std::runtime_error(strprintf("up-version (%d) fee estimate file", nVersionRequired));
993  }
994 
995  // Read fee estimates file into temporary variables so existing data
996  // structures aren't corrupted if there is an exception.
997  unsigned int nFileBestSeenHeight;
998  filein >> nFileBestSeenHeight;
999 
1000  if (nVersionRequired < 149900) {
1001  LogPrintf("%s: incompatible old fee estimation data (non-fatal). Version: %d\n", __func__, nVersionRequired);
1002  } else { // New format introduced in 149900
1003  unsigned int nFileHistoricalFirst, nFileHistoricalBest;
1004  filein >> nFileHistoricalFirst >> nFileHistoricalBest;
1005  if (nFileHistoricalFirst > nFileHistoricalBest || nFileHistoricalBest > nFileBestSeenHeight) {
1006  throw std::runtime_error("Corrupt estimates file. Historical block range for estimates is invalid");
1007  }
1008  std::vector<double> fileBuckets;
1009  filein >> Using<VectorFormatter<EncodedDoubleFormatter>>(fileBuckets);
1010  size_t numBuckets = fileBuckets.size();
1011  if (numBuckets <= 1 || numBuckets > 1000) {
1012  throw std::runtime_error("Corrupt estimates file. Must have between 2 and 1000 feerate buckets");
1013  }
1014 
1015  std::unique_ptr<TxConfirmStats> fileFeeStats(new TxConfirmStats(buckets, bucketMap, MED_BLOCK_PERIODS, MED_DECAY, MED_SCALE));
1016  std::unique_ptr<TxConfirmStats> fileShortStats(new TxConfirmStats(buckets, bucketMap, SHORT_BLOCK_PERIODS, SHORT_DECAY, SHORT_SCALE));
1017  std::unique_ptr<TxConfirmStats> fileLongStats(new TxConfirmStats(buckets, bucketMap, LONG_BLOCK_PERIODS, LONG_DECAY, LONG_SCALE));
1018  fileFeeStats->Read(filein, nVersionThatWrote, numBuckets);
1019  fileShortStats->Read(filein, nVersionThatWrote, numBuckets);
1020  fileLongStats->Read(filein, nVersionThatWrote, numBuckets);
1021 
1022  // Fee estimates file parsed correctly
1023  // Copy buckets from file and refresh our bucketmap
1024  buckets = fileBuckets;
1025  bucketMap.clear();
1026  for (unsigned int i = 0; i < buckets.size(); i++) {
1027  bucketMap[buckets[i]] = i;
1028  }
1029 
1030  // Destroy old TxConfirmStats and point to new ones that already reference buckets and bucketMap
1031  feeStats = std::move(fileFeeStats);
1032  shortStats = std::move(fileShortStats);
1033  longStats = std::move(fileLongStats);
1034 
1035  nBestSeenHeight = nFileBestSeenHeight;
1036  historicalFirst = nFileHistoricalFirst;
1037  historicalBest = nFileHistoricalBest;
1038  }
1039  }
1040  catch (const std::exception& e) {
1041  LogPrintf("CBlockPolicyEstimator::Read(): unable to read policy estimator data (non-fatal): %s\n",e.what());
1042  return false;
1043  }
1044  return true;
1045 }
1046 
1048 {
1049  const auto startclear{SteadyClock::now()};
1051  size_t num_entries = mapMemPoolTxs.size();
1052  // Remove every entry in mapMemPoolTxs
1053  while (!mapMemPoolTxs.empty()) {
1054  auto mi = mapMemPoolTxs.begin();
1055  _removeTx(mi->first, false); // this calls erase() on mapMemPoolTxs
1056  }
1057  const auto endclear{SteadyClock::now()};
1058  LogPrint(BCLog::ESTIMATEFEE, "Recorded %u unconfirmed txs from mempool in %.3fs\n", num_entries, Ticks<SecondsDouble>(endclear - startclear));
1059 }
1060 
1062 {
1063  auto file_time{fs::last_write_time(m_estimation_filepath)};
1064  auto now{fs::file_time_type::clock::now()};
1065  return std::chrono::duration_cast<std::chrono::hours>(now - file_time);
1066 }
1067 
1068 static std::set<double> MakeFeeSet(const CFeeRate& min_incremental_fee,
1069  double max_filter_fee_rate,
1070  double fee_filter_spacing)
1071 {
1072  std::set<double> fee_set;
1073 
1074  const CAmount min_fee_limit{std::max(CAmount(1), min_incremental_fee.GetFeePerK() / 2)};
1075  fee_set.insert(0);
1076  for (double bucket_boundary = min_fee_limit;
1077  bucket_boundary <= max_filter_fee_rate;
1078  bucket_boundary *= fee_filter_spacing) {
1079 
1080  fee_set.insert(bucket_boundary);
1081  }
1082 
1083  return fee_set;
1084 }
1085 
1087  : m_fee_set{MakeFeeSet(minIncrementalFee, MAX_FILTER_FEERATE, FEE_FILTER_SPACING)},
1088  insecure_rand{rng}
1089 {
1090 }
1091 
1093 {
1095  std::set<double>::iterator it = m_fee_set.lower_bound(currentMinFee);
1096  if (it == m_fee_set.end() ||
1097  (it != m_fee_set.begin() &&
1098  WITH_LOCK(m_insecure_rand_mutex, return insecure_rand.rand32()) % 3 != 0)) {
1099  --it;
1100  }
1101  return static_cast<CAmount>(*it);
1102 }
int64_t CAmount
Amount in satoshis (Can be negative)
Definition: amount.h:12
Non-refcounted RAII wrapper for FILE*.
Definition: streams.h:389
void processTransaction(const NewMempoolTransactionInfo &tx) EXCLUSIVE_LOCKS_REQUIRED(!m_cs_fee_estimator)
Process a transaction accepted to the mempool.
Definition: fees.cpp:592
static constexpr unsigned int LONG_SCALE
Definition: fees.h:159
static constexpr double SUCCESS_PCT
Require greater than 85% of X feerate transactions to be confirmed within Y blocks.
Definition: fees.h:173
static constexpr double MIN_BUCKET_FEERATE
Minimum and Maximum values for tracking feerates The MIN_BUCKET_FEERATE should just be set to the low...
Definition: fees.h:189
double estimateCombinedFee(unsigned int confTarget, double successThreshold, bool checkShorterHorizon, EstimationResult *result) const EXCLUSIVE_LOCKS_REQUIRED(m_cs_fee_estimator)
Helper for estimateSmartFee.
Definition: fees.cpp:804
static constexpr double FEE_SPACING
Spacing of FeeRate buckets We have to lump transactions into buckets based on feerate,...
Definition: fees.h:197
static const unsigned int OLDEST_ESTIMATE_HISTORY
Historical estimates that are older than this aren't valid.
Definition: fees.h:161
void Flush() EXCLUSIVE_LOCKS_REQUIRED(!m_cs_fee_estimator)
Drop still unconfirmed transactions and record current estimations, if the fee estimation file is pre...
Definition: fees.cpp:945
static constexpr double SUFFICIENT_FEETXS
Require an avg of 0.1 tx in the combined feerate bucket per block to have stat significance.
Definition: fees.h:178
static constexpr double MAX_BUCKET_FEERATE
Definition: fees.h:190
void FlushFeeEstimates() EXCLUSIVE_LOCKS_REQUIRED(!m_cs_fee_estimator)
Record current fee estimations.
Definition: fees.cpp:950
CFeeRate estimateSmartFee(int confTarget, FeeCalculation *feeCalc, bool conservative) const EXCLUSIVE_LOCKS_REQUIRED(!m_cs_fee_estimator)
Estimate feerate needed to get be included in a block within confTarget blocks.
Definition: fees.cpp:867
static constexpr unsigned int LONG_BLOCK_PERIODS
Track confirm delays up to 1008 blocks for long horizon.
Definition: fees.h:158
bool Write(AutoFile &fileout) const EXCLUSIVE_LOCKS_REQUIRED(!m_cs_fee_estimator)
Write estimation data to a file.
Definition: fees.cpp:960
static constexpr double SHORT_DECAY
Decay of .962 is a half-life of 18 blocks or about 3 hours.
Definition: fees.h:164
std::chrono::hours GetFeeEstimatorFileAge()
Calculates the age of the file, since last modified.
Definition: fees.cpp:1061
Mutex m_cs_fee_estimator
Definition: fees.h:275
unsigned int HighestTargetTracked(FeeEstimateHorizon horizon) const EXCLUSIVE_LOCKS_REQUIRED(!m_cs_fee_estimator)
Calculation of highest target that estimates are tracked for.
Definition: fees.cpp:759
bool _removeTx(const uint256 &hash, bool inBlock) EXCLUSIVE_LOCKS_REQUIRED(m_cs_fee_estimator)
A non-thread-safe helper for the removeTx function.
Definition: fees.cpp:524
static constexpr double LONG_DECAY
Decay of .99931 is a half-life of 1008 blocks or about 1 week.
Definition: fees.h:168
double estimateConservativeFee(unsigned int doubleTarget, EstimationResult *result) const EXCLUSIVE_LOCKS_REQUIRED(m_cs_fee_estimator)
Helper for estimateSmartFee.
Definition: fees.cpp:843
void TransactionRemovedFromMempool(const CTransactionRef &tx, MemPoolRemovalReason, uint64_t) override EXCLUSIVE_LOCKS_REQUIRED(!m_cs_fee_estimator)
Notifies listeners of a transaction leaving mempool.
Definition: fees.cpp:582
static constexpr double HALF_SUCCESS_PCT
Require greater than 60% of X feerate transactions to be confirmed within Y/2 blocks.
Definition: fees.h:171
static constexpr double MED_DECAY
Decay of .9952 is a half-life of 144 blocks or about 1 day.
Definition: fees.h:166
CFeeRate estimateFee(int confTarget) const EXCLUSIVE_LOCKS_REQUIRED(!m_cs_fee_estimator)
DEPRECATED.
Definition: fees.cpp:714
CBlockPolicyEstimator(const fs::path &estimation_filepath, const bool read_stale_estimates)
Create new BlockPolicyEstimator and initialize stats tracking classes with default values.
Definition: fees.cpp:539
unsigned int MaxUsableEstimate() const EXCLUSIVE_LOCKS_REQUIRED(m_cs_fee_estimator)
Calculation of highest target that reasonable estimate can be provided for.
Definition: fees.cpp:794
static constexpr unsigned int SHORT_SCALE
Definition: fees.h:153
unsigned int BlockSpan() const EXCLUSIVE_LOCKS_REQUIRED(m_cs_fee_estimator)
Number of blocks of data recorded while fee estimates have been running.
Definition: fees.cpp:776
void MempoolTransactionsRemovedForBlock(const std::vector< RemovedMempoolTransactionInfo > &txs_removed_for_block, unsigned int nBlockHeight) override EXCLUSIVE_LOCKS_REQUIRED(!m_cs_fee_estimator)
Definition: fees.cpp:587
bool Read(AutoFile &filein) EXCLUSIVE_LOCKS_REQUIRED(!m_cs_fee_estimator)
Read estimation data from a file.
Definition: fees.cpp:985
static constexpr unsigned int SHORT_BLOCK_PERIODS
Track confirm delays up to 12 blocks for short horizon.
Definition: fees.h:152
static constexpr double DOUBLE_SUCCESS_PCT
Require greater than 95% of X feerate transactions to be confirmed within 2 * Y blocks.
Definition: fees.h:175
unsigned int HistoricalBlockSpan() const EXCLUSIVE_LOCKS_REQUIRED(m_cs_fee_estimator)
Number of blocks of recorded fee estimate data represented in saved data file.
Definition: fees.cpp:784
void FlushUnconfirmed() EXCLUSIVE_LOCKS_REQUIRED(!m_cs_fee_estimator)
Empty mempool transactions on shutdown to record failure to confirm for txs still in mempool.
Definition: fees.cpp:1047
CFeeRate estimateRawFee(int confTarget, double successThreshold, FeeEstimateHorizon horizon, EstimationResult *result=nullptr) const EXCLUSIVE_LOCKS_REQUIRED(!m_cs_fee_estimator)
Return a specific fee estimate calculation with a given success threshold and time horizon,...
Definition: fees.cpp:723
static constexpr double SUFFICIENT_TXS_SHORT
Require an avg of 0.5 tx when using short decay since there are fewer blocks considered.
Definition: fees.h:180
bool processBlockTx(unsigned int nBlockHeight, const RemovedMempoolTransactionInfo &tx) EXCLUSIVE_LOCKS_REQUIRED(m_cs_fee_estimator)
Process a transaction confirmed in a block.
Definition: fees.cpp:637
static constexpr unsigned int MED_SCALE
Definition: fees.h:156
static constexpr unsigned int MED_BLOCK_PERIODS
Track confirm delays up to 48 blocks for medium horizon.
Definition: fees.h:155
bool removeTx(uint256 hash) EXCLUSIVE_LOCKS_REQUIRED(!m_cs_fee_estimator)
Remove a transaction from the mempool tracking stats for non BLOCK removal reasons.
Definition: fees.cpp:518
const fs::path m_estimation_filepath
Definition: fees.h:199
virtual ~CBlockPolicyEstimator()
void TransactionAddedToMempool(const NewMempoolTransactionInfo &tx, uint64_t) override EXCLUSIVE_LOCKS_REQUIRED(!m_cs_fee_estimator)
Overridden from CValidationInterface.
Definition: fees.cpp:577
void processBlock(const std::vector< RemovedMempoolTransactionInfo > &txs_removed_for_block, unsigned int nBlockHeight) EXCLUSIVE_LOCKS_REQUIRED(!m_cs_fee_estimator)
Process all the transactions that have been included in a block.
Definition: fees.cpp:665
Fee rate in satoshis per kilovirtualbyte: CAmount / kvB.
Definition: feerate.h:33
CAmount GetFeePerK() const
Return the fee in satoshis for a vsize of 1000 vbytes.
Definition: feerate.h:65
Fast randomness source.
Definition: random.h:145
const std::set< double > m_fee_set
Definition: fees.h:340
CAmount round(CAmount currentMinFee) EXCLUSIVE_LOCKS_REQUIRED(!m_insecure_rand_mutex)
Quantize a minimum fee for privacy purpose before broadcast.
Definition: fees.cpp:1092
FeeFilterRounder(const CFeeRate &min_incremental_fee, FastRandomContext &rng)
Create new FeeFilterRounder.
Definition: fees.cpp:1086
Mutex m_insecure_rand_mutex
Definition: fees.h:341
We will instantiate an instance of this class to track transactions that were included in a block.
Definition: fees.cpp:75
void removeTx(unsigned int entryHeight, unsigned int nBestSeenHeight, unsigned int bucketIndex, bool inBlock)
Remove a transaction from mempool tracking stats.
Definition: fees.cpp:481
std::vector< std::vector< double > > failAvg
Definition: fees.cpp:92
TxConfirmStats(const std::vector< double > &defaultBuckets, const std::map< double, unsigned int > &defaultBucketMap, unsigned int maxPeriods, double decay, unsigned int scale)
Create new TxConfirmStats.
Definition: fees.cpp:175
void Read(AutoFile &filein, int nFileVersion, size_t numBuckets)
Read saved state of estimation data from a file and replace all internal data structures and variable...
Definition: fees.cpp:417
unsigned int GetMaxConfirms() const
Return the max number of confirms we're tracking.
Definition: fees.cpp:162
void ClearCurrent(unsigned int nBlockHeight)
Roll the circular buffer for unconfirmed txs.
Definition: fees.cpp:204
unsigned int scale
Definition: fees.cpp:104
double decay
Definition: fees.cpp:101
void Record(int blocksToConfirm, double val)
Record a new transaction data point in the current block stats.
Definition: fees.cpp:213
void resizeInMemoryCounters(size_t newbuckets)
Definition: fees.cpp:194
std::vector< double > txCtAvg
Definition: fees.cpp:84
std::vector< int > oldUnconfTxs
Definition: fees.cpp:111
void UpdateMovingAverages()
Update our estimates by decaying our historical moving average and updating with the data gathered fr...
Definition: fees.cpp:227
const std::map< double, unsigned int > & bucketMap
Definition: fees.cpp:79
const std::vector< double > & buckets
Definition: fees.cpp:78
std::vector< std::vector< double > > confAvg
Definition: fees.cpp:88
void Write(AutoFile &fileout) const
Write state of estimation data to a file.
Definition: fees.cpp:407
std::vector< std::vector< int > > unconfTxs
Definition: fees.cpp:109
unsigned int NewTx(unsigned int nBlockHeight, double val)
Record a new transaction entering the mempool.
Definition: fees.cpp:473
std::vector< double > m_feerate_avg
Definition: fees.cpp:96
double EstimateMedianVal(int confTarget, double sufficientTxVal, double minSuccess, unsigned int nBlockHeight, EstimationResult *result=nullptr) const
Calculate a feerate estimate.
Definition: fees.cpp:241
Path class wrapper to block calls to the fs::path(std::string) implicit constructor and the fs::path:...
Definition: fs.h:33
path filename() const
Definition: fs.h:72
256-bit opaque blob.
Definition: uint256.h:106
static const int CLIENT_VERSION
bitcoind-res.rc includes this file, but it cannot cope with real c++ code.
Definition: clientversion.h:33
#define LogPrint(category,...)
Definition: logging.h:264
#define LogPrintf(...)
Definition: logging.h:245
MemPoolRemovalReason
Reason why a transaction was removed from the mempool, this is passed to the notification signal.
@ ESTIMATEFEE
Definition: logging.h:49
static std::string PathToString(const path &path)
Convert path object to a byte string.
Definition: fs.h:151
FILE * fopen(const fs::path &p, const char *mode)
Definition: fs.cpp:26
static constexpr double INF_FEERATE
Definition: fees.cpp:35
static std::set< double > MakeFeeSet(const CFeeRate &min_incremental_fee, double max_filter_fee_rate, double fee_filter_spacing)
Definition: fees.cpp:1068
std::string StringForFeeEstimateHorizon(FeeEstimateHorizon horizon)
Definition: fees.cpp:37
static constexpr std::chrono::hours MAX_FILE_AGE
fee_estimates.dat that are more than 60 hours (2.5 days) old will not be read, as fee estimates are b...
Definition: fees.h:33
FeeEstimateHorizon
Definition: fees.h:45
std::shared_ptr< const CTransaction > CTransactionRef
Definition: transaction.h:423
uint64_t EncodeDouble(double f) noexcept
Definition: serfloat.cpp:37
double DecodeDouble(uint64_t v) noexcept
Definition: serfloat.cpp:10
EstimatorBucket fail
Definition: fees.h:87
EstimatorBucket pass
Definition: fees.h:86
double decay
Definition: fees.h:88
unsigned int scale
Definition: fees.h:89
double totalConfirmed
Definition: fees.h:78
double end
Definition: fees.h:76
double leftMempool
Definition: fees.h:80
double start
Definition: fees.h:75
double withinTarget
Definition: fees.h:77
double inMempool
Definition: fees.h:79
int returnedTarget
Definition: fees.h:97
int desiredTarget
Definition: fees.h:96
FeeReason reason
Definition: fees.h:95
EstimationResult est
Definition: fees.h:94
const bool m_has_no_mempool_parents
const bool m_chainstate_is_current
const bool m_mempool_limit_bypassed
const CAmount m_fee
const unsigned int txHeight
const CTransactionRef m_tx
const int64_t m_virtual_transaction_size
The virtual transaction size.
#define AssertLockNotHeld(cs)
Definition: sync.h:147
#define LOCK(cs)
Definition: sync.h:257
#define WITH_LOCK(cs, code)
Run code while locking a mutex.
Definition: sync.h:301
#define strprintf
Format arguments and return the string or write to given std::ostream (see tinyformat::format doc for...
Definition: tinyformat.h:1162
AssertLockHeld(pool.cs)
assert(!tx.IsCoinBase())