You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

733 lines
24 KiB

  1. // Copyright (c) 2018-2020, Zpalmtree
  2. //
  3. // Please see the included LICENSE file for more information.
  4. import * as _ from 'lodash';
  5. import { sizeof } from 'object-sizeof';
  6. import { EventEmitter } from 'events';
  7. import { Config } from './Config';
  8. import { Daemon } from './Daemon';
  9. import { SubWallets } from './SubWallets';
  10. import { prettyPrintBytes } from './Utilities';
  11. import { LAST_KNOWN_BLOCK_HASHES_SIZE } from './Constants';
  12. import { SynchronizationStatus } from './SynchronizationStatus';
  13. import { WalletSynchronizerJSON } from './JsonSerialization';
  14. import { LogCategory, logger, LogLevel } from './Logger';
  15. import { underivePublicKey, generateKeyDerivation } from './CryptoWrapper';
  16. import {
  17. Block, RawCoinbaseTransaction, RawTransaction, Transaction,
  18. TransactionData, TransactionInput, TopBlock,
  19. } from './Types';
  20. /**
  21. * Decrypts blocks for our transactions and inputs
  22. * @noInheritDoc
  23. */
  24. export class WalletSynchronizer extends EventEmitter {
  25. public static fromJSON(json: WalletSynchronizerJSON): WalletSynchronizer {
  26. const walletSynchronizer = Object.create(WalletSynchronizer.prototype);
  27. return Object.assign(walletSynchronizer, {
  28. privateViewKey: json.privateViewKey,
  29. startHeight: json.startHeight,
  30. startTimestamp: json.startTimestamp,
  31. synchronizationStatus: SynchronizationStatus.fromJSON(json.transactionSynchronizerStatus),
  32. });
  33. }
  34. /**
  35. * The daemon instance to retrieve blocks from
  36. */
  37. private daemon: Daemon;
  38. /**
  39. * The timestamp to start taking blocks from
  40. */
  41. private startTimestamp: number;
  42. /**
  43. * The height to start taking blocks from
  44. */
  45. private startHeight: number;
  46. /**
  47. * The shared private view key of this wallet
  48. */
  49. private readonly privateViewKey: string;
  50. /**
  51. * Stores the progress of our synchronization
  52. */
  53. private synchronizationStatus: SynchronizationStatus = new SynchronizationStatus();
  54. /**
  55. * Used to find spend keys, inspect key images, etc
  56. */
  57. private subWallets: SubWallets;
  58. /**
  59. * Whether we are already downloading a chunk of blocks
  60. */
  61. private fetchingBlocks: boolean = false;
  62. /**
  63. * Stored blocks for later processing
  64. */
  65. private storedBlocks: Block[] = [];
  66. /**
  67. * Transactions that have disappeared from the pool and not appeared in a
  68. * block, and the amount of times they have failed this check.
  69. */
  70. private cancelledTransactionsFailCount: Map<string, number> = new Map();
  71. /**
  72. * Function to run on block download completion to ensure reset() works
  73. * correctly without blocks being stored after wiping them.
  74. */
  75. private finishedFunc: (() => void) | undefined = undefined;
  76. /**
  77. * Last time we fetched blocks from the daemon. If this goes over the
  78. * configured limit, we'll emit deadnode.
  79. */
  80. private lastDownloadedBlocks: Date = new Date();
  81. private config: Config = new Config();
  82. constructor(
  83. daemon: Daemon,
  84. subWallets: SubWallets,
  85. startTimestamp: number,
  86. startHeight: number,
  87. privateViewKey: string,
  88. config: Config,
  89. synchronizationStatus: SynchronizationStatus = new SynchronizationStatus()) {
  90. super();
  91. this.daemon = daemon;
  92. this.startTimestamp = startTimestamp;
  93. this.startHeight = startHeight;
  94. this.privateViewKey = privateViewKey;
  95. this.subWallets = subWallets;
  96. this.config = config;
  97. this.synchronizationStatus = synchronizationStatus;
  98. }
  99. public getScanHeights(): [number, number] {
  100. return [this.startHeight, this.startTimestamp];
  101. }
  102. /**
  103. * Initialize things we can't initialize from the JSON
  104. */
  105. public initAfterLoad(subWallets: SubWallets, daemon: Daemon, config: Config): void {
  106. this.subWallets = subWallets;
  107. this.daemon = daemon;
  108. this.storedBlocks = [];
  109. this.config = config;
  110. this.cancelledTransactionsFailCount = new Map();
  111. this.lastDownloadedBlocks = new Date();
  112. }
  113. /**
  114. * Convert from class to stringable type
  115. */
  116. public toJSON(): WalletSynchronizerJSON {
  117. return {
  118. privateViewKey: this.privateViewKey,
  119. startHeight: this.startHeight,
  120. startTimestamp: this.startTimestamp,
  121. transactionSynchronizerStatus: this.synchronizationStatus.toJSON(),
  122. };
  123. }
  124. public processBlock(
  125. block: Block,
  126. ourInputs: [string, TransactionInput][]): TransactionData {
  127. const txData: TransactionData = new TransactionData();
  128. if (this.config.scanCoinbaseTransactions) {
  129. const tx: Transaction | undefined = this.processCoinbaseTransaction(
  130. block, ourInputs,
  131. );
  132. if (tx !== undefined) {
  133. txData.transactionsToAdd.push(tx);
  134. }
  135. }
  136. for (const rawTX of block.transactions) {
  137. const [tx, keyImagesToMarkSpent] = this.processTransaction(
  138. block, ourInputs, rawTX,
  139. );
  140. if (tx !== undefined) {
  141. txData.transactionsToAdd.push(tx);
  142. txData.keyImagesToMarkSpent = txData.keyImagesToMarkSpent.concat(
  143. keyImagesToMarkSpent,
  144. );
  145. }
  146. }
  147. txData.inputsToAdd = ourInputs;
  148. return txData;
  149. }
  150. /**
  151. * Process transaction outputs of the given block. No external dependencies,
  152. * lets us easily swap out with a C++ replacement for SPEEEED
  153. *
  154. * @param block
  155. * @param privateViewKey
  156. * @param spendKeys Array of spend keys in the format [publicKey, privateKey]
  157. * @param isViewWallet
  158. * @param processCoinbaseTransactions
  159. */
  160. public async processBlockOutputs(
  161. block: Block,
  162. privateViewKey: string,
  163. spendKeys: [string, string][],
  164. isViewWallet: boolean,
  165. processCoinbaseTransactions: boolean): Promise<[string, TransactionInput][]> {
  166. let inputs: [string, TransactionInput][] = [];
  167. /* Process the coinbase tx if we're not skipping them for speed */
  168. if (processCoinbaseTransactions && block.coinbaseTransaction) {
  169. inputs = inputs.concat(await this.processTransactionOutputs(
  170. block.coinbaseTransaction, block.blockHeight,
  171. ));
  172. }
  173. /* Process the normal txs */
  174. for (const tx of block.transactions) {
  175. inputs = inputs.concat(await this.processTransactionOutputs(
  176. tx, block.blockHeight,
  177. ));
  178. }
  179. return inputs;
  180. }
  181. /**
  182. * Get the height of the sync process
  183. */
  184. public getHeight(): number {
  185. return this.synchronizationStatus.getHeight();
  186. }
  187. public reset(scanHeight: number, scanTimestamp: number): Promise<void> {
  188. return new Promise((resolve) => {
  189. const f = () => {
  190. this.startHeight = scanHeight;
  191. this.startTimestamp = scanTimestamp;
  192. /* Discard sync status */
  193. this.synchronizationStatus = new SynchronizationStatus(scanHeight - 1);
  194. this.storedBlocks = [];
  195. };
  196. if (this.fetchingBlocks) {
  197. this.finishedFunc = () => {
  198. f();
  199. resolve();
  200. this.finishedFunc = undefined;
  201. };
  202. } else {
  203. f();
  204. resolve();
  205. }
  206. });
  207. }
  208. public rewind(scanHeight: number): Promise<void> {
  209. return new Promise((resolve) => {
  210. const f = () => {
  211. this.startHeight = scanHeight;
  212. this.startTimestamp = 0;
  213. /* Discard sync status */
  214. this.synchronizationStatus = new SynchronizationStatus(scanHeight - 1);
  215. this.storedBlocks = [];
  216. };
  217. if (this.fetchingBlocks) {
  218. this.finishedFunc = () => {
  219. f();
  220. resolve();
  221. this.finishedFunc = undefined;
  222. };
  223. } else {
  224. f();
  225. resolve();
  226. }
  227. });
  228. }
  229. /**
  230. * Takes in hashes that we have previously sent. Returns transactions which
  231. * are no longer in the pool, and not in a block, and therefore have
  232. * returned to our wallet
  233. */
  234. public async findCancelledTransactions(transactionHashes: string[]): Promise<string[]> {
  235. /* This is the common case - don't waste time making a useless request
  236. to the daemon */
  237. if (_.isEmpty(transactionHashes)) {
  238. return [];
  239. }
  240. logger.log(
  241. 'Checking locked transactions',
  242. LogLevel.DEBUG,
  243. LogCategory.TRANSACTIONS,
  244. );
  245. const cancelled: string[] = await this.daemon.getCancelledTransactions(transactionHashes);
  246. const toRemove: string[] = [];
  247. for (const [hash, failCount] of this.cancelledTransactionsFailCount) {
  248. /* Hash still not found, increment fail count */
  249. if (cancelled.includes(hash)) {
  250. /* Failed too many times, cancel transaction, return funds to wallet */
  251. if (failCount === 10) {
  252. toRemove.push(hash);
  253. this.cancelledTransactionsFailCount.delete(hash);
  254. logger.log(
  255. `Unconfirmed transaction ${hash} is still not known by daemon after ${failCount} queries. ` +
  256. 'Assuming transaction got dropped from mempool, returning funds and removing unconfirmed transaction.',
  257. LogLevel.DEBUG,
  258. LogCategory.TRANSACTIONS,
  259. );
  260. } else {
  261. logger.log(
  262. `Unconfirmed transaction ${hash} is not known by daemon, query ${failCount + 1}.`,
  263. LogLevel.DEBUG,
  264. LogCategory.TRANSACTIONS,
  265. );
  266. this.cancelledTransactionsFailCount.set(hash, failCount + 1);
  267. }
  268. /* Hash has since been found, remove from fail count array */
  269. } else {
  270. logger.log(
  271. `Unconfirmed transaction ${hash} is known by daemon, removing from possibly cancelled transactions array.`,
  272. LogLevel.DEBUG,
  273. LogCategory.TRANSACTIONS,
  274. );
  275. this.cancelledTransactionsFailCount.delete(hash);
  276. }
  277. }
  278. for (const hash of cancelled) {
  279. /* Transaction with no history, first fail, add to map. */
  280. if (!this.cancelledTransactionsFailCount.has(hash)) {
  281. logger.log(
  282. `Unconfirmed transaction ${hash} is not known by daemon, query 1.`,
  283. LogLevel.DEBUG,
  284. LogCategory.TRANSACTIONS,
  285. );
  286. this.cancelledTransactionsFailCount.set(hash, 1);
  287. }
  288. }
  289. return toRemove;
  290. }
  291. /**
  292. * Retrieve blockCount blocks from the internal store. Does not remove
  293. * them.
  294. */
  295. public async fetchBlocks(blockCount: number): Promise<[Block[], boolean]> {
  296. let shouldSleep = false;
  297. /* Fetch more blocks if we haven't got any downloaded yet */
  298. if (this.storedBlocks.length === 0) {
  299. if (!this.fetchingBlocks) {
  300. logger.log(
  301. 'No blocks stored, attempting to fetch more.',
  302. LogLevel.DEBUG,
  303. LogCategory.SYNC,
  304. );
  305. }
  306. const [successOrBusy, shouldSleepTmp] = await this.downloadBlocks();
  307. shouldSleep = shouldSleepTmp;
  308. /* Not in the middle of fetching blocks. */
  309. if (!successOrBusy) {
  310. /* Seconds since we last got a block */
  311. const diff = (new Date().getTime() - this.lastDownloadedBlocks.getTime()) / 1000;
  312. if (diff > this.config.maxLastFetchedBlockInterval) {
  313. this.emit('deadnode');
  314. }
  315. } else {
  316. this.lastDownloadedBlocks = new Date();
  317. }
  318. }
  319. return [_.take(this.storedBlocks, blockCount), shouldSleep];
  320. }
  321. public dropBlock(blockHeight: number, blockHash: string): void {
  322. /* it's possible for this function to get ran twice.
  323. Need to make sure we don't remove more than the block we just
  324. processed. */
  325. if (this.storedBlocks.length >= 1 &&
  326. this.storedBlocks[0].blockHeight === blockHeight &&
  327. this.storedBlocks[0].blockHash === blockHash) {
  328. this.storedBlocks = _.drop(this.storedBlocks);
  329. this.synchronizationStatus.storeBlockHash(blockHeight, blockHash);
  330. }
  331. /* sizeof() gets a tad expensive... */
  332. if (blockHeight % 10 === 0 && this.shouldFetchMoreBlocks()) {
  333. /* Note - not awaiting here */
  334. this.downloadBlocks().then(([successOrBusy]) => {
  335. if (!successOrBusy) {
  336. /* Seconds since we last got a block */
  337. const diff = (new Date().getTime() - this.lastDownloadedBlocks.getTime()) / 1000;
  338. if (diff > this.config.maxLastFetchedBlockInterval) {
  339. this.emit('deadnode');
  340. }
  341. } else {
  342. this.lastDownloadedBlocks = new Date();
  343. }
  344. });
  345. }
  346. }
  347. public getBlockCheckpoints(): string[] {
  348. return this.synchronizationStatus.getBlockCheckpoints();
  349. }
  350. public getRecentBlockHashes(): string[] {
  351. return this.synchronizationStatus.getRecentBlockHashes();
  352. }
  353. private getStoredBlockCheckpoints(): string[] {
  354. const hashes = [];
  355. for (const block of this.storedBlocks) {
  356. /* Add to start of array - we want hashes in descending block height order */
  357. hashes.unshift(block.blockHash);
  358. }
  359. return _.take(hashes, LAST_KNOWN_BLOCK_HASHES_SIZE);
  360. }
  361. /**
  362. * Only retrieve more blocks if we're not getting close to the memory limit
  363. */
  364. private shouldFetchMoreBlocks(): boolean {
  365. /* Don't fetch more if we're already doing so */
  366. if (this.fetchingBlocks) {
  367. return false;
  368. }
  369. const ramUsage = sizeof(this.storedBlocks);
  370. if (ramUsage < this.config.blockStoreMemoryLimit) {
  371. logger.log(
  372. `Approximate ram usage of stored blocks: ${prettyPrintBytes(ramUsage)}, fetching more.`,
  373. LogLevel.DEBUG,
  374. LogCategory.SYNC,
  375. );
  376. return true;
  377. }
  378. return false;
  379. }
  380. private getWalletSyncDataHashes(): string[] {
  381. const unprocessedBlockHashes: string[] = this.getStoredBlockCheckpoints();
  382. const recentProcessedBlockHashes: string[] = this.synchronizationStatus.getRecentBlockHashes();
  383. const blockHashCheckpoints: string[] = this.synchronizationStatus.getBlockCheckpoints();
  384. const combined = unprocessedBlockHashes.concat(recentProcessedBlockHashes);
  385. /* Take the 50 most recent block hashes, along with the infrequent
  386. checkpoints, to handle deep forks. */
  387. return _.take(combined, LAST_KNOWN_BLOCK_HASHES_SIZE)
  388. .concat(blockHashCheckpoints);
  389. }
  390. /* Returns [successOrBusy, shouldSleep] */
  391. private async downloadBlocks(): Promise<[boolean, boolean]> {
  392. /* Middle of fetching blocks, wait for previous request to complete.
  393. * Don't need to sleep. */
  394. if (this.fetchingBlocks) {
  395. return [true, false];
  396. }
  397. this.fetchingBlocks = true;
  398. const localDaemonBlockCount: number = this.daemon.getLocalDaemonBlockCount();
  399. const walletBlockCount: number = this.getHeight();
  400. if (localDaemonBlockCount < walletBlockCount) {
  401. this.fetchingBlocks = false;
  402. return [true, true];
  403. }
  404. /* Get the checkpoints of the blocks we've got stored, so we can fetch
  405. later ones. Also use the checkpoints of the previously processed
  406. ones, in case we don't have any blocks yet. */
  407. const blockCheckpoints: string[] = this.getWalletSyncDataHashes();
  408. let blocks: Block[] = [];
  409. let topBlock: TopBlock | boolean;
  410. try {
  411. [blocks, topBlock] = await this.daemon.getWalletSyncData(
  412. blockCheckpoints, this.startHeight, this.startTimestamp,
  413. );
  414. } catch (err) {
  415. let strErr = "error in getWalletSyncData";
  416. if (err instanceof Error) strErr = err.toString();
  417. logger.log(
  418. 'Failed to get blocks from daemon: ' + strErr,
  419. LogLevel.DEBUG,
  420. LogCategory.SYNC,
  421. );
  422. if (this.finishedFunc) {
  423. this.finishedFunc();
  424. }
  425. this.fetchingBlocks = false;
  426. return [false, true];
  427. }
  428. if (typeof topBlock === 'object' && blocks.length === 0) {
  429. if (this.finishedFunc) {
  430. this.finishedFunc();
  431. }
  432. /* Synced, store the top block so sync status displays correctly if
  433. we are not scanning coinbase tx only blocks.
  434. Only store top block if we have finished processing stored
  435. blocks */
  436. if (this.storedBlocks.length === 0) {
  437. this.emit('heightchange', topBlock.height);
  438. this.synchronizationStatus.storeBlockHash(topBlock.height, topBlock.hash);
  439. }
  440. logger.log(
  441. 'Zero blocks received from daemon, fully synced',
  442. LogLevel.DEBUG,
  443. LogCategory.SYNC,
  444. );
  445. if (this.finishedFunc) {
  446. this.finishedFunc();
  447. }
  448. this.fetchingBlocks = false;
  449. return [true, true];
  450. }
  451. if (blocks.length === 0) {
  452. logger.log(
  453. 'Zero blocks received from daemon, possibly fully synced',
  454. LogLevel.DEBUG,
  455. LogCategory.SYNC,
  456. );
  457. if (this.finishedFunc) {
  458. this.finishedFunc();
  459. }
  460. this.fetchingBlocks = false;
  461. return [false, false];
  462. }
  463. /* Timestamp is transient and can change - block height is constant. */
  464. if (this.startTimestamp !== 0) {
  465. this.startTimestamp = 0;
  466. this.startHeight = blocks[0].blockHeight;
  467. this.subWallets.convertSyncTimestampToHeight(
  468. this.startTimestamp, this.startHeight,
  469. );
  470. }
  471. /* Add the new blocks to the store */
  472. this.storedBlocks = this.storedBlocks.concat(blocks);
  473. if (this.finishedFunc) {
  474. this.finishedFunc();
  475. }
  476. this.fetchingBlocks = false;
  477. return [true, false];
  478. }
  479. /**
  480. * Process the outputs of a transaction, and create inputs that are ours
  481. */
  482. private async processTransactionOutputs(
  483. rawTX: RawCoinbaseTransaction,
  484. blockHeight: number): Promise<[string, TransactionInput][]> {
  485. const inputs: [string, TransactionInput][] = [];
  486. if (rawTX.transactionPublicKey === undefined) {
  487. return inputs;
  488. }
  489. const derivation: string = await generateKeyDerivation(
  490. rawTX.transactionPublicKey, this.privateViewKey, this.config,
  491. );
  492. const spendKeys: string[] = this.subWallets.getPublicSpendKeys();
  493. for (const [outputIndex, output] of rawTX.keyOutputs.entries()) {
  494. /* Derive the spend key from the transaction, using the previous
  495. derivation */
  496. const derivedSpendKey = await underivePublicKey(
  497. derivation, outputIndex, output.key, this.config,
  498. );
  499. /* See if the derived spend key matches any of our spend keys */
  500. if (!_.includes(spendKeys, derivedSpendKey)) {
  501. continue;
  502. }
  503. /* The public spend key of the subwallet that owns this input */
  504. const ownerSpendKey = derivedSpendKey;
  505. /* Not spent yet! */
  506. const spendHeight: number = 0;
  507. const [keyImage, privateEphemeral] = await this.subWallets.getTxInputKeyImage(
  508. ownerSpendKey, derivation, outputIndex,
  509. );
  510. const txInput: TransactionInput = new TransactionInput(
  511. keyImage, output.amount, blockHeight,
  512. rawTX.transactionPublicKey, outputIndex, output.globalIndex,
  513. output.key, spendHeight, rawTX.unlockTime, rawTX.hash,
  514. privateEphemeral,
  515. );
  516. inputs.push([ownerSpendKey, txInput]);
  517. }
  518. return inputs;
  519. }
  520. private processCoinbaseTransaction(
  521. block: Block,
  522. ourInputs: [string, TransactionInput][]): Transaction | undefined {
  523. /* Should be guaranteed to be defined here */
  524. const rawTX: RawCoinbaseTransaction = block.coinbaseTransaction as RawCoinbaseTransaction;
  525. const transfers: Map<string, number> = new Map();
  526. const relevantInputs: [string, TransactionInput][]
  527. = _.filter(ourInputs, ([, input]) => {
  528. return input.parentTransactionHash === rawTX.hash;
  529. });
  530. for (const [publicSpendKey, input] of relevantInputs) {
  531. transfers.set(
  532. publicSpendKey,
  533. input.amount + (transfers.get(publicSpendKey) || 0),
  534. );
  535. }
  536. if (!_.isEmpty(transfers)) {
  537. /* Coinbase transaction have no fee */
  538. const fee: number = 0;
  539. const isCoinbaseTransaction: boolean = true;
  540. /* Coinbase transactions can't have payment ID's */
  541. const paymentID: string = '';
  542. return new Transaction(
  543. transfers, rawTX.hash, fee, block.blockHeight, block.blockTimestamp,
  544. paymentID, rawTX.unlockTime, isCoinbaseTransaction,
  545. );
  546. }
  547. return undefined;
  548. }
  549. private processTransaction(
  550. block: Block,
  551. ourInputs: [string, TransactionInput][],
  552. rawTX: RawTransaction): [Transaction | undefined, [string, string][]] {
  553. const transfers: Map<string, number> = new Map();
  554. const relevantInputs: [string, TransactionInput][]
  555. = _.filter(ourInputs, ([, input]) => {
  556. return input.parentTransactionHash === rawTX.hash;
  557. });
  558. for (const [publicSpendKey, input] of relevantInputs) {
  559. transfers.set(
  560. publicSpendKey,
  561. input.amount + (transfers.get(publicSpendKey) || 0),
  562. );
  563. }
  564. const spentKeyImages: [string, string][] = [];
  565. for (const input of rawTX.keyInputs) {
  566. const [found, publicSpendKey] = this.subWallets.getKeyImageOwner(
  567. input.keyImage,
  568. );
  569. if (found) {
  570. transfers.set(
  571. publicSpendKey,
  572. -input.amount + (transfers.get(publicSpendKey) || 0),
  573. );
  574. spentKeyImages.push([publicSpendKey, input.keyImage]);
  575. }
  576. }
  577. if (!_.isEmpty(transfers)) {
  578. const fee: number = _.sumBy(rawTX.keyInputs, 'amount') -
  579. _.sumBy(rawTX.keyOutputs, 'amount');
  580. const isCoinbaseTransaction: boolean = false;
  581. return [new Transaction(
  582. transfers, rawTX.hash, fee, block.blockHeight,
  583. block.blockTimestamp, rawTX.paymentID, rawTX.unlockTime,
  584. isCoinbaseTransaction,
  585. ), spentKeyImages];
  586. }
  587. return [undefined, []];
  588. }
  589. }