diff --git a/docker-compose-prod.yml b/docker-compose-prod.yml index 10b7b5ad..79d31f32 100644 --- a/docker-compose-prod.yml +++ b/docker-compose-prod.yml @@ -1,9 +1,9 @@ version: "3" services: federation: - image: sovryn/fed-tokenbridge:10.9 + image: sovryn/fed-tokenbridge:10.19 ports: - - 30303:30303 + - 4444:30303 volumes: - ./federator-env/${FED_ENV}/:/app/federator/config - ./federator-env/${FED_ENV}/db:/app/federator/db diff --git a/docker-compose-test.yml b/docker-compose-test.yml index 74541349..46f138ec 100644 --- a/docker-compose-test.yml +++ b/docker-compose-test.yml @@ -2,7 +2,7 @@ version: "3" services: federation: image: sovryn/fed-tokenbridge:10.9 - ports: + ports: - 30303:30303 volumes: - ./federator-env/${FED_ENV}/:/app/federator/config diff --git a/federator-env/mainnet-BSC-RSK/bmainnet.json b/federator-env/mainnet-BSC-RSK/bmainnet.json index 331df70d..e9ef6e83 100644 --- a/federator-env/mainnet-BSC-RSK/bmainnet.json +++ b/federator-env/mainnet-BSC-RSK/bmainnet.json @@ -1,9 +1,9 @@ { "bridge": "0xdfc7127593c8af1a17146893f10e08528f4c2aa7", - "federation": "0x6d212BB9059a1761592ed199Be9955f0dFBf141E", + "federation": "0x502fBCe27973d4bE1E69a4099046762251D005B4", "multiSig": "0xec3fabc3517e64e07669dd1d2d673f466f93a328", "allowTokens": "0xc4b5178Cc086E764568AdfB2dacCBB0d973e8132", "erc777Converter": "0x9D46B33171eA7124aEE472bFe61B5B7084B55069", - "host": "https://bsc-dataseed1.defibit.io/", - "fromBlock": 7912333 + "host": "https://bsc.sovryn.app/mainnet", + "fromBlock": 21118895 } diff --git a/federator-env/mainnet-BSC-RSK/config.js b/federator-env/mainnet-BSC-RSK/config.js index 12f50855..429f8bca 100644 --- a/federator-env/mainnet-BSC-RSK/config.js +++ b/federator-env/mainnet-BSC-RSK/config.js @@ -13,6 +13,10 @@ module.exports = { confirmations: 120, // Number of blocks before processing it, if working with ganache set as 0 privateKey: fs.readFileSync(`${__dirname}/federator.key`, 'utf8').trim(), storagePath: './db', + minimumPeerAmount: 2, + port: 30303, + signaturesTTL: 24 * 60 * 60, // In seconds + signatureRequestTimeoutMs: 2 * 60 * 1000, // In milliseconds federatorInstanceId: 'federatorInstanceId_replace_this', federatorAddress: 'federatorAddress_replace_this', telegramBot: { diff --git a/federator-env/mainnet-BSC-RSK/peers.config.js b/federator-env/mainnet-BSC-RSK/peers.config.js new file mode 100644 index 00000000..dca2fa40 --- /dev/null +++ b/federator-env/mainnet-BSC-RSK/peers.config.js @@ -0,0 +1,24 @@ +module.exports = { + "peers": [ + { + "address": "0xf963C7B3D8f6dAB6d1176702B94Ecdb75916770A", + "ip": "3.132.89.149", + "port": 4444 + }, + { + "address": "0x4b03E69D6962649573f2747c04F2dd9aB5494Cdb", + "ip": "3.140.153.73", + "port": 4444 + }, + { + "address": "0x642aA4Ab1F29c0E8877A99312494E2A0b623a682", + "ip": "18.189.59.47", + "port": 4444 + }, + { + "address": "0x57a247b871784E6e303CC29230Fa664fab15370D", + "ip": "18.116.158.70", + "port": 4444 + } + ] +}; \ No newline at end of file diff --git a/federator-env/mainnet-BSC-RSK/rskmainnet.json b/federator-env/mainnet-BSC-RSK/rskmainnet.json index 5cf97b60..5a3fd62f 100644 --- a/federator-env/mainnet-BSC-RSK/rskmainnet.json +++ b/federator-env/mainnet-BSC-RSK/rskmainnet.json @@ -1,9 +1,9 @@ { "bridge": "0x971b97c8cc82e7d27bc467c2dc3f219c6ee2e350", - "federation": "0xAb8F212eA333626a96999A865072095aDE95Eb5a", + "federation": "0xD1E45f51C8f09b139218fc75d26409096316971C", "multiSig": "0xee9ea57555d9533d71f6f77e0e480961f068a6c5", "allowTokens": "0x200FD7a1cCEa4651F15008cC99bF82d7461EFd3F", "erc777Converter": "0xc8149b1F15794D135Dfe2924955cb90709Ac7070", - "host": "http://18.221.155.102:4444/", - "fromBlock": 3398643 + "host": "https://mainnet.sovryn.app/rpc", + "fromBlock": 4614985 } diff --git a/federator-env/mainnet-ETH-RSK/config.js b/federator-env/mainnet-ETH-RSK/config.js index 3b304f71..1ebfeb6a 100644 --- a/federator-env/mainnet-ETH-RSK/config.js +++ b/federator-env/mainnet-ETH-RSK/config.js @@ -26,6 +26,10 @@ module.exports = { confirmations: 120, // Number of blocks before processing it, if working with ganache set as 0 privateKey: fs.readFileSync(`${__dirname}/federator.key`, 'utf8').trim(), storagePath: './db', + minimumPeerAmount: 2, + port: 30303, + signaturesTTL: 24 * 60 * 60, // In seconds + signatureRequestTimeoutMs: 2 * 60 * 1000, // In milliseconds federatorInstanceId: 'federatorInstanceId_replace_this', federatorAddress: 'federatorAddress_replace_this', etherscanApiKey: etherscanApiKey, diff --git a/federator-env/mainnet-ETH-RSK/mainnet.json b/federator-env/mainnet-ETH-RSK/mainnet.json index b763581c..75cbaa2d 100644 --- a/federator-env/mainnet-ETH-RSK/mainnet.json +++ b/federator-env/mainnet-ETH-RSK/mainnet.json @@ -1,9 +1,9 @@ { "bridge": "0x33c0d33a0d4312562ad622f91d12b0ac47366ee1", - "federation": "0xa38E6a92495A58A05969846a21d5cbb41Dc4dF24", + "federation": "0xD77b76A65a19715BDcB5eE223928af2919836A3E", "multiSig": "0x062c74f9d27b1178bb76186c1756128ccb3ccd2e", "allowTokens": "0xf9A59a649859A27d664C8bDb51fA53bCb268545C", "erc777Converter": "0xC0b2A9E31f69e4F0bC24584C678C582714a4fA1b", "host": "https://mainnet.infura.io/v3/f02caaf003d14ad7bee83f33eeda2f5a", - "fromBlock": 12110034 + "fromBlock": 15483239 } diff --git a/federator-env/mainnet-ETH-RSK/peers.config.js b/federator-env/mainnet-ETH-RSK/peers.config.js new file mode 100644 index 00000000..7632eeff --- /dev/null +++ b/federator-env/mainnet-ETH-RSK/peers.config.js @@ -0,0 +1,25 @@ +module.exports = { + "peers": [ + { + "address": "0xa420512B06B23d14Beb25Bae524a9B5F8789c45C", + "ip": "18.119.155.67", + "port": 4444 + }, + { + "address": "0xcD1b561207E20A7ccbcf004bb0a4bc897BA8F2eE", + "ip": "3.129.188.13", + "port": 4444 + }, + { + "address": "0x778898877A3277F7306b19879F426A86d078E115", + "ip": "3.132.239.65", + "port": 4444 + }, + { + "address": "0x8a2241ce21AfA71515a73f82D09E690b85603F35", + "ip": "3.134.242.47", + "port": 4444 + } + ] +} +; \ No newline at end of file diff --git a/federator-env/mainnet-ETH-RSK/rskmainnet.json b/federator-env/mainnet-ETH-RSK/rskmainnet.json index 062e4087..cbe6762c 100644 --- a/federator-env/mainnet-ETH-RSK/rskmainnet.json +++ b/federator-env/mainnet-ETH-RSK/rskmainnet.json @@ -1,9 +1,9 @@ { "bridge": "0x1ccad820b6d031b41c54f1f3da11c0d48b399581", - "federation": "0xCEB11D7e862eF324C2C6c0feBc8797D8be83084e", + "federation": "0x32593e4f7A4991C2fe17459DaE9920fd612855B4", "multiSig": "0xb64322e10b5ae1be121b8bb0dead560c53d9dbc3", "allowTokens": "0x7DC1D73C620cF8eB167eDD32942DA0d01B70adC0", "erc777Converter": "0xb86623c103843ccf75c6f0073d84bcfc0e34536c", - "host": "http://18.221.155.102:4444/", - "fromBlock": 3258718 + "host": "https://mainnet.sovryn.app/rpc", + "fromBlock": 4615325 } \ No newline at end of file diff --git a/federator-env/testnet-BSC-RSK/btestnet.json b/federator-env/testnet-BSC-RSK/btestnet.json index 00bad1ca..604f8bdf 100644 --- a/federator-env/testnet-BSC-RSK/btestnet.json +++ b/federator-env/testnet-BSC-RSK/btestnet.json @@ -4,6 +4,6 @@ "multiSig": "0x1d8cb60d35fcd42a8bd18d027386be9c0f9c509b", "allowTokens": "0xeb23e848ceca88b7d0c019c7186bb86cefadd0bd", "erc777Converter": "0x573CAF2cA648e22fE9721EDB5DdfBdF5645ffd18", - "host": "https://data-seed-prebsc-1-s1.binance.org:8545/", + "host": "https://bsc.sovryn.app/testnet", "fromBlock": 16600000 } diff --git a/federator-env/testnet-ETH-RSK/config.js b/federator-env/testnet-ETH-RSK/config.js index f464f5f0..c04d9011 100644 --- a/federator-env/testnet-ETH-RSK/config.js +++ b/federator-env/testnet-ETH-RSK/config.js @@ -29,7 +29,8 @@ module.exports = { runEvery: 2, // In minutes, confirmations: 120, // Number of blocks before processing it, if working with ganache set as 0 privateKey: fs.readFileSync(`${__dirname}/federator.key`, 'utf8').trim(), - signaturesTTL: 120, + signaturesTTL: 24 * 60 * 60, // In seconds + signatureRequestTimeoutMs: 2 * 60 * 1000, // In milliseconds storagePath: './db', federatorInstanceId: 'federatorInstanceId_replace_this', federatorAddress: 'federatorAddress_replace_this', diff --git a/federator-env/testnet-ETH-RSK/ropsten.json b/federator-env/testnet-ETH-RSK/ropsten.json index be50d2b8..f67aac7a 100644 --- a/federator-env/testnet-ETH-RSK/ropsten.json +++ b/federator-env/testnet-ETH-RSK/ropsten.json @@ -5,5 +5,5 @@ "allowTokens": "0x9bc4243880730a9bca69addb0f971700d39d1646", "erc777Converter": "0x8F3bbD7673485Cf4c43EF008e125575028fD43F1", "host": "https://ropsten.infura.io/v3/728946296ea64626941bb3d120d16333", - "fromBlock": 10222073 + "fromBlock": 12190673 } \ No newline at end of file diff --git a/federator-env/testnet-ETH-RSK/rsktestnet.json b/federator-env/testnet-ETH-RSK/rsktestnet.json index a0d7ffcc..113e11fd 100644 --- a/federator-env/testnet-ETH-RSK/rsktestnet.json +++ b/federator-env/testnet-ETH-RSK/rsktestnet.json @@ -5,5 +5,5 @@ "allowTokens": "0x918b9fd8c2e9cf5625ea00ca6cfa270a44050d01", "erc777Converter": "0x621d9Ce70Db000273Ddb3d50fa85732960a9E934", "host": "https://testnet2.sovryn.app/rpc", - "fromBlock": 1980020 + "fromBlock": 2962423 } diff --git a/federator-wallets-checker/Jenkinsfile b/federator-wallets-checker/Jenkinsfile index 1371c094..e273dbeb 100644 --- a/federator-wallets-checker/Jenkinsfile +++ b/federator-wallets-checker/Jenkinsfile @@ -45,7 +45,7 @@ def sendEmail(needFundsWalletsOuput, subject){ """ - emailext (from: "notifications@sovryn.app", to:"ororosovryn@protonmail.com, shlumperx@gmail.com, brave.pooh8@gmail.com, elan@remake.money", mimeType: 'text/html', subject: subject, body: emailBody) + emailext (to:"ororosovryn@protonmail.com, brave.pooh8@gmail.com, elan@remake.money", mimeType: 'text/html', subject: subject, body: emailBody) } diff --git a/how_to_build_fed_docker_image.md b/how_to_build_fed_docker_image.md index bf1c81ac..1df028fd 100644 --- a/how_to_build_fed_docker_image.md +++ b/how_to_build_fed_docker_image.md @@ -5,7 +5,7 @@ with github actions * create tag: ```git tag 8.5``` -* push tags: ```git push tag 8.5``` +* push tag: ```git push origin 8.5``` diff --git a/reset.sh b/reset.sh index 8e5960d7..1095eb42 100755 --- a/reset.sh +++ b/reset.sh @@ -4,5 +4,6 @@ find federator-env/testnet-BSC-RSK/db/. -type f -not -name 'failingTxIds.txt' -d find federator-env/testnet-ETH-RSK/db/. -type f -not -name 'failingTxIds.txt' -delete find federator-env/rinkeby-ETH-RSK/db/. -type f -not -name 'failingTxIds.txt' -delete rm -rf federator.log -git reset --hard origin/master -git pull +git reset --hard HEAD +git fetch +git checkout origin/signatures # test diff --git a/sovryn-token-bridge/federator/config/config.sample.js b/sovryn-token-bridge/federator/config/config.sample.js index 33de7a81..8bc11c8f 100644 --- a/sovryn-token-bridge/federator/config/config.sample.js +++ b/sovryn-token-bridge/federator/config/config.sample.js @@ -24,7 +24,7 @@ module.exports = { runEvery: 2, // In minutes, confirmations: 120, // Number of blocks before processing it, if working with ganache set as 0 privateKey: fs.readFileSync(`${__dirname}/federator.key`, 'utf8').trim(), - signaturesTTL: 120, // 2 minutes + signaturesTTL: 24 * 60 * 60, // In seconds storagePath: './db', federatorInstanceId: '', etherscanApiKey: etherscanApiKey, diff --git a/sovryn-token-bridge/federator/src/lib/Federator.js b/sovryn-token-bridge/federator/src/lib/Federator.js index 9ae31235..25abc3d6 100644 --- a/sovryn-token-bridge/federator/src/lib/Federator.js +++ b/sovryn-token-bridge/federator/src/lib/Federator.js @@ -145,12 +145,6 @@ module.exports = class Federator { // Loop around logs to process each of them async _processLogs(ctr, logs) { try { - const transactionSender = new TransactionSender( - this.sideWeb3, - this.logger, - this.config, - '' - ); const currentBlock = await this._getCurrentBlockNumber(); let newLastBlockNumber; @@ -161,9 +155,16 @@ module.exports = class Federator { const { _amount: amount, _symbol: symbol } = log.returnValues; if (this._isConfirmed(ctr, symbol, amount, currentBlock, log.blockNumber)) { - const signatures = await this._requestSignatureFromFederators(log); - this.logger.info('Collected enough signatures'); - await this._processLog(log, signatures); + const alreadyProcessed = await this._isAlreadyProcessed(log); + if (alreadyProcessed) { + this.logger.debug( + `Block: ${log.blockHash} Tx: ${log.transactionHash} token: ${symbol} is already processed (on Bridge) -- no need to get signatures` + ); + } else { + const signatures = await this._requestSignatureFromFederators(log); + this.logger.info('Collected enough signatures'); + await this._processLog(log, signatures); + } } else if (allLogsConfirmed) { newLastBlockNumber = log.blockNumber - 1; allLogsConfirmed = false; @@ -181,8 +182,16 @@ module.exports = class Federator { this.logger.info('Requesting other federators to sign event'); const timer = setTimeout( - () => reject("Didn't get enough signatures after 10 minutes timeout"), - 60000 + () => { + this.logger.warn("Timeout: Didn't get enough signatures after waiting"); + try { + listener.unsubscribe(); + } catch(e) { + this.logger.error(`Error subscribing from listener on timeout: ${e}`); + } + reject("Didn't get enough signatures before timeout") + }, + this.config.signatureRequestTimeoutMs || 60000 ); // Select correct message type depending on main on side federator @@ -198,6 +207,7 @@ module.exports = class Federator { }; const signatures = new Set(); + const signers = new Set(); const listener = this.network.net.onMessage(async (msg) => { if (msg.type === submissionType && msg.data.logId === log.id) { this.logger.info(`Submission received from ${msg.source.id}`); @@ -212,6 +222,28 @@ module.exports = class Federator { return; } + if (signers.has(signerAddress)) { + this.logger.warn( + `Signer ${signerAddress} has already submitted a signature` + ); + return; + } + signers.add(signerAddress); + + // Require 2 minutes or signaturesTTL/2 of buffer, whichever is smaller, to avoid sending the + // transactions with signatures that might expire before the transaction gets mined in the blockchain. + // 2 minutes might not be enough, but we need to start with something + const deadlineBufferSeconds = Math.min( + this.config.signaturesTTL / 2, + 120 + ); + if (!utils.validateDeadline(signatureData.deadline, deadlineBufferSeconds)) { + this.logger.warn( + `Deadline ${signatureData.deadline} has either passed or is too close` + ); + return; + } + signatures.add(signatureData); if (signatures.size >= this.config.minimumPeerAmount) { clearTimeout(timer); @@ -256,6 +288,21 @@ module.exports = class Federator { return ethers.utils.recoverAddress(digest, signature); } + async _isAlreadyProcessed(log) { + const { + _to: receiver, + _amount: amount, + } = log.returnValues; + let bridgeTransactionId = await this.sideBridgeContract.methods.getTransactionId( + log.blockHash, + log.transactionHash, + receiver, + amount, + log.logIndex, + ).call(); + return await this.sideBridgeContract.methods.processed(bridgeTransactionId).call(); + } + async _processLog(log, signatures) { const { _to: receiver, @@ -268,17 +315,11 @@ module.exports = class Federator { } = log.returnValues; // We check the status from the bridge first before bothering checking the Federation contract. - // Actually, a check from the bridge is all that we need (in principle) - let bridgeTransactionId = await this.sideBridgeContract.methods.getTransactionId( - log.blockHash, - log.transactionHash, - receiver, - amount, - log.logIndex, - ).call(); - this.logger.info('Bridge transaction id:', bridgeTransactionId); - let wasProcessed = await this.sideBridgeContract.methods.processed(bridgeTransactionId).call(); - this.logger.info('was processed (bridge):', wasProcessed); + // Actually, a check from the bridge is all that we need (in principle) -- but let's leave the other checks + // there too. + // Note that we don't really need to double-check here either, since we check this before requesting + // signatures from federators, but let's do it anyway for safety. + let wasProcessed = await this._isAlreadyProcessed(log); if (wasProcessed) { this.logger.debug( `Block: ${log.blockHash} Tx: ${log.transactionHash} token: ${symbol} was already processed (on Bridge)` @@ -543,13 +584,18 @@ module.exports = class Federator { } async signTransaction({ blockNumber, id }) { - const logs = await this.mainBridgeContract.getPastEvents('Cross', { + const allLogs = await this.mainBridgeContract.getPastEvents('Cross', { fromBlock: blockNumber, toBlock: blockNumber, - filter: { id }, }); + // Passing filter: { id } to getPastEvents won't do any filtering, we need to filter like this + const logs = allLogs.filter(log => log.id === id); - if (logs.length !== 1) throw new CustomError('Invalid return when searching for event'); + if (logs.length !== 1) { + this.logger.error(`Got ${logs.length} logs when expecting 1. Block number: ${blockNumber}, Log id: ${id}, Logs:`); + this.logger.error(logs); + throw new CustomError('Invalid return when searching for event'); + } const { _tokenAddress, _amount, _to, _symbol, _decimals, _granularity, _userData } = logs[0].returnValues; diff --git a/sovryn-token-bridge/federator/src/lib/GasServices.js b/sovryn-token-bridge/federator/src/lib/GasServices.js index 2664faf3..6550492b 100644 --- a/sovryn-token-bridge/federator/src/lib/GasServices.js +++ b/sovryn-token-bridge/federator/src/lib/GasServices.js @@ -77,8 +77,11 @@ module.exports = class GasServices { this.logger.info( `Process is waiting to calculate average gas of ${avgGasPeriodInterval} ms.` ); - this.logger.debug('Process is waiting to calculate average gas of ', avgGasPeriodInterval); - await utils.sleep(avgGasPeriodInterval, { logger: this.logger }); + // Temporarily only sleep for 5 min while waiting for gas prices + // this.logger.debug('Process is waiting to calculate average gas of ', avgGasPeriodInterval); + // await utils.sleep(avgGasPeriodInterval, { logger: this.logger }); + this.logger.debug('Process is waiting to calculate average gas for 5min'); + await utils.sleep(5 * 60 * 1000, { logger: this.logger }); } async runGasPriceService() { diff --git a/sovryn-token-bridge/federator/src/lib/TransactionSender.js b/sovryn-token-bridge/federator/src/lib/TransactionSender.js index 731c2cb8..11927717 100644 --- a/sovryn-token-bridge/federator/src/lib/TransactionSender.js +++ b/sovryn-token-bridge/federator/src/lib/TransactionSender.js @@ -110,7 +110,7 @@ module.exports = class TransactionSender { async createETHRawTransaction(from, to, data, value, chainId) { const nonce = await this.getNonce(from); - const gwei = 1_000_000_000; + const gwei = 1000000000; const priorityFee = 2; const sleepOnGas = this.config.sleepOnGas * 1000; //10 * 1000 ; // 10 Seconds const maxSleepOnGas = this.config.maxSleepOnGas; //12 diff --git a/sovryn-token-bridge/federator/src/lib/p2p.js b/sovryn-token-bridge/federator/src/lib/p2p.js index 697af3bf..c0dc945b 100644 --- a/sovryn-token-bridge/federator/src/lib/p2p.js +++ b/sovryn-token-bridge/federator/src/lib/p2p.js @@ -86,7 +86,8 @@ class P2p { } getPeerAmount() { - return this.net.nodes.length; + // This returns the peer amount without this node + return this.getNodeIds().length - 1; } isLeader() { diff --git a/sovryn-token-bridge/federator/src/lib/utils.js b/sovryn-token-bridge/federator/src/lib/utils.js index f3434df7..3954924d 100644 --- a/sovryn-token-bridge/federator/src/lib/utils.js +++ b/sovryn-token-bridge/federator/src/lib/utils.js @@ -126,8 +126,13 @@ function eliminateDuplicates(arrays) { return Array.from(set); } -function createTimestamp(secondesOffset) { - return Math.floor(Date.now() / 1000) + secondesOffset; +function createTimestamp(secondsOffset) { + return Math.floor(Date.now() / 1000) + secondsOffset; +} + +function validateDeadline(deadline, bufferSeconds = 0) { + const currentTimestamp = Math.floor(Date.now() / 1000); + return deadline >= currentTimestamp + bufferSeconds; } module.exports = { @@ -143,5 +148,6 @@ module.exports = { zeroHash: '0x0000000000000000000000000000000000000000000000000000000000000000', waitForReceipt: waitForReceipt, eliminateDuplicates: eliminateDuplicates, - createTimestamp: createTimestamp, + createTimestamp, + validateDeadline, }; diff --git a/sovryn-token-bridge/federator/src/main.js b/sovryn-token-bridge/federator/src/main.js index 74be7fa3..64abd126 100644 --- a/sovryn-token-bridge/federator/src/main.js +++ b/sovryn-token-bridge/federator/src/main.js @@ -156,7 +156,11 @@ async function startServices() { let fedAddress = config.federatorAddress; let peers = peersConfig.peers - let peersFiltered = peers.filter(peer => peer.address !== fedAddress); + let peersFiltered = peers.filter( + peer => ( + (peer.address || '').toLowerCase() !== (fedAddress || '').toLowerCase() + ) + ); console.log(peersFiltered); try { @@ -189,12 +193,22 @@ async function startServices() { } async function run() { - if (p2pNode.getPeerAmount() < config.minimumPeerAmount) { - logger.info('Waiting for enough peers'); + const numOtherPeers = p2pNode.getPeerAmount(); + let nodeId = ''; + let leaderId = ''; + try { + nodeId = p2pNode.net.networkId; + leaderId = p2pNode.getLeaderId(); + } catch (e) { + // just ignore now, this is only for debugging + } + if (numOtherPeers < config.minimumPeerAmount) { + logger.info(`Waiting for enough peers (now ${numOtherPeers}. node ${nodeId}, leader ${leaderId})`); return; } if (p2pNode.isLeader()) { + console.log(`This node is a leader -- handling iteration. (${numOtherPeers} other peers, node ${nodeId}, leader ${leaderId})`); try { console.log('before mainfed'); await mainFederator.run(); @@ -204,6 +218,8 @@ async function run() { logger.error('Unhandled Error on run()', err); process.exit(); } + } else { + console.log(`Not leader, just chilling. (${numOtherPeers} other peers, node ${nodeId}, leader ${leaderId})`) } } diff --git a/sovryn-token-bridge/federator/test/Federator.test.js b/sovryn-token-bridge/federator/test/Federator.test.js index 9564f324..59ebea53 100644 --- a/sovryn-token-bridge/federator/test/Federator.test.js +++ b/sovryn-token-bridge/federator/test/Federator.test.js @@ -36,8 +36,20 @@ const disableEtherscanGasPrices = (sender) => { .mockRejectedValue(new Error('expected etherscan error')); }; -function mockFederatorMethods(federator, methods) { - federator.mainWeb3.eth = federator.mainWeb3.eth.mockMethods(methods); +function mockFederatorMethods(federator, mainWeb3Methods, sideWeb3Methods) { + // create copies to avoid mutating the original + if (mainWeb3Methods) { + federator.mainWeb3 = { + ...federator.mainWeb3, + eth: federator.mainWeb3.eth.mockMethods(mainWeb3Methods) // this creates a copy + }; + } + if (sideWeb3Methods) { + federator.sideWeb3 = { + ...federator.sideWeb3, + eth: federator.sideWeb3.eth.mockMethods(sideWeb3Methods) // this creates a copy + }; + } disableEtherscanGasPrices(federator.transactionSender); } @@ -110,7 +122,9 @@ describe('Federator module tests', () => { mockFederatorMethods(federator, { getBlockNumber: () => Promise.resolve(currentBlock), - getId: () => Promise.resolve(1), + net: { + getId: () => Promise.resolve(1), + } }); const _processLogsSpy = jest.spyOn(federator, '_processLogs'); @@ -132,10 +146,10 @@ describe('Federator module tests', () => { const federator = new Federator(MAIN_FEDERATOR, testConfig, logger, {}, web3Mock); mockFederatorMethods(federator, { getBlockNumber: () => Promise.resolve(currentBlock), - getId: () => Promise.resolve(1), + net: { + getId: () => Promise.resolve(1), + } }); - federator.mainWeb3.eth.getBlockNumber = () => Promise.resolve(currentBlock); - federator.mainWeb3.eth.net.getId = () => Promise.resolve(1); const _processLogsSpy = jest.spyOn(federator, '_processLogs'); let result = await federator.run(); @@ -480,11 +494,56 @@ describe('Federator module tests', () => { }); describe('Signatures', () => { - const chainId = 1; + const mainChainId = 1; + const sideChainId = 3; const contractAddress = testConfig.sidechain.federation; - const wallets = []; + const txId = '0x7cfbaa6f05794922229e60c7c9695cc52cd13ed9ab1b88597626bd70bb8315d1'; + const wallets = [ + new ethers.Wallet(testConfig.privateKey), + new ethers.Wallet( + '0x20a3c8eb679bc7fe83e31754871c31a0cff0bf8d96edb8136f1b364753f720f1' + ), + new ethers.Wallet( + '0x6ff46791d809f5a588c1339dc065e38d2079eafa6ff32130a6d5686e0b6b60ea' + ), + ]; const signatures = []; + // Use this to create the federator instead of using the constructor -- it handles method mocking correctly + const createFederator = (mockP2p = {}, members = undefined) => { + const federator = new Federator(MAIN_FEDERATOR, testConfig, logger, mockP2p, web3Mock); + if (members) { + federator.members = members; + } + + const expectedConfirmations = 5760; + const onePage = 1001; + const currentBlock = testConfig.mainchain.fromBlock + onePage + expectedConfirmations; + mockFederatorMethods( + federator, + undefined, + { + getBlockNumber: () => Promise.resolve(currentBlock), + net: { + getId: () => Promise.resolve(sideChainId), + } + } + ); + return federator; + } + + const createSignature = async (wallet, deadline) => { + const payload = ethers.utils.solidityPack( + ['bytes32', 'uint256', 'address', 'uint256'], + [txId, sideChainId, contractAddress, deadline] + ); + const signature = await wallet.signMessage(ethers.utils.arrayify(payload)); + return { + signature, + deadline, + } + } + const logs = [ { logIndex: 2, @@ -555,31 +614,9 @@ describe('Federator module tests', () => { ]; beforeAll(async () => { - wallets.push(new ethers.Wallet(testConfig.privateKey)); - wallets.push( - new ethers.Wallet( - '0x20a3c8eb679bc7fe83e31754871c31a0cff0bf8d96edb8136f1b364753f720f1' - ) - ); - wallets.push( - new ethers.Wallet( - '0x6ff46791d809f5a588c1339dc065e38d2079eafa6ff32130a6d5686e0b6b60ea' - ) - ); - - const txId = '0x7cfbaa6f05794922229e60c7c9695cc52cd13ed9ab1b88597626bd70bb8315d1'; - const deadline = Math.floor(Date.now() / 1000) + 120; - const payload = ethers.utils.solidityPack( - ['bytes32', 'uint256', 'address', 'uint256'], - [txId, chainId, contractAddress, deadline] - ); for (let i = 0; i < wallets.length; i += 1) { - const signature = await wallets[i].signMessage(ethers.utils.arrayify(payload)); - signatures.push({ - signature, - deadline, - }); + signatures.push(await createSignature(wallets[i], deadline)); } }); @@ -605,8 +642,7 @@ describe('Federator module tests', () => { }, }; - const federator = new Federator(MAIN_FEDERATOR, testConfig, logger, mockP2p, web3Mock); - federator.members = [wallets[0].address, wallets[1].address, wallets[2].address]; + const federator = createFederator(mockP2p, [wallets[0].address, wallets[1].address, wallets[2].address]); const res = await federator._requestSignatureFromFederators(logs[0]); expect(res).toHaveLength(2); @@ -641,13 +677,84 @@ describe('Federator module tests', () => { }, }; + const federator = createFederator(mockP2p, [wallets[0].address, wallets[1].address, wallets[2].address]); + + const res = await federator._requestSignatureFromFederators(logs[0]); + // It won't timeout, but it will reject the second signature and return the first and third. + // Otherwise it would return first and second. + expect(res).toHaveLength(2); + expect(res[0]).toBe(signatures[0]); + expect(res[1]).toBe(signatures[1]); + }); + + it('should not use same signature twice (with timeout)', async () => { + const mockP2p = { + net: { + onMessage: (func) => { + this.callback = func; + return { unsubscribe: () => {} }; + }, + broadcast: (_, { log }) => { + this.callback({ + type: MAIN_SIGNATURE_SUBMISSION, + source: { id: 'TEST-ID' }, + data: { signatureData: signatures[0], logId: log.id }, + }); + this.callback({ + type: MAIN_SIGNATURE_SUBMISSION, + source: { id: 'TEST-ID' }, + data: { signatureData: signatures[0], logId: log.id }, + }); + }, + }, + }; + + const federator = createFederator(mockP2p, [wallets[0].address, wallets[1].address, wallets[2].address]); + + await expect(federator._requestSignatureFromFederators(logs[0])).rejects.toEqual( + "Didn't get enough signatures before timeout" + ); + }); + + it('should not use different signature from same federator twice', async () => { + const signatureWithDifferentDL = await createSignature( + wallets[0], + signatures[0].deadline + 1 + ); + const mockP2p = { + net: { + onMessage: (func) => { + this.callback = func; + return { unsubscribe: () => {} }; + }, + broadcast: (_, { log }) => { + this.callback({ + type: MAIN_SIGNATURE_SUBMISSION, + source: { id: 'TEST-ID' }, + data: { signatureData: signatures[0], logId: log.id }, + }); + this.callback({ + type: MAIN_SIGNATURE_SUBMISSION, + source: { id: 'TEST-ID' }, + data: { signatureData: signatureWithDifferentDL, logId: log.id }, + }); + this.callback({ + type: MAIN_SIGNATURE_SUBMISSION, + source: { id: 'TEST-ID' }, + data: { signatureData: signatures[2], logId: log.id }, + }); + }, + }, + }; + const federator = new Federator(MAIN_FEDERATOR, testConfig, logger, mockP2p, web3Mock); federator.members = [wallets[0].address, wallets[1].address, wallets[2].address]; const res = await federator._requestSignatureFromFederators(logs[0]); expect(res).toHaveLength(2); expect(res[0]).toBe(signatures[0]); - expect(res[1]).toBe(signatures[1]); + // If this is signatureWithDifferentDL, it has returned the wrong signature. + expect(res[1]).toBe(signatures[2]); }); it("should not use signature if doesn't match log id", async () => { @@ -677,8 +784,7 @@ describe('Federator module tests', () => { }, }; - const federator = new Federator(MAIN_FEDERATOR, testConfig, logger, mockP2p, web3Mock); - federator.members = [wallets[0].address, wallets[1].address, wallets[2].address]; + const federator = createFederator(mockP2p, [wallets[0].address, wallets[1].address, wallets[2].address]); const res = await federator._requestSignatureFromFederators(logs[0]); expect(res).toHaveLength(2); @@ -686,6 +792,133 @@ describe('Federator module tests', () => { expect(res[1]).toBe(signatures[2]); }); + it("should not use signature if the deadline does not match", async () => { + const mockP2p = { + net: { + onMessage: (func) => { + this.callback = func; + return { unsubscribe: () => {} }; + }, + broadcast: (_, { log }) => { + this.callback({ + type: MAIN_SIGNATURE_SUBMISSION, + source: { id: 'TEST-ID' }, + data: { signatureData: signatures[0], logId: log.id }, + }); + this.callback({ + type: MAIN_SIGNATURE_SUBMISSION, + source: { id: 'TEST-ID' }, + data: { + signatureData: { + ...signatures[1], + deadline: signatures[1].deadline + 1, + }, + logId: log.id + }, + }); + this.callback({ + type: MAIN_SIGNATURE_SUBMISSION, + source: { id: 'TEST-ID' }, + data: { signatureData: signatures[2], logId: log.id }, + }); + }, + }, + }; + + const federator = createFederator(mockP2p, [wallets[0].address, wallets[1].address, wallets[2].address]); + + const res = await federator._requestSignatureFromFederators(logs[0]); + expect(res).toHaveLength(2); + expect(res[0]).toBe(signatures[0]); + expect(res[1]).toBe(signatures[2]); + }); + + it("should not use signature if the deadline is late", async () => { + const signatureWithLateDL = await createSignature( + wallets[1], + utils.createTimestamp(-1), + ); + const mockP2p = { + net: { + onMessage: (func) => { + this.callback = func; + return { unsubscribe: () => {} }; + }, + broadcast: (_, { log }) => { + this.callback({ + type: MAIN_SIGNATURE_SUBMISSION, + source: { id: 'TEST-ID' }, + data: { signatureData: signatures[0], logId: log.id }, + }); + this.callback({ + type: MAIN_SIGNATURE_SUBMISSION, + source: { id: 'TEST-ID' }, + data: { + signatureData: signatureWithLateDL, + logId: log.id + }, + }); + this.callback({ + type: MAIN_SIGNATURE_SUBMISSION, + source: { id: 'TEST-ID' }, + data: { signatureData: signatures[2], logId: log.id }, + }); + }, + }, + }; + + const federator = createFederator(mockP2p, [wallets[0].address, wallets[1].address, wallets[2].address]); + + const res = await federator._requestSignatureFromFederators(logs[0]); + expect(res).toHaveLength(2); + expect(res[0]).toBe(signatures[0]); + // if res[1] is signatureWithLateDL, the test fails + expect(res[1]).toBe(signatures[2]); + }); + + it("should not use signature if the deadline is too close", async () => { + const signatureWithTooCloseDL = await createSignature( + wallets[1], + utils.createTimestamp(59), // it should require 60s buffer because signaturesTTL is 120 + ); + const mockP2p = { + net: { + onMessage: (func) => { + this.callback = func; + return { unsubscribe: () => {} }; + }, + broadcast: (_, { log }) => { + this.callback({ + type: MAIN_SIGNATURE_SUBMISSION, + source: { id: 'TEST-ID' }, + data: { signatureData: signatures[0], logId: log.id }, + }); + this.callback({ + type: MAIN_SIGNATURE_SUBMISSION, + source: { id: 'TEST-ID' }, + data: { + signatureData: signatureWithTooCloseDL, + logId: log.id + }, + }); + this.callback({ + type: MAIN_SIGNATURE_SUBMISSION, + source: { id: 'TEST-ID' }, + data: { signatureData: signatures[2], logId: log.id }, + }); + }, + }, + }; + + const federator = createFederator(mockP2p, [wallets[0].address, wallets[1].address, wallets[2].address]); + + const res = await federator._requestSignatureFromFederators(logs[0]); + expect(res).toHaveLength(2); + expect(res[0]).toBe(signatures[0]); + // if res[1] is signatureWithLateDL, the test fails + expect(res[1]).toBe(signatures[2]); + }); + it('should not use signature if not from federator', async () => { const mockP2p = { net: { @@ -713,8 +946,7 @@ describe('Federator module tests', () => { }, }; - const federator = new Federator(MAIN_FEDERATOR, testConfig, logger, mockP2p, web3Mock); - federator.members = [wallets[0].address, wallets[2].address]; + const federator = createFederator(mockP2p, [wallets[0].address, wallets[2].address]); const res = await federator._requestSignatureFromFederators(logs[0]); expect(res).toHaveLength(2); @@ -723,7 +955,7 @@ describe('Federator module tests', () => { }); it('should sign message', async () => { - const federator = new Federator(MAIN_FEDERATOR, testConfig, logger, {}, web3Mock); + const federator = createFederator(); federator.mainBridgeContract.getPastEvents = () => [logs[0]]; const { signatureData, logId } = await federator.signTransaction({ @@ -746,7 +978,7 @@ describe('Federator module tests', () => { }); it('should not sign message if no matching event fetched', async () => { - const federator = new Federator(MAIN_FEDERATOR, testConfig, logger, {}, web3Mock); + const federator = createFederator(); federator.mainBridgeContract.getPastEvents = () => []; await expect(federator.signTransaction({ blockNumber: 4, id: 123 })).rejects.toThrow( @@ -755,7 +987,7 @@ describe('Federator module tests', () => { }); it('should not sign message if multiple matching events fetched', async () => { - const federator = new Federator(MAIN_FEDERATOR, testConfig, logger, {}, web3Mock); + const federator = createFederator(); federator.mainBridgeContract.getPastEvents = () => logs; await expect(federator.signTransaction({ blockNumber: 4, id: 123 })).rejects.toThrow( @@ -764,7 +996,7 @@ describe('Federator module tests', () => { }); it('should not sign message if blocknumber not confirmed', async () => { - const federator = new Federator(MAIN_FEDERATOR, testConfig, logger, {}, web3Mock); + const federator = createFederator(); federator.mainBridgeContract.getPastEvents = () => [logs[0]]; const currentBlock = await federator._getCurrentBlockNumber(); diff --git a/sovryn-token-bridge/federator/test/config.json b/sovryn-token-bridge/federator/test/config.json index 474527b9..4ec69f44 100644 --- a/sovryn-token-bridge/federator/test/config.json +++ b/sovryn-token-bridge/federator/test/config.json @@ -21,6 +21,7 @@ "port": 30303, "runEvery": 1, "signaturesTTL": 120, + "signatureRequestTimeoutMs": 100, "sleepOnGas": 2, "maxSleepOnGas": 3, "confirmations": 0, diff --git a/start.sh b/start.sh index 9ed9b29e..c8e77f6d 100755 --- a/start.sh +++ b/start.sh @@ -60,8 +60,12 @@ if [ "$FED_ENV" = "rinkeby-ETH-RSK" ]; then echo $(cat /home/ubuntu/Bridge-SC/federator-env/$FED_ENV/rinkeby.json | jq --arg args "$ETHER_RPC" '."host"=$args') > /home/ubuntu/Bridge-SC/federator-env/$FED_ENV/rinkeby.json fi +echo "Enter Federator public address:" +read FED_ADDRESS -echo "starting federator on $ED_ENV.. this should take 30 sec, please wait" +sed -i 's/federatorAddress_replace_this/'"$FED_ADDRESS"'/g' /home/ubuntu/Bridge-SC/federator-env/$FED_ENV/config.js + +echo "starting federator on $FED_ENV.. this should take 90 sec, please wait" mkdir -p /home/ubuntu/Bridge-SC/federator-env/$FED_ENV/db echo "createing db folder.." echo "getting fed secret:" @@ -72,9 +76,9 @@ echo using key named: $FED_KEY_NAME cat << EOF > /home/ubuntu/Bridge-SC/federator-env/$FED_ENV/federator.key $FED_KEY EOF -echo "starting federator please wait..." +echo "starting federator please wait... this takes" nohup 2>&1 docker-compose -f docker-compose-prod.yml up > federator.log & -sleep 30 +sleep 90 echo "federator logs: /home/ubuntu/Bridge-SC/federator.log" rm -rf /home/ubuntu/Bridge-SC/federator-env/$FED_ENV/federator.key tail -f /home/ubuntu/Bridge-SC/federator.log \ No newline at end of file