diff --git a/docs/flashbots-data/dashboard.md b/docs/flashbots-data/dashboard.md deleted file mode 100644 index bd16dddfa..000000000 --- a/docs/flashbots-data/dashboard.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -title: Flashbots Transparency Dashboard ---- - -The [Flashbots Transparency Dashboard](https://transparency.flashbots.net/) is a public dashboard with metrics on Realised Extractable Value (REV) on Ethereum, as well as data related to the Flashbots MEV-Boost Relay. The dashboard includes data from the Ethereum mainnet since block #15537940 on September 15th 2022 and has a freshness delay of 1 week. Pre-merge MEV activity can be explored here: [MEV-Explore](https://explore.flashbots.net/). - -The Transparency Dashboard is a key component of our commitment to provide transparency on the wider MEV ecosystem and our products. It serves as a valuable resource for contributors and collaborators by providing data for research. Methodology and more information can be found in the [forum post](https://collective.flashbots.net/t/will-the-real-mev-please-stand-up/1686) introducing the dashboard. diff --git a/docs/flashbots-data/mev-inspect-py/data/arbitrages.md b/docs/flashbots-data/mev-inspect-py/data/arbitrages.md deleted file mode 100644 index 69c5890bb..000000000 --- a/docs/flashbots-data/mev-inspect-py/data/arbitrages.md +++ /dev/null @@ -1,28 +0,0 @@ ---- -title: arbitrages ---- -### description - -an **arbitrage** is an circuit of swaps - -arbitrages are joined to their corresponding swaps in the **arbitrage_swaps** table - -### fields - -| Column | Type | Nullable | Description | -|----------------------|-----------------------------|----------|-----------------------------------------------------------------------| -| id | character varying(256) | not null | unique id | -| created_at | timestamp without time zone | not null | when the entry was added to the database | -| block_number | numeric | not null | block number | -| transaction_hash | character varying(256) | not null | transaction hash | -| account_address | character varying(256) | not null | address that took the profit of the arb - can be a contract or an EOA | -| profit_token_address | character varying(256) | not null | token that profit was taken in | -| profit_amount | numeric | not null | gross profit - note: this does not account for miner payment | -| start_amount | numeric | not null | starting amount of the profit token | -| end_amount | numeric | not null | end amount of the profit token | - -:::note - -The "miner" nomenclature will be replaced in a future release to accurately reflect PoS Ethereum architecture. - -::: diff --git a/docs/flashbots-data/mev-inspect-py/data/classified_traces.md b/docs/flashbots-data/mev-inspect-py/data/classified_traces.md deleted file mode 100644 index 254d8254f..000000000 --- a/docs/flashbots-data/mev-inspect-py/data/classified_traces.md +++ /dev/null @@ -1,32 +0,0 @@ ---- -title: classified_traces ---- -### description - -a **classified trace** represents a single trace - -classified traces optionally include: -- a classification for the call (ex: transfer, swap, liquidation) -- decoded function name and inputs - -### fields - -| Column | Type | Nullable | Description | -|--------------------|-----------------------------|----------|-------------------------------------------------------------------------------------------| -| classified_at | timestamp without time zone | not null | when this trace was added to the database | -| block_number | numeric | not null | block number | -| transaction_hash | character varying(66) | not null | transaction hash | -| trace_address | integer[] | not null | trace address | -| trace_type | character varying(256) | not null | the type of call in the trace - one of `call`, `create`, `delegate_call`, `reward`, `suicide` | -| classification | character varying(256) | not null | classification for the trace - one of `unknown`, `swap`, `burn`, `transfer`, `liquidate` | -| protocol | character varying(256) | | the protocol associated with the trace | -| abi_name | character varying(1024) | | the ABI used to decode this trace | -| function_name | character varying(2048) | | the name of the function called | -| function_signature | character varying(2048) | | the signature of the function called | -| inputs | json | | inputs of the function | -| from_address | character varying(256) | | from address | -| to_address | character varying(256) | | to address | -| gas | numeric | | gas | -| value | numeric | | ETH value | -| gas_used | numeric | | gas used | -| error | character varying(256) | | error | diff --git a/docs/flashbots-data/mev-inspect-py/data/miner_payments.md b/docs/flashbots-data/mev-inspect-py/data/miner_payments.md deleted file mode 100644 index 590204b8e..000000000 --- a/docs/flashbots-data/mev-inspect-py/data/miner_payments.md +++ /dev/null @@ -1,31 +0,0 @@ ---- -title: miner_payments ---- -### description - -a **miner payment** is how much was paid to a miner in a given transaction - -miner payment includes payment through gas and coinbase transfers - -### fields - -| Column | Type | Nullable | Description | -|----------------------------------|-----------------------------|----------|--------------------------------------------------------| -| created_at | timestamp without time zone | not null | when the entry was added to the database | -| block_number | numeric | not null | block number | -| transaction_hash | character varying(66) | not null | transaction hash | -| transaction_index | numeric | not null | transaction index | -| miner_address | character varying(256) | not null | address of the miner | -| coinbase_transfer | numeric | not null | amount of ETH was paid as direct transfer to the miner | -| base_fee_per_gas | numeric | not null | base fee for this block | -| gas_price | numeric | not null | gas price (excludes coinbase transfer) | -| gas_price_with_coinbase_transfer | numeric | not null | gas price (includes coinbase transfer) | -| gas_used | numeric | not null | total gas used by the transaction | -| transaction_to_address | character varying(256) | | to address of the transaction | -| transaction_from_address | character varying(256) | | from address of the transaction | - -:::note - -The "miner" nomenclature will be replaced in a future release to accurately reflect PoS Ethereum architecture. - -::: diff --git a/docs/flashbots-data/mev-inspect-py/data/swaps.md b/docs/flashbots-data/mev-inspect-py/data/swaps.md deleted file mode 100644 index 2d6082ec2..000000000 --- a/docs/flashbots-data/mev-inspect-py/data/swaps.md +++ /dev/null @@ -1,24 +0,0 @@ ---- -title: swaps ---- -### description - -a **swap** is a swap between two ERC-20 tokens - -### fields -| Column | Type | Nullable | Description | -|-------------------|-----------------------------|----------|-------------------------------------------| -| created_at | timestamp without time zone | not null | when the entry was added to the database | -| block_number | numeric | not null | block number | -| transaction_hash | character varying(66) | not null | transaction hash | -| trace_address | integer[] | not null | trace address | -| abi_name | character varying(1024) | not null | name of the ABI used to decode the swap | -| pool_address | character varying(256) | not null | pool address | -| from_address | character varying(256) | not null | address where tokens are coming from | -| to_address | character varying(256) | not null | address where swapped tokens are going to | -| token_in_address | character varying(256) | not null | address of the token going in | -| token_in_amount | numeric | not null | amount of the token going in | -| token_out_address | character varying(256) | not null | address of the token going out | -| token_out_amount | numeric | not null | amount of the token going out | -| protocol | character varying(256) | | protocol | -| error | character varying(256) | | error | diff --git a/docs/flashbots-data/mev-inspect-py/data/transfers.md b/docs/flashbots-data/mev-inspect-py/data/transfers.md deleted file mode 100644 index 8e8786c8a..000000000 --- a/docs/flashbots-data/mev-inspect-py/data/transfers.md +++ /dev/null @@ -1,21 +0,0 @@ ---- -title: transfers ---- -### description - -a **transfer** is an ERC-20 token transfer - -### fields - -| Column | Type | Nullable | Description | -|------------------|-----------------------------|----------|---------------------------------------------| -| created_at | timestamp without time zone | not null | when the transfer was added to the database | -| block_number | numeric | not null | block number | -| transaction_hash | character varying(66) | not null | transaction hash | -| trace_address | character varying(256) | not null | trace address | -| from_address | character varying(256) | not null | who the tokens are transferring from | -| to_address | character varying(256) | not null | who the tokens are transferring to | -| token_address | character varying(256) | not null | token address | -| amount | numeric | not null | amount | -| protocol | character varying(256) | | protocol associated with the transfer | -| error | character varying(256) | | error | diff --git a/docs/flashbots-data/mev-inspect-py/exploring.md b/docs/flashbots-data/mev-inspect-py/exploring.md deleted file mode 100644 index ab638ef41..000000000 --- a/docs/flashbots-data/mev-inspect-py/exploring.md +++ /dev/null @@ -1,37 +0,0 @@ ---- -title: Exploring ---- - -All inspect output data is stored in Postgres. - -To connect to the local Postgres database for querying, launch a client container with: - -```sh -./mev db -``` - -When you see the prompt - -```sh -mev_inspect=# -``` - -You're ready to query! - -Try finding the total number of swaps decoded with UniswapV3Pool - -```sql -SELECT COUNT(*) FROM swaps WHERE abi_name='UniswapV3Pool'; -``` - -or top 10 arbs by gross profit that took profit in WETH - -```sql -SELECT * -FROM arbitrages -WHERE profit_token_address = '0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2' -ORDER BY profit_amount DESC -LIMIT 10; -``` - -Postgres tip: Enter `\x` to enter "Expanded display" mode which looks nicer for results with many columns diff --git a/docs/flashbots-data/mev-inspect-py/inspecting.md b/docs/flashbots-data/mev-inspect-py/inspecting.md deleted file mode 100644 index 041083f95..000000000 --- a/docs/flashbots-data/mev-inspect-py/inspecting.md +++ /dev/null @@ -1,75 +0,0 @@ ---- -title: Inspecting ---- - -Executing an inspection for a block will perform the following operations: - -- Retrieve traces, receipts, and block data from the RPC endpoint. -- Decode the traces using recognized ABIs. -- Extract structured objects such as transfers and swaps. -- Store all these elements in the database for future querying. - -## Examples - -### Inspect a single block - -Inspecting block [12914944](https://twitter.com/mevalphaleak/status/1420416437575901185): - -```sh -./mev inspect 12914944 -``` - -### Inspect many blocks - -Inspecting blocks 12914944 to 12914954: - -```sh -./mev inspect-many 12914944 12914954 -``` - -### Inspect all incoming blocks - -Start a block listener with: - -```sh -./mev listener start -``` - -By default, it will pick up wherever you left off. -If running for the first time, listener starts at the latest block. - -Tail logs for the listener with: - -```sh -./mev listener tail -``` - -And stop the listener with: - -```sh -./mev listener stop -``` - -### Backfilling - -For larger backfills, you can inspect many blocks in parallel using kubernetes - -To inspect blocks 12914944 to 12915044 divided across 10 worker pods: - -```sh -./mev backfill 12914944 12915044 10 -``` - -You can see worker pods spin up then complete by watching the status of all pods - -```sh -watch kubectl get pods -``` - -To watch the logs for a given pod, take its pod name using the above, then run: - -```sh -kubectl logs -f pod/mev-inspect-backfill-abcdefg -``` - -(where `mev-inspect-backfill-abcdefg` is your actual pod name) diff --git a/docs/flashbots-data/mev-inspect-py/overview.md b/docs/flashbots-data/mev-inspect-py/overview.md deleted file mode 100644 index 784bd6ce5..000000000 --- a/docs/flashbots-data/mev-inspect-py/overview.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -title: Overview ---- - -**mev-inspect-py** is an MEV inspector for Ethereum - -Given a block, mev-inspect finds: - -- Validator payments (gas + coinbase) -- Tokens transfers and profit -- Swaps and [arbitrages](https://twitter.com/bertcmiller/status/1427632028263059462) -- ...and more - -All data is stored in Postgres for querying and analysis - -To get started, check out the [quick start](/flashbots-data/mev-inspect-py/quick-start.md) - -To see which data inspect produces, check out the [data](/flashbots-data/mev-inspect-py/data/classified_traces.md) section - -Access and contribute to the public mev-inspect-py repository [here](https://github.com/flashbots/mev-inspect-py). diff --git a/docs/flashbots-data/mev-inspect-py/quick-start.md b/docs/flashbots-data/mev-inspect-py/quick-start.md deleted file mode 100644 index c75e74d65..000000000 --- a/docs/flashbots-data/mev-inspect-py/quick-start.md +++ /dev/null @@ -1,208 +0,0 @@ ---- -title: Quick Start ---- - -Let's use mev-inspect to find the same arbitrage as [MEV Alpha Leak](https://twitter.com/mevalphaleak/status/1420416437575901185) - -## Install - -mev-inspect-py is built to run on kubernetes locally and in production - -### Install dependencies - -First, setup a local kubernetes deployment - we use [Docker](https://www.docker.com/products/docker-desktop) and [kind](https://kind.sigs.k8s.io/docs/user/quick-start) - -If using kind, create a new cluster with: - -```sh -kind create cluster -``` - -Next, install the kubernetes CLI [`kubectl`](https://kubernetes.io/docs/tasks/tools/) - -Then, install [helm](https://helm.sh/docs/intro/install/) - helm is a package manager for kubernetes - -Lastly, setup [Tilt](https://docs.tilt.dev/install.html) which manages running and updating kubernetes resources locally - -### Start up - -Set an environment variable `RPC_URL` to an RPC for fetching blocks -Example: - -```sh -export RPC_URL="http://111.111.111.111:8546" -``` - -:::note - -mev-inspect-py currently requires and RPC with support for OpenEthereum / Erigon traces (not geth 😔) - -::: - -Next, start all servcies with: - -```sh -tilt up -``` - -Press "space" to see a browser of the services starting up - -On first startup, you'll need to apply database migrations. Apply with: - -```sh -./mev exec alembic upgrade head -``` - -### Tear down - -First stop the running tilt window with `Ctrl+C` - -Then run - -```sh -tilt down -``` - -## Inspect a block - -Using the [linked etherscan transaction](https://etherscan.io/tx/0xfcf4558f6432689ea57737fe63124a5ec39fd6ba6aaf198df13a825dd599bffc), we can see the block number is 12914944. - -To inspect this block, run - -```sh -./mev inspect 12914944 -``` - -## Connect to Postgres - -We'll connect to the Postgres database to see the data inspect found in that block - -Let's start up a client container connected to the DB: - -```sh -./mev db -``` - -When you see the prompt - -```sh -mev_inspect=# -``` - -You're ready to query! - -To make the data display nice, switch into "Expanded display" mode by running - -```sh -\x -``` - -## Query for arbitrage data - -Let's find that arbitrage by querying the `arbitrages` table: - -```sql -SELECT * -FROM arbitrages -WHERE - block_number = 12914944 AND - transaction_hash = '0xfcf4558f6432689ea57737fe63124a5ec39fd6ba6aaf198df13a825dd599bffc' -``` - -You should see output like this: - -```txt -id | ff2deb13-c2c1-4ef5-a6ff-0ca813a07d6b -created_at | 2021-09-27 15:26:58.193263 -account_address | 0x0000fee6275dab194ab538a01dd8b18b02b20000 -profit_token_address | 0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2 -block_number | 12914944 -transaction_hash | 0xfcf4558f6432689ea57737fe63124a5ec39fd6ba6aaf198df13a825dd599bffc -start_amount | 70287643212620210176 -end_amount | 123848351154563483804 -profit_amount | 53560707941943273628 -``` - -We can see this matches the original tweet description! - -The `profit_token_address` is the address for WETH, our `start_amount` is 70 WETH (assuming 18 decimals), and our `end_amount` is 123 WETH - -## Query for arbitrage swaps - -We can learn about the swaps involed in this arbitrage by joining against the `arbitrage_swaps` and `swaps` tables - -:::note -You'll need to switch in the id you got in the first query for arbitrage_id -::: - -```sql -SELECT s.* -FROM swaps s -JOIN arbitrage_swaps arb_swaps ON - s.transaction_hash = arb_swaps.swap_transaction_hash AND - s.trace_address = arb_swaps.swap_trace_address -WHERE arb_swaps.arbitrage_id = 'ff2deb13-c2c1-4ef5-a6ff-0ca813a07d6b'; -``` - -You should see output like this: - -```txt --[ RECORD 1 ]-----+------------------------------------------------------------------- -created_at | 2021-09-27 15:26:58.180131 -abi_name | UniswapV3Pool -transaction_hash | 0xfcf4558f6432689ea57737fe63124a5ec39fd6ba6aaf198df13a825dd599bffc -block_number | 12914944 -protocol | -pool_address | 0x88e6a0c2ddd26feeb64f039a2c41296fcb3f5640 -from_address | 0x7ec0b75a98997c927ace6d87958147a105147ea0 -to_address | 0x0000fee6275dab194ab538a01dd8b18b02b20000 -token_in_address | 0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48 -token_in_amount | 283588902010 -token_out_address | 0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2 -token_out_amount | 123848351154563483804 -trace_address | {0,0} -error | --[ RECORD 2 ]-----+------------------------------------------------------------------- -... -``` - -## Query for miner payment - -Lastly, we can see how much was paid to the miner for this transaction by querying by the transaction hash: - -```sql -SELECT * -FROM miner_payments -WHERE transaction_hash = '0xfcf4558f6432689ea57737fe63124a5ec39fd6ba6aaf198df13a825dd599bffc'; -``` - -You should see results like this: - -```txt -created_at | 2021-09-27 15:26:58.245444 -block_number | 12914944 -transaction_hash | 0xfcf4558f6432689ea57737fe63124a5ec39fd6ba6aaf198df13a825dd599bffc -transaction_index | 1 -miner_address | 0x5A0b54D5dc17e0AadC383d2db43B0a0D3E029c4c -coinbase_transfer | 48204637147748941824 -base_fee_per_gas | 0 -gas_price | 0 -gas_price_with_coinbase_transfer | 200463421638605 -gas_used | 240466 -transaction_to_address | 0x00000000454a11ca3a574738c0aab442b62d5d45 -transaction_from_address | 0xd80276cd0348e9b3c5d017e1f7529f0a785fec3a -``` - -`gas_price` is the gas price paid directly as gas (it includes the EIP-1559 base fee) - -`coinbase_transfer` is the amount of ETH paid directly as a transfer to the validator's address - -`gas_price_with_coinbase_transfer` is the gas price including both original gas and coinbase transfers - -So in total, this searcher paid 48.2 ETH to make 53.5 ETH for a net profit of 5.3 ETh - -## Next steps - -To see what other data is available for querying, check out the [data](/flashbots-data/mev-inspect-py/data/classified_traces) section - -To learn about inspecting blocks in bulk or listening for new blocks as they come in, check out the [inspecting](/flashbots-data/mev-inspect-py/inspecting) section diff --git a/docs/sidebars.js b/docs/sidebars.js index 3a7fba647..fdf0d28a1 100644 --- a/docs/sidebars.js +++ b/docs/sidebars.js @@ -68,28 +68,6 @@ module.exports = { } ], }, - { - "Flashbots Data": [ - { - 'MEV-Inspect': [ - 'flashbots-data/mev-inspect-py/overview', - 'flashbots-data/mev-inspect-py/quick-start', - 'flashbots-data/mev-inspect-py/inspecting', - 'flashbots-data/mev-inspect-py/exploring', - { - "Data": [ - 'flashbots-data/mev-inspect-py/data/classified_traces', - 'flashbots-data/mev-inspect-py/data/transfers', - 'flashbots-data/mev-inspect-py/data/swaps', - 'flashbots-data/mev-inspect-py/data/arbitrages', - 'flashbots-data/mev-inspect-py/data/miner_payments', - ], - }, - ] - }, - 'flashbots-data/dashboard', - ], - }, { "MEV-Boost": [ 'flashbots-mev-boost/introduction',