leaderboard_scraper.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "leaderboard_scraper.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "X9UggT1-0810",
        "outputId": "384bb59a-b6a5-4ca3-afa8-4d881dd2ee94"
      },
      "source": [
        "!pip install lxml\n",
        "!pip install selenium\n",
        "!pip install cssselect\n",
        "!apt-get update\n",
        "!apt install chromium-chromedriver\n",
        "!cp /usr/lib/chromium-browser/chromedriver /usr/bin"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: lxml in /usr/local/lib/python3.7/dist-packages (4.2.6)\n",
            "Requirement already satisfied: selenium in /usr/local/lib/python3.7/dist-packages (3.141.0)\n",
            "Requirement already satisfied: urllib3 in /usr/local/lib/python3.7/dist-packages (from selenium) (1.24.3)\n",
            "Requirement already satisfied: cssselect in /usr/local/lib/python3.7/dist-packages (1.1.0)\n",
            "Hit:1 http://ppa.launchpad.net/c2d4u.team/c2d4u4.0+/ubuntu bionic InRelease\n",
            "Hit:2 http://archive.ubuntu.com/ubuntu bionic InRelease\n",
            "Hit:3 http://archive.ubuntu.com/ubuntu bionic-updates InRelease\n",
            "Hit:4 http://ppa.launchpad.net/cran/libgit2/ubuntu bionic InRelease\n",
            "Hit:5 http://archive.ubuntu.com/ubuntu bionic-backports InRelease\n",
            "Hit:6 https://cloud.r-project.org/bin/linux/ubuntu bionic-cran40/ InRelease\n",
            "Hit:7 http://ppa.launchpad.net/deadsnakes/ppa/ubuntu bionic InRelease\n",
            "Hit:8 http://ppa.launchpad.net/graphics-drivers/ppa/ubuntu bionic InRelease\n",
            "Ign:9 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  InRelease\n",
            "Hit:10 http://security.ubuntu.com/ubuntu bionic-security InRelease\n",
            "Ign:11 https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64  InRelease\n",
            "Hit:12 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  Release\n",
            "Hit:13 https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64  Release\n",
            "Reading package lists... Done\n",
            "Reading package lists... Done\n",
            "Building dependency tree       \n",
            "Reading state information... Done\n",
            "chromium-chromedriver is already the newest version (91.0.4472.101-0ubuntu0.18.04.1).\n",
            "0 upgraded, 0 newly installed, 0 to remove and 58 not upgraded.\n",
            "cp: '/usr/lib/chromium-browser/chromedriver' and '/usr/bin/chromedriver' are the same file\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "N8W-Kg0c555w"
      },
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "\n",
        "from lxml import html\n",
        "from lxml.cssselect import CSSSelector\n",
        "\n",
        "# Create the web driver\n",
        "from selenium import webdriver\n",
        "from selenium.webdriver.common.by import By\n",
        "from selenium.webdriver.support.ui import WebDriverWait\n",
        "from selenium.webdriver.support import expected_conditions as EC\n",
        "from selenium.common.exceptions import TimeoutException\n",
        "\n",
        "import sys\n",
        "sys.path.insert(0,'/usr/lib/chromium-browser/chromedriver')\n",
        "from selenium import webdriver\n",
        "chrome_options = webdriver.ChromeOptions()\n",
        "chrome_options.add_argument('--headless')\n",
        "chrome_options.add_argument('--no-sandbox')\n",
        "chrome_options.add_argument('--disable-dev-shm-usage')\n",
        "driver = webdriver.Chrome('chromedriver',options=chrome_options)"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eEonYQcE57qP",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ca29e06d-d12c-4a92-afe1-1d4069d39a61"
      },
      "source": [
        "# Load the site on the web driver\n",
        "url = \"https://axie.zone/leaderboard\"\n",
        "driver.get(url)\n",
        "driver.implicitly_wait(10)\n",
        "\n",
        "# List to store links pointing to the player pages\n",
        "players = []\n",
        "\n",
        "try:\n",
        "  # Find the anchor tags in the leaderboard table\n",
        "  table_body_selector = \".leaderboard_table tbody\"\n",
        "  table_body = WebDriverWait(driver, 10).until(EC.presence_of_all_elements_located((By.CSS_SELECTOR, table_body_selector)))\n",
        "  \n",
        "  table_html = table_body[0].get_attribute('innerHTML')\n",
        "  table_tree = html.fromstring(table_html)\n",
        "\n",
        "  table_rows = table_tree.cssselect(\"tr\")\n",
        "\n",
        "  # Extract the href from the anchors found in the table\n",
        "  for row in table_rows:\n",
        "    players.append({\n",
        "        \"url\": \"https://axie.zone/\" + row.cssselect(\"a\")[0].get(\"href\"),\n",
        "        \"winrate\": int(row.cssselect(\"td:nth-child(6)\")[0].text.replace(\"%\", \"\"))\n",
        "    })\n",
        "\n",
        "  print(\"Retrieved\", len(players), \"URLs\")\n",
        "except TimeoutException:\n",
        "  print(\"Element not properly loaded\")\n",
        "\n",
        "print(players)"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Retrieved 100 URLs\n",
            "[{'url': 'https://axie.zone//profile?ron_addr=0x6f08189713d7c88b6f69b671c9ac5f0081cc9ba5', 'winrate': 54}, {'url': 'https://axie.zone//profile?ron_addr=0x7f7cdcb632d1641378086ce89ab4190b61677805', 'winrate': 54}, {'url': 'https://axie.zone//profile?ron_addr=0x833ce13db4f49cae55833b8671eddfe1f9a0a2db', 'winrate': 56}, {'url': 'https://axie.zone//profile?ron_addr=0xdabb232cb403ba23c44c8d593851c47ad74be52f', 'winrate': 49}, {'url': 'https://axie.zone//profile?ron_addr=0xfdaa118b4c3ca2ea10bdc89becda775c081cb7d6', 'winrate': 54}, {'url': 'https://axie.zone//profile?ron_addr=0xebb389d9593b86a964433f16f04cb408df67ad82', 'winrate': 55}, {'url': 'https://axie.zone//profile?ron_addr=0xc5bf891f862dcb690a6d7a5a935f0df6bd6d41ab', 'winrate': 58}, {'url': 'https://axie.zone//profile?ron_addr=0x9fc41da747be454189ed5b2c81755839b6607e80', 'winrate': 55}, {'url': 'https://axie.zone//profile?ron_addr=0xa56a0aae7d6da05d62be9638918e65a18702764c', 'winrate': 53}, {'url': 'https://axie.zone//profile?ron_addr=0x2c23b77b7e85c7f634c6db175153c96918355bab', 'winrate': 59}, {'url': 'https://axie.zone//profile?ron_addr=0xfd9f5b141d616f3fe6de5a06b7863cc10cef3f49', 'winrate': 53}, {'url': 'https://axie.zone//profile?ron_addr=0x333bd3dc062ac4ca316d33a4e5ddb4c693f5f699', 'winrate': 54}, {'url': 'https://axie.zone//profile?ron_addr=0xff6e9826ba25273a17de62a7d9d7a7c741c84fcd', 'winrate': 55}, {'url': 'https://axie.zone//profile?ron_addr=0xf07654297d2b4be58ded0c02966bf59b2ba7732b', 'winrate': 56}, {'url': 'https://axie.zone//profile?ron_addr=0xaf7a25fa490bc007835741459d8ab105594ede9f', 'winrate': 59}, {'url': 'https://axie.zone//profile?ron_addr=0x166e9db7b6702c001b209b3633b1949ec75f7b19', 'winrate': 56}, {'url': 'https://axie.zone//profile?ron_addr=0x33d41dbe6ba46177f2822e7650d708108f509c4f', 'winrate': 52}, {'url': 'https://axie.zone//profile?ron_addr=0xa90c58361c456d1feff8773ddab0f28b0bd09313', 'winrate': 52}, {'url': 'https://axie.zone//profile?ron_addr=0xa98dee8e00a93a36e62c6dbdaa89fab4ac9afb7b', 'winrate': 63}, {'url': 'https://axie.zone//profile?ron_addr=0x941fc0bb9015e9f0e0f2e2dc8481b49745657372', 'winrate': 51}, {'url': 'https://axie.zone//profile?ron_addr=0x531e58fbf300e34e289afbdf8fe3601b1fbbb895', 'winrate': 54}, {'url': 'https://axie.zone//profile?ron_addr=0x88ae0b96f1c66f3c8cf2e83b4022ceea82be3b52', 'winrate': 52}, {'url': 'https://axie.zone//profile?ron_addr=0x6fac1708d64d4e33494d44532ef883f273cf86ee', 'winrate': 53}, {'url': 'https://axie.zone//profile?ron_addr=0x830e72ca48e40b66c0a27b6cc36f3a80ee3fa1b3', 'winrate': 52}, {'url': 'https://axie.zone//profile?ron_addr=0x9678d3d212b1d241acc44c4a24c2058fec17fbce', 'winrate': 52}, {'url': 'https://axie.zone//profile?ron_addr=0x99296a9f9081bb914050a302cc8cd62ed72be3e6', 'winrate': 55}, {'url': 'https://axie.zone//profile?ron_addr=0xae4f5db1eda7c36b591e2c3a1dc7c20142808e19', 'winrate': 55}, {'url': 'https://axie.zone//profile?ron_addr=0x5be29b2a1b9cef7f79c4418ba7e16b10c4513bad', 'winrate': 55}, {'url': 'https://axie.zone//profile?ron_addr=0x2ed53cbe4650883944d3638f20c2d185cdde28d3', 'winrate': 53}, {'url': 'https://axie.zone//profile?ron_addr=0xcdca0236dc8c9781ed6a3c4c96e717074f2b5a51', 'winrate': 56}, {'url': 'https://axie.zone//profile?ron_addr=0x8dcc99daaeb84ac6764753c4a0d5afb291b79418', 'winrate': 59}, {'url': 'https://axie.zone//profile?ron_addr=0xd309ae634b0596f7922a6e66f3837bc5e70cc8de', 'winrate': 52}, {'url': 'https://axie.zone//profile?ron_addr=0xe0e2fd476a98838eebc713fbbfd82a7cff7abb4d', 'winrate': 54}, {'url': 'https://axie.zone//profile?ron_addr=0x8d55fbfd50d86fa7262b38e5686b9b2b5e6b4484', 'winrate': 53}, {'url': 'https://axie.zone//profile?ron_addr=0x850d798dd55da50ea83b60da2ac22c0bc8973c3a', 'winrate': 53}, {'url': 'https://axie.zone//profile?ron_addr=0x8f3cdd6b7c48333f72b3979c502fd6266db529ed', 'winrate': 52}, {'url': 'https://axie.zone//profile?ron_addr=0x2d49b8696534c168539bc8b428c9869a97f7486a', 'winrate': 55}, {'url': 'https://axie.zone//profile?ron_addr=0xf6abd4469be1896eb764d572a7dfa6fe393bd687', 'winrate': 52}, {'url': 'https://axie.zone//profile?ron_addr=0x965a436ffbe4a87636fd511e2d64d620f6ce50c6', 'winrate': 51}, {'url': 'https://axie.zone//profile?ron_addr=0xfbc3f6eb2f8c417a5fe362e431da7e6bee966465', 'winrate': 57}, {'url': 'https://axie.zone//profile?ron_addr=0x7cd377701b6e1f89c194b1fce442bc6fbc424aa3', 'winrate': 54}, {'url': 'https://axie.zone//profile?ron_addr=0xa1fe293c883c9f948ea7d4c3085688a6aae27637', 'winrate': 52}, {'url': 'https://axie.zone//profile?ron_addr=0x967a472c700661d9a70a859b0d9404f0dd1d59c3', 'winrate': 55}, {'url': 'https://axie.zone//profile?ron_addr=0x62b2c9ed21db11e787817c2a0465c3df35c615a9', 'winrate': 57}, {'url': 'https://axie.zone//profile?ron_addr=0xa57aa0418d79a40ba01e8eb0a6eae150bd2e8304', 'winrate': 53}, {'url': 'https://axie.zone//profile?ron_addr=0x70b5340fd9e743cab7d09050ddd82353355fc341', 'winrate': 52}, {'url': 'https://axie.zone//profile?ron_addr=0xc151798ffca08c8c8f95e247949b5ff1941ad5ec', 'winrate': 61}, {'url': 'https://axie.zone//profile?ron_addr=0xb1bfcdefec9e4233c44160371b401a7fc482aee5', 'winrate': 51}, {'url': 'https://axie.zone//profile?ron_addr=0x2833ed75172d0cb1840b804b65ae26d9a1675eda', 'winrate': 53}, {'url': 'https://axie.zone//profile?ron_addr=0x53ff4d423f451bd0793d08b4fe184a9204530ef6', 'winrate': 53}, {'url': 'https://axie.zone//profile?ron_addr=0x549cf67cba55137e05ab9d05a23041b163b81cf8', 'winrate': 54}, {'url': 'https://axie.zone//profile?ron_addr=0xab5162ba89f12b9ef6185f3af445e02c4597ce88', 'winrate': 52}, {'url': 'https://axie.zone//profile?ron_addr=0x03004c4892e3481c3f6fd9134156328e5c731818', 'winrate': 54}, {'url': 'https://axie.zone//profile?ron_addr=0xbe8ecb79228a015162a6dd0b22e186320d69ce0c', 'winrate': 52}, {'url': 'https://axie.zone//profile?ron_addr=0x9277567af37fd96407a04d34897a122e943c1ec4', 'winrate': 55}, {'url': 'https://axie.zone//profile?ron_addr=0x187887a05cab4877398aeda019228e67740f8dc9', 'winrate': 52}, {'url': 'https://axie.zone//profile?ron_addr=0x0fe51484e7a8bfb3b1be4b827f7c623b9a4e9e6f', 'winrate': 52}, {'url': 'https://axie.zone//profile?ron_addr=0x898052925300283aee0fe01568979d88687a2ef5', 'winrate': 53}, {'url': 'https://axie.zone//profile?ron_addr=0x54ae550c21949dacb5550de5f1b8a2b35e0ed987', 'winrate': 58}, {'url': 'https://axie.zone//profile?ron_addr=0x824dc08641704bed606d19f6be911d427cf59669', 'winrate': 56}, {'url': 'https://axie.zone//profile?ron_addr=0x5789d134181b229ec25be63d8eecafd4df765416', 'winrate': 52}, {'url': 'https://axie.zone//profile?ron_addr=0xedb4a8f6a7b9025fe00250a271d76b7a391dbd91', 'winrate': 52}, {'url': 'https://axie.zone//profile?ron_addr=0xdf0efd6173c36fe561b0c1939cf55d6bb9b89da0', 'winrate': 59}, {'url': 'https://axie.zone//profile?ron_addr=0x9ca23f836c8fc5c663e7b23a955afc182a6ca504', 'winrate': 54}, {'url': 'https://axie.zone//profile?ron_addr=0x142148e11e4ee5b7747fc5a4407d85984d5ec19e', 'winrate': 53}, {'url': 'https://axie.zone//profile?ron_addr=0x7ded284464835fb2df25fd310ec32b3dd47a968f', 'winrate': 54}, {'url': 'https://axie.zone//profile?ron_addr=0x17f8882d451ebc7df673f6ca1620cadf8832988f', 'winrate': 52}, {'url': 'https://axie.zone//profile?ron_addr=0xd8ec7508a7681c5450232402fbe5f6bffed75ce0', 'winrate': 52}, {'url': 'https://axie.zone//profile?ron_addr=0x5760382d0c8951d46253e0a2e0854603b3195065', 'winrate': 55}, {'url': 'https://axie.zone//profile?ron_addr=0xb5f51063ddfc8483c05af466e2ab019157798817', 'winrate': 52}, {'url': 'https://axie.zone//profile?ron_addr=0xa1210ae7ff47b2cc10dc36fdead4a00e050ab0ff', 'winrate': 65}, {'url': 'https://axie.zone//profile?ron_addr=0x104bed6ae4df99ff8df0b89c28d95dc3f585408f', 'winrate': 52}, {'url': 'https://axie.zone//profile?ron_addr=0xa1ca8c1f46fecfa8f9fa7945fe2bc9023a3f4638', 'winrate': 51}, {'url': 'https://axie.zone//profile?ron_addr=0x4e2f8fe642f1113a629a86ef7877177bb444a880', 'winrate': 55}, {'url': 'https://axie.zone//profile?ron_addr=0x08cd4334345cda78031969026fa992f6918db3ef', 'winrate': 52}, {'url': 'https://axie.zone//profile?ron_addr=0x111896c3134929d2170ed95b7ef8c98223f7930e', 'winrate': 52}, {'url': 'https://axie.zone//profile?ron_addr=0xa3d66a40f971bbbbabc8d018bffe13cfda9773a9', 'winrate': 52}, {'url': 'https://axie.zone//profile?ron_addr=0x28152c39709f0094aaf901d566cf282bb20dd29f', 'winrate': 52}, {'url': 'https://axie.zone//profile?ron_addr=0x93dafd6c400e608b7d3f65f50e6402b4bcc1e78c', 'winrate': 54}, {'url': 'https://axie.zone//profile?ron_addr=0x7e574900d3ea0d8b3573d94e5dc5d05de535512d', 'winrate': 53}, {'url': 'https://axie.zone//profile?ron_addr=0xfec38c0c9eef518567647c734cf3dec7f4933f61', 'winrate': 52}, {'url': 'https://axie.zone//profile?ron_addr=0x21526c10432bf5d28c46d2e6a04d9a26ea5a31ab', 'winrate': 55}, {'url': 'https://axie.zone//profile?ron_addr=0x4d58a08e2c0a8e2b0d9bd80a1f188a7cc85f7ce7', 'winrate': 61}, {'url': 'https://axie.zone//profile?ron_addr=0xfb6d533ef2c3475219565db040d6ae1c2267cdb8', 'winrate': 51}, {'url': 'https://axie.zone//profile?ron_addr=0x4dde97c94130c996adde66f7102b439503549718', 'winrate': 51}, {'url': 'https://axie.zone//profile?ron_addr=0xb9603b5ae4d10fce072432d99de572c1eb66b43e', 'winrate': 51}, {'url': 'https://axie.zone//profile?ron_addr=0x7744dcbf28e12ac400d316bdd0abe16fd7750269', 'winrate': 52}, {'url': 'https://axie.zone//profile?ron_addr=0x8fd4930834149633d9ac4ae9749149ab06660d8f', 'winrate': 53}, {'url': 'https://axie.zone//profile?ron_addr=0x0a66a61869be0f906d12e2aaa613d02ea53135ea', 'winrate': 52}, {'url': 'https://axie.zone//profile?ron_addr=0xbc6dc449b82eb01d7c9bbe1b2b4e084a558570c0', 'winrate': 53}, {'url': 'https://axie.zone//profile?ron_addr=0x31559ef95c97f8e1d662ad2beba7131b17a5d79c', 'winrate': 55}, {'url': 'https://axie.zone//profile?ron_addr=0x1da0b9e1cfb48a22d5f01e2ffe5bdc1baf90bff1', 'winrate': 54}, {'url': 'https://axie.zone//profile?ron_addr=0x9ebc1d6aa19f24686346d6c0ffc2102bd16e7abe', 'winrate': 52}, {'url': 'https://axie.zone//profile?ron_addr=0x1679206036fa5e870aef29eca5ea00dfb6ff90cf', 'winrate': 53}, {'url': 'https://axie.zone//profile?ron_addr=0xa5e898e17a177216c7b256e4fdfce8c89ed15a55', 'winrate': 51}, {'url': 'https://axie.zone//profile?ron_addr=0xb8da1590bb7fd691ed2273d0c4c144f83652b40d', 'winrate': 55}, {'url': 'https://axie.zone//profile?ron_addr=0xf13976baa8c63f616a3bf90bda0a70113fcb0c43', 'winrate': 52}, {'url': 'https://axie.zone//profile?ron_addr=0xbddd45875e2e54386bf3a90d0196e0daec1ae0bf', 'winrate': 52}, {'url': 'https://axie.zone//profile?ron_addr=0x89ac84f3b487840f996c4c6678d8cfc292e5e7a7', 'winrate': 54}, {'url': 'https://axie.zone//profile?ron_addr=0x6331b3b76834543f46d3cf376263c08ca22d725d', 'winrate': 52}]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ETjQxfer6k8O"
      },
      "source": [
        "import re\n",
        "\n",
        "# Used to extract the Axie parts from the user page\n",
        "def get_parts(players):\n",
        "  # List to store the scraped data\n",
        "  axie_data = []\n",
        "\n",
        "  # List for failed links\n",
        "  failed = []\n",
        "\n",
        "  # Counter that will be used to track the progress of the function\n",
        "  progress = 0\n",
        "\n",
        "  # Regex pattern to parse the parts\n",
        "  p = re.compile('(.*): (.*) (\\[.*\\])')\n",
        "\n",
        "  # Iterate through each of the player in the leaderboards\n",
        "  for player in players:\n",
        "    progress += 1\n",
        "    try:\n",
        "      # Load the player page on the driver\n",
        "      driver.get(player[\"url\"])\n",
        "      driver.implicitly_wait(10)\n",
        "\n",
        "      # Wait till the Axie info container loads\n",
        "      _ = WebDriverWait(driver, 20).until(EC.presence_of_all_elements_located((By.CSS_SELECTOR, \"#last_used_team_container > div\")))\n",
        "\n",
        "      # Get the element that wraps the last used team container\n",
        "      # This can be refactored to include the most used team of the player as well\n",
        "      axie_container = WebDriverWait(driver, 0).until(EC.presence_of_all_elements_located((By.CSS_SELECTOR, \"#last_used_team_container\")))\n",
        "\n",
        "      # Convert the selenium webelement into a LXML tree\n",
        "      axie_html = axie_container[0].get_attribute('innerHTML')\n",
        "      axie_tree = html.fromstring(axie_html)\n",
        "\n",
        "      # Get the Axie containers\n",
        "      info_containers = axie_tree.cssselect(\".search_result_wrapper\")\n",
        "\n",
        "      for info_container in info_containers:\n",
        "        axie_parts = {}\n",
        "\n",
        "        # Get the address of the owner\n",
        "        axie_parts[\"User_Addr\"] = player[\"url\"].replace(\"https://axie.zone/profile?ron_addr=\", \"\")\n",
        "\n",
        "        # Get the Axie class\n",
        "        axie_parts[\"Class\"] = info_container.cssselect(\".search_result\")[0].get('class').replace(\"search_result \", \"\").capitalize()\n",
        "\n",
        "        # Scrapes the parts from the Axie container\n",
        "        parts = info_container.cssselect(\".purity_part > div\")\n",
        "\n",
        "        for part in parts:\n",
        "          m = p.match(part.get('title'))\n",
        "          axie_parts[m.group(1)] = m.group(2)\n",
        "\n",
        "        # Add winrate\n",
        "        axie_parts[\"Winrate\"] = player[\"winrate\"]\n",
        "\n",
        "        # Add the scraped parts to the main list of data\n",
        "        axie_data.append(axie_parts)\n",
        "\n",
        "      # To keep track and now that our script is working, we would print a message\n",
        "      # containing a summary of the scraping's progress\n",
        "      print(\"(\", progress, \"/\", len(players) ,\") Retrieved\")\n",
        "    except Exception as e:\n",
        "      # We might also encounter an error where our collection script fails,\n",
        "      # in the rare case where this would happen we would need to manually\n",
        "      # revisit what went wrong and fix it.\n",
        "      failed.append(player)\n",
        "      print(\"(\", progress, \"/\", len(players) ,\") Error occured while retrieving\", player[\"url\"])\n",
        "      print(\"Error received:\", e)\n",
        "\n",
        "  if len(failed) != 0:\n",
        "    print(\"Retrying failed links\")\n",
        "    axie_data += get_parts(failed)\n",
        "\n",
        "  return axie_data\n",
        "  \n",
        "# Start the scrapping process\n",
        "res = get_parts(players)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZBqzbQZO-wdv"
      },
      "source": [
        "# Convert the python list into a dataframe\n",
        "df_res = pd.DataFrame(res)\n",
        "df_res.head()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ctCAz1ZdIAQb"
      },
      "source": [
        "# Export the dataframe into a csv file\n",
        "df_res.to_csv(\"leaderboard.csv\", index=False)"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}