From f51404665bba1489c7b25b6bb0223e7d20344325 Mon Sep 17 00:00:00 2001 From: ShadowNinja Date: Sat, 23 Jan 2016 17:42:29 -0500 Subject: [PATCH] Add persistent server storage with MongoDB --- README.md | 29 +++- config-example.py | 33 ++--- requirements.txt | 4 + server.py | 353 ++++++++++++++++++++++++++-------------------- 4 files changed, 240 insertions(+), 179 deletions(-) create mode 100644 requirements.txt diff --git a/README.md b/README.md index 0799ce0..cefc594 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ Setting up the webpage ---------------------- You will have to install node.js, doT.js and their dependencies to compile -the serverlist webpage template. +the server list webpage template. First install node.js, e.g.: @@ -62,10 +62,10 @@ Setting up the server # # OR: # apt-get install python3 python3-pip - 2. Install Flask, and APScheduler: + 2. Install required Python packages: # # You might have to use pip3 if your system defaults to Python 2 - # pip install APScheduler flask + # pip install -r requirements.txt 3. If using in production, install uwsgi and it's python plugin: @@ -75,16 +75,31 @@ Setting up the server # # OR: # pip install uwsgi - 4. Configure the server by changing options in config.py, which is a Flask - configuration file. + 4. Install, start, and enable MongoDB on boot: - 5. Start the server: + # pacman -S mongodb && systemctl enable mongodb --now + + 5. Configure the server by adding options to `config.py`. + See `config-example.py` for defaults. + + 6. Start the server: $ ./server.py $ # Or for production: - $ uwsgi -s /tmp/serverlist.sock --plugin python -w server:app --enable-threads + $ uwsgi -s /tmp/minetest-master.sock --plugin python -w server:app --enable-threads $ # Then configure according to http://flask.pocoo.org/docs/deploying/uwsgi/ + 7. (optional) Configure the proxy server, if any. You should make the server + load static files directly from the static directory. Also, `/list` + should be served from `list.json`. Example for nginx: + + root /path/to/server/static; + rewrite ^/list$ /list.json; + try_files $uri @uwsgi; + location @uwsgi { + uwsgi_pass ...; + } + License ------- diff --git a/config-example.py b/config-example.py index ee4782c..d009f0c 100644 --- a/config-example.py +++ b/config-example.py @@ -1,30 +1,31 @@ +from datetime import timedelta # Enables detailed tracebacks and an interactive Python console on errors. # Never use in production! -#DEBUG = True +DEBUG = False # Address for development server to listen on -#HOST = "127.0.0.1" +HOST = "127.0.0.1" # Port for development server to listen on -#PORT = 5000 +PORT = 5000 -# Makes the server more performant at sending static files when the -# server is being proxied by a server that supports X-Sendfile. -#USE_X_SENDFILE = True +# Amount of time after which servers are removed from the list if they haven't +# updated their listings. Note: By default Minetest servers only announce +# once every 5 minutes, so this should be more than that. +UPDATE_TIME = timedelta(minutes=6) -# File to store the JSON server list data in. -FILENAME = "list.json" - -# Amount of time, is seconds, after which servers are removed from the list -# if they haven't updated their listings. Note: By default Minetest servers -# only announce once every 5 minutes, so this should be more than 300. -PURGE_TIME = 350 - -# List of banned IP addresses -BANLIST = [] +# Amount of time after which servers are removed from the database if they +# haven't updated their listings. +PURGE_TIME = timedelta(days=30) # Creates server entries if a server sends an 'update' and there is no entry yet. # This should only be used to populate the server list after list.json was deleted. # This WILL cause problems such as mapgen, mods and privilege information missing from the list ALLOW_UPDATE_WITHOUT_OLD = False +# Number of days' data to factor into popularity calculation +POP_DAYS = 3 + +# Address of the MongoDB server. You can use domain sockets on unix. +MONGO_URI = "mongodb://localhost/minetest-master" + diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..7926ac8 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +APScheduler>=3 +Flask>=0.10 +Flask-PyMongo>=0.3 + diff --git a/server.py b/server.py index c9f5cdc..ff8e2d9 100755 --- a/server.py +++ b/server.py @@ -1,17 +1,28 @@ #!/usr/bin/env python3 import os, sys, json, time, socket +from datetime import date, datetime, timedelta from threading import Thread, RLock from apscheduler.schedulers.background import BackgroundScheduler from flask import Flask, request, send_from_directory +from flask.ext.pymongo import PyMongo, ASCENDING, DESCENDING +app = Flask(__name__, static_url_path = "") + +# Load configuration +app.config.from_pyfile("config-example.py") # Use example for defaults +if os.path.isfile(os.path.join(app.root_path, "config.py")): + app.config.from_pyfile("config.py") + +# Set up database +mongo = PyMongo(app) +# Set up scheduler sched = BackgroundScheduler(timezone="UTC") sched.start() -app = Flask(__name__, static_url_path = "") -app.config.from_pyfile("config.py") +# Views @app.route("/") def index(): @@ -20,10 +31,10 @@ def index(): @app.route("/list") def list(): - # We have to make sure that the list isn't cached, - # since the list isn't really static. - return send_from_directory(app.static_folder, app.config["FILENAME"], - cache_timeout=0) + # Make sure that the list isn't cached too long + return send_from_directory(app.static_folder, "list.json", + cache_timeout=0 if app.config["DEBUG"] else + app.config["UPDATE_TIME"].total_seconds() / 2) @app.route("/announce", methods=["GET", "POST"]) @@ -32,10 +43,7 @@ def announce(): if ip.startswith("::ffff:"): ip = ip[7:] - if ip in app.config["BANLIST"]: - return "Banned.", 403 - - data = request.form["json"] if request.method == "POST" else request.args["json"] + data = request.values["json"] if len(data) > 5000: return "JSON data is too big.", 413 @@ -51,8 +59,9 @@ def announce(): if not "action" in server: return "Missing action field.", 400 - if server["action"] == "start": - server["uptime"] = 0 + action = server["action"] + if action not in ("start", "update", "delete"): + return "Invalid action field.", 400 server["ip"] = ip @@ -64,9 +73,9 @@ def announce(): server["port"] = int(server["port"]) #### End compatability code #### - old = serverList.get(server["ip"], server["port"]) + old = serverList.get(ip, server["port"]) - if server["action"] == "delete": + if action == "delete": if not old: return "Server not found.", 500 serverList.remove(old) @@ -75,7 +84,7 @@ def announce(): elif not checkRequest(server): return "Invalid JSON data.", 400 - if server["action"] != "start" and not old: + if action == "update" and not old: if app.config["ALLOW_UPDATE_WITHOUT_OLD"]: old = server old["start"] = time.time() @@ -85,9 +94,10 @@ def announce(): else: return "Server to update not found.", 500 - server["update_time"] = time.time() + server["last_update"] = datetime.utcnow() - server["start"] = time.time() if server["action"] == "start" else old["start"] + del server["uptime"] + server["start"] = time.time() if action == "start" else old["start"] if "clients_list" in server: server["clients"] = len(server["clients_list"]) @@ -95,32 +105,57 @@ def announce(): server["clients_top"] = max(server["clients"], old["clients_top"]) if old else server["clients"] # Make sure that startup options are saved - if server["action"] != "start": + if action == "update": for field in ("dedicated", "rollback", "mapgen", "privs", "can_see_far_names", "mods"): if field in old: server[field] = old[field] - # Popularity - if old: - server["updates"] = old["updates"] + 1 - # This is actually a count of all the client numbers we've received, - # it includes clients that were on in the previous update. - server["total_clients"] = old["total_clients"] + server["clients"] + ## Popularity + # This should only include stats from the past few days, so storing + # the necessary data is fairly complicated. Three pieces of + # information are required: the time the server's been connected + # (number of updates), the number of clients reported, and a + # timestamp so that this information can expire. This is stored in + # the format {date: (updates, clients)}. + total_updates = 1 + # This is the total of all the client numbers we've received, + # it includes clients that were on in previous updates. + total_clients = server["clients"] + today = date.today() + # Key must be a string + today_key = str(today.toordinal()) + if old and type(old["updates"]) == dict: + server["updates"] = {} + updates = server["updates"] + # Copy over only recent updates, and set the total counter + pop_days = app.config["POP_DAYS"] + for d, data in old["updates"].items(): + if date.fromordinal(int(d)) >= \ + today - timedelta(days=pop_days): + updates[d] = data + total_updates += data[0] + total_clients += data[1] + + if today_key in updates: + updates[today_key][0] += 1 + updates[today_key][1] += server["clients"] + else: + updates[today_key] = (1, server["clients"]) else: - server["updates"] = 1 - server["total_clients"] = server["clients"] - server["pop_v"] = server["total_clients"] / server["updates"] + server["updates"] = {today_key: (1, server["clients"])} + server["pop_v"] = total_clients / total_updates + + # Keep server record ID + if old: + server["_id"] = old["_id"] finishRequestAsync(server) return "Thanks, your request has been filed.", 202 -def purgeOld(): - serverList.purgeOld() - -sched.add_job(purgeOld, "interval", seconds=60, coalesce=True, max_instances=1) +# Utilities # Returns ping time in seconds (up), False (down), or None (error). def serverUp(info): @@ -155,7 +190,6 @@ def serverUp(info): "clients": (True, "int"), "clients_max": (True, "int"), - "uptime": (True, "int"), "game_time": (True, "int"), "lag": (False, "float"), @@ -193,7 +227,7 @@ def checkRequest(server): # Accept strings in boolean fields but convert it to a # boolean, because old servers sent some booleans as strings. if data[1] == "bool" and type(server[name]).__name__ == "str": - server[name] = True if server[name].lower() in ["true", "1"] else False + server[name] = True if server[name].lower() in ("true", "1") else False continue # clients_max was sent as a string instead of an integer if name == "clients_max" and type(server[name]).__name__ == "str": @@ -206,6 +240,8 @@ def checkRequest(server): for item in server[name]: if type(item).__name__ != data[2]: return False + if "_id" in server: + return False return True @@ -247,140 +283,145 @@ def asyncFinishThread(server): del server["action"] - serverList.update(server) + with app.app_context(): + serverList.updateServer(server) class ServerList: def __init__(self): - self.list = [] - self.maxServers = 0 - self.maxClients = 0 - self.lock = RLock() - self.load() - self.purgeOld() - - def getWithIndex(self, ip, port): - with self.lock: - for i, server in enumerate(self.list): - if server["ip"] == ip and server["port"] == port: - return (i, server) - return (None, None) + self.last_cache_update = 0 + with app.app_context(): + mongo.db.meta.create_index("key", unique=True, name="key") + self.info = mongo.db.meta.find_one({"key": "list_info"}) or {} + + mongo.db.servers.create_index([("ip", ASCENDING), ("port", ASCENDING)], + unique=True, + name="address") + mongo.db.servers.create_index("points", name="points") + mongo.db.servers.create_index("last_update", + expireAfterSeconds=app.config["PURGE_TIME"].total_seconds(), + name="expiration") + + if "max_servers" not in self.info: + self.info["max_servers"] = 0 + if "max_clients" not in self.info: + self.info["max_clients"] = 0 + + def __del__(self): + with app.app_context(): + self.updateMeta() + + def getPoints(self, server): + points = 0 + + # 1 per client without a guest or all-numeric name. + if "clients_list" in server: + for name in server["clients_list"]: + if not name.startswith("Guest") and \ + not name.isdigit(): + points += 1 + else: + # Old server + points = server["clients"] / 4 + + # Penalize highly loaded servers to improve player distribution. + # Note: This doesn't just make more than 16 players stop + # increasing your points, it can actually reduce your points + # if you have guests/all-numerics. + if server["clients"] > 16: + points -= server["clients"] - 16 + + # 1 per month of age, limited to 8 + points += min(8, server["game_time"] / (60*60*24*30)) + + # 1/2 per average client, limited to 4 + points += min(4, server["pop_v"] / 2) + + # -8 for unrealistic max_clients + if server["clients_max"] >= 128: + points -= 8 + + # -8 per second of ping over 0.4s + if server["ping"] > 0.4: + points -= (server["ping"] - 0.4) * 8 + + # Up to -8 for less than an hour of uptime (penalty linearly decreasing) + HOUR_SECS = 60 * 60 + uptime = server["uptime"] + if uptime < HOUR_SECS: + points -= ((HOUR_SECS - uptime) / HOUR_SECS) * 8 + + return points + + def updateCache(self): + servers = mongo.db.servers.find({"last_update": + {"$gt": datetime.utcnow() - app.config["UPDATE_TIME"]} + }).sort("points", DESCENDING) + server_list = [] + num_clients = 0 + for server in servers: + del server["_id"] + del server["last_update"] + del server["updates"] + server["uptime"] = time.time() - server["start"] + server_list.append(server) + num_clients += server["clients"] + + info = self.info + info["max_servers"] = max(len(server_list), info["max_servers"]) + info["max_clients"] = max(num_clients, info["max_clients"]) + + with open(os.path.join("static", "list.json"), "w") as fd: + json.dump({ + "total": {"servers": len(server_list), "clients": num_clients}, + "total_max": {"servers": info["max_servers"], "clients": info["max_clients"]}, + "list": server_list + }, + fd, + indent = "\t" if app.config["DEBUG"] else None + ) + + self.updateMeta() + + self.last_cache_update = time.time() + + # Update if servers are likely to have expired + def updateCacheIfNeeded(self): + if time.time() - self.last_cache_update < \ + app.config["UPDATE_TIME"].total_seconds() / 2: + return + with app.app_context(): + self.updateCache() + + def updateMeta(self): + if not "key" in self.info: + self.info["key"] = "list_info" + mongo.db.meta.replace_one({"key": "list_info"}, + self.info, upsert=True) def get(self, ip, port): - i, server = self.getWithIndex(ip, port) - return server + return mongo.db.servers.find_one({"ip": ip, "port": port}) def remove(self, server): - with self.lock: - try: - self.list.remove(server) - except: - pass - - def sort(self): - def server_points(server): - points = 0 - - # 1 per client, but only 1/8 per client with a guest - # or all-numeric name. - if "clients_list" in server: - for name in server["clients_list"]: - if name.startswith("Guest") or \ - name.isdigit(): - points += 1/8 - else: - points += 1 - else: - # Old server - points = server["clients"] / 4 - - # Penalize highly loaded servers to improve player distribution. - # Note: This doesn't just make more than 16 players stop - # increasing your points, it can actually reduce your points - # if you have guests/all-numerics. - if server["clients"] > 16: - points -= server["clients"] - 16 - - # 1 per month of age, limited to 8 - points += min(8, server["game_time"] / (60*60*24*30)) - - # 1/2 per average client, limited to 4 - points += min(4, server["pop_v"] / 2) - - # -8 for unrealistic max_clients - if server["clients_max"] >= 128: - points -= 8 - - # -8 per second of ping over 0.4s - if server["ping"] > 0.4: - points -= (server["ping"] - 0.4) * 8 - - # Up to -8 for less than an hour of uptime (penalty linearly decreasing) - HOUR_SECS = 60 * 60 - uptime = server["uptime"] - if uptime < HOUR_SECS: - points -= ((HOUR_SECS - uptime) / HOUR_SECS) * 8 - - return points - - with self.lock: - self.list.sort(key=server_points, reverse=True) - - def purgeOld(self): - with self.lock: - for server in self.list: - if server["update_time"] < time.time() - app.config["PURGE_TIME"]: - self.list.remove(server) - self.save() - - def load(self): - try: - with open(os.path.join("static", app.config["FILENAME"]), "r") as fd: - data = json.load(fd) - except FileNotFoundError: - return - - if not data: - return - - with self.lock: - self.list = data["list"] - self.maxServers = data["total_max"]["servers"] - self.maxClients = data["total_max"]["clients"] - - def save(self): - with self.lock: - servers = len(self.list) - clients = 0 - for server in self.list: - clients += server["clients"] - - self.maxServers = max(servers, self.maxServers) - self.maxClients = max(clients, self.maxClients) - - with open(os.path.join("static", app.config["FILENAME"]), "w") as fd: - json.dump({ - "total": {"servers": servers, "clients": clients}, - "total_max": {"servers": self.maxServers, "clients": self.maxClients}, - "list": self.list - }, - fd, - indent = "\t" if app.config["DEBUG"] else None - ) - - def update(self, server): - with self.lock: - i, old = self.getWithIndex(server["ip"], server["port"]) - if i is not None: - self.list[i] = server - else: - self.list.append(server) - - self.sort() - self.save() + mongo.db.servers.delete_one({"_id": server["_id"]}) + self.updateCache() + + def updateServer(self, server): + server["points"] = self.getPoints(server) + if "_id" in server: + mongo.db.servers.replace_one({"_id": server["_id"]}, + server) + else: + mongo.db.servers.insert_one(server) + self.updateCache() serverList = ServerList() +sched.add_job(serverList.updateCacheIfNeeded, "interval", + seconds=app.config["UPDATE_TIME"].total_seconds() / 2, coalesce=True, + max_instances=1) + if __name__ == "__main__": + serverList.updateCacheIfNeeded() app.run(host = app.config["HOST"], port = app.config["PORT"])