X-Git-Url: https://git.xonotic.org/?a=blobdiff_plain;f=xonstat%2Fviews%2Fsubmission.py;h=26d35b34400fbea98d7057265c8e69b35d412d06;hb=364dba8255d6343f126613f8435085ff3e07ea67;hp=6792fc75511d946e97fb1a31475a5204231c2a4a;hpb=7bb73ec899cfdff2f1cac0783ab650ebf7365c8c;p=xonotic%2Fxonstat.git diff --git a/xonstat/views/submission.py b/xonstat/views/submission.py index 6792fc7..26d35b3 100644 --- a/xonstat/views/submission.py +++ b/xonstat/views/submission.py @@ -1,94 +1,268 @@ +import calendar +import collections import datetime import logging -import os -import pyramid.httpexceptions import re -import time -import sqlalchemy.sql.expression as expr -from pyramid.response import Response + +import pyramid.httpexceptions from sqlalchemy import Sequence from sqlalchemy.orm.exc import MultipleResultsFound, NoResultFound -from xonstat.elo import process_elos -from xonstat.models import * +from xonstat.elo import EloProcessor +from xonstat.models import DBSession, Server, Map, Game, PlayerGameStat, PlayerWeaponStat +from xonstat.models import PlayerRank, PlayerCaptime +from xonstat.models import TeamGameStat, PlayerGameAnticheat, Player, Hashkey, PlayerNick from xonstat.util import strip_colors, qfont_decode, verify_request, weapon_map - log = logging.getLogger(__name__) -def parse_stats_submission(body): - """ - Parses the POST request body for a stats submission - """ - # storage vars for the request body - game_meta = {} - events = {} - players = [] - teams = [] +class Submission(object): + """Parses an incoming POST request for stats submissions.""" + + def __init__(self, body, headers): + # a copy of the HTTP headers + self.headers = headers + + # a copy of the HTTP POST body + self.body = body + + # the submission code version (from the server) + self.version = None + + # the revision string of the server + self.revision = None + + # the game type played + self.game_type_cd = None + + # the active game mod + self.mod = None + + # the name of the map played + self.map_name = None + + # unique identifier (string) for a match on a given server + self.match_id = None + + # the name of the server + self.server_name = None + + # the number of cvars that were changed to be different than default + self.impure_cvar_changes = None + + # the port number the game server is listening on + self.port_number = None + + # how long the game lasted + self.duration = None - # we're not in either stanza to start - in_P = in_Q = False + # which ladder is being used, if any + self.ladder = None - for line in body.split('\n'): + # players involved in the match (humans, bots, and spectators) + self.players = [] + + # raw team events + self.teams = [] + + # the parsing deque (we use this to allow peeking) + self.q = collections.deque(self.body.split("\n")) + + ############################################################################################ + # Below this point are fields useful in determining if the submission is valid or + # performance optimizations that save us from looping over the events over and over again. + ############################################################################################ + + # humans who played in the match + self.humans = [] + + # bots who played in the match + self.bots = [] + + # distinct weapons that we have seen fired + self.weapons = set() + + # has a human player fired a shot? + self.human_fired_weapon = False + + # does any human have a non-zero score? + self.human_nonzero_score = False + + # does any human have a fastest cap? + self.human_fastest = False + + def next_item(self): + """Returns the next key:value pair off the queue.""" try: - (key, value) = line.strip().split(' ', 1) - - # Server (S) and Nick (n) fields can have international characters. - if key in 'S' 'n': - value = unicode(value, 'utf-8') - - if key not in 'P' 'Q' 'n' 'e' 't' 'i': - game_meta[key] = value - - if key == 'Q' or key == 'P': - #log.debug('Found a {0}'.format(key)) - #log.debug('in_Q: {0}'.format(in_Q)) - #log.debug('in_P: {0}'.format(in_P)) - #log.debug('events: {0}'.format(events)) - - # check where we were before and append events accordingly - if in_Q and len(events) > 0: - #log.debug('creating a team (Q) entry') - teams.append(events) - events = {} - elif in_P and len(events) > 0: - #log.debug('creating a player (P) entry') - players.append(events) - events = {} - - if key == 'P': - #log.debug('key == P') - in_P = True - in_Q = False - elif key == 'Q': - #log.debug('key == Q') - in_P = False - in_Q = True - - events[key] = value - - if key == 'e': - (subkey, subvalue) = value.split(' ', 1) - events[subkey] = subvalue - if key == 'n': - events[key] = value - if key == 't': - events[key] = value + items = self.q.popleft().strip().split(' ', 1) + if len(items) == 1: + # Some keys won't have values, like 'L' records where the server isn't actually + # participating in any ladders. These can be safely ignored. + return None, None + else: + return items except: - # no key/value pair - move on to the next line - pass - - # add the last entity we were working on - if in_P and len(events) > 0: - players.append(events) - elif in_Q and len(events) > 0: - teams.append(events) + return None, None + + def add_weapon_fired(self, sub_key): + """Adds a weapon to the set of weapons fired during the match (a set).""" + self.weapons.add(sub_key.split("-")[1]) + + @staticmethod + def is_human_player(player): + """ + Determines if a given set of events correspond with a non-bot + """ + return not player['P'].startswith('bot') + + @staticmethod + def played_in_game(player): + """ + Determines if a given set of player events correspond with a player who + played in the game (matches 1 and scoreboardvalid 1) + """ + return 'matches' in player and 'scoreboardvalid' in player + + def parse_player(self, key, pid): + """Construct a player events listing from the submission.""" + + # all of the keys related to player records + player_keys = ['i', 'n', 't', 'e'] + + player = {key: pid} + + player_fired_weapon = False + player_nonzero_score = False + player_fastest = False + + # Consume all following 'i' 'n' 't' 'e' records + while len(self.q) > 0: + (key, value) = self.next_item() + if key is None and value is None: + continue + elif key == 'e': + (sub_key, sub_value) = value.split(' ', 1) + player[sub_key] = sub_value + + if sub_key.endswith("cnt-fired"): + player_fired_weapon = True + self.add_weapon_fired(sub_key) + elif sub_key == 'scoreboard-score' and int(sub_value) != 0: + player_nonzero_score = True + elif sub_key == 'scoreboard-fastest': + player_fastest = True + elif key == 'n': + player[key] = unicode(value, 'utf-8') + elif key in player_keys: + player[key] = value + else: + # something we didn't expect - put it back on the deque + self.q.appendleft("{} {}".format(key, value)) + break + + played = self.played_in_game(player) + human = self.is_human_player(player) + + if played and human: + self.humans.append(player) + + if player_fired_weapon: + self.human_fired_weapon = True + + if player_nonzero_score: + self.human_nonzero_score = True + + if player_fastest: + self.human_fastest = True + + elif played and not human: + self.bots.append(player) + + self.players.append(player) + + def parse_team(self, key, tid): + """Construct a team events listing from the submission.""" + team = {key: tid} + + # Consume all following 'e' records + while len(self.q) > 0 and self.q[0].startswith('e'): + (_, value) = self.next_item() + (sub_key, sub_value) = value.split(' ', 1) + team[sub_key] = sub_value + + self.teams.append(team) + + def parse(self): + """Parses the request body into instance variables.""" + while len(self.q) > 0: + (key, value) = self.next_item() + if key is None and value is None: + continue + elif key == 'V': + self.version = value + elif key == 'R': + self.revision = value + elif key == 'G': + self.game_type_cd = value + elif key == 'O': + self.mod = value + elif key == 'M': + self.map_name = value + elif key == 'I': + self.match_id = value + elif key == 'S': + self.server_name = unicode(value, 'utf-8') + elif key == 'C': + self.impure_cvar_changes = int(value) + elif key == 'U': + self.port_number = int(value) + elif key == 'D': + self.duration = datetime.timedelta(seconds=int(round(float(value)))) + elif key == 'L': + self.ladder = value + elif key == 'Q': + self.parse_team(key, value) + elif key == 'P': + self.parse_player(key, value) + else: + raise Exception("Invalid submission") + + return self + + def __repr__(self): + """Debugging representation of a submission.""" + return "game_type_cd: {}, mod: {}, players: {}, humans: {}, bots: {}, weapons: {}".format( + self.game_type_cd, self.mod, len(self.players), len(self.humans), len(self.bots), + self.weapons) + + +def elo_submission_category(submission): + """Determines the Elo category purely by what is in the submission data.""" + mod = submission.mod + + vanilla_allowed_weapons = {"shotgun", "devastator", "blaster", "mortar", "vortex", "electro", + "arc", "hagar", "crylink", "machinegun"} + insta_allowed_weapons = {"vaporizer", "blaster"} + overkill_allowed_weapons = {"hmg", "vortex", "shotgun", "blaster", "machinegun", "rpc"} + + if mod == "Xonotic": + if len(submission.weapons - vanilla_allowed_weapons) == 0: + return "vanilla" + elif mod == "InstaGib": + if len(submission.weapons - insta_allowed_weapons) == 0: + return "insta" + elif mod == "Overkill": + if len(submission.weapons - overkill_allowed_weapons) == 0: + return "overkill" + else: + return "general" - return (game_meta, players, teams) + return "general" -def is_blank_game(gametype, players): - """Determine if this is a blank game or not. A blank game is either: +def is_blank_game(submission): + """ + Determine if this is a blank game or not. A blank game is either: 1) a match that ended in the warmup stage, where accuracy events are not present (for non-CTS games) @@ -105,40 +279,24 @@ def is_blank_game(gametype, players): 1) a match in which no player made a positive or negative score """ - r = re.compile(r'acc-.*-cnt-fired') - flg_nonzero_score = False - flg_acc_events = False - flg_fastest_lap = False - - for events in players: - if is_real_player(events) and played_in_game(events): - for (key,value) in events.items(): - if key == 'scoreboard-score' and value != 0: - flg_nonzero_score = True - if r.search(key): - flg_acc_events = True - if key == 'scoreboard-fastest': - flg_fastest_lap = True - - if gametype == 'cts': - return not flg_fastest_lap - elif gametype == 'nb': - return not flg_nonzero_score + if submission.game_type_cd == 'cts': + return not submission.human_fastest + elif submission.game_type_cd == 'nb': + return not submission.human_nonzero_score else: - return not (flg_nonzero_score and flg_acc_events) + return not (submission.human_nonzero_score and submission.human_fired_weapon) -def get_remote_addr(request): - """Get the Xonotic server's IP address""" - if 'X-Forwarded-For' in request.headers: - return request.headers['X-Forwarded-For'] - else: - return request.remote_addr +def has_required_metadata(submission): + """Determines if a submission has all the required metadata fields.""" + return (submission.game_type_cd is not None + and submission.map_name is not None + and submission.match_id is not None + and submission.server_name is not None) -def is_supported_gametype(gametype, version): - """Whether a gametype is supported or not""" - is_supported = False +def is_supported_gametype(submission): + """Determines if a submission is of a valid and supported game type.""" # if the type can be supported, but with version constraints, uncomment # here and add the restriction for a specific version below @@ -150,6 +308,7 @@ def is_supported_gametype(gametype, version): 'cts', 'dm', 'dom', + 'duel', 'ft', 'freezetag', 'ka', 'keepaway', 'kh', @@ -160,63 +319,77 @@ def is_supported_gametype(gametype, version): 'tdm', ) - if gametype in supported_game_types: - is_supported = True - else: - is_supported = False + is_supported = submission.game_type_cd in supported_game_types # some game types were buggy before revisions, thus this additional filter - if gametype == 'ca' and version <= 5: + if submission.game_type_cd == 'ca' and submission.version <= 5: is_supported = False return is_supported -def do_precondition_checks(request, game_meta, raw_players): - """Precondition checks for ALL gametypes. - These do not require a database connection.""" - if not has_required_metadata(game_meta): - log.debug("ERROR: Required game meta missing") - raise pyramid.httpexceptions.HTTPUnprocessableEntity("Missing game meta") - +def has_minimum_real_players(settings, submission): + """ + Determines if the submission has enough human players to store in the database. The minimum + setting comes from the config file under the setting xonstat.minimum_real_players. + """ try: - version = int(game_meta['V']) + minimum_required_players = int(settings.get("xonstat.minimum_required_players")) except: - log.debug("ERROR: Required game meta invalid") - raise pyramid.httpexceptions.HTTPUnprocessableEntity("Invalid game meta") + minimum_required_players = 2 - if not is_supported_gametype(game_meta['G'], version): - log.debug("ERROR: Unsupported gametype") - raise pyramid.httpexceptions.HTTPOk("OK") + return len(submission.human_players) >= minimum_required_players - if not has_minimum_real_players(request.registry.settings, raw_players): - log.debug("ERROR: Not enough real players") - raise pyramid.httpexceptions.HTTPOk("OK") - if is_blank_game(game_meta['G'], raw_players): - log.debug("ERROR: Blank game") - raise pyramid.httpexceptions.HTTPOk("OK") +def do_precondition_checks(settings, submission): + """Precondition checks for ALL gametypes. These do not require a database connection.""" + if not has_required_metadata(submission): + msg = "Missing required game metadata" + log.debug(msg) + raise pyramid.httpexceptions.HTTPUnprocessableEntity( + body=msg, + content_type="text/plain" + ) + if submission.version is None: + msg = "Invalid or incorrect game metadata provided" + log.debug(msg) + raise pyramid.httpexceptions.HTTPUnprocessableEntity( + body=msg, + content_type="text/plain" + ) -def is_real_player(events): - """ - Determines if a given set of events correspond with a non-bot - """ - if not events['P'].startswith('bot'): - return True - else: - return False + if not is_supported_gametype(submission): + msg = "Unsupported game type ({})".format(submission.game_type_cd) + log.debug(msg) + raise pyramid.httpexceptions.HTTPOk( + body=msg, + content_type="text/plain" + ) + if not has_minimum_real_players(settings, submission): + msg = "Not enough real players" + log.debug(msg) + raise pyramid.httpexceptions.HTTPOk( + body=msg, + content_type="text/plain" + ) -def played_in_game(events): - """ - Determines if a given set of player events correspond with a player who - played in the game (matches 1 and scoreboardvalid 1) - """ - if 'matches' in events and 'scoreboardvalid' in events: - return True + if is_blank_game(submission): + msg = "Blank game" + log.debug(msg) + raise pyramid.httpexceptions.HTTPOk( + body=msg, + content_type="text/plain" + ) + + +def get_remote_addr(request): + """Get the Xonotic server's IP address""" + if 'X-Forwarded-For' in request.headers: + return request.headers['X-Forwarded-For'] else: - return False + return request.remote_addr def num_real_players(player_events): @@ -233,60 +406,14 @@ def num_real_players(player_events): return real_players -def has_minimum_real_players(settings, player_events): - """ - Determines if the collection of player events has enough "real" players - to store in the database. The minimum setting comes from the config file - under the setting xonstat.minimum_real_players. - """ - flg_has_min_real_players = True - - try: - minimum_required_players = int( - settings['xonstat.minimum_required_players']) - except: - minimum_required_players = 2 - - real_players = num_real_players(player_events) - - if real_players < minimum_required_players: - flg_has_min_real_players = False - - return flg_has_min_real_players - - -def has_required_metadata(metadata): - """ - Determines if a give set of metadata has enough data to create a game, - server, and map with. - """ - flg_has_req_metadata = True - - if 'G' not in metadata or\ - 'M' not in metadata or\ - 'I' not in metadata or\ - 'S' not in metadata: - flg_has_req_metadata = False - - return flg_has_req_metadata - - def should_do_weapon_stats(game_type_cd): """True of the game type should record weapon stats. False otherwise.""" - if game_type_cd in 'cts': - return False - else: - return True + return game_type_cd not in {'cts'} -def should_do_elos(game_type_cd): +def gametype_elo_eligible(game_type_cd): """True of the game type should process Elos. False otherwise.""" - elo_game_types = ('duel', 'dm', 'ca', 'ctf', 'tdm', 'ka', 'ft') - - if game_type_cd in elo_game_types: - return True - else: - return False + return game_type_cd in {'duel', 'dm', 'ca', 'ctf', 'tdm', 'ka', 'ft'} def register_new_nick(session, player, new_nick): @@ -350,86 +477,94 @@ def update_fastest_cap(session, player_id, game_id, map_id, captime, mod): session.flush() -def get_or_create_server(session, name, hashkey, ip_addr, revision, port, - impure_cvars): +def update_server(server, name, hashkey, ip_addr, port, revision, impure_cvars): """ - Find a server by name or create one if not found. Parameters: - - session - SQLAlchemy database session factory - name - server name of the server to be found or created - hashkey - server hashkey - ip_addr - the IP address of the server - revision - the xonotic revision number - port - the port number of the server - impure_cvars - the number of impure cvar changes + Updates the server in the given DB session, if needed. + + :param server: The found server instance. + :param name: The incoming server name. + :param hashkey: The incoming server hashkey. + :param ip_addr: The incoming server IP address. + :param port: The incoming server port. + :param revision: The incoming server revision. + :param impure_cvars: The incoming number of impure server cvars. + :return: bool """ - server = None - + # ensure the two int attributes are actually ints try: port = int(port) except: port = None - try: + try: impure_cvars = int(impure_cvars) except: impure_cvars = 0 - # finding by hashkey is preferred, but if not we will fall - # back to using name only, which can result in dupes - if hashkey is not None: - servers = session.query(Server).\ - filter_by(hashkey=hashkey).\ - order_by(expr.desc(Server.create_dt)).limit(1).all() - - if len(servers) > 0: - server = servers[0] - log.debug("Found existing server {0} by hashkey ({1})".format( - server.server_id, server.hashkey)) - else: - servers = session.query(Server).\ - filter_by(name=name).\ - order_by(expr.desc(Server.create_dt)).limit(1).all() + updated = False + if name and server.name != name: + server.name = name + updated = True + if hashkey and server.hashkey != hashkey: + server.hashkey = hashkey + updated = True + if ip_addr and server.ip_addr != ip_addr: + server.ip_addr = ip_addr + updated = True + if port and server.port != port: + server.port = port + updated = True + if revision and server.revision != revision: + server.revision = revision + updated = True + if impure_cvars and server.impure_cvars != impure_cvars: + server.impure_cvars = impure_cvars + server.pure_ind = True if impure_cvars == 0 else False + updated = True - if len(servers) > 0: - server = servers[0] - log.debug("Found existing server {0} by name".format(server.server_id)) + return updated - # still haven't found a server by hashkey or name, so we need to create one - if server is None: - server = Server(name=name, hashkey=hashkey) - session.add(server) - session.flush() - log.debug("Created server {0} with hashkey {1}".format( - server.server_id, server.hashkey)) - # detect changed fields - if server.name != name: - server.name = name - session.add(server) +def get_or_create_server(session, name, hashkey, ip_addr, revision, port, impure_cvars): + """ + Find a server by name or create one if not found. Parameters: - if server.hashkey != hashkey: - server.hashkey = hashkey - session.add(server) + session - SQLAlchemy database session factory + name - server name of the server to be found or created + hashkey - server hashkey + ip_addr - the IP address of the server + revision - the xonotic revision number + port - the port number of the server + impure_cvars - the number of impure cvar changes + """ + servers_q = DBSession.query(Server).filter(Server.active_ind) - if server.ip_addr != ip_addr: - server.ip_addr = ip_addr - session.add(server) + if hashkey: + # if the hashkey is provided, we'll use that + servers_q = servers_q.filter((Server.name == name) or (Server.hashkey == hashkey)) + else: + # otherwise, it is just by name + servers_q = servers_q.filter(Server.name == name) - if server.port != port: - server.port = port - session.add(server) + # order by the hashkey, which means any hashkey match will appear first if there are multiple + servers = servers_q.order_by(Server.hashkey, Server.create_dt).all() - if server.revision != revision: - server.revision = revision + if len(servers) == 0: + server = Server(name=name, hashkey=hashkey) session.add(server) + session.flush() + log.debug("Created server {} with hashkey {}.".format(server.server_id, server.hashkey)) + else: + server = servers[0] + if len(servers) == 1: + log.info("Found existing server {}.".format(server.server_id)) - if server.impure_cvars != impure_cvars: - server.impure_cvars = impure_cvars - if impure_cvars > 0: - server.pure_ind = False - else: - server.pure_ind = True + elif len(servers) > 1: + server_id_list = ", ".join(["{}".format(s.server_id) for s in servers]) + log.warn("Multiple servers found ({})! Using the first one ({})." + .format(server_id_list, server.server_id)) + + if update_server(server, name, hashkey, ip_addr, port, revision, impure_cvars): session.add(server) return server @@ -486,6 +621,11 @@ def create_game(session, start_dt, game_type_cd, server_id, map_id, game.match_id = match_id game.mod = mod[:64] + # There is some drift between start_dt (provided by app) and create_dt + # (default in the database), so we'll make them the same until this is + # resolved. + game.create_dt = start_dt + try: game.duration = datetime.timedelta(seconds=int(round(float(duration)))) except: @@ -521,7 +661,7 @@ def get_or_create_player(session=None, hashkey=None, nick=None): nick - nick of the player (in case of a first time create) """ # if we have a bot - if re.search('^bot#\d+$', hashkey) or re.search('^bot#\d+#', hashkey): + if re.search('^bot#\d+', hashkey): player = session.query(Player).filter_by(player_id=1).one() # if we have an untracked player elif re.search('^player#\d+$', hashkey): @@ -752,6 +892,7 @@ def create_team_stat(session, game, events): for (key,value) in events.items(): if key == 'scoreboard-score': teamstat.score = int(round(float(value))) if key == 'scoreboard-caps': teamstat.caps = int(value) + if key == 'scoreboard-goals': teamstat.caps = int(value) if key == 'scoreboard-rounds': teamstat.rounds = int(value) session.add(teamstat) @@ -830,12 +971,20 @@ def create_weapon_stats(session, game_meta, game, player, pgstat, events): return pwstats -def create_elos(session, game): - """Elo handler for all game types.""" - try: - process_elos(game, session) - except Exception as e: - log.debug('Error (non-fatal): elo processing failed.') +def get_ranks(session, player_ids, game_type_cd): + """ + Gets the rank entries for all players in the given list, returning a dict + of player_id -> PlayerRank instance. The rank entry corresponds to the + game type of the parameter passed in as well. + """ + ranks = {} + for pr in session.query(PlayerRank).\ + filter(PlayerRank.player_id.in_(player_ids)).\ + filter(PlayerRank.game_type_cd == game_type_cd).\ + all(): + ranks[pr.player_id] = pr + + return ranks def submit_stats(request): @@ -899,6 +1048,8 @@ def submit_stats(request): # keep track of the players we've seen player_ids = [] + pgstats = [] + hashkeys = {} for events in raw_players: player = get_or_create_player( session = session, @@ -907,11 +1058,14 @@ def submit_stats(request): pgstat = create_game_stat(session, game_meta, game, server, gmap, player, events) + pgstats.append(pgstat) if player.player_id > 1: - anticheats = create_anticheats(session, pgstat, game, player, - events) + anticheats = create_anticheats(session, pgstat, game, player, events) + + if player.player_id > 2: player_ids.append(player.player_id) + hashkeys[player.player_id] = events['P'] if should_do_weapon_stats(game_type_cd) and player.player_id > 1: pwstats = create_weapon_stats(session, game_meta, game, player, @@ -926,12 +1080,30 @@ def submit_stats(request): except Exception as e: raise e - if should_do_elos(game_type_cd): - create_elos(session, game) + if server.elo_ind and gametype_elo_eligible(game_type_cd): + ep = EloProcessor(session, game, pgstats) + ep.save(session) session.commit() log.debug('Success! Stats recorded.') - return Response('200 OK') + + # ranks are fetched after we've done the "real" processing + ranks = get_ranks(session, player_ids, game_type_cd) + + # plain text response + request.response.content_type = 'text/plain' + + return { + "now" : calendar.timegm(datetime.datetime.utcnow().timetuple()), + "server" : server, + "game" : game, + "gmap" : gmap, + "player_ids" : player_ids, + "hashkeys" : hashkeys, + "elos" : ep.wip, + "ranks" : ranks, + } + except Exception as e: if session: session.rollback()