4 import pyramid.httpexceptions
7 from pyramid.response import Response
8 from sqlalchemy import Sequence
9 from sqlalchemy.orm.exc import MultipleResultsFound, NoResultFound
10 from xonstat.d0_blind_id import d0_blind_id_verify
11 from xonstat.elo import process_elos
12 from xonstat.models import *
13 from xonstat.util import strip_colors, qfont_decode
15 log = logging.getLogger(__name__)
18 def is_blank_game(players):
19 """Determine if this is a blank game or not. A blank game is either:
21 1) a match that ended in the warmup stage, where accuracy events are not
24 2) a match in which no player made a positive or negative score AND was
27 r = re.compile(r'acc-.*-cnt-fired')
28 flg_nonzero_score = False
29 flg_acc_events = False
31 for events in players:
32 if is_real_player(events):
33 for (key,value) in events.items():
34 if key == 'scoreboard-score' and value != 0:
35 flg_nonzero_score = True
39 return not (flg_nonzero_score and flg_acc_events)
41 def get_remote_addr(request):
42 """Get the Xonotic server's IP address"""
43 if 'X-Forwarded-For' in request.headers:
44 return request.headers['X-Forwarded-For']
46 return request.remote_addr
49 def is_supported_gametype(gametype):
50 """Whether a gametype is supported or not"""
53 if gametype == 'cts' or gametype == 'lms':
59 def verify_request(request):
61 (idfp, status) = d0_blind_id_verify(
62 sig=request.headers['X-D0-Blind-Id-Detached-Signature'],
64 postdata=request.body)
66 log.debug('\nidfp: {0}\nstatus: {1}'.format(idfp, status))
74 def num_real_players(player_events, count_bots=False):
76 Returns the number of real players (those who played
77 and are on the scoreboard).
81 for events in player_events:
82 if is_real_player(events, count_bots):
88 def has_minimum_real_players(settings, player_events):
90 Determines if the collection of player events has enough "real" players
91 to store in the database. The minimum setting comes from the config file
92 under the setting xonstat.minimum_real_players.
94 flg_has_min_real_players = True
97 minimum_required_players = int(
98 settings['xonstat.minimum_required_players'])
100 minimum_required_players = 2
102 real_players = num_real_players(player_events)
104 if real_players < minimum_required_players:
105 flg_has_min_real_players = False
107 return flg_has_min_real_players
110 def verify_requests(settings):
112 Determines whether or not to verify requests using the blind_id algorithm
115 val_verify_requests = settings['xonstat.verify_requests']
116 if val_verify_requests == "true":
117 flg_verify_requests = True
119 flg_verify_requests = False
121 flg_verify_requests = True
123 return flg_verify_requests
126 def has_required_metadata(metadata):
128 Determines if a give set of metadata has enough data to create a game,
129 server, and map with.
131 flg_has_req_metadata = True
133 if 'T' not in metadata or\
134 'G' not in metadata or\
135 'M' not in metadata or\
136 'I' not in metadata or\
138 flg_has_req_metadata = False
140 return flg_has_req_metadata
143 def is_real_player(events, count_bots=False):
145 Determines if a given set of player events correspond with a player who
147 1) is not a bot (P event does not look like a bot)
148 2) played in the game (matches 1)
149 3) was present at the end of the game (scoreboardvalid 1)
151 Returns True if the player meets the above conditions, and false otherwise.
155 # removing 'joins' here due to bug, but it should be here
156 if 'matches' in events and 'scoreboardvalid' in events:
157 if (events['P'].startswith('bot') and count_bots) or \
158 not events['P'].startswith('bot'):
164 def register_new_nick(session, player, new_nick):
166 Change the player record's nick to the newly found nick. Store the old
167 nick in the player_nicks table for that player.
169 session - SQLAlchemy database session factory
170 player - player record whose nick is changing
171 new_nick - the new nickname
173 # see if that nick already exists
174 stripped_nick = strip_colors(qfont_decode(player.nick))
176 player_nick = session.query(PlayerNick).filter_by(
177 player_id=player.player_id, stripped_nick=stripped_nick).one()
178 except NoResultFound, e:
179 # player_id/stripped_nick not found, create one
180 # but we don't store "Anonymous Player #N"
181 if not re.search('^Anonymous Player #\d+$', player.nick):
182 player_nick = PlayerNick()
183 player_nick.player_id = player.player_id
184 player_nick.stripped_nick = stripped_nick
185 player_nick.nick = player.nick
186 session.add(player_nick)
188 # We change to the new nick regardless
189 player.nick = new_nick
190 player.stripped_nick = strip_colors(qfont_decode(new_nick))
194 def update_fastest_cap(session, player_id, game_id, map_id, captime):
196 Check the fastest cap time for the player and map. If there isn't
197 one, insert one. If there is, check if the passed time is faster.
200 # we don't record fastest cap times for bots or anonymous players
204 # see if a cap entry exists already
205 # then check to see if the new captime is faster
207 cur_fastest_cap = session.query(PlayerCaptime).filter_by(
208 player_id=player_id, map_id=map_id).one()
210 # current captime is faster, so update
211 if captime < cur_fastest_cap.fastest_cap:
212 cur_fastest_cap.fastest_cap = captime
213 cur_fastest_cap.game_id = game_id
214 cur_fastest_cap.create_dt = datetime.datetime.utcnow()
215 session.add(cur_fastest_cap)
217 except NoResultFound, e:
218 # none exists, so insert
219 cur_fastest_cap = PlayerCaptime(player_id, game_id, map_id, captime)
220 session.add(cur_fastest_cap)
224 def get_or_create_server(session=None, name=None, hashkey=None, ip_addr=None,
227 Find a server by name or create one if not found. Parameters:
229 session - SQLAlchemy database session factory
230 name - server name of the server to be found or created
231 hashkey - server hashkey
234 # find one by that name, if it exists
235 server = session.query(Server).filter_by(name=name).one()
238 if server.hashkey != hashkey:
239 server.hashkey = hashkey
242 # store new IP address
243 if server.ip_addr != ip_addr:
244 server.ip_addr = ip_addr
248 if server.revision != revision:
249 server.revision = revision
252 log.debug("Found existing server {0}".format(server.server_id))
254 except MultipleResultsFound, e:
255 # multiple found, so also filter by hashkey
256 server = session.query(Server).filter_by(name=name).\
257 filter_by(hashkey=hashkey).one()
258 log.debug("Found existing server {0}".format(server.server_id))
260 except NoResultFound, e:
261 # not found, create one
262 server = Server(name=name, hashkey=hashkey)
265 log.debug("Created server {0} with hashkey {1}".format(
266 server.server_id, server.hashkey))
271 def get_or_create_map(session=None, name=None):
273 Find a map by name or create one if not found. Parameters:
275 session - SQLAlchemy database session factory
276 name - map name of the map to be found or created
279 # find one by the name, if it exists
280 gmap = session.query(Map).filter_by(name=name).one()
281 log.debug("Found map id {0}: {1}".format(gmap.map_id,
283 except NoResultFound, e:
284 gmap = Map(name=name)
287 log.debug("Created map id {0}: {1}".format(gmap.map_id,
289 except MultipleResultsFound, e:
290 # multiple found, so use the first one but warn
292 gmaps = session.query(Map).filter_by(name=name).order_by(
295 log.debug("Found map id {0}: {1} but found \
296 multiple".format(gmap.map_id, gmap.name))
301 def create_game(session=None, start_dt=None, game_type_cd=None,
302 server_id=None, map_id=None, winner=None, match_id=None,
305 Creates a game. Parameters:
307 session - SQLAlchemy database session factory
308 start_dt - when the game started (datetime object)
309 game_type_cd - the game type of the game being played
310 server_id - server identifier of the server hosting the game
311 map_id - map on which the game was played
312 winner - the team id of the team that won
314 seq = Sequence('games_game_id_seq')
315 game_id = session.execute(seq)
316 game = Game(game_id=game_id, start_dt=start_dt, game_type_cd=game_type_cd,
317 server_id=server_id, map_id=map_id, winner=winner)
318 game.match_id = match_id
321 game.duration = datetime.timedelta(seconds=int(round(float(duration))))
326 session.query(Game).filter(Game.server_id==server_id).\
327 filter(Game.match_id==match_id).one()
329 log.debug("Error: game with same server and match_id found! Ignoring.")
331 # if a game under the same server and match_id found,
332 # this is a duplicate game and can be ignored
333 raise pyramid.httpexceptions.HTTPOk('OK')
334 except NoResultFound, e:
335 # server_id/match_id combination not found. game is ok to insert
338 log.debug("Created game id {0} on server {1}, map {2} at \
339 {3}".format(game.game_id,
340 server_id, map_id, start_dt))
345 def get_or_create_player(session=None, hashkey=None, nick=None):
347 Finds a player by hashkey or creates a new one (along with a
348 corresponding hashkey entry. Parameters:
350 session - SQLAlchemy database session factory
351 hashkey - hashkey of the player to be found or created
352 nick - nick of the player (in case of a first time create)
355 if re.search('^bot#\d+$', hashkey) or re.search('^bot#\d+#', hashkey):
356 player = session.query(Player).filter_by(player_id=1).one()
357 # if we have an untracked player
358 elif re.search('^player#\d+$', hashkey):
359 player = session.query(Player).filter_by(player_id=2).one()
360 # else it is a tracked player
362 # see if the player is already in the database
363 # if not, create one and the hashkey along with it
365 hk = session.query(Hashkey).filter_by(
366 hashkey=hashkey).one()
367 player = session.query(Player).filter_by(
368 player_id=hk.player_id).one()
369 log.debug("Found existing player {0} with hashkey {1}".format(
370 player.player_id, hashkey))
376 # if nick is given to us, use it. If not, use "Anonymous Player"
377 # with a suffix added for uniqueness.
379 player.nick = nick[:128]
380 player.stripped_nick = strip_colors(qfont_decode(nick[:128]))
382 player.nick = "Anonymous Player #{0}".format(player.player_id)
383 player.stripped_nick = player.nick
385 hk = Hashkey(player_id=player.player_id, hashkey=hashkey)
387 log.debug("Created player {0} ({2}) with hashkey {1}".format(
388 player.player_id, hashkey, player.nick.encode('utf-8')))
392 def create_player_game_stat(session=None, player=None,
393 game=None, player_events=None):
395 Creates game statistics for a given player in a given game. Parameters:
397 session - SQLAlchemy session factory
398 player - Player record of the player who owns the stats
399 game - Game record for the game to which the stats pertain
400 player_events - dictionary for the actual stats that need to be transformed
403 # in here setup default values (e.g. if game type is CTF then
404 # set kills=0, score=0, captures=0, pickups=0, fckills=0, etc
405 # TODO: use game's create date here instead of now()
406 seq = Sequence('player_game_stats_player_game_stat_id_seq')
407 pgstat_id = session.execute(seq)
408 pgstat = PlayerGameStat(player_game_stat_id=pgstat_id,
409 create_dt=datetime.datetime.utcnow())
411 # set player id from player record
412 pgstat.player_id = player.player_id
414 #set game id from game record
415 pgstat.game_id = game.game_id
417 # all games have a score and every player has an alivetime
419 pgstat.alivetime = datetime.timedelta(seconds=0)
421 if game.game_type_cd == 'dm' or game.game_type_cd == 'tdm' or game.game_type_cd == 'duel':
425 elif game.game_type_cd == 'ctf':
431 pgstat.carrier_frags = 0
433 for (key,value) in player_events.items():
435 pgstat.nick = value[:128]
436 pgstat.stripped_nick = strip_colors(qfont_decode(pgstat.nick))
437 if key == 't': pgstat.team = int(value)
439 pgstat.rank = int(value)
440 # to support older servers who don't send scoreboardpos values
441 if pgstat.scoreboardpos is None:
442 pgstat.scoreboardpos = pgstat.rank
443 if key == 'alivetime':
444 pgstat.alivetime = datetime.timedelta(seconds=int(round(float(value))))
445 if key == 'scoreboard-drops': pgstat.drops = int(value)
446 if key == 'scoreboard-returns': pgstat.returns = int(value)
447 if key == 'scoreboard-fckills': pgstat.carrier_frags = int(value)
448 if key == 'scoreboard-pickups': pgstat.pickups = int(value)
449 if key == 'scoreboard-caps': pgstat.captures = int(value)
450 if key == 'scoreboard-score': pgstat.score = int(value)
451 if key == 'scoreboard-deaths': pgstat.deaths = int(value)
452 if key == 'scoreboard-kills': pgstat.kills = int(value)
453 if key == 'scoreboard-suicides': pgstat.suicides = int(value)
454 if key == 'scoreboard-captime':
455 pgstat.fastest_cap = datetime.timedelta(seconds=float(value)/100)
456 if key == 'avglatency': pgstat.avg_latency = float(value)
457 if key == 'teamrank': pgstat.teamrank = int(value)
458 if key == 'scoreboardpos': pgstat.scoreboardpos = int(value)
460 # check to see if we had a name, and if
461 # not use an anonymous handle
462 if pgstat.nick == None:
463 pgstat.nick = "Anonymous Player"
464 pgstat.stripped_nick = "Anonymous Player"
466 # otherwise process a nick change
467 elif pgstat.nick != player.nick and player.player_id > 2:
468 register_new_nick(session, player, pgstat.nick)
470 # if the player is ranked #1 and it is a team game, set the game's winner
471 # to be the team of that player
472 # FIXME: this is a hack, should be using the 'W' field (not present)
473 if pgstat.rank == 1 and pgstat.team:
474 game.winner = pgstat.team
482 def create_player_weapon_stats(session=None, player=None,
483 game=None, pgstat=None, player_events=None, game_meta=None):
485 Creates accuracy records for each weapon used by a given player in a
486 given game. Parameters:
488 session - SQLAlchemy session factory object
489 player - Player record who owns the weapon stats
490 game - Game record in which the stats were created
491 pgstat - Corresponding PlayerGameStat record for these weapon stats
492 player_events - dictionary containing the raw weapon values that need to be
494 game_meta - dictionary of game metadata (only used for stats version info)
498 # Version 1 of stats submissions doubled the data sent.
499 # To counteract this we divide the data by 2 only for
500 # POSTs coming from version 1.
502 version = int(game_meta['V'])
505 log.debug('NOTICE: found a version 1 request, halving the weapon stats...')
511 for (key,value) in player_events.items():
512 matched = re.search("acc-(.*?)-cnt-fired", key)
514 weapon_cd = matched.group(1)
515 seq = Sequence('player_weapon_stats_player_weapon_stats_id_seq')
516 pwstat_id = session.execute(seq)
517 pwstat = PlayerWeaponStat()
518 pwstat.player_weapon_stats_id = pwstat_id
519 pwstat.player_id = player.player_id
520 pwstat.game_id = game.game_id
521 pwstat.player_game_stat_id = pgstat.player_game_stat_id
522 pwstat.weapon_cd = weapon_cd
524 if 'n' in player_events:
525 pwstat.nick = player_events['n']
527 pwstat.nick = player_events['P']
529 if 'acc-' + weapon_cd + '-cnt-fired' in player_events:
530 pwstat.fired = int(round(float(
531 player_events['acc-' + weapon_cd + '-cnt-fired'])))
532 if 'acc-' + weapon_cd + '-fired' in player_events:
533 pwstat.max = int(round(float(
534 player_events['acc-' + weapon_cd + '-fired'])))
535 if 'acc-' + weapon_cd + '-cnt-hit' in player_events:
536 pwstat.hit = int(round(float(
537 player_events['acc-' + weapon_cd + '-cnt-hit'])))
538 if 'acc-' + weapon_cd + '-hit' in player_events:
539 pwstat.actual = int(round(float(
540 player_events['acc-' + weapon_cd + '-hit'])))
541 if 'acc-' + weapon_cd + '-frags' in player_events:
542 pwstat.frags = int(round(float(
543 player_events['acc-' + weapon_cd + '-frags'])))
546 pwstat.fired = pwstat.fired/2
547 pwstat.max = pwstat.max/2
548 pwstat.hit = pwstat.hit/2
549 pwstat.actual = pwstat.actual/2
550 pwstat.frags = pwstat.frags/2
553 pwstats.append(pwstat)
558 def parse_body(request):
560 Parses the POST request body for a stats submission
562 # storage vars for the request body
568 for line in request.body.split('\n'):
570 (key, value) = line.strip().split(' ', 1)
572 # Server (S) and Nick (n) fields can have international characters.
573 # We convert to UTF-8.
575 value = unicode(value, 'utf-8')
577 if key in 'V' 'T' 'G' 'M' 'S' 'C' 'R' 'W' 'I' 'D':
578 game_meta[key] = value
581 # if we were working on a player record already, append
582 # it and work on a new one (only set team info)
583 if len(player_events) != 0:
584 players.append(player_events)
587 player_events[key] = value
590 (subkey, subvalue) = value.split(' ', 1)
591 player_events[subkey] = subvalue
593 player_events[key] = value
595 player_events[key] = value
597 # no key/value pair - move on to the next line
600 # add the last player we were working on
601 if len(player_events) > 0:
602 players.append(player_events)
604 return (game_meta, players)
607 def create_player_stats(session=None, player=None, game=None,
608 player_events=None, game_meta=None):
610 Creates player game and weapon stats according to what type of player
612 pgstat = create_player_game_stat(session=session,
613 player=player, game=game, player_events=player_events)
615 # fastest cap "upsert"
616 if game.game_type_cd == 'ctf' and pgstat.fastest_cap is not None:
617 update_fastest_cap(session, pgstat.player_id, game.game_id,
618 game.map_id, pgstat.fastest_cap)
620 # bots don't get weapon stats. sorry, bots!
621 if not re.search('^bot#\d+$', player_events['P']):
622 create_player_weapon_stats(session=session,
623 player=player, game=game, pgstat=pgstat,
624 player_events=player_events, game_meta=game_meta)
627 def stats_submit(request):
629 Entry handler for POST stats submissions.
632 # placeholder for the actual session
635 log.debug("\n----- BEGIN REQUEST BODY -----\n" + request.body +
636 "----- END REQUEST BODY -----\n\n")
638 (idfp, status) = verify_request(request)
639 if verify_requests(request.registry.settings):
641 log.debug("ERROR: Unverified request")
642 raise pyramid.httpexceptions.HTTPUnauthorized("Unverified request")
644 (game_meta, players) = parse_body(request)
646 if not has_required_metadata(game_meta):
647 log.debug("ERROR: Required game meta missing")
648 raise pyramid.httpexceptions.HTTPUnprocessableEntity("Missing game meta")
650 if not is_supported_gametype(game_meta['G']):
651 log.debug("ERROR: Unsupported gametype")
652 raise pyramid.httpexceptions.HTTPOk("OK")
654 if not has_minimum_real_players(request.registry.settings, players):
655 log.debug("ERROR: Not enough real players")
656 raise pyramid.httpexceptions.HTTPOk("OK")
658 if is_blank_game(players):
659 log.debug("ERROR: Blank game")
660 raise pyramid.httpexceptions.HTTPOk("OK")
662 # the "duel" gametype is fake
663 if num_real_players(players, count_bots=True) == 2 and \
664 game_meta['G'] == 'dm':
665 game_meta['G'] = 'duel'
668 # fix for DTG, who didn't #ifdef WATERMARK to set the revision info
670 revision = game_meta['R']
674 #----------------------------------------------------------------------
675 # This ends the "precondition" section of sanity checks. All
676 # functions not requiring a database connection go ABOVE HERE.
677 #----------------------------------------------------------------------
678 session = DBSession()
680 server = get_or_create_server(session=session, hashkey=idfp,
681 name=game_meta['S'], revision=revision,
682 ip_addr=get_remote_addr(request))
684 gmap = get_or_create_map(session=session, name=game_meta['M'])
686 # duration is optional
688 duration = game_meta['D']
692 game = create_game(session=session,
693 start_dt=datetime.datetime.utcnow(),
694 #start_dt=datetime.datetime(
695 #*time.gmtime(float(game_meta['T']))[:6]),
696 server_id=server.server_id, game_type_cd=game_meta['G'],
697 map_id=gmap.map_id, match_id=game_meta['I'],
700 # find or create a record for each player
701 # and add stats for each if they were present at the end
703 for player_events in players:
704 if 'n' in player_events:
705 nick = player_events['n']
709 if 'matches' in player_events and 'scoreboardvalid' \
711 player = get_or_create_player(session=session,
712 hashkey=player_events['P'], nick=nick)
713 log.debug('Creating stats for %s' % player_events['P'])
714 create_player_stats(session=session, player=player, game=game,
715 player_events=player_events, game_meta=game_meta)
719 process_elos(game, session)
720 except Exception as e:
721 log.debug('Error (non-fatal): elo processing failed.')
724 log.debug('Success! Stats recorded.')
725 return Response('200 OK')
726 except Exception as e: