api

requirments.txt

@@ -1,3 +1,7 @@
 pytest
 click
-psycopg2
+psycopg2-binary
+ratelimit
+backoff
+fastapi
+python-multipart

sqitch/deploy/player-stats.sql

@@ -5,87 +5,87 @@ BEGIN;
 
   CREATE TABLE IF NOT EXISTS data.player_stats (
     id TEXT,
-    gameId TEXT,
+    game_id TEXT,
-    gameFinishedAt TIMESTAMP,
+    game_finished_at TIMESTAMP,
-    steam64Id TEXT,
+    steam64_id TEXT,
     name TEXT,
     preaim NUMERIC,
-    reactionTime NUMERIC,
+    reaction_time NUMERIC,
     accuracy NUMERIC,
-    accuracyEnemySpotted NUMERIC,
+    accuracy_enemy_spotted NUMERIC,
-    accuracyHead NUMERIC,
+    accuracy_head NUMERIC,
-    shotsFiredEnemySpotted NUMERIC,
+    shots_fired_enemy_spotted NUMERIC,
-    shotsFired NUMERIC,
+    shots_fired NUMERIC,
-    shotsHitEnemySpotted NUMERIC,
+    shots_hit_enemy_spotted NUMERIC,
-    shotsHitFriend NUMERIC,
+    shots_hit_friend NUMERIC,
-    shotsHitFriendHead NUMERIC,
+    shots_Hit_Friend_Head NUMERIC,
-    shotsHitFoe NUMERIC,
+    shots_Hit_Foe NUMERIC,
-    shotsHitFoeHead NUMERIC,
+    shots_Hit_Foe_Head NUMERIC,
-    utilityOnDeathAvg NUMERIC,
+    utility_On_Death_Avg NUMERIC,
-    heFoesDamageAvg NUMERIC,
+    he_foes_damage_avg NUMERIC,
-    heFriendsDamageAvg NUMERIC,
+    he_friends_damage_avg NUMERIC,
-    heThrown NUMERIC,
+    he_thrown NUMERIC,
-    molotovThrown NUMERIC,
+    molotov_thrown NUMERIC,
-    smokeThrown NUMERIC,
+    smoke_thrown NUMERIC,
-    smokeThrownCT NUMERIC,
+    smoke_thrown_ct NUMERIC,
-    smokeThrownCTGood NUMERIC,
+    smoke_thrown_ct_good NUMERIC,
-    smokeThrownCTGoodRatio NUMERIC,
+    smoke_thrown_ct_good_ratio NUMERIC,
-    smokeThrownCTFoes NUMERIC,
+    smoke_thrown_ct_foes NUMERIC,
-    counterStrafingShotsAll NUMERIC,
+    counter_strafing_shots_all NUMERIC,
-    counterStrafingShotsBad NUMERIC,
+    counter_strafing_shots_bad NUMERIC,
-    counterStrafingShotsGood NUMERIC,
+    counter_strafing_shots_good NUMERIC,
-    counterStrafingShotsGoodRatio NUMERIC,
+    counter_strafing_shots_good_ratio NUMERIC,
-    flashbangHitFoe NUMERIC,
+    flashbang_hit_foe NUMERIC,
-    flashbangLeadingToKill NUMERIC,
+    flashbang_leading_to_kill NUMERIC,
-    flashbangHitFoeAvgDuration NUMERIC,
+    flashbang_hit_foe_avg_duration NUMERIC,
-    flashbangHitFriend NUMERIC,
+    flashbang_hit_friend NUMERIC,
-    flashbangThrown NUMERIC,
+    flashbang_thrown NUMERIC,
-    flashAssist NUMERIC,
+    flash_assist NUMERIC,
     score NUMERIC,
-    initialTeamNumber NUMERIC,
+    initial_Team_Number NUMERIC,
     mvps NUMERIC,
-    ctRoundsWon NUMERIC,
+    ct_rounds_won NUMERIC,
-    ctRoundsLost NUMERIC,
+    ct_rounds_lost NUMERIC,
-    tRoundsWon NUMERIC,
+    t_rounds_won NUMERIC,
-    tRoundsLost NUMERIC,
+    t_rounds_lost NUMERIC,
-    sprayAccuracy NUMERIC,
+    spray_accuracy NUMERIC,
-    molotovFoesDamageAvg NUMERIC,
+    molotov_foes_damage_avg NUMERIC,
-    molotovFriendsDamageAvg NUMERIC,
+    molotov_friends_damage_avg NUMERIC,
     color NUMERIC,
-    totalKills NUMERIC,
+    total_kills NUMERIC,
-    totalDeaths NUMERIC,
+    total_deaths NUMERIC,
-    kdRatio NUMERIC,
+    kd_ratio NUMERIC,
     multi2k NUMERIC,
     multi3k NUMERIC,
     multi4k NUMERIC,
     multi5k NUMERIC,
-    hltvRating NUMERIC,
+    hltv_rating NUMERIC,
     hsp NUMERIC,
-    roundsSurvived NUMERIC,
+    rounds_survived NUMERIC,
-    roundsSurvivedPercentage NUMERIC,
+    rounds_survived_percentage NUMERIC,
     dpr NUMERIC,
-    totalAssists NUMERIC,
+    total_assists NUMERIC,
-    totalDamage NUMERIC,
+    total_damage NUMERIC,
-    tradeKillOpportunities NUMERIC,
+    trade_kill_opportunities NUMERIC,
-    tradeKillAttempts NUMERIC,
+    trade_kill_attempts NUMERIC,
-    tradeKillsSucceeded NUMERIC,
+    trade_kills_succeeded NUMERIC,
-    tradeKillAttemptsPercentage NUMERIC,
+    trade_kill_attempts_percentage NUMERIC,
-    tradeKillsSuccessPercentage NUMERIC,
+    trade_kills_success_percentage NUMERIC,
-    tradeKillOpportunitiesPerRound NUMERIC,
+    trade_kill_opportunities_per_round NUMERIC,
-    tradedDeathOpportunities NUMERIC,
+    traded_death_opportunities NUMERIC,
-    tradedDeathAttempts NUMERIC,
+    traded_death_attempts NUMERIC,
-    tradedDeathAttemptsPercentage NUMERIC,
+    traded_death_attempts_percentage NUMERIC,
-    tradedDeathsSucceeded NUMERIC,
+    traded_deaths_succeeded NUMERIC,
-    tradedDeathsSuccessPercentage NUMERIC,
+    traded_deaths_success_percentage NUMERIC,
-    tradedDeathsOpportunitiesPerRound NUMERIC,
+    traded_deaths_opportunities_per_round NUMERIC,
-    leetifyRating NUMERIC,
+    leetify_rating NUMERIC,
-    personalPerformanceRating NUMERIC,
+    personal_performance_rating NUMERIC,
-    ctLeetifyRating NUMERIC,
+    ct_leetify_rating NUMERIC,
-    tLeetifyRating NUMERIC,
+    t_leetify_rating NUMERIC,
-    leetifyUserId TEXT,
+    leetify_user_id TEXT,
-    isCollector BOOL,
+    is_collector BOOL,
-    isProPlan BOOL,
+    is_pro_plan BOOL,
-    isLeetifyStaff BOOL
+    is_leetify_staff BOOL
   );
 
 COMMIT;

sqitch/deploy/profile-game.sql

@@ -4,24 +4,24 @@
 BEGIN;
 
   CREATE TABLE IF NOT EXISTS data.profile_game (
-     leetifyUserId TEXT,
+     leetify_user_id TEXT,
-     ctLeetifyRating NUMERIC,
+     ct_Leetify_rating NUMERIC,
-     ctLeetifyRatingRounds NUMERIC,
+     ct_Leetify_rating_rounds NUMERIC,
-     dataSource NUMERIC,
+     data_source TEXT,
      elo NUMERIC,
-     gameFinishedAt TIMESTAMP,
+     game_finished_at TIMESTAMP,
-     gameId TEXT,
+     game_id TEXT,
-     isCs2 BOOL,
+     is_cs2 BOOL,
-     mapName TEXT,
+     map_name TEXT,
-     matchResult TEXT,
+     match_result TEXT,
      scores TEXT, -- has to be extracted from array
-     skillLevel NUMERIC,
+     skill_level NUMERIC,
-     tLeetifyRating NUMERIC,
+     t_leetify_rating NUMERIC,
-     tLeetifyRatingRounds NUMERIC,
+     t_leetify_rating_rounds NUMERIC,
      deaths NUMERIC,
-     hasBannedPlayer BOOL,
+     has_banned_player BOOL,
      kills NUMERIC,
-     partySize NUMERIC
+     party_size NUMERIC
   );
 
 

sqitch/deploy/profile-meta.sql

@@ -4,12 +4,13 @@
 BEGIN;
 
   CREATE TABLE IF NOT EXISTS data.profile_meta (
-    steam64Id TEXT,
+    name TEXT,
-    isCollector BOOL,
+    steam64_id TEXT,
-    isLeetifyStaff BOOL,
+    is_collector BOOL,
-    isProPlan BOOL,
+    is_leetify_staff BOOL,
-    leetifyUserId TEXT,
+    is_pro_plan BOOL,
-    faceitNickname TEXT
+    leetify_user_id TEXT,
+    faceit_nickname TEXT
   );
 
 COMMIT;

sqitch/verify/player-stats.sql

@@ -3,26 +3,31 @@
 BEGIN;
 
 -- XXX Add verifications here.
-SELECT id, gameId, gameFinishedAt, steam64Id, name, preaim, reactionTime,
+SELECT id, game_id, game_finished_at, steam64_id, name, preaim, reaction_time,
-  accuracy, accuracyEnemySpotted, accuracyHead, shotsFiredEnemySpotted,
+    accuracy, accuracy_enemy_spotted, accuracy_head, shots_fired_enemy_spotted,
-  shotsFired, shotsHitEnemySpotted, shotsHitFriend, shotsHitFriendHead,
+    shots_fired, shots_hit_enemy_spotted, shots_hit_friend,
-  shotsHitFoe, shotsHitFoeHead, utilityOnDeathAvg, heFoesDamageAvg,
+    shots_Hit_Friend_Head, shots_Hit_Foe, shots_Hit_Foe_Head,
-  heFriendsDamageAvg, heThrown, molotovThrown, smokeThrown,
+    utility_On_Death_Avg, he_foes_damage_avg, he_friends_damage_avg, he_thrown,
-  smokeThrownCT, smokeThrownCTGood, smokeThrownCTGoodRatio, smokeThrownCTFoes,
+    molotov_thrown, smoke_thrown, smoke_thrown_ct, smoke_thrown_ct_good,
-  counterStrafingShotsAll, counterStrafingShotsBad, counterStrafingShotsGood,
+    smoke_thrown_ct_good_ratio, smoke_thrown_ct_foes,
-  counterStrafingShotsGoodRatio, flashbangHitFoe, flashbangLeadingToKill,
+    counter_strafing_shots_all, counter_strafing_shots_bad,
-  flashbangHitFoeAvgDuration, flashbangHitFriend, flashbangThrown, flashAssist,
+    counter_strafing_shots_good, counter_strafing_shots_good_ratio,
-  score, initialTeamNumber, mvps, ctRoundsWon, ctRoundsLost, tRoundsWon,
+    flashbang_hit_foe, flashbang_leading_to_kill,
-  tRoundsLost, sprayAccuracy, molotovFoesDamageAvg, molotovFriendsDamageAvg,
+    flashbang_hit_foe_avg_duration, flashbang_hit_friend, flashbang_thrown,
-  color, totalKills, totalDeaths, kdRatio, multi2k, multi3k, multi4k, multi5k,
+    flash_assist, score, initial_Team_Number, mvps, ct_rounds_won,
-  hltvRating, hsp, roundsSurvived, roundsSurvivedPercentage, dpr, totalAssists,
+    ct_rounds_lost, t_rounds_won, t_rounds_lost, spray_accuracy,
-  totalDamage, tradeKillOpportunities, tradeKillAttempts, tradeKillsSucceeded,
+    molotov_foes_damage_avg, molotov_friends_damage_avg, color,
-  tradeKillAttemptsPercentage, tradeKillsSuccessPercentage,
+    total_kills, total_deaths, kd_ratio, multi2k, multi3k, multi4k, multi5k,
-  tradeKillOpportunitiesPerRound, tradedDeathOpportunities,
+    hltv_rating, hsp, rounds_survived, rounds_survived_percentage, dpr,
-  tradedDeathAttempts, tradedDeathAttemptsPercentage, tradedDeathsSucceeded,
+    total_assists, total_damage, trade_kill_opportunities, trade_kill_attempts,
-  tradedDeathsSuccessPercentage, leetifyRating, personalPerformanceRating,
+    trade_kills_succeeded, trade_kill_attempts_percentage,
-  ctLeetifyRating, tLeetifyRating, leetifyUserId, isCollector, isProPlan,
+    trade_kills_success_percentage, trade_kill_opportunities_per_round,
-  isLeetifyStaff
+    traded_death_opportunities, traded_death_attempts,
+    traded_death_attempts_percentage, traded_deaths_succeeded,
+    traded_deaths_success_percentage, traded_deaths_opportunities_per_round,
+    leetify_rating, personal_performance_rating, ct_leetify_rating,
+    t_leetify_rating, leetify_user_id,
+    is_collector, is_pro_plan, is_leetify_staff
 FROM data.player_stats;
 
 ROLLBACK;

sqitch/verify/profile-game.sql

@@ -3,10 +3,10 @@
 BEGIN;
 
 -- XXX Add verifications here.
-SELECT leetifyUserId, ctLeetifyRating, ctLeetifyRatingRounds, dataSource, elo,
+SELECT leetify_user_id, ct_Leetify_rating, ct_Leetify_rating_rounds,
-      gameFinishedAt, gameId, isCs2, mapName, matchResult, scores, skillLevel,
+  data_source, elo, game_finished_at, game_id, is_cs2, map_name, match_result,
-      tLeetifyRating, tLeetifyRatingRounds, deaths, hasBannedPlayer,
+  scores, skill_level, t_leetify_rating, t_leetify_rating_rounds, deaths,
-      kills, partySize
+  has_banned_player, kills, party_size
 FROM data.profile_game
 WHERE FALSE;
 

sqitch/verify/profile-meta.sql

@@ -2,8 +2,8 @@
 
 BEGIN;
 
-SELECT steam64Id, isCollector, isLeetifyStaff,
+SELECT name, steam64_id, is_collector, is_leetify_staff, is_pro_plan,
-       isProPlan, leetifyUserId, faceitNickname
+    leetify_user_id, faceit_nickname
 FROM data.profile_meta
 WHERE FALSE;
 

src/api.py

@@ -0,0 +1,55 @@
+from fastapi import FastAPI, status
+from transform import get_db_cols
+from datetime import datetime
+import db
+
+
+app = FastAPI()
+
+
+@app.get("/players")
+async def get_players():
+    cols = get_db_cols("profile_meta")
+    profiles = db.get_all_profiles()
+    profiles = map(
+        lambda profile: {cols[i]: profile[i] for i, _ in enumerate(profile)},
+        profiles,
+    )
+
+    return list(profiles)
+
+
+@app.get("/players/{player_id}/games")
+async def get_player_games(
+    player_id,
+    begin: datetime | None = None,
+    end: datetime | None = None,
+):
+    print(begin)
+    print(end)
+
+    game_ids = db.get_profile_game_ids(player_id, begin=begin, end=end)
+    print(len(game_ids))
+    if len(game_ids) == 0:
+        return status.HTTP_404_NOT_FOUND
+
+    return {"game_ids": game_ids}
+
+
+@app.get("/players/{player_id}/player_stats/")
+async def get_player_stats(player_id: str, game_ids_param: str | None = None):
+    if not game_ids_param or len(game_ids_param) == 0:
+        game_ids = db.get_profile_game_ids(player_id)
+    else:
+        game_ids = [id.strip('" ') for id in game_ids_param.split(",")]
+
+    print(len(game_ids))
+    cols = get_db_cols("player_stats")
+    stats = db.get_player_stats(game_ids)
+    print(len(stats))
+    stats = map(
+        lambda stat: {cols[i]: stat[i] for i, _ in enumerate(stat)},
+        stats,
+    )
+
+    return {"player_id": player_id, "player_stats": list(stats)}

src/db.py

@@ -0,0 +1,149 @@
+import psycopg2 as pg
+from os import environ as env
+from transform import vals_in_order, get_cols
+
+
+def connect():
+    print("connecting")
+    return pg.connect(
+        dbname=env.get("POSTGRES_DB"),
+        user=env.get("POSTGRES_USER"),
+        password=env.get("POSTGRES_PASSWORD"),
+        host=env.get("POSTGRES_HOST"),
+        port=env.get("POSTGRES_PORT"),
+    )
+
+
+def connect_by_default(func, conn_func=connect):
+    def wrapper(*args, **kargs):
+        if not kargs.get("conn"):
+            kargs["conn"] = conn_func()
+        return func(*args, **kargs)
+
+    return wrapper
+
+
+@connect_by_default
+def get_all_profile_names(conn=None):
+    with conn.cursor() as curs:
+        curs.execute("""SELECT name, leetify_user_id FROM data.profile_meta;""")
+        data = curs.fetchall()
+
+    return data
+
+
+@connect_by_default
+def get_all_profiles(conn=None):
+    with conn.cursor() as curs:
+        curs.execute("SELECT * from data.profile_meta;")
+        data = curs.fetchall()
+
+    return data
+
+
+@connect_by_default
+def get_profile_game_count(user_id, conn=None):
+    with conn.cursor() as curs:
+        curs.execute(
+            """SELECT count(*) FROM data.profile_game
+               WHERE leetify_user_id = %s;""",
+            (user_id,),
+        )
+        data = curs.fetchone()
+
+    if not data:
+        data = [0]
+    return data[0]
+
+
+@connect_by_default
+def get_profile_game_ids(user_id, begin=None, end=None, conn=None):
+    sql = """
+            SELECT game_id FROM data.profile_game
+            WHERE leetify_user_id = %s"""
+    vals = [user_id]
+    if begin:
+        sql += """
+            AND game_finished_at >= (%s)
+        """
+        vals.append(begin)
+    if end:
+        sql += """
+            AND game_finished_at < (%s)
+        """
+        vals.append(end)
+    sql += ";"
+    with conn.cursor() as curs:
+        curs.execute(sql, (*vals,))
+        data = curs.fetchall()
+
+    return [row[0] for row in data]
+
+
+@connect_by_default
+def get_player_stats(game_ids, conn=None):
+    sql = """
+            SELECT * FROM data.player_stats
+            WHERE game_id = ANY(%s);
+            """
+    with conn.cursor() as curs:
+        curs.execute(sql, (game_ids,))
+        data = curs.fetchall()
+
+    return data
+
+
+@connect_by_default
+def insert_player_stats(stats, conn=None):
+    cols_api, cols_db = get_cols("player_stats")
+    if type(cols_api) != list or type(cols_db) != list:
+        raise Exception(
+            "could not get columns for player_stats:", cols_api, cols_db
+        )
+
+    vals = map(lambda game: vals_in_order(game, cols_api), stats)
+
+    with conn.cursor() as curs:
+        sql = f"""
+              INSERT into data.player_stats ({', '.join(cols_db)})
+              VALUES ({', '.join(['%s'] * len(cols_db))}); """
+        curs.executemany(sql, vals)
+    conn.commit()
+
+
+@connect_by_default
+def insert_profile_meta(meta, conn=None):
+    cols_api, cols_db = get_cols("profile_meta")
+    if type(cols_api) != list or type(cols_db) != list:
+        raise Exception(
+            "could not get columns for profile_meta:", cols_api, cols_db
+        )
+
+    vals = vals_in_order(meta, cols_api)
+    sql = f"""
+    INSERT INTO data.profile_meta ({', '.join(cols_db)})
+    VALUES ({', '.join(['%s'] * len(cols_db))});
+    """
+
+    with conn.cursor() as curs:
+        curs.execute(sql, vals)
+    conn.commit()
+
+
+@connect_by_default
+def insert_profile_games(games, conn=None):
+    cols_api, cols_db = get_cols("profile_game")
+    if type(cols_api) != list or type(cols_db) != list:
+        raise Exception(
+            "could not get columns for profile_game:", cols_api, cols_db
+        )
+
+    vals = map(lambda game: vals_in_order(game, cols_api), games)
+    sql = f"""
+        INSERT into data.profile_game ({', '.join(cols_db)})
+        VALUES ({', '.join(['%s'] * len(cols_db))});
+    """
+
+    with conn.cursor() as curs:
+        curs.executemany(sql, vals)
+    conn.commit()

src/extract.py

@@ -1,12 +1,100 @@
-import psycopg2 as pg
+from leetify import Leetify
-from os import environ as env
+from transform import (
+    get_cols,
+    meta_from_profile,
+    games_from_profile,
+    player_stats_from_game,
+)
+import db
+import logging
 
 
-def connect():
+@db.connect_by_default
-    return pg.connect(
+def extract_profile_meta(id, api=Leetify(), conn=None):
-        dbname=env.get("POSTGRES_DB"),
+    cols_api, cols_db = get_cols("profile_meta")
-        user=env.get("POSTGRES_USER"),
+    if type(cols_api) != list or type(cols_db) != list:
-        password=env.get("POSTGRES_PASSWORD"),
+        raise Exception(
-        host=env.get("POSTGRES_HOST"),
+            "could not get columns for profile_meta:", cols_api, cols_db
-        port=env.get("POSTGRES_PORT"),
+        )
-    )
+
+    profile = api.get_profile(id)
+    meta = meta_from_profile(profile)
+
+    db.insert_profile_meta(meta, conn=conn)
+
+
+@db.connect_by_default
+def extract_profile_games(profile, conn=None):
+    cols_api, cols_db = get_cols("profile_games")
+    if type(cols_api) != list or type(cols_db) != list:
+        raise Exception(
+            "could not get columns for profile_game:", cols_api, cols_db
+        )
+
+    games = games_from_profile(profile)
+
+    db.insert_profile_games(games, conn)
+
+
+@db.connect_by_default
+def extract_player_stats(game, conn=None):
+    stats = player_stats_from_game(game)
+
+    db.insert_player_stats(stats, conn=conn)
+
+
+@db.connect_by_default
+def insert_new_profile_games(games, user_id, conn=None):
+    game_ids = db.get_profile_game_ids(user_id, conn=conn)
+
+    game_ids = set(game_ids)
+    new_games = [game for game in games if game.get("gameId") not in game_ids]
+
+    logging.info(f"Inserting {len(new_games)} for: {user_id}")
+    db.insert_profile_games(new_games, conn=conn)
+
+    return new_games
+
+
+@db.connect_by_default
+def insert_player_stats(game_ids, api=Leetify(), conn=None):
+    logging.info(f"Inserting player_stats from {len(game_ids)} games")
+    print_every = int(len(game_ids) / 10)
+    for i, id in enumerate(game_ids):
+        if i % print_every == 0:
+            logging.info(f"Inserted player_stats from {i}/{len(game_ids)} games")
+        game = api.get_match(id)
+        extract_player_stats(game, conn=conn)
+
+
+def get_all(api=Leetify()):
+    conn = db.connect()
+
+    data = db.get_all_profile_names(conn=conn)
+    game_ids = []
+    for name, id in data:
+        logging.info(f"Getting data for {name}: {id}")
+        profile = api.get_profile(id)
+        profile_games = games_from_profile(profile)
+
+        api_games = len(profile_games)
+        db_games = db.get_profile_game_count(id, conn=conn)
+
+        if api_games > db_games:
+            logging.info(f"Getting new games for {name}: {id}")
+            new_games = insert_new_profile_games(profile_games, id, conn=conn)
+            new_game_ids = [game.get("gameId") for game in new_games]
+            game_ids.extend(new_game_ids)
+        elif api_games < db_games:
+            logging.error(f"API returned less games then in DB for {name}: {id}")
+
+        logging.info(f"Games synced with Leetify for {name}: {id}")
+
+    logging.info(f"Updating player_stats with {len(game_ids)} new games")
+    insert_player_stats(game_ids, conn=conn)
+    logging.info("DONE Updating player_stats with new games")
+
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.INFO)
+    get_all()

src/leetify/core.py

@@ -1,4 +1,7 @@
+from ratelimit.decorators import sleep_and_retry
 import requests
+from ratelimit import limits, RateLimitException
+from backoff import expo, on_exception
 
 
 class Leetify:
@@ -6,6 +9,9 @@ class Leetify:
     profile_base_url = f"{api_base_url}/profile"
     match_base_url = f"{api_base_url}/games"
 
+    @on_exception(expo, Exception, max_tries=3)
+    @sleep_and_retry
+    @limits(1, 5)
     def __get_page(self, url: str) -> dict:
         resp = requests.get(url)
         return resp.json()

src/transform.py

@@ -1,9 +1,15 @@
+from types import NoneType
 from typing import TypeVar
 
 T = TypeVar("T")
 U = TypeVar("U")
 
 
+def vals_in_order(data: dict, keys: list):
+    vals = [data[key] for key in keys]
+    return vals
+
+
 def extract_cols(data: dict, cols: list[str]) -> dict:
     return {key: data.get(key) for key in cols}
 
@@ -12,116 +18,274 @@ def score_to_text(score: list[int]) -> str:
     return "-".join(map(str, score))
 
 
+## returns column names if the form:
+## {"<table_name>": ([<names from api>]}, [<names from db>])}
+def get_cols(table_name=None):
+    cols = {
+        "profile_game": (
+            [
+                "leetifyUserId",
+                "ctLeetifyRating",
+                "ctLeetifyRatingRounds",
+                "dataSource",
+                "elo",
+                "gameFinishedAt",
+                "gameId",
+                "isCs2",
+                "mapName",
+                "matchResult",
+                "scores",
+                "skillLevel",
+                "tLeetifyRating",
+                "tLeetifyRatingRounds",
+                "deaths",
+                "hasBannedPlayer",
+                "kills",
+                "partySize",
+            ],
+            [
+                "leetify_user_id",
+                "ct_Leetify_rating",
+                "ct_Leetify_rating_rounds",
+                "data_source",
+                "elo",
+                "game_finished_at",
+                "game_id",
+                "is_cs2",
+                "map_name",
+                "match_result",
+                "scores",
+                "skill_level",
+                "t_leetify_rating",
+                "t_leetify_rating_rounds",
+                "deaths",
+                "has_banned_player",
+                "kills",
+                "party_size",
+            ],
+        ),
+        "player_stats": (
+            [
+                "id",
+                "gameId",
+                "gameFinishedAt",
+                "steam64Id",
+                "name",
+                "preaim",
+                "reactionTime",
+                "accuracy",
+                "accuracyEnemySpotted",
+                "accuracyHead",
+                "shotsFiredEnemySpotted",
+                "shotsFired",
+                "shotsHitEnemySpotted",
+                "shotsHitFriend",
+                "shotsHitFriendHead",
+                "shotsHitFoe",
+                "shotsHitFoeHead",
+                "utilityOnDeathAvg",
+                "heFoesDamageAvg",
+                "heFriendsDamageAvg",
+                "heThrown",
+                "molotovThrown",
+                "smokeThrown",
+                "smokeThrownCT",
+                "smokeThrownCTGood",
+                "smokeThrownCTGoodRatio",
+                "smokeThrownCTFoes",
+                "counterStrafingShotsAll",
+                "counterStrafingShotsBad",
+                "counterStrafingShotsGood",
+                "counterStrafingShotsGoodRatio",
+                "flashbangHitFoe",
+                "flashbangLeadingToKill",
+                "flashbangHitFoeAvgDuration",
+                "flashbangHitFriend",
+                "flashbangThrown",
+                "flashAssist",
+                "score",
+                "initialTeamNumber",
+                "mvps",
+                "ctRoundsWon",
+                "ctRoundsLost",
+                "tRoundsWon",
+                "tRoundsLost",
+                "sprayAccuracy",
+                "molotovFoesDamageAvg",
+                "molotovFriendsDamageAvg",
+                "color",
+                "totalKills",
+                "totalDeaths",
+                "kdRatio",
+                "multi2k",
+                "multi3k",
+                "multi4k",
+                "multi5k",
+                "hltvRating",
+                "hsp",
+                "roundsSurvived",
+                "roundsSurvivedPercentage",
+                "dpr",
+                "totalAssists",
+                "totalDamage",
+                "tradeKillOpportunities",
+                "tradeKillAttempts",
+                "tradeKillsSucceeded",
+                "tradeKillAttemptsPercentage",
+                "tradeKillsSuccessPercentage",
+                "tradeKillOpportunitiesPerRound",
+                "tradedDeathOpportunities",
+                "tradedDeathAttempts",
+                "tradedDeathAttemptsPercentage",
+                "tradedDeathsSucceeded",
+                "tradedDeathsSuccessPercentage",
+                "tradedDeathsOpportunitiesPerRound",
+                "leetifyRating",
+                "personalPerformanceRating",
+                "ctLeetifyRating",
+                "tLeetifyRating",
+                "leetifyUserId",
+                "isCollector",
+                "isProPlan",
+                "isLeetifyStaff",
+            ],
+            [
+                "id",
+                "game_id",
+                "game_finished_at",
+                "steam64_id",
+                "name",
+                "preaim",
+                "reaction_time",
+                "accuracy",
+                "accuracy_enemy_spotted",
+                "accuracy_head",
+                "shots_fired_enemy_spotted",
+                "shots_fired",
+                "shots_hit_enemy_spotted",
+                "shots_hit_friend",
+                "shots_Hit_Friend_Head",
+                "shots_Hit_Foe",
+                "shots_Hit_Foe_Head",
+                "utility_On_Death_Avg",
+                "he_foes_damage_avg",
+                "he_friends_damage_avg",
+                "he_thrown",
+                "molotov_thrown",
+                "smoke_thrown",
+                "smoke_thrown_ct",
+                "smoke_thrown_ct_good",
+                "smoke_thrown_ct_good_ratio",
+                "smoke_thrown_ct_foes",
+                "counter_strafing_shots_all",
+                "counter_strafing_shots_bad",
+                "counter_strafing_shots_good",
+                "counter_strafing_shots_good_ratio",
+                "flashbang_hit_foe",
+                "flashbang_leading_to_kill",
+                "flashbang_hit_foe_avg_duration",
+                "flashbang_hit_friend",
+                "flashbang_thrown",
+                "flash_assist",
+                "score",
+                "initial_Team_Number",
+                "mvps",
+                "ct_rounds_won",
+                "ct_rounds_lost",
+                "t_rounds_won",
+                "t_rounds_lost",
+                "spray_accuracy",
+                "molotov_foes_damage_avg",
+                "molotov_friends_damage_avg",
+                "color",
+                "total_kills",
+                "total_deaths",
+                "kd_ratio",
+                "multi2k",
+                "multi3k",
+                "multi4k",
+                "multi5k",
+                "hltv_rating",
+                "hsp",
+                "rounds_survived",
+                "rounds_survived_percentage",
+                "dpr",
+                "total_assists",
+                "total_damage",
+                "trade_kill_opportunities",
+                "trade_kill_attempts",
+                "trade_kills_succeeded",
+                "trade_kill_attempts_percentage",
+                "trade_kills_success_percentage",
+                "trade_kill_opportunities_per_round",
+                "traded_death_opportunities",
+                "traded_death_attempts",
+                "traded_death_attempts_percentage",
+                "traded_deaths_succeeded",
+                "traded_deaths_success_percentage",
+                "traded_deaths_opportunities_per_round",
+                "leetify_rating",
+                "personal_performance_rating",
+                "ct_leetify_rating",
+                "t_leetify_rating",
+                "leetify_user_id",
+                "is_collector",
+                "is_pro_plan",
+                "is_leetify_staff",
+            ],
+        ),
+        "profile_meta": (
+            [
+                "name",
+                "steam64Id",
+                "isCollector",
+                "isLeetifyStaff",
+                "isProPlan",
+                "leetifyUserId",
+                "faceitNickname",
+            ],
+            [
+                "name",
+                "steam64_id",
+                "is_collector",
+                "is_leetify_staff",
+                "is_pro_plan",
+                "leetify_user_id",
+                "faceit_nickname",
+            ],
+        ),
+    }
+    if table_name:
+        return cols[table_name]
+    return cols
+
+
+def get_api_cols(table_name: str | NoneType = None):
+    col_pos = 0
+    if table_name:
+        return get_cols(table_name)[col_pos]
+    return {key: val[col_pos] for key, val in get_cols().items()}
+
+
+def get_db_cols(table_name=None):
+    col_pos = 1
+    if table_name:
+        return get_cols(table_name)[col_pos]
+    return {key: val[col_pos] for key, val in get_cols().items()}
+
+
 # maybe could get columns form db
 def cols_from_player_stats(player_stats: dict) -> dict:
-    cols = [
+    cols = get_api_cols("player_stats")
-        "id",
+    if type(cols) != list:
-        "gameId",
+        raise Exception("could not get api columns for player_stats")
-        "gameFinishedAt",
-        "steam64Id",
-        "name",
-        "preaim",
-        "reactionTime",
-        "accuracy",
-        "accuracyEnemySpotted",
-        "accuracyHead",
-        "shotsFiredEnemySpotted",
-        "shotsFired",
-        "shotsHitEnemySpotted",
-        "shotsHitFriend",
-        "shotsHitFriendHead",
-        "shotsHitFoe",
-        "shotsHitFoeHead",
-        "utilityOnDeathAvg",
-        "heFoesDamageAvg",
-        "heFriendsDamageAvg",
-        "heThrown",
-        "molotovThrown",
-        "smokeThrown",
-        "smokeThrownCT",
-        "smokeThrownCTGood",
-        "smokeThrownCTGoodRatio",
-        "smokeThrownCTFoes",
-        "counterStrafingShotsAll",
-        "counterStrafingShotsBad",
-        "counterStrafingShotsGood",
-        "counterStrafingShotsGoodRatio",
-        "flashbangHitFoe",
-        "flashbangLeadingToKill",
-        "flashbangHitFoeAvgDuration",
-        "flashbangHitFriend",
-        "flashbangThrown",
-        "flashAssist",
-        "score",
-        "initialTeamNumber",
-        "mvps",
-        "ctRoundsWon",
-        "ctRoundsLost",
-        "tRoundsWon",
-        "tRoundsLost",
-        "sprayAccuracy",
-        "molotovFoesDamageAvg",
-        "molotovFriendsDamageAvg",
-        "color",
-        "totalKills",
-        "totalDeaths",
-        "kdRatio",
-        "multi2k",
-        "multi3k",
-        "multi4k",
-        "multi5k",
-        "hltvRating",
-        "hsp",
-        "roundsSurvived",
-        "roundsSurvivedPercentage",
-        "dpr",
-        "totalAssists",
-        "totalDamage",
-        "tradeKillOpportunities",
-        "tradeKillAttempts",
-        "tradeKillsSucceeded",
-        "tradeKillAttemptsPercentage",
-        "tradeKillsSuccessPercentage",
-        "tradeKillOpportunitiesPerRound",
-        "tradedDeathOpportunities",
-        "tradedDeathAttempts",
-        "tradedDeathAttemptsPercentage",
-        "tradedDeathsSucceeded",
-        "tradedDeathsSuccessPercentage",
-        "tradedDeathsOpportunitiesPerRound",
-        "leetifyRating",
-        "personalPerformanceRating",
-        "ctLeetifyRating",
-        "tLeetifyRating",
-        "leetifyUserId",
-        "isCollector",
-        "isProPlan",
-        "isLeetifyStaff",
-    ]
-
     return extract_cols(player_stats, cols)
 
 
 def cols_from_profile_game(game: dict) -> dict:
-    cols = [
+    cols = get_api_cols("profile_game")
-        "ctLeetifyRating",
+    if type(cols) != list:
-        "ctLeetifyRatingRounds",
+        raise Exception("could not get api columns for profile_game")
-        "dataSource",
-        "elo",
-        "gameFinishedAt",
-        "gameId",
-        "isCs2",
-        "mapName",
-        "matchResult",
-        "scores",
-        "skillLevel",
-        "tLeetifyRating",
-        "tLeetifyRatingRounds",
-        "deaths",
-        "hasBannedPlayer",
-        "kills",
-        "partySize",
-    ]
     return extract_cols(game, cols)
 
 
@@ -130,15 +294,9 @@ def meta_from_profile(profile: dict) -> dict:
     if not meta:
         raise Exception("Could not get profile metadata", profile)
 
-    cols = [
+    cols = get_api_cols("profile_meta")
-        "name",
+    if type(cols) != list:
-        "steam64Id",
+        raise Exception("could not get api columns for profile_meta")
-        "isCollector",
-        "isLeetifyStaff",
-        "isProPlan",
-        "leetifyUserId",
-        "faceitNickname",
-    ]
 
     return extract_cols(meta, cols)
 
@@ -151,7 +309,7 @@ def insert_value(data: dict[T, U], key: T, value: U) -> dict[T, U]:
 def convert_game_scores(game: dict) -> dict:
     score = game.get("scores")
     if not score:
-        raise Exception("Could not get score from prfile game", game)
+        raise Exception("Could not get score from profile game", game)
 
     score = score_to_text(score)
     game["scores"] = score
@@ -183,6 +341,5 @@ def player_stats_from_game(game: dict) -> list[dict]:
     if not stats:
         raise Exception("Could not get stats from game", game)
 
-
+    stats = map(cols_from_player_stats, stats)
-    stats = map(cols_from_player_stats,stats)
     return list(stats)