refactored functions to use centeralized column name definition

requirments.txt

@@ -1,3 +1,5 @@
 pytest
 click
-psycopg2
+psycopg2-binary
+ratelimit
+backoff

sqitch/deploy/player-stats.sql

@@ -5,87 +5,87 @@ BEGIN;
 
   CREATE TABLE IF NOT EXISTS data.player_stats (
     id TEXT,
-    gameId TEXT,
-    gameFinishedAt TIMESTAMP,
-    steam64Id TEXT,
+    game_id TEXT,
+    game_finished_at TIMESTAMP,
+    steam64_id TEXT,
     name TEXT,
     preaim NUMERIC,
-    reactionTime NUMERIC,
+    reaction_time NUMERIC,
     accuracy NUMERIC,
-    accuracyEnemySpotted NUMERIC,
-    accuracyHead NUMERIC,
-    shotsFiredEnemySpotted NUMERIC,
-    shotsFired NUMERIC,
-    shotsHitEnemySpotted NUMERIC,
-    shotsHitFriend NUMERIC,
-    shotsHitFriendHead NUMERIC,
-    shotsHitFoe NUMERIC,
-    shotsHitFoeHead NUMERIC,
-    utilityOnDeathAvg NUMERIC,
-    heFoesDamageAvg NUMERIC,
-    heFriendsDamageAvg NUMERIC,
-    heThrown NUMERIC,
-    molotovThrown NUMERIC,
-    smokeThrown NUMERIC,
-    smokeThrownCT NUMERIC,
-    smokeThrownCTGood NUMERIC,
-    smokeThrownCTGoodRatio NUMERIC,
-    smokeThrownCTFoes NUMERIC,
-    counterStrafingShotsAll NUMERIC,
-    counterStrafingShotsBad NUMERIC,
-    counterStrafingShotsGood NUMERIC,
-    counterStrafingShotsGoodRatio NUMERIC,
-    flashbangHitFoe NUMERIC,
-    flashbangLeadingToKill NUMERIC,
-    flashbangHitFoeAvgDuration NUMERIC,
-    flashbangHitFriend NUMERIC,
-    flashbangThrown NUMERIC,
-    flashAssist NUMERIC,
+    accuracy_enemy_spotted NUMERIC,
+    accuracy_head NUMERIC,
+    shots_fired_enemy_spotted NUMERIC,
+    shots_fired NUMERIC,
+    shots_hit_enemy_spotted NUMERIC,
+    shots_hit_friend NUMERIC,
+    shots_Hit_Friend_Head NUMERIC,
+    shots_Hit_Foe NUMERIC,
+    shots_Hit_Foe_Head NUMERIC,
+    utility_On_Death_Avg NUMERIC,
+    he_foes_damage_avg NUMERIC,
+    he_friends_damage_avg NUMERIC,
+    he_thrown NUMERIC,
+    molotov_thrown NUMERIC,
+    smoke_thrown NUMERIC,
+    smoke_thrown_ct NUMERIC,
+    smoke_thrown_ct_good NUMERIC,
+    smoke_thrown_ct_good_ratio NUMERIC,
+    smoke_thrown_ct_foes NUMERIC,
+    counter_strafing_shots_all NUMERIC,
+    counter_strafing_shots_bad NUMERIC,
+    counter_strafing_shots_good NUMERIC,
+    counter_strafing_shots_good_ratio NUMERIC,
+    flashbang_hit_foe NUMERIC,
+    flashbang_leading_to_kill NUMERIC,
+    flashbang_hit_foe_avg_duration NUMERIC,
+    flashbang_hit_friend NUMERIC,
+    flashbang_thrown NUMERIC,
+    flash_assist NUMERIC,
     score NUMERIC,
-    initialTeamNumber NUMERIC,
+    initial_Team_Number NUMERIC,
     mvps NUMERIC,
-    ctRoundsWon NUMERIC,
-    ctRoundsLost NUMERIC,
-    tRoundsWon NUMERIC,
-    tRoundsLost NUMERIC,
-    sprayAccuracy NUMERIC,
-    molotovFoesDamageAvg NUMERIC,
-    molotovFriendsDamageAvg NUMERIC,
+    ct_rounds_won NUMERIC,
+    ct_rounds_lost NUMERIC,
+    t_rounds_won NUMERIC,
+    t_rounds_lost NUMERIC,
+    spray_accuracy NUMERIC,
+    molotov_foes_damage_avg NUMERIC,
+    molotov_friends_damage_avg NUMERIC,
     color NUMERIC,
-    totalKills NUMERIC,
-    totalDeaths NUMERIC,
-    kdRatio NUMERIC,
+    total_kills NUMERIC,
+    total_deaths NUMERIC,
+    kd_ratio NUMERIC,
     multi2k NUMERIC,
     multi3k NUMERIC,
     multi4k NUMERIC,
     multi5k NUMERIC,
-    hltvRating NUMERIC,
+    hltv_rating NUMERIC,
     hsp NUMERIC,
-    roundsSurvived NUMERIC,
-    roundsSurvivedPercentage NUMERIC,
+    rounds_survived NUMERIC,
+    rounds_survived_percentage NUMERIC,
     dpr NUMERIC,
-    totalAssists NUMERIC,
-    totalDamage NUMERIC,
-    tradeKillOpportunities NUMERIC,
-    tradeKillAttempts NUMERIC,
-    tradeKillsSucceeded NUMERIC,
-    tradeKillAttemptsPercentage NUMERIC,
-    tradeKillsSuccessPercentage NUMERIC,
-    tradeKillOpportunitiesPerRound NUMERIC,
-    tradedDeathOpportunities NUMERIC,
-    tradedDeathAttempts NUMERIC,
-    tradedDeathAttemptsPercentage NUMERIC,
-    tradedDeathsSucceeded NUMERIC,
-    tradedDeathsSuccessPercentage NUMERIC,
-    tradedDeathsOpportunitiesPerRound NUMERIC,
-    leetifyRating NUMERIC,
-    personalPerformanceRating NUMERIC,
-    ctLeetifyRating NUMERIC,
-    tLeetifyRating NUMERIC,
-    leetifyUserId TEXT,
-    isCollector BOOL,
-    isProPlan BOOL,
-    isLeetifyStaff BOOL
+    total_assists NUMERIC,
+    total_damage NUMERIC,
+    trade_kill_opportunities NUMERIC,
+    trade_kill_attempts NUMERIC,
+    trade_kills_succeeded NUMERIC,
+    trade_kill_attempts_percentage NUMERIC,
+    trade_kills_success_percentage NUMERIC,
+    trade_kill_opportunities_per_round NUMERIC,
+    traded_death_opportunities NUMERIC,
+    traded_death_attempts NUMERIC,
+    traded_death_attempts_percentage NUMERIC,
+    traded_deaths_succeeded NUMERIC,
+    traded_deaths_success_percentage NUMERIC,
+    traded_deaths_opportunities_per_round NUMERIC,
+    leetify_rating NUMERIC,
+    personal_performance_rating NUMERIC,
+    ct_leetify_rating NUMERIC,
+    t_leetify_rating NUMERIC,
+    leetify_user_id TEXT,
+    is_collector BOOL,
+    is_pro_plan BOOL,
+    is_leetify_staff BOOL
   );
 
 COMMIT;

sqitch/deploy/profile-game.sql

@@ -4,24 +4,24 @@
 BEGIN;
 
   CREATE TABLE IF NOT EXISTS data.profile_game (
-     leetifyUserId TEXT,
-     ctLeetifyRating NUMERIC,
-     ctLeetifyRatingRounds NUMERIC,
-     dataSource NUMERIC,
+     leetify_user_id TEXT,
+     ct_Leetify_rating NUMERIC,
+     ct_Leetify_rating_rounds NUMERIC,
+     data_source TEXT,
      elo NUMERIC,
-     gameFinishedAt TIMESTAMP,
-     gameId TEXT,
-     isCs2 BOOL,
-     mapName TEXT,
-     matchResult TEXT,
+     game_finished_at TIMESTAMP,
+     game_id TEXT,
+     is_cs2 BOOL,
+     map_name TEXT,
+     match_result TEXT,
      scores TEXT, -- has to be extracted from array
-     skillLevel NUMERIC,
-     tLeetifyRating NUMERIC,
-     tLeetifyRatingRounds NUMERIC,
+     skill_level NUMERIC,
+     t_leetify_rating NUMERIC,
+     t_leetify_rating_rounds NUMERIC,
      deaths NUMERIC,
-     hasBannedPlayer BOOL,
+     has_banned_player BOOL,
      kills NUMERIC,
-     partySize NUMERIC
+     party_size NUMERIC
   );
 
 

sqitch/deploy/profile-meta.sql

@@ -4,12 +4,13 @@
 BEGIN;
 
   CREATE TABLE IF NOT EXISTS data.profile_meta (
-    steam64Id TEXT,
-    isCollector BOOL,
-    isLeetifyStaff BOOL,
-    isProPlan BOOL,
-    leetifyUserId TEXT,
-    faceitNickname TEXT
+    name TEXT,
+    steam64_id TEXT,
+    is_collector BOOL,
+    is_leetify_staff BOOL,
+    is_pro_plan BOOL,
+    leetify_user_id TEXT,
+    faceit_nickname TEXT
   );
 
 COMMIT;

sqitch/verify/player-stats.sql

@@ -3,26 +3,31 @@
 BEGIN;
 
 -- XXX Add verifications here.
-SELECT id, gameId, gameFinishedAt, steam64Id, name, preaim, reactionTime,
-  accuracy, accuracyEnemySpotted, accuracyHead, shotsFiredEnemySpotted,
-  shotsFired, shotsHitEnemySpotted, shotsHitFriend, shotsHitFriendHead,
-  shotsHitFoe, shotsHitFoeHead, utilityOnDeathAvg, heFoesDamageAvg,
-  heFriendsDamageAvg, heThrown, molotovThrown, smokeThrown,
-  smokeThrownCT, smokeThrownCTGood, smokeThrownCTGoodRatio, smokeThrownCTFoes,
-  counterStrafingShotsAll, counterStrafingShotsBad, counterStrafingShotsGood,
-  counterStrafingShotsGoodRatio, flashbangHitFoe, flashbangLeadingToKill,
-  flashbangHitFoeAvgDuration, flashbangHitFriend, flashbangThrown, flashAssist,
-  score, initialTeamNumber, mvps, ctRoundsWon, ctRoundsLost, tRoundsWon,
-  tRoundsLost, sprayAccuracy, molotovFoesDamageAvg, molotovFriendsDamageAvg,
-  color, totalKills, totalDeaths, kdRatio, multi2k, multi3k, multi4k, multi5k,
-  hltvRating, hsp, roundsSurvived, roundsSurvivedPercentage, dpr, totalAssists,
-  totalDamage, tradeKillOpportunities, tradeKillAttempts, tradeKillsSucceeded,
-  tradeKillAttemptsPercentage, tradeKillsSuccessPercentage,
-  tradeKillOpportunitiesPerRound, tradedDeathOpportunities,
-  tradedDeathAttempts, tradedDeathAttemptsPercentage, tradedDeathsSucceeded,
-  tradedDeathsSuccessPercentage, leetifyRating, personalPerformanceRating,
-  ctLeetifyRating, tLeetifyRating, leetifyUserId, isCollector, isProPlan,
-  isLeetifyStaff
+SELECT id, game_id, game_finished_at, steam64_id, name, preaim, reaction_time,
+    accuracy, accuracy_enemy_spotted, accuracy_head, shots_fired_enemy_spotted,
+    shots_fired, shots_hit_enemy_spotted, shots_hit_friend,
+    shots_Hit_Friend_Head, shots_Hit_Foe, shots_Hit_Foe_Head,
+    utility_On_Death_Avg, he_foes_damage_avg, he_friends_damage_avg, he_thrown,
+    molotov_thrown, smoke_thrown, smoke_thrown_ct, smoke_thrown_ct_good,
+    smoke_thrown_ct_good_ratio, smoke_thrown_ct_foes,
+    counter_strafing_shots_all, counter_strafing_shots_bad,
+    counter_strafing_shots_good, counter_strafing_shots_good_ratio,
+    flashbang_hit_foe, flashbang_leading_to_kill,
+    flashbang_hit_foe_avg_duration, flashbang_hit_friend, flashbang_thrown,
+    flash_assist, score, initial_Team_Number, mvps, ct_rounds_won,
+    ct_rounds_lost, t_rounds_won, t_rounds_lost, spray_accuracy,
+    molotov_foes_damage_avg, molotov_friends_damage_avg, color,
+    total_kills, total_deaths, kd_ratio, multi2k, multi3k, multi4k, multi5k,
+    hltv_rating, hsp, rounds_survived, rounds_survived_percentage, dpr,
+    total_assists, total_damage, trade_kill_opportunities, trade_kill_attempts,
+    trade_kills_succeeded, trade_kill_attempts_percentage,
+    trade_kills_success_percentage, trade_kill_opportunities_per_round,
+    traded_death_opportunities, traded_death_attempts,
+    traded_death_attempts_percentage, traded_deaths_succeeded,
+    traded_deaths_success_percentage, traded_deaths_opportunities_per_round,
+    leetify_rating, personal_performance_rating, ct_leetify_rating,
+    t_leetify_rating, leetify_user_id,
+    is_collector, is_pro_plan, is_leetify_staff
 FROM data.player_stats;
 
 ROLLBACK;

sqitch/verify/profile-game.sql

@@ -3,10 +3,10 @@
 BEGIN;
 
 -- XXX Add verifications here.
-SELECT leetifyUserId, ctLeetifyRating, ctLeetifyRatingRounds, dataSource, elo,
-      gameFinishedAt, gameId, isCs2, mapName, matchResult, scores, skillLevel,
-      tLeetifyRating, tLeetifyRatingRounds, deaths, hasBannedPlayer,
-      kills, partySize
+SELECT leetify_user_id, ct_Leetify_rating, ct_Leetify_rating_rounds,
+  data_source, elo, game_finished_at, game_id, is_cs2, map_name, match_result,
+  scores, skill_level, t_leetify_rating, t_leetify_rating_rounds, deaths,
+  has_banned_player, kills, party_size
 FROM data.profile_game
 WHERE FALSE;
 

sqitch/verify/profile-meta.sql

@@ -2,8 +2,8 @@
 
 BEGIN;
 
-SELECT steam64Id, isCollector, isLeetifyStaff,
-       isProPlan, leetifyUserId, faceitNickname
+SELECT name, steam64_id, is_collector, is_leetify_staff, is_pro_plan,
+    leetify_user_id, faceit_nickname
 FROM data.profile_meta
 WHERE FALSE;
 

src/db.py

@@ -0,0 +1,79 @@
+import psycopg2 as pg
+from os import environ as env
+from transform import vals_in_order, get_cols
+
+
+def connect():
+    print("connecting")
+    return pg.connect(
+        dbname=env.get("POSTGRES_DB"),
+        user=env.get("POSTGRES_USER"),
+        password=env.get("POSTGRES_PASSWORD"),
+        host=env.get("POSTGRES_HOST"),
+        port=env.get("POSTGRES_PORT"),
+    )
+
+
+def connect_by_default(func, conn_func=connect):
+    def wrapper(*args, **kargs):
+        if not kargs.get("conn"):
+            kargs["conn"] = conn_func()
+        return func(*args, **kargs)
+
+    return wrapper
+
+
+@connect_by_default
+def get_all_profile_names(conn=None):
+    with conn.cursor() as curs:
+        curs.execute("""SELECT name, leetify_user_id FROM data.profile_meta;""")
+        data = curs.fetchall()
+
+    return data
+
+
+@connect_by_default
+def get_profile_game_count(user_id, conn=None):
+    with conn.cursor() as curs:
+        curs.execute(
+            """SELECT count(*) FROM data.profile_game
+               WHERE leetify_user_id = %s;""",
+            (user_id,),
+        )
+        data = curs.fetchone()
+
+    if not data:
+        data = [0]
+    return data[0]
+
+
+@connect_by_default
+def get_profile_game_ids(user_id, conn=None):
+    with conn.cursor() as curs:
+        curs.execute(
+            """SELECT game_id FROM data.profile_game
+               WHERE leetify_user_id = %s;""",
+            (user_id,),
+        )
+        data = curs.fetchall()
+
+    return [row[0] for row in data]
+
+
+@connect_by_default
+def insert_profile_games(games, conn=None):
+    cols_api, cols_db = get_cols("profile_game")
+    if type(cols_api) != list or type(cols_db) != list:
+        raise Exception(
+            "could not get columns for profile_game:", cols_api, cols_db
+        )
+
+    vals = map(lambda game: vals_in_order(game, cols_api), games)
+
+    with conn.cursor() as curs:
+        sql = f"""
+            INSERT into data.profile_game ({', '.join(cols_db)})
+            VALUES ({', '.join(['%s'] * len(cols_db))});
+        """
+        curs.executemany(sql, vals)
+    conn.commit()

src/extract.py

@@ -1,12 +1,132 @@
-import psycopg2 as pg
-from os import environ as env
+from leetify import Leetify
+from transform import (
+    get_cols,
+    meta_from_profile,
+    games_from_profile,
+    player_stats_from_game,
+    vals_in_order,
+)
+import db
+import logging
+
+## TODO seperate out loading from extraction
+## this currently handles getting data from api and loading into db
 
 
-def connect():
-    return pg.connect(
-        dbname=env.get("POSTGRES_DB"),
-        user=env.get("POSTGRES_USER"),
-        password=env.get("POSTGRES_PASSWORD"),
-        host=env.get("POSTGRES_HOST"),
-        port=env.get("POSTGRES_PORT"),
+@db.connect_by_default
+def extract_profile_meta(id, api=Leetify(), conn=None):
+    cols_api, cols_db = get_cols("profile_meta")
+    if type(cols_api) != list or type(cols_db) != list:
+        raise Exception(
+            "could not get columns for profile_meta:", cols_api, cols_db
         )
+
+    profile = api.get_profile(id)
+    meta = meta_from_profile(profile)
+    vals = vals_in_order(meta, cols_api)
+
+    with conn.cursor() as curs:
+        sql = f"""
+        INSERT INTO data.profile_meta ({', '.join(cols_db)})
+        VALUES ({', '.join(['%s'] * len(cols_db))});
+        """
+        curs.execute(sql, vals)
+    conn.commit()
+
+
+@db.connect_by_default
+def extract_profile_games(profile, conn=None):
+    cols_api, cols_db = get_cols("profile_games")
+    if type(cols_api) != list or type(cols_db) != list:
+        raise Exception(
+            "could not get columns for profile_game:", cols_api, cols_db
+        )
+
+    games = games_from_profile(profile)
+    games = map(lambda game: vals_in_order(game, cols_api), games)
+
+    with conn.cursor() as curs:
+        curs.executemany(
+            f"""
+            INSERT into data.profile_game ({', '.join(cols_db)})
+            VALUES ({', '.join(['%s'] * len(cols_db))});
+            """,
+            games,
+        )
+    conn.commit()
+
+
+@db.connect_by_default
+def extract_player_stats(game, conn=None):
+    cols_api, cols_db = get_cols("player_stats")
+    if type(cols_api) != list or type(cols_db) != list:
+        raise Exception(
+            "could not get columns for player_stats:", cols_api, cols_db
+        )
+
+    stats = player_stats_from_game(game)
+    vals = map(lambda game: vals_in_order(game, cols_api), stats)
+
+    with conn.cursor() as curs:
+        sql = f"""
+              INSERT into data.player_stats ({', '.join(cols_db)})
+              VALUES ({', '.join(['%s'] * len(cols_db))}); """
+        curs.executemany(sql, vals)
+    conn.commit()
+
+
+@db.connect_by_default
+def insert_new_profile_games(games, user_id, conn=None):
+    game_ids = db.get_profile_game_ids(user_id, conn=conn)
+
+    game_ids = set(game_ids)
+    new_games = [game for game in games if game.get("gameId") not in game_ids]
+
+    logging.info(f"Inserting {len(new_games)} for: {user_id}")
+    db.insert_profile_games(new_games, conn=conn)
+
+    return new_games
+
+
+@db.connect_by_default
+def insert_player_stats(game_ids, api=Leetify(), conn=None):
+    logging.info(f"Inserting player_stats from {len(game_ids)} games")
+    print_every = int(len(game_ids) / 10)
+    for i, id in enumerate(game_ids):
+        if i % print_every == 0:
+            logging.info(f"Inserted player_stats from {i}/{len(game_ids)} games")
+        game = api.get_match(id)
+        extract_player_stats(game, conn=conn)
+
+
+def get_all(api=Leetify()):
+    conn = db.connect()
+
+    data = db.get_all_profile_names(conn=conn)
+    game_ids = []
+    for name, id in data:
+        logging.info(f"Getting data for {name}: {id}")
+        profile = api.get_profile(id)
+        profile_games = games_from_profile(profile)
+
+        api_games = len(profile_games)
+        db_games = db.get_profile_game_count(id, conn=conn)
+
+        if api_games > db_games:
+            logging.info(f"Getting new games for {name}: {id}")
+            new_games = insert_new_profile_games(profile_games, id, conn=conn)
+            new_game_ids = [game.get("gameId") for game in new_games]
+            game_ids.extend(new_game_ids)
+        elif api_games < db_games:
+            logging.error(f"API returned less games then in DB for {name}: {id}")
+
+        logging.info(f"Games synced with Leetify for {name}: {id}")
+
+    logging.info(f"Updating player_stats with {len(game_ids)} new games")
+    insert_player_stats(game_ids, conn=conn)
+    logging.info("DONE Updating player_stats with new games")
+
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.INFO)
+    get_all()

src/leetify/core.py

@@ -1,4 +1,7 @@
+from ratelimit.decorators import sleep_and_retry
 import requests
+from ratelimit import limits, RateLimitException
+from backoff import expo, on_exception
 
 
 class Leetify:
@@ -6,6 +9,9 @@ class Leetify:
     profile_base_url = f"{api_base_url}/profile"
     match_base_url = f"{api_base_url}/games"
 
+    @on_exception(expo, Exception, max_tries=3)
+    @sleep_and_retry
+    @limits(1, 5)
     def __get_page(self, url: str) -> dict:
         resp = requests.get(url)
         return resp.json()

src/transform.py

@@ -1,9 +1,15 @@
+from types import NoneType
 from typing import TypeVar
 
 T = TypeVar("T")
 U = TypeVar("U")
 
 
+def vals_in_order(data: dict, keys: list):
+    vals = [data[key] for key in keys]
+    return vals
+
+
 def extract_cols(data: dict, cols: list[str]) -> dict:
     return {key: data.get(key) for key in cols}
 
@@ -12,9 +18,54 @@ def score_to_text(score: list[int]) -> str:
     return "-".join(map(str, score))
 
 
-# maybe could get columns form db
-def cols_from_player_stats(player_stats: dict) -> dict:
-    cols = [
+## returns column names if the form:
+## {"<table_name>": ([<names from api>]}, [<names from db>])}
+def get_cols(table_name=None):
+    cols = {
+        "profile_game": (
+            [
+                "leetifyUserId",
+                "ctLeetifyRating",
+                "ctLeetifyRatingRounds",
+                "dataSource",
+                "elo",
+                "gameFinishedAt",
+                "gameId",
+                "isCs2",
+                "mapName",
+                "matchResult",
+                "scores",
+                "skillLevel",
+                "tLeetifyRating",
+                "tLeetifyRatingRounds",
+                "deaths",
+                "hasBannedPlayer",
+                "kills",
+                "partySize",
+            ],
+            [
+                "leetify_user_id",
+                "ct_Leetify_rating",
+                "ct_Leetify_rating_rounds",
+                "data_source",
+                "elo",
+                "game_finished_at",
+                "game_id",
+                "is_cs2",
+                "map_name",
+                "match_result",
+                "scores",
+                "skill_level",
+                "t_leetify_rating",
+                "t_leetify_rating_rounds",
+                "deaths",
+                "has_banned_player",
+                "kills",
+                "party_size",
+            ],
+        ),
+        "player_stats": (
+            [
                 "id",
                 "gameId",
                 "gameFinishedAt",
@@ -97,31 +148,144 @@ def cols_from_player_stats(player_stats: dict) -> dict:
                 "isCollector",
                 "isProPlan",
                 "isLeetifyStaff",
-    ]
+            ],
+            [
+                "id",
+                "game_id",
+                "game_finished_at",
+                "steam64_id",
+                "name",
+                "preaim",
+                "reaction_time",
+                "accuracy",
+                "accuracy_enemy_spotted",
+                "accuracy_head",
+                "shots_fired_enemy_spotted",
+                "shots_fired",
+                "shots_hit_enemy_spotted",
+                "shots_hit_friend",
+                "shots_Hit_Friend_Head",
+                "shots_Hit_Foe",
+                "shots_Hit_Foe_Head",
+                "utility_On_Death_Avg",
+                "he_foes_damage_avg",
+                "he_friends_damage_avg",
+                "he_thrown",
+                "molotov_thrown",
+                "smoke_thrown",
+                "smoke_thrown_ct",
+                "smoke_thrown_ct_good",
+                "smoke_thrown_ct_good_ratio",
+                "smoke_thrown_ct_foes",
+                "counter_strafing_shots_all",
+                "counter_strafing_shots_bad",
+                "counter_strafing_shots_good",
+                "counter_strafing_shots_good_ratio",
+                "flashbang_hit_foe",
+                "flashbang_leading_to_kill",
+                "flashbang_hit_foe_avg_duration",
+                "flashbang_hit_friend",
+                "flashbang_thrown",
+                "flash_assist",
+                "score",
+                "initial_Team_Number",
+                "mvps",
+                "ct_rounds_won",
+                "ct_rounds_lost",
+                "t_rounds_won",
+                "t_rounds_lost",
+                "spray_accuracy",
+                "molotov_foes_damage_avg",
+                "molotov_friends_damage_avg",
+                "color",
+                "total_kills",
+                "total_deaths",
+                "kd_ratio",
+                "multi2k",
+                "multi3k",
+                "multi4k",
+                "multi5k",
+                "hltv_rating",
+                "hsp",
+                "rounds_survived",
+                "rounds_survived_percentage",
+                "dpr",
+                "total_assists",
+                "total_damage",
+                "trade_kill_opportunities",
+                "trade_kill_attempts",
+                "trade_kills_succeeded",
+                "trade_kill_attempts_percentage",
+                "trade_kills_success_percentage",
+                "trade_kill_opportunities_per_round",
+                "traded_death_opportunities",
+                "traded_death_attempts",
+                "traded_death_attempts_percentage",
+                "traded_deaths_succeeded",
+                "traded_deaths_success_percentage",
+                "traded_deaths_opportunities_per_round",
+                "leetify_rating",
+                "personal_performance_rating",
+                "ct_leetify_rating",
+                "t_leetify_rating",
+                "leetify_user_id",
+                "is_collector",
+                "is_pro_plan",
+                "is_leetify_staff",
+            ],
+        ),
+        "profile_meta": (
+            [
+                "name",
+                "steam64Id",
+                "isCollector",
+                "isLeetifyStaff",
+                "isProPlan",
+                "leetifyUserId",
+                "faceitNickname",
+            ],
+            [
+                "name",
+                "steam64_id",
+                "is_collector",
+                "is_leetify_staff",
+                "is_pro_plan",
+                "leetify_user_id",
+                "faceit_nickname",
+            ],
+        ),
+    }
+    if table_name:
+        return cols[table_name]
+    return cols
 
+
+def get_api_cols(table_name: str | NoneType = None):
+    col_pos = 0
+    if table_name:
+        return get_cols(table_name)[col_pos]
+    return {key: val[col_pos] for key, val in get_cols().items()}
+
+
+def get_db_cols(table_name=None):
+    col_pos = 1
+    if table_name:
+        return get_cols(table_name)[col_pos]
+    return {key: val[col_pos] for key, val in get_cols().items()}
+
+
+# maybe could get columns form db
+def cols_from_player_stats(player_stats: dict) -> dict:
+    cols = get_api_cols("player_stats")
+    if type(cols) != list:
+        raise Exception("could not get api columns for player_stats")
     return extract_cols(player_stats, cols)
 
 
 def cols_from_profile_game(game: dict) -> dict:
-    cols = [
-        "ctLeetifyRating",
-        "ctLeetifyRatingRounds",
-        "dataSource",
-        "elo",
-        "gameFinishedAt",
-        "gameId",
-        "isCs2",
-        "mapName",
-        "matchResult",
-        "scores",
-        "skillLevel",
-        "tLeetifyRating",
-        "tLeetifyRatingRounds",
-        "deaths",
-        "hasBannedPlayer",
-        "kills",
-        "partySize",
-    ]
+    cols = get_api_cols("profile_game")
+    if type(cols) != list:
+        raise Exception("could not get api columns for profile_game")
     return extract_cols(game, cols)
 
 
@@ -130,15 +294,9 @@ def meta_from_profile(profile: dict) -> dict:
     if not meta:
         raise Exception("Could not get profile metadata", profile)
 
-    cols = [
-        "name",
-        "steam64Id",
-        "isCollector",
-        "isLeetifyStaff",
-        "isProPlan",
-        "leetifyUserId",
-        "faceitNickname",
-    ]
+    cols = get_api_cols("profile_meta")
+    if type(cols) != list:
+        raise Exception("could not get api columns for profile_meta")
 
     return extract_cols(meta, cols)
 
@@ -151,7 +309,7 @@ def insert_value(data: dict[T, U], key: T, value: U) -> dict[T, U]:
 def convert_game_scores(game: dict) -> dict:
     score = game.get("scores")
     if not score:
-        raise Exception("Could not get score from prfile game", game)
+        raise Exception("Could not get score from profile game", game)
 
     score = score_to_text(score)
     game["scores"] = score
@@ -183,6 +341,5 @@ def player_stats_from_game(game: dict) -> list[dict]:
     if not stats:
         raise Exception("Could not get stats from game", game)
 
-
-    stats = map(cols_from_player_stats,stats)
+    stats = map(cols_from_player_stats, stats)
     return list(stats)