refactored functions to use centeralized column name definition

.aliases

@@ -0,0 +1 @@
+alias sqitchl='SQITCH_PASSWORD=$POSTGRES_PASSWORD sqitch -u $POSTGRES_USER -p $POSTGRES_PORT -h 127.0.0.1'

.gitignore

@@ -1,3 +1,4 @@
+tmp/
 # ---> Python
 # Byte-compiled / optimized / DLL files
 __pycache__/

docker-compose.yml

@@ -0,0 +1,11 @@
+services:
+  db:
+    image: postgres
+    volumes:
+      - ./tmp/${COMPOSE_PROJECT_NAME}/db:/var/lib/postgresql/data
+    environment:
+      - POSTGRES_DB=${POSTGRES_DB}
+      - POSTGRES_USER=${POSTGRES_USER}
+      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
+    ports:
+      - "${POSTGRES_PORT}:5432"

pytest.ini

@@ -0,0 +1,4 @@
+[pytest]  
+pythonpath = .
+addopts = --cov=src --cov-report html
+norecursedirs = tmp*

requirments.txt

@@ -0,0 +1,5 @@
+pytest
+click
+psycopg2-binary
+ratelimit
+backoff

sqitch.conf

@@ -0,0 +1,8 @@
+[core]
+	engine = pg
+	top_dir = sqitch
+	# plan_file = sqitch/sqitch.plan
+# [engine "pg"]
+	# target = db:pg:
+	# registry = sqitch
+	# client = psql

sqitch/deploy/data-schema.sql

@@ -0,0 +1,7 @@
+-- Deploy leetify-data:data-schema to pg
+
+BEGIN;
+
+  CREATE SCHEMA data;
+
+COMMIT;

sqitch/deploy/player-stats.sql

@@ -0,0 +1,91 @@
+-- Deploy leetify-data:player-stats to pg
+-- requires: data-schema
+
+BEGIN;
+
+  CREATE TABLE IF NOT EXISTS data.player_stats (
+    id TEXT,
+    game_id TEXT,
+    game_finished_at TIMESTAMP,
+    steam64_id TEXT,
+    name TEXT,
+    preaim NUMERIC,
+    reaction_time NUMERIC,
+    accuracy NUMERIC,
+    accuracy_enemy_spotted NUMERIC,
+    accuracy_head NUMERIC,
+    shots_fired_enemy_spotted NUMERIC,
+    shots_fired NUMERIC,
+    shots_hit_enemy_spotted NUMERIC,
+    shots_hit_friend NUMERIC,
+    shots_Hit_Friend_Head NUMERIC,
+    shots_Hit_Foe NUMERIC,
+    shots_Hit_Foe_Head NUMERIC,
+    utility_On_Death_Avg NUMERIC,
+    he_foes_damage_avg NUMERIC,
+    he_friends_damage_avg NUMERIC,
+    he_thrown NUMERIC,
+    molotov_thrown NUMERIC,
+    smoke_thrown NUMERIC,
+    smoke_thrown_ct NUMERIC,
+    smoke_thrown_ct_good NUMERIC,
+    smoke_thrown_ct_good_ratio NUMERIC,
+    smoke_thrown_ct_foes NUMERIC,
+    counter_strafing_shots_all NUMERIC,
+    counter_strafing_shots_bad NUMERIC,
+    counter_strafing_shots_good NUMERIC,
+    counter_strafing_shots_good_ratio NUMERIC,
+    flashbang_hit_foe NUMERIC,
+    flashbang_leading_to_kill NUMERIC,
+    flashbang_hit_foe_avg_duration NUMERIC,
+    flashbang_hit_friend NUMERIC,
+    flashbang_thrown NUMERIC,
+    flash_assist NUMERIC,
+    score NUMERIC,
+    initial_Team_Number NUMERIC,
+    mvps NUMERIC,
+    ct_rounds_won NUMERIC,
+    ct_rounds_lost NUMERIC,
+    t_rounds_won NUMERIC,
+    t_rounds_lost NUMERIC,
+    spray_accuracy NUMERIC,
+    molotov_foes_damage_avg NUMERIC,
+    molotov_friends_damage_avg NUMERIC,
+    color NUMERIC,
+    total_kills NUMERIC,
+    total_deaths NUMERIC,
+    kd_ratio NUMERIC,
+    multi2k NUMERIC,
+    multi3k NUMERIC,
+    multi4k NUMERIC,
+    multi5k NUMERIC,
+    hltv_rating NUMERIC,
+    hsp NUMERIC,
+    rounds_survived NUMERIC,
+    rounds_survived_percentage NUMERIC,
+    dpr NUMERIC,
+    total_assists NUMERIC,
+    total_damage NUMERIC,
+    trade_kill_opportunities NUMERIC,
+    trade_kill_attempts NUMERIC,
+    trade_kills_succeeded NUMERIC,
+    trade_kill_attempts_percentage NUMERIC,
+    trade_kills_success_percentage NUMERIC,
+    trade_kill_opportunities_per_round NUMERIC,
+    traded_death_opportunities NUMERIC,
+    traded_death_attempts NUMERIC,
+    traded_death_attempts_percentage NUMERIC,
+    traded_deaths_succeeded NUMERIC,
+    traded_deaths_success_percentage NUMERIC,
+    traded_deaths_opportunities_per_round NUMERIC,
+    leetify_rating NUMERIC,
+    personal_performance_rating NUMERIC,
+    ct_leetify_rating NUMERIC,
+    t_leetify_rating NUMERIC,
+    leetify_user_id TEXT,
+    is_collector BOOL,
+    is_pro_plan BOOL,
+    is_leetify_staff BOOL
+  );
+
+COMMIT;

sqitch/deploy/profile-game.sql

@@ -0,0 +1,28 @@
+-- Deploy leetify-data:profile-game to pg
+-- requires: data-schema
+
+BEGIN;
+
+  CREATE TABLE IF NOT EXISTS data.profile_game (
+     leetify_user_id TEXT,
+     ct_Leetify_rating NUMERIC,
+     ct_Leetify_rating_rounds NUMERIC,
+     data_source TEXT,
+     elo NUMERIC,
+     game_finished_at TIMESTAMP,
+     game_id TEXT,
+     is_cs2 BOOL,
+     map_name TEXT,
+     match_result TEXT,
+     scores TEXT, -- has to be extracted from array
+     skill_level NUMERIC,
+     t_leetify_rating NUMERIC,
+     t_leetify_rating_rounds NUMERIC,
+     deaths NUMERIC,
+     has_banned_player BOOL,
+     kills NUMERIC,
+     party_size NUMERIC
+  );
+
+
+COMMIT;

sqitch/deploy/profile-meta.sql

@@ -0,0 +1,16 @@
+-- Deploy leetify-data:profile-meta to pg
+-- requires: data-schema
+
+BEGIN;
+
+  CREATE TABLE IF NOT EXISTS data.profile_meta (
+    name TEXT,
+    steam64_id TEXT,
+    is_collector BOOL,
+    is_leetify_staff BOOL,
+    is_pro_plan BOOL,
+    leetify_user_id TEXT,
+    faceit_nickname TEXT
+  );
+
+COMMIT;

sqitch/revert/data-schema.sql

@@ -0,0 +1,7 @@
+-- Revert leetify-data:data-schema from pg
+
+BEGIN;
+
+  DROP SCHEMA data;
+
+COMMIT;

sqitch/revert/player-stats.sql

@@ -0,0 +1,8 @@
+-- Revert leetify-data:player-stats from pg
+
+BEGIN;
+
+-- XXX Add DDLs here.
+DROP TABLE data.player_stats;
+
+COMMIT;

sqitch/revert/profile-game.sql

@@ -0,0 +1,7 @@
+-- Revert leetify-data:profile-game from pg
+
+BEGIN;
+
+  DROP TABLE data.profile_game;
+
+COMMIT;

sqitch/revert/profile-meta.sql

@@ -0,0 +1,8 @@
+-- Revert leetify-data:profile-meta from pg
+
+BEGIN;
+
+
+  DROP TABLE data.profile_meta;
+
+COMMIT;

sqitch/sqitch.plan

@@ -0,0 +1,7 @@
+%syntax-version=1.0.0
+%project=leetify-data
+
+data-schema 2024-01-25T03:19:31Z andrei <andrei@tower> # adding schema for data
+profile-meta [data-schema] 2024-01-25T03:28:08Z andrei <andrei@tower> # add table for profile metadata
+player-stats 2024-01-26T23:39:00Z andrei <andrei@tower> # Add player-stats table
+profile-game [data-schema] 2024-01-27T02:15:46Z andrei <andrei@tower> # add profile_game table

sqitch/verify/data-schema.sql

@@ -0,0 +1,7 @@
+-- Verify leetify-data:data-schema on pg
+
+BEGIN;
+
+  SELECT pg_catalog.has_schema_privilege('data', 'usage');
+
+ROLLBACK;

sqitch/verify/player-stats.sql

@@ -0,0 +1,33 @@
+-- Verify leetify-data:player-stats on pg
+
+BEGIN;
+
+-- XXX Add verifications here.
+SELECT id, game_id, game_finished_at, steam64_id, name, preaim, reaction_time,
+    accuracy, accuracy_enemy_spotted, accuracy_head, shots_fired_enemy_spotted,
+    shots_fired, shots_hit_enemy_spotted, shots_hit_friend,
+    shots_Hit_Friend_Head, shots_Hit_Foe, shots_Hit_Foe_Head,
+    utility_On_Death_Avg, he_foes_damage_avg, he_friends_damage_avg, he_thrown,
+    molotov_thrown, smoke_thrown, smoke_thrown_ct, smoke_thrown_ct_good,
+    smoke_thrown_ct_good_ratio, smoke_thrown_ct_foes,
+    counter_strafing_shots_all, counter_strafing_shots_bad,
+    counter_strafing_shots_good, counter_strafing_shots_good_ratio,
+    flashbang_hit_foe, flashbang_leading_to_kill,
+    flashbang_hit_foe_avg_duration, flashbang_hit_friend, flashbang_thrown,
+    flash_assist, score, initial_Team_Number, mvps, ct_rounds_won,
+    ct_rounds_lost, t_rounds_won, t_rounds_lost, spray_accuracy,
+    molotov_foes_damage_avg, molotov_friends_damage_avg, color,
+    total_kills, total_deaths, kd_ratio, multi2k, multi3k, multi4k, multi5k,
+    hltv_rating, hsp, rounds_survived, rounds_survived_percentage, dpr,
+    total_assists, total_damage, trade_kill_opportunities, trade_kill_attempts,
+    trade_kills_succeeded, trade_kill_attempts_percentage,
+    trade_kills_success_percentage, trade_kill_opportunities_per_round,
+    traded_death_opportunities, traded_death_attempts,
+    traded_death_attempts_percentage, traded_deaths_succeeded,
+    traded_deaths_success_percentage, traded_deaths_opportunities_per_round,
+    leetify_rating, personal_performance_rating, ct_leetify_rating,
+    t_leetify_rating, leetify_user_id,
+    is_collector, is_pro_plan, is_leetify_staff
+FROM data.player_stats;
+
+ROLLBACK;

sqitch/verify/profile-game.sql

@@ -0,0 +1,13 @@
+-- Verify leetify-data:profile-game on pg
+
+BEGIN;
+
+-- XXX Add verifications here.
+SELECT leetify_user_id, ct_Leetify_rating, ct_Leetify_rating_rounds,
+  data_source, elo, game_finished_at, game_id, is_cs2, map_name, match_result,
+  scores, skill_level, t_leetify_rating, t_leetify_rating_rounds, deaths,
+  has_banned_player, kills, party_size
+FROM data.profile_game
+WHERE FALSE;
+
+ROLLBACK;

sqitch/verify/profile-meta.sql

@@ -0,0 +1,10 @@
+-- Verify leetify-data:profile-meta on pg
+
+BEGIN;
+
+SELECT name, steam64_id, is_collector, is_leetify_staff, is_pro_plan,
+    leetify_user_id, faceit_nickname
+FROM data.profile_meta
+WHERE FALSE;
+
+ROLLBACK;

src/db.py

@@ -0,0 +1,79 @@
+import psycopg2 as pg
+from os import environ as env
+from transform import vals_in_order, get_cols
+
+
+def connect():
+    print("connecting")
+    return pg.connect(
+        dbname=env.get("POSTGRES_DB"),
+        user=env.get("POSTGRES_USER"),
+        password=env.get("POSTGRES_PASSWORD"),
+        host=env.get("POSTGRES_HOST"),
+        port=env.get("POSTGRES_PORT"),
+    )
+
+
+def connect_by_default(func, conn_func=connect):
+    def wrapper(*args, **kargs):
+        if not kargs.get("conn"):
+            kargs["conn"] = conn_func()
+        return func(*args, **kargs)
+
+    return wrapper
+
+
+@connect_by_default
+def get_all_profile_names(conn=None):
+    with conn.cursor() as curs:
+        curs.execute("""SELECT name, leetify_user_id FROM data.profile_meta;""")
+        data = curs.fetchall()
+
+    return data
+
+
+@connect_by_default
+def get_profile_game_count(user_id, conn=None):
+    with conn.cursor() as curs:
+        curs.execute(
+            """SELECT count(*) FROM data.profile_game
+               WHERE leetify_user_id = %s;""",
+            (user_id,),
+        )
+        data = curs.fetchone()
+
+    if not data:
+        data = [0]
+    return data[0]
+
+
+@connect_by_default
+def get_profile_game_ids(user_id, conn=None):
+    with conn.cursor() as curs:
+        curs.execute(
+            """SELECT game_id FROM data.profile_game
+               WHERE leetify_user_id = %s;""",
+            (user_id,),
+        )
+        data = curs.fetchall()
+
+    return [row[0] for row in data]
+
+
+@connect_by_default
+def insert_profile_games(games, conn=None):
+    cols_api, cols_db = get_cols("profile_game")
+    if type(cols_api) != list or type(cols_db) != list:
+        raise Exception(
+            "could not get columns for profile_game:", cols_api, cols_db
+        )
+
+    vals = map(lambda game: vals_in_order(game, cols_api), games)
+
+    with conn.cursor() as curs:
+        sql = f"""
+            INSERT into data.profile_game ({', '.join(cols_db)})
+            VALUES ({', '.join(['%s'] * len(cols_db))});
+        """
+        curs.executemany(sql, vals)
+    conn.commit()

src/extract.py

@@ -0,0 +1,132 @@
+from leetify import Leetify
+from transform import (
+    get_cols,
+    meta_from_profile,
+    games_from_profile,
+    player_stats_from_game,
+    vals_in_order,
+)
+import db
+import logging
+
+## TODO seperate out loading from extraction
+## this currently handles getting data from api and loading into db
+
+
+@db.connect_by_default
+def extract_profile_meta(id, api=Leetify(), conn=None):
+    cols_api, cols_db = get_cols("profile_meta")
+    if type(cols_api) != list or type(cols_db) != list:
+        raise Exception(
+            "could not get columns for profile_meta:", cols_api, cols_db
+        )
+
+    profile = api.get_profile(id)
+    meta = meta_from_profile(profile)
+    vals = vals_in_order(meta, cols_api)
+
+    with conn.cursor() as curs:
+        sql = f"""
+        INSERT INTO data.profile_meta ({', '.join(cols_db)})
+        VALUES ({', '.join(['%s'] * len(cols_db))});
+        """
+        curs.execute(sql, vals)
+    conn.commit()
+
+
+@db.connect_by_default
+def extract_profile_games(profile, conn=None):
+    cols_api, cols_db = get_cols("profile_games")
+    if type(cols_api) != list or type(cols_db) != list:
+        raise Exception(
+            "could not get columns for profile_game:", cols_api, cols_db
+        )
+
+    games = games_from_profile(profile)
+    games = map(lambda game: vals_in_order(game, cols_api), games)
+
+    with conn.cursor() as curs:
+        curs.executemany(
+            f"""
+            INSERT into data.profile_game ({', '.join(cols_db)})
+            VALUES ({', '.join(['%s'] * len(cols_db))});
+            """,
+            games,
+        )
+    conn.commit()
+
+
+@db.connect_by_default
+def extract_player_stats(game, conn=None):
+    cols_api, cols_db = get_cols("player_stats")
+    if type(cols_api) != list or type(cols_db) != list:
+        raise Exception(
+            "could not get columns for player_stats:", cols_api, cols_db
+        )
+
+    stats = player_stats_from_game(game)
+    vals = map(lambda game: vals_in_order(game, cols_api), stats)
+
+    with conn.cursor() as curs:
+        sql = f"""
+              INSERT into data.player_stats ({', '.join(cols_db)})
+              VALUES ({', '.join(['%s'] * len(cols_db))}); """
+        curs.executemany(sql, vals)
+    conn.commit()
+
+
+@db.connect_by_default
+def insert_new_profile_games(games, user_id, conn=None):
+    game_ids = db.get_profile_game_ids(user_id, conn=conn)
+
+    game_ids = set(game_ids)
+    new_games = [game for game in games if game.get("gameId") not in game_ids]
+
+    logging.info(f"Inserting {len(new_games)} for: {user_id}")
+    db.insert_profile_games(new_games, conn=conn)
+
+    return new_games
+
+
+@db.connect_by_default
+def insert_player_stats(game_ids, api=Leetify(), conn=None):
+    logging.info(f"Inserting player_stats from {len(game_ids)} games")
+    print_every = int(len(game_ids) / 10)
+    for i, id in enumerate(game_ids):
+        if i % print_every == 0:
+            logging.info(f"Inserted player_stats from {i}/{len(game_ids)} games")
+        game = api.get_match(id)
+        extract_player_stats(game, conn=conn)
+
+
+def get_all(api=Leetify()):
+    conn = db.connect()
+
+    data = db.get_all_profile_names(conn=conn)
+    game_ids = []
+    for name, id in data:
+        logging.info(f"Getting data for {name}: {id}")
+        profile = api.get_profile(id)
+        profile_games = games_from_profile(profile)
+
+        api_games = len(profile_games)
+        db_games = db.get_profile_game_count(id, conn=conn)
+
+        if api_games > db_games:
+            logging.info(f"Getting new games for {name}: {id}")
+            new_games = insert_new_profile_games(profile_games, id, conn=conn)
+            new_game_ids = [game.get("gameId") for game in new_games]
+            game_ids.extend(new_game_ids)
+        elif api_games < db_games:
+            logging.error(f"API returned less games then in DB for {name}: {id}")
+
+        logging.info(f"Games synced with Leetify for {name}: {id}")
+
+    logging.info(f"Updating player_stats with {len(game_ids)} new games")
+    insert_player_stats(game_ids, conn=conn)
+    logging.info("DONE Updating player_stats with new games")
+
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.INFO)
+    get_all()

src/leetify/__init__.py

@@ -0,0 +1 @@
+from .core import *

src/leetify/core.py

@@ -0,0 +1,25 @@
+from ratelimit.decorators import sleep_and_retry
+import requests
+from ratelimit import limits, RateLimitException
+from backoff import expo, on_exception
+
+
+class Leetify:
+    api_base_url = "https://api.leetify.com/api"
+    profile_base_url = f"{api_base_url}/profile"
+    match_base_url = f"{api_base_url}/games"
+
+    @on_exception(expo, Exception, max_tries=3)
+    @sleep_and_retry
+    @limits(1, 5)
+    def __get_page(self, url: str) -> dict:
+        resp = requests.get(url)
+        return resp.json()
+
+    def get_profile(self, id: str) -> dict:
+        url = f"{self.profile_base_url}/{id}"
+        return self.__get_page(url)
+
+    def get_match(self, id: str) -> dict:
+        url = f"{self.match_base_url}/{id}"
+        return self.__get_page(url)

src/transform.py

@@ -0,0 +1,345 @@
+from types import NoneType
+from typing import TypeVar
+
+T = TypeVar("T")
+U = TypeVar("U")
+
+
+def vals_in_order(data: dict, keys: list):
+    vals = [data[key] for key in keys]
+    return vals
+
+
+def extract_cols(data: dict, cols: list[str]) -> dict:
+    return {key: data.get(key) for key in cols}
+
+
+def score_to_text(score: list[int]) -> str:
+    return "-".join(map(str, score))
+
+
+## returns column names if the form:
+## {"<table_name>": ([<names from api>]}, [<names from db>])}
+def get_cols(table_name=None):
+    cols = {
+        "profile_game": (
+            [
+                "leetifyUserId",
+                "ctLeetifyRating",
+                "ctLeetifyRatingRounds",
+                "dataSource",
+                "elo",
+                "gameFinishedAt",
+                "gameId",
+                "isCs2",
+                "mapName",
+                "matchResult",
+                "scores",
+                "skillLevel",
+                "tLeetifyRating",
+                "tLeetifyRatingRounds",
+                "deaths",
+                "hasBannedPlayer",
+                "kills",
+                "partySize",
+            ],
+            [
+                "leetify_user_id",
+                "ct_Leetify_rating",
+                "ct_Leetify_rating_rounds",
+                "data_source",
+                "elo",
+                "game_finished_at",
+                "game_id",
+                "is_cs2",
+                "map_name",
+                "match_result",
+                "scores",
+                "skill_level",
+                "t_leetify_rating",
+                "t_leetify_rating_rounds",
+                "deaths",
+                "has_banned_player",
+                "kills",
+                "party_size",
+            ],
+        ),
+        "player_stats": (
+            [
+                "id",
+                "gameId",
+                "gameFinishedAt",
+                "steam64Id",
+                "name",
+                "preaim",
+                "reactionTime",
+                "accuracy",
+                "accuracyEnemySpotted",
+                "accuracyHead",
+                "shotsFiredEnemySpotted",
+                "shotsFired",
+                "shotsHitEnemySpotted",
+                "shotsHitFriend",
+                "shotsHitFriendHead",
+                "shotsHitFoe",
+                "shotsHitFoeHead",
+                "utilityOnDeathAvg",
+                "heFoesDamageAvg",
+                "heFriendsDamageAvg",
+                "heThrown",
+                "molotovThrown",
+                "smokeThrown",
+                "smokeThrownCT",
+                "smokeThrownCTGood",
+                "smokeThrownCTGoodRatio",
+                "smokeThrownCTFoes",
+                "counterStrafingShotsAll",
+                "counterStrafingShotsBad",
+                "counterStrafingShotsGood",
+                "counterStrafingShotsGoodRatio",
+                "flashbangHitFoe",
+                "flashbangLeadingToKill",
+                "flashbangHitFoeAvgDuration",
+                "flashbangHitFriend",
+                "flashbangThrown",
+                "flashAssist",
+                "score",
+                "initialTeamNumber",
+                "mvps",
+                "ctRoundsWon",
+                "ctRoundsLost",
+                "tRoundsWon",
+                "tRoundsLost",
+                "sprayAccuracy",
+                "molotovFoesDamageAvg",
+                "molotovFriendsDamageAvg",
+                "color",
+                "totalKills",
+                "totalDeaths",
+                "kdRatio",
+                "multi2k",
+                "multi3k",
+                "multi4k",
+                "multi5k",
+                "hltvRating",
+                "hsp",
+                "roundsSurvived",
+                "roundsSurvivedPercentage",
+                "dpr",
+                "totalAssists",
+                "totalDamage",
+                "tradeKillOpportunities",
+                "tradeKillAttempts",
+                "tradeKillsSucceeded",
+                "tradeKillAttemptsPercentage",
+                "tradeKillsSuccessPercentage",
+                "tradeKillOpportunitiesPerRound",
+                "tradedDeathOpportunities",
+                "tradedDeathAttempts",
+                "tradedDeathAttemptsPercentage",
+                "tradedDeathsSucceeded",
+                "tradedDeathsSuccessPercentage",
+                "tradedDeathsOpportunitiesPerRound",
+                "leetifyRating",
+                "personalPerformanceRating",
+                "ctLeetifyRating",
+                "tLeetifyRating",
+                "leetifyUserId",
+                "isCollector",
+                "isProPlan",
+                "isLeetifyStaff",
+            ],
+            [
+                "id",
+                "game_id",
+                "game_finished_at",
+                "steam64_id",
+                "name",
+                "preaim",
+                "reaction_time",
+                "accuracy",
+                "accuracy_enemy_spotted",
+                "accuracy_head",
+                "shots_fired_enemy_spotted",
+                "shots_fired",
+                "shots_hit_enemy_spotted",
+                "shots_hit_friend",
+                "shots_Hit_Friend_Head",
+                "shots_Hit_Foe",
+                "shots_Hit_Foe_Head",
+                "utility_On_Death_Avg",
+                "he_foes_damage_avg",
+                "he_friends_damage_avg",
+                "he_thrown",
+                "molotov_thrown",
+                "smoke_thrown",
+                "smoke_thrown_ct",
+                "smoke_thrown_ct_good",
+                "smoke_thrown_ct_good_ratio",
+                "smoke_thrown_ct_foes",
+                "counter_strafing_shots_all",
+                "counter_strafing_shots_bad",
+                "counter_strafing_shots_good",
+                "counter_strafing_shots_good_ratio",
+                "flashbang_hit_foe",
+                "flashbang_leading_to_kill",
+                "flashbang_hit_foe_avg_duration",
+                "flashbang_hit_friend",
+                "flashbang_thrown",
+                "flash_assist",
+                "score",
+                "initial_Team_Number",
+                "mvps",
+                "ct_rounds_won",
+                "ct_rounds_lost",
+                "t_rounds_won",
+                "t_rounds_lost",
+                "spray_accuracy",
+                "molotov_foes_damage_avg",
+                "molotov_friends_damage_avg",
+                "color",
+                "total_kills",
+                "total_deaths",
+                "kd_ratio",
+                "multi2k",
+                "multi3k",
+                "multi4k",
+                "multi5k",
+                "hltv_rating",
+                "hsp",
+                "rounds_survived",
+                "rounds_survived_percentage",
+                "dpr",
+                "total_assists",
+                "total_damage",
+                "trade_kill_opportunities",
+                "trade_kill_attempts",
+                "trade_kills_succeeded",
+                "trade_kill_attempts_percentage",
+                "trade_kills_success_percentage",
+                "trade_kill_opportunities_per_round",
+                "traded_death_opportunities",
+                "traded_death_attempts",
+                "traded_death_attempts_percentage",
+                "traded_deaths_succeeded",
+                "traded_deaths_success_percentage",
+                "traded_deaths_opportunities_per_round",
+                "leetify_rating",
+                "personal_performance_rating",
+                "ct_leetify_rating",
+                "t_leetify_rating",
+                "leetify_user_id",
+                "is_collector",
+                "is_pro_plan",
+                "is_leetify_staff",
+            ],
+        ),
+        "profile_meta": (
+            [
+                "name",
+                "steam64Id",
+                "isCollector",
+                "isLeetifyStaff",
+                "isProPlan",
+                "leetifyUserId",
+                "faceitNickname",
+            ],
+            [
+                "name",
+                "steam64_id",
+                "is_collector",
+                "is_leetify_staff",
+                "is_pro_plan",
+                "leetify_user_id",
+                "faceit_nickname",
+            ],
+        ),
+    }
+    if table_name:
+        return cols[table_name]
+    return cols
+
+
+def get_api_cols(table_name: str | NoneType = None):
+    col_pos = 0
+    if table_name:
+        return get_cols(table_name)[col_pos]
+    return {key: val[col_pos] for key, val in get_cols().items()}
+
+
+def get_db_cols(table_name=None):
+    col_pos = 1
+    if table_name:
+        return get_cols(table_name)[col_pos]
+    return {key: val[col_pos] for key, val in get_cols().items()}
+
+
+# maybe could get columns form db
+def cols_from_player_stats(player_stats: dict) -> dict:
+    cols = get_api_cols("player_stats")
+    if type(cols) != list:
+        raise Exception("could not get api columns for player_stats")
+    return extract_cols(player_stats, cols)
+
+
+def cols_from_profile_game(game: dict) -> dict:
+    cols = get_api_cols("profile_game")
+    if type(cols) != list:
+        raise Exception("could not get api columns for profile_game")
+    return extract_cols(game, cols)
+
+
+def meta_from_profile(profile: dict) -> dict:
+    meta = profile.get("meta")
+    if not meta:
+        raise Exception("Could not get profile metadata", profile)
+
+    cols = get_api_cols("profile_meta")
+    if type(cols) != list:
+        raise Exception("could not get api columns for profile_meta")
+
+    return extract_cols(meta, cols)
+
+
+def insert_value(data: dict[T, U], key: T, value: U) -> dict[T, U]:
+    data[key] = value
+    return data
+
+
+def convert_game_scores(game: dict) -> dict:
+    score = game.get("scores")
+    if not score:
+        raise Exception("Could not get score from profile game", game)
+
+    score = score_to_text(score)
+    game["scores"] = score
+    return game
+
+
+def games_from_profile(profile: dict) -> list:
+    games = profile.get("games")
+    # emtpy list is falsy but an acceptable value
+    if not games and not type(games) == list:
+        raise Exception("Could not get games from profile", profile)
+
+    meta = profile.get("meta")
+    if not meta:
+        raise Exception("Could not get profile metadata", profile)
+
+    id = meta.get("leetifyUserId")
+    if not id:
+        raise Exception("Could not get id from profile metadata", meta, profile)
+
+    games = map(cols_from_profile_game, games)
+    games = map(lambda game: insert_value(game, "leetifyUserId", id), games)
+    games = map(convert_game_scores, games)
+    return list(games)
+
+
+def player_stats_from_game(game: dict) -> list[dict]:
+    stats = game.get("playerStats")
+    if not stats:
+        raise Exception("Could not get stats from game", game)
+
+    stats = map(cols_from_player_stats, stats)
+    return list(stats)