From 2786d73d0558de4e968e1ba319feae959fe04021 Mon Sep 17 00:00:00 2001 From: vytal <80233205+SeanCole02@users.noreply.github.com> Date: Tue, 22 Oct 2024 13:45:17 +0100 Subject: [PATCH 1/8] Update main.py More robust handling of connection errors/loss on schema fetch --- tf2utilities/main.py | 99 +++++++++++++++++++++++--------------------- 1 file changed, 52 insertions(+), 47 deletions(-) diff --git a/tf2utilities/main.py b/tf2utilities/main.py index 58ec74f..0372e53 100644 --- a/tf2utilities/main.py +++ b/tf2utilities/main.py @@ -1,3 +1,4 @@ +import requests.exceptions from tf2utilities.schema import Schema from threading import Thread import time @@ -25,52 +26,56 @@ def updater(self): # Gets the schema from the TF2 API def getSchema(self): if self.apiKey is not None: - raw = { - "schema": Schema.getOverview(self.apiKey) | {"items": Schema.getItems(self.apiKey), "paintkits": Schema.getPaintKits()}, - "items_game": Schema.getItemsGame() - } + try: + while 1: + raw = { + "schema": Schema.getOverview(self.apiKey) | {"items": Schema.getItems(self.apiKey), "paintkits": Schema.getPaintKits()}, + "items_game": Schema.getItemsGame() + } + if self.lite: + del raw["schema"]["originNames"] + del raw["schema"]["item_sets"] + del raw["schema"]["item_levels"] + del raw["schema"]["string_lookups"] - if self.lite: - del raw["schema"]["originNames"] - del raw["schema"]["item_sets"] - del raw["schema"]["item_levels"] - del raw["schema"]["string_lookups"] + del raw["items_game"]["game_info"] + del raw["items_game"]["qualities"] + del raw["items_game"]["colors"] + del raw["items_game"]["rarities"] + del raw["items_game"]["equip_regions_list"] + del raw["items_game"]["equip_conflicts"] + del raw["items_game"]["quest_objective_conditions"] + del raw["items_game"]["item_series_types"] + del raw["items_game"]["item_collections"] + del raw["items_game"]["operations"] + del raw["items_game"]["prefabs"] + del raw["items_game"]["attributes"] + del raw["items_game"]["item_criteria_templates"] + del raw["items_game"]["random_attribute_templates"] + del raw["items_game"]["lootlist_job_template_definitions"] + del raw["items_game"]["item_sets"] + del raw["items_game"]["client_loot_lists"] + del raw["items_game"]["revolving_loot_lists"] + del raw["items_game"]["recipes"] + del raw["items_game"]["achievement_rewards"] + del raw["items_game"]["attribute_controlled_attached_particles"] + del raw["items_game"]["armory_data"] + del raw["items_game"]["item_levels"] + del raw["items_game"]["kill_eater_score_types"] + del raw["items_game"]["mvm_maps"] + del raw["items_game"]["mvm_tours"] + del raw["items_game"]["matchmaking_categories"] + del raw["items_game"]["maps"] + del raw["items_game"]["master_maps_list"] + del raw["items_game"]["steam_packages"] + del raw["items_game"]["community_market_item_remaps"] + del raw["items_game"]["war_definitions"] - del raw["items_game"]["game_info"] - del raw["items_game"]["qualities"] - del raw["items_game"]["colors"] - del raw["items_game"]["rarities"] - del raw["items_game"]["equip_regions_list"] - del raw["items_game"]["equip_conflicts"] - del raw["items_game"]["quest_objective_conditions"] - del raw["items_game"]["item_series_types"] - del raw["items_game"]["item_collections"] - del raw["items_game"]["operations"] - del raw["items_game"]["prefabs"] - del raw["items_game"]["attributes"] - del raw["items_game"]["item_criteria_templates"] - del raw["items_game"]["random_attribute_templates"] - del raw["items_game"]["lootlist_job_template_definitions"] - del raw["items_game"]["item_sets"] - del raw["items_game"]["client_loot_lists"] - del raw["items_game"]["revolving_loot_lists"] - del raw["items_game"]["recipes"] - del raw["items_game"]["achievement_rewards"] - del raw["items_game"]["attribute_controlled_attached_particles"] - del raw["items_game"]["armory_data"] - del raw["items_game"]["item_levels"] - del raw["items_game"]["kill_eater_score_types"] - del raw["items_game"]["mvm_maps"] - del raw["items_game"]["mvm_tours"] - del raw["items_game"]["matchmaking_categories"] - del raw["items_game"]["maps"] - del raw["items_game"]["master_maps_list"] - del raw["items_game"]["steam_packages"] - del raw["items_game"]["community_market_item_remaps"] - del raw["items_game"]["war_definitions"] - - schema = {"time": time.time(), "raw": raw} - if isinstance(schema, dict): - self.schema = Schema(schema) - else: - raise Exception("Schema is not a dict.") + schema = {"time": time.time(), "raw": raw} + if isinstance(schema, dict): + self.schema = Schema(schema) + else: + raise Exception("Schema is not a dict.") + break + except requests.exceptions.ConnectionError: + time.sleep(10) From 101c134a9cfa106ed43e51c1fc0f49ef88c0ab17 Mon Sep 17 00:00:00 2001 From: vytal <80233205+SeanCole02@users.noreply.github.com> Date: Tue, 22 Oct 2024 17:30:05 +0100 Subject: [PATCH 2/8] Update main.py --- tf2utilities/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tf2utilities/main.py b/tf2utilities/main.py index 0372e53..39a5ea5 100644 --- a/tf2utilities/main.py +++ b/tf2utilities/main.py @@ -77,5 +77,5 @@ def getSchema(self): else: raise Exception("Schema is not a dict.") break - except requests.exceptions.ConnectionError: + except: time.sleep(10) From a9f9583054689c3b36aee404421d070fe3dbe3b5 Mon Sep 17 00:00:00 2001 From: vytal <80233205+SeanCole02@users.noreply.github.com> Date: Tue, 22 Oct 2024 17:51:04 +0100 Subject: [PATCH 3/8] Update main.py --- tf2utilities/main.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tf2utilities/main.py b/tf2utilities/main.py index 39a5ea5..a6a3050 100644 --- a/tf2utilities/main.py +++ b/tf2utilities/main.py @@ -26,8 +26,8 @@ def updater(self): # Gets the schema from the TF2 API def getSchema(self): if self.apiKey is not None: - try: - while 1: + while 1: + try: raw = { "schema": Schema.getOverview(self.apiKey) | {"items": Schema.getItems(self.apiKey), "paintkits": Schema.getPaintKits()}, "items_game": Schema.getItemsGame() @@ -77,5 +77,5 @@ def getSchema(self): else: raise Exception("Schema is not a dict.") break - except: - time.sleep(10) + except: + time.sleep(10) From e10118a22f849b0ef7131cf9c8a3359499aa1fec Mon Sep 17 00:00:00 2001 From: Sean Cole Date: Wed, 8 Oct 2025 06:50:00 +0800 Subject: [PATCH 4/8] Improved schema fetch reliability --- tf2utilities/main.py | 43 ++++++++-- tf2utilities/schema.py | 187 +++++++++++++++++++++++++++++++---------- tf2utilities/webapi.py | 32 +++++-- 3 files changed, 202 insertions(+), 60 deletions(-) diff --git a/tf2utilities/main.py b/tf2utilities/main.py index a6a3050..ad8f9a1 100644 --- a/tf2utilities/main.py +++ b/tf2utilities/main.py @@ -2,6 +2,10 @@ from tf2utilities.schema import Schema from threading import Thread import time +import logging + +# Configure logging +logger = logging.getLogger(__name__) class TF2: @@ -23,15 +27,21 @@ def updater(self): time.sleep(self.updateTime) - # Gets the schema from the TF2 API + # Gets the schema from the TF2 API with backup fallback def getSchema(self): if self.apiKey is not None: - while 1: + maxAttempts = 3 + lastError = None + + for attempt in range(maxAttempts): try: + logger.info(f"Fetching schema from API (attempt {attempt + 1}/{maxAttempts})...") + raw = { "schema": Schema.getOverview(self.apiKey) | {"items": Schema.getItems(self.apiKey), "paintkits": Schema.getPaintKits()}, "items_game": Schema.getItemsGame() } + if self.lite: del raw["schema"]["originNames"] del raw["schema"]["item_sets"] @@ -74,8 +84,29 @@ def getSchema(self): schema = {"time": time.time(), "raw": raw} if isinstance(schema, dict): self.schema = Schema(schema) + # Save successful schema as backup + Schema.saveBackup(schema) + logger.info("Schema fetched successfully from API") + return else: - raise Exception("Schema is not a dict.") - break - except: - time.sleep(10) + raise Exception("Schema is not a dict.") + + except Exception as e: + lastError = e + logger.error(f"Failed to fetch schema (attempt {attempt + 1}/{maxAttempts}): {str(e)}") + if attempt < maxAttempts - 1: + delay = 10 * (attempt + 1) + logger.info(f"Retrying in {delay} seconds...") + time.sleep(delay) + + # All attempts failed, try loading from backup + logger.warning(f"All {maxAttempts} attempts to fetch schema from API failed. Trying backup schema...") + backupSchema = Schema.loadBackup() + + if backupSchema is not None: + logger.info("Using backup schema") + self.schema = Schema(backupSchema) + else: + # No backup available, raise the error + logger.error("No backup schema available. Cannot initialize TF2 library.") + raise Exception(f"Failed to fetch schema after {maxAttempts} attempts and no backup available. Last error: {str(lastError)}") diff --git a/tf2utilities/schema.py b/tf2utilities/schema.py index cea0ddb..1078e55 100644 --- a/tf2utilities/schema.py +++ b/tf2utilities/schema.py @@ -1,10 +1,20 @@ from tf2utilities.webapi import WebRequest from tf2utilities.sku import SKU -import requests +import requests import time import math import vdf import re +import json +import os +import logging + +# Configure logging +logger = logging.getLogger(__name__) + +# Backup schema directory +BACKUP_DIR = os.path.join(os.path.expanduser('~'), '.tf2utilities') +BACKUP_FILE = os.path.join(BACKUP_DIR, 'schema_backup.json') # munitionCrate = { @@ -367,6 +377,42 @@ def __init__(self, data): self.paints = self.getPaints() + # Save schema to backup file + @staticmethod + def saveBackup(schemaData): + try: + # Create backup directory if it doesn't exist + os.makedirs(BACKUP_DIR, exist_ok=True) + + # Save schema to file + with open(BACKUP_FILE, 'w') as f: + json.dump(schemaData, f) + + logger.info(f"Schema backup saved to {BACKUP_FILE}") + return True + except Exception as e: + logger.error(f"Failed to save schema backup: {str(e)}") + return False + + + # Load schema from backup file + @staticmethod + def loadBackup(): + try: + if not os.path.exists(BACKUP_FILE): + logger.warning("No backup schema file found") + return None + + with open(BACKUP_FILE, 'r') as f: + schemaData = json.load(f) + + logger.info(f"Schema backup loaded from {BACKUP_FILE}") + return schemaData + except Exception as e: + logger.error(f"Failed to load schema backup: {str(e)}") + return None + + def getItemByNameWithThe(self, name): items = self.raw["schema"]["items"] @@ -1441,9 +1487,17 @@ def getOverview(apiKey): "key": apiKey, "language": "en" } - overview = WebRequest('GET', 'GetSchemaOverview', 'v0001', input)["result"] - del overview["status"] - return overview + try: + result = WebRequest('GET', 'GetSchemaOverview', 'v0001', input) + if result and "result" in result: + overview = result["result"] + del overview["status"] + return overview + else: + raise Exception("Invalid response structure from Steam API") + except Exception as e: + logger.error(f"Failed to get schema overview: {str(e)}") + raise # Gets schema items @@ -1452,42 +1506,69 @@ def getItems(apiKey): return getAllSchemaItems(apiKey) - # Gets skins / paintkits from TF2 protodefs + # Gets skins / paintkits from TF2 protodefs with retry logic @staticmethod def getPaintKits(): - response = requests.get('https://raw.githubusercontent.com/SteamDatabase/GameTracking-TF2/master/tf/resource/tf_proto_obj_defs_english.txt', timeout=10) - if response.status_code == 200: - parsed = vdf.loads(response.text) - protodefs = parsed["lang"]["Tokens"] - paintkits = [] - for protodef in protodefs: - if protodef not in protodefs: continue - parts = protodef[0:protodef.index(' ')].split('_') - if len(parts) != 3: continue - type = parts[0] - if type != "9": continue - DEF = parts[1] - name = protodefs[protodef] - if name.startswith(DEF + ':'): continue - paintkits.append({"id": DEF, "name": name}) - paintkits = sorted(paintkits, key=lambda x:int(x["id"])) - paintkitsObj = {} - for paintKit in paintkits: - paintKitName = paintKit["name"] - if paintKitName not in [paintkitsObj[paintkit] for paintkit in paintkitsObj]: - paintkitsObj[paintKit["id"]] = paintKit["name"] - return paintkitsObj - else: - raise Exception("Failed to get paintkits.") + maxRetries = 3 + lastException = None + + for attempt in range(maxRetries): + try: + response = requests.get('https://raw.githubusercontent.com/SteamDatabase/GameTracking-TF2/master/tf/resource/tf_proto_obj_defs_english.txt', timeout=10) + response.raise_for_status() + + parsed = vdf.loads(response.text) + protodefs = parsed["lang"]["Tokens"] + paintkits = [] + for protodef in protodefs: + if protodef not in protodefs: continue + parts = protodef[0:protodef.index(' ')].split('_') + if len(parts) != 3: continue + type = parts[0] + if type != "9": continue + DEF = parts[1] + name = protodefs[protodef] + if name.startswith(DEF + ':'): continue + paintkits.append({"id": DEF, "name": name}) + paintkits = sorted(paintkits, key=lambda x:int(x["id"])) + paintkitsObj = {} + for paintKit in paintkits: + paintKitName = paintKit["name"] + if paintKitName not in [paintkitsObj[paintkit] for paintkit in paintkitsObj]: + paintkitsObj[paintKit["id"]] = paintKit["name"] + return paintkitsObj + except Exception as e: + lastException = e + if attempt < maxRetries - 1: + delay = 2 ** attempt + logger.warning(f'Failed to get paintkits (attempt {attempt + 1}/{maxRetries}): {str(e)}. Retrying in {delay}s...') + time.sleep(delay) + continue + + logger.error(f"Failed to get paintkits after {maxRetries} attempts") + raise Exception(f"Failed to get paintkits after {maxRetries} retries. Last error: {str(lastException)}") @staticmethod def getItemsGame(): - response = requests.get('https://raw.githubusercontent.com/SteamDatabase/GameTracking-TF2/master/tf/scripts/items/items_game.txt', timeout=10) - if response.status_code == 200: - return vdf.loads(response.text)["items_game"] - else: - raise Exception("Failed to get items game.") + maxRetries = 3 + lastException = None + + for attempt in range(maxRetries): + try: + response = requests.get('https://raw.githubusercontent.com/SteamDatabase/GameTracking-TF2/master/tf/scripts/items/items_game.txt', timeout=10) + response.raise_for_status() + return vdf.loads(response.text)["items_game"] + except Exception as e: + lastException = e + if attempt < maxRetries - 1: + delay = 2 ** attempt + logger.warning(f'Failed to get items_game (attempt {attempt + 1}/{maxRetries}): {str(e)}. Retrying in {delay}s...') + time.sleep(delay) + continue + + logger.error(f"Failed to get items_game after {maxRetries} attempts") + raise Exception(f"Failed to get items_game after {maxRetries} retries. Last error: {str(lastException)}") # Creates data object used for initializing class @@ -1495,20 +1576,34 @@ def toJSON(self): return {"time": time.time(), "raw": self.raw} -# Recursive function that requests all schema items +# Recursive function that requests all schema items with error handling def getAllSchemaItems(apiKey): - input = { - "key": apiKey, - "language": "en" - } - result = WebRequest('GET', 'GetSchemaItems', 'v0001', input) - items = result["result"]["items"] - while "next" in result["result"]: + try: input = { "key": apiKey, - "language": "en", - "start": result["result"]["next"] + "language": "en" } result = WebRequest('GET', 'GetSchemaItems', 'v0001', input) - items = items + result["result"]["items"] - return items + + if not result or "result" not in result or "items" not in result["result"]: + raise Exception("Invalid response structure from Steam API") + + items = result["result"]["items"] + + while "next" in result["result"]: + input = { + "key": apiKey, + "language": "en", + "start": result["result"]["next"] + } + result = WebRequest('GET', 'GetSchemaItems', 'v0001', input) + + if not result or "result" not in result or "items" not in result["result"]: + raise Exception("Invalid response structure from Steam API during pagination") + + items = items + result["result"]["items"] + + return items + except Exception as e: + logger.error(f"Failed to get all schema items: {str(e)}") + raise diff --git a/tf2utilities/webapi.py b/tf2utilities/webapi.py index eedc2ea..fa6fb78 100644 --- a/tf2utilities/webapi.py +++ b/tf2utilities/webapi.py @@ -1,27 +1,43 @@ import requests import time +import logging +# Configure logging +logger = logging.getLogger(__name__) -# Sends a request to the Steam API + +# Sends a request to the Steam API with exponential backoff def WebRequest(httpMethod, method, version, input): url = 'https://api.steampowered.com' face = 'IEconItems_440' maxRetries = 5 + baseDelay = 1 if httpMethod == "GET": - for _ in range(maxRetries): + lastException = None + for attempt in range(maxRetries): try: response = requests.get(f'{url}/{face}/{method}/{version}', params=input, timeout=10) response.raise_for_status() result = response.json() if not result: - raise Exception(('Steam API returned an empty response.')) - + raise Exception('Steam API returned an empty response.') + return result - except: - time.sleep(1) - continue + except Exception as e: + lastException = e + if attempt < maxRetries - 1: + # Exponential backoff: 1s, 2s, 4s, 8s + delay = baseDelay * (2 ** attempt) + logger.warning(f'Steam API request failed (attempt {attempt + 1}/{maxRetries}): {str(e)}. Retrying in {delay}s...') + time.sleep(delay) + continue + else: + logger.error(f'Steam API request failed after {maxRetries} attempts: {str(e)}') + + # If we've exhausted all retries, raise the last exception + raise Exception(f'Steam API request failed after {maxRetries} retries. Last error: {str(lastException)}') else: - raise Exception(('Unknown Steam API http method.')) + raise Exception('Unknown Steam API http method.') From e09ee2b1b9a8bf8ba3d0f87de3dae045f253d44e Mon Sep 17 00:00:00 2001 From: Sean Cole Date: Wed, 8 Oct 2025 07:03:11 +0800 Subject: [PATCH 5/8] name and sku conversion optimization 1 --- tf2utilities/.claude/settings.local.json | 9 + tf2utilities/schema.py | 107 ++++--- tf2utilities/sku.py | 347 +++++++++++++---------- 3 files changed, 265 insertions(+), 198 deletions(-) create mode 100644 tf2utilities/.claude/settings.local.json diff --git a/tf2utilities/.claude/settings.local.json b/tf2utilities/.claude/settings.local.json new file mode 100644 index 0000000..d910e75 --- /dev/null +++ b/tf2utilities/.claude/settings.local.json @@ -0,0 +1,9 @@ +{ + "permissions": { + "allow": [ + "Read(//mnt/c/Users/vytal/OneDrive/Desktop/Liquid GH/python-tf2-utilities/**)" + ], + "deny": [], + "ask": [] + } +} diff --git a/tf2utilities/schema.py b/tf2utilities/schema.py index 1078e55..567bb07 100644 --- a/tf2utilities/schema.py +++ b/tf2utilities/schema.py @@ -8,6 +8,7 @@ import json import os import logging +from functools import lru_cache # Configure logging logger = logging.getLogger(__name__) @@ -376,6 +377,15 @@ def __init__(self, data): self.paintkits = self.getPaintKitsList() self.paints = self.getPaints() + # Build lookup indexes for O(1) item searches + self._buildItemIndexes() + + # Pre-compute lowercase lookups for performance + self._effectsLower = {effect.lower(): effectId for effect, effectId in self.effects.items()} + self._qualitiesLower = {qual.lower(): qualId for qual, qualId in self.qualities.items()} + self._paintkitsLower = {pk.lower(): pkId for pk, pkId in self.paintkits.items()} + self._paintsLower = {paint.lower(): paintId for paint, paintId in self.paints.items()} + # Save schema to backup file @staticmethod @@ -413,22 +423,39 @@ def loadBackup(): return None - def getItemByNameWithThe(self, name): + # Build lookup indexes for fast item searches + def _buildItemIndexes(self): + """Build hash maps for O(1) item lookups by name""" + self._itemsByName = {} + self._itemsByNameWithoutThe = {} + items = self.raw["schema"]["items"] - + for item in items: - if name.lower().replace("the ", "").strip() == item["item_name"].lower().replace("the ", ""): - if item["item_name"] == "Name Tag" and item["defindex"] == 2093: - # skip and let it find Name Tag with defindex 5020 - continue + # Skip Name Tag with defindex 2093 (use 5020 instead) + if item["item_name"] == "Name Tag" and item["defindex"] == 2093: + continue - if item["item_quality"] == 0: - # skip if Stock Quality - continue + # Skip Stock Quality items + if item["item_quality"] == 0: + continue - return item + item_name_lower = item["item_name"].lower() - return None + # Index by exact name (lowercase) + if item_name_lower not in self._itemsByName: + self._itemsByName[item_name_lower] = item + + # Index by name without "the " prefix + name_without_the = item_name_lower.replace("the ", "").strip() + if name_without_the not in self._itemsByNameWithoutThe: + self._itemsByNameWithoutThe[name_without_the] = item + + + def getItemByNameWithThe(self, name): + """Fast O(1) lookup using pre-built index""" + name_normalized = name.lower().replace("the ", "").strip() + return self._itemsByNameWithoutThe.get(name_normalized) # Gets sku @@ -436,8 +463,21 @@ def getSkuFromName(self, name): return SKU.fromObject(self.getItemObjectFromName(name)) - # Gets sku item object + # Cached wrapper for getItemObjectFromName + @lru_cache(maxsize=2048) + def _getItemObjectFromNameCached(self, name): + """Cached version of getItemObjectFromName for repeated lookups""" + return self._getItemObjectFromNameUncached(name) + + + # Public method with caching def getItemObjectFromName(self, name): + """Gets item object from name with caching for performance""" + return self._getItemObjectFromNameCached(name) + + + # Gets sku item object (internal uncached version) + def _getItemObjectFromNameUncached(self, name): name = name.lower() item = { "defindex": None, @@ -538,13 +578,12 @@ def getItemObjectFromName(self, name): qualitySearch = name.replace(ex, "").strip() break - # Get all qualities + # Get all qualities (optimized with pre-computed lowercase lookups) schema = self.raw["schema"] if not any(ex in qualitySearch for ex in exception): # Make sure qualitySearch does not includes in the exception list # example: "Haunted Ghosts Vintage Tyrolean" - will skip this - for qualityC in self.qualities: - quality = qualityC.lower() + for quality, qualityId in self._qualitiesLower.items(): if quality == "collector's" and "collector's" in qualitySearch and 'chemistry set' in qualitySearch: # Skip setting quality if item is Collector's Chemistrt Set continue @@ -554,14 +593,13 @@ def getItemObjectFromName(self, name): if qualitySearch.startswith(quality): name = name.replace(quality, "").strip() item["quality2"] = item.get("quality2") or item.get("quality") - item["quality"] = self.qualities[qualityC] + item["quality"] = qualityId break - # Check for effects + # Check for effects (optimized with pre-computed lowercase lookups) excludeAtomic = True if any(excludeName in name for excludeName in ["bonk! atomic punch", "atomic accolade"]) else False - for effectC in self.effects: - effect = effectC.lower() + for effect, effectId in self._effectsLower.items(): if effect == "stardust" and "starduster" in name: sub = name.replace("stardust", "").strip() if "starduster" not in sub: @@ -590,7 +628,7 @@ def getItemObjectFromName(self, name): # Skip Frostbite effect if name include Faunted Braken continue - if effect == "hot": + if effect == "hot": # shotgun # shot to hell # plaid potshotter # shot in the dark if not item.get("wear"): @@ -615,7 +653,7 @@ def getItemObjectFromName(self, name): continue if effect in name: name = name.replace(effect, "", 1).strip() - item["effect"] = self.effects[effectC] + item["effect"] = effectId if item["effect"] == 4: if item["quality"] is None: item["quality"] = 5 @@ -626,8 +664,7 @@ def getItemObjectFromName(self, name): break if item.get("wear"): - for paintkitC in self.paintkits: - paintkit = paintkitC.lower() + for paintkit, paintkitId in self._paintkitsLower.items(): if "mk.ii" in name and "mk.ii" not in paintkit: continue if "(green)" in name and "(green)" not in paintkit: @@ -636,7 +673,7 @@ def getItemObjectFromName(self, name): continue if paintkit in name: name = name.replace(paintkit, "").replace("|", "").strip() - item["paintkit"] = self.paintkits[paintkitC] + item["paintkit"] = paintkitId if item.get("effect") is not None: if item.get("quality") == 5 and item.get("quality2") == 11: if not isExplicitElevatedStrange: @@ -684,11 +721,10 @@ def getItemObjectFromName(self, name): if "(paint: " in name: name = name.replace("(paint: ", "").replace(")", "").strip() - for paintC in self.paints: - paint = paintC.lower() + for paint, paintId in self._paintsLower.items(): if paint in name: name = name.replace(paint, "").strip() - item["paint"] = self.paints[paintC] + item["paint"] = paintId break if "kit fabricator" in name and item["killstreak"] > 1: @@ -851,21 +887,8 @@ def getItemByDefindex(self, defindex): # Gets schema item by item name def getItemByItemName(self, name): - items = self.raw["schema"]["items"] - - for item in items: - if name.lower() == item["item_name"].lower(): - if item["item_name"] == "Name Tag" and item["defindex"] == 2093: - # skip and let it find Name Tag with defindex 5020 - continue - - if item["item_quality"] == 0: - # skip if Stock Quality - continue - - return item - - return None + """Fast O(1) lookup using pre-built index""" + return self._itemsByName.get(name.lower()) # Gets schema item by sku diff --git a/tf2utilities/sku.py b/tf2utilities/sku.py index d56c8ae..3b36db7 100644 --- a/tf2utilities/sku.py +++ b/tf2utilities/sku.py @@ -1,170 +1,205 @@ +# Constants for attribute defindexes (for fast lookups) +ATTR_KILLSTREAK = 2025 +ATTR_AUSTRALIUM = 2027 +ATTR_EFFECT = 134 +ATTR_FESTIVE = 2053 +ATTR_PAINTKIT = 834 +ATTR_WEAR = 749 +ATTR_QUALITY2 = 214 +ATTR_CRAFTNUMBER = 229 +ATTR_CRATESERIES = 187 +ATTR_TARGET = 2012 +ATTR_PAINT = 142 + +# Template for item objects (shared to reduce allocations) +ITEM_TEMPLATE = { + "defindex": 0, + "quality": 0, + "craftable": True, + "tradable": True, + "killstreak": 0, + "australium": False, + "effect": None, + "festive": False, + "paintkit": None, + "wear": None, + "quality2": None, + "craftnumber": None, + "crateseries": None, + "target": None, + "output": None, + "outputQuality": None, + "paint": None +} + + class SKU: - # Convert SKU to item object + # Convert SKU to item object (optimized) @staticmethod def fromString(sku): - TEMPLATE = { - "defindex": 0, - "quality": 0, - "craftable": True, - "tradable": True, - "killstreak": 0, - "australium": False, - "effect": None, - "festive": False, - "paintkit": None, - "wear": None, - "quality2": None, - "craftnumber": None, - "crateseries": None, - "target": None, - "output": None, - "outputQuality": None, - "paint": None - } - attributes = {} - + result = ITEM_TEMPLATE.copy() + parts = sku.split(";") - partsCount = len(parts) - if partsCount > 0: - if str(parts[0]).isnumeric(): - attributes["defindex"] = int(parts[0]) - parts.pop(0) + # Parse defindex (first part) + if parts and parts[0].isnumeric(): + result["defindex"] = int(parts[0]) + parts = parts[1:] - if partsCount > 0: - if str(parts[0]).isnumeric(): - attributes["quality"] = int(parts[0]) - parts.pop(0) + # Parse quality (second part) + if parts and parts[0].isnumeric(): + result["quality"] = int(parts[0]) + parts = parts[1:] + # Parse remaining attributes for part in parts: - attribute = str(part.replace("-", "")) - - if attribute == "uncraftable": - attributes["craftable"] = False - elif attribute in ["untradeable", "untradable"]: - attributes["tradable"] = False - elif attribute == "australium": - attributes["australium"] = True - elif attribute == "festive": - attributes["festive"] = True - elif attribute == "strange": - attributes["quality2"] = 11 - elif attribute.startswith("kt") and attribute[2:].isnumeric(): - attributes["killstreak"] = int(attribute[2:]) - elif attribute.startswith("u") and attribute[1:].isnumeric(): - attributes["effect"] = int(attribute[1:]) - elif attribute.startswith("pk") and attribute[2:].isnumeric(): - attributes["paintkit"] = int(attribute[2:]) - elif attribute.startswith("w") and attribute[1:].isnumeric(): - attributes["wear"] = int(attribute[1:]) - elif attribute.startswith("td") and attribute[2:].isnumeric(): - attributes["target"] = int(attribute[2:]) - elif attribute.startswith("n") and attribute[1:].isnumeric(): - attributes["craftnumber"] = int(attribute[1:]) - elif attribute.startswith("c") and attribute[1:].isnumeric(): - attributes["crateseries"] = int(attribute[1:]) - elif attribute.startswith("od") and attribute[2:].isnumeric(): - attributes["output"] = int(attribute[2:]) - elif attribute.startswith("oq") and attribute[2:].isnumeric(): - attributes["outputQuality"] = int(attribute[2:]) - elif attribute.startswith("p") and attribute[1:].isnumeric(): - attributes["paint"] = int(attribute[1:]) - - for attr in TEMPLATE: - if attr in attributes: TEMPLATE[attr] = attributes[attr] - - return TEMPLATE - - - # Convert item object to SKU + part_stripped = part.replace("-", "") + + # Simple string flags + if part_stripped == "uncraftable": + result["craftable"] = False + elif part_stripped in ("untradeable", "untradable"): + result["tradable"] = False + elif part_stripped == "australium": + result["australium"] = True + elif part_stripped == "festive": + result["festive"] = True + elif part_stripped == "strange": + result["quality2"] = 11 + # Prefixed numeric attributes + elif len(part_stripped) > 2: + prefix = part_stripped[:2] + suffix = part_stripped[2:] + if suffix.isnumeric(): + value = int(suffix) + if prefix == "kt": + result["killstreak"] = value + elif prefix == "pk": + result["paintkit"] = value + elif prefix == "td": + result["target"] = value + elif prefix == "od": + result["output"] = value + elif prefix == "oq": + result["outputQuality"] = value + # Single-char prefix attributes + if len(part_stripped) > 1: + prefix = part_stripped[0] + suffix = part_stripped[1:] + if suffix.isnumeric(): + value = int(suffix) + if prefix == "u": + result["effect"] = value + elif prefix == "w": + result["wear"] = value + elif prefix == "n": + result["craftnumber"] = value + elif prefix == "c": + result["crateseries"] = value + elif prefix == "p": + result["paint"] = value + + return result + + + # Convert item object to SKU (optimized) @staticmethod def fromObject(item): - TEMPLATE = { - "defindex": 0, - "quality": 0, - "craftable": True, - "tradable": True, - "killstreak": 0, - "australium": False, - "effect": None, - "festive": False, - "paintkit": None, - "wear": None, - "quality2": None, - "craftnumber": None, - "crateseries": None, - "target": None, - "output": None, - "outputQuality": None, - "paint": None - } - for attr in TEMPLATE: - if attr in item: TEMPLATE[attr] = item[attr] - - sku = f'{item["defindex"]};{item["quality"]}' - - if item.get("effect"): sku += f";u{item['effect']}" - if item.get("australium") is True: sku += ";australium" - if item.get("craftable") is False: sku += ";uncraftable" - if item.get("tradable") is False: sku += ";untradable" - if item.get("wear"): sku += f";w{item['wear']}" - if item.get("paintkit") and isinstance(item['paintkit'], int): sku += f";pk{item['paintkit']}" - if item.get("quality2") and item["quality2"] == 11: sku += ";strange" - if item.get("killstreak") and isinstance(item['killstreak'], int) and item["killstreak"] != 0: sku += f";kt-{item['killstreak']}" - if item.get("target"): sku += f";td-{item['target']}" - if item.get("festive") is True: sku += ';festive' - if item.get("craftnumber"): sku += f";n{item['craftnumber']}" - if item.get("crateseries"): sku += f";c{item['crateseries']}" - if item.get("output"): sku += f";od-{item['output']}" - if item.get("outputQuality"): sku += f";oq{item['outputQuality']}" - if item.get("paint"): sku += f";p{item['paint']}" - - return sku + # Use list for faster string building + parts = [str(item.get("defindex", 0)), str(item.get("quality", 0))] + + # Append parts in order for consistent SKU format + if item.get("effect"): + parts.append(f"u{item['effect']}") + if item.get("australium") is True: + parts.append("australium") + if item.get("craftable") is False: + parts.append("uncraftable") + if item.get("tradable") is False: + parts.append("untradable") + if item.get("wear"): + parts.append(f"w{item['wear']}") + if item.get("paintkit") and isinstance(item['paintkit'], int): + parts.append(f"pk{item['paintkit']}") + if item.get("quality2") == 11: + parts.append("strange") + + killstreak = item.get("killstreak") + if killstreak and isinstance(killstreak, int) and killstreak != 0: + parts.append(f"kt-{killstreak}") + + if item.get("target"): + parts.append(f"td-{item['target']}") + if item.get("festive") is True: + parts.append("festive") + if item.get("craftnumber"): + parts.append(f"n{item['craftnumber']}") + if item.get("crateseries"): + parts.append(f"c{item['crateseries']}") + if item.get("output"): + parts.append(f"od-{item['output']}") + if item.get("outputQuality"): + parts.append(f"oq{item['outputQuality']}") + if item.get("paint"): + parts.append(f"p{item['paint']}") + + return ";".join(parts) @staticmethod def fromAPI(item): - TEMPLATE = { - "defindex": 0, - "quality": 0, - "craftable": True, - "tradable": True, - "killstreak": 0, - "australium": False, - "effect": None, - "festive": False, - "paintkit": None, - "wear": None, - "quality2": None, - "craftnumber": None, - "crateseries": None, - "target": None, - "output": None, - "outputQuality": None, - "paint": None - } - - TEMPLATE["defindex"] = item["defindex"] - TEMPLATE["quality"] = item["quality"] - if item.get("flag_cannot_craft"): TEMPLATE["craftable"] = False - if item.get("flag_cannot_trade"): TEMPLATE["tradable"] = False - if item.get("attributes"): - for attribute in item["attributes"]: - if int(attribute["defindex"]) == 2025: TEMPLATE["killstreak"] = attribute["float_value"] - if int(attribute["defindex"]) == 2027: TEMPLATE["australium"] = True if attribute['float_value'] == 1 else False - if int(attribute["defindex"]) == 134: TEMPLATE["effect"] = attribute['float_value'] - if int(attribute["defindex"]) == 2053: TEMPLATE["festive"] = True if attribute['float_value'] == 1 else False - if int(attribute["defindex"]) == 834: TEMPLATE["paintkit"] = attribute["float_value"] - if int(attribute["defindex"]) == 749: TEMPLATE["wear"] = attribute["float_value"] - if int(attribute["defindex"]) == 214 and item['quality'] == 5: TEMPLATE["quality2"] = attribute["value"] - if int(attribute["defindex"]) == 229: TEMPLATE["craftnumber"] = attribute["value"] - if int(attribute["defindex"]) == 187: TEMPLATE["crateseries"] = attribute["float_value"] - if 2000 <= int(attribute["defindex"]) <= 2009 and attribute.get("attributes"): - for attr in attribute["attributes"]: - if int(attr["defindex"]) == 2012: TEMPLATE["target"] = attr["float_value"] - if attribute.get("is_output") and attribute["is_output"] is True: - TEMPLATE["output"] = attribute["itemdef"] - TEMPLATE["outputQuality"] = attribute["quantity"] - if int(attribute["defindex"]) == 142: TEMPLATE["paint"] = attribute["float_value"] - - return SKU.fromObject(TEMPLATE) + """Convert API item to SKU (optimized with attribute lookup table)""" + result = ITEM_TEMPLATE.copy() + + result["defindex"] = item["defindex"] + result["quality"] = item["quality"] + + if item.get("flag_cannot_craft"): + result["craftable"] = False + if item.get("flag_cannot_trade"): + result["tradable"] = False + + attributes = item.get("attributes") + if attributes: + item_quality = item['quality'] + + for attribute in attributes: + # Convert defindex once per iteration + attr_defindex = int(attribute["defindex"]) + + # Use dictionary lookup for faster attribute processing + if attr_defindex == ATTR_KILLSTREAK: + result["killstreak"] = attribute["float_value"] + elif attr_defindex == ATTR_AUSTRALIUM: + result["australium"] = attribute['float_value'] == 1 + elif attr_defindex == ATTR_EFFECT: + result["effect"] = attribute['float_value'] + elif attr_defindex == ATTR_FESTIVE: + result["festive"] = attribute['float_value'] == 1 + elif attr_defindex == ATTR_PAINTKIT: + result["paintkit"] = attribute["float_value"] + elif attr_defindex == ATTR_WEAR: + result["wear"] = attribute["float_value"] + elif attr_defindex == ATTR_QUALITY2 and item_quality == 5: + result["quality2"] = attribute["value"] + elif attr_defindex == ATTR_CRAFTNUMBER: + result["craftnumber"] = attribute["value"] + elif attr_defindex == ATTR_CRATESERIES: + result["crateseries"] = attribute["float_value"] + elif attr_defindex == ATTR_PAINT: + result["paint"] = attribute["float_value"] + elif 2000 <= attr_defindex <= 2009: + # Handle nested attributes for target + nested_attrs = attribute.get("attributes") + if nested_attrs: + for attr in nested_attrs: + if int(attr["defindex"]) == ATTR_TARGET: + result["target"] = attr["float_value"] + break + + # Handle output items + if attribute.get("is_output") is True: + result["output"] = attribute["itemdef"] + result["outputQuality"] = attribute["quantity"] + + return SKU.fromObject(result) From 6480549f5a6e79bb5e230e39b9b9c89f816bb1af Mon Sep 17 00:00:00 2001 From: vytal <80233205+SeanCole02@users.noreply.github.com> Date: Mon, 27 Oct 2025 06:29:17 +0800 Subject: [PATCH 6/8] Change exception from list to set in schema.py Refactor exception variable from list to set for quality names, added haunted mist as an exception --- tf2utilities/schema.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tf2utilities/schema.py b/tf2utilities/schema.py index 567bb07..24d1405 100644 --- a/tf2utilities/schema.py +++ b/tf2utilities/schema.py @@ -558,7 +558,7 @@ def _getItemObjectFromNameUncached(self, name): item["festive"] = True # Try to find quality name in name - exception = [ + exception = { 'haunted ghosts', 'haunted phantasm jr', 'haunted phantasm', @@ -569,8 +569,9 @@ def _getItemObjectFromNameUncached(self, name): 'vintage merryweather', 'haunted kraken', 'haunted forever!', - 'haunted cremation' - ] + 'haunted cremation', + "haunted mist", + } qualitySearch = name for ex in exception: From b41f4fdf293edfc2bb89285532999c87ad0158f9 Mon Sep 17 00:00:00 2001 From: Sean Cole Date: Mon, 27 Oct 2025 06:32:59 +0800 Subject: [PATCH 7/8] updated haunted exceptions --- tf2utilities/schema.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tf2utilities/schema.py b/tf2utilities/schema.py index 567bb07..24d1405 100644 --- a/tf2utilities/schema.py +++ b/tf2utilities/schema.py @@ -558,7 +558,7 @@ def _getItemObjectFromNameUncached(self, name): item["festive"] = True # Try to find quality name in name - exception = [ + exception = { 'haunted ghosts', 'haunted phantasm jr', 'haunted phantasm', @@ -569,8 +569,9 @@ def _getItemObjectFromNameUncached(self, name): 'vintage merryweather', 'haunted kraken', 'haunted forever!', - 'haunted cremation' - ] + 'haunted cremation', + "haunted mist", + } qualitySearch = name for ex in exception: From 5de947397d696cdf5c630cd2a492d9d64660844e Mon Sep 17 00:00:00 2001 From: Sean Cole Date: Wed, 17 Dec 2025 14:37:00 +0800 Subject: [PATCH 8/8] frostbite fit exception --- tf2utilities/schema.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tf2utilities/schema.py b/tf2utilities/schema.py index 24d1405..6d81132 100644 --- a/tf2utilities/schema.py +++ b/tf2utilities/schema.py @@ -625,8 +625,8 @@ def _getItemObjectFromNameUncached(self, name): if effect == "haunted" and "haunted kraken" in name: # Skip Haunted effect if name include Haunted Kraken continue - if effect == "frostbite" and "frostbite bonnet" in name: - # Skip Frostbite effect if name include Faunted Braken + if effect == "frostbite" and ("frostbite bonnet" in name or "frostbite fit" in name): + # Skip Frostbite effect for items with Frostbite in their name continue if effect == "hot":