404 lines
20 KiB
Python
404 lines
20 KiB
Python
"""
|
|
AI HTTP server interface for AI training
|
|
|
|
Opens a TCP socket to listen for incoming connections from an AI client.
|
|
When a new connection is established, the server intialise a new training session :
|
|
- Model Score is set to 0
|
|
For the duration of the training session, two mapSets are created, one with hard maps and one with all of the other maps.
|
|
When the AI client sends a request for a new game or validation game,
|
|
the server loads a map based on the saved best score or from the hard mapSet if a validation game is requested,
|
|
and sends it to the client as a matrix of block types strings.
|
|
The client then sends back a list of moves to be executed by the game engine. E
|
|
Each item of the list is a different model givin a string representing a move from GameState.PLAYER_MOVEMENTS to execute in the simulation.
|
|
The server then executes the moves in the game engine for each model
|
|
and sends back the new player position or game scores
|
|
(calculated here using formula with weights) if the game is finished to the client.
|
|
This process is repeated until all models have finished the game.
|
|
The client will then train the model based on their scores
|
|
and the server set the Model Score to the best score of the models
|
|
and shows in a separate thread the best model series of moves on the previous map.
|
|
The client then sends a new request for a new game and the process repeats.
|
|
The simulation runs in a pool of threads to allow multiple models to be trained in parallel.
|
|
The handler makes sure an AI client is not stuck and only allows a maximum number
|
|
of moves to be executed in a single game simulation, based on the map min_moves value.
|
|
"""
|
|
|
|
import json
|
|
import math
|
|
import os
|
|
import random
|
|
import socketserver
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from copy import deepcopy
|
|
from threading import Condition, Thread
|
|
from time import sleep
|
|
|
|
from game.handlers.aiSimulation import AiSimulation
|
|
from game.game import Game
|
|
from game.gameState import GameState
|
|
from game.handlers.gameHandler import GameHandler
|
|
from game.map import Map
|
|
from game.mapReader import MapReader
|
|
from game.mapSet import MapSet
|
|
|
|
class AiHttpGameHandler(GameHandler):
|
|
DEFAULT_HOST = "127.0.0.1"
|
|
DEFAULT_PORT = 8765
|
|
VALIDATION_DIFFICULTY = 90
|
|
REPLAY_FRAME_DELAY = 0.1
|
|
# Scoring weights TOTAL MUST BE 100
|
|
COINS_WEIGHT = 50
|
|
STEPS_WEIGHT = 20
|
|
DISTANCE_WEIGHT = 10
|
|
WIN_WEIGHT = 20
|
|
PLAY_ALL_MAPS_PER_GENERATION = False # Are all maps played during a generation or only MAPS_PER_GENERATION maps per generation are played
|
|
MAPS_PER_GENERATION = 3 # Evaluate across x maps per generation. Only used if PLAY_ALL_MAPS_PER_GENERATION is False. If True, all maps are played in a generation.
|
|
MAP_PASS_MIN_SCORE = 20 # Minimum score to consider a map passed. Only used if PLAY_ALL_MAPS_PER_GENERATION is False. If True, all maps are played in a generation.
|
|
MAP_DIFFICULTY_VARIANCE = 5 # Lower value will make the map selection more variable, higher value will make the map selection more consistent with the current performance. Only used if PLAY_ALL_MAPS_PER_GENERATION is False. A value too high may produce no evolving behavior
|
|
CONSISTENCY_PENALTY_WEIGHT = 0.4 # Penalty for inconsistent map scores
|
|
|
|
def __init__(self, host: str | None = None, port: int | None = None,
|
|
max_workers: int | None = None):
|
|
super().__init__()
|
|
self.host = host or os.getenv("AI_HOST", self.DEFAULT_HOST)
|
|
self.port = port or int(os.getenv("AI_PORT", self.DEFAULT_PORT))
|
|
self.max_workers = max_workers or int(os.getenv("AI_MAX_WORKERS", "4"))
|
|
self.map_reader = MapReader()
|
|
self.map_set = MapSet(self.map_reader, self.map_reader.get_map_names())
|
|
self.validation_maps = [
|
|
game_map for game_map in self.map_set.maps
|
|
if game_map.difficulty >= self.VALIDATION_DIFFICULTY
|
|
] or list(self.map_set.maps)
|
|
|
|
if (self.COINS_WEIGHT + self.DISTANCE_WEIGHT + self.STEPS_WEIGHT + self.WIN_WEIGHT) != 100:
|
|
raise ValueError("Scoring weights must sum to 100.")
|
|
|
|
def run(self) -> None:
|
|
handler = self
|
|
|
|
class RequestHandler(socketserver.StreamRequestHandler):
|
|
def handle(self) -> None:
|
|
session = {
|
|
"performance": 0.0,
|
|
"simulations": None,
|
|
"best_run": None,
|
|
"replay_condition": Condition(),
|
|
"client_connected": True,
|
|
"stagnant_generations": 0,
|
|
"map_round": 0,
|
|
"cumulative_scores": {},
|
|
"last_results": {}
|
|
}
|
|
replay_thread = Thread(target=handler._replay_best_runs, args=(session,))
|
|
replay_thread.start()
|
|
print(f"AI client connected: {self.client_address}", flush=True)
|
|
try:
|
|
for raw_request in self.rfile:
|
|
try:
|
|
request = json.loads(raw_request)
|
|
response = handler._handle_request(request, session)
|
|
except (json.JSONDecodeError, TypeError, ValueError, KeyError) as error:
|
|
response = {"ok": False, "type": "error", "error": str(error)}
|
|
compact_response = json.dumps(response, separators=(",", ":"))
|
|
self.wfile.write((compact_response + "\n").encode("utf-8"))
|
|
self.wfile.flush()
|
|
finally:
|
|
with session["replay_condition"]:
|
|
session["client_connected"] = False
|
|
session["replay_condition"].notify_all()
|
|
replay_thread.join()
|
|
print(f"AI client disconnected: {self.client_address}", flush=True)
|
|
|
|
class ThreadingTCPServer(socketserver.ThreadingTCPServer):
|
|
allow_reuse_address = True
|
|
|
|
with ThreadingTCPServer((self.host, self.port), RequestHandler) as server:
|
|
self.port = server.server_address[1]
|
|
print(f"AI TCP server listening on {self.host}:{self.port}")
|
|
server.serve_forever()
|
|
|
|
def _handle_request(self, request: dict, session: dict | None = None) -> dict:
|
|
if not isinstance(request, dict):
|
|
raise ValueError("Each request must be a JSON object.")
|
|
request_type = request.get("type")
|
|
if request_type == "ping":
|
|
return {"ok": True, "type": "pong"}
|
|
if request_type == "simulate":
|
|
return self._simulate(request, session)
|
|
if request_type == "step":
|
|
return self._step(request, session)
|
|
raise ValueError("Unknown request type. Expected 'ping', 'simulate', or 'step'.")
|
|
|
|
def _simulate(self, request: dict, session: dict | None = None) -> dict:
|
|
models = request.get("models")
|
|
if not isinstance(models, list) or not models:
|
|
raise ValueError("'models' must be a non-empty list.")
|
|
if not all(isinstance(model, dict) for model in models):
|
|
raise ValueError("Each item in 'models' must be a JSON object.")
|
|
validation = bool(request.get("validation", False))
|
|
if session is None:
|
|
raise ValueError("Simulation requests must belong to a TCP session.")
|
|
if session.get("simulations"):
|
|
raise ValueError("A simulation is already active in this session.")
|
|
|
|
# Initialize generation round tracking and select 3 maps upfront if starting fresh
|
|
if session.get("map_round", 0) == 0:
|
|
session["generation"] = session.get("generation", 0) + 1
|
|
session["map_scores"] = {model.get("id"): [] for model in models}
|
|
|
|
# Pre-select the 3 maps for this entire generation based on current performance
|
|
if validation:
|
|
session["generation_maps"] = [
|
|
random.choice(self.validation_maps)
|
|
for _ in range(self.MAPS_PER_GENERATION)
|
|
]
|
|
else:
|
|
perf = session.get("performance", 0.0)
|
|
if self.PLAY_ALL_MAPS_PER_GENERATION:
|
|
session["generation_maps"] = self.map_set.get_maps()
|
|
else:
|
|
session["generation_maps"] = sorted(self.map_set.get_maps_by_difficulty(round(perf), self.MAPS_PER_GENERATION), key=lambda m: m.difficulty)
|
|
|
|
# Fetch the pre-selected map for the current round
|
|
map_round = session.get("map_round", 0)
|
|
selected_map = session["generation_maps"][map_round]
|
|
|
|
session["last_result"] = None
|
|
|
|
simulations = {}
|
|
for model in models:
|
|
model_id = model.get("id")
|
|
if not isinstance(model_id, str) or not model_id:
|
|
raise ValueError("Each model must have a non-empty string 'id'.")
|
|
if model_id in simulations:
|
|
raise ValueError(f"Duplicate model id '{model_id}'.")
|
|
simulation = AiSimulation(selected_map)
|
|
simulation.start()
|
|
simulations[model_id] = simulation
|
|
|
|
session["simulations"] = simulations
|
|
session["map"] = selected_map
|
|
return {
|
|
"ok": True,
|
|
"type": "move_request",
|
|
"map": self._map_payload(selected_map),
|
|
"results": self._results(simulations),
|
|
}
|
|
|
|
def _step(self, request: dict, session: dict | None) -> dict:
|
|
if session is None:
|
|
raise ValueError("No simulation is active. Send a 'simulate' request first.")
|
|
if not session.get("simulations"):
|
|
if session.get("last_result") is not None:
|
|
return session["last_result"]
|
|
raise ValueError("No simulation is active. Send a 'simulate' request first.")
|
|
moves = request.get("moves")
|
|
# print(
|
|
# "Step payload contains moves for: "
|
|
# f"{list(moves) if isinstance(moves, dict) else type(moves).__name__}",
|
|
# flush=True,
|
|
# )
|
|
if not isinstance(moves, dict):
|
|
raise ValueError("'moves' must be an object mapping model ids to moves.")
|
|
simulations = session["simulations"]
|
|
if set(moves) != set(simulations):
|
|
raise ValueError("'moves' must contain exactly one move for every model.")
|
|
for model_id, move in moves.items():
|
|
if move is not None and (not isinstance(move, str) or move not in GameState.PLAYER_MOVEMENTS):
|
|
raise ValueError(f"Invalid move for model '{model_id}'.")
|
|
# print("Dispatching step to simulation workers", flush=True)
|
|
|
|
def step_model(item):
|
|
model_id, move = item
|
|
simulation = simulations[model_id]
|
|
# print(f"Starting step for model '{model_id}'", flush=True)
|
|
result = simulation.step(move)
|
|
result["score"] = self._score_simulation(simulation, result)
|
|
# print(f"Finished step for model '{model_id}'", flush=True)
|
|
return model_id, result
|
|
|
|
with ThreadPoolExecutor(max_workers=min(self.max_workers, len(simulations))) as executor:
|
|
results = dict(executor.map(step_model, moves.items()))
|
|
# print(f"Completed step for models: {list(moves)}", flush=True)
|
|
all_finished = all(result["finished"] for result in results.values())
|
|
response = {"ok": True, "type": "move_request", "results": results}
|
|
|
|
if all_finished:
|
|
session["map_round"] += 1
|
|
|
|
# Accumulate scores for this map round
|
|
for model_id, res in results.items():
|
|
session["map_scores"][model_id].append(res["score"])
|
|
|
|
# Check if we have completed all maps for this generation
|
|
if session["map_round"] < len(session["generation_maps"]):
|
|
# More maps remaining in this generation! Tell client to simulate next map.
|
|
response["type"] = "map_round_complete"
|
|
response["map_round"] = session["map_round"]
|
|
session["simulations"] = None
|
|
session["last_result"] = response
|
|
else:
|
|
# All maps completed -> finalize generation results (Average scores)
|
|
response["type"] = "simulation_result"
|
|
final_results = {}
|
|
|
|
# Calculate final scores for each model based on their score on every map
|
|
for model_id, scores in session["map_scores"].items():
|
|
# Calculate Mean Squared Error from perfection (100)
|
|
squared_errors = [(100.0 - score) ** 2 for score in scores]
|
|
mse = sum(squared_errors) / len(squared_errors)
|
|
rmse = math.sqrt(mse)
|
|
|
|
# Final score penalizes variance/drops sharply based on RMSE
|
|
final_score = round(max(0.0, min(100.0, 100.0 - rmse)), 2)
|
|
|
|
# Attach final structural properties from the last results set
|
|
final_results[model_id] = deepcopy(results[model_id])
|
|
final_results[model_id]["score"] = final_score
|
|
|
|
response["results"] = final_results
|
|
best_score = max(res["score"] for res in final_results.values())
|
|
best_model_id = max(final_results, key=lambda mid: final_results[mid]["score"])
|
|
response["score"] = best_score
|
|
|
|
# --- Performance Update ---
|
|
# Average score of the 90th percentile of the final results
|
|
sorted_scores = sorted((res["score"] for res in final_results.values()), reverse=True)
|
|
top_90_count = max(1, round(len(sorted_scores) * 0.9))
|
|
top_90_scores = sorted_scores[:top_90_count]
|
|
avg_top_90_score = sum(top_90_scores) / len(top_90_scores)
|
|
performance_delta = (avg_top_90_score - self.MAP_PASS_MIN_SCORE) / self.MAP_DIFFICULTY_VARIANCE
|
|
performance_delta = max(0.05, performance_delta)
|
|
session["performance"] = max(0.0, session.get("performance", 0.0) + performance_delta)
|
|
# print(f"Generation {session['generation']} completed on difficulty {map_difficulty}. Best score: {best_score:.2f}. Updated performance: {session['performance']:.2f}", flush=True)
|
|
|
|
# Reset round counter for the next generation
|
|
session["map_round"] = 0
|
|
session["last_result"] = response
|
|
session["simulations"] = None
|
|
session["best_run"] = (
|
|
deepcopy(session["map"]),
|
|
list(simulations[best_model_id].moves),
|
|
session["generation"],
|
|
session["performance"],
|
|
)
|
|
replay_condition = session.get("replay_condition")
|
|
if replay_condition is not None:
|
|
with replay_condition:
|
|
replay_condition.notify_all()
|
|
return response
|
|
|
|
def _replay_best_runs(self, session: dict) -> None:
|
|
condition = session["replay_condition"]
|
|
while True:
|
|
with condition:
|
|
condition.wait_for(
|
|
lambda: session["best_run"] is not None
|
|
or not session["client_connected"]
|
|
)
|
|
best_run = session["best_run"]
|
|
session["best_run"] = None
|
|
if best_run is None:
|
|
return
|
|
self._show_best_run(*best_run)
|
|
|
|
def _show_best_run(self, game_map, moves: list[str], generation: int, performance: float) -> None:
|
|
game = Game(game_map, GameState())
|
|
remaining_moves = iter(moves)
|
|
|
|
def on_input_request() -> str:
|
|
try:
|
|
return next(remaining_moves)
|
|
except StopIteration:
|
|
game.stop_game()
|
|
return -1
|
|
|
|
def on_position_update() -> None:
|
|
game_string = game.to_string()
|
|
frame_lines = []
|
|
for index, line in enumerate(game_string.splitlines()):
|
|
if index == 0:
|
|
line += "\tGeneration: " + str(generation)
|
|
elif index == 1:
|
|
line += "\tPerformance: " + str(round(performance, 2))
|
|
elif index == 2:
|
|
line += "\tCoins: " + str(game.gameState.coins)
|
|
elif index == 3:
|
|
line += "\tMoves: " + str(game.gameState.movesCount)
|
|
elif index == 4:
|
|
line += "\tMove: " + moves[game.gameState.movesCount - 1] if len(moves) >= game.gameState.movesCount else "\tMove: None"
|
|
frame_lines.append(line)
|
|
print("\033[2J\033[H\n\n\n" + "\n".join(frame_lines), flush=True)
|
|
sleep(self.REPLAY_FRAME_DELAY)
|
|
|
|
game.set_waiting_for_input_callback(on_input_request)
|
|
game.set_position_update_callback(on_position_update)
|
|
game.start_game()
|
|
|
|
def _results(self, simulations: dict) -> dict:
|
|
results = {}
|
|
for model_id, simulation in simulations.items():
|
|
result = simulation.result()
|
|
result["score"] = self._score_simulation(simulation, result)
|
|
results[model_id] = result
|
|
return results
|
|
|
|
def _select_map(self, performance: float, validation: bool):
|
|
if validation:
|
|
return random.choice(self.validation_maps)
|
|
maps = self.map_set.get_maps_by_difficulty(round(performance), 2)
|
|
return random.choice(maps)
|
|
|
|
def _score_simulation(self, simulation: AiSimulation, result: dict) -> int:
|
|
game_map = simulation.game.map
|
|
raw_score = self._score_result(game_map, result)
|
|
max_score = self._get_map_max_score(game_map)
|
|
if max_score == 0:
|
|
return 0
|
|
percentage = (raw_score / max_score) * 100.0
|
|
return round(max(0.0, min(100.0, percentage)))
|
|
|
|
def _get_map_max_score(self, game_map: Map) -> int:
|
|
result = {}
|
|
result["position"] = game_map.get_first_block_type_position("finish")
|
|
result["coins"] = game_map.count_blocks_of_type("coin")
|
|
result["won"] = True
|
|
result["moves"] = game_map.min_moves
|
|
return self._score_result(game_map, result)
|
|
|
|
def _score_result(self, game_map: Map, result: dict) -> int:
|
|
finish = game_map.get_first_block_type_position("finish")
|
|
spawn = game_map.get_first_block_type_position("spawn_point")
|
|
total_coins = game_map.count_blocks_of_type("coin")
|
|
|
|
if finish is None or spawn is None:
|
|
distance_progress = 0.0
|
|
else:
|
|
start_distance = abs(spawn[0] - finish[0]) + abs(spawn[1] - finish[1])
|
|
remaining_distance = abs(result["position"][0] - finish[0]) + abs(result["position"][1] - finish[1])
|
|
distance_progress = 1.0 if start_distance == 0 else max(
|
|
0.0, min(1.0, 1.0 - remaining_distance / start_distance)
|
|
)
|
|
steps_progress = 0.0 if result["moves"] == 0 else min(1.0, 1.0 - result["moves"] / game_map.min_moves)
|
|
|
|
coin_progress = 0.0 if total_coins == 0 else min(1.0, result["coins"] / total_coins)
|
|
return round(
|
|
self.COINS_WEIGHT * coin_progress
|
|
+ self.DISTANCE_WEIGHT * distance_progress
|
|
+ self.STEPS_WEIGHT * steps_progress
|
|
+ self.WIN_WEIGHT * result["won"]
|
|
)
|
|
|
|
@staticmethod
|
|
def _map_payload(game_map: Map) -> dict:
|
|
return {
|
|
"name": game_map.name,
|
|
"difficulty": game_map.difficulty,
|
|
"layout": game_map.to_type_dict(),
|
|
}
|
|
|
|
def on_position_update(self) -> None:
|
|
pass
|
|
|
|
def on_input_request(self) -> str:
|
|
raise RuntimeError("AI input is supplied one turn at a time in 'step' requests.") |