Implementation (Python basic)

This implementation demonstrates Q-Learning in a simple 3x3 grid world. The agent's goal is to navigate from any starting state to the goal state (2,2) while maximizing cumulative rewards and avoiding penalties.
import numpy as np
import random # For epsilon-greedy action selection

print("--- Q-Learning Example (Gridworld) ---")

# 1. Define the Environment:
# R = Rewards matrix for a 3x3 grid.
# States are represented by (row, col) tuples.
#   Columns: (0,0) (0,1) (0,2)
#   Rows:
#   0: [ 0,  0, -1]  (-1 indicates a penalty state)
#   1: [ 0,  0,  0]
#   2: [-1,  0, 10]  (10 indicates a large reward, which is our goal state (2,2))
R = np.array([
    [0, 0, -1],
    [0, 0, 0],
    [-1, 0, 10]
])

# Define possible actions an agent can take: Up, Down, Left, Right.
# These are represented as changes in (row, col) coordinates.
actions = [(-1, 0), (1, 0), (0, -1), (0, 1)] # Index mapping: 0=Up, 1=Down, 2=Left, 3=Right
num_actions = len(actions)
num_states_r, num_states_c = R.shape # Get the dimensions of the grid (3 rows, 3 columns)

# 2. Initialize Q-table:
# The Q-table stores the Q-value for each possible (state, action) pair.
# Its dimensions are (number_of_rows, number_of_columns, number_of_actions).
# Initially, all Q-values are set to zero, as the agent has no experience.
Q = np.zeros((num_states_r, num_states_c, num_actions), dtype=float)

# 3. Hyperparameters for Q-Learning:
gamma = 0.9   # Discount factor: Determines the importance of future rewards. A value closer to 1 makes the agent more farsighted.
alpha = 0.1   # Learning rate: Controls how much new information (from the Bellman equation) updates the old Q-value.
epsilon = 0.2 # Epsilon for epsilon-greedy action selection:
              #   - With probability `epsilon`, the agent explores (chooses a random action).
              #   - With probability `1 - epsilon`, the agent exploits (chooses the action with the highest Q-value).
n_episodes = 1000 # Number of training episodes. Each episode is a complete run from a start state to a goal/terminal state.

# 4. Helper Functions:
# Checks if a given state (row, col) is within the grid boundaries.
def is_valid_state(state):
    return 0 <= state[0] < num_states_r and 0 <= state[1] < num_states_c

# Calculates the next state given a current state and an action index.
# If the chosen action would lead outside the grid boundaries, the agent remains in its current state.
def get_next_state(current_state, action_idx):
    action = actions[action_idx] # Get the (row_change, col_change) for the chosen action
    next_s = (current_state[0] + action[0], current_state[1] + action[1])
    return next_s if is_valid_state(next_s) else current_state # Return valid next state or current state

# 5. Q-Learning Algorithm Training Loop:
print("\nStarting Q-Learning training...")
for episode in range(n_episodes):
    # Start each episode from a random valid state.
    # We ensure the starting state is not the goal state (2,2) itself for meaningful learning.
    start_r, start_c = random.randint(0, num_states_r - 1), random.randint(0, num_states_c - 1)
    current_state = (start_r, start_c)
    if current_state == (2,2): # If randomly started at goal, move to (0,0)
        current_state = (0,0)

    # Loop within an episode until the goal state (2,2) is reached
    # or a maximum number of steps is exceeded (to prevent infinite loops in case of poor policy).
    max_steps_per_episode = 100
    step = 0
    while current_state != (2, 2) and step < max_steps_per_episode:
        # Epsilon-greedy action selection:
        if random.uniform(0, 1) < epsilon:
            action_idx = random.randint(0, num_actions - 1) # Explore: choose a random action
        else:
            # Exploit: choose the action with the highest Q-value for the current state.
            # `np.a