function epsilon_greedy(state, estimates, exploration_rate):
   if random_number() < exploration_rate:

      action = choose_untried_action(state)
   else:
      action = choose_best_known_action(state, estimates)
   reward = take_action(action)
   update estimates[state][action] using reward

   return action
