Как правильно делать Q-обучение и/или что лучше использовать для генетического алгоритма игры в Змейку?

Question

[Mirozr]

Мой код я скину в комментарии, он не вместился (если понадобиться могу ещё доскинуть код без ООП).

Ошибка на данный момент:

Traceback (most recent call last):
  File "", line 748, in <module>
    game.game(model=snake.model, calculate_reward=snake.calculate_reward, custom_loss=snake.custom_loss, neural_network=snake.neural_network, bui
ld=snake.build_neural_network, snake=snake, optimizer=0.4)
  File "", line 618, in game
    self.starter(snake, optimizer)
  File "", line 472, in starter
    self.game(model=snake.model, calculate_reward=snake.calculate_reward,
  File "", line 503, in game
    snake.set_weights(genetic_result[1])
  File "", line 706, in set_weights
    return self.model.set_weights(weights)
  File "", line 1797, in set_weights
    raise ValueError(
ValueError: You called `set_weights(weights)` on layer "sequential" with a weight list of length 0, but the layer was expecting 4 weights. Provided weights: []...

Comments

[Mirozr]

class Genetic:
    def __init__(self):
        self.population_size = 20
        self.population = []
        self.v_crossover = 0.8
        self.v_mutation = 0.2
        self.generics = 300
        self.generic = 0 
        self.get_ready = False
        self.elite = 2
        self.cicle = 0

    def add(self, hromosom):
        self.population.append(hromosom)
        if self.cicle != self.population_size:
            self.cicle += 1
        else:
            self.generic += 1

    def calculate_elite(self, rewards, mobs):
        elit = []
        elit_rewards = []
        max_rate = 0
        pointer = 0
        for index, reward in enumerate(rewards):
            if pointer <= self.elite:
                if reward > max_rate:
                    max_rate = reward
                    elit_rewards.append([index, reward])
                    pointer += 1

        for i in elit_rewards:
            elit.append(mobs[i[0]])

        return elit

    def crossover(self, hromosoms_of_elite):
        childs = []
        for hromosom in range(0, len(hromosoms_of_elite), 2):
            if hromosom + 1 < len(hromosoms_of_elite):
                child = []
                for p in range(min(len(hromosoms_of_elite[hromosom]), len(hromosoms_of_elite[hromosom + 1]))):
                    gene = []
                    for w in range(min(len(hromosoms_of_elite[hromosom][p]), len(hromosoms_of_elite[hromosom + 1][p]))):
                        if self.v_crossover * 10 >= random.randint(0, 11):
                            gene.append(hromosoms_of_elite[hromosom + 1][p][w])
                        else:
                            gene.append(hromosoms_of_elite[hromosom][p][w])
                    child.append(gene)
                childs.append(child)

        return childs

    def mutation(self, hromosoms):
        childs = []
        for hromosom in hromosoms:
            for p in range(len(hromosom)):
                for w in range(len(hromosom[p])):
                    if self.v_mutation * 10 >= random.randint(0, 11):
                        if 0 == random.randint(0, 1):
                            hromosoms[hromosom][p][w] -= 2.75
                        else:
                            hromosoms[hromosom][p][w] += 2.75
            childs.append(hromosoms[hromosom])

        return childs

    def get_child(self, rewards, snakes):

        if self.generic == self.generics:
            self.get_ready = True
            return [False, self.get_ready]

        if self.cicle == self.population_size:
            return [True, self.mutation(self.crossover(self.calculate_elite(rewards, snakes)))]

class Network:
    def get_weights(self):
        return self.model.get_weights()
    def set_weights(self, weights):
        return self.model.set_weights(weights)
    def get_Total_Weights(self):
        return self.model.count_params()

    def calculate_reward(self, food_aet=False, score=0, hit_wall=False, hit_self=False, hit_self_dur=False):
        if food_aet:
            print('Молодець')
            return 1.0 * score
        elif hit_self_dur:
            print('Дebilchik!')
            return -5.0
        elif hit_wall or hit_self:
            print('Дурачок!')
            return -0.9
        else:
            return 0.0

    def custom_loss(self, y_true, y_pred, reward):
        loss = y_true * (reward - y_pred) + (1 - y_true) * (reward + y_pred)
        return loss

    def build_neural_network(self, optimizer, squere, num_actions=4):
        self.model.add(Flatten(input_shape=(squere**2, )))
        self.model.add(Dense(units=4, activation="linear"))
        self.model.add(Dense(units=num_actions, activation="linear"))
        self.model.compile(loss=self.custom_loss,
                           optimizer=keras.optimizers.Adam(learning_rate=optimizer))

    def neural_network(self, displaylist, model):
        direct = model.predict(np.expand_dims(displaylist, axis=0), verbose=0)

        return np.argmax(direct)


    def __init__(self):
        self.model = keras.Sequential()


genetic = Genetic()
game = Game(genetic)
snake = Network()

game.game(model=snake.model, calculate_reward=snake.calculate_reward, custom_loss=snake.custom_loss, neural_network=snake.neural_network, build=snake.build_neural_network, snake=snake, optimizer=0.4)

[Mirozr]

class Game:
    def is_true(self):
        self.food_pos = [random.randrange(1, (self.frame_size_x // 10)) * 10,
                         random.randrange(1, (self.frame_size_y // 10)) * 10]
        for body in self.snake_body:
            if self.food_pos[0] == body[0] and self.food_pos[1] == body[1]:
                self.is_true()
        return self.food_pos

    def add_reward(self, rew, food_aet=False, score=0, hit_wall=False, hit_self=False, hit_self_dur=False):
        return rew(food_aet=food_aet, score=score, hit_wall=hit_wall, hit_self=hit_self, hit_self_dur=hit_self_dur)

    def compile(self, optimizer, patience, screen, model, custom_loss, reward, epochs=1):
        model.compile(loss=lambda y_true, y_pred: custom_loss(y_true, y_pred, reward),
                      optimizer=keras.optimizers.Adam(optimizer))

        history = model.fit(np.expand_dims(screen, axis=0), np.zeros((1, 4)), epochs=epochs, verbose=0,
                            callbacks=[keras.callbacks.EarlyStopping(monitor='loss', patience=patience)],
                            sample_weight=np.array([reward]))

    def starter(self, snake, optimizer):
        self.game(model=snake.model, calculate_reward=snake.calculate_reward,
                  custom_loss=snake.custom_loss, neural_network=snake.neural_network,
                  build=snake.build_neural_network, snake=snake, optimizer=optimizer)

    def game(self, model, calculate_reward, custom_loss, neural_network, build, snake, optimizer, difficulty=500):
        direction = 'RIGHT'
        change_to = direction

        individ = []

        if not self.create:
            self.rewards.append(self.reward)
            for weights in snake.get_weights():
                individ.append([np.array(weights)])
            self.individuum.append(individ)
            self.genetic.add(individ)

        self.reward = 0

        self.snake_pos = [30, 50]
        self.snake_body = [[30, 50], [30 - 10, 50], [30 - (2 * 10), 50]]

        self.food_pos = [self.snake_pos[0] + 10, 50]
        self.food_spawn = True

        print(self.rewards)
        genetic_result = self.genetic.get_child(rewards=self.rewards, snakes=self.individuum)

        if genetic_result != None and genetic_result[0]:
            print(f'Поколение: {self.genetic.generic}')
            print(f'Лутший результат: {genetic_result[1]}')
            snake.set_weights(genetic_result[1])
        elif genetic_result != None and genetic_result[1]:
            print("Все поколения пройдены!")
            print(f'Лутший результат: {self.genetic.population[self.genetic.population.index(max(self.genetic.population))]}')

        while True:
            self.snake_on_eat = False

            screen = []

            for i in range(self.squere ** 2):
                if self.snake_body[0][0] - self.metrix[i][0] == self.food_pos[0] and self.snake_body[0][1] - \
                        self.metrix[i][1] == self.food_pos[1]:
                    screen.append(1)
                    print("I`m here")
                elif self.snake_body[0][0] - self.metrix[i][0] <= 0 or self.snake_body[0][1] - self.metrix[i][1] <= 0 or self.snake_body[0][0] - self.metrix[i][0] >= self.frame_size_x or self.snake_body[0][1] - self.metrix[i][1] >= self.frame_size_y or \
                        self.snake_body[0][0] - self.metrix[i][0] == self.snake_body[0][0] and self.snake_body[0][1] - \
                        self.metrix[i][1] == self.snake_body[0][1] or self.snake_body[0][0] - self.metrix[i][0] == \
                        self.snake_body[0][0] - 10 and self.snake_body[0][1] - self.metrix[i][1] == self.snake_body[0][
                    1] or self.snake_body[0][0] - self.metrix[i][0] == self.snake_body[0][0] + 10 and \
                        self.snake_body[0][1] - self.metrix[i][1] == self.snake_body[0][1] or self.snake_body[0][0] - \
                        self.metrix[i][0] == self.snake_body[0][0] and self.snake_body[0][1] - self.metrix[i][1] == \
                        self.snake_body[0][1] - 10 or self.snake_body[0][0] - self.metrix[i][0] == self.snake_body[0][
                    0] and self.snake_body[0][1] - self.metrix[i][1] == self.snake_body[0][1] + 10:
                    screen.append(-1)
                else:
                    screen.append(0)

            screen = np.array(screen)


            if self.create:
                build(optimizer, self.squere)
                self.create = False

            direct = neural_network(np.expand_dims(screen, axis=0)[0], model)

            print(snake.get_weights()[0], '\n', snake.get_weights()[1])

            for event in pygame.event.get():
                if event.type == pygame.QUIT:
                    pygame.quit()
                    sys.exit()
                elif event.type == pygame.KEYDOWN:
                    if event.key == ord('k'):
                        self.score = 0
                        self.reward += self.add_reward(rew=calculate_reward, hit_self_dur=True)
                        self.compile(optimizer=optimizer, patience=80, screen=screen, model=model, custom_loss=custom_loss, reward=self.reward)
                        self.starter(snake, optimizer)

                    elif event.key == ord("w"):
                        direct = 0
                    elif event.key == ord("s"):
                        direct = 1
                    elif event.key == ord("a"):
                        direct = 2
                    elif event.key == ord("d"):
                        direct = 3

            if direct == 0 and change_to != 'DOWN':
                change_to = 'UP'

            if direct == 1 and change_to != 'UP':
                change_to = 'DOWN'

            if direct == 2 and change_to != 'RIGHT':
                change_to = 'LEFT'

            if direct == 3 and change_to != 'LEFT':
                change_to = 'RIGHT'


            if change_to == 'UP':
                direction = 'UP'
            if change_to == 'DOWN':
                direction = 'DOWN'
            if change_to == 'LEFT':
                direction = 'LEFT'
            if change_to == 'RIGHT':
                direction = 'RIGHT'

            if direction == 'UP':
                self.snake_pos[1] -= 10
            if direction == 'DOWN':
                self.snake_pos[1] += 10
            if direction == 'LEFT':
                self.snake_pos[0] -= 10
            if direction == 'RIGHT':
                self.snake_pos[0] += 10

            self.snake_body.insert(0, list(self.snake_pos))
            if self.snake_pos[0] == self.food_pos[0] and self.snake_pos[1] == self.food_pos[1]:
                self.score += 1
                self.snake_on_eat = True
                self.reward = self.add_reward(rew=calculate_reward, food_aet=self.snake_on_eat, score=self.score)
                self.compile(optimizer=optimizer, patience=80, screen=screen, model=model, custom_loss=custom_loss, reward=self.reward)
                self.food_spawn = False
            else:
                self.snake_body.pop()

            if not self.food_spawn:
                self.is_true()
            self.food_spawn = True

            self.game_window.fill(self.black)
            for pos in self.snake_body:
                pygame.draw.rect(self.game_window, self.red, pygame.Rect(pos[0], pos[1], 10, 10))

            pygame.draw.rect(self.game_window, self.white, pygame.Rect(self.food_pos[0], self.food_pos[1], 10, 10))

            if self.snake_pos[0] < 0 or self.snake_pos[0] > self.frame_size_x - 10:
                print(f'Онполучил: {self.score}\nВін врізався в стінку!')
                self.score = 0
                self.reward += self.add_reward(rew=calculate_reward, hit_wall=True)
                self.compile(optimizer=optimizer, patience=80, screen=screen, model=model, custom_loss=custom_loss, reward=self.reward)
                self.starter(snake, optimizer)
            if self.snake_pos[1] < 0 or self.snake_pos[1] > self.frame_size_y - 10:
                print(f'Он получил: {self.score}\nВін врізався в стінку!!')
                self.score = 0
                self.reward += self.add_reward(rew=calculate_reward, hit_wall=True)
                self.compile(optimizer=optimizer, patience=80, screen=screen, model=model, custom_loss=custom_loss, reward=self.reward)
                self.starter(snake, optimizer)

            for block in self.snake_body[1:]:
                if self.snake_pos[0] == block[0] and self.snake_pos[1] == block[1]:
                    print(f'Він отримав: {self.score}\nВін сам себе вбив!')
                    self.score = 0
                    self.reward += self.add_reward(rew=calculate_reward, hit_self_dur=True)
                    self.compile(optimizer=optimizer, patience=80, screen=screen, model=model, custom_loss=custom_loss, reward=self.reward)
                    self.starter(snake, optimizer)

            pygame.display.update()

            self.fps_controller.tick(difficulty)

[Mirozr]

def __init__(self, genetic, squere=3):
        self.squere = squere

        self.shape = 0
        self.frame_size_x = 210
        self.frame_size_y = 210

        self.check_errors = pygame.init()

        if self.check_errors[1] > 0:
            print(f'[!] Had {self.check_errors[1]} errors when initialising game, exiting...')
            sys.exit(-1)
        else:
            print('[+] Game successfully initialised')

        pygame.display.set_caption('Snake Eater')
        self.game_window = pygame.display.set_mode((self.frame_size_x, self.frame_size_y))

        self.black = pygame.Color(0, 0, 0)
        self.white = pygame.Color(254, 254, 254)
        self.red = pygame.Color(255, 0, 0)
        self.green = pygame.Color(0, 255, 0)
        self.blue = pygame.Color(0, 0, 255)

        self.fps_controller = pygame.time.Clock()

        self.snake_pos = [100, 50]
        self.snake_body = [[100, 50], [100 - 10, 50], [100 - (2 * 10), 50]]
        self.snake_on_eat = False

        self.food_pos = [self.snake_pos[0] + 10, 50]  # [random.randrange(1, (self.frame_size_x // 10)) * 10, random.randrange(1, (self.frame_size_y // 10)) * 10]
        self.food_spawn = True

        self.score = 0
        self.create = True

        self.reward = 0

        self.genetic = genetic

        self.rewards = []
        self.individuum = []

        self.screen1 = np.zeros((21, 21, 3), dtype=int)

        self.dict_len = {}

        for i in range(22):
            if i % 2 != 0 and i != 0 and i != 1:
                self.dict_len[i] = -10 * ((i - 1) // 2)

        self.metrix = []

        y = 10

        for i in range(1, squere+1):
            x = self.dict_len[squere]
            for j in range(1, squere+1):
                self.metrix.append([x, y])
                x += 10
            y -= 10