sorarebuddy/sorare_utils/update_service.go

package sorare_utils

import (
	"context"

	"git.lehouerou.net/laurent/sorare"
	"git.lehouerou.net/laurent/sorare/football"
	"git.lehouerou.net/laurent/sorare/graphql"
	"github.com/jackc/pgx/v5/pgtype"
	gql "github.com/llehouerou/go-graphql-client"
	"github.com/pkg/errors"
	"github.com/rs/zerolog/log"
	"github.com/samber/lo"

	"git.lehouerou.net/laurent/sorarebuddy/model"
)

type UpdateService struct {
	s  *sorare.Sorare
	db *model.Queries
}

func NewUpdateService(s *sorare.Sorare, db *model.Queries) *UpdateService {
	return &UpdateService{s: s, db: db}
}

func (u *UpdateService) InitSyncDatabase(ctx context.Context) error {

	sfixtures, err := u.s.Football.So5.So5Fixtures.Get(ctx, football.So5FixturesParams{
		// AasmStates: []string{"started"},
	})
	if err != nil {
		return err
	}
	log.Debug().Msgf("fixtures: %v", sfixtures)

	batchFixtures := u.db.CreateOrUpdateFixtures(
		ctx,
		lo.Map(sfixtures, func(fixture football.So5Fixture, index int) model.CreateOrUpdateFixturesParams {
			return model.CreateOrUpdateFixturesParams{
				Slug:        fixture.Slug,
				DisplayName: fixture.DisplayName,
				State:       fixture.AasmState,
				StartDate:   pgtype.Timestamptz{Time: fixture.StartDate, Valid: true},
				EndDate:     pgtype.Timestamptz{Time: fixture.EndDate, Valid: true},
				GameWeek:    int32(fixture.GameWeek),
			}
		}),
	)
	var batcherr error
	batchFixtures.Exec(func(_ int, err error) {
		if err != nil {
			batcherr = err
			batchFixtures.Close()

		}
	})
	if batcherr != nil {
		return err
	}

	log.Debug().Msgf("created %d fixtures", len(sfixtures))

	fixtures, err := u.db.GetAllFixtures(ctx)
	if err != nil {
		return err
	}
	games, err := GetGamesFromFixtures(
		ctx,
		u.s,
		lo.Map(fixtures, func(fixture model.Fixture, index int) string {
			return fixture.Slug
		}),
	)
	log.Info().Msgf("found %d games to process", len(games))

	log.Debug().Msgf("getting players for each game...")
	var gamePlayers []model.CreateOrUpdateGamePlayersParams
	playerSlugsByGameMap := make(map[string][]string)
	for _, game := range games {
		gameWithFormation, err := u.s.Football.Game.Get(ctx, graphql.IdParams{Id: gql.ID(game.Id.Value)})
		if err != nil {
			return errors.Wrapf(err, "getting game with formation %s", game.Id.Value)
		}
		newplayers := ExtractPlayersFromGameWithFormation(gameWithFormation)
		log.Debug().Msgf("\t%s -> %d players", game.String(), len(newplayers))
		playerSlugsByGameMap[game.Id.Value] = lo.Map(
			newplayers,
			func(player model.CreateOrUpdateGamePlayersParams, index int) string {
				return player.PlayerSlug
			},
		)
		gamePlayers = append(gamePlayers, newplayers...)
	}

	playerSlugs := lo.Uniq(
		lo.Filter(lo.Map(gamePlayers, func(player model.CreateOrUpdateGamePlayersParams, index int) string {
			return player.PlayerSlug
		}), func(slug string, index int) bool {
			return slug != ""
		}),
	)

	log.Debug().Msgf("getting players...")
	var players []football.Player
	for i, chunk := range lo.Chunk(playerSlugs, 80) {
		log.Debug().Msgf("\tbatch %d/%d", i+1, (len(playerSlugs)/80)+1)
		p, err := u.s.Football.Players.Get(ctx, graphql.SlugsParams{Slugs: chunk})
		if err != nil {
			return errors.Wrapf(err, "getting players batch %d", i)
		}
		players = append(players, p...)
	}
	log.Debug().Msgf("found %d players", len(players))

	teamSlugs := ExtractTeamSlugsFromPlayersAndGames(players, games)
	log.Debug().Msgf("extracted %d unique team slugs from games and players", len(teamSlugs))

	log.Debug().Msgf("getting clubs...")
	var clubs []football.Club
	for i, chunk := range lo.Chunk(teamSlugs, 100) {
		log.Debug().Msgf("\tbatch %d/%d", i+1, (len(teamSlugs)/100)+1)
		t, err := u.s.Football.Clubs.Get(ctx, graphql.SlugsParams{Slugs: chunk})
		if err != nil {
			return err
		}
		clubs = append(clubs, t...)
	}
	log.Debug().Msgf("found %d clubs", len(clubs))
	var nationalTeams []football.NationalTeam
	slugsLeft := lo.Without(teamSlugs, lo.Map(clubs, func(club football.Club, index int) string {
		return club.Slug
	})...)
	log.Debug().Msgf("getting national teams...")
	log.Debug().Msgf("slugs left: %d", len(slugsLeft))
	for i, chunk := range lo.Chunk(slugsLeft, 100) {
		log.Debug().Msgf("\tbatch %d/%d", i+1, (len(teamSlugs)/100)+1)
		t, err := u.s.Football.NationalTeams.Get(ctx, graphql.SlugsParams{Slugs: chunk})
		if err != nil {
			return err
		}
		nationalTeams = append(nationalTeams, t...)
	}
	log.Debug().Msgf("found %d national teams", len(nationalTeams))

	competitionSlugs := ExtractCompetitionSlugsFromPlayersGamesAndClubs(players, games, clubs)
	log.Debug().Msgf("extracted %d unique competition slugs from players, games and clubs", len(competitionSlugs))
	log.Debug().Msgf("getting competitions...")
	var competitions []football.Competition
	for _, slug := range competitionSlugs {
		log.Debug().Msgf("\tcompetition %s", slug)
		c, err := u.s.Football.Competition.Get(ctx, graphql.SlugParams{Slug: slug})
		if err != nil {
			return err
		}
		competitions = append(competitions, c)
	}
	log.Debug().Msgf("found %d competitions", len(competitions))

	countrySlugs := ExtractCountrySlugsFromPlayersCompetitionsClubsAndNationalTeams(
		players,
		competitions,
		clubs,
		nationalTeams,
	)
	log.Debug().
		Msgf("extracted %d unique country slugs from players, competitions, clubs and national teams", len(countrySlugs))
	log.Debug().Msgf("getting countries...")
	var countries []sorare.Country
	for i, chunk := range lo.Chunk(countrySlugs, 100) {
		log.Debug().Msgf("\tbatch %d/%d", i+1, (len(countrySlugs)/100)+1)
		c, err := u.s.Countries.Get(ctx, graphql.SlugsParams{Slugs: chunk})
		if err != nil {
			return err
		}
		countries = append(countries, c...)
	}
	log.Debug().Msgf("found %d countries", len(countries))

	scores := make(map[string][]football.PlayerScore)
	for gameId, playerSlugs := range playerSlugsByGameMap {
		log.Debug().Msgf("getting scores for game %s...", gameId)
		var gameScores []football.PlayerScore
		for i, chunk := range lo.Chunk(playerSlugs, 80) {
			log.Debug().Msgf("\tbatch %d/%d", i+1, (len(playerSlugs)/80)+1)
			s, err := u.s.Football.PlayersGameScores(gql.ID(gameId)).Get(ctx, graphql.SlugsParams{Slugs: chunk})
			if err != nil {
				return errors.Wrapf(err, "getting scores for game %s", gameId)
			}
			gameScores = append(gameScores, s...)
		}
		for _, score := range gameScores {
			log.Debug().Msgf("\t%s -> %s", score.Slug, score.So5Score.Score)
		}
		scores[gameId] = gameScores
	}

	log.Debug().Msg("inserting countries into db...")
	batchCountries := u.db.CreateOrUpdateCountries(
		ctx,
		lo.Map(countries, func(country sorare.Country, index int) model.CreateOrUpdateCountriesParams {
			return model.CreateOrUpdateCountriesParams{
				Slug:            country.Slug,
				Code:            country.Code,
				DisplayName:     country.Name,
				ThreeLetterCode: country.ThreeLetterCode,
				FlagFlat64Url:   country.FlagFlat64Url,
				FlagFlat32Url:   country.FlagFlat32Url,
				FlagRound64Url:  country.FlagRound64Url,
				FlagRound32Url:  country.FlagRound32Url,
			}
		}),
	)
	batcherr = nil
	batchCountries.Exec(func(_ int, err error) {
		if err != nil {
			batcherr = err
			batchCountries.Close()
		}
	})
	if batcherr != nil {
		return err
	}
	log.Debug().Msgf("%d countries inserted", len(countries))

	log.Debug().Msg("inserting competitions into db...")

	batchCompetitions := u.db.CreateOrUpdateCompetitions(
		ctx,
		lo.Map(competitions, func(competition football.Competition, index int) model.CreateOrUpdateCompetitionsParams {
			return model.CreateOrUpdateCompetitionsParams{
				Slug:              competition.Slug,
				CompetitionFormat: competition.Format,
				CompetitionType:   competition.Type,
				DisplayName:       competition.DisplayName,
				PictureUrl:        competition.PictureUrl,
				LogoUrl:           competition.LogoUrl,
				CountrySlug:       competition.Country.Slug,
			}
		}),
	)
	batcherr = nil
	batchCompetitions.Exec(func(_ int, err error) {
		if err != nil {
			batcherr = err
			batchCompetitions.Close()
		}
	})
	if batcherr != nil {
		return errors.Wrap(batcherr, "inserting competitions")
	}

	log.Debug().Msgf("%d competitions inserted", len(competitions))

	log.Debug().Msg("inserting teams into db...")
	batchTeams := u.db.CreateOrUpdateTeams(ctx, lo.Union(
		lo.Map(clubs, func(club football.Club, index int) model.CreateOrUpdateTeamsParams {
			return model.CreateOrUpdateTeamsParams{
				Slug:        club.Slug,
				DisplayName: club.Name,
				CountrySlug: club.Country.Slug,
				DomesticLeagueSlug: func() *string {
					if club.DomesticLeague.Slug == "" {
						return nil
					}
					return &club.DomesticLeague.Slug
				}(),
				ShortName:  club.ShortName,
				PictureUrl: club.PictureUrl,
				TeamType:   "club",
			}
		}),
		lo.Map(nationalTeams, func(nationalTeam football.NationalTeam, index int) model.CreateOrUpdateTeamsParams {
			return model.CreateOrUpdateTeamsParams{
				Slug:               nationalTeam.Slug,
				DisplayName:        nationalTeam.Name,
				CountrySlug:        nationalTeam.Country.Slug,
				DomesticLeagueSlug: nil,
				ShortName:          nationalTeam.ShortName,
				PictureUrl:         nationalTeam.PictureUrl,
				TeamType:           "national",
			}
		}),
	))
	batcherr = nil
	batchTeams.Exec(func(_ int, err error) {
		if err != nil {
			batcherr = err
			batchTeams.Close()
		}
	})
	if batcherr != nil {
		return errors.Wrap(batcherr, "inserting teams into db")
	}
	log.Debug().Msgf("%d teams inserted", len(clubs)+len(nationalTeams))

	log.Debug().Msg("inserting games into db...")
	batchGames := u.db.CreateOrUpdateGames(
		ctx,
		lo.Map(games, func(game football.Game, index int) model.CreateOrUpdateGamesParams {
			return NewCreateOrUpdateGamesParamsFromSorare(game)
		}),
	)
	batcherr = nil
	batchGames.Exec(func(_ int, err error) {
		if err != nil {
			batcherr = err
			batchGames.Close()
		}
	})
	if batcherr != nil {
		return err
	}
	log.Debug().Msgf("%d games inserted", len(games))

	log.Debug().Msg("inserting players into db...")
	batchPlayers := u.db.CreateOrUpdatePlayers(
		ctx,
		lo.Map(players, func(player football.Player, index int) model.CreateOrUpdatePlayersParams {
			res := model.CreateOrUpdatePlayersParams{
				Slug:          player.Slug,
				DisplayName:   player.DisplayName,
				BirthDate:     player.BirthDate,
				CountrySlug:   player.Country.Slug,
				AvatarUrl:     player.AvatarUrl,
				FieldPosition: string(player.Position),
				Status:        string(player.PlayingStatus),
				ShirtNumber:   int32(player.ShirtNumber),
			}
			for _, competition := range player.ActiveClub.ActiveCompetitions {
				if competition.Format == "DOMESTIC_LEAGUE" {
					res.DomesticLeagueSlug = &competition.Slug
				}
			}
			if player.ActiveClub.Slug != "" {
				res.TeamSlug = &player.ActiveClub.Slug
			}
			return res
		}),
	)
	batcherr = nil
	batchPlayers.Exec(func(_ int, err error) {
		if err != nil {
			batcherr = err
			batchPlayers.Close()
		}
	})
	if batcherr != nil {
		return errors.Wrap(batcherr, "inserting players")
	}
	log.Debug().Msgf("%d players inserted", len(players))

	log.Debug().Msg("inserting game players into db...")
	batchGamePlayers := u.db.CreateOrUpdateGamePlayers(
		ctx,
		gamePlayers,
	)
	batcherr = nil
	batchGamePlayers.Exec(func(_ int, err error) {
		if err != nil {
			batcherr = err
			batchGamePlayers.Close()
		}
	})
	if batcherr != nil {
		return errors.Wrap(err, "inserting game players")
	}
	log.Debug().Msgf("%d game players inserted", len(gamePlayers))

	log.Debug().Msg("inserting game player scores into db...")
	batchGamePlayerScores := u.db.CreateOrUpdateGamePlayerScores(
		ctx,
		lo.Union(
			lo.MapToSlice(
				scores,
				func(gameId string, scores []football.PlayerScore) []model.CreateOrUpdateGamePlayerScoresParams {
					return lo.Map(scores, func(score football.PlayerScore, index int) model.CreateOrUpdateGamePlayerScoresParams {
						return NewCreateOrUpdateGamePlayerScoresParamsFromSorare(gameId, score)
					})

				},
			)...),
	)
	batcherr = nil
	batchGamePlayerScores.Exec(func(i int, err error) {
		if err != nil {
			batcherr = err
			batchGamePlayerScores.Close()
		}
	})
	if batcherr != nil {
		return errors.Wrap(batcherr, "inserting game player scores")
	}
	log.Debug().Msgf("game player scores inserted")

	return nil
}

func ExtractTeamSlugsFromPlayersAndGames(players []football.Player, games []football.Game) []string {
	return lo.Uniq(lo.Union(
		ExtractTeamSlugsFromPlayers(players),
		ExtractTeamSlugsFromGames(games),
	))
}

func ExtractTeamSlugsFromPlayers(players []football.Player) []string {
	return lo.Uniq(lo.Filter(lo.Map(players, func(player football.Player, index int) string {
		return player.ActiveClub.Slug
	}), func(slug string, index int) bool {
		return slug != ""
	}))
}

func ExtractTeamSlugsFromGames(games []football.Game) []string {
	var res []string
	res = lo.Map(games, func(game football.Game, index int) string {
		return game.AwayTeam.Team.Slug
	})
	res = append(res, lo.Map(games, func(game football.Game, index int) string {
		return game.HomeTeam.Team.Slug
	})...)
	res = lo.Filter(res, func(slug string, index int) bool {
		return slug != ""
	})
	return lo.Uniq(res)
}

func ExtractCountrySlugsFromCompetitions(competitions []football.Competition) []string {
	return lo.Uniq(lo.Filter(lo.Map(competitions, func(competition football.Competition, index int) string {
		return competition.Country.Slug
	}), func(slug string, index int) bool {
		return slug != ""
	}))
}

func ExtractCountrySlugsFromTeams(teams []football.Team) []string {
	return lo.Uniq(lo.Filter(lo.Map(teams, func(team football.Team, index int) string {
		return team.Country.Slug
	}), func(slug string, index int) bool {
		return slug != ""
	}))
}

func ExtractCountrySlugsFromPlayersCompetitionsClubsAndNationalTeams(
	players []football.Player,
	competitions []football.Competition,
	clubs []football.Club,
	nationalTeams []football.NationalTeam,
) []string {
	return lo.Uniq(lo.Union(
		ExtractCountrySlugsFromPlayers(players),
		ExtractCountrySlugsFromCompetitions(competitions),
		ExtractCountrySlugsFromTeams(lo.Map(clubs, func(club football.Club, index int) football.Team {
			return club.Team
		})),
		ExtractCountrySlugsFromTeams(
			lo.Map(nationalTeams, func(nationalTeam football.NationalTeam, index int) football.Team {
				return nationalTeam.Team
			}),
		),
	))
}

func ExtractCountrySlugsFromPlayers(players []football.Player) []string {
	return lo.Uniq(lo.Filter(lo.Map(players, func(player football.Player, index int) string {
		return player.Country.Slug
	}), func(slug string, index int) bool {
		return slug != ""
	}))

}

func ExtractCompetitionSlugsFromPlayers(players []football.Player) []string {
	return lo.Uniq(lo.Filter(lo.Map(players, func(player football.Player, index int) string {
		var res string
		for _, competition := range player.ActiveClub.ActiveCompetitions {
			if competition.Format == "DOMESTIC_LEAGUE" {
				res = competition.Slug
			}
		}
		return res
	}), func(slug string, index int) bool {
		return slug != ""
	}))
}

func ExtractCompetitionSlugsFromGames(games []football.Game) []string {
	return lo.Uniq(lo.Filter(lo.Map(games, func(game football.Game, index int) string {
		return game.Competition.Slug
	}), func(slug string, index int) bool {
		return slug != ""
	}))
}

func ExtractCompetitionSlugsFromClubs(clubs []football.Club) []string {
	return lo.Uniq(lo.Filter(lo.Map(clubs, func(club football.Club, index int) string {
		return club.DomesticLeague.Slug
	}), func(slug string, index int) bool {
		return slug != ""
	}))
}

func ExtractCompetitionSlugsFromPlayersGamesAndClubs(
	players []football.Player,
	games []football.Game,
	clubs []football.Club,
) []string {
	return lo.Uniq(lo.Union(
		ExtractCompetitionSlugsFromPlayers(players),
		ExtractCompetitionSlugsFromGames(games),
		ExtractCompetitionSlugsFromClubs(clubs),
	))
}