package sorare_utils import ( "context" "time" "git.lehouerou.net/laurent/sorare" "git.lehouerou.net/laurent/sorare/football" "git.lehouerou.net/laurent/sorare/graphql" gql "github.com/llehouerou/go-graphql-client" "github.com/pkg/errors" "github.com/rs/zerolog/log" "github.com/samber/lo" "git.lehouerou.net/laurent/sorarebuddy/db" "git.lehouerou.net/laurent/sorarebuddy/model" ) type Updater struct { s *sorare.Sorare db *db.Client countryUpdater *CountryUpdater competitionSlugsToRead []string teamSlugsToRead []string playerSlugsToRead []string gamesFromFixtureSlugToRead []string gamesToRead []string gameFormationsToRead []string gameScoresToReadFromMap map[string][]string competitionCache []football.Competition clubCache []football.Club nationalTeamCache []football.NationalTeam playerCache []football.Player gameCache []football.Game gamePlayersCache []model.GamePlayer gameScoreCache []model.GamePlayerScore } type updaterOptions struct { UpdateOnlyMissingCountries bool UpdateOnlyMissingCompetitions bool UpdateOnlyMissingTeams bool UpdateOnlyMissingPlayers bool } type UpdaterOption func(*updaterOptions) *updaterOptions func WithUpdateOnlyMissingCountries(value bool) UpdaterOption { return func(o *updaterOptions) *updaterOptions { o.UpdateOnlyMissingCountries = value return o } } func WithUpdateOnlyMissingCompetitions(value bool) UpdaterOption { return func(o *updaterOptions) *updaterOptions { o.UpdateOnlyMissingCompetitions = value return o } } func WithUpdateOnlyMissingTeams(value bool) UpdaterOption { return func(o *updaterOptions) *updaterOptions { o.UpdateOnlyMissingTeams = value return o } } func WithUpdateOnlyMissingPlayers(value bool) UpdaterOption { return func(o *updaterOptions) *updaterOptions { o.UpdateOnlyMissingPlayers = value return o } } func NewUpdater(s *sorare.Sorare, db *db.Client, opts ...UpdaterOption) *Updater { return &Updater{ s: s, db: db, countryUpdater: NewCountryUpdater(s, db), gameScoresToReadFromMap: make(map[string][]string), } } func (u *Updater) Reset() { u.competitionSlugsToRead = nil u.teamSlugsToRead = nil u.playerSlugsToRead = nil u.gamesFromFixtureSlugToRead = nil u.gamesToRead = nil u.gameFormationsToRead = nil for k := range u.gameScoresToReadFromMap { delete(u.gameScoresToReadFromMap, k) } u.competitionCache = nil u.clubCache = nil u.nationalTeamCache = nil u.playerCache = nil u.gameCache = nil u.gamePlayersCache = nil u.gameScoreCache = nil } func (u *Updater) Update(ctx context.Context, opts ...UpdaterOption) error { options := updaterOptions{ UpdateOnlyMissingCountries: true, UpdateOnlyMissingCompetitions: true, UpdateOnlyMissingTeams: false, UpdateOnlyMissingPlayers: false, } for _, opt := range opts { options = *opt(&options) } if err := u.readGamesFromFixture(ctx); err != nil { return errors.Wrap(err, "reading games from fixture") } if err := u.readGames(ctx); err != nil { return errors.Wrap(err, "reading games") } if err := u.readGamePlayers(ctx); err != nil { return errors.Wrap(err, "reading game players") } if err := u.readGameScoresFromMap(ctx); err != nil { return errors.Wrap(err, "reading game scores") } if err := u.readPlayers(ctx, options.UpdateOnlyMissingPlayers); err != nil { return errors.Wrap(err, "reading players") } if err := u.readTeams(ctx, options.UpdateOnlyMissingTeams); err != nil { return errors.Wrap(err, "reading teams") } if err := u.readCompetitions(ctx, options.UpdateOnlyMissingCompetitions); err != nil { return errors.Wrap(err, "reading competitions") } if err := u.countryUpdater.Read(ctx, options.UpdateOnlyMissingCountries); err != nil { return errors.Wrap(err, "reading countries") } if err := u.countryUpdater.Write(ctx); err != nil { return errors.Wrap(err, "writing countries") } if err := u.writeCompetitions(ctx); err != nil { return errors.Wrap(err, "writing competitions") } if err := u.writeTeams(ctx); err != nil { return errors.Wrap(err, "writing teams") } if err := u.writePlayers(ctx); err != nil { return errors.Wrap(err, "writing players") } if err := u.writeGames(ctx); err != nil { return errors.Wrap(err, "writing games") } if err := u.writeGamePlayers(ctx); err != nil { return errors.Wrap(err, "writing game players") } if err := u.writeGameScores(ctx); err != nil { return errors.Wrap(err, "writing game scores") } return nil } func (u *Updater) AddCountriesToRead(slugs ...string) { u.countryUpdater.AddSlugsToRead(slugs) } func (u *Updater) AddCompetitionsToRead(slugs ...string) { slugs = lo.Filter(slugs, func(slug string, index int) bool { return slug != "" }) u.competitionSlugsToRead = lo.Uniq(append(u.competitionSlugsToRead, slugs...)) } func (u *Updater) readCompetitions(ctx context.Context, onlyMissings bool) error { if len(u.competitionSlugsToRead) == 0 { log.Debug().Msg("no competitions to read") return nil } log.Debug().Msgf("reading %d competitions...", len(u.competitionSlugsToRead)) slugs := u.competitionSlugsToRead u.competitionSlugsToRead = nil if onlyMissings { log.Debug().Msgf("filtering competitions not in db...") slugsNotInDb, err := u.db.Competitions.GetCompetitionSlugsNotInDb(ctx, slugs) if err != nil { return errors.Wrap(err, "getting competitions not in db") } slugs = slugsNotInDb log.Debug().Msgf("%d competitions not in db", len(slugs)) } log.Debug().Msgf("getting competitions...") u.competitionCache = nil for _, slug := range slugs { log.Debug().Msgf("\tcompetition %s", slug) c, err := u.s.Football.Competition.Get(ctx, graphql.SlugParams{Slug: slug}) if err != nil { return err } u.competitionCache = append(u.competitionCache, c) } log.Debug().Msgf("found %d competitions", len(u.competitionCache)) u.AddCountriesToRead(lo.Map(u.competitionCache, func(c football.Competition, index int) string { return c.Country.Slug })...) return nil } func (u *Updater) writeCompetitions(ctx context.Context) error { log.Debug().Msg("inserting competitions into db...") err := u.db.Competitions.CreateOrUpdateMany( ctx, lo.Map(u.competitionCache, func(competition football.Competition, index int) model.Competition { return model.Competition{ Slug: competition.Slug, CompetitionFormat: competition.Format, CompetitionType: competition.Type, DisplayName: competition.DisplayName, PictureUrl: competition.PictureUrl, LogoUrl: competition.LogoUrl, CountrySlug: competition.Country.Slug, } }), ) if err != nil { return errors.Wrap(err, "inserting competitions") } log.Debug().Msgf("%d competitions inserted", len(u.competitionCache)) u.competitionCache = nil return nil } func (u *Updater) AddTeamsToRead(slugs ...string) { slugs = lo.Filter(slugs, func(slug string, index int) bool { return slug != "" }) u.teamSlugsToRead = lo.Uniq(append(u.teamSlugsToRead, slugs...)) } func (u *Updater) readTeams(ctx context.Context, onlyMissings bool) error { if len(u.teamSlugsToRead) == 0 { log.Debug().Msg("no teams to read") return nil } log.Debug().Msgf("reading %d teams...", len(u.teamSlugsToRead)) slugs := u.teamSlugsToRead u.teamSlugsToRead = nil log.Debug().Msgf("getting clubs...") if onlyMissings { log.Debug().Msgf("filtering clubs not in db...") slugsNotInDb, err := u.db.Teams.GetTeamSlugsNotInDb(ctx, slugs) if err != nil { return errors.Wrap(err, "getting teams not in db") } slugs = slugsNotInDb log.Debug().Msgf("%d clubs not in db", len(slugs)) } u.clubCache = nil for i, chunk := range lo.Chunk(slugs, 100) { log.Debug().Msgf("\tbatch %d/%d", i+1, (len(slugs)/100)+1) t, err := u.s.Football.Clubs.Get(ctx, graphql.SlugsParams{Slugs: chunk}) if err != nil { return err } u.clubCache = append(u.clubCache, t...) } log.Debug().Msgf("found %d clubs", len(u.clubCache)) slugsLeft := lo.Without(slugs, lo.Map(u.clubCache, func(club football.Club, index int) string { return club.Slug })...) u.nationalTeamCache = nil log.Debug().Msgf("getting national teams...") log.Debug().Msgf("slugs left: %d", len(slugsLeft)) for i, chunk := range lo.Chunk(slugsLeft, 100) { log.Debug().Msgf("\tbatch %d/%d", i+1, (len(slugsLeft)/100)+1) t, err := u.s.Football.NationalTeams.Get(ctx, graphql.SlugsParams{Slugs: chunk}) if err != nil { return err } u.nationalTeamCache = append(u.nationalTeamCache, t...) } log.Debug().Msgf("found %d national teams", len(u.nationalTeamCache)) u.AddCompetitionsToRead(lo.Map(u.clubCache, func(club football.Club, index int) string { return club.DomesticLeague.Slug })...) u.AddCountriesToRead(lo.Map(u.clubCache, func(club football.Club, index int) string { return club.Country.Slug })...) u.AddCountriesToRead(lo.Map(u.nationalTeamCache, func(nationalTeam football.NationalTeam, index int) string { return nationalTeam.Country.Slug })...) return nil } func (u *Updater) writeTeams(ctx context.Context) error { log.Debug().Msg("inserting teams into db...") err := u.db.Teams.CreateOrUpdateMany(ctx, append( lo.Map(u.clubCache, func(club football.Club, index int) model.Team { return model.Team{ Slug: club.Slug, DisplayName: club.Name, CountrySlug: club.Country.Slug, DomesticLeagueSlug: func() *string { if club.DomesticLeague.Slug == "" { return nil } return &club.DomesticLeague.Slug }(), ShortName: club.ShortName, PictureUrl: club.PictureUrl, TeamType: "club", } }), lo.Map(u.nationalTeamCache, func(nationalTeam football.NationalTeam, index int) model.Team { return model.Team{ Slug: nationalTeam.Slug, DisplayName: nationalTeam.Name, CountrySlug: nationalTeam.Country.Slug, DomesticLeagueSlug: nil, ShortName: nationalTeam.ShortName, PictureUrl: nationalTeam.PictureUrl, TeamType: "national", } })..., )) if err != nil { return errors.Wrap(err, "inserting teams") } log.Debug().Msgf("%d teams inserted", len(u.clubCache)+len(u.nationalTeamCache)) u.clubCache = nil u.nationalTeamCache = nil return nil } func (u *Updater) AddPlayersToRead(slugs ...string) { slugs = lo.Filter(slugs, func(slug string, index int) bool { return slug != "" }) u.playerSlugsToRead = lo.Uniq(append(u.playerSlugsToRead, slugs...)) } func (u *Updater) readPlayers(ctx context.Context, onlyMissings bool) error { if len(u.playerSlugsToRead) == 0 { log.Debug().Msg("no players to read") return nil } slugs := u.playerSlugsToRead u.playerSlugsToRead = nil log.Debug().Msgf("updating %d players", len(slugs)) if onlyMissings { log.Debug().Msgf("filtering players not in db...") slugsNotInDb, err := u.db.Players.GetPlayerSlugsNotInDb(ctx, slugs) if err != nil { return errors.Wrap(err, "getting players not in db") } slugs = slugsNotInDb log.Debug().Msgf("%d players not in db", len(slugs)) } log.Debug().Msgf("getting players from sorare...") u.playerCache = nil for i, chunk := range lo.Chunk(slugs, 65) { log.Debug().Msgf("\tbatch %d/%d", i+1, (len(slugs)/65)+1) p, err := u.s.Football.Players.Get(ctx, graphql.SlugsParams{Slugs: chunk}) if err != nil { return errors.Wrapf(err, "getting players batch %d", i) } u.playerCache = append(u.playerCache, p...) } log.Debug().Msgf("found %d players", len(u.playerCache)) u.AddTeamsToRead(lo.FlatMap(u.playerCache, func(player football.Player, index int) []string { var res []string res = append(res, player.ActiveClub.Slug) res = append(res, player.ActiveNationalTeam.Slug) res = append(res, lo.Map(player.Memberships, func(membership football.Membership, index int) string { if membership.MembershipTeam.Club.Slug != "" { return membership.MembershipTeam.Club.Slug } else if membership.MembershipTeam.NationalTeam.Slug != "" { return membership.MembershipTeam.NationalTeam.Slug } return "" })...) return res })...) u.AddCompetitionsToRead(lo.Map(u.playerCache, func(player football.Player, index int) string { var res string for _, competition := range player.ActiveClub.ActiveCompetitions { if competition.Format == "DOMESTIC_LEAGUE" { res = competition.Slug } } return res })...) u.AddCountriesToRead(lo.Map(u.playerCache, func(player football.Player, index int) string { return player.Country.Slug })...) return nil } func (u *Updater) writePlayers(ctx context.Context) error { log.Debug().Msg("inserting players into db...") err := u.db.Players.CreateOrUpdateMany( ctx, lo.Map(u.playerCache, func(player football.Player, index int) model.Player { res := model.Player{ Slug: player.Slug, DisplayName: player.DisplayName, BirthDate: player.BirthDate, CountrySlug: player.Country.Slug, AvatarUrl: player.AvatarUrl, FieldPosition: string(player.Position), Status: string(player.PlayingStatus), ShirtNumber: int(player.ShirtNumber), ActiveNationalTeamSlug: func() *string { if player.ActiveNationalTeam.Slug == "" { return nil } return &player.ActiveNationalTeam.Slug }(), } for _, competition := range player.ActiveClub.ActiveCompetitions { if competition.Format == "DOMESTIC_LEAGUE" { res.DomesticLeagueSlug = &competition.Slug } } if player.ActiveClub.Slug != "" { res.TeamSlug = &player.ActiveClub.Slug } return res }), ) if err != nil { return errors.Wrap(err, "inserting players") } log.Debug().Msgf("%d players inserted", len(u.playerCache)) log.Debug().Msgf("inserting players card supply into db...") err = u.db.CardSupplies.CreateOrUpdateMany( ctx, lo.FlatMap(u.playerCache, func(player football.Player, index int) []model.CardSupply { var res []model.CardSupply for _, supply := range player.CardSupply { res = append(res, model.CardSupply{ PlayerSlug: player.Slug, SeasonStartYear: supply.Season.StartYear, Limited: supply.Limited, Rare: supply.Rare, SuperRare: supply.SuperRare, Unique: supply.Unique, LastUpdated: time.Now(), }) } return res }), ) if err != nil { return errors.Wrap(err, "inserting players card supply") } log.Debug().Msgf("%d players card supply inserted", len(u.playerCache)) log.Debug().Msgf("inserting players club_memberships into db...") err = u.db.Memberships.CreateOrUpdateMany( ctx, lo.FlatMap(u.playerCache, func(player football.Player, index int) []model.Membership { var res []model.Membership for _, membership := range player.Memberships { new := model.Membership{ Id: membership.Id.Value, PlayerSlug: player.Slug, StartDate: membership.StartDate, EndDate: membership.EndDate, } if membership.MembershipTeam.TypeName == "Club" { new.TeamSlug = membership.MembershipTeam.Club.Slug new.MembershipType = "club" } else if membership.MembershipTeam.TypeName == "NationalTeam" { new.TeamSlug = membership.MembershipTeam.NationalTeam.Slug new.MembershipType = "national" } else { continue } res = append(res, new) } return res }), ) if err != nil { return errors.Wrap(err, "inserting players club_memberships") } log.Debug().Msgf("%d players club_memberships inserted", len(u.playerCache)) u.playerCache = nil return nil } func (u *Updater) AddGamesFromFixtureToRead(slugs ...string) { slugs = lo.Filter(slugs, func(slug string, index int) bool { return slug != "" }) u.gamesFromFixtureSlugToRead = lo.Uniq(append(u.gamesFromFixtureSlugToRead, slugs...)) } func (u *Updater) readGamesFromFixture(ctx context.Context) error { if len(u.gamesFromFixtureSlugToRead) == 0 { log.Debug().Msg("no games from fixture to read") return nil } slugs := u.gamesFromFixtureSlugToRead u.gamesFromFixtureSlugToRead = nil log.Debug().Msgf("updating games for fixtures %v", slugs) u.gameCache = nil for _, slug := range slugs { log.Debug().Msgf("getting games for fixture %s...", slug) g, err := u.s.Football.So5.FixtureGames(slug).Get(ctx, graphql.EmptyParams{}) if err != nil { return errors.Wrapf(err, "getting games for fixture %s", slug) } g = lo.Filter(g, func(game football.Game, index int) bool { return game.Id.Value != "" }) log.Debug().Msgf("found %d games", len(g)) u.gameCache = append(u.gameCache, g...) } u.gameCache = lo.Filter(u.gameCache, func(game football.Game, index int) bool { return game.So5Fixture.Slug != "" }) u.gameCache = lo.UniqBy(u.gameCache, func(game football.Game) string { return game.Id.Value }) u.AddTeamsToRead(lo.Union( lo.Map(u.gameCache, func(game football.Game, index int) string { return game.AwayTeam.Team.Slug }), lo.Map(u.gameCache, func(game football.Game, index int) string { return game.HomeTeam.Team.Slug }), )...) u.AddCompetitionsToRead(lo.Map(u.gameCache, func(game football.Game, index int) string { return game.Competition.Slug })...) u.AddGameFormationsToRead(lo.Map(u.gameCache, func(game football.Game, index int) string { return game.Id.Value })...) return nil } func (u *Updater) writeGames(ctx context.Context) error { log.Debug().Msg("inserting games into db...") err := u.db.Games.CreateOrUpdateMany( ctx, lo.Map(u.gameCache, func(game football.Game, index int) model.Game { return NewGameFromSorare(game) }), ) if err != nil { return errors.Wrap(err, "inserting games") } log.Debug().Msgf("%d games inserted", len(u.gameCache)) u.gameCache = nil return nil } func (u *Updater) AddGameFormationsToRead(ids ...string) { ids = lo.Filter(ids, func(id string, index int) bool { return id != "" }) u.gameFormationsToRead = lo.Uniq(append(u.gameFormationsToRead, ids...)) } func (u *Updater) readGamePlayers(ctx context.Context) error { if len(u.gameFormationsToRead) == 0 { log.Debug().Msg("no game formations to read") return nil } ids := u.gameFormationsToRead u.gameFormationsToRead = nil playerSlugsByGameMap := make(map[string][]string) for _, chunk := range lo.Chunk(ids, 50) { gamesWithFormation, err := u.s.Football.GamesFormation.Get(ctx, chunk) if err != nil { return errors.Wrapf(err, "getting games with formation for games %v", ids) } for _, game := range gamesWithFormation { newplayers := model.ExtractPlayersFromGameWithFormation(game) log.Debug().Msgf("\t%s -> %d players", game.Id.Value, len(newplayers)) playerSlugsByGameMap[game.Id.Value] = lo.Map( newplayers, func(player model.GamePlayer, index int) string { return player.PlayerSlug }, ) u.gamePlayersCache = append(u.gamePlayersCache, newplayers...) } } u.gamePlayersCache = lo.UniqBy(u.gamePlayersCache, func(player model.GamePlayer) string { return player.GameId + "-" + player.PlayerSlug }) u.AddPlayersToRead(lo.Map(u.gamePlayersCache, func(player model.GamePlayer, index int) string { return player.PlayerSlug })...) u.AddGameScoresFromMapToRead(playerSlugsByGameMap) return nil } func (u *Updater) writeGamePlayers(ctx context.Context) error { log.Debug().Msg("inserting game players into db...") err := u.db.GamePlayers.CreateOrUpdateMany( ctx, u.gamePlayersCache, ) if err != nil { return errors.Wrap(err, "inserting game players") } log.Debug().Msgf("%d game players inserted", len(u.gamePlayersCache)) u.gamePlayersCache = nil return nil } func (u *Updater) AddGameScoresFromMapToRead(gameScores map[string][]string) { for gameId, playerSlugs := range gameScores { u.gameScoresToReadFromMap[gameId] = append(u.gameScoresToReadFromMap[gameId], playerSlugs...) } for gameId, playerSlugs := range u.gameScoresToReadFromMap { u.gameScoresToReadFromMap[gameId] = lo.Uniq(playerSlugs) } } func (u *Updater) readGameScoresFromMap(ctx context.Context) error { if len(u.gameScoresToReadFromMap) == 0 { log.Debug().Msg("no game scores to read") return nil } for gameId, playerSlugs := range u.gameScoresToReadFromMap { if len(playerSlugs) == 0 { delete(u.gameScoresToReadFromMap, gameId) } } gameIdList := lo.MapToSlice(u.gameScoresToReadFromMap, func(key string, value []string) string { return key }) for _, gameIds := range lo.Chunk(gameIdList, 2) { log.Debug().Msgf("getting scores for games %v", gameIds) params := make(map[string][]string) for _, gameId := range gameIds { if u.gameScoresToReadFromMap[gameId] != nil && len(u.gameScoresToReadFromMap[gameId]) > 0 { params[gameId] = u.gameScoresToReadFromMap[gameId] } } if len(params) == 0 { continue } scores, err := u.s.Football.GamesScores.Get(ctx, params) if err != nil { return errors.Wrap(err, "getting scores") } scores = lo.Filter(scores, func(score football.So5Score, index int) bool { return score.Player.Slug != "" && score.Game.Id.Value != "" }) u.gameScoreCache = append(u.gameScoreCache, lo.Map( scores, func(score football.So5Score, index int) model.GamePlayerScore { return NewGamePlayerScoreFromSorare(score.Game.Id.Value, score) }, )...) } for k := range u.gameScoresToReadFromMap { delete(u.gameScoresToReadFromMap, k) } return nil } func (u *Updater) writeGameScores(ctx context.Context) error { log.Debug().Msg("inserting game scores from map into db...") err := u.db.GamePlayerScores.CreateOrUpdateMany( ctx, u.gameScoreCache, ) if err != nil { return errors.Wrap(err, "inserting game scores from map") } log.Debug().Msgf("%d game scores from map inserted", len(u.gameScoreCache)) u.gameScoreCache = nil return nil } func (u *Updater) AddGamesToRead(ids ...string) { ids = lo.Filter(ids, func(id string, index int) bool { return id != "" }) u.gamesToRead = lo.Uniq(append(u.gamesToRead, ids...)) } func (u *Updater) readGames(ctx context.Context) error { if len(u.gamesToRead) == 0 { log.Debug().Msg("no games to read") return nil } ids := u.gamesToRead u.gamesToRead = nil log.Debug().Msgf("updating games %v", ids) u.gameCache = nil for _, id := range ids { g, err := u.s.Football.Game.Get(ctx, graphql.IdParams{Id: gql.ID(id)}) if err != nil { return errors.Wrapf(err, "getting game %s", id) } u.gameCache = append(u.gameCache, g.Game) } u.gameCache = lo.Filter(u.gameCache, func(game football.Game, index int) bool { return game.So5Fixture.Slug != "" }) u.gameCache = lo.UniqBy(u.gameCache, func(game football.Game) string { return game.Id.Value }) u.AddTeamsToRead(lo.Union( lo.Map(u.gameCache, func(game football.Game, index int) string { return game.AwayTeam.Team.Slug }), lo.Map(u.gameCache, func(game football.Game, index int) string { return game.HomeTeam.Team.Slug }), )...) u.AddCompetitionsToRead(lo.Map(u.gameCache, func(game football.Game, index int) string { return game.Competition.Slug })...) u.AddGameFormationsToRead(lo.Map(u.gameCache, func(game football.Game, index int) string { return game.Id.Value })...) return nil }