package sorare_utils import ( "context" "git.lehouerou.net/laurent/sorare" "git.lehouerou.net/laurent/sorare/football" "git.lehouerou.net/laurent/sorare/graphql" "github.com/jackc/pgx/v5/pgtype" gql "github.com/llehouerou/go-graphql-client" "github.com/pkg/errors" "github.com/rs/zerolog/log" "github.com/samber/lo" "git.lehouerou.net/laurent/sorarebuddy/model" ) type UpdateService struct { s *sorare.Sorare db *model.Queries } func NewUpdateService(s *sorare.Sorare, db *model.Queries) *UpdateService { return &UpdateService{s: s, db: db} } func (u *UpdateService) InitSyncDatabase(ctx context.Context) error { sfixtures, err := u.s.Football.So5.So5Fixtures.Get(ctx, football.So5FixturesParams{ AasmStates: []string{"started"}, }) if err != nil { return err } log.Debug().Msgf("fixtures: %v", sfixtures) cnt, err := u.db.CreateFixtures( ctx, lo.Map(sfixtures, func(fixture football.So5Fixture, index int) model.CreateFixturesParams { return model.CreateFixturesParams{ Slug: fixture.Slug, DisplayName: fixture.DisplayName, State: fixture.AasmState, StartDate: pgtype.Timestamptz{Time: fixture.StartDate, Valid: true}, EndDate: pgtype.Timestamptz{Time: fixture.EndDate, Valid: true}, GameWeek: int32(fixture.GameWeek), } }), ) if err != nil { return err } log.Debug().Msgf("created %d fixtures", cnt) fixtures, err := u.db.GetAllFixtures(ctx) if err != nil { return err } games, err := GetGamesFromFixtures( ctx, u.s, lo.Map(fixtures, func(fixture model.Fixture, index int) string { return fixture.Slug }), ) log.Info().Msgf("found %d games to process", len(games)) log.Debug().Msgf("getting players for each game...") var gamePlayers []model.BatchInsertGamePlayersParams for _, game := range games { gameWithFormation, err := u.s.Football.Game.Get(ctx, graphql.IdParams{Id: gql.ID(game.Id.Value)}) if err != nil { return errors.Wrapf(err, "getting game with formation %s", game.Id.Value) } newplayers := ExtractPlayersFromGameWithFormation(gameWithFormation) log.Debug().Msgf("\t%s -> %d players", game.String(), len(newplayers)) gamePlayers = append(gamePlayers, newplayers...) } playerSlugs := lo.Uniq( lo.Filter(lo.Map(gamePlayers, func(player model.BatchInsertGamePlayersParams, index int) string { return player.PlayerSlug }), func(slug string, index int) bool { return slug != "" }), ) log.Debug().Msgf("getting players...") var players []football.Player for i, chunk := range lo.Chunk(playerSlugs, 80) { log.Debug().Msgf("\tbatch %d/%d", i+1, (len(playerSlugs)/80)+1) p, err := u.s.Football.Players.Get(ctx, graphql.SlugsParams{Slugs: chunk}) if err != nil { return errors.Wrapf(err, "getting players batch %d", i) } players = append(players, p...) } log.Debug().Msgf("found %d players", len(players)) teamSlugs := ExtractTeamSlugsFromPlayersAndGames(players, games) log.Debug().Msgf("extracted %d unique team slugs from games and players", len(teamSlugs)) log.Debug().Msgf("getting clubs...") var clubs []football.Club for i, chunk := range lo.Chunk(teamSlugs, 100) { log.Debug().Msgf("\tbatch %d/%d", i+1, (len(teamSlugs)/100)+1) t, err := u.s.Football.Clubs.Get(ctx, graphql.SlugsParams{Slugs: chunk}) if err != nil { return err } clubs = append(clubs, t...) } log.Debug().Msgf("found %d clubs", len(clubs)) var nationalTeams []football.NationalTeam slugsLeft := lo.Without(teamSlugs, lo.Map(clubs, func(club football.Club, index int) string { return club.Slug })...) log.Debug().Msgf("getting national teams...") log.Debug().Msgf("slugs left: %d", len(slugsLeft)) for i, chunk := range lo.Chunk(slugsLeft, 100) { log.Debug().Msgf("\tbatch %d/%d", i+1, (len(teamSlugs)/100)+1) t, err := u.s.Football.NationalTeams.Get(ctx, graphql.SlugsParams{Slugs: chunk}) if err != nil { return err } nationalTeams = append(nationalTeams, t...) } log.Debug().Msgf("found %d national teams", len(nationalTeams)) competitionSlugs := ExtractCompetitionSlugsFromPlayersGamesAndClubs(players, games, clubs) log.Debug().Msgf("extracted %d unique competition slugs from players, games and clubs", len(competitionSlugs)) log.Debug().Msgf("getting competitions...") var competitions []football.Competition for _, slug := range competitionSlugs { log.Debug().Msgf("\tcompetition %s", slug) c, err := u.s.Football.Competition.Get(ctx, graphql.SlugParams{Slug: slug}) if err != nil { return err } competitions = append(competitions, c) } log.Debug().Msgf("found %d competitions", len(competitions)) countrySlugs := ExtractCountrySlugsFromPlayersCompetitionsClubsAndNationalTeams( players, competitions, clubs, nationalTeams, ) log.Debug(). Msgf("extracted %d unique country slugs from players, competitions, clubs and national teams", len(countrySlugs)) log.Debug().Msgf("getting countries...") var countries []sorare.Country for i, chunk := range lo.Chunk(countrySlugs, 100) { log.Debug().Msgf("\tbatch %d/%d", i+1, (len(countrySlugs)/100)+1) c, err := u.s.Countries.Get(ctx, graphql.SlugsParams{Slugs: chunk}) if err != nil { return err } countries = append(countries, c...) } log.Debug().Msgf("found %d countries", len(countries)) log.Debug().Msg("inserting countries into db...") batchCountries := u.db.CreateOrUpdateCountries( ctx, lo.Map(countries, func(country sorare.Country, index int) model.CreateOrUpdateCountriesParams { return model.CreateOrUpdateCountriesParams{ Slug: country.Slug, Code: country.Code, DisplayName: country.Name, ThreeLetterCode: country.ThreeLetterCode, FlagFlat64Url: country.FlagFlat64Url, FlagFlat32Url: country.FlagFlat32Url, FlagRound64Url: country.FlagRound64Url, FlagRound32Url: country.FlagRound32Url, } }), ) var batcherr error batchCountries.Exec(func(_ int, err error) { if err != nil { batcherr = err batchCountries.Close() } }) if batcherr != nil { return err } log.Debug().Msgf("%d countries inserted", cnt) log.Debug().Msg("inserting competitions into db...") batchCompetitions := u.db.CreateOrUpdateCompetitions( ctx, lo.Map(competitions, func(competition football.Competition, index int) model.CreateOrUpdateCompetitionsParams { return model.CreateOrUpdateCompetitionsParams{ Slug: competition.Slug, CompetitionFormat: competition.Format, CompetitionType: competition.Type, DisplayName: competition.DisplayName, PictureUrl: competition.PictureUrl, LogoUrl: competition.LogoUrl, CountrySlug: competition.Country.Slug, } }), ) batchCompetitions.Exec(func(_ int, err error) { if err != nil { batcherr = err batchCompetitions.Close() } }) if batcherr != nil { return errors.Wrap(batcherr, "inserting competitions") } log.Debug().Msgf("%d competitions inserted", len(competitions)) log.Debug().Msg("inserting teams into db...") cnt, err = u.db.BatchInsertTeams(ctx, lo.Union( lo.Map(clubs, func(club football.Club, index int) model.BatchInsertTeamsParams { return model.BatchInsertTeamsParams{ Slug: club.Slug, DisplayName: club.Name, CountrySlug: club.Country.Slug, DomesticLeagueSlug: func() *string { if club.DomesticLeague.Slug == "" { return nil } return &club.DomesticLeague.Slug }(), ShortName: club.ShortName, PictureUrl: club.PictureUrl, TeamType: "club", } }), lo.Map(nationalTeams, func(nationalTeam football.NationalTeam, index int) model.BatchInsertTeamsParams { return model.BatchInsertTeamsParams{ Slug: nationalTeam.Slug, DisplayName: nationalTeam.Name, CountrySlug: nationalTeam.Country.Slug, DomesticLeagueSlug: nil, ShortName: nationalTeam.ShortName, PictureUrl: nationalTeam.PictureUrl, TeamType: "national", } }), )) if err != nil { return err } log.Debug().Msgf("%d teams inserted", cnt) log.Debug().Msg("inserting games into db...") cnt, err = u.db.BatchInsertGames( ctx, lo.Map(games, func(game football.Game, index int) model.BatchInsertGamesParams { return NewBatchInsertGamesParamsFromSorare(game) }), ) if err != nil { return err } log.Debug().Msgf("%d games inserted", cnt) log.Debug().Msg("inserting players into db...") cnt, err = u.db.BatchInsertPlayers( ctx, lo.Map(players, func(player football.Player, index int) model.BatchInsertPlayersParams { res := model.BatchInsertPlayersParams{ Slug: player.Slug, DisplayName: player.DisplayName, BirthDate: player.BirthDate, CountrySlug: player.Country.Slug, AvatarUrl: player.AvatarUrl, FieldPosition: string(player.Position), Status: string(player.PlayingStatus), ShirtNumber: int32(player.ShirtNumber), } for _, competition := range player.ActiveClub.ActiveCompetitions { if competition.Format == "DOMESTIC_LEAGUE" { res.DomesticLeagueSlug = &competition.Slug } } if player.ActiveClub.Slug != "" { res.TeamSlug = &player.ActiveClub.Slug } return res }), ) if err != nil { return errors.Wrap(err, "inserting players") } log.Debug().Msgf("%d players inserted", cnt) log.Debug().Msg("inserting game players into db...") cnt, err = u.db.BatchInsertGamePlayers(ctx, gamePlayers) if err != nil { return errors.Wrap(err, "inserting game players") } log.Debug().Msgf("%d game players inserted", cnt) return nil } func ExtractPlayersFromGameWithFormation( gameWithFormation football.GameWithFormation, ) []model.BatchInsertGamePlayersParams { var res []model.BatchInsertGamePlayersParams for _, p := range gameWithFormation.HomeFormation.Bench { res = append(res, model.BatchInsertGamePlayersParams{ GameID: gameWithFormation.Id.Value, PlayerSlug: p.Slug, TeamSlug: gameWithFormation.HomeTeam.Team.Slug, Status: "bench", }) } for _, p := range gameWithFormation.HomeFormation.StartingLineup { for _, q := range p { res = append(res, model.BatchInsertGamePlayersParams{ GameID: gameWithFormation.Id.Value, PlayerSlug: q.Slug, TeamSlug: gameWithFormation.HomeTeam.Team.Slug, Status: "starting", }) } } for _, p := range gameWithFormation.AwayFormation.Bench { res = append(res, model.BatchInsertGamePlayersParams{ GameID: gameWithFormation.Id.Value, PlayerSlug: p.Slug, TeamSlug: gameWithFormation.AwayTeam.Team.Slug, Status: "bench", }) } for _, p := range gameWithFormation.AwayFormation.StartingLineup { for _, q := range p { res = append(res, model.BatchInsertGamePlayersParams{ GameID: gameWithFormation.Id.Value, PlayerSlug: q.Slug, TeamSlug: gameWithFormation.AwayTeam.Team.Slug, Status: "starting", }) } } return res } func ExtractTeamSlugsFromPlayersAndGames(players []football.Player, games []football.Game) []string { return lo.Uniq(lo.Union( ExtractTeamSlugsFromPlayers(players), ExtractTeamSlugsFromGames(games), )) } func ExtractTeamSlugsFromPlayers(players []football.Player) []string { return lo.Uniq(lo.Filter(lo.Map(players, func(player football.Player, index int) string { return player.ActiveClub.Slug }), func(slug string, index int) bool { return slug != "" })) } func ExtractTeamSlugsFromGames(games []football.Game) []string { var res []string res = lo.Map(games, func(game football.Game, index int) string { return game.AwayTeam.Team.Slug }) res = append(res, lo.Map(games, func(game football.Game, index int) string { return game.HomeTeam.Team.Slug })...) res = lo.Filter(res, func(slug string, index int) bool { return slug != "" }) return lo.Uniq(res) } func ExtractCountrySlugsFromCompetitions(competitions []football.Competition) []string { return lo.Uniq(lo.Filter(lo.Map(competitions, func(competition football.Competition, index int) string { return competition.Country.Slug }), func(slug string, index int) bool { return slug != "" })) } func ExtractCountrySlugsFromTeams(teams []football.Team) []string { return lo.Uniq(lo.Filter(lo.Map(teams, func(team football.Team, index int) string { return team.Country.Slug }), func(slug string, index int) bool { return slug != "" })) } func ExtractCountrySlugsFromPlayersCompetitionsClubsAndNationalTeams( players []football.Player, competitions []football.Competition, clubs []football.Club, nationalTeams []football.NationalTeam, ) []string { return lo.Uniq(lo.Union( ExtractCountrySlugsFromPlayers(players), ExtractCountrySlugsFromCompetitions(competitions), ExtractCountrySlugsFromTeams(lo.Map(clubs, func(club football.Club, index int) football.Team { return club.Team })), ExtractCountrySlugsFromTeams( lo.Map(nationalTeams, func(nationalTeam football.NationalTeam, index int) football.Team { return nationalTeam.Team }), ), )) } func ExtractCountrySlugsFromPlayers(players []football.Player) []string { return lo.Uniq(lo.Filter(lo.Map(players, func(player football.Player, index int) string { return player.Country.Slug }), func(slug string, index int) bool { return slug != "" })) } func ExtractCompetitionSlugsFromPlayers(players []football.Player) []string { return lo.Uniq(lo.Filter(lo.Map(players, func(player football.Player, index int) string { var res string for _, competition := range player.ActiveClub.ActiveCompetitions { if competition.Format == "DOMESTIC_LEAGUE" { res = competition.Slug } } return res }), func(slug string, index int) bool { return slug != "" })) } func ExtractCompetitionSlugsFromGames(games []football.Game) []string { return lo.Uniq(lo.Filter(lo.Map(games, func(game football.Game, index int) string { return game.Competition.Slug }), func(slug string, index int) bool { return slug != "" })) } func ExtractCompetitionSlugsFromClubs(clubs []football.Club) []string { return lo.Uniq(lo.Filter(lo.Map(clubs, func(club football.Club, index int) string { return club.DomesticLeague.Slug }), func(slug string, index int) bool { return slug != "" })) } func ExtractCompetitionSlugsFromPlayersGamesAndClubs( players []football.Player, games []football.Game, clubs []football.Club, ) []string { return lo.Uniq(lo.Union( ExtractCompetitionSlugsFromPlayers(players), ExtractCompetitionSlugsFromGames(games), ExtractCompetitionSlugsFromClubs(clubs), )) }