514 lines
15 KiB
Go
514 lines
15 KiB
Go
package sorare_utils
|
|
|
|
import (
|
|
"context"
|
|
|
|
"git.lehouerou.net/laurent/sorare"
|
|
"git.lehouerou.net/laurent/sorare/football"
|
|
"git.lehouerou.net/laurent/sorare/graphql"
|
|
"github.com/jackc/pgx/v5/pgtype"
|
|
gql "github.com/llehouerou/go-graphql-client"
|
|
"github.com/pkg/errors"
|
|
"github.com/rs/zerolog/log"
|
|
"github.com/samber/lo"
|
|
|
|
"git.lehouerou.net/laurent/sorarebuddy/model"
|
|
)
|
|
|
|
type UpdateService struct {
|
|
s *sorare.Sorare
|
|
db *model.Queries
|
|
}
|
|
|
|
func NewUpdateService(s *sorare.Sorare, db *model.Queries) *UpdateService {
|
|
return &UpdateService{s: s, db: db}
|
|
}
|
|
|
|
func (u *UpdateService) InitSyncDatabase(ctx context.Context) error {
|
|
|
|
sfixtures, err := u.s.Football.So5.So5Fixtures.Get(ctx, football.So5FixturesParams{
|
|
// AasmStates: []string{"started"},
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
log.Debug().Msgf("fixtures: %v", sfixtures)
|
|
|
|
batchFixtures := u.db.CreateOrUpdateFixtures(
|
|
ctx,
|
|
lo.Map(sfixtures, func(fixture football.So5Fixture, index int) model.CreateOrUpdateFixturesParams {
|
|
return model.CreateOrUpdateFixturesParams{
|
|
Slug: fixture.Slug,
|
|
DisplayName: fixture.DisplayName,
|
|
State: fixture.AasmState,
|
|
StartDate: pgtype.Timestamptz{Time: fixture.StartDate, Valid: true},
|
|
EndDate: pgtype.Timestamptz{Time: fixture.EndDate, Valid: true},
|
|
GameWeek: int32(fixture.GameWeek),
|
|
}
|
|
}),
|
|
)
|
|
var batcherr error
|
|
batchFixtures.Exec(func(_ int, err error) {
|
|
if err != nil {
|
|
batcherr = err
|
|
batchFixtures.Close()
|
|
|
|
}
|
|
})
|
|
if batcherr != nil {
|
|
return err
|
|
}
|
|
|
|
log.Debug().Msgf("created %d fixtures", len(sfixtures))
|
|
|
|
fixtures, err := u.db.GetAllFixtures(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
games, err := GetGamesFromFixtures(
|
|
ctx,
|
|
u.s,
|
|
lo.Map(fixtures, func(fixture model.Fixture, index int) string {
|
|
return fixture.Slug
|
|
}),
|
|
)
|
|
log.Info().Msgf("found %d games to process", len(games))
|
|
|
|
log.Debug().Msgf("getting players for each game...")
|
|
var gamePlayers []model.CreateOrUpdateGamePlayersParams
|
|
playerSlugsByGameMap := make(map[string][]string)
|
|
for _, game := range games {
|
|
gameWithFormation, err := u.s.Football.Game.Get(ctx, graphql.IdParams{Id: gql.ID(game.Id.Value)})
|
|
if err != nil {
|
|
return errors.Wrapf(err, "getting game with formation %s", game.Id.Value)
|
|
}
|
|
newplayers := ExtractPlayersFromGameWithFormation(gameWithFormation)
|
|
log.Debug().Msgf("\t%s -> %d players", game.String(), len(newplayers))
|
|
playerSlugsByGameMap[game.Id.Value] = lo.Map(
|
|
newplayers,
|
|
func(player model.CreateOrUpdateGamePlayersParams, index int) string {
|
|
return player.PlayerSlug
|
|
},
|
|
)
|
|
gamePlayers = append(gamePlayers, newplayers...)
|
|
}
|
|
|
|
playerSlugs := lo.Uniq(
|
|
lo.Filter(lo.Map(gamePlayers, func(player model.CreateOrUpdateGamePlayersParams, index int) string {
|
|
return player.PlayerSlug
|
|
}), func(slug string, index int) bool {
|
|
return slug != ""
|
|
}),
|
|
)
|
|
|
|
log.Debug().Msgf("getting players...")
|
|
var players []football.Player
|
|
for i, chunk := range lo.Chunk(playerSlugs, 80) {
|
|
log.Debug().Msgf("\tbatch %d/%d", i+1, (len(playerSlugs)/80)+1)
|
|
p, err := u.s.Football.Players.Get(ctx, graphql.SlugsParams{Slugs: chunk})
|
|
if err != nil {
|
|
return errors.Wrapf(err, "getting players batch %d", i)
|
|
}
|
|
players = append(players, p...)
|
|
}
|
|
log.Debug().Msgf("found %d players", len(players))
|
|
|
|
teamSlugs := ExtractTeamSlugsFromPlayersAndGames(players, games)
|
|
log.Debug().Msgf("extracted %d unique team slugs from games and players", len(teamSlugs))
|
|
|
|
log.Debug().Msgf("getting clubs...")
|
|
var clubs []football.Club
|
|
for i, chunk := range lo.Chunk(teamSlugs, 100) {
|
|
log.Debug().Msgf("\tbatch %d/%d", i+1, (len(teamSlugs)/100)+1)
|
|
t, err := u.s.Football.Clubs.Get(ctx, graphql.SlugsParams{Slugs: chunk})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
clubs = append(clubs, t...)
|
|
}
|
|
log.Debug().Msgf("found %d clubs", len(clubs))
|
|
var nationalTeams []football.NationalTeam
|
|
slugsLeft := lo.Without(teamSlugs, lo.Map(clubs, func(club football.Club, index int) string {
|
|
return club.Slug
|
|
})...)
|
|
log.Debug().Msgf("getting national teams...")
|
|
log.Debug().Msgf("slugs left: %d", len(slugsLeft))
|
|
for i, chunk := range lo.Chunk(slugsLeft, 100) {
|
|
log.Debug().Msgf("\tbatch %d/%d", i+1, (len(teamSlugs)/100)+1)
|
|
t, err := u.s.Football.NationalTeams.Get(ctx, graphql.SlugsParams{Slugs: chunk})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
nationalTeams = append(nationalTeams, t...)
|
|
}
|
|
log.Debug().Msgf("found %d national teams", len(nationalTeams))
|
|
|
|
competitionSlugs := ExtractCompetitionSlugsFromPlayersGamesAndClubs(players, games, clubs)
|
|
log.Debug().Msgf("extracted %d unique competition slugs from players, games and clubs", len(competitionSlugs))
|
|
log.Debug().Msgf("getting competitions...")
|
|
var competitions []football.Competition
|
|
for _, slug := range competitionSlugs {
|
|
log.Debug().Msgf("\tcompetition %s", slug)
|
|
c, err := u.s.Football.Competition.Get(ctx, graphql.SlugParams{Slug: slug})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
competitions = append(competitions, c)
|
|
}
|
|
log.Debug().Msgf("found %d competitions", len(competitions))
|
|
|
|
countrySlugs := ExtractCountrySlugsFromPlayersCompetitionsClubsAndNationalTeams(
|
|
players,
|
|
competitions,
|
|
clubs,
|
|
nationalTeams,
|
|
)
|
|
log.Debug().
|
|
Msgf("extracted %d unique country slugs from players, competitions, clubs and national teams", len(countrySlugs))
|
|
log.Debug().Msgf("getting countries...")
|
|
var countries []sorare.Country
|
|
for i, chunk := range lo.Chunk(countrySlugs, 100) {
|
|
log.Debug().Msgf("\tbatch %d/%d", i+1, (len(countrySlugs)/100)+1)
|
|
c, err := u.s.Countries.Get(ctx, graphql.SlugsParams{Slugs: chunk})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
countries = append(countries, c...)
|
|
}
|
|
log.Debug().Msgf("found %d countries", len(countries))
|
|
|
|
scores := make(map[string][]football.PlayerScore)
|
|
for gameId, playerSlugs := range playerSlugsByGameMap {
|
|
log.Debug().Msgf("getting scores for game %s...", gameId)
|
|
var gameScores []football.PlayerScore
|
|
for i, chunk := range lo.Chunk(playerSlugs, 80) {
|
|
log.Debug().Msgf("\tbatch %d/%d", i+1, (len(playerSlugs)/80)+1)
|
|
s, err := u.s.Football.PlayersGameScores(gql.ID(gameId)).Get(ctx, graphql.SlugsParams{Slugs: chunk})
|
|
if err != nil {
|
|
return errors.Wrapf(err, "getting scores for game %s", gameId)
|
|
}
|
|
gameScores = append(gameScores, s...)
|
|
}
|
|
for _, score := range gameScores {
|
|
log.Debug().Msgf("\t%s -> %s", score.Slug, score.So5Score.Score)
|
|
}
|
|
scores[gameId] = gameScores
|
|
}
|
|
|
|
log.Debug().Msg("inserting countries into db...")
|
|
batchCountries := u.db.CreateOrUpdateCountries(
|
|
ctx,
|
|
lo.Map(countries, func(country sorare.Country, index int) model.CreateOrUpdateCountriesParams {
|
|
return model.CreateOrUpdateCountriesParams{
|
|
Slug: country.Slug,
|
|
Code: country.Code,
|
|
DisplayName: country.Name,
|
|
ThreeLetterCode: country.ThreeLetterCode,
|
|
FlagFlat64Url: country.FlagFlat64Url,
|
|
FlagFlat32Url: country.FlagFlat32Url,
|
|
FlagRound64Url: country.FlagRound64Url,
|
|
FlagRound32Url: country.FlagRound32Url,
|
|
}
|
|
}),
|
|
)
|
|
batcherr = nil
|
|
batchCountries.Exec(func(_ int, err error) {
|
|
if err != nil {
|
|
batcherr = err
|
|
batchCountries.Close()
|
|
}
|
|
})
|
|
if batcherr != nil {
|
|
return err
|
|
}
|
|
log.Debug().Msgf("%d countries inserted", len(countries))
|
|
|
|
log.Debug().Msg("inserting competitions into db...")
|
|
|
|
batchCompetitions := u.db.CreateOrUpdateCompetitions(
|
|
ctx,
|
|
lo.Map(competitions, func(competition football.Competition, index int) model.CreateOrUpdateCompetitionsParams {
|
|
return model.CreateOrUpdateCompetitionsParams{
|
|
Slug: competition.Slug,
|
|
CompetitionFormat: competition.Format,
|
|
CompetitionType: competition.Type,
|
|
DisplayName: competition.DisplayName,
|
|
PictureUrl: competition.PictureUrl,
|
|
LogoUrl: competition.LogoUrl,
|
|
CountrySlug: competition.Country.Slug,
|
|
}
|
|
}),
|
|
)
|
|
batcherr = nil
|
|
batchCompetitions.Exec(func(_ int, err error) {
|
|
if err != nil {
|
|
batcherr = err
|
|
batchCompetitions.Close()
|
|
}
|
|
})
|
|
if batcherr != nil {
|
|
return errors.Wrap(batcherr, "inserting competitions")
|
|
}
|
|
|
|
log.Debug().Msgf("%d competitions inserted", len(competitions))
|
|
|
|
log.Debug().Msg("inserting teams into db...")
|
|
batchTeams := u.db.CreateOrUpdateTeams(ctx, lo.Union(
|
|
lo.Map(clubs, func(club football.Club, index int) model.CreateOrUpdateTeamsParams {
|
|
return model.CreateOrUpdateTeamsParams{
|
|
Slug: club.Slug,
|
|
DisplayName: club.Name,
|
|
CountrySlug: club.Country.Slug,
|
|
DomesticLeagueSlug: func() *string {
|
|
if club.DomesticLeague.Slug == "" {
|
|
return nil
|
|
}
|
|
return &club.DomesticLeague.Slug
|
|
}(),
|
|
ShortName: club.ShortName,
|
|
PictureUrl: club.PictureUrl,
|
|
TeamType: "club",
|
|
}
|
|
}),
|
|
lo.Map(nationalTeams, func(nationalTeam football.NationalTeam, index int) model.CreateOrUpdateTeamsParams {
|
|
return model.CreateOrUpdateTeamsParams{
|
|
Slug: nationalTeam.Slug,
|
|
DisplayName: nationalTeam.Name,
|
|
CountrySlug: nationalTeam.Country.Slug,
|
|
DomesticLeagueSlug: nil,
|
|
ShortName: nationalTeam.ShortName,
|
|
PictureUrl: nationalTeam.PictureUrl,
|
|
TeamType: "national",
|
|
}
|
|
}),
|
|
))
|
|
batcherr = nil
|
|
batchTeams.Exec(func(_ int, err error) {
|
|
if err != nil {
|
|
batcherr = err
|
|
batchTeams.Close()
|
|
}
|
|
})
|
|
if batcherr != nil {
|
|
return errors.Wrap(batcherr, "inserting teams into db")
|
|
}
|
|
log.Debug().Msgf("%d teams inserted", len(clubs)+len(nationalTeams))
|
|
|
|
log.Debug().Msg("inserting games into db...")
|
|
batchGames := u.db.CreateOrUpdateGames(
|
|
ctx,
|
|
lo.Map(games, func(game football.Game, index int) model.CreateOrUpdateGamesParams {
|
|
return NewCreateOrUpdateGamesParamsFromSorare(game)
|
|
}),
|
|
)
|
|
batcherr = nil
|
|
batchGames.Exec(func(_ int, err error) {
|
|
if err != nil {
|
|
batcherr = err
|
|
batchGames.Close()
|
|
}
|
|
})
|
|
if batcherr != nil {
|
|
return err
|
|
}
|
|
log.Debug().Msgf("%d games inserted", len(games))
|
|
|
|
log.Debug().Msg("inserting players into db...")
|
|
batchPlayers := u.db.CreateOrUpdatePlayers(
|
|
ctx,
|
|
lo.Map(players, func(player football.Player, index int) model.CreateOrUpdatePlayersParams {
|
|
res := model.CreateOrUpdatePlayersParams{
|
|
Slug: player.Slug,
|
|
DisplayName: player.DisplayName,
|
|
BirthDate: player.BirthDate,
|
|
CountrySlug: player.Country.Slug,
|
|
AvatarUrl: player.AvatarUrl,
|
|
FieldPosition: string(player.Position),
|
|
Status: string(player.PlayingStatus),
|
|
ShirtNumber: int32(player.ShirtNumber),
|
|
}
|
|
for _, competition := range player.ActiveClub.ActiveCompetitions {
|
|
if competition.Format == "DOMESTIC_LEAGUE" {
|
|
res.DomesticLeagueSlug = &competition.Slug
|
|
}
|
|
}
|
|
if player.ActiveClub.Slug != "" {
|
|
res.TeamSlug = &player.ActiveClub.Slug
|
|
}
|
|
return res
|
|
}),
|
|
)
|
|
batcherr = nil
|
|
batchPlayers.Exec(func(_ int, err error) {
|
|
if err != nil {
|
|
batcherr = err
|
|
batchPlayers.Close()
|
|
}
|
|
})
|
|
if batcherr != nil {
|
|
return errors.Wrap(batcherr, "inserting players")
|
|
}
|
|
log.Debug().Msgf("%d players inserted", len(players))
|
|
|
|
log.Debug().Msg("inserting game players into db...")
|
|
batchGamePlayers := u.db.CreateOrUpdateGamePlayers(
|
|
ctx,
|
|
gamePlayers,
|
|
)
|
|
batcherr = nil
|
|
batchGamePlayers.Exec(func(_ int, err error) {
|
|
if err != nil {
|
|
batcherr = err
|
|
batchGamePlayers.Close()
|
|
}
|
|
})
|
|
if batcherr != nil {
|
|
return errors.Wrap(err, "inserting game players")
|
|
}
|
|
log.Debug().Msgf("%d game players inserted", len(gamePlayers))
|
|
|
|
log.Debug().Msg("inserting game player scores into db...")
|
|
batchGamePlayerScores := u.db.CreateOrUpdateGamePlayerScores(
|
|
ctx,
|
|
lo.Union(
|
|
lo.MapToSlice(
|
|
scores,
|
|
func(gameId string, scores []football.PlayerScore) []model.CreateOrUpdateGamePlayerScoresParams {
|
|
return lo.Map(scores, func(score football.PlayerScore, index int) model.CreateOrUpdateGamePlayerScoresParams {
|
|
return NewCreateOrUpdateGamePlayerScoresParamsFromSorare(gameId, score)
|
|
})
|
|
|
|
},
|
|
)...),
|
|
)
|
|
batcherr = nil
|
|
batchGamePlayerScores.Exec(func(i int, err error) {
|
|
if err != nil {
|
|
batcherr = err
|
|
batchGamePlayerScores.Close()
|
|
}
|
|
})
|
|
if batcherr != nil {
|
|
return errors.Wrap(batcherr, "inserting game player scores")
|
|
}
|
|
log.Debug().Msgf("game player scores inserted")
|
|
|
|
return nil
|
|
}
|
|
|
|
func ExtractTeamSlugsFromPlayersAndGames(players []football.Player, games []football.Game) []string {
|
|
return lo.Uniq(lo.Union(
|
|
ExtractTeamSlugsFromPlayers(players),
|
|
ExtractTeamSlugsFromGames(games),
|
|
))
|
|
}
|
|
|
|
func ExtractTeamSlugsFromPlayers(players []football.Player) []string {
|
|
return lo.Uniq(lo.Filter(lo.Map(players, func(player football.Player, index int) string {
|
|
return player.ActiveClub.Slug
|
|
}), func(slug string, index int) bool {
|
|
return slug != ""
|
|
}))
|
|
}
|
|
|
|
func ExtractTeamSlugsFromGames(games []football.Game) []string {
|
|
var res []string
|
|
res = lo.Map(games, func(game football.Game, index int) string {
|
|
return game.AwayTeam.Team.Slug
|
|
})
|
|
res = append(res, lo.Map(games, func(game football.Game, index int) string {
|
|
return game.HomeTeam.Team.Slug
|
|
})...)
|
|
res = lo.Filter(res, func(slug string, index int) bool {
|
|
return slug != ""
|
|
})
|
|
return lo.Uniq(res)
|
|
}
|
|
|
|
func ExtractCountrySlugsFromCompetitions(competitions []football.Competition) []string {
|
|
return lo.Uniq(lo.Filter(lo.Map(competitions, func(competition football.Competition, index int) string {
|
|
return competition.Country.Slug
|
|
}), func(slug string, index int) bool {
|
|
return slug != ""
|
|
}))
|
|
}
|
|
|
|
func ExtractCountrySlugsFromTeams(teams []football.Team) []string {
|
|
return lo.Uniq(lo.Filter(lo.Map(teams, func(team football.Team, index int) string {
|
|
return team.Country.Slug
|
|
}), func(slug string, index int) bool {
|
|
return slug != ""
|
|
}))
|
|
}
|
|
|
|
func ExtractCountrySlugsFromPlayersCompetitionsClubsAndNationalTeams(
|
|
players []football.Player,
|
|
competitions []football.Competition,
|
|
clubs []football.Club,
|
|
nationalTeams []football.NationalTeam,
|
|
) []string {
|
|
return lo.Uniq(lo.Union(
|
|
ExtractCountrySlugsFromPlayers(players),
|
|
ExtractCountrySlugsFromCompetitions(competitions),
|
|
ExtractCountrySlugsFromTeams(lo.Map(clubs, func(club football.Club, index int) football.Team {
|
|
return club.Team
|
|
})),
|
|
ExtractCountrySlugsFromTeams(
|
|
lo.Map(nationalTeams, func(nationalTeam football.NationalTeam, index int) football.Team {
|
|
return nationalTeam.Team
|
|
}),
|
|
),
|
|
))
|
|
}
|
|
|
|
func ExtractCountrySlugsFromPlayers(players []football.Player) []string {
|
|
return lo.Uniq(lo.Filter(lo.Map(players, func(player football.Player, index int) string {
|
|
return player.Country.Slug
|
|
}), func(slug string, index int) bool {
|
|
return slug != ""
|
|
}))
|
|
|
|
}
|
|
|
|
func ExtractCompetitionSlugsFromPlayers(players []football.Player) []string {
|
|
return lo.Uniq(lo.Filter(lo.Map(players, func(player football.Player, index int) string {
|
|
var res string
|
|
for _, competition := range player.ActiveClub.ActiveCompetitions {
|
|
if competition.Format == "DOMESTIC_LEAGUE" {
|
|
res = competition.Slug
|
|
}
|
|
}
|
|
return res
|
|
}), func(slug string, index int) bool {
|
|
return slug != ""
|
|
}))
|
|
}
|
|
|
|
func ExtractCompetitionSlugsFromGames(games []football.Game) []string {
|
|
return lo.Uniq(lo.Filter(lo.Map(games, func(game football.Game, index int) string {
|
|
return game.Competition.Slug
|
|
}), func(slug string, index int) bool {
|
|
return slug != ""
|
|
}))
|
|
}
|
|
|
|
func ExtractCompetitionSlugsFromClubs(clubs []football.Club) []string {
|
|
return lo.Uniq(lo.Filter(lo.Map(clubs, func(club football.Club, index int) string {
|
|
return club.DomesticLeague.Slug
|
|
}), func(slug string, index int) bool {
|
|
return slug != ""
|
|
}))
|
|
}
|
|
|
|
func ExtractCompetitionSlugsFromPlayersGamesAndClubs(
|
|
players []football.Player,
|
|
games []football.Game,
|
|
clubs []football.Club,
|
|
) []string {
|
|
return lo.Uniq(lo.Union(
|
|
ExtractCompetitionSlugsFromPlayers(players),
|
|
ExtractCompetitionSlugsFromGames(games),
|
|
ExtractCompetitionSlugsFromClubs(clubs),
|
|
))
|
|
}
|