462 lines
14 KiB
Go
462 lines
14 KiB
Go
package sorare_utils
|
|
|
|
import (
|
|
"context"
|
|
|
|
"git.lehouerou.net/laurent/sorare"
|
|
"git.lehouerou.net/laurent/sorare/football"
|
|
"git.lehouerou.net/laurent/sorare/graphql"
|
|
"github.com/jackc/pgx/v5/pgtype"
|
|
gql "github.com/llehouerou/go-graphql-client"
|
|
"github.com/pkg/errors"
|
|
"github.com/rs/zerolog/log"
|
|
"github.com/samber/lo"
|
|
|
|
"git.lehouerou.net/laurent/sorarebuddy/model"
|
|
)
|
|
|
|
type UpdateService struct {
|
|
s *sorare.Sorare
|
|
db *model.Queries
|
|
}
|
|
|
|
func NewUpdateService(s *sorare.Sorare, db *model.Queries) *UpdateService {
|
|
return &UpdateService{s: s, db: db}
|
|
}
|
|
|
|
func (u *UpdateService) InitSyncDatabase(ctx context.Context) error {
|
|
|
|
sfixtures, err := u.s.Football.So5.So5Fixtures.Get(ctx, football.So5FixturesParams{
|
|
AasmStates: []string{"started"},
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
log.Debug().Msgf("fixtures: %v", sfixtures)
|
|
|
|
cnt, err := u.db.CreateFixtures(
|
|
ctx,
|
|
lo.Map(sfixtures, func(fixture football.So5Fixture, index int) model.CreateFixturesParams {
|
|
return model.CreateFixturesParams{
|
|
Slug: fixture.Slug,
|
|
DisplayName: fixture.DisplayName,
|
|
State: fixture.AasmState,
|
|
StartDate: pgtype.Timestamptz{Time: fixture.StartDate, Valid: true},
|
|
EndDate: pgtype.Timestamptz{Time: fixture.EndDate, Valid: true},
|
|
GameWeek: int32(fixture.GameWeek),
|
|
}
|
|
}),
|
|
)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
log.Debug().Msgf("created %d fixtures", cnt)
|
|
|
|
fixtures, err := u.db.GetAllFixtures(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
games, err := GetGamesFromFixtures(
|
|
ctx,
|
|
u.s,
|
|
lo.Map(fixtures, func(fixture model.Fixture, index int) string {
|
|
return fixture.Slug
|
|
}),
|
|
)
|
|
log.Info().Msgf("found %d games to process", len(games))
|
|
|
|
log.Debug().Msgf("getting players for each game...")
|
|
var gamePlayers []model.BatchInsertGamePlayersParams
|
|
for _, game := range games {
|
|
gameWithFormation, err := u.s.Football.Game.Get(ctx, graphql.IdParams{Id: gql.ID(game.Id.Value)})
|
|
if err != nil {
|
|
return errors.Wrapf(err, "getting game with formation %s", game.Id.Value)
|
|
}
|
|
newplayers := ExtractPlayersFromGameWithFormation(gameWithFormation)
|
|
log.Debug().Msgf("\t%s -> %d players", game.String(), len(newplayers))
|
|
gamePlayers = append(gamePlayers, newplayers...)
|
|
}
|
|
|
|
playerSlugs := lo.Uniq(
|
|
lo.Filter(lo.Map(gamePlayers, func(player model.BatchInsertGamePlayersParams, index int) string {
|
|
return player.PlayerSlug
|
|
}), func(slug string, index int) bool {
|
|
return slug != ""
|
|
}),
|
|
)
|
|
|
|
log.Debug().Msgf("getting players...")
|
|
var players []football.Player
|
|
for i, chunk := range lo.Chunk(playerSlugs, 80) {
|
|
log.Debug().Msgf("\tbatch %d/%d", i+1, (len(playerSlugs)/80)+1)
|
|
p, err := u.s.Football.Players.Get(ctx, graphql.SlugsParams{Slugs: chunk})
|
|
if err != nil {
|
|
return errors.Wrapf(err, "getting players batch %d", i)
|
|
}
|
|
players = append(players, p...)
|
|
}
|
|
log.Debug().Msgf("found %d players", len(players))
|
|
|
|
teamSlugs := ExtractTeamSlugsFromPlayersAndGames(players, games)
|
|
log.Debug().Msgf("extracted %d unique team slugs from games and players", len(teamSlugs))
|
|
|
|
log.Debug().Msgf("getting clubs...")
|
|
var clubs []football.Club
|
|
for i, chunk := range lo.Chunk(teamSlugs, 100) {
|
|
log.Debug().Msgf("\tbatch %d/%d", i+1, (len(teamSlugs)/100)+1)
|
|
t, err := u.s.Football.Clubs.Get(ctx, graphql.SlugsParams{Slugs: chunk})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
clubs = append(clubs, t...)
|
|
}
|
|
log.Debug().Msgf("found %d clubs", len(clubs))
|
|
var nationalTeams []football.NationalTeam
|
|
slugsLeft := lo.Without(teamSlugs, lo.Map(clubs, func(club football.Club, index int) string {
|
|
return club.Slug
|
|
})...)
|
|
log.Debug().Msgf("getting national teams...")
|
|
log.Debug().Msgf("slugs left: %d", len(slugsLeft))
|
|
for i, chunk := range lo.Chunk(slugsLeft, 100) {
|
|
log.Debug().Msgf("\tbatch %d/%d", i+1, (len(teamSlugs)/100)+1)
|
|
t, err := u.s.Football.NationalTeams.Get(ctx, graphql.SlugsParams{Slugs: chunk})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
nationalTeams = append(nationalTeams, t...)
|
|
}
|
|
log.Debug().Msgf("found %d national teams", len(nationalTeams))
|
|
|
|
competitionSlugs := ExtractCompetitionSlugsFromPlayersGamesAndClubs(players, games, clubs)
|
|
log.Debug().Msgf("extracted %d unique competition slugs from players, games and clubs", len(competitionSlugs))
|
|
log.Debug().Msgf("getting competitions...")
|
|
var competitions []football.Competition
|
|
for _, slug := range competitionSlugs {
|
|
log.Debug().Msgf("\tcompetition %s", slug)
|
|
c, err := u.s.Football.Competition.Get(ctx, graphql.SlugParams{Slug: slug})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
competitions = append(competitions, c)
|
|
}
|
|
log.Debug().Msgf("found %d competitions", len(competitions))
|
|
|
|
countrySlugs := ExtractCountrySlugsFromPlayersCompetitionsClubsAndNationalTeams(
|
|
players,
|
|
competitions,
|
|
clubs,
|
|
nationalTeams,
|
|
)
|
|
log.Debug().
|
|
Msgf("extracted %d unique country slugs from players, competitions, clubs and national teams", len(countrySlugs))
|
|
log.Debug().Msgf("getting countries...")
|
|
var countries []sorare.Country
|
|
for i, chunk := range lo.Chunk(countrySlugs, 100) {
|
|
log.Debug().Msgf("\tbatch %d/%d", i+1, (len(countrySlugs)/100)+1)
|
|
c, err := u.s.Countries.Get(ctx, graphql.SlugsParams{Slugs: chunk})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
countries = append(countries, c...)
|
|
}
|
|
log.Debug().Msgf("found %d countries", len(countries))
|
|
|
|
log.Debug().Msg("inserting countries into db...")
|
|
cnt, err = u.db.CreateCountries(
|
|
ctx,
|
|
lo.Map(countries, func(country sorare.Country, index int) model.CreateCountriesParams {
|
|
return model.CreateCountriesParams{
|
|
Slug: country.Slug,
|
|
Code: country.Code,
|
|
DisplayName: country.Name,
|
|
ThreeLetterCode: country.ThreeLetterCode,
|
|
FlagFlat64Url: country.FlagFlat64Url,
|
|
FlagFlat32Url: country.FlagFlat32Url,
|
|
FlagRound64Url: country.FlagRound64Url,
|
|
FlagRound32Url: country.FlagRound32Url,
|
|
}
|
|
}),
|
|
)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
log.Debug().Msgf("%d countries inserted", cnt)
|
|
|
|
log.Debug().Msg("inserting competitions into db...")
|
|
|
|
var batcherr error
|
|
batch := u.db.CreateOrUpdateCompetitions(
|
|
ctx,
|
|
lo.Map(competitions, func(competition football.Competition, index int) model.CreateOrUpdateCompetitionsParams {
|
|
return model.CreateOrUpdateCompetitionsParams{
|
|
Slug: competition.Slug,
|
|
CompetitionFormat: competition.Format,
|
|
CompetitionType: competition.Type,
|
|
DisplayName: competition.DisplayName,
|
|
PictureUrl: competition.PictureUrl,
|
|
LogoUrl: competition.LogoUrl,
|
|
CountrySlug: competition.Country.Slug,
|
|
}
|
|
}),
|
|
)
|
|
batch.Exec(func(_ int, err error) {
|
|
if err != nil {
|
|
batcherr = err
|
|
batch.Close()
|
|
}
|
|
})
|
|
if batcherr != nil {
|
|
return errors.Wrap(batcherr, "inserting competitions")
|
|
}
|
|
|
|
log.Debug().Msgf("%d competitions inserted", len(competitions))
|
|
|
|
log.Debug().Msg("inserting teams into db...")
|
|
cnt, err = u.db.BatchInsertTeams(ctx, lo.Union(
|
|
lo.Map(clubs, func(club football.Club, index int) model.BatchInsertTeamsParams {
|
|
return model.BatchInsertTeamsParams{
|
|
Slug: club.Slug,
|
|
DisplayName: club.Name,
|
|
CountrySlug: club.Country.Slug,
|
|
DomesticLeagueSlug: func() *string {
|
|
if club.DomesticLeague.Slug == "" {
|
|
return nil
|
|
}
|
|
return &club.DomesticLeague.Slug
|
|
}(),
|
|
ShortName: club.ShortName,
|
|
PictureUrl: club.PictureUrl,
|
|
TeamType: "club",
|
|
}
|
|
}),
|
|
lo.Map(nationalTeams, func(nationalTeam football.NationalTeam, index int) model.BatchInsertTeamsParams {
|
|
return model.BatchInsertTeamsParams{
|
|
Slug: nationalTeam.Slug,
|
|
DisplayName: nationalTeam.Name,
|
|
CountrySlug: nationalTeam.Country.Slug,
|
|
DomesticLeagueSlug: nil,
|
|
ShortName: nationalTeam.ShortName,
|
|
PictureUrl: nationalTeam.PictureUrl,
|
|
TeamType: "national",
|
|
}
|
|
}),
|
|
))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
log.Debug().Msgf("%d teams inserted", cnt)
|
|
|
|
log.Debug().Msg("inserting games into db...")
|
|
cnt, err = u.db.BatchInsertGames(
|
|
ctx,
|
|
lo.Map(games, func(game football.Game, index int) model.BatchInsertGamesParams {
|
|
return NewBatchInsertGamesParamsFromSorare(game)
|
|
}),
|
|
)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
log.Debug().Msgf("%d games inserted", cnt)
|
|
|
|
log.Debug().Msg("inserting players into db...")
|
|
cnt, err = u.db.BatchInsertPlayers(
|
|
ctx,
|
|
lo.Map(players, func(player football.Player, index int) model.BatchInsertPlayersParams {
|
|
res := model.BatchInsertPlayersParams{
|
|
Slug: player.Slug,
|
|
DisplayName: player.DisplayName,
|
|
BirthDate: player.BirthDate,
|
|
CountrySlug: player.Country.Slug,
|
|
AvatarUrl: player.AvatarUrl,
|
|
FieldPosition: string(player.Position),
|
|
Status: string(player.PlayingStatus),
|
|
ShirtNumber: int32(player.ShirtNumber),
|
|
}
|
|
for _, competition := range player.ActiveClub.ActiveCompetitions {
|
|
if competition.Format == "DOMESTIC_LEAGUE" {
|
|
res.DomesticLeagueSlug = &competition.Slug
|
|
}
|
|
}
|
|
if player.ActiveClub.Slug != "" {
|
|
res.TeamSlug = &player.ActiveClub.Slug
|
|
}
|
|
return res
|
|
}),
|
|
)
|
|
if err != nil {
|
|
return errors.Wrap(err, "inserting players")
|
|
}
|
|
log.Debug().Msgf("%d players inserted", cnt)
|
|
|
|
log.Debug().Msg("inserting game players into db...")
|
|
cnt, err = u.db.BatchInsertGamePlayers(ctx, gamePlayers)
|
|
if err != nil {
|
|
return errors.Wrap(err, "inserting game players")
|
|
}
|
|
log.Debug().Msgf("%d game players inserted", cnt)
|
|
|
|
return nil
|
|
}
|
|
|
|
func ExtractPlayersFromGameWithFormation(
|
|
gameWithFormation football.GameWithFormation,
|
|
) []model.BatchInsertGamePlayersParams {
|
|
var res []model.BatchInsertGamePlayersParams
|
|
for _, p := range gameWithFormation.HomeFormation.Bench {
|
|
res = append(res, model.BatchInsertGamePlayersParams{
|
|
GameID: gameWithFormation.Id.Value,
|
|
PlayerSlug: p.Slug,
|
|
TeamSlug: gameWithFormation.HomeTeam.Team.Slug,
|
|
Status: "bench",
|
|
})
|
|
}
|
|
for _, p := range gameWithFormation.HomeFormation.StartingLineup {
|
|
for _, q := range p {
|
|
res = append(res, model.BatchInsertGamePlayersParams{
|
|
GameID: gameWithFormation.Id.Value,
|
|
PlayerSlug: q.Slug,
|
|
TeamSlug: gameWithFormation.HomeTeam.Team.Slug,
|
|
Status: "starting",
|
|
})
|
|
}
|
|
}
|
|
for _, p := range gameWithFormation.AwayFormation.Bench {
|
|
res = append(res, model.BatchInsertGamePlayersParams{
|
|
GameID: gameWithFormation.Id.Value,
|
|
PlayerSlug: p.Slug,
|
|
TeamSlug: gameWithFormation.AwayTeam.Team.Slug,
|
|
Status: "bench",
|
|
})
|
|
}
|
|
for _, p := range gameWithFormation.AwayFormation.StartingLineup {
|
|
for _, q := range p {
|
|
res = append(res, model.BatchInsertGamePlayersParams{
|
|
GameID: gameWithFormation.Id.Value,
|
|
PlayerSlug: q.Slug,
|
|
TeamSlug: gameWithFormation.AwayTeam.Team.Slug,
|
|
Status: "starting",
|
|
})
|
|
}
|
|
}
|
|
|
|
return res
|
|
|
|
}
|
|
|
|
func ExtractTeamSlugsFromPlayersAndGames(players []football.Player, games []football.Game) []string {
|
|
return lo.Uniq(lo.Union(
|
|
ExtractTeamSlugsFromPlayers(players),
|
|
ExtractTeamSlugsFromGames(games),
|
|
))
|
|
}
|
|
|
|
func ExtractTeamSlugsFromPlayers(players []football.Player) []string {
|
|
return lo.Uniq(lo.Filter(lo.Map(players, func(player football.Player, index int) string {
|
|
return player.ActiveClub.Slug
|
|
}), func(slug string, index int) bool {
|
|
return slug != ""
|
|
}))
|
|
}
|
|
|
|
func ExtractTeamSlugsFromGames(games []football.Game) []string {
|
|
var res []string
|
|
res = lo.Map(games, func(game football.Game, index int) string {
|
|
return game.AwayTeam.Team.Slug
|
|
})
|
|
res = append(res, lo.Map(games, func(game football.Game, index int) string {
|
|
return game.HomeTeam.Team.Slug
|
|
})...)
|
|
res = lo.Filter(res, func(slug string, index int) bool {
|
|
return slug != ""
|
|
})
|
|
return lo.Uniq(res)
|
|
}
|
|
|
|
func ExtractCountrySlugsFromCompetitions(competitions []football.Competition) []string {
|
|
return lo.Uniq(lo.Filter(lo.Map(competitions, func(competition football.Competition, index int) string {
|
|
return competition.Country.Slug
|
|
}), func(slug string, index int) bool {
|
|
return slug != ""
|
|
}))
|
|
}
|
|
|
|
func ExtractCountrySlugsFromTeams(teams []football.Team) []string {
|
|
return lo.Uniq(lo.Filter(lo.Map(teams, func(team football.Team, index int) string {
|
|
return team.Country.Slug
|
|
}), func(slug string, index int) bool {
|
|
return slug != ""
|
|
}))
|
|
}
|
|
|
|
func ExtractCountrySlugsFromPlayersCompetitionsClubsAndNationalTeams(
|
|
players []football.Player,
|
|
competitions []football.Competition,
|
|
clubs []football.Club,
|
|
nationalTeams []football.NationalTeam,
|
|
) []string {
|
|
return lo.Uniq(lo.Union(
|
|
ExtractCountrySlugsFromPlayers(players),
|
|
ExtractCountrySlugsFromCompetitions(competitions),
|
|
ExtractCountrySlugsFromTeams(lo.Map(clubs, func(club football.Club, index int) football.Team {
|
|
return club.Team
|
|
})),
|
|
ExtractCountrySlugsFromTeams(
|
|
lo.Map(nationalTeams, func(nationalTeam football.NationalTeam, index int) football.Team {
|
|
return nationalTeam.Team
|
|
}),
|
|
),
|
|
))
|
|
}
|
|
|
|
func ExtractCountrySlugsFromPlayers(players []football.Player) []string {
|
|
return lo.Uniq(lo.Filter(lo.Map(players, func(player football.Player, index int) string {
|
|
return player.Country.Slug
|
|
}), func(slug string, index int) bool {
|
|
return slug != ""
|
|
}))
|
|
|
|
}
|
|
|
|
func ExtractCompetitionSlugsFromPlayers(players []football.Player) []string {
|
|
return lo.Uniq(lo.Filter(lo.Map(players, func(player football.Player, index int) string {
|
|
var res string
|
|
for _, competition := range player.ActiveClub.ActiveCompetitions {
|
|
if competition.Format == "DOMESTIC_LEAGUE" {
|
|
res = competition.Slug
|
|
}
|
|
}
|
|
return res
|
|
}), func(slug string, index int) bool {
|
|
return slug != ""
|
|
}))
|
|
}
|
|
|
|
func ExtractCompetitionSlugsFromGames(games []football.Game) []string {
|
|
return lo.Uniq(lo.Filter(lo.Map(games, func(game football.Game, index int) string {
|
|
return game.Competition.Slug
|
|
}), func(slug string, index int) bool {
|
|
return slug != ""
|
|
}))
|
|
}
|
|
|
|
func ExtractCompetitionSlugsFromClubs(clubs []football.Club) []string {
|
|
return lo.Uniq(lo.Filter(lo.Map(clubs, func(club football.Club, index int) string {
|
|
return club.DomesticLeague.Slug
|
|
}), func(slug string, index int) bool {
|
|
return slug != ""
|
|
}))
|
|
}
|
|
|
|
func ExtractCompetitionSlugsFromPlayersGamesAndClubs(
|
|
players []football.Player,
|
|
games []football.Game,
|
|
clubs []football.Club,
|
|
) []string {
|
|
return lo.Uniq(lo.Union(
|
|
ExtractCompetitionSlugsFromPlayers(players),
|
|
ExtractCompetitionSlugsFromGames(games),
|
|
ExtractCompetitionSlugsFromClubs(clubs),
|
|
))
|
|
}
|