From 3affe1f2e300a1b34aad5fb7931790a08f7080a1 Mon Sep 17 00:00:00 2001 From: Laurent Le Houerou Date: Fri, 22 Mar 2024 11:40:34 +0000 Subject: [PATCH] managing initial game players sync. changing competition creation to batchexec. adding player initial sync --- db/migrations/00001_init.sql | 4 +- model/batch.go | 93 ++++++++++ model/competition.sql.go | 54 ------ model/copyfrom.go | 68 ++++++-- model/db.go | 1 + model/game_player.sql.go | 50 ++++++ model/models.go | 4 +- model/player.sql.go | 23 +++ model/sql/competition.sql | 6 +- model/sql/game_player.sql | 27 +++ model/sql/player.sql | 3 + sorare_utils/fixture.go | 71 -------- sorare_utils/game.go | 37 ++++ sorare_utils/game_player.go | 1 + sorare_utils/update_service.go | 301 ++++++++++++++++++++++++++++----- 15 files changed, 553 insertions(+), 190 deletions(-) create mode 100644 model/batch.go create mode 100644 model/game_player.sql.go create mode 100644 model/player.sql.go create mode 100644 model/sql/game_player.sql create mode 100644 model/sql/player.sql create mode 100644 sorare_utils/game.go create mode 100644 sorare_utils/game_player.go diff --git a/db/migrations/00001_init.sql b/db/migrations/00001_init.sql index 267ed85..884d3c9 100644 --- a/db/migrations/00001_init.sql +++ b/db/migrations/00001_init.sql @@ -87,9 +87,9 @@ birth_date DATE NOT NULL, country_slug TEXT NOT NULL, FOREIGN KEY (country_slug) REFERENCES countries (slug), - team_slug TEXT NOT NULL, + team_slug TEXT, FOREIGN KEY (team_slug) REFERENCES teams (slug), - domestic_league_slug TEXT NOT NULL, + domestic_league_slug TEXT, FOREIGN KEY (domestic_league_slug) REFERENCES competitions (slug), avatar_url TEXT NOT NULL, field_position TEXT NOT NULL, diff --git a/model/batch.go b/model/batch.go new file mode 100644 index 0000000..e36144f --- /dev/null +++ b/model/batch.go @@ -0,0 +1,93 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.25.0 +// source: batch.go + +package model + +import ( + "context" + "errors" + + "github.com/jackc/pgx/v5" +) + +var ( + ErrBatchAlreadyClosed = errors.New("batch already closed") +) + +const createOrUpdateCompetitions = `-- name: CreateOrUpdateCompetitions :batchexec + INSERT INTO competitions ( + slug, + display_name, + country_slug, + competition_format, + competition_type, + picture_url, + logo_url + ) + VALUES ($1, $2, $3, $4, $5, $6, $7) ON CONFLICT (slug) + DO + UPDATE + SET display_name = EXCLUDED.display_name, + competition_format = EXCLUDED.competition_format, + competition_type = EXCLUDED.competition_type, + picture_url = EXCLUDED.picture_url, + logo_url = EXCLUDED.logo_url, + country_slug = EXCLUDED.country_slug +` + +type CreateOrUpdateCompetitionsBatchResults struct { + br pgx.BatchResults + tot int + closed bool +} + +type CreateOrUpdateCompetitionsParams struct { + Slug string + DisplayName string + CountrySlug string + CompetitionFormat string + CompetitionType string + PictureUrl string + LogoUrl string +} + +func (q *Queries) CreateOrUpdateCompetitions(ctx context.Context, arg []CreateOrUpdateCompetitionsParams) *CreateOrUpdateCompetitionsBatchResults { + batch := &pgx.Batch{} + for _, a := range arg { + vals := []interface{}{ + a.Slug, + a.DisplayName, + a.CountrySlug, + a.CompetitionFormat, + a.CompetitionType, + a.PictureUrl, + a.LogoUrl, + } + batch.Queue(createOrUpdateCompetitions, vals...) + } + br := q.db.SendBatch(ctx, batch) + return &CreateOrUpdateCompetitionsBatchResults{br, len(arg), false} +} + +func (b *CreateOrUpdateCompetitionsBatchResults) Exec(f func(int, error)) { + defer b.br.Close() + for t := 0; t < b.tot; t++ { + if b.closed { + if f != nil { + f(t, ErrBatchAlreadyClosed) + } + continue + } + _, err := b.br.Exec() + if f != nil { + f(t, err) + } + } +} + +func (b *CreateOrUpdateCompetitionsBatchResults) Close() error { + b.closed = true + return b.br.Close() +} diff --git a/model/competition.sql.go b/model/competition.sql.go index 5595951..a8d3797 100644 --- a/model/competition.sql.go +++ b/model/competition.sql.go @@ -9,60 +9,6 @@ import ( "context" ) -type BatchInsertCompetitionsParams struct { - Slug string - DisplayName string - CountrySlug string - CompetitionFormat string - CompetitionType string - PictureUrl string - LogoUrl string -} - -const createOrUpdateCompetition = `-- name: CreateOrUpdateCompetition :exec - INSERT INTO competitions ( - slug, - display_name, - country_slug, - competition_format, - competition_type, - picture_url, - logo_url - ) - VALUES ($1, $2, $3, $4, $5, $6, $7) ON CONFLICT (slug) - DO - UPDATE - SET display_name = EXCLUDED.display_name, - competition_format = EXCLUDED.competition_format, - competition_type = EXCLUDED.competition_type, - picture_url = EXCLUDED.picture_url, - logo_url = EXCLUDED.logo_url, - country_slug = EXCLUDED.country_slug -` - -type CreateOrUpdateCompetitionParams struct { - Slug string - DisplayName string - CountrySlug string - CompetitionFormat string - CompetitionType string - PictureUrl string - LogoUrl string -} - -func (q *Queries) CreateOrUpdateCompetition(ctx context.Context, arg CreateOrUpdateCompetitionParams) error { - _, err := q.db.Exec(ctx, createOrUpdateCompetition, - arg.Slug, - arg.DisplayName, - arg.CountrySlug, - arg.CompetitionFormat, - arg.CompetitionType, - arg.PictureUrl, - arg.LogoUrl, - ) - return err -} - const getAllCompetitions = `-- name: GetAllCompetitions :many SELECT competitions.slug, competitions.display_name, competitions.country_slug, competitions.competition_format, competitions.competition_type, competitions.picture_url, competitions.logo_url, competitions.zone_id, zones.id, zones.display_name, diff --git a/model/copyfrom.go b/model/copyfrom.go index 29aca41..1e12cc0 100644 --- a/model/copyfrom.go +++ b/model/copyfrom.go @@ -9,13 +9,13 @@ import ( "context" ) -// iteratorForBatchInsertCompetitions implements pgx.CopyFromSource. -type iteratorForBatchInsertCompetitions struct { - rows []BatchInsertCompetitionsParams +// iteratorForBatchInsertGamePlayers implements pgx.CopyFromSource. +type iteratorForBatchInsertGamePlayers struct { + rows []BatchInsertGamePlayersParams skippedFirstNextCall bool } -func (r *iteratorForBatchInsertCompetitions) Next() bool { +func (r *iteratorForBatchInsertGamePlayers) Next() bool { if len(r.rows) == 0 { return false } @@ -27,24 +27,21 @@ func (r *iteratorForBatchInsertCompetitions) Next() bool { return len(r.rows) > 0 } -func (r iteratorForBatchInsertCompetitions) Values() ([]interface{}, error) { +func (r iteratorForBatchInsertGamePlayers) Values() ([]interface{}, error) { return []interface{}{ - r.rows[0].Slug, - r.rows[0].DisplayName, - r.rows[0].CountrySlug, - r.rows[0].CompetitionFormat, - r.rows[0].CompetitionType, - r.rows[0].PictureUrl, - r.rows[0].LogoUrl, + r.rows[0].GameID, + r.rows[0].PlayerSlug, + r.rows[0].Status, + r.rows[0].TeamSlug, }, nil } -func (r iteratorForBatchInsertCompetitions) Err() error { +func (r iteratorForBatchInsertGamePlayers) Err() error { return nil } -func (q *Queries) BatchInsertCompetitions(ctx context.Context, arg []BatchInsertCompetitionsParams) (int64, error) { - return q.db.CopyFrom(ctx, []string{"competitions"}, []string{"slug", "display_name", "country_slug", "competition_format", "competition_type", "picture_url", "logo_url"}, &iteratorForBatchInsertCompetitions{rows: arg}) +func (q *Queries) BatchInsertGamePlayers(ctx context.Context, arg []BatchInsertGamePlayersParams) (int64, error) { + return q.db.CopyFrom(ctx, []string{"game_players"}, []string{"game_id", "player_slug", "status", "team_slug"}, &iteratorForBatchInsertGamePlayers{rows: arg}) } // iteratorForBatchInsertGames implements pgx.CopyFromSource. @@ -97,6 +94,47 @@ func (q *Queries) BatchInsertGames(ctx context.Context, arg []BatchInsertGamesPa return q.db.CopyFrom(ctx, []string{"games"}, []string{"id", "date", "coverage_status", "low_coverage", "minutes", "period_type", "scored", "status", "competition_slug", "fixture_slug", "away_team_slug", "away_goals", "away_extra_time_score", "away_penalty_score", "home_team_slug", "home_goals", "home_extra_time_score", "home_penalty_score", "winner_team_slug"}, &iteratorForBatchInsertGames{rows: arg}) } +// iteratorForBatchInsertPlayers implements pgx.CopyFromSource. +type iteratorForBatchInsertPlayers struct { + rows []BatchInsertPlayersParams + skippedFirstNextCall bool +} + +func (r *iteratorForBatchInsertPlayers) Next() bool { + if len(r.rows) == 0 { + return false + } + if !r.skippedFirstNextCall { + r.skippedFirstNextCall = true + return true + } + r.rows = r.rows[1:] + return len(r.rows) > 0 +} + +func (r iteratorForBatchInsertPlayers) Values() ([]interface{}, error) { + return []interface{}{ + r.rows[0].Slug, + r.rows[0].DisplayName, + r.rows[0].BirthDate, + r.rows[0].CountrySlug, + r.rows[0].TeamSlug, + r.rows[0].DomesticLeagueSlug, + r.rows[0].AvatarUrl, + r.rows[0].FieldPosition, + r.rows[0].Status, + r.rows[0].ShirtNumber, + }, nil +} + +func (r iteratorForBatchInsertPlayers) Err() error { + return nil +} + +func (q *Queries) BatchInsertPlayers(ctx context.Context, arg []BatchInsertPlayersParams) (int64, error) { + return q.db.CopyFrom(ctx, []string{"players"}, []string{"slug", "display_name", "birth_date", "country_slug", "team_slug", "domestic_league_slug", "avatar_url", "field_position", "status", "shirt_number"}, &iteratorForBatchInsertPlayers{rows: arg}) +} + // iteratorForBatchInsertTeams implements pgx.CopyFromSource. type iteratorForBatchInsertTeams struct { rows []BatchInsertTeamsParams diff --git a/model/db.go b/model/db.go index 38ffa40..cca127f 100644 --- a/model/db.go +++ b/model/db.go @@ -16,6 +16,7 @@ type DBTX interface { Query(context.Context, string, ...interface{}) (pgx.Rows, error) QueryRow(context.Context, string, ...interface{}) pgx.Row CopyFrom(ctx context.Context, tableName pgx.Identifier, columnNames []string, rowSrc pgx.CopyFromSource) (int64, error) + SendBatch(context.Context, *pgx.Batch) pgx.BatchResults } func New(db DBTX) *Queries { diff --git a/model/game_player.sql.go b/model/game_player.sql.go new file mode 100644 index 0000000..c9be146 --- /dev/null +++ b/model/game_player.sql.go @@ -0,0 +1,50 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.25.0 +// source: game_player.sql + +package model + +import ( + "context" +) + +type BatchInsertGamePlayersParams struct { + GameID string + PlayerSlug string + Status string + TeamSlug string +} + +const createOrUpdateGamePlayer = `-- name: CreateOrUpdateGamePlayer :exec +INSERT INTO game_players( + game_id, + player_slug, + status, + team_slug) +VALUES( + $1, + $2, + $3, + $4) + ON CONFLICT (game_id, player_slug) DO UPDATE + SET status = $3, + team_slug = $4 +` + +type CreateOrUpdateGamePlayerParams struct { + GameID string + PlayerSlug string + Status string + TeamSlug string +} + +func (q *Queries) CreateOrUpdateGamePlayer(ctx context.Context, arg CreateOrUpdateGamePlayerParams) error { + _, err := q.db.Exec(ctx, createOrUpdateGamePlayer, + arg.GameID, + arg.PlayerSlug, + arg.Status, + arg.TeamSlug, + ) + return err +} diff --git a/model/models.go b/model/models.go index 61dfd7f..5f58680 100644 --- a/model/models.go +++ b/model/models.go @@ -140,8 +140,8 @@ type Player struct { DisplayName string BirthDate time.Time CountrySlug string - TeamSlug string - DomesticLeagueSlug string + TeamSlug *string + DomesticLeagueSlug *string AvatarUrl string FieldPosition string Status string diff --git a/model/player.sql.go b/model/player.sql.go new file mode 100644 index 0000000..6654c8d --- /dev/null +++ b/model/player.sql.go @@ -0,0 +1,23 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.25.0 +// source: player.sql + +package model + +import ( + "time" +) + +type BatchInsertPlayersParams struct { + Slug string + DisplayName string + BirthDate time.Time + CountrySlug string + TeamSlug *string + DomesticLeagueSlug *string + AvatarUrl string + FieldPosition string + Status string + ShirtNumber int32 +} diff --git a/model/sql/competition.sql b/model/sql/competition.sql index ea3e0f3..fad7167 100644 --- a/model/sql/competition.sql +++ b/model/sql/competition.sql @@ -1,8 +1,4 @@ --- name: BatchInsertCompetitions :copyfrom - INSERT INTO competitions (slug, display_name, country_slug, competition_format, competition_type, picture_url, logo_url) - VALUES ($1, $2, $3, $4, $5, $6, $7); - --- name: CreateOrUpdateCompetition :exec +-- name: CreateOrUpdateCompetitions :batchexec INSERT INTO competitions ( slug, display_name, diff --git a/model/sql/game_player.sql b/model/sql/game_player.sql new file mode 100644 index 0000000..9c1b5b9 --- /dev/null +++ b/model/sql/game_player.sql @@ -0,0 +1,27 @@ +-- name: BatchInsertGamePlayers :copyfrom +INSERT INTO game_players( + game_id, + player_slug, + status, + team_slug) +VALUES( + $1, + $2, + $3, + $4); + +-- name: CreateOrUpdateGamePlayer :exec +INSERT INTO game_players( + game_id, + player_slug, + status, + team_slug) +VALUES( + $1, + $2, + $3, + $4) + ON CONFLICT (game_id, player_slug) DO UPDATE + SET status = $3, + team_slug = $4; + diff --git a/model/sql/player.sql b/model/sql/player.sql new file mode 100644 index 0000000..405b212 --- /dev/null +++ b/model/sql/player.sql @@ -0,0 +1,3 @@ +-- name: BatchInsertPlayers :copyfrom +INSERT INTO players (slug, display_name, birth_date, country_slug, team_slug, domestic_league_slug, avatar_url, field_position, status, shirt_number) +VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10); diff --git a/sorare_utils/fixture.go b/sorare_utils/fixture.go index 2b7fcfd..dbe2ba1 100644 --- a/sorare_utils/fixture.go +++ b/sorare_utils/fixture.go @@ -37,74 +37,3 @@ func GetGamesFromFixtures( log.Debug().Msgf("found total of %d games after filtering duplicate ids", len(games)) return games, nil } - -func ExtractTeamSlugsFromGames(games []football.Game) []string { - var res []string - res = lo.Map(games, func(game football.Game, index int) string { - return game.AwayTeam.Team.Slug - }) - res = append(res, lo.Map(games, func(game football.Game, index int) string { - return game.HomeTeam.Team.Slug - })...) - res = lo.Filter(res, func(slug string, index int) bool { - return slug != "" - }) - return lo.Uniq(res) -} - -func ExtractCountrySlugsFromCompetitions(competitions []football.Competition) []string { - return lo.Uniq(lo.Filter(lo.Map(competitions, func(competition football.Competition, index int) string { - return competition.Country.Slug - }), func(slug string, index int) bool { - return slug != "" - })) -} - -func ExtractCountrySlugsFromTeams(teams []football.Team) []string { - return lo.Uniq(lo.Filter(lo.Map(teams, func(team football.Team, index int) string { - return team.Country.Slug - }), func(slug string, index int) bool { - return slug != "" - })) -} - -func ExtractCountrySlugsFromCompetitionsClubsAndNationalTeams( - competitions []football.Competition, - clubs []football.Club, - nationalTeams []football.NationalTeam, -) []string { - return lo.Uniq(lo.Union( - ExtractCountrySlugsFromCompetitions(competitions), - ExtractCountrySlugsFromTeams(lo.Map(clubs, func(club football.Club, index int) football.Team { - return club.Team - })), - ExtractCountrySlugsFromTeams( - lo.Map(nationalTeams, func(nationalTeam football.NationalTeam, index int) football.Team { - return nationalTeam.Team - }), - ), - )) -} - -func ExtractCompetitionSlugsFromGames(games []football.Game) []string { - return lo.Uniq(lo.Filter(lo.Map(games, func(game football.Game, index int) string { - return game.Competition.Slug - }), func(slug string, index int) bool { - return slug != "" - })) -} - -func ExtractCompetitionSlugsFromClubs(clubs []football.Club) []string { - return lo.Uniq(lo.Filter(lo.Map(clubs, func(club football.Club, index int) string { - return club.DomesticLeague.Slug - }), func(slug string, index int) bool { - return slug != "" - })) -} - -func ExtractCompetitionSlugsFromGamesAndClubs(games []football.Game, clubs []football.Club) []string { - return lo.Uniq(lo.Union( - ExtractCompetitionSlugsFromGames(games), - ExtractCompetitionSlugsFromClubs(clubs), - )) -} diff --git a/sorare_utils/game.go b/sorare_utils/game.go new file mode 100644 index 0000000..d6da8c6 --- /dev/null +++ b/sorare_utils/game.go @@ -0,0 +1,37 @@ +package sorare_utils + +import ( + "git.lehouerou.net/laurent/sorare/football" + "github.com/jackc/pgx/v5/pgtype" + + "git.lehouerou.net/laurent/sorarebuddy/model" +) + +func NewBatchInsertGamesParamsFromSorare(game football.Game) model.BatchInsertGamesParams { + return model.BatchInsertGamesParams{ + ID: game.Id.Value, + Date: pgtype.Timestamptz{Time: game.Date, Valid: true}, + CoverageStatus: game.CoverageStatus, + LowCoverage: game.LowCoverage, + Minutes: int32(game.Minute), + PeriodType: game.PeriodType, + Scored: game.Scored, + Status: game.Status, + CompetitionSlug: game.Competition.Slug, + FixtureSlug: game.So5Fixture.Slug, + AwayTeamSlug: game.AwayTeam.Team.Slug, + AwayGoals: int32(game.AwayGoals), + AwayExtraTimeScore: int32(game.ExtraTimeScoreAway), + AwayPenaltyScore: int32(game.PenaltyScoreAway), + HomeTeamSlug: game.HomeTeam.Team.Slug, + HomeGoals: int32(game.HomeGoals), + HomeExtraTimeScore: int32(game.ExtraTimeScoreHome), + HomePenaltyScore: int32(game.PenaltyScoreHome), + WinnerTeamSlug: func() *string { + if game.Winner.Team.Slug == "" { + return nil + } + return &game.Winner.Team.Slug + }(), + } +} diff --git a/sorare_utils/game_player.go b/sorare_utils/game_player.go new file mode 100644 index 0000000..e410f01 --- /dev/null +++ b/sorare_utils/game_player.go @@ -0,0 +1 @@ +package sorare_utils diff --git a/sorare_utils/update_service.go b/sorare_utils/update_service.go index 1214a9d..0471028 100644 --- a/sorare_utils/update_service.go +++ b/sorare_utils/update_service.go @@ -7,6 +7,8 @@ import ( "git.lehouerou.net/laurent/sorare/football" "git.lehouerou.net/laurent/sorare/graphql" "github.com/jackc/pgx/v5/pgtype" + gql "github.com/llehouerou/go-graphql-client" + "github.com/pkg/errors" "github.com/rs/zerolog/log" "github.com/samber/lo" @@ -24,7 +26,9 @@ func NewUpdateService(s *sorare.Sorare, db *model.Queries) *UpdateService { func (u *UpdateService) InitSyncDatabase(ctx context.Context) error { - sfixtures, err := u.s.Football.So5.So5Fixtures.Get(ctx, football.So5FixturesParams{}) + sfixtures, err := u.s.Football.So5.So5Fixtures.Get(ctx, football.So5FixturesParams{ + AasmStates: []string{"started"}, + }) if err != nil { return err } @@ -62,13 +66,45 @@ func (u *UpdateService) InitSyncDatabase(ctx context.Context) error { ) log.Info().Msgf("found %d games to process", len(games)) - teamSlugs := ExtractTeamSlugsFromGames(games) - log.Debug().Msgf("extracted %d unique team slugs from games", len(teamSlugs)) + log.Debug().Msgf("getting players for each game...") + var gamePlayers []model.BatchInsertGamePlayersParams + for _, game := range games { + gameWithFormation, err := u.s.Football.Game.Get(ctx, graphql.IdParams{Id: gql.ID(game.Id.Value)}) + if err != nil { + return errors.Wrapf(err, "getting game with formation %s", game.Id.Value) + } + newplayers := ExtractPlayersFromGameWithFormation(gameWithFormation) + log.Debug().Msgf("\t%s -> %d players", game.String(), len(newplayers)) + gamePlayers = append(gamePlayers, newplayers...) + } + + playerSlugs := lo.Uniq( + lo.Filter(lo.Map(gamePlayers, func(player model.BatchInsertGamePlayersParams, index int) string { + return player.PlayerSlug + }), func(slug string, index int) bool { + return slug != "" + }), + ) + + log.Debug().Msgf("getting players...") + var players []football.Player + for i, chunk := range lo.Chunk(playerSlugs, 80) { + log.Debug().Msgf("\tbatch %d/%d", i+1, (len(playerSlugs)/80)+1) + p, err := u.s.Football.Players.Get(ctx, graphql.SlugsParams{Slugs: chunk}) + if err != nil { + return errors.Wrapf(err, "getting players batch %d", i) + } + players = append(players, p...) + } + log.Debug().Msgf("found %d players", len(players)) + + teamSlugs := ExtractTeamSlugsFromPlayersAndGames(players, games) + log.Debug().Msgf("extracted %d unique team slugs from games and players", len(teamSlugs)) log.Debug().Msgf("getting clubs...") var clubs []football.Club for i, chunk := range lo.Chunk(teamSlugs, 100) { - log.Debug().Msgf("\tbatch %d/%d", i+1, len(teamSlugs)/100) + log.Debug().Msgf("\tbatch %d/%d", i+1, (len(teamSlugs)/100)+1) t, err := u.s.Football.Clubs.Get(ctx, graphql.SlugsParams{Slugs: chunk}) if err != nil { return err @@ -83,7 +119,7 @@ func (u *UpdateService) InitSyncDatabase(ctx context.Context) error { log.Debug().Msgf("getting national teams...") log.Debug().Msgf("slugs left: %d", len(slugsLeft)) for i, chunk := range lo.Chunk(slugsLeft, 100) { - log.Debug().Msgf("\tbatch %d/%d", i+1, len(teamSlugs)/100) + log.Debug().Msgf("\tbatch %d/%d", i+1, (len(teamSlugs)/100)+1) t, err := u.s.Football.NationalTeams.Get(ctx, graphql.SlugsParams{Slugs: chunk}) if err != nil { return err @@ -92,8 +128,8 @@ func (u *UpdateService) InitSyncDatabase(ctx context.Context) error { } log.Debug().Msgf("found %d national teams", len(nationalTeams)) - competitionSlugs := ExtractCompetitionSlugsFromGamesAndClubs(games, clubs) - log.Debug().Msgf("extracted %d unique competition slugs from games and clubs", len(competitionSlugs)) + competitionSlugs := ExtractCompetitionSlugsFromPlayersGamesAndClubs(players, games, clubs) + log.Debug().Msgf("extracted %d unique competition slugs from players, games and clubs", len(competitionSlugs)) log.Debug().Msgf("getting competitions...") var competitions []football.Competition for _, slug := range competitionSlugs { @@ -106,16 +142,18 @@ func (u *UpdateService) InitSyncDatabase(ctx context.Context) error { } log.Debug().Msgf("found %d competitions", len(competitions)) - countrySlugs := ExtractCountrySlugsFromCompetitionsClubsAndNationalTeams( + countrySlugs := ExtractCountrySlugsFromPlayersCompetitionsClubsAndNationalTeams( + players, competitions, clubs, nationalTeams, ) - log.Debug().Msgf("extracted %d unique country slugs from competitions, clubs and national teams", len(countrySlugs)) + log.Debug(). + Msgf("extracted %d unique country slugs from players, competitions, clubs and national teams", len(countrySlugs)) log.Debug().Msgf("getting countries...") var countries []sorare.Country for i, chunk := range lo.Chunk(countrySlugs, 100) { - log.Debug().Msgf("\tbatch %d/%d", i+1, len(countrySlugs)/100) + log.Debug().Msgf("\tbatch %d/%d", i+1, (len(countrySlugs)/100)+1) c, err := u.s.Countries.Get(ctx, graphql.SlugsParams{Slugs: chunk}) if err != nil { return err @@ -147,10 +185,11 @@ func (u *UpdateService) InitSyncDatabase(ctx context.Context) error { log.Debug().Msg("inserting competitions into db...") - _, err = u.db.BatchInsertCompetitions( + var batcherr error + batch := u.db.CreateOrUpdateCompetitions( ctx, - lo.Map(competitions, func(competition football.Competition, index int) model.BatchInsertCompetitionsParams { - return model.BatchInsertCompetitionsParams{ + lo.Map(competitions, func(competition football.Competition, index int) model.CreateOrUpdateCompetitionsParams { + return model.CreateOrUpdateCompetitionsParams{ Slug: competition.Slug, CompetitionFormat: competition.Format, CompetitionType: competition.Type, @@ -161,8 +200,14 @@ func (u *UpdateService) InitSyncDatabase(ctx context.Context) error { } }), ) - if err != nil { - return err + batch.Exec(func(_ int, err error) { + if err != nil { + batcherr = err + batch.Close() + } + }) + if batcherr != nil { + return errors.Wrap(batcherr, "inserting competitions") } log.Debug().Msgf("%d competitions inserted", len(competitions)) @@ -206,37 +251,211 @@ func (u *UpdateService) InitSyncDatabase(ctx context.Context) error { cnt, err = u.db.BatchInsertGames( ctx, lo.Map(games, func(game football.Game, index int) model.BatchInsertGamesParams { - return model.BatchInsertGamesParams{ - ID: game.Id.Value, - Date: pgtype.Timestamptz{Time: game.Date, Valid: true}, - CoverageStatus: game.CoverageStatus, - LowCoverage: game.LowCoverage, - Minutes: int32(game.Minute), - PeriodType: game.PeriodType, - Scored: game.Scored, - Status: game.Status, - CompetitionSlug: game.Competition.Slug, - FixtureSlug: game.So5Fixture.Slug, - AwayTeamSlug: game.AwayTeam.Team.Slug, - AwayGoals: int32(game.AwayGoals), - AwayExtraTimeScore: int32(game.ExtraTimeScoreAway), - AwayPenaltyScore: int32(game.PenaltyScoreAway), - HomeTeamSlug: game.HomeTeam.Team.Slug, - HomeGoals: int32(game.HomeGoals), - HomeExtraTimeScore: int32(game.ExtraTimeScoreHome), - HomePenaltyScore: int32(game.PenaltyScoreHome), - WinnerTeamSlug: func() *string { - if game.Winner.Team.Slug == "" { - return nil - } - return &game.Winner.Team.Slug - }(), - } + return NewBatchInsertGamesParamsFromSorare(game) }), ) if err != nil { return err } log.Debug().Msgf("%d games inserted", cnt) + + log.Debug().Msg("inserting players into db...") + cnt, err = u.db.BatchInsertPlayers( + ctx, + lo.Map(players, func(player football.Player, index int) model.BatchInsertPlayersParams { + res := model.BatchInsertPlayersParams{ + Slug: player.Slug, + DisplayName: player.DisplayName, + BirthDate: player.BirthDate, + CountrySlug: player.Country.Slug, + AvatarUrl: player.AvatarUrl, + FieldPosition: string(player.Position), + Status: string(player.PlayingStatus), + ShirtNumber: int32(player.ShirtNumber), + } + for _, competition := range player.ActiveClub.ActiveCompetitions { + if competition.Format == "DOMESTIC_LEAGUE" { + res.DomesticLeagueSlug = &competition.Slug + } + } + if player.ActiveClub.Slug != "" { + res.TeamSlug = &player.ActiveClub.Slug + } + return res + }), + ) + if err != nil { + return errors.Wrap(err, "inserting players") + } + log.Debug().Msgf("%d players inserted", cnt) + + log.Debug().Msg("inserting game players into db...") + cnt, err = u.db.BatchInsertGamePlayers(ctx, gamePlayers) + if err != nil { + return errors.Wrap(err, "inserting game players") + } + log.Debug().Msgf("%d game players inserted", cnt) + return nil } + +func ExtractPlayersFromGameWithFormation( + gameWithFormation football.GameWithFormation, +) []model.BatchInsertGamePlayersParams { + var res []model.BatchInsertGamePlayersParams + for _, p := range gameWithFormation.HomeFormation.Bench { + res = append(res, model.BatchInsertGamePlayersParams{ + GameID: gameWithFormation.Id.Value, + PlayerSlug: p.Slug, + TeamSlug: gameWithFormation.HomeTeam.Team.Slug, + Status: "bench", + }) + } + for _, p := range gameWithFormation.HomeFormation.StartingLineup { + for _, q := range p { + res = append(res, model.BatchInsertGamePlayersParams{ + GameID: gameWithFormation.Id.Value, + PlayerSlug: q.Slug, + TeamSlug: gameWithFormation.HomeTeam.Team.Slug, + Status: "starting", + }) + } + } + for _, p := range gameWithFormation.AwayFormation.Bench { + res = append(res, model.BatchInsertGamePlayersParams{ + GameID: gameWithFormation.Id.Value, + PlayerSlug: p.Slug, + TeamSlug: gameWithFormation.AwayTeam.Team.Slug, + Status: "bench", + }) + } + for _, p := range gameWithFormation.AwayFormation.StartingLineup { + for _, q := range p { + res = append(res, model.BatchInsertGamePlayersParams{ + GameID: gameWithFormation.Id.Value, + PlayerSlug: q.Slug, + TeamSlug: gameWithFormation.AwayTeam.Team.Slug, + Status: "starting", + }) + } + } + + return res + +} + +func ExtractTeamSlugsFromPlayersAndGames(players []football.Player, games []football.Game) []string { + return lo.Uniq(lo.Union( + ExtractTeamSlugsFromPlayers(players), + ExtractTeamSlugsFromGames(games), + )) +} + +func ExtractTeamSlugsFromPlayers(players []football.Player) []string { + return lo.Uniq(lo.Filter(lo.Map(players, func(player football.Player, index int) string { + return player.ActiveClub.Slug + }), func(slug string, index int) bool { + return slug != "" + })) +} + +func ExtractTeamSlugsFromGames(games []football.Game) []string { + var res []string + res = lo.Map(games, func(game football.Game, index int) string { + return game.AwayTeam.Team.Slug + }) + res = append(res, lo.Map(games, func(game football.Game, index int) string { + return game.HomeTeam.Team.Slug + })...) + res = lo.Filter(res, func(slug string, index int) bool { + return slug != "" + }) + return lo.Uniq(res) +} + +func ExtractCountrySlugsFromCompetitions(competitions []football.Competition) []string { + return lo.Uniq(lo.Filter(lo.Map(competitions, func(competition football.Competition, index int) string { + return competition.Country.Slug + }), func(slug string, index int) bool { + return slug != "" + })) +} + +func ExtractCountrySlugsFromTeams(teams []football.Team) []string { + return lo.Uniq(lo.Filter(lo.Map(teams, func(team football.Team, index int) string { + return team.Country.Slug + }), func(slug string, index int) bool { + return slug != "" + })) +} + +func ExtractCountrySlugsFromPlayersCompetitionsClubsAndNationalTeams( + players []football.Player, + competitions []football.Competition, + clubs []football.Club, + nationalTeams []football.NationalTeam, +) []string { + return lo.Uniq(lo.Union( + ExtractCountrySlugsFromPlayers(players), + ExtractCountrySlugsFromCompetitions(competitions), + ExtractCountrySlugsFromTeams(lo.Map(clubs, func(club football.Club, index int) football.Team { + return club.Team + })), + ExtractCountrySlugsFromTeams( + lo.Map(nationalTeams, func(nationalTeam football.NationalTeam, index int) football.Team { + return nationalTeam.Team + }), + ), + )) +} + +func ExtractCountrySlugsFromPlayers(players []football.Player) []string { + return lo.Uniq(lo.Filter(lo.Map(players, func(player football.Player, index int) string { + return player.Country.Slug + }), func(slug string, index int) bool { + return slug != "" + })) + +} + +func ExtractCompetitionSlugsFromPlayers(players []football.Player) []string { + return lo.Uniq(lo.Filter(lo.Map(players, func(player football.Player, index int) string { + var res string + for _, competition := range player.ActiveClub.ActiveCompetitions { + if competition.Format == "DOMESTIC_LEAGUE" { + res = competition.Slug + } + } + return res + }), func(slug string, index int) bool { + return slug != "" + })) +} + +func ExtractCompetitionSlugsFromGames(games []football.Game) []string { + return lo.Uniq(lo.Filter(lo.Map(games, func(game football.Game, index int) string { + return game.Competition.Slug + }), func(slug string, index int) bool { + return slug != "" + })) +} + +func ExtractCompetitionSlugsFromClubs(clubs []football.Club) []string { + return lo.Uniq(lo.Filter(lo.Map(clubs, func(club football.Club, index int) string { + return club.DomesticLeague.Slug + }), func(slug string, index int) bool { + return slug != "" + })) +} + +func ExtractCompetitionSlugsFromPlayersGamesAndClubs( + players []football.Player, + games []football.Game, + clubs []football.Club, +) []string { + return lo.Uniq(lo.Union( + ExtractCompetitionSlugsFromPlayers(players), + ExtractCompetitionSlugsFromGames(games), + ExtractCompetitionSlugsFromClubs(clubs), + )) +}