managing initial game players sync. changing competition creation to batchexec. adding player initial sync

This commit is contained in:
Laurent Le Houerou 2024-03-22 11:40:34 +00:00
parent 717ae339dc
commit 3affe1f2e3
15 changed files with 553 additions and 190 deletions

View File

@ -87,9 +87,9 @@
birth_date DATE NOT NULL, birth_date DATE NOT NULL,
country_slug TEXT NOT NULL, country_slug TEXT NOT NULL,
FOREIGN KEY (country_slug) REFERENCES countries (slug), FOREIGN KEY (country_slug) REFERENCES countries (slug),
team_slug TEXT NOT NULL, team_slug TEXT,
FOREIGN KEY (team_slug) REFERENCES teams (slug), FOREIGN KEY (team_slug) REFERENCES teams (slug),
domestic_league_slug TEXT NOT NULL, domestic_league_slug TEXT,
FOREIGN KEY (domestic_league_slug) REFERENCES competitions (slug), FOREIGN KEY (domestic_league_slug) REFERENCES competitions (slug),
avatar_url TEXT NOT NULL, avatar_url TEXT NOT NULL,
field_position TEXT NOT NULL, field_position TEXT NOT NULL,

93
model/batch.go Normal file
View File

@ -0,0 +1,93 @@
// Code generated by sqlc. DO NOT EDIT.
// versions:
// sqlc v1.25.0
// source: batch.go
package model
import (
"context"
"errors"
"github.com/jackc/pgx/v5"
)
var (
ErrBatchAlreadyClosed = errors.New("batch already closed")
)
const createOrUpdateCompetitions = `-- name: CreateOrUpdateCompetitions :batchexec
INSERT INTO competitions (
slug,
display_name,
country_slug,
competition_format,
competition_type,
picture_url,
logo_url
)
VALUES ($1, $2, $3, $4, $5, $6, $7) ON CONFLICT (slug)
DO
UPDATE
SET display_name = EXCLUDED.display_name,
competition_format = EXCLUDED.competition_format,
competition_type = EXCLUDED.competition_type,
picture_url = EXCLUDED.picture_url,
logo_url = EXCLUDED.logo_url,
country_slug = EXCLUDED.country_slug
`
type CreateOrUpdateCompetitionsBatchResults struct {
br pgx.BatchResults
tot int
closed bool
}
type CreateOrUpdateCompetitionsParams struct {
Slug string
DisplayName string
CountrySlug string
CompetitionFormat string
CompetitionType string
PictureUrl string
LogoUrl string
}
func (q *Queries) CreateOrUpdateCompetitions(ctx context.Context, arg []CreateOrUpdateCompetitionsParams) *CreateOrUpdateCompetitionsBatchResults {
batch := &pgx.Batch{}
for _, a := range arg {
vals := []interface{}{
a.Slug,
a.DisplayName,
a.CountrySlug,
a.CompetitionFormat,
a.CompetitionType,
a.PictureUrl,
a.LogoUrl,
}
batch.Queue(createOrUpdateCompetitions, vals...)
}
br := q.db.SendBatch(ctx, batch)
return &CreateOrUpdateCompetitionsBatchResults{br, len(arg), false}
}
func (b *CreateOrUpdateCompetitionsBatchResults) Exec(f func(int, error)) {
defer b.br.Close()
for t := 0; t < b.tot; t++ {
if b.closed {
if f != nil {
f(t, ErrBatchAlreadyClosed)
}
continue
}
_, err := b.br.Exec()
if f != nil {
f(t, err)
}
}
}
func (b *CreateOrUpdateCompetitionsBatchResults) Close() error {
b.closed = true
return b.br.Close()
}

View File

@ -9,60 +9,6 @@ import (
"context" "context"
) )
type BatchInsertCompetitionsParams struct {
Slug string
DisplayName string
CountrySlug string
CompetitionFormat string
CompetitionType string
PictureUrl string
LogoUrl string
}
const createOrUpdateCompetition = `-- name: CreateOrUpdateCompetition :exec
INSERT INTO competitions (
slug,
display_name,
country_slug,
competition_format,
competition_type,
picture_url,
logo_url
)
VALUES ($1, $2, $3, $4, $5, $6, $7) ON CONFLICT (slug)
DO
UPDATE
SET display_name = EXCLUDED.display_name,
competition_format = EXCLUDED.competition_format,
competition_type = EXCLUDED.competition_type,
picture_url = EXCLUDED.picture_url,
logo_url = EXCLUDED.logo_url,
country_slug = EXCLUDED.country_slug
`
type CreateOrUpdateCompetitionParams struct {
Slug string
DisplayName string
CountrySlug string
CompetitionFormat string
CompetitionType string
PictureUrl string
LogoUrl string
}
func (q *Queries) CreateOrUpdateCompetition(ctx context.Context, arg CreateOrUpdateCompetitionParams) error {
_, err := q.db.Exec(ctx, createOrUpdateCompetition,
arg.Slug,
arg.DisplayName,
arg.CountrySlug,
arg.CompetitionFormat,
arg.CompetitionType,
arg.PictureUrl,
arg.LogoUrl,
)
return err
}
const getAllCompetitions = `-- name: GetAllCompetitions :many const getAllCompetitions = `-- name: GetAllCompetitions :many
SELECT competitions.slug, competitions.display_name, competitions.country_slug, competitions.competition_format, competitions.competition_type, competitions.picture_url, competitions.logo_url, competitions.zone_id, SELECT competitions.slug, competitions.display_name, competitions.country_slug, competitions.competition_format, competitions.competition_type, competitions.picture_url, competitions.logo_url, competitions.zone_id,
zones.id, zones.display_name, zones.id, zones.display_name,

View File

@ -9,13 +9,13 @@ import (
"context" "context"
) )
// iteratorForBatchInsertCompetitions implements pgx.CopyFromSource. // iteratorForBatchInsertGamePlayers implements pgx.CopyFromSource.
type iteratorForBatchInsertCompetitions struct { type iteratorForBatchInsertGamePlayers struct {
rows []BatchInsertCompetitionsParams rows []BatchInsertGamePlayersParams
skippedFirstNextCall bool skippedFirstNextCall bool
} }
func (r *iteratorForBatchInsertCompetitions) Next() bool { func (r *iteratorForBatchInsertGamePlayers) Next() bool {
if len(r.rows) == 0 { if len(r.rows) == 0 {
return false return false
} }
@ -27,24 +27,21 @@ func (r *iteratorForBatchInsertCompetitions) Next() bool {
return len(r.rows) > 0 return len(r.rows) > 0
} }
func (r iteratorForBatchInsertCompetitions) Values() ([]interface{}, error) { func (r iteratorForBatchInsertGamePlayers) Values() ([]interface{}, error) {
return []interface{}{ return []interface{}{
r.rows[0].Slug, r.rows[0].GameID,
r.rows[0].DisplayName, r.rows[0].PlayerSlug,
r.rows[0].CountrySlug, r.rows[0].Status,
r.rows[0].CompetitionFormat, r.rows[0].TeamSlug,
r.rows[0].CompetitionType,
r.rows[0].PictureUrl,
r.rows[0].LogoUrl,
}, nil }, nil
} }
func (r iteratorForBatchInsertCompetitions) Err() error { func (r iteratorForBatchInsertGamePlayers) Err() error {
return nil return nil
} }
func (q *Queries) BatchInsertCompetitions(ctx context.Context, arg []BatchInsertCompetitionsParams) (int64, error) { func (q *Queries) BatchInsertGamePlayers(ctx context.Context, arg []BatchInsertGamePlayersParams) (int64, error) {
return q.db.CopyFrom(ctx, []string{"competitions"}, []string{"slug", "display_name", "country_slug", "competition_format", "competition_type", "picture_url", "logo_url"}, &iteratorForBatchInsertCompetitions{rows: arg}) return q.db.CopyFrom(ctx, []string{"game_players"}, []string{"game_id", "player_slug", "status", "team_slug"}, &iteratorForBatchInsertGamePlayers{rows: arg})
} }
// iteratorForBatchInsertGames implements pgx.CopyFromSource. // iteratorForBatchInsertGames implements pgx.CopyFromSource.
@ -97,6 +94,47 @@ func (q *Queries) BatchInsertGames(ctx context.Context, arg []BatchInsertGamesPa
return q.db.CopyFrom(ctx, []string{"games"}, []string{"id", "date", "coverage_status", "low_coverage", "minutes", "period_type", "scored", "status", "competition_slug", "fixture_slug", "away_team_slug", "away_goals", "away_extra_time_score", "away_penalty_score", "home_team_slug", "home_goals", "home_extra_time_score", "home_penalty_score", "winner_team_slug"}, &iteratorForBatchInsertGames{rows: arg}) return q.db.CopyFrom(ctx, []string{"games"}, []string{"id", "date", "coverage_status", "low_coverage", "minutes", "period_type", "scored", "status", "competition_slug", "fixture_slug", "away_team_slug", "away_goals", "away_extra_time_score", "away_penalty_score", "home_team_slug", "home_goals", "home_extra_time_score", "home_penalty_score", "winner_team_slug"}, &iteratorForBatchInsertGames{rows: arg})
} }
// iteratorForBatchInsertPlayers implements pgx.CopyFromSource.
type iteratorForBatchInsertPlayers struct {
rows []BatchInsertPlayersParams
skippedFirstNextCall bool
}
func (r *iteratorForBatchInsertPlayers) Next() bool {
if len(r.rows) == 0 {
return false
}
if !r.skippedFirstNextCall {
r.skippedFirstNextCall = true
return true
}
r.rows = r.rows[1:]
return len(r.rows) > 0
}
func (r iteratorForBatchInsertPlayers) Values() ([]interface{}, error) {
return []interface{}{
r.rows[0].Slug,
r.rows[0].DisplayName,
r.rows[0].BirthDate,
r.rows[0].CountrySlug,
r.rows[0].TeamSlug,
r.rows[0].DomesticLeagueSlug,
r.rows[0].AvatarUrl,
r.rows[0].FieldPosition,
r.rows[0].Status,
r.rows[0].ShirtNumber,
}, nil
}
func (r iteratorForBatchInsertPlayers) Err() error {
return nil
}
func (q *Queries) BatchInsertPlayers(ctx context.Context, arg []BatchInsertPlayersParams) (int64, error) {
return q.db.CopyFrom(ctx, []string{"players"}, []string{"slug", "display_name", "birth_date", "country_slug", "team_slug", "domestic_league_slug", "avatar_url", "field_position", "status", "shirt_number"}, &iteratorForBatchInsertPlayers{rows: arg})
}
// iteratorForBatchInsertTeams implements pgx.CopyFromSource. // iteratorForBatchInsertTeams implements pgx.CopyFromSource.
type iteratorForBatchInsertTeams struct { type iteratorForBatchInsertTeams struct {
rows []BatchInsertTeamsParams rows []BatchInsertTeamsParams

View File

@ -16,6 +16,7 @@ type DBTX interface {
Query(context.Context, string, ...interface{}) (pgx.Rows, error) Query(context.Context, string, ...interface{}) (pgx.Rows, error)
QueryRow(context.Context, string, ...interface{}) pgx.Row QueryRow(context.Context, string, ...interface{}) pgx.Row
CopyFrom(ctx context.Context, tableName pgx.Identifier, columnNames []string, rowSrc pgx.CopyFromSource) (int64, error) CopyFrom(ctx context.Context, tableName pgx.Identifier, columnNames []string, rowSrc pgx.CopyFromSource) (int64, error)
SendBatch(context.Context, *pgx.Batch) pgx.BatchResults
} }
func New(db DBTX) *Queries { func New(db DBTX) *Queries {

50
model/game_player.sql.go Normal file
View File

@ -0,0 +1,50 @@
// Code generated by sqlc. DO NOT EDIT.
// versions:
// sqlc v1.25.0
// source: game_player.sql
package model
import (
"context"
)
type BatchInsertGamePlayersParams struct {
GameID string
PlayerSlug string
Status string
TeamSlug string
}
const createOrUpdateGamePlayer = `-- name: CreateOrUpdateGamePlayer :exec
INSERT INTO game_players(
game_id,
player_slug,
status,
team_slug)
VALUES(
$1,
$2,
$3,
$4)
ON CONFLICT (game_id, player_slug) DO UPDATE
SET status = $3,
team_slug = $4
`
type CreateOrUpdateGamePlayerParams struct {
GameID string
PlayerSlug string
Status string
TeamSlug string
}
func (q *Queries) CreateOrUpdateGamePlayer(ctx context.Context, arg CreateOrUpdateGamePlayerParams) error {
_, err := q.db.Exec(ctx, createOrUpdateGamePlayer,
arg.GameID,
arg.PlayerSlug,
arg.Status,
arg.TeamSlug,
)
return err
}

View File

@ -140,8 +140,8 @@ type Player struct {
DisplayName string DisplayName string
BirthDate time.Time BirthDate time.Time
CountrySlug string CountrySlug string
TeamSlug string TeamSlug *string
DomesticLeagueSlug string DomesticLeagueSlug *string
AvatarUrl string AvatarUrl string
FieldPosition string FieldPosition string
Status string Status string

23
model/player.sql.go Normal file
View File

@ -0,0 +1,23 @@
// Code generated by sqlc. DO NOT EDIT.
// versions:
// sqlc v1.25.0
// source: player.sql
package model
import (
"time"
)
type BatchInsertPlayersParams struct {
Slug string
DisplayName string
BirthDate time.Time
CountrySlug string
TeamSlug *string
DomesticLeagueSlug *string
AvatarUrl string
FieldPosition string
Status string
ShirtNumber int32
}

View File

@ -1,8 +1,4 @@
-- name: BatchInsertCompetitions :copyfrom -- name: CreateOrUpdateCompetitions :batchexec
INSERT INTO competitions (slug, display_name, country_slug, competition_format, competition_type, picture_url, logo_url)
VALUES ($1, $2, $3, $4, $5, $6, $7);
-- name: CreateOrUpdateCompetition :exec
INSERT INTO competitions ( INSERT INTO competitions (
slug, slug,
display_name, display_name,

27
model/sql/game_player.sql Normal file
View File

@ -0,0 +1,27 @@
-- name: BatchInsertGamePlayers :copyfrom
INSERT INTO game_players(
game_id,
player_slug,
status,
team_slug)
VALUES(
$1,
$2,
$3,
$4);
-- name: CreateOrUpdateGamePlayer :exec
INSERT INTO game_players(
game_id,
player_slug,
status,
team_slug)
VALUES(
$1,
$2,
$3,
$4)
ON CONFLICT (game_id, player_slug) DO UPDATE
SET status = $3,
team_slug = $4;

3
model/sql/player.sql Normal file
View File

@ -0,0 +1,3 @@
-- name: BatchInsertPlayers :copyfrom
INSERT INTO players (slug, display_name, birth_date, country_slug, team_slug, domestic_league_slug, avatar_url, field_position, status, shirt_number)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10);

View File

@ -37,74 +37,3 @@ func GetGamesFromFixtures(
log.Debug().Msgf("found total of %d games after filtering duplicate ids", len(games)) log.Debug().Msgf("found total of %d games after filtering duplicate ids", len(games))
return games, nil return games, nil
} }
func ExtractTeamSlugsFromGames(games []football.Game) []string {
var res []string
res = lo.Map(games, func(game football.Game, index int) string {
return game.AwayTeam.Team.Slug
})
res = append(res, lo.Map(games, func(game football.Game, index int) string {
return game.HomeTeam.Team.Slug
})...)
res = lo.Filter(res, func(slug string, index int) bool {
return slug != ""
})
return lo.Uniq(res)
}
func ExtractCountrySlugsFromCompetitions(competitions []football.Competition) []string {
return lo.Uniq(lo.Filter(lo.Map(competitions, func(competition football.Competition, index int) string {
return competition.Country.Slug
}), func(slug string, index int) bool {
return slug != ""
}))
}
func ExtractCountrySlugsFromTeams(teams []football.Team) []string {
return lo.Uniq(lo.Filter(lo.Map(teams, func(team football.Team, index int) string {
return team.Country.Slug
}), func(slug string, index int) bool {
return slug != ""
}))
}
func ExtractCountrySlugsFromCompetitionsClubsAndNationalTeams(
competitions []football.Competition,
clubs []football.Club,
nationalTeams []football.NationalTeam,
) []string {
return lo.Uniq(lo.Union(
ExtractCountrySlugsFromCompetitions(competitions),
ExtractCountrySlugsFromTeams(lo.Map(clubs, func(club football.Club, index int) football.Team {
return club.Team
})),
ExtractCountrySlugsFromTeams(
lo.Map(nationalTeams, func(nationalTeam football.NationalTeam, index int) football.Team {
return nationalTeam.Team
}),
),
))
}
func ExtractCompetitionSlugsFromGames(games []football.Game) []string {
return lo.Uniq(lo.Filter(lo.Map(games, func(game football.Game, index int) string {
return game.Competition.Slug
}), func(slug string, index int) bool {
return slug != ""
}))
}
func ExtractCompetitionSlugsFromClubs(clubs []football.Club) []string {
return lo.Uniq(lo.Filter(lo.Map(clubs, func(club football.Club, index int) string {
return club.DomesticLeague.Slug
}), func(slug string, index int) bool {
return slug != ""
}))
}
func ExtractCompetitionSlugsFromGamesAndClubs(games []football.Game, clubs []football.Club) []string {
return lo.Uniq(lo.Union(
ExtractCompetitionSlugsFromGames(games),
ExtractCompetitionSlugsFromClubs(clubs),
))
}

37
sorare_utils/game.go Normal file
View File

@ -0,0 +1,37 @@
package sorare_utils
import (
"git.lehouerou.net/laurent/sorare/football"
"github.com/jackc/pgx/v5/pgtype"
"git.lehouerou.net/laurent/sorarebuddy/model"
)
func NewBatchInsertGamesParamsFromSorare(game football.Game) model.BatchInsertGamesParams {
return model.BatchInsertGamesParams{
ID: game.Id.Value,
Date: pgtype.Timestamptz{Time: game.Date, Valid: true},
CoverageStatus: game.CoverageStatus,
LowCoverage: game.LowCoverage,
Minutes: int32(game.Minute),
PeriodType: game.PeriodType,
Scored: game.Scored,
Status: game.Status,
CompetitionSlug: game.Competition.Slug,
FixtureSlug: game.So5Fixture.Slug,
AwayTeamSlug: game.AwayTeam.Team.Slug,
AwayGoals: int32(game.AwayGoals),
AwayExtraTimeScore: int32(game.ExtraTimeScoreAway),
AwayPenaltyScore: int32(game.PenaltyScoreAway),
HomeTeamSlug: game.HomeTeam.Team.Slug,
HomeGoals: int32(game.HomeGoals),
HomeExtraTimeScore: int32(game.ExtraTimeScoreHome),
HomePenaltyScore: int32(game.PenaltyScoreHome),
WinnerTeamSlug: func() *string {
if game.Winner.Team.Slug == "" {
return nil
}
return &game.Winner.Team.Slug
}(),
}
}

View File

@ -0,0 +1 @@
package sorare_utils

View File

@ -7,6 +7,8 @@ import (
"git.lehouerou.net/laurent/sorare/football" "git.lehouerou.net/laurent/sorare/football"
"git.lehouerou.net/laurent/sorare/graphql" "git.lehouerou.net/laurent/sorare/graphql"
"github.com/jackc/pgx/v5/pgtype" "github.com/jackc/pgx/v5/pgtype"
gql "github.com/llehouerou/go-graphql-client"
"github.com/pkg/errors"
"github.com/rs/zerolog/log" "github.com/rs/zerolog/log"
"github.com/samber/lo" "github.com/samber/lo"
@ -24,7 +26,9 @@ func NewUpdateService(s *sorare.Sorare, db *model.Queries) *UpdateService {
func (u *UpdateService) InitSyncDatabase(ctx context.Context) error { func (u *UpdateService) InitSyncDatabase(ctx context.Context) error {
sfixtures, err := u.s.Football.So5.So5Fixtures.Get(ctx, football.So5FixturesParams{}) sfixtures, err := u.s.Football.So5.So5Fixtures.Get(ctx, football.So5FixturesParams{
AasmStates: []string{"started"},
})
if err != nil { if err != nil {
return err return err
} }
@ -62,13 +66,45 @@ func (u *UpdateService) InitSyncDatabase(ctx context.Context) error {
) )
log.Info().Msgf("found %d games to process", len(games)) log.Info().Msgf("found %d games to process", len(games))
teamSlugs := ExtractTeamSlugsFromGames(games) log.Debug().Msgf("getting players for each game...")
log.Debug().Msgf("extracted %d unique team slugs from games", len(teamSlugs)) var gamePlayers []model.BatchInsertGamePlayersParams
for _, game := range games {
gameWithFormation, err := u.s.Football.Game.Get(ctx, graphql.IdParams{Id: gql.ID(game.Id.Value)})
if err != nil {
return errors.Wrapf(err, "getting game with formation %s", game.Id.Value)
}
newplayers := ExtractPlayersFromGameWithFormation(gameWithFormation)
log.Debug().Msgf("\t%s -> %d players", game.String(), len(newplayers))
gamePlayers = append(gamePlayers, newplayers...)
}
playerSlugs := lo.Uniq(
lo.Filter(lo.Map(gamePlayers, func(player model.BatchInsertGamePlayersParams, index int) string {
return player.PlayerSlug
}), func(slug string, index int) bool {
return slug != ""
}),
)
log.Debug().Msgf("getting players...")
var players []football.Player
for i, chunk := range lo.Chunk(playerSlugs, 80) {
log.Debug().Msgf("\tbatch %d/%d", i+1, (len(playerSlugs)/80)+1)
p, err := u.s.Football.Players.Get(ctx, graphql.SlugsParams{Slugs: chunk})
if err != nil {
return errors.Wrapf(err, "getting players batch %d", i)
}
players = append(players, p...)
}
log.Debug().Msgf("found %d players", len(players))
teamSlugs := ExtractTeamSlugsFromPlayersAndGames(players, games)
log.Debug().Msgf("extracted %d unique team slugs from games and players", len(teamSlugs))
log.Debug().Msgf("getting clubs...") log.Debug().Msgf("getting clubs...")
var clubs []football.Club var clubs []football.Club
for i, chunk := range lo.Chunk(teamSlugs, 100) { for i, chunk := range lo.Chunk(teamSlugs, 100) {
log.Debug().Msgf("\tbatch %d/%d", i+1, len(teamSlugs)/100) log.Debug().Msgf("\tbatch %d/%d", i+1, (len(teamSlugs)/100)+1)
t, err := u.s.Football.Clubs.Get(ctx, graphql.SlugsParams{Slugs: chunk}) t, err := u.s.Football.Clubs.Get(ctx, graphql.SlugsParams{Slugs: chunk})
if err != nil { if err != nil {
return err return err
@ -83,7 +119,7 @@ func (u *UpdateService) InitSyncDatabase(ctx context.Context) error {
log.Debug().Msgf("getting national teams...") log.Debug().Msgf("getting national teams...")
log.Debug().Msgf("slugs left: %d", len(slugsLeft)) log.Debug().Msgf("slugs left: %d", len(slugsLeft))
for i, chunk := range lo.Chunk(slugsLeft, 100) { for i, chunk := range lo.Chunk(slugsLeft, 100) {
log.Debug().Msgf("\tbatch %d/%d", i+1, len(teamSlugs)/100) log.Debug().Msgf("\tbatch %d/%d", i+1, (len(teamSlugs)/100)+1)
t, err := u.s.Football.NationalTeams.Get(ctx, graphql.SlugsParams{Slugs: chunk}) t, err := u.s.Football.NationalTeams.Get(ctx, graphql.SlugsParams{Slugs: chunk})
if err != nil { if err != nil {
return err return err
@ -92,8 +128,8 @@ func (u *UpdateService) InitSyncDatabase(ctx context.Context) error {
} }
log.Debug().Msgf("found %d national teams", len(nationalTeams)) log.Debug().Msgf("found %d national teams", len(nationalTeams))
competitionSlugs := ExtractCompetitionSlugsFromGamesAndClubs(games, clubs) competitionSlugs := ExtractCompetitionSlugsFromPlayersGamesAndClubs(players, games, clubs)
log.Debug().Msgf("extracted %d unique competition slugs from games and clubs", len(competitionSlugs)) log.Debug().Msgf("extracted %d unique competition slugs from players, games and clubs", len(competitionSlugs))
log.Debug().Msgf("getting competitions...") log.Debug().Msgf("getting competitions...")
var competitions []football.Competition var competitions []football.Competition
for _, slug := range competitionSlugs { for _, slug := range competitionSlugs {
@ -106,16 +142,18 @@ func (u *UpdateService) InitSyncDatabase(ctx context.Context) error {
} }
log.Debug().Msgf("found %d competitions", len(competitions)) log.Debug().Msgf("found %d competitions", len(competitions))
countrySlugs := ExtractCountrySlugsFromCompetitionsClubsAndNationalTeams( countrySlugs := ExtractCountrySlugsFromPlayersCompetitionsClubsAndNationalTeams(
players,
competitions, competitions,
clubs, clubs,
nationalTeams, nationalTeams,
) )
log.Debug().Msgf("extracted %d unique country slugs from competitions, clubs and national teams", len(countrySlugs)) log.Debug().
Msgf("extracted %d unique country slugs from players, competitions, clubs and national teams", len(countrySlugs))
log.Debug().Msgf("getting countries...") log.Debug().Msgf("getting countries...")
var countries []sorare.Country var countries []sorare.Country
for i, chunk := range lo.Chunk(countrySlugs, 100) { for i, chunk := range lo.Chunk(countrySlugs, 100) {
log.Debug().Msgf("\tbatch %d/%d", i+1, len(countrySlugs)/100) log.Debug().Msgf("\tbatch %d/%d", i+1, (len(countrySlugs)/100)+1)
c, err := u.s.Countries.Get(ctx, graphql.SlugsParams{Slugs: chunk}) c, err := u.s.Countries.Get(ctx, graphql.SlugsParams{Slugs: chunk})
if err != nil { if err != nil {
return err return err
@ -147,10 +185,11 @@ func (u *UpdateService) InitSyncDatabase(ctx context.Context) error {
log.Debug().Msg("inserting competitions into db...") log.Debug().Msg("inserting competitions into db...")
_, err = u.db.BatchInsertCompetitions( var batcherr error
batch := u.db.CreateOrUpdateCompetitions(
ctx, ctx,
lo.Map(competitions, func(competition football.Competition, index int) model.BatchInsertCompetitionsParams { lo.Map(competitions, func(competition football.Competition, index int) model.CreateOrUpdateCompetitionsParams {
return model.BatchInsertCompetitionsParams{ return model.CreateOrUpdateCompetitionsParams{
Slug: competition.Slug, Slug: competition.Slug,
CompetitionFormat: competition.Format, CompetitionFormat: competition.Format,
CompetitionType: competition.Type, CompetitionType: competition.Type,
@ -161,8 +200,14 @@ func (u *UpdateService) InitSyncDatabase(ctx context.Context) error {
} }
}), }),
) )
if err != nil { batch.Exec(func(_ int, err error) {
return err if err != nil {
batcherr = err
batch.Close()
}
})
if batcherr != nil {
return errors.Wrap(batcherr, "inserting competitions")
} }
log.Debug().Msgf("%d competitions inserted", len(competitions)) log.Debug().Msgf("%d competitions inserted", len(competitions))
@ -206,37 +251,211 @@ func (u *UpdateService) InitSyncDatabase(ctx context.Context) error {
cnt, err = u.db.BatchInsertGames( cnt, err = u.db.BatchInsertGames(
ctx, ctx,
lo.Map(games, func(game football.Game, index int) model.BatchInsertGamesParams { lo.Map(games, func(game football.Game, index int) model.BatchInsertGamesParams {
return model.BatchInsertGamesParams{ return NewBatchInsertGamesParamsFromSorare(game)
ID: game.Id.Value,
Date: pgtype.Timestamptz{Time: game.Date, Valid: true},
CoverageStatus: game.CoverageStatus,
LowCoverage: game.LowCoverage,
Minutes: int32(game.Minute),
PeriodType: game.PeriodType,
Scored: game.Scored,
Status: game.Status,
CompetitionSlug: game.Competition.Slug,
FixtureSlug: game.So5Fixture.Slug,
AwayTeamSlug: game.AwayTeam.Team.Slug,
AwayGoals: int32(game.AwayGoals),
AwayExtraTimeScore: int32(game.ExtraTimeScoreAway),
AwayPenaltyScore: int32(game.PenaltyScoreAway),
HomeTeamSlug: game.HomeTeam.Team.Slug,
HomeGoals: int32(game.HomeGoals),
HomeExtraTimeScore: int32(game.ExtraTimeScoreHome),
HomePenaltyScore: int32(game.PenaltyScoreHome),
WinnerTeamSlug: func() *string {
if game.Winner.Team.Slug == "" {
return nil
}
return &game.Winner.Team.Slug
}(),
}
}), }),
) )
if err != nil { if err != nil {
return err return err
} }
log.Debug().Msgf("%d games inserted", cnt) log.Debug().Msgf("%d games inserted", cnt)
log.Debug().Msg("inserting players into db...")
cnt, err = u.db.BatchInsertPlayers(
ctx,
lo.Map(players, func(player football.Player, index int) model.BatchInsertPlayersParams {
res := model.BatchInsertPlayersParams{
Slug: player.Slug,
DisplayName: player.DisplayName,
BirthDate: player.BirthDate,
CountrySlug: player.Country.Slug,
AvatarUrl: player.AvatarUrl,
FieldPosition: string(player.Position),
Status: string(player.PlayingStatus),
ShirtNumber: int32(player.ShirtNumber),
}
for _, competition := range player.ActiveClub.ActiveCompetitions {
if competition.Format == "DOMESTIC_LEAGUE" {
res.DomesticLeagueSlug = &competition.Slug
}
}
if player.ActiveClub.Slug != "" {
res.TeamSlug = &player.ActiveClub.Slug
}
return res
}),
)
if err != nil {
return errors.Wrap(err, "inserting players")
}
log.Debug().Msgf("%d players inserted", cnt)
log.Debug().Msg("inserting game players into db...")
cnt, err = u.db.BatchInsertGamePlayers(ctx, gamePlayers)
if err != nil {
return errors.Wrap(err, "inserting game players")
}
log.Debug().Msgf("%d game players inserted", cnt)
return nil return nil
} }
func ExtractPlayersFromGameWithFormation(
gameWithFormation football.GameWithFormation,
) []model.BatchInsertGamePlayersParams {
var res []model.BatchInsertGamePlayersParams
for _, p := range gameWithFormation.HomeFormation.Bench {
res = append(res, model.BatchInsertGamePlayersParams{
GameID: gameWithFormation.Id.Value,
PlayerSlug: p.Slug,
TeamSlug: gameWithFormation.HomeTeam.Team.Slug,
Status: "bench",
})
}
for _, p := range gameWithFormation.HomeFormation.StartingLineup {
for _, q := range p {
res = append(res, model.BatchInsertGamePlayersParams{
GameID: gameWithFormation.Id.Value,
PlayerSlug: q.Slug,
TeamSlug: gameWithFormation.HomeTeam.Team.Slug,
Status: "starting",
})
}
}
for _, p := range gameWithFormation.AwayFormation.Bench {
res = append(res, model.BatchInsertGamePlayersParams{
GameID: gameWithFormation.Id.Value,
PlayerSlug: p.Slug,
TeamSlug: gameWithFormation.AwayTeam.Team.Slug,
Status: "bench",
})
}
for _, p := range gameWithFormation.AwayFormation.StartingLineup {
for _, q := range p {
res = append(res, model.BatchInsertGamePlayersParams{
GameID: gameWithFormation.Id.Value,
PlayerSlug: q.Slug,
TeamSlug: gameWithFormation.AwayTeam.Team.Slug,
Status: "starting",
})
}
}
return res
}
func ExtractTeamSlugsFromPlayersAndGames(players []football.Player, games []football.Game) []string {
return lo.Uniq(lo.Union(
ExtractTeamSlugsFromPlayers(players),
ExtractTeamSlugsFromGames(games),
))
}
func ExtractTeamSlugsFromPlayers(players []football.Player) []string {
return lo.Uniq(lo.Filter(lo.Map(players, func(player football.Player, index int) string {
return player.ActiveClub.Slug
}), func(slug string, index int) bool {
return slug != ""
}))
}
func ExtractTeamSlugsFromGames(games []football.Game) []string {
var res []string
res = lo.Map(games, func(game football.Game, index int) string {
return game.AwayTeam.Team.Slug
})
res = append(res, lo.Map(games, func(game football.Game, index int) string {
return game.HomeTeam.Team.Slug
})...)
res = lo.Filter(res, func(slug string, index int) bool {
return slug != ""
})
return lo.Uniq(res)
}
func ExtractCountrySlugsFromCompetitions(competitions []football.Competition) []string {
return lo.Uniq(lo.Filter(lo.Map(competitions, func(competition football.Competition, index int) string {
return competition.Country.Slug
}), func(slug string, index int) bool {
return slug != ""
}))
}
func ExtractCountrySlugsFromTeams(teams []football.Team) []string {
return lo.Uniq(lo.Filter(lo.Map(teams, func(team football.Team, index int) string {
return team.Country.Slug
}), func(slug string, index int) bool {
return slug != ""
}))
}
func ExtractCountrySlugsFromPlayersCompetitionsClubsAndNationalTeams(
players []football.Player,
competitions []football.Competition,
clubs []football.Club,
nationalTeams []football.NationalTeam,
) []string {
return lo.Uniq(lo.Union(
ExtractCountrySlugsFromPlayers(players),
ExtractCountrySlugsFromCompetitions(competitions),
ExtractCountrySlugsFromTeams(lo.Map(clubs, func(club football.Club, index int) football.Team {
return club.Team
})),
ExtractCountrySlugsFromTeams(
lo.Map(nationalTeams, func(nationalTeam football.NationalTeam, index int) football.Team {
return nationalTeam.Team
}),
),
))
}
func ExtractCountrySlugsFromPlayers(players []football.Player) []string {
return lo.Uniq(lo.Filter(lo.Map(players, func(player football.Player, index int) string {
return player.Country.Slug
}), func(slug string, index int) bool {
return slug != ""
}))
}
func ExtractCompetitionSlugsFromPlayers(players []football.Player) []string {
return lo.Uniq(lo.Filter(lo.Map(players, func(player football.Player, index int) string {
var res string
for _, competition := range player.ActiveClub.ActiveCompetitions {
if competition.Format == "DOMESTIC_LEAGUE" {
res = competition.Slug
}
}
return res
}), func(slug string, index int) bool {
return slug != ""
}))
}
func ExtractCompetitionSlugsFromGames(games []football.Game) []string {
return lo.Uniq(lo.Filter(lo.Map(games, func(game football.Game, index int) string {
return game.Competition.Slug
}), func(slug string, index int) bool {
return slug != ""
}))
}
func ExtractCompetitionSlugsFromClubs(clubs []football.Club) []string {
return lo.Uniq(lo.Filter(lo.Map(clubs, func(club football.Club, index int) string {
return club.DomesticLeague.Slug
}), func(slug string, index int) bool {
return slug != ""
}))
}
func ExtractCompetitionSlugsFromPlayersGamesAndClubs(
players []football.Player,
games []football.Game,
clubs []football.Club,
) []string {
return lo.Uniq(lo.Union(
ExtractCompetitionSlugsFromPlayers(players),
ExtractCompetitionSlugsFromGames(games),
ExtractCompetitionSlugsFromClubs(clubs),
))
}