managing initial game players sync. changing competition creation to batchexec. adding player initial sync
This commit is contained in:
parent
717ae339dc
commit
3affe1f2e3
@ -87,9 +87,9 @@
|
||||
birth_date DATE NOT NULL,
|
||||
country_slug TEXT NOT NULL,
|
||||
FOREIGN KEY (country_slug) REFERENCES countries (slug),
|
||||
team_slug TEXT NOT NULL,
|
||||
team_slug TEXT,
|
||||
FOREIGN KEY (team_slug) REFERENCES teams (slug),
|
||||
domestic_league_slug TEXT NOT NULL,
|
||||
domestic_league_slug TEXT,
|
||||
FOREIGN KEY (domestic_league_slug) REFERENCES competitions (slug),
|
||||
avatar_url TEXT NOT NULL,
|
||||
field_position TEXT NOT NULL,
|
||||
|
93
model/batch.go
Normal file
93
model/batch.go
Normal file
@ -0,0 +1,93 @@
|
||||
// Code generated by sqlc. DO NOT EDIT.
|
||||
// versions:
|
||||
// sqlc v1.25.0
|
||||
// source: batch.go
|
||||
|
||||
package model
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrBatchAlreadyClosed = errors.New("batch already closed")
|
||||
)
|
||||
|
||||
const createOrUpdateCompetitions = `-- name: CreateOrUpdateCompetitions :batchexec
|
||||
INSERT INTO competitions (
|
||||
slug,
|
||||
display_name,
|
||||
country_slug,
|
||||
competition_format,
|
||||
competition_type,
|
||||
picture_url,
|
||||
logo_url
|
||||
)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7) ON CONFLICT (slug)
|
||||
DO
|
||||
UPDATE
|
||||
SET display_name = EXCLUDED.display_name,
|
||||
competition_format = EXCLUDED.competition_format,
|
||||
competition_type = EXCLUDED.competition_type,
|
||||
picture_url = EXCLUDED.picture_url,
|
||||
logo_url = EXCLUDED.logo_url,
|
||||
country_slug = EXCLUDED.country_slug
|
||||
`
|
||||
|
||||
type CreateOrUpdateCompetitionsBatchResults struct {
|
||||
br pgx.BatchResults
|
||||
tot int
|
||||
closed bool
|
||||
}
|
||||
|
||||
type CreateOrUpdateCompetitionsParams struct {
|
||||
Slug string
|
||||
DisplayName string
|
||||
CountrySlug string
|
||||
CompetitionFormat string
|
||||
CompetitionType string
|
||||
PictureUrl string
|
||||
LogoUrl string
|
||||
}
|
||||
|
||||
func (q *Queries) CreateOrUpdateCompetitions(ctx context.Context, arg []CreateOrUpdateCompetitionsParams) *CreateOrUpdateCompetitionsBatchResults {
|
||||
batch := &pgx.Batch{}
|
||||
for _, a := range arg {
|
||||
vals := []interface{}{
|
||||
a.Slug,
|
||||
a.DisplayName,
|
||||
a.CountrySlug,
|
||||
a.CompetitionFormat,
|
||||
a.CompetitionType,
|
||||
a.PictureUrl,
|
||||
a.LogoUrl,
|
||||
}
|
||||
batch.Queue(createOrUpdateCompetitions, vals...)
|
||||
}
|
||||
br := q.db.SendBatch(ctx, batch)
|
||||
return &CreateOrUpdateCompetitionsBatchResults{br, len(arg), false}
|
||||
}
|
||||
|
||||
func (b *CreateOrUpdateCompetitionsBatchResults) Exec(f func(int, error)) {
|
||||
defer b.br.Close()
|
||||
for t := 0; t < b.tot; t++ {
|
||||
if b.closed {
|
||||
if f != nil {
|
||||
f(t, ErrBatchAlreadyClosed)
|
||||
}
|
||||
continue
|
||||
}
|
||||
_, err := b.br.Exec()
|
||||
if f != nil {
|
||||
f(t, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (b *CreateOrUpdateCompetitionsBatchResults) Close() error {
|
||||
b.closed = true
|
||||
return b.br.Close()
|
||||
}
|
@ -9,60 +9,6 @@ import (
|
||||
"context"
|
||||
)
|
||||
|
||||
type BatchInsertCompetitionsParams struct {
|
||||
Slug string
|
||||
DisplayName string
|
||||
CountrySlug string
|
||||
CompetitionFormat string
|
||||
CompetitionType string
|
||||
PictureUrl string
|
||||
LogoUrl string
|
||||
}
|
||||
|
||||
const createOrUpdateCompetition = `-- name: CreateOrUpdateCompetition :exec
|
||||
INSERT INTO competitions (
|
||||
slug,
|
||||
display_name,
|
||||
country_slug,
|
||||
competition_format,
|
||||
competition_type,
|
||||
picture_url,
|
||||
logo_url
|
||||
)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7) ON CONFLICT (slug)
|
||||
DO
|
||||
UPDATE
|
||||
SET display_name = EXCLUDED.display_name,
|
||||
competition_format = EXCLUDED.competition_format,
|
||||
competition_type = EXCLUDED.competition_type,
|
||||
picture_url = EXCLUDED.picture_url,
|
||||
logo_url = EXCLUDED.logo_url,
|
||||
country_slug = EXCLUDED.country_slug
|
||||
`
|
||||
|
||||
type CreateOrUpdateCompetitionParams struct {
|
||||
Slug string
|
||||
DisplayName string
|
||||
CountrySlug string
|
||||
CompetitionFormat string
|
||||
CompetitionType string
|
||||
PictureUrl string
|
||||
LogoUrl string
|
||||
}
|
||||
|
||||
func (q *Queries) CreateOrUpdateCompetition(ctx context.Context, arg CreateOrUpdateCompetitionParams) error {
|
||||
_, err := q.db.Exec(ctx, createOrUpdateCompetition,
|
||||
arg.Slug,
|
||||
arg.DisplayName,
|
||||
arg.CountrySlug,
|
||||
arg.CompetitionFormat,
|
||||
arg.CompetitionType,
|
||||
arg.PictureUrl,
|
||||
arg.LogoUrl,
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
const getAllCompetitions = `-- name: GetAllCompetitions :many
|
||||
SELECT competitions.slug, competitions.display_name, competitions.country_slug, competitions.competition_format, competitions.competition_type, competitions.picture_url, competitions.logo_url, competitions.zone_id,
|
||||
zones.id, zones.display_name,
|
||||
|
@ -9,13 +9,13 @@ import (
|
||||
"context"
|
||||
)
|
||||
|
||||
// iteratorForBatchInsertCompetitions implements pgx.CopyFromSource.
|
||||
type iteratorForBatchInsertCompetitions struct {
|
||||
rows []BatchInsertCompetitionsParams
|
||||
// iteratorForBatchInsertGamePlayers implements pgx.CopyFromSource.
|
||||
type iteratorForBatchInsertGamePlayers struct {
|
||||
rows []BatchInsertGamePlayersParams
|
||||
skippedFirstNextCall bool
|
||||
}
|
||||
|
||||
func (r *iteratorForBatchInsertCompetitions) Next() bool {
|
||||
func (r *iteratorForBatchInsertGamePlayers) Next() bool {
|
||||
if len(r.rows) == 0 {
|
||||
return false
|
||||
}
|
||||
@ -27,24 +27,21 @@ func (r *iteratorForBatchInsertCompetitions) Next() bool {
|
||||
return len(r.rows) > 0
|
||||
}
|
||||
|
||||
func (r iteratorForBatchInsertCompetitions) Values() ([]interface{}, error) {
|
||||
func (r iteratorForBatchInsertGamePlayers) Values() ([]interface{}, error) {
|
||||
return []interface{}{
|
||||
r.rows[0].Slug,
|
||||
r.rows[0].DisplayName,
|
||||
r.rows[0].CountrySlug,
|
||||
r.rows[0].CompetitionFormat,
|
||||
r.rows[0].CompetitionType,
|
||||
r.rows[0].PictureUrl,
|
||||
r.rows[0].LogoUrl,
|
||||
r.rows[0].GameID,
|
||||
r.rows[0].PlayerSlug,
|
||||
r.rows[0].Status,
|
||||
r.rows[0].TeamSlug,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (r iteratorForBatchInsertCompetitions) Err() error {
|
||||
func (r iteratorForBatchInsertGamePlayers) Err() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (q *Queries) BatchInsertCompetitions(ctx context.Context, arg []BatchInsertCompetitionsParams) (int64, error) {
|
||||
return q.db.CopyFrom(ctx, []string{"competitions"}, []string{"slug", "display_name", "country_slug", "competition_format", "competition_type", "picture_url", "logo_url"}, &iteratorForBatchInsertCompetitions{rows: arg})
|
||||
func (q *Queries) BatchInsertGamePlayers(ctx context.Context, arg []BatchInsertGamePlayersParams) (int64, error) {
|
||||
return q.db.CopyFrom(ctx, []string{"game_players"}, []string{"game_id", "player_slug", "status", "team_slug"}, &iteratorForBatchInsertGamePlayers{rows: arg})
|
||||
}
|
||||
|
||||
// iteratorForBatchInsertGames implements pgx.CopyFromSource.
|
||||
@ -97,6 +94,47 @@ func (q *Queries) BatchInsertGames(ctx context.Context, arg []BatchInsertGamesPa
|
||||
return q.db.CopyFrom(ctx, []string{"games"}, []string{"id", "date", "coverage_status", "low_coverage", "minutes", "period_type", "scored", "status", "competition_slug", "fixture_slug", "away_team_slug", "away_goals", "away_extra_time_score", "away_penalty_score", "home_team_slug", "home_goals", "home_extra_time_score", "home_penalty_score", "winner_team_slug"}, &iteratorForBatchInsertGames{rows: arg})
|
||||
}
|
||||
|
||||
// iteratorForBatchInsertPlayers implements pgx.CopyFromSource.
|
||||
type iteratorForBatchInsertPlayers struct {
|
||||
rows []BatchInsertPlayersParams
|
||||
skippedFirstNextCall bool
|
||||
}
|
||||
|
||||
func (r *iteratorForBatchInsertPlayers) Next() bool {
|
||||
if len(r.rows) == 0 {
|
||||
return false
|
||||
}
|
||||
if !r.skippedFirstNextCall {
|
||||
r.skippedFirstNextCall = true
|
||||
return true
|
||||
}
|
||||
r.rows = r.rows[1:]
|
||||
return len(r.rows) > 0
|
||||
}
|
||||
|
||||
func (r iteratorForBatchInsertPlayers) Values() ([]interface{}, error) {
|
||||
return []interface{}{
|
||||
r.rows[0].Slug,
|
||||
r.rows[0].DisplayName,
|
||||
r.rows[0].BirthDate,
|
||||
r.rows[0].CountrySlug,
|
||||
r.rows[0].TeamSlug,
|
||||
r.rows[0].DomesticLeagueSlug,
|
||||
r.rows[0].AvatarUrl,
|
||||
r.rows[0].FieldPosition,
|
||||
r.rows[0].Status,
|
||||
r.rows[0].ShirtNumber,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (r iteratorForBatchInsertPlayers) Err() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (q *Queries) BatchInsertPlayers(ctx context.Context, arg []BatchInsertPlayersParams) (int64, error) {
|
||||
return q.db.CopyFrom(ctx, []string{"players"}, []string{"slug", "display_name", "birth_date", "country_slug", "team_slug", "domestic_league_slug", "avatar_url", "field_position", "status", "shirt_number"}, &iteratorForBatchInsertPlayers{rows: arg})
|
||||
}
|
||||
|
||||
// iteratorForBatchInsertTeams implements pgx.CopyFromSource.
|
||||
type iteratorForBatchInsertTeams struct {
|
||||
rows []BatchInsertTeamsParams
|
||||
|
@ -16,6 +16,7 @@ type DBTX interface {
|
||||
Query(context.Context, string, ...interface{}) (pgx.Rows, error)
|
||||
QueryRow(context.Context, string, ...interface{}) pgx.Row
|
||||
CopyFrom(ctx context.Context, tableName pgx.Identifier, columnNames []string, rowSrc pgx.CopyFromSource) (int64, error)
|
||||
SendBatch(context.Context, *pgx.Batch) pgx.BatchResults
|
||||
}
|
||||
|
||||
func New(db DBTX) *Queries {
|
||||
|
50
model/game_player.sql.go
Normal file
50
model/game_player.sql.go
Normal file
@ -0,0 +1,50 @@
|
||||
// Code generated by sqlc. DO NOT EDIT.
|
||||
// versions:
|
||||
// sqlc v1.25.0
|
||||
// source: game_player.sql
|
||||
|
||||
package model
|
||||
|
||||
import (
|
||||
"context"
|
||||
)
|
||||
|
||||
type BatchInsertGamePlayersParams struct {
|
||||
GameID string
|
||||
PlayerSlug string
|
||||
Status string
|
||||
TeamSlug string
|
||||
}
|
||||
|
||||
const createOrUpdateGamePlayer = `-- name: CreateOrUpdateGamePlayer :exec
|
||||
INSERT INTO game_players(
|
||||
game_id,
|
||||
player_slug,
|
||||
status,
|
||||
team_slug)
|
||||
VALUES(
|
||||
$1,
|
||||
$2,
|
||||
$3,
|
||||
$4)
|
||||
ON CONFLICT (game_id, player_slug) DO UPDATE
|
||||
SET status = $3,
|
||||
team_slug = $4
|
||||
`
|
||||
|
||||
type CreateOrUpdateGamePlayerParams struct {
|
||||
GameID string
|
||||
PlayerSlug string
|
||||
Status string
|
||||
TeamSlug string
|
||||
}
|
||||
|
||||
func (q *Queries) CreateOrUpdateGamePlayer(ctx context.Context, arg CreateOrUpdateGamePlayerParams) error {
|
||||
_, err := q.db.Exec(ctx, createOrUpdateGamePlayer,
|
||||
arg.GameID,
|
||||
arg.PlayerSlug,
|
||||
arg.Status,
|
||||
arg.TeamSlug,
|
||||
)
|
||||
return err
|
||||
}
|
@ -140,8 +140,8 @@ type Player struct {
|
||||
DisplayName string
|
||||
BirthDate time.Time
|
||||
CountrySlug string
|
||||
TeamSlug string
|
||||
DomesticLeagueSlug string
|
||||
TeamSlug *string
|
||||
DomesticLeagueSlug *string
|
||||
AvatarUrl string
|
||||
FieldPosition string
|
||||
Status string
|
||||
|
23
model/player.sql.go
Normal file
23
model/player.sql.go
Normal file
@ -0,0 +1,23 @@
|
||||
// Code generated by sqlc. DO NOT EDIT.
|
||||
// versions:
|
||||
// sqlc v1.25.0
|
||||
// source: player.sql
|
||||
|
||||
package model
|
||||
|
||||
import (
|
||||
"time"
|
||||
)
|
||||
|
||||
type BatchInsertPlayersParams struct {
|
||||
Slug string
|
||||
DisplayName string
|
||||
BirthDate time.Time
|
||||
CountrySlug string
|
||||
TeamSlug *string
|
||||
DomesticLeagueSlug *string
|
||||
AvatarUrl string
|
||||
FieldPosition string
|
||||
Status string
|
||||
ShirtNumber int32
|
||||
}
|
@ -1,8 +1,4 @@
|
||||
-- name: BatchInsertCompetitions :copyfrom
|
||||
INSERT INTO competitions (slug, display_name, country_slug, competition_format, competition_type, picture_url, logo_url)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7);
|
||||
|
||||
-- name: CreateOrUpdateCompetition :exec
|
||||
-- name: CreateOrUpdateCompetitions :batchexec
|
||||
INSERT INTO competitions (
|
||||
slug,
|
||||
display_name,
|
||||
|
27
model/sql/game_player.sql
Normal file
27
model/sql/game_player.sql
Normal file
@ -0,0 +1,27 @@
|
||||
-- name: BatchInsertGamePlayers :copyfrom
|
||||
INSERT INTO game_players(
|
||||
game_id,
|
||||
player_slug,
|
||||
status,
|
||||
team_slug)
|
||||
VALUES(
|
||||
$1,
|
||||
$2,
|
||||
$3,
|
||||
$4);
|
||||
|
||||
-- name: CreateOrUpdateGamePlayer :exec
|
||||
INSERT INTO game_players(
|
||||
game_id,
|
||||
player_slug,
|
||||
status,
|
||||
team_slug)
|
||||
VALUES(
|
||||
$1,
|
||||
$2,
|
||||
$3,
|
||||
$4)
|
||||
ON CONFLICT (game_id, player_slug) DO UPDATE
|
||||
SET status = $3,
|
||||
team_slug = $4;
|
||||
|
3
model/sql/player.sql
Normal file
3
model/sql/player.sql
Normal file
@ -0,0 +1,3 @@
|
||||
-- name: BatchInsertPlayers :copyfrom
|
||||
INSERT INTO players (slug, display_name, birth_date, country_slug, team_slug, domestic_league_slug, avatar_url, field_position, status, shirt_number)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10);
|
@ -37,74 +37,3 @@ func GetGamesFromFixtures(
|
||||
log.Debug().Msgf("found total of %d games after filtering duplicate ids", len(games))
|
||||
return games, nil
|
||||
}
|
||||
|
||||
func ExtractTeamSlugsFromGames(games []football.Game) []string {
|
||||
var res []string
|
||||
res = lo.Map(games, func(game football.Game, index int) string {
|
||||
return game.AwayTeam.Team.Slug
|
||||
})
|
||||
res = append(res, lo.Map(games, func(game football.Game, index int) string {
|
||||
return game.HomeTeam.Team.Slug
|
||||
})...)
|
||||
res = lo.Filter(res, func(slug string, index int) bool {
|
||||
return slug != ""
|
||||
})
|
||||
return lo.Uniq(res)
|
||||
}
|
||||
|
||||
func ExtractCountrySlugsFromCompetitions(competitions []football.Competition) []string {
|
||||
return lo.Uniq(lo.Filter(lo.Map(competitions, func(competition football.Competition, index int) string {
|
||||
return competition.Country.Slug
|
||||
}), func(slug string, index int) bool {
|
||||
return slug != ""
|
||||
}))
|
||||
}
|
||||
|
||||
func ExtractCountrySlugsFromTeams(teams []football.Team) []string {
|
||||
return lo.Uniq(lo.Filter(lo.Map(teams, func(team football.Team, index int) string {
|
||||
return team.Country.Slug
|
||||
}), func(slug string, index int) bool {
|
||||
return slug != ""
|
||||
}))
|
||||
}
|
||||
|
||||
func ExtractCountrySlugsFromCompetitionsClubsAndNationalTeams(
|
||||
competitions []football.Competition,
|
||||
clubs []football.Club,
|
||||
nationalTeams []football.NationalTeam,
|
||||
) []string {
|
||||
return lo.Uniq(lo.Union(
|
||||
ExtractCountrySlugsFromCompetitions(competitions),
|
||||
ExtractCountrySlugsFromTeams(lo.Map(clubs, func(club football.Club, index int) football.Team {
|
||||
return club.Team
|
||||
})),
|
||||
ExtractCountrySlugsFromTeams(
|
||||
lo.Map(nationalTeams, func(nationalTeam football.NationalTeam, index int) football.Team {
|
||||
return nationalTeam.Team
|
||||
}),
|
||||
),
|
||||
))
|
||||
}
|
||||
|
||||
func ExtractCompetitionSlugsFromGames(games []football.Game) []string {
|
||||
return lo.Uniq(lo.Filter(lo.Map(games, func(game football.Game, index int) string {
|
||||
return game.Competition.Slug
|
||||
}), func(slug string, index int) bool {
|
||||
return slug != ""
|
||||
}))
|
||||
}
|
||||
|
||||
func ExtractCompetitionSlugsFromClubs(clubs []football.Club) []string {
|
||||
return lo.Uniq(lo.Filter(lo.Map(clubs, func(club football.Club, index int) string {
|
||||
return club.DomesticLeague.Slug
|
||||
}), func(slug string, index int) bool {
|
||||
return slug != ""
|
||||
}))
|
||||
}
|
||||
|
||||
func ExtractCompetitionSlugsFromGamesAndClubs(games []football.Game, clubs []football.Club) []string {
|
||||
return lo.Uniq(lo.Union(
|
||||
ExtractCompetitionSlugsFromGames(games),
|
||||
ExtractCompetitionSlugsFromClubs(clubs),
|
||||
))
|
||||
}
|
||||
|
37
sorare_utils/game.go
Normal file
37
sorare_utils/game.go
Normal file
@ -0,0 +1,37 @@
|
||||
package sorare_utils
|
||||
|
||||
import (
|
||||
"git.lehouerou.net/laurent/sorare/football"
|
||||
"github.com/jackc/pgx/v5/pgtype"
|
||||
|
||||
"git.lehouerou.net/laurent/sorarebuddy/model"
|
||||
)
|
||||
|
||||
func NewBatchInsertGamesParamsFromSorare(game football.Game) model.BatchInsertGamesParams {
|
||||
return model.BatchInsertGamesParams{
|
||||
ID: game.Id.Value,
|
||||
Date: pgtype.Timestamptz{Time: game.Date, Valid: true},
|
||||
CoverageStatus: game.CoverageStatus,
|
||||
LowCoverage: game.LowCoverage,
|
||||
Minutes: int32(game.Minute),
|
||||
PeriodType: game.PeriodType,
|
||||
Scored: game.Scored,
|
||||
Status: game.Status,
|
||||
CompetitionSlug: game.Competition.Slug,
|
||||
FixtureSlug: game.So5Fixture.Slug,
|
||||
AwayTeamSlug: game.AwayTeam.Team.Slug,
|
||||
AwayGoals: int32(game.AwayGoals),
|
||||
AwayExtraTimeScore: int32(game.ExtraTimeScoreAway),
|
||||
AwayPenaltyScore: int32(game.PenaltyScoreAway),
|
||||
HomeTeamSlug: game.HomeTeam.Team.Slug,
|
||||
HomeGoals: int32(game.HomeGoals),
|
||||
HomeExtraTimeScore: int32(game.ExtraTimeScoreHome),
|
||||
HomePenaltyScore: int32(game.PenaltyScoreHome),
|
||||
WinnerTeamSlug: func() *string {
|
||||
if game.Winner.Team.Slug == "" {
|
||||
return nil
|
||||
}
|
||||
return &game.Winner.Team.Slug
|
||||
}(),
|
||||
}
|
||||
}
|
1
sorare_utils/game_player.go
Normal file
1
sorare_utils/game_player.go
Normal file
@ -0,0 +1 @@
|
||||
package sorare_utils
|
@ -7,6 +7,8 @@ import (
|
||||
"git.lehouerou.net/laurent/sorare/football"
|
||||
"git.lehouerou.net/laurent/sorare/graphql"
|
||||
"github.com/jackc/pgx/v5/pgtype"
|
||||
gql "github.com/llehouerou/go-graphql-client"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/rs/zerolog/log"
|
||||
"github.com/samber/lo"
|
||||
|
||||
@ -24,7 +26,9 @@ func NewUpdateService(s *sorare.Sorare, db *model.Queries) *UpdateService {
|
||||
|
||||
func (u *UpdateService) InitSyncDatabase(ctx context.Context) error {
|
||||
|
||||
sfixtures, err := u.s.Football.So5.So5Fixtures.Get(ctx, football.So5FixturesParams{})
|
||||
sfixtures, err := u.s.Football.So5.So5Fixtures.Get(ctx, football.So5FixturesParams{
|
||||
AasmStates: []string{"started"},
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@ -62,13 +66,45 @@ func (u *UpdateService) InitSyncDatabase(ctx context.Context) error {
|
||||
)
|
||||
log.Info().Msgf("found %d games to process", len(games))
|
||||
|
||||
teamSlugs := ExtractTeamSlugsFromGames(games)
|
||||
log.Debug().Msgf("extracted %d unique team slugs from games", len(teamSlugs))
|
||||
log.Debug().Msgf("getting players for each game...")
|
||||
var gamePlayers []model.BatchInsertGamePlayersParams
|
||||
for _, game := range games {
|
||||
gameWithFormation, err := u.s.Football.Game.Get(ctx, graphql.IdParams{Id: gql.ID(game.Id.Value)})
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "getting game with formation %s", game.Id.Value)
|
||||
}
|
||||
newplayers := ExtractPlayersFromGameWithFormation(gameWithFormation)
|
||||
log.Debug().Msgf("\t%s -> %d players", game.String(), len(newplayers))
|
||||
gamePlayers = append(gamePlayers, newplayers...)
|
||||
}
|
||||
|
||||
playerSlugs := lo.Uniq(
|
||||
lo.Filter(lo.Map(gamePlayers, func(player model.BatchInsertGamePlayersParams, index int) string {
|
||||
return player.PlayerSlug
|
||||
}), func(slug string, index int) bool {
|
||||
return slug != ""
|
||||
}),
|
||||
)
|
||||
|
||||
log.Debug().Msgf("getting players...")
|
||||
var players []football.Player
|
||||
for i, chunk := range lo.Chunk(playerSlugs, 80) {
|
||||
log.Debug().Msgf("\tbatch %d/%d", i+1, (len(playerSlugs)/80)+1)
|
||||
p, err := u.s.Football.Players.Get(ctx, graphql.SlugsParams{Slugs: chunk})
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "getting players batch %d", i)
|
||||
}
|
||||
players = append(players, p...)
|
||||
}
|
||||
log.Debug().Msgf("found %d players", len(players))
|
||||
|
||||
teamSlugs := ExtractTeamSlugsFromPlayersAndGames(players, games)
|
||||
log.Debug().Msgf("extracted %d unique team slugs from games and players", len(teamSlugs))
|
||||
|
||||
log.Debug().Msgf("getting clubs...")
|
||||
var clubs []football.Club
|
||||
for i, chunk := range lo.Chunk(teamSlugs, 100) {
|
||||
log.Debug().Msgf("\tbatch %d/%d", i+1, len(teamSlugs)/100)
|
||||
log.Debug().Msgf("\tbatch %d/%d", i+1, (len(teamSlugs)/100)+1)
|
||||
t, err := u.s.Football.Clubs.Get(ctx, graphql.SlugsParams{Slugs: chunk})
|
||||
if err != nil {
|
||||
return err
|
||||
@ -83,7 +119,7 @@ func (u *UpdateService) InitSyncDatabase(ctx context.Context) error {
|
||||
log.Debug().Msgf("getting national teams...")
|
||||
log.Debug().Msgf("slugs left: %d", len(slugsLeft))
|
||||
for i, chunk := range lo.Chunk(slugsLeft, 100) {
|
||||
log.Debug().Msgf("\tbatch %d/%d", i+1, len(teamSlugs)/100)
|
||||
log.Debug().Msgf("\tbatch %d/%d", i+1, (len(teamSlugs)/100)+1)
|
||||
t, err := u.s.Football.NationalTeams.Get(ctx, graphql.SlugsParams{Slugs: chunk})
|
||||
if err != nil {
|
||||
return err
|
||||
@ -92,8 +128,8 @@ func (u *UpdateService) InitSyncDatabase(ctx context.Context) error {
|
||||
}
|
||||
log.Debug().Msgf("found %d national teams", len(nationalTeams))
|
||||
|
||||
competitionSlugs := ExtractCompetitionSlugsFromGamesAndClubs(games, clubs)
|
||||
log.Debug().Msgf("extracted %d unique competition slugs from games and clubs", len(competitionSlugs))
|
||||
competitionSlugs := ExtractCompetitionSlugsFromPlayersGamesAndClubs(players, games, clubs)
|
||||
log.Debug().Msgf("extracted %d unique competition slugs from players, games and clubs", len(competitionSlugs))
|
||||
log.Debug().Msgf("getting competitions...")
|
||||
var competitions []football.Competition
|
||||
for _, slug := range competitionSlugs {
|
||||
@ -106,16 +142,18 @@ func (u *UpdateService) InitSyncDatabase(ctx context.Context) error {
|
||||
}
|
||||
log.Debug().Msgf("found %d competitions", len(competitions))
|
||||
|
||||
countrySlugs := ExtractCountrySlugsFromCompetitionsClubsAndNationalTeams(
|
||||
countrySlugs := ExtractCountrySlugsFromPlayersCompetitionsClubsAndNationalTeams(
|
||||
players,
|
||||
competitions,
|
||||
clubs,
|
||||
nationalTeams,
|
||||
)
|
||||
log.Debug().Msgf("extracted %d unique country slugs from competitions, clubs and national teams", len(countrySlugs))
|
||||
log.Debug().
|
||||
Msgf("extracted %d unique country slugs from players, competitions, clubs and national teams", len(countrySlugs))
|
||||
log.Debug().Msgf("getting countries...")
|
||||
var countries []sorare.Country
|
||||
for i, chunk := range lo.Chunk(countrySlugs, 100) {
|
||||
log.Debug().Msgf("\tbatch %d/%d", i+1, len(countrySlugs)/100)
|
||||
log.Debug().Msgf("\tbatch %d/%d", i+1, (len(countrySlugs)/100)+1)
|
||||
c, err := u.s.Countries.Get(ctx, graphql.SlugsParams{Slugs: chunk})
|
||||
if err != nil {
|
||||
return err
|
||||
@ -147,10 +185,11 @@ func (u *UpdateService) InitSyncDatabase(ctx context.Context) error {
|
||||
|
||||
log.Debug().Msg("inserting competitions into db...")
|
||||
|
||||
_, err = u.db.BatchInsertCompetitions(
|
||||
var batcherr error
|
||||
batch := u.db.CreateOrUpdateCompetitions(
|
||||
ctx,
|
||||
lo.Map(competitions, func(competition football.Competition, index int) model.BatchInsertCompetitionsParams {
|
||||
return model.BatchInsertCompetitionsParams{
|
||||
lo.Map(competitions, func(competition football.Competition, index int) model.CreateOrUpdateCompetitionsParams {
|
||||
return model.CreateOrUpdateCompetitionsParams{
|
||||
Slug: competition.Slug,
|
||||
CompetitionFormat: competition.Format,
|
||||
CompetitionType: competition.Type,
|
||||
@ -161,8 +200,14 @@ func (u *UpdateService) InitSyncDatabase(ctx context.Context) error {
|
||||
}
|
||||
}),
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
batch.Exec(func(_ int, err error) {
|
||||
if err != nil {
|
||||
batcherr = err
|
||||
batch.Close()
|
||||
}
|
||||
})
|
||||
if batcherr != nil {
|
||||
return errors.Wrap(batcherr, "inserting competitions")
|
||||
}
|
||||
|
||||
log.Debug().Msgf("%d competitions inserted", len(competitions))
|
||||
@ -206,37 +251,211 @@ func (u *UpdateService) InitSyncDatabase(ctx context.Context) error {
|
||||
cnt, err = u.db.BatchInsertGames(
|
||||
ctx,
|
||||
lo.Map(games, func(game football.Game, index int) model.BatchInsertGamesParams {
|
||||
return model.BatchInsertGamesParams{
|
||||
ID: game.Id.Value,
|
||||
Date: pgtype.Timestamptz{Time: game.Date, Valid: true},
|
||||
CoverageStatus: game.CoverageStatus,
|
||||
LowCoverage: game.LowCoverage,
|
||||
Minutes: int32(game.Minute),
|
||||
PeriodType: game.PeriodType,
|
||||
Scored: game.Scored,
|
||||
Status: game.Status,
|
||||
CompetitionSlug: game.Competition.Slug,
|
||||
FixtureSlug: game.So5Fixture.Slug,
|
||||
AwayTeamSlug: game.AwayTeam.Team.Slug,
|
||||
AwayGoals: int32(game.AwayGoals),
|
||||
AwayExtraTimeScore: int32(game.ExtraTimeScoreAway),
|
||||
AwayPenaltyScore: int32(game.PenaltyScoreAway),
|
||||
HomeTeamSlug: game.HomeTeam.Team.Slug,
|
||||
HomeGoals: int32(game.HomeGoals),
|
||||
HomeExtraTimeScore: int32(game.ExtraTimeScoreHome),
|
||||
HomePenaltyScore: int32(game.PenaltyScoreHome),
|
||||
WinnerTeamSlug: func() *string {
|
||||
if game.Winner.Team.Slug == "" {
|
||||
return nil
|
||||
}
|
||||
return &game.Winner.Team.Slug
|
||||
}(),
|
||||
}
|
||||
return NewBatchInsertGamesParamsFromSorare(game)
|
||||
}),
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
log.Debug().Msgf("%d games inserted", cnt)
|
||||
|
||||
log.Debug().Msg("inserting players into db...")
|
||||
cnt, err = u.db.BatchInsertPlayers(
|
||||
ctx,
|
||||
lo.Map(players, func(player football.Player, index int) model.BatchInsertPlayersParams {
|
||||
res := model.BatchInsertPlayersParams{
|
||||
Slug: player.Slug,
|
||||
DisplayName: player.DisplayName,
|
||||
BirthDate: player.BirthDate,
|
||||
CountrySlug: player.Country.Slug,
|
||||
AvatarUrl: player.AvatarUrl,
|
||||
FieldPosition: string(player.Position),
|
||||
Status: string(player.PlayingStatus),
|
||||
ShirtNumber: int32(player.ShirtNumber),
|
||||
}
|
||||
for _, competition := range player.ActiveClub.ActiveCompetitions {
|
||||
if competition.Format == "DOMESTIC_LEAGUE" {
|
||||
res.DomesticLeagueSlug = &competition.Slug
|
||||
}
|
||||
}
|
||||
if player.ActiveClub.Slug != "" {
|
||||
res.TeamSlug = &player.ActiveClub.Slug
|
||||
}
|
||||
return res
|
||||
}),
|
||||
)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "inserting players")
|
||||
}
|
||||
log.Debug().Msgf("%d players inserted", cnt)
|
||||
|
||||
log.Debug().Msg("inserting game players into db...")
|
||||
cnt, err = u.db.BatchInsertGamePlayers(ctx, gamePlayers)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "inserting game players")
|
||||
}
|
||||
log.Debug().Msgf("%d game players inserted", cnt)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func ExtractPlayersFromGameWithFormation(
|
||||
gameWithFormation football.GameWithFormation,
|
||||
) []model.BatchInsertGamePlayersParams {
|
||||
var res []model.BatchInsertGamePlayersParams
|
||||
for _, p := range gameWithFormation.HomeFormation.Bench {
|
||||
res = append(res, model.BatchInsertGamePlayersParams{
|
||||
GameID: gameWithFormation.Id.Value,
|
||||
PlayerSlug: p.Slug,
|
||||
TeamSlug: gameWithFormation.HomeTeam.Team.Slug,
|
||||
Status: "bench",
|
||||
})
|
||||
}
|
||||
for _, p := range gameWithFormation.HomeFormation.StartingLineup {
|
||||
for _, q := range p {
|
||||
res = append(res, model.BatchInsertGamePlayersParams{
|
||||
GameID: gameWithFormation.Id.Value,
|
||||
PlayerSlug: q.Slug,
|
||||
TeamSlug: gameWithFormation.HomeTeam.Team.Slug,
|
||||
Status: "starting",
|
||||
})
|
||||
}
|
||||
}
|
||||
for _, p := range gameWithFormation.AwayFormation.Bench {
|
||||
res = append(res, model.BatchInsertGamePlayersParams{
|
||||
GameID: gameWithFormation.Id.Value,
|
||||
PlayerSlug: p.Slug,
|
||||
TeamSlug: gameWithFormation.AwayTeam.Team.Slug,
|
||||
Status: "bench",
|
||||
})
|
||||
}
|
||||
for _, p := range gameWithFormation.AwayFormation.StartingLineup {
|
||||
for _, q := range p {
|
||||
res = append(res, model.BatchInsertGamePlayersParams{
|
||||
GameID: gameWithFormation.Id.Value,
|
||||
PlayerSlug: q.Slug,
|
||||
TeamSlug: gameWithFormation.AwayTeam.Team.Slug,
|
||||
Status: "starting",
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return res
|
||||
|
||||
}
|
||||
|
||||
func ExtractTeamSlugsFromPlayersAndGames(players []football.Player, games []football.Game) []string {
|
||||
return lo.Uniq(lo.Union(
|
||||
ExtractTeamSlugsFromPlayers(players),
|
||||
ExtractTeamSlugsFromGames(games),
|
||||
))
|
||||
}
|
||||
|
||||
func ExtractTeamSlugsFromPlayers(players []football.Player) []string {
|
||||
return lo.Uniq(lo.Filter(lo.Map(players, func(player football.Player, index int) string {
|
||||
return player.ActiveClub.Slug
|
||||
}), func(slug string, index int) bool {
|
||||
return slug != ""
|
||||
}))
|
||||
}
|
||||
|
||||
func ExtractTeamSlugsFromGames(games []football.Game) []string {
|
||||
var res []string
|
||||
res = lo.Map(games, func(game football.Game, index int) string {
|
||||
return game.AwayTeam.Team.Slug
|
||||
})
|
||||
res = append(res, lo.Map(games, func(game football.Game, index int) string {
|
||||
return game.HomeTeam.Team.Slug
|
||||
})...)
|
||||
res = lo.Filter(res, func(slug string, index int) bool {
|
||||
return slug != ""
|
||||
})
|
||||
return lo.Uniq(res)
|
||||
}
|
||||
|
||||
func ExtractCountrySlugsFromCompetitions(competitions []football.Competition) []string {
|
||||
return lo.Uniq(lo.Filter(lo.Map(competitions, func(competition football.Competition, index int) string {
|
||||
return competition.Country.Slug
|
||||
}), func(slug string, index int) bool {
|
||||
return slug != ""
|
||||
}))
|
||||
}
|
||||
|
||||
func ExtractCountrySlugsFromTeams(teams []football.Team) []string {
|
||||
return lo.Uniq(lo.Filter(lo.Map(teams, func(team football.Team, index int) string {
|
||||
return team.Country.Slug
|
||||
}), func(slug string, index int) bool {
|
||||
return slug != ""
|
||||
}))
|
||||
}
|
||||
|
||||
func ExtractCountrySlugsFromPlayersCompetitionsClubsAndNationalTeams(
|
||||
players []football.Player,
|
||||
competitions []football.Competition,
|
||||
clubs []football.Club,
|
||||
nationalTeams []football.NationalTeam,
|
||||
) []string {
|
||||
return lo.Uniq(lo.Union(
|
||||
ExtractCountrySlugsFromPlayers(players),
|
||||
ExtractCountrySlugsFromCompetitions(competitions),
|
||||
ExtractCountrySlugsFromTeams(lo.Map(clubs, func(club football.Club, index int) football.Team {
|
||||
return club.Team
|
||||
})),
|
||||
ExtractCountrySlugsFromTeams(
|
||||
lo.Map(nationalTeams, func(nationalTeam football.NationalTeam, index int) football.Team {
|
||||
return nationalTeam.Team
|
||||
}),
|
||||
),
|
||||
))
|
||||
}
|
||||
|
||||
func ExtractCountrySlugsFromPlayers(players []football.Player) []string {
|
||||
return lo.Uniq(lo.Filter(lo.Map(players, func(player football.Player, index int) string {
|
||||
return player.Country.Slug
|
||||
}), func(slug string, index int) bool {
|
||||
return slug != ""
|
||||
}))
|
||||
|
||||
}
|
||||
|
||||
func ExtractCompetitionSlugsFromPlayers(players []football.Player) []string {
|
||||
return lo.Uniq(lo.Filter(lo.Map(players, func(player football.Player, index int) string {
|
||||
var res string
|
||||
for _, competition := range player.ActiveClub.ActiveCompetitions {
|
||||
if competition.Format == "DOMESTIC_LEAGUE" {
|
||||
res = competition.Slug
|
||||
}
|
||||
}
|
||||
return res
|
||||
}), func(slug string, index int) bool {
|
||||
return slug != ""
|
||||
}))
|
||||
}
|
||||
|
||||
func ExtractCompetitionSlugsFromGames(games []football.Game) []string {
|
||||
return lo.Uniq(lo.Filter(lo.Map(games, func(game football.Game, index int) string {
|
||||
return game.Competition.Slug
|
||||
}), func(slug string, index int) bool {
|
||||
return slug != ""
|
||||
}))
|
||||
}
|
||||
|
||||
func ExtractCompetitionSlugsFromClubs(clubs []football.Club) []string {
|
||||
return lo.Uniq(lo.Filter(lo.Map(clubs, func(club football.Club, index int) string {
|
||||
return club.DomesticLeague.Slug
|
||||
}), func(slug string, index int) bool {
|
||||
return slug != ""
|
||||
}))
|
||||
}
|
||||
|
||||
func ExtractCompetitionSlugsFromPlayersGamesAndClubs(
|
||||
players []football.Player,
|
||||
games []football.Game,
|
||||
clubs []football.Club,
|
||||
) []string {
|
||||
return lo.Uniq(lo.Union(
|
||||
ExtractCompetitionSlugsFromPlayers(players),
|
||||
ExtractCompetitionSlugsFromGames(games),
|
||||
ExtractCompetitionSlugsFromClubs(clubs),
|
||||
))
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user