migrate from copyfrom to batchexec for inserts during init of db

This commit is contained in:
Laurent Le Houerou 2024-03-23 11:39:37 +00:00
parent 9494bbe760
commit 12cf5dd6e1
14 changed files with 362 additions and 321 deletions

2
go.mod

@ -4,6 +4,7 @@ go 1.22.1
require (
github.com/jackc/pgx/v5 v5.5.5
github.com/llehouerou/go-graphql-client v0.9.6
github.com/shopspring/decimal v1.3.1
github.com/spf13/cobra v1.8.0
github.com/spf13/viper v1.18.2
@ -18,7 +19,6 @@ require (
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/klauspost/compress v1.17.0 // indirect
github.com/llehouerou/go-graphql-client v0.9.6 // indirect
github.com/magiconair/properties v1.8.7 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.19 // indirect

@ -8,6 +8,7 @@ package model
import (
"context"
"errors"
"time"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgtype"
@ -232,3 +233,284 @@ func (b *CreateOrUpdateFixturesBatchResults) Close() error {
b.closed = true
return b.br.Close()
}
const createOrUpdateGamePlayers = `-- name: CreateOrUpdateGamePlayers :batchexec
INSERT INTO game_players(
game_id,
player_slug,
status,
team_slug)
VALUES(
$1,
$2,
$3,
$4)
ON CONFLICT (game_id, player_slug) DO UPDATE
SET status = $3,
team_slug = $4
`
type CreateOrUpdateGamePlayersBatchResults struct {
br pgx.BatchResults
tot int
closed bool
}
type CreateOrUpdateGamePlayersParams struct {
GameID string
PlayerSlug string
Status string
TeamSlug string
}
func (q *Queries) CreateOrUpdateGamePlayers(ctx context.Context, arg []CreateOrUpdateGamePlayersParams) *CreateOrUpdateGamePlayersBatchResults {
batch := &pgx.Batch{}
for _, a := range arg {
vals := []interface{}{
a.GameID,
a.PlayerSlug,
a.Status,
a.TeamSlug,
}
batch.Queue(createOrUpdateGamePlayers, vals...)
}
br := q.db.SendBatch(ctx, batch)
return &CreateOrUpdateGamePlayersBatchResults{br, len(arg), false}
}
func (b *CreateOrUpdateGamePlayersBatchResults) Exec(f func(int, error)) {
defer b.br.Close()
for t := 0; t < b.tot; t++ {
if b.closed {
if f != nil {
f(t, ErrBatchAlreadyClosed)
}
continue
}
_, err := b.br.Exec()
if f != nil {
f(t, err)
}
}
}
func (b *CreateOrUpdateGamePlayersBatchResults) Close() error {
b.closed = true
return b.br.Close()
}
const createOrUpdateGames = `-- name: CreateOrUpdateGames :batchexec
INSERT INTO games (id, date, coverage_status, low_coverage, minutes, period_type, scored, status, competition_slug, fixture_slug, away_team_slug, away_goals, away_extra_time_score, away_penalty_score, home_team_slug, home_goals, home_extra_time_score, home_penalty_score, winner_team_slug)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19)
ON CONFLICT (id) DO UPDATE
SET date = EXCLUDED.date, coverage_status = EXCLUDED.coverage_status, low_coverage = EXCLUDED.low_coverage, minutes = EXCLUDED.minutes, period_type = EXCLUDED.period_type, scored = EXCLUDED.scored, status = EXCLUDED.status, competition_slug = EXCLUDED.competition_slug, fixture_slug = EXCLUDED.fixture_slug, away_team_slug = EXCLUDED.away_team_slug, away_goals = EXCLUDED.away_goals, away_extra_time_score = EXCLUDED.away_extra_time_score, away_penalty_score = EXCLUDED.away_penalty_score, home_team_slug = EXCLUDED.home_team_slug, home_goals = EXCLUDED.home_goals, home_extra_time_score = EXCLUDED.home_extra_time_score, home_penalty_score = EXCLUDED.home_penalty_score, winner_team_slug = EXCLUDED.winner_team_slug
`
type CreateOrUpdateGamesBatchResults struct {
br pgx.BatchResults
tot int
closed bool
}
type CreateOrUpdateGamesParams struct {
ID string
Date pgtype.Timestamptz
CoverageStatus string
LowCoverage bool
Minutes int32
PeriodType string
Scored bool
Status string
CompetitionSlug string
FixtureSlug string
AwayTeamSlug string
AwayGoals int32
AwayExtraTimeScore int32
AwayPenaltyScore int32
HomeTeamSlug string
HomeGoals int32
HomeExtraTimeScore int32
HomePenaltyScore int32
WinnerTeamSlug *string
}
func (q *Queries) CreateOrUpdateGames(ctx context.Context, arg []CreateOrUpdateGamesParams) *CreateOrUpdateGamesBatchResults {
batch := &pgx.Batch{}
for _, a := range arg {
vals := []interface{}{
a.ID,
a.Date,
a.CoverageStatus,
a.LowCoverage,
a.Minutes,
a.PeriodType,
a.Scored,
a.Status,
a.CompetitionSlug,
a.FixtureSlug,
a.AwayTeamSlug,
a.AwayGoals,
a.AwayExtraTimeScore,
a.AwayPenaltyScore,
a.HomeTeamSlug,
a.HomeGoals,
a.HomeExtraTimeScore,
a.HomePenaltyScore,
a.WinnerTeamSlug,
}
batch.Queue(createOrUpdateGames, vals...)
}
br := q.db.SendBatch(ctx, batch)
return &CreateOrUpdateGamesBatchResults{br, len(arg), false}
}
func (b *CreateOrUpdateGamesBatchResults) Exec(f func(int, error)) {
defer b.br.Close()
for t := 0; t < b.tot; t++ {
if b.closed {
if f != nil {
f(t, ErrBatchAlreadyClosed)
}
continue
}
_, err := b.br.Exec()
if f != nil {
f(t, err)
}
}
}
func (b *CreateOrUpdateGamesBatchResults) Close() error {
b.closed = true
return b.br.Close()
}
const createOrUpdatePlayers = `-- name: CreateOrUpdatePlayers :batchexec
INSERT INTO players (slug, display_name, birth_date, country_slug, team_slug, domestic_league_slug, avatar_url, field_position, status, shirt_number)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
ON CONFLICT (slug) DO UPDATE
SET display_name = $2, birth_date = $3, country_slug = $4, team_slug = $5, domestic_league_slug = $6, avatar_url = $7, field_position = $8, status = $9, shirt_number = $10
`
type CreateOrUpdatePlayersBatchResults struct {
br pgx.BatchResults
tot int
closed bool
}
type CreateOrUpdatePlayersParams struct {
Slug string
DisplayName string
BirthDate time.Time
CountrySlug string
TeamSlug *string
DomesticLeagueSlug *string
AvatarUrl string
FieldPosition string
Status string
ShirtNumber int32
}
func (q *Queries) CreateOrUpdatePlayers(ctx context.Context, arg []CreateOrUpdatePlayersParams) *CreateOrUpdatePlayersBatchResults {
batch := &pgx.Batch{}
for _, a := range arg {
vals := []interface{}{
a.Slug,
a.DisplayName,
a.BirthDate,
a.CountrySlug,
a.TeamSlug,
a.DomesticLeagueSlug,
a.AvatarUrl,
a.FieldPosition,
a.Status,
a.ShirtNumber,
}
batch.Queue(createOrUpdatePlayers, vals...)
}
br := q.db.SendBatch(ctx, batch)
return &CreateOrUpdatePlayersBatchResults{br, len(arg), false}
}
func (b *CreateOrUpdatePlayersBatchResults) Exec(f func(int, error)) {
defer b.br.Close()
for t := 0; t < b.tot; t++ {
if b.closed {
if f != nil {
f(t, ErrBatchAlreadyClosed)
}
continue
}
_, err := b.br.Exec()
if f != nil {
f(t, err)
}
}
}
func (b *CreateOrUpdatePlayersBatchResults) Close() error {
b.closed = true
return b.br.Close()
}
const createOrUpdateTeams = `-- name: CreateOrUpdateTeams :batchexec
INSERT INTO teams (slug, display_name, country_slug, domestic_league_slug, short_name, picture_url, team_type)
VALUES ($1, $2, $3, $4, $5, $6, $7)
ON CONFLICT (slug) DO UPDATE
SET display_name = $2, country_slug = $3, domestic_league_slug = $4, short_name = $5, picture_url = $6, team_type = $7
`
type CreateOrUpdateTeamsBatchResults struct {
br pgx.BatchResults
tot int
closed bool
}
type CreateOrUpdateTeamsParams struct {
Slug string
DisplayName string
CountrySlug string
DomesticLeagueSlug *string
ShortName string
PictureUrl string
TeamType string
}
func (q *Queries) CreateOrUpdateTeams(ctx context.Context, arg []CreateOrUpdateTeamsParams) *CreateOrUpdateTeamsBatchResults {
batch := &pgx.Batch{}
for _, a := range arg {
vals := []interface{}{
a.Slug,
a.DisplayName,
a.CountrySlug,
a.DomesticLeagueSlug,
a.ShortName,
a.PictureUrl,
a.TeamType,
}
batch.Queue(createOrUpdateTeams, vals...)
}
br := q.db.SendBatch(ctx, batch)
return &CreateOrUpdateTeamsBatchResults{br, len(arg), false}
}
func (b *CreateOrUpdateTeamsBatchResults) Exec(f func(int, error)) {
defer b.br.Close()
for t := 0; t < b.tot; t++ {
if b.closed {
if f != nil {
f(t, ErrBatchAlreadyClosed)
}
continue
}
_, err := b.br.Exec()
if f != nil {
f(t, err)
}
}
}
func (b *CreateOrUpdateTeamsBatchResults) Close() error {
b.closed = true
return b.br.Close()
}

@ -1,174 +0,0 @@
// Code generated by sqlc. DO NOT EDIT.
// versions:
// sqlc v1.25.0
// source: copyfrom.go
package model
import (
"context"
)
// iteratorForBatchInsertGamePlayers implements pgx.CopyFromSource.
type iteratorForBatchInsertGamePlayers struct {
rows []BatchInsertGamePlayersParams
skippedFirstNextCall bool
}
func (r *iteratorForBatchInsertGamePlayers) Next() bool {
if len(r.rows) == 0 {
return false
}
if !r.skippedFirstNextCall {
r.skippedFirstNextCall = true
return true
}
r.rows = r.rows[1:]
return len(r.rows) > 0
}
func (r iteratorForBatchInsertGamePlayers) Values() ([]interface{}, error) {
return []interface{}{
r.rows[0].GameID,
r.rows[0].PlayerSlug,
r.rows[0].Status,
r.rows[0].TeamSlug,
}, nil
}
func (r iteratorForBatchInsertGamePlayers) Err() error {
return nil
}
func (q *Queries) BatchInsertGamePlayers(ctx context.Context, arg []BatchInsertGamePlayersParams) (int64, error) {
return q.db.CopyFrom(ctx, []string{"game_players"}, []string{"game_id", "player_slug", "status", "team_slug"}, &iteratorForBatchInsertGamePlayers{rows: arg})
}
// iteratorForBatchInsertGames implements pgx.CopyFromSource.
type iteratorForBatchInsertGames struct {
rows []BatchInsertGamesParams
skippedFirstNextCall bool
}
func (r *iteratorForBatchInsertGames) Next() bool {
if len(r.rows) == 0 {
return false
}
if !r.skippedFirstNextCall {
r.skippedFirstNextCall = true
return true
}
r.rows = r.rows[1:]
return len(r.rows) > 0
}
func (r iteratorForBatchInsertGames) Values() ([]interface{}, error) {
return []interface{}{
r.rows[0].ID,
r.rows[0].Date,
r.rows[0].CoverageStatus,
r.rows[0].LowCoverage,
r.rows[0].Minutes,
r.rows[0].PeriodType,
r.rows[0].Scored,
r.rows[0].Status,
r.rows[0].CompetitionSlug,
r.rows[0].FixtureSlug,
r.rows[0].AwayTeamSlug,
r.rows[0].AwayGoals,
r.rows[0].AwayExtraTimeScore,
r.rows[0].AwayPenaltyScore,
r.rows[0].HomeTeamSlug,
r.rows[0].HomeGoals,
r.rows[0].HomeExtraTimeScore,
r.rows[0].HomePenaltyScore,
r.rows[0].WinnerTeamSlug,
}, nil
}
func (r iteratorForBatchInsertGames) Err() error {
return nil
}
func (q *Queries) BatchInsertGames(ctx context.Context, arg []BatchInsertGamesParams) (int64, error) {
return q.db.CopyFrom(ctx, []string{"games"}, []string{"id", "date", "coverage_status", "low_coverage", "minutes", "period_type", "scored", "status", "competition_slug", "fixture_slug", "away_team_slug", "away_goals", "away_extra_time_score", "away_penalty_score", "home_team_slug", "home_goals", "home_extra_time_score", "home_penalty_score", "winner_team_slug"}, &iteratorForBatchInsertGames{rows: arg})
}
// iteratorForBatchInsertPlayers implements pgx.CopyFromSource.
type iteratorForBatchInsertPlayers struct {
rows []BatchInsertPlayersParams
skippedFirstNextCall bool
}
func (r *iteratorForBatchInsertPlayers) Next() bool {
if len(r.rows) == 0 {
return false
}
if !r.skippedFirstNextCall {
r.skippedFirstNextCall = true
return true
}
r.rows = r.rows[1:]
return len(r.rows) > 0
}
func (r iteratorForBatchInsertPlayers) Values() ([]interface{}, error) {
return []interface{}{
r.rows[0].Slug,
r.rows[0].DisplayName,
r.rows[0].BirthDate,
r.rows[0].CountrySlug,
r.rows[0].TeamSlug,
r.rows[0].DomesticLeagueSlug,
r.rows[0].AvatarUrl,
r.rows[0].FieldPosition,
r.rows[0].Status,
r.rows[0].ShirtNumber,
}, nil
}
func (r iteratorForBatchInsertPlayers) Err() error {
return nil
}
func (q *Queries) BatchInsertPlayers(ctx context.Context, arg []BatchInsertPlayersParams) (int64, error) {
return q.db.CopyFrom(ctx, []string{"players"}, []string{"slug", "display_name", "birth_date", "country_slug", "team_slug", "domestic_league_slug", "avatar_url", "field_position", "status", "shirt_number"}, &iteratorForBatchInsertPlayers{rows: arg})
}
// iteratorForBatchInsertTeams implements pgx.CopyFromSource.
type iteratorForBatchInsertTeams struct {
rows []BatchInsertTeamsParams
skippedFirstNextCall bool
}
func (r *iteratorForBatchInsertTeams) Next() bool {
if len(r.rows) == 0 {
return false
}
if !r.skippedFirstNextCall {
r.skippedFirstNextCall = true
return true
}
r.rows = r.rows[1:]
return len(r.rows) > 0
}
func (r iteratorForBatchInsertTeams) Values() ([]interface{}, error) {
return []interface{}{
r.rows[0].Slug,
r.rows[0].DisplayName,
r.rows[0].CountrySlug,
r.rows[0].DomesticLeagueSlug,
r.rows[0].ShortName,
r.rows[0].PictureUrl,
r.rows[0].TeamType,
}, nil
}
func (r iteratorForBatchInsertTeams) Err() error {
return nil
}
func (q *Queries) BatchInsertTeams(ctx context.Context, arg []BatchInsertTeamsParams) (int64, error) {
return q.db.CopyFrom(ctx, []string{"teams"}, []string{"slug", "display_name", "country_slug", "domestic_league_slug", "short_name", "picture_url", "team_type"}, &iteratorForBatchInsertTeams{rows: arg})
}

@ -15,7 +15,6 @@ type DBTX interface {
Exec(context.Context, string, ...interface{}) (pgconn.CommandTag, error)
Query(context.Context, string, ...interface{}) (pgx.Rows, error)
QueryRow(context.Context, string, ...interface{}) pgx.Row
CopyFrom(ctx context.Context, tableName pgx.Identifier, columnNames []string, rowSrc pgx.CopyFromSource) (int64, error)
SendBatch(context.Context, *pgx.Batch) pgx.BatchResults
}

@ -5,28 +5,4 @@
package model
import (
"github.com/jackc/pgx/v5/pgtype"
)
type BatchInsertGamesParams struct {
ID string
Date pgtype.Timestamptz
CoverageStatus string
LowCoverage bool
Minutes int32
PeriodType string
Scored bool
Status string
CompetitionSlug string
FixtureSlug string
AwayTeamSlug string
AwayGoals int32
AwayExtraTimeScore int32
AwayPenaltyScore int32
HomeTeamSlug string
HomeGoals int32
HomeExtraTimeScore int32
HomePenaltyScore int32
WinnerTeamSlug *string
}
import ()

@ -5,46 +5,4 @@
package model
import (
"context"
)
type BatchInsertGamePlayersParams struct {
GameID string
PlayerSlug string
Status string
TeamSlug string
}
const createOrUpdateGamePlayer = `-- name: CreateOrUpdateGamePlayer :exec
INSERT INTO game_players(
game_id,
player_slug,
status,
team_slug)
VALUES(
$1,
$2,
$3,
$4)
ON CONFLICT (game_id, player_slug) DO UPDATE
SET status = $3,
team_slug = $4
`
type CreateOrUpdateGamePlayerParams struct {
GameID string
PlayerSlug string
Status string
TeamSlug string
}
func (q *Queries) CreateOrUpdateGamePlayer(ctx context.Context, arg CreateOrUpdateGamePlayerParams) error {
_, err := q.db.Exec(ctx, createOrUpdateGamePlayer,
arg.GameID,
arg.PlayerSlug,
arg.Status,
arg.TeamSlug,
)
return err
}
import ()

@ -5,19 +5,4 @@
package model
import (
"time"
)
type BatchInsertPlayersParams struct {
Slug string
DisplayName string
BirthDate time.Time
CountrySlug string
TeamSlug *string
DomesticLeagueSlug *string
AvatarUrl string
FieldPosition string
Status string
ShirtNumber int32
}
import ()

@ -1,4 +1,6 @@
-- name: BatchInsertGames :copyfrom
-- name: CreateOrUpdateGames :batchexec
INSERT INTO games (id, date, coverage_status, low_coverage, minutes, period_type, scored, status, competition_slug, fixture_slug, away_team_slug, away_goals, away_extra_time_score, away_penalty_score, home_team_slug, home_goals, home_extra_time_score, home_penalty_score, winner_team_slug)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19);
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19)
ON CONFLICT (id) DO UPDATE
SET date = EXCLUDED.date, coverage_status = EXCLUDED.coverage_status, low_coverage = EXCLUDED.low_coverage, minutes = EXCLUDED.minutes, period_type = EXCLUDED.period_type, scored = EXCLUDED.scored, status = EXCLUDED.status, competition_slug = EXCLUDED.competition_slug, fixture_slug = EXCLUDED.fixture_slug, away_team_slug = EXCLUDED.away_team_slug, away_goals = EXCLUDED.away_goals, away_extra_time_score = EXCLUDED.away_extra_time_score, away_penalty_score = EXCLUDED.away_penalty_score, home_team_slug = EXCLUDED.home_team_slug, home_goals = EXCLUDED.home_goals, home_extra_time_score = EXCLUDED.home_extra_time_score, home_penalty_score = EXCLUDED.home_penalty_score, winner_team_slug = EXCLUDED.winner_team_slug;

@ -1,16 +1,4 @@
-- name: BatchInsertGamePlayers :copyfrom
INSERT INTO game_players(
game_id,
player_slug,
status,
team_slug)
VALUES(
$1,
$2,
$3,
$4);
-- name: CreateOrUpdateGamePlayer :exec
-- name: CreateOrUpdateGamePlayers :batchexec
INSERT INTO game_players(
game_id,
player_slug,

@ -1,3 +1,5 @@
-- name: BatchInsertPlayers :copyfrom
-- name: CreateOrUpdatePlayers :batchexec
INSERT INTO players (slug, display_name, birth_date, country_slug, team_slug, domestic_league_slug, avatar_url, field_position, status, shirt_number)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10);
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
ON CONFLICT (slug) DO UPDATE
SET display_name = $2, birth_date = $3, country_slug = $4, team_slug = $5, domestic_league_slug = $6, avatar_url = $7, field_position = $8, status = $9, shirt_number = $10;

@ -1,3 +1,5 @@
-- name: BatchInsertTeams :copyfrom
-- name: CreateOrUpdateTeams :batchexec
INSERT INTO teams (slug, display_name, country_slug, domestic_league_slug, short_name, picture_url, team_type)
VALUES ($1, $2, $3, $4, $5, $6, $7);
VALUES ($1, $2, $3, $4, $5, $6, $7)
ON CONFLICT (slug) DO UPDATE
SET display_name = $2, country_slug = $3, domestic_league_slug = $4, short_name = $5, picture_url = $6, team_type = $7;

@ -6,13 +6,3 @@
package model
import ()
type BatchInsertTeamsParams struct {
Slug string
DisplayName string
CountrySlug string
DomesticLeagueSlug *string
ShortName string
PictureUrl string
TeamType string
}

@ -7,8 +7,8 @@ import (
"git.lehouerou.net/laurent/sorarebuddy/model"
)
func NewBatchInsertGamesParamsFromSorare(game football.Game) model.BatchInsertGamesParams {
return model.BatchInsertGamesParams{
func NewCreateOrUpdateGamesParamsFromSorare(game football.Game) model.CreateOrUpdateGamesParams {
return model.CreateOrUpdateGamesParams{
ID: game.Id.Value,
Date: pgtype.Timestamptz{Time: game.Date, Valid: true},
CoverageStatus: game.CoverageStatus,

@ -75,7 +75,7 @@ func (u *UpdateService) InitSyncDatabase(ctx context.Context) error {
log.Info().Msgf("found %d games to process", len(games))
log.Debug().Msgf("getting players for each game...")
var gamePlayers []model.BatchInsertGamePlayersParams
var gamePlayers []model.CreateOrUpdateGamePlayersParams
for _, game := range games {
gameWithFormation, err := u.s.Football.Game.Get(ctx, graphql.IdParams{Id: gql.ID(game.Id.Value)})
if err != nil {
@ -87,7 +87,7 @@ func (u *UpdateService) InitSyncDatabase(ctx context.Context) error {
}
playerSlugs := lo.Uniq(
lo.Filter(lo.Map(gamePlayers, func(player model.BatchInsertGamePlayersParams, index int) string {
lo.Filter(lo.Map(gamePlayers, func(player model.CreateOrUpdateGamePlayersParams, index int) string {
return player.PlayerSlug
}), func(slug string, index int) bool {
return slug != ""
@ -228,9 +228,9 @@ func (u *UpdateService) InitSyncDatabase(ctx context.Context) error {
log.Debug().Msgf("%d competitions inserted", len(competitions))
log.Debug().Msg("inserting teams into db...")
cnt, err := u.db.BatchInsertTeams(ctx, lo.Union(
lo.Map(clubs, func(club football.Club, index int) model.BatchInsertTeamsParams {
return model.BatchInsertTeamsParams{
batchTeams := u.db.CreateOrUpdateTeams(ctx, lo.Union(
lo.Map(clubs, func(club football.Club, index int) model.CreateOrUpdateTeamsParams {
return model.CreateOrUpdateTeamsParams{
Slug: club.Slug,
DisplayName: club.Name,
CountrySlug: club.Country.Slug,
@ -245,8 +245,8 @@ func (u *UpdateService) InitSyncDatabase(ctx context.Context) error {
TeamType: "club",
}
}),
lo.Map(nationalTeams, func(nationalTeam football.NationalTeam, index int) model.BatchInsertTeamsParams {
return model.BatchInsertTeamsParams{
lo.Map(nationalTeams, func(nationalTeam football.NationalTeam, index int) model.CreateOrUpdateTeamsParams {
return model.CreateOrUpdateTeamsParams{
Slug: nationalTeam.Slug,
DisplayName: nationalTeam.Name,
CountrySlug: nationalTeam.Country.Slug,
@ -257,28 +257,42 @@ func (u *UpdateService) InitSyncDatabase(ctx context.Context) error {
}
}),
))
if err != nil {
return err
batcherr = nil
batchTeams.Exec(func(_ int, err error) {
if err != nil {
batcherr = err
batchTeams.Close()
}
})
if batcherr != nil {
return errors.Wrap(batcherr, "inserting teams into db")
}
log.Debug().Msgf("%d teams inserted", cnt)
log.Debug().Msgf("%d teams inserted", len(clubs)+len(nationalTeams))
log.Debug().Msg("inserting games into db...")
cnt, err = u.db.BatchInsertGames(
batchGames := u.db.CreateOrUpdateGames(
ctx,
lo.Map(games, func(game football.Game, index int) model.BatchInsertGamesParams {
return NewBatchInsertGamesParamsFromSorare(game)
lo.Map(games, func(game football.Game, index int) model.CreateOrUpdateGamesParams {
return NewCreateOrUpdateGamesParamsFromSorare(game)
}),
)
if err != nil {
batcherr = nil
batchGames.Exec(func(_ int, err error) {
if err != nil {
batcherr = err
batchGames.Close()
}
})
if batcherr != nil {
return err
}
log.Debug().Msgf("%d games inserted", cnt)
log.Debug().Msgf("%d games inserted", len(games))
log.Debug().Msg("inserting players into db...")
cnt, err = u.db.BatchInsertPlayers(
batchPlayers := u.db.CreateOrUpdatePlayers(
ctx,
lo.Map(players, func(player football.Player, index int) model.BatchInsertPlayersParams {
res := model.BatchInsertPlayersParams{
lo.Map(players, func(player football.Player, index int) model.CreateOrUpdatePlayersParams {
res := model.CreateOrUpdatePlayersParams{
Slug: player.Slug,
DisplayName: player.DisplayName,
BirthDate: player.BirthDate,
@ -299,27 +313,44 @@ func (u *UpdateService) InitSyncDatabase(ctx context.Context) error {
return res
}),
)
if err != nil {
return errors.Wrap(err, "inserting players")
batcherr = nil
batchPlayers.Exec(func(_ int, err error) {
if err != nil {
batcherr = err
batchPlayers.Close()
}
})
if batcherr != nil {
return errors.Wrap(batcherr, "inserting players")
}
log.Debug().Msgf("%d players inserted", cnt)
log.Debug().Msgf("%d players inserted", len(players))
log.Debug().Msg("inserting game players into db...")
cnt, err = u.db.BatchInsertGamePlayers(ctx, gamePlayers)
if err != nil {
batchGamePlayers := u.db.CreateOrUpdateGamePlayers(
ctx,
gamePlayers,
)
batcherr = nil
batchGamePlayers.Exec(func(_ int, err error) {
if err != nil {
batcherr = err
batchGamePlayers.Close()
}
})
if batcherr != nil {
return errors.Wrap(err, "inserting game players")
}
log.Debug().Msgf("%d game players inserted", cnt)
log.Debug().Msgf("%d game players inserted", len(gamePlayers))
return nil
}
func ExtractPlayersFromGameWithFormation(
gameWithFormation football.GameWithFormation,
) []model.BatchInsertGamePlayersParams {
var res []model.BatchInsertGamePlayersParams
) []model.CreateOrUpdateGamePlayersParams {
var res []model.CreateOrUpdateGamePlayersParams
for _, p := range gameWithFormation.HomeFormation.Bench {
res = append(res, model.BatchInsertGamePlayersParams{
res = append(res, model.CreateOrUpdateGamePlayersParams{
GameID: gameWithFormation.Id.Value,
PlayerSlug: p.Slug,
TeamSlug: gameWithFormation.HomeTeam.Team.Slug,
@ -328,7 +359,7 @@ func ExtractPlayersFromGameWithFormation(
}
for _, p := range gameWithFormation.HomeFormation.StartingLineup {
for _, q := range p {
res = append(res, model.BatchInsertGamePlayersParams{
res = append(res, model.CreateOrUpdateGamePlayersParams{
GameID: gameWithFormation.Id.Value,
PlayerSlug: q.Slug,
TeamSlug: gameWithFormation.HomeTeam.Team.Slug,
@ -337,7 +368,7 @@ func ExtractPlayersFromGameWithFormation(
}
}
for _, p := range gameWithFormation.AwayFormation.Bench {
res = append(res, model.BatchInsertGamePlayersParams{
res = append(res, model.CreateOrUpdateGamePlayersParams{
GameID: gameWithFormation.Id.Value,
PlayerSlug: p.Slug,
TeamSlug: gameWithFormation.AwayTeam.Team.Slug,
@ -346,7 +377,7 @@ func ExtractPlayersFromGameWithFormation(
}
for _, p := range gameWithFormation.AwayFormation.StartingLineup {
for _, q := range p {
res = append(res, model.BatchInsertGamePlayersParams{
res = append(res, model.CreateOrUpdateGamePlayersParams{
GameID: gameWithFormation.Id.Value,
PlayerSlug: q.Slug,
TeamSlug: gameWithFormation.AwayTeam.Team.Slug,