managing initial game players sync. changing competition creation to batchexec. adding player initial sync

db/migrations/00001_init.sql

@@ -87,9 +87,9 @@
           birth_date DATE NOT NULL,
           country_slug TEXT NOT NULL,
           FOREIGN KEY (country_slug) REFERENCES countries (slug),
-          team_slug TEXT NOT NULL,
+          team_slug TEXT,
           FOREIGN KEY (team_slug) REFERENCES teams (slug),
-          domestic_league_slug TEXT NOT NULL,
+          domestic_league_slug TEXT,
           FOREIGN KEY (domestic_league_slug) REFERENCES competitions (slug),
           avatar_url TEXT NOT NULL,
           field_position TEXT NOT NULL,

model/batch.go

@@ -0,0 +1,93 @@
+// Code generated by sqlc. DO NOT EDIT.
+// versions:
+//   sqlc v1.25.0
+// source: batch.go
+
+package model
+
+import (
+	"context"
+	"errors"
+
+	"github.com/jackc/pgx/v5"
+)
+
+var (
+	ErrBatchAlreadyClosed = errors.New("batch already closed")
+)
+
+const createOrUpdateCompetitions = `-- name: CreateOrUpdateCompetitions :batchexec
+   INSERT INTO competitions (
+          slug,
+          display_name,
+          country_slug,
+          competition_format,
+          competition_type,
+          picture_url,
+          logo_url
+          )
+   VALUES ($1, $2, $3, $4, $5, $6, $7) ON CONFLICT (slug)
+       DO
+   UPDATE
+      SET display_name = EXCLUDED.display_name,
+          competition_format = EXCLUDED.competition_format,
+          competition_type = EXCLUDED.competition_type,
+          picture_url = EXCLUDED.picture_url,
+          logo_url = EXCLUDED.logo_url,
+          country_slug = EXCLUDED.country_slug
+`
+
+type CreateOrUpdateCompetitionsBatchResults struct {
+	br     pgx.BatchResults
+	tot    int
+	closed bool
+}
+
+type CreateOrUpdateCompetitionsParams struct {
+	Slug              string
+	DisplayName       string
+	CountrySlug       string
+	CompetitionFormat string
+	CompetitionType   string
+	PictureUrl        string
+	LogoUrl           string
+}
+
+func (q *Queries) CreateOrUpdateCompetitions(ctx context.Context, arg []CreateOrUpdateCompetitionsParams) *CreateOrUpdateCompetitionsBatchResults {
+	batch := &pgx.Batch{}
+	for _, a := range arg {
+		vals := []interface{}{
+			a.Slug,
+			a.DisplayName,
+			a.CountrySlug,
+			a.CompetitionFormat,
+			a.CompetitionType,
+			a.PictureUrl,
+			a.LogoUrl,
+		}
+		batch.Queue(createOrUpdateCompetitions, vals...)
+	}
+	br := q.db.SendBatch(ctx, batch)
+	return &CreateOrUpdateCompetitionsBatchResults{br, len(arg), false}
+}
+
+func (b *CreateOrUpdateCompetitionsBatchResults) Exec(f func(int, error)) {
+	defer b.br.Close()
+	for t := 0; t < b.tot; t++ {
+		if b.closed {
+			if f != nil {
+				f(t, ErrBatchAlreadyClosed)
+			}
+			continue
+		}
+		_, err := b.br.Exec()
+		if f != nil {
+			f(t, err)
+		}
+	}
+}
+
+func (b *CreateOrUpdateCompetitionsBatchResults) Close() error {
+	b.closed = true
+	return b.br.Close()
+}

model/competition.sql.go

@@ -9,60 +9,6 @@ import (
 	"context"
 )
 
-type BatchInsertCompetitionsParams struct {
-	Slug              string
-	DisplayName       string
-	CountrySlug       string
-	CompetitionFormat string
-	CompetitionType   string
-	PictureUrl        string
-	LogoUrl           string
-}
-
-const createOrUpdateCompetition = `-- name: CreateOrUpdateCompetition :exec
-   INSERT INTO competitions (
-          slug,
-          display_name,
-          country_slug,
-          competition_format,
-          competition_type,
-          picture_url,
-          logo_url
-          )
-   VALUES ($1, $2, $3, $4, $5, $6, $7) ON CONFLICT (slug)
-       DO
-   UPDATE
-      SET display_name = EXCLUDED.display_name,
-          competition_format = EXCLUDED.competition_format,
-          competition_type = EXCLUDED.competition_type,
-          picture_url = EXCLUDED.picture_url,
-          logo_url = EXCLUDED.logo_url,
-          country_slug = EXCLUDED.country_slug
-`
-
-type CreateOrUpdateCompetitionParams struct {
-	Slug              string
-	DisplayName       string
-	CountrySlug       string
-	CompetitionFormat string
-	CompetitionType   string
-	PictureUrl        string
-	LogoUrl           string
-}
-
-func (q *Queries) CreateOrUpdateCompetition(ctx context.Context, arg CreateOrUpdateCompetitionParams) error {
-	_, err := q.db.Exec(ctx, createOrUpdateCompetition,
-		arg.Slug,
-		arg.DisplayName,
-		arg.CountrySlug,
-		arg.CompetitionFormat,
-		arg.CompetitionType,
-		arg.PictureUrl,
-		arg.LogoUrl,
-	)
-	return err
-}
-
 const getAllCompetitions = `-- name: GetAllCompetitions :many
    SELECT competitions.slug, competitions.display_name, competitions.country_slug, competitions.competition_format, competitions.competition_type, competitions.picture_url, competitions.logo_url, competitions.zone_id,
           zones.id, zones.display_name,

model/copyfrom.go

@@ -9,13 +9,13 @@ import (
 	"context"
 )
 
-// iteratorForBatchInsertCompetitions implements pgx.CopyFromSource.
-type iteratorForBatchInsertCompetitions struct {
-	rows                 []BatchInsertCompetitionsParams
+// iteratorForBatchInsertGamePlayers implements pgx.CopyFromSource.
+type iteratorForBatchInsertGamePlayers struct {
+	rows                 []BatchInsertGamePlayersParams
 	skippedFirstNextCall bool
 }
 
-func (r *iteratorForBatchInsertCompetitions) Next() bool {
+func (r *iteratorForBatchInsertGamePlayers) Next() bool {
 	if len(r.rows) == 0 {
 		return false
 	}
@@ -27,24 +27,21 @@ func (r *iteratorForBatchInsertCompetitions) Next() bool {
 	return len(r.rows) > 0
 }
 
-func (r iteratorForBatchInsertCompetitions) Values() ([]interface{}, error) {
+func (r iteratorForBatchInsertGamePlayers) Values() ([]interface{}, error) {
 	return []interface{}{
-		r.rows[0].Slug,
-		r.rows[0].DisplayName,
-		r.rows[0].CountrySlug,
-		r.rows[0].CompetitionFormat,
-		r.rows[0].CompetitionType,
-		r.rows[0].PictureUrl,
-		r.rows[0].LogoUrl,
+		r.rows[0].GameID,
+		r.rows[0].PlayerSlug,
+		r.rows[0].Status,
+		r.rows[0].TeamSlug,
 	}, nil
 }
 
-func (r iteratorForBatchInsertCompetitions) Err() error {
+func (r iteratorForBatchInsertGamePlayers) Err() error {
 	return nil
 }
 
-func (q *Queries) BatchInsertCompetitions(ctx context.Context, arg []BatchInsertCompetitionsParams) (int64, error) {
-	return q.db.CopyFrom(ctx, []string{"competitions"}, []string{"slug", "display_name", "country_slug", "competition_format", "competition_type", "picture_url", "logo_url"}, &iteratorForBatchInsertCompetitions{rows: arg})
+func (q *Queries) BatchInsertGamePlayers(ctx context.Context, arg []BatchInsertGamePlayersParams) (int64, error) {
+	return q.db.CopyFrom(ctx, []string{"game_players"}, []string{"game_id", "player_slug", "status", "team_slug"}, &iteratorForBatchInsertGamePlayers{rows: arg})
 }
 
 // iteratorForBatchInsertGames implements pgx.CopyFromSource.
@@ -97,6 +94,47 @@ func (q *Queries) BatchInsertGames(ctx context.Context, arg []BatchInsertGamesPa
 	return q.db.CopyFrom(ctx, []string{"games"}, []string{"id", "date", "coverage_status", "low_coverage", "minutes", "period_type", "scored", "status", "competition_slug", "fixture_slug", "away_team_slug", "away_goals", "away_extra_time_score", "away_penalty_score", "home_team_slug", "home_goals", "home_extra_time_score", "home_penalty_score", "winner_team_slug"}, &iteratorForBatchInsertGames{rows: arg})
 }
 
+// iteratorForBatchInsertPlayers implements pgx.CopyFromSource.
+type iteratorForBatchInsertPlayers struct {
+	rows                 []BatchInsertPlayersParams
+	skippedFirstNextCall bool
+}
+
+func (r *iteratorForBatchInsertPlayers) Next() bool {
+	if len(r.rows) == 0 {
+		return false
+	}
+	if !r.skippedFirstNextCall {
+		r.skippedFirstNextCall = true
+		return true
+	}
+	r.rows = r.rows[1:]
+	return len(r.rows) > 0
+}
+
+func (r iteratorForBatchInsertPlayers) Values() ([]interface{}, error) {
+	return []interface{}{
+		r.rows[0].Slug,
+		r.rows[0].DisplayName,
+		r.rows[0].BirthDate,
+		r.rows[0].CountrySlug,
+		r.rows[0].TeamSlug,
+		r.rows[0].DomesticLeagueSlug,
+		r.rows[0].AvatarUrl,
+		r.rows[0].FieldPosition,
+		r.rows[0].Status,
+		r.rows[0].ShirtNumber,
+	}, nil
+}
+
+func (r iteratorForBatchInsertPlayers) Err() error {
+	return nil
+}
+
+func (q *Queries) BatchInsertPlayers(ctx context.Context, arg []BatchInsertPlayersParams) (int64, error) {
+	return q.db.CopyFrom(ctx, []string{"players"}, []string{"slug", "display_name", "birth_date", "country_slug", "team_slug", "domestic_league_slug", "avatar_url", "field_position", "status", "shirt_number"}, &iteratorForBatchInsertPlayers{rows: arg})
+}
+
 // iteratorForBatchInsertTeams implements pgx.CopyFromSource.
 type iteratorForBatchInsertTeams struct {
 	rows                 []BatchInsertTeamsParams

model/db.go

@@ -16,6 +16,7 @@ type DBTX interface {
 	Query(context.Context, string, ...interface{}) (pgx.Rows, error)
 	QueryRow(context.Context, string, ...interface{}) pgx.Row
 	CopyFrom(ctx context.Context, tableName pgx.Identifier, columnNames []string, rowSrc pgx.CopyFromSource) (int64, error)
+	SendBatch(context.Context, *pgx.Batch) pgx.BatchResults
 }
 
 func New(db DBTX) *Queries {

model/game_player.sql.go

@@ -0,0 +1,50 @@
+// Code generated by sqlc. DO NOT EDIT.
+// versions:
+//   sqlc v1.25.0
+// source: game_player.sql
+
+package model
+
+import (
+	"context"
+)
+
+type BatchInsertGamePlayersParams struct {
+	GameID     string
+	PlayerSlug string
+	Status     string
+	TeamSlug   string
+}
+
+const createOrUpdateGamePlayer = `-- name: CreateOrUpdateGamePlayer :exec
+INSERT INTO game_players(
+   game_id,
+   player_slug,
+   status,
+   team_slug)
+VALUES(
+    $1,
+    $2,
+    $3,
+    $4)
+  ON CONFLICT (game_id, player_slug) DO UPDATE
+  SET status = $3,
+      team_slug = $4
+`
+
+type CreateOrUpdateGamePlayerParams struct {
+	GameID     string
+	PlayerSlug string
+	Status     string
+	TeamSlug   string
+}
+
+func (q *Queries) CreateOrUpdateGamePlayer(ctx context.Context, arg CreateOrUpdateGamePlayerParams) error {
+	_, err := q.db.Exec(ctx, createOrUpdateGamePlayer,
+		arg.GameID,
+		arg.PlayerSlug,
+		arg.Status,
+		arg.TeamSlug,
+	)
+	return err
+}

model/models.go

@@ -140,8 +140,8 @@ type Player struct {
 	DisplayName        string
 	BirthDate          time.Time
 	CountrySlug        string
-	TeamSlug           string
-	DomesticLeagueSlug string
+	TeamSlug           *string
+	DomesticLeagueSlug *string
 	AvatarUrl          string
 	FieldPosition      string
 	Status             string

model/player.sql.go

@@ -0,0 +1,23 @@
+// Code generated by sqlc. DO NOT EDIT.
+// versions:
+//   sqlc v1.25.0
+// source: player.sql
+
+package model
+
+import (
+	"time"
+)
+
+type BatchInsertPlayersParams struct {
+	Slug               string
+	DisplayName        string
+	BirthDate          time.Time
+	CountrySlug        string
+	TeamSlug           *string
+	DomesticLeagueSlug *string
+	AvatarUrl          string
+	FieldPosition      string
+	Status             string
+	ShirtNumber        int32
+}

model/sql/competition.sql

@@ -1,8 +1,4 @@
--- name: BatchInsertCompetitions :copyfrom
-   INSERT INTO competitions (slug, display_name, country_slug, competition_format, competition_type, picture_url, logo_url)
-   VALUES ($1, $2, $3, $4, $5, $6, $7);
-
--- name: CreateOrUpdateCompetition :exec
+-- name: CreateOrUpdateCompetitions :batchexec
    INSERT INTO competitions (
           slug,
           display_name,

model/sql/game_player.sql

@@ -0,0 +1,27 @@
+-- name: BatchInsertGamePlayers :copyfrom
+INSERT INTO game_players(
+   game_id,
+   player_slug,
+   status,
+   team_slug)
+VALUES(
+   $1,
+   $2,
+   $3,
+   $4);
+
+-- name: CreateOrUpdateGamePlayer :exec
+INSERT INTO game_players(
+   game_id,
+   player_slug,
+   status,
+   team_slug)
+VALUES(
+    $1,
+    $2,
+    $3,
+    $4)
+  ON CONFLICT (game_id, player_slug) DO UPDATE
+  SET status = $3,
+      team_slug = $4;
+

model/sql/player.sql

@@ -0,0 +1,3 @@
+-- name: BatchInsertPlayers :copyfrom
+INSERT INTO players (slug, display_name, birth_date, country_slug, team_slug, domestic_league_slug, avatar_url, field_position, status, shirt_number)
+VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10);

sorare_utils/fixture.go

@@ -37,74 +37,3 @@ func GetGamesFromFixtures(
 	log.Debug().Msgf("found total of %d games after filtering duplicate ids", len(games))
 	return games, nil
 }
-
-func ExtractTeamSlugsFromGames(games []football.Game) []string {
-	var res []string
-	res = lo.Map(games, func(game football.Game, index int) string {
-		return game.AwayTeam.Team.Slug
-	})
-	res = append(res, lo.Map(games, func(game football.Game, index int) string {
-		return game.HomeTeam.Team.Slug
-	})...)
-	res = lo.Filter(res, func(slug string, index int) bool {
-		return slug != ""
-	})
-	return lo.Uniq(res)
-}
-
-func ExtractCountrySlugsFromCompetitions(competitions []football.Competition) []string {
-	return lo.Uniq(lo.Filter(lo.Map(competitions, func(competition football.Competition, index int) string {
-		return competition.Country.Slug
-	}), func(slug string, index int) bool {
-		return slug != ""
-	}))
-}
-
-func ExtractCountrySlugsFromTeams(teams []football.Team) []string {
-	return lo.Uniq(lo.Filter(lo.Map(teams, func(team football.Team, index int) string {
-		return team.Country.Slug
-	}), func(slug string, index int) bool {
-		return slug != ""
-	}))
-}
-
-func ExtractCountrySlugsFromCompetitionsClubsAndNationalTeams(
-	competitions []football.Competition,
-	clubs []football.Club,
-	nationalTeams []football.NationalTeam,
-) []string {
-	return lo.Uniq(lo.Union(
-		ExtractCountrySlugsFromCompetitions(competitions),
-		ExtractCountrySlugsFromTeams(lo.Map(clubs, func(club football.Club, index int) football.Team {
-			return club.Team
-		})),
-		ExtractCountrySlugsFromTeams(
-			lo.Map(nationalTeams, func(nationalTeam football.NationalTeam, index int) football.Team {
-				return nationalTeam.Team
-			}),
-		),
-	))
-}
-
-func ExtractCompetitionSlugsFromGames(games []football.Game) []string {
-	return lo.Uniq(lo.Filter(lo.Map(games, func(game football.Game, index int) string {
-		return game.Competition.Slug
-	}), func(slug string, index int) bool {
-		return slug != ""
-	}))
-}
-
-func ExtractCompetitionSlugsFromClubs(clubs []football.Club) []string {
-	return lo.Uniq(lo.Filter(lo.Map(clubs, func(club football.Club, index int) string {
-		return club.DomesticLeague.Slug
-	}), func(slug string, index int) bool {
-		return slug != ""
-	}))
-}
-
-func ExtractCompetitionSlugsFromGamesAndClubs(games []football.Game, clubs []football.Club) []string {
-	return lo.Uniq(lo.Union(
-		ExtractCompetitionSlugsFromGames(games),
-		ExtractCompetitionSlugsFromClubs(clubs),
-	))
-}

sorare_utils/game.go

@@ -0,0 +1,37 @@
+package sorare_utils
+
+import (
+	"git.lehouerou.net/laurent/sorare/football"
+	"github.com/jackc/pgx/v5/pgtype"
+
+	"git.lehouerou.net/laurent/sorarebuddy/model"
+)
+
+func NewBatchInsertGamesParamsFromSorare(game football.Game) model.BatchInsertGamesParams {
+	return model.BatchInsertGamesParams{
+		ID:                 game.Id.Value,
+		Date:               pgtype.Timestamptz{Time: game.Date, Valid: true},
+		CoverageStatus:     game.CoverageStatus,
+		LowCoverage:        game.LowCoverage,
+		Minutes:            int32(game.Minute),
+		PeriodType:         game.PeriodType,
+		Scored:             game.Scored,
+		Status:             game.Status,
+		CompetitionSlug:    game.Competition.Slug,
+		FixtureSlug:        game.So5Fixture.Slug,
+		AwayTeamSlug:       game.AwayTeam.Team.Slug,
+		AwayGoals:          int32(game.AwayGoals),
+		AwayExtraTimeScore: int32(game.ExtraTimeScoreAway),
+		AwayPenaltyScore:   int32(game.PenaltyScoreAway),
+		HomeTeamSlug:       game.HomeTeam.Team.Slug,
+		HomeGoals:          int32(game.HomeGoals),
+		HomeExtraTimeScore: int32(game.ExtraTimeScoreHome),
+		HomePenaltyScore:   int32(game.PenaltyScoreHome),
+		WinnerTeamSlug: func() *string {
+			if game.Winner.Team.Slug == "" {
+				return nil
+			}
+			return &game.Winner.Team.Slug
+		}(),
+	}
+}

sorare_utils/game_player.go

@@ -0,0 +1 @@
+package sorare_utils

sorare_utils/update_service.go

@@ -7,6 +7,8 @@ import (
 	"git.lehouerou.net/laurent/sorare/football"
 	"git.lehouerou.net/laurent/sorare/graphql"
 	"github.com/jackc/pgx/v5/pgtype"
+	gql "github.com/llehouerou/go-graphql-client"
+	"github.com/pkg/errors"
 	"github.com/rs/zerolog/log"
 	"github.com/samber/lo"
 
@@ -24,7 +26,9 @@ func NewUpdateService(s *sorare.Sorare, db *model.Queries) *UpdateService {
 
 func (u *UpdateService) InitSyncDatabase(ctx context.Context) error {
 
-	sfixtures, err := u.s.Football.So5.So5Fixtures.Get(ctx, football.So5FixturesParams{})
+	sfixtures, err := u.s.Football.So5.So5Fixtures.Get(ctx, football.So5FixturesParams{
+		AasmStates: []string{"started"},
+	})
 	if err != nil {
 		return err
 	}
@@ -62,13 +66,45 @@ func (u *UpdateService) InitSyncDatabase(ctx context.Context) error {
 	)
 	log.Info().Msgf("found %d games to process", len(games))
 
-	teamSlugs := ExtractTeamSlugsFromGames(games)
-	log.Debug().Msgf("extracted %d unique team slugs from games", len(teamSlugs))
+	log.Debug().Msgf("getting players for each game...")
+	var gamePlayers []model.BatchInsertGamePlayersParams
+	for _, game := range games {
+		gameWithFormation, err := u.s.Football.Game.Get(ctx, graphql.IdParams{Id: gql.ID(game.Id.Value)})
+		if err != nil {
+			return errors.Wrapf(err, "getting game with formation %s", game.Id.Value)
+		}
+		newplayers := ExtractPlayersFromGameWithFormation(gameWithFormation)
+		log.Debug().Msgf("\t%s -> %d players", game.String(), len(newplayers))
+		gamePlayers = append(gamePlayers, newplayers...)
+	}
+
+	playerSlugs := lo.Uniq(
+		lo.Filter(lo.Map(gamePlayers, func(player model.BatchInsertGamePlayersParams, index int) string {
+			return player.PlayerSlug
+		}), func(slug string, index int) bool {
+			return slug != ""
+		}),
+	)
+
+	log.Debug().Msgf("getting players...")
+	var players []football.Player
+	for i, chunk := range lo.Chunk(playerSlugs, 80) {
+		log.Debug().Msgf("\tbatch %d/%d", i+1, (len(playerSlugs)/80)+1)
+		p, err := u.s.Football.Players.Get(ctx, graphql.SlugsParams{Slugs: chunk})
+		if err != nil {
+			return errors.Wrapf(err, "getting players batch %d", i)
+		}
+		players = append(players, p...)
+	}
+	log.Debug().Msgf("found %d players", len(players))
+
+	teamSlugs := ExtractTeamSlugsFromPlayersAndGames(players, games)
+	log.Debug().Msgf("extracted %d unique team slugs from games and players", len(teamSlugs))
 
 	log.Debug().Msgf("getting clubs...")
 	var clubs []football.Club
 	for i, chunk := range lo.Chunk(teamSlugs, 100) {
-		log.Debug().Msgf("\tbatch %d/%d", i+1, len(teamSlugs)/100)
+		log.Debug().Msgf("\tbatch %d/%d", i+1, (len(teamSlugs)/100)+1)
 		t, err := u.s.Football.Clubs.Get(ctx, graphql.SlugsParams{Slugs: chunk})
 		if err != nil {
 			return err
@@ -83,7 +119,7 @@ func (u *UpdateService) InitSyncDatabase(ctx context.Context) error {
 	log.Debug().Msgf("getting national teams...")
 	log.Debug().Msgf("slugs left: %d", len(slugsLeft))
 	for i, chunk := range lo.Chunk(slugsLeft, 100) {
-		log.Debug().Msgf("\tbatch %d/%d", i+1, len(teamSlugs)/100)
+		log.Debug().Msgf("\tbatch %d/%d", i+1, (len(teamSlugs)/100)+1)
 		t, err := u.s.Football.NationalTeams.Get(ctx, graphql.SlugsParams{Slugs: chunk})
 		if err != nil {
 			return err
@@ -92,8 +128,8 @@ func (u *UpdateService) InitSyncDatabase(ctx context.Context) error {
 	}
 	log.Debug().Msgf("found %d national teams", len(nationalTeams))
 
-	competitionSlugs := ExtractCompetitionSlugsFromGamesAndClubs(games, clubs)
-	log.Debug().Msgf("extracted %d unique competition slugs from games and clubs", len(competitionSlugs))
+	competitionSlugs := ExtractCompetitionSlugsFromPlayersGamesAndClubs(players, games, clubs)
+	log.Debug().Msgf("extracted %d unique competition slugs from players, games and clubs", len(competitionSlugs))
 	log.Debug().Msgf("getting competitions...")
 	var competitions []football.Competition
 	for _, slug := range competitionSlugs {
@@ -106,16 +142,18 @@ func (u *UpdateService) InitSyncDatabase(ctx context.Context) error {
 	}
 	log.Debug().Msgf("found %d competitions", len(competitions))
 
-	countrySlugs := ExtractCountrySlugsFromCompetitionsClubsAndNationalTeams(
+	countrySlugs := ExtractCountrySlugsFromPlayersCompetitionsClubsAndNationalTeams(
+		players,
 		competitions,
 		clubs,
 		nationalTeams,
 	)
-	log.Debug().Msgf("extracted %d unique country slugs from competitions, clubs and national teams", len(countrySlugs))
+	log.Debug().
+		Msgf("extracted %d unique country slugs from players, competitions, clubs and national teams", len(countrySlugs))
 	log.Debug().Msgf("getting countries...")
 	var countries []sorare.Country
 	for i, chunk := range lo.Chunk(countrySlugs, 100) {
-		log.Debug().Msgf("\tbatch %d/%d", i+1, len(countrySlugs)/100)
+		log.Debug().Msgf("\tbatch %d/%d", i+1, (len(countrySlugs)/100)+1)
 		c, err := u.s.Countries.Get(ctx, graphql.SlugsParams{Slugs: chunk})
 		if err != nil {
 			return err
@@ -147,10 +185,11 @@ func (u *UpdateService) InitSyncDatabase(ctx context.Context) error {
 
 	log.Debug().Msg("inserting competitions into db...")
 
-	_, err = u.db.BatchInsertCompetitions(
+	var batcherr error
+	batch := u.db.CreateOrUpdateCompetitions(
 		ctx,
-		lo.Map(competitions, func(competition football.Competition, index int) model.BatchInsertCompetitionsParams {
-			return model.BatchInsertCompetitionsParams{
+		lo.Map(competitions, func(competition football.Competition, index int) model.CreateOrUpdateCompetitionsParams {
+			return model.CreateOrUpdateCompetitionsParams{
 				Slug:              competition.Slug,
 				CompetitionFormat: competition.Format,
 				CompetitionType:   competition.Type,
@@ -161,8 +200,14 @@ func (u *UpdateService) InitSyncDatabase(ctx context.Context) error {
 			}
 		}),
 	)
+	batch.Exec(func(_ int, err error) {
 		if err != nil {
-		return err
+			batcherr = err
+			batch.Close()
+		}
+	})
+	if batcherr != nil {
+		return errors.Wrap(batcherr, "inserting competitions")
 	}
 
 	log.Debug().Msgf("%d competitions inserted", len(competitions))
@@ -206,37 +251,211 @@ func (u *UpdateService) InitSyncDatabase(ctx context.Context) error {
 	cnt, err = u.db.BatchInsertGames(
 		ctx,
 		lo.Map(games, func(game football.Game, index int) model.BatchInsertGamesParams {
-			return model.BatchInsertGamesParams{
-				ID:                 game.Id.Value,
-				Date:               pgtype.Timestamptz{Time: game.Date, Valid: true},
-				CoverageStatus:     game.CoverageStatus,
-				LowCoverage:        game.LowCoverage,
-				Minutes:            int32(game.Minute),
-				PeriodType:         game.PeriodType,
-				Scored:             game.Scored,
-				Status:             game.Status,
-				CompetitionSlug:    game.Competition.Slug,
-				FixtureSlug:        game.So5Fixture.Slug,
-				AwayTeamSlug:       game.AwayTeam.Team.Slug,
-				AwayGoals:          int32(game.AwayGoals),
-				AwayExtraTimeScore: int32(game.ExtraTimeScoreAway),
-				AwayPenaltyScore:   int32(game.PenaltyScoreAway),
-				HomeTeamSlug:       game.HomeTeam.Team.Slug,
-				HomeGoals:          int32(game.HomeGoals),
-				HomeExtraTimeScore: int32(game.ExtraTimeScoreHome),
-				HomePenaltyScore:   int32(game.PenaltyScoreHome),
-				WinnerTeamSlug: func() *string {
-					if game.Winner.Team.Slug == "" {
-						return nil
-					}
-					return &game.Winner.Team.Slug
-				}(),
-			}
+			return NewBatchInsertGamesParamsFromSorare(game)
 		}),
 	)
 	if err != nil {
 		return err
 	}
 	log.Debug().Msgf("%d games inserted", cnt)
+
+	log.Debug().Msg("inserting players into db...")
+	cnt, err = u.db.BatchInsertPlayers(
+		ctx,
+		lo.Map(players, func(player football.Player, index int) model.BatchInsertPlayersParams {
+			res := model.BatchInsertPlayersParams{
+				Slug:          player.Slug,
+				DisplayName:   player.DisplayName,
+				BirthDate:     player.BirthDate,
+				CountrySlug:   player.Country.Slug,
+				AvatarUrl:     player.AvatarUrl,
+				FieldPosition: string(player.Position),
+				Status:        string(player.PlayingStatus),
+				ShirtNumber:   int32(player.ShirtNumber),
+			}
+			for _, competition := range player.ActiveClub.ActiveCompetitions {
+				if competition.Format == "DOMESTIC_LEAGUE" {
+					res.DomesticLeagueSlug = &competition.Slug
+				}
+			}
+			if player.ActiveClub.Slug != "" {
+				res.TeamSlug = &player.ActiveClub.Slug
+			}
+			return res
+		}),
+	)
+	if err != nil {
+		return errors.Wrap(err, "inserting players")
+	}
+	log.Debug().Msgf("%d players inserted", cnt)
+
+	log.Debug().Msg("inserting game players into db...")
+	cnt, err = u.db.BatchInsertGamePlayers(ctx, gamePlayers)
+	if err != nil {
+		return errors.Wrap(err, "inserting game players")
+	}
+	log.Debug().Msgf("%d game players inserted", cnt)
+
 	return nil
 }
+
+func ExtractPlayersFromGameWithFormation(
+	gameWithFormation football.GameWithFormation,
+) []model.BatchInsertGamePlayersParams {
+	var res []model.BatchInsertGamePlayersParams
+	for _, p := range gameWithFormation.HomeFormation.Bench {
+		res = append(res, model.BatchInsertGamePlayersParams{
+			GameID:     gameWithFormation.Id.Value,
+			PlayerSlug: p.Slug,
+			TeamSlug:   gameWithFormation.HomeTeam.Team.Slug,
+			Status:     "bench",
+		})
+	}
+	for _, p := range gameWithFormation.HomeFormation.StartingLineup {
+		for _, q := range p {
+			res = append(res, model.BatchInsertGamePlayersParams{
+				GameID:     gameWithFormation.Id.Value,
+				PlayerSlug: q.Slug,
+				TeamSlug:   gameWithFormation.HomeTeam.Team.Slug,
+				Status:     "starting",
+			})
+		}
+	}
+	for _, p := range gameWithFormation.AwayFormation.Bench {
+		res = append(res, model.BatchInsertGamePlayersParams{
+			GameID:     gameWithFormation.Id.Value,
+			PlayerSlug: p.Slug,
+			TeamSlug:   gameWithFormation.AwayTeam.Team.Slug,
+			Status:     "bench",
+		})
+	}
+	for _, p := range gameWithFormation.AwayFormation.StartingLineup {
+		for _, q := range p {
+			res = append(res, model.BatchInsertGamePlayersParams{
+				GameID:     gameWithFormation.Id.Value,
+				PlayerSlug: q.Slug,
+				TeamSlug:   gameWithFormation.AwayTeam.Team.Slug,
+				Status:     "starting",
+			})
+		}
+	}
+
+	return res
+
+}
+
+func ExtractTeamSlugsFromPlayersAndGames(players []football.Player, games []football.Game) []string {
+	return lo.Uniq(lo.Union(
+		ExtractTeamSlugsFromPlayers(players),
+		ExtractTeamSlugsFromGames(games),
+	))
+}
+
+func ExtractTeamSlugsFromPlayers(players []football.Player) []string {
+	return lo.Uniq(lo.Filter(lo.Map(players, func(player football.Player, index int) string {
+		return player.ActiveClub.Slug
+	}), func(slug string, index int) bool {
+		return slug != ""
+	}))
+}
+
+func ExtractTeamSlugsFromGames(games []football.Game) []string {
+	var res []string
+	res = lo.Map(games, func(game football.Game, index int) string {
+		return game.AwayTeam.Team.Slug
+	})
+	res = append(res, lo.Map(games, func(game football.Game, index int) string {
+		return game.HomeTeam.Team.Slug
+	})...)
+	res = lo.Filter(res, func(slug string, index int) bool {
+		return slug != ""
+	})
+	return lo.Uniq(res)
+}
+
+func ExtractCountrySlugsFromCompetitions(competitions []football.Competition) []string {
+	return lo.Uniq(lo.Filter(lo.Map(competitions, func(competition football.Competition, index int) string {
+		return competition.Country.Slug
+	}), func(slug string, index int) bool {
+		return slug != ""
+	}))
+}
+
+func ExtractCountrySlugsFromTeams(teams []football.Team) []string {
+	return lo.Uniq(lo.Filter(lo.Map(teams, func(team football.Team, index int) string {
+		return team.Country.Slug
+	}), func(slug string, index int) bool {
+		return slug != ""
+	}))
+}
+
+func ExtractCountrySlugsFromPlayersCompetitionsClubsAndNationalTeams(
+	players []football.Player,
+	competitions []football.Competition,
+	clubs []football.Club,
+	nationalTeams []football.NationalTeam,
+) []string {
+	return lo.Uniq(lo.Union(
+		ExtractCountrySlugsFromPlayers(players),
+		ExtractCountrySlugsFromCompetitions(competitions),
+		ExtractCountrySlugsFromTeams(lo.Map(clubs, func(club football.Club, index int) football.Team {
+			return club.Team
+		})),
+		ExtractCountrySlugsFromTeams(
+			lo.Map(nationalTeams, func(nationalTeam football.NationalTeam, index int) football.Team {
+				return nationalTeam.Team
+			}),
+		),
+	))
+}
+
+func ExtractCountrySlugsFromPlayers(players []football.Player) []string {
+	return lo.Uniq(lo.Filter(lo.Map(players, func(player football.Player, index int) string {
+		return player.Country.Slug
+	}), func(slug string, index int) bool {
+		return slug != ""
+	}))
+
+}
+
+func ExtractCompetitionSlugsFromPlayers(players []football.Player) []string {
+	return lo.Uniq(lo.Filter(lo.Map(players, func(player football.Player, index int) string {
+		var res string
+		for _, competition := range player.ActiveClub.ActiveCompetitions {
+			if competition.Format == "DOMESTIC_LEAGUE" {
+				res = competition.Slug
+			}
+		}
+		return res
+	}), func(slug string, index int) bool {
+		return slug != ""
+	}))
+}
+
+func ExtractCompetitionSlugsFromGames(games []football.Game) []string {
+	return lo.Uniq(lo.Filter(lo.Map(games, func(game football.Game, index int) string {
+		return game.Competition.Slug
+	}), func(slug string, index int) bool {
+		return slug != ""
+	}))
+}
+
+func ExtractCompetitionSlugsFromClubs(clubs []football.Club) []string {
+	return lo.Uniq(lo.Filter(lo.Map(clubs, func(club football.Club, index int) string {
+		return club.DomesticLeague.Slug
+	}), func(slug string, index int) bool {
+		return slug != ""
+	}))
+}
+
+func ExtractCompetitionSlugsFromPlayersGamesAndClubs(
+	players []football.Player,
+	games []football.Game,
+	clubs []football.Club,
+) []string {
+	return lo.Uniq(lo.Union(
+		ExtractCompetitionSlugsFromPlayers(players),
+		ExtractCompetitionSlugsFromGames(games),
+		ExtractCompetitionSlugsFromClubs(clubs),
+	))
+}