refactor db initialization (wip)

cmd/console/root.go

@@ -12,6 +12,7 @@ import (
 
 	"git.lehouerou.net/laurent/sorarebuddy/cmd/common"
 	"git.lehouerou.net/laurent/sorarebuddy/model"
+	"git.lehouerou.net/laurent/sorarebuddy/sorare_utils"
 )
 
 var Cmd = &cobra.Command{
@@ -73,129 +74,78 @@ func run(cmd *cobra.Command, _ []string) error {
 	if err != nil {
 		return err
 	}
+	games, err := sorare_utils.GetGamesFromFixtures(
+		ctx,
+		s,
+		lo.Map(fixtures, func(fixture model.Fixture, index int) string {
+			return fixture.Slug
+		}),
+	)
+	log.Info().Msgf("found %d games to process", len(games))
 
-	var games []football.Game
-	for _, fixture := range fixtures {
-		log.Debug().Msgf("fixture: %v", fixture)
-		g, err := s.Football.So5.FixtureGames(fixture.Slug).Get(ctx, graphql.EmptyParams{})
-		if err != nil {
-			return err
-		}
-		log.Debug().Msgf("games: %d", len(g))
-		games = append(games, g...)
-	}
-	log.Debug().Msgf("games: %d", len(games))
-
-	var teamSlugs []string
-	teamSlugs = lo.Map(games, func(game football.Game, index int) string {
-		return game.AwayTeam.Team.Slug
-	})
-	teamSlugs = append(teamSlugs, lo.Map(games, func(game football.Game, index int) string {
-		return game.HomeTeam.Team.Slug
-	})...)
-	teamSlugs = lo.Filter(teamSlugs, func(slug string, index int) bool {
-		return slug != ""
-	})
-	teamSlugs = lo.Uniq(teamSlugs)
+	teamSlugs := sorare_utils.ExtractTeamSlugsFromGames(games)
+	log.Debug().Msgf("extracted %d unique team slugs from games", len(teamSlugs))
 
+	log.Debug().Msgf("getting clubs...")
 	var clubs []football.Club
-	for _, chunk := range lo.Chunk(teamSlugs, 100) {
+	for i, chunk := range lo.Chunk(teamSlugs, 100) {
+		log.Debug().Msgf("\tbatch %d/%d", i+1, len(teamSlugs)/100)
 		t, err := s.Football.Clubs.Get(ctx, graphql.SlugsParams{Slugs: chunk})
 		if err != nil {
 			return err
 		}
 		clubs = append(clubs, t...)
 	}
-
+	log.Debug().Msgf("found %d clubs", len(clubs))
 	var nationalTeams []football.NationalTeam
-	for _, chunk := range lo.Chunk(lo.Without(teamSlugs, lo.Map(clubs, func(club football.Club, index int) string {
+	slugsLeft := lo.Without(teamSlugs, lo.Map(clubs, func(club football.Club, index int) string {
 		return club.Slug
-	})...), 100) {
+	})...)
+	log.Debug().Msgf("getting national teams...")
+	log.Debug().Msgf("slugs left: %d", len(slugsLeft))
+	for i, chunk := range lo.Chunk(slugsLeft, 100) {
+		log.Debug().Msgf("\tbatch %d/%d", i+1, len(teamSlugs)/100)
 		t, err := s.Football.NationalTeams.Get(ctx, graphql.SlugsParams{Slugs: chunk})
 		if err != nil {
 			return err
 		}
 		nationalTeams = append(nationalTeams, t...)
 	}
+	log.Debug().Msgf("found %d national teams", len(nationalTeams))
 
-	var teamsToInsert []model.CreateTeamsParams
-	teamsToInsert = append(teamsToInsert, lo.Map(clubs, func(club football.Club, index int) model.CreateTeamsParams {
-		return model.CreateTeamsParams{
-			Slug:        club.Slug,
-			DisplayName: club.Name,
-			CountrySlug: club.Country.Slug,
-			DomesticLeagueSlug: func() *string {
-				if club.DomesticLeague.Slug == "" {
-					return nil
-				}
-				return &club.DomesticLeague.Slug
-			}(),
-			ShortName:  club.ShortName,
-			PictureUrl: club.PictureUrl,
-			TeamType:   "club",
-		}
-	})...)
-	teamsToInsert = append(
-		teamsToInsert,
-		lo.Map(nationalTeams, func(nationalTeam football.NationalTeam, index int) model.CreateTeamsParams {
-			return model.CreateTeamsParams{
-				Slug:               nationalTeam.Slug,
-				DisplayName:        nationalTeam.Name,
-				CountrySlug:        nationalTeam.Country.Slug,
-				DomesticLeagueSlug: nil,
-				ShortName:          nationalTeam.ShortName,
-				PictureUrl:         nationalTeam.PictureUrl,
-				TeamType:           "national",
-			}
-		})...)
-
-	competitionSlugs := lo.Map(games, func(game football.Game, index int) string {
-		return game.Competition.Slug
-	})
-	competitionSlugs = append(
-		competitionSlugs,
-		lo.Map(teamsToInsert, func(team model.CreateTeamsParams, index int) string {
-			if team.DomesticLeagueSlug == nil {
-				return ""
-			}
-			return *team.DomesticLeagueSlug
-		})...)
-	competitionSlugs = lo.Filter(competitionSlugs, func(slug string, index int) bool {
-		return slug != ""
-	})
-	competitionSlugs = lo.Uniq(competitionSlugs)
-
+	competitionSlugs := sorare_utils.ExtractCompetitionSlugsFromGamesAndClubs(games, clubs)
+	log.Debug().Msgf("extracted %d unique competition slugs from games and clubs", len(competitionSlugs))
+	log.Debug().Msgf("getting competitions...")
 	var competitions []football.Competition
 	for _, slug := range competitionSlugs {
-		log.Debug().Msgf("competition: %s", slug)
+		log.Debug().Msgf("\tcompetition %s", slug)
 		c, err := s.Football.Competition.Get(ctx, graphql.SlugParams{Slug: slug})
 		if err != nil {
 			return err
 		}
 		competitions = append(competitions, c)
 	}
+	log.Debug().Msgf("found %d competitions", len(competitions))
 
-	var countrySlugs []string
-	countrySlugs = append(countrySlugs, lo.Map(teamsToInsert, func(team model.CreateTeamsParams, index int) string {
-		return team.CountrySlug
-	})...)
-	countrySlugs = append(countrySlugs, lo.Map(competitions, func(competition football.Competition, index int) string {
-		return competition.Country.Slug
-	})...)
-	countrySlugs = lo.Filter(countrySlugs, func(slug string, index int) bool {
-		return slug != ""
-	})
-	countrySlugs = lo.Uniq(countrySlugs)
-
+	countrySlugs := sorare_utils.ExtractCountrySlugsFromCompetitionsClubsAndNationalTeams(
+		competitions,
+		clubs,
+		nationalTeams,
+	)
+	log.Debug().Msgf("extracted %d unique country slugs from competitions, clubs and national teams", len(countrySlugs))
+	log.Debug().Msgf("getting countries...")
 	var countries []sorare.Country
-	for _, chunk := range lo.Chunk(countrySlugs, 100) {
+	for i, chunk := range lo.Chunk(countrySlugs, 100) {
+		log.Debug().Msgf("\tbatch %d/%d", i+1, len(countrySlugs)/100)
 		c, err := s.Countries.Get(ctx, graphql.SlugsParams{Slugs: chunk})
 		if err != nil {
 			return err
 		}
 		countries = append(countries, c...)
 	}
+	log.Debug().Msgf("found %d countries", len(countries))
 
+	log.Debug().Msg("inserting countries into db...")
 	cnt, err = db.CreateCountries(
 		ctx,
 		lo.Map(countries, func(country sorare.Country, index int) model.CreateCountriesParams {
@@ -214,12 +164,14 @@ func run(cmd *cobra.Command, _ []string) error {
 	if err != nil {
 		return err
 	}
-	log.Debug().Msgf("created %d countries", cnt)
+	log.Debug().Msgf("%d countries inserted", cnt)
 
-	log.Debug().Msgf("competitions: %d", len(competitions))
-	for _, competition := range competitions {
+	log.Debug().Msg("inserting competitions into db...")
 
-		err := db.CreateOrUpdateCompetition(ctx, model.CreateOrUpdateCompetitionParams{
+	_, err = db.BatchInsertCompetitions(
+		ctx,
+		lo.Map(competitions, func(competition football.Competition, index int) model.BatchInsertCompetitionsParams {
+			return model.BatchInsertCompetitionsParams{
 				Slug:              competition.Slug,
 				CompetitionFormat: competition.Format,
 				CompetitionType:   competition.Type,
@@ -227,20 +179,57 @@ func run(cmd *cobra.Command, _ []string) error {
 				PictureUrl:        competition.PictureUrl,
 				LogoUrl:           competition.LogoUrl,
 				CountrySlug:       competition.Country.Slug,
-		})
+			}
+		}),
+	)
 	if err != nil {
 		return err
 	}
-	}
 
-	cnt, err = db.CreateTeams(ctx, teamsToInsert)
+	log.Debug().Msgf("%d competitions inserted", len(competitions))
+
+	log.Debug().Msg("inserting teams into db...")
+	cnt, err = db.BatchInsertTeams(ctx, lo.Union(
+		lo.Map(clubs, func(club football.Club, index int) model.BatchInsertTeamsParams {
+			return model.BatchInsertTeamsParams{
+				Slug:        club.Slug,
+				DisplayName: club.Name,
+				CountrySlug: club.Country.Slug,
+				DomesticLeagueSlug: func() *string {
+					if club.DomesticLeague.Slug == "" {
+						return nil
+					}
+					return &club.DomesticLeague.Slug
+				}(),
+				ShortName:  club.ShortName,
+				PictureUrl: club.PictureUrl,
+				TeamType:   "club",
+			}
+		}),
+		lo.Map(nationalTeams, func(nationalTeam football.NationalTeam, index int) model.BatchInsertTeamsParams {
+			return model.BatchInsertTeamsParams{
+				Slug:               nationalTeam.Slug,
+				DisplayName:        nationalTeam.Name,
+				CountrySlug:        nationalTeam.Country.Slug,
+				DomesticLeagueSlug: nil,
+				ShortName:          nationalTeam.ShortName,
+				PictureUrl:         nationalTeam.PictureUrl,
+				TeamType:           "national",
+			}
+		}),
+	))
 	if err != nil {
 		return err
 	}
-	log.Debug().Msgf("created %d teams", cnt)
+	log.Debug().Msgf("%d teams inserted", cnt)
 
-	cnt, err = db.CreateGames(ctx, lo.Map(games, func(game football.Game, index int) model.CreateGamesParams {
-		return model.CreateGamesParams{
+	log.Debug().Msg("inserting games into db...")
+	cnt, err = db.BatchInsertGames(
+		ctx,
+		lo.Map(games, func(game football.Game, index int) model.BatchInsertGamesParams {
+			return model.BatchInsertGamesParams{
+				ID:                 game.Id.Value,
+				Date:               pgtype.Timestamptz{Time: game.Date, Valid: true},
 				CoverageStatus:     game.CoverageStatus,
 				LowCoverage:        game.LowCoverage,
 				Minutes:            int32(game.Minute),
@@ -264,12 +253,12 @@ func run(cmd *cobra.Command, _ []string) error {
 					return &game.Winner.Team.Slug
 				}(),
 			}
-	}))
+		}),
+	)
 	if err != nil {
 		return err
 	}
-
-	log.Debug().Msgf("created %d games", cnt)
+	log.Debug().Msgf("%d games inserted", cnt)
 
 	// log.Debug().Msg("start sequence completed. waiting for shutdown request")
 	// <-ctx.Done()

model/competition.sql.go

@@ -9,6 +9,16 @@ import (
 	"context"
 )
 
+type BatchInsertCompetitionsParams struct {
+	Slug              string
+	DisplayName       string
+	CountrySlug       string
+	CompetitionFormat string
+	CompetitionType   string
+	PictureUrl        string
+	LogoUrl           string
+}
+
 const createOrUpdateCompetition = `-- name: CreateOrUpdateCompetition :exec
    INSERT INTO competitions (
           slug,
@@ -40,7 +50,6 @@ type CreateOrUpdateCompetitionParams struct {
 	LogoUrl           string
 }
 
-// Active: 1709890109198@@192.168.1.250@5436@sorare
 func (q *Queries) CreateOrUpdateCompetition(ctx context.Context, arg CreateOrUpdateCompetitionParams) error {
 	_, err := q.db.Exec(ctx, createOrUpdateCompetition,
 		arg.Slug,

model/copyfrom.go

@@ -9,6 +9,132 @@ import (
 	"context"
 )
 
+// iteratorForBatchInsertCompetitions implements pgx.CopyFromSource.
+type iteratorForBatchInsertCompetitions struct {
+	rows                 []BatchInsertCompetitionsParams
+	skippedFirstNextCall bool
+}
+
+func (r *iteratorForBatchInsertCompetitions) Next() bool {
+	if len(r.rows) == 0 {
+		return false
+	}
+	if !r.skippedFirstNextCall {
+		r.skippedFirstNextCall = true
+		return true
+	}
+	r.rows = r.rows[1:]
+	return len(r.rows) > 0
+}
+
+func (r iteratorForBatchInsertCompetitions) Values() ([]interface{}, error) {
+	return []interface{}{
+		r.rows[0].Slug,
+		r.rows[0].DisplayName,
+		r.rows[0].CountrySlug,
+		r.rows[0].CompetitionFormat,
+		r.rows[0].CompetitionType,
+		r.rows[0].PictureUrl,
+		r.rows[0].LogoUrl,
+	}, nil
+}
+
+func (r iteratorForBatchInsertCompetitions) Err() error {
+	return nil
+}
+
+func (q *Queries) BatchInsertCompetitions(ctx context.Context, arg []BatchInsertCompetitionsParams) (int64, error) {
+	return q.db.CopyFrom(ctx, []string{"competitions"}, []string{"slug", "display_name", "country_slug", "competition_format", "competition_type", "picture_url", "logo_url"}, &iteratorForBatchInsertCompetitions{rows: arg})
+}
+
+// iteratorForBatchInsertGames implements pgx.CopyFromSource.
+type iteratorForBatchInsertGames struct {
+	rows                 []BatchInsertGamesParams
+	skippedFirstNextCall bool
+}
+
+func (r *iteratorForBatchInsertGames) Next() bool {
+	if len(r.rows) == 0 {
+		return false
+	}
+	if !r.skippedFirstNextCall {
+		r.skippedFirstNextCall = true
+		return true
+	}
+	r.rows = r.rows[1:]
+	return len(r.rows) > 0
+}
+
+func (r iteratorForBatchInsertGames) Values() ([]interface{}, error) {
+	return []interface{}{
+		r.rows[0].ID,
+		r.rows[0].Date,
+		r.rows[0].CoverageStatus,
+		r.rows[0].LowCoverage,
+		r.rows[0].Minutes,
+		r.rows[0].PeriodType,
+		r.rows[0].Scored,
+		r.rows[0].Status,
+		r.rows[0].CompetitionSlug,
+		r.rows[0].FixtureSlug,
+		r.rows[0].AwayTeamSlug,
+		r.rows[0].AwayGoals,
+		r.rows[0].AwayExtraTimeScore,
+		r.rows[0].AwayPenaltyScore,
+		r.rows[0].HomeTeamSlug,
+		r.rows[0].HomeGoals,
+		r.rows[0].HomeExtraTimeScore,
+		r.rows[0].HomePenaltyScore,
+		r.rows[0].WinnerTeamSlug,
+	}, nil
+}
+
+func (r iteratorForBatchInsertGames) Err() error {
+	return nil
+}
+
+func (q *Queries) BatchInsertGames(ctx context.Context, arg []BatchInsertGamesParams) (int64, error) {
+	return q.db.CopyFrom(ctx, []string{"games"}, []string{"id", "date", "coverage_status", "low_coverage", "minutes", "period_type", "scored", "status", "competition_slug", "fixture_slug", "away_team_slug", "away_goals", "away_extra_time_score", "away_penalty_score", "home_team_slug", "home_goals", "home_extra_time_score", "home_penalty_score", "winner_team_slug"}, &iteratorForBatchInsertGames{rows: arg})
+}
+
+// iteratorForBatchInsertTeams implements pgx.CopyFromSource.
+type iteratorForBatchInsertTeams struct {
+	rows                 []BatchInsertTeamsParams
+	skippedFirstNextCall bool
+}
+
+func (r *iteratorForBatchInsertTeams) Next() bool {
+	if len(r.rows) == 0 {
+		return false
+	}
+	if !r.skippedFirstNextCall {
+		r.skippedFirstNextCall = true
+		return true
+	}
+	r.rows = r.rows[1:]
+	return len(r.rows) > 0
+}
+
+func (r iteratorForBatchInsertTeams) Values() ([]interface{}, error) {
+	return []interface{}{
+		r.rows[0].Slug,
+		r.rows[0].DisplayName,
+		r.rows[0].CountrySlug,
+		r.rows[0].DomesticLeagueSlug,
+		r.rows[0].ShortName,
+		r.rows[0].PictureUrl,
+		r.rows[0].TeamType,
+	}, nil
+}
+
+func (r iteratorForBatchInsertTeams) Err() error {
+	return nil
+}
+
+func (q *Queries) BatchInsertTeams(ctx context.Context, arg []BatchInsertTeamsParams) (int64, error) {
+	return q.db.CopyFrom(ctx, []string{"teams"}, []string{"slug", "display_name", "country_slug", "domestic_league_slug", "short_name", "picture_url", "team_type"}, &iteratorForBatchInsertTeams{rows: arg})
+}
+
 // iteratorForCreateCountries implements pgx.CopyFromSource.
 type iteratorForCreateCountries struct {
 	rows                 []CreateCountriesParams
@@ -85,91 +211,3 @@ func (r iteratorForCreateFixtures) Err() error {
 func (q *Queries) CreateFixtures(ctx context.Context, arg []CreateFixturesParams) (int64, error) {
 	return q.db.CopyFrom(ctx, []string{"fixtures"}, []string{"slug", "display_name", "state", "start_date", "end_date", "game_week"}, &iteratorForCreateFixtures{rows: arg})
 }
-
-// iteratorForCreateGames implements pgx.CopyFromSource.
-type iteratorForCreateGames struct {
-	rows                 []CreateGamesParams
-	skippedFirstNextCall bool
-}
-
-func (r *iteratorForCreateGames) Next() bool {
-	if len(r.rows) == 0 {
-		return false
-	}
-	if !r.skippedFirstNextCall {
-		r.skippedFirstNextCall = true
-		return true
-	}
-	r.rows = r.rows[1:]
-	return len(r.rows) > 0
-}
-
-func (r iteratorForCreateGames) Values() ([]interface{}, error) {
-	return []interface{}{
-		r.rows[0].ID,
-		r.rows[0].Date,
-		r.rows[0].CoverageStatus,
-		r.rows[0].LowCoverage,
-		r.rows[0].Minutes,
-		r.rows[0].PeriodType,
-		r.rows[0].Scored,
-		r.rows[0].Status,
-		r.rows[0].CompetitionSlug,
-		r.rows[0].FixtureSlug,
-		r.rows[0].AwayTeamSlug,
-		r.rows[0].AwayGoals,
-		r.rows[0].AwayExtraTimeScore,
-		r.rows[0].AwayPenaltyScore,
-		r.rows[0].HomeTeamSlug,
-		r.rows[0].HomeGoals,
-		r.rows[0].HomeExtraTimeScore,
-		r.rows[0].HomePenaltyScore,
-		r.rows[0].WinnerTeamSlug,
-	}, nil
-}
-
-func (r iteratorForCreateGames) Err() error {
-	return nil
-}
-
-func (q *Queries) CreateGames(ctx context.Context, arg []CreateGamesParams) (int64, error) {
-	return q.db.CopyFrom(ctx, []string{"games"}, []string{"id", "date", "coverage_status", "low_coverage", "minutes", "period_type", "scored", "status", "competition_slug", "fixture_slug", "away_team_slug", "away_goals", "away_extra_time_score", "away_penalty_score", "home_team_slug", "home_goals", "home_extra_time_score", "home_penalty_score", "winner_team_slug"}, &iteratorForCreateGames{rows: arg})
-}
-
-// iteratorForCreateTeams implements pgx.CopyFromSource.
-type iteratorForCreateTeams struct {
-	rows                 []CreateTeamsParams
-	skippedFirstNextCall bool
-}
-
-func (r *iteratorForCreateTeams) Next() bool {
-	if len(r.rows) == 0 {
-		return false
-	}
-	if !r.skippedFirstNextCall {
-		r.skippedFirstNextCall = true
-		return true
-	}
-	r.rows = r.rows[1:]
-	return len(r.rows) > 0
-}
-
-func (r iteratorForCreateTeams) Values() ([]interface{}, error) {
-	return []interface{}{
-		r.rows[0].Slug,
-		r.rows[0].DisplayName,
-		r.rows[0].CountrySlug,
-		r.rows[0].DomesticLeagueSlug,
-		r.rows[0].ShortName,
-		r.rows[0].PictureUrl,
-		r.rows[0].TeamType,
-	}, nil
-}
-
-func (r iteratorForCreateTeams) Err() error {
-	return nil
-}
-
-func (q *Queries) CreateTeams(ctx context.Context, arg []CreateTeamsParams) (int64, error) {
-	return q.db.CopyFrom(ctx, []string{"teams"}, []string{"slug", "display_name", "country_slug", "domestic_league_slug", "short_name", "picture_url", "team_type"}, &iteratorForCreateTeams{rows: arg})
-}

model/game.sql.go

@@ -9,7 +9,7 @@ import (
 	"github.com/jackc/pgx/v5/pgtype"
 )
 
-type CreateGamesParams struct {
+type BatchInsertGamesParams struct {
 	ID                 string
 	Date               pgtype.Timestamptz
 	CoverageStatus     string

model/sql/competition.sql

@@ -1,3 +1,7 @@
+-- name: BatchInsertCompetitions :copyfrom
+   INSERT INTO competitions (slug, display_name, country_slug, competition_format, competition_type, picture_url, logo_url)
+   VALUES ($1, $2, $3, $4, $5, $6, $7);
+
 -- name: CreateOrUpdateCompetition :exec
    INSERT INTO competitions (
           slug,

model/sql/game.sql

@@ -1,4 +1,4 @@
--- name: CreateGames :copyfrom
+-- name: BatchInsertGames :copyfrom
 INSERT INTO games (id, date, coverage_status, low_coverage, minutes, period_type, scored, status, competition_slug, fixture_slug, away_team_slug, away_goals, away_extra_time_score, away_penalty_score, home_team_slug, home_goals, home_extra_time_score, home_penalty_score, winner_team_slug)
 VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19);
 

model/sql/team.sql

@@ -1,3 +1,3 @@
--- name: CreateTeams :copyfrom
+-- name: BatchInsertTeams :copyfrom
 INSERT INTO teams (slug, display_name, country_slug, domestic_league_slug, short_name, picture_url, team_type)
 VALUES ($1, $2, $3, $4, $5, $6, $7);

model/sql/test.sql

@@ -1,2 +0,0 @@
-select * from competitions
-join games on games.id = competitions.game_id

model/team.sql.go

@@ -7,7 +7,7 @@ package model
 
 import ()
 
-type CreateTeamsParams struct {
+type BatchInsertTeamsParams struct {
 	Slug               string
 	DisplayName        string
 	CountrySlug        string

sorare_utils/fixture.go

@@ -0,0 +1,110 @@
+package sorare_utils
+
+import (
+	"context"
+
+	"git.lehouerou.net/laurent/sorare"
+	"git.lehouerou.net/laurent/sorare/football"
+	"git.lehouerou.net/laurent/sorare/graphql"
+	"github.com/pkg/errors"
+	"github.com/rs/zerolog/log"
+	"github.com/samber/lo"
+)
+
+func GetGamesFromFixtures(
+	ctx context.Context,
+	s *sorare.Sorare,
+	fixtureSlugs []string,
+) ([]football.Game, error) {
+	var games []football.Game
+	for _, slug := range fixtureSlugs {
+		log.Debug().Msgf("getting games for fixture %s...", slug)
+		g, err := s.Football.So5.FixtureGames(slug).Get(ctx, graphql.EmptyParams{})
+		if err != nil {
+			return nil, errors.Wrapf(err, "getting games for fixture %s", slug)
+		}
+		log.Debug().Msgf("found %d games", len(g))
+		games = append(games, g...)
+	}
+	log.Debug().Msgf("found total of %d games before filtering", len(games))
+	games = lo.Filter(games, func(game football.Game, index int) bool {
+		return game.Id.Value != ""
+	})
+	log.Debug().Msgf("found total of %d games after filtering empty ids", len(games))
+	games = lo.UniqBy(games, func(game football.Game) string {
+		return game.Id.Value
+	})
+	log.Debug().Msgf("found total of %d games after filtering duplicate ids", len(games))
+	return games, nil
+}
+
+func ExtractTeamSlugsFromGames(games []football.Game) []string {
+	var res []string
+	res = lo.Map(games, func(game football.Game, index int) string {
+		return game.AwayTeam.Team.Slug
+	})
+	res = append(res, lo.Map(games, func(game football.Game, index int) string {
+		return game.HomeTeam.Team.Slug
+	})...)
+	res = lo.Filter(res, func(slug string, index int) bool {
+		return slug != ""
+	})
+	return lo.Uniq(res)
+}
+
+func ExtractCountrySlugsFromCompetitions(competitions []football.Competition) []string {
+	return lo.Uniq(lo.Filter(lo.Map(competitions, func(competition football.Competition, index int) string {
+		return competition.Country.Slug
+	}), func(slug string, index int) bool {
+		return slug != ""
+	}))
+}
+
+func ExtractCountrySlugsFromTeams(teams []football.Team) []string {
+	return lo.Uniq(lo.Filter(lo.Map(teams, func(team football.Team, index int) string {
+		return team.Country.Slug
+	}), func(slug string, index int) bool {
+		return slug != ""
+	}))
+}
+
+func ExtractCountrySlugsFromCompetitionsClubsAndNationalTeams(
+	competitions []football.Competition,
+	clubs []football.Club,
+	nationalTeams []football.NationalTeam,
+) []string {
+	return lo.Uniq(lo.Union(
+		ExtractCountrySlugsFromCompetitions(competitions),
+		ExtractCountrySlugsFromTeams(lo.Map(clubs, func(club football.Club, index int) football.Team {
+			return club.Team
+		})),
+		ExtractCountrySlugsFromTeams(
+			lo.Map(nationalTeams, func(nationalTeam football.NationalTeam, index int) football.Team {
+				return nationalTeam.Team
+			}),
+		),
+	))
+}
+
+func ExtractCompetitionSlugsFromGames(games []football.Game) []string {
+	return lo.Uniq(lo.Filter(lo.Map(games, func(game football.Game, index int) string {
+		return game.Competition.Slug
+	}), func(slug string, index int) bool {
+		return slug != ""
+	}))
+}
+
+func ExtractCompetitionSlugsFromClubs(clubs []football.Club) []string {
+	return lo.Uniq(lo.Filter(lo.Map(clubs, func(club football.Club, index int) string {
+		return club.DomesticLeague.Slug
+	}), func(slug string, index int) bool {
+		return slug != ""
+	}))
+}
+
+func ExtractCompetitionSlugsFromGamesAndClubs(games []football.Game, clubs []football.Club) []string {
+	return lo.Uniq(lo.Union(
+		ExtractCompetitionSlugsFromGames(games),
+		ExtractCompetitionSlugsFromClubs(clubs),
+	))
+}