From 122e83480ebb932dcaa54becbec645732a310f53 Mon Sep 17 00:00:00 2001 From: Laurent Le Houerou Date: Thu, 21 Mar 2024 17:57:17 +0000 Subject: [PATCH] refactor db initialization (wip) --- cmd/console/root.go | 265 ++++++++++++++++++-------------------- model/competition.sql.go | 11 +- model/copyfrom.go | 214 +++++++++++++++++------------- model/country.sql.go | 2 +- model/game.sql.go | 2 +- model/sql/competition.sql | 4 + model/sql/game.sql | 2 +- model/sql/team.sql | 2 +- model/sql/test.sql | 2 - model/team.sql.go | 2 +- sorare_utils/fixture.go | 110 ++++++++++++++++ 11 files changed, 382 insertions(+), 234 deletions(-) delete mode 100644 model/sql/test.sql create mode 100644 sorare_utils/fixture.go diff --git a/cmd/console/root.go b/cmd/console/root.go index 8010d57..14981c2 100644 --- a/cmd/console/root.go +++ b/cmd/console/root.go @@ -12,6 +12,7 @@ import ( "git.lehouerou.net/laurent/sorarebuddy/cmd/common" "git.lehouerou.net/laurent/sorarebuddy/model" + "git.lehouerou.net/laurent/sorarebuddy/sorare_utils" ) var Cmd = &cobra.Command{ @@ -73,129 +74,78 @@ func run(cmd *cobra.Command, _ []string) error { if err != nil { return err } + games, err := sorare_utils.GetGamesFromFixtures( + ctx, + s, + lo.Map(fixtures, func(fixture model.Fixture, index int) string { + return fixture.Slug + }), + ) + log.Info().Msgf("found %d games to process", len(games)) - var games []football.Game - for _, fixture := range fixtures { - log.Debug().Msgf("fixture: %v", fixture) - g, err := s.Football.So5.FixtureGames(fixture.Slug).Get(ctx, graphql.EmptyParams{}) - if err != nil { - return err - } - log.Debug().Msgf("games: %d", len(g)) - games = append(games, g...) - } - log.Debug().Msgf("games: %d", len(games)) - - var teamSlugs []string - teamSlugs = lo.Map(games, func(game football.Game, index int) string { - return game.AwayTeam.Team.Slug - }) - teamSlugs = append(teamSlugs, lo.Map(games, func(game football.Game, index int) string { - return game.HomeTeam.Team.Slug - })...) - teamSlugs = lo.Filter(teamSlugs, func(slug string, index int) bool { - return slug != "" - }) - teamSlugs = lo.Uniq(teamSlugs) + teamSlugs := sorare_utils.ExtractTeamSlugsFromGames(games) + log.Debug().Msgf("extracted %d unique team slugs from games", len(teamSlugs)) + log.Debug().Msgf("getting clubs...") var clubs []football.Club - for _, chunk := range lo.Chunk(teamSlugs, 100) { + for i, chunk := range lo.Chunk(teamSlugs, 100) { + log.Debug().Msgf("\tbatch %d/%d", i+1, len(teamSlugs)/100) t, err := s.Football.Clubs.Get(ctx, graphql.SlugsParams{Slugs: chunk}) if err != nil { return err } clubs = append(clubs, t...) } - + log.Debug().Msgf("found %d clubs", len(clubs)) var nationalTeams []football.NationalTeam - for _, chunk := range lo.Chunk(lo.Without(teamSlugs, lo.Map(clubs, func(club football.Club, index int) string { + slugsLeft := lo.Without(teamSlugs, lo.Map(clubs, func(club football.Club, index int) string { return club.Slug - })...), 100) { + })...) + log.Debug().Msgf("getting national teams...") + log.Debug().Msgf("slugs left: %d", len(slugsLeft)) + for i, chunk := range lo.Chunk(slugsLeft, 100) { + log.Debug().Msgf("\tbatch %d/%d", i+1, len(teamSlugs)/100) t, err := s.Football.NationalTeams.Get(ctx, graphql.SlugsParams{Slugs: chunk}) if err != nil { return err } nationalTeams = append(nationalTeams, t...) } + log.Debug().Msgf("found %d national teams", len(nationalTeams)) - var teamsToInsert []model.CreateTeamsParams - teamsToInsert = append(teamsToInsert, lo.Map(clubs, func(club football.Club, index int) model.CreateTeamsParams { - return model.CreateTeamsParams{ - Slug: club.Slug, - DisplayName: club.Name, - CountrySlug: club.Country.Slug, - DomesticLeagueSlug: func() *string { - if club.DomesticLeague.Slug == "" { - return nil - } - return &club.DomesticLeague.Slug - }(), - ShortName: club.ShortName, - PictureUrl: club.PictureUrl, - TeamType: "club", - } - })...) - teamsToInsert = append( - teamsToInsert, - lo.Map(nationalTeams, func(nationalTeam football.NationalTeam, index int) model.CreateTeamsParams { - return model.CreateTeamsParams{ - Slug: nationalTeam.Slug, - DisplayName: nationalTeam.Name, - CountrySlug: nationalTeam.Country.Slug, - DomesticLeagueSlug: nil, - ShortName: nationalTeam.ShortName, - PictureUrl: nationalTeam.PictureUrl, - TeamType: "national", - } - })...) - - competitionSlugs := lo.Map(games, func(game football.Game, index int) string { - return game.Competition.Slug - }) - competitionSlugs = append( - competitionSlugs, - lo.Map(teamsToInsert, func(team model.CreateTeamsParams, index int) string { - if team.DomesticLeagueSlug == nil { - return "" - } - return *team.DomesticLeagueSlug - })...) - competitionSlugs = lo.Filter(competitionSlugs, func(slug string, index int) bool { - return slug != "" - }) - competitionSlugs = lo.Uniq(competitionSlugs) - + competitionSlugs := sorare_utils.ExtractCompetitionSlugsFromGamesAndClubs(games, clubs) + log.Debug().Msgf("extracted %d unique competition slugs from games and clubs", len(competitionSlugs)) + log.Debug().Msgf("getting competitions...") var competitions []football.Competition for _, slug := range competitionSlugs { - log.Debug().Msgf("competition: %s", slug) + log.Debug().Msgf("\tcompetition %s", slug) c, err := s.Football.Competition.Get(ctx, graphql.SlugParams{Slug: slug}) if err != nil { return err } competitions = append(competitions, c) } + log.Debug().Msgf("found %d competitions", len(competitions)) - var countrySlugs []string - countrySlugs = append(countrySlugs, lo.Map(teamsToInsert, func(team model.CreateTeamsParams, index int) string { - return team.CountrySlug - })...) - countrySlugs = append(countrySlugs, lo.Map(competitions, func(competition football.Competition, index int) string { - return competition.Country.Slug - })...) - countrySlugs = lo.Filter(countrySlugs, func(slug string, index int) bool { - return slug != "" - }) - countrySlugs = lo.Uniq(countrySlugs) - + countrySlugs := sorare_utils.ExtractCountrySlugsFromCompetitionsClubsAndNationalTeams( + competitions, + clubs, + nationalTeams, + ) + log.Debug().Msgf("extracted %d unique country slugs from competitions, clubs and national teams", len(countrySlugs)) + log.Debug().Msgf("getting countries...") var countries []sorare.Country - for _, chunk := range lo.Chunk(countrySlugs, 100) { + for i, chunk := range lo.Chunk(countrySlugs, 100) { + log.Debug().Msgf("\tbatch %d/%d", i+1, len(countrySlugs)/100) c, err := s.Countries.Get(ctx, graphql.SlugsParams{Slugs: chunk}) if err != nil { return err } countries = append(countries, c...) } + log.Debug().Msgf("found %d countries", len(countries)) + log.Debug().Msg("inserting countries into db...") cnt, err = db.CreateCountries( ctx, lo.Map(countries, func(country sorare.Country, index int) model.CreateCountriesParams { @@ -214,62 +164,101 @@ func run(cmd *cobra.Command, _ []string) error { if err != nil { return err } - log.Debug().Msgf("created %d countries", cnt) + log.Debug().Msgf("%d countries inserted", cnt) - log.Debug().Msgf("competitions: %d", len(competitions)) - for _, competition := range competitions { + log.Debug().Msg("inserting competitions into db...") - err := db.CreateOrUpdateCompetition(ctx, model.CreateOrUpdateCompetitionParams{ - Slug: competition.Slug, - CompetitionFormat: competition.Format, - CompetitionType: competition.Type, - DisplayName: competition.DisplayName, - PictureUrl: competition.PictureUrl, - LogoUrl: competition.LogoUrl, - CountrySlug: competition.Country.Slug, - }) - if err != nil { - return err - } - } - - cnt, err = db.CreateTeams(ctx, teamsToInsert) - if err != nil { - return err - } - log.Debug().Msgf("created %d teams", cnt) - - cnt, err = db.CreateGames(ctx, lo.Map(games, func(game football.Game, index int) model.CreateGamesParams { - return model.CreateGamesParams{ - CoverageStatus: game.CoverageStatus, - LowCoverage: game.LowCoverage, - Minutes: int32(game.Minute), - PeriodType: game.PeriodType, - Scored: game.Scored, - Status: game.Status, - CompetitionSlug: game.Competition.Slug, - FixtureSlug: game.So5Fixture.Slug, - AwayTeamSlug: game.AwayTeam.Team.Slug, - AwayGoals: int32(game.AwayGoals), - AwayExtraTimeScore: int32(game.ExtraTimeScoreAway), - AwayPenaltyScore: int32(game.PenaltyScoreAway), - HomeTeamSlug: game.HomeTeam.Team.Slug, - HomeGoals: int32(game.HomeGoals), - HomeExtraTimeScore: int32(game.ExtraTimeScoreHome), - HomePenaltyScore: int32(game.PenaltyScoreHome), - WinnerTeamSlug: func() *string { - if game.Winner.Team.Slug == "" { - return nil - } - return &game.Winner.Team.Slug - }(), - } - })) + _, err = db.BatchInsertCompetitions( + ctx, + lo.Map(competitions, func(competition football.Competition, index int) model.BatchInsertCompetitionsParams { + return model.BatchInsertCompetitionsParams{ + Slug: competition.Slug, + CompetitionFormat: competition.Format, + CompetitionType: competition.Type, + DisplayName: competition.DisplayName, + PictureUrl: competition.PictureUrl, + LogoUrl: competition.LogoUrl, + CountrySlug: competition.Country.Slug, + } + }), + ) if err != nil { return err } - log.Debug().Msgf("created %d games", cnt) + log.Debug().Msgf("%d competitions inserted", len(competitions)) + + log.Debug().Msg("inserting teams into db...") + cnt, err = db.BatchInsertTeams(ctx, lo.Union( + lo.Map(clubs, func(club football.Club, index int) model.BatchInsertTeamsParams { + return model.BatchInsertTeamsParams{ + Slug: club.Slug, + DisplayName: club.Name, + CountrySlug: club.Country.Slug, + DomesticLeagueSlug: func() *string { + if club.DomesticLeague.Slug == "" { + return nil + } + return &club.DomesticLeague.Slug + }(), + ShortName: club.ShortName, + PictureUrl: club.PictureUrl, + TeamType: "club", + } + }), + lo.Map(nationalTeams, func(nationalTeam football.NationalTeam, index int) model.BatchInsertTeamsParams { + return model.BatchInsertTeamsParams{ + Slug: nationalTeam.Slug, + DisplayName: nationalTeam.Name, + CountrySlug: nationalTeam.Country.Slug, + DomesticLeagueSlug: nil, + ShortName: nationalTeam.ShortName, + PictureUrl: nationalTeam.PictureUrl, + TeamType: "national", + } + }), + )) + if err != nil { + return err + } + log.Debug().Msgf("%d teams inserted", cnt) + + log.Debug().Msg("inserting games into db...") + cnt, err = db.BatchInsertGames( + ctx, + lo.Map(games, func(game football.Game, index int) model.BatchInsertGamesParams { + return model.BatchInsertGamesParams{ + ID: game.Id.Value, + Date: pgtype.Timestamptz{Time: game.Date, Valid: true}, + CoverageStatus: game.CoverageStatus, + LowCoverage: game.LowCoverage, + Minutes: int32(game.Minute), + PeriodType: game.PeriodType, + Scored: game.Scored, + Status: game.Status, + CompetitionSlug: game.Competition.Slug, + FixtureSlug: game.So5Fixture.Slug, + AwayTeamSlug: game.AwayTeam.Team.Slug, + AwayGoals: int32(game.AwayGoals), + AwayExtraTimeScore: int32(game.ExtraTimeScoreAway), + AwayPenaltyScore: int32(game.PenaltyScoreAway), + HomeTeamSlug: game.HomeTeam.Team.Slug, + HomeGoals: int32(game.HomeGoals), + HomeExtraTimeScore: int32(game.ExtraTimeScoreHome), + HomePenaltyScore: int32(game.PenaltyScoreHome), + WinnerTeamSlug: func() *string { + if game.Winner.Team.Slug == "" { + return nil + } + return &game.Winner.Team.Slug + }(), + } + }), + ) + if err != nil { + return err + } + log.Debug().Msgf("%d games inserted", cnt) // log.Debug().Msg("start sequence completed. waiting for shutdown request") // <-ctx.Done() diff --git a/model/competition.sql.go b/model/competition.sql.go index 2deb2fc..5595951 100644 --- a/model/competition.sql.go +++ b/model/competition.sql.go @@ -9,6 +9,16 @@ import ( "context" ) +type BatchInsertCompetitionsParams struct { + Slug string + DisplayName string + CountrySlug string + CompetitionFormat string + CompetitionType string + PictureUrl string + LogoUrl string +} + const createOrUpdateCompetition = `-- name: CreateOrUpdateCompetition :exec INSERT INTO competitions ( slug, @@ -40,7 +50,6 @@ type CreateOrUpdateCompetitionParams struct { LogoUrl string } -// Active: 1709890109198@@192.168.1.250@5436@sorare func (q *Queries) CreateOrUpdateCompetition(ctx context.Context, arg CreateOrUpdateCompetitionParams) error { _, err := q.db.Exec(ctx, createOrUpdateCompetition, arg.Slug, diff --git a/model/copyfrom.go b/model/copyfrom.go index a4da5b9..29aca41 100644 --- a/model/copyfrom.go +++ b/model/copyfrom.go @@ -9,6 +9,132 @@ import ( "context" ) +// iteratorForBatchInsertCompetitions implements pgx.CopyFromSource. +type iteratorForBatchInsertCompetitions struct { + rows []BatchInsertCompetitionsParams + skippedFirstNextCall bool +} + +func (r *iteratorForBatchInsertCompetitions) Next() bool { + if len(r.rows) == 0 { + return false + } + if !r.skippedFirstNextCall { + r.skippedFirstNextCall = true + return true + } + r.rows = r.rows[1:] + return len(r.rows) > 0 +} + +func (r iteratorForBatchInsertCompetitions) Values() ([]interface{}, error) { + return []interface{}{ + r.rows[0].Slug, + r.rows[0].DisplayName, + r.rows[0].CountrySlug, + r.rows[0].CompetitionFormat, + r.rows[0].CompetitionType, + r.rows[0].PictureUrl, + r.rows[0].LogoUrl, + }, nil +} + +func (r iteratorForBatchInsertCompetitions) Err() error { + return nil +} + +func (q *Queries) BatchInsertCompetitions(ctx context.Context, arg []BatchInsertCompetitionsParams) (int64, error) { + return q.db.CopyFrom(ctx, []string{"competitions"}, []string{"slug", "display_name", "country_slug", "competition_format", "competition_type", "picture_url", "logo_url"}, &iteratorForBatchInsertCompetitions{rows: arg}) +} + +// iteratorForBatchInsertGames implements pgx.CopyFromSource. +type iteratorForBatchInsertGames struct { + rows []BatchInsertGamesParams + skippedFirstNextCall bool +} + +func (r *iteratorForBatchInsertGames) Next() bool { + if len(r.rows) == 0 { + return false + } + if !r.skippedFirstNextCall { + r.skippedFirstNextCall = true + return true + } + r.rows = r.rows[1:] + return len(r.rows) > 0 +} + +func (r iteratorForBatchInsertGames) Values() ([]interface{}, error) { + return []interface{}{ + r.rows[0].ID, + r.rows[0].Date, + r.rows[0].CoverageStatus, + r.rows[0].LowCoverage, + r.rows[0].Minutes, + r.rows[0].PeriodType, + r.rows[0].Scored, + r.rows[0].Status, + r.rows[0].CompetitionSlug, + r.rows[0].FixtureSlug, + r.rows[0].AwayTeamSlug, + r.rows[0].AwayGoals, + r.rows[0].AwayExtraTimeScore, + r.rows[0].AwayPenaltyScore, + r.rows[0].HomeTeamSlug, + r.rows[0].HomeGoals, + r.rows[0].HomeExtraTimeScore, + r.rows[0].HomePenaltyScore, + r.rows[0].WinnerTeamSlug, + }, nil +} + +func (r iteratorForBatchInsertGames) Err() error { + return nil +} + +func (q *Queries) BatchInsertGames(ctx context.Context, arg []BatchInsertGamesParams) (int64, error) { + return q.db.CopyFrom(ctx, []string{"games"}, []string{"id", "date", "coverage_status", "low_coverage", "minutes", "period_type", "scored", "status", "competition_slug", "fixture_slug", "away_team_slug", "away_goals", "away_extra_time_score", "away_penalty_score", "home_team_slug", "home_goals", "home_extra_time_score", "home_penalty_score", "winner_team_slug"}, &iteratorForBatchInsertGames{rows: arg}) +} + +// iteratorForBatchInsertTeams implements pgx.CopyFromSource. +type iteratorForBatchInsertTeams struct { + rows []BatchInsertTeamsParams + skippedFirstNextCall bool +} + +func (r *iteratorForBatchInsertTeams) Next() bool { + if len(r.rows) == 0 { + return false + } + if !r.skippedFirstNextCall { + r.skippedFirstNextCall = true + return true + } + r.rows = r.rows[1:] + return len(r.rows) > 0 +} + +func (r iteratorForBatchInsertTeams) Values() ([]interface{}, error) { + return []interface{}{ + r.rows[0].Slug, + r.rows[0].DisplayName, + r.rows[0].CountrySlug, + r.rows[0].DomesticLeagueSlug, + r.rows[0].ShortName, + r.rows[0].PictureUrl, + r.rows[0].TeamType, + }, nil +} + +func (r iteratorForBatchInsertTeams) Err() error { + return nil +} + +func (q *Queries) BatchInsertTeams(ctx context.Context, arg []BatchInsertTeamsParams) (int64, error) { + return q.db.CopyFrom(ctx, []string{"teams"}, []string{"slug", "display_name", "country_slug", "domestic_league_slug", "short_name", "picture_url", "team_type"}, &iteratorForBatchInsertTeams{rows: arg}) +} + // iteratorForCreateCountries implements pgx.CopyFromSource. type iteratorForCreateCountries struct { rows []CreateCountriesParams @@ -85,91 +211,3 @@ func (r iteratorForCreateFixtures) Err() error { func (q *Queries) CreateFixtures(ctx context.Context, arg []CreateFixturesParams) (int64, error) { return q.db.CopyFrom(ctx, []string{"fixtures"}, []string{"slug", "display_name", "state", "start_date", "end_date", "game_week"}, &iteratorForCreateFixtures{rows: arg}) } - -// iteratorForCreateGames implements pgx.CopyFromSource. -type iteratorForCreateGames struct { - rows []CreateGamesParams - skippedFirstNextCall bool -} - -func (r *iteratorForCreateGames) Next() bool { - if len(r.rows) == 0 { - return false - } - if !r.skippedFirstNextCall { - r.skippedFirstNextCall = true - return true - } - r.rows = r.rows[1:] - return len(r.rows) > 0 -} - -func (r iteratorForCreateGames) Values() ([]interface{}, error) { - return []interface{}{ - r.rows[0].ID, - r.rows[0].Date, - r.rows[0].CoverageStatus, - r.rows[0].LowCoverage, - r.rows[0].Minutes, - r.rows[0].PeriodType, - r.rows[0].Scored, - r.rows[0].Status, - r.rows[0].CompetitionSlug, - r.rows[0].FixtureSlug, - r.rows[0].AwayTeamSlug, - r.rows[0].AwayGoals, - r.rows[0].AwayExtraTimeScore, - r.rows[0].AwayPenaltyScore, - r.rows[0].HomeTeamSlug, - r.rows[0].HomeGoals, - r.rows[0].HomeExtraTimeScore, - r.rows[0].HomePenaltyScore, - r.rows[0].WinnerTeamSlug, - }, nil -} - -func (r iteratorForCreateGames) Err() error { - return nil -} - -func (q *Queries) CreateGames(ctx context.Context, arg []CreateGamesParams) (int64, error) { - return q.db.CopyFrom(ctx, []string{"games"}, []string{"id", "date", "coverage_status", "low_coverage", "minutes", "period_type", "scored", "status", "competition_slug", "fixture_slug", "away_team_slug", "away_goals", "away_extra_time_score", "away_penalty_score", "home_team_slug", "home_goals", "home_extra_time_score", "home_penalty_score", "winner_team_slug"}, &iteratorForCreateGames{rows: arg}) -} - -// iteratorForCreateTeams implements pgx.CopyFromSource. -type iteratorForCreateTeams struct { - rows []CreateTeamsParams - skippedFirstNextCall bool -} - -func (r *iteratorForCreateTeams) Next() bool { - if len(r.rows) == 0 { - return false - } - if !r.skippedFirstNextCall { - r.skippedFirstNextCall = true - return true - } - r.rows = r.rows[1:] - return len(r.rows) > 0 -} - -func (r iteratorForCreateTeams) Values() ([]interface{}, error) { - return []interface{}{ - r.rows[0].Slug, - r.rows[0].DisplayName, - r.rows[0].CountrySlug, - r.rows[0].DomesticLeagueSlug, - r.rows[0].ShortName, - r.rows[0].PictureUrl, - r.rows[0].TeamType, - }, nil -} - -func (r iteratorForCreateTeams) Err() error { - return nil -} - -func (q *Queries) CreateTeams(ctx context.Context, arg []CreateTeamsParams) (int64, error) { - return q.db.CopyFrom(ctx, []string{"teams"}, []string{"slug", "display_name", "country_slug", "domestic_league_slug", "short_name", "picture_url", "team_type"}, &iteratorForCreateTeams{rows: arg}) -} diff --git a/model/country.sql.go b/model/country.sql.go index e6ad665..a7fcd2f 100644 --- a/model/country.sql.go +++ b/model/country.sql.go @@ -40,7 +40,7 @@ const createOrUpdateCountry = `-- name: CreateOrUpdateCountry :exec flag_flat_64_url = EXCLUDED.flag_flat_64_url, flag_flat_32_url = EXCLUDED.flag_flat_32_url, flag_round_64_url = EXCLUDED.flag_round_64_url, - flag_round_32_url = EXCLUDED.flag_round_32_url + flag_round_32_url = EXCLUDED.flag_round_32_url ` type CreateOrUpdateCountryParams struct { diff --git a/model/game.sql.go b/model/game.sql.go index c082fad..c60b47a 100644 --- a/model/game.sql.go +++ b/model/game.sql.go @@ -9,7 +9,7 @@ import ( "github.com/jackc/pgx/v5/pgtype" ) -type CreateGamesParams struct { +type BatchInsertGamesParams struct { ID string Date pgtype.Timestamptz CoverageStatus string diff --git a/model/sql/competition.sql b/model/sql/competition.sql index d020a67..ea3e0f3 100644 --- a/model/sql/competition.sql +++ b/model/sql/competition.sql @@ -1,3 +1,7 @@ +-- name: BatchInsertCompetitions :copyfrom + INSERT INTO competitions (slug, display_name, country_slug, competition_format, competition_type, picture_url, logo_url) + VALUES ($1, $2, $3, $4, $5, $6, $7); + -- name: CreateOrUpdateCompetition :exec INSERT INTO competitions ( slug, diff --git a/model/sql/game.sql b/model/sql/game.sql index 60d07f0..9f74f50 100644 --- a/model/sql/game.sql +++ b/model/sql/game.sql @@ -1,4 +1,4 @@ --- name: CreateGames :copyfrom +-- name: BatchInsertGames :copyfrom INSERT INTO games (id, date, coverage_status, low_coverage, minutes, period_type, scored, status, competition_slug, fixture_slug, away_team_slug, away_goals, away_extra_time_score, away_penalty_score, home_team_slug, home_goals, home_extra_time_score, home_penalty_score, winner_team_slug) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19); diff --git a/model/sql/team.sql b/model/sql/team.sql index c82c820..2f8c764 100644 --- a/model/sql/team.sql +++ b/model/sql/team.sql @@ -1,3 +1,3 @@ --- name: CreateTeams :copyfrom +-- name: BatchInsertTeams :copyfrom INSERT INTO teams (slug, display_name, country_slug, domestic_league_slug, short_name, picture_url, team_type) VALUES ($1, $2, $3, $4, $5, $6, $7); diff --git a/model/sql/test.sql b/model/sql/test.sql deleted file mode 100644 index c9fedec..0000000 --- a/model/sql/test.sql +++ /dev/null @@ -1,2 +0,0 @@ -select * from competitions -join games on games.id = competitions.game_id diff --git a/model/team.sql.go b/model/team.sql.go index 11ffca3..29fcba9 100644 --- a/model/team.sql.go +++ b/model/team.sql.go @@ -7,7 +7,7 @@ package model import () -type CreateTeamsParams struct { +type BatchInsertTeamsParams struct { Slug string DisplayName string CountrySlug string diff --git a/sorare_utils/fixture.go b/sorare_utils/fixture.go new file mode 100644 index 0000000..2b7fcfd --- /dev/null +++ b/sorare_utils/fixture.go @@ -0,0 +1,110 @@ +package sorare_utils + +import ( + "context" + + "git.lehouerou.net/laurent/sorare" + "git.lehouerou.net/laurent/sorare/football" + "git.lehouerou.net/laurent/sorare/graphql" + "github.com/pkg/errors" + "github.com/rs/zerolog/log" + "github.com/samber/lo" +) + +func GetGamesFromFixtures( + ctx context.Context, + s *sorare.Sorare, + fixtureSlugs []string, +) ([]football.Game, error) { + var games []football.Game + for _, slug := range fixtureSlugs { + log.Debug().Msgf("getting games for fixture %s...", slug) + g, err := s.Football.So5.FixtureGames(slug).Get(ctx, graphql.EmptyParams{}) + if err != nil { + return nil, errors.Wrapf(err, "getting games for fixture %s", slug) + } + log.Debug().Msgf("found %d games", len(g)) + games = append(games, g...) + } + log.Debug().Msgf("found total of %d games before filtering", len(games)) + games = lo.Filter(games, func(game football.Game, index int) bool { + return game.Id.Value != "" + }) + log.Debug().Msgf("found total of %d games after filtering empty ids", len(games)) + games = lo.UniqBy(games, func(game football.Game) string { + return game.Id.Value + }) + log.Debug().Msgf("found total of %d games after filtering duplicate ids", len(games)) + return games, nil +} + +func ExtractTeamSlugsFromGames(games []football.Game) []string { + var res []string + res = lo.Map(games, func(game football.Game, index int) string { + return game.AwayTeam.Team.Slug + }) + res = append(res, lo.Map(games, func(game football.Game, index int) string { + return game.HomeTeam.Team.Slug + })...) + res = lo.Filter(res, func(slug string, index int) bool { + return slug != "" + }) + return lo.Uniq(res) +} + +func ExtractCountrySlugsFromCompetitions(competitions []football.Competition) []string { + return lo.Uniq(lo.Filter(lo.Map(competitions, func(competition football.Competition, index int) string { + return competition.Country.Slug + }), func(slug string, index int) bool { + return slug != "" + })) +} + +func ExtractCountrySlugsFromTeams(teams []football.Team) []string { + return lo.Uniq(lo.Filter(lo.Map(teams, func(team football.Team, index int) string { + return team.Country.Slug + }), func(slug string, index int) bool { + return slug != "" + })) +} + +func ExtractCountrySlugsFromCompetitionsClubsAndNationalTeams( + competitions []football.Competition, + clubs []football.Club, + nationalTeams []football.NationalTeam, +) []string { + return lo.Uniq(lo.Union( + ExtractCountrySlugsFromCompetitions(competitions), + ExtractCountrySlugsFromTeams(lo.Map(clubs, func(club football.Club, index int) football.Team { + return club.Team + })), + ExtractCountrySlugsFromTeams( + lo.Map(nationalTeams, func(nationalTeam football.NationalTeam, index int) football.Team { + return nationalTeam.Team + }), + ), + )) +} + +func ExtractCompetitionSlugsFromGames(games []football.Game) []string { + return lo.Uniq(lo.Filter(lo.Map(games, func(game football.Game, index int) string { + return game.Competition.Slug + }), func(slug string, index int) bool { + return slug != "" + })) +} + +func ExtractCompetitionSlugsFromClubs(clubs []football.Club) []string { + return lo.Uniq(lo.Filter(lo.Map(clubs, func(club football.Club, index int) string { + return club.DomesticLeague.Slug + }), func(slug string, index int) bool { + return slug != "" + })) +} + +func ExtractCompetitionSlugsFromGamesAndClubs(games []football.Game, clubs []football.Club) []string { + return lo.Uniq(lo.Union( + ExtractCompetitionSlugsFromGames(games), + ExtractCompetitionSlugsFromClubs(clubs), + )) +}