From 717ae339dc40d0d869a8f7d04b92014ee63e55cb Mon Sep 17 00:00:00 2001 From: Laurent Le Houerou Date: Fri, 22 Mar 2024 05:59:47 +0000 Subject: [PATCH] move db initialization into UpdateService --- cmd/console/root.go | 220 +----------------------------- sorare_utils/update_service.go | 242 +++++++++++++++++++++++++++++++++ 2 files changed, 245 insertions(+), 217 deletions(-) create mode 100644 sorare_utils/update_service.go diff --git a/cmd/console/root.go b/cmd/console/root.go index 14981c2..b0a0231 100644 --- a/cmd/console/root.go +++ b/cmd/console/root.go @@ -2,12 +2,7 @@ package main import ( "git.lehouerou.net/laurent/sorare" - "git.lehouerou.net/laurent/sorare/football" - "git.lehouerou.net/laurent/sorare/graphql" - "github.com/jackc/pgx/v5/pgtype" "github.com/pkg/errors" - "github.com/rs/zerolog/log" - "github.com/samber/lo" "github.com/spf13/cobra" "git.lehouerou.net/laurent/sorarebuddy/cmd/common" @@ -45,220 +40,11 @@ func run(cmd *cobra.Command, _ []string) error { return errors.New("db not found in context") } - sfixtures, err := s.Football.So5.So5Fixtures.Get(ctx, football.So5FixturesParams{}) + us := sorare_utils.NewUpdateService(s, db) + err := us.InitSyncDatabase(ctx) if err != nil { - return err + return errors.Wrap(err, "initializing database") } - log.Debug().Msgf("fixtures: %v", sfixtures) - - cnt, err := db.CreateFixtures( - ctx, - lo.Map(sfixtures, func(fixture football.So5Fixture, index int) model.CreateFixturesParams { - return model.CreateFixturesParams{ - Slug: fixture.Slug, - DisplayName: fixture.DisplayName, - State: fixture.AasmState, - StartDate: pgtype.Timestamptz{Time: fixture.StartDate, Valid: true}, - EndDate: pgtype.Timestamptz{Time: fixture.EndDate, Valid: true}, - GameWeek: int32(fixture.GameWeek), - } - }), - ) - if err != nil { - return err - } - - log.Debug().Msgf("created %d fixtures", cnt) - - fixtures, err := db.GetAllFixtures(ctx) - if err != nil { - return err - } - games, err := sorare_utils.GetGamesFromFixtures( - ctx, - s, - lo.Map(fixtures, func(fixture model.Fixture, index int) string { - return fixture.Slug - }), - ) - log.Info().Msgf("found %d games to process", len(games)) - - teamSlugs := sorare_utils.ExtractTeamSlugsFromGames(games) - log.Debug().Msgf("extracted %d unique team slugs from games", len(teamSlugs)) - - log.Debug().Msgf("getting clubs...") - var clubs []football.Club - for i, chunk := range lo.Chunk(teamSlugs, 100) { - log.Debug().Msgf("\tbatch %d/%d", i+1, len(teamSlugs)/100) - t, err := s.Football.Clubs.Get(ctx, graphql.SlugsParams{Slugs: chunk}) - if err != nil { - return err - } - clubs = append(clubs, t...) - } - log.Debug().Msgf("found %d clubs", len(clubs)) - var nationalTeams []football.NationalTeam - slugsLeft := lo.Without(teamSlugs, lo.Map(clubs, func(club football.Club, index int) string { - return club.Slug - })...) - log.Debug().Msgf("getting national teams...") - log.Debug().Msgf("slugs left: %d", len(slugsLeft)) - for i, chunk := range lo.Chunk(slugsLeft, 100) { - log.Debug().Msgf("\tbatch %d/%d", i+1, len(teamSlugs)/100) - t, err := s.Football.NationalTeams.Get(ctx, graphql.SlugsParams{Slugs: chunk}) - if err != nil { - return err - } - nationalTeams = append(nationalTeams, t...) - } - log.Debug().Msgf("found %d national teams", len(nationalTeams)) - - competitionSlugs := sorare_utils.ExtractCompetitionSlugsFromGamesAndClubs(games, clubs) - log.Debug().Msgf("extracted %d unique competition slugs from games and clubs", len(competitionSlugs)) - log.Debug().Msgf("getting competitions...") - var competitions []football.Competition - for _, slug := range competitionSlugs { - log.Debug().Msgf("\tcompetition %s", slug) - c, err := s.Football.Competition.Get(ctx, graphql.SlugParams{Slug: slug}) - if err != nil { - return err - } - competitions = append(competitions, c) - } - log.Debug().Msgf("found %d competitions", len(competitions)) - - countrySlugs := sorare_utils.ExtractCountrySlugsFromCompetitionsClubsAndNationalTeams( - competitions, - clubs, - nationalTeams, - ) - log.Debug().Msgf("extracted %d unique country slugs from competitions, clubs and national teams", len(countrySlugs)) - log.Debug().Msgf("getting countries...") - var countries []sorare.Country - for i, chunk := range lo.Chunk(countrySlugs, 100) { - log.Debug().Msgf("\tbatch %d/%d", i+1, len(countrySlugs)/100) - c, err := s.Countries.Get(ctx, graphql.SlugsParams{Slugs: chunk}) - if err != nil { - return err - } - countries = append(countries, c...) - } - log.Debug().Msgf("found %d countries", len(countries)) - - log.Debug().Msg("inserting countries into db...") - cnt, err = db.CreateCountries( - ctx, - lo.Map(countries, func(country sorare.Country, index int) model.CreateCountriesParams { - return model.CreateCountriesParams{ - Slug: country.Slug, - Code: country.Code, - DisplayName: country.Name, - ThreeLetterCode: country.ThreeLetterCode, - FlagFlat64Url: country.FlagFlat64Url, - FlagFlat32Url: country.FlagFlat32Url, - FlagRound64Url: country.FlagRound64Url, - FlagRound32Url: country.FlagRound32Url, - } - }), - ) - if err != nil { - return err - } - log.Debug().Msgf("%d countries inserted", cnt) - - log.Debug().Msg("inserting competitions into db...") - - _, err = db.BatchInsertCompetitions( - ctx, - lo.Map(competitions, func(competition football.Competition, index int) model.BatchInsertCompetitionsParams { - return model.BatchInsertCompetitionsParams{ - Slug: competition.Slug, - CompetitionFormat: competition.Format, - CompetitionType: competition.Type, - DisplayName: competition.DisplayName, - PictureUrl: competition.PictureUrl, - LogoUrl: competition.LogoUrl, - CountrySlug: competition.Country.Slug, - } - }), - ) - if err != nil { - return err - } - - log.Debug().Msgf("%d competitions inserted", len(competitions)) - - log.Debug().Msg("inserting teams into db...") - cnt, err = db.BatchInsertTeams(ctx, lo.Union( - lo.Map(clubs, func(club football.Club, index int) model.BatchInsertTeamsParams { - return model.BatchInsertTeamsParams{ - Slug: club.Slug, - DisplayName: club.Name, - CountrySlug: club.Country.Slug, - DomesticLeagueSlug: func() *string { - if club.DomesticLeague.Slug == "" { - return nil - } - return &club.DomesticLeague.Slug - }(), - ShortName: club.ShortName, - PictureUrl: club.PictureUrl, - TeamType: "club", - } - }), - lo.Map(nationalTeams, func(nationalTeam football.NationalTeam, index int) model.BatchInsertTeamsParams { - return model.BatchInsertTeamsParams{ - Slug: nationalTeam.Slug, - DisplayName: nationalTeam.Name, - CountrySlug: nationalTeam.Country.Slug, - DomesticLeagueSlug: nil, - ShortName: nationalTeam.ShortName, - PictureUrl: nationalTeam.PictureUrl, - TeamType: "national", - } - }), - )) - if err != nil { - return err - } - log.Debug().Msgf("%d teams inserted", cnt) - - log.Debug().Msg("inserting games into db...") - cnt, err = db.BatchInsertGames( - ctx, - lo.Map(games, func(game football.Game, index int) model.BatchInsertGamesParams { - return model.BatchInsertGamesParams{ - ID: game.Id.Value, - Date: pgtype.Timestamptz{Time: game.Date, Valid: true}, - CoverageStatus: game.CoverageStatus, - LowCoverage: game.LowCoverage, - Minutes: int32(game.Minute), - PeriodType: game.PeriodType, - Scored: game.Scored, - Status: game.Status, - CompetitionSlug: game.Competition.Slug, - FixtureSlug: game.So5Fixture.Slug, - AwayTeamSlug: game.AwayTeam.Team.Slug, - AwayGoals: int32(game.AwayGoals), - AwayExtraTimeScore: int32(game.ExtraTimeScoreAway), - AwayPenaltyScore: int32(game.PenaltyScoreAway), - HomeTeamSlug: game.HomeTeam.Team.Slug, - HomeGoals: int32(game.HomeGoals), - HomeExtraTimeScore: int32(game.ExtraTimeScoreHome), - HomePenaltyScore: int32(game.PenaltyScoreHome), - WinnerTeamSlug: func() *string { - if game.Winner.Team.Slug == "" { - return nil - } - return &game.Winner.Team.Slug - }(), - } - }), - ) - if err != nil { - return err - } - log.Debug().Msgf("%d games inserted", cnt) // log.Debug().Msg("start sequence completed. waiting for shutdown request") // <-ctx.Done() diff --git a/sorare_utils/update_service.go b/sorare_utils/update_service.go new file mode 100644 index 0000000..1214a9d --- /dev/null +++ b/sorare_utils/update_service.go @@ -0,0 +1,242 @@ +package sorare_utils + +import ( + "context" + + "git.lehouerou.net/laurent/sorare" + "git.lehouerou.net/laurent/sorare/football" + "git.lehouerou.net/laurent/sorare/graphql" + "github.com/jackc/pgx/v5/pgtype" + "github.com/rs/zerolog/log" + "github.com/samber/lo" + + "git.lehouerou.net/laurent/sorarebuddy/model" +) + +type UpdateService struct { + s *sorare.Sorare + db *model.Queries +} + +func NewUpdateService(s *sorare.Sorare, db *model.Queries) *UpdateService { + return &UpdateService{s: s, db: db} +} + +func (u *UpdateService) InitSyncDatabase(ctx context.Context) error { + + sfixtures, err := u.s.Football.So5.So5Fixtures.Get(ctx, football.So5FixturesParams{}) + if err != nil { + return err + } + log.Debug().Msgf("fixtures: %v", sfixtures) + + cnt, err := u.db.CreateFixtures( + ctx, + lo.Map(sfixtures, func(fixture football.So5Fixture, index int) model.CreateFixturesParams { + return model.CreateFixturesParams{ + Slug: fixture.Slug, + DisplayName: fixture.DisplayName, + State: fixture.AasmState, + StartDate: pgtype.Timestamptz{Time: fixture.StartDate, Valid: true}, + EndDate: pgtype.Timestamptz{Time: fixture.EndDate, Valid: true}, + GameWeek: int32(fixture.GameWeek), + } + }), + ) + if err != nil { + return err + } + + log.Debug().Msgf("created %d fixtures", cnt) + + fixtures, err := u.db.GetAllFixtures(ctx) + if err != nil { + return err + } + games, err := GetGamesFromFixtures( + ctx, + u.s, + lo.Map(fixtures, func(fixture model.Fixture, index int) string { + return fixture.Slug + }), + ) + log.Info().Msgf("found %d games to process", len(games)) + + teamSlugs := ExtractTeamSlugsFromGames(games) + log.Debug().Msgf("extracted %d unique team slugs from games", len(teamSlugs)) + + log.Debug().Msgf("getting clubs...") + var clubs []football.Club + for i, chunk := range lo.Chunk(teamSlugs, 100) { + log.Debug().Msgf("\tbatch %d/%d", i+1, len(teamSlugs)/100) + t, err := u.s.Football.Clubs.Get(ctx, graphql.SlugsParams{Slugs: chunk}) + if err != nil { + return err + } + clubs = append(clubs, t...) + } + log.Debug().Msgf("found %d clubs", len(clubs)) + var nationalTeams []football.NationalTeam + slugsLeft := lo.Without(teamSlugs, lo.Map(clubs, func(club football.Club, index int) string { + return club.Slug + })...) + log.Debug().Msgf("getting national teams...") + log.Debug().Msgf("slugs left: %d", len(slugsLeft)) + for i, chunk := range lo.Chunk(slugsLeft, 100) { + log.Debug().Msgf("\tbatch %d/%d", i+1, len(teamSlugs)/100) + t, err := u.s.Football.NationalTeams.Get(ctx, graphql.SlugsParams{Slugs: chunk}) + if err != nil { + return err + } + nationalTeams = append(nationalTeams, t...) + } + log.Debug().Msgf("found %d national teams", len(nationalTeams)) + + competitionSlugs := ExtractCompetitionSlugsFromGamesAndClubs(games, clubs) + log.Debug().Msgf("extracted %d unique competition slugs from games and clubs", len(competitionSlugs)) + log.Debug().Msgf("getting competitions...") + var competitions []football.Competition + for _, slug := range competitionSlugs { + log.Debug().Msgf("\tcompetition %s", slug) + c, err := u.s.Football.Competition.Get(ctx, graphql.SlugParams{Slug: slug}) + if err != nil { + return err + } + competitions = append(competitions, c) + } + log.Debug().Msgf("found %d competitions", len(competitions)) + + countrySlugs := ExtractCountrySlugsFromCompetitionsClubsAndNationalTeams( + competitions, + clubs, + nationalTeams, + ) + log.Debug().Msgf("extracted %d unique country slugs from competitions, clubs and national teams", len(countrySlugs)) + log.Debug().Msgf("getting countries...") + var countries []sorare.Country + for i, chunk := range lo.Chunk(countrySlugs, 100) { + log.Debug().Msgf("\tbatch %d/%d", i+1, len(countrySlugs)/100) + c, err := u.s.Countries.Get(ctx, graphql.SlugsParams{Slugs: chunk}) + if err != nil { + return err + } + countries = append(countries, c...) + } + log.Debug().Msgf("found %d countries", len(countries)) + + log.Debug().Msg("inserting countries into db...") + cnt, err = u.db.CreateCountries( + ctx, + lo.Map(countries, func(country sorare.Country, index int) model.CreateCountriesParams { + return model.CreateCountriesParams{ + Slug: country.Slug, + Code: country.Code, + DisplayName: country.Name, + ThreeLetterCode: country.ThreeLetterCode, + FlagFlat64Url: country.FlagFlat64Url, + FlagFlat32Url: country.FlagFlat32Url, + FlagRound64Url: country.FlagRound64Url, + FlagRound32Url: country.FlagRound32Url, + } + }), + ) + if err != nil { + return err + } + log.Debug().Msgf("%d countries inserted", cnt) + + log.Debug().Msg("inserting competitions into db...") + + _, err = u.db.BatchInsertCompetitions( + ctx, + lo.Map(competitions, func(competition football.Competition, index int) model.BatchInsertCompetitionsParams { + return model.BatchInsertCompetitionsParams{ + Slug: competition.Slug, + CompetitionFormat: competition.Format, + CompetitionType: competition.Type, + DisplayName: competition.DisplayName, + PictureUrl: competition.PictureUrl, + LogoUrl: competition.LogoUrl, + CountrySlug: competition.Country.Slug, + } + }), + ) + if err != nil { + return err + } + + log.Debug().Msgf("%d competitions inserted", len(competitions)) + + log.Debug().Msg("inserting teams into db...") + cnt, err = u.db.BatchInsertTeams(ctx, lo.Union( + lo.Map(clubs, func(club football.Club, index int) model.BatchInsertTeamsParams { + return model.BatchInsertTeamsParams{ + Slug: club.Slug, + DisplayName: club.Name, + CountrySlug: club.Country.Slug, + DomesticLeagueSlug: func() *string { + if club.DomesticLeague.Slug == "" { + return nil + } + return &club.DomesticLeague.Slug + }(), + ShortName: club.ShortName, + PictureUrl: club.PictureUrl, + TeamType: "club", + } + }), + lo.Map(nationalTeams, func(nationalTeam football.NationalTeam, index int) model.BatchInsertTeamsParams { + return model.BatchInsertTeamsParams{ + Slug: nationalTeam.Slug, + DisplayName: nationalTeam.Name, + CountrySlug: nationalTeam.Country.Slug, + DomesticLeagueSlug: nil, + ShortName: nationalTeam.ShortName, + PictureUrl: nationalTeam.PictureUrl, + TeamType: "national", + } + }), + )) + if err != nil { + return err + } + log.Debug().Msgf("%d teams inserted", cnt) + + log.Debug().Msg("inserting games into db...") + cnt, err = u.db.BatchInsertGames( + ctx, + lo.Map(games, func(game football.Game, index int) model.BatchInsertGamesParams { + return model.BatchInsertGamesParams{ + ID: game.Id.Value, + Date: pgtype.Timestamptz{Time: game.Date, Valid: true}, + CoverageStatus: game.CoverageStatus, + LowCoverage: game.LowCoverage, + Minutes: int32(game.Minute), + PeriodType: game.PeriodType, + Scored: game.Scored, + Status: game.Status, + CompetitionSlug: game.Competition.Slug, + FixtureSlug: game.So5Fixture.Slug, + AwayTeamSlug: game.AwayTeam.Team.Slug, + AwayGoals: int32(game.AwayGoals), + AwayExtraTimeScore: int32(game.ExtraTimeScoreAway), + AwayPenaltyScore: int32(game.PenaltyScoreAway), + HomeTeamSlug: game.HomeTeam.Team.Slug, + HomeGoals: int32(game.HomeGoals), + HomeExtraTimeScore: int32(game.ExtraTimeScoreHome), + HomePenaltyScore: int32(game.PenaltyScoreHome), + WinnerTeamSlug: func() *string { + if game.Winner.Team.Slug == "" { + return nil + } + return &game.Winner.Team.Slug + }(), + } + }), + ) + if err != nil { + return err + } + log.Debug().Msgf("%d games inserted", cnt) + return nil +}