package main import ( "git.lehouerou.net/laurent/sorare" "git.lehouerou.net/laurent/sorare/football" "git.lehouerou.net/laurent/sorare/graphql" "github.com/jackc/pgx/v5/pgtype" "github.com/pkg/errors" "github.com/rs/zerolog/log" "github.com/samber/lo" "github.com/spf13/cobra" "git.lehouerou.net/laurent/sorarebuddy/cmd/common" "git.lehouerou.net/laurent/sorarebuddy/model" "git.lehouerou.net/laurent/sorarebuddy/sorare_utils" ) var Cmd = &cobra.Command{ Use: "console", Short: "console", Long: `console`, RunE: run, PersistentPreRunE: common.CmdPreRunE, } func main() { common.Start(Cmd) } func init() { common.InitParams(Cmd) } func run(cmd *cobra.Command, _ []string) error { //var wg conc.WaitGroup ctx := cmd.Context() s, ok := ctx.Value(common.SorareContextKey).(*sorare.Sorare) if !ok { return errors.New("sorare not found in context") } db, ok := ctx.Value(common.DbContextKey).(*model.Queries) if !ok { return errors.New("db not found in context") } sfixtures, err := s.Football.So5.So5Fixtures.Get(ctx, football.So5FixturesParams{}) if err != nil { return err } log.Debug().Msgf("fixtures: %v", sfixtures) cnt, err := db.CreateFixtures( ctx, lo.Map(sfixtures, func(fixture football.So5Fixture, index int) model.CreateFixturesParams { return model.CreateFixturesParams{ Slug: fixture.Slug, DisplayName: fixture.DisplayName, State: fixture.AasmState, StartDate: pgtype.Timestamptz{Time: fixture.StartDate, Valid: true}, EndDate: pgtype.Timestamptz{Time: fixture.EndDate, Valid: true}, GameWeek: int32(fixture.GameWeek), } }), ) if err != nil { return err } log.Debug().Msgf("created %d fixtures", cnt) fixtures, err := db.GetAllFixtures(ctx) if err != nil { return err } games, err := sorare_utils.GetGamesFromFixtures( ctx, s, lo.Map(fixtures, func(fixture model.Fixture, index int) string { return fixture.Slug }), ) log.Info().Msgf("found %d games to process", len(games)) teamSlugs := sorare_utils.ExtractTeamSlugsFromGames(games) log.Debug().Msgf("extracted %d unique team slugs from games", len(teamSlugs)) log.Debug().Msgf("getting clubs...") var clubs []football.Club for i, chunk := range lo.Chunk(teamSlugs, 100) { log.Debug().Msgf("\tbatch %d/%d", i+1, len(teamSlugs)/100) t, err := s.Football.Clubs.Get(ctx, graphql.SlugsParams{Slugs: chunk}) if err != nil { return err } clubs = append(clubs, t...) } log.Debug().Msgf("found %d clubs", len(clubs)) var nationalTeams []football.NationalTeam slugsLeft := lo.Without(teamSlugs, lo.Map(clubs, func(club football.Club, index int) string { return club.Slug })...) log.Debug().Msgf("getting national teams...") log.Debug().Msgf("slugs left: %d", len(slugsLeft)) for i, chunk := range lo.Chunk(slugsLeft, 100) { log.Debug().Msgf("\tbatch %d/%d", i+1, len(teamSlugs)/100) t, err := s.Football.NationalTeams.Get(ctx, graphql.SlugsParams{Slugs: chunk}) if err != nil { return err } nationalTeams = append(nationalTeams, t...) } log.Debug().Msgf("found %d national teams", len(nationalTeams)) competitionSlugs := sorare_utils.ExtractCompetitionSlugsFromGamesAndClubs(games, clubs) log.Debug().Msgf("extracted %d unique competition slugs from games and clubs", len(competitionSlugs)) log.Debug().Msgf("getting competitions...") var competitions []football.Competition for _, slug := range competitionSlugs { log.Debug().Msgf("\tcompetition %s", slug) c, err := s.Football.Competition.Get(ctx, graphql.SlugParams{Slug: slug}) if err != nil { return err } competitions = append(competitions, c) } log.Debug().Msgf("found %d competitions", len(competitions)) countrySlugs := sorare_utils.ExtractCountrySlugsFromCompetitionsClubsAndNationalTeams( competitions, clubs, nationalTeams, ) log.Debug().Msgf("extracted %d unique country slugs from competitions, clubs and national teams", len(countrySlugs)) log.Debug().Msgf("getting countries...") var countries []sorare.Country for i, chunk := range lo.Chunk(countrySlugs, 100) { log.Debug().Msgf("\tbatch %d/%d", i+1, len(countrySlugs)/100) c, err := s.Countries.Get(ctx, graphql.SlugsParams{Slugs: chunk}) if err != nil { return err } countries = append(countries, c...) } log.Debug().Msgf("found %d countries", len(countries)) log.Debug().Msg("inserting countries into db...") cnt, err = db.CreateCountries( ctx, lo.Map(countries, func(country sorare.Country, index int) model.CreateCountriesParams { return model.CreateCountriesParams{ Slug: country.Slug, Code: country.Code, DisplayName: country.Name, ThreeLetterCode: country.ThreeLetterCode, FlagFlat64Url: country.FlagFlat64Url, FlagFlat32Url: country.FlagFlat32Url, FlagRound64Url: country.FlagRound64Url, FlagRound32Url: country.FlagRound32Url, } }), ) if err != nil { return err } log.Debug().Msgf("%d countries inserted", cnt) log.Debug().Msg("inserting competitions into db...") _, err = db.BatchInsertCompetitions( ctx, lo.Map(competitions, func(competition football.Competition, index int) model.BatchInsertCompetitionsParams { return model.BatchInsertCompetitionsParams{ Slug: competition.Slug, CompetitionFormat: competition.Format, CompetitionType: competition.Type, DisplayName: competition.DisplayName, PictureUrl: competition.PictureUrl, LogoUrl: competition.LogoUrl, CountrySlug: competition.Country.Slug, } }), ) if err != nil { return err } log.Debug().Msgf("%d competitions inserted", len(competitions)) log.Debug().Msg("inserting teams into db...") cnt, err = db.BatchInsertTeams(ctx, lo.Union( lo.Map(clubs, func(club football.Club, index int) model.BatchInsertTeamsParams { return model.BatchInsertTeamsParams{ Slug: club.Slug, DisplayName: club.Name, CountrySlug: club.Country.Slug, DomesticLeagueSlug: func() *string { if club.DomesticLeague.Slug == "" { return nil } return &club.DomesticLeague.Slug }(), ShortName: club.ShortName, PictureUrl: club.PictureUrl, TeamType: "club", } }), lo.Map(nationalTeams, func(nationalTeam football.NationalTeam, index int) model.BatchInsertTeamsParams { return model.BatchInsertTeamsParams{ Slug: nationalTeam.Slug, DisplayName: nationalTeam.Name, CountrySlug: nationalTeam.Country.Slug, DomesticLeagueSlug: nil, ShortName: nationalTeam.ShortName, PictureUrl: nationalTeam.PictureUrl, TeamType: "national", } }), )) if err != nil { return err } log.Debug().Msgf("%d teams inserted", cnt) log.Debug().Msg("inserting games into db...") cnt, err = db.BatchInsertGames( ctx, lo.Map(games, func(game football.Game, index int) model.BatchInsertGamesParams { return model.BatchInsertGamesParams{ ID: game.Id.Value, Date: pgtype.Timestamptz{Time: game.Date, Valid: true}, CoverageStatus: game.CoverageStatus, LowCoverage: game.LowCoverage, Minutes: int32(game.Minute), PeriodType: game.PeriodType, Scored: game.Scored, Status: game.Status, CompetitionSlug: game.Competition.Slug, FixtureSlug: game.So5Fixture.Slug, AwayTeamSlug: game.AwayTeam.Team.Slug, AwayGoals: int32(game.AwayGoals), AwayExtraTimeScore: int32(game.ExtraTimeScoreAway), AwayPenaltyScore: int32(game.PenaltyScoreAway), HomeTeamSlug: game.HomeTeam.Team.Slug, HomeGoals: int32(game.HomeGoals), HomeExtraTimeScore: int32(game.ExtraTimeScoreHome), HomePenaltyScore: int32(game.PenaltyScoreHome), WinnerTeamSlug: func() *string { if game.Winner.Team.Slug == "" { return nil } return &game.Winner.Team.Slug }(), } }), ) if err != nil { return err } log.Debug().Msgf("%d games inserted", cnt) // log.Debug().Msg("start sequence completed. waiting for shutdown request") // <-ctx.Done() // log.Debug().Msg("shutdown requested") // wg.Wait() // log.Debug().Msg("shutdown complete") return nil }