move db initialization into UpdateService
This commit is contained in:
parent
122e83480e
commit
717ae339dc
@ -2,12 +2,7 @@ package main
|
||||
|
||||
import (
|
||||
"git.lehouerou.net/laurent/sorare"
|
||||
"git.lehouerou.net/laurent/sorare/football"
|
||||
"git.lehouerou.net/laurent/sorare/graphql"
|
||||
"github.com/jackc/pgx/v5/pgtype"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/rs/zerolog/log"
|
||||
"github.com/samber/lo"
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"git.lehouerou.net/laurent/sorarebuddy/cmd/common"
|
||||
@ -45,220 +40,11 @@ func run(cmd *cobra.Command, _ []string) error {
|
||||
return errors.New("db not found in context")
|
||||
}
|
||||
|
||||
sfixtures, err := s.Football.So5.So5Fixtures.Get(ctx, football.So5FixturesParams{})
|
||||
us := sorare_utils.NewUpdateService(s, db)
|
||||
err := us.InitSyncDatabase(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
return errors.Wrap(err, "initializing database")
|
||||
}
|
||||
log.Debug().Msgf("fixtures: %v", sfixtures)
|
||||
|
||||
cnt, err := db.CreateFixtures(
|
||||
ctx,
|
||||
lo.Map(sfixtures, func(fixture football.So5Fixture, index int) model.CreateFixturesParams {
|
||||
return model.CreateFixturesParams{
|
||||
Slug: fixture.Slug,
|
||||
DisplayName: fixture.DisplayName,
|
||||
State: fixture.AasmState,
|
||||
StartDate: pgtype.Timestamptz{Time: fixture.StartDate, Valid: true},
|
||||
EndDate: pgtype.Timestamptz{Time: fixture.EndDate, Valid: true},
|
||||
GameWeek: int32(fixture.GameWeek),
|
||||
}
|
||||
}),
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
log.Debug().Msgf("created %d fixtures", cnt)
|
||||
|
||||
fixtures, err := db.GetAllFixtures(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
games, err := sorare_utils.GetGamesFromFixtures(
|
||||
ctx,
|
||||
s,
|
||||
lo.Map(fixtures, func(fixture model.Fixture, index int) string {
|
||||
return fixture.Slug
|
||||
}),
|
||||
)
|
||||
log.Info().Msgf("found %d games to process", len(games))
|
||||
|
||||
teamSlugs := sorare_utils.ExtractTeamSlugsFromGames(games)
|
||||
log.Debug().Msgf("extracted %d unique team slugs from games", len(teamSlugs))
|
||||
|
||||
log.Debug().Msgf("getting clubs...")
|
||||
var clubs []football.Club
|
||||
for i, chunk := range lo.Chunk(teamSlugs, 100) {
|
||||
log.Debug().Msgf("\tbatch %d/%d", i+1, len(teamSlugs)/100)
|
||||
t, err := s.Football.Clubs.Get(ctx, graphql.SlugsParams{Slugs: chunk})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
clubs = append(clubs, t...)
|
||||
}
|
||||
log.Debug().Msgf("found %d clubs", len(clubs))
|
||||
var nationalTeams []football.NationalTeam
|
||||
slugsLeft := lo.Without(teamSlugs, lo.Map(clubs, func(club football.Club, index int) string {
|
||||
return club.Slug
|
||||
})...)
|
||||
log.Debug().Msgf("getting national teams...")
|
||||
log.Debug().Msgf("slugs left: %d", len(slugsLeft))
|
||||
for i, chunk := range lo.Chunk(slugsLeft, 100) {
|
||||
log.Debug().Msgf("\tbatch %d/%d", i+1, len(teamSlugs)/100)
|
||||
t, err := s.Football.NationalTeams.Get(ctx, graphql.SlugsParams{Slugs: chunk})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
nationalTeams = append(nationalTeams, t...)
|
||||
}
|
||||
log.Debug().Msgf("found %d national teams", len(nationalTeams))
|
||||
|
||||
competitionSlugs := sorare_utils.ExtractCompetitionSlugsFromGamesAndClubs(games, clubs)
|
||||
log.Debug().Msgf("extracted %d unique competition slugs from games and clubs", len(competitionSlugs))
|
||||
log.Debug().Msgf("getting competitions...")
|
||||
var competitions []football.Competition
|
||||
for _, slug := range competitionSlugs {
|
||||
log.Debug().Msgf("\tcompetition %s", slug)
|
||||
c, err := s.Football.Competition.Get(ctx, graphql.SlugParams{Slug: slug})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
competitions = append(competitions, c)
|
||||
}
|
||||
log.Debug().Msgf("found %d competitions", len(competitions))
|
||||
|
||||
countrySlugs := sorare_utils.ExtractCountrySlugsFromCompetitionsClubsAndNationalTeams(
|
||||
competitions,
|
||||
clubs,
|
||||
nationalTeams,
|
||||
)
|
||||
log.Debug().Msgf("extracted %d unique country slugs from competitions, clubs and national teams", len(countrySlugs))
|
||||
log.Debug().Msgf("getting countries...")
|
||||
var countries []sorare.Country
|
||||
for i, chunk := range lo.Chunk(countrySlugs, 100) {
|
||||
log.Debug().Msgf("\tbatch %d/%d", i+1, len(countrySlugs)/100)
|
||||
c, err := s.Countries.Get(ctx, graphql.SlugsParams{Slugs: chunk})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
countries = append(countries, c...)
|
||||
}
|
||||
log.Debug().Msgf("found %d countries", len(countries))
|
||||
|
||||
log.Debug().Msg("inserting countries into db...")
|
||||
cnt, err = db.CreateCountries(
|
||||
ctx,
|
||||
lo.Map(countries, func(country sorare.Country, index int) model.CreateCountriesParams {
|
||||
return model.CreateCountriesParams{
|
||||
Slug: country.Slug,
|
||||
Code: country.Code,
|
||||
DisplayName: country.Name,
|
||||
ThreeLetterCode: country.ThreeLetterCode,
|
||||
FlagFlat64Url: country.FlagFlat64Url,
|
||||
FlagFlat32Url: country.FlagFlat32Url,
|
||||
FlagRound64Url: country.FlagRound64Url,
|
||||
FlagRound32Url: country.FlagRound32Url,
|
||||
}
|
||||
}),
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
log.Debug().Msgf("%d countries inserted", cnt)
|
||||
|
||||
log.Debug().Msg("inserting competitions into db...")
|
||||
|
||||
_, err = db.BatchInsertCompetitions(
|
||||
ctx,
|
||||
lo.Map(competitions, func(competition football.Competition, index int) model.BatchInsertCompetitionsParams {
|
||||
return model.BatchInsertCompetitionsParams{
|
||||
Slug: competition.Slug,
|
||||
CompetitionFormat: competition.Format,
|
||||
CompetitionType: competition.Type,
|
||||
DisplayName: competition.DisplayName,
|
||||
PictureUrl: competition.PictureUrl,
|
||||
LogoUrl: competition.LogoUrl,
|
||||
CountrySlug: competition.Country.Slug,
|
||||
}
|
||||
}),
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
log.Debug().Msgf("%d competitions inserted", len(competitions))
|
||||
|
||||
log.Debug().Msg("inserting teams into db...")
|
||||
cnt, err = db.BatchInsertTeams(ctx, lo.Union(
|
||||
lo.Map(clubs, func(club football.Club, index int) model.BatchInsertTeamsParams {
|
||||
return model.BatchInsertTeamsParams{
|
||||
Slug: club.Slug,
|
||||
DisplayName: club.Name,
|
||||
CountrySlug: club.Country.Slug,
|
||||
DomesticLeagueSlug: func() *string {
|
||||
if club.DomesticLeague.Slug == "" {
|
||||
return nil
|
||||
}
|
||||
return &club.DomesticLeague.Slug
|
||||
}(),
|
||||
ShortName: club.ShortName,
|
||||
PictureUrl: club.PictureUrl,
|
||||
TeamType: "club",
|
||||
}
|
||||
}),
|
||||
lo.Map(nationalTeams, func(nationalTeam football.NationalTeam, index int) model.BatchInsertTeamsParams {
|
||||
return model.BatchInsertTeamsParams{
|
||||
Slug: nationalTeam.Slug,
|
||||
DisplayName: nationalTeam.Name,
|
||||
CountrySlug: nationalTeam.Country.Slug,
|
||||
DomesticLeagueSlug: nil,
|
||||
ShortName: nationalTeam.ShortName,
|
||||
PictureUrl: nationalTeam.PictureUrl,
|
||||
TeamType: "national",
|
||||
}
|
||||
}),
|
||||
))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
log.Debug().Msgf("%d teams inserted", cnt)
|
||||
|
||||
log.Debug().Msg("inserting games into db...")
|
||||
cnt, err = db.BatchInsertGames(
|
||||
ctx,
|
||||
lo.Map(games, func(game football.Game, index int) model.BatchInsertGamesParams {
|
||||
return model.BatchInsertGamesParams{
|
||||
ID: game.Id.Value,
|
||||
Date: pgtype.Timestamptz{Time: game.Date, Valid: true},
|
||||
CoverageStatus: game.CoverageStatus,
|
||||
LowCoverage: game.LowCoverage,
|
||||
Minutes: int32(game.Minute),
|
||||
PeriodType: game.PeriodType,
|
||||
Scored: game.Scored,
|
||||
Status: game.Status,
|
||||
CompetitionSlug: game.Competition.Slug,
|
||||
FixtureSlug: game.So5Fixture.Slug,
|
||||
AwayTeamSlug: game.AwayTeam.Team.Slug,
|
||||
AwayGoals: int32(game.AwayGoals),
|
||||
AwayExtraTimeScore: int32(game.ExtraTimeScoreAway),
|
||||
AwayPenaltyScore: int32(game.PenaltyScoreAway),
|
||||
HomeTeamSlug: game.HomeTeam.Team.Slug,
|
||||
HomeGoals: int32(game.HomeGoals),
|
||||
HomeExtraTimeScore: int32(game.ExtraTimeScoreHome),
|
||||
HomePenaltyScore: int32(game.PenaltyScoreHome),
|
||||
WinnerTeamSlug: func() *string {
|
||||
if game.Winner.Team.Slug == "" {
|
||||
return nil
|
||||
}
|
||||
return &game.Winner.Team.Slug
|
||||
}(),
|
||||
}
|
||||
}),
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
log.Debug().Msgf("%d games inserted", cnt)
|
||||
|
||||
// log.Debug().Msg("start sequence completed. waiting for shutdown request")
|
||||
// <-ctx.Done()
|
||||
|
||||
242
sorare_utils/update_service.go
Normal file
242
sorare_utils/update_service.go
Normal file
@ -0,0 +1,242 @@
|
||||
package sorare_utils
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"git.lehouerou.net/laurent/sorare"
|
||||
"git.lehouerou.net/laurent/sorare/football"
|
||||
"git.lehouerou.net/laurent/sorare/graphql"
|
||||
"github.com/jackc/pgx/v5/pgtype"
|
||||
"github.com/rs/zerolog/log"
|
||||
"github.com/samber/lo"
|
||||
|
||||
"git.lehouerou.net/laurent/sorarebuddy/model"
|
||||
)
|
||||
|
||||
type UpdateService struct {
|
||||
s *sorare.Sorare
|
||||
db *model.Queries
|
||||
}
|
||||
|
||||
func NewUpdateService(s *sorare.Sorare, db *model.Queries) *UpdateService {
|
||||
return &UpdateService{s: s, db: db}
|
||||
}
|
||||
|
||||
func (u *UpdateService) InitSyncDatabase(ctx context.Context) error {
|
||||
|
||||
sfixtures, err := u.s.Football.So5.So5Fixtures.Get(ctx, football.So5FixturesParams{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
log.Debug().Msgf("fixtures: %v", sfixtures)
|
||||
|
||||
cnt, err := u.db.CreateFixtures(
|
||||
ctx,
|
||||
lo.Map(sfixtures, func(fixture football.So5Fixture, index int) model.CreateFixturesParams {
|
||||
return model.CreateFixturesParams{
|
||||
Slug: fixture.Slug,
|
||||
DisplayName: fixture.DisplayName,
|
||||
State: fixture.AasmState,
|
||||
StartDate: pgtype.Timestamptz{Time: fixture.StartDate, Valid: true},
|
||||
EndDate: pgtype.Timestamptz{Time: fixture.EndDate, Valid: true},
|
||||
GameWeek: int32(fixture.GameWeek),
|
||||
}
|
||||
}),
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
log.Debug().Msgf("created %d fixtures", cnt)
|
||||
|
||||
fixtures, err := u.db.GetAllFixtures(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
games, err := GetGamesFromFixtures(
|
||||
ctx,
|
||||
u.s,
|
||||
lo.Map(fixtures, func(fixture model.Fixture, index int) string {
|
||||
return fixture.Slug
|
||||
}),
|
||||
)
|
||||
log.Info().Msgf("found %d games to process", len(games))
|
||||
|
||||
teamSlugs := ExtractTeamSlugsFromGames(games)
|
||||
log.Debug().Msgf("extracted %d unique team slugs from games", len(teamSlugs))
|
||||
|
||||
log.Debug().Msgf("getting clubs...")
|
||||
var clubs []football.Club
|
||||
for i, chunk := range lo.Chunk(teamSlugs, 100) {
|
||||
log.Debug().Msgf("\tbatch %d/%d", i+1, len(teamSlugs)/100)
|
||||
t, err := u.s.Football.Clubs.Get(ctx, graphql.SlugsParams{Slugs: chunk})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
clubs = append(clubs, t...)
|
||||
}
|
||||
log.Debug().Msgf("found %d clubs", len(clubs))
|
||||
var nationalTeams []football.NationalTeam
|
||||
slugsLeft := lo.Without(teamSlugs, lo.Map(clubs, func(club football.Club, index int) string {
|
||||
return club.Slug
|
||||
})...)
|
||||
log.Debug().Msgf("getting national teams...")
|
||||
log.Debug().Msgf("slugs left: %d", len(slugsLeft))
|
||||
for i, chunk := range lo.Chunk(slugsLeft, 100) {
|
||||
log.Debug().Msgf("\tbatch %d/%d", i+1, len(teamSlugs)/100)
|
||||
t, err := u.s.Football.NationalTeams.Get(ctx, graphql.SlugsParams{Slugs: chunk})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
nationalTeams = append(nationalTeams, t...)
|
||||
}
|
||||
log.Debug().Msgf("found %d national teams", len(nationalTeams))
|
||||
|
||||
competitionSlugs := ExtractCompetitionSlugsFromGamesAndClubs(games, clubs)
|
||||
log.Debug().Msgf("extracted %d unique competition slugs from games and clubs", len(competitionSlugs))
|
||||
log.Debug().Msgf("getting competitions...")
|
||||
var competitions []football.Competition
|
||||
for _, slug := range competitionSlugs {
|
||||
log.Debug().Msgf("\tcompetition %s", slug)
|
||||
c, err := u.s.Football.Competition.Get(ctx, graphql.SlugParams{Slug: slug})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
competitions = append(competitions, c)
|
||||
}
|
||||
log.Debug().Msgf("found %d competitions", len(competitions))
|
||||
|
||||
countrySlugs := ExtractCountrySlugsFromCompetitionsClubsAndNationalTeams(
|
||||
competitions,
|
||||
clubs,
|
||||
nationalTeams,
|
||||
)
|
||||
log.Debug().Msgf("extracted %d unique country slugs from competitions, clubs and national teams", len(countrySlugs))
|
||||
log.Debug().Msgf("getting countries...")
|
||||
var countries []sorare.Country
|
||||
for i, chunk := range lo.Chunk(countrySlugs, 100) {
|
||||
log.Debug().Msgf("\tbatch %d/%d", i+1, len(countrySlugs)/100)
|
||||
c, err := u.s.Countries.Get(ctx, graphql.SlugsParams{Slugs: chunk})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
countries = append(countries, c...)
|
||||
}
|
||||
log.Debug().Msgf("found %d countries", len(countries))
|
||||
|
||||
log.Debug().Msg("inserting countries into db...")
|
||||
cnt, err = u.db.CreateCountries(
|
||||
ctx,
|
||||
lo.Map(countries, func(country sorare.Country, index int) model.CreateCountriesParams {
|
||||
return model.CreateCountriesParams{
|
||||
Slug: country.Slug,
|
||||
Code: country.Code,
|
||||
DisplayName: country.Name,
|
||||
ThreeLetterCode: country.ThreeLetterCode,
|
||||
FlagFlat64Url: country.FlagFlat64Url,
|
||||
FlagFlat32Url: country.FlagFlat32Url,
|
||||
FlagRound64Url: country.FlagRound64Url,
|
||||
FlagRound32Url: country.FlagRound32Url,
|
||||
}
|
||||
}),
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
log.Debug().Msgf("%d countries inserted", cnt)
|
||||
|
||||
log.Debug().Msg("inserting competitions into db...")
|
||||
|
||||
_, err = u.db.BatchInsertCompetitions(
|
||||
ctx,
|
||||
lo.Map(competitions, func(competition football.Competition, index int) model.BatchInsertCompetitionsParams {
|
||||
return model.BatchInsertCompetitionsParams{
|
||||
Slug: competition.Slug,
|
||||
CompetitionFormat: competition.Format,
|
||||
CompetitionType: competition.Type,
|
||||
DisplayName: competition.DisplayName,
|
||||
PictureUrl: competition.PictureUrl,
|
||||
LogoUrl: competition.LogoUrl,
|
||||
CountrySlug: competition.Country.Slug,
|
||||
}
|
||||
}),
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
log.Debug().Msgf("%d competitions inserted", len(competitions))
|
||||
|
||||
log.Debug().Msg("inserting teams into db...")
|
||||
cnt, err = u.db.BatchInsertTeams(ctx, lo.Union(
|
||||
lo.Map(clubs, func(club football.Club, index int) model.BatchInsertTeamsParams {
|
||||
return model.BatchInsertTeamsParams{
|
||||
Slug: club.Slug,
|
||||
DisplayName: club.Name,
|
||||
CountrySlug: club.Country.Slug,
|
||||
DomesticLeagueSlug: func() *string {
|
||||
if club.DomesticLeague.Slug == "" {
|
||||
return nil
|
||||
}
|
||||
return &club.DomesticLeague.Slug
|
||||
}(),
|
||||
ShortName: club.ShortName,
|
||||
PictureUrl: club.PictureUrl,
|
||||
TeamType: "club",
|
||||
}
|
||||
}),
|
||||
lo.Map(nationalTeams, func(nationalTeam football.NationalTeam, index int) model.BatchInsertTeamsParams {
|
||||
return model.BatchInsertTeamsParams{
|
||||
Slug: nationalTeam.Slug,
|
||||
DisplayName: nationalTeam.Name,
|
||||
CountrySlug: nationalTeam.Country.Slug,
|
||||
DomesticLeagueSlug: nil,
|
||||
ShortName: nationalTeam.ShortName,
|
||||
PictureUrl: nationalTeam.PictureUrl,
|
||||
TeamType: "national",
|
||||
}
|
||||
}),
|
||||
))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
log.Debug().Msgf("%d teams inserted", cnt)
|
||||
|
||||
log.Debug().Msg("inserting games into db...")
|
||||
cnt, err = u.db.BatchInsertGames(
|
||||
ctx,
|
||||
lo.Map(games, func(game football.Game, index int) model.BatchInsertGamesParams {
|
||||
return model.BatchInsertGamesParams{
|
||||
ID: game.Id.Value,
|
||||
Date: pgtype.Timestamptz{Time: game.Date, Valid: true},
|
||||
CoverageStatus: game.CoverageStatus,
|
||||
LowCoverage: game.LowCoverage,
|
||||
Minutes: int32(game.Minute),
|
||||
PeriodType: game.PeriodType,
|
||||
Scored: game.Scored,
|
||||
Status: game.Status,
|
||||
CompetitionSlug: game.Competition.Slug,
|
||||
FixtureSlug: game.So5Fixture.Slug,
|
||||
AwayTeamSlug: game.AwayTeam.Team.Slug,
|
||||
AwayGoals: int32(game.AwayGoals),
|
||||
AwayExtraTimeScore: int32(game.ExtraTimeScoreAway),
|
||||
AwayPenaltyScore: int32(game.PenaltyScoreAway),
|
||||
HomeTeamSlug: game.HomeTeam.Team.Slug,
|
||||
HomeGoals: int32(game.HomeGoals),
|
||||
HomeExtraTimeScore: int32(game.ExtraTimeScoreHome),
|
||||
HomePenaltyScore: int32(game.PenaltyScoreHome),
|
||||
WinnerTeamSlug: func() *string {
|
||||
if game.Winner.Team.Slug == "" {
|
||||
return nil
|
||||
}
|
||||
return &game.Winner.Team.Slug
|
||||
}(),
|
||||
}
|
||||
}),
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
log.Debug().Msgf("%d games inserted", cnt)
|
||||
return nil
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user