refactor db initialization (wip)
This commit is contained in:
parent
c0fc470d72
commit
122e83480e
@ -12,6 +12,7 @@ import (
|
|||||||
|
|
||||||
"git.lehouerou.net/laurent/sorarebuddy/cmd/common"
|
"git.lehouerou.net/laurent/sorarebuddy/cmd/common"
|
||||||
"git.lehouerou.net/laurent/sorarebuddy/model"
|
"git.lehouerou.net/laurent/sorarebuddy/model"
|
||||||
|
"git.lehouerou.net/laurent/sorarebuddy/sorare_utils"
|
||||||
)
|
)
|
||||||
|
|
||||||
var Cmd = &cobra.Command{
|
var Cmd = &cobra.Command{
|
||||||
@ -73,129 +74,78 @@ func run(cmd *cobra.Command, _ []string) error {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
games, err := sorare_utils.GetGamesFromFixtures(
|
||||||
|
ctx,
|
||||||
|
s,
|
||||||
|
lo.Map(fixtures, func(fixture model.Fixture, index int) string {
|
||||||
|
return fixture.Slug
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
log.Info().Msgf("found %d games to process", len(games))
|
||||||
|
|
||||||
var games []football.Game
|
teamSlugs := sorare_utils.ExtractTeamSlugsFromGames(games)
|
||||||
for _, fixture := range fixtures {
|
log.Debug().Msgf("extracted %d unique team slugs from games", len(teamSlugs))
|
||||||
log.Debug().Msgf("fixture: %v", fixture)
|
|
||||||
g, err := s.Football.So5.FixtureGames(fixture.Slug).Get(ctx, graphql.EmptyParams{})
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
log.Debug().Msgf("games: %d", len(g))
|
|
||||||
games = append(games, g...)
|
|
||||||
}
|
|
||||||
log.Debug().Msgf("games: %d", len(games))
|
|
||||||
|
|
||||||
var teamSlugs []string
|
|
||||||
teamSlugs = lo.Map(games, func(game football.Game, index int) string {
|
|
||||||
return game.AwayTeam.Team.Slug
|
|
||||||
})
|
|
||||||
teamSlugs = append(teamSlugs, lo.Map(games, func(game football.Game, index int) string {
|
|
||||||
return game.HomeTeam.Team.Slug
|
|
||||||
})...)
|
|
||||||
teamSlugs = lo.Filter(teamSlugs, func(slug string, index int) bool {
|
|
||||||
return slug != ""
|
|
||||||
})
|
|
||||||
teamSlugs = lo.Uniq(teamSlugs)
|
|
||||||
|
|
||||||
|
log.Debug().Msgf("getting clubs...")
|
||||||
var clubs []football.Club
|
var clubs []football.Club
|
||||||
for _, chunk := range lo.Chunk(teamSlugs, 100) {
|
for i, chunk := range lo.Chunk(teamSlugs, 100) {
|
||||||
|
log.Debug().Msgf("\tbatch %d/%d", i+1, len(teamSlugs)/100)
|
||||||
t, err := s.Football.Clubs.Get(ctx, graphql.SlugsParams{Slugs: chunk})
|
t, err := s.Football.Clubs.Get(ctx, graphql.SlugsParams{Slugs: chunk})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
clubs = append(clubs, t...)
|
clubs = append(clubs, t...)
|
||||||
}
|
}
|
||||||
|
log.Debug().Msgf("found %d clubs", len(clubs))
|
||||||
var nationalTeams []football.NationalTeam
|
var nationalTeams []football.NationalTeam
|
||||||
for _, chunk := range lo.Chunk(lo.Without(teamSlugs, lo.Map(clubs, func(club football.Club, index int) string {
|
slugsLeft := lo.Without(teamSlugs, lo.Map(clubs, func(club football.Club, index int) string {
|
||||||
return club.Slug
|
return club.Slug
|
||||||
})...), 100) {
|
})...)
|
||||||
|
log.Debug().Msgf("getting national teams...")
|
||||||
|
log.Debug().Msgf("slugs left: %d", len(slugsLeft))
|
||||||
|
for i, chunk := range lo.Chunk(slugsLeft, 100) {
|
||||||
|
log.Debug().Msgf("\tbatch %d/%d", i+1, len(teamSlugs)/100)
|
||||||
t, err := s.Football.NationalTeams.Get(ctx, graphql.SlugsParams{Slugs: chunk})
|
t, err := s.Football.NationalTeams.Get(ctx, graphql.SlugsParams{Slugs: chunk})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
nationalTeams = append(nationalTeams, t...)
|
nationalTeams = append(nationalTeams, t...)
|
||||||
}
|
}
|
||||||
|
log.Debug().Msgf("found %d national teams", len(nationalTeams))
|
||||||
|
|
||||||
var teamsToInsert []model.CreateTeamsParams
|
competitionSlugs := sorare_utils.ExtractCompetitionSlugsFromGamesAndClubs(games, clubs)
|
||||||
teamsToInsert = append(teamsToInsert, lo.Map(clubs, func(club football.Club, index int) model.CreateTeamsParams {
|
log.Debug().Msgf("extracted %d unique competition slugs from games and clubs", len(competitionSlugs))
|
||||||
return model.CreateTeamsParams{
|
log.Debug().Msgf("getting competitions...")
|
||||||
Slug: club.Slug,
|
|
||||||
DisplayName: club.Name,
|
|
||||||
CountrySlug: club.Country.Slug,
|
|
||||||
DomesticLeagueSlug: func() *string {
|
|
||||||
if club.DomesticLeague.Slug == "" {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
return &club.DomesticLeague.Slug
|
|
||||||
}(),
|
|
||||||
ShortName: club.ShortName,
|
|
||||||
PictureUrl: club.PictureUrl,
|
|
||||||
TeamType: "club",
|
|
||||||
}
|
|
||||||
})...)
|
|
||||||
teamsToInsert = append(
|
|
||||||
teamsToInsert,
|
|
||||||
lo.Map(nationalTeams, func(nationalTeam football.NationalTeam, index int) model.CreateTeamsParams {
|
|
||||||
return model.CreateTeamsParams{
|
|
||||||
Slug: nationalTeam.Slug,
|
|
||||||
DisplayName: nationalTeam.Name,
|
|
||||||
CountrySlug: nationalTeam.Country.Slug,
|
|
||||||
DomesticLeagueSlug: nil,
|
|
||||||
ShortName: nationalTeam.ShortName,
|
|
||||||
PictureUrl: nationalTeam.PictureUrl,
|
|
||||||
TeamType: "national",
|
|
||||||
}
|
|
||||||
})...)
|
|
||||||
|
|
||||||
competitionSlugs := lo.Map(games, func(game football.Game, index int) string {
|
|
||||||
return game.Competition.Slug
|
|
||||||
})
|
|
||||||
competitionSlugs = append(
|
|
||||||
competitionSlugs,
|
|
||||||
lo.Map(teamsToInsert, func(team model.CreateTeamsParams, index int) string {
|
|
||||||
if team.DomesticLeagueSlug == nil {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
return *team.DomesticLeagueSlug
|
|
||||||
})...)
|
|
||||||
competitionSlugs = lo.Filter(competitionSlugs, func(slug string, index int) bool {
|
|
||||||
return slug != ""
|
|
||||||
})
|
|
||||||
competitionSlugs = lo.Uniq(competitionSlugs)
|
|
||||||
|
|
||||||
var competitions []football.Competition
|
var competitions []football.Competition
|
||||||
for _, slug := range competitionSlugs {
|
for _, slug := range competitionSlugs {
|
||||||
log.Debug().Msgf("competition: %s", slug)
|
log.Debug().Msgf("\tcompetition %s", slug)
|
||||||
c, err := s.Football.Competition.Get(ctx, graphql.SlugParams{Slug: slug})
|
c, err := s.Football.Competition.Get(ctx, graphql.SlugParams{Slug: slug})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
competitions = append(competitions, c)
|
competitions = append(competitions, c)
|
||||||
}
|
}
|
||||||
|
log.Debug().Msgf("found %d competitions", len(competitions))
|
||||||
|
|
||||||
var countrySlugs []string
|
countrySlugs := sorare_utils.ExtractCountrySlugsFromCompetitionsClubsAndNationalTeams(
|
||||||
countrySlugs = append(countrySlugs, lo.Map(teamsToInsert, func(team model.CreateTeamsParams, index int) string {
|
competitions,
|
||||||
return team.CountrySlug
|
clubs,
|
||||||
})...)
|
nationalTeams,
|
||||||
countrySlugs = append(countrySlugs, lo.Map(competitions, func(competition football.Competition, index int) string {
|
)
|
||||||
return competition.Country.Slug
|
log.Debug().Msgf("extracted %d unique country slugs from competitions, clubs and national teams", len(countrySlugs))
|
||||||
})...)
|
log.Debug().Msgf("getting countries...")
|
||||||
countrySlugs = lo.Filter(countrySlugs, func(slug string, index int) bool {
|
|
||||||
return slug != ""
|
|
||||||
})
|
|
||||||
countrySlugs = lo.Uniq(countrySlugs)
|
|
||||||
|
|
||||||
var countries []sorare.Country
|
var countries []sorare.Country
|
||||||
for _, chunk := range lo.Chunk(countrySlugs, 100) {
|
for i, chunk := range lo.Chunk(countrySlugs, 100) {
|
||||||
|
log.Debug().Msgf("\tbatch %d/%d", i+1, len(countrySlugs)/100)
|
||||||
c, err := s.Countries.Get(ctx, graphql.SlugsParams{Slugs: chunk})
|
c, err := s.Countries.Get(ctx, graphql.SlugsParams{Slugs: chunk})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
countries = append(countries, c...)
|
countries = append(countries, c...)
|
||||||
}
|
}
|
||||||
|
log.Debug().Msgf("found %d countries", len(countries))
|
||||||
|
|
||||||
|
log.Debug().Msg("inserting countries into db...")
|
||||||
cnt, err = db.CreateCountries(
|
cnt, err = db.CreateCountries(
|
||||||
ctx,
|
ctx,
|
||||||
lo.Map(countries, func(country sorare.Country, index int) model.CreateCountriesParams {
|
lo.Map(countries, func(country sorare.Country, index int) model.CreateCountriesParams {
|
||||||
@ -214,62 +164,101 @@ func run(cmd *cobra.Command, _ []string) error {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
log.Debug().Msgf("created %d countries", cnt)
|
log.Debug().Msgf("%d countries inserted", cnt)
|
||||||
|
|
||||||
log.Debug().Msgf("competitions: %d", len(competitions))
|
log.Debug().Msg("inserting competitions into db...")
|
||||||
for _, competition := range competitions {
|
|
||||||
|
|
||||||
err := db.CreateOrUpdateCompetition(ctx, model.CreateOrUpdateCompetitionParams{
|
_, err = db.BatchInsertCompetitions(
|
||||||
Slug: competition.Slug,
|
ctx,
|
||||||
CompetitionFormat: competition.Format,
|
lo.Map(competitions, func(competition football.Competition, index int) model.BatchInsertCompetitionsParams {
|
||||||
CompetitionType: competition.Type,
|
return model.BatchInsertCompetitionsParams{
|
||||||
DisplayName: competition.DisplayName,
|
Slug: competition.Slug,
|
||||||
PictureUrl: competition.PictureUrl,
|
CompetitionFormat: competition.Format,
|
||||||
LogoUrl: competition.LogoUrl,
|
CompetitionType: competition.Type,
|
||||||
CountrySlug: competition.Country.Slug,
|
DisplayName: competition.DisplayName,
|
||||||
})
|
PictureUrl: competition.PictureUrl,
|
||||||
if err != nil {
|
LogoUrl: competition.LogoUrl,
|
||||||
return err
|
CountrySlug: competition.Country.Slug,
|
||||||
}
|
}
|
||||||
}
|
}),
|
||||||
|
)
|
||||||
cnt, err = db.CreateTeams(ctx, teamsToInsert)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
log.Debug().Msgf("created %d teams", cnt)
|
|
||||||
|
|
||||||
cnt, err = db.CreateGames(ctx, lo.Map(games, func(game football.Game, index int) model.CreateGamesParams {
|
|
||||||
return model.CreateGamesParams{
|
|
||||||
CoverageStatus: game.CoverageStatus,
|
|
||||||
LowCoverage: game.LowCoverage,
|
|
||||||
Minutes: int32(game.Minute),
|
|
||||||
PeriodType: game.PeriodType,
|
|
||||||
Scored: game.Scored,
|
|
||||||
Status: game.Status,
|
|
||||||
CompetitionSlug: game.Competition.Slug,
|
|
||||||
FixtureSlug: game.So5Fixture.Slug,
|
|
||||||
AwayTeamSlug: game.AwayTeam.Team.Slug,
|
|
||||||
AwayGoals: int32(game.AwayGoals),
|
|
||||||
AwayExtraTimeScore: int32(game.ExtraTimeScoreAway),
|
|
||||||
AwayPenaltyScore: int32(game.PenaltyScoreAway),
|
|
||||||
HomeTeamSlug: game.HomeTeam.Team.Slug,
|
|
||||||
HomeGoals: int32(game.HomeGoals),
|
|
||||||
HomeExtraTimeScore: int32(game.ExtraTimeScoreHome),
|
|
||||||
HomePenaltyScore: int32(game.PenaltyScoreHome),
|
|
||||||
WinnerTeamSlug: func() *string {
|
|
||||||
if game.Winner.Team.Slug == "" {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
return &game.Winner.Team.Slug
|
|
||||||
}(),
|
|
||||||
}
|
|
||||||
}))
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Debug().Msgf("created %d games", cnt)
|
log.Debug().Msgf("%d competitions inserted", len(competitions))
|
||||||
|
|
||||||
|
log.Debug().Msg("inserting teams into db...")
|
||||||
|
cnt, err = db.BatchInsertTeams(ctx, lo.Union(
|
||||||
|
lo.Map(clubs, func(club football.Club, index int) model.BatchInsertTeamsParams {
|
||||||
|
return model.BatchInsertTeamsParams{
|
||||||
|
Slug: club.Slug,
|
||||||
|
DisplayName: club.Name,
|
||||||
|
CountrySlug: club.Country.Slug,
|
||||||
|
DomesticLeagueSlug: func() *string {
|
||||||
|
if club.DomesticLeague.Slug == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return &club.DomesticLeague.Slug
|
||||||
|
}(),
|
||||||
|
ShortName: club.ShortName,
|
||||||
|
PictureUrl: club.PictureUrl,
|
||||||
|
TeamType: "club",
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
lo.Map(nationalTeams, func(nationalTeam football.NationalTeam, index int) model.BatchInsertTeamsParams {
|
||||||
|
return model.BatchInsertTeamsParams{
|
||||||
|
Slug: nationalTeam.Slug,
|
||||||
|
DisplayName: nationalTeam.Name,
|
||||||
|
CountrySlug: nationalTeam.Country.Slug,
|
||||||
|
DomesticLeagueSlug: nil,
|
||||||
|
ShortName: nationalTeam.ShortName,
|
||||||
|
PictureUrl: nationalTeam.PictureUrl,
|
||||||
|
TeamType: "national",
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
log.Debug().Msgf("%d teams inserted", cnt)
|
||||||
|
|
||||||
|
log.Debug().Msg("inserting games into db...")
|
||||||
|
cnt, err = db.BatchInsertGames(
|
||||||
|
ctx,
|
||||||
|
lo.Map(games, func(game football.Game, index int) model.BatchInsertGamesParams {
|
||||||
|
return model.BatchInsertGamesParams{
|
||||||
|
ID: game.Id.Value,
|
||||||
|
Date: pgtype.Timestamptz{Time: game.Date, Valid: true},
|
||||||
|
CoverageStatus: game.CoverageStatus,
|
||||||
|
LowCoverage: game.LowCoverage,
|
||||||
|
Minutes: int32(game.Minute),
|
||||||
|
PeriodType: game.PeriodType,
|
||||||
|
Scored: game.Scored,
|
||||||
|
Status: game.Status,
|
||||||
|
CompetitionSlug: game.Competition.Slug,
|
||||||
|
FixtureSlug: game.So5Fixture.Slug,
|
||||||
|
AwayTeamSlug: game.AwayTeam.Team.Slug,
|
||||||
|
AwayGoals: int32(game.AwayGoals),
|
||||||
|
AwayExtraTimeScore: int32(game.ExtraTimeScoreAway),
|
||||||
|
AwayPenaltyScore: int32(game.PenaltyScoreAway),
|
||||||
|
HomeTeamSlug: game.HomeTeam.Team.Slug,
|
||||||
|
HomeGoals: int32(game.HomeGoals),
|
||||||
|
HomeExtraTimeScore: int32(game.ExtraTimeScoreHome),
|
||||||
|
HomePenaltyScore: int32(game.PenaltyScoreHome),
|
||||||
|
WinnerTeamSlug: func() *string {
|
||||||
|
if game.Winner.Team.Slug == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return &game.Winner.Team.Slug
|
||||||
|
}(),
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
log.Debug().Msgf("%d games inserted", cnt)
|
||||||
|
|
||||||
// log.Debug().Msg("start sequence completed. waiting for shutdown request")
|
// log.Debug().Msg("start sequence completed. waiting for shutdown request")
|
||||||
// <-ctx.Done()
|
// <-ctx.Done()
|
||||||
|
@ -9,6 +9,16 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
type BatchInsertCompetitionsParams struct {
|
||||||
|
Slug string
|
||||||
|
DisplayName string
|
||||||
|
CountrySlug string
|
||||||
|
CompetitionFormat string
|
||||||
|
CompetitionType string
|
||||||
|
PictureUrl string
|
||||||
|
LogoUrl string
|
||||||
|
}
|
||||||
|
|
||||||
const createOrUpdateCompetition = `-- name: CreateOrUpdateCompetition :exec
|
const createOrUpdateCompetition = `-- name: CreateOrUpdateCompetition :exec
|
||||||
INSERT INTO competitions (
|
INSERT INTO competitions (
|
||||||
slug,
|
slug,
|
||||||
@ -40,7 +50,6 @@ type CreateOrUpdateCompetitionParams struct {
|
|||||||
LogoUrl string
|
LogoUrl string
|
||||||
}
|
}
|
||||||
|
|
||||||
// Active: 1709890109198@@192.168.1.250@5436@sorare
|
|
||||||
func (q *Queries) CreateOrUpdateCompetition(ctx context.Context, arg CreateOrUpdateCompetitionParams) error {
|
func (q *Queries) CreateOrUpdateCompetition(ctx context.Context, arg CreateOrUpdateCompetitionParams) error {
|
||||||
_, err := q.db.Exec(ctx, createOrUpdateCompetition,
|
_, err := q.db.Exec(ctx, createOrUpdateCompetition,
|
||||||
arg.Slug,
|
arg.Slug,
|
||||||
|
@ -9,6 +9,132 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// iteratorForBatchInsertCompetitions implements pgx.CopyFromSource.
|
||||||
|
type iteratorForBatchInsertCompetitions struct {
|
||||||
|
rows []BatchInsertCompetitionsParams
|
||||||
|
skippedFirstNextCall bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *iteratorForBatchInsertCompetitions) Next() bool {
|
||||||
|
if len(r.rows) == 0 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if !r.skippedFirstNextCall {
|
||||||
|
r.skippedFirstNextCall = true
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
r.rows = r.rows[1:]
|
||||||
|
return len(r.rows) > 0
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r iteratorForBatchInsertCompetitions) Values() ([]interface{}, error) {
|
||||||
|
return []interface{}{
|
||||||
|
r.rows[0].Slug,
|
||||||
|
r.rows[0].DisplayName,
|
||||||
|
r.rows[0].CountrySlug,
|
||||||
|
r.rows[0].CompetitionFormat,
|
||||||
|
r.rows[0].CompetitionType,
|
||||||
|
r.rows[0].PictureUrl,
|
||||||
|
r.rows[0].LogoUrl,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r iteratorForBatchInsertCompetitions) Err() error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (q *Queries) BatchInsertCompetitions(ctx context.Context, arg []BatchInsertCompetitionsParams) (int64, error) {
|
||||||
|
return q.db.CopyFrom(ctx, []string{"competitions"}, []string{"slug", "display_name", "country_slug", "competition_format", "competition_type", "picture_url", "logo_url"}, &iteratorForBatchInsertCompetitions{rows: arg})
|
||||||
|
}
|
||||||
|
|
||||||
|
// iteratorForBatchInsertGames implements pgx.CopyFromSource.
|
||||||
|
type iteratorForBatchInsertGames struct {
|
||||||
|
rows []BatchInsertGamesParams
|
||||||
|
skippedFirstNextCall bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *iteratorForBatchInsertGames) Next() bool {
|
||||||
|
if len(r.rows) == 0 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if !r.skippedFirstNextCall {
|
||||||
|
r.skippedFirstNextCall = true
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
r.rows = r.rows[1:]
|
||||||
|
return len(r.rows) > 0
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r iteratorForBatchInsertGames) Values() ([]interface{}, error) {
|
||||||
|
return []interface{}{
|
||||||
|
r.rows[0].ID,
|
||||||
|
r.rows[0].Date,
|
||||||
|
r.rows[0].CoverageStatus,
|
||||||
|
r.rows[0].LowCoverage,
|
||||||
|
r.rows[0].Minutes,
|
||||||
|
r.rows[0].PeriodType,
|
||||||
|
r.rows[0].Scored,
|
||||||
|
r.rows[0].Status,
|
||||||
|
r.rows[0].CompetitionSlug,
|
||||||
|
r.rows[0].FixtureSlug,
|
||||||
|
r.rows[0].AwayTeamSlug,
|
||||||
|
r.rows[0].AwayGoals,
|
||||||
|
r.rows[0].AwayExtraTimeScore,
|
||||||
|
r.rows[0].AwayPenaltyScore,
|
||||||
|
r.rows[0].HomeTeamSlug,
|
||||||
|
r.rows[0].HomeGoals,
|
||||||
|
r.rows[0].HomeExtraTimeScore,
|
||||||
|
r.rows[0].HomePenaltyScore,
|
||||||
|
r.rows[0].WinnerTeamSlug,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r iteratorForBatchInsertGames) Err() error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (q *Queries) BatchInsertGames(ctx context.Context, arg []BatchInsertGamesParams) (int64, error) {
|
||||||
|
return q.db.CopyFrom(ctx, []string{"games"}, []string{"id", "date", "coverage_status", "low_coverage", "minutes", "period_type", "scored", "status", "competition_slug", "fixture_slug", "away_team_slug", "away_goals", "away_extra_time_score", "away_penalty_score", "home_team_slug", "home_goals", "home_extra_time_score", "home_penalty_score", "winner_team_slug"}, &iteratorForBatchInsertGames{rows: arg})
|
||||||
|
}
|
||||||
|
|
||||||
|
// iteratorForBatchInsertTeams implements pgx.CopyFromSource.
|
||||||
|
type iteratorForBatchInsertTeams struct {
|
||||||
|
rows []BatchInsertTeamsParams
|
||||||
|
skippedFirstNextCall bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *iteratorForBatchInsertTeams) Next() bool {
|
||||||
|
if len(r.rows) == 0 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if !r.skippedFirstNextCall {
|
||||||
|
r.skippedFirstNextCall = true
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
r.rows = r.rows[1:]
|
||||||
|
return len(r.rows) > 0
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r iteratorForBatchInsertTeams) Values() ([]interface{}, error) {
|
||||||
|
return []interface{}{
|
||||||
|
r.rows[0].Slug,
|
||||||
|
r.rows[0].DisplayName,
|
||||||
|
r.rows[0].CountrySlug,
|
||||||
|
r.rows[0].DomesticLeagueSlug,
|
||||||
|
r.rows[0].ShortName,
|
||||||
|
r.rows[0].PictureUrl,
|
||||||
|
r.rows[0].TeamType,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r iteratorForBatchInsertTeams) Err() error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (q *Queries) BatchInsertTeams(ctx context.Context, arg []BatchInsertTeamsParams) (int64, error) {
|
||||||
|
return q.db.CopyFrom(ctx, []string{"teams"}, []string{"slug", "display_name", "country_slug", "domestic_league_slug", "short_name", "picture_url", "team_type"}, &iteratorForBatchInsertTeams{rows: arg})
|
||||||
|
}
|
||||||
|
|
||||||
// iteratorForCreateCountries implements pgx.CopyFromSource.
|
// iteratorForCreateCountries implements pgx.CopyFromSource.
|
||||||
type iteratorForCreateCountries struct {
|
type iteratorForCreateCountries struct {
|
||||||
rows []CreateCountriesParams
|
rows []CreateCountriesParams
|
||||||
@ -85,91 +211,3 @@ func (r iteratorForCreateFixtures) Err() error {
|
|||||||
func (q *Queries) CreateFixtures(ctx context.Context, arg []CreateFixturesParams) (int64, error) {
|
func (q *Queries) CreateFixtures(ctx context.Context, arg []CreateFixturesParams) (int64, error) {
|
||||||
return q.db.CopyFrom(ctx, []string{"fixtures"}, []string{"slug", "display_name", "state", "start_date", "end_date", "game_week"}, &iteratorForCreateFixtures{rows: arg})
|
return q.db.CopyFrom(ctx, []string{"fixtures"}, []string{"slug", "display_name", "state", "start_date", "end_date", "game_week"}, &iteratorForCreateFixtures{rows: arg})
|
||||||
}
|
}
|
||||||
|
|
||||||
// iteratorForCreateGames implements pgx.CopyFromSource.
|
|
||||||
type iteratorForCreateGames struct {
|
|
||||||
rows []CreateGamesParams
|
|
||||||
skippedFirstNextCall bool
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *iteratorForCreateGames) Next() bool {
|
|
||||||
if len(r.rows) == 0 {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
if !r.skippedFirstNextCall {
|
|
||||||
r.skippedFirstNextCall = true
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
r.rows = r.rows[1:]
|
|
||||||
return len(r.rows) > 0
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r iteratorForCreateGames) Values() ([]interface{}, error) {
|
|
||||||
return []interface{}{
|
|
||||||
r.rows[0].ID,
|
|
||||||
r.rows[0].Date,
|
|
||||||
r.rows[0].CoverageStatus,
|
|
||||||
r.rows[0].LowCoverage,
|
|
||||||
r.rows[0].Minutes,
|
|
||||||
r.rows[0].PeriodType,
|
|
||||||
r.rows[0].Scored,
|
|
||||||
r.rows[0].Status,
|
|
||||||
r.rows[0].CompetitionSlug,
|
|
||||||
r.rows[0].FixtureSlug,
|
|
||||||
r.rows[0].AwayTeamSlug,
|
|
||||||
r.rows[0].AwayGoals,
|
|
||||||
r.rows[0].AwayExtraTimeScore,
|
|
||||||
r.rows[0].AwayPenaltyScore,
|
|
||||||
r.rows[0].HomeTeamSlug,
|
|
||||||
r.rows[0].HomeGoals,
|
|
||||||
r.rows[0].HomeExtraTimeScore,
|
|
||||||
r.rows[0].HomePenaltyScore,
|
|
||||||
r.rows[0].WinnerTeamSlug,
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r iteratorForCreateGames) Err() error {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (q *Queries) CreateGames(ctx context.Context, arg []CreateGamesParams) (int64, error) {
|
|
||||||
return q.db.CopyFrom(ctx, []string{"games"}, []string{"id", "date", "coverage_status", "low_coverage", "minutes", "period_type", "scored", "status", "competition_slug", "fixture_slug", "away_team_slug", "away_goals", "away_extra_time_score", "away_penalty_score", "home_team_slug", "home_goals", "home_extra_time_score", "home_penalty_score", "winner_team_slug"}, &iteratorForCreateGames{rows: arg})
|
|
||||||
}
|
|
||||||
|
|
||||||
// iteratorForCreateTeams implements pgx.CopyFromSource.
|
|
||||||
type iteratorForCreateTeams struct {
|
|
||||||
rows []CreateTeamsParams
|
|
||||||
skippedFirstNextCall bool
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *iteratorForCreateTeams) Next() bool {
|
|
||||||
if len(r.rows) == 0 {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
if !r.skippedFirstNextCall {
|
|
||||||
r.skippedFirstNextCall = true
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
r.rows = r.rows[1:]
|
|
||||||
return len(r.rows) > 0
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r iteratorForCreateTeams) Values() ([]interface{}, error) {
|
|
||||||
return []interface{}{
|
|
||||||
r.rows[0].Slug,
|
|
||||||
r.rows[0].DisplayName,
|
|
||||||
r.rows[0].CountrySlug,
|
|
||||||
r.rows[0].DomesticLeagueSlug,
|
|
||||||
r.rows[0].ShortName,
|
|
||||||
r.rows[0].PictureUrl,
|
|
||||||
r.rows[0].TeamType,
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r iteratorForCreateTeams) Err() error {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (q *Queries) CreateTeams(ctx context.Context, arg []CreateTeamsParams) (int64, error) {
|
|
||||||
return q.db.CopyFrom(ctx, []string{"teams"}, []string{"slug", "display_name", "country_slug", "domestic_league_slug", "short_name", "picture_url", "team_type"}, &iteratorForCreateTeams{rows: arg})
|
|
||||||
}
|
|
||||||
|
@ -40,7 +40,7 @@ const createOrUpdateCountry = `-- name: CreateOrUpdateCountry :exec
|
|||||||
flag_flat_64_url = EXCLUDED.flag_flat_64_url,
|
flag_flat_64_url = EXCLUDED.flag_flat_64_url,
|
||||||
flag_flat_32_url = EXCLUDED.flag_flat_32_url,
|
flag_flat_32_url = EXCLUDED.flag_flat_32_url,
|
||||||
flag_round_64_url = EXCLUDED.flag_round_64_url,
|
flag_round_64_url = EXCLUDED.flag_round_64_url,
|
||||||
flag_round_32_url = EXCLUDED.flag_round_32_url
|
flag_round_32_url = EXCLUDED.flag_round_32_url
|
||||||
`
|
`
|
||||||
|
|
||||||
type CreateOrUpdateCountryParams struct {
|
type CreateOrUpdateCountryParams struct {
|
||||||
|
@ -9,7 +9,7 @@ import (
|
|||||||
"github.com/jackc/pgx/v5/pgtype"
|
"github.com/jackc/pgx/v5/pgtype"
|
||||||
)
|
)
|
||||||
|
|
||||||
type CreateGamesParams struct {
|
type BatchInsertGamesParams struct {
|
||||||
ID string
|
ID string
|
||||||
Date pgtype.Timestamptz
|
Date pgtype.Timestamptz
|
||||||
CoverageStatus string
|
CoverageStatus string
|
||||||
|
@ -1,3 +1,7 @@
|
|||||||
|
-- name: BatchInsertCompetitions :copyfrom
|
||||||
|
INSERT INTO competitions (slug, display_name, country_slug, competition_format, competition_type, picture_url, logo_url)
|
||||||
|
VALUES ($1, $2, $3, $4, $5, $6, $7);
|
||||||
|
|
||||||
-- name: CreateOrUpdateCompetition :exec
|
-- name: CreateOrUpdateCompetition :exec
|
||||||
INSERT INTO competitions (
|
INSERT INTO competitions (
|
||||||
slug,
|
slug,
|
||||||
|
@ -1,4 +1,4 @@
|
|||||||
-- name: CreateGames :copyfrom
|
-- name: BatchInsertGames :copyfrom
|
||||||
INSERT INTO games (id, date, coverage_status, low_coverage, minutes, period_type, scored, status, competition_slug, fixture_slug, away_team_slug, away_goals, away_extra_time_score, away_penalty_score, home_team_slug, home_goals, home_extra_time_score, home_penalty_score, winner_team_slug)
|
INSERT INTO games (id, date, coverage_status, low_coverage, minutes, period_type, scored, status, competition_slug, fixture_slug, away_team_slug, away_goals, away_extra_time_score, away_penalty_score, home_team_slug, home_goals, home_extra_time_score, home_penalty_score, winner_team_slug)
|
||||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19);
|
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19);
|
||||||
|
|
||||||
|
@ -1,3 +1,3 @@
|
|||||||
-- name: CreateTeams :copyfrom
|
-- name: BatchInsertTeams :copyfrom
|
||||||
INSERT INTO teams (slug, display_name, country_slug, domestic_league_slug, short_name, picture_url, team_type)
|
INSERT INTO teams (slug, display_name, country_slug, domestic_league_slug, short_name, picture_url, team_type)
|
||||||
VALUES ($1, $2, $3, $4, $5, $6, $7);
|
VALUES ($1, $2, $3, $4, $5, $6, $7);
|
||||||
|
@ -1,2 +0,0 @@
|
|||||||
select * from competitions
|
|
||||||
join games on games.id = competitions.game_id
|
|
@ -7,7 +7,7 @@ package model
|
|||||||
|
|
||||||
import ()
|
import ()
|
||||||
|
|
||||||
type CreateTeamsParams struct {
|
type BatchInsertTeamsParams struct {
|
||||||
Slug string
|
Slug string
|
||||||
DisplayName string
|
DisplayName string
|
||||||
CountrySlug string
|
CountrySlug string
|
||||||
|
110
sorare_utils/fixture.go
Normal file
110
sorare_utils/fixture.go
Normal file
@ -0,0 +1,110 @@
|
|||||||
|
package sorare_utils
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
|
||||||
|
"git.lehouerou.net/laurent/sorare"
|
||||||
|
"git.lehouerou.net/laurent/sorare/football"
|
||||||
|
"git.lehouerou.net/laurent/sorare/graphql"
|
||||||
|
"github.com/pkg/errors"
|
||||||
|
"github.com/rs/zerolog/log"
|
||||||
|
"github.com/samber/lo"
|
||||||
|
)
|
||||||
|
|
||||||
|
func GetGamesFromFixtures(
|
||||||
|
ctx context.Context,
|
||||||
|
s *sorare.Sorare,
|
||||||
|
fixtureSlugs []string,
|
||||||
|
) ([]football.Game, error) {
|
||||||
|
var games []football.Game
|
||||||
|
for _, slug := range fixtureSlugs {
|
||||||
|
log.Debug().Msgf("getting games for fixture %s...", slug)
|
||||||
|
g, err := s.Football.So5.FixtureGames(slug).Get(ctx, graphql.EmptyParams{})
|
||||||
|
if err != nil {
|
||||||
|
return nil, errors.Wrapf(err, "getting games for fixture %s", slug)
|
||||||
|
}
|
||||||
|
log.Debug().Msgf("found %d games", len(g))
|
||||||
|
games = append(games, g...)
|
||||||
|
}
|
||||||
|
log.Debug().Msgf("found total of %d games before filtering", len(games))
|
||||||
|
games = lo.Filter(games, func(game football.Game, index int) bool {
|
||||||
|
return game.Id.Value != ""
|
||||||
|
})
|
||||||
|
log.Debug().Msgf("found total of %d games after filtering empty ids", len(games))
|
||||||
|
games = lo.UniqBy(games, func(game football.Game) string {
|
||||||
|
return game.Id.Value
|
||||||
|
})
|
||||||
|
log.Debug().Msgf("found total of %d games after filtering duplicate ids", len(games))
|
||||||
|
return games, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func ExtractTeamSlugsFromGames(games []football.Game) []string {
|
||||||
|
var res []string
|
||||||
|
res = lo.Map(games, func(game football.Game, index int) string {
|
||||||
|
return game.AwayTeam.Team.Slug
|
||||||
|
})
|
||||||
|
res = append(res, lo.Map(games, func(game football.Game, index int) string {
|
||||||
|
return game.HomeTeam.Team.Slug
|
||||||
|
})...)
|
||||||
|
res = lo.Filter(res, func(slug string, index int) bool {
|
||||||
|
return slug != ""
|
||||||
|
})
|
||||||
|
return lo.Uniq(res)
|
||||||
|
}
|
||||||
|
|
||||||
|
func ExtractCountrySlugsFromCompetitions(competitions []football.Competition) []string {
|
||||||
|
return lo.Uniq(lo.Filter(lo.Map(competitions, func(competition football.Competition, index int) string {
|
||||||
|
return competition.Country.Slug
|
||||||
|
}), func(slug string, index int) bool {
|
||||||
|
return slug != ""
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
func ExtractCountrySlugsFromTeams(teams []football.Team) []string {
|
||||||
|
return lo.Uniq(lo.Filter(lo.Map(teams, func(team football.Team, index int) string {
|
||||||
|
return team.Country.Slug
|
||||||
|
}), func(slug string, index int) bool {
|
||||||
|
return slug != ""
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
func ExtractCountrySlugsFromCompetitionsClubsAndNationalTeams(
|
||||||
|
competitions []football.Competition,
|
||||||
|
clubs []football.Club,
|
||||||
|
nationalTeams []football.NationalTeam,
|
||||||
|
) []string {
|
||||||
|
return lo.Uniq(lo.Union(
|
||||||
|
ExtractCountrySlugsFromCompetitions(competitions),
|
||||||
|
ExtractCountrySlugsFromTeams(lo.Map(clubs, func(club football.Club, index int) football.Team {
|
||||||
|
return club.Team
|
||||||
|
})),
|
||||||
|
ExtractCountrySlugsFromTeams(
|
||||||
|
lo.Map(nationalTeams, func(nationalTeam football.NationalTeam, index int) football.Team {
|
||||||
|
return nationalTeam.Team
|
||||||
|
}),
|
||||||
|
),
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
|
func ExtractCompetitionSlugsFromGames(games []football.Game) []string {
|
||||||
|
return lo.Uniq(lo.Filter(lo.Map(games, func(game football.Game, index int) string {
|
||||||
|
return game.Competition.Slug
|
||||||
|
}), func(slug string, index int) bool {
|
||||||
|
return slug != ""
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
func ExtractCompetitionSlugsFromClubs(clubs []football.Club) []string {
|
||||||
|
return lo.Uniq(lo.Filter(lo.Map(clubs, func(club football.Club, index int) string {
|
||||||
|
return club.DomesticLeague.Slug
|
||||||
|
}), func(slug string, index int) bool {
|
||||||
|
return slug != ""
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
func ExtractCompetitionSlugsFromGamesAndClubs(games []football.Game, clubs []football.Club) []string {
|
||||||
|
return lo.Uniq(lo.Union(
|
||||||
|
ExtractCompetitionSlugsFromGames(games),
|
||||||
|
ExtractCompetitionSlugsFromClubs(clubs),
|
||||||
|
))
|
||||||
|
}
|
Loading…
Reference in New Issue
Block a user