From 412f5f75ca5f9da0b95c9b7f9f7ddf49b0093b22 Mon Sep 17 00:00:00 2001 From: Jannik Clausen <12862103+masseelch@users.noreply.github.com> Date: Wed, 6 Jul 2022 10:34:39 +0200 Subject: [PATCH] add docs about auto-increment counter "bug" in MySQL versions < 8.0 and how to handle it (#2722) --- dialect/sql/schema/atlas.go | 95 ++++++++++++++++++++++++++++------ dialect/sql/schema/migrate.go | 2 +- dialect/sql/schema/mysql.go | 2 +- doc/md/versioned-migrations.md | 56 ++++++++++++++++++++ 4 files changed, 136 insertions(+), 19 deletions(-) diff --git a/dialect/sql/schema/atlas.go b/dialect/sql/schema/atlas.go index 4599ba9a1..f1134e502 100644 --- a/dialect/sql/schema/atlas.go +++ b/dialect/sql/schema/atlas.go @@ -227,6 +227,55 @@ func (a *Atlas) NamedDiff(ctx context.Context, name string, tables ...*Table) er return migrate.NewPlanner(nil, a.dir, opts...).WritePlan(plan) } +// VerifyTableRange ensures, that the defined autoincrement starting value is set for each table as defined by the +// TypTable. This is necessary for MySQL versions < 8.0. In those versions the defined starting value for AUTOINCREMENT +// columns was stored in memory, and when a server restarts happens and there are no rows yet in a table, the defined +// starting value is lost, which will result in incorrect behavior when working with global unique ids. Calling this +// method on service start ensures the information are correct and are set again, if they aren't. For MySQL versions > 8 +// calling this method is only required once after the upgrade. +func (a *Atlas) VerifyTableRange(ctx context.Context, tables []*Table) error { + if a.driver != nil { + var err error + a.sqlDialect, err = a.entDialect(a.driver) + if err != nil { + return err + } + } else { + c, err := sqlclient.OpenURL(ctx, a.url) + if err != nil { + return err + } + defer c.Close() + a.sqlDialect, err = a.entDialect(entsql.OpenDB(a.dialect, c.DB)) + if err != nil { + return err + } + } + defer func() { + a.sqlDialect = nil + }() + vr, ok := a.sqlDialect.(verifyRanger) + if !ok { + return nil + } + types, err := a.loadTypes(ctx, a.sqlDialect) + if err != nil { + // In most cases this means the table does not exist, which in turn + // indicates the user does not use global unique ids. + return err + } + for _, t := range tables { + id := indexOf(types, t.Name) + if id == -1 { + continue + } + if err := vr.verifyRange(ctx, a.sqlDialect, t, int64(id<<32)); err != nil { + return err + } + } + return nil +} + type ( // Differ is the interface that wraps the Diff method. Differ interface { @@ -661,25 +710,11 @@ func (a *Atlas) plan(ctx context.Context, conn dialect.ExecQuerier, name string, } var types []string if a.universalID { - // Fetch pre-existing type allocations. - exists, err := a.sqlDialect.tableExist(ctx, conn, TypeTable) - if err != nil { + types, err = a.loadTypes(ctx, conn) + if err != nil && !errors.Is(err, errTypeTableNotFound) { return nil, err } - if exists { - rows := &entsql.Rows{} - query, args := entsql.Dialect(a.dialect). - Select("type").From(entsql.Table(TypeTable)).OrderBy(entsql.Asc("id")).Query() - if err := conn.Query(ctx, query, args, rows); err != nil { - return nil, fmt.Errorf("query types table: %w", err) - } - defer rows.Close() - a.types = nil - if err := entsql.ScanSlice(rows, &a.types); err != nil { - return nil, err - } - } - types = a.types + a.types = types } desired, err := a.StateReader(tables...).ReadState(ctx) if err != nil { @@ -710,6 +745,32 @@ func (a *Atlas) plan(ctx context.Context, conn dialect.ExecQuerier, name string, return plan, nil } +var errTypeTableNotFound = errors.New("ent_type table not found") + +// loadTypes loads the currently saved range allocations from the TypeTable. +func (a *Atlas) loadTypes(ctx context.Context, conn dialect.ExecQuerier) ([]string, error) { + // Fetch pre-existing type allocations. + exists, err := a.sqlDialect.tableExist(ctx, conn, TypeTable) + if err != nil { + return nil, err + } + if !exists { + return nil, errTypeTableNotFound + } + rows := &entsql.Rows{} + query, args := entsql.Dialect(a.dialect). + Select("type").From(entsql.Table(TypeTable)).OrderBy(entsql.Asc("id")).Query() + if err := conn.Query(ctx, query, args, rows); err != nil { + return nil, fmt.Errorf("query types table: %w", err) + } + defer rows.Close() + var types []string + if err := entsql.ScanSlice(rows, &types); err != nil { + return nil, err + } + return types, nil +} + type db struct{ dialect.ExecQuerier } func (d *db) QueryContext(ctx context.Context, query string, args ...interface{}) (*sql.Rows, error) { diff --git a/dialect/sql/schema/migrate.go b/dialect/sql/schema/migrate.go index b666feb24..37ce2d0ab 100644 --- a/dialect/sql/schema/migrate.go +++ b/dialect/sql/schema/migrate.go @@ -635,5 +635,5 @@ type fkRenamer interface { // verifyRanger wraps the method for verifying global-id range correctness. type verifyRanger interface { - verifyRange(context.Context, dialect.Tx, *Table, int64) error + verifyRange(context.Context, dialect.ExecQuerier, *Table, int64) error } diff --git a/dialect/sql/schema/mysql.go b/dialect/sql/schema/mysql.go index fddf1e2da..3398b70b7 100644 --- a/dialect/sql/schema/mysql.go +++ b/dialect/sql/schema/mysql.go @@ -141,7 +141,7 @@ func (d *MySQL) setRange(ctx context.Context, conn dialect.ExecQuerier, t *Table return conn.Exec(ctx, fmt.Sprintf("ALTER TABLE `%s` AUTO_INCREMENT = %d", t.Name, value), []interface{}{}, nil) } -func (d *MySQL) verifyRange(ctx context.Context, tx dialect.Tx, t *Table, expected int64) error { +func (d *MySQL) verifyRange(ctx context.Context, tx dialect.ExecQuerier, t *Table, expected int64) error { if expected == 0 { return nil } diff --git a/doc/md/versioned-migrations.md b/doc/md/versioned-migrations.md index dc35f3e1c..07c5c51c6 100644 --- a/doc/md/versioned-migrations.md +++ b/doc/md/versioned-migrations.md @@ -108,6 +108,62 @@ If you want to inspect an existing database and compute the diff against your En migration mode. ::: +### A Word on Global Unique IDs + +**This section only applies to MySQL users using the [global unique id](migrate.md/#universal-ids) feature.** + +When using the global unique ids, Ent allocates a range of `1<<32` integer values for each table. This is done by giving +the first table an autoincrement starting value of `1`, the second one the starting value `4294967296`, the third one +`8589934592`, and so on. The order in which the tables receive the starting value is saved in an extra table +called `ent_types`. With MySQL 5.6 and 5.7, the autoincrement starting value is only saved in +memory ([docs](https://dev.mysql.com/doc/refman/8.0/en/innodb-auto-increment-handling.html), **InnoDB AUTO_INCREMENT +Counter Initialization** header) and re-calculated on startup by looking at the last inserted id for any table. Now, if +you happen to have a table with no rows yet, the autoincrement starting value is set to 0 for every table without any +entries. With the online migration feature this wasn't an issue, because the migration engine looked at the `ent_types` +tables and made sure to update the counter, if it wasn't set correctly. However, with versioned migration, this is no +longer the case. In oder to ensure, that everything is set up correctly after a server restart, make sure to call +the `VerifyTableRange` method on the Atlas struct: + +```go +package main + +import ( + "context" + "log" + + "/ent" + "/ent/migrate" + "entgo.io/ent/dialect/sql" + "entgo.io/ent/dialect/sql/schema" + + _ "github.com/go-sql-driver/mysql" +) + +func main() { + drv, err := sql.Open("mysql", "user:pass@tcp(localhost:3306)/ent") + if err != nil { + log.Fatalf("failed opening connection to mysql: %v", err) + } + defer drv.Close() + // Verify the type allocation range. + m, err := schema.NewMigrate(drv, nil) + if err != nil { + log.Fatalf("failed creating migrate: %v", err) + } + if err := m.VerifyTableRange(context.Background(), migrate.Tables); err != nil { + log.Fatalf("failed verifyint range allocations: %v", err) + } + client := ent.NewClient(ent.Driver(drv)) + // ... do stuff with the client +} +``` + +:::caution Important +After an upgrade to MySQL 8 from a previous version, you still have to run the method once to update the starting +values. Since MySQL 8 the counter is no longer only stored in memory, meaning subsequent calls to the method are no +longer needed after the first one. +::: + ## Apply Migrations The Atlas migration engine does not support applying the migration files onto a database yet, therefore to manage and