From b6c185a6606e0f51afd52477a99585d51d7bd711 Mon Sep 17 00:00:00 2001 From: Jannik Clausen <12862103+masseelch@users.noreply.github.com> Date: Fri, 19 Aug 2022 14:42:54 +0200 Subject: [PATCH] =?UTF-8?q?dialect/sql/schema:=20make=20use=20of=20new=20a?= =?UTF-8?q?tlas=20migration=20directories=20and=20r=E2=80=A6=20(#2873)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * dialect/sql/schema: make use of new atlas migration directories and remove examples for custom formatters * typo --- dialect/sql/schema/atlas.go | 16 +- dialect/sql/schema/migrate_test.go | 31 ++- doc/md/versioned-migrations.mdx | 341 +++++++++++++++++------ entc/integration/migrate/migrate_test.go | 2 +- go.mod | 2 +- go.sum | 4 +- 6 files changed, 306 insertions(+), 90 deletions(-) diff --git a/dialect/sql/schema/atlas.go b/dialect/sql/schema/atlas.go index 6a56ca97d..dc1eef783 100644 --- a/dialect/sql/schema/atlas.go +++ b/dialect/sql/schema/atlas.go @@ -310,7 +310,6 @@ func (f DiffFunc) Diff(current, desired *schema.Schema) ([]schema.Change, error) // return changes, nil // }) // }) -// func WithDiffHook(hooks ...DiffHook) MigrateOption { return func(a *Atlas) { a.diffHooks = append(a.diffHooks, hooks...) @@ -321,7 +320,6 @@ func WithDiffHook(hooks ...DiffHook) MigrateOption { // returned by the Differ before executing migration planning. // // SkipChanges(schema.DropTable|schema.DropColumn) -// func WithSkipChanges(skip ChangeKind) MigrateOption { return func(a *Atlas) { a.skip = skip @@ -488,7 +486,6 @@ func (f ApplyFunc) Apply(ctx context.Context, conn dialect.ExecQuerier, plan *mi // return next.Apply(ctx, conn, plan) // }) // }) -// func WithApplyHook(hooks ...ApplyHook) MigrateOption { return func(a *Atlas) { a.applyHook = append(a.applyHook, hooks...) @@ -608,7 +605,18 @@ func (a *Atlas) init() error { a.diffHooks = append(a.diffHooks, withoutForeignKeys) } if a.dir != nil && a.fmt == nil { - a.fmt = sqltool.GolangMigrateFormatter + switch a.dir.(type) { + case *sqltool.GooseDir: + a.fmt = sqltool.GooseFormatter + case *sqltool.DBMateDir: + a.fmt = sqltool.DBMateFormatter + case *sqltool.FlywayDir: + a.fmt = sqltool.FlywayFormatter + case *sqltool.LiquibaseDir: + a.fmt = sqltool.LiquibaseFormatter + default: // migrate.LocalDir, sqltool.GolangMigrateDir and custom ones + a.fmt = sqltool.GolangMigrateFormatter + } } if a.mode == ModeReplay { // ModeReplay requires a migration directory. diff --git a/dialect/sql/schema/migrate_test.go b/dialect/sql/schema/migrate_test.go index 8a375e5e0..1ee2d96e0 100644 --- a/dialect/sql/schema/migrate_test.go +++ b/dialect/sql/schema/migrate_test.go @@ -16,6 +16,7 @@ import ( "ariga.io/atlas/sql/migrate" "ariga.io/atlas/sql/schema" + "ariga.io/atlas/sql/sqltool" "entgo.io/ent/dialect" "entgo.io/ent/dialect/sql" "entgo.io/ent/schema/field" @@ -72,6 +73,34 @@ func TestMigrateHookAddTable(t *testing.T) { require.NoError(t, err) } +func TestMigrate_Formatter(t *testing.T) { + db, _, err := sqlmock.New() + require.NoError(t, err) + + // If no formatter is given it will be set according to the given migration directory implementation. + for _, tt := range []struct { + dir migrate.Dir + fmt migrate.Formatter + }{ + {&migrate.LocalDir{}, sqltool.GolangMigrateFormatter}, + {&sqltool.GolangMigrateDir{}, sqltool.GolangMigrateFormatter}, + {&sqltool.GooseDir{}, sqltool.GooseFormatter}, + {&sqltool.DBMateDir{}, sqltool.DBMateFormatter}, + {&sqltool.FlywayDir{}, sqltool.FlywayFormatter}, + {&sqltool.LiquibaseDir{}, sqltool.LiquibaseFormatter}, + {struct{ migrate.Dir }{}, sqltool.GolangMigrateFormatter}, // default one if migration dir is unknown + } { + m, err := NewMigrate(sql.OpenDB("", db), WithDir(tt.dir)) + require.NoError(t, err) + require.Equal(t, tt.fmt, m.fmt) + } + + // If a formatter is given, it is not overridden. + m, err := NewMigrate(sql.OpenDB("", db), WithDir(&migrate.LocalDir{}), WithFormatter(migrate.DefaultFormatter)) + require.NoError(t, err) + require.Equal(t, migrate.DefaultFormatter, m.fmt) +} + func TestMigrate_Diff(t *testing.T) { ctx := context.Background() @@ -94,7 +123,7 @@ func TestMigrate_Diff(t *testing.T) { p = t.TempDir() d, err = migrate.NewLocalDir(p) require.NoError(t, err) - m, err = NewMigrate(db, WithDir(d), WithSumFile()) + m, err = NewMigrate(db, WithDir(d)) require.NoError(t, err) require.NoError(t, m.Diff(ctx, &Table{Name: "users"})) requireFileEqual(t, filepath.Join(p, v+"_changes.up.sql"), "-- create \"users\" table\nCREATE TABLE `users` (, PRIMARY KEY ());\n") diff --git a/doc/md/versioned-migrations.mdx b/doc/md/versioned-migrations.mdx index d23675b4f..9f1c4f9e6 100644 --- a/doc/md/versioned-migrations.mdx +++ b/doc/md/versioned-migrations.mdx @@ -120,6 +120,64 @@ docker run --name migration --rm -p 5432:5432 -e POSTGRES_PASSWORD=pass -e POSTG 2\. Create a `main.go` file under the `migrate/ent` package and customize the migration generation for your project. + + + +```go title="ent/migrate/main.go" +//go:build ignore + +package main + +import ( + "context" + "log" + "os" + + "/ent/migrate" + + atlas "ariga.io/atlas/sql/migrate" + "entgo.io/ent/dialect" + "entgo.io/ent/dialect/sql/schema" + _ "github.com/go-sql-driver/mysql" +) + +func main() { + ctx := context.Background() + // Create a local migration directory able to understand Atlas migration file format for replay. + dir, err := atlas.NewLocalDir("ent/migrate/migrations") + if err != nil { + log.Fatalf("failed creating atlas migration directory: %v", err) + } + // Migrate diff options. + opts := []schema.MigrateOption{ + schema.WithDir(dir), // provide migration directory + schema.WithMigrationMode(schema.ModeReplay), // provide migration mode + schema.WithDialect(dialect.MySQL), // Ent dialect to use + schema.WithFormatter(atlas.DefaultFormatter), + } + if len(os.Args) != 2 { + log.Fatalln("migration name is required. Use: 'go run -mod=mod ent/migrate/main.go '") + } + // Generate migrations using Atlas support for MySQL (note the Ent dialect option passed above). + err = migrate.NamedDiff(ctx, "mysql://root:pass@localhost:3306/test", os.Args[1], opts...) + if err != nil { + log.Fatalf("failed generating migration file: %v", err) + } +} +``` + + + + ```go title="ent/migrate/main.go" //go:build ignore @@ -140,7 +198,7 @@ import ( func main() { ctx := context.Background() - // Create a local migration directory able to understand golang-migrate migration files for replay. + // Create a local migration directory able to understand golang-migrate migration file format for replay. dir, err := sqltool.NewGolangMigrateDir("ent/migrate/migrations") if err != nil { log.Fatalf("failed creating atlas migration directory: %v", err) @@ -162,6 +220,189 @@ func main() { } ``` + + + +```go title="ent/migrate/main.go" +//go:build ignore + +package main + +import ( + "context" + "log" + "os" + + "/ent/migrate" + + "ariga.io/atlas/sql/sqltool" + "entgo.io/ent/dialect" + "entgo.io/ent/dialect/sql/schema" + _ "github.com/go-sql-driver/mysql" +) + +func main() { + ctx := context.Background() + // Create a local migration directory able to understand goose migration file format for replay. + dir, err := sqltool.NewGooseDir("ent/migrate/migrations") + if err != nil { + log.Fatalf("failed creating atlas migration directory: %v", err) + } + // Migrate diff options. + opts := []schema.MigrateOption{ + schema.WithDir(dir), // provide migration directory + schema.WithMigrationMode(schema.ModeReplay), // provide migration mode + schema.WithDialect(dialect.MySQL), // Ent dialect to use + } + if len(os.Args) != 2 { + log.Fatalln("migration name is required. Use: 'go run -mod=mod ent/migrate/main.go '") + } + // Generate migrations using Atlas support for MySQL (note the Ent dialect option passed above). + err = migrate.NamedDiff(ctx, "mysql://root:pass@localhost:3306/test", os.Args[1], opts...) + if err != nil { + log.Fatalf("failed generating migration file: %v", err) + } +} +``` + + + + +```go title="ent/migrate/main.go" +//go:build ignore + +package main + +import ( + "context" + "log" + "os" + + "/ent/migrate" + + "ariga.io/atlas/sql/sqltool" + "entgo.io/ent/dialect" + "entgo.io/ent/dialect/sql/schema" + _ "github.com/go-sql-driver/mysql" +) + +func main() { + ctx := context.Background() + // Create a local migration directory able to understand dbmate migration file format for replay. + dir, err := sqltool.NewDBMateDir("ent/migrate/migrations") + if err != nil { + log.Fatalf("failed creating atlas migration directory: %v", err) + } + // Migrate diff options. + opts := []schema.MigrateOption{ + schema.WithDir(dir), // provide migration directory + schema.WithMigrationMode(schema.ModeReplay), // provide migration mode + schema.WithDialect(dialect.MySQL), // Ent dialect to use + } + if len(os.Args) != 2 { + log.Fatalln("migration name is required. Use: 'go run -mod=mod ent/migrate/main.go '") + } + // Generate migrations using Atlas support for MySQL (note the Ent dialect option passed above). + err = migrate.NamedDiff(ctx, "mysql://root:pass@localhost:3306/test", os.Args[1], opts...) + if err != nil { + log.Fatalf("failed generating migration file: %v", err) + } +} +``` + + + + +```go title="ent/migrate/main.go" +//go:build ignore + +package main + +import ( + "context" + "log" + "os" + + "/ent/migrate" + + "ariga.io/atlas/sql/sqltool" + "entgo.io/ent/dialect" + "entgo.io/ent/dialect/sql/schema" + _ "github.com/go-sql-driver/mysql" +) + +func main() { + ctx := context.Background() + // Create a local migration directory able to understand Flyway migration file format for replay. + dir, err := sqltool.NewFlywayDir("ent/migrate/migrations") + if err != nil { + log.Fatalf("failed creating atlas migration directory: %v", err) + } + // Migrate diff options. + opts := []schema.MigrateOption{ + schema.WithDir(dir), // provide migration directory + schema.WithMigrationMode(schema.ModeReplay), // provide migration mode + schema.WithDialect(dialect.MySQL), // Ent dialect to use + } + if len(os.Args) != 2 { + log.Fatalln("migration name is required. Use: 'go run -mod=mod ent/migrate/main.go '") + } + // Generate migrations using Atlas support for MySQL (note the Ent dialect option passed above). + err = migrate.NamedDiff(ctx, "mysql://root:pass@localhost:3306/test", os.Args[1], opts...) + if err != nil { + log.Fatalf("failed generating migration file: %v", err) + } +} +``` + + + + +```go title="ent/migrate/main.go" +//go:build ignore + +package main + +import ( + "context" + "log" + "os" + + "/ent/migrate" + + "ariga.io/atlas/sql/sqltool" + "entgo.io/ent/dialect" + "entgo.io/ent/dialect/sql/schema" + _ "github.com/go-sql-driver/mysql" +) + +func main() { + ctx := context.Background() + // Create a local migration directory able to understand Liquibase migration file format for replay. + dir, err := sqltool.NewLiquibaseDir("ent/migrate/migrations") + if err != nil { + log.Fatalf("failed creating atlas migration directory: %v", err) + } + // Migrate diff options. + opts := []schema.MigrateOption{ + schema.WithDir(dir), // provide migration directory + schema.WithMigrationMode(schema.ModeReplay), // provide migration mode + schema.WithDialect(dialect.MySQL), // Ent dialect to use + } + if len(os.Args) != 2 { + log.Fatalln("migration name is required. Use: 'go run -mod=mod ent/migrate/main.go '") + } + // Generate migrations using Atlas support for MySQL (note the Ent dialect option passed above). + err = migrate.NamedDiff(ctx, "mysql://root:pass@localhost:3306/test", os.Args[1], opts...) + if err != nil { + log.Fatalf("failed generating migration file: %v", err) + } +} +``` + + + + 3\. Trigger migration generation by executing `go run -mod=mod ent/migrate/main.go ` from the root of the project. For example: @@ -307,16 +548,27 @@ longer needed after the first one. ## Apply Migrations -The Atlas migration engine does not support applying the migration files onto a database yet, therefore to manage and -execute the generated migration files, you have to rely on an external tool (or execute them by hand). By default, Atlas -generates one "up" and one "down" migration file for the computed diff. These files are compatible with the popular -[golang-migrate/migrate](https://github.com/golang-migrate/migrate) package, and you can use that tool to manage the -migrations in your deployments. +The Atlas migration engine has only experimental support applying the migration files onto a database yet. Therefore, +to manage and execute the generated migration files for production systems it is recommended you use on an external tool +(or execute them by hand). By default, Ent generates one "up" and one "down" migration file for the computed diff. These +files are compatible with the popular [golang-migrate/migrate](https://github.com/golang-migrate/migrate) package, and +you can use that tool to manage the migrations in your deployments. ```shell migrate -source file://migrations -database mysql://root:pass@tcp(localhost:3306)/test up ``` +:::note + +If you use golang-migrate with MySQL, you need to add the `multiStatements` parameter to `true` as in the example below +and then take the DSN we used in the documents with the param applied. + +``` +"user:password@tcp(host:port)/dbname?multiStatements=true" +``` + +::: + ## Moving from Auto-Migration to Versioned Migrations In case you already have an Ent application in production and want to switch over from auto migration to the new @@ -354,81 +606,8 @@ migration file. ### Configure the tool you use to manage migrations to consider this file as applied In case of `golang-migrate` this can be done by forcing your database version as -described [here](https://github.com/golang-migrate/migrate/blob/master/GETTING_STARTED.md#forcing-your-database-version) -. - -## Use a Custom Formatter - -Atlas' migration engine comes with great customizability. By the use of a custom `Formatter` you can generate the -migration files in a format compatible with other migration management tools and Atlas has built-in support for the -following four: - -1. [golang-migrate/migrate](https://github.com/golang-migrate/migrate) -2. [pressly/goose](https://github.com/pressly/goose) -3. [Flyway](https://flywaydb.org/) -4. [Liquibase](https://www.liquibase.org/) - -Please be aware, that migration directory replay currently only supports `golang-migrate/migrate` formatted files. -Attempting to replay migration files of other tools might work, but is not officially supported (yet). - -```go -//go:build ignore - -package main - -import ( - "context" - "log" - - "/ent/migrate" - - atlas "ariga.io/atlas/sql/migrate" - _ "ariga.io/atlas/sql/mysql" - "ariga.io/atlas/sql/sqltool" - "entgo.io/ent/dialect" - "entgo.io/ent/dialect/sql/schema" - _ "github.com/go-sql-driver/mysql" -) - -func main() { - ctx := context.Background() - // Create a local migration directory. - dir, err := atlas.NewLocalDir("ent/migrate/migrations") - if err != nil { - log.Fatalf("failed creating atlas migration directory: %v", err) - } - // Write migration diff. - opts := []schema.MigrateOption{ - schema.WithDir(dir), // provide migration directory - schema.WithMigrationMode(schema.ModeReplay), // provide migration mode - schema.WithDialect(dialect.MySQL), // Ent dialect to use - // highlight-start - // Choose one of the below. - schema.WithFormatter(sqltool.GolangMigrateFormatter), - schema.WithFormatter(sqltool.GooseFormatter), - schema.WithFormatter(sqltool.FlywayFormatter), - schema.WithFormatter(sqltool.LiquibaseFormatter), - // highlight-end - } - if len(os.Args) != 2 { - log.Fatalln("migration name is required. Use: 'go run -mod=mod ent/migrate/main.go '") - } - // Generate migrations using Atlas support for MySQL (note the Ent dialect option passed above). - err = migrate.NamedDiff(ctx, "mysql://root:pass@localhost:3306/test", os.Args[1], opts...) - if err != nil { - log.Fatalf("failed generating migration file: %v", err) - } -} -``` - -### Note for using golang-migrate - -If you use golang-migrate with MySQL, you need to add the `multiStatements` parameter to `true` as in the example below -and then take the DSN we used in the documents with the param applied. - -``` -"user:password@tcp(host:port)/dbname?multiStatements=true" -``` +described +[here](https://github.com/golang-migrate/migrate/blob/master/GETTING_STARTED.md#forcing-your-database-version). ## Atlas migration directory integrity file diff --git a/entc/integration/migrate/migrate_test.go b/entc/integration/migrate/migrate_test.go index 6d3443345..29c316275 100644 --- a/entc/integration/migrate/migrate_test.go +++ b/entc/integration/migrate/migrate_test.go @@ -282,7 +282,7 @@ func Versioned(t *testing.T, drv sql.ExecQuerier, devURL string, client *version require.ErrorIs(t, client.Schema.Diff(ctx, opts...), migrate.ErrChecksumMismatch) // Diffing by replaying on the current connection -> not clean. - hf, err := migrate.HashSum(dir) + hf, err := dir.Checksum() require.NoError(t, err) require.NoError(t, migrate.WriteSumFile(dir, hf)) require.ErrorAs(t, client.Schema.Diff(ctx, opts...), &migrate.NotCleanError{}) diff --git a/go.mod b/go.mod index e56595d89..c5db67a1e 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module entgo.io/ent go 1.18 require ( - ariga.io/atlas v0.6.2-0.20220819082710-8934a5c3f6e6 + ariga.io/atlas v0.6.2-0.20220819114704-2060066abac7 github.com/DATA-DOG/go-sqlmock v1.5.0 github.com/go-openapi/inflect v0.19.0 github.com/go-sql-driver/mysql v1.6.0 diff --git a/go.sum b/go.sum index c41da1481..a9b613fc3 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -ariga.io/atlas v0.6.2-0.20220819082710-8934a5c3f6e6 h1:Abt1n0exvwB+4kN9Kuq7zET3Fd7sZCuImy323997wYg= -ariga.io/atlas v0.6.2-0.20220819082710-8934a5c3f6e6/go.mod h1:ft47uSh5hWGDCmQC9DsztZg6Xk+KagM5Ts/mZYKb9JE= +ariga.io/atlas v0.6.2-0.20220819114704-2060066abac7 h1:qhVEfrV5Z9XZyQJxgogBq6c2pjWUxGLU7bvFeEDY0DA= +ariga.io/atlas v0.6.2-0.20220819114704-2060066abac7/go.mod h1:ft47uSh5hWGDCmQC9DsztZg6Xk+KagM5Ts/mZYKb9JE= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60=