From 6802bf68f872c7258ea0bab94da2df5348dc5abd Mon Sep 17 00:00:00 2001 From: "Giau. Tran Minh" Date: Tue, 5 May 2026 15:27:47 +0000 Subject: [PATCH] upload first, then store the key as virtual fk --- blob.go | 122 +++++++++++++ entc/gen/graph.go | 6 + entc/gen/template.go | 2 - entc/gen/template/client.tmpl | 40 +---- entc/gen/template/dialect/sql/create.tmpl | 160 +++++------------ entc/gen/template/dialect/sql/decode.tmpl | 12 ++ entc/gen/template/dialect/sql/ent.tmpl | 3 + entc/gen/template/dialect/sql/meta.tmpl | 27 +++ entc/gen/template/dialect/sql/update.tmpl | 88 +++------- entc/gen/template/ent.tmpl | 46 ++--- entc/gen/template/meta.tmpl | 6 + entc/gen/template/runtime.tmpl | 14 ++ entc/gen/type.go | 42 +++++ entc/integration/blob/blob.go | 10 -- entc/integration/blob/encrypt.go | 5 - entc/integration/blob_test.go | 86 ++++------ entc/integration/ent/client.go | 41 +---- entc/integration/ent/document.go | 111 +++++------- entc/integration/ent/document/document.go | 19 ++ entc/integration/ent/document_create.go | 181 +++++--------------- entc/integration/ent/document_update.go | 82 +++------ entc/integration/ent/migrate/schema.go | 2 + entc/integration/ent/runtime.go | 5 + entc/integration/ent/schema/document.go | 9 +- entc/integration/ent/template/blob_key.tmpl | 9 - entc/integration/gremlin/ent/client.go | 26 +-- entc/integration/privacy/ent/client.go | 12 +- entc/integration/privacy/ent/entc.go | 2 + entc/load/schema.go | 2 + schema/field/field.go | 54 +++--- 30 files changed, 539 insertions(+), 685 deletions(-) create mode 100644 blob.go delete mode 100644 entc/integration/ent/template/blob_key.tmpl diff --git a/blob.go b/blob.go new file mode 100644 index 000000000..d09f86c00 --- /dev/null +++ b/blob.go @@ -0,0 +1,122 @@ +// Copyright 2019-present Facebook Inc. All rights reserved. +// This source code is licensed under the Apache 2.0 license found +// in the LICENSE file in the root directory of this source tree. + +package ent + +import ( + "context" + "errors" + "io" + "io/fs" +) + +// Blob defines the interface for blob storage operations. +// Implementations should return [io/fs.ErrNotExist] (or an error wrapping it) +// from NewReader when the requested key does not exist. +type Blob interface { + // NewReader opens a reader for the given key. + NewReader(ctx context.Context, key string) (io.ReadCloser, error) + // NewWriter opens a writer for the given key. + NewWriter(ctx context.Context, key string) (io.WriteCloser, error) + // Close releases any resources held by the bucket. + Close() error +} + +// BlobReader returns a reader for the given key from the blob bucket. +// The returned reader closes both the underlying reader and the bucket. +// Returns nil, nil if the blob does not exist (fs.ErrNotExist). +func BlobReader(ctx context.Context, b Blob, key string) (io.ReadCloser, error) { + switch r, err := b.NewReader(ctx, key); { + case errors.Is(err, fs.ErrNotExist): + return nil, b.Close() + case err != nil: + return nil, errors.Join(err, b.Close()) + default: + return &blobReadCloser{ReadCloser: r, bucket: b}, nil + } +} + +// blobReadCloser wraps an io.ReadCloser to also close the bucket on Close. +type blobReadCloser struct { + io.ReadCloser + bucket Blob +} + +func (r *blobReadCloser) Close() error { + return errors.Join(r.ReadCloser.Close(), r.bucket.Close()) +} + +// BlobWriter returns a writer for the given key in the blob bucket. +// The returned writer closes both the underlying writer and the bucket. +func BlobWriter(ctx context.Context, b Blob, key string) (io.WriteCloser, error) { + w, err := b.NewWriter(ctx, key) + if err != nil { + return nil, errors.Join(err, b.Close()) + } + return &blobWriteCloser{WriteCloser: w, bucket: b}, nil +} + +// blobWriteCloser wraps an io.WriteCloser to also close the bucket on Close. +type blobWriteCloser struct { + io.WriteCloser + bucket Blob +} + +func (w *blobWriteCloser) Close() error { + return errors.Join(w.WriteCloser.Close(), w.bucket.Close()) +} + +// BlobBulkWriter manages blob bucket lifecycles for write operations. +// It lazily opens buckets per field and provides a Close method +// to release all resources when done. +type BlobBulkWriter struct { + opener func(context.Context, string) (Blob, error) + buckets map[string]Blob +} + +// NewBlobBulkWriter creates a writer that uses opener to lazily open buckets. +func NewBlobBulkWriter(opener func(context.Context, string) (Blob, error)) *BlobBulkWriter { + return &BlobBulkWriter{ + buckets: make(map[string]Blob), + opener: opener, + } +} + +// Close closes all open buckets. +func (w *BlobBulkWriter) Close() error { + var errs []error + for _, b := range w.buckets { + errs = append(errs, b.Close()) + } + return errors.Join(errs...) +} + +// Write writes r to the blob at key for the given field. The bucket is opened +// lazily on first use and reused for subsequent writes to the same field. +func (w *BlobBulkWriter) Write(ctx context.Context, field, key string, r io.Reader) error { + b, err := w.bucket(ctx, field) + if err != nil { + return err + } + wr, err := b.NewWriter(ctx, key) + if err != nil { + return err + } + if _, err := io.Copy(wr, r); err != nil { + return errors.Join(err, wr.Close()) + } + return wr.Close() +} + +func (w *BlobBulkWriter) bucket(ctx context.Context, field string) (Blob, error) { + if b, ok := w.buckets[field]; ok { + return b, nil + } + b, err := w.opener(ctx, field) + if err != nil { + return nil, err + } + w.buckets[field] = b + return b, nil +} diff --git a/entc/gen/graph.go b/entc/gen/graph.go index 6a29e8ff9..cf77b3c36 100644 --- a/entc/gen/graph.go +++ b/entc/gen/graph.go @@ -172,6 +172,9 @@ func NewGraph(c *Config, schemas ...*load.Schema) (g *Graph, err error) { for _, t := range g.Nodes { check(t.setupFKs(), "set %q foreign-keys", t.Name) } + for _, t := range g.Nodes { + t.setupBlobKeys() + } for i := range schemas { g.addIndexes(schemas[i]) } @@ -661,6 +664,9 @@ func (g *Graph) Tables() (all []*schema.Table, err error) { table.AddColumn(f.Column()) } } + for _, bk := range n.BlobKeys { + table.AddColumn(bk.Field.Column()) + } switch { case tables[table.Name] == nil: tables[table.Name] = table diff --git a/entc/gen/template.go b/entc/gen/template.go index 666b29f65..930a012bd 100644 --- a/entc/gen/template.go +++ b/entc/gen/template.go @@ -181,8 +181,6 @@ var ( deletedTemplates = []string{"config.go", "context.go"} // patterns for extending partial-templates (included by other templates). partialPatterns = [...]string{ - "blob/key/*", - "blob/key/*/*", "client/additional/*", "client/additional/*/*", "config/*/*", diff --git a/entc/gen/template/client.tmpl b/entc/gen/template/client.tmpl index 74b327bd2..dca216fe1 100644 --- a/entc/gen/template/client.tmpl +++ b/entc/gen/template/client.tmpl @@ -160,24 +160,8 @@ func Driver(driver dialect.Driver) Option { {{- end }} {{- if $hasBlobNodes }} -// Blob defines the interface for blob storage operations. -// Implementations should return [io/fs.ErrNotExist] (or an error wrapping it) -// from NewReader when the requested key does not exist. -// -// Blob writes are not transactional with the database. If a bulk operation -// fails partway through, already-written blobs will be cleaned up on a -// best-effort basis using the Delete method. -type Blob interface { - // NewReader opens a reader for the given key. - NewReader(ctx context.Context, key string) (io.ReadCloser, error) - // NewWriter opens a writer for the given key. - NewWriter(ctx context.Context, key string) (io.WriteCloser, error) - // Delete removes the blob stored at key. It should return nil - // if the key does not exist. - Delete(ctx context.Context, key string) error - // Close releases any resources held by the bucket. - Close() error -} +// Blob is an alias for the [ent.Blob] interface defined in the entgo.io/ent package. +type Blob = ent.Blob // BlobOpeners configures how blob buckets are opened for each entity type. // Each field is a function that opens a blob bucket for the given field name. @@ -195,26 +179,6 @@ func WithBlobOpeners(openers BlobOpeners) Option { c.blobOpeners = openers } } - -// blobReadCloser wraps an io.ReadCloser to also close the parent bucket on Close. -type blobReadCloser struct { - io.ReadCloser - bucket Blob -} - -func (r *blobReadCloser) Close() error { - return errors.Join(r.ReadCloser.Close(), r.bucket.Close()) -} - -// blobWriteCloser wraps an io.WriteCloser to also close the parent bucket on Close. -type blobWriteCloser struct { - io.WriteCloser - bucket Blob -} - -func (w *blobWriteCloser) Close() error { - return errors.Join(w.WriteCloser.Close(), w.bucket.Close()) -} {{- end }} // Open opens a database/sql.DB specified by the driver name and diff --git a/entc/gen/template/dialect/sql/create.tmpl b/entc/gen/template/dialect/sql/create.tmpl index e8546a7bb..a4cbb6b9e 100644 --- a/entc/gen/template/dialect/sql/create.tmpl +++ b/entc/gen/template/dialect/sql/create.tmpl @@ -22,11 +22,35 @@ func ({{ $receiver }} *{{ $builder }}) sqlSave(ctx context.Context) (*{{ $.Name return nil, err } {{- end }} + {{- if $.HasBlobFields }} + _blobs := ent.NewBlobBulkWriter({{ $mutation }}.blobOpeners.{{ $.Name }}) + {{- range $f := $.BlobFields }} + if r, ok := {{ $mutation }}.{{ $f.StructField }}(); ok { + {{- if $f.HasBlobKey }} + key, err := {{ $.Package }}.{{ $f.BlobKeyName }}(ctx) + if err != nil { + return nil, errors.Join(fmt.Errorf("{{ $pkg }}: generating blob key for {{ $f.Name }}: %w", err), _blobs.Close()) + } + {{- else }} + key := uuid.NewString() + {{- end }} + if err := _blobs.Write(ctx, {{ $.Package }}.{{ $f.Constant }}, key, r); err != nil { + return nil, errors.Join(fmt.Errorf("{{ $pkg }}: writing blob for {{ $f.Name }}: %w", err), _blobs.Close()) + } + _node.{{ $f.Name }}_key = &key + _spec.SetField("{{ $f.Name }}_key", field.TypeString, key) + } + {{- end }} + {{- end }} if err := sqlgraph.CreateNode(ctx, {{ $receiver }}.driver, _spec); err != nil { if sqlgraph.IsConstraintError(err) { err = &ConstraintError{msg: err.Error(), wrap: err} } + {{- if $.HasBlobFields }} + return nil, errors.Join(err, _blobs.Close()) + {{- else }} return nil, err + {{- end }} } {{- if $.HasCompositeID }} {{- else if or $.ID.HasValueScanner $.ID.Type.ValueScanner (not $.ID.Type.Numeric) }} @@ -78,68 +102,9 @@ func ({{ $receiver }} *{{ $builder }}) sqlSave(ctx context.Context) (*{{ $.Name {{ $mutation }}.done = true {{- end }} {{- if $.HasBlobFields }} - {{- $blobFields := $.BlobFields }} - {{- if gt (len $blobFields) 1 }} - type blobWritten struct { field string; key string } - var _blobWritten []blobWritten - _blobCleanup := func(ctx context.Context) error { - var errs []error - for _, bw := range _blobWritten { - b, err := {{ $mutation }}.blobOpeners.{{ $.Name }}(ctx, bw.field) - if err != nil { - errs = append(errs, err) - continue - } - errs = append(errs, b.Delete(ctx, bw.key), b.Close()) - } - return errors.Join(errs...) - } - {{- end }} - {{- range $i, $f := $blobFields }} - if r, ok := {{ $mutation }}.{{ $f.StructField }}(); ok { - b, err := {{ $mutation }}.blobOpeners.{{ $.Name }}(ctx, {{ $.Package }}.{{ $f.Constant }}) - if err != nil { - {{- if gt (len $blobFields) 1 }} - return nil, errors.Join(fmt.Errorf("{{ $pkg }}: opening blob bucket for {{ $f.Name }}: %w", err), _blobCleanup(ctx)) - {{- else }} - return nil, fmt.Errorf("{{ $pkg }}: opening blob bucket for {{ $f.Name }}: %w", err) - {{- end }} - } - key, err := _node.{{ $f.StructField }}Key(ctx) - if err != nil { - {{- if gt (len $blobFields) 1 }} - return nil, errors.Join(fmt.Errorf("{{ $pkg }}: blob key for {{ $f.Name }}: %w", err), b.Close(), _blobCleanup(ctx)) - {{- else }} - return nil, errors.Join(fmt.Errorf("{{ $pkg }}: blob key for {{ $f.Name }}: %w", err), b.Close()) - {{- end }} - } - w, err := b.NewWriter(ctx, key) - if err != nil { - {{- if gt (len $blobFields) 1 }} - return nil, errors.Join(fmt.Errorf("{{ $pkg }}: creating writer for {{ $f.Name }}: %w", err), b.Close(), _blobCleanup(ctx)) - {{- else }} - return nil, errors.Join(fmt.Errorf("{{ $pkg }}: creating writer for {{ $f.Name }}: %w", err), b.Close()) - {{- end }} - } - if _, err := io.Copy(w, r); err != nil { - {{- if gt (len $blobFields) 1 }} - return nil, errors.Join(fmt.Errorf("{{ $pkg }}: writing blob for {{ $f.Name }}: %w", err), w.Close(), b.Close(), _blobCleanup(ctx)) - {{- else }} - return nil, errors.Join(fmt.Errorf("{{ $pkg }}: writing blob for {{ $f.Name }}: %w", err), w.Close(), b.Close()) - {{- end }} - } - if err := errors.Join(w.Close(), b.Close()); err != nil { - {{- if gt (len $blobFields) 1 }} - return nil, errors.Join(fmt.Errorf("{{ $pkg }}: closing blob for {{ $f.Name }}: %w", err), _blobCleanup(ctx)) - {{- else }} - return nil, fmt.Errorf("{{ $pkg }}: closing blob for {{ $f.Name }}: %w", err) - {{- end }} - } - {{- if gt (len $blobFields) 1 }} - _blobWritten = append(_blobWritten, blobWritten{field: {{ $.Package }}.{{ $f.Constant }}, key: key}) - {{- end }} - } - {{- end }} + if err := _blobs.Close(); err != nil { + return nil, err + } {{- end }} return _node, nil } @@ -240,31 +205,7 @@ func ({{ $receiver }} *{{ $builder }}) Save(ctx context.Context) ([]*{{ $.Name } nodes := make([]*{{ $.Name }}, len({{ $receiver }}.builders)) mutators := make([]Mutator, len({{ $receiver }}.builders)) {{- if $.HasBlobFields }} - {{- range $f := $.BlobFields }} - var _blob{{ $f.StructField }} Blob - var _blobKeys{{ $f.StructField }} []string - {{- end }} - closeBlobs := func() error { - var errs []error - {{- range $f := $.BlobFields }} - if _blob{{ $f.StructField }} != nil { - errs = append(errs, _blob{{ $f.StructField }}.Close()) - } - {{- end }} - return errors.Join(errs...) - } - cleanupBlobs := func(ctx context.Context) error { - var errs []error - {{- range $f := $.BlobFields }} - if _blob{{ $f.StructField }} != nil { - for _, key := range _blobKeys{{ $f.StructField }} { - errs = append(errs, _blob{{ $f.StructField }}.Delete(ctx, key)) - } - } - {{- end }} - errs = append(errs, closeBlobs()) - return errors.Join(errs...) - } + _blobs := ent.NewBlobBulkWriter({{ $receiver }}.builders[0].mutation.blobOpeners.{{ $.Name }}) {{- end }} for i := range {{ $receiver }}.builders { func(i int, root context.Context) { @@ -288,6 +229,23 @@ func ({{ $receiver }} *{{ $builder }}) Save(ctx context.Context) ([]*{{ $.Name } return nil, err } {{- end }} + {{- range $f := $.BlobFields }} + if r, ok := mutation.{{ $f.StructField }}(); ok { + {{- if $f.HasBlobKey }} + key, err := {{ $.Package }}.{{ $f.BlobKeyName }}(ctx) + if err != nil { + return nil, fmt.Errorf("{{ $pkg }}: generating blob key for {{ $f.Name }}: %w", err) + } + {{- else }} + key := uuid.NewString() + {{- end }} + if err := _blobs.Write(ctx, {{ $.Package }}.{{ $f.Constant }}, key, r); err != nil { + return nil, fmt.Errorf("{{ $pkg }}: writing blob for {{ $f.Name }}: %w", err) + } + nodes[i].{{ $f.Name }}_key = &key + specs[i].SetField("{{ $f.Name }}_key", field.TypeString, key) + } + {{- end }} if i < len(mutators)-1 { _, err = mutators[i+1].Mutate(root, {{ $receiver }}.builders[i+1].mutation) } else { @@ -341,30 +299,6 @@ func ({{ $receiver }} *{{ $builder }}) Save(ctx context.Context) ([]*{{ $.Name } {{- end }} {{- end }} mutation.done = true - {{- range $f := $.BlobFields }} - if r, ok := mutation.{{ $f.StructField }}(); ok { - if _blob{{ $f.StructField }} == nil { - if _blob{{ $f.StructField }}, err = mutation.blobOpeners.{{ $.Name }}(ctx, {{ $.Package }}.{{ $f.Constant }}); err != nil { - return nil, fmt.Errorf("{{ $pkg }}: opening blob bucket for {{ $f.Name }}: %w", err) - } - } - key, err := nodes[i].{{ $f.StructField }}Key(ctx) - if err != nil { - return nil, fmt.Errorf("{{ $pkg }}: blob key for {{ $f.Name }}: %w", err) - } - w, err := _blob{{ $f.StructField }}.NewWriter(ctx, key) - if err != nil { - return nil, fmt.Errorf("{{ $pkg }}: creating writer for {{ $f.Name }}: %w", err) - } - if _, err := io.Copy(w, r); err != nil { - return nil, errors.Join(fmt.Errorf("{{ $pkg }}: writing blob for {{ $f.Name }}: %w", err), w.Close()) - } - if err := w.Close(); err != nil { - return nil, fmt.Errorf("{{ $pkg }}: closing writer for {{ $f.Name }}: %w", err) - } - _blobKeys{{ $f.StructField }} = append(_blobKeys{{ $f.StructField }}, key) - } - {{- end }} return nodes[i], nil }) for i := len(builder.hooks) - 1; i >= 0; i-- { @@ -376,14 +310,14 @@ func ({{ $receiver }} *{{ $builder }}) Save(ctx context.Context) ([]*{{ $.Name } if len(mutators) > 0 { if _, err := mutators[0].Mutate(ctx, {{ $receiver }}.builders[0].mutation); err != nil { {{- if $.HasBlobFields }} - return nil, errors.Join(err, cleanupBlobs(ctx)) + return nil, errors.Join(err, _blobs.Close()) {{- else }} return nil, err {{- end }} } } {{- if $.HasBlobFields }} - return nodes, closeBlobs() + return nodes, _blobs.Close() {{- else }} return nodes, nil {{- end }} diff --git a/entc/gen/template/dialect/sql/decode.tmpl b/entc/gen/template/dialect/sql/decode.tmpl index bd2c096e0..29bfde2d7 100644 --- a/entc/gen/template/dialect/sql/decode.tmpl +++ b/entc/gen/template/dialect/sql/decode.tmpl @@ -56,6 +56,10 @@ func (*{{ $.Name }}) scanValues(columns []string) ([]any, error) { case {{ $.Package }}.ForeignKeys[{{ $i }}]: // {{ $f.Name }} values[i] = {{ if not $f.UserDefined }}new(sql.NullInt64){{ else }}{{ $f.NewScanType }}{{ end }} {{- end }} + {{- range $i, $bk := $.BlobKeys }} + case {{ $.Package }}.BlobKeys[{{ $i }}]: // {{ $bk.Field.Name }} + values[i] = new(sql.NullString) + {{- end }} default: {{- /* In case of unknown column that was added by a modifier, predicate, etc., fallback to any. */}} values[i] = new(sql.UnknownType) @@ -111,6 +115,14 @@ func ({{ $receiver }} *{{ $.Name }}) assignValues(columns []string, values []any } {{- end }} {{- end }} + {{- range $i, $bk := $.BlobKeys }} + case {{ $.Package }}.BlobKeys[{{ $i }}]: + if value, ok := values[i].(*sql.NullString); !ok { + return fmt.Errorf("unexpected type %T for field {{ $bk.Field.Name }}", values[i]) + } else if value.Valid { + {{ $receiver }}.{{ $bk.StructField }} = &value.String + } + {{- end }} default: {{- /* In case of no match, allow getting this value by its name. */}} {{ $receiver }}.selectValues.Set(columns[i], values[i]) diff --git a/entc/gen/template/dialect/sql/ent.tmpl b/entc/gen/template/dialect/sql/ent.tmpl index 030551c0d..e38ff7c19 100644 --- a/entc/gen/template/dialect/sql/ent.tmpl +++ b/entc/gen/template/dialect/sql/ent.tmpl @@ -12,6 +12,9 @@ in the LICENSE file in the root directory of this source tree. {{- $f := $fk.Field }} {{ $fk.StructField }} {{ if $f.Nillable }}*{{ end }}{{ $f.Type }} {{- end }} + {{- range $bk := $.BlobKeys }} + {{ $bk.StructField }} *string + {{- end }} selectValues sql.SelectValues {{- /* Allow adding struct fields by ent extensions or user templates.*/}} {{- with $tmpls := matchTemplate "dialect/sql/model/fields/*" }} diff --git a/entc/gen/template/dialect/sql/meta.tmpl b/entc/gen/template/dialect/sql/meta.tmpl index eb8802386..adb3c5f51 100644 --- a/entc/gen/template/dialect/sql/meta.tmpl +++ b/entc/gen/template/dialect/sql/meta.tmpl @@ -53,6 +53,9 @@ in the LICENSE file in the root directory of this source tree. {{ $f.Constant }}, {{- end }} {{- end }} + {{- range $bk := $.BlobKeys }} + "{{ $bk.Field.Name }}", + {{- end }} } {{/* If any of the edges owns a foreign-key */}} {{ with $.UnexportedForeignKeys }} @@ -65,6 +68,23 @@ in the LICENSE file in the root directory of this source tree. } {{ end }} + {{ with $.BlobKeys }} + // BlobKeys holds the SQL columns for blob storage keys. + var BlobKeys = []string{ + {{- range $bk := . }} + "{{ $bk.Field.Name }}", + {{- end }} + } + {{ end }} + + {{- range $f := $.BlobFields }} + {{- if $f.HasBlobKey }} + // {{ $f.BlobKeyName }} generates the blob storage key for the {{ $f.Name }} field. + // It is set by the runtime/init package from the schema descriptor. + var {{ $f.BlobKeyName }} func(context.Context) (string, error) + {{- end }} + {{- end }} + {{ with $.NumM2M }} var ( {{- range $e := $.Edges }} @@ -94,6 +114,13 @@ func ValidColumn(column string) bool { } } {{- end }} + {{- with $.BlobKeys }} + for i := range BlobKeys { + if column == BlobKeys[i] { + return true + } + } + {{- end }} {{- with $.DeprecatedFields }} for _, f := range [...]string{ {{- range . }}{{ .Constant }},{{ end }} } { if column == f { diff --git a/entc/gen/template/dialect/sql/update.tmpl b/entc/gen/template/dialect/sql/update.tmpl index 1e52c5a7b..8666e6f4f 100644 --- a/entc/gen/template/dialect/sql/update.tmpl +++ b/entc/gen/template/dialect/sql/update.tmpl @@ -169,6 +169,25 @@ func ({{ $receiver }} *{{ $builder }}) sqlSave(ctx context.Context) (_node {{ if {{- xtemplate $tmpl $ }} {{- end }} {{- end }} + {{- if and $one $.HasBlobFields }} + _blobs := ent.NewBlobBulkWriter({{ $mutation }}.blobOpeners.{{ $.Name }}) + {{- range $f := $.BlobFields }} + if r, ok := {{ $mutation }}.{{ $f.StructField }}(); ok { + {{- if $f.HasBlobKey }} + key, err := {{ $.Package }}.{{ $f.BlobKeyName }}(ctx) + if err != nil { + return nil, errors.Join(fmt.Errorf("{{ $pkg }}: generating blob key for {{ $f.Name }}: %w", err), _blobs.Close()) + } + {{- else }} + key := uuid.NewString() + {{- end }} + if err := _blobs.Write(ctx, {{ $.Package }}.{{ $f.Constant }}, key, r); err != nil { + return nil, errors.Join(fmt.Errorf("{{ $pkg }}: writing blob for {{ $f.Name }}: %w", err), _blobs.Close()) + } + _spec.SetField("{{ $f.Name }}_key", field.TypeString, key) + } + {{- end }} + {{- end }} {{- if $one }} _node = &{{ $.Name }}{config: {{ $receiver }}.config} _spec.Assign = _node.assignValues @@ -184,72 +203,17 @@ func ({{ $receiver }} *{{ $builder }}) sqlSave(ctx context.Context) (_node {{ if } else if sqlgraph.IsConstraintError(err) { err = &ConstraintError{msg: err.Error(), wrap: err} } + {{- if and $one $.HasBlobFields }} + return {{ $zero }}, errors.Join(err, _blobs.Close()) + {{- else }} return {{ $zero }}, err + {{- end }} } {{ $mutation }}.done = true {{- if and $one $.HasBlobFields }} - {{- $blobFields := $.BlobFields }} - {{- if gt (len $blobFields) 1 }} - type blobWritten struct { field string; key string } - var _blobWritten []blobWritten - _blobCleanup := func(ctx context.Context) error { - var errs []error - for _, bw := range _blobWritten { - b, err := {{ $mutation }}.blobOpeners.{{ $.Name }}(ctx, bw.field) - if err != nil { - errs = append(errs, err) - continue - } - errs = append(errs, b.Delete(ctx, bw.key), b.Close()) - } - return errors.Join(errs...) - } - {{- end }} - {{- range $i, $f := $blobFields }} - if r, ok := {{ $mutation }}.{{ $f.StructField }}(); ok { - b, err := {{ $mutation }}.blobOpeners.{{ $.Name }}(ctx, {{ $.Package }}.{{ $f.Constant }}) - if err != nil { - {{- if gt (len $blobFields) 1 }} - return nil, errors.Join(fmt.Errorf("{{ $pkg }}: opening blob bucket for {{ $f.Name }}: %w", err), _blobCleanup(ctx)) - {{- else }} - return nil, fmt.Errorf("{{ $pkg }}: opening blob bucket for {{ $f.Name }}: %w", err) - {{- end }} - } - key, err := _node.{{ $f.StructField }}Key(ctx) - if err != nil { - {{- if gt (len $blobFields) 1 }} - return nil, errors.Join(fmt.Errorf("{{ $pkg }}: blob key for {{ $f.Name }}: %w", err), b.Close(), _blobCleanup(ctx)) - {{- else }} - return nil, errors.Join(fmt.Errorf("{{ $pkg }}: blob key for {{ $f.Name }}: %w", err), b.Close()) - {{- end }} - } - w, err := b.NewWriter(ctx, key) - if err != nil { - {{- if gt (len $blobFields) 1 }} - return nil, errors.Join(fmt.Errorf("{{ $pkg }}: creating writer for {{ $f.Name }}: %w", err), b.Close(), _blobCleanup(ctx)) - {{- else }} - return nil, errors.Join(fmt.Errorf("{{ $pkg }}: creating writer for {{ $f.Name }}: %w", err), b.Close()) - {{- end }} - } - if _, err := io.Copy(w, r); err != nil { - {{- if gt (len $blobFields) 1 }} - return nil, errors.Join(fmt.Errorf("{{ $pkg }}: writing blob for {{ $f.Name }}: %w", err), w.Close(), b.Close(), _blobCleanup(ctx)) - {{- else }} - return nil, errors.Join(fmt.Errorf("{{ $pkg }}: writing blob for {{ $f.Name }}: %w", err), w.Close(), b.Close()) - {{- end }} - } - if err := errors.Join(w.Close(), b.Close()); err != nil { - {{- if gt (len $blobFields) 1 }} - return nil, errors.Join(fmt.Errorf("{{ $pkg }}: closing blob for {{ $f.Name }}: %w", err), _blobCleanup(ctx)) - {{- else }} - return nil, fmt.Errorf("{{ $pkg }}: closing blob for {{ $f.Name }}: %w", err) - {{- end }} - } - {{- if gt (len $blobFields) 1 }} - _blobWritten = append(_blobWritten, blobWritten{field: {{ $.Package }}.{{ $f.Constant }}, key: key}) - {{- end }} - } - {{- end }} + if err := _blobs.Close(); err != nil { + return nil, err + } {{- end }} return _node, nil } diff --git a/entc/gen/template/ent.tmpl b/entc/gen/template/ent.tmpl index af6f93aa8..1e2f80ad0 100644 --- a/entc/gen/template/ent.tmpl +++ b/entc/gen/template/ent.tmpl @@ -102,29 +102,17 @@ type {{ $edgesType }} struct { {{ end }} {{- range $f := $.BlobFields }} - // {{ $f.StructField }} opens a reader for the "{{ $f.Name }}" field from blob storage. + // {{ $f.StructField }}Reader opens a reader for the "{{ $f.Name }}" field from blob storage. // The caller must close the returned reader when done. - // It returns nil, nil if the blob does not exist. - func ({{ $receiver }} *{{ $.Name }}) {{ $f.StructField }}(ctx context.Context) (io.ReadCloser, error) { - key, err := {{ $receiver }}.{{ $f.StructField }}Key(ctx) - if err != nil { - return nil, fmt.Errorf("{{ $pkg }}: blob key for {{ $f.Name }}: %w", err) + func ({{ $receiver }} *{{ $.Name }}) {{ $f.StructField }}Reader(ctx context.Context) (io.ReadCloser, error) { + if {{ $receiver }}.{{ $f.Name }}_key == nil || *{{ $receiver }}.{{ $f.Name }}_key == "" { + return nil, fmt.Errorf("{{ $pkg }}: {{ $.Name }}.{{ $f.Name }}_key is nil or empty") } b, err := {{ $receiver }}.blobOpeners.{{ $.Name }}(ctx, {{ $.Package }}.{{ $f.Constant }}) if err != nil { - return nil, fmt.Errorf("{{ $pkg }}: opening blob bucket for {{ $f.Name }}: %w", err) - } - switch r, err := b.NewReader(ctx, key); { - case errors.Is(err, fs.ErrNotExist): - if err := b.Close(); err != nil { - return nil, fmt.Errorf("{{ $pkg }}: closing blob bucket for {{ $f.Name }}: %w", err) - } - return nil, nil - case err != nil: - return nil, errors.Join(fmt.Errorf("{{ $pkg }}: creating reader for {{ $f.Name }}: %w", err), b.Close()) - default: - return &blobReadCloser{ReadCloser: r, bucket: b}, nil + return nil, err } + return ent.BlobReader(ctx, b, *{{ $receiver }}.{{ $f.Name }}_key) } // {{ $f.StructField }}Writer opens a writer for the "{{ $f.Name }}" field in blob storage. @@ -132,29 +120,15 @@ type {{ $edgesType }} struct { // also releases the underlying bucket resources. // Writing via this method does not go through the mutation pipeline. func ({{ $receiver }} *{{ $.Name }}) {{ $f.StructField }}Writer(ctx context.Context) (io.WriteCloser, error) { - key, err := {{ $receiver }}.{{ $f.StructField }}Key(ctx) - if err != nil { - return nil, fmt.Errorf("{{ $pkg }}: blob key for {{ $f.Name }}: %w", err) + if {{ $receiver }}.{{ $f.Name }}_key == nil || *{{ $receiver }}.{{ $f.Name }}_key == "" { + return nil, fmt.Errorf("{{ $pkg }}: {{ $.Name }}.{{ $f.Name }}_key is nil or empty") } b, err := {{ $receiver }}.blobOpeners.{{ $.Name }}(ctx, {{ $.Package }}.{{ $f.Constant }}) if err != nil { - return nil, fmt.Errorf("{{ $pkg }}: opening blob bucket for {{ $f.Name }}: %w", err) + return nil, err } - w, err := b.NewWriter(ctx, key) - if err != nil { - return nil, errors.Join(fmt.Errorf("{{ $pkg }}: creating writer for {{ $f.Name }}: %w", err), b.Close()) - } - return &blobWriteCloser{WriteCloser: w, bucket: b}, nil + return ent.BlobWriter(ctx, b, *{{ $receiver }}.{{ $f.Name }}_key) } - - {{- with $tmpls := matchTemplate (printf "blob/key/%s/%s" $.Name $f.Name) (printf "blob/key/%s" $.Name) }} - {{- xtemplate (index $tmpls 0) (extend $ "Field" $f) }} - {{- else }} - // {{ $f.StructField }}Key returns the blob storage key for the "{{ $f.Name }}" field. - func ({{ $receiver }} *{{ $.Name }}) {{ $f.StructField }}Key(context.Context) (string, error) { - return fmt.Sprintf("%s/%v/%s", "{{ $.Table }}", {{ $receiver }}.ID, "{{ $f.Name }}"), nil - } - {{- end }} {{- end }} {{- if not $.IsView }} diff --git a/entc/gen/template/meta.tmpl b/entc/gen/template/meta.tmpl index 4a58b78bf..f4d1f52c2 100644 --- a/entc/gen/template/meta.tmpl +++ b/entc/gen/template/meta.tmpl @@ -96,6 +96,12 @@ const ( {{ $name }} {{ printf "func (%s) error" $type }} {{- end }} {{- end }} + {{- range $f := $.BlobFields }} + {{- if $f.HasBlobKey }} + // {{ $f.BlobKeyName }} generates the blob storage key for the "{{ $f.Name }}" field. + {{ $f.BlobKeyName }} func(context.Context) (string, error) + {{- end }} + {{- end }} {{- if $.HasValueScanner }} // ValueScanner of all {{ $.Name }} fields. ValueScanner struct { diff --git a/entc/gen/template/runtime.tmpl b/entc/gen/template/runtime.tmpl index 87982abcc..cc80d0603 100644 --- a/entc/gen/template/runtime.tmpl +++ b/entc/gen/template/runtime.tmpl @@ -228,6 +228,20 @@ func init() { {{- end }} {{- end }} {{- end }} + {{- range $i, $f := $n.BlobFields }} + {{- if $f.HasBlobKey }} + {{- $bkName := print $pkg "." $f.BlobKeyName }} + {{- $desc := print $pkg "BlobDesc" $f.StructField }} + // {{ $desc }} is the schema descriptor for {{ $f.Name }} blob field. + {{- if $f.Position.MixedIn }} + {{ $desc }} := {{ print $pkg "MixinFields" $f.Position.MixinIndex }}[{{ $f.Position.Index }}].Descriptor() + {{- else }} + {{ $desc }} := {{ $pkg }}Fields[{{ $f.Position.Index }}].Descriptor() + {{- end }} + // {{ $bkName }} generates the blob storage key for the {{ $f.Name }} field. + {{ $bkName }} = {{ $desc }}.BlobKey + {{- end }} + {{- end }} {{- end }} {{- end }} } diff --git a/entc/gen/type.go b/entc/gen/type.go index 3c5ec4ab4..ee52ebbce 100644 --- a/entc/gen/type.go +++ b/entc/gen/type.go @@ -53,6 +53,8 @@ type ( // ForeignKeys are the foreign-keys that resides in the type table. ForeignKeys []*ForeignKey foreignKeys map[string]struct{} + // BlobKeys are the implicit key columns for blob-stored fields. + BlobKeys []*BlobKey // Annotations that were defined for the field in the schema. // The mapping is from the Annotation.Name() to a JSON decoded object. Annotations Annotations @@ -197,6 +199,14 @@ type ( // UserDefined bool } + // BlobKey holds the information for blob key columns. Similar to a foreign-key, + // it is an implicit string column that stores the storage key for a blob field. + BlobKey struct { + // Field is the implicit string column that stores the blob key. + Field *Field + // BlobField is the blob field this key belongs to. + BlobField *Field + } // Enum holds the enum information for schema enums in codegen. Enum struct { // Name is the Go name of the enum. @@ -760,6 +770,25 @@ func (t *Type) setupFKs() error { return nil } +// setupBlobKeys creates implicit key columns for all blob-stored fields. +func (t *Type) setupBlobKeys() { + for _, f := range t.Fields { + if !f.IsBlob() { + continue + } + t.BlobKeys = append(t.BlobKeys, &BlobKey{ + Field: &Field{ + typ: t, + Name: f.Name + "_key", + Type: &field.TypeInfo{Type: field.TypeString}, + Nillable: true, + Optional: f.Optional, + }, + BlobField: f, + }) + } +} + // setupFieldEdge check the field-edge validity and configures it and its foreign-key. func (t *Type) setupFieldEdge(fk *ForeignKey, fkOwner *Edge, fkName string) error { tf, ok := t.fields[fkName] @@ -1359,6 +1388,14 @@ func (f Field) IsBlob() bool { return f.Type != nil && f.Type.Type == field.TypeBlob } +// HasBlobKey reports whether this blob field has a user-defined key function. +func (f Field) HasBlobKey() bool { + return f.def != nil && f.def.BlobKey +} + +// BlobKeyName returns the variable name of the key generator for this blob field. +func (f Field) BlobKeyName() string { return "New" + pascal(f.Name) + "Key" } + // IsEdgeField reports if the given field is an edge-field (i.e. a foreign-key) // that was referenced by one of the edges. func (f Field) IsEdgeField() bool { return f.fk != nil } @@ -2210,6 +2247,11 @@ func (f ForeignKey) StructField() string { return f.Field.Name } +// StructField returns the struct field name of the blob key. +func (bk BlobKey) StructField() string { + return bk.Field.Name +} + // Rel is a relation type of an edge. type Rel int diff --git a/entc/integration/blob/blob.go b/entc/integration/blob/blob.go index 253a87c7d..130fb8ee2 100644 --- a/entc/integration/blob/blob.go +++ b/entc/integration/blob/blob.go @@ -70,16 +70,6 @@ func (b *GoBucket) NewWriter(ctx context.Context, key string) (io.WriteCloser, e return b.b.NewWriter(ctx, key, b.writerOpts) } -// Delete removes the blob stored at key. -// It returns nil if the key does not exist. -func (b *GoBucket) Delete(ctx context.Context, key string) error { - err := b.b.Delete(ctx, key) - if gcerrors.Code(err) == gcerrors.NotFound { - return nil - } - return err -} - // Close releases the underlying gocloud bucket resources. func (b *GoBucket) Close() error { return b.b.Close() diff --git a/entc/integration/blob/encrypt.go b/entc/integration/blob/encrypt.go index fee64e364..e085c4d87 100644 --- a/entc/integration/blob/encrypt.go +++ b/entc/integration/blob/encrypt.go @@ -119,11 +119,6 @@ func (e *Encrypted) NewWriter(ctx context.Context, key string) (io.WriteCloser, return cipher.StreamWriter{S: cipher.NewCTR(block, iv), W: wc}, nil } -// Delete removes the blob stored at key. -func (e *Encrypted) Delete(ctx context.Context, key string) error { - return e.inner.Delete(ctx, key) -} - // Close releases the underlying bucket resources. func (e *Encrypted) Close() error { return e.inner.Close() diff --git a/entc/integration/blob_test.go b/entc/integration/blob_test.go index 3a6013dee..29a7a077d 100644 --- a/entc/integration/blob_test.go +++ b/entc/integration/blob_test.go @@ -7,8 +7,6 @@ package integration import ( "bytes" "context" - "crypto/sha256" - "encoding/hex" "fmt" "io" "os" @@ -104,7 +102,7 @@ func TestBlobCreateAndRead(t *testing.T) { SaveX(ctx) // Read the blob back through the entity method. - got := blobContent(t, doc.Content, ctx) + got := blobContent(t, doc.ContentReader, ctx) require.Equal(t, data, got) } @@ -122,7 +120,7 @@ func TestBlobQueryAndRead(t *testing.T) { queried := client.Document.GetX(ctx, created.ID) // Read blob content from the queried entity. - got := blobContent(t, queried.Content, ctx) + got := blobContent(t, queried.ContentReader, ctx) require.Equal(t, data, got) } @@ -143,7 +141,7 @@ func TestBlobUpdateData(t *testing.T) { SaveX(ctx) // Read the new blob content. - got := blobContent(t, doc.Content, ctx) + got := blobContent(t, doc.ContentReader, ctx) require.Equal(t, v2, got) } @@ -180,7 +178,7 @@ func TestBlobMultipleDocuments(t *testing.T) { // Read each document and verify content is correct. for i, id := range ids { doc := client.Document.GetX(ctx, id) - got := blobContent(t, doc.Content, ctx) + got := blobContent(t, doc.ContentReader, ctx) require.Equal(t, contents[i], string(got), "document %d content mismatch", i) } } @@ -201,7 +199,7 @@ func TestBlobBulkCreate(t *testing.T) { // Verify each document's blob can be read back with correct data. for i, doc := range docs { - got := blobContent(t, doc.Content, ctx) + got := blobContent(t, doc.ContentReader, ctx) require.Equal(t, []byte(strings.Repeat("x", i+1)), got) } } @@ -216,7 +214,7 @@ func TestBlobThumbnailCreateAndRead(t *testing.T) { SetThumbnail(bytes.NewReader(thumbData)). SaveX(ctx) - got := blobContent(t, doc.Thumbnail, ctx) + got := blobContent(t, doc.ThumbnailReader, ctx) require.Equal(t, thumbData, got) } @@ -232,11 +230,11 @@ func TestBlobBothFields(t *testing.T) { SaveX(ctx) // Read content. - cGot := blobContent(t, doc.Content, ctx) + cGot := blobContent(t, doc.ContentReader, ctx) require.Equal(t, contentData, cGot) // Read thumbnail. - tGot := blobContent(t, doc.Thumbnail, ctx) + tGot := blobContent(t, doc.ThumbnailReader, ctx) require.Equal(t, thumbData, tGot) // Update only thumbnail, content should remain unchanged. @@ -245,10 +243,10 @@ func TestBlobBothFields(t *testing.T) { SetThumbnail(bytes.NewReader(newThumb)). SaveX(ctx) - cGot2 := blobContent(t, doc.Content, ctx) + cGot2 := blobContent(t, doc.ContentReader, ctx) require.Equal(t, contentData, cGot2) - tGot2 := blobContent(t, doc.Thumbnail, ctx) + tGot2 := blobContent(t, doc.ThumbnailReader, ctx) require.Equal(t, newThumb, tGot2) } @@ -267,10 +265,10 @@ func TestBlobBulkCreateBothFields(t *testing.T) { require.Len(t, docs, 3) for i, doc := range docs { - cGot := blobContent(t, doc.Content, ctx) + cGot := blobContent(t, doc.ContentReader, ctx) require.Equal(t, []byte(strings.Repeat("c", i+1)), cGot) - tGot := blobContent(t, doc.Thumbnail, ctx) + tGot := blobContent(t, doc.ThumbnailReader, ctx) require.Equal(t, []byte(strings.Repeat("t", i+1)), tGot) } } @@ -293,7 +291,7 @@ func TestBlobWriter(t *testing.T) { require.NoError(t, w.Close()) // Read back should reflect the overwritten data. - got := blobContent(t, doc.Content, ctx) + got := blobContent(t, doc.ContentReader, ctx) require.Equal(t, []byte("overwritten via writer"), got) } @@ -308,7 +306,7 @@ func TestBlobReader(t *testing.T) { SaveX(ctx) // Read back via Content (io.ReadCloser). - r, err := doc.Content(ctx) + r, err := doc.ContentReader(ctx) require.NoError(t, err) defer r.Close() @@ -337,30 +335,10 @@ func TestBlobWriterThenReader(t *testing.T) { require.NoError(t, w.Close()) // Read via reader. - got := blobContent(t, doc.Content, ctx) + got := blobContent(t, doc.ContentReader, ctx) require.Equal(t, []byte("hello world"), got) } -func TestBlobKey(t *testing.T) { - client, ctx, _ := setupBlob(t) - - doc := client.Document.Create(). - SetName("key-doc"). - SetContent(bytes.NewReader([]byte("content"))). - SetThumbnail(bytes.NewReader([]byte("thumb"))). - SaveX(ctx) - - // Content key follows the convention {table}/{id}/{field}. - contentKey, err := doc.ContentKey(ctx) - require.NoError(t, err) - require.Equal(t, fmt.Sprintf("documents/%d/content", doc.ID), contentKey) - // Thumbnail key uses a custom hash-based format (template override). - thumbnailKey, err := doc.ThumbnailKey(ctx) - require.NoError(t, err) - h := sha256.Sum256([]byte(fmt.Sprintf("documents/%d/thumbnail", doc.ID))) - require.Equal(t, hex.EncodeToString(h[:]), thumbnailKey) -} - func TestBlobWriterOptions(t *testing.T) { // Verify that a custom opener with WriterOptions works for roundtrip. dir := blobDir(t) @@ -382,7 +360,7 @@ func TestBlobWriterOptions(t *testing.T) { SetThumbnail(bytes.NewReader([]byte("thumb"))). SaveX(ctx) - got := blobContent(t, doc.Content, ctx) + got := blobContent(t, doc.ContentReader, ctx) require.Equal(t, data, got) } @@ -408,13 +386,13 @@ func TestBlobWriterOptionsApplied(t *testing.T) { SaveX(ctx) // Verify the data was written successfully and can be read back. - got := blobContent(t, doc.Content, ctx) + got := blobContent(t, doc.ContentReader, ctx) require.Equal(t, data, got) // Update also uses WriterOpts. v2 := []byte("updated with writer options") doc = doc.Update().SetContent(bytes.NewReader(v2)).SaveX(ctx) - got = blobContent(t, doc.Content, ctx) + got = blobContent(t, doc.ContentReader, ctx) require.Equal(t, v2, got) } @@ -459,17 +437,17 @@ func TestBlobEncryption(t *testing.T) { SaveX(acmeCtx) // Reading with the same tenant returns decrypted plaintext. - got := blobContent(t, doc.Content, acmeCtx) + got := blobContent(t, doc.ContentReader, acmeCtx) require.Equal(t, plaintext, got) - gotThumb := blobContent(t, doc.Thumbnail, acmeCtx) + gotThumb := blobContent(t, doc.ThumbnailReader, acmeCtx) require.Equal(t, []byte("secret-thumb"), gotThumb) // Verify the raw file on disk is NOT plaintext (it's encrypted). - contentKey, err := doc.ContentKey(acmeCtx) + matches, err := filepath.Glob(filepath.Join(dir, "documents", "*", "content")) require.NoError(t, err) - rawPath := filepath.Join(dir, "documents", contentKey) - raw, err := os.ReadFile(rawPath) + require.Len(t, matches, 1) + raw, err := os.ReadFile(matches[0]) require.NoError(t, err) require.NotEqual(t, plaintext, raw, "raw data on disk should be encrypted") // The raw data should be longer than plaintext (16-byte IV prepended). @@ -478,19 +456,19 @@ func TestBlobEncryption(t *testing.T) { // A different tenant ("evil") cannot decrypt acme's data — the derived key // differs, so AES-CTR produces garbage (not the original plaintext). evilCtx := blob.WithTenant(context.Background(), "evil") - evilData := blobContent(t, doc.Content, evilCtx) + evilData := blobContent(t, doc.ContentReader, evilCtx) require.NotEqual(t, plaintext, evilData, "different tenant must not read acme's plaintext") // No tenant in context → error. noTenantCtx := context.Background() - _, err = doc.Content(noTenantCtx) + _, err = doc.ContentReader(noTenantCtx) require.Error(t, err) require.Contains(t, err.Error(), "requires a tenant") // Update works through encryption too. v2 := []byte("updated secret content") doc = doc.Update().SetContent(bytes.NewReader(v2)).SaveX(acmeCtx) - got = blobContent(t, doc.Content, acmeCtx) + got = blobContent(t, doc.ContentReader, acmeCtx) require.Equal(t, v2, got) // ContentWriter also encrypts with tenant key. @@ -500,7 +478,7 @@ func TestBlobEncryption(t *testing.T) { require.NoError(t, err) require.NoError(t, w.Close()) - got = blobContent(t, doc.Content, acmeCtx) + got = blobContent(t, doc.ContentReader, acmeCtx) require.Equal(t, []byte("written via writer"), got) } @@ -529,7 +507,7 @@ func TestBlobPrefix(t *testing.T) { SaveX(ctx) // Read through the entity — uses the same opener. - got := blobContent(t, doc.Content, ctx) + got := blobContent(t, doc.ContentReader, ctx) require.Equal(t, data, got) // Create a second client with the default opener — reading should return nil (not found). @@ -538,7 +516,7 @@ func TestBlobPrefix(t *testing.T) { ) t.Cleanup(func() { defaultClient.Close() }) doc2 := defaultClient.Document.Query().OnlyX(ctx) - got = blobContent(t, doc2.Content, ctx) + got = blobContent(t, doc2.ContentReader, ctx) require.Nil(t, got, "expected nil when reading without prefix") } @@ -569,7 +547,7 @@ func TestBlobPrefixUpdate(t *testing.T) { updated := []byte("updated under prefix") doc = doc.Update().SetContent(bytes.NewReader(updated)).SaveX(ctx) - got := blobContent(t, doc.Content, ctx) + got := blobContent(t, doc.ContentReader, ctx) require.Equal(t, updated, got) } @@ -596,7 +574,7 @@ func TestBlobPrefixBulkCreate(t *testing.T) { ).SaveX(ctx) for i, doc := range docs { - got := blobContent(t, doc.Content, ctx) + got := blobContent(t, doc.ContentReader, ctx) require.Equal(t, []byte(fmt.Sprintf("b%d", i+1)), got) } } @@ -632,6 +610,6 @@ func TestBlobPrefixWriterReader(t *testing.T) { require.NoError(t, w.Close()) // Read via Content. - got := blobContent(t, doc.Content, ctx) + got := blobContent(t, doc.ContentReader, ctx) require.Equal(t, []byte("writer-prefixed"), got) } diff --git a/entc/integration/ent/client.go b/entc/integration/ent/client.go index 5d64f7c84..fd921a259 100644 --- a/entc/integration/ent/client.go +++ b/entc/integration/ent/client.go @@ -10,7 +10,6 @@ import ( "context" "errors" "fmt" - "io" "log" "reflect" @@ -181,24 +180,8 @@ func Driver(driver dialect.Driver) Option { } } -// Blob defines the interface for blob storage operations. -// Implementations should return [io/fs.ErrNotExist] (or an error wrapping it) -// from NewReader when the requested key does not exist. -// -// Blob writes are not transactional with the database. If a bulk operation -// fails partway through, already-written blobs will be cleaned up on a -// best-effort basis using the Delete method. -type Blob interface { - // NewReader opens a reader for the given key. - NewReader(ctx context.Context, key string) (io.ReadCloser, error) - // NewWriter opens a writer for the given key. - NewWriter(ctx context.Context, key string) (io.WriteCloser, error) - // Delete removes the blob stored at key. It should return nil - // if the key does not exist. - Delete(ctx context.Context, key string) error - // Close releases any resources held by the bucket. - Close() error -} +// Blob is an alias for the [ent.Blob] interface defined in the entgo.io/ent package. +type Blob = ent.Blob // BlobOpeners configures how blob buckets are opened for each entity type. // Each field is a function that opens a blob bucket for the given field name. @@ -213,26 +196,6 @@ func WithBlobOpeners(openers BlobOpeners) Option { } } -// blobReadCloser wraps an io.ReadCloser to also close the parent bucket on Close. -type blobReadCloser struct { - io.ReadCloser - bucket Blob -} - -func (r *blobReadCloser) Close() error { - return errors.Join(r.ReadCloser.Close(), r.bucket.Close()) -} - -// blobWriteCloser wraps an io.WriteCloser to also close the parent bucket on Close. -type blobWriteCloser struct { - io.WriteCloser - bucket Blob -} - -func (w *blobWriteCloser) Close() error { - return errors.Join(w.WriteCloser.Close(), w.bucket.Close()) -} - // Open opens a database/sql.DB specified by the driver name and // the data source name, and returns a new client attached to it. // Optional parameters can be added for configuring the client. diff --git a/entc/integration/ent/document.go b/entc/integration/ent/document.go index d84aae436..99aa64113 100644 --- a/entc/integration/ent/document.go +++ b/entc/integration/ent/document.go @@ -8,12 +8,8 @@ package ent import ( "context" - "crypto/sha256" - "encoding/hex" - "errors" "fmt" "io" - "io/fs" "strings" "entgo.io/ent" @@ -27,8 +23,10 @@ type Document struct { // ID of the ent. ID int `json:"id,omitempty"` // Name holds the value of the "name" field. - Name string `json:"name,omitempty"` - selectValues sql.SelectValues + Name string `json:"name,omitempty"` + content_key *string + thumbnail_key *string + selectValues sql.SelectValues } // scanValues returns the types for scanning values from sql.Rows. @@ -40,6 +38,10 @@ func (*Document) scanValues(columns []string) ([]any, error) { values[i] = new(sql.NullInt64) case document.FieldName: values[i] = new(sql.NullString) + case document.BlobKeys[0]: // content_key + values[i] = new(sql.NullString) + case document.BlobKeys[1]: // thumbnail_key + values[i] = new(sql.NullString) default: values[i] = new(sql.UnknownType) } @@ -67,6 +69,18 @@ func (_m *Document) assignValues(columns []string, values []any) error { } else if value.Valid { _m.Name = value.String } + case document.BlobKeys[0]: + if value, ok := values[i].(*sql.NullString); !ok { + return fmt.Errorf("unexpected type %T for field content_key", values[i]) + } else if value.Valid { + _m.content_key = &value.String + } + case document.BlobKeys[1]: + if value, ok := values[i].(*sql.NullString); !ok { + return fmt.Errorf("unexpected type %T for field thumbnail_key", values[i]) + } else if value.Valid { + _m.thumbnail_key = &value.String + } default: _m.selectValues.Set(columns[i], values[i]) } @@ -80,29 +94,17 @@ func (_m *Document) Value(name string) (ent.Value, error) { return _m.selectValues.Get(name) } -// Content opens a reader for the "content" field from blob storage. +// ContentReader opens a reader for the "content" field from blob storage. // The caller must close the returned reader when done. -// It returns nil, nil if the blob does not exist. -func (_m *Document) Content(ctx context.Context) (io.ReadCloser, error) { - key, err := _m.ContentKey(ctx) - if err != nil { - return nil, fmt.Errorf("ent: blob key for content: %w", err) +func (_m *Document) ContentReader(ctx context.Context) (io.ReadCloser, error) { + if _m.content_key == nil || *_m.content_key == "" { + return nil, fmt.Errorf("ent: Document.content_key is nil or empty") } b, err := _m.blobOpeners.Document(ctx, document.FieldContent) if err != nil { - return nil, fmt.Errorf("ent: opening blob bucket for content: %w", err) - } - switch r, err := b.NewReader(ctx, key); { - case errors.Is(err, fs.ErrNotExist): - if err := b.Close(); err != nil { - return nil, fmt.Errorf("ent: closing blob bucket for content: %w", err) - } - return nil, nil - case err != nil: - return nil, errors.Join(fmt.Errorf("ent: creating reader for content: %w", err), b.Close()) - default: - return &blobReadCloser{ReadCloser: r, bucket: b}, nil + return nil, err } + return ent.BlobReader(ctx, b, *_m.content_key) } // ContentWriter opens a writer for the "content" field in blob storage. @@ -110,49 +112,27 @@ func (_m *Document) Content(ctx context.Context) (io.ReadCloser, error) { // also releases the underlying bucket resources. // Writing via this method does not go through the mutation pipeline. func (_m *Document) ContentWriter(ctx context.Context) (io.WriteCloser, error) { - key, err := _m.ContentKey(ctx) - if err != nil { - return nil, fmt.Errorf("ent: blob key for content: %w", err) + if _m.content_key == nil || *_m.content_key == "" { + return nil, fmt.Errorf("ent: Document.content_key is nil or empty") } b, err := _m.blobOpeners.Document(ctx, document.FieldContent) if err != nil { - return nil, fmt.Errorf("ent: opening blob bucket for content: %w", err) + return nil, err } - w, err := b.NewWriter(ctx, key) - if err != nil { - return nil, errors.Join(fmt.Errorf("ent: creating writer for content: %w", err), b.Close()) - } - return &blobWriteCloser{WriteCloser: w, bucket: b}, nil + return ent.BlobWriter(ctx, b, *_m.content_key) } -// ContentKey returns the blob storage key for the "content" field. -func (_m *Document) ContentKey(context.Context) (string, error) { - return fmt.Sprintf("%s/%v/%s", "documents", _m.ID, "content"), nil -} - -// Thumbnail opens a reader for the "thumbnail" field from blob storage. +// ThumbnailReader opens a reader for the "thumbnail" field from blob storage. // The caller must close the returned reader when done. -// It returns nil, nil if the blob does not exist. -func (_m *Document) Thumbnail(ctx context.Context) (io.ReadCloser, error) { - key, err := _m.ThumbnailKey(ctx) - if err != nil { - return nil, fmt.Errorf("ent: blob key for thumbnail: %w", err) +func (_m *Document) ThumbnailReader(ctx context.Context) (io.ReadCloser, error) { + if _m.thumbnail_key == nil || *_m.thumbnail_key == "" { + return nil, fmt.Errorf("ent: Document.thumbnail_key is nil or empty") } b, err := _m.blobOpeners.Document(ctx, document.FieldThumbnail) if err != nil { - return nil, fmt.Errorf("ent: opening blob bucket for thumbnail: %w", err) - } - switch r, err := b.NewReader(ctx, key); { - case errors.Is(err, fs.ErrNotExist): - if err := b.Close(); err != nil { - return nil, fmt.Errorf("ent: closing blob bucket for thumbnail: %w", err) - } - return nil, nil - case err != nil: - return nil, errors.Join(fmt.Errorf("ent: creating reader for thumbnail: %w", err), b.Close()) - default: - return &blobReadCloser{ReadCloser: r, bucket: b}, nil + return nil, err } + return ent.BlobReader(ctx, b, *_m.thumbnail_key) } // ThumbnailWriter opens a writer for the "thumbnail" field in blob storage. @@ -160,25 +140,14 @@ func (_m *Document) Thumbnail(ctx context.Context) (io.ReadCloser, error) { // also releases the underlying bucket resources. // Writing via this method does not go through the mutation pipeline. func (_m *Document) ThumbnailWriter(ctx context.Context) (io.WriteCloser, error) { - key, err := _m.ThumbnailKey(ctx) - if err != nil { - return nil, fmt.Errorf("ent: blob key for thumbnail: %w", err) + if _m.thumbnail_key == nil || *_m.thumbnail_key == "" { + return nil, fmt.Errorf("ent: Document.thumbnail_key is nil or empty") } b, err := _m.blobOpeners.Document(ctx, document.FieldThumbnail) if err != nil { - return nil, fmt.Errorf("ent: opening blob bucket for thumbnail: %w", err) + return nil, err } - w, err := b.NewWriter(ctx, key) - if err != nil { - return nil, errors.Join(fmt.Errorf("ent: creating writer for thumbnail: %w", err), b.Close()) - } - return &blobWriteCloser{WriteCloser: w, bucket: b}, nil -} - -// ThumbnailKey returns a hash-based blob storage key for the "thumbnail" field. -func (_m *Document) ThumbnailKey(context.Context) (string, error) { - h := sha256.Sum256(fmt.Appendf(nil, "%s/%v/%s", "documents", _m.ID, "thumbnail")) - return hex.EncodeToString(h[:]), nil + return ent.BlobWriter(ctx, b, *_m.thumbnail_key) } // Update returns a builder for updating this Document. diff --git a/entc/integration/ent/document/document.go b/entc/integration/ent/document/document.go index bb5d803bc..13c1c3993 100644 --- a/entc/integration/ent/document/document.go +++ b/entc/integration/ent/document/document.go @@ -7,6 +7,8 @@ package document import ( + "context" + "entgo.io/ent/dialect/sql" ) @@ -29,8 +31,20 @@ const ( var Columns = []string{ FieldID, FieldName, + "content_key", + "thumbnail_key", } +// BlobKeys holds the SQL columns for blob storage keys. +var BlobKeys = []string{ + "content_key", + "thumbnail_key", +} + +// NewContentKey generates the blob storage key for the content field. +// It is set by the runtime/init package from the schema descriptor. +var NewContentKey func(context.Context) (string, error) + // ValidColumn reports if the column name is valid (part of the table columns). func ValidColumn(column string) bool { for i := range Columns { @@ -38,6 +52,11 @@ func ValidColumn(column string) bool { return true } } + for i := range BlobKeys { + if column == BlobKeys[i] { + return true + } + } return false } diff --git a/entc/integration/ent/document_create.go b/entc/integration/ent/document_create.go index 8e6ed8328..210ca1456 100644 --- a/entc/integration/ent/document_create.go +++ b/entc/integration/ent/document_create.go @@ -12,10 +12,12 @@ import ( "fmt" "io" + "entgo.io/ent" "entgo.io/ent/dialect/sql" "entgo.io/ent/dialect/sql/sqlgraph" "entgo.io/ent/entc/integration/ent/document" "entgo.io/ent/schema/field" + "github.com/google/uuid" ) // DocumentCreate is the builder for creating a Document entity. @@ -95,74 +97,38 @@ func (_c *DocumentCreate) sqlSave(ctx context.Context) (*Document, error) { return nil, err } _node, _spec := _c.createSpec() + _blobs := ent.NewBlobBulkWriter(_c.mutation.blobOpeners.Document) + if r, ok := _c.mutation.Content(); ok { + key, err := document.NewContentKey(ctx) + if err != nil { + return nil, errors.Join(fmt.Errorf("ent: generating blob key for content: %w", err), _blobs.Close()) + } + if err := _blobs.Write(ctx, document.FieldContent, key, r); err != nil { + return nil, errors.Join(fmt.Errorf("ent: writing blob for content: %w", err), _blobs.Close()) + } + _node.content_key = &key + _spec.SetField("content_key", field.TypeString, key) + } + if r, ok := _c.mutation.Thumbnail(); ok { + key := uuid.NewString() + if err := _blobs.Write(ctx, document.FieldThumbnail, key, r); err != nil { + return nil, errors.Join(fmt.Errorf("ent: writing blob for thumbnail: %w", err), _blobs.Close()) + } + _node.thumbnail_key = &key + _spec.SetField("thumbnail_key", field.TypeString, key) + } if err := sqlgraph.CreateNode(ctx, _c.driver, _spec); err != nil { if sqlgraph.IsConstraintError(err) { err = &ConstraintError{msg: err.Error(), wrap: err} } - return nil, err + return nil, errors.Join(err, _blobs.Close()) } id := _spec.ID.Value.(int64) _node.ID = int(id) _c.mutation.id = &_node.ID _c.mutation.done = true - type blobWritten struct { - field string - key string - } - var _blobWritten []blobWritten - _blobCleanup := func(ctx context.Context) error { - var errs []error - for _, bw := range _blobWritten { - b, err := _c.mutation.blobOpeners.Document(ctx, bw.field) - if err != nil { - errs = append(errs, err) - continue - } - errs = append(errs, b.Delete(ctx, bw.key), b.Close()) - } - return errors.Join(errs...) - } - if r, ok := _c.mutation.Content(); ok { - b, err := _c.mutation.blobOpeners.Document(ctx, document.FieldContent) - if err != nil { - return nil, errors.Join(fmt.Errorf("ent: opening blob bucket for content: %w", err), _blobCleanup(ctx)) - } - key, err := _node.ContentKey(ctx) - if err != nil { - return nil, errors.Join(fmt.Errorf("ent: blob key for content: %w", err), b.Close(), _blobCleanup(ctx)) - } - w, err := b.NewWriter(ctx, key) - if err != nil { - return nil, errors.Join(fmt.Errorf("ent: creating writer for content: %w", err), b.Close(), _blobCleanup(ctx)) - } - if _, err := io.Copy(w, r); err != nil { - return nil, errors.Join(fmt.Errorf("ent: writing blob for content: %w", err), w.Close(), b.Close(), _blobCleanup(ctx)) - } - if err := errors.Join(w.Close(), b.Close()); err != nil { - return nil, errors.Join(fmt.Errorf("ent: closing blob for content: %w", err), _blobCleanup(ctx)) - } - _blobWritten = append(_blobWritten, blobWritten{field: document.FieldContent, key: key}) - } - if r, ok := _c.mutation.Thumbnail(); ok { - b, err := _c.mutation.blobOpeners.Document(ctx, document.FieldThumbnail) - if err != nil { - return nil, errors.Join(fmt.Errorf("ent: opening blob bucket for thumbnail: %w", err), _blobCleanup(ctx)) - } - key, err := _node.ThumbnailKey(ctx) - if err != nil { - return nil, errors.Join(fmt.Errorf("ent: blob key for thumbnail: %w", err), b.Close(), _blobCleanup(ctx)) - } - w, err := b.NewWriter(ctx, key) - if err != nil { - return nil, errors.Join(fmt.Errorf("ent: creating writer for thumbnail: %w", err), b.Close(), _blobCleanup(ctx)) - } - if _, err := io.Copy(w, r); err != nil { - return nil, errors.Join(fmt.Errorf("ent: writing blob for thumbnail: %w", err), w.Close(), b.Close(), _blobCleanup(ctx)) - } - if err := errors.Join(w.Close(), b.Close()); err != nil { - return nil, errors.Join(fmt.Errorf("ent: closing blob for thumbnail: %w", err), _blobCleanup(ctx)) - } - _blobWritten = append(_blobWritten, blobWritten{field: document.FieldThumbnail, key: key}) + if err := _blobs.Close(); err != nil { + return nil, err } return _node, nil } @@ -344,35 +310,7 @@ func (_c *DocumentCreateBulk) Save(ctx context.Context) ([]*Document, error) { specs := make([]*sqlgraph.CreateSpec, len(_c.builders)) nodes := make([]*Document, len(_c.builders)) mutators := make([]Mutator, len(_c.builders)) - var _blobContent Blob - var _blobKeysContent []string - var _blobThumbnail Blob - var _blobKeysThumbnail []string - closeBlobs := func() error { - var errs []error - if _blobContent != nil { - errs = append(errs, _blobContent.Close()) - } - if _blobThumbnail != nil { - errs = append(errs, _blobThumbnail.Close()) - } - return errors.Join(errs...) - } - cleanupBlobs := func(ctx context.Context) error { - var errs []error - if _blobContent != nil { - for _, key := range _blobKeysContent { - errs = append(errs, _blobContent.Delete(ctx, key)) - } - } - if _blobThumbnail != nil { - for _, key := range _blobKeysThumbnail { - errs = append(errs, _blobThumbnail.Delete(ctx, key)) - } - } - errs = append(errs, closeBlobs()) - return errors.Join(errs...) - } + _blobs := ent.NewBlobBulkWriter(_c.builders[0].mutation.blobOpeners.Document) for i := range _c.builders { func(i int, root context.Context) { builder := _c.builders[i] @@ -387,6 +325,25 @@ func (_c *DocumentCreateBulk) Save(ctx context.Context) ([]*Document, error) { builder.mutation = mutation var err error nodes[i], specs[i] = builder.createSpec() + if r, ok := mutation.Content(); ok { + key, err := document.NewContentKey(ctx) + if err != nil { + return nil, fmt.Errorf("ent: generating blob key for content: %w", err) + } + if err := _blobs.Write(ctx, document.FieldContent, key, r); err != nil { + return nil, fmt.Errorf("ent: writing blob for content: %w", err) + } + nodes[i].content_key = &key + specs[i].SetField("content_key", field.TypeString, key) + } + if r, ok := mutation.Thumbnail(); ok { + key := uuid.NewString() + if err := _blobs.Write(ctx, document.FieldThumbnail, key, r); err != nil { + return nil, fmt.Errorf("ent: writing blob for thumbnail: %w", err) + } + nodes[i].thumbnail_key = &key + specs[i].SetField("thumbnail_key", field.TypeString, key) + } if i < len(mutators)-1 { _, err = mutators[i+1].Mutate(root, _c.builders[i+1].mutation) } else { @@ -408,50 +365,6 @@ func (_c *DocumentCreateBulk) Save(ctx context.Context) ([]*Document, error) { nodes[i].ID = int(id) } mutation.done = true - if r, ok := mutation.Content(); ok { - if _blobContent == nil { - if _blobContent, err = mutation.blobOpeners.Document(ctx, document.FieldContent); err != nil { - return nil, fmt.Errorf("ent: opening blob bucket for content: %w", err) - } - } - key, err := nodes[i].ContentKey(ctx) - if err != nil { - return nil, fmt.Errorf("ent: blob key for content: %w", err) - } - w, err := _blobContent.NewWriter(ctx, key) - if err != nil { - return nil, fmt.Errorf("ent: creating writer for content: %w", err) - } - if _, err := io.Copy(w, r); err != nil { - return nil, errors.Join(fmt.Errorf("ent: writing blob for content: %w", err), w.Close()) - } - if err := w.Close(); err != nil { - return nil, fmt.Errorf("ent: closing writer for content: %w", err) - } - _blobKeysContent = append(_blobKeysContent, key) - } - if r, ok := mutation.Thumbnail(); ok { - if _blobThumbnail == nil { - if _blobThumbnail, err = mutation.blobOpeners.Document(ctx, document.FieldThumbnail); err != nil { - return nil, fmt.Errorf("ent: opening blob bucket for thumbnail: %w", err) - } - } - key, err := nodes[i].ThumbnailKey(ctx) - if err != nil { - return nil, fmt.Errorf("ent: blob key for thumbnail: %w", err) - } - w, err := _blobThumbnail.NewWriter(ctx, key) - if err != nil { - return nil, fmt.Errorf("ent: creating writer for thumbnail: %w", err) - } - if _, err := io.Copy(w, r); err != nil { - return nil, errors.Join(fmt.Errorf("ent: writing blob for thumbnail: %w", err), w.Close()) - } - if err := w.Close(); err != nil { - return nil, fmt.Errorf("ent: closing writer for thumbnail: %w", err) - } - _blobKeysThumbnail = append(_blobKeysThumbnail, key) - } return nodes[i], nil }) for i := len(builder.hooks) - 1; i >= 0; i-- { @@ -462,10 +375,10 @@ func (_c *DocumentCreateBulk) Save(ctx context.Context) ([]*Document, error) { } if len(mutators) > 0 { if _, err := mutators[0].Mutate(ctx, _c.builders[0].mutation); err != nil { - return nil, errors.Join(err, cleanupBlobs(ctx)) + return nil, errors.Join(err, _blobs.Close()) } } - return nodes, closeBlobs() + return nodes, _blobs.Close() } // SaveX is like Save, but panics if an error occurs. diff --git a/entc/integration/ent/document_update.go b/entc/integration/ent/document_update.go index 082ae048d..09989adc4 100644 --- a/entc/integration/ent/document_update.go +++ b/entc/integration/ent/document_update.go @@ -12,11 +12,13 @@ import ( "fmt" "io" + "entgo.io/ent" "entgo.io/ent/dialect/sql" "entgo.io/ent/dialect/sql/sqlgraph" "entgo.io/ent/entc/integration/ent/document" "entgo.io/ent/entc/integration/ent/predicate" "entgo.io/ent/schema/field" + "github.com/google/uuid" ) // DocumentUpdate is the builder for updating Document entities. @@ -238,6 +240,24 @@ func (_u *DocumentUpdateOne) sqlSave(ctx context.Context) (_node *Document, err _spec.SetField(document.FieldName, field.TypeString, value) } _spec.AddModifiers(_u.modifiers...) + _blobs := ent.NewBlobBulkWriter(_u.mutation.blobOpeners.Document) + if r, ok := _u.mutation.Content(); ok { + key, err := document.NewContentKey(ctx) + if err != nil { + return nil, errors.Join(fmt.Errorf("ent: generating blob key for content: %w", err), _blobs.Close()) + } + if err := _blobs.Write(ctx, document.FieldContent, key, r); err != nil { + return nil, errors.Join(fmt.Errorf("ent: writing blob for content: %w", err), _blobs.Close()) + } + _spec.SetField("content_key", field.TypeString, key) + } + if r, ok := _u.mutation.Thumbnail(); ok { + key := uuid.NewString() + if err := _blobs.Write(ctx, document.FieldThumbnail, key, r); err != nil { + return nil, errors.Join(fmt.Errorf("ent: writing blob for thumbnail: %w", err), _blobs.Close()) + } + _spec.SetField("thumbnail_key", field.TypeString, key) + } _node = &Document{config: _u.config} _spec.Assign = _node.assignValues _spec.ScanValues = _node.scanValues @@ -247,67 +267,11 @@ func (_u *DocumentUpdateOne) sqlSave(ctx context.Context) (_node *Document, err } else if sqlgraph.IsConstraintError(err) { err = &ConstraintError{msg: err.Error(), wrap: err} } - return nil, err + return nil, errors.Join(err, _blobs.Close()) } _u.mutation.done = true - type blobWritten struct { - field string - key string - } - var _blobWritten []blobWritten - _blobCleanup := func(ctx context.Context) error { - var errs []error - for _, bw := range _blobWritten { - b, err := _u.mutation.blobOpeners.Document(ctx, bw.field) - if err != nil { - errs = append(errs, err) - continue - } - errs = append(errs, b.Delete(ctx, bw.key), b.Close()) - } - return errors.Join(errs...) - } - if r, ok := _u.mutation.Content(); ok { - b, err := _u.mutation.blobOpeners.Document(ctx, document.FieldContent) - if err != nil { - return nil, errors.Join(fmt.Errorf("ent: opening blob bucket for content: %w", err), _blobCleanup(ctx)) - } - key, err := _node.ContentKey(ctx) - if err != nil { - return nil, errors.Join(fmt.Errorf("ent: blob key for content: %w", err), b.Close(), _blobCleanup(ctx)) - } - w, err := b.NewWriter(ctx, key) - if err != nil { - return nil, errors.Join(fmt.Errorf("ent: creating writer for content: %w", err), b.Close(), _blobCleanup(ctx)) - } - if _, err := io.Copy(w, r); err != nil { - return nil, errors.Join(fmt.Errorf("ent: writing blob for content: %w", err), w.Close(), b.Close(), _blobCleanup(ctx)) - } - if err := errors.Join(w.Close(), b.Close()); err != nil { - return nil, errors.Join(fmt.Errorf("ent: closing blob for content: %w", err), _blobCleanup(ctx)) - } - _blobWritten = append(_blobWritten, blobWritten{field: document.FieldContent, key: key}) - } - if r, ok := _u.mutation.Thumbnail(); ok { - b, err := _u.mutation.blobOpeners.Document(ctx, document.FieldThumbnail) - if err != nil { - return nil, errors.Join(fmt.Errorf("ent: opening blob bucket for thumbnail: %w", err), _blobCleanup(ctx)) - } - key, err := _node.ThumbnailKey(ctx) - if err != nil { - return nil, errors.Join(fmt.Errorf("ent: blob key for thumbnail: %w", err), b.Close(), _blobCleanup(ctx)) - } - w, err := b.NewWriter(ctx, key) - if err != nil { - return nil, errors.Join(fmt.Errorf("ent: creating writer for thumbnail: %w", err), b.Close(), _blobCleanup(ctx)) - } - if _, err := io.Copy(w, r); err != nil { - return nil, errors.Join(fmt.Errorf("ent: writing blob for thumbnail: %w", err), w.Close(), b.Close(), _blobCleanup(ctx)) - } - if err := errors.Join(w.Close(), b.Close()); err != nil { - return nil, errors.Join(fmt.Errorf("ent: closing blob for thumbnail: %w", err), _blobCleanup(ctx)) - } - _blobWritten = append(_blobWritten, blobWritten{field: document.FieldThumbnail, key: key}) + if err := _blobs.Close(); err != nil { + return nil, err } return _node, nil } diff --git a/entc/integration/ent/migrate/schema.go b/entc/integration/ent/migrate/schema.go index afed48afe..984346793 100644 --- a/entc/integration/ent/migrate/schema.go +++ b/entc/integration/ent/migrate/schema.go @@ -94,6 +94,8 @@ var ( DocumentsColumns = []*schema.Column{ {Name: "id", Type: field.TypeInt, Increment: true}, {Name: "name", Type: field.TypeString}, + {Name: "content_key", Type: field.TypeString}, + {Name: "thumbnail_key", Type: field.TypeString}, } // DocumentsTable holds the schema information for the "documents" table. DocumentsTable = &schema.Table{ diff --git a/entc/integration/ent/runtime.go b/entc/integration/ent/runtime.go index 613595ade..5acbef3d3 100644 --- a/entc/integration/ent/runtime.go +++ b/entc/integration/ent/runtime.go @@ -15,6 +15,7 @@ import ( "entgo.io/ent/dialect/sql" "entgo.io/ent/entc/integration/ent/card" + "entgo.io/ent/entc/integration/ent/document" "entgo.io/ent/entc/integration/ent/exvaluescan" "entgo.io/ent/entc/integration/ent/fieldtype" "entgo.io/ent/entc/integration/ent/file" @@ -65,6 +66,10 @@ func init() { card.NameValidator = cardDescName.Validators[0].(func(string) error) documentFields := schema.Document{}.Fields() _ = documentFields + // documentBlobDescContent is the schema descriptor for content blob field. + documentBlobDescContent := documentFields[1].Descriptor() + // document.NewContentKey generates the blob storage key for the content field. + document.NewContentKey = documentBlobDescContent.BlobKey exvaluescanFields := schema.ExValueScan{}.Fields() _ = exvaluescanFields // exvaluescanDescBinary is the schema descriptor for binary field. diff --git a/entc/integration/ent/schema/document.go b/entc/integration/ent/schema/document.go index 8cf5026e7..1ad7f2714 100644 --- a/entc/integration/ent/schema/document.go +++ b/entc/integration/ent/schema/document.go @@ -5,8 +5,13 @@ package schema import ( + "context" + "fmt" + "entgo.io/ent" "entgo.io/ent/schema/field" + + "github.com/google/uuid" ) // Document holds the schema definition for the Document entity. @@ -18,7 +23,9 @@ type Document struct { func (Document) Fields() []ent.Field { return []ent.Field{ field.String("name"), - field.Blob("content"), + field.Blob("content").Key(func(_ context.Context) (string, error) { + return fmt.Sprintf("documents/%s/content", uuid.NewString()), nil + }), field.Blob("thumbnail"), } } diff --git a/entc/integration/ent/template/blob_key.tmpl b/entc/integration/ent/template/blob_key.tmpl deleted file mode 100644 index 8a428b062..000000000 --- a/entc/integration/ent/template/blob_key.tmpl +++ /dev/null @@ -1,9 +0,0 @@ -{{ define "blob/key/Document/thumbnail" }} - {{ $receiver := $.Receiver }} - {{ $f := $.Scope.Field }} - // {{ $f.StructField }}Key returns a hash-based blob storage key for the "{{ $f.Name }}" field. - func ({{ $receiver }} *{{ $.Name }}) {{ $f.StructField }}Key(context.Context) (string, error) { - h := sha256.Sum256(fmt.Appendf(nil, "%s/%v/%s", "{{ $.Table }}", {{ $receiver }}.ID, "{{ $f.Name }}")) - return hex.EncodeToString(h[:]), nil - } -{{ end }} diff --git a/entc/integration/gremlin/ent/client.go b/entc/integration/gremlin/ent/client.go index 846da8259..ecad54dbb 100644 --- a/entc/integration/gremlin/ent/client.go +++ b/entc/integration/gremlin/ent/client.go @@ -15,11 +15,11 @@ import ( "net/url" "reflect" - "entgo.io/ent" "entgo.io/ent/dialect" "entgo.io/ent/dialect/gremlin" "entgo.io/ent/dialect/gremlin/graph/dsl" "entgo.io/ent/dialect/gremlin/graph/dsl/g" + "entgo.io/ent/entc/integration/ent" "entgo.io/ent/entc/integration/gremlin/ent/api" "entgo.io/ent/entc/integration/gremlin/ent/builder" "entgo.io/ent/entc/integration/gremlin/ent/card" @@ -176,24 +176,8 @@ func Driver(driver dialect.Driver) Option { } } -// Blob defines the interface for blob storage operations. -// Implementations should return [io/fs.ErrNotExist] (or an error wrapping it) -// from NewReader when the requested key does not exist. -// -// Blob writes are not transactional with the database. If a bulk operation -// fails partway through, already-written blobs will be cleaned up on a -// best-effort basis using the Delete method. -type Blob interface { - // NewReader opens a reader for the given key. - NewReader(ctx context.Context, key string) (io.ReadCloser, error) - // NewWriter opens a writer for the given key. - NewWriter(ctx context.Context, key string) (io.WriteCloser, error) - // Delete removes the blob stored at key. It should return nil - // if the key does not exist. - Delete(ctx context.Context, key string) error - // Close releases any resources held by the bucket. - Close() error -} +// Blob is an alias for the [ent.Blob] interface defined in the entgo.io/ent package. +type Blob = ent.Blob // BlobOpeners configures how blob buckets are opened for each entity type. // Each field is a function that opens a blob bucket for the given field name. @@ -208,7 +192,7 @@ func WithBlobOpeners(openers BlobOpeners) Option { } } -// blobReadCloser wraps an io.ReadCloser to also close the parent bucket on Close. +// blobReadCloser wraps an io.ReadCloser to also close additional resources on Close. type blobReadCloser struct { io.ReadCloser bucket Blob @@ -218,7 +202,7 @@ func (r *blobReadCloser) Close() error { return errors.Join(r.ReadCloser.Close(), r.bucket.Close()) } -// blobWriteCloser wraps an io.WriteCloser to also close the parent bucket on Close. +// blobWriteCloser wraps an io.WriteCloser to also close additional resources on Close. type blobWriteCloser struct { io.WriteCloser bucket Blob diff --git a/entc/integration/privacy/ent/client.go b/entc/integration/privacy/ent/client.go index 3fd1f7512..6e500264a 100644 --- a/entc/integration/privacy/ent/client.go +++ b/entc/integration/privacy/ent/client.go @@ -16,8 +16,6 @@ import ( "entgo.io/ent" "entgo.io/ent/entc/integration/privacy/ent/migrate" - "net/http" - "entgo.io/ent/dialect" "entgo.io/ent/dialect/sql" "entgo.io/ent/dialect/sql/sqlgraph" @@ -65,8 +63,7 @@ type ( // hooks to execute on mutations. hooks *hooks // interceptors to execute on queries. - inters *inters - HTTPClient *http.Client + inters *inters } // Option function to configure the client. Option func(*config) @@ -110,13 +107,6 @@ func Driver(driver dialect.Driver) Option { } } -// HTTPClient configures the HTTPClient. -func HTTPClient(v *http.Client) Option { - return func(c *config) { - c.HTTPClient = v - } -} - // Open opens a database/sql.DB specified by the driver name and // the data source name, and returns a new client attached to it. // Optional parameters can be added for configuring the client. diff --git a/entc/integration/privacy/ent/entc.go b/entc/integration/privacy/ent/entc.go index 57ceb572f..a3892ff13 100644 --- a/entc/integration/privacy/ent/entc.go +++ b/entc/integration/privacy/ent/entc.go @@ -41,3 +41,5 @@ func main() { log.Fatalf("running ent codegen: %v", err) } } + +<<<<<<< \ No newline at end of file diff --git a/entc/load/schema.go b/entc/load/schema.go index e11a33a1a..e596ec9eb 100644 --- a/entc/load/schema.go +++ b/entc/load/schema.go @@ -63,6 +63,7 @@ type Field struct { Comment string `json:"comment,omitempty"` Deprecated bool `json:"deprecated,omitempty"` DeprecatedReason string `json:"deprecated_reason,omitempty"` + BlobKey bool `json:"blob_key,omitempty"` } // Edge represents an ent.Edge that was loaded from a complied user package. @@ -144,6 +145,7 @@ func NewField(fd *field.Descriptor) (*Field, error) { Comment: fd.Comment, Deprecated: fd.Deprecated, DeprecatedReason: fd.DeprecatedReason, + BlobKey: fd.BlobKey != nil, } for _, at := range fd.Annotations { sf.addAnnotation(at) diff --git a/schema/field/field.go b/schema/field/field.go index 3e52faef1..be759a478 100644 --- a/schema/field/field.go +++ b/schema/field/field.go @@ -5,6 +5,7 @@ package field import ( + "context" "database/sql" "database/sql/driver" "encoding" @@ -1485,6 +1486,18 @@ func (b *blobBuilder) Deprecated(reason ...string) *blobBuilder { return b } +// Key sets the function used to generate the blob storage key. +// The function is called at create/update time to produce a unique key +// for storing the blob data. If not set, a random key is generated. +// +// field.Blob("content").Key(func(ctx context.Context) (string, error) { +// return uuid.NewString(), nil +// }) +func (b *blobBuilder) Key(fn func(context.Context) (string, error)) *blobBuilder { + b.desc.BlobKey = fn + return b +} + // Descriptor implements the ent.Field interface by returning its descriptor. func (b *blobBuilder) Descriptor() *Descriptor { return b.desc @@ -1492,26 +1505,27 @@ func (b *blobBuilder) Descriptor() *Descriptor { // A Descriptor for field configuration. type Descriptor struct { - Tag string // struct tag. - Size int // varchar size. - Name string // field name. - Info *TypeInfo // field type info. - ValueScanner any // custom field codec. - Unique bool // unique index of field. - Nillable bool // nillable struct field. - Optional bool // nullable field in database. - Immutable bool // create only field. - Default any // default value on create. - UpdateDefault any // default value on update. - Validators []any // validator functions. - StorageKey string // sql column or gremlin property. - Enums []struct{ N, V string } // enum values. - Sensitive bool // sensitive info string field. - SchemaType map[string]string // override the schema type. - Annotations []schema.Annotation // field annotations. - Comment string // field comment. - Deprecated bool // mark the field as deprecated. - DeprecatedReason string // deprecation reason. + Tag string // struct tag. + Size int // varchar size. + Name string // field name. + Info *TypeInfo // field type info. + ValueScanner any // custom field codec. + Unique bool // unique index of field. + Nillable bool // nillable struct field. + Optional bool // nullable field in database. + Immutable bool // create only field. + Default any // default value on create. + UpdateDefault any // default value on update. + Validators []any // validator functions. + StorageKey string // sql column or gremlin property. + Enums []struct{ N, V string } // enum values. + Sensitive bool // sensitive info string field. + SchemaType map[string]string // override the schema type. + Annotations []schema.Annotation // field annotations. + Comment string // field comment. + Deprecated bool // mark the field as deprecated. + DeprecatedReason string // deprecation reason. + BlobKey func(context.Context) (string, error) // blob key generation function: func(context.Context) (string, error). Err error }