From 9a069a3ca3bd0c11842836c1467bfe415d47ee42 Mon Sep 17 00:00:00 2001 From: blockchainluffy Date: Tue, 25 Mar 2025 16:53:47 +0530 Subject: [PATCH 1/9] initial commit --- .../gnosis/database/sql/schemas/gnosiskeyper.sql | 8 ++++++-- rolling-shutter/medley/db/metadb.go | 16 ++++++++++++++++ rolling-shutter/medley/db/sqlc.go | 11 ++++++++++- 3 files changed, 32 insertions(+), 3 deletions(-) diff --git a/rolling-shutter/keyperimpl/gnosis/database/sql/schemas/gnosiskeyper.sql b/rolling-shutter/keyperimpl/gnosis/database/sql/schemas/gnosiskeyper.sql index e0928c5b6..df4a270ec 100644 --- a/rolling-shutter/keyperimpl/gnosis/database/sql/schemas/gnosiskeyper.sql +++ b/rolling-shutter/keyperimpl/gnosis/database/sql/schemas/gnosiskeyper.sql @@ -1,4 +1,4 @@ --- schema-version: gnosiskeyper-1 -- +-- schema-version: gnosiskeyper-2 -- -- Please change the version above if you make incompatible changes to -- the schema. We'll use this to check we're using the right schema. @@ -66,4 +66,8 @@ CREATE TABLE validator_registrations_synced_until( enforce_one_row bool PRIMARY KEY DEFAULT true, block_hash bytea NOT NULL, block_number bigint NOT NULL CHECK (block_number >= 0) -); \ No newline at end of file +); + +-- ALTER TABLE validator_registrations +-- DROP CONSTRAINT validator_registrations_pkey CASCADE, +-- ADD PRIMARY KEY (block_number, tx_index, log_index, validator_index); \ No newline at end of file diff --git a/rolling-shutter/medley/db/metadb.go b/rolling-shutter/medley/db/metadb.go index e139cb2cb..73ccaf7f7 100644 --- a/rolling-shutter/medley/db/metadb.go +++ b/rolling-shutter/medley/db/metadb.go @@ -3,6 +3,7 @@ package db import ( "context" "fmt" + "strconv" "github.com/jackc/pgx/v4" "github.com/pkg/errors" @@ -49,6 +50,7 @@ func ValidateSchemaVersion(ctx context.Context, tx pgx.Tx, definitionName string func expectMetaKeyVal(ctx context.Context, tx pgx.Tx, key, val string) error { haveVal, err := New(tx).GetMeta(ctx, key) + println(haveVal, "metainf value") if err == pgx.ErrNoRows { return errors.Wrapf(ErrKeyNotFound, "key: %s", key) } else if err != nil { @@ -67,3 +69,17 @@ func expectMetaKeyVal(ctx context.Context, tx pgx.Tx, key, val string) error { func ValidateDatabaseVersion(ctx context.Context, tx pgx.Tx, version string) error { return expectMetaKeyVal(ctx, tx, DatabaseVersionKey, version) } + +func GetSchemaVersion(ctx context.Context, tx pgx.Tx, definitionName string, schema Schema) (int, error) { + haveVal, err := New(tx).GetMeta(ctx, MakeSchemaVersionKey(definitionName, schema.Name)) + if err == pgx.ErrNoRows { + return 0, nil + } else if err != nil { + return 0, err + } + version, err := strconv.ParseInt(haveVal, 10, 0) + if err != nil { + return 0, err + } + return int(version), nil +} diff --git a/rolling-shutter/medley/db/sqlc.go b/rolling-shutter/medley/db/sqlc.go index fe769d8bb..018ef5a36 100644 --- a/rolling-shutter/medley/db/sqlc.go +++ b/rolling-shutter/medley/db/sqlc.go @@ -2,6 +2,7 @@ package db import ( "context" + "fmt" "io/fs" "path" "strings" @@ -31,6 +32,7 @@ type entry struct { // The schema file will then be stored in the object's state // together with the passed in `version`. func ParseSQLC(filesystem fs.FS, sqlcPath string, version int) ([]Schema, error) { + println(sqlcPath, "sqlcpath") b, err := fs.ReadFile(filesystem, sqlcPath) if err != nil { return nil, err @@ -69,10 +71,16 @@ func ParseSQLC(filesystem fs.FS, sqlcPath string, version int) ([]Schema, error) if !isSQL { continue } + pathstr := path.Join(schemaDirPath, i.Name()) + if version != 1 { + pathstr = path.Join(schemaDirPath, fmt.Sprintf("v%d", version), i.Name()) + } + + println(base, "base", pathstr, "pathstr") schema := Schema{ Version: version, Name: base, - Path: path.Join(schemaDirPath, i.Name()), + Path: pathstr, } schemas = append(schemas, schema) } @@ -103,6 +111,7 @@ func NewSQLCDefinition(filesystem fs.FS, sqlcPath string, name string, version i if err != nil { return nil, err } + print("name", name) return &SQLC{ schemas: schemas, filesystem: filesystem, From e3c342115e5f2bc083b49df65e9981c537515651 Mon Sep 17 00:00:00 2001 From: blockchainluffy Date: Tue, 1 Apr 2025 14:46:38 +0530 Subject: [PATCH 2/9] feat/added db migration for sqlc definition --- rolling-shutter/go.mod | 2 +- rolling-shutter/go.sum | 4 +- .../keyperimpl/gnosis/database/definition.go | 2 +- .../migrations/V2_validatorRegistrations.sql | 6 ++ .../database/sql/schemas/gnosiskeyper.sql | 5 +- rolling-shutter/medley/db/definitions.go | 45 +++++--- rolling-shutter/medley/db/meta.sqlc.gen.go | 14 +++ rolling-shutter/medley/db/metadb.go | 12 ++- .../medley/db/sql/queries/meta.sql | 3 + rolling-shutter/medley/db/sqlc.go | 102 +++++++++++++++++- 10 files changed, 172 insertions(+), 23 deletions(-) create mode 100644 rolling-shutter/keyperimpl/gnosis/database/sql/migrations/V2_validatorRegistrations.sql diff --git a/rolling-shutter/go.mod b/rolling-shutter/go.mod index d8b7e6903..4e09e69cd 100644 --- a/rolling-shutter/go.mod +++ b/rolling-shutter/go.mod @@ -40,7 +40,7 @@ require ( github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.13.0 github.com/stretchr/testify v1.9.0 - github.com/supranational/blst v0.3.12 + github.com/supranational/blst v0.3.14 github.com/tendermint/go-amino v0.16.0 github.com/tendermint/tendermint v0.37.0-rc2 go.opentelemetry.io/otel v1.27.0 diff --git a/rolling-shutter/go.sum b/rolling-shutter/go.sum index d3f37a9dc..2b24483b9 100644 --- a/rolling-shutter/go.sum +++ b/rolling-shutter/go.sum @@ -929,8 +929,8 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8= github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= -github.com/supranational/blst v0.3.12 h1:Vfas2U2CFHhniv2QkUm2OVa1+pGTdqtpqm9NnhUUbZ8= -github.com/supranational/blst v0.3.12/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= +github.com/supranational/blst v0.3.14 h1:xNMoHRJOTwMn63ip6qoWJ2Ymgvj7E2b9jY2FAwY+qRo= +github.com/supranational/blst v0.3.14/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= github.com/syndtr/goleveldb v1.0.1-0.20220614013038-64ee5596c38a h1:1ur3QoCqvE5fl+nylMaIr9PVV1w343YRDtsy+Rwu7XI= github.com/syndtr/goleveldb v1.0.1-0.20220614013038-64ee5596c38a/go.mod h1:RRCYJbIwD5jmqPI9XoAFR0OcDxqUctll6zUj/+B4S48= github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= diff --git a/rolling-shutter/keyperimpl/gnosis/database/definition.go b/rolling-shutter/keyperimpl/gnosis/database/definition.go index e83d8021f..fa74379e1 100644 --- a/rolling-shutter/keyperimpl/gnosis/database/definition.go +++ b/rolling-shutter/keyperimpl/gnosis/database/definition.go @@ -17,7 +17,7 @@ var files embed.FS var Definition db.Definition func init() { - def, err := db.NewSQLCDefinition(files, "sql/", "gnosiskeyper", 1) + def, err := db.NewSQLCDefinition(files, "sql/", "gnosiskeyper", 2) if err != nil { log.Fatal().Err(err).Msg("failed to initialize DB metadata") } diff --git a/rolling-shutter/keyperimpl/gnosis/database/sql/migrations/V2_validatorRegistrations.sql b/rolling-shutter/keyperimpl/gnosis/database/sql/migrations/V2_validatorRegistrations.sql new file mode 100644 index 000000000..a253e6c55 --- /dev/null +++ b/rolling-shutter/keyperimpl/gnosis/database/sql/migrations/V2_validatorRegistrations.sql @@ -0,0 +1,6 @@ +-- schema-version: gnosiskeyper-2 -- +-- migrations need to start from V2... as file name, as the V1 was initial schema + +ALTER TABLE validator_registrations + DROP CONSTRAINT validator_registrations_pkey CASCADE, + ADD PRIMARY KEY (block_number, tx_index, log_index, validator_index); \ No newline at end of file diff --git a/rolling-shutter/keyperimpl/gnosis/database/sql/schemas/gnosiskeyper.sql b/rolling-shutter/keyperimpl/gnosis/database/sql/schemas/gnosiskeyper.sql index df4a270ec..cd04083be 100644 --- a/rolling-shutter/keyperimpl/gnosis/database/sql/schemas/gnosiskeyper.sql +++ b/rolling-shutter/keyperimpl/gnosis/database/sql/schemas/gnosiskeyper.sql @@ -1,4 +1,4 @@ --- schema-version: gnosiskeyper-2 -- +-- schema-version: gnosiskeyper-1 -- -- Please change the version above if you make incompatible changes to -- the schema. We'll use this to check we're using the right schema. @@ -68,6 +68,3 @@ CREATE TABLE validator_registrations_synced_until( block_number bigint NOT NULL CHECK (block_number >= 0) ); --- ALTER TABLE validator_registrations --- DROP CONSTRAINT validator_registrations_pkey CASCADE, --- ADD PRIMARY KEY (block_number, tx_index, log_index, validator_index); \ No newline at end of file diff --git a/rolling-shutter/medley/db/definitions.go b/rolling-shutter/medley/db/definitions.go index 021a28ca7..a4ba0138f 100644 --- a/rolling-shutter/medley/db/definitions.go +++ b/rolling-shutter/medley/db/definitions.go @@ -17,14 +17,23 @@ import ( // pins the database to a specific role, e.g. "keyper-test" or "snapshot-keyper-production" // in order to prevent later usage of the database with commands that fulfill a different role. func InitDB(ctx context.Context, dbpool *pgxpool.Pool, role string, definition Definition) error { + // First check if schema exists and is valid err := dbpool.BeginFunc(WrapContext(ctx, definition.Validate)) if err == nil { shdb.AddConnectionInfo(log.Info(), dbpool).Msg("database already exists") return nil - } else if errors.Is(err, ErrValueMismatch) { + } else if errors.Is(err, ErrNeedsMigation) { + // Schema exists, just run migrations + shdb.AddConnectionInfo(log.Info(), dbpool).Msg("database exists, checking for migrations") + err = dbpool.BeginFunc(WrapContext(ctx, definition.Migrate)) + if err != nil { + return errors.Wrap(err, "failed to apply migrations") + } + return nil + } else if !errors.Is(err, ErrValueMismatch) && !errors.Is(err, ErrKeyNotFound) { return err } - + // Schema doesn't exist or is invalid, create it err = dbpool.BeginFunc(WrapContext(ctx, definition.Create)) if err != nil { return err @@ -35,19 +44,20 @@ func InitDB(ctx context.Context, dbpool *pgxpool.Pool, role string, definition D return err } - // For the outer DB initialisation, also set the database version to - // the overall "role", so that e.g. a snapshot keyper database won't be - // used by another keyper implementation, no matter if the schemas - // are compatible - err = dbpool.BeginFunc( - ctx, - func(tx pgx.Tx) error { - return InsertDBVersion(ctx, tx, role) - }, - ) + // Run any migrations after initial creation + err = dbpool.BeginFunc(WrapContext(ctx, definition.Migrate)) + if err != nil { + return errors.Wrap(err, "failed to apply migrations") + } + + // Set the database role + err = dbpool.BeginFunc(ctx, func(tx pgx.Tx) error { + return InsertDBVersion(ctx, tx, role) + }) if err != nil { return err } + shdb.AddConnectionInfo(log.Info(), dbpool).Msg("database initialized") return nil } @@ -120,11 +130,22 @@ func (d AggregateDefinition) Validate(ctx context.Context, tx pgx.Tx) error { return nil } +func (d AggregateDefinition) Migrate(ctx context.Context, tx pgx.Tx) error { + for def := range d.defs { + err := def.Migrate(ctx, tx) + if err != nil { + return errors.Wrapf(err, "migration failed for definition '%s'", def.Name()) + } + } + return nil +} + type Definition interface { Name() string Create(context.Context, pgx.Tx) error Init(context.Context, pgx.Tx) error Validate(context.Context, pgx.Tx) error + Migrate(context.Context, pgx.Tx) error } type Schema struct { diff --git a/rolling-shutter/medley/db/meta.sqlc.gen.go b/rolling-shutter/medley/db/meta.sqlc.gen.go index 5390a3604..c64079864 100644 --- a/rolling-shutter/medley/db/meta.sqlc.gen.go +++ b/rolling-shutter/medley/db/meta.sqlc.gen.go @@ -33,3 +33,17 @@ func (q *Queries) InsertMeta(ctx context.Context, arg InsertMetaParams) error { _, err := q.db.Exec(ctx, insertMeta, arg.Key, arg.Value) return err } + +const updateMeta = `-- name: UpdateMeta :exec +UPDATE meta_inf SET value = $1 WHERE key = $2 +` + +type UpdateMetaParams struct { + Value string + Key string +} + +func (q *Queries) UpdateMeta(ctx context.Context, arg UpdateMetaParams) error { + _, err := q.db.Exec(ctx, updateMeta, arg.Value, arg.Key) + return err +} diff --git a/rolling-shutter/medley/db/metadb.go b/rolling-shutter/medley/db/metadb.go index 73ccaf7f7..7a2cf6ed7 100644 --- a/rolling-shutter/medley/db/metadb.go +++ b/rolling-shutter/medley/db/metadb.go @@ -13,6 +13,7 @@ import ( var ( ErrValueMismatch = errors.New("database has unexpected value") ErrKeyNotFound = errors.New("key does not exist") + ErrNeedsMigation = errors.New("needs migration") ) var DatabaseVersionKey string = "database-version" @@ -43,6 +44,13 @@ func insertMetaInf(ctx context.Context, tx pgx.Tx, key, val string) error { }) } +func UpdateSchemaVersion(ctx context.Context, tx pgx.Tx, defName string, schema Schema) error { + return New(tx).UpdateMeta(ctx, UpdateMetaParams{ + Key: MakeSchemaVersionKey(defName, schema.Name), + Value: fmt.Sprint(schema.Version), + }) +} + // ValidateSchemaVersion checks that the database schema is compatible. func ValidateSchemaVersion(ctx context.Context, tx pgx.Tx, definitionName string, schema Schema) error { return expectMetaKeyVal(ctx, tx, MakeSchemaVersionKey(definitionName, schema.Name), fmt.Sprint(schema.Version)) @@ -50,12 +58,14 @@ func ValidateSchemaVersion(ctx context.Context, tx pgx.Tx, definitionName string func expectMetaKeyVal(ctx context.Context, tx pgx.Tx, key, val string) error { haveVal, err := New(tx).GetMeta(ctx, key) - println(haveVal, "metainf value") if err == pgx.ErrNoRows { return errors.Wrapf(ErrKeyNotFound, "key: %s", key) } else if err != nil { return errors.Wrapf(err, "failed to get key '%s' from meta_inf table", key) } + if haveVal < val { + return errors.Wrapf(ErrNeedsMigation, "expected %s, have %s", val, haveVal) + } if haveVal != val { return errors.Wrapf(ErrValueMismatch, "expected %s, have %s", val, haveVal) } diff --git a/rolling-shutter/medley/db/sql/queries/meta.sql b/rolling-shutter/medley/db/sql/queries/meta.sql index a1111c571..b1496e7d4 100644 --- a/rolling-shutter/medley/db/sql/queries/meta.sql +++ b/rolling-shutter/medley/db/sql/queries/meta.sql @@ -3,3 +3,6 @@ INSERT INTO meta_inf (key, value) VALUES ($1, $2); -- name: GetMeta :one SELECT value FROM meta_inf WHERE key = $1; + +-- name: UpdateMeta :exec +UPDATE meta_inf SET value = $1 WHERE key = $2; \ No newline at end of file diff --git a/rolling-shutter/medley/db/sqlc.go b/rolling-shutter/medley/db/sqlc.go index 018ef5a36..e6b0b4c61 100644 --- a/rolling-shutter/medley/db/sqlc.go +++ b/rolling-shutter/medley/db/sqlc.go @@ -5,10 +5,12 @@ import ( "fmt" "io/fs" "path" + "sort" "strings" "github.com/jackc/pgx/v4" "github.com/pkg/errors" + "github.com/rs/zerolog/log" "gopkg.in/yaml.v3" ) @@ -32,7 +34,6 @@ type entry struct { // The schema file will then be stored in the object's state // together with the passed in `version`. func ParseSQLC(filesystem fs.FS, sqlcPath string, version int) ([]Schema, error) { - println(sqlcPath, "sqlcpath") b, err := fs.ReadFile(filesystem, sqlcPath) if err != nil { return nil, err @@ -76,7 +77,6 @@ func ParseSQLC(filesystem fs.FS, sqlcPath string, version int) ([]Schema, error) pathstr = path.Join(schemaDirPath, fmt.Sprintf("v%d", version), i.Name()) } - println(base, "base", pathstr, "pathstr") schema := Schema{ Version: version, Name: base, @@ -116,6 +116,7 @@ func NewSQLCDefinition(filesystem fs.FS, sqlcPath string, name string, version i schemas: schemas, filesystem: filesystem, name: name, + sqlcPath: sqlcPath, }, nil } @@ -125,6 +126,7 @@ type SQLC struct { schemas []Schema filesystem fs.FS name string + sqlcPath string } func (d *SQLC) Name() string { @@ -147,7 +149,10 @@ func (d *SQLC) Create(ctx context.Context, tx pgx.Tx) error { return errors.Wrapf(err, "failed to execute SQL statements for definition '%s'", d.Name()) } } + for _, schema := range d.schemas { + // this is initial creation of db, so create version as one here + schema.Version = 1 err := InsertSchemaVersion(ctx, tx, d.Name(), schema) if err != nil { return err @@ -179,3 +184,96 @@ func (d *SQLC) sqlCreateStatements() []string { } return sqlStatements } + +func (d *SQLC) LoadMigrations() ([]Migration, error) { + migrationsPath := path.Join(path.Dir(d.sqlcPath), "migrations") + entries, err := fs.ReadDir(d.filesystem, migrationsPath) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return nil, nil + } + return nil, errors.Wrap(err, "error reading migrations directory") + } + + var migrations []Migration + for _, entry := range entries { + if entry.IsDir() { + continue + } + name := entry.Name() + if !strings.HasSuffix(name, ".sql") { + continue + } + + var fileversion int + _, err := fmt.Sscanf(name, "V%d_", &fileversion) + if err != nil { + continue + } + + _, err = fs.ReadFile(d.filesystem, path.Join(migrationsPath, name)) + if err != nil { + return nil, errors.Wrapf(err, "failed to read migration %s", name) + } + + migrations = append(migrations, Migration{ + Version: fileversion, + Path: path.Join(migrationsPath, name), + Up: true, + }) + } + + sort.Slice(migrations, func(i, j int) bool { + return migrations[i].Version < migrations[j].Version + }) + + return migrations, nil +} + +func (d *SQLC) Migrate(ctx context.Context, tx pgx.Tx) error { + for _, schema := range d.schemas { + // Get current version from meta-inf + version, err := GetSchemaVersion(ctx, tx, d.Name(), schema) + if err != nil { + return err + } + + migrations, err := d.LoadMigrations() + if err != nil { + return errors.Wrap(err, "failed to load migrations") + } + + // Apply only migrations that are newer than current version + for _, migration := range migrations { + println(migration.Path, migration.Version, "migration") + if migration.Version <= version { + continue + } + + content, err := fs.ReadFile(d.filesystem, migration.Path) + if err != nil { + return errors.Wrapf(err, "failed to read migration file %s", migration.Path) + } + + log.Info(). + Str("definition", d.Name()). + Int("from_version", version). + Int("to_version", migration.Version). + Str("path", migration.Path). + Msg("applying migration") + + _, err = tx.Exec(ctx, string(content)) + if err != nil { + return errors.Wrapf(err, "failed to apply migration %d", migration.Version) + } + + // Update version after each successful migration + err = UpdateSchemaVersion(ctx, tx, d.Name(), schema) + if err != nil { + return errors.Wrapf(err, "failed to update schema version to %d", migration.Version) + } + } + } + + return nil +} From 03464b2272cc608f57d8f6e851325d8b4aadd887 Mon Sep 17 00:00:00 2001 From: blockchainluffy Date: Tue, 1 Apr 2025 15:26:11 +0530 Subject: [PATCH 3/9] tested db migration to run on when db is initialised and when it is already initialised --- rolling-shutter/medley/db/definitions.go | 2 +- rolling-shutter/medley/db/sqlc.go | 11 +---------- 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/rolling-shutter/medley/db/definitions.go b/rolling-shutter/medley/db/definitions.go index a4ba0138f..182e1b4c3 100644 --- a/rolling-shutter/medley/db/definitions.go +++ b/rolling-shutter/medley/db/definitions.go @@ -30,7 +30,7 @@ func InitDB(ctx context.Context, dbpool *pgxpool.Pool, role string, definition D return errors.Wrap(err, "failed to apply migrations") } return nil - } else if !errors.Is(err, ErrValueMismatch) && !errors.Is(err, ErrKeyNotFound) { + } else if errors.Is(err, ErrValueMismatch) { return err } // Schema doesn't exist or is invalid, create it diff --git a/rolling-shutter/medley/db/sqlc.go b/rolling-shutter/medley/db/sqlc.go index e6b0b4c61..fa4bf0889 100644 --- a/rolling-shutter/medley/db/sqlc.go +++ b/rolling-shutter/medley/db/sqlc.go @@ -65,22 +65,14 @@ func ParseSQLC(filesystem fs.FS, sqlcPath string, version int) ([]Schema, error) if i.IsDir() { continue } - // TODO: allow database migrations. - // For now assume only schemas, no migrations. - // However sqlc does recognize migration files from different tools. base, isSQL := strings.CutSuffix(i.Name(), ".sql") if !isSQL { continue } - pathstr := path.Join(schemaDirPath, i.Name()) - if version != 1 { - pathstr = path.Join(schemaDirPath, fmt.Sprintf("v%d", version), i.Name()) - } - schema := Schema{ Version: version, Name: base, - Path: pathstr, + Path: path.Join(schemaDirPath, i.Name()), } schemas = append(schemas, schema) } @@ -111,7 +103,6 @@ func NewSQLCDefinition(filesystem fs.FS, sqlcPath string, name string, version i if err != nil { return nil, err } - print("name", name) return &SQLC{ schemas: schemas, filesystem: filesystem, From 964601d6c8a90c762ea4767a94ea4b305ca263c9 Mon Sep 17 00:00:00 2001 From: blockchainluffy Date: Tue, 8 Apr 2025 12:26:32 +0530 Subject: [PATCH 4/9] chore: fix typo and updated gnosis migration to not cascade --- .../database/sql/migrations/V2_validatorRegistrations.sql | 2 +- rolling-shutter/medley/db/definitions.go | 2 +- rolling-shutter/medley/db/metadb.go | 8 ++++---- rolling-shutter/medley/db/sqlc.go | 1 - 4 files changed, 6 insertions(+), 7 deletions(-) diff --git a/rolling-shutter/keyperimpl/gnosis/database/sql/migrations/V2_validatorRegistrations.sql b/rolling-shutter/keyperimpl/gnosis/database/sql/migrations/V2_validatorRegistrations.sql index a253e6c55..9aac91159 100644 --- a/rolling-shutter/keyperimpl/gnosis/database/sql/migrations/V2_validatorRegistrations.sql +++ b/rolling-shutter/keyperimpl/gnosis/database/sql/migrations/V2_validatorRegistrations.sql @@ -2,5 +2,5 @@ -- migrations need to start from V2... as file name, as the V1 was initial schema ALTER TABLE validator_registrations - DROP CONSTRAINT validator_registrations_pkey CASCADE, + DROP CONSTRAINT validator_registrations_pkey, ADD PRIMARY KEY (block_number, tx_index, log_index, validator_index); \ No newline at end of file diff --git a/rolling-shutter/medley/db/definitions.go b/rolling-shutter/medley/db/definitions.go index 182e1b4c3..8f3469295 100644 --- a/rolling-shutter/medley/db/definitions.go +++ b/rolling-shutter/medley/db/definitions.go @@ -22,7 +22,7 @@ func InitDB(ctx context.Context, dbpool *pgxpool.Pool, role string, definition D if err == nil { shdb.AddConnectionInfo(log.Info(), dbpool).Msg("database already exists") return nil - } else if errors.Is(err, ErrNeedsMigation) { + } else if errors.Is(err, ErrNeedsMigration) { // Schema exists, just run migrations shdb.AddConnectionInfo(log.Info(), dbpool).Msg("database exists, checking for migrations") err = dbpool.BeginFunc(WrapContext(ctx, definition.Migrate)) diff --git a/rolling-shutter/medley/db/metadb.go b/rolling-shutter/medley/db/metadb.go index 7a2cf6ed7..9951a7173 100644 --- a/rolling-shutter/medley/db/metadb.go +++ b/rolling-shutter/medley/db/metadb.go @@ -11,9 +11,9 @@ import ( ) var ( - ErrValueMismatch = errors.New("database has unexpected value") - ErrKeyNotFound = errors.New("key does not exist") - ErrNeedsMigation = errors.New("needs migration") + ErrValueMismatch = errors.New("database has unexpected value") + ErrKeyNotFound = errors.New("key does not exist") + ErrNeedsMigration = errors.New("needs migration") ) var DatabaseVersionKey string = "database-version" @@ -64,7 +64,7 @@ func expectMetaKeyVal(ctx context.Context, tx pgx.Tx, key, val string) error { return errors.Wrapf(err, "failed to get key '%s' from meta_inf table", key) } if haveVal < val { - return errors.Wrapf(ErrNeedsMigation, "expected %s, have %s", val, haveVal) + return errors.Wrapf(ErrNeedsMigration, "expected %s, have %s", val, haveVal) } if haveVal != val { return errors.Wrapf(ErrValueMismatch, "expected %s, have %s", val, haveVal) diff --git a/rolling-shutter/medley/db/sqlc.go b/rolling-shutter/medley/db/sqlc.go index fa4bf0889..8732c7a39 100644 --- a/rolling-shutter/medley/db/sqlc.go +++ b/rolling-shutter/medley/db/sqlc.go @@ -236,7 +236,6 @@ func (d *SQLC) Migrate(ctx context.Context, tx pgx.Tx) error { // Apply only migrations that are newer than current version for _, migration := range migrations { - println(migration.Path, migration.Version, "migration") if migration.Version <= version { continue } From 34e270f71b1a1a583349d8fcad54757a74a7de8d Mon Sep 17 00:00:00 2001 From: blockchainluffy Date: Mon, 14 Apr 2025 16:00:17 +0530 Subject: [PATCH 5/9] add versioning to sqlc definition rather than in schema definition --- .../chainobserver/db/collator/definition.go | 2 +- .../chainobserver/db/keyper/definition.go | 2 +- .../chainobserver/db/sync/definition.go | 2 +- rolling-shutter/keyper/database/definition.go | 2 +- .../keyperimpl/gnosis/database/definition.go | 2 +- .../shutterservice/database/definition.go | 2 +- rolling-shutter/medley/db/definition.go | 2 +- rolling-shutter/medley/db/definitions.go | 5 +-- rolling-shutter/medley/db/metadb.go | 12 ++--- rolling-shutter/medley/db/sqlc.go | 44 ++++++++++++++----- .../snapshot/database/definition.go | 2 +- 11 files changed, 48 insertions(+), 29 deletions(-) diff --git a/rolling-shutter/chainobserver/db/collator/definition.go b/rolling-shutter/chainobserver/db/collator/definition.go index 07f06a1d8..35d31392d 100644 --- a/rolling-shutter/chainobserver/db/collator/definition.go +++ b/rolling-shutter/chainobserver/db/collator/definition.go @@ -16,7 +16,7 @@ var files embed.FS var Definition db.Definition func init() { - def, err := db.NewSQLCDefinition(files, "sql/", "chainobscollator", 1) + def, err := db.NewSQLCDefinition(files, "sql/", "chainobscollator") if err != nil { log.Fatal().Err(err).Msg("failed to initialize DB metadata") } diff --git a/rolling-shutter/chainobserver/db/keyper/definition.go b/rolling-shutter/chainobserver/db/keyper/definition.go index 26e7ddf8e..790a4ac45 100644 --- a/rolling-shutter/chainobserver/db/keyper/definition.go +++ b/rolling-shutter/chainobserver/db/keyper/definition.go @@ -16,7 +16,7 @@ var files embed.FS var Definition db.Definition func init() { - def, err := db.NewSQLCDefinition(files, "sql/", "chainobskeyper", 1) + def, err := db.NewSQLCDefinition(files, "sql/", "chainobskeyper") if err != nil { log.Fatal().Err(err).Msg("failed to initialize DB metadata") } diff --git a/rolling-shutter/chainobserver/db/sync/definition.go b/rolling-shutter/chainobserver/db/sync/definition.go index 196dc3c08..6cd6eaadf 100644 --- a/rolling-shutter/chainobserver/db/sync/definition.go +++ b/rolling-shutter/chainobserver/db/sync/definition.go @@ -16,7 +16,7 @@ var Definition db.Definition func init() { var err error - Definition, err = db.NewSQLCDefinition(files, "sql/", "chainobssync", 1) + Definition, err = db.NewSQLCDefinition(files, "sql/", "chainobssync") if err != nil { log.Fatal().Err(err).Msg("failed to initialize DB metadata") } diff --git a/rolling-shutter/keyper/database/definition.go b/rolling-shutter/keyper/database/definition.go index 0e69d1c8f..1594f20ef 100644 --- a/rolling-shutter/keyper/database/definition.go +++ b/rolling-shutter/keyper/database/definition.go @@ -21,7 +21,7 @@ var files embed.FS var Definition db.Definition func init() { - sqlcDB, err := db.NewSQLCDefinition(files, "sql/", "keyper", 1) + sqlcDB, err := db.NewSQLCDefinition(files, "sql/", "keyper") if err != nil { log.Fatal().Err(err).Msg("failed to initialize DB") } diff --git a/rolling-shutter/keyperimpl/gnosis/database/definition.go b/rolling-shutter/keyperimpl/gnosis/database/definition.go index fa74379e1..4e8b1ec8d 100644 --- a/rolling-shutter/keyperimpl/gnosis/database/definition.go +++ b/rolling-shutter/keyperimpl/gnosis/database/definition.go @@ -17,7 +17,7 @@ var files embed.FS var Definition db.Definition func init() { - def, err := db.NewSQLCDefinition(files, "sql/", "gnosiskeyper", 2) + def, err := db.NewSQLCDefinition(files, "sql/", "gnosiskeyper") if err != nil { log.Fatal().Err(err).Msg("failed to initialize DB metadata") } diff --git a/rolling-shutter/keyperimpl/shutterservice/database/definition.go b/rolling-shutter/keyperimpl/shutterservice/database/definition.go index 8f9cd7b3c..374a1fbe4 100644 --- a/rolling-shutter/keyperimpl/shutterservice/database/definition.go +++ b/rolling-shutter/keyperimpl/shutterservice/database/definition.go @@ -17,7 +17,7 @@ var files embed.FS var Definition db.Definition func init() { - def, err := db.NewSQLCDefinition(files, "sql/", "shutterservicekeyper", 1) + def, err := db.NewSQLCDefinition(files, "sql/", "shutterservicekeyper") if err != nil { log.Fatal().Err(err).Msg("failed to initialize DB metadata") } diff --git a/rolling-shutter/medley/db/definition.go b/rolling-shutter/medley/db/definition.go index 1b50ef56f..a4b095ee3 100644 --- a/rolling-shutter/medley/db/definition.go +++ b/rolling-shutter/medley/db/definition.go @@ -14,7 +14,7 @@ var metaDefinition Definition func init() { var err error - metaDefinition, err = NewSQLCDefinition(files, "sql/", "meta", 1) + metaDefinition, err = NewSQLCDefinition(files, "sql/", "meta") if err != nil { log.Fatal().Err(err).Msg("failed to initialize DB metadata") } diff --git a/rolling-shutter/medley/db/definitions.go b/rolling-shutter/medley/db/definitions.go index 8f3469295..fbd7314be 100644 --- a/rolling-shutter/medley/db/definitions.go +++ b/rolling-shutter/medley/db/definitions.go @@ -149,9 +149,8 @@ type Definition interface { } type Schema struct { - Version int - Name string - Path string + Name string + Path string } type Migration struct { diff --git a/rolling-shutter/medley/db/metadb.go b/rolling-shutter/medley/db/metadb.go index 9951a7173..ae4e1b86a 100644 --- a/rolling-shutter/medley/db/metadb.go +++ b/rolling-shutter/medley/db/metadb.go @@ -31,8 +31,8 @@ func InsertDBVersion(ctx context.Context, tx pgx.Tx, version string) error { return insertMetaInf(ctx, tx, DatabaseVersionKey, version) } -func InsertSchemaVersion(ctx context.Context, tx pgx.Tx, definitionName string, schema Schema) error { - return insertMetaInf(ctx, tx, MakeSchemaVersionKey(definitionName, schema.Name), fmt.Sprint(schema.Version)) +func InsertSchemaVersion(ctx context.Context, tx pgx.Tx, definitionName string, schema Schema, version int) error { + return insertMetaInf(ctx, tx, MakeSchemaVersionKey(definitionName, schema.Name), fmt.Sprint(version)) } func insertMetaInf(ctx context.Context, tx pgx.Tx, key, val string) error { @@ -44,16 +44,16 @@ func insertMetaInf(ctx context.Context, tx pgx.Tx, key, val string) error { }) } -func UpdateSchemaVersion(ctx context.Context, tx pgx.Tx, defName string, schema Schema) error { +func UpdateSchemaVersion(ctx context.Context, tx pgx.Tx, defName string, schema Schema, version int) error { return New(tx).UpdateMeta(ctx, UpdateMetaParams{ Key: MakeSchemaVersionKey(defName, schema.Name), - Value: fmt.Sprint(schema.Version), + Value: fmt.Sprint(version), }) } // ValidateSchemaVersion checks that the database schema is compatible. -func ValidateSchemaVersion(ctx context.Context, tx pgx.Tx, definitionName string, schema Schema) error { - return expectMetaKeyVal(ctx, tx, MakeSchemaVersionKey(definitionName, schema.Name), fmt.Sprint(schema.Version)) +func ValidateSchemaVersion(ctx context.Context, tx pgx.Tx, definitionName string, schema Schema, version int) error { + return expectMetaKeyVal(ctx, tx, MakeSchemaVersionKey(definitionName, schema.Name), fmt.Sprint(version)) } func expectMetaKeyVal(ctx context.Context, tx pgx.Tx, key, val string) error { diff --git a/rolling-shutter/medley/db/sqlc.go b/rolling-shutter/medley/db/sqlc.go index 8732c7a39..8969cae45 100644 --- a/rolling-shutter/medley/db/sqlc.go +++ b/rolling-shutter/medley/db/sqlc.go @@ -33,7 +33,7 @@ type entry struct { // paths of all the schema definition directories. // The schema file will then be stored in the object's state // together with the passed in `version`. -func ParseSQLC(filesystem fs.FS, sqlcPath string, version int) ([]Schema, error) { +func ParseSQLC(filesystem fs.FS, sqlcPath string) ([]Schema, error) { b, err := fs.ReadFile(filesystem, sqlcPath) if err != nil { return nil, err @@ -70,9 +70,8 @@ func ParseSQLC(filesystem fs.FS, sqlcPath string, version int) ([]Schema, error) continue } schema := Schema{ - Version: version, - Name: base, - Path: path.Join(schemaDirPath, i.Name()), + Name: base, + Path: path.Join(schemaDirPath, i.Name()), } schemas = append(schemas, schema) } @@ -80,7 +79,7 @@ func ParseSQLC(filesystem fs.FS, sqlcPath string, version int) ([]Schema, error) return schemas, nil } -func NewSQLCDefinition(filesystem fs.FS, sqlcPath string, name string, version int) (*SQLC, error) { +func NewSQLCDefinition(filesystem fs.FS, sqlcPath string, name string) (*SQLC, error) { p := path.Clean(sqlcPath) des, err := fs.ReadDir(filesystem, p) if errors.Is(err, ErrNotADirectory) { @@ -99,16 +98,22 @@ func NewSQLCDefinition(filesystem fs.FS, sqlcPath string, name string, version i if foundPath == "" { return nil, errors.Errorf("SQLC file '%s' does not exists", p) } - schemas, err := ParseSQLC(filesystem, foundPath, version) + schemas, err := ParseSQLC(filesystem, foundPath) if err != nil { return nil, err } - return &SQLC{ + + sqlcdef := &SQLC{ schemas: schemas, filesystem: filesystem, name: name, sqlcPath: sqlcPath, - }, nil + } + sqlcdef.version, err = sqlcdef.GetLatestMigrationVersion() + if err != nil { + return nil, err + } + return sqlcdef, err } // SQLC implements the `Definition` interface and keeps @@ -118,6 +123,7 @@ type SQLC struct { filesystem fs.FS name string sqlcPath string + version int } func (d *SQLC) Name() string { @@ -143,8 +149,7 @@ func (d *SQLC) Create(ctx context.Context, tx pgx.Tx) error { for _, schema := range d.schemas { // this is initial creation of db, so create version as one here - schema.Version = 1 - err := InsertSchemaVersion(ctx, tx, d.Name(), schema) + err := InsertSchemaVersion(ctx, tx, d.Name(), schema, 1) if err != nil { return err } @@ -156,7 +161,7 @@ func (d *SQLC) Create(ctx context.Context, tx pgx.Tx) error { // with the schema versions of it's schema definitions. func (d *SQLC) Validate(ctx context.Context, tx pgx.Tx) error { for _, schema := range d.schemas { - err := ValidateSchemaVersion(ctx, tx, d.Name(), schema) + err := ValidateSchemaVersion(ctx, tx, d.Name(), schema, d.version) if err != nil { return err } @@ -258,7 +263,7 @@ func (d *SQLC) Migrate(ctx context.Context, tx pgx.Tx) error { } // Update version after each successful migration - err = UpdateSchemaVersion(ctx, tx, d.Name(), schema) + err = UpdateSchemaVersion(ctx, tx, d.Name(), schema, migration.Version) if err != nil { return errors.Wrapf(err, "failed to update schema version to %d", migration.Version) } @@ -267,3 +272,18 @@ func (d *SQLC) Migrate(ctx context.Context, tx pgx.Tx) error { return nil } + +func (d *SQLC) GetLatestMigrationVersion() (int, error) { + migrations, err := d.LoadMigrations() + if err != nil { + return 0, err + } + + if len(migrations) == 0 { + return 1, nil // Return 1 if no migrations exist + } + + // Since migrations are already sorted in LoadMigrations(), + // we can just return the version of the last migration + return migrations[len(migrations)-1].Version, nil +} diff --git a/rolling-shutter/snapshot/database/definition.go b/rolling-shutter/snapshot/database/definition.go index 6ff3c3b04..c2a7c9cd0 100644 --- a/rolling-shutter/snapshot/database/definition.go +++ b/rolling-shutter/snapshot/database/definition.go @@ -15,7 +15,7 @@ var files embed.FS var Definition db.Definition func init() { - def, err := db.NewSQLCDefinition(files, "sql/", "snapshot", 22) + def, err := db.NewSQLCDefinition(files, "sql/", "snapshot") if err != nil { log.Fatal().Err(err).Msg("failed to initialize DB metadata") } From bfdfa25463ad52f2f9223cf9c22f7e7ef6b05b35 Mon Sep 17 00:00:00 2001 From: blockchainluffy Date: Tue, 15 Apr 2025 12:23:57 +0530 Subject: [PATCH 6/9] updated ValidateSchema to include migration check --- rolling-shutter/medley/db/metadb.go | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/rolling-shutter/medley/db/metadb.go b/rolling-shutter/medley/db/metadb.go index ae4e1b86a..b489a4e5e 100644 --- a/rolling-shutter/medley/db/metadb.go +++ b/rolling-shutter/medley/db/metadb.go @@ -53,7 +53,24 @@ func UpdateSchemaVersion(ctx context.Context, tx pgx.Tx, defName string, schema // ValidateSchemaVersion checks that the database schema is compatible. func ValidateSchemaVersion(ctx context.Context, tx pgx.Tx, definitionName string, schema Schema, version int) error { - return expectMetaKeyVal(ctx, tx, MakeSchemaVersionKey(definitionName, schema.Name), fmt.Sprint(version)) + key := MakeSchemaVersionKey(definitionName, schema.Name) + haveVal, err := New(tx).GetMeta(ctx, key) + if err == pgx.ErrNoRows { + return errors.Wrapf(ErrKeyNotFound, "key: %s", key) + } else if err != nil { + return errors.Wrapf(err, "failed to get key '%s' from meta_inf table", key) + } + haveVersion, err := strconv.ParseInt(haveVal, 10, 0) + if err != nil { + return errors.Wrapf(err, "failed to convert version '%s' from meta_inf table", key) + } + if int(haveVersion) < version { + return errors.Wrapf(ErrNeedsMigration, "expected %d, have %d", version, haveVersion) + } + if int(haveVersion) != version { + return errors.Wrapf(ErrValueMismatch, "expected %d, have %d", version, haveVersion) + } + return nil } func expectMetaKeyVal(ctx context.Context, tx pgx.Tx, key, val string) error { @@ -63,9 +80,6 @@ func expectMetaKeyVal(ctx context.Context, tx pgx.Tx, key, val string) error { } else if err != nil { return errors.Wrapf(err, "failed to get key '%s' from meta_inf table", key) } - if haveVal < val { - return errors.Wrapf(ErrNeedsMigration, "expected %s, have %s", val, haveVal) - } if haveVal != val { return errors.Wrapf(ErrValueMismatch, "expected %s, have %s", val, haveVal) } From e031da47d946fa76f83e997643238fccdc1a0907 Mon Sep 17 00:00:00 2001 From: blockchainluffy <47202792+blockchainluffy@users.noreply.github.com> Date: Wed, 28 May 2025 15:57:54 +0530 Subject: [PATCH 7/9] Update rolling-shutter/medley/db/metadb.go Co-authored-by: jannikluhn --- rolling-shutter/medley/db/metadb.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rolling-shutter/medley/db/metadb.go b/rolling-shutter/medley/db/metadb.go index b489a4e5e..b45dc7e00 100644 --- a/rolling-shutter/medley/db/metadb.go +++ b/rolling-shutter/medley/db/metadb.go @@ -65,7 +65,7 @@ func ValidateSchemaVersion(ctx context.Context, tx pgx.Tx, definitionName string return errors.Wrapf(err, "failed to convert version '%s' from meta_inf table", key) } if int(haveVersion) < version { - return errors.Wrapf(ErrNeedsMigration, "expected %d, have %d", version, haveVersion) + return errors.Wrapf(ErrNeedsMigration, "expected version %d, have %d", version, haveVersion) } if int(haveVersion) != version { return errors.Wrapf(ErrValueMismatch, "expected %d, have %d", version, haveVersion) From fa12275c7f68f9a4535d760a2917846b8e4b2370 Mon Sep 17 00:00:00 2001 From: blockchainluffy <47202792+blockchainluffy@users.noreply.github.com> Date: Wed, 28 May 2025 15:58:04 +0530 Subject: [PATCH 8/9] Update rolling-shutter/medley/db/metadb.go Co-authored-by: jannikluhn --- rolling-shutter/medley/db/metadb.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rolling-shutter/medley/db/metadb.go b/rolling-shutter/medley/db/metadb.go index b45dc7e00..0787c8724 100644 --- a/rolling-shutter/medley/db/metadb.go +++ b/rolling-shutter/medley/db/metadb.go @@ -68,7 +68,7 @@ func ValidateSchemaVersion(ctx context.Context, tx pgx.Tx, definitionName string return errors.Wrapf(ErrNeedsMigration, "expected version %d, have %d", version, haveVersion) } if int(haveVersion) != version { - return errors.Wrapf(ErrValueMismatch, "expected %d, have %d", version, haveVersion) + return errors.Wrapf(ErrValueMismatch, "expected version %d, have %d", version, haveVersion) } return nil } From 30bcc256084893ad5bb4ef0c370b2e0538a4e791 Mon Sep 17 00:00:00 2001 From: blockchainluffy Date: Wed, 28 May 2025 17:32:51 +0530 Subject: [PATCH 9/9] fix: PR comments - add migration file version validation, code refactoring --- rolling-shutter/medley/db/definitions.go | 5 ++++- rolling-shutter/medley/db/metadb.go | 24 +++++++++--------------- rolling-shutter/medley/db/sqlc.go | 5 +++++ 3 files changed, 18 insertions(+), 16 deletions(-) diff --git a/rolling-shutter/medley/db/definitions.go b/rolling-shutter/medley/db/definitions.go index fbd7314be..f94f36b4b 100644 --- a/rolling-shutter/medley/db/definitions.go +++ b/rolling-shutter/medley/db/definitions.go @@ -50,7 +50,10 @@ func InitDB(ctx context.Context, dbpool *pgxpool.Pool, role string, definition D return errors.Wrap(err, "failed to apply migrations") } - // Set the database role + // For the outer DB initialisation, also set the database version to + // the overall "role", so that e.g. a snapshot keyper database won't be + // used by another keyper implementation, no matter if the schemas + // are compatible err = dbpool.BeginFunc(ctx, func(tx pgx.Tx) error { return InsertDBVersion(ctx, tx, role) }) diff --git a/rolling-shutter/medley/db/metadb.go b/rolling-shutter/medley/db/metadb.go index 0787c8724..b76f6008f 100644 --- a/rolling-shutter/medley/db/metadb.go +++ b/rolling-shutter/medley/db/metadb.go @@ -53,21 +53,14 @@ func UpdateSchemaVersion(ctx context.Context, tx pgx.Tx, defName string, schema // ValidateSchemaVersion checks that the database schema is compatible. func ValidateSchemaVersion(ctx context.Context, tx pgx.Tx, definitionName string, schema Schema, version int) error { - key := MakeSchemaVersionKey(definitionName, schema.Name) - haveVal, err := New(tx).GetMeta(ctx, key) - if err == pgx.ErrNoRows { - return errors.Wrapf(ErrKeyNotFound, "key: %s", key) - } else if err != nil { - return errors.Wrapf(err, "failed to get key '%s' from meta_inf table", key) - } - haveVersion, err := strconv.ParseInt(haveVal, 10, 0) + haveVersion, err := GetSchemaVersion(ctx, tx, definitionName, schema) if err != nil { - return errors.Wrapf(err, "failed to convert version '%s' from meta_inf table", key) + return err } - if int(haveVersion) < version { + if haveVersion < version { return errors.Wrapf(ErrNeedsMigration, "expected version %d, have %d", version, haveVersion) } - if int(haveVersion) != version { + if haveVersion != version { return errors.Wrapf(ErrValueMismatch, "expected version %d, have %d", version, haveVersion) } return nil @@ -95,15 +88,16 @@ func ValidateDatabaseVersion(ctx context.Context, tx pgx.Tx, version string) err } func GetSchemaVersion(ctx context.Context, tx pgx.Tx, definitionName string, schema Schema) (int, error) { - haveVal, err := New(tx).GetMeta(ctx, MakeSchemaVersionKey(definitionName, schema.Name)) + key := MakeSchemaVersionKey(definitionName, schema.Name) + haveVal, err := New(tx).GetMeta(ctx, key) if err == pgx.ErrNoRows { - return 0, nil + return 0, errors.Wrapf(ErrKeyNotFound, "key: %s", key) } else if err != nil { - return 0, err + return 0, errors.Wrapf(err, "failed to get key '%s' from meta_inf table", key) } version, err := strconv.ParseInt(haveVal, 10, 0) if err != nil { - return 0, err + return 0, errors.Wrapf(err, "failed to convert version '%s' from meta_inf table", key) } return int(version), nil } diff --git a/rolling-shutter/medley/db/sqlc.go b/rolling-shutter/medley/db/sqlc.go index 8969cae45..3144af393 100644 --- a/rolling-shutter/medley/db/sqlc.go +++ b/rolling-shutter/medley/db/sqlc.go @@ -207,6 +207,11 @@ func (d *SQLC) LoadMigrations() ([]Migration, error) { continue } + // We don't support migrations before version 2 + if fileversion < 2 { + return nil, errors.Errorf("migration file %s has version %d, which is less than 2", name, fileversion) + } + _, err = fs.ReadFile(d.filesystem, path.Join(migrationsPath, name)) if err != nil { return nil, errors.Wrapf(err, "failed to read migration %s", name)