From c635e6bfc5c60dc9c1323421f919341e1fce24b9 Mon Sep 17 00:00:00 2001 From: Jorge Sardina Date: Fri, 6 Jun 2025 14:19:43 +0200 Subject: [PATCH 1/6] raw tables in schema --- .../native/native_powersync_database.dart | 4 ++ .../database/web/web_powersync_database.dart | 1 + packages/powersync_core/lib/src/schema.dart | 59 ++++++++++++++++++- .../lib/src/sync/streaming_sync.dart | 5 +- .../lib/src/web/sync_worker.dart | 1 + .../test/utils/abstract_test_utils.dart | 1 + 6 files changed, 68 insertions(+), 3 deletions(-) diff --git a/packages/powersync_core/lib/src/database/native/native_powersync_database.dart b/packages/powersync_core/lib/src/database/native/native_powersync_database.dart index 2f846fd5..af1ac201 100644 --- a/packages/powersync_core/lib/src/database/native/native_powersync_database.dart +++ b/packages/powersync_core/lib/src/database/native/native_powersync_database.dart @@ -247,6 +247,7 @@ class PowerSyncDatabaseImpl options, crudMutex.shared, syncMutex.shared, + schema, ), debugName: 'Sync ${database.openFactory.path}', onError: receiveUnhandledErrors.sendPort, @@ -290,6 +291,7 @@ class _PowerSyncDatabaseIsolateArgs { final ResolvedSyncOptions options; final SerializedMutex crudMutex; final SerializedMutex syncMutex; + final Schema schema; _PowerSyncDatabaseIsolateArgs( this.sPort, @@ -297,6 +299,7 @@ class _PowerSyncDatabaseIsolateArgs { this.options, this.crudMutex, this.syncMutex, + this.schema, ); } @@ -392,6 +395,7 @@ Future _syncIsolate(_PowerSyncDatabaseIsolateArgs args) async { final storage = BucketStorage(connection); final sync = StreamingSyncImplementation( adapter: storage, + schema: args.schema, connector: InternalConnector( getCredentialsCached: getCredentialsCached, prefetchCredentials: prefetchCredentials, diff --git a/packages/powersync_core/lib/src/database/web/web_powersync_database.dart b/packages/powersync_core/lib/src/database/web/web_powersync_database.dart index 6b40a6a2..fb410caa 100644 --- a/packages/powersync_core/lib/src/database/web/web_powersync_database.dart +++ b/packages/powersync_core/lib/src/database/web/web_powersync_database.dart @@ -141,6 +141,7 @@ class PowerSyncDatabaseImpl sync = StreamingSyncImplementation( adapter: storage, + schema: schema, connector: InternalConnector.wrap(connector, this), crudUpdateTriggerStream: crudStream, options: options, diff --git a/packages/powersync_core/lib/src/schema.dart b/packages/powersync_core/lib/src/schema.dart index 4892ee6c..3f8722ef 100644 --- a/packages/powersync_core/lib/src/schema.dart +++ b/packages/powersync_core/lib/src/schema.dart @@ -8,10 +8,11 @@ import 'schema_logic.dart'; class Schema { /// List of tables in the schema. final List tables; + final List rawTables; - const Schema(this.tables); + const Schema(this.tables, {this.rawTables = const []}); - Map toJson() => {'tables': tables}; + Map toJson() => {'raw_tables': rawTables, 'tables': tables}; void validate() { Set tableNames = {}; @@ -315,6 +316,60 @@ class Column { Map toJson() => {'name': name, 'type': type.sqlite}; } +class RawTable { + final String + name; // TODO: it does not need to be the same name as the raw table + final PendingStatement put; + final PendingStatement delete; + + const RawTable( + this.name, + this.put, + this.delete, + ); + + Map toJson() => { + 'name': name, + 'put': put, + 'delete': delete, + }; +} + +class PendingStatement { + final String sql; + final List params; + + PendingStatement({required this.sql, required this.params}); + + Map toJson() => { + 'sql': sql, + 'params': params, + }; +} + +sealed class PendingStatementValue { + dynamic toJson(); +} + +class PendingStmtValueColumn extends PendingStatementValue { + final String column; + PendingStmtValueColumn(this.column); + + @override + dynamic toJson() { + return { + 'Column': column, + }; + } +} + +class PendingStmtValueId extends PendingStatementValue { + @override + dynamic toJson() { + return 'Id'; + } +} + /// Type of column. enum ColumnType { /// TEXT column. diff --git a/packages/powersync_core/lib/src/sync/streaming_sync.dart b/packages/powersync_core/lib/src/sync/streaming_sync.dart index b2e2f5bc..82527fd8 100644 --- a/packages/powersync_core/lib/src/sync/streaming_sync.dart +++ b/packages/powersync_core/lib/src/sync/streaming_sync.dart @@ -5,6 +5,7 @@ import 'dart:typed_data'; import 'package:http/http.dart' as http; import 'package:logging/logging.dart'; import 'package:meta/meta.dart'; +import 'package:powersync_core/powersync_core.dart'; import 'package:powersync_core/src/abort_controller.dart'; import 'package:powersync_core/src/exceptions.dart'; import 'package:powersync_core/src/log_internal.dart'; @@ -32,6 +33,7 @@ abstract interface class StreamingSync { @internal class StreamingSyncImplementation implements StreamingSync { + final Schema? schema; //TODO(SkillDevs): pass in all implementations final BucketStorage adapter; final InternalConnector connector; final ResolvedSyncOptions options; @@ -62,6 +64,7 @@ class StreamingSyncImplementation implements StreamingSync { String? clientId; StreamingSyncImplementation({ + required this.schema, required this.adapter, required this.connector, required this.crudUpdateTriggerStream, @@ -592,7 +595,7 @@ final class _ActiveRustStreamingIteration { 'start', convert.json.encode({ 'parameters': sync.options.params, - 'schema': 'TODO: Pass-through schema (probably in serialized form)', + 'schema': sync.schema, }), ); assert(_completedStream.isCompleted, 'Should have started streaming'); diff --git a/packages/powersync_core/lib/src/web/sync_worker.dart b/packages/powersync_core/lib/src/web/sync_worker.dart index b5e8ed63..5ee3c4ca 100644 --- a/packages/powersync_core/lib/src/web/sync_worker.dart +++ b/packages/powersync_core/lib/src/web/sync_worker.dart @@ -264,6 +264,7 @@ class _SyncRunner { sync = StreamingSyncImplementation( adapter: WebBucketStorage(database), + schema: null, connector: InternalConnector( getCredentialsCached: client.channel.credentialsCallback, prefetchCredentials: ({required bool invalidate}) async { diff --git a/packages/powersync_core/test/utils/abstract_test_utils.dart b/packages/powersync_core/test/utils/abstract_test_utils.dart index 3ea4a319..b2dac843 100644 --- a/packages/powersync_core/test/utils/abstract_test_utils.dart +++ b/packages/powersync_core/test/utils/abstract_test_utils.dart @@ -153,6 +153,7 @@ extension MockSync on PowerSyncDatabase { }) { final impl = StreamingSyncImplementation( adapter: BucketStorage(this), + schema: null, client: client, options: ResolvedSyncOptions(options), connector: InternalConnector.wrap(connector, this), From af052b07d2a2696296bc2ed51bb53a39f473a480 Mon Sep 17 00:00:00 2001 From: David Martos Date: Sat, 7 Jun 2025 01:00:40 +0200 Subject: [PATCH 2/6] fix tests --- .../powersync_core/test/in_memory_sync_test.dart | 13 +++++++------ .../test/utils/abstract_test_utils.dart | 2 +- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/packages/powersync_core/test/in_memory_sync_test.dart b/packages/powersync_core/test/in_memory_sync_test.dart index 9455bc94..c735d817 100644 --- a/packages/powersync_core/test/in_memory_sync_test.dart +++ b/packages/powersync_core/test/in_memory_sync_test.dart @@ -1,4 +1,5 @@ import 'dart:async'; +import 'dart:convert'; import 'package:async/async.dart'; import 'package:logging/logging.dart'; @@ -172,7 +173,7 @@ void _declareTests(String name, SyncOptions options) { 'object_type': 'a', 'object_id': '1', 'checksum': 0, - 'data': {}, + 'data': '{}', } ], } @@ -187,7 +188,7 @@ void _declareTests(String name, SyncOptions options) { 'object_type': 'b', 'object_id': '1', 'checksum': 0, - 'data': {}, + 'data': '{}', } ], } @@ -229,7 +230,7 @@ void _declareTests(String name, SyncOptions options) { 'data': [ { 'checksum': priority + 10, - 'data': {'name': 'test', 'email': 'email'}, + 'data': json.encode({'name': 'test', 'email': 'email'}), 'op': 'PUT', 'op_id': '${operationId++}', 'object_id': 'prio$priority', @@ -411,7 +412,7 @@ void _declareTests(String name, SyncOptions options) { 'data': [ { 'checksum': 0, - 'data': {'name': 'from local', 'email': 'local@example.org'}, + 'data': json.encode({'name': 'from local', 'email': 'local@example.org'}), 'op': 'PUT', 'op_id': '1', 'object_id': '1', @@ -419,7 +420,7 @@ void _declareTests(String name, SyncOptions options) { }, { 'checksum': 0, - 'data': {'name': 'additional', 'email': ''}, + 'data': json.encode({'name': 'additional', 'email': ''}), 'op': 'PUT', 'op_id': '2', 'object_id': '2', @@ -477,7 +478,7 @@ void _declareTests(String name, SyncOptions options) { 'object_type': bucket, 'object_id': '$lastOpId', 'checksum': 0, - 'data': {}, + 'data': '{}', } ], } diff --git a/packages/powersync_core/test/utils/abstract_test_utils.dart b/packages/powersync_core/test/utils/abstract_test_utils.dart index b2dac843..b0ebd7ba 100644 --- a/packages/powersync_core/test/utils/abstract_test_utils.dart +++ b/packages/powersync_core/test/utils/abstract_test_utils.dart @@ -153,7 +153,7 @@ extension MockSync on PowerSyncDatabase { }) { final impl = StreamingSyncImplementation( adapter: BucketStorage(this), - schema: null, + schema: schema, client: client, options: ResolvedSyncOptions(options), connector: InternalConnector.wrap(connector, this), From bf9f1e9e18bb097f3cdec454d8ecf0fce390e4a8 Mon Sep 17 00:00:00 2001 From: David Martos Date: Sun, 8 Jun 2025 01:21:29 +0200 Subject: [PATCH 3/6] manual schema management --- .../native/native_powersync_database.dart | 37 ++++++++--- .../lib/src/database/powersync_database.dart | 64 ++++++++++++------- .../powersync_database_impl_stub.dart | 15 +++-- .../lib/src/database/powersync_db_mixin.dart | 38 ++++++++++- .../database/web/web_powersync_database.dart | 28 ++++++-- 5 files changed, 140 insertions(+), 42 deletions(-) diff --git a/packages/powersync_core/lib/src/database/native/native_powersync_database.dart b/packages/powersync_core/lib/src/database/native/native_powersync_database.dart index af1ac201..f4be54e3 100644 --- a/packages/powersync_core/lib/src/database/native/native_powersync_database.dart +++ b/packages/powersync_core/lib/src/database/native/native_powersync_database.dart @@ -44,6 +44,9 @@ class PowerSyncDatabaseImpl @override SqliteDatabase database; + @override + bool manualSchemaManagement; + @override @protected late Future isInitialized; @@ -76,6 +79,7 @@ class PowerSyncDatabaseImpl required String path, int maxReaders = SqliteDatabase.defaultMaxReaders, Logger? logger, + bool manualSchemaManagement = false, @Deprecated("Use [PowerSyncDatabase.withFactory] instead.") // ignore: deprecated_member_use_from_same_package SqliteConnectionSetup? sqliteSetup}) { @@ -83,8 +87,13 @@ class PowerSyncDatabaseImpl DefaultSqliteOpenFactory factory = // ignore: deprecated_member_use_from_same_package PowerSyncOpenFactory(path: path, sqliteSetup: sqliteSetup); - return PowerSyncDatabaseImpl.withFactory(factory, - schema: schema, maxReaders: maxReaders, logger: logger); + return PowerSyncDatabaseImpl.withFactory( + factory, + schema: schema, + maxReaders: maxReaders, + logger: logger, + manualSchemaManagement: manualSchemaManagement, + ); } /// Open a [PowerSyncDatabase] with a [PowerSyncOpenFactory]. @@ -96,13 +105,19 @@ class PowerSyncDatabaseImpl /// /// [logger] defaults to [autoLogger], which logs to the console in debug builds. factory PowerSyncDatabaseImpl.withFactory( - DefaultSqliteOpenFactory openFactory, - {required Schema schema, - int maxReaders = SqliteDatabase.defaultMaxReaders, - Logger? logger}) { + DefaultSqliteOpenFactory openFactory, { + required Schema schema, + int maxReaders = SqliteDatabase.defaultMaxReaders, + Logger? logger, + bool manualSchemaManagement = false, + }) { final db = SqliteDatabase.withFactory(openFactory, maxReaders: maxReaders); return PowerSyncDatabaseImpl.withDatabase( - schema: schema, database: db, logger: logger); + schema: schema, + database: db, + logger: logger, + manualSchemaManagement: manualSchemaManagement, + ); } /// Open a PowerSyncDatabase on an existing [SqliteDatabase]. @@ -110,8 +125,12 @@ class PowerSyncDatabaseImpl /// Migrations are run on the database when this constructor is called. /// /// [logger] defaults to [autoLogger], which logs to the console in debug builds.s - PowerSyncDatabaseImpl.withDatabase( - {required this.schema, required this.database, Logger? logger}) { + PowerSyncDatabaseImpl.withDatabase({ + required this.schema, + required this.database, + Logger? logger, + this.manualSchemaManagement = false, + }) { this.logger = logger ?? autoLogger; isInitialized = baseInit(); } diff --git a/packages/powersync_core/lib/src/database/powersync_database.dart b/packages/powersync_core/lib/src/database/powersync_database.dart index 95543ce8..eb220758 100644 --- a/packages/powersync_core/lib/src/database/powersync_database.dart +++ b/packages/powersync_core/lib/src/database/powersync_database.dart @@ -32,19 +32,23 @@ abstract class PowerSyncDatabase /// A maximum of [maxReaders] concurrent read transactions are allowed. /// /// [logger] defaults to [autoLogger], which logs to the console in debug builds. - factory PowerSyncDatabase( - {required Schema schema, - required String path, - Logger? logger, - @Deprecated("Use [PowerSyncDatabase.withFactory] instead.") - // ignore: deprecated_member_use_from_same_package - SqliteConnectionSetup? sqliteSetup}) { + factory PowerSyncDatabase({ + required Schema schema, + required String path, + Logger? logger, + bool manualSchemaManagement = false, + @Deprecated("Use [PowerSyncDatabase.withFactory] instead.") + // ignore: deprecated_member_use_from_same_package + SqliteConnectionSetup? sqliteSetup, + }) { return PowerSyncDatabaseImpl( - schema: schema, - path: path, - logger: logger, - // ignore: deprecated_member_use_from_same_package - sqliteSetup: sqliteSetup); + schema: schema, + path: path, + manualSchemaManagement: manualSchemaManagement, + logger: logger, + // ignore: deprecated_member_use_from_same_package + sqliteSetup: sqliteSetup, + ); } /// Open a [PowerSyncDatabase] with a [PowerSyncOpenFactory]. @@ -55,12 +59,20 @@ abstract class PowerSyncDatabase /// Subclass [PowerSyncOpenFactory] to add custom logic to this process. /// /// [logger] defaults to [autoLogger], which logs to the console in debug builds. - factory PowerSyncDatabase.withFactory(DefaultSqliteOpenFactory openFactory, - {required Schema schema, - int maxReaders = SqliteDatabase.defaultMaxReaders, - Logger? logger}) { - return PowerSyncDatabaseImpl.withFactory(openFactory, - schema: schema, maxReaders: maxReaders, logger: logger); + factory PowerSyncDatabase.withFactory( + DefaultSqliteOpenFactory openFactory, { + required Schema schema, + int maxReaders = SqliteDatabase.defaultMaxReaders, + bool manualSchemaManagement = false, + Logger? logger, + }) { + return PowerSyncDatabaseImpl.withFactory( + openFactory, + schema: schema, + maxReaders: maxReaders, + manualSchemaManagement: manualSchemaManagement, + logger: logger, + ); } /// Open a PowerSyncDatabase on an existing [SqliteDatabase]. @@ -68,11 +80,17 @@ abstract class PowerSyncDatabase /// Migrations are run on the database when this constructor is called. /// /// [logger] defaults to [autoLogger], which logs to the console in debug builds. - factory PowerSyncDatabase.withDatabase( - {required Schema schema, - required SqliteDatabase database, - Logger? loggers}) { + factory PowerSyncDatabase.withDatabase({ + required Schema schema, + required SqliteDatabase database, + bool manualSchemaManagement = false, + Logger? logger, + }) { return PowerSyncDatabaseImpl.withDatabase( - schema: schema, database: database, logger: loggers); + schema: schema, + database: database, + manualSchemaManagement: manualSchemaManagement, + logger: logger, + ); } } diff --git a/packages/powersync_core/lib/src/database/powersync_database_impl_stub.dart b/packages/powersync_core/lib/src/database/powersync_database_impl_stub.dart index 2a795497..ee3ab2af 100644 --- a/packages/powersync_core/lib/src/database/powersync_database_impl_stub.dart +++ b/packages/powersync_core/lib/src/database/powersync_database_impl_stub.dart @@ -32,6 +32,9 @@ class PowerSyncDatabaseImpl @override SqliteDatabase get database => throw UnimplementedError(); + @override + bool get manualSchemaManagement => throw UnimplementedError(); + @override Future get isInitialized => throw UnimplementedError(); @@ -53,6 +56,7 @@ class PowerSyncDatabaseImpl {required Schema schema, required String path, int maxReaders = SqliteDatabase.defaultMaxReaders, + bool manualSchemaManagement = false, Logger? logger, @Deprecated("Use [PowerSyncDatabase.withFactory] instead.") // ignore: deprecated_member_use_from_same_package @@ -72,6 +76,7 @@ class PowerSyncDatabaseImpl DefaultSqliteOpenFactory openFactory, { required Schema schema, int maxReaders = SqliteDatabase.defaultMaxReaders, + bool manualSchemaManagement = false, Logger? logger, }) { throw UnimplementedError(); @@ -82,10 +87,12 @@ class PowerSyncDatabaseImpl /// Migrations are run on the database when this constructor is called. /// /// [logger] defaults to [autoLogger], which logs to the console in debug builds.s - factory PowerSyncDatabaseImpl.withDatabase( - {required Schema schema, - required SqliteDatabase database, - Logger? logger}) { + factory PowerSyncDatabaseImpl.withDatabase({ + required Schema schema, + required SqliteDatabase database, + bool manualSchemaManagement = false, + Logger? logger, + }) { throw UnimplementedError(); } diff --git a/packages/powersync_core/lib/src/database/powersync_db_mixin.dart b/packages/powersync_core/lib/src/database/powersync_db_mixin.dart index 808efc71..ae63823e 100644 --- a/packages/powersync_core/lib/src/database/powersync_db_mixin.dart +++ b/packages/powersync_core/lib/src/database/powersync_db_mixin.dart @@ -38,6 +38,10 @@ mixin PowerSyncDatabaseMixin implements SqliteConnection { /// Use [attachedLogger] to propagate logs to [Logger.root] for custom logging. Logger get logger; + bool get manualSchemaManagement; + + bool _manualSchemaManagementCompleted = false; + @Deprecated("This field is unused, pass params to connect() instead") Map? clientParams; @@ -110,10 +114,36 @@ mixin PowerSyncDatabaseMixin implements SqliteConnection { statusStream = statusStreamController.stream; updates = powerSyncUpdateNotifications(database.updates); + _manualSchemaManagementCompleted = false; + await database.initialize(); await _checkVersion(); await database.execute('SELECT powersync_init()'); - await updateSchema(schema); + + if (!manualSchemaManagement) { + // Create the internal db schema + await updateSchema(schema); + await _afterSchemaReady(); + } + } + + Future markSchemaAsReady() async { + await isInitialized; + _manualSchemaManagementCompleted = true; + + await _afterSchemaReady(); + } + + void _assertSchemaIsReady() { + if (!manualSchemaManagement || _manualSchemaManagementCompleted) { + return; + } + + throw AssertionError( + 'In manual schema management mode, you need to mark the powersync database as ready'); + } + + Future _afterSchemaReady() async { await _updateHasSynced(); } @@ -289,6 +319,8 @@ mixin PowerSyncDatabaseMixin implements SqliteConnection { // the lock for the connection. await initialize(); + _assertSchemaIsReady(); + final resolvedOptions = ResolvedSyncOptions.resolve( options, crudThrottleTime: crudThrottleTime, @@ -452,6 +484,7 @@ mixin PowerSyncDatabaseMixin implements SqliteConnection { /// Get an unique id for this client. /// This id is only reset when the database is deleted. Future getClientId() async { + _assertSchemaIsReady(); // TODO(skilldevs): Needed? final row = await get('SELECT powersync_client_id() as client_id'); return row['client_id'] as String; } @@ -459,6 +492,7 @@ mixin PowerSyncDatabaseMixin implements SqliteConnection { /// Get upload queue size estimate and count. Future getUploadQueueStats( {bool includeSize = false}) async { + _assertSchemaIsReady(); if (includeSize) { final row = await getOptional( 'SELECT SUM(cast(data as blob) + 20) as size, count(*) as count FROM ps_crud'); @@ -486,6 +520,7 @@ mixin PowerSyncDatabaseMixin implements SqliteConnection { /// data by transaction. One batch may contain data from multiple transactions, /// and a single transaction may be split over multiple batches. Future getCrudBatch({int limit = 100}) async { + _assertSchemaIsReady(); final rows = await getAll( 'SELECT id, tx_id, data FROM ps_crud ORDER BY id ASC LIMIT ?', [limit + 1]); @@ -532,6 +567,7 @@ mixin PowerSyncDatabaseMixin implements SqliteConnection { /// Unlike [getCrudBatch], this only returns data from a single transaction at a time. /// All data for the transaction is loaded into memory. Future getNextCrudTransaction() async { + _assertSchemaIsReady(); return await readTransaction((tx) async { final first = await tx.getOptional( 'SELECT id, tx_id, data FROM ps_crud ORDER BY id ASC LIMIT 1'); diff --git a/packages/powersync_core/lib/src/database/web/web_powersync_database.dart b/packages/powersync_core/lib/src/database/web/web_powersync_database.dart index fb410caa..8132d8ce 100644 --- a/packages/powersync_core/lib/src/database/web/web_powersync_database.dart +++ b/packages/powersync_core/lib/src/database/web/web_powersync_database.dart @@ -38,6 +38,9 @@ class PowerSyncDatabaseImpl @override SqliteDatabase database; + @override + bool manualSchemaManagement; + @override @protected late Future isInitialized; @@ -69,14 +72,20 @@ class PowerSyncDatabaseImpl {required Schema schema, required String path, int maxReaders = SqliteDatabase.defaultMaxReaders, + bool manualSchemaManagement = false, Logger? logger, @Deprecated("Use [PowerSyncDatabase.withFactory] instead.") // ignore: deprecated_member_use_from_same_package SqliteConnectionSetup? sqliteSetup}) { // ignore: deprecated_member_use_from_same_package DefaultSqliteOpenFactory factory = PowerSyncOpenFactory(path: path); - return PowerSyncDatabaseImpl.withFactory(factory, - maxReaders: maxReaders, logger: logger, schema: schema); + return PowerSyncDatabaseImpl.withFactory( + factory, + maxReaders: maxReaders, + logger: logger, + schema: schema, + manualSchemaManagement: manualSchemaManagement, + ); } /// Open a [PowerSyncDatabase] with a [PowerSyncOpenFactory]. @@ -91,10 +100,15 @@ class PowerSyncDatabaseImpl DefaultSqliteOpenFactory openFactory, {required Schema schema, int maxReaders = SqliteDatabase.defaultMaxReaders, + bool manualSchemaManagement = false, Logger? logger}) { final db = SqliteDatabase.withFactory(openFactory, maxReaders: 1); return PowerSyncDatabaseImpl.withDatabase( - schema: schema, logger: logger, database: db); + schema: schema, + manualSchemaManagement: manualSchemaManagement, + logger: logger, + database: db, + ); } /// Open a PowerSyncDatabase on an existing [SqliteDatabase]. @@ -102,8 +116,12 @@ class PowerSyncDatabaseImpl /// Migrations are run on the database when this constructor is called. /// /// [logger] defaults to [autoLogger], which logs to the console in debug builds. - PowerSyncDatabaseImpl.withDatabase( - {required this.schema, required this.database, Logger? logger}) { + PowerSyncDatabaseImpl.withDatabase({ + required this.schema, + required this.database, + this.manualSchemaManagement = false, + Logger? logger, + }) { if (logger != null) { this.logger = logger; } else { From 54cf1b9bc1865ea764c7acfc876f629491f122d6 Mon Sep 17 00:00:00 2001 From: David Martos Date: Sun, 8 Jun 2025 12:12:20 +0200 Subject: [PATCH 4/6] name params --- packages/powersync_core/lib/src/schema.dart | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/powersync_core/lib/src/schema.dart b/packages/powersync_core/lib/src/schema.dart index 3f8722ef..575aacc6 100644 --- a/packages/powersync_core/lib/src/schema.dart +++ b/packages/powersync_core/lib/src/schema.dart @@ -322,11 +322,11 @@ class RawTable { final PendingStatement put; final PendingStatement delete; - const RawTable( - this.name, - this.put, - this.delete, - ); + const RawTable({ + required this.name, + required this.put, + required this.delete, + }); Map toJson() => { 'name': name, From d287e3d5fa16d98e5db7173dfb29bc69dff62891 Mon Sep 17 00:00:00 2001 From: Jorge Sardina Date: Wed, 11 Jun 2025 12:22:20 +0200 Subject: [PATCH 5/6] pass schema to sync worker --- .../native/native_powersync_database.dart | 9 ++++--- .../database/web/web_powersync_database.dart | 3 ++- .../lib/src/sync/streaming_sync.dart | 7 +++-- .../lib/src/web/sync_controller.dart | 5 +++- .../lib/src/web/sync_worker.dart | 27 ++++++++++++------- .../lib/src/web/sync_worker_protocol.dart | 6 ++++- .../test/utils/abstract_test_utils.dart | 6 +++-- 7 files changed, 41 insertions(+), 22 deletions(-) diff --git a/packages/powersync_core/lib/src/database/native/native_powersync_database.dart b/packages/powersync_core/lib/src/database/native/native_powersync_database.dart index f4be54e3..94e3669b 100644 --- a/packages/powersync_core/lib/src/database/native/native_powersync_database.dart +++ b/packages/powersync_core/lib/src/database/native/native_powersync_database.dart @@ -1,4 +1,5 @@ import 'dart:async'; +import 'dart:convert'; import 'dart:isolate'; import 'package:meta/meta.dart'; @@ -266,7 +267,7 @@ class PowerSyncDatabaseImpl options, crudMutex.shared, syncMutex.shared, - schema, + jsonEncode(schema), ), debugName: 'Sync ${database.openFactory.path}', onError: receiveUnhandledErrors.sendPort, @@ -310,7 +311,7 @@ class _PowerSyncDatabaseIsolateArgs { final ResolvedSyncOptions options; final SerializedMutex crudMutex; final SerializedMutex syncMutex; - final Schema schema; + final String schemaJson; _PowerSyncDatabaseIsolateArgs( this.sPort, @@ -318,7 +319,7 @@ class _PowerSyncDatabaseIsolateArgs { this.options, this.crudMutex, this.syncMutex, - this.schema, + this.schemaJson, ); } @@ -414,7 +415,7 @@ Future _syncIsolate(_PowerSyncDatabaseIsolateArgs args) async { final storage = BucketStorage(connection); final sync = StreamingSyncImplementation( adapter: storage, - schema: args.schema, + schemaJson: args.schemaJson, connector: InternalConnector( getCredentialsCached: getCredentialsCached, prefetchCredentials: prefetchCredentials, diff --git a/packages/powersync_core/lib/src/database/web/web_powersync_database.dart b/packages/powersync_core/lib/src/database/web/web_powersync_database.dart index 8132d8ce..a879d9e0 100644 --- a/packages/powersync_core/lib/src/database/web/web_powersync_database.dart +++ b/packages/powersync_core/lib/src/database/web/web_powersync_database.dart @@ -1,4 +1,5 @@ import 'dart:async'; +import 'dart:convert'; import 'package:meta/meta.dart'; import 'package:http/browser_client.dart'; import 'package:logging/logging.dart'; @@ -159,7 +160,7 @@ class PowerSyncDatabaseImpl sync = StreamingSyncImplementation( adapter: storage, - schema: schema, + schemaJson: jsonEncode(schema), connector: InternalConnector.wrap(connector, this), crudUpdateTriggerStream: crudStream, options: options, diff --git a/packages/powersync_core/lib/src/sync/streaming_sync.dart b/packages/powersync_core/lib/src/sync/streaming_sync.dart index 82527fd8..b4dbe068 100644 --- a/packages/powersync_core/lib/src/sync/streaming_sync.dart +++ b/packages/powersync_core/lib/src/sync/streaming_sync.dart @@ -5,7 +5,6 @@ import 'dart:typed_data'; import 'package:http/http.dart' as http; import 'package:logging/logging.dart'; import 'package:meta/meta.dart'; -import 'package:powersync_core/powersync_core.dart'; import 'package:powersync_core/src/abort_controller.dart'; import 'package:powersync_core/src/exceptions.dart'; import 'package:powersync_core/src/log_internal.dart'; @@ -33,7 +32,7 @@ abstract interface class StreamingSync { @internal class StreamingSyncImplementation implements StreamingSync { - final Schema? schema; //TODO(SkillDevs): pass in all implementations + final String schemaJson; final BucketStorage adapter; final InternalConnector connector; final ResolvedSyncOptions options; @@ -64,7 +63,7 @@ class StreamingSyncImplementation implements StreamingSync { String? clientId; StreamingSyncImplementation({ - required this.schema, + required this.schemaJson, required this.adapter, required this.connector, required this.crudUpdateTriggerStream, @@ -595,7 +594,7 @@ final class _ActiveRustStreamingIteration { 'start', convert.json.encode({ 'parameters': sync.options.params, - 'schema': sync.schema, + 'schema': convert.json.decode(sync.schemaJson), }), ); assert(_completedStream.isCompleted, 'Should have started streaming'); diff --git a/packages/powersync_core/lib/src/web/sync_controller.dart b/packages/powersync_core/lib/src/web/sync_controller.dart index 0c26252e..7f05cff3 100644 --- a/packages/powersync_core/lib/src/web/sync_controller.dart +++ b/packages/powersync_core/lib/src/web/sync_controller.dart @@ -113,6 +113,9 @@ class SyncWorkerHandle implements StreamingSync { @override Future streamingSync() async { await _channel.startSynchronization( - database.database.openFactory.path, ResolvedSyncOptions(options)); + database.database.openFactory.path, + ResolvedSyncOptions(options), + database.schema, + ); } } diff --git a/packages/powersync_core/lib/src/web/sync_worker.dart b/packages/powersync_core/lib/src/web/sync_worker.dart index 5ee3c4ca..ddc4eaf0 100644 --- a/packages/powersync_core/lib/src/web/sync_worker.dart +++ b/packages/powersync_core/lib/src/web/sync_worker.dart @@ -45,12 +45,16 @@ class _SyncWorker { }); } - _SyncRunner referenceSyncTask( - String databaseIdentifier, SyncOptions options, _ConnectedClient client) { + _SyncRunner referenceSyncTask(String databaseIdentifier, SyncOptions options, + String schemaJson, _ConnectedClient client) { return _requestedSyncTasks.putIfAbsent(databaseIdentifier, () { return _SyncRunner(databaseIdentifier); }) - ..registerClient(client, options); + ..registerClient( + client, + options, + schemaJson, + ); } } @@ -86,8 +90,8 @@ class _ConnectedClient { }, ); - _runner = _worker.referenceSyncTask( - request.databaseName, recoveredOptions, this); + _runner = _worker.referenceSyncTask(request.databaseName, + recoveredOptions, request.schemaJson, this); return (JSObject(), null); case SyncWorkerMessageType.abortSynchronization: _runner?.disconnectClient(this); @@ -128,6 +132,7 @@ class _ConnectedClient { class _SyncRunner { final String identifier; ResolvedSyncOptions options = ResolvedSyncOptions(SyncOptions()); + String schemaJson = '{}'; final StreamGroup<_RunnerEvent> _group = StreamGroup(); final StreamController<_RunnerEvent> _mainEvents = StreamController(); @@ -146,10 +151,12 @@ class _SyncRunner { case _AddConnection( :final client, :final options, + :final schemaJson, ): connections.add(client); final (newOptions, reconnect) = this.options.applyFrom(options); this.options = newOptions; + this.schemaJson = schemaJson; if (sync == null) { await _requestDatabase(client); @@ -264,7 +271,7 @@ class _SyncRunner { sync = StreamingSyncImplementation( adapter: WebBucketStorage(database), - schema: null, + schemaJson: client._runner!.schemaJson, connector: InternalConnector( getCredentialsCached: client.channel.credentialsCallback, prefetchCredentials: ({required bool invalidate}) async { @@ -287,8 +294,9 @@ class _SyncRunner { sync!.streamingSync(); } - void registerClient(_ConnectedClient client, SyncOptions options) { - _mainEvents.add(_AddConnection(client, options)); + void registerClient( + _ConnectedClient client, SyncOptions options, String schemaJson) { + _mainEvents.add(_AddConnection(client, options, schemaJson)); } /// Remove a client, disconnecting if no clients remain.. @@ -307,8 +315,9 @@ sealed class _RunnerEvent {} final class _AddConnection implements _RunnerEvent { final _ConnectedClient client; final SyncOptions options; + final String schemaJson; - _AddConnection(this.client, this.options); + _AddConnection(this.client, this.options, this.schemaJson); } final class _RemoveConnection implements _RunnerEvent { diff --git a/packages/powersync_core/lib/src/web/sync_worker_protocol.dart b/packages/powersync_core/lib/src/web/sync_worker_protocol.dart index 2b859e53..3c64d90f 100644 --- a/packages/powersync_core/lib/src/web/sync_worker_protocol.dart +++ b/packages/powersync_core/lib/src/web/sync_worker_protocol.dart @@ -3,6 +3,7 @@ import 'dart:convert'; import 'dart:js_interop'; import 'package:logging/logging.dart'; +import 'package:powersync_core/src/schema.dart'; import 'package:powersync_core/src/sync/options.dart'; import 'package:web/web.dart'; @@ -71,6 +72,7 @@ extension type StartSynchronization._(JSObject _) implements JSObject { required int requestId, required int retryDelayMs, required String implementationName, + required String schemaJson, String? syncParamsEncoded, }); @@ -79,6 +81,7 @@ extension type StartSynchronization._(JSObject _) implements JSObject { external int get crudThrottleTimeMs; external int? get retryDelayMs; external String? get implementationName; + external String get schemaJson; external String? get syncParamsEncoded; } @@ -410,7 +413,7 @@ final class WorkerCommunicationChannel { } Future startSynchronization( - String databaseName, ResolvedSyncOptions options) async { + String databaseName, ResolvedSyncOptions options, Schema schema) async { final (id, completion) = _newRequest(); port.postMessage(SyncWorkerMessage( type: SyncWorkerMessageType.startSynchronization.name, @@ -420,6 +423,7 @@ final class WorkerCommunicationChannel { retryDelayMs: options.retryDelay.inMilliseconds, requestId: id, implementationName: options.source.syncImplementation.name, + schemaJson: jsonEncode(schema), syncParamsEncoded: switch (options.source.params) { null => null, final params => jsonEncode(params), diff --git a/packages/powersync_core/test/utils/abstract_test_utils.dart b/packages/powersync_core/test/utils/abstract_test_utils.dart index b0ebd7ba..f402bb5f 100644 --- a/packages/powersync_core/test/utils/abstract_test_utils.dart +++ b/packages/powersync_core/test/utils/abstract_test_utils.dart @@ -1,3 +1,5 @@ +import 'dart:convert'; + import 'package:http/http.dart'; import 'package:logging/logging.dart'; import 'package:powersync_core/powersync_core.dart'; @@ -74,7 +76,7 @@ abstract mixin class TestPowerSyncFactory implements PowerSyncOpenFactory { schema: schema, database: SqliteDatabase.singleConnection( SqliteConnection.synchronousWrapper(raw)), - loggers: logger, + logger: logger, ); } } @@ -153,7 +155,7 @@ extension MockSync on PowerSyncDatabase { }) { final impl = StreamingSyncImplementation( adapter: BucketStorage(this), - schema: schema, + schemaJson: jsonEncode(schema), client: client, options: ResolvedSyncOptions(options), connector: InternalConnector.wrap(connector, this), From 58de160ae028bd669ea7ca99993ca70349c8ac4e Mon Sep 17 00:00:00 2001 From: Jorge Sardina Date: Wed, 11 Jun 2025 12:37:25 +0200 Subject: [PATCH 6/6] review comments --- .../lib/src/database/powersync_db_mixin.dart | 14 +++++++------- packages/powersync_core/lib/src/schema.dart | 18 +++++++++++------- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/packages/powersync_core/lib/src/database/powersync_db_mixin.dart b/packages/powersync_core/lib/src/database/powersync_db_mixin.dart index ae63823e..64744797 100644 --- a/packages/powersync_core/lib/src/database/powersync_db_mixin.dart +++ b/packages/powersync_core/lib/src/database/powersync_db_mixin.dart @@ -134,12 +134,12 @@ mixin PowerSyncDatabaseMixin implements SqliteConnection { await _afterSchemaReady(); } - void _assertSchemaIsReady() { + void _checkSchemaIsReady() { if (!manualSchemaManagement || _manualSchemaManagementCompleted) { return; } - throw AssertionError( + throw StateError( 'In manual schema management mode, you need to mark the powersync database as ready'); } @@ -319,7 +319,7 @@ mixin PowerSyncDatabaseMixin implements SqliteConnection { // the lock for the connection. await initialize(); - _assertSchemaIsReady(); + _checkSchemaIsReady(); final resolvedOptions = ResolvedSyncOptions.resolve( options, @@ -484,7 +484,7 @@ mixin PowerSyncDatabaseMixin implements SqliteConnection { /// Get an unique id for this client. /// This id is only reset when the database is deleted. Future getClientId() async { - _assertSchemaIsReady(); // TODO(skilldevs): Needed? + _checkSchemaIsReady(); // TODO(skilldevs): Needed? final row = await get('SELECT powersync_client_id() as client_id'); return row['client_id'] as String; } @@ -492,7 +492,7 @@ mixin PowerSyncDatabaseMixin implements SqliteConnection { /// Get upload queue size estimate and count. Future getUploadQueueStats( {bool includeSize = false}) async { - _assertSchemaIsReady(); + _checkSchemaIsReady(); if (includeSize) { final row = await getOptional( 'SELECT SUM(cast(data as blob) + 20) as size, count(*) as count FROM ps_crud'); @@ -520,7 +520,7 @@ mixin PowerSyncDatabaseMixin implements SqliteConnection { /// data by transaction. One batch may contain data from multiple transactions, /// and a single transaction may be split over multiple batches. Future getCrudBatch({int limit = 100}) async { - _assertSchemaIsReady(); + _checkSchemaIsReady(); final rows = await getAll( 'SELECT id, tx_id, data FROM ps_crud ORDER BY id ASC LIMIT ?', [limit + 1]); @@ -567,7 +567,7 @@ mixin PowerSyncDatabaseMixin implements SqliteConnection { /// Unlike [getCrudBatch], this only returns data from a single transaction at a time. /// All data for the transaction is loaded into memory. Future getNextCrudTransaction() async { - _assertSchemaIsReady(); + _checkSchemaIsReady(); return await readTransaction((tx) async { final first = await tx.getOptional( 'SELECT id, tx_id, data FROM ps_crud ORDER BY id ASC LIMIT 1'); diff --git a/packages/powersync_core/lib/src/schema.dart b/packages/powersync_core/lib/src/schema.dart index 575aacc6..d5273c38 100644 --- a/packages/powersync_core/lib/src/schema.dart +++ b/packages/powersync_core/lib/src/schema.dart @@ -316,9 +316,8 @@ class Column { Map toJson() => {'name': name, 'type': type.sqlite}; } -class RawTable { - final String - name; // TODO: it does not need to be the same name as the raw table +final class RawTable { + final String name; final PendingStatement put; final PendingStatement delete; @@ -335,7 +334,7 @@ class RawTable { }; } -class PendingStatement { +final class PendingStatement { final String sql; final List params; @@ -348,12 +347,15 @@ class PendingStatement { } sealed class PendingStatementValue { + factory PendingStatementValue.id() = _PendingStmtValueId; + factory PendingStatementValue.column(String column) = _PendingStmtValueColumn; + dynamic toJson(); } -class PendingStmtValueColumn extends PendingStatementValue { +class _PendingStmtValueColumn implements PendingStatementValue { final String column; - PendingStmtValueColumn(this.column); + const _PendingStmtValueColumn(this.column); @override dynamic toJson() { @@ -363,7 +365,9 @@ class PendingStmtValueColumn extends PendingStatementValue { } } -class PendingStmtValueId extends PendingStatementValue { +class _PendingStmtValueId implements PendingStatementValue { + const _PendingStmtValueId(); + @override dynamic toJson() { return 'Id';