From 08d6dfc3e7920169df8d1d7f4aa92eef6ddb1479 Mon Sep 17 00:00:00 2001 From: JaffaKetchup Date: Wed, 14 Feb 2024 23:02:13 +0000 Subject: [PATCH] Add support to ObjectBox backend for root statistics Removed v6 -> v7 migrator Removed `RootManagement` --- lib/flutter_map_tile_caching.dart | 3 - lib/src/backend/impls/objectbox/backend.dart | 14 +- lib/src/backend/impls/objectbox/worker.dart | 38 +++ lib/src/backend/interfaces/backend.dart | 16 ++ lib/src/misc/exts.dart | 29 -- lib/src/root/directory.dart | 13 +- lib/src/root/manage.dart | 45 ---- lib/src/root/migrator.dart | 266 +------------------ lib/src/root/statistics.dart | 59 +--- 9 files changed, 81 insertions(+), 402 deletions(-) delete mode 100644 lib/src/misc/exts.dart delete mode 100644 lib/src/root/manage.dart diff --git a/lib/flutter_map_tile_caching.dart b/lib/flutter_map_tile_caching.dart index 0161d451..98d8a2c1 100644 --- a/lib/flutter_map_tile_caching.dart +++ b/lib/flutter_map_tile_caching.dart @@ -29,7 +29,6 @@ import 'package:isar/isar.dart'; import 'package:latlong2/latlong.dart'; import 'package:meta/meta.dart'; import 'package:path/path.dart' as path; -import 'package:path_provider/path_provider.dart'; import 'package:stream_transform/stream_transform.dart'; import 'package:watcher/watcher.dart'; @@ -38,7 +37,6 @@ import 'src/bulk_download/instance.dart'; import 'src/bulk_download/rate_limited_stream.dart'; import 'src/bulk_download/tile_loops/shared.dart'; import 'src/errors/browsing.dart'; -import 'src/misc/exts.dart'; import 'src/misc/int_extremes.dart'; import 'src/misc/obscure_query_params.dart'; import 'src/providers/image_provider.dart'; @@ -64,7 +62,6 @@ part 'src/regions/recovered_region.dart'; part 'src/regions/rectangle.dart'; part 'src/root/directory.dart'; part 'src/root/import.dart'; -part 'src/root/manage.dart'; part 'src/root/migrator.dart'; part 'src/root/recovery.dart'; part 'src/root/statistics.dart'; diff --git a/lib/src/backend/impls/objectbox/backend.dart b/lib/src/backend/impls/objectbox/backend.dart index 2bdf9237..e889a1b2 100644 --- a/lib/src/backend/impls/objectbox/backend.dart +++ b/lib/src/backend/impls/objectbox/backend.dart @@ -89,7 +89,7 @@ class _ObjectBoxBackendImpl implements FMTCObjectBoxBackendInternal { Future?> _sendCmd({ required _WorkerCmdType type, - required Map args, + Map args = const {}, }) async { expectInitialised; @@ -199,6 +199,18 @@ class _ObjectBoxBackendImpl implements FMTCObjectBoxBackendInternal { FMTCBackendAccess.internal = null; } + @override + Future> listStores() async => + (await _sendCmd(type: _WorkerCmdType.storeExists))!['stores']; + + @override + Future rootSize() async => + (await _sendCmd(type: _WorkerCmdType.rootSize))!['size']; + + @override + Future rootLength() async => + (await _sendCmd(type: _WorkerCmdType.rootLength))!['length']; + @override Future storeExists({ required String storeName, diff --git a/lib/src/backend/impls/objectbox/worker.dart b/lib/src/backend/impls/objectbox/worker.dart index 67dbae7a..e5df4fba 100644 --- a/lib/src/backend/impls/objectbox/worker.dart +++ b/lib/src/backend/impls/objectbox/worker.dart @@ -6,6 +6,9 @@ part of 'backend.dart'; enum _WorkerCmdType { initialise_, // Only valid as a response destroy_, // Only valid as a request + listStores, + rootSize, + rootLength, storeExists, createStore, resetStore, @@ -132,6 +135,41 @@ Future _worker( // TODO: Consider final message Isolate.exit(); + case _WorkerCmdType.listStores: + final query = root + .box() + .query() + .build() + .property(ObjectBoxStore_.name); + + sendRes(id: cmd.id, data: {'stores': query.find()}); + + query.close(); + + break; + case _WorkerCmdType.rootSize: + final query = root + .box() + .query() + .build() + .property(ObjectBoxStore_.size); + + sendRes( + id: cmd.id, + data: {'size': query.find().sum / 1024}, // Convert to KiB + ); + + query.close(); + + break; + case _WorkerCmdType.rootLength: + final query = root.box().query().build(); + + sendRes(id: cmd.id, data: {'length': query.count()}); + + query.close(); + + break; case _WorkerCmdType.storeExists: final query = root .box() diff --git a/lib/src/backend/interfaces/backend.dart b/lib/src/backend/interfaces/backend.dart index ddb6db3c..1986d954 100644 --- a/lib/src/backend/interfaces/backend.dart +++ b/lib/src/backend/interfaces/backend.dart @@ -70,6 +70,22 @@ abstract interface class FMTCBackendInternal with FMTCBackendAccess { /// initialised. Directory? get rootDirectory; + /// {@template fmtc.backend.listStores} + /// List all the available stores + /// {@endtemplate} + Future> listStores(); + + /// {@template fmtc.backend.rootSize} + /// Retrieve the total number of KiBs of all tiles' bytes (not 'real total' + /// size) from all stores + /// {@endtemplate} + Future rootSize(); + + /// {@template fmtc.backend.rootLength} + /// Retrieve the total number of tiles in all stores + /// {@endtemplate} + Future rootLength(); + /// {@template fmtc.backend.storeExists} /// Check whether the specified store currently exists /// {@endtemplate} diff --git a/lib/src/misc/exts.dart b/lib/src/misc/exts.dart deleted file mode 100644 index d9484144..00000000 --- a/lib/src/misc/exts.dart +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright © Luka S (JaffaKetchup) under GPL-v3 -// A full license can be found at .\LICENSE - -import 'dart:io'; - -import 'package:meta/meta.dart'; -import 'package:path/path.dart' as p; - -@internal -extension DirectoryExtensions on Directory { - String operator >(String sub) => p.join( - absolute.path, - sub, - ); - - Directory operator >>(String sub) => Directory( - p.join( - absolute.path, - sub, - ), - ); - - File operator >>>(String name) => File( - p.join( - absolute.path, - name, - ), - ); -} diff --git a/lib/src/root/directory.dart b/lib/src/root/directory.dart index 8e03967a..30e639d2 100644 --- a/lib/src/root/directory.dart +++ b/lib/src/root/directory.dart @@ -18,19 +18,16 @@ only. It will be removed in a future version. ) typedef RootDirectory = FMTCRoot; -/// Provides access to management, statistics, recovery, migration (and the -/// import functionality) on the intitialised root. +/// Provides access to statistics, recovery, migration (and the import +/// functionality) on the intitialised root. +/// +/// Management services are not provided here, instead use methods on the backend +/// directly. /// /// Note that this does not provide direct access to any [FMTCStore]s. abstract class FMTCRoot { const FMTCRoot._(); - /// Manage the root's representation on the filesystem - /// - /// To create, initialise FMTC. Assume that FMTC is ready after initialisation - /// and before [RootManagement.delete] is called. - static RootManagement get manage => const RootManagement._(); - /// Get statistics about this root (and all sub-stores) static RootStats get stats => const RootStats._(); diff --git a/lib/src/root/manage.dart b/lib/src/root/manage.dart deleted file mode 100644 index e372ce1d..00000000 --- a/lib/src/root/manage.dart +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright © Luka S (JaffaKetchup) under GPL-v3 -// A full license can be found at .\LICENSE - -part of flutter_map_tile_caching; - -/// Manages a [FMTCRoot]'s representation on the filesystem, such as -/// creation and deletion -class RootManagement { - const RootManagement._(); - - /// Unintialise/close open databases, and delete the root directory and its - /// contents - /// - /// This will remove all traces of this root from the user's device. Use with - /// caution! - Future delete() async { - await FMTCRegistry.instance.uninitialise(delete: true); - await FMTC.instance.rootDirectory.directory.delete(recursive: true); - FMTC._instance = null; - } - - /// Reset the root directory, database, and stores - /// - /// Internally calls [delete] then re-initialises FMTC with the same root - /// directory, [FMTCSettings], and debug mode. Other setup is lost: need to - /// further customise the [FlutterMapTileCaching.initialise]? Use [delete], - /// then re-initialise yourself. - /// - /// This will remove all traces of this root from the user's device. Use with - /// caution! - /// - /// Returns the new [FlutterMapTileCaching] instance. - Future reset() async { - final directory = FMTC.instance.rootDirectory.directory.absolute.path; - final settings = FMTC.instance.settings; - final debugMode = FMTC.instance.debugMode; - - await delete(); - return FMTC.initialise( - rootDirectory: directory, - settings: settings, - debugMode: debugMode, - ); - } -} diff --git a/lib/src/root/migrator.dart b/lib/src/root/migrator.dart index 68a93fd0..fd3159cc 100644 --- a/lib/src/root/migrator.dart +++ b/lib/src/root/migrator.dart @@ -1,273 +1,11 @@ // Copyright © Luka S (JaffaKetchup) under GPL-v3 // A full license can be found at .\LICENSE -// ignore_for_file: comment_references - part of flutter_map_tile_caching; /// Manage migration for file structure across FMTC versions +/// +/// There is no migration available to v9. class RootMigrator { const RootMigrator._(); - - /// Migrates a v6 file structure to a v7 structure - /// - /// Note that this method can be slow on large tilesets, so it's best to offer - /// a choice to your users as to whether they would like to migrate, or just - /// lose all stored tiles. - /// - /// Checks within `getApplicationDocumentsDirectory()` and - /// `getTemporaryDirectory()` for a directory named 'fmtc'. Alternatively, - /// specify a [customDirectory] to search for 'fmtc' within. - /// - /// In order to migrate the tiles to the new format, [urlTemplates] must be - /// used. Pass every URL template used to store any of the tiles that might be - /// in the store. Specifying an empty list will use the preset OSM tile servers - /// only. - /// - /// Set [deleteOldStructure] to `false` to keep the old structure. If a store - /// exists with the same name, it will not be overwritten, and the - /// [deleteOldStructure] parameter will be followed regardless. - /// - /// Only supports placeholders in the normal flutter_map form, those that meet - /// the RegEx: `\{ *([\w_-]+) *\}`. Only supports tiles that were sanitised - /// with the default sanitiser included in FMTC. - /// - /// Recovery information and cached statistics will be lost. - /// - /// Returns `null` if no structure root was found, otherwise a [Map] of the - /// store names to the number of failed tiles (tiles that could not be matched - /// to any of the [urlTemplates]), or `null` if it was skipped because there - /// was an existing store with the same name. A successful migration will have - /// all values 0. - Future?> fromV6({ - required List urlTemplates, - Directory? customDirectory, - bool deleteOldStructure = true, - }) async { - // Prepare the migration regular expressions - final placeholderRegex = RegExp(r'\{ *([\w_-]+) *\}'); - final matchables = [ - ...[ - 'https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png', - 'https://tile.openstreetmap.org/{z}/{x}/{y}.png', - ...urlTemplates, - ].map((url) { - final sanitised = _defaultFilesystemSanitiser(url).validOutput; - - return [ - sanitised.replaceAll('.', r'\.').replaceAll(placeholderRegex, '.+?'), - sanitised, - url, - ]; - }), - ]; - - // Search for the previous structure - final normal = (await getApplicationDocumentsDirectory()) >> 'fmtc'; - final temporary = (await getTemporaryDirectory()) >> 'fmtc'; - final custom = customDirectory == null ? null : customDirectory >> 'fmtc'; - final root = await normal.exists() - ? normal - : await temporary.exists() - ? temporary - : custom == null - ? null - : await custom.exists() - ? custom - : null; - if (root == null) return null; - - // Delete recovery files and cached statistics - if (deleteOldStructure) { - final oldRecovery = root >> 'recovery'; - if (await oldRecovery.exists()) await oldRecovery.delete(recursive: true); - final oldStats = root >> 'stats'; - if (await oldStats.exists()) await oldStats.delete(recursive: true); - } - - // Don't continue migration if there are no stores - final oldStores = root >> 'stores'; - if (!await oldStores.exists()) return {}; - - // Prepare results map - final Map results = {}; - - // Migrate stores - await for (final storeDirectory - in oldStores.list().whereType()) { - final name = path.basename(storeDirectory.absolute.path); - results[name] = 0; - - // Ignore this store if a counterpart already exists - if (FMTC.instance(name).manage.ready) { - results[name] = null; - continue; - } - await FMTC.instance(name).manage.createAsync(); - final store = FMTCRegistry.instance(name); - - // Migrate tiles in transaction batches of 250 - await for (final List tiles - in (storeDirectory >> 'tiles').list().whereType().slices(250)) { - await store.writeTxn( - () async => store.tiles.putAll( - (await Future.wait( - tiles.map( - (f) async { - final filename = path.basename(f.absolute.path); - final Map placeholderValues = {}; - - for (final e in matchables) { - if (!RegExp('^${e[0]}\$', multiLine: true) - .hasMatch(filename)) { - continue; - } - - String filenameChangable = filename; - List filenameSplit = filename.split('')..add(''); - - for (final match in placeholderRegex.allMatches(e[1])) { - final templateValue = - e[1].substring(match.start, match.end); - final afterChar = (e[1].split('')..add(''))[match.end]; - - final memory = StringBuffer(); - int i = match.start; - for (; filenameSplit[i] != afterChar; i++) { - memory.write(filenameSplit[i]); - } - filenameChangable = filenameChangable.replaceRange( - match.start, - i, - templateValue, - ); - filenameSplit = filenameChangable.split('')..add(''); - - placeholderValues[templateValue.substring( - 1, - templateValue.length - 1, - )] = memory.toString(); - } - - return DbTile( - url: e[2].replaceAllMapped( - TileProvider.templatePlaceholderElement, - (match) { - final value = placeholderValues[match.group(1)!]; - if (value != null) return value; - throw ArgumentError( - 'Missing value for placeholder: {${match.group(1)}}', - ); - }, - ), - bytes: await f.readAsBytes(), - ); - } - - results[name] = results[name]! + 1; - return null; - }, - ), - )) - .nonNulls - .toList(), - ), - ); - } - - // Migrate metadata - await store.writeTxn( - () async => store.metadata.putAll( - await (storeDirectory >> 'metadata') - .list() - .whereType() - .asyncMap( - (f) async => DbMetadata( - name: path.basename(f.absolute.path).split('.metadata')[0], - data: await f.readAsString(), - ), - ) - .toList(), - ), - ); - } - - // Delete store files - if (deleteOldStructure && await oldStores.exists()) { - await oldStores.delete(recursive: true); - } - - return results; - } -} - -//! OLD FILESYSTEM SANITISER CODE !// - -_FilesystemSanitiserResult _defaultFilesystemSanitiser(String input) { - final List errorMessages = []; - String validOutput = input; - - // Apply other character rules with general RegExp - validOutput = validOutput.replaceAll(RegExp(r'[\\\\/\:\*\?\"\<\>\|]'), '_'); - if (validOutput != input) { - errorMessages - .add('The name cannot contain invalid characters: \'[NUL]\\/:*?"<>|\''); - } - - // Trim - validOutput = validOutput.trim(); - if (validOutput != input) { - errorMessages.add('The name cannot contain leading and/or trailing spaces'); - } - - // Ensure is not empty - if (validOutput.isEmpty) { - errorMessages.add('The name cannot be empty'); - validOutput = '_'; - } - - // Ensure is not just '.' - if (validOutput.replaceAll('.', '').isEmpty) { - errorMessages.add('The name cannot consist of only periods (.)'); - validOutput = validOutput.replaceAll('.', '_'); - } - - // Reduce string to under 255 chars (keeps end) - if (validOutput.length > 255) { - validOutput = validOutput.substring(validOutput.length - 255); - if (validOutput != input) { - errorMessages.add('The name cannot contain more than 255 characters'); - } - } - - return _FilesystemSanitiserResult( - validOutput: validOutput, - errorMessages: errorMessages, - ); -} - -class _FilesystemSanitiserResult { - final String validOutput; - final List errorMessages; - - _FilesystemSanitiserResult({ - required this.validOutput, - this.errorMessages = const [], - }); - - @override - String toString() => - 'FilesystemSanitiserResult(validOutput: $validOutput, errorMessages: $errorMessages)'; - - @override - bool operator ==(Object other) { - if (identical(this, other)) return true; - - return other is _FilesystemSanitiserResult && - other.validOutput == validOutput && - listEquals(other.errorMessages, errorMessages); - } - - @override - int get hashCode => Object.hash(validOutput, errorMessages); } diff --git a/lib/src/root/statistics.dart b/lib/src/root/statistics.dart index f38a32ea..527d0e0a 100644 --- a/lib/src/root/statistics.dart +++ b/lib/src/root/statistics.dart @@ -7,60 +7,15 @@ part of flutter_map_tile_caching; class RootStats { const RootStats._(); - FMTCRegistry get _registry => FMTCRegistry.instance; + /// {@macro fmtc.backend.listStores} + Future> get storesAvailable async => + FMTCBackendAccess.internal.listStores().then((s) => s.map(FMTCStore.new)); - /// List all the available [FMTCStore]s synchronously - /// - /// Prefer [storesAvailableAsync] to avoid blocking the UI thread. Otherwise, - /// this has slightly better performance. - List get storesAvailable => _registry.storeDatabases.values - .map((e) => FMTCStore._(e.descriptorSync.name, autoCreate: false)) - .toList(); - - /// List all the available [FMTCStore]s asynchronously - Future> get storesAvailableAsync => Future.wait( - _registry.storeDatabases.values.map( - (e) async => - FMTCStore._((await e.descriptor).name, autoCreate: false), - ), - ); - - /// Retrieve the total size of all stored tiles in kibibytes (KiB) - /// synchronously - /// - /// Prefer [rootSizeAsync] to avoid blocking the UI thread. Otherwise, this - /// has slightly better performance. - /// - /// Internally sums up the size of all stores (using [StoreStats.storeSize]). - double get rootSize => - storesAvailable.map((e) => e.stats.storeSize).sum / 1024; + /// {@macro fmtc.backend.rootSize} + Future get rootSize async => FMTCBackendAccess.internal.rootSize(); - /// Retrieve the total size of all stored tiles in kibibytes (KiB) - /// asynchronously - /// - /// Internally sums up the size of all stores (using - /// [StoreStats.storeSizeAsync]). - Future get rootSizeAsync async => - (await Future.wait(storesAvailable.map((e) => e.stats.storeSizeAsync))) - .sum / - 1024; - - /// Retrieve the number of all stored tiles synchronously - /// - /// Prefer [rootLengthAsync] to avoid blocking the UI thread. Otherwise, this - /// has slightly better performance. - /// - /// Internally sums up the length of all stores (using - /// [StoreStats.storeLength]). - int get rootLength => storesAvailable.map((e) => e.stats.storeLength).sum; - - /// Retrieve the number of all stored tiles asynchronously - /// - /// Internally sums up the length of all stores (using - /// [StoreStats.storeLengthAsync]). - Future get rootLengthAsync async => - (await Future.wait(storesAvailable.map((e) => e.stats.storeLengthAsync))) - .sum; + /// {@macro fmtc.backend.rootLength} + Future get rootLength async => FMTCBackendAccess.internal.rootLength(); /// Watch for changes in the current root ///