From 038644d129f36f28344611938c54d8d49529ff03 Mon Sep 17 00:00:00 2001 From: Peter Sorotokin Date: Fri, 10 Jan 2025 12:24:46 -0800 Subject: [PATCH] Generic Storage interface for use on all client and on the server. (#834) Signed-off-by: Peter Sorotokin --- gradle/libs.versions.toml | 6 + ...orageTest.kt => AndroidBaseStorageTest.kt} | 0 identity/build.gradle.kts | 30 + .../storage/testStorageList.android.kt | 50 ++ .../storage/android/AndroidStorage.kt | 64 +++ .../storage/android/AndroidStorageTable.kt | 214 +++++++ .../storage/testStorageList.android.kt | 14 + .../identity/storage/sqlite/SqliteStorage.kt | 50 ++ .../storage/sqlite/SqliteStorageTable.kt | 241 ++++++++ .../identity/storage/testStorageList.apple.kt | 66 +++ .../storage/KeyExistsStorageException.kt | 3 + .../storage/NoRecordStorageException.kt | 3 + .../com/android/identity/storage/Storage.kt | 23 + .../identity/storage/StorageException.kt | 3 + .../android/identity/storage/StorageTable.kt | 127 +++++ .../identity/storage/StorageTableSpec.kt | 76 +++ .../identity/storage/base/BaseStorage.kt | 114 ++++ .../identity/storage/base/BaseStorageTable.kt | 55 ++ .../storage/base/SqlStatementMaker.kt | 193 +++++++ .../storage/ephemeral/EphemeralStorage.kt | 13 + .../ephemeral/EphemeralStorageTable.kt | 211 +++++++ .../android/identity/storage/StorageTest.kt | 539 ++++++++++++++++++ .../identity/storage/testStorageList.kt | 16 + .../com/android/identity/crypto/CryptoJvm.kt | 28 +- .../android/identity/crypto/EcPublicKeyJvm.kt | 20 +- .../identity/storage/jdbc/JdbcStorage.kt | 111 ++++ .../identity/storage/jdbc/JdbcStorageTable.kt | 233 ++++++++ .../identity/storage/GenericStorageTest.kt | 0 .../identity/storage/testStorageList.jvm.kt | 55 ++ 29 files changed, 2542 insertions(+), 16 deletions(-) rename identity-android/src/androidTest/java/com/android/identity/android/storage/{AndroidStorageTest.kt => AndroidBaseStorageTest.kt} (100%) create mode 100644 identity/src/androidInstrumentedTest/kotlin/com/android/identity/storage/testStorageList.android.kt create mode 100644 identity/src/androidMain/kotlin/com/android/identity/storage/android/AndroidStorage.kt create mode 100644 identity/src/androidMain/kotlin/com/android/identity/storage/android/AndroidStorageTable.kt create mode 100644 identity/src/androidUnitTest/kotlin/com/android/identity/storage/testStorageList.android.kt create mode 100644 identity/src/appleMain/kotlin/com/android/identity/storage/sqlite/SqliteStorage.kt create mode 100644 identity/src/appleMain/kotlin/com/android/identity/storage/sqlite/SqliteStorageTable.kt create mode 100644 identity/src/appleTest/kotlin/com/android/identity/storage/testStorageList.apple.kt create mode 100644 identity/src/commonMain/kotlin/com/android/identity/storage/KeyExistsStorageException.kt create mode 100644 identity/src/commonMain/kotlin/com/android/identity/storage/NoRecordStorageException.kt create mode 100644 identity/src/commonMain/kotlin/com/android/identity/storage/Storage.kt create mode 100644 identity/src/commonMain/kotlin/com/android/identity/storage/StorageException.kt create mode 100644 identity/src/commonMain/kotlin/com/android/identity/storage/StorageTable.kt create mode 100644 identity/src/commonMain/kotlin/com/android/identity/storage/StorageTableSpec.kt create mode 100644 identity/src/commonMain/kotlin/com/android/identity/storage/base/BaseStorage.kt create mode 100644 identity/src/commonMain/kotlin/com/android/identity/storage/base/BaseStorageTable.kt create mode 100644 identity/src/commonMain/kotlin/com/android/identity/storage/base/SqlStatementMaker.kt create mode 100644 identity/src/commonMain/kotlin/com/android/identity/storage/ephemeral/EphemeralStorage.kt create mode 100644 identity/src/commonMain/kotlin/com/android/identity/storage/ephemeral/EphemeralStorageTable.kt create mode 100644 identity/src/commonTest/kotlin/com/android/identity/storage/StorageTest.kt create mode 100644 identity/src/commonTest/kotlin/com/android/identity/storage/testStorageList.kt create mode 100644 identity/src/jvmMain/kotlin/com/android/identity/storage/jdbc/JdbcStorage.kt create mode 100644 identity/src/jvmMain/kotlin/com/android/identity/storage/jdbc/JdbcStorageTable.kt rename identity/src/{commonTest => jvmTest}/kotlin/com/android/identity/storage/GenericStorageTest.kt (100%) create mode 100644 identity/src/jvmTest/kotlin/com/android/identity/storage/testStorageList.jvm.kt diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 639a84ea0..89f65451c 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -48,6 +48,7 @@ zxing = "3.5.3" gretty = "4.1.4" hsqldb = "2.7.2" mysql = "8.0.16" +postgresql = "42.7.4" compose-junit4 = "1.6.8" compose-test-manifest = "1.6.8" androidx-fragment = "1.8.0" @@ -61,6 +62,7 @@ cameraLifecycle = "1.3.4" buildconfig = "5.3.5" qrose = "1.0.1" easyqrscan = "0.2.0" +androidx-sqlite = "2.5.0-alpha12" [libraries] face-detection = { module = "com.google.mlkit:face-detection", version.ref = "faceDetection" } @@ -135,6 +137,10 @@ jetbrains-lifecycle-viewmodel-compose = { module = "org.jetbrains.androidx.lifec camera-lifecycle = { group = "androidx.camera", name = "camera-lifecycle", version.ref = "cameraLifecycle" } qrose = { group = "io.github.alexzhirkevich", name = "qrose", version.ref="qrose"} easyqrscan = { module = "io.github.kalinjul.easyqrscan:scanner", version.ref = "easyqrscan" } +androidx-sqlite = { module="androidx.sqlite:sqlite", version.ref = "androidx-sqlite" } +androidx-sqlite-framework = { module="androidx.sqlite:sqlite-framework", version.ref = "androidx-sqlite" } +androidx-sqlite-bundled = { module="androidx.sqlite:sqlite-bundled", version.ref = "androidx-sqlite" } +postgresql = { module="org.postgresql:postgresql", version.ref = "postgresql" } [bundles] google-play-services = ["play-services-base", "play-services-basement", "play-services-tasks"] diff --git a/identity-android/src/androidTest/java/com/android/identity/android/storage/AndroidStorageTest.kt b/identity-android/src/androidTest/java/com/android/identity/android/storage/AndroidBaseStorageTest.kt similarity index 100% rename from identity-android/src/androidTest/java/com/android/identity/android/storage/AndroidStorageTest.kt rename to identity-android/src/androidTest/java/com/android/identity/android/storage/AndroidBaseStorageTest.kt diff --git a/identity/build.gradle.kts b/identity/build.gradle.kts index 685e6d661..9334f5f25 100644 --- a/identity/build.gradle.kts +++ b/identity/build.gradle.kts @@ -113,14 +113,44 @@ kotlin { } } + val appleMain by getting { + dependencies { + // This dependency is needed for SqliteStorage implementation. + // KMP-compatible version is still alpha and it is not compatible with + // other androidx packages, particularly androidx.work that we use in wallet. + // TODO: once compatibility issues are resolved, SqliteStorage and this + // dependency can be moved into commonMain. + implementation(libs.androidx.sqlite) + } + } + + val jvmTest by getting { + dependencies { + implementation(libs.hsqldb) + implementation(libs.mysql) + implementation(libs.postgresql) + } + } + val androidInstrumentedTest by getting { + dependsOn(commonTest) dependencies { + implementation(libs.androidx.sqlite) + implementation(libs.androidx.sqlite.framework) + implementation(libs.androidx.sqlite.bundled) implementation(libs.androidx.test.junit) implementation(libs.androidx.espresso.core) implementation(libs.compose.junit4) } } + val appleTest by getting { + dependencies { + implementation(libs.androidx.sqlite) + implementation(libs.androidx.sqlite.framework) + implementation(libs.androidx.sqlite.bundled) + } + } } } diff --git a/identity/src/androidInstrumentedTest/kotlin/com/android/identity/storage/testStorageList.android.kt b/identity/src/androidInstrumentedTest/kotlin/com/android/identity/storage/testStorageList.android.kt new file mode 100644 index 000000000..efb497a05 --- /dev/null +++ b/identity/src/androidInstrumentedTest/kotlin/com/android/identity/storage/testStorageList.android.kt @@ -0,0 +1,50 @@ +package com.android.identity.storage + +import android.app.Instrumentation +import androidx.test.platform.app.InstrumentationRegistry +import com.android.identity.storage.android.AndroidStorage +import com.android.identity.storage.ephemeral.EphemeralStorage +import kotlinx.datetime.Clock +import java.io.File + +/** + * Creates a list of empty [Storage] objects for testing. + */ +actual fun createTransientStorageList(testClock: Clock): List { + return listOf( + EphemeralStorage(testClock), + /* + TODO: this can be enabled once SqliteStorage is moved into commonMain + com.android.identity.storage.sqlite.SqliteStorage( + connection = AndroidSQLiteDriver().open(":memory:"), + clock = testClock + ), + com.android.identity.storage.sqlite.SqliteStorage( + connection = BundledSQLiteDriver().open(":memory:"), + clock = testClock, + // bundled sqlite crashes when used with Dispatchers.IO + coroutineContext = newSingleThreadContext("DB") + ), + */ + AndroidStorage( + databasePath = null, + clock = testClock, + keySize = 3 + ) + ) +} + +val knownNames = mutableSetOf() + +actual fun createPersistentStorage(name: String, testClock: Clock): Storage? { + val context = InstrumentationRegistry.getInstrumentation().context + val dbFile = context.getDatabasePath("$name.db") + if (knownNames.add(name)) { + dbFile.delete() + } + return AndroidStorage( + databasePath = dbFile.absolutePath, + clock = testClock, + keySize = 3 + ) +} \ No newline at end of file diff --git a/identity/src/androidMain/kotlin/com/android/identity/storage/android/AndroidStorage.kt b/identity/src/androidMain/kotlin/com/android/identity/storage/android/AndroidStorage.kt new file mode 100644 index 000000000..9543d83df --- /dev/null +++ b/identity/src/androidMain/kotlin/com/android/identity/storage/android/AndroidStorage.kt @@ -0,0 +1,64 @@ +package com.android.identity.storage.android + +import android.database.sqlite.SQLiteDatabase +import com.android.identity.storage.Storage +import com.android.identity.storage.base.BaseStorage +import com.android.identity.storage.base.BaseStorageTable +import com.android.identity.storage.StorageTableSpec +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.async +import kotlinx.datetime.Clock +import kotlin.coroutines.CoroutineContext + +/** + * [Storage] implementation based on Android [SQLiteDatabase] API. + */ +class AndroidStorage: BaseStorage { + private val coroutineContext: CoroutineContext + private val databaseFactory: () -> SQLiteDatabase + internal val keySize: Int + private var database: SQLiteDatabase? = null + + constructor( + database: SQLiteDatabase, + clock: Clock, + coroutineContext: CoroutineContext = Dispatchers.IO, + keySize: Int = 9 + ): super(clock) { + this.database = database + databaseFactory = { throw IllegalStateException("unexpected call") } + this.coroutineContext = coroutineContext + this.keySize = keySize + } + + constructor( + databasePath: String?, + clock: Clock, + coroutineContext: CoroutineContext = Dispatchers.IO, + keySize: Int = 9 + ): super(clock) { + databaseFactory = { + SQLiteDatabase.openOrCreateDatabase(databasePath ?: ":memory:", null) + } + this.coroutineContext = coroutineContext + this.keySize = keySize + } + + override suspend fun createTable(tableSpec: StorageTableSpec): BaseStorageTable { + if (database == null) { + database = databaseFactory() + } + val table = AndroidStorageTable(this, tableSpec) + table.init() + return table + } + + internal suspend fun withDatabase( + block: suspend CoroutineScope.(database: SQLiteDatabase) -> T + ): T { + return CoroutineScope(coroutineContext).async { + block(database!!) + }.await() + } +} \ No newline at end of file diff --git a/identity/src/androidMain/kotlin/com/android/identity/storage/android/AndroidStorageTable.kt b/identity/src/androidMain/kotlin/com/android/identity/storage/android/AndroidStorageTable.kt new file mode 100644 index 000000000..711c6cd96 --- /dev/null +++ b/identity/src/androidMain/kotlin/com/android/identity/storage/android/AndroidStorageTable.kt @@ -0,0 +1,214 @@ +package com.android.identity.storage.android + +import android.content.ContentValues +import android.database.AbstractWindowedCursor +import android.database.CursorWindow +import android.os.Build +import com.android.identity.storage.KeyExistsStorageException +import com.android.identity.storage.NoRecordStorageException +import com.android.identity.storage.base.BaseStorageTable +import com.android.identity.storage.StorageTableSpec +import com.android.identity.storage.base.SqlStatementMaker +import com.android.identity.util.toBase64Url +import kotlinx.datetime.Instant +import kotlinx.io.bytestring.ByteString +import kotlin.random.Random + +internal class AndroidStorageTable( + private val owner: AndroidStorage, + spec: StorageTableSpec +): BaseStorageTable(spec) { + private val sql = SqlStatementMaker( + spec, + textType = "TEXT", + blobType = "BLOB", + longType = "INTEGER", + useReturningClause = false, + collationCharset = null + ) + + suspend fun init() { + owner.withDatabase { database -> + database.execSQL(sql.createTableStatement) + } + } + + override suspend fun get(key: String, partitionId: String?): ByteString? { + checkPartition(partitionId) + return owner.withDatabase { database -> + val cursor = database.query( + sql.tableName, + arrayOf("data"), + sql.conditionWithExpiration(owner.clock.now().epochSeconds), + whereArgs(key, partitionId), + null, + null, + null + ) + // TODO: Older OS versions don't support setting the cursor window size. + // What should we do with older OS versions? + // Also note that a large window size may lead to longer delays when loading from the + // database. + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) { + // The default window size of 2MB which is not the limit we want to be + // constrained by. + (cursor as? AbstractWindowedCursor)?.window = CursorWindow( + "Larger Window", CURSOR_WINDOW_SIZE) + } + if (cursor.moveToFirst()) { + val bytes = cursor.getBlob(0) + cursor.close() + ByteString(bytes) + } else { + cursor.close() + null + } + } + } + + override suspend fun insert( + key: String?, + data: ByteString, + partitionId: String?, + expiration: Instant + ): String { + if (key != null) { + checkKey(key) + } + checkPartition(partitionId) + checkExpiration(expiration) + return owner.withDatabase { database -> + if (key != null && spec.supportExpiration) { + // if there is an entry with this key, but it is expired, it needs to be purged. + // Purging expired keys does not interfere with operation atomicity + database.delete( + sql.tableName, + sql.purgeExpiredWithIdCondition(owner.clock.now().epochSeconds), + whereArgs(key, partitionId) + ) + } + var newKey: String + var done = false + do { + newKey = key ?: Random.nextBytes(owner.keySize).toBase64Url() + val values = ContentValues().apply { + put("id", newKey) + if (spec.supportPartitions) { + put("partitionId", partitionId) + } + if (spec.supportExpiration) { + put("expiration", expiration.epochSeconds) + } + put("data", data.toByteArray()) + } + val rowId = database.insert(sql.tableName, null, values) + if (rowId >= 0) { + done = true + } else if (key != null) { + throw KeyExistsStorageException( + "Record with ${recordDescription(key, partitionId)} already exists") + } + } while (!done) + newKey + } + } + + override suspend fun update( + key: String, + data: ByteString, + partitionId: String?, + expiration: Instant? + ) { + checkPartition(partitionId) + if (expiration != null) { + checkExpiration(expiration) + } + owner.withDatabase { database -> + val nowSeconds = owner.clock.now().epochSeconds + val values = ContentValues().apply { + if (expiration != null) { + put("expiration", expiration.epochSeconds) + } + put("data", data.toByteArray()) + } + val count = database.update( + sql.tableName, + values, + sql.conditionWithExpiration(nowSeconds), + whereArgs(key, partitionId) + ) + if (count != 1) { + throw NoRecordStorageException( + "No record with ${recordDescription(key, partitionId)}") + } + } + } + + override suspend fun delete(key: String, partitionId: String?): Boolean { + checkPartition(partitionId) + return owner.withDatabase { database -> + val nowSeconds = owner.clock.now().epochSeconds + val count = database.delete( + sql.tableName, + sql.conditionWithExpiration(nowSeconds), + whereArgs(key, partitionId) + ) + count > 0 + } + } + + override suspend fun deleteAll() { + owner.withDatabase { database -> + database.execSQL(sql.deleteAllStatement) + } + } + + override suspend fun enumerate( + partitionId: String?, + afterKey: String?, + limit: Int + ): List { + checkPartition(partitionId) + checkLimit(limit) + if (limit == 0) { + return listOf() + } + return owner.withDatabase { database -> + val cursor = database.query( + sql.tableName, + arrayOf("id"), + sql.enumerateConditionWithExpiration(owner.clock.now().epochSeconds), + whereArgs(afterKey ?: "", partitionId), + null, + null, + "id", + if (limit < Int.MAX_VALUE) "0, $limit" else null + ) + val list = mutableListOf() + while (cursor.moveToNext()) { + list.add(cursor.getString(0)) + } + cursor.close() + list + } + } + + override suspend fun purgeExpired() { + owner.withDatabase { database -> + database.execSQL(sql.purgeExpiredStatement + .replace("?", owner.clock.now().epochSeconds.toString())) + } + } + + private fun whereArgs(key: String, partitionId: String?): Array { + return if (spec.supportPartitions) { + arrayOf(key, partitionId!!) + } else { + arrayOf(key) + } + } + + companion object { + const val CURSOR_WINDOW_SIZE = 5 * 1024 * 1024L + } +} \ No newline at end of file diff --git a/identity/src/androidUnitTest/kotlin/com/android/identity/storage/testStorageList.android.kt b/identity/src/androidUnitTest/kotlin/com/android/identity/storage/testStorageList.android.kt new file mode 100644 index 000000000..b0e388716 --- /dev/null +++ b/identity/src/androidUnitTest/kotlin/com/android/identity/storage/testStorageList.android.kt @@ -0,0 +1,14 @@ +package com.android.identity.storage + +import com.android.identity.storage.ephemeral.EphemeralStorage +import kotlinx.datetime.Clock + +actual fun createTransientStorageList(testClock: Clock): List { + return listOf( + EphemeralStorage(testClock) + ) +} + +actual fun createPersistentStorage(name: String, testClock: Clock): Storage? { + return null +} \ No newline at end of file diff --git a/identity/src/appleMain/kotlin/com/android/identity/storage/sqlite/SqliteStorage.kt b/identity/src/appleMain/kotlin/com/android/identity/storage/sqlite/SqliteStorage.kt new file mode 100644 index 000000000..51011d4f8 --- /dev/null +++ b/identity/src/appleMain/kotlin/com/android/identity/storage/sqlite/SqliteStorage.kt @@ -0,0 +1,50 @@ +package com.android.identity.storage.sqlite + +import androidx.sqlite.SQLiteConnection +import com.android.identity.storage.Storage +import com.android.identity.storage.base.BaseStorage +import com.android.identity.storage.base.BaseStorageTable +import com.android.identity.storage.StorageTableSpec +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.IO +import kotlinx.coroutines.async +import kotlinx.datetime.Clock +import kotlin.coroutines.CoroutineContext + +/** + * [Storage] implementation based on Kotlin Multiplatform [SQLiteConnection] API. + * + * One limitation of [SQLiteConnection] APIs is that there is no way to get result from + * `UPDATE` and `DELETE` SQL statements. This can be worked around either using + * SQLite-specific `RETURNING` clause (without breaking atomicity) or by additional + * `SELECT` statements (this does break atomicity). + * + * Note: currently we use this implementation only for iOS as there are multiple problems + * using this code on Android: + * - required androidx.sqlite library version (2.5.0-alpha12) conflicts with some + * other commonly used Android libraries. + * - implementation supplied by AndroidSQLiteDriver lacks support for SQLite-specific + * `RETURNING` clause and thus it is not possible to guarantee truly atomic operations + * (most notably insertions with unique keys and correct return value from deletions). + */ +class SqliteStorage( + private val connection: SQLiteConnection, + clock: Clock = Clock.System, + private val coroutineContext: CoroutineContext = Dispatchers.IO, + internal val keySize: Int = 9 +): BaseStorage(clock) { + override suspend fun createTable(tableSpec: StorageTableSpec): BaseStorageTable { + val table = SqliteStorageTable(this, tableSpec) + table.init() + return table + } + + internal suspend fun withConnection( + block: suspend CoroutineScope.(connection: SQLiteConnection) -> T + ): T { + return CoroutineScope(coroutineContext).async { + block(connection) + }.await() + } +} \ No newline at end of file diff --git a/identity/src/appleMain/kotlin/com/android/identity/storage/sqlite/SqliteStorageTable.kt b/identity/src/appleMain/kotlin/com/android/identity/storage/sqlite/SqliteStorageTable.kt new file mode 100644 index 000000000..c701f8263 --- /dev/null +++ b/identity/src/appleMain/kotlin/com/android/identity/storage/sqlite/SqliteStorageTable.kt @@ -0,0 +1,241 @@ +package com.android.identity.storage.sqlite + +import androidx.sqlite.SQLiteException +import androidx.sqlite.execSQL +import androidx.sqlite.use +import com.android.identity.storage.KeyExistsStorageException +import com.android.identity.storage.NoRecordStorageException +import com.android.identity.storage.base.BaseStorageTable +import com.android.identity.storage.StorageTableSpec +import com.android.identity.storage.base.SqlStatementMaker +import com.android.identity.util.toBase64Url +import kotlinx.datetime.Instant +import kotlinx.io.bytestring.ByteString +import kotlin.random.Random + +class SqliteStorageTable( + private val owner: SqliteStorage, + spec: StorageTableSpec +): BaseStorageTable(spec) { + private var sql = SqlStatementMaker( + spec = spec, + textType = "TEXT", + blobType = "BLOB", + longType = "INTEGER", + // NB: SQLiteConnection API without RETURNING clause support lacks a way to faithfully + // implement some operations atomically. RETURNING clause is not currently supported + // when using Android (non-bundled) driver, so using this implementation on Android would + // require bundled driver. AndroidStorage implementation (which uses non-multiplatform + // sqlite APIs) is probably a better alternative. + useReturningClause = true, + collationCharset = null + ) + + internal suspend fun init() { + owner.withConnection { connection -> + connection.execSQL(sql.createTableStatement) + } + } + + override suspend fun get(key: String, partitionId: String?): ByteString? { + checkPartition(partitionId) + return owner.withConnection { connection -> + connection.prepare(sql.getStatement).use { statement -> + statement.bindText(1, key) + var index = 2 + if (spec.supportPartitions) { + statement.bindText(index++, partitionId!!) + } + if (spec.supportExpiration) { + statement.bindLong(index, owner.clock.now().epochSeconds) + } + if (statement.step()) { + ByteString(statement.getBlob(0)) + } else { + null + } + } + } + } + + override suspend fun insert( + key: String?, + data: ByteString, + partitionId: String?, + expiration: Instant + ): String { + if (key != null) { + checkKey(key) + } + checkPartition(partitionId) + checkExpiration(expiration) + return owner.withConnection { connection -> + var newKey: String + if (key != null && spec.supportExpiration) { + // if there is an entry with this key, but it is expired, it needs to be purged. + // Purging expired keys does not interfere with operation atomicity + connection.prepare(sql.purgeExpiredWithIdStatement).use { statement -> + statement.bindText(1, key) + var index = 2 + if (spec.supportPartitions) { + statement.bindText(index++, partitionId!!) + } + statement.bindLong(index, owner.clock.now().epochSeconds) + statement.step() + } + } + var done = false + // Loop until the key we generated is unique (if key is not null, we only loop once). + do { + newKey = key ?: Random.nextBytes(owner.keySize).toBase64Url() + connection.prepare(sql.insertStatement).use { statement -> + var index = 1 + if (spec.supportPartitions) { + statement.bindText(index++, partitionId!!) + } + statement.bindText(index++, newKey) + if (spec.supportExpiration) { + statement.bindLong(index++, expiration.epochSeconds) + } + statement.bindBlob(index, data.toByteArray()) + try { + // This statement will throw an exception if the key already exists. + // Unfortunately, different implementations throw different exceptions. + // When key is null (i.e. a unique one must be generated), retry, otherwise, + // fail. + statement.step() + done = true + } catch (err: SQLiteException) { /* apple */ + // nothing + } catch (err: RuntimeException) { + val errorName = err::class.simpleName + if (errorName != "SQLiteConstraintException" /* android */ + && errorName != "SQLException" /* bundled */ + ) { + throw err + } + } + if (!done && key != null) { + throw KeyExistsStorageException( + "Record with ${recordDescription(key, partitionId)} already exists" + ) + } + } + } while(!done) + newKey + } + } + + override suspend fun update( + key: String, + data: ByteString, + partitionId: String?, + expiration: Instant? + ) { + checkPartition(partitionId) + if (expiration != null) { + checkExpiration(expiration) + } + owner.withConnection { connection -> + val nowSeconds = owner.clock.now().epochSeconds + val committed = connection.prepare( + if (expiration == null) { + sql.updateStatement + } else { + sql.updateWithExpirationStatement + } + ).use { statement -> + statement.bindBlob(1, data.toByteArray()) + var index = 2 + if (expiration != null) { + statement.bindLong(index++, expiration.epochSeconds) + } + statement.bindText(index++, key) + if (spec.supportPartitions) { + statement.bindText(index++, partitionId!!) + } + if (spec.supportExpiration) { + statement.bindLong(index, nowSeconds) + } + statement.step() + } + if (!committed) { + throw NoRecordStorageException( + "No record with ${recordDescription(key, partitionId)}") + } + } + } + + override suspend fun delete(key: String, partitionId: String?): Boolean { + checkPartition(partitionId) + return owner.withConnection { connection -> + val nowSeconds = owner.clock.now().epochSeconds + connection.prepare(sql.deleteStatement).use { statement -> + statement.bindText(1, key) + var index = 2 + if (spec.supportPartitions) { + statement.bindText(index++, partitionId!!) + } + if (spec.supportExpiration) { + statement.bindLong(index, nowSeconds) + } + statement.step() + } + } + } + + override suspend fun deleteAll() { + return owner.withConnection { connection -> + connection.prepare(sql.deleteAllStatement).use { statement -> + statement.step() + } + } + } + + override suspend fun enumerate( + partitionId: String?, + afterKey: String?, + limit: Int + ): List { + checkPartition(partitionId) + checkLimit(limit) + if (limit == 0) { + return listOf() + } + return owner.withConnection { connection -> + connection.prepare( + if (limit < Int.MAX_VALUE) { + sql.enumerateWithLimitStatement + } else { + sql.enumerateStatement + } + ).use { statement -> + var index = 1 + statement.bindText(index++, afterKey ?: "") + if (spec.supportPartitions) { + statement.bindText(index++, partitionId!!) + } + if (spec.supportExpiration) { + statement.bindLong(index++, owner.clock.now().epochSeconds) + } + if (limit < Int.MAX_VALUE) { + statement.bindInt(index, limit) + } + val list = mutableListOf() + while (statement.step()) { + list.add(statement.getText(0)) + } + list + } + } + } + + override suspend fun purgeExpired() { + return owner.withConnection { connection -> + connection.prepare(sql.purgeExpiredStatement).use { statement -> + statement.bindLong(1, owner.clock.now().epochSeconds) + statement.step() + } + } + } +} \ No newline at end of file diff --git a/identity/src/appleTest/kotlin/com/android/identity/storage/testStorageList.apple.kt b/identity/src/appleTest/kotlin/com/android/identity/storage/testStorageList.apple.kt new file mode 100644 index 000000000..bde883c25 --- /dev/null +++ b/identity/src/appleTest/kotlin/com/android/identity/storage/testStorageList.apple.kt @@ -0,0 +1,66 @@ +package com.android.identity.storage + +import androidx.sqlite.driver.NativeSQLiteDriver +import androidx.sqlite.driver.bundled.BundledSQLiteDriver +import com.android.identity.storage.ephemeral.EphemeralStorage +import com.android.identity.storage.sqlite.SqliteStorage +import kotlinx.cinterop.ExperimentalForeignApi +import kotlinx.coroutines.DelicateCoroutinesApi +import kotlinx.coroutines.ExperimentalCoroutinesApi +import kotlinx.coroutines.newSingleThreadContext +import kotlinx.datetime.Clock +import platform.Foundation.NSCachesDirectory +import platform.Foundation.NSFileManager +import platform.Foundation.NSSearchPathForDirectoriesInDomains +import platform.Foundation.NSUserDomainMask + +@OptIn(ExperimentalCoroutinesApi::class, DelicateCoroutinesApi::class) +actual fun createTransientStorageList(testClock: Clock): List { + val bundledDb = BundledSQLiteDriver().open(":memory:") + val nativeDb = NativeSQLiteDriver().open(":memory:") + return listOf( + EphemeralStorage(testClock), + SqliteStorage( + connection = nativeDb, + clock = testClock, + // native sqlite crashes when used with Dispatchers.IO + coroutineContext = newSingleThreadContext("DB") + ), + SqliteStorage( + connection = bundledDb, + clock = testClock, + // bundled sqlite crashes when used with Dispatchers.IO + coroutineContext = newSingleThreadContext("DB") + ) + ) +} + +val knownNames = mutableSetOf() + +@OptIn( + ExperimentalCoroutinesApi::class, + DelicateCoroutinesApi::class, + ExperimentalForeignApi::class +) +actual fun createPersistentStorage(name: String, testClock: Clock): Storage? { + val paths = NSSearchPathForDirectoriesInDomains( + directory = NSCachesDirectory, + domainMask = NSUserDomainMask, + expandTilde = true + ) + if (paths.isEmpty()) { + throw IllegalStateException("No caches directory") + } + val dbPath = "${paths[0]}/$name.db" + if (knownNames.add(name)) { + if (NSFileManager.defaultManager.fileExistsAtPath(dbPath)) { + NSFileManager.defaultManager.removeItemAtPath(dbPath, null) + } + } + return SqliteStorage( + connection = NativeSQLiteDriver().open(dbPath), + clock = testClock, + // native sqlite crashes when used with Dispatchers.IO + coroutineContext = newSingleThreadContext("DB") + ) +} \ No newline at end of file diff --git a/identity/src/commonMain/kotlin/com/android/identity/storage/KeyExistsStorageException.kt b/identity/src/commonMain/kotlin/com/android/identity/storage/KeyExistsStorageException.kt new file mode 100644 index 000000000..a8b946348 --- /dev/null +++ b/identity/src/commonMain/kotlin/com/android/identity/storage/KeyExistsStorageException.kt @@ -0,0 +1,3 @@ +package com.android.identity.storage + +class KeyExistsStorageException(message: String): StorageException(message) \ No newline at end of file diff --git a/identity/src/commonMain/kotlin/com/android/identity/storage/NoRecordStorageException.kt b/identity/src/commonMain/kotlin/com/android/identity/storage/NoRecordStorageException.kt new file mode 100644 index 000000000..e5dfc0d91 --- /dev/null +++ b/identity/src/commonMain/kotlin/com/android/identity/storage/NoRecordStorageException.kt @@ -0,0 +1,3 @@ +package com.android.identity.storage + +class NoRecordStorageException(message: String): StorageException(message) \ No newline at end of file diff --git a/identity/src/commonMain/kotlin/com/android/identity/storage/Storage.kt b/identity/src/commonMain/kotlin/com/android/identity/storage/Storage.kt new file mode 100644 index 000000000..871399848 --- /dev/null +++ b/identity/src/commonMain/kotlin/com/android/identity/storage/Storage.kt @@ -0,0 +1,23 @@ +package com.android.identity.storage + +/** + * Storage (in most cases persistent) that holds data items. Collection of items are organized + * in named [StorageTable]s. + */ +interface Storage { + /** + * Get the table with specific name and features. + * + * In order to avoid situation where several parts of the app define a table with the + * same name, this method throws [IllegalArgumentException] when there are multiple + * [StorageTableSpec] objects that define a table with the same name. + */ + suspend fun getTable(spec: StorageTableSpec): StorageTable + + /** + * Reclaim the storage occupied by expired entries across all tables in this [Storage] + * object (even if these tables were never accessed using [getTable] in this + * session). + */ + suspend fun purgeExpired() +} \ No newline at end of file diff --git a/identity/src/commonMain/kotlin/com/android/identity/storage/StorageException.kt b/identity/src/commonMain/kotlin/com/android/identity/storage/StorageException.kt new file mode 100644 index 000000000..2503e7cca --- /dev/null +++ b/identity/src/commonMain/kotlin/com/android/identity/storage/StorageException.kt @@ -0,0 +1,3 @@ +package com.android.identity.storage + +sealed class StorageException(message: String): Exception(message) \ No newline at end of file diff --git a/identity/src/commonMain/kotlin/com/android/identity/storage/StorageTable.kt b/identity/src/commonMain/kotlin/com/android/identity/storage/StorageTable.kt new file mode 100644 index 000000000..0978b74ee --- /dev/null +++ b/identity/src/commonMain/kotlin/com/android/identity/storage/StorageTable.kt @@ -0,0 +1,127 @@ +package com.android.identity.storage + +import com.android.identity.storage.base.BaseStorageTable +import kotlinx.datetime.Instant +import kotlinx.io.bytestring.ByteString + +/** + * A storage unit that holds a collection of items. An item is a [ByteString] indexed by a + * unique key. + * + * [BaseStorageTable] has two optional features: partitioning and expiration. + * + * When the table is partitioned, each item is actually indexed by a key pair (partitionId, key). + * Keys are unique only within a particular partition. + * + * When item expiration is enabled, each item can be given optional expiration time. An item is + * (conceptually) silently and automatically deleted once clock time goes past expiration time + * (in other words, an item still exists at exactly the expiration time). + */ +interface StorageTable { + /** + * Gets data. + * + * This gets data previously stored with [StorageTable.insert]. + * + * - [key] is the key used to identify the data. + * - [partitionId] secondary key. If partitioning is supported + * (see [StorageTableSpec.supportExpiration]), it must be non-null. If partitioning is not + * supported it must be null. + * + * Returns the stored data or `null` if there is no data for the given key (including the case + * when the record has expired). + */ + suspend fun get(key: String, partitionId: String? = null): ByteString? + + /** + * Stores new data. + * + * The data can later be retrieved using [StorageTable.get]. + * + * - [key] the key used to identify the data. If null, and new unique key will be generated. + * if not null, the key (or (partitionId,key) pair if partitioning is enabled) must be unique + * and [KeyExistsStorageException] is thrown if the key (or (partitionId,key) pair) already + * exists in the table. + * - [partitionId] secondary key. If partitioning is supported + * (see [StorageTableSpec.supportExpiration]), it must be non-null. If partitioning is not + * supported it must be null. + * - [expiration] if expiration is not supported (see [StorageTableSpec]), this must be + * [Instant.DISTANT_FUTURE] which is default value. If expiration is supported this should + * be last moment of time when the newly-created record still exists. Expired records are + * not accessible using [Storage] APIs, and the storage they occupy can be reclaimed + * at any moment. + * - [data] the data to store. + * + * Returns the key for the newly-inserted record. Generated keys only contain ASCII + * alphanumeric characters. + */ + suspend fun insert( + key: String?, + data: ByteString, + partitionId: String? = null, + expiration: Instant = Instant.DISTANT_FUTURE + ): String + + /** + * Updates data that is already stored in the engine. + * + * The data can later be retrieved using [get]. The record with the given key (or + * (partitionId, key) pair when partitions are enabled) must exist in the database or + * [NoRecordStorageException] will be thrown. + * + * - [key] the key used to identify the data. + * - [partitionId] secondary key. If partitioning is supported + * (see [StorageTableSpec.supportExpiration]), it must be non-null. If partitioning is not + * supported it must be null. + * - [expiration] if expiration is not supported (see [StorageTableSpec]), this must be `null`. + * Otherwise, if expiration is given, it is updated, and if it is `null`, it is left as it + * was before. + * - [data] the data to store. + */ + suspend fun update( + key: String, + data: ByteString, + partitionId: String? = null, + expiration: Instant? = null + ) + + /** + * Deletes data. + * + * - [key] the key used to identify the data. + * - [partitionId] secondary key. If partitioning is supported in [StorageTableSpec] this must + * be non-null. If partitioning is not supported it must be null. + * + * Returns `true` if the record was found and successfully deleted. Returns `false` if + * the record was not found (including the case when it is expired). + */ + suspend fun delete( + key: String, + partitionId: String? = null, + ): Boolean + + /** + * Deletes all data previously stored in this table. + */ + suspend fun deleteAll() + + /** + * Enumerate keys of the records with given table and partitionId in key lexicographic order. + * + * - [partitionId] secondary key. If partitioning is supported + * (see [StorageTableSpec.supportExpiration]), it must be non-null. If partitioning is not + * supported it must be null. + * - [afterKey] if given only keys that follow the given key lexicographically are returned. If + * not given, enumeration starts from the lexicographically first key. + * - [limit] if given, no more than the given number of keys are returned. + * + * To enumerate a large table completely in manageable chunks, specify the desired [limit] + * to repeated [enumerate] calls and pass last key from the previously returned list as + * [afterKey]. + */ + suspend fun enumerate( + partitionId: String? = null, + afterKey: String? = null, + limit: Int = Int.MAX_VALUE + ): List +} \ No newline at end of file diff --git a/identity/src/commonMain/kotlin/com/android/identity/storage/StorageTableSpec.kt b/identity/src/commonMain/kotlin/com/android/identity/storage/StorageTableSpec.kt new file mode 100644 index 000000000..1a35de871 --- /dev/null +++ b/identity/src/commonMain/kotlin/com/android/identity/storage/StorageTableSpec.kt @@ -0,0 +1,76 @@ +package com.android.identity.storage + +import com.android.identity.cbor.Cbor +import com.android.identity.cbor.CborMap +import com.android.identity.cbor.DataItem +import com.android.identity.cbor.toDataItem +import com.android.identity.storage.base.BaseStorageTable +import kotlinx.io.bytestring.ByteString + +/** + * [StorageTable]'s name and features. + * + * NB: Once the table is created for the first time, its features must stay the same. + * - [name] name of the table, 60 characters at most, ASCII letters, digits or + * underscore, must start with a letter. Must be a unique name when compared + * with other table names in non-case-sensitive manner. + * - [supportPartitions] true if partitions are supported + * - [supportExpiration] true if expiration is supported + * - [schemaVersion] (optional) schema version of this table which typically defines + * the format of the data stored in the table. Initially it is set to 0. When the + * table is loaded for the first time, schema version is compared with the version + * saved in the storage and if there is a discrepancy, [schemaUpgrade] method is + * called. By default, this method throws [IllegalStateException], but it could, + * for instance just call [StorageTable.deleteAll] to wipe out incompatible data + * or it could change table data to make it compatible with the new schema. + */ +open class StorageTableSpec( + val name: String, + val supportPartitions: Boolean, + val supportExpiration: Boolean, + val schemaVersion: Long = 0 +) { + open suspend fun schemaUpgrade(oldTable: BaseStorageTable) { + throw IllegalStateException("Schema change is not supported for '${oldTable.spec.name}'") + } + + override fun equals(other: Any?): Boolean { + if (this === other) { + return true + } + if (other !is StorageTableSpec) { + return false + } + return name == other.name && supportPartitions == other.supportPartitions && + supportExpiration == other.supportExpiration && schemaVersion == other.schemaVersion + } + + override fun hashCode(): Int { + var result = name.hashCode() + result = 31 * result + supportPartitions.hashCode() + result = 31 * result + supportExpiration.hashCode() + result = 31 * result + schemaVersion.hashCode() + return result + } + + internal fun encodeToByteString(): ByteString { + val map = mutableMapOf() + map["name".toDataItem()] = name.toDataItem() + map["supportPartitions".toDataItem()] = supportPartitions.toDataItem() + map["supportExpiration".toDataItem()] = supportExpiration.toDataItem() + map["schemaVersion".toDataItem()] = schemaVersion.toDataItem() + return ByteString(Cbor.encode(CborMap(map))) + } + + companion object { + internal fun decodeByteString(data: ByteString): StorageTableSpec { + val map = Cbor.decode(data.toByteArray()) + return StorageTableSpec( + name = map["name"].asTstr, + supportPartitions = map["supportPartitions"].asBoolean, + supportExpiration = map["supportExpiration"].asBoolean, + schemaVersion = map["schemaVersion"].asNumber + ) + } + } +} \ No newline at end of file diff --git a/identity/src/commonMain/kotlin/com/android/identity/storage/base/BaseStorage.kt b/identity/src/commonMain/kotlin/com/android/identity/storage/base/BaseStorage.kt new file mode 100644 index 000000000..f553b7ee1 --- /dev/null +++ b/identity/src/commonMain/kotlin/com/android/identity/storage/base/BaseStorage.kt @@ -0,0 +1,114 @@ +package com.android.identity.storage.base + +import com.android.identity.storage.Storage +import com.android.identity.storage.StorageTable +import com.android.identity.storage.StorageTableSpec +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import kotlinx.datetime.Clock +import kotlinx.datetime.Instant + +/** + * Base class implementing common functionality for various [Storage] implementations. + */ +abstract class BaseStorage(val clock: Clock): Storage { + private val lock = Mutex() + private var schemaTable: BaseStorageTable? = null + + // NB: the key to the tableMap is *lowercased* table name as we want to detect collisions + // between table names in non-case sensitive manner (some implementations are case-sensitive + // and others are not). Otherwise we keep original table case for clarity. + private val tableMap = mutableMapOf() + + override suspend fun getTable(spec: StorageTableSpec): StorageTable { + if (spec.name.length > MAX_TABLE_NAME_LENGTH) { + throw IllegalArgumentException("Table name is too long") + } + if (!spec.name.matches(safeNameRegex)) { + throw IllegalArgumentException("Table name contains prohibited characters") + } + return lock.withLock { + ensureTablesLoaded() + val tableMap = this.tableMap + val existing = tableMap[spec.name.lowercase()] + if (existing == null) { + // Table never existed + val newTable = createTable(spec) + tableMap[spec.name.lowercase()] = TableEntry(newTable, spec) + schemaTable!!.insert(key = spec.name, data = spec.encodeToByteString()) + return@withLock newTable + } + if (existing.spec != null && existing.spec !== spec) { + throw IllegalArgumentException("Multiple table specs for table '${spec.name}'") + } + if (existing.table.spec == spec) { + // Known table with up-to-date schema + existing.spec = spec + existing.table + } else { + // Known table that needs to be upgraded + spec.schemaUpgrade(existing.table) + val upgradedTable = createTable(spec) + tableMap[spec.name.lowercase()] = TableEntry(upgradedTable, spec) + schemaTable!!.update(key = spec.name, data = spec.encodeToByteString()) + upgradedTable + } + } + } + + override suspend fun purgeExpired() { + val tablesToPurge = lock.withLock { + ensureTablesLoaded() + tableMap.values.filter { it.table.spec.supportExpiration }.map {it.table} .toList() + } + for (table in tablesToPurge) { + table.purgeExpired() + } + } + + private suspend fun ensureTablesLoaded() { + check(lock.isLocked) + if (schemaTable == null) { + val schemaTable = createTable(SchemaTableSpec) + this.schemaTable = schemaTable + tableMap.putAll(schemaTable.enumerate().map { name -> + val storedSpec = StorageTableSpec.decodeByteString(schemaTable.get(name)!!) + check(storedSpec.name == name) + Pair(name.lowercase(), TableEntry(createTable(storedSpec), null)) + }) + } + } + + protected abstract suspend fun createTable(tableSpec: StorageTableSpec): BaseStorageTable + + object SchemaTableSpec: StorageTableSpec( + name = "_SCHEMA", + supportPartitions = false, + supportExpiration = false + ) { + override suspend fun schemaUpgrade(oldTable: BaseStorageTable) { + throw IllegalStateException("Schema table can be never upgraded") + } + } + + protected object StoppedClock: Clock { + override fun now(): Instant = Instant.DISTANT_PAST + } + + private class TableEntry( + val table: BaseStorageTable, + // Keep the reference to the spec which was used to instantiate the table to detect + // duplicate specs for the same name. + var spec: StorageTableSpec? = null + ) + + companion object { + private val safeNameRegex = Regex("^[a-zA-Z][a-zA-Z0-9_]*\$") + + const val MAX_KEY_SIZE = 1024 + // NB: MySQL does not allow table names longer than 64 characters and we need 2 + // characters for the prefix. Without prefix we'd have to exclude SQL keywords from + // valid table names. + const val MAX_TABLE_NAME_LENGTH = 60 + } +} \ No newline at end of file diff --git a/identity/src/commonMain/kotlin/com/android/identity/storage/base/BaseStorageTable.kt b/identity/src/commonMain/kotlin/com/android/identity/storage/base/BaseStorageTable.kt new file mode 100644 index 000000000..a74f58e8e --- /dev/null +++ b/identity/src/commonMain/kotlin/com/android/identity/storage/base/BaseStorageTable.kt @@ -0,0 +1,55 @@ +package com.android.identity.storage.base + +import com.android.identity.storage.StorageTable +import com.android.identity.storage.StorageTableSpec +import kotlinx.datetime.Instant +import kotlinx.io.bytestring.ByteString + +abstract class BaseStorageTable(val spec: StorageTableSpec): StorageTable { + /** Reclaim storage that is taken up by the expired entries. */ + abstract suspend fun purgeExpired() + + protected fun checkExpiration(expiration: Instant) { + if (!this.spec.supportExpiration && expiration < Instant.DISTANT_FUTURE) { + throw IllegalArgumentException("Expiration is not supported") + } + } + + protected fun checkPartition(partitionId: String?) { + if (this.spec.supportPartitions) { + if (partitionId == null) { + throw IllegalArgumentException("partitionId is required") + } + if (partitionId.length > BaseStorage.MAX_KEY_SIZE) { + throw IllegalArgumentException("partitionId is too long") + } + } else { + if (partitionId != null) { + throw IllegalArgumentException("Partitioning is not supported") + } + } + } + + protected fun checkKey(key: String) { + if (key.isEmpty()) { + throw IllegalArgumentException("Empty key is not allowed") + } + if (key.length > BaseStorage.MAX_KEY_SIZE) { + throw IllegalArgumentException("Key is too long") + } + } + + protected fun checkLimit(limit: Int) { + if (limit < 0) { + throw IllegalArgumentException("Negative limit: $limit") + } + } + + protected fun recordDescription(key: String, partitionId: String?): String { + return if (spec.supportPartitions) { + "partitionId='$partitionId' key='$key' (table '${spec.name}')" + } else { + "key='$key' (table '${spec.name}')" + } + } +} \ No newline at end of file diff --git a/identity/src/commonMain/kotlin/com/android/identity/storage/base/SqlStatementMaker.kt b/identity/src/commonMain/kotlin/com/android/identity/storage/base/SqlStatementMaker.kt new file mode 100644 index 000000000..c35305815 --- /dev/null +++ b/identity/src/commonMain/kotlin/com/android/identity/storage/base/SqlStatementMaker.kt @@ -0,0 +1,193 @@ +package com.android.identity.storage.base + +import com.android.identity.storage.StorageTableSpec + +class SqlStatementMaker( + val spec: StorageTableSpec, + val textType: String, + val blobType: String, + val longType: String, + val useReturningClause: Boolean, + collationCharset: String? +) { + val tableName = "Mz${spec.name}" + + private val collation = if (collationCharset != null) "COLLATE latin1_bin" else "" + + private val returning get() = if (useReturningClause) { + "RETURNING 1" + } else { + "" + } + + private val expirationCondition get() = if (spec.supportExpiration) { + " AND expiration >= ?" + } else { + "" + } + + private val partitionCondition get() = if (spec.supportPartitions) { + " AND partitionId = ?" + } else { + "" + } + + private val partitionDef = if (spec.supportPartitions) { + "partitionId $textType $collation," + } else { + "" + } + + private val expirationDef = if (spec.supportExpiration) { + "expiration $longType NOT NULL," + } else { + "" + } + + private val primaryKeyDef = if (spec.supportPartitions) { + "PRIMARY KEY(partitionId, id)" + } else { + "PRIMARY KEY(id)" + } + + val createTableStatement = + """ + CREATE TABLE IF NOT EXISTS $tableName ( + $partitionDef + id $textType $collation, + $expirationDef + data $blobType, + $primaryKeyDef + ) + """.trimIndent() + + val getStatement get() = + """ + SELECT data + FROM $tableName + WHERE (id = ? $partitionCondition $expirationCondition) + """.trimIndent() + + /** + * SQL condition for the record with an id, a partition (if needed) and with + * expiration check (if needed) already injected. + * + * This is needed for older Android Sqlite APIs that have no way to inject non-string + * parameters. + */ + fun conditionWithExpiration(nowSeconds: Long): String { + val expirationCheck = if (spec.supportExpiration) { + expirationCondition.replace("?", nowSeconds.toString()) + } else { + "" + } + return "id = ? $partitionCondition $expirationCheck" + } + + val purgeExpiredWithIdStatement = + """ + DELETE + FROM $tableName + WHERE (id = ? $partitionCondition AND expiration < ?) + """ + + /** + * SQL condition for the expired record with an id, and a partition (if needed). + * + * This is needed for older Android Sqlite APIs that have no way to inject non-string + * parameters. + */ + fun purgeExpiredWithIdCondition(timeSeconds: Long): String { + return "id = ? $partitionCondition AND expiration < $timeSeconds" + } + + val insertStatement: String = run { + val names = StringBuilder() + val values = StringBuilder() + if (spec.supportPartitions) { + names.append("partitionId, ") + values.append("?, ") + } + names.append("id") + values.append("?") + if (spec.supportExpiration) { + names.append(", expiration") + values.append(", ?") + } + names.append(", data") + values.append(", ?") + "INSERT INTO $tableName ($names) VALUES($values)" + } + + val updateStatement = + """ + UPDATE $tableName SET data = ? + WHERE (id = ? $partitionCondition $expirationCondition) + $returning + """.trimIndent() + + val updateWithExpirationStatement = + """ + UPDATE $tableName SET data = ?, expiration = ? + WHERE (id = ? $partitionCondition $expirationCondition) + $returning + """.trimIndent() + + val enumerateStatement = + """ + SELECT id + FROM $tableName + WHERE (id > ? $partitionCondition $expirationCondition) + ORDER BY id + """.trimIndent() + + val enumerateWithLimitStatement = + """ + SELECT id + FROM $tableName + WHERE (id > ? $partitionCondition $expirationCondition) + ORDER BY id + LIMIT ? + """.trimIndent() + + /** + * SQL condition for the record with an id, a partition (if needed) and with + * expiration check (if needed) already injected. + * + * This is needed for older Android Sqlite APIs that have no way to inject non-string + * parameters. + */ + fun enumerateConditionWithExpiration(nowSeconds: Long): String { + val expirationCheck = if (spec.supportExpiration) { + expirationCondition.replace("?", nowSeconds.toString()) + } else { + "" + } + return "id > ? $partitionCondition $expirationCheck" + } + + val deleteOrUpdateCheckStatement = + """ + SELECT 1 FROM $tableName + WHERE (id = ? $partitionCondition $expirationCondition) + """.trimIndent() + + val deleteStatement = + """ + DELETE FROM $tableName + WHERE (id = ? $partitionCondition $expirationCondition) + $returning + """.trimIndent() + + val deleteAllStatement = + """ + DELETE FROM $tableName + """.trimIndent() + + val purgeExpiredStatement = + """ + DELETE + FROM $tableName + WHERE (expiration < ?) + """.trimIndent() +} \ No newline at end of file diff --git a/identity/src/commonMain/kotlin/com/android/identity/storage/ephemeral/EphemeralStorage.kt b/identity/src/commonMain/kotlin/com/android/identity/storage/ephemeral/EphemeralStorage.kt new file mode 100644 index 000000000..90941951f --- /dev/null +++ b/identity/src/commonMain/kotlin/com/android/identity/storage/ephemeral/EphemeralStorage.kt @@ -0,0 +1,13 @@ +package com.android.identity.storage.ephemeral + +import com.android.identity.storage.base.BaseStorage +import com.android.identity.storage.base.BaseStorageTable +import com.android.identity.storage.StorageTableSpec +import kotlinx.datetime.Clock + +class EphemeralStorage(clock: Clock = Clock.System) : BaseStorage(clock) { + override suspend fun createTable(tableSpec: StorageTableSpec): BaseStorageTable { + val clockToUse = if (tableSpec.supportExpiration) clock else StoppedClock + return EphemeralStorageTable(tableSpec, clockToUse) + } +} \ No newline at end of file diff --git a/identity/src/commonMain/kotlin/com/android/identity/storage/ephemeral/EphemeralStorageTable.kt b/identity/src/commonMain/kotlin/com/android/identity/storage/ephemeral/EphemeralStorageTable.kt new file mode 100644 index 000000000..e239da31e --- /dev/null +++ b/identity/src/commonMain/kotlin/com/android/identity/storage/ephemeral/EphemeralStorageTable.kt @@ -0,0 +1,211 @@ +package com.android.identity.storage.ephemeral + +import com.android.identity.storage.KeyExistsStorageException +import com.android.identity.storage.NoRecordStorageException +import com.android.identity.storage.base.BaseStorageTable +import com.android.identity.storage.StorageTableSpec +import com.android.identity.util.toBase64Url +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import kotlinx.datetime.Clock +import kotlinx.datetime.Instant +import kotlinx.io.bytestring.ByteString +import kotlin.math.abs +import kotlin.random.Random + +internal class EphemeralStorageTable( + spec: StorageTableSpec, + private val clock: Clock +): BaseStorageTable(spec) { + private val lock = Mutex() + private var storedData = mutableListOf() + private var earliestExpiration: Instant = Instant.DISTANT_FUTURE + + override suspend fun get(key: String, partitionId: String?): ByteString? { + checkPartition(partitionId) + return lock.withLock { + val index = storedData.binarySearch(Item(partitionId, key)) + if (index < 0) { + null + } else { + val data = storedData[index] + if (data.expired(clock.now())) null else data.value + } + } + } + + override suspend fun insert( + key: String?, + data: ByteString, + partitionId: String?, + expiration: Instant + ): String { + checkPartition(partitionId) + checkExpiration(expiration) + if (key != null) { + checkKey(key) + } + return lock.withLock { + var index: Int + var keyToUse = key + if (keyToUse == null) { + do { + keyToUse = Random.Default.nextBytes(9).toBase64Url() + index = storedData.binarySearch(Item(partitionId, keyToUse)) + } while (index >= 0) + } else { + index = storedData.binarySearch(Item(partitionId, keyToUse)) + if (index >= 0) { + val item = storedData[index] + if (item.expired(clock.now())) { + // Stale entry, can be reused + updateEarliestExpiration(expiration) + item.expiration = expiration + item.value = data + return@withLock keyToUse + } + throw KeyExistsStorageException( + "Record with ${recordDescription(key!!, partitionId)} already exists" + ) + } + } + check(index < 0) + updateEarliestExpiration(expiration) + storedData.add(-index - 1, Item(partitionId, keyToUse!!, data, expiration)) + keyToUse + } + } + + override suspend fun update( + key: String, + data: ByteString, + partitionId: String?, + expiration: Instant? + ) { + checkPartition(partitionId) + if (expiration != null) { + checkExpiration(expiration) + } + lock.withLock { + val index = storedData.binarySearch(Item(partitionId, key)) + if (index < 0) { + throw NoRecordStorageException( + "No record with ${recordDescription(key, partitionId)}") + } + val item = storedData[index] + if (item.expired(clock.now())) { + throw NoRecordStorageException( + "No record with ${recordDescription(key, partitionId)} (expired)") + } + item.value = data + if (expiration != null) { + updateEarliestExpiration(expiration) + item.expiration = expiration + } + } + } + + override suspend fun delete(key: String, partitionId: String?): Boolean { + checkPartition(partitionId) + return lock.withLock { + val index = storedData.binarySearch(Item(partitionId, key)) + if (index < 0 || storedData[index].expired(clock.now())) { + false + } else { + storedData.removeAt(index) + true + } + } + } + + override suspend fun deleteAll() { + lock.withLock { + storedData.clear() + } + } + + override suspend fun enumerate( + partitionId: String?, + afterKey: String?, + limit: Int + ): List { + checkPartition(partitionId) + checkLimit(limit) + if (limit == 0) { + return listOf() + } + return lock.withLock { + var index = if (afterKey == null) { + val spot = storedData.binarySearch(Item(partitionId, "")) + if (spot > 0) spot else -(spot + 1) + } else { + abs(storedData.binarySearch(Item(partitionId, afterKey)) + 1) + } + val now = clock.now() + val keyList = mutableListOf() + while (keyList.size < limit && index < storedData.size) { + val data = storedData[index] + if (data.partitionId != partitionId) { + break + } + if (!data.expired(now)) { + keyList.add(data.key) + } + index++ + } + keyList.toList() + } + } + + private fun updateEarliestExpiration(expiration: Instant) { + if (earliestExpiration > expiration) { + earliestExpiration = expiration + } + } + + override suspend fun purgeExpired() { + if (!spec.supportExpiration) { + throw IllegalStateException("This table does not support expiration") + } + lock.withLock { + val now = clock.now() + if (earliestExpiration < now) { + earliestExpiration = Instant.DISTANT_FUTURE + val unexpired = mutableListOf() + for (item in storedData) { + if (!item.expired(now)) { + updateEarliestExpiration(item.expiration) + unexpired.add(item) + } + } + storedData = unexpired + } + } + } + + private class Item( + val partitionId: String?, + val key: String, + var value: ByteString = EMPTY, + var expiration: Instant = Instant.DISTANT_FUTURE + ): Comparable { + override fun compareTo(other: Item): Int { + val c = if (partitionId == null) { + if (other.partitionId == null) 0 else -1 + } else if (other.partitionId == null) { + 1 + } else { + partitionId.compareTo(other.partitionId) + } + return if (c != 0) c else key.compareTo(other.key) + } + + fun expired(now: Instant): Boolean { + return expiration < now + } + } + + companion object { + val EMPTY = ByteString() + } +} \ No newline at end of file diff --git a/identity/src/commonTest/kotlin/com/android/identity/storage/StorageTest.kt b/identity/src/commonTest/kotlin/com/android/identity/storage/StorageTest.kt new file mode 100644 index 000000000..bf2b3ca1e --- /dev/null +++ b/identity/src/commonTest/kotlin/com/android/identity/storage/StorageTest.kt @@ -0,0 +1,539 @@ +package com.android.identity.storage + +import com.android.identity.storage.base.BaseStorageTable +import com.android.identity.util.toHex +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.ExperimentalCoroutinesApi +import kotlinx.coroutines.async +import kotlinx.coroutines.delay +import kotlinx.coroutines.newFixedThreadPoolContext +import kotlin.test.Test +import kotlinx.coroutines.runBlocking +import kotlinx.datetime.Clock +import kotlinx.datetime.Instant +import kotlinx.io.bytestring.ByteString +import kotlinx.io.bytestring.decodeToString +import kotlinx.io.bytestring.encodeToByteString +import kotlin.random.Random +import kotlin.test.BeforeTest +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertFalse +import kotlin.test.assertNotEquals +import kotlin.test.assertNotNull +import kotlin.test.assertNull +import kotlin.test.assertTrue +import kotlin.time.Duration.Companion.minutes + +class StorageTest { + @BeforeTest + fun resetSharedState() { + TestClock.time = Instant.DISTANT_PAST + } + + @Test + fun testSimple() { + withTable { table -> + assertEquals(0, table.enumerate().size.toLong()) + assertNull(table.get("foo")) + + val data = "Foobar".encodeToByteString() + table.insert(data = data, key = "foo") + assertEquals(table.get("foo"), data) + assertEquals(1, table.enumerate().size.toLong()) + assertEquals("foo", table.enumerate().iterator().next()) + assertNull(table.get("bar")) + + val data2 = "Buzz".encodeToByteString() + table.insert(data = data2, key = "bar") + assertEquals(table.get("bar"), data2) + assertEquals(2, table.enumerate().size.toLong()) + table.delete("foo") + assertNull(table.get("foo")) + assertNotNull(table.get("bar")) + assertEquals(1, table.enumerate().size.toLong()) + table.delete("bar") + assertNull(table.get("bar")) + assertEquals(0, table.enumerate().size.toLong()) + } + } + + @Test + fun testGetReturnsCorrectEntry() { + withTable(supportPartitions = true) { table -> + table.insert(key = null, "bad1".encodeToByteString(), partitionId = "client0") + val key = table.insert(key = null, "good".encodeToByteString(), partitionId = "client1") + table.insert(key = null, "bad2".encodeToByteString(), partitionId = "client1") + table.insert(key = null, "bad3".encodeToByteString(), partitionId = "client3") + assertNotEquals("", key) + val data = table.get(key = key, partitionId = "client1") + assertEquals("good", data!!.decodeToString()) + } + } + + @Test + fun testEnumerate() { + withTable(supportPartitions = true) { table -> + (0..19).forEach { i -> + table.insert( + key = null, + partitionId = "client0", + data = "bad$i".encodeToByteString() + ) + table.insert( + key = null, + partitionId = "client2", + data = "bad$i".encodeToByteString() + ) + } + val keys = (0..19).map { i -> + table.insert( + key = null, + partitionId = "client1", + data = "good$i".encodeToByteString() + ) + }.toSet() + assertEquals(20, keys.size) + val chunk1 = table.enumerate( + partitionId = "client1", + limit = 9 + ) + val chunk2 = table.enumerate( + partitionId = "client1", + afterKey = chunk1.last(), + limit = 14 + ) + assertEquals(9, chunk1.size) + assertEquals(11, chunk2.size) + assertEquals(keys.sorted(), chunk1 + chunk2) + for (key in keys) { + val data = table.get(partitionId = "client1", key = key)!!.decodeToString() + assertTrue(data.startsWith("good")) + } + assertEquals(listOf(), table.enumerate(partitionId = "client1", limit = 0)) + } + } + + @Test + fun testUpdate() { + withTable(supportPartitions = true) { table -> + val key = table.insert( + key = null, + partitionId = "client1", + data = "bad".encodeToByteString() + ) + table.update(key = key, partitionId = "client1", data = "good".encodeToByteString()) + val data = table.get(partitionId = "client1", key = key) + assertEquals("good", data!!.decodeToString()) + } + } + + @Test + fun testUpdateExpiration() { + withTable(supportPartitions = true, supportExpiration = true) { table -> + TestClock.time = Instant.parse("2024-12-20T11:35:00Z") + val key1 = table.insert( + key = null, + partitionId = "client1", + expiration = TestClock.now() + 2.minutes, + data = "entry1".encodeToByteString() + ) + val key2 = table.insert( + key = null, + partitionId = "client1", + expiration = TestClock.now() + 2.minutes, + data = "entry2".encodeToByteString() + ) + table.update( + key = key1, + partitionId = "client1", + expiration = TestClock.now() + 10.minutes, + data = "updated1".encodeToByteString() + ) + TestClock.time += 5.minutes + assertEquals( + "updated1".encodeToByteString(), + table.get(partitionId = "client1", key = key1) + ) + assertNull(table.get(partitionId = "client1", key = key2)) + assertEquals(listOf(key1), table.enumerate(partitionId = "client1")) + } + } + + @Test + fun testDelete() { + withTable(supportPartitions = true) { table -> + val key = table.insert(key = null, partitionId = "client1", data = "data".encodeToByteString()) + assertFalse(table.delete(partitionId = "client0", key = key)) + assertFalse(table.delete(partitionId = "client1", key = "#fake_key")) + assertTrue(table.delete(partitionId = "client1", key = key)) + val data = table.get(partitionId = "client1", key = key) + assertNull(data) + } + } + + @Test + fun testExpiration() { + withStorage { storage -> + val tableSpec = StorageTableSpec( + name = "TestExpiration${uniqueSuffix()}", + supportPartitions = false, + supportExpiration = true + ) + val table = storage.getTable(tableSpec) + TestClock.time = Instant.parse("2024-12-20T11:35:00Z") + val time3 = TestClock.time + 3.minutes + val time5 = TestClock.time + 5.minutes + val time6 = TestClock.time + 6.minutes + val time8 = TestClock.time + 8.minutes + val id8 = table.insert(null, "8.minutes".encodeToByteString(), expiration = time8) + val id3 = table.insert(null, "3.minutes".encodeToByteString(), expiration = time3) + val id5 = table.insert(null, "5.minutes".encodeToByteString(), expiration = time5) + assertEquals(setOf(id3, id5, id8), table.enumerate().toSet()) + TestClock.time = time5 + assertEquals(setOf(id5, id8), table.enumerate().toSet()) + table.insert(key = id3, data = "reused id".encodeToByteString(), expiration = time5) + assertEquals(setOf(id3, id5, id8), table.enumerate().toSet()) + TestClock.time = time6 + assertEquals(setOf(id8), table.enumerate().toSet()) + assertEquals("8.minutes", table.get(id8)!!.decodeToString()) + storage.purgeExpired() + assertEquals(setOf(id8), table.enumerate().toSet()) + } + } + + @Test + fun testExpirationWithPartitions() { + withStorage { storage -> + val tableSpec = StorageTableSpec( + name = "TestExpirationPartitions${uniqueSuffix()}", + supportPartitions = true, + supportExpiration = true + ) + val table = storage.getTable(tableSpec) + TestClock.time = Instant.parse("2024-12-20T11:35:00Z") + val id8 = table.insert(null, "8.minutes".encodeToByteString(), + partitionId = "A", expiration = TestClock.time + 8.minutes) + val id3 = table.insert(null, "3.minutes".encodeToByteString(), + partitionId = "B", expiration = TestClock.time + 3.minutes) + assertEquals(setOf(id8), table.enumerate("A").toSet()) + assertEquals("8.minutes", table.get(id8,"A")!!.decodeToString()) + assertNull(table.get(id8,"B")) + assertEquals(setOf(id3), table.enumerate("B").toSet()) + assertEquals("3.minutes", table.get(id3,"B")!!.decodeToString()) + assertNull(table.get(id3,"A")) + + TestClock.time += 4.minutes + assertEquals(setOf(id8), table.enumerate("A").toSet()) + assertEquals("8.minutes", table.get(id8,"A")!!.decodeToString()) + assertNull(table.get(id8,"B")) + assertEquals(setOf(), table.enumerate("B").toSet()) + assertNull(table.get(id3,"B")) + assertNull(table.get(id3,"A")) + } + } + + @Test + fun testDuplicateKey() { + withTable { table -> + table.insert(key = "foo", "bar".encodeToByteString()) + assertFailsWith(KeyExistsStorageException::class) { + table.insert(key = "foo", "buz".encodeToByteString()) + } + } + } + + @Test + fun testKeysCaseSensitive() { + withTable(supportPartitions = true) { table -> + table.insert(key = "foo", partitionId = "a", data = "bar".encodeToByteString()) + assertEquals("bar".encodeToByteString(), table.get(key = "foo", partitionId = "a")) + assertNull(table.get(key = "FOO", partitionId = "A")) + assertNull(table.get(key = "FOO", partitionId = "a")) + assertNull(table.get(key = "foo", partitionId = "A")) + assertFailsWith(NoRecordStorageException::class) { + table.update(key = "FOO", partitionId = "A", data = "buz".encodeToByteString()) + } + assertFailsWith(NoRecordStorageException::class) { + table.update(key = "FOO", partitionId = "a", data = "buz".encodeToByteString()) + } + assertFailsWith(NoRecordStorageException::class) { + table.update(key = "foo", partitionId = "A", data = "buz".encodeToByteString()) + } + // Non-duplicates + table.insert(key = "FOO", partitionId = "A", data = "q".encodeToByteString()) + table.insert(key = "foo", partitionId = "A", data = "r".encodeToByteString()) + table.insert(key = "FOO", partitionId = "a", data = "s".encodeToByteString()) + } + } + + @Test + fun testUpdateNonexistent() { + withTable { table -> + assertFailsWith(NoRecordStorageException::class) { + table.update(key = "foo", "buz".encodeToByteString()) + } + } + } + + @OptIn(ExperimentalCoroutinesApi::class) + @Test + fun testConcurrentInsertion() { + withTable { table -> + val result = async(newFixedThreadPoolContext(nThreads = 10, name = "inserter")) { + (0..99).map { coroutineId -> + async { + (0..99).associate { dataId -> + val value = "$coroutineId:$dataId" + val key = table.insert(key = null, value.encodeToByteString()) + delay(Random.nextLong(2)) // a bit of jitter + Pair(key, value) + } + } + } + } + val allInsertedEntries = mutableMapOf() + result.await().map { insertingJob -> + insertingJob.await() + }.forEach { insertedEntries -> + allInsertedEntries.putAll(insertedEntries) + } + + assertEquals(allInsertedEntries.keys.toSet(), table.enumerate().toSet()) + for ((key, value) in allInsertedEntries.entries) { + assertEquals(value, table.get(key)!!.decodeToString()) + } + } + } + + @Test + fun testBasicPersistence() { + val storageName = "testBasicPersistence" + val storage1 = createPersistentStorage(storageName, TestClock) ?: return + val tableSpec = StorageTableSpec("table", false, false) + runBlocking { + val table1 = storage1.getTable(tableSpec) + val key = table1.insert(key = null, data = "Hello, world!".encodeToByteString()) + val storage2 = createPersistentStorage(storageName, TestClock)!! + val table2 = storage2.getTable(tableSpec) + assertEquals("Hello, world!", table2.get(key)!!.decodeToString()) + } + } + + @Test + fun testSchemaUpdate() { + val storageName = "testSchemaUpdate" + val storage1 = createPersistentStorage(storageName, TestClock) ?: return + val spec1 = StorageTableSpec( + name = "table", + supportPartitions = false, + supportExpiration = false + ) + val spec1Uppercase = StorageTableSpec( + name = "TABLE", + supportPartitions = false, + supportExpiration = false + ) + val spec2 = object : StorageTableSpec( + name = "table", + supportPartitions = false, + supportExpiration = false, + schemaVersion = 1 + ) { + override suspend fun schemaUpgrade(oldTable: BaseStorageTable) { + val ids = oldTable.enumerate() + for (id in ids) { + val oldValue = oldTable.get(id)!!.decodeToString() + val newValue = oldValue.replace("old", "new") + oldTable.update(id, newValue.encodeToByteString()) + } + } + } + runBlocking { + val table1 = storage1.getTable(spec1) + val key1 = table1.insert(key = null, data = "old value 1".encodeToByteString()) + val key2 = table1.insert(key = null, data = "old value 2".encodeToByteString()) + val storage2 = createPersistentStorage(storageName, TestClock)!! + assertFailsWith(IllegalStateException::class) { + storage2.getTable(spec1Uppercase) + } + val storage3 = createPersistentStorage(storageName, TestClock)!! + val table3 = storage3.getTable(spec2) + assertEquals("new value 1", table3.get(key1)!!.decodeToString()) + assertEquals("new value 2", table3.get(key2)!!.decodeToString()) + val storage4 = createPersistentStorage(storageName, TestClock)!! + assertFailsWith(IllegalStateException::class) { + storage4.getTable(spec1) + } + } + } + + @Test + fun testTableName() { + withStorage { storage -> + storage.getTable(StorageTableSpec("foo", true, true)) + // Duplicate name ( + assertFailsWith(IllegalArgumentException::class) { + storage.getTable(StorageTableSpec("foo", true, true)) + } + // Duplicate name (case-insensitive comparison) + assertFailsWith(IllegalArgumentException::class) { + storage.getTable(StorageTableSpec("Foo", true, true)) + } + // Name is not too long + storage.getTable(StorageTableSpec(LONG_NAME_60, true, true)) + // Name is too long + assertFailsWith(IllegalArgumentException::class) { + storage.getTable(StorageTableSpec(LONG_NAME_60 + "X", true, true)) + } + // Name does not start with a letter + assertFailsWith(IllegalArgumentException::class) { + storage.getTable(StorageTableSpec("1", true, true)) + } + // Name contains illegal character + assertFailsWith(IllegalArgumentException::class) { + storage.getTable(StorageTableSpec("foo@", true, true)) + } + } + } + + @Test + fun testKeyValidity() { + withTable(supportPartitions = true) { table -> + val longKey = Random.nextBytes(512).toHex() + val longPartition = Random.nextBytes(512).toHex() + assertEquals(1024, longKey.length) + // no error + table.insert( + key = longKey, + partitionId = longPartition, + data = "data".encodeToByteString() + ) + // key is too long + assertFailsWith(IllegalArgumentException::class) { + table.insert( + key = longKey + "X", + partitionId = longPartition, + data = "data".encodeToByteString() + ) + } + // partition is too long + assertFailsWith(IllegalArgumentException::class) { + table.insert( + key = longKey, + partitionId = longPartition + "X", + data = "data".encodeToByteString() + ) + } + } + } + + @Test + fun testLargeData() { + withTable { table -> + val data = ByteString(Random.nextBytes(LARGE_DATA_SIZE)) + val key = table.insert(key = null, data = data) + assertEquals(data, table.get(key)) + } + } + + @Test + fun testPartitionIdRequired() { + withTable(supportPartitions = true) { table -> + // partition was not given when it is required + assertFailsWith(IllegalArgumentException::class) { + table.insert(key = null, data = "data".encodeToByteString()) + } + } + } + + @Test + fun testNoPartitionSupported() { + withTable(supportPartitions = false) { table -> + // partition is given when it is not supported + assertFailsWith(IllegalArgumentException::class) { + table.insert(key = null, partitionId = "A", data = "data".encodeToByteString()) + } + } + } + + @Test + fun testNoExpirationSupported() { + withTable(supportExpiration = false) { table -> + // expiration is given when it is not supported + assertFailsWith(IllegalArgumentException::class) { + val time = TestClock.time + 10.minutes + table.insert(key = null, expiration = time, data = "data".encodeToByteString()) + } + } + } + + @Test + fun testEnumerateNegativeLimit() { + withTable() { table -> + assertFailsWith(IllegalArgumentException::class) { + table.enumerate(limit = -5) + } + } + } + + @Test + fun testDeleteAll() { + withTable(supportExpiration = true, supportPartitions = true) { table -> + val time = TestClock.time + 10.minutes + val data = "data".encodeToByteString() + val key1 = table.insert(key = null, partitionId = "A", data = data) + val key2 = table.insert(key = null, partitionId = "A", expiration = time, data = data) + val key3 = table.insert(key = null, partitionId = "A", expiration = time, data = data) + assertEquals(setOf(key1, key2, key3), table.enumerate(partitionId = "A").toSet()) + assertEquals(data, table.get(key = key1, partitionId = "A")) + assertEquals(data, table.get(key = key2, partitionId = "A")) + assertEquals(data, table.get(key = key3, partitionId = "A")) + table.deleteAll() + assertEquals(setOf(), table.enumerate(partitionId = "A").toSet()) + assertNull(table.get(key = key1, partitionId = "A")) + assertNull(table.get(key = key2, partitionId = "A")) + assertNull(table.get(key = key3, partitionId = "A")) + } + } + + private fun withStorage(block: suspend CoroutineScope.(storage: Storage) -> Unit) { + for (storage in transientStorageList) { + runBlocking { + block(storage) + } + } + } + + private fun withTable( + supportExpiration: Boolean = false, + supportPartitions: Boolean = false, + block: suspend CoroutineScope.(table: StorageTable) -> Unit + ) { + val tableName = "TestTable${uniqueSuffix()}" + val tableSpec = StorageTableSpec(tableName, supportPartitions, supportExpiration) + withStorage { storage -> + block(storage.getTable(tableSpec)) + } + } + + private fun uniqueSuffix(): String { + val timestamp = Clock.System.now().epochSeconds.toString(36) + val count = uniqueCount++.toString(36) + return "_${timestamp}_${count}" + } + + object TestClock: Clock { + internal var time: Instant = Instant.DISTANT_PAST + override fun now(): Instant = time + } + + companion object { + var uniqueCount: Long = 0 + const val LARGE_DATA_SIZE = 4 * 1024 * 1024 // 4Mb + const val LONG_NAME_60 = "A12345678901234567890123456789012345678901234567890123456789" + val transientStorageList by lazy { + createTransientStorageList(TestClock) + } + } +} \ No newline at end of file diff --git a/identity/src/commonTest/kotlin/com/android/identity/storage/testStorageList.kt b/identity/src/commonTest/kotlin/com/android/identity/storage/testStorageList.kt new file mode 100644 index 000000000..18719f661 --- /dev/null +++ b/identity/src/commonTest/kotlin/com/android/identity/storage/testStorageList.kt @@ -0,0 +1,16 @@ +package com.android.identity.storage + +import kotlinx.datetime.Clock + +/** + * Creates a list of empty transient [Storage] objects for testing. + */ +expect fun createTransientStorageList(testClock: Clock): List + +/** + * Creates a persistent [Storage] object for testing if supported by this platform. + * + * Passing the same name will connect [Storage] to the same storage area. First time a + * particular name is used (during process lifetime), the storage area will be cleared. + */ +expect fun createPersistentStorage(name: String, testClock: Clock): Storage? \ No newline at end of file diff --git a/identity/src/javaSharedMain/kotlin/com/android/identity/crypto/CryptoJvm.kt b/identity/src/javaSharedMain/kotlin/com/android/identity/crypto/CryptoJvm.kt index 8caa8d900..c9b3341ab 100644 --- a/identity/src/javaSharedMain/kotlin/com/android/identity/crypto/CryptoJvm.kt +++ b/identity/src/javaSharedMain/kotlin/com/android/identity/crypto/CryptoJvm.kt @@ -93,6 +93,7 @@ actual object Crypto { ) init { + Security.removeProvider(BouncyCastleProvider.PROVIDER_NAME) Security.insertProviderAt(BouncyCastleProvider(), 1) } @@ -329,7 +330,12 @@ actual object Crypto { signature.toDerEncoded() } } - Signature.getInstance(signatureAlgorithm).run { + val signatureImpl = if (publicKey.curve.requireBouncyCastle) { + Signature.getInstance(signatureAlgorithm, BouncyCastleProvider.PROVIDER_NAME) + } else { + Signature.getInstance(signatureAlgorithm) + } + signatureImpl.run { initVerify(publicKey.javaPublicKey) update(message) verify(rawSignature) @@ -739,13 +745,8 @@ actual object Crypto { .replace("-----BEGIN PUBLIC KEY-----", "") .replace("-----END PUBLIC KEY-----", "") .trim()) - val kf = when (curve) { - EcCurve.ED448, - EcCurve.ED25519 -> KeyFactory.getInstance("EdDSA", BouncyCastleProvider.PROVIDER_NAME) - EcCurve.X25519, - EcCurve.X448 -> KeyFactory.getInstance("XDH", BouncyCastleProvider.PROVIDER_NAME) - else -> KeyFactory.getInstance("EC", BouncyCastleProvider.PROVIDER_NAME) - } + // Always use BouncyCastle, publicKeyJava.toEcPublicKey below would choke on anything else. + val kf = KeyFactory.getInstance(curve.javaKeyAlgorithm, BouncyCastleProvider.PROVIDER_NAME) val spec = X509EncodedKeySpec(encoded) val publicKeyJava = kf.generatePublic(spec) return publicKeyJava.toEcPublicKey(curve) @@ -769,13 +770,10 @@ actual object Crypto { .replace("-----BEGIN PRIVATE KEY-----", "") .replace("-----END PRIVATE KEY-----", "") .trim()) - val kf = when (publicKey.curve) { - EcCurve.ED448, - EcCurve.ED25519 -> KeyFactory.getInstance("EdDSA", BouncyCastleProvider.PROVIDER_NAME) - EcCurve.X25519, - EcCurve.X448 -> KeyFactory.getInstance("XDH", BouncyCastleProvider.PROVIDER_NAME) - else -> KeyFactory.getInstance("EC", BouncyCastleProvider.PROVIDER_NAME) - } + // Always use BouncyCastle, privateKeyJava.toEcPrivateKey below would + // choke on anything else. + val kf = KeyFactory.getInstance( + publicKey.curve.javaKeyAlgorithm, BouncyCastleProvider.PROVIDER_NAME) val spec = PKCS8EncodedKeySpec(encoded) val privateKeyJava = kf.generatePrivate(spec) return privateKeyJava.toEcPrivateKey(publicKey.javaPublicKey, publicKey.curve) diff --git a/identity/src/javaSharedMain/kotlin/com/android/identity/crypto/EcPublicKeyJvm.kt b/identity/src/javaSharedMain/kotlin/com/android/identity/crypto/EcPublicKeyJvm.kt index b83d9ec28..5da145813 100644 --- a/identity/src/javaSharedMain/kotlin/com/android/identity/crypto/EcPublicKeyJvm.kt +++ b/identity/src/javaSharedMain/kotlin/com/android/identity/crypto/EcPublicKeyJvm.kt @@ -54,6 +54,20 @@ private val ED448_X509_ENCODED_PREFIX = private val X448_X509_ENCODED_PREFIX = byteArrayOf(0x30, 0x42, 0x30, 0x05, 0x06, 0x03, 0x2b, 0x65, 0x6f, 0x03, 0x39, 0x00) +val EcCurve.requireBouncyCastle: Boolean + get() = when (this) { + EcCurve.BRAINPOOLP256R1, EcCurve.BRAINPOOLP384R1, + EcCurve.BRAINPOOLP320R1, EcCurve.BRAINPOOLP512R1 -> true + else -> false + } + +val EcCurve.javaKeyAlgorithm: String + get() = when(this) { + EcCurve.ED448, EcCurve.ED25519 -> "EdDSA" + EcCurve.X25519, EcCurve.X448 -> "XDH" + else -> "EC" + } + val EcPublicKey.javaPublicKey: PublicKey get() = when (this) { is EcPublicKeyDoubleCoordinate -> { @@ -71,7 +85,11 @@ val EcPublicKey.javaPublicKey: PublicKey val ecParameters = params.getParameterSpec(ECParameterSpec::class.java) val ecPoint = ECPoint(bx, by) val keySpec = ECPublicKeySpec(ecPoint, ecParameters) - val kf = KeyFactory.getInstance("EC") + val kf = if (curve.requireBouncyCastle) { + KeyFactory.getInstance("EC", BouncyCastleProvider.PROVIDER_NAME) + } else { + KeyFactory.getInstance("EC") + } kf.generatePublic(keySpec) } catch (e: Exception) { throw IllegalStateException("Unexpected error", e) diff --git a/identity/src/jvmMain/kotlin/com/android/identity/storage/jdbc/JdbcStorage.kt b/identity/src/jvmMain/kotlin/com/android/identity/storage/jdbc/JdbcStorage.kt new file mode 100644 index 000000000..02b8a16da --- /dev/null +++ b/identity/src/jvmMain/kotlin/com/android/identity/storage/jdbc/JdbcStorage.kt @@ -0,0 +1,111 @@ +package com.android.identity.storage.jdbc + +import com.android.identity.storage.base.BaseStorage +import com.android.identity.storage.base.BaseStorageTable +import com.android.identity.storage.StorageTableSpec +import com.android.identity.storage.base.SqlStatementMaker +import kotlinx.datetime.Clock +import kotlinx.datetime.Instant +import java.sql.Connection +import java.sql.DriverManager +import java.util.ArrayDeque +import java.util.concurrent.Executor +import java.util.concurrent.Executors +import kotlin.coroutines.resume +import kotlin.coroutines.resumeWithException +import kotlin.coroutines.suspendCoroutine +import kotlin.time.Duration.Companion.minutes + +class JdbcStorage( + private val jdbc: String, + private val user: String = "", + private val password: String = "", + clock: Clock = Clock.System, + private val executor: Executor = Executors.newFixedThreadPool(4), + internal val keySize: Int = 12 /* exposed for testing only */ +): BaseStorage(clock) { + private val connectionPool = ArrayDeque() + + override suspend fun createTable(tableSpec: StorageTableSpec): BaseStorageTable { + val sql = if (jdbc.startsWith("jdbc:mysql:")) { + SqlStatementMaker( + spec = tableSpec, + textType = "VARCHAR($MAX_KEY_SIZE)", + blobType = "LONGBLOB", // avoids 64k limit + longType = "BIGINT", + useReturningClause = false, + collationCharset = "latin1_bin" + ) + } else if (jdbc.startsWith("jdbc:postgresql:")) { + SqlStatementMaker( + spec = tableSpec, + textType = "TEXT", + blobType = "BYTEA", + longType = "BIGINT", + useReturningClause = false, + collationCharset = null + ) + } else { + SqlStatementMaker( + spec = tableSpec, + textType = "VARCHAR($MAX_KEY_SIZE)", + blobType = "BLOB", + longType = "BIGINT", + useReturningClause = false, + collationCharset = null + ) + } + val table = JdbcStorageTable(this, sql) + table.init() + return table + } + + internal suspend fun withConnection(block: (connection: Connection) -> T): T { + return suspendCoroutine { continuation -> + executor.execute { + val staleConnections = mutableListOf() + // only real clock makes sense here + val connectionExpiration = Clock.System.now() - MAX_CONNECTION_LIFE + val connection = synchronized(connectionPool) { + while (connectionPool.isNotEmpty()) { + val poolEntry = connectionPool.removeFirst() + if (poolEntry.timeLastUsed > connectionExpiration) { + return@synchronized poolEntry.connection + } + staleConnections.add(poolEntry.connection) + } + null + } ?: DriverManager.getConnection(jdbc, user, password) + try { + val result = block(connection) + continuation.resume(result) + synchronized(connectionPool) { + connectionPool.add( + ConnectionPoolEntry( + connection = connection, + timeLastUsed = Clock.System.now() + ) + ) + } + } catch (error: Throwable) { + continuation.resumeWithException(error) + // Close after exceptions instead of returning to the pool + connection.close() + } + for (staleConnection in staleConnections) { + try { + staleConnection.close() + } catch (err: Throwable) { + // ignore all errors + } + } + } + } + } + + class ConnectionPoolEntry(val connection: Connection, val timeLastUsed: Instant) + + companion object { + val MAX_CONNECTION_LIFE = 3.minutes + } +} \ No newline at end of file diff --git a/identity/src/jvmMain/kotlin/com/android/identity/storage/jdbc/JdbcStorageTable.kt b/identity/src/jvmMain/kotlin/com/android/identity/storage/jdbc/JdbcStorageTable.kt new file mode 100644 index 000000000..5c66c4f87 --- /dev/null +++ b/identity/src/jvmMain/kotlin/com/android/identity/storage/jdbc/JdbcStorageTable.kt @@ -0,0 +1,233 @@ +package com.android.identity.storage.jdbc + +import com.android.identity.storage.KeyExistsStorageException +import com.android.identity.storage.NoRecordStorageException +import com.android.identity.storage.base.BaseStorageTable +import com.android.identity.storage.base.SqlStatementMaker +import com.android.identity.util.toBase64Url +import kotlinx.datetime.Instant +import kotlinx.io.bytestring.ByteString +import java.sql.SQLException +import kotlin.random.Random + +class JdbcStorageTable( + private val owner: JdbcStorage, + private val sql: SqlStatementMaker +): BaseStorageTable(sql.spec) { + internal suspend fun init() { + owner.withConnection { connection -> + connection.createStatement().execute(sql.createTableStatement) + } + } + + override suspend fun get(key: String, partitionId: String?): ByteString? { + checkPartition(partitionId) + return owner.withConnection { connection -> + val statement = connection.prepareStatement(sql.getStatement) + var index = 1 + statement.setString(index++, key) + if (spec.supportPartitions) { + statement.setString(index++, partitionId!!) + } + if (spec.supportExpiration) { + statement.setLong(index, owner.clock.now().epochSeconds) + } + val resultSet = statement.executeQuery() + if (resultSet.next()) { + val bytes = resultSet.getBytes(1) + ByteString(bytes) + } else { + null + } + } + } + + + override suspend fun insert( + key: String?, + data: ByteString, + partitionId: String?, + expiration: Instant + ): String { + checkPartition(partitionId) + checkExpiration(expiration) + if (key != null) { + checkKey(key) + } + return owner.withConnection { connection -> + var newKey: String + if (key != null && spec.supportExpiration) { + // if there is an entry with this key, but it is expired, it needs to be purged + // purging expired keys does not interfere with operation atomicity + val purge = connection.prepareStatement(sql.purgeExpiredWithIdStatement) + purge.setString(1, key) + var index = 2 + if (spec.supportPartitions) { + purge.setString(index++, partitionId) + } + purge.setLong(index, owner.clock.now().epochSeconds) + purge.executeUpdate() + } + var tries = 0 + // Loop until the key we generated is unique (if key is not null, we only loop once). + while (true) { + newKey = key ?: Random.nextBytes(owner.keySize).toBase64Url() + val values = StringBuilder("?, ?") + if (spec.supportPartitions) { + values.append(", ?") + } + if (spec.supportExpiration) { + values.append(", ?") + } + val statement = connection.prepareStatement(sql.insertStatement) + var index = 1 + if (spec.supportPartitions) { + statement.setString(index++, partitionId!!) + } + statement.setString(index++, newKey) + if (spec.supportExpiration) { + statement.setLong(index++, expiration.epochSeconds) + } + statement.setBytes(index, data.toByteArray()) + val count = try { + // This statement will throw an exception if the key already exists. + // When key is null (i.e. a unique one must be generated), retry, otherwise, + // fail. + statement.executeUpdate() + } catch (err: SQLException) { + // NB: we are using a very generic exception to detect key collision error + // (every jdbc driver seem to have its own flavor, unfortunately). + // After enough tries, we should conclude that the problem is likely to + // be something else, otherwise this will become an infinite loop. + if (++tries > 21) { + throw err + } + if (key == null) { + continue + } else { + throw KeyExistsStorageException( + "Record with ${recordDescription(key, partitionId)} already exists" + ) + } + } + if (count != 1) { + throw IllegalStateException( + "Expected SQL INSERT statement to return 1, got $count") + } + statement.close() + break + } + newKey + } + } + + override suspend fun update( + key: String, + data: ByteString, + partitionId: String?, + expiration: Instant? + ) { + checkPartition(partitionId) + if (expiration != null) { + checkExpiration(expiration) + } + owner.withConnection { connection -> + val statement = connection.prepareStatement( + if (expiration != null) { + sql.updateWithExpirationStatement + } else { + sql.updateStatement + } + ) + statement.setBytes(1, data.toByteArray()) + var index = 2 + if (expiration != null) { + statement.setLong(index++, expiration.epochSeconds) + } + statement.setString(index++, key) + if (spec.supportPartitions) { + statement.setString(index++, partitionId!!) + } + if (spec.supportExpiration) { + statement.setLong(index, owner.clock.now().epochSeconds) + } + val count = statement.executeUpdate() + if (count != 1) { + throw NoRecordStorageException( + "No record with ${recordDescription(key, partitionId)}") + } + } + } + + override suspend fun delete(key: String, partitionId: String?): Boolean { + checkPartition(partitionId) + return owner.withConnection { connection -> + val statement = connection.prepareStatement(sql.deleteStatement) + statement.setString(1, key) + var index = 2 + if (spec.supportPartitions) { + statement.setString(index++, partitionId!!) + } + if (spec.supportExpiration) { + statement.setLong(index, owner.clock.now().epochSeconds) + } + val count = statement.executeUpdate() + count > 0 + } + } + + override suspend fun deleteAll() { + return owner.withConnection { connection -> + connection.prepareStatement(sql.deleteAllStatement).executeUpdate() + } + } + + override suspend fun enumerate( + partitionId: String?, + afterKey: String?, + limit: Int + ): List { + checkPartition(partitionId) + checkLimit(limit) + if (limit == 0) { + return listOf() + } + return owner.withConnection { connection -> + val statement = connection.prepareStatement( + if (limit < Int.MAX_VALUE) { + sql.enumerateWithLimitStatement + } else { + sql.enumerateStatement + } + ) + var index = 1 + statement.setString(index++, afterKey ?: "") + if (spec.supportPartitions) { + statement.setString(index++, partitionId!!) + } + if (spec.supportExpiration) { + statement.setLong(index++, owner.clock.now().epochSeconds) + } + if (limit < Int.MAX_VALUE) { + statement.setInt(index, limit) + } + val resultSet = statement.executeQuery() + val list = mutableListOf() + while (resultSet.next()) { + list.add(resultSet.getString(1)) + } + list + } + } + + override suspend fun purgeExpired() { + if (!spec.supportExpiration) { + throw IllegalStateException("This table does not support expiration") + } + owner.withConnection { connection -> + val purge = connection.prepareStatement(sql.purgeExpiredStatement) + purge.setLong(1, owner.clock.now().epochSeconds) + purge.executeUpdate() + } + } +} \ No newline at end of file diff --git a/identity/src/commonTest/kotlin/com/android/identity/storage/GenericStorageTest.kt b/identity/src/jvmTest/kotlin/com/android/identity/storage/GenericStorageTest.kt similarity index 100% rename from identity/src/commonTest/kotlin/com/android/identity/storage/GenericStorageTest.kt rename to identity/src/jvmTest/kotlin/com/android/identity/storage/GenericStorageTest.kt diff --git a/identity/src/jvmTest/kotlin/com/android/identity/storage/testStorageList.jvm.kt b/identity/src/jvmTest/kotlin/com/android/identity/storage/testStorageList.jvm.kt new file mode 100644 index 000000000..7effb9c88 --- /dev/null +++ b/identity/src/jvmTest/kotlin/com/android/identity/storage/testStorageList.jvm.kt @@ -0,0 +1,55 @@ +package com.android.identity.storage + +import com.android.identity.storage.base.BaseStorage +import com.android.identity.storage.ephemeral.EphemeralStorage +import com.android.identity.storage.jdbc.JdbcStorage +import kotlinx.datetime.Clock + +var count: Int = 0 + +/** + * Creates a list of empty [Storage] objects for testing. + */ +actual fun createTransientStorageList(testClock: Clock): List { + org.hsqldb.jdbc.JDBCDriver() + com.mysql.cj.jdbc.Driver() + org.postgresql.Driver() + return listOf( + EphemeralStorage(testClock), + JdbcStorage( + jdbc = "jdbc:hsqldb:mem:tmp${count++}", + clock = testClock, + keySize = 3), + /* + // This can be enabled if MySQL installation is available for testing. + // Steps to initialize suitable database: + // CREATE USER 'wallet'@'localhost' IDENTIFIED BY 'XP4xpGNz' + // CREATE DATABASE wallet; + // GRANT ALL PRIVILEGES ON wallet.* TO 'wallet'@'localhost'; + JdbcStorage( + jdbc = "jdbc:mysql://localhost:3306/wallet?serverTimezone=UTC", + user = "wallet", + password = "XP4xpGNz", + clock = testClock, + keySize = 3 + ), + */ + /* + // This can be enabled if Postgresql installation is available for testing: + JdbcStorage( + jdbc = "jdbc:postgresql://localhost:5432/test", + user = "test", + password = "", + clock = testClock, + keySize = 3 + ) + */ + ) +} + +actual fun createPersistentStorage(name: String, testClock: Clock): Storage? { + return JdbcStorage( + jdbc = "jdbc:hsqldb:mem:p${name}", + clock = testClock, + keySize = 3) +} \ No newline at end of file