diff --git a/src/TokenCleanup/TokenCleanup.cs b/src/TokenCleanup/TokenCleanup.cs
index 34c3f20..30e8519 100644
--- a/src/TokenCleanup/TokenCleanup.cs
+++ b/src/TokenCleanup/TokenCleanup.cs
@@ -135,7 +135,6 @@ public async Task RemoveExpiredGrantsAsync()
                             // TODO: Device Flow cleanup
                             var expired = context.PersistedGrants
                                 .Where(x => x.Expiration < DateTime.UtcNow)
-                                .OrderBy(x => x.Key)
                                 .Take(_options.TokenCleanupBatchSize)
                                 .ToArray();
 
diff --git a/test/IntegrationTests/FakeOperationalStoreNotification.cs b/test/IntegrationTests/FakeOperationalStoreNotification.cs
new file mode 100644
index 0000000..3907c81
--- /dev/null
+++ b/test/IntegrationTests/FakeOperationalStoreNotification.cs
@@ -0,0 +1,14 @@
+using System.Collections.Generic;
+using System.Threading.Tasks;
+using IdentityServer4.EntityFramework.Entities;
+
+namespace IdentityServer4.EntityFramework.IntegrationTests
+{
+    public class FakeOperationalStoreNotification : IOperationalStoreNotification
+    {
+        public Task PersistedGrantsRemovedAsync(IEnumerable<PersistedGrant> persistedGrants)
+        {
+            return Task.CompletedTask;
+        }
+    }
+}
diff --git a/test/IntegrationTests/TokenCleanup/TokenCleanupTests.cs b/test/IntegrationTests/TokenCleanup/TokenCleanupTests.cs
new file mode 100644
index 0000000..db4ebf4
--- /dev/null
+++ b/test/IntegrationTests/TokenCleanup/TokenCleanupTests.cs
@@ -0,0 +1,79 @@
+using IdentityServer4.EntityFramework.DbContexts;
+using IdentityServer4.EntityFramework.Entities;
+using IdentityServer4.EntityFramework.Interfaces;
+using IdentityServer4.EntityFramework.Options;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.DependencyInjection;
+using System;
+using System.Linq;
+using System.Threading.Tasks;
+using Xunit;
+
+namespace IdentityServer4.EntityFramework.IntegrationTests.TokenCleanup
+{
+    public class TokenCleanupTests : IntegrationTest<TokenCleanupTests, PersistedGrantDbContext, OperationalStoreOptions>
+    {
+        private readonly DateTime _now;
+
+        public TokenCleanupTests(DatabaseProviderFixture<PersistedGrantDbContext> fixture) : base(fixture)
+        {
+            _now = DateTime.UtcNow;
+            foreach (var options in TestDatabaseProviders.SelectMany(x => x.Select(y => (DbContextOptions<PersistedGrantDbContext>)y)).ToList())
+            {
+                using (var context = new PersistedGrantDbContext(options, StoreOptions))
+                {
+                    context.Database.EnsureCreated();
+                }
+            }
+        }
+
+        [Theory, MemberData(nameof(TestDatabaseProviders))]
+        public async Task RemoveExpiredGrantsAsync_ExpiredTokensCleanedUp(DbContextOptions<PersistedGrantDbContext> options)
+        {
+            using (var context = new PersistedGrantDbContext(options, StoreOptions))
+            {
+                var grants = new PersistedGrant[100];
+                for (var i = 0; i < grants.Length; i++)
+                {
+                    grants[i] = new PersistedGrant
+                    {
+                        Key = Guid.NewGuid().ToString("N"),
+                        Type = i % 4 == 0 ? "authorization_code" : "refresh_token",
+                        SubjectId = i.ToString(),
+                        ClientId = "test-client",
+                        CreationTime = _now.AddDays(-60),
+                        // 34 tokens should be cleaned up.
+                        Expiration = _now.AddDays(i % 3 == 0 ? -1 : 1),
+                        Data = "grant data"
+                    };
+                }
+                context.PersistedGrants.AddRange(grants);
+                await context.SaveChangesAsync();
+            }
+
+            var customStoreOptions = new OperationalStoreOptions
+            {
+                TokenCleanupBatchSize = 5
+            };
+            using (var context = new PersistedGrantDbContext(options, StoreOptions))
+            {
+                var serviceCollection = new ServiceCollection();
+                serviceCollection.AddSingleton<IPersistedGrantDbContext>(context);
+                serviceCollection.AddSingleton<IOperationalStoreNotification, FakeOperationalStoreNotification>();
+
+                var cleanup = new EntityFramework.TokenCleanup(
+                    serviceCollection.BuildServiceProvider(),
+                    new FakeLogger<EntityFramework.TokenCleanup>(),
+                    customStoreOptions);
+
+                await cleanup.RemoveExpiredGrantsAsync();
+            }
+
+            using (var context = new PersistedGrantDbContext(options, StoreOptions))
+            {
+                var grants = await context.PersistedGrants.ToListAsync();
+                Assert.Equal(100 - 34, grants.Count);
+            }
+        }
+    }
+}