diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java b/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java index 516bb75d5..1237567b9 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/InMemoryDao.java @@ -29,6 +29,10 @@ public InMemoryDao(Config conf) { @Override public Iterator> get(MemorySegment from, MemorySegment to) { + if (new MemSegComparatorNull().compare(from, to) > 0) { + return reverseIter(from, to); + } + SortedMap> dataSlice; if (from == null && to == null) { @@ -44,6 +48,11 @@ public Iterator> get(MemorySegment from, MemorySegment to) return new SSTableIterator(dataSlice.values().iterator(), controller, from, to); } + private Iterator> reverseIter(MemorySegment fromRightSide, MemorySegment toLeftSide) { + return new SSTableIterator(getMemTable().reversed().values().iterator(), + controller, fromRightSide, toLeftSide, true, new MemSegComparatorNull().reversed()); + } + @Override public Entry get(MemorySegment key) { if (key == null) { diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/IteratorUtils.java b/src/main/java/ru/vk/itmo/test/kachmareugene/IteratorUtils.java new file mode 100644 index 000000000..9d4214c87 --- /dev/null +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/IteratorUtils.java @@ -0,0 +1,38 @@ +package ru.vk.itmo.test.kachmareugene; + +import ru.vk.itmo.Entry; + +import java.lang.foreign.MemorySegment; +import java.util.SortedMap; + +public class IteratorUtils { + private IteratorUtils() { + } + + public static void insertNew(SortedMap mp, + SSTablesController controller, SSTableRowInfo info, MemorySegment to) { + Entry kv = controller.getRow(info); + + if (kv == null) { + return; + } + + if (!mp.containsKey(kv.key())) { + mp.put(kv.key(), info); + return; + } + SSTableRowInfo old = mp.get(kv.key()); + + SSTableRowInfo oldInfo = old.ssTableInd > info.ssTableInd ? info : old; + SSTableRowInfo newInfo = old.ssTableInd < info.ssTableInd ? info : old; + + mp.put(controller.getRow(newInfo).key(), newInfo); + + // tail recursion + if (oldInfo.isReversedToIter) { + insertNew(mp, controller, controller.getPrevInfo(oldInfo, to), to); + } else { + insertNew(mp, controller, controller.getNextInfo(oldInfo, to), to); + } + } +} diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterator.java b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterator.java index 6d53ce9b3..9b74a6207 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterator.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableIterator.java @@ -14,17 +14,20 @@ public class SSTableIterator implements Iterator> { private final Iterator> memTableIterator; private final SSTablesController controller; - private final Comparator comp = new MemSegComparatorNull(); - private final SortedMap mp = new TreeMap<>(comp); + private final Comparator comp; + private final SortedMap mp; private final MemorySegment from; private final MemorySegment to; private Entry head; private Entry keeper; + private boolean isReversed; public SSTableIterator(Iterator> it, SSTablesController controller, MemorySegment from, MemorySegment to) { - memTableIterator = it; + this.memTableIterator = it; this.controller = controller; + this.comp = new MemSegComparatorNull(); + this.mp = new TreeMap<>(comp); this.from = from; this.to = to; @@ -32,33 +35,25 @@ public SSTableIterator(Iterator> it, SSTablesController con positioningIterator(); } - private void insertNew(SSTableRowInfo info) { - Entry kv = controller.getRow(info); - - if (kv == null) { - return; - } - - if (!mp.containsKey(kv.key())) { - mp.put(kv.key(), info); - return; - } - SSTableRowInfo old = mp.get(kv.key()); - - SSTableRowInfo oldInfo = old.ssTableInd > info.ssTableInd ? info : old; - SSTableRowInfo newInfo = old.ssTableInd < info.ssTableInd ? info : old; + public SSTableIterator(Iterator> it, SSTablesController controller, + MemorySegment from, MemorySegment to, boolean isReversed, Comparator comp) { + this.memTableIterator = it; + this.controller = controller; + this.comp = comp; + this.mp = new TreeMap<>(comp); - mp.put(controller.getRow(newInfo).key(), newInfo); + this.from = from; + this.to = to; + this.isReversed = isReversed; - // tail recursion - insertNew(controller.getNextInfo(oldInfo, to)); + positioningIterator(); } private void positioningIterator() { - List rawData = controller.firstGreaterKeys(from); + List rawData = controller.firstKeys(from, isReversed); for (var info : rawData) { - insertNew(info); + IteratorUtils.insertNew(mp, controller, info, to); } } @@ -138,7 +133,13 @@ private void changeState() { } private void updateMp(MemorySegment key) { - insertNew(controller.getNextInfo(mp.remove(key), to)); + if (isReversed) { + IteratorUtils.insertNew(mp, controller, + controller.getPrevInfo(mp.remove(key), to), to); + return; + } + IteratorUtils.insertNew(mp, controller, + controller.getNextInfo(mp.remove(key), to), to); } private boolean isBetween(MemorySegment who) { @@ -150,8 +151,7 @@ private boolean isBetween(MemorySegment who) { private Map.Entry getFirstMin() { Map.Entry minSStablesEntry = mp.firstEntry(); while (!mp.isEmpty() && !isBetween(minSStablesEntry.getKey())) { - mp.remove(minSStablesEntry.getKey()); - insertNew(controller.getNextInfo(minSStablesEntry.getValue(), to)); + updateMp(minSStablesEntry.getKey()); minSStablesEntry = mp.firstEntry(); } return minSStablesEntry; diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableRowInfo.java b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableRowInfo.java index 309a26400..ee9e16301 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableRowInfo.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTableRowInfo.java @@ -7,6 +7,7 @@ public class SSTableRowInfo { long rowShift; private final long valueSize; int ssTableInd; + boolean isReversedToIter; public SSTableRowInfo(long keyOffset, long keySize, long valueOffset, long valueSize, int ssTableInd, long rowShift) { @@ -18,6 +19,17 @@ public SSTableRowInfo(long keyOffset, long keySize, long valueOffset, this.rowShift = rowShift; } + public SSTableRowInfo(long keyOffset, long keySize, long valueOffset, + long valueSize, int ssTableInd, long rowShift, boolean isReversedToIter) { + this.keyOffset = keyOffset; + this.valueOffset = valueOffset; + this.keySize = keySize; + this.valueSize = valueSize; + this.ssTableInd = ssTableInd; + this.rowShift = rowShift; + this.isReversedToIter = isReversedToIter; + } + public boolean isDeletedData() { return valueSize < 0; } diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java index 7341db9e3..00be25143 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/SSTablesController.java @@ -71,47 +71,34 @@ private List openFiles(Path dir, String fileNamePref, List } } - private boolean greaterThen(long keyOffset, long keySize, - MemorySegment mapped, MemorySegment key) { - - return segComp.compare(key, mapped.asSlice(keyOffset, keySize)) > 0; - } - - //Gives offset for line in index file - private long searchKeyInFile(int ind, MemorySegment mapped, MemorySegment key) { - long l = -1; - long r = getNumberOfEntries(mapped); - - while (r - l > 1) { - long mid = (l + r) / 2; - SSTableRowInfo info = createRowInfo(ind, mid); - if (greaterThen(info.keyOffset, info.keySize, mapped, key)) { - l = mid; - } else { - r = mid; - } - } - return r == getNumberOfEntries(mapped) ? -1 : r; - } - //return - List ordered form the latest created sstable to the first. - public List firstGreaterKeys(MemorySegment key) { + public List firstKeys(MemorySegment key, boolean isReversed) { List ans = new ArrayList<>(); for (int i = ssTables.size() - 1; i >= 0; i--) { long entryIndexesLine = 0; if (key != null) { - entryIndexesLine = searchKeyInFile(i, ssTables.get(i), key); + if (isReversed) { + entryIndexesLine = Utils.searchKeyInFileReversed(this, i, + getNumberOfEntries(ssTables.get(i)), + ssTables.get(i), key, segComp); + } else { + entryIndexesLine = Utils.searchKeyInFile(this, i, + getNumberOfEntries(ssTables.get(i)), ssTables.get(i), key, segComp); + } } if (entryIndexesLine < 0) { continue; } - ans.add(createRowInfo(i, entryIndexesLine)); + SSTableRowInfo row = createRowInfo(i, entryIndexesLine); + row.isReversedToIter = isReversed; + + ans.add(row); } return ans; } - private SSTableRowInfo createRowInfo(int ind, final long rowIndex) { + public SSTableRowInfo createRowInfo(int ind, final long rowIndex) { long start = ssTables.get(ind).get(ValueLayout.JAVA_LONG_UNALIGNED, rowIndex * ONE_LINE_SIZE + Long.BYTES); long size = ssTables.get(ind).get(ValueLayout.JAVA_LONG_UNALIGNED, rowIndex * ONE_LINE_SIZE + Long.BYTES * 2); @@ -122,7 +109,8 @@ private SSTableRowInfo createRowInfo(int ind, final long rowIndex) { public SSTableRowInfo searchInSStables(MemorySegment key) { for (int i = ssTables.size() - 1; i >= 0; i--) { - long ind = searchKeyInFile(i, ssTables.get(i), key); + long ind = Utils.searchKeyInFile(this, i, + getNumberOfEntries(ssTables.get(i)), ssTables.get(i), key, segComp); if (ind >= 0) { return createRowInfo(i, ind); } @@ -160,6 +148,18 @@ public SSTableRowInfo getNextInfo(SSTableRowInfo info, MemorySegment maxKey) { return null; } + public SSTableRowInfo getPrevInfo(SSTableRowInfo info, MemorySegment minKey) { + for (long t = info.rowShift - 1; t >= 0; t--) { + var inf = createRowInfo(info.ssTableInd, t); + + Entry row = getRow(inf); + if (segComp.compare(row.key(), minKey) > 0) { + return inf; + } + } + return null; + } + private long getNumberOfEntries(MemorySegment memSeg) { return memSeg.get(ValueLayout.JAVA_LONG_UNALIGNED, 0); } diff --git a/src/main/java/ru/vk/itmo/test/kachmareugene/Utils.java b/src/main/java/ru/vk/itmo/test/kachmareugene/Utils.java index 77886d9d3..913c8b521 100644 --- a/src/main/java/ru/vk/itmo/test/kachmareugene/Utils.java +++ b/src/main/java/ru/vk/itmo/test/kachmareugene/Utils.java @@ -72,4 +72,56 @@ public static MemorySegment getValueOrNull(Entry kv) { } return value; } + + private static boolean greaterThen(Comparator segComp, long keyOffset, long keySize, + MemorySegment mapped, MemorySegment key) { + + return segComp.compare(key, mapped.asSlice(keyOffset, keySize)) > 0; + } + + private static boolean greaterEqThen(Comparator segComp, long keyOffset, long keySize, + MemorySegment mapped, MemorySegment key) { + + return segComp.compare(key, mapped.asSlice(keyOffset, keySize)) >= 0; + } + + //Gives offset for line in index file + public static long searchKeyInFile(SSTablesController controller, + int ind, long numberOfEntries, + MemorySegment mapped, MemorySegment key, + Comparator segComp) { + long l = -1; + long r = numberOfEntries; + + while (r - l > 1) { + long mid = (l + r) / 2; + SSTableRowInfo info = controller.createRowInfo(ind, mid); + if (greaterThen(segComp, info.keyOffset, info.keySize, mapped, key)) { + l = mid; + } else { + r = mid; + } + } + return r == numberOfEntries ? -1 : r; + } + + public static long searchKeyInFileReversed(SSTablesController controller, + int ind, long numberOfEntries, + MemorySegment mapped, MemorySegment key, + Comparator segComp) { + long l = -1; + long r = numberOfEntries; + + while (r - l > 1) { + long mid = (l + r) / 2; + SSTableRowInfo info = controller.createRowInfo(ind, mid); + if (greaterEqThen(segComp, info.keyOffset, info.keySize, mapped, key)) { + l = mid; + } else { + r = mid; + } + } + return l; + } + } diff --git a/src/test/java/ru/vk/itmo/ReverseIteratorTest.java b/src/test/java/ru/vk/itmo/ReverseIteratorTest.java new file mode 100644 index 000000000..93ffb58b7 --- /dev/null +++ b/src/test/java/ru/vk/itmo/ReverseIteratorTest.java @@ -0,0 +1,158 @@ +package ru.vk.itmo; + +import ru.vk.itmo.test.DaoFactory; + +import java.io.IOException; +import java.util.List; + +public class ReverseIteratorTest extends BaseTest { + @DaoTest(stage = 4, maxStage = 4) + void memTableCheckReverse(Dao> dao) throws IOException { + + List> entries = entries(100); + for (Entry entry : entries) { + dao.upsert(entry); + } + + assertSame(dao.get(keyAt(100), keyAt(1)), + entries.reversed().subList(0, entries.size() - 2)); + } + + @DaoTest(stage = 4, maxStage = 4) + void memTableCheckOrdinary(Dao> dao) throws IOException { + + List> entries = entries(100); + for (Entry entry : entries) { + dao.upsert(entry); + } + + assertSame(dao.all(), entries); + } + + @DaoTest(stage = 4, maxStage = 4) + void ssTablesCheckReverse(Dao> dao) throws IOException { + + List> entries = entries(100); + for (Entry entry : entries) { + dao.upsert(entry); + } + + dao.close(); + dao = DaoFactory.Factory.reopen(dao); + + assertSame(dao.get(keyAt(100), keyAt(1)), + entries.reversed().subList(0, entries.size() - 2)); + } + + @DaoTest(stage = 4, maxStage = 4) + void ssTablesCheckOrdinary(Dao> dao) throws IOException { + + List> entries = entries(100); + for (Entry entry : entries) { + dao.upsert(entry); + } + dao.close(); + dao = DaoFactory.Factory.reopen(dao); + + assertSame(dao.all(), entries); + } + + @DaoTest(stage = 4) + void compactionReverse(Dao> dao) throws IOException { + List> entries = entries(100); + List> firstHalf = entries.subList(0, 50); + List> lastHalf = entries.subList(50, 100); + + for (Entry entry : firstHalf) { + dao.upsert(entry); + } + dao.compact(); + dao.close(); + + dao = DaoFactory.Factory.reopen(dao); + for (Entry entry : lastHalf) { + dao.upsert(entry); + } + assertSame(dao.get(keyAt(100), keyAt(1)), + entries.reversed().subList(0, entries.size() - 2)); + + dao.flush(); + assertSame(dao.get(keyAt(100), keyAt(1)), + entries.reversed().subList(0, entries.size() - 2)); + + dao.close(); + dao = DaoFactory.Factory.reopen(dao); + assertSame(dao.get(keyAt(100), keyAt(1)), + entries.reversed().subList(0, entries.size() - 2)); + + + dao.compact(); + dao.close(); + dao = DaoFactory.Factory.reopen(dao); + assertSame(dao.get(keyAt(100), keyAt(1)), + entries.reversed().subList(0, entries.size() - 2)); + + } + + + @DaoTest(stage = 4) + void compactionOrdinary(Dao> dao) throws IOException { + List> entries = entries(100); + List> firstHalf = entries.subList(0, 50); + List> lastHalf = entries.subList(50, 100); + + for (Entry entry : firstHalf) { + dao.upsert(entry); + } + dao.compact(); + dao.close(); + + dao = DaoFactory.Factory.reopen(dao); + for (Entry entry : lastHalf) { + dao.upsert(entry); + } + assertSame(dao.all(), entries); + + dao.flush(); + assertSame(dao.all(), entries); + + dao.close(); + dao = DaoFactory.Factory.reopen(dao); + assertSame(dao.all(), entries); + + dao.compact(); + dao.close(); + dao = DaoFactory.Factory.reopen(dao); + assertSame(dao.all(), entries); + } + + @DaoTest(stage = 1) + void testOrder(Dao> dao) { + dao.upsert(entry("b", "b")); + dao.upsert(entry("aa", "aa")); + dao.upsert(entry("", "")); + + assertSame( + dao.get("b", ""), + + entry("b", "b"), + entry("aa", "aa") + ); + } + + @DaoTest(stage = 1) + void testFullRange(Dao> dao) { + dao.upsert(entry("a", "b")); + + dao.upsert(entry("b", "b")); + dao.upsert(entry("c", "c")); + + assertSame( + dao.get("z", ""), + + entry("c", "c"), + entry("b", "b"), + entry("a", "b") + ); + } +}