feat: DH-18364: Add Property to Disable Core DataIndex (#6538)

deephaven · Jan 8, 2025 · ed87a6c · ed87a6c
1 parent fc40e1d
commit ed87a6c
Show file tree

Hide file tree

Showing 2 changed files with 14 additions and 1 deletion.
diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/QueryTable.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/QueryTable.java
@@ -196,6 +196,14 @@ public interface MemoizableOperation<T extends DynamicNode & NotificationStepRec
      */
     static boolean USE_REDIRECTED_COLUMNS_FOR_SELECT =
             Configuration.getInstance().getBooleanWithDefault("QueryTable.redirectSelect", false);
+
+    /**
+     * If set to true, then permit where filters to use a data index, when applicable. If false, data indexes are not
+     * used even if present.
+     */
+    public static boolean USE_DATA_INDEX_FOR_WHERE =
+            Configuration.getInstance().getBooleanWithDefault("QueryTable.useDataIndexForWhere", true);
+
     /**
      * For a static select(), we would prefer to flatten the table to avoid using memory unnecessarily (because the data
      * may be spread out across many blocks depending on the input RowSet). However, the select() can become slower
@@ -1225,7 +1233,7 @@ public Table where(Filter filter) {
     }
 
     private void initializeAndPrioritizeFilters(@NotNull final WhereFilter... filters) {
-        final DataIndexer dataIndexer = DataIndexer.existingOf(rowSet);
+        final DataIndexer dataIndexer = USE_DATA_INDEX_FOR_WHERE ? DataIndexer.existingOf(rowSet) : null;
         final int numFilters = filters.length;
         final BitSet priorityFilterIndexes = new BitSet(numFilters);
 

diff --git a/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java b/engine/table/src/main/java/io/deephaven/engine/table/impl/select/MatchFilter.java
@@ -15,6 +15,7 @@
 import io.deephaven.engine.table.Table;
 import io.deephaven.engine.table.TableDefinition;
 import io.deephaven.engine.table.impl.QueryCompilerRequestProcessor;
+import io.deephaven.engine.table.impl.QueryTable;
 import io.deephaven.engine.table.impl.chunkfilter.ChunkFilter;
 import io.deephaven.engine.table.impl.chunkfilter.ChunkMatchFilterFactory;
 import io.deephaven.engine.table.impl.lang.QueryLanguageFunctionUtils;
@@ -252,6 +253,10 @@ public SafeCloseable beginOperation(@NotNull final Table sourceTable) {
         if (initialDependenciesGathered || dataIndex != null) {
             throw new IllegalStateException("Inputs already initialized, use copy() instead of re-using a WhereFilter");
         }
+        if (!QueryTable.USE_DATA_INDEX_FOR_WHERE) {
+            return () -> {
+            };
+        }
         try (final SafeCloseable ignored = sourceTable.isRefreshing() ? LivenessScopeStack.open() : null) {
             dataIndex = DataIndexer.getDataIndex(sourceTable, columnName);
             if (dataIndex != null && dataIndex.isRefreshing()) {