refactor: handle delete of lucene index entries in a backward compati…

…ble way

lucene/src/main/java/com/orientechnologies/lucene/analyzer/OLucenePerFieldAnalyzerWrapper.java

@@ -2,6 +2,7 @@
 
 import static com.orientechnologies.lucene.engine.OLuceneIndexEngineAbstract.RID;
 
+import com.orientechnologies.lucene.builder.OLuceneIndexType;
 import java.util.HashMap;
 import java.util.Map;
 import org.apache.lucene.analysis.Analyzer;
@@ -45,6 +46,7 @@ public OLucenePerFieldAnalyzerWrapper(
     this.fieldAnalyzers.putAll(fieldAnalyzers);
 
     this.fieldAnalyzers.put(RID, new KeywordAnalyzer());
+    this.fieldAnalyzers.put(OLuceneIndexType.RID_HASH, new KeywordAnalyzer());
     this.fieldAnalyzers.put("_CLASS", new KeywordAnalyzer());
     this.fieldAnalyzers.put("_CLUSTER", new KeywordAnalyzer());
     this.fieldAnalyzers.put("_JSON", new KeywordAnalyzer());

lucene/src/main/java/com/orientechnologies/lucene/builder/OLuceneDocumentBuilder.java

@@ -20,7 +20,8 @@
 
 import static com.orientechnologies.lucene.builder.OLuceneIndexType.createField;
 import static com.orientechnologies.lucene.builder.OLuceneIndexType.createFields;
-import static com.orientechnologies.lucene.engine.OLuceneIndexEngineAbstract.RID;
+import static com.orientechnologies.lucene.builder.OLuceneIndexType.createIdField;
+import static com.orientechnologies.lucene.builder.OLuceneIndexType.createOldIdField;
 
 import com.orientechnologies.orient.core.db.record.OIdentifiable;
 import com.orientechnologies.orient.core.index.OCompositeKey;
@@ -51,7 +52,7 @@ public Document build(
       final Map<String, Boolean> fieldsToStore,
       final ODocument metadata) {
     final Document doc = new Document();
-    this.addDefaultFieldsToDocument(definition, value, doc);
+    this.addDefaultFieldsToDocument(definition, key, value, doc);
 
     final List<Object> formattedKey = formatKeys(definition, key);
     int counter = 0;
@@ -71,9 +72,10 @@ public Document build(
   }
 
   private void addDefaultFieldsToDocument(
-      OIndexDefinition definition, OIdentifiable value, Document doc) {
+      OIndexDefinition definition, Object key, OIdentifiable value, Document doc) {
     if (value != null) {
-      doc.add(createField(RID, value.getIdentity().toString(), Field.Store.YES));
+      doc.add(createOldIdField(value));
+      doc.add(createIdField(value, key));
       doc.add(createField("_CLUSTER", "" + value.getIdentity().getClusterId(), Field.Store.YES));
       doc.add(createField("_CLASS", definition.getClassName(), Field.Store.YES));
     }

lucene/src/main/java/com/orientechnologies/lucene/builder/OLuceneIndexType.java

@@ -30,6 +30,8 @@
 import java.util.Base64;
 import java.util.Date;
 import java.util.List;
+import java.util.Locale;
+import org.apache.lucene.document.Document;
 import org.apache.lucene.document.DoubleDocValuesField;
 import org.apache.lucene.document.DoublePoint;
 import org.apache.lucene.document.Field;
@@ -43,28 +45,53 @@
 import org.apache.lucene.document.TextField;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.BooleanClause;
-import org.apache.lucene.search.BooleanClause.Occur;
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.util.BytesRef;
 
 /** Created by enricorisa on 21/03/14. */
 public class OLuceneIndexType {
+  public static final String RID_HASH = "_RID_HASH";
+
   public static Field createField(
       final String fieldName, final Object value, final Field.Store store /*,Field.Index index*/) {
-    if (fieldName.equalsIgnoreCase(OLuceneIndexEngineAbstract.RID)) {
-      StringField ridField = new StringField(fieldName, value.toString(), store);
-      return ridField;
-    }
     // metadata fields: _CLASS, _CLUSTER
     if (fieldName.startsWith("_CLASS") || fieldName.startsWith("_CLUSTER")) {
-      StringField ridField = new StringField(fieldName, value.toString(), store);
-      return ridField;
+      return new StringField(fieldName, value.toString(), store);
     }
     return new TextField(fieldName, value.toString(), Field.Store.YES);
   }
 
+  public static String extractId(Document doc) {
+    String value = doc.get(RID_HASH);
+    if (value != null) {
+      int pos = value.indexOf("|");
+      if (pos > 0) {
+        return value.substring(0, pos);
+      } else {
+        return value;
+      }
+    } else {
+      return null;
+    }
+  }
+
+  public static Field createIdField(final OIdentifiable id, final Object key) {
+    return new StringField(RID_HASH, genValueId(id, key), Field.Store.YES);
+  }
+
+  public static Field createOldIdField(final OIdentifiable id) {
+    return new StringField(
+        OLuceneIndexEngineAbstract.RID, id.getIdentity().toString(), Field.Store.YES);
+  }
+
+  public static String genValueId(final OIdentifiable id, final Object key) {
+    String value = id.getIdentity().toString() + "|";
+    value += hashKey(key);
+    return value;
+  }
+
   public static List<Field> createFields(
       String fieldName, Object value, Field.Store store, Boolean sort) {
     List<Field> fields = new ArrayList<>();
@@ -96,7 +123,6 @@ public static List<Field> createFields(
       fields.add(new SortedDocValuesField(fieldName, new BytesRef(value.toString())));
     }
     fields.add(new TextField(fieldName, value.toString(), Field.Store.YES));
-    fields.add(new StringField(fieldName + "__orient_key_hash", hashKey(value), Field.Store.YES));
     return fields;
   }
 
@@ -133,6 +159,10 @@ public static Query createQueryId(OIdentifiable value) {
     return new TermQuery(new Term(OLuceneIndexEngineAbstract.RID, value.getIdentity().toString()));
   }
 
+  public static Query createQueryId(OIdentifiable value, Object key) {
+    return new TermQuery(new Term(RID_HASH, genValueId(value, key)));
+  }
+
   public static String hashKey(Object key) {
     try {
       String keyString;
@@ -154,17 +184,24 @@ public static String hashKey(Object key) {
 
   public static Query createDeleteQuery(
       OIdentifiable value, List<String> fields, Object key, ODocument metadata) {
-    final BooleanQuery.Builder queryBuilder = new BooleanQuery.Builder();
+
+    // TODO Implementation of Composite keys with Collection
+    final BooleanQuery.Builder filter = new BooleanQuery.Builder();
+    final BooleanQuery.Builder builder = new BooleanQuery.Builder();
+    // TODO: Condition on Id and field key only for backward compatibility
     if (value != null) {
-      queryBuilder.add(createQueryId(value), BooleanClause.Occur.MUST);
+      builder.add(createQueryId(value), BooleanClause.Occur.MUST);
     }
-    // TODO Implementation of Composite keys with Collection
-    if (!(key instanceof OCompositeKey)) {
-      String field = fields.iterator().next();
-      // TODO: make or query for backward compatibility
-      queryBuilder.add(
-          new TermQuery(new Term(field + "__orient_key_hash", hashKey(key))), Occur.MUST);
+    String field = fields.iterator().next();
+    builder.add(
+        new TermQuery(new Term(field, key.toString().toLowerCase(Locale.ENGLISH))),
+        BooleanClause.Occur.MUST);
+
+    filter.add(builder.build(), BooleanClause.Occur.SHOULD);
+    if (value != null) {
+      filter.add(createQueryId(value, key), BooleanClause.Occur.SHOULD);
     }
-    return queryBuilder.build();
+
+    return filter.build();
   }
 }

lucene/src/main/java/com/orientechnologies/lucene/engine/OLuceneFullTextIndexEngine.java

@@ -219,8 +219,8 @@ public Document buildDocument(Object key, OIdentifiable value) {
 
   private static Document putInManualindex(Object key, OIdentifiable oIdentifiable) {
     Document doc = new Document();
-    doc.add(
-        OLuceneIndexType.createField(RID, oIdentifiable.getIdentity().toString(), Field.Store.YES));
+    doc.add(OLuceneIndexType.createOldIdField(oIdentifiable));
+    doc.add(OLuceneIndexType.createIdField(oIdentifiable, key));
 
     if (key instanceof OCompositeKey) {
 

lucene/src/main/java/com/orientechnologies/spatial/engine/OLuceneSpatialIndexEngineAbstract.java

@@ -140,9 +140,8 @@ protected Document newGeoDocument(OIdentifiable oIdentifiable, Shape shape, ODoc
     ft.setStored(true);
 
     Document doc = new Document();
-
-    doc.add(
-        OLuceneIndexType.createField(RID, oIdentifiable.getIdentity().toString(), Field.Store.YES));
+    doc.add(OLuceneIndexType.createOldIdField(oIdentifiable));
+    doc.add(OLuceneIndexType.createIdField(oIdentifiable, shapeDoc));
 
     for (IndexableField f : strategy.createIndexableFields(shape)) {
       doc.add(f);