Merge remote-tracking branch 'origin/pypi2021'

oaqa · Mar 2, 2023 · d847ac1 · d847ac1
2 parents fa768e8 + 467c642
commit d847ac1
Show file tree

Hide file tree

Showing 4 changed files with 10 additions and 10 deletions.
diff --git a/flexneuart/ir_datasets/spacy.py b/flexneuart/ir_datasets/spacy.py
@@ -39,7 +39,6 @@ def __init__(self, input_fields : list,
         :param  model_name     a name of the spacy model to use, e.g., en_core_web_sm
         :param  stop_word_file  the name of the stop word file
         :param  remove_punct  a bool flag indicating if the punctuation tokens need to be removed
-        :param  sent_split    a bool flag indicating if sentence splitting is necessary
         :param  keep_only_alpha_num a bool flag indicating if we need to keep only alpha-numeric characters
         :param  enable_pos    a bool flag that enables POS tagging (which, e.g., can improve lemmatization)
         """

diff --git a/flexneuart/text_proc/parse.py b/flexneuart/text_proc/parse.py
@@ -42,7 +42,7 @@ def __init__(self, model_name):
         :param model_name: a name of the spacy model to use, e.g., en_core_web_sm
         """
         self._nlp = spacy.load(model_name, disable=[SPACY_NER, SPACY_PARSER, SPACY_POS])
-        self._nlp.add_pipe(self._nlp.create_pipe("sentencizer"))
+        self._nlp.add_pipe("sentencizer")
 
     def __call__(self, text):
         """A thin wrapper that merely calls spacy.
@@ -111,8 +111,7 @@ def __init__(self, model_name, stop_words,
 
         self._nlp = spacy.load(model_name, disable=disable_list)
         if sent_split:
-            sentencizer = self._nlp.create_pipe("sentencizer")
-            self._nlp.add_pipe(sentencizer)
+            self._nlp.add_pipe("sentencizer")
 
         self._removePunct = remove_punct
         self._stopWords = frozenset([w.lower() for w in stop_words])

diff --git a/requirements.txt b/requirements.txt
@@ -16,8 +16,7 @@ torch
 torchtext
 
 transformers
-# we fix Spacy version, though in the future we would have to upgrade too
-spacy==2.2.3
+spacy
 sentence-transformers
 sentencepiece
 krovetzstemmer
@@ -30,6 +29,9 @@ typing-extensions
 
 # Colbert specific
 ujson
-# Deeberta specific, might fail with older
-# protobuf versions.
-protobuf==3.20
+
+# Deeberta specific, might fail with older protobuf versions.
+# but let's not install it by default, actually newer 
+# transformer versions should install protbuf on its own
+#protobuf==3.20
+
diff --git a/scripts/data_convert/ir_datasets/sample_configs/trec_robust04.json b/scripts/data_convert/ir_datasets/sample_configs/trec_robust04.json
@@ -48,7 +48,7 @@
         ]
     },
     {
-        "part_name": "all",
+        "part_name": "queries_all",
         "dataset_name": "trec-robust04",
         "is_query": true,
         "src_attributes": [