Added Emanuele's fix for the filter rules.

buildmaster/meta/ATLAS_TOPDIFF_DILEPT_8TEV_TTRAPNORM.yaml

@@ -1,4 +1,4 @@
 ndata:  	  5                          
 nsys:   	  5                          
 setname:	  "ATLAS_TOPDIFF_DILEPT_8TEV_TTRAPNORM"      
-proctype:	  "HQP_YQ" 
+proctype:	  "HQP_YQQ" 

nnpdfcpp/data/NNLOCFAC/CF_QCD_CMS_TTB_DIFF_13TEV_2016_2L_TTMNORM_DEN.dat

@@ -13,4 +13,3 @@ Warnings:
 1.07030	0.00000
 1.07030	0.00000
 1.07030	0.00000
-1.07030	0.00000

nnpdfcpp/data/NNLOCFAC/CF_QCD_CMS_TTB_DIFF_13TEV_2016_2L_TTMNORM_NUM.dat

@@ -13,4 +13,3 @@ Warnings:
 1.08225	0.00000
 1.09178	0.00000
 1.09284	0.00000
-1.12079	0.00000

nnpdfcpp/data/commondata/DATA_ATLAS_TOPDIFF_DILEPT_8TEV_TTRAPNORM.dat

@@ -1,6 +1,6 @@
 ATLAS_TOPDIFF_DILEPT_8TEV_TTRAPNORM	5	5
-   1	HQP_YQ	2.000000000000e-01	3.003289000000e+04	8.000000000000e+03	8.210000000000e-01	0.000000000000e+00	2.880822776869e-02	3.508919338452e+00	-8.379445396709e-03	-1.020638903375e+00	-8.308848565496e-03	-1.012040020158e+00	-1.007667443919e-03	-1.227365948744e-01	-8.276176108275e-05	-1.008060427317e-02	
-   2	HQP_YQ	6.000000000000e-01	3.003289000000e+04	8.000000000000e+03	7.210000000000e-01	0.000000000000e+00	-2.065837286933e-02	-2.865238955524e+00	-1.798747836570e-02	-2.494795889834e+00	-5.635869564862e-03	-7.816740034483e-01	-1.753133177919e-03	-2.431530066462e-01	-1.070081468065e-04	-1.484162923807e-02	
-   3	HQP_YQ	1.000000000000e+00	3.003289000000e+04	8.000000000000e+03	4.990000000000e-01	0.000000000000e+00	-9.477603208718e-03	-1.899319280304e+00	1.411259972037e-02	2.828176296668e+00	-1.278787511604e-02	-2.562700424056e+00	-6.230360112141e-05	-1.248569160750e-02	5.901562153626e-06	1.182677786298e-03	
-   4	HQP_YQ	1.600000000000e+00	3.003289000000e+04	8.000000000000e+03	2.060000000000e-01	0.000000000000e+00	8.295367941452e-04	4.026877641482e-01	4.398008172997e-03	2.134955423785e+00	2.136815949654e-03	1.037289295948e+00	-9.026705106575e-03	-4.381895682804e+00	-3.641501526012e-04	-1.767719187384e-01	
-   5	HQP_YQ	2.400000000000e+00	3.003289000000e+04	8.000000000000e+03	2.260000000000e-02	0.000000000000e+00	2.082642466025e-04	9.215232150554e-01	-4.309558781916e-04	-1.906884416777e+00	-1.712566084069e-04	-7.577726035702e-01	-1.393760275119e-03	-6.167080863358e+00	2.552594073529e-03	1.129466404216e+01	
+   1	HQP_YQQ	2.000000000000e-01	3.003289000000e+04	8.000000000000e+03	8.210000000000e-01	0.000000000000e+00	2.880822776869e-02	3.508919338452e+00	-8.379445396709e-03	-1.020638903375e+00	-8.308848565496e-03	-1.012040020158e+00	-1.007667443919e-03	-1.227365948744e-01	-8.276176108275e-05	-1.008060427317e-02	
+   2	HQP_YQQ	6.000000000000e-01	3.003289000000e+04	8.000000000000e+03	7.210000000000e-01	0.000000000000e+00	-2.065837286933e-02	-2.865238955524e+00	-1.798747836570e-02	-2.494795889834e+00	-5.635869564862e-03	-7.816740034483e-01	-1.753133177919e-03	-2.431530066462e-01	-1.070081468065e-04	-1.484162923807e-02	
+   3	HQP_YQQ	1.000000000000e+00	3.003289000000e+04	8.000000000000e+03	4.990000000000e-01	0.000000000000e+00	-9.477603208718e-03	-1.899319280304e+00	1.411259972037e-02	2.828176296668e+00	-1.278787511604e-02	-2.562700424056e+00	-6.230360112141e-05	-1.248569160750e-02	5.901562153626e-06	1.182677786298e-03	
+   4	HQP_YQQ	1.600000000000e+00	3.003289000000e+04	8.000000000000e+03	2.060000000000e-01	0.000000000000e+00	8.295367941452e-04	4.026877641482e-01	4.398008172997e-03	2.134955423785e+00	2.136815949654e-03	1.037289295948e+00	-9.026705106575e-03	-4.381895682804e+00	-3.641501526012e-04	-1.767719187384e-01	
+   5	HQP_YQQ	2.400000000000e+00	3.003289000000e+04	8.000000000000e+03	2.260000000000e-02	0.000000000000e+00	2.082642466025e-04	9.215232150554e-01	-4.309558781916e-04	-1.906884416777e+00	-1.712566084069e-04	-7.577726035702e-01	-1.393760275119e-03	-6.167080863358e+00	2.552594073529e-03	1.129466404216e+01	

validphys2/src/validphys/cuts/filters.yaml

@@ -1,3 +1,39 @@
+- dataset: ATLAS_TTBARZ_13TEV_PTZNORM
+  reason: |
+   We remove the last bin of the normalised distribution because it is
+   linearly dependent on the others
+  rule: "p_T<300.0"
+
+- dataset: ATLAS_TTBAR_13TEV_HADRONIC_TTMNORM
+  reason: |
+   We remove the last bin of the normalised distribution because it is
+   linearly dependent on the others
+  rule: "MQQ<1101"
+
+- dataset: CMS_TTBARZ_13TEV_PTZNORM
+  reason: |
+   We remove the last bin of the normalised distribution because it is
+   linearly dependent on the others
+  rule: "p_T<250.0"
+
+- dataset: CMS_TTBAR_13TEV_LJETS_2D_TTM_ABSYTTNORM
+  reason: |
+   We remove the last bin of the normalised distribution because it is
+   linearly dependent on the others
+  rule: "(yQQ<1.4) or (mu2<820000)"
+
+- dataset: ATLAS_TTBAR_13TEV_HADRONIC_2D_TTM_ABSYTTNORM
+  reason: |
+   We remove the last bin of the normalised distribution because it is
+   linearly dependent on the others
+  rule: "(yQQ<1.4) or (mu2<3610000)"
+
+- dataset: ATLAS_TTBAR_13TEV_TTMNORM
+  reason: |
+   We remove the last bin of the normalised distribution because it is
+   linearly dependent on the others
+  rule: "MQQ<1600"
+
 - dataset: ATLAS_TTB_DIFF_8TEV_LJ_TPTNORM
   reason: |
    We remove the last bin of the normalised distribution because it is 
@@ -22,6 +58,18 @@
    linearly dependent on the others
   rule: "MQQ<1350"
 
+- dataset: ATLAS_TOPDIFF_DILEPT_8TEV_TTRAPNORM
+  reason: |
+   We remove the last bin of the normalised distribution because it is 
+   linearly dependent on the others
+  rule: "yQQ<2.00"
+
+- dataset: ATLAS_TOPDIFF_DILEPT_8TEV_TTMNORM
+  reason: |
+   We remove the last bin of the normalised distribution because it is 
+   linearly dependent on the others
+  rule: "MQQ<1000"
+
 - dataset: CMSTOPDIFF8TEVTRAPNORM
   reason: |
    We remove the last bin of the normalised distribution because it is
@@ -42,18 +90,10 @@
     for the NNLO jet project.
   rule: eta < 0.3
 
-- dataset: ATLAS1JET11_NEW_SCALE
-  reason: |
-    We keep only the first rapidity bin since the chi2 to the whole set of
-    rapidity bins is too large (known problems with correlation matrix).
-    This cut will be removed in the new bunch of fits that we are going to do
-    for the NNLO jet project.
-  rule: eta < 0.3
-
 - dataset: CMS_1JET_8TEV
   reason: |
     We keep only the bins with pTjet>74 GeV because fixed-order theory does
-    not provide a good description of the data for smalle values of the	jet
+    not provide a good description of the data for smalle values of the jet
     transverse momentum.
   rule: "p_T2 >= 5476"
 
@@ -104,15 +144,15 @@
     for which the asymmetry is small is unfeasible. So we can only fit points
     at NNLO for which the value of the asymmetry is not too small.
   PTO: NNLO
-  rule: "central_value >= 0.03"
+  rule: "fabs(central_value) >= 0.03"
 
-#- dataset: ATLAS_WP_JET_8TEV_PT
-  #reason: Avoid the region where small-pT resummation effects become important.
-  #rule: "p_T >= 30"
+- dataset: ATLAS_WP_JET_8TEV_PT
+  reason: Avoid the region where small-pT resummation effects become important.
+  rule: "p_T >= 30"
 
-#- dataset: ATLAS_WM_JET_8TEV_PT
-  #reason: Avoid the region where small-pT resummation effects become important.
-  #rule: "p_T >= 30"
+- dataset: ATLAS_WM_JET_8TEV_PT
+  reason: Avoid the region where small-pT resummation effects become important.
+  rule: "p_T >= 30"
 
 - dataset: ATLASZPT7TEV
   reason: Avoid the region where resummation effects become important.
@@ -159,67 +199,67 @@
     max_M: 116.0
   rule: M <= max_M
 
-#- dataset: CMSDY2D11
-  #reason: |
-    #Removes data points for which the NNLO K-factors are very large and
-    #thus the NLO calculation is unreliable.
-  #PTO: NNLO-
-  #local_variables:
-    #M: sqrt(M2)
-    #max_M: 200.0
-  #rule: M <= max_M
-
-#- dataset: CMSDY2D11
-  #reason: Remove data points for which electroweak corrections are large.
-  #PTO: NNLO-
-  #local_variables:
-    #M: sqrt(M2)
-    #min_M: 30.0
-    #max_rapidity: 2.2
-  #rule: M >= min_M and etay <= max_rapidity
-
-#- dataset: CMSDY2D11
-  #reason: Remove data points for which electroweak corrections are large.
-  #PTO: NNLO
-  #local_variables:
-    #M: sqrt(M2)
-    #max_rapidity: 2.2
-    #max_M: 200.0
-  #rule: M <= max_M and etay <= max_rapidity
-
-#- dataset: CMSDY2D12
-  #reason: |
-    #Removes data points for which the NNLO K-factors are very large and
-    #thus the NLO calculation is unreliable.
-  #PTO: NNLO-
-  #local_variables:
-    #M: sqrt(M2)
-    #max_M: 200.0
-  #rule: M <= max_M
-
-#- dataset: CMSDY2D12
-  #reason: Remove data points for which electroweak corrections are large.
-  #PTO: NNLO-
-  #local_variables:
-    #M: sqrt(M2)
-    #min_M: 30.0
-    #max_rapidity: 2.2
-  #rule: M >= min_M and etay <= max_rapidity
-
-#- dataset: CMSDY2D12
-  #reason: Remove data points for which electroweak corrections are large.
-  #PTO: NNLO
-  #local_variables:
-    #M: sqrt(M2)
-    #max_rapidity: 2.2
-    #max_M: 200.0
-  #rule: M <= max_M and etay <= max_rapidity
-
-#- dataset: ATLASZHIGHMASS49FB
-  #reason: Avoid the region where resummation effects become important.
-  #local_variables:
-    #max_M: 200.0
-  #rule: M_ll <= max_M
+- dataset: CMSDY2D11
+  reason: |
+    Removes data points for which the NNLO K-factors are very large and
+    thus the NLO calculation is unreliable.
+  PTO: NNLO-
+  local_variables:
+    M: sqrt(M2)
+    max_M: 200.0
+  rule: M <= max_M
+
+- dataset: CMSDY2D11
+  reason: Remove data points for which electroweak corrections are large.
+  PTO: NNLO-
+  local_variables:
+    M: sqrt(M2)
+    min_M: 30.0
+    max_rapidity: 2.2
+  rule: M >= min_M and etay <= max_rapidity
+
+- dataset: CMSDY2D11
+  reason: Remove data points for which electroweak corrections are large.
+  PTO: NNLO
+  local_variables:
+    M: sqrt(M2)
+    max_rapidity: 2.2
+    max_M: 200.0
+  rule: M <= max_M and etay <= max_rapidity
+
+- dataset: CMSDY2D12
+  reason: |
+    Removes data points for which the NNLO K-factors are very large and
+    thus the NLO calculation is unreliable.
+  PTO: NNLO-
+  local_variables:
+    M: sqrt(M2)
+    max_M: 200.0
+  rule: M <= max_M
+
+- dataset: CMSDY2D12
+  reason: Remove data points for which electroweak corrections are large.
+  PTO: NNLO-
+  local_variables:
+    M: sqrt(M2)
+    min_M: 30.0
+    max_rapidity: 2.2
+  rule: M >= min_M and etay <= max_rapidity
+
+- dataset: CMSDY2D12
+  reason: Remove data points for which electroweak corrections are large.
+  PTO: NNLO
+  local_variables:
+    M: sqrt(M2)
+    max_rapidity: 2.2
+    max_M: 200.0
+  rule: M <= max_M and etay <= max_rapidity
+
+- dataset: ATLASZHIGHMASS49FB
+  reason: Avoid the region where resummation effects become important.
+  local_variables:
+    max_M: 200.0
+  rule: M_ll <= max_M
 
 # - dataset: LHCBLOWMASS37PB
 #   rule: "pT <= maxCMSDY2Dminv"
@@ -258,57 +298,57 @@
     maxY: 0.663
   rule: "tau <= maxTau and fabs(y/ymax) <= maxY"
 
-#- dataset: DYE605_dw
-  #reason: |
-    #Remove data points for which the fixed-order perturbative expansion is not
-    #reliable since resummation effects are large. A justification of these cuts
-    #can be found in arXiv:1507.01006.
-  #process_type: DYP
-  #local_variables:
-    #tau: M2 / sqrts**2
-    #ymax: -0.5 * log(tau)
-    #maxTau: 0.080
-    #maxY: 0.663
-  #rule: "tau <= maxTau and fabs(y/ymax) <= maxY"
-
-#- dataset: DYE605_sh
-  #reason: |
-    #Remove data points for which the fixed-order perturbative expansion is not
-    #reliable since resummation effects are large. A justification of these cuts
-    #can be found in arXiv:1507.01006.
-  #process_type: DYP
-  #local_variables:
-    #tau: M2 / sqrts**2
-    #ymax: -0.5 * log(tau)
-    #maxTau: 0.080
-    #maxY: 0.663
-  #rule: "tau <= maxTau and fabs(y/ymax) <= maxY"
-
-#- dataset: DYE605_dw_ite
-  #reason: |
-    #Remove data points for which the fixed-order perturbative expansion is not
-    #reliable since resummation effects are large. A justification of these cuts
-    #can be found in arXiv:1507.01006.
-  #process_type: DYP
-  #local_variables:
-    #tau: M2 / sqrts**2
-    #ymax: -0.5 * log(tau)
-    #maxTau: 0.080
-    #maxY: 0.663
-  #rule: "tau <= maxTau and fabs(y/ymax) <= maxY"
-
-#- dataset: DYE605_sh_ite
-  #reason: |
-    #Remove data points for which the fixed-order perturbative expansion is not
-    #reliable since resummation effects are large. A justification of these cuts
-    #can be found in arXiv:1507.01006.
-  #process_type: DYP
-  #local_variables:
-    #tau: M2 / sqrts**2
-    #ymax: -0.5 * log(tau)
-    #maxTau: 0.080
-    #maxY: 0.663
-  #rule: "tau <= maxTau and fabs(y/ymax) <= maxY"
+- dataset: DYE605_dw
+  reason: |
+    Remove data points for which the fixed-order perturbative expansion is not
+    reliable since resummation effects are large. A justification of these cuts
+    can be found in arXiv:1507.01006.
+  process_type: DYP
+  local_variables:
+    tau: M2 / sqrts**2
+    ymax: -0.5 * log(tau)
+    maxTau: 0.080
+    maxY: 0.663
+  rule: "tau <= maxTau and fabs(y/ymax) <= maxY"
+
+- dataset: DYE605_sh
+  reason: |
+    Remove data points for which the fixed-order perturbative expansion is not
+    reliable since resummation effects are large. A justification of these cuts
+    can be found in arXiv:1507.01006.
+  process_type: DYP
+  local_variables:
+    tau: M2 / sqrts**2
+    ymax: -0.5 * log(tau)
+    maxTau: 0.080
+    maxY: 0.663
+  rule: "tau <= maxTau and fabs(y/ymax) <= maxY"
+
+- dataset: DYE605_dw_ite
+  reason: |
+    Remove data points for which the fixed-order perturbative expansion is not
+    reliable since resummation effects are large. A justification of these cuts
+    can be found in arXiv:1507.01006.
+  process_type: DYP
+  local_variables:
+    tau: M2 / sqrts**2
+    ymax: -0.5 * log(tau)
+    maxTau: 0.080
+    maxY: 0.663
+  rule: "tau <= maxTau and fabs(y/ymax) <= maxY"
+
+- dataset: DYE605_sh_ite
+  reason: |
+    Remove data points for which the fixed-order perturbative expansion is not
+    reliable since resummation effects are large. A justification of these cuts
+    can be found in arXiv:1507.01006.
+  process_type: DYP
+  local_variables:
+    tau: M2 / sqrts**2
+    ymax: -0.5 * log(tau)
+    maxTau: 0.080
+    maxY: 0.663
+  rule: "tau <= maxTau and fabs(y/ymax) <= maxY"
 
 - process_type: DIS_ALL
   reason: Generalised cut for all DIS processes
@@ -350,26 +390,3 @@
   FNS: FONLL-C
   IC: True
   rule: "Q2 > 8"
-
-# Below are cuts that are specific to the PDF4EFT fits
-
-- dataset: CMSDY2D11
-  reason: |
-    Remove points at large values of the rapidity. The theoretical description
-    of these points is poor, even at NNLO QCD + NLO EW. Removing these points
-    results in a significant improvement in this dataset's chi2. Note that this
-    cut is part of the standard NNPDF cuts for this dataset.
-  rule: etay <= 2.2
-
-- dataset: CMSDY2D11
-  reason: |
-    Specific to PDF4EFT fits. Cut points that go beyond LO kinematic limit in
-    rapidity for highest invariant mass bin.
-  rule: etay < 1.5 if M2 == 7.225e+05 else True
-
-- dataset: ATLASDY2D8TEV
-  reason: |
-    Specific to PDF4EFT fits. Cut points that go beyond LO kinematic limit in
-    rapidity for highest invariant mass bin.
-  rule: etay < 1.68 if M2 == 1.0e+06 else True
-