adding year month to disk join where clause (#5443)

--------- Co-authored-by: myersCody <[email protected]> Co-authored-by: Cody Myers <[email protected]>
project-koku · Jan 14, 2025 · 38946b4 · 38946b4
1 parent fd3f76e
commit 38946b4
Show file tree

Hide file tree

Showing 2 changed files with 30 additions and 16 deletions.
diff --git a/koku/masu/database/trino_sql/reporting_ocpawscostlineitem_daily_summary.sql b/koku/masu/database/trino_sql/reporting_ocpawscostlineitem_daily_summary.sql
@@ -684,7 +684,6 @@ JOIN hive.{{schema | sqlsafe}}.aws_openshift_daily_resource_matched_temp as aws
 JOIN hive.{{schema | sqlsafe}}.aws_openshift_disk_capacities_temp AS aws_disk
     ON aws_disk.usage_start = aws.usage_start
     AND aws_disk.resource_id = aws.resource_id
-    AND aws_disk.ocp_source = {{ocp_source_uuid}}
 WHERE ocp.source = {{ocp_source_uuid}}
     AND ocp.year = {{year}}
     AND lpad(ocp.month, 2, '0') = {{month}} -- Zero pad the month when fewer than 2 characters
@@ -697,6 +696,9 @@ WHERE ocp.source = {{ocp_source_uuid}}
     AND aws.data_transfer_direction IS NULL
     AND ocp.namespace != 'Storage unattributed'
     AND aws.resource_id_matched = True
+    AND aws_disk.year = {{year}}
+    AND aws_disk.month = {{month}}
+    AND aws_disk.ocp_source = {{ocp_source_uuid}}
 GROUP BY aws.uuid, ocp.namespace, ocp.pod_labels, ocp.volume_labels
 {% endif %}
 ;
@@ -749,15 +751,17 @@ WITH cte_total_pv_capacity as (
             aws.resource_id as aws_resource_id
         FROM hive.{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp
         JOIN hive.{{schema | sqlsafe}}.aws_openshift_daily_resource_matched_temp as aws
-        ON (aws.usage_start = ocp.usage_start)
-        AND strpos(aws.resource_id, ocp.csi_volume_handle) > 0
-        AND ocp.csi_volume_handle is not null
-        AND ocp.csi_volume_handle != ''
-        WHERE ocp.source = {{ocp_source_uuid}}
-            AND ocp.year = {{year}}
+            ON (aws.usage_start = ocp.usage_start)
+            AND strpos(aws.resource_id, ocp.csi_volume_handle) > 0
+            AND ocp.csi_volume_handle is not null
+            AND ocp.csi_volume_handle != ''
+        WHERE ocp.year = {{year}}
             AND lpad(ocp.month, 2, '0') = {{month}}
             AND ocp.usage_start >= {{start_date}}
             AND ocp.usage_start < date_add('day', 1, {{end_date}})
+            AND aws.ocp_source = {{ocp_source_uuid}}
+            AND aws.year = {{year}}
+            AND aws.month = {{month}}
         GROUP BY ocp.persistentvolume, aws.resource_id
     ) as combined_requests group by aws_resource_id
 )
@@ -802,7 +806,6 @@ JOIN hive.{{schema | sqlsafe}}.aws_openshift_daily_resource_matched_temp as aws
 JOIN hive.{{schema | sqlsafe}}.aws_openshift_disk_capacities_temp AS aws_disk
     ON aws_disk.usage_start = aws.usage_start
     AND aws_disk.resource_id = aws.resource_id
-    AND aws_disk.ocp_source = {{ocp_source_uuid}}
 LEFT JOIN cte_total_pv_capacity as pv_cap
     ON pv_cap.aws_resource_id = aws.resource_id
 WHERE ocp.source = {{ocp_source_uuid}}
@@ -817,6 +820,9 @@ WHERE ocp.source = {{ocp_source_uuid}}
     AND aws.data_transfer_direction IS NULL
     AND ocp.namespace != 'Storage unattributed'
     AND aws_disk.capacity != pv_cap.total_pv_capacity -- prevent inserting zero cost rows
+    AND aws_disk.year = {{year}}
+    AND aws_disk.month = {{month}}
+    AND aws_disk.ocp_source = {{ocp_source_uuid}}
 GROUP BY aws.uuid, aws.resource_id
 {% endif %}
 ;
@@ -933,7 +939,9 @@ SELECT aws.uuid as aws_uuid,
     LEFT JOIN hive.{{schema | sqlsafe}}.aws_openshift_disk_capacities_temp AS aws_disk
         ON aws_disk.usage_start = aws.usage_start
         AND aws_disk.resource_id = aws.resource_id
-        AND aws_disk.ocp_source = {{ocp_source_uuid}}
+        AND aws_disk.year = aws.year
+        AND aws_disk.month = aws.month
+        AND aws_disk.ocp_source = aws.ocp_source
     WHERE ocp.source = {{ocp_source_uuid}}
         AND ocp.year = {{year}}
         AND lpad(ocp.month, 2, '0') = {{month}} -- Zero pad the month when fewer than 2 characters

diff --git a/koku/masu/database/trino_sql/reporting_ocpazurecostlineitem_daily_summary.sql b/koku/masu/database/trino_sql/reporting_ocpazurecostlineitem_daily_summary.sql
@@ -445,7 +445,6 @@ SELECT cast(uuid() as varchar) as azure_uuid,
     JOIN hive.{{schema | sqlsafe}}.azure_openshift_disk_capacities_temp AS az_disk
         ON az_disk.usage_start = azure.usage_start
         AND az_disk.resource_id = azure.resource_id
-        AND az_disk.ocp_source = {{ocp_source_uuid}}
     WHERE ocp.source = {{ocp_source_uuid}}
         AND ocp.year = {{year}}
         AND lpad(ocp.month, 2, '0') = {{month}} -- Zero pad the month when fewer than 2 characters
@@ -458,6 +457,9 @@ SELECT cast(uuid() as varchar) as azure_uuid,
         AND azure.year = {{year}}
         AND azure.month = {{month}}
         AND ocp.namespace != 'Storage unattributed'
+        AND az_disk.year = {{year}}
+        AND az_disk.month = {{month}}
+        AND az_disk.ocp_source = {{ocp_source_uuid}}
     GROUP BY azure.uuid, ocp.namespace, ocp.data_source, ocp.pod_labels, ocp.volume_labels
 -- The endif needs to come before the ; when using sqlparse
 {% endif %}
@@ -508,11 +510,13 @@ WITH cte_total_pv_capacity as (
             OR
                 (lower(ocp.csi_volume_handle) = lower(azure.resource_id))
             )
-        WHERE ocp.source = {{ocp_source_uuid}}
-            AND ocp.year = {{year}}
+        WHERE ocp.year = {{year}}
             AND lpad(ocp.month, 2, '0') = {{month}}
             AND ocp.usage_start >= {{start_date}}
             AND ocp.usage_start < date_add('day', 1, {{end_date}})
+            AND azure.ocp_source = {{ocp_source_uuid}}
+            AND azure.year = {{year}}
+            AND azure.month = {{month}}
         GROUP BY ocp.persistentvolume, azure.resource_id
     ) as combined_requests group by azure_resource_id
 )
@@ -551,7 +555,6 @@ SELECT cast(uuid() as varchar) as azure_uuid, -- need a new uuid or it will dedu
     JOIN hive.{{schema | sqlsafe}}.azure_openshift_disk_capacities_temp AS az_disk
         ON az_disk.usage_start = azure.usage_start
         AND az_disk.resource_id = azure.resource_id
-        AND az_disk.ocp_source = {{ocp_source_uuid}}
     LEFT JOIN cte_total_pv_capacity as pv_cap
         ON pv_cap.azure_resource_id = azure.resource_id
     WHERE ocp.source = {{ocp_source_uuid}}
@@ -563,6 +566,9 @@ SELECT cast(uuid() as varchar) as azure_uuid, -- need a new uuid or it will dedu
         AND azure.year = {{year}}
         AND azure.month = {{month}}
         AND ocp.namespace != 'Storage unattributed'
+        AND az_disk.year = {{year}}
+        AND az_disk.month = {{month}}
+        AND az_disk.ocp_source = {{ocp_source_uuid}}
     GROUP BY azure.uuid, ocp.data_source, azure.resource_id
 {% endif %}
 ;
@@ -665,9 +671,9 @@ SELECT azure.uuid as azure_uuid,
             )
     LEFT JOIN hive.{{schema | sqlsafe}}.azure_openshift_disk_capacities_temp as disk_cap
         ON azure.resource_id = disk_cap.resource_id
-        AND azure.ocp_source = disk_cap.ocp_source
-        AND azure.year = disk_cap.year
-        AND azure.month = disk_cap.month
+        AND disk_cap.year = azure.year
+        AND disk_cap.month = azure.month
+        AND disk_cap.ocp_source = azure.ocp_source
     WHERE ocp.source = {{ocp_source_uuid}}
         AND ocp.year = {{year}}
         AND lpad(ocp.month, 2, '0') = {{month}} -- Zero pad the month when fewer than 2 characters