From 14359f142779090123d31feaca31424f6670fd5e Mon Sep 17 00:00:00 2001 From: Courtney Holcomb Date: Wed, 13 Nov 2024 13:44:25 -0800 Subject: [PATCH] WIP - testing custom offset SQL --- .../test_cumulative_metric_rendering.py | 8 +- ...cumulative_metric_grain_to_date__plan0.sql | 204 +++++++++++------- ..._metric_grain_to_date__plan0_optimized.sql | 22 +- ..._with_offset_to_grain__plan0_optimized.sql | 71 ++---- ...ic_with_offset_window__plan0_optimized.sql | 46 +--- 5 files changed, 160 insertions(+), 191 deletions(-) diff --git a/tests_metricflow/query_rendering/test_cumulative_metric_rendering.py b/tests_metricflow/query_rendering/test_cumulative_metric_rendering.py index 0dbfae51fc..871e21288d 100644 --- a/tests_metricflow/query_rendering/test_cumulative_metric_rendering.py +++ b/tests_metricflow/query_rendering/test_cumulative_metric_rendering.py @@ -241,13 +241,7 @@ def test_cumulative_metric_grain_to_date( """Tests rendering a query against a grain_to_date cumulative metric.""" query_spec = MetricFlowQuerySpec( metric_specs=(MetricSpec(element_name="revenue_mtd"),), - time_dimension_specs=( - TimeDimensionSpec( - element_name="ds", - entity_links=(), - time_granularity=ExpandedTimeGranularity.from_time_granularity(TimeGranularity.MONTH), - ), - ), + time_dimension_specs=(MTD_SPEC_DAY,), ) render_and_check( diff --git a/tests_metricflow/snapshots/test_cumulative_metric_rendering.py/SqlQueryPlan/DuckDB/test_cumulative_metric_grain_to_date__plan0.sql b/tests_metricflow/snapshots/test_cumulative_metric_rendering.py/SqlQueryPlan/DuckDB/test_cumulative_metric_grain_to_date__plan0.sql index cdccdf772a..31e30fbe2b 100644 --- a/tests_metricflow/snapshots/test_cumulative_metric_rendering.py/SqlQueryPlan/DuckDB/test_cumulative_metric_grain_to_date__plan0.sql +++ b/tests_metricflow/snapshots/test_cumulative_metric_rendering.py/SqlQueryPlan/DuckDB/test_cumulative_metric_grain_to_date__plan0.sql @@ -6,89 +6,141 @@ sql_engine: DuckDB --- -- Compute Metrics via Expressions SELECT - subq_3.ds__month - , subq_3.txn_revenue AS revenue_mtd + subq_6.metric_time__day + , subq_6.txn_revenue AS revenue_mtd FROM ( -- Aggregate Measures SELECT - subq_2.ds__month - , SUM(subq_2.txn_revenue) AS txn_revenue + subq_5.metric_time__day + , SUM(subq_5.txn_revenue) AS txn_revenue FROM ( - -- Pass Only Elements: ['txn_revenue', 'ds__month'] + -- Pass Only Elements: ['txn_revenue', 'metric_time__day'] SELECT - subq_1.ds__month - , subq_1.txn_revenue + subq_4.metric_time__day + , subq_4.txn_revenue FROM ( - -- Metric Time Dimension 'ds' + -- Join Self Over Time Range SELECT - subq_0.ds__day - , subq_0.ds__week - , subq_0.ds__month - , subq_0.ds__quarter - , subq_0.ds__year - , subq_0.ds__extract_year - , subq_0.ds__extract_quarter - , subq_0.ds__extract_month - , subq_0.ds__extract_day - , subq_0.ds__extract_dow - , subq_0.ds__extract_doy - , subq_0.revenue_instance__ds__day - , subq_0.revenue_instance__ds__week - , subq_0.revenue_instance__ds__month - , subq_0.revenue_instance__ds__quarter - , subq_0.revenue_instance__ds__year - , subq_0.revenue_instance__ds__extract_year - , subq_0.revenue_instance__ds__extract_quarter - , subq_0.revenue_instance__ds__extract_month - , subq_0.revenue_instance__ds__extract_day - , subq_0.revenue_instance__ds__extract_dow - , subq_0.revenue_instance__ds__extract_doy - , subq_0.ds__day AS metric_time__day - , subq_0.ds__week AS metric_time__week - , subq_0.ds__month AS metric_time__month - , subq_0.ds__quarter AS metric_time__quarter - , subq_0.ds__year AS metric_time__year - , subq_0.ds__extract_year AS metric_time__extract_year - , subq_0.ds__extract_quarter AS metric_time__extract_quarter - , subq_0.ds__extract_month AS metric_time__extract_month - , subq_0.ds__extract_day AS metric_time__extract_day - , subq_0.ds__extract_dow AS metric_time__extract_dow - , subq_0.ds__extract_doy AS metric_time__extract_doy - , subq_0.user - , subq_0.revenue_instance__user - , subq_0.txn_revenue + subq_2.metric_time__day AS metric_time__day + , subq_1.ds__day AS ds__day + , subq_1.ds__week AS ds__week + , subq_1.ds__month AS ds__month + , subq_1.ds__quarter AS ds__quarter + , subq_1.ds__year AS ds__year + , subq_1.ds__extract_year AS ds__extract_year + , subq_1.ds__extract_quarter AS ds__extract_quarter + , subq_1.ds__extract_month AS ds__extract_month + , subq_1.ds__extract_day AS ds__extract_day + , subq_1.ds__extract_dow AS ds__extract_dow + , subq_1.ds__extract_doy AS ds__extract_doy + , subq_1.revenue_instance__ds__day AS revenue_instance__ds__day + , subq_1.revenue_instance__ds__week AS revenue_instance__ds__week + , subq_1.revenue_instance__ds__month AS revenue_instance__ds__month + , subq_1.revenue_instance__ds__quarter AS revenue_instance__ds__quarter + , subq_1.revenue_instance__ds__year AS revenue_instance__ds__year + , subq_1.revenue_instance__ds__extract_year AS revenue_instance__ds__extract_year + , subq_1.revenue_instance__ds__extract_quarter AS revenue_instance__ds__extract_quarter + , subq_1.revenue_instance__ds__extract_month AS revenue_instance__ds__extract_month + , subq_1.revenue_instance__ds__extract_day AS revenue_instance__ds__extract_day + , subq_1.revenue_instance__ds__extract_dow AS revenue_instance__ds__extract_dow + , subq_1.revenue_instance__ds__extract_doy AS revenue_instance__ds__extract_doy + , subq_1.metric_time__week AS metric_time__week + , subq_1.metric_time__month AS metric_time__month + , subq_1.metric_time__quarter AS metric_time__quarter + , subq_1.metric_time__year AS metric_time__year + , subq_1.metric_time__extract_year AS metric_time__extract_year + , subq_1.metric_time__extract_quarter AS metric_time__extract_quarter + , subq_1.metric_time__extract_month AS metric_time__extract_month + , subq_1.metric_time__extract_day AS metric_time__extract_day + , subq_1.metric_time__extract_dow AS metric_time__extract_dow + , subq_1.metric_time__extract_doy AS metric_time__extract_doy + , subq_1.user AS user + , subq_1.revenue_instance__user AS revenue_instance__user + , subq_1.txn_revenue AS txn_revenue FROM ( - -- Read Elements From Semantic Model 'revenue' + -- Time Spine SELECT - revenue_src_28000.revenue AS txn_revenue - , DATE_TRUNC('day', revenue_src_28000.created_at) AS ds__day - , DATE_TRUNC('week', revenue_src_28000.created_at) AS ds__week - , DATE_TRUNC('month', revenue_src_28000.created_at) AS ds__month - , DATE_TRUNC('quarter', revenue_src_28000.created_at) AS ds__quarter - , DATE_TRUNC('year', revenue_src_28000.created_at) AS ds__year - , EXTRACT(year FROM revenue_src_28000.created_at) AS ds__extract_year - , EXTRACT(quarter FROM revenue_src_28000.created_at) AS ds__extract_quarter - , EXTRACT(month FROM revenue_src_28000.created_at) AS ds__extract_month - , EXTRACT(day FROM revenue_src_28000.created_at) AS ds__extract_day - , EXTRACT(isodow FROM revenue_src_28000.created_at) AS ds__extract_dow - , EXTRACT(doy FROM revenue_src_28000.created_at) AS ds__extract_doy - , DATE_TRUNC('day', revenue_src_28000.created_at) AS revenue_instance__ds__day - , DATE_TRUNC('week', revenue_src_28000.created_at) AS revenue_instance__ds__week - , DATE_TRUNC('month', revenue_src_28000.created_at) AS revenue_instance__ds__month - , DATE_TRUNC('quarter', revenue_src_28000.created_at) AS revenue_instance__ds__quarter - , DATE_TRUNC('year', revenue_src_28000.created_at) AS revenue_instance__ds__year - , EXTRACT(year FROM revenue_src_28000.created_at) AS revenue_instance__ds__extract_year - , EXTRACT(quarter FROM revenue_src_28000.created_at) AS revenue_instance__ds__extract_quarter - , EXTRACT(month FROM revenue_src_28000.created_at) AS revenue_instance__ds__extract_month - , EXTRACT(day FROM revenue_src_28000.created_at) AS revenue_instance__ds__extract_day - , EXTRACT(isodow FROM revenue_src_28000.created_at) AS revenue_instance__ds__extract_dow - , EXTRACT(doy FROM revenue_src_28000.created_at) AS revenue_instance__ds__extract_doy - , revenue_src_28000.user_id AS user - , revenue_src_28000.user_id AS revenue_instance__user - FROM ***************************.fct_revenue revenue_src_28000 - ) subq_0 - ) subq_1 - ) subq_2 + subq_3.ds AS metric_time__day + FROM ***************************.mf_time_spine subq_3 + ) subq_2 + INNER JOIN ( + -- Metric Time Dimension 'ds' + SELECT + subq_0.ds__day + , subq_0.ds__week + , subq_0.ds__month + , subq_0.ds__quarter + , subq_0.ds__year + , subq_0.ds__extract_year + , subq_0.ds__extract_quarter + , subq_0.ds__extract_month + , subq_0.ds__extract_day + , subq_0.ds__extract_dow + , subq_0.ds__extract_doy + , subq_0.revenue_instance__ds__day + , subq_0.revenue_instance__ds__week + , subq_0.revenue_instance__ds__month + , subq_0.revenue_instance__ds__quarter + , subq_0.revenue_instance__ds__year + , subq_0.revenue_instance__ds__extract_year + , subq_0.revenue_instance__ds__extract_quarter + , subq_0.revenue_instance__ds__extract_month + , subq_0.revenue_instance__ds__extract_day + , subq_0.revenue_instance__ds__extract_dow + , subq_0.revenue_instance__ds__extract_doy + , subq_0.ds__day AS metric_time__day + , subq_0.ds__week AS metric_time__week + , subq_0.ds__month AS metric_time__month + , subq_0.ds__quarter AS metric_time__quarter + , subq_0.ds__year AS metric_time__year + , subq_0.ds__extract_year AS metric_time__extract_year + , subq_0.ds__extract_quarter AS metric_time__extract_quarter + , subq_0.ds__extract_month AS metric_time__extract_month + , subq_0.ds__extract_day AS metric_time__extract_day + , subq_0.ds__extract_dow AS metric_time__extract_dow + , subq_0.ds__extract_doy AS metric_time__extract_doy + , subq_0.user + , subq_0.revenue_instance__user + , subq_0.txn_revenue + FROM ( + -- Read Elements From Semantic Model 'revenue' + SELECT + revenue_src_28000.revenue AS txn_revenue + , DATE_TRUNC('day', revenue_src_28000.created_at) AS ds__day + , DATE_TRUNC('week', revenue_src_28000.created_at) AS ds__week + , DATE_TRUNC('month', revenue_src_28000.created_at) AS ds__month + , DATE_TRUNC('quarter', revenue_src_28000.created_at) AS ds__quarter + , DATE_TRUNC('year', revenue_src_28000.created_at) AS ds__year + , EXTRACT(year FROM revenue_src_28000.created_at) AS ds__extract_year + , EXTRACT(quarter FROM revenue_src_28000.created_at) AS ds__extract_quarter + , EXTRACT(month FROM revenue_src_28000.created_at) AS ds__extract_month + , EXTRACT(day FROM revenue_src_28000.created_at) AS ds__extract_day + , EXTRACT(isodow FROM revenue_src_28000.created_at) AS ds__extract_dow + , EXTRACT(doy FROM revenue_src_28000.created_at) AS ds__extract_doy + , DATE_TRUNC('day', revenue_src_28000.created_at) AS revenue_instance__ds__day + , DATE_TRUNC('week', revenue_src_28000.created_at) AS revenue_instance__ds__week + , DATE_TRUNC('month', revenue_src_28000.created_at) AS revenue_instance__ds__month + , DATE_TRUNC('quarter', revenue_src_28000.created_at) AS revenue_instance__ds__quarter + , DATE_TRUNC('year', revenue_src_28000.created_at) AS revenue_instance__ds__year + , EXTRACT(year FROM revenue_src_28000.created_at) AS revenue_instance__ds__extract_year + , EXTRACT(quarter FROM revenue_src_28000.created_at) AS revenue_instance__ds__extract_quarter + , EXTRACT(month FROM revenue_src_28000.created_at) AS revenue_instance__ds__extract_month + , EXTRACT(day FROM revenue_src_28000.created_at) AS revenue_instance__ds__extract_day + , EXTRACT(isodow FROM revenue_src_28000.created_at) AS revenue_instance__ds__extract_dow + , EXTRACT(doy FROM revenue_src_28000.created_at) AS revenue_instance__ds__extract_doy + , revenue_src_28000.user_id AS user + , revenue_src_28000.user_id AS revenue_instance__user + FROM ***************************.fct_revenue revenue_src_28000 + ) subq_0 + ) subq_1 + ON + ( + subq_1.metric_time__day <= subq_2.metric_time__day + ) AND ( + subq_1.metric_time__day >= DATE_TRUNC('month', subq_2.metric_time__day) + ) + ) subq_4 + ) subq_5 GROUP BY - subq_2.ds__month -) subq_3 + subq_5.metric_time__day +) subq_6 diff --git a/tests_metricflow/snapshots/test_cumulative_metric_rendering.py/SqlQueryPlan/DuckDB/test_cumulative_metric_grain_to_date__plan0_optimized.sql b/tests_metricflow/snapshots/test_cumulative_metric_rendering.py/SqlQueryPlan/DuckDB/test_cumulative_metric_grain_to_date__plan0_optimized.sql index 08be0535c2..4fdd243606 100644 --- a/tests_metricflow/snapshots/test_cumulative_metric_rendering.py/SqlQueryPlan/DuckDB/test_cumulative_metric_grain_to_date__plan0_optimized.sql +++ b/tests_metricflow/snapshots/test_cumulative_metric_rendering.py/SqlQueryPlan/DuckDB/test_cumulative_metric_grain_to_date__plan0_optimized.sql @@ -4,14 +4,22 @@ docstring: Tests rendering a query against a grain_to_date cumulative metric. sql_engine: DuckDB --- --- Read Elements From Semantic Model 'revenue' --- Metric Time Dimension 'ds' --- Pass Only Elements: ['txn_revenue', 'ds__month'] +-- Join Self Over Time Range +-- Pass Only Elements: ['txn_revenue', 'metric_time__day'] -- Aggregate Measures -- Compute Metrics via Expressions SELECT - DATE_TRUNC('month', created_at) AS ds__month - , SUM(revenue) AS revenue_mtd -FROM ***************************.fct_revenue revenue_src_28000 + subq_10.ds AS metric_time__day + , SUM(revenue_src_28000.revenue) AS revenue_mtd +FROM ***************************.mf_time_spine subq_10 +INNER JOIN + ***************************.fct_revenue revenue_src_28000 +ON + -- for each day, sum everything from the start of the month to today + ( + DATE_TRUNC('day', revenue_src_28000.created_at) <= subq_10.ds + ) AND ( + DATE_TRUNC('day', revenue_src_28000.created_at) >= DATE_TRUNC('month', subq_10.ds) + ) GROUP BY - DATE_TRUNC('month', created_at) + subq_10.ds diff --git a/tests_metricflow/snapshots/test_derived_metric_rendering.py/SqlQueryPlan/DuckDB/test_derived_metric_with_offset_to_grain__plan0_optimized.sql b/tests_metricflow/snapshots/test_derived_metric_rendering.py/SqlQueryPlan/DuckDB/test_derived_metric_with_offset_to_grain__plan0_optimized.sql index d7456bd9df..68bd3ddce4 100644 --- a/tests_metricflow/snapshots/test_derived_metric_rendering.py/SqlQueryPlan/DuckDB/test_derived_metric_with_offset_to_grain__plan0_optimized.sql +++ b/tests_metricflow/snapshots/test_derived_metric_rendering.py/SqlQueryPlan/DuckDB/test_derived_metric_with_offset_to_grain__plan0_optimized.sql @@ -1,59 +1,16 @@ -test_name: test_derived_metric_with_offset_to_grain -test_filename: test_derived_metric_rendering.py -sql_engine: DuckDB ---- --- Compute Metrics via Expressions SELECT - metric_time__day - , bookings - bookings_at_start_of_month AS bookings_growth_since_start_of_month -FROM ( - -- Combine Aggregated Outputs + subq_22.ds AS metric_time__day + , SUM(subq_20.bookings) AS bookings_at_start_of_month +FROM ***************************.mf_time_spine subq_22 +INNER JOIN ( + -- Read Elements From Semantic Model 'bookings_source' + -- Metric Time Dimension 'ds' SELECT - COALESCE(subq_18.metric_time__day, subq_26.metric_time__day) AS metric_time__day - , MAX(subq_18.bookings) AS bookings - , MAX(subq_26.bookings_at_start_of_month) AS bookings_at_start_of_month - FROM ( - -- Aggregate Measures - -- Compute Metrics via Expressions - SELECT - metric_time__day - , SUM(bookings) AS bookings - FROM ( - -- Read Elements From Semantic Model 'bookings_source' - -- Metric Time Dimension 'ds' - -- Pass Only Elements: ['bookings', 'metric_time__day'] - SELECT - DATE_TRUNC('day', ds) AS metric_time__day - , 1 AS bookings - FROM ***************************.fct_bookings bookings_source_src_28000 - ) subq_16 - GROUP BY - metric_time__day - ) subq_18 - FULL OUTER JOIN ( - -- Join to Time Spine Dataset - -- Pass Only Elements: ['bookings', 'metric_time__day'] - -- Aggregate Measures - -- Compute Metrics via Expressions - SELECT - subq_22.ds AS metric_time__day - , SUM(subq_20.bookings) AS bookings_at_start_of_month - FROM ***************************.mf_time_spine subq_22 - INNER JOIN ( - -- Read Elements From Semantic Model 'bookings_source' - -- Metric Time Dimension 'ds' - SELECT - DATE_TRUNC('day', ds) AS metric_time__day - , 1 AS bookings - FROM ***************************.fct_bookings bookings_source_src_28000 - ) subq_20 - ON - DATE_TRUNC('month', subq_22.ds) = subq_20.metric_time__day - GROUP BY - subq_22.ds - ) subq_26 - ON - subq_18.metric_time__day = subq_26.metric_time__day - GROUP BY - COALESCE(subq_18.metric_time__day, subq_26.metric_time__day) -) subq_27 + DATE_TRUNC('day', ds) AS metric_time__day + , 1 AS bookings + FROM ***************************.fct_bookings bookings_source_src_28000 +) subq_20 +ON + DATE_TRUNC('month', subq_22.ds) = subq_20.metric_time__day +GROUP BY + subq_22.ds \ No newline at end of file diff --git a/tests_metricflow/snapshots/test_derived_metric_rendering.py/SqlQueryPlan/DuckDB/test_derived_metric_with_offset_window__plan0_optimized.sql b/tests_metricflow/snapshots/test_derived_metric_rendering.py/SqlQueryPlan/DuckDB/test_derived_metric_with_offset_window__plan0_optimized.sql index 18cde7417b..edf55be2c4 100644 --- a/tests_metricflow/snapshots/test_derived_metric_rendering.py/SqlQueryPlan/DuckDB/test_derived_metric_with_offset_window__plan0_optimized.sql +++ b/tests_metricflow/snapshots/test_derived_metric_rendering.py/SqlQueryPlan/DuckDB/test_derived_metric_with_offset_window__plan0_optimized.sql @@ -1,40 +1,4 @@ -test_name: test_derived_metric_with_offset_window -test_filename: test_derived_metric_rendering.py -sql_engine: DuckDB ---- --- Compute Metrics via Expressions -SELECT - metric_time__day - , bookings - bookings_2_weeks_ago AS bookings_growth_2_weeks -FROM ( - -- Combine Aggregated Outputs - SELECT - COALESCE(subq_18.metric_time__day, subq_26.metric_time__day) AS metric_time__day - , MAX(subq_18.bookings) AS bookings - , MAX(subq_26.bookings_2_weeks_ago) AS bookings_2_weeks_ago - FROM ( - -- Aggregate Measures - -- Compute Metrics via Expressions - SELECT - metric_time__day - , SUM(bookings) AS bookings - FROM ( - -- Read Elements From Semantic Model 'bookings_source' - -- Metric Time Dimension 'ds' - -- Pass Only Elements: ['bookings', 'metric_time__day'] - SELECT - DATE_TRUNC('day', ds) AS metric_time__day - , 1 AS bookings - FROM ***************************.fct_bookings bookings_source_src_28000 - ) subq_16 - GROUP BY - metric_time__day - ) subq_18 - FULL OUTER JOIN ( - -- Join to Time Spine Dataset - -- Pass Only Elements: ['bookings', 'metric_time__day'] - -- Aggregate Measures - -- Compute Metrics via Expressions + SELECT subq_22.ds AS metric_time__day , SUM(subq_20.bookings) AS bookings_2_weeks_ago @@ -50,10 +14,4 @@ FROM ( ON subq_22.ds - INTERVAL 14 day = subq_20.metric_time__day GROUP BY - subq_22.ds - ) subq_26 - ON - subq_18.metric_time__day = subq_26.metric_time__day - GROUP BY - COALESCE(subq_18.metric_time__day, subq_26.metric_time__day) -) subq_27 + subq_22.ds \ No newline at end of file