Merge pull request #300 from ScalefreeCOM/test

Pre-Release 1.8.2

macros/internal/metadata_processing/metadata_processing.yml

@@ -0,0 +1,21 @@
+version: 2
+
+macros: 
+  - name: yaml_metadata_parser
+    description: A macro to parse yaml-metadata into single parameters. Used in top-level front-end macros.
+    arguments: 
+      - name: name
+        type: string
+        description: The name of the parameter you want to extract of the yaml-metadata. 
+      - name: yaml_metadata
+        type: string
+        description: The yaml-string that holds the definition of other parameters. Needs to be in yaml format. 
+      - name: parameter
+        type: variable
+        description: The forwarded parameter of the top-level macro. This is used, if the yaml-metadata is none. 
+      - name: required
+        type: boolean
+        description: Whether this parameter is required for the top-level macro. Default is False. 
+      - name: documentation
+        type: string
+        description: A string that holds documentation of this parameter. 

macros/internal/metadata_processing/yaml_metadata_parser.sql

@@ -0,0 +1,28 @@
+{% macro yaml_metadata_parser(name=none, yaml_metadata=none, parameter=none, required=False, documentation=none) %}
+
+    {% if datavault4dbt.is_something(yaml_metadata) %}
+        {%- set metadata_dict = fromyaml(yaml_metadata) -%}
+        {% if name in metadata_dict.keys() %}
+            {% set return_value = metadata_dict.get(name) %}
+            {% if datavault4dbt.is_something(parameter)%}
+                {{ log("[" ~ this ~ "] Parameter '" ~ name ~ "' defined both in yaml-metadata and separately. Value from yaml-metadata will be used, and separate parameter is ignored.", info=False) }}
+            {% endif %}
+        {% elif datavault4dbt.is_something(parameter) %}
+            {% set return_value = parameter %}
+            {{ log("[" ~ this ~ "] yaml-metadata given, but parameter '" ~ name ~ "' not defined in there. Applying '" ~ parameter ~ "' which is either a parameter passed separately or the default value.", info=False) }}
+        {% elif required %}
+            {{ exceptions.raise_compiler_error("[" ~ this ~ "] Error: yaml-metadata given, but required parameter '" ~ name ~ "' not defined in there or outside in the parameter. \n Description of parameter '" ~ name ~ "': \n" ~ documentation ) }}
+        {% else %}
+            {% set return_value = None %}
+        {% endif %}
+    {% elif datavault4dbt.is_something(parameter) %}
+        {% set return_value = parameter %}
+    {% elif required %}
+        {{ exceptions.raise_compiler_error("[" ~ this ~ "] Error: Required parameter '" ~ name ~ "' not defined. Define it either directly, or inside yaml-metadata. \n Description of parameter '" ~ name ~ "': \n" ~ documentation ) }}
+    {% else %}
+        {% set return_value = None %}
+    {% endif %}
+
+    {{ return(return_value) }}
+
+{% endmacro %}

macros/internal/helpers/stage_processing_macros.sql → macros/staging/stage_processing_macros.sql

@@ -54,24 +54,27 @@
                 {# Do nothing. No source column required. #}    
             {%- elif value is mapping and value.is_hashdiff -%}
                 {%- do extracted_input_columns.append(value['columns']) -%}
-            {%- elif value is mapping and 'this_column_name' in value.keys() -%}
-                {%- if datavault4dbt.is_list(value['this_column_name'])-%}
-                    {%- for column in value['this_column_name'] -%}
-                        {%- do extracted_input_columns.append(column) -%}
-                    {%- endfor -%}
-                {%- else -%}
-                    {%- do extracted_input_columns.append(value['this_column_name']) -%}
-                {%- endif -%}
             {%- else -%}
                 {%- do extracted_input_columns.append(value) -%}
             {%- endif -%}
         {%- endfor -%}
-
-        {%- do return(extracted_input_columns) -%}
+
+    {%- elif datavault4dbt.is_list(columns_dict) -%}
+        {% for prejoin in columns_dict %}
+            {%- if datavault4dbt.is_list(prejoin['this_column_name'])-%}
+                {%- for column in prejoin['this_column_name'] -%}
+                    {%- do extracted_input_columns.append(column) -%}
+                {%- endfor -%}
+            {%- else -%}
+                {%- do extracted_input_columns.append(prejoin['this_column_name']) -%}
+            {%- endif -%}
+        {% endfor %}
     {%- else -%}
         {%- do return([]) -%}
     {%- endif -%}
 
+    {%- do return(extracted_input_columns) -%}
+
 {%- endmacro -%}
 
 
@@ -123,4 +126,89 @@
         {%- endif %}
     {%- endfor -%}
 
-{%- endmacro -%}
+{%- endmacro -%}
+
+
+{%- macro process_prejoined_columns(prejoined_columns=none) -%}
+    {# Check if the old syntax is used for prejoined columns
+        If so parse it to new list syntax #}
+
+    {% if datavault4dbt.is_list(prejoined_columns) %}
+        {% do return(prejoined_columns) %}
+    {% else %}
+        {% set output = [] %}
+
+        {% for key, value in prejoined_columns.items() %}
+            {% set ref_model = value.get('ref_model') %}
+            {% set src_name = value.get('src_name') %}
+            {% set src_table = value.get('src_table') %}
+            {%- if 'operator' not in value.keys() -%}  
+                {%- do value.update({'operator': 'AND'}) -%}
+                {%- set operator = 'AND' -%}
+            {%- else -%}
+                {%- set operator = value.get('operator') -%}
+            {%- endif -%}
+
+    {% set match_criteria = (
+            ref_model and output | selectattr('ref_model', 'equalto', ref_model) or
+            src_name and output | selectattr('src_name', 'equalto', src_name) | selectattr('src_table', 'equalto', src_table)
+        ) | selectattr('this_column_name', 'equalto', value.this_column_name)
+        | selectattr('ref_column_name', 'equalto', value.ref_column_name)
+        | selectattr('operator', 'equalto', value.operator)
+        | list | first %}
+
+            {% if match_criteria %}
+                {% do match_criteria['extract_columns'].append(value.bk) %}
+                {% do match_criteria['aliases'].append(key) %}
+            {% else %}
+                {% set new_item = {
+                    'extract_columns': [value.bk],
+                    'aliases': [key],
+                    'this_column_name': value.this_column_name,
+                    'ref_column_name': value.ref_column_name,
+                    'operator': operator
+                } %}
+
+                {% if ref_model %}
+                    {% do new_item.update({'ref_model': ref_model}) %}
+                {% elif src_name and src_table %}
+                    {% do new_item.update({'src_name': src_name, 'src_table': src_table}) %}
+                {% endif %}
+
+                {% do output.append(new_item) %}
+            {% endif %}
+        {% endfor %}
+    {% endif %}
+
+    {%- do return(output) -%}
+
+{%- endmacro -%}
+
+
+{%- macro extract_prejoin_column_names(prejoined_columns=none) -%}
+
+    {%- set extracted_column_names = [] -%}
+
+    {% if not datavault4dbt.is_something(prejoined_columns) %}
+        {%- do return(extracted_column_names) -%}
+    {% endif %}
+
+    {% for prejoin in prejoined_columns %}
+        {% if datavault4dbt.is_list(prejoin['aliases']) %}
+            {% for alias in prejoin['aliases'] %}
+                {%- do extracted_column_names.append(alias) -%}
+            {% endfor %}
+        {% elif datavault4dbt.is_something(prejoin['aliases']) %}
+            {%- do extracted_column_names.append(prejoin['aliases']) -%}
+        {% elif datavault4dbt.is_list(prejoin['extract_columns']) %}
+            {% for column in prejoin['extract_columns'] %}
+                {%- do extracted_column_names.append(column) -%}
+            {% endfor %}
+        {% else %}
+            {%- do extracted_column_names.append(prejoin['extract_columns']) -%}
+        {% endif %}
+    {%- endfor -%}
+
+    {%- do return(extracted_column_names) -%}
+
+{%- endmacro -%}

macros/staging/staging.yml

@@ -0,0 +1,23 @@
+version: 2
+
+macros:
+  - name: process_prejoined_columns
+    description: >
+      A macro to process prejoined columns. If a list of dictioniaries(new syntax) is provided it will do nothing and return the list.
+      If a dictionary of dictionaries if provided(old syntax) it will be transformed to the new syntax.
+      When multiple columns are to be extracted from the same prejoin-target and with the same conditions(columns and operator) they will be combined into one item.
+    arguments:
+      - name: prejoined_columns
+        type: list or dictionary
+        description: The value of the prejoined_columns as defined in the yaml_metadata of the stage-model.
+
+  - name: extract_prejoin_column_names
+    description: >
+      A macro to extract the names of the prejoined columns of each staging-model.
+      Takes a list of prejoins and will add the aliases of the prejoins to the return-list.
+      If no aliases are present it will return the names of the extracted columns.
+      Returns an empty list if the passed parameter is empty.
+    arguments:
+      - name: prejoined_columns
+        type: list
+        description: The prejoined_columns as process by the process_prejoined_columns-macro