More tweaks to defillama + dagster + blockbatch_markers invalid field…

… bug fix (#1241)

Makefile

@@ -78,7 +78,7 @@ sphinx-serve: .makemarkers/sphinx-docs
 # ----------------------------------------------------------------------------#
 
 IMAGE_TAG = ghcr.io/lithium323/op-analytics:v20250110.2
-IMAGE_TAG_DAGSTER = ghcr.io/lithium323/op-analytics-dagster:v20250110.002
+IMAGE_TAG_DAGSTER = ghcr.io/lithium323/op-analytics-dagster:v20250110.006
 
 .PHONY: uv-build
 uv-build:

ddl/bigquery/dailydata_defillama.revenue_protocols_metadata_v1.sql

@@ -0,0 +1,9 @@
+CREATE EXTERNAL TABLE `oplabs-tools-data.dailydata_defillama.revenue_protocols_metadata_v1`
+WITH PARTITION COLUMNS (dt DATE) 
+OPTIONS (
+    format = 'PARQUET',
+    uris = ['gs://oplabs-tools-data-sink/defillama/revenue_protocols_metadata_v1/*'],
+    hive_partition_uri_prefix = 'gs://oplabs-tools-data-sink/defillama/revenue_protocols_metadata_v1',
+    require_hive_partition_filter = true,
+    decimal_target_types = ["NUMERIC", "BIGNUMERIC"]
+)

ddl/bigquery/dailydata_defillama.dexs_protocols_metadata_v1.sql → ddl/bigquery/dailydata_defillama.volume_protocols_metadata_v1.sql

@@ -1,9 +1,9 @@
-CREATE EXTERNAL TABLE `oplabs-tools-data.dailydata_defillama.dexs_protocols_metadata_v1`
+CREATE EXTERNAL TABLE `oplabs-tools-data.dailydata_defillama.volume_protocols_metadata_v1`
 WITH PARTITION COLUMNS (dt DATE) 
 OPTIONS (
     format = 'PARQUET',
     uris = ['gs://oplabs-tools-data-sink/defillama/dexs_protocols_metadata_v1/*'],
-    hive_partition_uri_prefix = 'gs://oplabs-tools-data-sink/defillama/dexs_protocols_metadata_v1',
+    hive_partition_uri_prefix = 'gs://oplabs-tools-data-sink/defillama/volume_protocols_metadata_v1',
     require_hive_partition_filter = true,
     decimal_target_types = ["NUMERIC", "BIGNUMERIC"]
 )

ddl/clickhouse_oplabs/defillama.revenue_protocols_metadata_v1.sql

@@ -0,0 +1,26 @@
+CREATE DATABASE IF NOT EXISTS defillama;
+
+CREATE TABLE IF NOT EXISTS defillama.revenue_protocols_metadata_v1 (
+    `dt` Date,
+    `name` String,
+    `defillamaId` String,
+    `displayName` Nullable(String),
+    `module` Nullable(String),
+    `category` Nullable(String),
+    `logo` Nullable(String),
+    `chains` Array(Nullable(String)),
+    `protocolType` Nullable(String),
+    `methodologyURL` Nullable(String),
+    `methodology` Array(
+        Tuple(
+            key Nullable(String),
+            value Nullable(String)
+        )
+    ),
+    `latestFetchIsOk` Nullable(Bool),
+    `slug` Nullable(String),
+    `id` Nullable(String),
+    `parentProtocol` Nullable(String)
+) ENGINE = ReplacingMergeTree(dt)
+ORDER BY
+    (dt, name, defillamaId)

ddl/clickhouse_oplabs/defillama.dexs_protocols_metadata_v1.sql → ddl/clickhouse_oplabs/defillama.volume_protocols_metadata_v1.sql

@@ -1,6 +1,6 @@
 CREATE DATABASE IF NOT EXISTS defillama;
 
-CREATE TABLE IF NOT EXISTS defillama.dexs_protocols_metadata_v1 (
+CREATE TABLE IF NOT EXISTS defillama.volume_protocols_metadata_v1 (
     `dt` Date,
     `name` String,
     `defillamaId` String,

helm/dagster/INSTRUCTIONS.md

@@ -0,0 +1,38 @@
+
+# Installing the dagster helm chart
+
+Here is the procedure that was used to install the dagster helm chart.
+```
+$ helm repo add dagster https://dagster-io.github.io/helm
+$ helm show values dagster/dagster > helm/dagster/values.yaml
+$ kubectl create namespace dagster
+$ helm upgrade --install dagster dagster/dagster -f helm/dagster/values.yaml -n dagster
+```
+
+To expose the dagster webserver we created a self-signed certificate and used to configure
+a traefik ingressroute.
+```
+$ openssl genpkey -algorithm RSA -out cert.key -pkeyopt rsa_keygen_bits:2048
+$ openssl req -new -key cert.key -out cert.csr
+$ openssl x509 -req -in cert.csr -signkey cert.key -out cert.crt -days 365
+$ kubectl create secret tls dagster-cert-tls --cert=cert.crt --key=cert.key -n dagster
+$ kubectl apply -f helm/dagster/ingressroute.yaml -n dagster
+```
+
+To provide access to secrets from job kubernetes pods we created a service account 
+and secret provider.
+```
+$ kubectl apply -f helm/dagster/secret-provider.yaml
+$ kubectl apply -f helm/dagster/service-account.yaml
+```
+
+Refer to the Google Cloud documentation for more information on how to grant secrets access to
+a kubernetes service account.
+
+# Updating the dagster helm chart
+
+When the dagster definitions docker image changes we need to update the image tag in values.yaml
+and re-deploy the helm chart
+```
+$ helm upgrade dagster dagster/dagster -f helm/dagster/values.yaml -n dagster
+```

helm/dagster/ingressroute.yaml

@@ -0,0 +1,16 @@
+---
+apiVersion: traefik.containo.us/v1alpha1
+kind: IngressRoute
+metadata:
+  name: dagster
+spec:
+  entryPoints:
+  - websecure
+  routes:
+  - kind: Rule
+    match: Host(`dagster.primary.data.tools.oplabs.cloud`)
+    services:
+    - name: dagster-dagster-webserver
+      port: 80
+  tls:
+    secretName: dagster-cert-tls

helm/dagster/secret-provider.yaml

@@ -0,0 +1,11 @@
+apiVersion: secrets-store.csi.x-k8s.io/v1
+kind: SecretProviderClass
+metadata:
+  name: dagster-secret-provider-class
+  namespace: dagster
+spec:
+  provider: gke
+  parameters:
+    secrets: |
+      - resourceName: "projects/oplabs-tools-data/secrets/op-analytics-python-runner/versions/3"
+        path: "op-analytics-vault.txt"

helm/dagster/service-account.yaml

@@ -0,0 +1,5 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: dagster-service-account
+  namespace: dagster