chore: main merge

.github/workflows/pr-issue-validator.yaml

@@ -29,6 +29,10 @@ jobs:
     steps:
     - name: Checkout repository
       uses: actions/checkout@v2
+      with:
+        ref: ${{ github.event.pull_request.head.sha }}
+        fetch-depth: 0
+
 
     - name: Validate Issue Reference
       env:
@@ -150,3 +154,84 @@ jobs:
             gh pr edit $PRNUM --remove-label "PR:Ready-to-Review"
             exit 1
         fi
+    - name: Check SQL file format and duplicates
+      shell: bash
+      env: 
+          pr_no: ${{ github.event.pull_request.number }}
+          GH_TOKEN: ${{ github.token }}
+      run: |
+        
+        # Fetch the latest changes from the main branch
+        git fetch origin main
+
+        # Get the list of changed files
+        git diff origin/main...HEAD --name-only > diff
+
+        echo "Changed files:"
+        cat diff
+
+        echo "Changed SQL files-:"
+        # Filter SQL files from the list of changed files
+        awk '/scripts\/sql\//' diff
+
+        # Count the number of changed SQL files in the 'scripts/sql' directory
+        count=$(awk '/scripts\/sql\//' diff | wc -l)
+
+        # Check if no SQL files were changed
+        if [[ $count == "0" ]]; then 
+          echo "No SQL files were added, Exiting from this action."
+          exit 0
+        fi
+
+        # Iterate through each changed SQL file
+        for filename in $(awk '/scripts\/sql\//' diff); do
+          echo "Checking File: $filename"
+          
+          # Check if the SQL file name is in the correct format (i.e., it ends with either '.up.sql' or '.down.sql')
+          if [[ "$filename" =~ \.(up|down)\.sql$ ]]; then
+
+            # Print a message that the file name is in the correct format
+            echo "File name: $filename is in the correct format"
+          else
+            # Print an error message
+            echo "Error: The SQL file name is not in the correct format: $filename."
+
+            # Post a comment on a GitHub pull request with the error message
+            gh pr comment $pr_no --body "The SQL file name: $filename is not in the correct format."
+
+            # Exit the script with a non-zero status code
+            exit 1 
+          fi
+
+          # Navigate to the SQL files directory
+          sql_dir="scripts/sql"
+          echo "Current directory: $(pwd)"
+          cd "$sql_dir"
+          echo "SQL files directory: $(pwd)"
+          
+          # Extract the migration number from the SQL file name
+          migration_no=$(echo "$filename" | cut -d "/" -f 3 | cut -d "_" -f 1)
+          echo "Migration Number: $migration_no"
+          
+          # Count the number of files with the same migration number
+          migration_files_present_of_this_no=$(ls | cut -d "_" -f 1 | grep -w -c "$migration_no")
+
+          # Navigate back to the original directory
+          cd ../..
+          
+          # Check the conditions based on the number of files with the same migration number
+          if [[ $migration_files_present_of_this_no == "2" ]]; then
+            echo "All looks good for this migration number."
+          elif [[ $migration_files_present_of_this_no == "1" ]]; then
+            # Only one file is present for this migration number
+            echo "Only single migration file was present for migration no.: $migration_no. either up or down migration is missing! EXITING"
+            gh pr comment $pr_no --body "Error: Only a single migration file was present for this number: $migration_no."
+            exit 1
+          else 
+            # Migration number is repeated
+            echo "Error: Migration number is repeated."
+            gh pr comment $pr_no --body "Error: The SQL file number: $migration_no is duplicated"
+            exit 1
+          fi
+        done
+

App.go

@@ -21,6 +21,7 @@ import (
 	"context"
 	"crypto/tls"
 	"fmt"
+	"github.com/devtron-labs/common-lib/middlewares"
 	"log"
 	"net/http"
 	"os"
@@ -101,6 +102,8 @@ func (app *App) Start() {
 	server := &http.Server{Addr: fmt.Sprintf(":%d", port), Handler: authMiddleware.Authorizer(app.sessionManager2, user.WhitelistChecker)(app.MuxRouter.Router)}
 	app.MuxRouter.Router.Use(app.loggingMiddleware.LoggingMiddleware)
 	app.MuxRouter.Router.Use(middleware.PrometheusMiddleware)
+	app.MuxRouter.Router.Use(middlewares.Recovery)
+
 	if tracerProvider != nil {
 		app.MuxRouter.Router.Use(otelmux.Middleware(otel.OTEL_ORCHESTRASTOR_SERVICE_NAME))
 	}

Wire.go

@@ -22,6 +22,7 @@ package main
 
 import (
 	"github.com/devtron-labs/authenticator/middleware"
+	cloudProviderIdentifier "github.com/devtron-labs/common-lib/cloud-provider-identifier"
 	pubsub1 "github.com/devtron-labs/common-lib/pubsub-lib"
 	util4 "github.com/devtron-labs/common-lib/utils/k8s"
 	"github.com/devtron-labs/devtron/api/apiToken"
@@ -65,6 +66,7 @@ import (
 	"github.com/devtron-labs/devtron/client/gitSensor"
 	"github.com/devtron-labs/devtron/client/grafana"
 	"github.com/devtron-labs/devtron/client/lens"
+	"github.com/devtron-labs/devtron/client/proxy"
 	"github.com/devtron-labs/devtron/client/telemetry"
 	"github.com/devtron-labs/devtron/internal/sql/repository"
 	app2 "github.com/devtron-labs/devtron/internal/sql/repository/app"
@@ -144,6 +146,7 @@ func InitializeApp() (*App, error) {
 		sso.SsoConfigWireSet,
 		cluster.ClusterWireSet,
 		dashboard.DashboardWireSet,
+		proxy.ProxyWireSet,
 		client.HelmAppWireSet,
 		k8s.K8sApplicationWireSet,
 		chartRepo.ChartRepositoryWireSet,
@@ -674,6 +677,9 @@ func InitializeApp() (*App, error) {
 		wire.Bind(new(restHandler.TelemetryRestHandler), new(*restHandler.TelemetryRestHandlerImpl)),
 		telemetry.NewPosthogClient,
 
+		cloudProviderIdentifier.NewProviderIdentifierServiceImpl,
+		wire.Bind(new(cloudProviderIdentifier.ProviderIdentifierService), new(*cloudProviderIdentifier.ProviderIdentifierServiceImpl)),
+
 		telemetry.NewTelemetryEventClientImplExtended,
 		wire.Bind(new(telemetry.TelemetryEventClient), new(*telemetry.TelemetryEventClientImplExtended)),
 

api/appStore/deployment/CommonDeploymentRestHandler.go

@@ -93,7 +93,7 @@ func (handler *CommonDeploymentRestHandlerImpl) getAppOfferingMode(installedAppI
 		}
 		installedAppDto, err = handler.appStoreDeploymentServiceC.GetInstalledAppByClusterNamespaceAndName(appIdentifier.ClusterId, appIdentifier.Namespace, appIdentifier.ReleaseName)
 		if err != nil {
-			err = &util.ApiError{HttpStatusCode: http.StatusInternalServerError, UserMessage: "unable to find app in database"}
+			err = &util.ApiError{HttpStatusCode: http.StatusBadRequest, UserMessage: "unable to find app in database"}
 			return appOfferingMode, installedAppDto, err
 		}
 		// this is the case when hyperion apps does not linked yet
@@ -119,7 +119,7 @@ func (handler *CommonDeploymentRestHandlerImpl) getAppOfferingMode(installedAppI
 		}
 		installedAppDto, err = handler.appStoreDeploymentServiceC.GetInstalledAppByInstalledAppId(installedAppId)
 		if err != nil {
-			err = &util.ApiError{HttpStatusCode: http.StatusInternalServerError, UserMessage: "unable to find app in database"}
+			err = &util.ApiError{HttpStatusCode: http.StatusBadRequest, UserMessage: "unable to find app in database"}
 			return appOfferingMode, installedAppDto, err
 		}
 	} else {

api/cluster/EnvironmentRestHandler.go

@@ -66,7 +66,7 @@ type EnvironmentRestHandlerImpl struct {
 	validator                         *validator.Validate
 	enforcer                          casbin.Enforcer
 	deleteService                     delete2.DeleteService
-	k8sUtil                           *k8s2.K8sUtil
+	k8sUtil                           *k8s2.K8sServiceImpl
 	cfg                               *bean.Config
 }
 
@@ -75,7 +75,7 @@ type ClusterReachableResponse struct {
 	ClusterName      string `json:"clusterName"`
 }
 
-func NewEnvironmentRestHandlerImpl(svc request.EnvironmentService, logger *zap.SugaredLogger, userService user.UserService, validator *validator.Validate, enforcer casbin.Enforcer, deleteService delete2.DeleteService, k8sUtil *k8s2.K8sUtil, k8sCommonService k8s.K8sCommonService) *EnvironmentRestHandlerImpl {
+func NewEnvironmentRestHandlerImpl(svc request.EnvironmentService, logger *zap.SugaredLogger, userService user.UserService, validator *validator.Validate, enforcer casbin.Enforcer, deleteService delete2.DeleteService, k8sUtil *k8s2.K8sServiceImpl, k8sCommonService k8s.K8sCommonService) *EnvironmentRestHandlerImpl {
 	cfg := &bean.Config{}
 	err := env.Parse(cfg)
 	if err != nil {

api/helm-app/HelmAppService.go

@@ -84,7 +84,7 @@ type HelmAppServiceImpl struct {
 	installedAppRepository               repository.InstalledAppRepository
 	appRepository                        app.AppRepository
 	clusterRepository                    clusterRepository.ClusterRepository
-	K8sUtil                              *k8s.K8sUtil
+	K8sUtil                              *k8s.K8sServiceImpl
 	helmReleaseConfig                    *HelmReleaseConfig
 }
 
@@ -94,7 +94,7 @@ func NewHelmAppServiceImpl(Logger *zap.SugaredLogger, clusterService cluster.Clu
 	appStoreApplicationVersionRepository appStoreDiscoverRepository.AppStoreApplicationVersionRepository,
 	environmentService cluster.EnvironmentService, pipelineRepository pipelineConfig.PipelineRepository,
 	installedAppRepository repository.InstalledAppRepository, appRepository app.AppRepository,
-	clusterRepository clusterRepository.ClusterRepository, K8sUtil *k8s.K8sUtil,
+	clusterRepository clusterRepository.ClusterRepository, K8sUtil *k8s.K8sServiceImpl,
 	helmReleaseConfig *HelmReleaseConfig) *HelmAppServiceImpl {
 	return &HelmAppServiceImpl{
 		logger:                               Logger,

api/k8s/application/k8sApplicationRestHandler.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"github.com/devtron-labs/common-lib/utils"
 	"net/http"
 	"strconv"
 	"strings"
@@ -512,8 +513,17 @@ func (handler *K8sApplicationRestHandlerImpl) DeleteResource(w http.ResponseWrit
 
 	resource, err := handler.k8sApplicationService.DeleteResourceWithAudit(r.Context(), &request, userId)
 	if err != nil {
+		errCode := http.StatusInternalServerError
+		if apiErr, ok := err.(*utils.ApiError); ok {
+			errCode = apiErr.HttpStatusCode
+			switch errCode {
+			case http.StatusNotFound:
+				errorMessage := "resource not found"
+				err = fmt.Errorf("%s: %w", errorMessage, err)
+			}
+		}
 		handler.logger.Errorw("error in deleting resource", "err", err)
-		common.WriteJsonResp(w, err, resource, http.StatusInternalServerError)
+		common.WriteJsonResp(w, err, resource, errCode)
 		return
 	}
 	common.WriteJsonResp(w, nil, resource, http.StatusOK)

api/restHandler/ExternalCiRestHandler.go

@@ -100,6 +100,13 @@ func (impl ExternalCiRestHandlerImpl) HandleExternalCiWebhook(w http.ResponseWri
 		common.WriteJsonResp(w, err, nil, http.StatusInternalServerError)
 		return
 	}
+
+	err = impl.validator.Struct(ciArtifactReq)
+	if err != nil {
+		impl.logger.Errorw("validation err, HandleExternalCiWebhook", "err", err, "payload", ciArtifactReq)
+		common.WriteJsonResp(w, err, nil, http.StatusBadRequest)
+		return
+	}
 	_, err = impl.webhookService.HandleExternalCiWebhook(externalCiId, ciArtifactReq, impl.checkExternalCiDeploymentAuth, token)
 	if err != nil {
 		impl.logger.Errorw("service err, HandleExternalCiWebhook", "err", err, "payload", req)

api/restHandler/PipelineTriggerRestHandler.go

@@ -129,7 +129,10 @@ func (handler PipelineTriggerRestHandlerImpl) OverrideConfig(w http.ResponseWrit
 	}
 	ctx := context.WithValue(r.Context(), "token", acdToken)
 	_, span := otel.Tracer("orchestrator").Start(ctx, "workflowDagExecutor.ManualCdTrigger")
-	mergeResp, err := handler.workflowDagExecutor.ManualCdTrigger(&overrideRequest, ctx)
+	triggerContext := pipeline.TriggerContext{
+		Context: ctx,
+	}
+	mergeResp, err := handler.workflowDagExecutor.ManualCdTrigger(triggerContext, &overrideRequest)
 	span.End()
 	if err != nil {
 		handler.logger.Errorw("request err, OverrideConfig", "err", err, "payload", overrideRequest)
@@ -224,7 +227,10 @@ func (handler PipelineTriggerRestHandlerImpl) StartStopApp(w http.ResponseWriter
 		return
 	}
 	ctx := context.WithValue(r.Context(), "token", acdToken)
-	mergeResp, err := handler.workflowDagExecutor.StopStartApp(&overrideRequest, ctx)
+	triggerContext := pipeline.TriggerContext{
+		Context: ctx,
+	}
+	mergeResp, err := handler.workflowDagExecutor.StopStartApp(triggerContext, &overrideRequest)
 	if err != nil {
 		handler.logger.Errorw("service err, StartStopApp", "err", err, "payload", overrideRequest)
 		common.WriteJsonResp(w, err, nil, http.StatusInternalServerError)

api/router/pubsub/ApplicationStatusHandler.go

@@ -24,6 +24,7 @@ import (
 	"fmt"
 	"github.com/devtron-labs/common-lib/pubsub-lib/model"
 	"github.com/devtron-labs/devtron/pkg/app"
+	"k8s.io/utils/pointer"
 	"time"
 
 	"github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig"
@@ -74,7 +75,7 @@ func NewApplicationStatusHandlerImpl(logger *zap.SugaredLogger, pubsubClient *pu
 	}
 	err := appStatusUpdateHandlerImpl.Subscribe()
 	if err != nil {
-		//logger.Error("err", err)
+		// logger.Error("err", err)
 		return nil
 	}
 	err = appStatusUpdateHandlerImpl.SubscribeDeleteStatus()
@@ -91,7 +92,6 @@ type ApplicationDetail struct {
 
 func (impl *ApplicationStatusHandlerImpl) Subscribe() error {
 	callback := func(msg *model.PubSubMsg) {
-		impl.logger.Debugw("APP_STATUS_UPDATE_REQ", "stage", "raw", "data", msg.Data)
 		applicationDetail := ApplicationDetail{}
 		err := json.Unmarshal([]byte(msg.Data), &applicationDetail)
 		if err != nil {
@@ -109,10 +109,10 @@ func (impl *ApplicationStatusHandlerImpl) Subscribe() error {
 		_, err = impl.pipelineRepository.GetArgoPipelineByArgoAppName(app.ObjectMeta.Name)
 		if err != nil && err == pg.ErrNoRows {
 			impl.logger.Infow("this app not found in pipeline table looking in installed_apps table", "appName", app.ObjectMeta.Name)
-			//if not found in pipeline table then search in installed_apps table
+			// if not found in pipeline table then search in installed_apps table
 			gitOpsDeployedAppNames, err := impl.installedAppRepository.GetAllGitOpsDeploymentAppName()
 			if err != nil && err == pg.ErrNoRows {
-				//no installed_apps found
+				// no installed_apps found
 				impl.logger.Errorw("no installed apps found", "err", err)
 				return
 			} else if err != nil {
@@ -127,17 +127,17 @@ func (impl *ApplicationStatusHandlerImpl) Subscribe() error {
 				devtronGitOpsAppName = app.ObjectMeta.Name
 			}
 			if slices.Contains(gitOpsDeployedAppNames, devtronGitOpsAppName) {
-				//app found in installed_apps table hence setting flag to true
+				// app found in installed_apps table hence setting flag to true
 				isAppStoreApplication = true
 			} else {
-				//app neither found in installed_apps nor in pipeline table hence returning
+				// app neither found in installed_apps nor in pipeline table hence returning
 				return
 			}
 		}
 		isSucceeded, pipelineOverride, err := impl.appService.UpdateDeploymentStatusAndCheckIsSucceeded(app, applicationDetail.StatusTime, isAppStoreApplication)
 		if err != nil {
 			impl.logger.Errorw("error on application status update", "err", err, "msg", string(msg.Data))
-			//TODO - check update for charts - fix this call
+			// TODO - check update for charts - fix this call
 			if err == pg.ErrNoRows {
 				// if not found in charts (which is for devtron apps) try to find in installed app (which is for devtron charts)
 				_, err := impl.installedAppService.UpdateInstalledAppVersionStatus(app)
@@ -153,7 +153,10 @@ func (impl *ApplicationStatusHandlerImpl) Subscribe() error {
 		// invoke DagExecutor, for cd success which will trigger post stage if exist.
 		if isSucceeded {
 			impl.logger.Debugw("git hash history", "list", app.Status.History)
-			err = impl.workflowDagExecutor.HandleDeploymentSuccessEvent(pipelineOverride)
+			triggerContext := pipeline.TriggerContext{
+				ReferenceId: pointer.String(msg.MsgId),
+			}
+			err = impl.workflowDagExecutor.HandleDeploymentSuccessEvent(triggerContext, pipelineOverride)
 			if err != nil {
 				impl.logger.Errorw("deployment success event error", "pipelineOverride", pipelineOverride, "err", err)
 				return
@@ -162,7 +165,13 @@ func (impl *ApplicationStatusHandlerImpl) Subscribe() error {
 		impl.logger.Debugw("application status update completed", "app", app.Name)
 	}
 
-	err := impl.pubsubClient.Subscribe(pubsub.APPLICATION_STATUS_UPDATE_TOPIC, callback)
+	// add required logging here
+	var loggerFunc pubsub.LoggerFunc = func(msg model.PubSubMsg) (string, []interface{}) {
+		return "", nil
+	}
+
+	validations := impl.workflowDagExecutor.GetTriggerValidateFuncs()
+	err := impl.pubsubClient.Subscribe(pubsub.APPLICATION_STATUS_UPDATE_TOPIC, callback, loggerFunc, validations...)
 	if err != nil {
 		impl.logger.Error(err)
 		return err
@@ -173,7 +182,6 @@ func (impl *ApplicationStatusHandlerImpl) Subscribe() error {
 func (impl *ApplicationStatusHandlerImpl) SubscribeDeleteStatus() error {
 	callback := func(msg *model.PubSubMsg) {
 
-		impl.logger.Debugw("APP_STATUS_DELETE_REQ", "stage", "raw", "data", msg.Data)
 		applicationDetail := ApplicationDetail{}
 		err := json.Unmarshal([]byte(msg.Data), &applicationDetail)
 		if err != nil {
@@ -191,7 +199,18 @@ func (impl *ApplicationStatusHandlerImpl) SubscribeDeleteStatus() error {
 			impl.logger.Errorw("error in updating pipeline delete status", "err", err, "appName", app.Name)
 		}
 	}
-	err := impl.pubsubClient.Subscribe(pubsub.APPLICATION_STATUS_DELETE_TOPIC, callback)
+
+	// add required logging here
+	var loggerFunc pubsub.LoggerFunc = func(msg model.PubSubMsg) (string, []interface{}) {
+		applicationDetail := ApplicationDetail{}
+		err := json.Unmarshal([]byte(msg.Data), &applicationDetail)
+		if err != nil {
+			return "unmarshal error on app delete status", []interface{}{"err", err}
+		}
+		return "got message for application status delete", []interface{}{"appName", applicationDetail.Application.Name, "namespace", applicationDetail.Application.Namespace, "deleteTimestamp", applicationDetail.Application.DeletionTimestamp}
+	}
+
+	err := impl.pubsubClient.Subscribe(pubsub.APPLICATION_STATUS_DELETE_TOPIC, callback, loggerFunc)
 	if err != nil {
 		impl.logger.Errorw("error in subscribing to argo application status delete topic", "err", err)
 		return err
@@ -210,7 +229,7 @@ func (impl *ApplicationStatusHandlerImpl) updateArgoAppDeleteStatus(app *v1alpha
 		return errors.New("invalid nats message, pipeline already deleted")
 	}
 	if err == pg.ErrNoRows {
-		//Helm app deployed using argocd
+		// Helm app deployed using argocd
 		var gitHash string
 		if app.Operation != nil && app.Operation.Sync != nil {
 			gitHash = app.Operation.Sync.Revision
@@ -229,7 +248,7 @@ func (impl *ApplicationStatusHandlerImpl) updateArgoAppDeleteStatus(app *v1alpha
 			impl.logger.Errorw("App not found in database", "installedAppId", model.InstalledAppId, "err", err)
 			return fmt.Errorf("app not found in database %s", err)
 		} else if installedApp.DeploymentAppDeleteRequest == false {
-			//TODO 4465 remove app from log after final RCA
+			// TODO 4465 remove app from log after final RCA
 			impl.logger.Infow("Deployment delete not requested for app, not deleting app from DB", "appName", app.Name, "app", app)
 			return nil
 		}