diff --git a/.github/workflows/version-release.yml b/.github/workflows/version-release.yml index e0cfe68..1c144f1 100644 --- a/.github/workflows/version-release.yml +++ b/.github/workflows/version-release.yml @@ -12,7 +12,7 @@ jobs: env: DOCKER_REGISTRY: hub.docker.com DOCKER_REPOSITORY: kriten - DOCKER_PLATFORM: linux/amd64 + DOCKER_PLATFORM: linux/amd64,linux/arm64 steps: - name: Check out the repo uses: actions/checkout@v3 @@ -23,6 +23,12 @@ jobs: command: init --parseDependency --parseInternal swagWersion: 1.8.12 + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Log in to Docker Hub uses: docker/login-action@v2 with: @@ -31,12 +37,12 @@ jobs: - name: Extract metadata (tags, labels) for Docker id: meta - uses: docker/metadata-action@v4 + uses: docker/metadata-action@v3 with: images: ${{ secrets.DOCKER_USERNAME }}/${{ env.DOCKER_REPOSITORY }} - name: Build and push Docker image - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v6 with: context: . push: true diff --git a/.gitignore b/.gitignore index 75b87c8..3ca6946 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,6 @@ brainiac-core # Dependency directories (remove the comment below to include it) # vendor/ + +# Local inventory file +.env diff --git a/controllers/runner.go b/controllers/runner.go index 118a50b..6cd2df9 100644 --- a/controllers/runner.go +++ b/controllers/runner.go @@ -42,6 +42,13 @@ func (rc *RunnerController) SetRunnerRoutes(rg *gin.RouterGroup, config config.C r.PATCH("/:id", rc.UpdateRunner) r.PUT("/:id", rc.UpdateRunner) r.DELETE("/:id", rc.DeleteRunner) + + { + r.GET("/:id/secret", rc.GetSecret) + r.POST("/:id/secret", rc.UpdateSecret) + r.PUT("/:id/secret", rc.UpdateSecret) + r.DELETE("/:id/secret", rc.DeleteSecret) + } } } @@ -150,7 +157,7 @@ func (rc *RunnerController) CreateRunner(ctx *gin.Context) { audit.EventTarget = runner.Name - configMap, err := rc.RunnerService.CreateRunner(runner) + runnerData, err := rc.RunnerService.CreateRunner(runner) if err != nil { if errors.IsAlreadyExists(err) { rc.AuditService.CreateAudit(audit) @@ -164,7 +171,7 @@ func (rc *RunnerController) CreateRunner(ctx *gin.Context) { audit.Status = "success" rc.AuditService.CreateAudit(audit) - ctx.JSON(http.StatusOK, configMap.Data) + ctx.JSON(http.StatusOK, runnerData) } // UpdateRunner godoc @@ -194,7 +201,7 @@ func (rc *RunnerController) UpdateRunner(ctx *gin.Context) { return } - configMap, err := rc.RunnerService.UpdateRunner(runner) + runnerData, err := rc.RunnerService.UpdateRunner(runner) if err != nil { if errors.IsNotFound(err) { rc.AuditService.CreateAudit(audit) @@ -208,7 +215,7 @@ func (rc *RunnerController) UpdateRunner(ctx *gin.Context) { audit.Status = "success" rc.AuditService.CreateAudit(audit) - ctx.JSON(http.StatusOK, configMap.Data) + ctx.JSON(http.StatusOK, runnerData) } // DeleteRunner godoc @@ -246,3 +253,107 @@ func (rc *RunnerController) DeleteRunner(ctx *gin.Context) { rc.AuditService.CreateAudit(audit) ctx.JSON(http.StatusOK, gin.H{"msg": "runner deleted successfully"}) } + +// GetSecret godoc +// +// @Summary Get secret +// @Description Get secret associated with runner (passwords are obfuscated) +// @Tags runners +// @Accept json +// @Produce json +// @Param id path string true "Runner name" +// @Success 200 {object} map[string]interface{} +// @Failure 400 {object} helpers.HTTPError +// @Failure 404 {object} helpers.HTTPError +// @Failure 500 {object} helpers.HTTPError +// @Router /tasks/{id}/secret [get] +// @Security Bearer +func (rc *RunnerController) GetSecret(ctx *gin.Context) { + runnerName := ctx.Param("id") + audit := rc.AuditService.InitialiseAuditLog(ctx, "get_secret", rc.AuditCategory, runnerName) + secret, err := rc.RunnerService.GetSecret(runnerName) + + if err != nil { + rc.AuditService.CreateAudit(audit) + ctx.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + + if secret == nil { + ctx.JSON(http.StatusOK, gin.H{"msg": "secret not found"}) + return + } + + audit.Status = "success" + rc.AuditService.CreateAudit(audit) + ctx.JSON(http.StatusOK, secret) +} + +// GetSecret godoc +// +// @Summary Update secret +// @Description Update secret associated with runner +// @Tags runners +// @Accept json +// @Produce json +// @Param id path string true "runner name" +// @Success 200 {object} map[string]interface{} +// @Failure 400 {object} helpers.HTTPError +// @Failure 404 {object} helpers.HTTPError +// @Failure 500 {object} helpers.HTTPError +// @Router /runners/{id}/secret [get] +// @Security Bearer +func (rc *RunnerController) UpdateSecret(ctx *gin.Context) { + runnerName := ctx.Param("id") + audit := rc.AuditService.InitialiseAuditLog(ctx, "update_secret", rc.AuditCategory, runnerName) + var secret map[string]string + + if err := ctx.BindJSON(&secret); err != nil { + rc.AuditService.CreateAudit(audit) + ctx.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + + secretStored, err := rc.RunnerService.UpdateSecret(runnerName, secret) + + if err != nil { + rc.AuditService.CreateAudit(audit) + ctx.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + + audit.Status = "success" + rc.AuditService.CreateAudit(audit) + ctx.JSON(http.StatusOK, secretStored) +} + +// DeleteSecret godoc +// +// @Summary Delete secret +// @Description Remove secret associated with runner +// @Tags runners +// @Accept json +// @Produce json +// @Param id path string true "Runner name" +// @Success 200 {object} map[string]interface{} +// @Failure 400 {object} helpers.HTTPError +// @Failure 404 {object} helpers.HTTPError +// @Failure 500 {object} helpers.HTTPError +// @Router /runners/{id}/schema [delete] +// @Security Bearer +func (rc *RunnerController) DeleteSecret(ctx *gin.Context) { + runnerName := ctx.Param("id") + audit := rc.AuditService.InitialiseAuditLog(ctx, "delete_secret", rc.AuditCategory, runnerName) + + err := rc.RunnerService.DeleteSecret(runnerName) + + if err != nil { + rc.AuditService.CreateAudit(audit) + ctx.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) + return + } + + audit.Status = "success" + rc.AuditService.CreateAudit(audit) + ctx.JSON(http.StatusOK, gin.H{"msg": "secret deleted successfully"}) +} diff --git a/controllers/task.go b/controllers/task.go index dcebb83..2e00c8a 100644 --- a/controllers/task.go +++ b/controllers/task.go @@ -49,11 +49,6 @@ func (tc *TaskController) SetTaskRoutes(rg *gin.RouterGroup, config config.Confi r.POST("/:id/schema", tc.UpdateSchema) r.PUT("/:id/schema", tc.UpdateSchema) r.DELETE("/:id/schema", tc.DeleteSchema) - - r.GET("/:id/secret", tc.GetSecret) - r.POST("/:id/secret", tc.UpdateSecret) - r.PUT("/:id/secret", tc.UpdateSecret) - r.DELETE("/:id/secret", tc.DeleteSecret) } } @@ -117,7 +112,7 @@ func (tc *TaskController) GetTask(ctx *gin.Context) { taskName := ctx.Param("id") audit := tc.AuditService.InitialiseAuditLog(ctx, "get", tc.AuditCategory, taskName) // username := ctx.MustGet("username").(string) - task, secret, err := tc.TaskService.GetTask(taskName) + task, err := tc.TaskService.GetTask(taskName) if err != nil { tc.AuditService.CreateAudit(audit) @@ -132,7 +127,6 @@ func (tc *TaskController) GetTask(ctx *gin.Context) { } audit.Status = "success" - task["secret"] = secret // ctx.JSON(http.StatusOK, gin.H{"msg": "task retrieved successfully", "value": task, "secret": secret}) tc.AuditService.CreateAudit(audit) ctx.JSON(http.StatusOK, task) @@ -163,7 +157,7 @@ func (tc *TaskController) CreateTask(ctx *gin.Context) { } audit.EventTarget = task.Name - taskConfig, secret, err := tc.TaskService.CreateTask(task) + taskConfig, err := tc.TaskService.CreateTask(task) if err != nil { if errors.IsAlreadyExists(err) { tc.AuditService.CreateAudit(audit) @@ -176,7 +170,6 @@ func (tc *TaskController) CreateTask(ctx *gin.Context) { } audit.Status = "success" - taskConfig["secret"] = secret tc.AuditService.CreateAudit(audit) ctx.JSON(http.StatusOK, taskConfig) } @@ -207,7 +200,7 @@ func (tc *TaskController) UpdateTask(ctx *gin.Context) { return } - taskConfig, secret, err := tc.TaskService.UpdateTask(task) + taskConfig, err := tc.TaskService.UpdateTask(task) if err != nil { if errors.IsNotFound(err) { tc.AuditService.CreateAudit(audit) @@ -220,7 +213,6 @@ func (tc *TaskController) UpdateTask(ctx *gin.Context) { } audit.Status = "success" tc.AuditService.CreateAudit(audit) - taskConfig["secret"] = secret ctx.JSON(http.StatusOK, taskConfig) } @@ -362,107 +354,3 @@ func (tc *TaskController) DeleteSchema(ctx *gin.Context) { tc.AuditService.CreateAudit(audit) ctx.JSON(http.StatusOK, gin.H{"msg": "schema deleted successfully"}) } - -// GetSecret godoc -// -// @Summary Get secret -// @Description Get secret associated to a specific task (passwords are obfuscated) -// @Tags tasks -// @Accept json -// @Produce json -// @Param id path string true "Task name" -// @Success 200 {object} map[string]interface{} -// @Failure 400 {object} helpers.HTTPError -// @Failure 404 {object} helpers.HTTPError -// @Failure 500 {object} helpers.HTTPError -// @Router /tasks/{id}/secret [get] -// @Security Bearer -func (tc *TaskController) GetSecret(ctx *gin.Context) { - taskName := ctx.Param("id") - audit := tc.AuditService.InitialiseAuditLog(ctx, "get_secret", tc.AuditCategory, taskName) - secret, err := tc.TaskService.GetSecret(taskName) - - if err != nil { - tc.AuditService.CreateAudit(audit) - ctx.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) - return - } - - if secret == nil { - ctx.JSON(http.StatusOK, gin.H{"msg": "secret not found"}) - return - } - - audit.Status = "success" - tc.AuditService.CreateAudit(audit) - ctx.JSON(http.StatusOK, secret) -} - -// GetSecret godoc -// -// @Summary Update secret -// @Description Update secret associated to a specific task -// @Tags tasks -// @Accept json -// @Produce json -// @Param id path string true "Task name" -// @Success 200 {object} map[string]interface{} -// @Failure 400 {object} helpers.HTTPError -// @Failure 404 {object} helpers.HTTPError -// @Failure 500 {object} helpers.HTTPError -// @Router /tasks/{id}/secret [get] -// @Security Bearer -func (tc *TaskController) UpdateSecret(ctx *gin.Context) { - taskName := ctx.Param("id") - audit := tc.AuditService.InitialiseAuditLog(ctx, "update_secret", tc.AuditCategory, taskName) - var secret map[string]string - - if err := ctx.BindJSON(&secret); err != nil { - tc.AuditService.CreateAudit(audit) - ctx.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) - return - } - - secretStored, err := tc.TaskService.UpdateSecret(taskName, secret) - - if err != nil { - tc.AuditService.CreateAudit(audit) - ctx.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) - return - } - - audit.Status = "success" - tc.AuditService.CreateAudit(audit) - ctx.JSON(http.StatusOK, secretStored) -} - -// DeleteSecret godoc -// -// @Summary Delete secret -// @Description Remove secret associated to a specific task -// @Tags tasks -// @Accept json -// @Produce json -// @Param id path string true "Task name" -// @Success 200 {object} map[string]interface{} -// @Failure 400 {object} helpers.HTTPError -// @Failure 404 {object} helpers.HTTPError -// @Failure 500 {object} helpers.HTTPError -// @Router /tasks/{id}/schema [delete] -// @Security Bearer -func (tc *TaskController) DeleteSecret(ctx *gin.Context) { - taskName := ctx.Param("id") - audit := tc.AuditService.InitialiseAuditLog(ctx, "delete_secret", tc.AuditCategory, taskName) - - err := tc.TaskService.DeleteSecret(taskName) - - if err != nil { - tc.AuditService.CreateAudit(audit) - ctx.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) - return - } - - audit.Status = "success" - tc.AuditService.CreateAudit(audit) - ctx.JSON(http.StatusOK, gin.H{"msg": "secret deleted successfully"}) -} diff --git a/controllers/tokens.go b/controllers/tokens.go index 50eaca4..855cb0c 100644 --- a/controllers/tokens.go +++ b/controllers/tokens.go @@ -9,7 +9,7 @@ import ( "net/http" "github.com/gin-gonic/gin" - "github.com/satori/go.uuid" + uuid "github.com/satori/go.uuid" ) type ApiTokenController struct { diff --git a/helpers/k8s.go b/helpers/k8s.go index e955891..7ee71a8 100644 --- a/helpers/k8s.go +++ b/helpers/k8s.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/json" + "fmt" "io" "kriten/config" "kriten/models" @@ -11,7 +12,6 @@ import ( batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -58,7 +58,6 @@ func CreateOrUpdateConfigMap(kube config.KubeConfig, data map[string]string, ope } if err != nil { - log.Println(err) return nil, err } @@ -97,7 +96,6 @@ func GetSecret(kube config.KubeConfig, secretName string) (*corev1.Secret, error context.TODO(), secretName, metav1.GetOptions{}) if err != nil { - log.Println(err) return nil, err } @@ -135,9 +133,6 @@ func DeleteSecret(kube config.KubeConfig, name string) error { context.TODO(), name, metav1.DeleteOptions{}) if err != nil { - if errors.IsNotFound(err) { - return nil - } log.Println(err) return err } @@ -202,8 +197,8 @@ func GetJob(kube config.KubeConfig, name string) (*batchv1.Job, error) { } // TODO: Too many arguments, will need a rework -func CreateJob(kube config.KubeConfig, name string, runnerImage string, owner string, extraVars string, command string, gitURL string, gitBranch string) (string, error) { - job := JobObject(name, kube, runnerImage, owner, extraVars, command, gitURL, gitBranch) +func CreateJob(kube config.KubeConfig, name string, runnerName string, runnerImage string, owner string, extraVars string, command string, gitURL string, gitBranch string) (string, error) { + job := JobObject(name, kube, runnerName, runnerImage, owner, extraVars, command, gitURL, gitBranch) job, err := kube.Clientset.BatchV1().Jobs( kube.Namespace).Create( @@ -231,8 +226,10 @@ func ListPods(kube config.KubeConfig, labelSelector string) (*corev1.PodList, er } // TODO: Need to implement logs for init-containers -func GetLogs(kube config.KubeConfig, podName string) (string, error) { - podLogOpts := corev1.PodLogOptions{} +func GetLogs(kube config.KubeConfig, podName string, containerName string) (string, error) { + podLogOpts := corev1.PodLogOptions{ + Container: containerName, + } req := kube.Clientset.CoreV1().Pods(kube.Namespace).GetLogs(podName, &podLogOpts) @@ -253,11 +250,21 @@ func GetLogs(kube config.KubeConfig, podName string) (string, error) { return buf.String(), nil } -func JobObject(name string, kube config.KubeConfig, image string, owner string, extraVars string, command string, gitURL string, gitBranch string) *batchv1.Job { - var ttlSeconds int32 = int32(kube.JobsTTL) +func JobObject(name string, + kube config.KubeConfig, + runnerName string, + image string, owner string, + extraVars string, + command string, + gitURL string, + gitBranch string) *batchv1.Job { + + var ttlSeconds = int32(kube.JobsTTL) var backoffLimit int32 = 1 - optional_secret := true + optionalSecret := true + + initCmd := fmt.Sprintf("git clone -b %s %s . ; git ls-remote", gitBranch, gitURL) env := []corev1.EnvVar{} // Append extra vars to environment variables only if provided @@ -289,8 +296,8 @@ func JobObject(name string, kube config.KubeConfig, image string, owner string, Name: "secret", VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: name, - Optional: &optional_secret, + SecretName: runnerName, + Optional: &optionalSecret, }, }, }, @@ -330,9 +337,9 @@ func JobObject(name string, kube config.KubeConfig, image string, owner string, { SecretRef: &corev1.SecretEnvSource{ LocalObjectReference: corev1.LocalObjectReference{ - Name: name, + Name: runnerName, }, - Optional: &optional_secret, + Optional: &optionalSecret, }, }, }, @@ -344,14 +351,9 @@ func JobObject(name string, kube config.KubeConfig, image string, owner string, Image: "bitnami/git", ImagePullPolicy: corev1.PullIfNotPresent, Command: []string{ - "git", - }, - Args: []string{ - "clone", - "-b", - gitBranch, - gitURL, - ".", + "sh", + "-c", + initCmd, }, WorkingDir: "/mnt/repo", VolumeMounts: []corev1.VolumeMount{ @@ -428,6 +430,7 @@ func CreateOrUpdateCronJob(kube config.KubeConfig, cronjob models.CronJob, runne jobObj := JobObject(cronjob.Task, kube, + runner.Data["name"], runner.Data["image"], cronjob.Owner, extraVars, diff --git a/models/role.go b/models/role.go index e64a3b2..873319e 100644 --- a/models/role.go +++ b/models/role.go @@ -4,7 +4,7 @@ import ( "time" "github.com/lib/pq" - "github.com/satori/go.uuid" + uuid "github.com/satori/go.uuid" ) type Role struct { diff --git a/models/runner.go b/models/runner.go index 7c8a6b4..d27512e 100644 --- a/models/runner.go +++ b/models/runner.go @@ -1,9 +1,10 @@ package models type Runner struct { - Name string `json:"name" binding:"required"` - Image string `json:"image" binding:"required"` - GitURL string `json:"gitURL" binding:"required"` - Token string `json:"token"` - Branch string `json:"branch"` + Secret map[string]string `json:"secret,omitempty"` + Name string `json:"name" binding:"required"` + Image string `json:"image" binding:"required"` + GitURL string `json:"gitURL" binding:"required"` + Token string `json:"token"` + Branch string `json:"branch"` } diff --git a/models/task.go b/models/task.go index c3c394c..c746c73 100644 --- a/models/task.go +++ b/models/task.go @@ -1,10 +1,9 @@ package models type Task struct { + Schema map[string]interface{} `json:"schema,omitempty"` Name string `json:"name" binding:"required"` Runner string `json:"runner" binding:"required"` Command string `json:"command" binding:"required"` Synchronous bool `json:"synchronous"` - Secret map[string]string `json:"secret,omitempty"` - Schema map[string]interface{} `json:"schema,omitempty"` } diff --git a/models/token.go b/models/token.go index 55a4b86..96ab8e4 100644 --- a/models/token.go +++ b/models/token.go @@ -3,13 +3,13 @@ package models import ( "time" - "github.com/satori/go.uuid" + uuid "github.com/satori/go.uuid" ) type ApiToken struct { ID uuid.UUID `gorm:"column:id;type:uuid;default:gen_random_uuid()" json:"id"` - Key string `json:"key"` Owner uuid.UUID `gorm:"type:uuid" json:"owner"` + Key string `json:"key,omitempty"` Description string `json:"description,omitempty"` CreatedAt time.Time `json:"created_at"` UpdatedAt time.Time `json:"updated_at"` diff --git a/services/job_svc.go b/services/job_svc.go index 1cf0a4c..c35909f 100644 --- a/services/job_svc.go +++ b/services/job_svc.go @@ -141,11 +141,23 @@ func (j *JobServiceImpl) GetJob(username string, jobID string) (models.Job, erro var logs string for _, pod := range pods.Items { // TODO: this will only retrieve logs for now, can be extended if needed - log, err := helpers.GetLogs(j.config.Kube, pod.Name) - if err != nil { - return jobStatus, err + logs += "## init container logs\n" + for c := range pod.Spec.InitContainers { + log, err := helpers.GetLogs(j.config.Kube, pod.Name, pod.Spec.InitContainers[c].Name) + if err != nil { + return jobStatus, err + } + logs += log + } + + logs += "##\n\n application container logs \n" + for c := range pod.Spec.Containers { + log, err := helpers.GetLogs(j.config.Kube, pod.Name, pod.Spec.Containers[c].Name) + if err != nil { + return jobStatus, err + } + logs += log } - logs = logs + log } @@ -207,20 +219,20 @@ func (j *JobServiceImpl) CreateJob(username string, taskName string, extraVars s if gitBranch == "" { gitBranch = "main" } - - secret, err := helpers.GetSecret(j.config.Kube, runnerName) + tokenObjName := runnerName + "-token" + token, err := helpers.GetSecret(j.config.Kube, tokenObjName) if err != nil { if !kerrors.IsNotFound(err) { return jobStatus, err } } else { - gitToken := string(secret.Data["token"]) + gitToken := string(token.Data["token"]) if gitToken != "" { gitURL = strings.Replace(gitURL, "://", "://"+gitToken+":@", 1) } } - jobID, err := helpers.CreateJob(j.config.Kube, taskName, runnerImage, username, extraVars, task.Data["command"], gitURL, gitBranch) + jobID, err := helpers.CreateJob(j.config.Kube, taskName, runnerName, runnerImage, username, extraVars, task.Data["command"], gitURL, gitBranch) jobStatus.ID = jobID diff --git a/services/runner_svc.go b/services/runner_svc.go index 22ead07..124465a 100644 --- a/services/runner_svc.go +++ b/services/runner_svc.go @@ -9,18 +9,20 @@ import ( "time" "golang.org/x/exp/slices" - v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" ) type RunnerService interface { ListRunners([]string) ([]map[string]string, error) - GetRunner(string) (map[string]string, error) - CreateRunner(models.Runner) (*v1.ConfigMap, error) - UpdateRunner(models.Runner) (*v1.ConfigMap, error) + GetRunner(string) (*models.Runner, error) + CreateRunner(models.Runner) (*models.Runner, error) + UpdateRunner(models.Runner) (*models.Runner, error) DeleteRunner(string) error GetAdminGroups(string) (string, error) ListAllJobs() ([]models.Job, error) + GetSecret(string) (map[string]string, error) + UpdateSecret(string, map[string]string) (map[string]string, error) + DeleteSecret(string) error } type RunnerServiceImpl struct { @@ -46,7 +48,7 @@ func (r *RunnerServiceImpl) ListRunners(authList []string) ([]map[string]string, } for _, configMap := range configMaps.Items { - // TODO: we don't currently have a way of definying what is a Runner configmap so I'm checking if it has an Image field + // TODO: we don't currently have a way to identify what is a Runner configmap so I'm checking if it has an Image field // This will be changed when runners will live in a separate namespace if configMap.Data["image"] != "" { if authList[0] != "*" { @@ -62,55 +64,85 @@ func (r *RunnerServiceImpl) ListRunners(authList []string) ([]map[string]string, return runnersList, nil } -func (r *RunnerServiceImpl) GetRunner(name string) (map[string]string, error) { +func (r *RunnerServiceImpl) GetRunner(name string) (*models.Runner, error) { + var runnerData models.Runner configMap, err := helpers.GetConfigMap(r.config.Kube, name) if err != nil { - return nil, err + return &runnerData, err } if configMap.Data["image"] == "" { return nil, fmt.Errorf("runner %s not found", name) } - secret, err := helpers.GetSecret(r.config.Kube, name) + b, _ := json.Marshal(configMap.Data) + _ = json.Unmarshal(b, &runnerData) + + tokenObjName := name + "-token" + token, err := r.GetSecret(tokenObjName) if err != nil { - if errors.IsNotFound(err) { - return configMap.Data, nil + if !errors.IsNotFound(err) { + return &runnerData, err } - return nil, err + } else { + runnerData.Token = token["token"] + } + + secretCleared, err := r.GetSecret(name) + if err != nil { + if !errors.IsNotFound(err) { + return &runnerData, err + } + } else { + runnerData.Secret = secretCleared } - configMap.Data["token"] = string(secret.Data["token"]) - return configMap.Data, nil + return &runnerData, nil } -func (r *RunnerServiceImpl) CreateRunner(runner models.Runner) (*v1.ConfigMap, error) { +func (r *RunnerServiceImpl) CreateRunner(runner models.Runner) (*models.Runner, error) { b, _ := json.Marshal(runner) var data map[string]string _ = json.Unmarshal(b, &data) delete(data, "token") + delete(data, "secret") if data["branch"] == "" { data["branch"] = "main" } - configMap, err := helpers.CreateOrUpdateConfigMap(r.config.Kube, data, "create") + _, err := helpers.CreateOrUpdateConfigMap(r.config.Kube, data, "create") + if err != nil { + return nil, err + } + // runner contains two types of secrets: git repo token and custom secrets, to be stored + // in separate k8s secrets. token will be stored under runner name, secrets as runner name + secrets. if runner.Token != "" { - secret := make(map[string]string) - secret["token"] = runner.Token - _, err = helpers.CreateOrUpdateSecret(r.config.Kube, runner.Name, secret, "create") + tokenObjName := runner.Name + "-token" + token := make(map[string]string) + token["token"] = runner.Token + _, err = helpers.CreateOrUpdateSecret(r.config.Kube, tokenObjName, token, "create") + + if err != nil { + return nil, err + } + } + + if runner.Secret != nil { + _, err = r.UpdateSecret(runner.Name, runner.Secret) if err != nil { return nil, err } } - return configMap, err + runnerData, err := r.GetRunner(runner.Name) + return runnerData, err } -func (r *RunnerServiceImpl) UpdateRunner(runner models.Runner) (*v1.ConfigMap, error) { +func (r *RunnerServiceImpl) UpdateRunner(runner models.Runner) (*models.Runner, error) { _, err := helpers.GetConfigMap(r.config.Kube, runner.Name) if err != nil { return nil, err @@ -120,19 +152,21 @@ func (r *RunnerServiceImpl) UpdateRunner(runner models.Runner) (*v1.ConfigMap, e var data map[string]string _ = json.Unmarshal(b, &data) delete(data, "token") + delete(data, "secret") - configMap, err := helpers.CreateOrUpdateConfigMap(r.config.Kube, data, "update") + _, err = helpers.CreateOrUpdateConfigMap(r.config.Kube, data, "update") if err != nil { return nil, err } - if runner.Token != "" { - secret := make(map[string]string) - secret["token"] = runner.Token + tokenObjName := runner.Name + "-token" + if runner.Token != "" && runner.Token != "************" { + token := make(map[string]string) + token["token"] = runner.Token operation := "update" // default operation is 'update', try to get the Secret first: if it's not found we need to create it // e.g. Someone created a Task without a secret and is adding one with update - _, err = helpers.GetSecret(r.config.Kube, runner.Name) + _, err = r.GetSecret(tokenObjName) if err != nil { if errors.IsNotFound(err) { operation = "create" @@ -140,18 +174,32 @@ func (r *RunnerServiceImpl) UpdateRunner(runner models.Runner) (*v1.ConfigMap, e return nil, err } } - _, err := helpers.CreateOrUpdateSecret(r.config.Kube, runner.Name, secret, operation) + _, err := helpers.CreateOrUpdateSecret(r.config.Kube, tokenObjName, token, operation) if err != nil { return nil, err } - } else { - err = helpers.DeleteSecret(r.config.Kube, runner.Name) + } else if runner.Token == "" { + err = helpers.DeleteSecret(r.config.Kube, tokenObjName) if err != nil && !errors.IsNotFound(err) { return nil, err } } - return configMap, err + if runner.Secret != nil { + _, err = r.UpdateSecret(runner.Name, runner.Secret) + + if err != nil { + return nil, err + } + + } + + updatedRunner, err := r.GetRunner(runner.Name) + if err != nil { + return nil, err + } + return updatedRunner, err + } func (r *RunnerServiceImpl) DeleteRunner(name string) error { @@ -215,3 +263,81 @@ func (r *RunnerServiceImpl) ListAllJobs() ([]models.Job, error) { return jobsRet, nil } + +func (r *RunnerServiceImpl) GetSecret(name string) (map[string]string, error) { + secretCleaned := make(map[string]string) + secret, err := helpers.GetSecret(r.config.Kube, name) + + if err != nil { + return nil, err + } + + for key := range secret.Data { + secretCleaned[key] = "************" + + } + return secretCleaned, nil +} + +func (r *RunnerServiceImpl) UpdateSecret(name string, secret map[string]string) (map[string]string, error) { + secretCleaned := make(map[string]string) + secretCurrent := make(map[string]string) + var operation string + + secretObj, err := helpers.GetSecret(r.config.Kube, name) + if err != nil && !errors.IsNotFound(err) { + return nil, err + } + // converting k8s secret from v1.Secret into map[string]string + + if secretObj != nil { + for k, v := range secretObj.Data { + secretCurrent[k] = string(v) + } + + operation = "update" + } else { + operation = "create" + } + + for k, v := range secret { + v2, ok := secretCurrent[k] + + if v != "" && v != v2 { + if v != "************" { + secretCurrent[k] = v + } + } else if v == "" && ok { + delete(secretCurrent, k) + } + } + + if len(secretCurrent) != 0 { + secretNew, err := helpers.CreateOrUpdateSecret(r.config.Kube, name, secretCurrent, operation) + if err != nil { + return secretCleaned, err + } + + for key := range secretNew.Data { + secretCleaned[key] = "************" + + } + return secretCleaned, nil + + } else { + err := helpers.DeleteSecret(r.config.Kube, name) + if err != nil { + return secretCleaned, err + } + return secretCleaned, nil + } +} + +func (r *RunnerServiceImpl) DeleteSecret(name string) error { + err := helpers.DeleteSecret(r.config.Kube, name) + if err != nil && !errors.IsNotFound(err) { + return err + } + + return nil +} diff --git a/services/task_svc.go b/services/task_svc.go index d1d3068..9a39800 100644 --- a/services/task_svc.go +++ b/services/task_svc.go @@ -15,22 +15,17 @@ import ( "github.com/go-openapi/strfmt" "github.com/go-openapi/validate" "golang.org/x/exp/slices" - - "k8s.io/apimachinery/pkg/api/errors" ) type TaskService interface { ListTasks([]string) ([]map[string]string, error) - GetTask(string) (map[string]interface{}, map[string]string, error) - CreateTask(models.Task) (map[string]interface{}, map[string]string, error) - UpdateTask(models.Task) (map[string]interface{}, map[string]string, error) + GetTask(string) (*models.Task, error) + CreateTask(models.Task) (*models.Task, error) + UpdateTask(models.Task) (*models.Task, error) DeleteTask(string) error GetSchema(string) (map[string]interface{}, error) DeleteSchema(string) error UpdateSchema(string, map[string]interface{}) (map[string]interface{}, error) - GetSecret(string) (map[string]string, error) - UpdateSecret(string, map[string]string) (map[string]string, error) - DeleteSecret(string) error } type TaskServiceImpl struct { @@ -69,60 +64,51 @@ func (t *TaskServiceImpl) ListTasks(authList []string) ([]map[string]string, err return tasksList, nil } -func (t *TaskServiceImpl) GetTask(name string) (map[string]interface{}, map[string]string, error) { +func (t *TaskServiceImpl) GetTask(name string) (*models.Task, error) { + var taskData models.Task configMap, err := helpers.GetConfigMap(t.config.Kube, name) if err != nil { - return nil, nil, err + return nil, err } if configMap.Data["runner"] == "" { - return nil, nil, fmt.Errorf("task %s not found", name) + return nil, fmt.Errorf("task %s not found", name) } // TODO: this is a temporary solution to return synchronous as a boolean b, _ := json.Marshal(configMap.Data) - var data map[string]interface{} - _ = json.Unmarshal(b, &data) - data["synchronous"], _ = strconv.ParseBool(configMap.Data["synchronous"]) + + _ = json.Unmarshal(b, &taskData) + taskData.Synchronous, _ = strconv.ParseBool(configMap.Data["synchronous"]) if configMap.Data["schema"] != "" { var jsonData map[string]interface{} err = json.Unmarshal([]byte(configMap.Data["schema"]), &jsonData) if err != nil { - return nil, nil, err - } - data["schema"] = jsonData - } - - secretCleared, err := t.GetSecret(name) - if err != nil { - if errors.IsNotFound(err) { - return data, nil, nil + return nil, err } - return data, nil, err + taskData.Schema = jsonData } - return data, secretCleared, nil + return &taskData, nil } -func (t *TaskServiceImpl) CreateTask(task models.Task) (map[string]interface{}, map[string]string, error) { +func (t *TaskServiceImpl) CreateTask(task models.Task) (*models.Task, error) { var jsonData []byte - var configuredTask map[string]interface{} - var secretCleared map[string]string runner, err := helpers.GetConfigMap(t.config.Kube, task.Runner) if err != nil || runner.Data["image"] == "" { - return nil, nil, fmt.Errorf("error retrieving runner %s, please specify an existing runner", task.Runner) + return nil, fmt.Errorf("error retrieving runner %s, please specify an existing runner", task.Runner) } if task.Schema != nil { jsonData, err = json.Marshal(task.Schema) if err != nil { - return nil, nil, err + return nil, err } err = ValidateSchema(jsonData) if err != nil { - return nil, nil, err + return nil, err } } @@ -136,48 +122,38 @@ func (t *TaskServiceImpl) CreateTask(task models.Task) (map[string]interface{}, _, err = helpers.CreateOrUpdateConfigMap(t.config.Kube, data, "create") if err != nil { - return nil, nil, err - } - - if task.Secret != nil { - _, err = t.UpdateSecret(task.Name, task.Secret) - - if err != nil { - return nil, nil, err - } + return nil, err } - configuredTask, secretCleared, err = t.GetTask(task.Name) + configuredTask, err := t.GetTask(task.Name) if err != nil { - return nil, nil, err + return nil, err } - return configuredTask, secretCleared, err + return configuredTask, err } -func (t *TaskServiceImpl) UpdateTask(task models.Task) (map[string]interface{}, map[string]string, error) { +func (t *TaskServiceImpl) UpdateTask(task models.Task) (*models.Task, error) { var jsonData []byte - var configuredTask map[string]interface{} - var secretCleared map[string]string _, err := helpers.GetConfigMap(t.config.Kube, task.Name) if err != nil { - return nil, nil, err + return nil, err } runner, err := helpers.GetConfigMap(t.config.Kube, task.Runner) if err != nil || runner.Data["image"] == "" { - return nil, nil, fmt.Errorf("error retrieving runner %s, please specify an existing runner", task.Runner) + return nil, fmt.Errorf("error retrieving runner %s, please specify an existing runner", task.Runner) } if task.Schema != nil { jsonData, err = json.Marshal(task.Schema) if err != nil { - return nil, nil, err + return nil, err } err = ValidateSchema(jsonData) if err != nil { - return nil, nil, err + return nil, err } } @@ -187,34 +163,17 @@ func (t *TaskServiceImpl) UpdateTask(task models.Task) (map[string]interface{}, _ = json.Unmarshal(b, &data) data["synchronous"] = strconv.FormatBool(task.Synchronous) data["schema"] = string(jsonData) - delete(data, "secret") _, err = helpers.CreateOrUpdateConfigMap(t.config.Kube, data, "update") if err != nil { - return nil, nil, err - } - - //var secret *v1.Secret - if task.Secret != nil { - - _, err = t.UpdateSecret(task.Name, task.Secret) - - if err != nil { - return nil, nil, err - } - - } else { - err = helpers.DeleteSecret(t.config.Kube, task.Name) - if err != nil && !errors.IsNotFound(err) { - return nil, nil, err - } + return nil, err } - configuredTask, secretCleared, err = t.GetTask(task.Name) + configuredTask, err := t.GetTask(task.Name) if err != nil { - return nil, nil, err + return nil, err } - return configuredTask, secretCleared, err + return configuredTask, err } @@ -224,11 +183,6 @@ func (t *TaskServiceImpl) DeleteTask(name string) error { return err } - err = helpers.DeleteSecret(t.config.Kube, name) - if err != nil && !errors.IsNotFound(err) { - return err - } - return nil } @@ -283,14 +237,12 @@ func (t *TaskServiceImpl) UpdateSchema(taskName string, schema map[string]interf } func (t *TaskServiceImpl) DeleteSchema(name string) error { - task, _, err := t.GetTask(name) + task, err := t.GetTask(name) if err != nil { return err } - log.Println(task) - log.Println(task["schema"]) - if task["schema"] == "" { + if task.Schema == nil { return nil } @@ -331,88 +283,3 @@ func ValidateSchema(schema []byte) error { return nil } - -func (t *TaskServiceImpl) GetSecret(name string) (map[string]string, error) { - secretCleaned := make(map[string]string) - secret, err := helpers.GetSecret(t.config.Kube, name) - if err != nil && !errors.IsNotFound(err) { - return nil, err - } else if errors.IsNotFound(err) { - return nil, nil - } - - for key := range secret.Data { - secretCleaned[key] = "************" - - } - - return secretCleaned, nil -} - -func (t *TaskServiceImpl) UpdateSecret(name string, secret map[string]string) (map[string]string, error) { - secretCleaned := make(map[string]string) - secretCurrent := make(map[string]string) - var operation string - - secretObj, err := helpers.GetSecret(t.config.Kube, name) - if err != nil && !errors.IsNotFound(err) { - return nil, err - } - // converting k8s secret from v1.Secret into map[string]string - - if secretObj != nil { - - for k, v := range secretObj.Data { - secretCurrent[k] = string(v) - } - - operation = "update" - - } else { - operation = "create" - } - - for k, v := range secret { - - v2, ok := secretCurrent[k] - - if v != "" || v != v2 { - if v != "************" { - secretCurrent[k] = v - } - - } else if v == "" && ok { - delete(secretCurrent, k) - } - } - - if len(secretCurrent) != 0 { - secretNew, err := helpers.CreateOrUpdateSecret(t.config.Kube, name, secretCurrent, operation) - if err != nil { - return secretCleaned, err - } - - for key := range secretNew.Data { - secretCleaned[key] = "*************" - - } - return secretCleaned, nil - - } else { - err := helpers.DeleteSecret(t.config.Kube, name) - if err != nil { - return secretCleaned, err - } - return secretCleaned, nil - } -} - -func (t *TaskServiceImpl) DeleteSecret(name string) error { - - err := helpers.DeleteSecret(t.config.Kube, name) - if err != nil && !errors.IsNotFound(err) { - return err - } - - return nil -} diff --git a/services/tokens_svc.go b/services/tokens_svc.go index b3ebb69..f21b172 100644 --- a/services/tokens_svc.go +++ b/services/tokens_svc.go @@ -39,7 +39,9 @@ func NewApiTokenService(database *gorm.DB, config config.Config) ApiTokenService func (u *ApiTokenServiceImpl) ListApiTokens(userid uuid.UUID) ([]models.ApiToken, error) { var apiTokens []models.ApiToken - res := u.db.Where("owner = ?", userid).Find(&apiTokens) + res := u.db.Select("id", "owner", "expires", "created_at", "updated_at", "description", "enabled"). + Where("owner = ?", userid). + Find(&apiTokens) if res.Error != nil { return apiTokens, res.Error @@ -70,7 +72,11 @@ func (u *ApiTokenServiceImpl) ListAllApiTokens(authList []string) ([]models.ApiT func (u *ApiTokenServiceImpl) GetApiToken(id string) (models.ApiToken, error) { var apiToken models.ApiToken - res := u.db.Where("id = ?", id).Find(&apiToken) + + res := u.db.Select("id", "owner", "expires", "created_at", "updated_at", "description", "enabled"). + Where("id = ?", id). + Find(&apiToken) + if res.Error != nil { return models.ApiToken{}, res.Error } @@ -87,15 +93,17 @@ func (u *ApiTokenServiceImpl) CreateApiToken(apiToken models.ApiToken) (models.A if err != nil { return models.ApiToken{}, err } - + var tokenEnabled = true apiToken.Key = helpers.GenerateHMAC(u.config.APISecret, key) // if No value is passed, initialise to Zero value if apiToken.Expires == nil { apiToken.Expires = new(time.Time) } + + // nil pointer dereference via tokenEnabled bool var if apiToken.Enabled == nil { - *apiToken.Enabled = true + apiToken.Enabled = &tokenEnabled } res := u.db.Create(&apiToken)