add upload chunk file

go.mod

@@ -8,6 +8,7 @@ require (
 	github.com/go-playground/validator/v10 v10.11.0
 	github.com/goccy/go-json v0.9.10 // indirect
 	github.com/google/go-cmp v0.5.6 // indirect
+	github.com/google/uuid v1.3.0
 	github.com/mattn/go-colorable v0.1.12 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/pelletier/go-toml/v2 v2.0.2 // indirect

go.sum

@@ -27,6 +27,8 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
 github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
+github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=

realtime-advanced/go.mod

@@ -22,9 +22,9 @@ require (
 	github.com/pelletier/go-toml/v2 v2.0.1 // indirect
 	github.com/ugorji/go/codec v1.2.7 // indirect
 	golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 // indirect
-	golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 // indirect
-	golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f // indirect
-	golang.org/x/text v0.3.8 // indirect
+	golang.org/x/net v0.7.0 // indirect
+	golang.org/x/sys v0.5.0 // indirect
+	golang.org/x/text v0.7.0 // indirect
 	google.golang.org/protobuf v1.28.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 )

realtime-advanced/go.sum

@@ -61,19 +61,20 @@ github.com/ugorji/go/codec v1.2.7 h1:YPXUKf7fYbp/y8xloBqZOw2qaVggbfwMlI8WM3wZUJ0
 github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY=
 golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 h1:/UOmuWzQfxxo9UtlXMwuQU8CMgg1eZXqTRwkSQJWKOI=
 golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 h1:qWPm9rbaAMKs8Bq/9LRpbMqxWRVUAQwMI9fVrssnTfw=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g=
+golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f h1:v4INt8xihDGvnrfjMDVXGxw9wrfxYyCjk0KbXjhR55s=
-golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.8 h1:nAL+RVCQ9uMn3vJZbV+MRnydTJFPf8qqY42YiA6MrqY=
-golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
+golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo=
+golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

upload-file/chunk/client/client.go

@@ -0,0 +1,75 @@
+package main
+
+import (
+	"bytes"
+	"crypto/md5"
+	"encoding/json"
+	"fmt"
+	"io"
+	"log"
+	"math"
+	"net/http"
+	"os"
+	"path/filepath"
+
+	"github.com/gin-gonic/examples/upload-file/chunk/model"
+	"github.com/google/uuid"
+)
+
+func main() {
+	filePath := "your slice chunk upload file path"
+	fileName := filepath.Base(filePath)
+
+	fileInfo, err := os.Stat(filePath)
+	if err != nil {
+		log.Fatalf("file stat fail: %v\n", err)
+		return
+	}
+
+	const chunkSize = 1 << (10 * 2) * 30
+
+	num := math.Ceil(float64(fileInfo.Size()) / float64(chunkSize))
+
+	fi, err := os.OpenFile(filePath, os.O_RDONLY, os.ModePerm)
+	if err != nil {
+		log.Fatalf("open file fail: %v\n", err)
+		return
+	}
+
+	fileId := uuid.NewString()
+
+	fileKeys := make([]string, 0)
+	for i := 1; i <= int(num); i++ {
+		file := make([]byte, chunkSize)
+		fi.Seek((int64(i)-1)*chunkSize, 0)
+		if len(file) > int(fileInfo.Size()-(int64(i)-1)*chunkSize) {
+			file = make([]byte, fileInfo.Size()-(int64(i)-1)*chunkSize)
+		}
+		fi.Read(file)
+
+		key := fmt.Sprintf("%x", md5.Sum(file))
+
+		fileKeys = append(fileKeys, key)
+
+		req := model.ChunkFileRequest{
+			FileId:    fileId,
+			FileName:  fileName,
+			FileIndex: i,
+			FileCount: int(num),
+			FileKey:   key,
+			FileKeys:  fileKeys,
+			File:      file,
+		}
+		body, _ := json.Marshal(req)
+
+		res, err := http.Post("http://127.0.0.1:8080/chunkUploadFile", "application/json", bytes.NewBuffer(body))
+
+		if err != nil {
+			log.Fatalf("http post fail: %v", err)
+			return
+		}
+		defer res.Body.Close()
+		msg, _ := io.ReadAll(res.Body)
+		fmt.Println(string(msg))
+	}
+}

upload-file/chunk/model/chunk_model.go

@@ -0,0 +1,127 @@
+package model
+
+import (
+	"crypto/md5"
+	"errors"
+	"fmt"
+	"net/http"
+	"os"
+	"path/filepath"
+
+	"github.com/gin-gonic/gin"
+)
+
+type ChunkFileRequest struct {
+	FileId    string   `json:"fileId"`    // client create uuid
+	FileName  string   `json:"fileName"`  // file name
+	FileIndex int      `json:"fileIndex"` // file index
+	FileCount int      `json:"fileCount"` // file slice size
+	FileKeys  []string `json:"fileKeys"`  // file slice all key md5 (accumulation file key if fileIndex == fileCount then merge after verification)
+	FileKey   string   `json:"fileKey"`   // file now key to md5 - if server read the slice to md5 eq key not eq then fail
+	File      []byte   `json:"file"`      // now file
+}
+
+func (cf *ChunkFileRequest) BindingForm(ctx *gin.Context) error {
+	if err := ctx.ShouldBind(cf); err != nil {
+		return err
+	}
+
+	return cf.md5()
+}
+
+func (cf *ChunkFileRequest) md5() error {
+	hash := fmt.Sprintf("%x", md5.Sum(cf.File))
+	if hash != cf.FileKey {
+		return errors.New("current file slice key error")
+	}
+	return nil
+}
+
+func (cf *ChunkFileRequest) SaveUploadedFile(tempPath, path string) (string, error) {
+	tempFolder := filepath.Join(tempPath, cf.FileId)
+
+	_, err := os.Stat(tempFolder)
+	if os.IsNotExist(err) {
+		err := os.MkdirAll(tempFolder, os.ModePerm)
+		if err != nil {
+			return "", err
+		}
+	}
+
+	out, err := os.Create(filepath.Join(tempFolder, cf.FileKey))
+	if err != nil {
+		return "", err
+	}
+	defer out.Close()
+	if _, err := out.Write(cf.File); err != nil {
+		return "", err
+	}
+
+	fmt.Println(cf.FileIndex, cf.FileCount)
+	if cf.FileIndex != cf.FileCount {
+		return "", nil
+	}
+	for _, fileKey := range cf.FileKeys {
+		tempFile := filepath.Join(tempFolder, fileKey)
+		if _, err := os.Stat(tempFile); err != nil {
+			return "", errors.New("file " + fileKey + " is emtpy")
+		}
+	}
+
+	base := filepath.Dir(path)
+	if _, err := os.Stat(base); err != nil {
+		if os.IsNotExist(err) {
+			err := os.MkdirAll(base, os.ModePerm)
+			if err != nil {
+				return "", err
+			}
+		}
+	}
+
+	file, err := os.OpenFile(path, os.O_RDWR|os.O_APPEND|os.O_CREATE, 0664)
+	if err != nil {
+		return "", err
+	}
+
+	defer file.Close()
+
+	for _, fileKey := range cf.FileKeys {
+		tempFile := filepath.Join(tempFolder, fileKey)
+		bt, err := os.ReadFile(tempFile)
+		if err != nil {
+			return "", err
+		}
+		file.Write(bt)
+	}
+
+	return tempFolder, nil
+}
+
+// request method is json
+// param: fileId
+// param: fileName
+// param: fileIndex the file slice index
+// param: fileCount the file slice size
+// param: fileKeys the file slice all file key md5 (accumulation file key if fileIndex == fileCount then merge after verification)
+// param: fileKey  now file slice key md5
+// param: file     now slice file
+func ChunkUploadFile(ctx *gin.Context) {
+	var cf ChunkFileRequest
+
+	if err := cf.BindingForm(ctx); err != nil {
+		ctx.JSON(http.StatusBadRequest, gin.H{"code": "400", "msg": "bad file param", "err": err.Error()})
+		return
+	}
+
+	tempFolder, err := cf.SaveUploadedFile("./temp", "./uploads/"+cf.FileName)
+	if err != nil {
+		ctx.JSON(http.StatusServiceUnavailable, gin.H{"code": "503", "msg": "bad save upload file", "err": err.Error()})
+		return
+	}
+	ctx.JSON(http.StatusOK, gin.H{"code": "200", "msg": "success"})
+	if tempFolder != "" {
+		defer func(tempFolder string) {
+			os.RemoveAll(tempFolder)
+		}(tempFolder)
+	}
+}

upload-file/chunk/server/server.go

@@ -0,0 +1,12 @@
+package main
+
+import (
+	"github.com/gin-gonic/examples/upload-file/chunk/model"
+	"github.com/gin-gonic/gin"
+)
+
+func main() {
+	g := gin.Default()
+	g.POST("/chunkUploadFile", model.ChunkUploadFile)
+	g.Run(":8080")
+}