Download empty files (Fixes Azure#102)

zachomedia · Feb 26, 2019 · b56f637 · b56f637
1 parent 0263ee6
commit b56f637
Show file tree

Hide file tree

Showing 2 changed files with 42 additions and 35 deletions.
diff --git a/pipeline/pipeline.go b/pipeline/pipeline.go
@@ -229,6 +229,11 @@ func ConstructPartsQueue(size uint64, blockSize uint64, sourceURI string, target
 	var bsib = blockSize
 	numOfBlocks = int((size + (bsib - 1)) / bsib)
 
+	// Have at least 1 block.
+	if numOfBlocks == 0 {
+		numOfBlocks = 1
+	}
+
 	parts = make([]Part, numOfBlocks)
 
 	var curFileOffset uint64

diff --git a/sources/http.go b/sources/http.go
@@ -170,52 +170,54 @@ func (f *HTTPSource) ExecuteReader(partitionsQ chan pipeline.PartsPartition, par
 			continue
 		}
 
-		util.RetriableOperation(func(r int) error {
-			if req, err = http.NewRequest("GET", p.SourceURI, nil); err != nil {
-				log.Fatal(err)
-			}
+		if p.BytesToRead > 0 {
+			util.RetriableOperation(func(r int) error {
+				if req, err = http.NewRequest("GET", p.SourceURI, nil); err != nil {
+					log.Fatal(err)
+				}
 
-			header := fmt.Sprintf("bytes=%v-%v", p.Offset, p.Offset-1+uint64(p.BytesToRead))
-			req.Header.Set("Range", header)
-			req.Header.Set("User-Agent", internal.UserAgentStr)
+				header := fmt.Sprintf("bytes=%v-%v", p.Offset, p.Offset-1+uint64(p.BytesToRead))
+				req.Header.Set("Range", header)
+				req.Header.Set("User-Agent", internal.UserAgentStr)
 
-			if res, err = f.HTTPClient.Do(req); err != nil || res.StatusCode != 206 {
-				var status int
-				if res != nil {
-					status = res.StatusCode
-					err = fmt.Errorf("Invalid status code in the response. Status: %v Bytes: %v", status, header)
-				}
+				if res, err = f.HTTPClient.Do(req); err != nil || res.StatusCode != 206 {
+					var status int
+					if res != nil {
+						status = res.StatusCode
+						err = fmt.Errorf("Invalid status code in the response. Status: %v Bytes: %v", status, header)
+					}
 
-				if res != nil && res.Body != nil {
-					res.Body.Close()
-				}
+					if res != nil && res.Body != nil {
+						res.Body.Close()
+					}
 
-				util.PrintfIfDebug("ExecuteReader -> blockid:%v toread:%v status:%v err:%v head:%v", p.BlockID, p.BytesToRead, status, err, header)
+					util.PrintfIfDebug("ExecuteReader -> blockid:%v toread:%v status:%v err:%v head:%v", p.BlockID, p.BytesToRead, status, err, header)
 
-				return err
-			}
+					return err
+				}
 
-			//p.Data, err = ioutil.ReadAll(res.Body)
-			p.GetBuffer()
-			_, err := io.ReadAtLeast(res.Body, p.Data, int(p.BytesToRead))
+				//p.Data, err = ioutil.ReadAll(res.Body)
+				p.GetBuffer()
+				_, err := io.ReadAtLeast(res.Body, p.Data, int(p.BytesToRead))
 
-			if err != nil && err != io.ErrUnexpectedEOF {
-				return err
-			}
+				if err != nil && err != io.ErrUnexpectedEOF {
+					return err
+				}
 
-			res.Body.Close()
-			if err != nil {
-				return err
-			}
+				res.Body.Close()
+				if err != nil {
+					return err
+				}
 
-			if f.includeMD5 {
-				p.MD5()
-			}
+				if f.includeMD5 {
+					p.MD5()
+				}
 
-			util.PrintfIfDebug("ExecuteReader -> blockid:%v toread:%v status:%v read:%v head:%v", p.BlockID, p.BytesToRead, res.StatusCode, res.ContentLength, header)
+				util.PrintfIfDebug("ExecuteReader -> blockid:%v toread:%v status:%v read:%v head:%v", p.BlockID, p.BytesToRead, res.StatusCode, res.ContentLength, header)
 
-			return nil
-		})
+				return nil
+			})
+		}
 
 		readPartsQ <- p
 		blocksHandled++