Golang : How to split or chunking a file to smaller pieces?

Problem :

You have a big file and you want to split / chunk the file into smaller pieces. How to do that in Golang?

Solution :

1. Calculate the size of each chunk that you want to split the file into.
2. Read the file content into a buffer(created according to the calculated chunk size).
3. Write the buffer content into individual pieces.

Here you go :

 package main

 import (
 "fmt"
 "io/ioutil"
 "math"
 "os"
 "strconv"
 )

 func main() {

 fileToBeChunked := "./somebigfile"

 file, err := os.Open(fileToBeChunked)

 if err != nil {
 fmt.Println(err)
 os.Exit(1)
 }

 defer file.Close()

 fileInfo, _ := file.Stat()

 var fileSize int64 = fileInfo.Size()

 const fileChunk = 1 * (1 << 20) // 1 MB, change this to your requirement

 // calculate total number of parts the file will be chunked into

 totalPartsNum := uint64(math.Ceil(float64(fileSize) / float64(fileChunk)))

 fmt.Printf("Splitting to %d pieces.\n", totalPartsNum)

 for i := uint64(0); i < totalPartsNum; i++ {

 partSize := int(math.Min(fileChunk, float64(fileSize-int64(i*fileChunk))))
 partBuffer := make([]byte, partSize)

 file.Read(partBuffer)

 // write to disk
 fileName := "somebigfile_" + strconv.FormatUint(i, 10)
 _, err := os.Create(fileName)

 if err != nil {
 fmt.Println(err)
 os.Exit(1)
 }

 // write/save buffer to disk
 ioutil.WriteFile(fileName, partBuffer, os.ModeAppend)

 fmt.Println("Split to : ", fileName)
 }
 }

Sample output :

Splitting to 5 pieces.

Split to : somebigfile_1

Split to : somebigfile_2

Split to : somebigfile_3

Split to : somebigfile_4

References :

https://www.socketloop.com/tutorials/golang-convert-cast-int64-to-string

https://www.socketloop.com/tutorials/golang-upload-big-file-larger-than-100mb-to-aws-s3-with-multipart-upload