Resuming downloads

This page provides an overview of how download resumption is implemented

Resuming downloads is a useful feature that saves bandwidth and time by allowing the retrieval process to continue from where it left off, in case of interruptions or disconnections.

It calculates where to restart by determining the startBlock when processing download requests. This computation occurs in the calculateShardParams method, which interacts with the fileHandler to establish the file size. The derived size is then employed to call the Seek() method of the DownloadRequest. This method adjusts the offset field of the DownloadRequest, which is subsequently used to calculate the startBlock.

For local files, the system opens them in append mode if the file already exists and has a size that's greater than 0.

This resumption feature is illustrated in the code sections below. These functions together facilitate the operation of the download resumption process:

• calculateShardsParams(): Executes the actual resumption logic by calculating shard parameters. If there's an offset (indicating an existing incomplete file), it adjusts the startBlock accordingly.

func (req *DownloadRequest) calculateShardsParams(
	fRef *fileref.FileRef, remotePathCB string) (chunksPerShard int64, err error) {

	size := fRef.ActualFileSize
	if req.contentMode == DOWNLOAD_CONTENT_THUMB {
		if fRef.ActualThumbnailSize == 0 {
			return 0, errors.New("invalid_request", "Thumbnail does not exist")
		}
		size = fRef.ActualThumbnailSize
	}
	req.size = size
	req.encryptedKey = fRef.EncryptedKey
	req.chunkSize = int(fRef.ChunkSize)

	effectivePerShardSize := (size + int64(req.datashards) - 1) / int64(req.datashards)
	effectiveBlockSize := fRef.ChunkSize
	if fRef.EncryptedKey != "" {
		effectiveBlockSize -= EncryptionHeaderSize + EncryptedDataPaddingSize
	}

	req.effectiveBlockSize = int(effectiveBlockSize)

	chunksPerShard = (effectivePerShardSize + effectiveBlockSize - 1) / effectiveBlockSize

	info, err := req.fileHandler.Stat()
	if err != nil {
		return 0, err
	}
	_, err = req.Seek(info.Size(), io.SeekStart)
	if err != nil {
		return 0, err
	}

	effectiveChunkSize := effectiveBlockSize * int64(req.datashards)
	if req.offset > 0 {
		req.startBlock += req.offset / effectiveChunkSize
	}

	if req.endBlock == 0 || req.endBlock > chunksPerShard {
		req.endBlock = chunksPerShard
	}

	if req.startBlock >= req.endBlock {
		err = errors.New("invalid_block_num", "start block should be less than end block")
		return 0, err
	}

	return
}

• prepareAndOpenLocalFile(): Handles the local file, checking if it exists, and opening in append mode or creating it as needed.

func (a *Allocation) prepareAndOpenLocalFile(localPath string, remotePath string) (*os.File, string, bool, error) {
	var toKeep bool

	if !a.isInitialized() {
		return nil, "", toKeep, notInitialized
	}

	var localFilePath string

	// If the localPath has a file extension, treat it as a file. Otherwise, treat it as a directory.
	if filepath.Ext(localPath) != "" {
		localFilePath = localPath
	} else {
		localFileName := filepath.Base(remotePath)
		localFilePath = filepath.Join(localPath, localFileName)
	}

	// Create necessary directories if they do not exist
	dir := filepath.Dir(localFilePath)
	if _, err := os.Stat(dir); os.IsNotExist(err) {
		if err := os.MkdirAll(dir, 0744); err != nil {
			return nil, "", toKeep, err
		}
	}

	var f *os.File
	info, err := os.Stat(localFilePath)
	if errors.Is(err, os.ErrNotExist) {
		f, err = os.OpenFile(localFilePath, os.O_WRONLY|os.O_CREATE, 0644)
		if err != nil {
			return nil, "", toKeep, errors.Wrap(err, "Can't create local file")
		}
	} else {
		f, err = os.OpenFile(localFilePath, os.O_WRONLY|os.O_APPEND, 0644)
		if err != nil {
			return nil, "", toKeep, errors.Wrap(err, "Can't open local file in append mode")
		}
		if info.Size() > 0 {
			toKeep = true
		}
	}

	return f, localFilePath, toKeep, nil
}