Skip to content

Download

src.utils.download.get_temporary_file(path: Path, ext: str = '.drive') -> tuple[Path, int]

Check for an existing temporary file representing a provided file path and ext.

Parameters:

Name Type Description Default
path Path

Path to the file we plan to download.

required
ext str

Temporary file extension to use, e.g. '.drive' or '.amazon'

'.drive'

Returns:

Type Description
tuple[Path, int]

A tuple containing a path to the temporary file and the current size of that file.

Source code in src\utils\download.py
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
def get_temporary_file(path: Path, ext: str = '.drive') -> tuple[Path, int]:
    """Check for an existing temporary file representing a provided file path and ext.

    Args:
        path: Path to the file we plan to download.
        ext: Temporary file extension to use, e.g. '.drive' or '.amazon'

    Returns:
        A tuple containing a path to the temporary file and the current size of that file.
    """

    # Look for an existing temporary file
    temp = path.with_suffix(ext)
    for p in os.listdir(path.parent):
        file = path.parent / p
        if file.is_dir():
            continue
        if temp.name in file.name:
            return file, file.stat().st_size

    # Create a new temporary file
    f = NamedTemporaryFile(prefix=temp.name, dir=temp.parent, delete=False)
    file = f.name
    f.close()
    return Path(file), 0

src.utils.download.download_file(file: Path, res: requests.Response, path: Path = None, callback: Optional[Callable] = None, chunk_size: int = 1024 * 1024) -> bool

Download file from streamed response as a temporary file, then rename to its final path.

Parameters:

Name Type Description Default
file Path

Path of the temporary file.

required
res Response

Download request.

required
path Path

Final path to save the completed temporary file.

None
callback Callable | None

Callback to update download progress.

None
chunk_size int

Amount of bytes to download before calling callback.

1024 * 1024

Returns:

Type Description
bool

True if download completed successfully, otherwise False.

Source code in src\utils\download.py
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
def download_file(
    file: Path,
    res: requests.Response,
    path: Path = None,
    callback: Optional[Callable] = None,
    chunk_size: int = 1024 * 1024
) -> bool:
    """Download file from streamed response as a temporary file, then rename to its final path.

    Args:
        file: Path of the temporary file.
        res: Download request.
        path: Final path to save the completed temporary file.
        callback: Callback to update download progress.
        chunk_size: Amount of bytes to download before calling `callback`.

    Returns:
        True if download completed successfully, otherwise False.
    """
    # Establish initial progress
    path = path or file
    total = int(res.headers.get("Content-Length") or 1)
    current = file.stat().st_size

    # Open the temporary file and write each chunk
    with open(file, "ab") as f:
        try:
            for chunk in res.iter_content(chunk_size=chunk_size):
                if not chunk:
                    raise OSError('Bad chunk detected!')
                if res.status_code != 200:
                    raise requests.RequestException(
                        request=res.request,
                        response=res)
                f.write(chunk)

                # Execute progress callback if provided
                if callback:
                    current += int(chunk_size)
                    callback(current, total)

        # Download failure
        except Exception as e:
            print(f'Download failed: {file}\n{e}')
            return False
        del res

    # Rename the temporary file, unpack it
    try:
        if file != path:
            shutil.move(file, path)
        unpack_archive(path)
    except Exception as e:
        print(f'Unpacking failed: {path}\n{e}')
        return False
    return True