File Uploader Result

class onetl.file.file_uploader.result.UploadResult(*, successful: FileSet[RemoteFile] = None, failed: FileSet[FailedLocalFile] = None, skipped: FileSet[LocalPath] = None, missing: FileSet[LocalPath] = None)

Representation of file upload result.

Container for file paths, divided into certain categories:

• successful

• failed

• skipped

• missing

Examples

Upload files

from onetl.impl import LocalPath, RemoteFile, FailedLocalFile
from onetl.file import FileUploader, UploadResult

uploader = FileUploader(target_path="/remote", ...)

uploaded_files = uploader.run(
    [
        "/local/file1",
        "/local/file2",
        "/failed/file",
        "/existing/file",
        "/missing/file",
    ]
)

assert uploaded_files == UploadResult(
    successful={
        RemoteFile("/remote/file1"),
        RemoteFile("/remote/file2"),
    },
    failed={FailedLocalFile("/failed/file")},
    skipped={LocalPath("/existing/file")},
    missing={LocalPath("/missing/file")},
)

property details: str

Return detailed information about files in the result object

Examples

from onetl.impl import RemoteFile, LocalPath
from onet.file.file_result import FileResult

file_result1 = FileResult(
    successful={LocalPath("/local/file"), LocalPath("/local/another.file")},
    failed={
        FailedRemoteFile(
            path="/remote/file1",
            exception=NotAFileError("'/remote/file1' is not a file"),
        ),
        FailedRemoteFile(
            path="/remote/file2",
            exception=FileMissingError("'/remote/file2' does not exist"),
        ),
    },
    skipped={LocalPath("/skipped/file1"), LocalPath("/skipped/file2")},
    missing={LocalPath("/missing/file1"), LocalPath("/missing/file2")},
)

details1 = """
    Total: 8 files (10.4 MB)

    Successful 2 files (30.7 kB):
        '/successful1' (10.2 kB)
        '/successful2' (20.5 kB)

    Failed 2 files (10MB):
        '/remote/file1' (1 MB)
            NotAFileError("'/remote/file1' is not a file")

        '/remote/file2' (9 MB)
            FileMissingError("'/remote/file2' does not exist")

    Skipped 2 files (15 kB):
        '/skipped/file1' (10kB)
        '/skipped/file2' (5 kB)

    Missing 2 files:
        '/missing/file1'
        '/missing/file2'
"""

assert file_result1.details == details1

file_result2 = FileResult()
details2 = """
    No successful files

    No failed files

    No skipped files

    No missing files
"""

assert file_result2.details == details2

dict(*, include: AbstractSetIntStr | MappingIntStrAny | None = None, exclude: AbstractSetIntStr | MappingIntStrAny | None = None, by_alias: bool = False, skip_defaults: bool | None = None, exclude_unset: bool = False, exclude_defaults: bool = False, exclude_none: bool = False) → DictStrAny

Generate a dictionary representation of the model, optionally specifying which fields to include or exclude.

field failed: FileSet[FailedLocalFile] [Optional]

File paths (local) which were not uploaded because of some failure

property failed_count: int

Get number of failed files

Examples

from onetl.impl import RemoteFile
from onet.file.file_result import FileResult

file_result = FileResult(
    failed={RemoteFile("/some/file"), RemoteFile("/some/another.file")},
)

assert file_result.failed_count == 2

property failed_size: int

Get size (in bytes) of failed files

Examples

from onetl.impl import RemoteFile
from onet.file.file_result import FileResult

file_result = FileResult(
    failed={RemoteFile("/some/file"), RemoteFile("/some/another.file")},
)

assert file_result.failed_size == 1_000_000  # in bytes

property is_empty: bool

Returns True if there are no files in successful, failed and skipped attributes

Examples

from onetl.impl import LocalPath
from onet.file.file_result import FileResult

file_result1 = FileResult()
assert file_result1.is_empty

file_result2 = FileResult(
    successful={LocalPath("/local/file"), LocalPath("/local/another.file")},
)
assert not file_result2.is_empty

json(*, include: AbstractSetIntStr | MappingIntStrAny | None = None, exclude: AbstractSetIntStr | MappingIntStrAny | None = None, by_alias: bool = False, skip_defaults: bool | None = None, exclude_unset: bool = False, exclude_defaults: bool = False, exclude_none: bool = False, encoder: Callable[[Any], Any] | None = None, models_as_dict: bool = True, **dumps_kwargs: Any) → str

Generate a JSON representation of the model, include and exclude arguments as per dict().

encoder is an optional function to supply as default to json.dumps(), other arguments as per json.dumps().

field missing: FileSet[LocalPath] [Optional]

File paths (local) which are not present in the local file system

property missing_count: int

Get number of missing files

Examples

from onetl.impl import LocalPath
from onet.file.file_result import FileResult

file_result = FileResult(
    missing={LocalPath("/some/file"), LocalPath("/some/another.file")},
)

assert file_result.missing_count == 2

raise_if_contains_zero_size() → None

Raise exception if successful attribute contains a file with zero size

Raises:

ZeroFileSizeError

successful file set contains a file with zero size

Examples

from onetl.impl import RemoteFile, LocalPath
from onet.file.file_result import FileResult

file_result = FileResult(
    successful={
        LocalPath("/local/empty1.file"),
        LocalPath("/local/empty2.file"),
        LocalPath("/local/normal.file"),
    },
)

file_result.raise_if_contains_zero_size()
# will raise ZeroFileSizeError('''
#    2 files out of 3 have zero size:
#        '/local/empty1.file'
#        '/local/empty2.file'
# ''')

raise_if_empty() → None

Raise exception if there are no files in successful, failed and skipped attributes

Raises:

EmptyFilesError

successful, failed and skipped file sets are empty

Examples

from onetl.impl import RemoteFile, LocalPath
from onet.file.file_result import FileResult

file_result = FileResult()

file_result.raise_if_empty()
# will raise EmptyFilesError("There are no files in the result")

raise_if_failed() → None

Raise exception if there are some files in failed attribute

Raises:

FailedFilesError

failed file set is not empty

Examples

from onetl.impl import RemoteFile, LocalPath
from onet.file.file_result import FileResult

files_with_exception = [
    FailedRemoteFile(
        path="/remote/file1",
        exception=NotAFileError("'/remote/file1' is not a file"),
    ),
    FailedRemoteFile(
        path="/remote/file2",
        exception=FileMissingError("'/remote/file2' does not exist"),
    ),
]

file_result = FileResult(failed=files_with_exception)

file_result.raise_if_failed()
# will raise FailedFilesError('''
#    Failed 2 files (10MB):
#        '/remote/file1' (1 MB)
#           NotAFileError("'/remote/file1' is not a file")
#
#        '/remote/file2' (9 MB)
#           FileMissingError("'/remote/file2' does not exist")
# ''')

raise_if_missing() → None

Raise exception if there are some files in missing attribute

Raises:

MissingFilesError

missing file set is not empty

Examples

from onetl.impl import RemoteFile, LocalPath
from onet.file.file_result import FileResult

file_result = FileResult(
    missing={
        LocalPath("/missing/file1"),
        LocalPath("/missing/file2"),
    },
)

file_result.raise_if_missing()
# will raise MissingFilesError('''
#    Missing 2 files:
#        '/missing/file1'
#        '/missing/file2'
# ''')

raise_if_skipped() → None

Raise exception if there are some files in skipped attribute

Raises:

SkippedFilesError

skipped file set is not empty

Examples

from onetl.impl import RemoteFile, LocalPath
from onet.file.file_result import FileResult

file_result = FileResult(
    skipped={LocalPath("/skipped/file1"), LocalPath("/skipped/file2")},
)

file_result.raise_if_skipped()
# will raise SkippedFilesError('''
#    Skipped 2 files (15 kB):
#        '/skipped/file1' (10kB)
#        '/skipped/file2' (5 kB)
# ''')

field skipped: FileSet[LocalPath] [Optional]

File paths (local) which were skipped because of some reason

property skipped_count: int

Get number of skipped files

Examples

from onetl.impl import LocalPath
from onet.file.file_result import FileResult

file_result = FileResult(
    skipped={LocalPath("/some/file"), LocalPath("/some/another.file")},
)

assert file_result.skipped_count == 2

property skipped_size: int

Get size (in bytes) of skipped files

Examples

from onetl.impl import LocalPath
from onet.file.file_result import FileResult

file_result = FileResult(
    skipped={LocalPath("/some/file"), LocalPath("/some/another.file")},
)

assert file_result.skipped_size == 1_000_000  # in bytes

field successful: FileSet[RemoteFile] [Optional]

File paths (remote) which were uploaded successfully

property successful_count: int

Get number of successful files

Examples

from onetl.impl import LocalPath
from onet.file.file_result import FileResult

file_result = FileResult(
    successful={LocalPath("/some/file"), LocalPath("/some/another.file")},
)

assert file_result.successful_count == 2

property successful_size: int

Get size (in bytes) of successful files

Examples

from onetl.impl import LocalPath
from onet.file.file_result import FileResult

file_result = FileResult(
    successful={LocalPath("/some/file"), LocalPath("/some/another.file")},
)

assert file_result.successful_size == 1_000_000  # in bytes

property summary: str

Return short summary about files in the result object

Examples

from onetl.impl import RemoteFile, LocalPath
from onet.file.file_result import FileResult

file_result1 = FileResult(
    successful={LocalPath("/local/file"), LocalPath("/local/another.file")},
    failed={RemoteFile("/remote/file"), RemoteFile("/remote/another.file")},
    skipped={LocalPath("/skipped/file")},
    missing={LocalPath("/missing/file")},
)

result = """
    Total: 8 files (10.4 MB)

    Successful: 2 files (30.7 kB)

    Failed: 2 files (10MB)

    Skipped: 2 files (15 kB)

    Missing: 2 files
"""

assert file_result1.summary == result

file_result2 = FileResult()
assert file_result1.summary == "No files"

property total_count: int

Get total number of all files

Examples

from onetl.impl import RemoteFile, LocalPath
from onet.file.file_result import FileResult

file_result = FileResult(
    successful={LocalPath("/local/file"), LocalPath("/local/another.file")},
    failed={RemoteFile("/remote/file"), RemoteFile("/remote/another.file")},
    skipped={LocalPath("/skipped/file")},
    missing={LocalPath("/missing/file")},
)

assert file_result.total_count == 6

property total_size: int

Get total size (in bytes) of all files

Examples

from onetl.impl import RemoteFile, LocalPath
from onet.file.file_result import FileResult

file_result = FileResult(
    successful={LocalPath("/local/file"), LocalPath("/local/another.file")},
    failed={RemoteFile("/remote/file"), RemoteFile("/remote/another.file")},
    skipped={LocalPath("/skipped/file")},
    missing={LocalPath("/missing/file")},
)

assert file_result.total_size == 10_000_000  # in bytes