FileSystem#

class FileSystem[source]#

Bases: ABC

Abstraction for a local or remote file system.

This can be subclassed to handle different cloud storage providers, etc.

__init__()#

Methods

__init__()

copy_from(src_uri, dst_path)

Copy a source file to a local destination.

copy_to(src_path, dst_uri)

Copy a local source file to a destination.

file_exists(uri[, include_dir])

Check if a file exists.

get_file_system(uri[, mode])

Return FileSystem that should be used for the given URI/mode pair.

last_modified(uri)

Get the last modified date of a file.

list_paths(uri[, ext])

List paths rooted at URI.

local_path(uri, download_dir)

Return the path where a local copy should be stored.

matches_uri(uri, mode)

Returns True if this FS can be used for the given URI/mode pair.

read_bytes(uri)

Read contents of URI to bytes.

read_str(uri)

Read contents of URI to a string.

sync_from_dir(src_dir_uri, dst_dir[, delete])

Syncs a source directory to a local destination directory.

sync_to_dir(src_dir, dst_dir_uri[, delete])

Syncs a local source directory to a destination directory.

write_bytes(uri, data)

Write bytes in data to URI.

write_str(uri, data)

Write string in data to URI.
abstract static copy_from(src_uri: str, dst_path: str)[source]#

Copy a source file to a local destination.

If the FileSystem is remote, this involves downloading.

Parameters

  • src_uri (str) – uri of source that can be copied from by this FileSystem

  • dst_path (str) – local path to destination file

abstract static copy_to(src_path: str, dst_uri: str)[source]#

Copy a local source file to a destination.

If the FileSystem is remote, this involves uploading.

Parameters

  • src_path (str) – local path to source file

  • dst_uri (str) – uri of destination that can be copied to by this FileSystem

abstract static file_exists(uri: str, include_dir: bool = True) → bool[source]#

Check if a file exists.

Parameters

  • uri (str) – The URI to check

  • include_dir (bool) – Include directories in check, if this file_system supports directory reads. Otherwise only return true if a single file exists at the URI.

Return type

bool

static get_file_system(uri: str, mode: str = 'r') → FileSystem[source]#

Return FileSystem that should be used for the given URI/mode pair.

Parameters

  • uri (str) – URI of file

  • mode (str) – mode to open file in, ‘r’ or ‘w’

Return type

FileSystem

abstract static last_modified(uri: str) → datetime.datetime | None[source]#

Get the last modified date of a file.

Parameters

uri (str) – the URI of the file

Returns

the last modified date in UTC of a file or None if this FileSystem does not support this operation.

Return type

datetime.datetime | None

abstract static list_paths(uri: str, ext: str | None = None) → list[str][source]#

List paths rooted at URI.

Optionally only includes paths with a certain file extension.

Parameters

  • uri (str) – the URI of a directory

  • ext (str | None) – the optional file extension to filter by

Return type

list[str]

abstract static local_path(uri: str, download_dir: str) → str[source]#

Return the path where a local copy should be stored.

Parameters

  • uri (str) – the URI of the file to be copied

  • download_dir (str) – path of the local directory in which files should be copied

Return type

str

abstract static matches_uri(uri: str, mode: str) → bool[source]#

Returns True if this FS can be used for the given URI/mode pair.

Parameters

  • uri (str) – URI of file

  • mode (str) – mode to open file in, ‘r’ or ‘w’

Return type

bool

abstract static read_bytes(uri: str) → bytes[source]#

Read contents of URI to bytes.

Parameters

uri (str) –

Return type

bytes

abstract static read_str(uri: str) → str[source]#

Read contents of URI to a string.

Parameters

uri (str) –

Return type

str

abstract static sync_from_dir(src_dir_uri: str, dst_dir: str, delete: bool = False)[source]#

Syncs a source directory to a local destination directory.

If the FileSystem is remote, this involves downloading.

Parameters

  • src_dir_uri (str) – source directory that can be synced from by this FileSystem

  • dst_dir (str) – A local destination directory

  • delete (bool) – True if the destination should be deleted first.

abstract static sync_to_dir(src_dir: str, dst_dir_uri: str, delete: bool = False)[source]#

Syncs a local source directory to a destination directory.

If the FileSystem is remote, this involves uploading.

Parameters

  • src_dir (str) – local source directory to sync from

  • dst_dir_uri (str) – A destination directory that can be synced to by this FileSystem

  • delete (bool) – True if the destination should be deleted first.

abstract static write_bytes(uri: str, data: bytes)[source]#

Write bytes in data to URI.

Parameters
abstract static write_str(uri: str, data: str)[source]#

Write string in data to URI.

Parameters

  • uri (str) –

  • data (str) –