S3FileSystem#

class S3FileSystem[source]#

Bases: FileSystem

A FileSystem for interacting with files stored on AWS S3.

Uses Everett configuration of form: ` [AWS_S3] requester_pays=True `

__init__()#

Methods

__init__()

copy_from(src_uri, dst_path)

Copy a source file to a local destination.

copy_to(src_path, dst_uri)

Copy a local source file to a destination.

file_exists(uri[, include_dir])

Check if a file exists.

get_client()

get_file_system(uri[, mode])

Return FileSystem that should be used for the given URI/mode pair.

get_request_payer()

get_session()

last_modified(uri)

Get the last modified date of a file.

list_paths(uri[, ext, delimiter])

List paths rooted at URI.

local_path(uri, download_dir)

Return the path where a local copy should be stored.

matches_uri(uri, mode)

Returns True if this FS can be used for the given URI/mode pair.

parse_uri(uri)

Parse bucket name and key from an S3 URI.

read_bytes(uri)

Read contents of URI to bytes.

read_str(uri)

Read contents of URI to a string.

sync_from_dir(src_dir_uri, dst_dir[, delete])

Syncs a source directory to a local destination directory.

sync_to_dir(src_dir, dst_dir_uri[, delete])

Syncs a local source directory to a destination directory.

write_bytes(uri, data)

Write bytes in data to URI.

write_str(uri, data)

Write string in data to URI.
static copy_from(src_uri: str, dst_path: str) → None[source]#

Copy a source file to a local destination.

If the FileSystem is remote, this involves downloading.

Parameters:

  • src_uri (str) – uri of source that can be copied from by this FileSystem

  • dst_path (str) – local path to destination file

Return type:

None

static copy_to(src_path: str, dst_uri: str) → None[source]#

Copy a local source file to a destination.

If the FileSystem is remote, this involves uploading.

Parameters:

  • src_path (str) – local path to source file

  • dst_uri (str) – uri of destination that can be copied to by this FileSystem

Return type:

None

static file_exists(uri: str, include_dir: bool = True) → bool[source]#

Check if a file exists.

Parameters:

  • uri (str) – The URI to check

  • include_dir (bool) – Include directories in check, if this file_system supports directory reads. Otherwise only return true if a single file exists at the URI.

Return type:

bool

static get_client()[source]#
static get_file_system(uri: str, mode: str = 'r') → FileSystem#

Return FileSystem that should be used for the given URI/mode pair.

Parameters:

  • uri (str) – URI of file

  • mode (str) – mode to open file in, ‘r’ or ‘w’

Return type:

FileSystem

static get_request_payer() → str[source]#
Return type:

str

static get_session()[source]#
static last_modified(uri: str) → datetime[source]#

Get the last modified date of a file.

Parameters:

uri (str) – the URI of the file

Returns:

the last modified date in UTC of a file or None if this FileSystem does not support this operation.

Return type:

datetime

static list_paths(uri: str, ext: str = '', delimiter: str = '/') → list[str][source]#

List paths rooted at URI.

Optionally only includes paths with a certain file extension.

Parameters:

  • uri (str) – the URI of a directory

  • ext (str) – the optional file extension to filter by

  • delimiter (str) –

Return type:

list[str]

static local_path(uri: str, download_dir: str) → None[source]#

Return the path where a local copy should be stored.

Parameters:

  • uri (str) – the URI of the file to be copied

  • download_dir (str) – path of the local directory in which files should be copied

Return type:

None

static matches_uri(uri: str, mode: str) → bool[source]#

Returns True if this FS can be used for the given URI/mode pair.

Parameters:

  • uri (str) – URI of file

  • mode (str) – mode to open file in, ‘r’ or ‘w’

Return type:

bool

static parse_uri(uri: str) → tuple[str, str][source]#

Parse bucket name and key from an S3 URI.

Parameters:

uri (str) –

Return type:

tuple[str, str]

static read_bytes(uri: str) → bytes[source]#

Read contents of URI to bytes.

Parameters:

uri (str) –

Return type:

bytes

static read_str(uri: str) → str[source]#

Read contents of URI to a string.

Parameters:

uri (str) –

Return type:

str

static sync_from_dir(src_dir_uri: str, dst_dir: str, delete: bool = False) → None[source]#

Syncs a source directory to a local destination directory.

If the FileSystem is remote, this involves downloading.

Parameters:

  • src_dir_uri (str) – source directory that can be synced from by this FileSystem

  • dst_dir (str) – A local destination directory

  • delete (bool) – True if the destination should be deleted first.

Return type:

None

static sync_to_dir(src_dir: str, dst_dir_uri: str, delete: bool = False) → None[source]#

Syncs a local source directory to a destination directory.

If the FileSystem is remote, this involves uploading.

Parameters:

  • src_dir (str) – local source directory to sync from

  • dst_dir_uri (str) – A destination directory that can be synced to by this FileSystem

  • delete (bool) – True if the destination should be deleted first.

Return type:

None

static write_bytes(uri: str, data: bytes) → None[source]#

Write bytes in data to URI.

Parameters:
Return type:

None

static write_str(uri: str, data: str) → None[source]#

Write string in data to URI.

Parameters:

  • uri (str) –

  • data (str) –

Return type:

None