Traktor/myenv/Lib/site-packages/fsspec/implementations/tests/test_dirfs.py
2024-05-23 01:57:24 +02:00

589 lines
18 KiB
Python

import pytest
from fsspec.asyn import AsyncFileSystem
from fsspec.implementations.dirfs import DirFileSystem
from fsspec.spec import AbstractFileSystem
PATH = "path/to/dir"
ARGS = ["foo", "bar"]
KWARGS = {"baz": "baz", "qux": "qux"}
@pytest.fixture
def make_fs(mocker):
def _make_fs(async_impl=False, asynchronous=False):
attrs = {
"sep": "/",
"async_impl": async_impl,
"_strip_protocol": lambda path: path,
}
if async_impl:
attrs["asynchronous"] = asynchronous
cls = AsyncFileSystem
else:
cls = AbstractFileSystem
fs = mocker.MagicMock(spec=cls, **attrs)
return fs
return _make_fs
@pytest.fixture(
params=[
pytest.param(False, id="sync"),
pytest.param(True, id="async"),
]
)
def fs(make_fs, request):
return make_fs(async_impl=request.param)
@pytest.fixture
def asyncfs(make_fs):
return make_fs(async_impl=True, asynchronous=True)
@pytest.fixture
def make_dirfs():
def _make_dirfs(fs, asynchronous=False):
return DirFileSystem(PATH, fs, asynchronous=asynchronous)
return _make_dirfs
@pytest.fixture
def dirfs(make_dirfs, fs):
return make_dirfs(fs)
@pytest.fixture
def adirfs(make_dirfs, asyncfs):
return make_dirfs(asyncfs, asynchronous=True)
def test_dirfs(fs, asyncfs):
DirFileSystem("path", fs)
DirFileSystem("path", asyncfs, asynchronous=True)
with pytest.raises(ValueError):
DirFileSystem("path", asyncfs)
with pytest.raises(ValueError):
DirFileSystem("path", fs, asynchronous=True)
@pytest.mark.parametrize(
"root, rel, full",
[
("", "", ""),
("", "foo", "foo"),
("root", "", "root"),
("root", "foo", "root/foo"),
],
)
def test_path(fs, root, rel, full):
dirfs = DirFileSystem(root, fs)
assert dirfs._join(rel) == full
assert dirfs._relpath(full) == rel
def test_sep(mocker, dirfs):
sep = mocker.Mock()
dirfs.fs.sep = sep
assert dirfs.sep == sep
@pytest.mark.asyncio
async def test_set_session(mocker, adirfs):
adirfs.fs.set_session = mocker.AsyncMock()
assert (
await adirfs.set_session(*ARGS, **KWARGS) == adirfs.fs.set_session.return_value
)
adirfs.fs.set_session.assert_called_once_with(*ARGS, **KWARGS)
@pytest.mark.asyncio
async def test_async_rm_file(adirfs):
await adirfs._rm_file("file", **KWARGS)
adirfs.fs._rm_file.assert_called_once_with(f"{PATH}/file", **KWARGS)
def test_rm_file(dirfs):
dirfs.rm_file("file", **KWARGS)
dirfs.fs.rm_file.assert_called_once_with("path/to/dir/file", **KWARGS)
@pytest.mark.asyncio
async def test_async_rm(adirfs):
await adirfs._rm("file", *ARGS, **KWARGS)
adirfs.fs._rm.assert_called_once_with("path/to/dir/file", *ARGS, **KWARGS)
def test_rm(dirfs):
dirfs.rm("file", *ARGS, **KWARGS)
dirfs.fs.rm.assert_called_once_with("path/to/dir/file", *ARGS, **KWARGS)
@pytest.mark.asyncio
async def test_async_cp_file(adirfs):
await adirfs._cp_file("one", "two", **KWARGS)
adirfs.fs._cp_file.assert_called_once_with(f"{PATH}/one", f"{PATH}/two", **KWARGS)
def test_cp_file(dirfs):
dirfs.cp_file("one", "two", **KWARGS)
dirfs.fs.cp_file.assert_called_once_with(f"{PATH}/one", f"{PATH}/two", **KWARGS)
@pytest.mark.asyncio
async def test_async_copy(adirfs):
await adirfs._copy("one", "two", *ARGS, **KWARGS)
adirfs.fs._copy.assert_called_once_with(
f"{PATH}/one", f"{PATH}/two", *ARGS, **KWARGS
)
def test_copy(dirfs):
dirfs.copy("one", "two", *ARGS, **KWARGS)
dirfs.fs.copy.assert_called_once_with(f"{PATH}/one", f"{PATH}/two", *ARGS, **KWARGS)
@pytest.mark.asyncio
async def test_async_pipe(adirfs):
await adirfs._pipe("file", *ARGS, **KWARGS)
adirfs.fs._pipe.assert_called_once_with(f"{PATH}/file", *ARGS, **KWARGS)
def test_pipe(dirfs):
dirfs.pipe("file", *ARGS, **KWARGS)
dirfs.fs.pipe.assert_called_once_with(f"{PATH}/file", *ARGS, **KWARGS)
@pytest.mark.asyncio
async def test_async_pipe_file(adirfs):
await adirfs._pipe_file("file", *ARGS, **KWARGS)
adirfs.fs._pipe_file.assert_called_once_with(f"{PATH}/file", *ARGS, **KWARGS)
def test_pipe_file(dirfs):
dirfs.pipe_file("file", *ARGS, **KWARGS)
dirfs.fs.pipe_file.assert_called_once_with(f"{PATH}/file", *ARGS, **KWARGS)
@pytest.mark.asyncio
async def test_async_cat_file(adirfs):
assert (
await adirfs._cat_file("file", *ARGS, **KWARGS)
== adirfs.fs._cat_file.return_value
)
adirfs.fs._cat_file.assert_called_once_with(f"{PATH}/file", *ARGS, **KWARGS)
def test_cat_file(dirfs):
assert dirfs.cat_file("file", *ARGS, **KWARGS) == dirfs.fs.cat_file.return_value
dirfs.fs.cat_file.assert_called_once_with(f"{PATH}/file", *ARGS, **KWARGS)
@pytest.mark.asyncio
async def test_async_cat(adirfs):
assert await adirfs._cat("file", *ARGS, **KWARGS) == adirfs.fs._cat.return_value
adirfs.fs._cat.assert_called_once_with(f"{PATH}/file", *ARGS, **KWARGS)
def test_cat(dirfs):
assert dirfs.cat("file", *ARGS, **KWARGS) == dirfs.fs.cat.return_value
dirfs.fs.cat.assert_called_once_with(f"{PATH}/file", *ARGS, **KWARGS)
@pytest.mark.asyncio
async def test_async_cat_list(adirfs):
adirfs.fs._cat.return_value = {f"{PATH}/one": "foo", f"{PATH}/two": "bar"}
assert await adirfs._cat(["one", "two"], *ARGS, **KWARGS) == {
"one": "foo",
"two": "bar",
}
adirfs.fs._cat.assert_called_once_with(
[f"{PATH}/one", f"{PATH}/two"], *ARGS, **KWARGS
)
def test_cat_list(dirfs):
dirfs.fs.cat.return_value = {f"{PATH}/one": "foo", f"{PATH}/two": "bar"}
assert dirfs.cat(["one", "two"], *ARGS, **KWARGS) == {"one": "foo", "two": "bar"}
dirfs.fs.cat.assert_called_once_with(
[f"{PATH}/one", f"{PATH}/two"], *ARGS, **KWARGS
)
@pytest.mark.asyncio
async def test_async_put_file(adirfs):
await adirfs._put_file("local", "file", **KWARGS)
adirfs.fs._put_file.assert_called_once_with("local", f"{PATH}/file", **KWARGS)
def test_put_file(dirfs):
dirfs.put_file("local", "file", **KWARGS)
dirfs.fs.put_file.assert_called_once_with("local", f"{PATH}/file", **KWARGS)
@pytest.mark.asyncio
async def test_async_put(adirfs):
await adirfs._put("local", "file", **KWARGS)
adirfs.fs._put.assert_called_once_with("local", f"{PATH}/file", **KWARGS)
def test_put(dirfs):
dirfs.put("local", "file", **KWARGS)
dirfs.fs.put.assert_called_once_with("local", f"{PATH}/file", **KWARGS)
@pytest.mark.asyncio
async def test_async_get_file(adirfs):
await adirfs._get_file("file", "local", **KWARGS)
adirfs.fs._get_file.assert_called_once_with(f"{PATH}/file", "local", **KWARGS)
def test_get_file(dirfs):
dirfs.get_file("file", "local", **KWARGS)
dirfs.fs.get_file.assert_called_once_with(f"{PATH}/file", "local", **KWARGS)
@pytest.mark.asyncio
async def test_async_get(adirfs):
await adirfs._get("file", "local", **KWARGS)
adirfs.fs._get.assert_called_once_with(f"{PATH}/file", "local", **KWARGS)
def test_get(dirfs):
dirfs.get("file", "local", **KWARGS)
dirfs.fs.get.assert_called_once_with(f"{PATH}/file", "local", **KWARGS)
@pytest.mark.asyncio
async def test_async_isfile(adirfs):
assert await adirfs._isfile("file") == adirfs.fs._isfile.return_value
adirfs.fs._isfile.assert_called_once_with(f"{PATH}/file")
def test_isfile(dirfs):
assert dirfs.isfile("file") == dirfs.fs.isfile.return_value
dirfs.fs.isfile.assert_called_once_with(f"{PATH}/file")
@pytest.mark.asyncio
async def test_async_isdir(adirfs):
assert await adirfs._isdir("file") == adirfs.fs._isdir.return_value
adirfs.fs._isdir.assert_called_once_with(f"{PATH}/file")
def test_isdir(dirfs):
assert dirfs.isdir("file") == dirfs.fs.isdir.return_value
dirfs.fs.isdir.assert_called_once_with(f"{PATH}/file")
@pytest.mark.asyncio
async def test_async_size(adirfs):
assert await adirfs._size("file") == adirfs.fs._size.return_value
adirfs.fs._size.assert_called_once_with(f"{PATH}/file")
def test_size(dirfs):
assert dirfs.size("file") == dirfs.fs.size.return_value
dirfs.fs.size.assert_called_once_with(f"{PATH}/file")
@pytest.mark.asyncio
async def test_async_exists(adirfs):
assert await adirfs._exists("file") == adirfs.fs._exists.return_value
adirfs.fs._exists.assert_called_once_with(f"{PATH}/file")
def test_exists(dirfs):
assert dirfs.exists("file") == dirfs.fs.exists.return_value
dirfs.fs.exists.assert_called_once_with(f"{PATH}/file")
@pytest.mark.asyncio
async def test_async_info(adirfs):
assert await adirfs._info("file", **KWARGS) == adirfs.fs._info.return_value
adirfs.fs._info.assert_called_once_with(f"{PATH}/file", **KWARGS)
def test_info(dirfs):
assert dirfs.info("file", **KWARGS) == dirfs.fs.info.return_value
dirfs.fs.info.assert_called_once_with(f"{PATH}/file", **KWARGS)
@pytest.mark.asyncio
async def test_async_ls(adirfs):
adirfs.fs._ls.return_value = [f"{PATH}/file"]
assert await adirfs._ls("file", detail=False, **KWARGS) == ["file"]
adirfs.fs._ls.assert_called_once_with(f"{PATH}/file", detail=False, **KWARGS)
def test_ls(dirfs):
dirfs.fs.ls.return_value = [f"{PATH}/file"]
assert dirfs.ls("file", detail=False, **KWARGS) == ["file"]
dirfs.fs.ls.assert_called_once_with(f"{PATH}/file", detail=False, **KWARGS)
@pytest.mark.asyncio
async def test_async_ls_detail(adirfs):
adirfs.fs._ls.return_value = [{"name": f"{PATH}/file", "foo": "bar"}]
assert await adirfs._ls("file", detail=True, **KWARGS) == [
{"name": "file", "foo": "bar"}
]
adirfs.fs._ls.assert_called_once_with(f"{PATH}/file", detail=True, **KWARGS)
def test_ls_detail(dirfs):
dirfs.fs.ls.return_value = [{"name": f"{PATH}/file", "foo": "bar"}]
assert dirfs.ls("file", detail=True, **KWARGS) == [{"name": "file", "foo": "bar"}]
dirfs.fs.ls.assert_called_once_with(f"{PATH}/file", detail=True, **KWARGS)
@pytest.mark.asyncio
async def test_async_walk(adirfs, mocker):
async def _walk(path, *args, **kwargs):
yield (f"{PATH}/root", ["foo", "bar"], ["baz", "qux"])
adirfs.fs._walk = mocker.MagicMock()
adirfs.fs._walk.side_effect = _walk
actual = []
async for entry in adirfs._walk("root", *ARGS, **KWARGS):
actual.append(entry) # noqa: PERF402
assert actual == [("root", ["foo", "bar"], ["baz", "qux"])]
adirfs.fs._walk.assert_called_once_with(f"{PATH}/root", *ARGS, **KWARGS)
def test_walk(dirfs):
dirfs.fs.walk.return_value = iter(
[(f"{PATH}/root", ["foo", "bar"], ["baz", "qux"])]
)
assert list(dirfs.walk("root", *ARGS, **KWARGS)) == [
("root", ["foo", "bar"], ["baz", "qux"])
]
dirfs.fs.walk.assert_called_once_with(f"{PATH}/root", *ARGS, **KWARGS)
@pytest.mark.asyncio
async def test_async_glob(adirfs):
adirfs.fs._glob.return_value = [f"{PATH}/one", f"{PATH}/two"]
assert await adirfs._glob("*", **KWARGS) == ["one", "two"]
adirfs.fs._glob.assert_called_once_with(f"{PATH}/*", **KWARGS)
def test_glob(dirfs):
dirfs.fs.glob.return_value = [f"{PATH}/one", f"{PATH}/two"]
assert dirfs.glob("*", **KWARGS) == ["one", "two"]
dirfs.fs.glob.assert_called_once_with(f"{PATH}/*", **KWARGS)
def test_glob_with_protocol(dirfs):
dirfs.fs.glob.return_value = [f"{PATH}/one", f"{PATH}/two"]
assert dirfs.glob("dir://*", **KWARGS) == ["one", "two"]
dirfs.fs.glob.assert_called_once_with(f"{PATH}/*", **KWARGS)
@pytest.mark.asyncio
async def test_async_glob_detail(adirfs):
adirfs.fs._glob.return_value = {
f"{PATH}/one": {"foo": "bar"},
f"{PATH}/two": {"baz": "qux"},
}
assert await adirfs._glob("*", detail=True, **KWARGS) == {
"one": {"foo": "bar"},
"two": {"baz": "qux"},
}
adirfs.fs._glob.assert_called_once_with(f"{PATH}/*", detail=True, **KWARGS)
def test_glob_detail(dirfs):
dirfs.fs.glob.return_value = {
f"{PATH}/one": {"foo": "bar"},
f"{PATH}/two": {"baz": "qux"},
}
assert dirfs.glob("*", detail=True, **KWARGS) == {
"one": {"foo": "bar"},
"two": {"baz": "qux"},
}
dirfs.fs.glob.assert_called_once_with(f"{PATH}/*", detail=True, **KWARGS)
@pytest.mark.asyncio
async def test_async_du(adirfs):
adirfs.fs._du.return_value = 1234
assert await adirfs._du("file", *ARGS, **KWARGS) == 1234
adirfs.fs._du.assert_called_once_with(f"{PATH}/file", *ARGS, **KWARGS)
def test_du(dirfs):
dirfs.fs.du.return_value = 1234
assert dirfs.du("file", *ARGS, **KWARGS) == 1234
dirfs.fs.du.assert_called_once_with(f"{PATH}/file", *ARGS, **KWARGS)
@pytest.mark.asyncio
async def test_async_du_granular(adirfs):
adirfs.fs._du.return_value = {f"{PATH}/dir/one": 1, f"{PATH}/dir/two": 2}
assert await adirfs._du("dir", *ARGS, total=False, **KWARGS) == {
"dir/one": 1,
"dir/two": 2,
}
adirfs.fs._du.assert_called_once_with(f"{PATH}/dir", *ARGS, total=False, **KWARGS)
def test_du_granular(dirfs):
dirfs.fs.du.return_value = {f"{PATH}/dir/one": 1, f"{PATH}/dir/two": 2}
assert dirfs.du("dir", *ARGS, total=False, **KWARGS) == {"dir/one": 1, "dir/two": 2}
dirfs.fs.du.assert_called_once_with(f"{PATH}/dir", *ARGS, total=False, **KWARGS)
@pytest.mark.asyncio
async def test_async_find(adirfs):
adirfs.fs._find.return_value = [f"{PATH}/dir/one", f"{PATH}/dir/two"]
assert await adirfs._find("dir", *ARGS, **KWARGS) == ["dir/one", "dir/two"]
adirfs.fs._find.assert_called_once_with(f"{PATH}/dir", *ARGS, **KWARGS)
def test_find(dirfs):
dirfs.fs.find.return_value = [f"{PATH}/dir/one", f"{PATH}/dir/two"]
assert dirfs.find("dir", *ARGS, **KWARGS) == ["dir/one", "dir/two"]
dirfs.fs.find.assert_called_once_with(f"{PATH}/dir", *ARGS, **KWARGS)
@pytest.mark.asyncio
async def test_async_find_detail(adirfs):
adirfs.fs._find.return_value = {
f"{PATH}/dir/one": {"foo": "bar"},
f"{PATH}/dir/two": {"baz": "qux"},
}
assert await adirfs._find("dir", *ARGS, detail=True, **KWARGS) == {
"dir/one": {"foo": "bar"},
"dir/two": {"baz": "qux"},
}
adirfs.fs._find.assert_called_once_with(f"{PATH}/dir", *ARGS, detail=True, **KWARGS)
def test_find_detail(dirfs):
dirfs.fs.find.return_value = {
f"{PATH}/dir/one": {"foo": "bar"},
f"{PATH}/dir/two": {"baz": "qux"},
}
assert dirfs.find("dir", *ARGS, detail=True, **KWARGS) == {
"dir/one": {"foo": "bar"},
"dir/two": {"baz": "qux"},
}
dirfs.fs.find.assert_called_once_with(f"{PATH}/dir", *ARGS, detail=True, **KWARGS)
@pytest.mark.asyncio
async def test_async_expand_path(adirfs):
adirfs.fs._expand_path.return_value = [f"{PATH}/file"]
assert await adirfs._expand_path("*", *ARGS, **KWARGS) == ["file"]
adirfs.fs._expand_path.assert_called_once_with(f"{PATH}/*", *ARGS, **KWARGS)
def test_expand_path(dirfs):
dirfs.fs.expand_path.return_value = [f"{PATH}/file"]
assert dirfs.expand_path("*", *ARGS, **KWARGS) == ["file"]
dirfs.fs.expand_path.assert_called_once_with(f"{PATH}/*", *ARGS, **KWARGS)
@pytest.mark.asyncio
async def test_async_expand_path_list(adirfs):
adirfs.fs._expand_path.return_value = [f"{PATH}/1file", f"{PATH}/2file"]
assert await adirfs._expand_path(["1*", "2*"], *ARGS, **KWARGS) == [
"1file",
"2file",
]
adirfs.fs._expand_path.assert_called_once_with(
[f"{PATH}/1*", f"{PATH}/2*"], *ARGS, **KWARGS
)
def test_expand_path_list(dirfs):
dirfs.fs.expand_path.return_value = [f"{PATH}/1file", f"{PATH}/2file"]
assert dirfs.expand_path(["1*", "2*"], *ARGS, **KWARGS) == ["1file", "2file"]
dirfs.fs.expand_path.assert_called_once_with(
[f"{PATH}/1*", f"{PATH}/2*"], *ARGS, **KWARGS
)
@pytest.mark.asyncio
async def test_async_mkdir(adirfs):
await adirfs._mkdir("dir", *ARGS, **KWARGS)
adirfs.fs._mkdir.assert_called_once_with(f"{PATH}/dir", *ARGS, **KWARGS)
def test_mkdir(dirfs):
dirfs.mkdir("dir", *ARGS, **KWARGS)
dirfs.fs.mkdir.assert_called_once_with(f"{PATH}/dir", *ARGS, **KWARGS)
@pytest.mark.asyncio
async def test_async_makedirs(adirfs):
await adirfs._makedirs("dir", *ARGS, **KWARGS)
adirfs.fs._makedirs.assert_called_once_with(f"{PATH}/dir", *ARGS, **KWARGS)
def test_makedirs(dirfs):
dirfs.makedirs("dir", *ARGS, **KWARGS)
dirfs.fs.makedirs.assert_called_once_with(f"{PATH}/dir", *ARGS, **KWARGS)
def test_rmdir(mocker, dirfs):
dirfs.fs.rmdir = mocker.Mock()
dirfs.rmdir("dir")
dirfs.fs.rmdir.assert_called_once_with(f"{PATH}/dir")
def test_mv(mocker, dirfs):
dirfs.fs.mv = mocker.Mock()
dirfs.mv("one", "two", **KWARGS)
dirfs.fs.mv.assert_called_once_with(f"{PATH}/one", f"{PATH}/two", **KWARGS)
def test_touch(mocker, dirfs):
dirfs.fs.touch = mocker.Mock()
dirfs.touch("file", **KWARGS)
dirfs.fs.touch.assert_called_once_with(f"{PATH}/file", **KWARGS)
def test_created(mocker, dirfs):
dirfs.fs.created = mocker.Mock(return_value="date")
assert dirfs.created("file") == "date"
dirfs.fs.created.assert_called_once_with(f"{PATH}/file")
def test_modified(mocker, dirfs):
dirfs.fs.modified = mocker.Mock(return_value="date")
assert dirfs.modified("file") == "date"
dirfs.fs.modified.assert_called_once_with(f"{PATH}/file")
def test_sign(mocker, dirfs):
dirfs.fs.sign = mocker.Mock(return_value="url")
assert dirfs.sign("file", *ARGS, **KWARGS) == "url"
dirfs.fs.sign.assert_called_once_with(f"{PATH}/file", *ARGS, **KWARGS)
def test_open(mocker, dirfs):
dirfs.fs.open = mocker.Mock()
assert dirfs.open("file", *ARGS, **KWARGS) == dirfs.fs.open.return_value
dirfs.fs.open.assert_called_once_with(f"{PATH}/file", *ARGS, **KWARGS)
def test_from_url(m):
from fsspec.core import url_to_fs
m.pipe("inner/file", b"data")
fs, _ = url_to_fs("dir::memory://inner")
assert fs.ls("", False) == ["file"]
assert fs.ls("", True)[0]["name"] == "file"
assert fs.cat("file") == b"data"