Skip to content

Commit f25e36f

Browse files
committed
feat: set fsspec as default source (#1023)
* feat: add fsspec as required dependency (#1021) * fsspec requirements * simplify fsspec import * use loop property * correctly create schemes list * remove deprecated handlers from docs * simplify source selection * return object source * pickle executor * rename test * test more handlers * option to check writeable file-like object * rename test * explicitly set handler * fix s3 source * rename test * Revert "fix s3 source" This reverts commit e76fdbb. * sesparate PR for s3 fix (#1024) * strip file:// * rename test * rename tests * add aiohttp skip * attempt to parse windows paths * test ci * Revert "test ci" This reverts commit 4c1c8a5. * rename test * remove fsspec from test * remove *_handler options * update defaults * do not override default s3 * do not use fsspec for multiprocessing * rename test * fix not selecting object source * missing import * normalize doc * remove helper * never return None as source * remove unnecessary xrootd source default override since fsspec is default now * rename test * add empty class to pass old pickle test
1 parent 9b4bc3a commit f25e36f

13 files changed

+70
-217
lines changed

src/uproot/__init__.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,6 @@
7272
isort:skip_file
7373
"""
7474

75-
7675
from uproot.version import __version__
7776
import uproot.const
7877
import uproot.extras
@@ -92,6 +91,7 @@
9291
from uproot.source.xrootd import MultithreadedXRootDSource
9392
from uproot.source.s3 import S3Source
9493
from uproot.source.object import ObjectSource
94+
from uproot.source.fsspec import FSSpecSource
9595
from uproot.source.cursor import Cursor
9696
from uproot.source.futures import TrivialExecutor
9797
from uproot.source.futures import ThreadPoolExecutor

src/uproot/_dask.py

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -142,11 +142,6 @@ def dask(
142142
Options (type; default):
143143
144144
* handler (:doc:`uproot.source.chunk.Source` class; None)
145-
* file_handler (:doc:`uproot.source.chunk.Source` class; None) (Deprecated: Use `handler` instead. If set, this will take precedence over `handler`)
146-
* xrootd_handler (:doc:`uproot.source.chunk.Source` class; None) (Deprecated: Use `handler` instead. If set, this will take precedence over `handler`)
147-
* s3_handler (:doc:`uproot.source.chunk.Source` class; None) (Deprecated: Use `handler` instead. If set, this will take precedence over `handler`)
148-
* http_handler (:doc:`uproot.source.chunk.Source` class; None) (Deprecated: Use `handler` instead. If set, this will take precedence over `handler`)
149-
* object_handler (:doc:`uproot.source.chunk.Source` class; None) (Deprecated: Use `handler` instead. If set, this will take precedence over `handler`)
150145
* timeout (float for HTTP, int for XRootD; 30)
151146
* max_num_elements (None or int; None)
152147
* num_workers (int; 1)

src/uproot/_util.py

Lines changed: 24 additions & 137 deletions
Original file line numberDiff line numberDiff line change
@@ -369,63 +369,30 @@ def file_path_to_source_class(file_path, options):
369369
Use a file path to get the :doc:`uproot.source.chunk.Source` class that would read it.
370370
371371
Returns a tuple of (class, file_path) where the class is a subclass of :doc:`uproot.source.chunk.Source`.
372-
373-
The "handler" option is the preferred way to specify a custom source class.
374-
The "*_handler" options are for backwards compatibility and will override the "handler" option if set.
375372
"""
373+
376374
import uproot.source.chunk
377375

378376
file_path = regularize_path(file_path)
379377

380-
out = options["handler"]
381-
if out is not None:
382-
if not (isinstance(out, type) and issubclass(out, uproot.source.chunk.Source)):
383-
raise TypeError(
384-
f"'handler' is not a class object inheriting from Source: {out!r}"
385-
)
386-
# check if "object_handler" is set
387-
if (
388-
options["object_handler"] is not None
389-
or options["file_handler"] is not None
390-
or options["xrootd_handler"] is not None
391-
or options["s3_handler"] is not None
392-
or options["http_handler"] is not None
378+
source_cls = options["handler"]
379+
if source_cls is not None:
380+
if not (
381+
isinstance(source_cls, type)
382+
and issubclass(source_cls, uproot.source.chunk.Source)
393383
):
394-
# These options will override the "handler" option for backwards compatibility
395-
warnings.warn(
396-
"""In version 5.2.0, the '*_handler' argument ('http_handler`, 's3_handler', etc.) will be removed from 'uproot.open'. Use 'handler' instead.""",
397-
stacklevel=1,
384+
raise TypeError(
385+
f"'handler' is not a class object inheriting from Source: {source_cls!r}"
398386
)
399-
else:
400-
return out, file_path
387+
return source_cls, file_path
401388

402389
if (
403390
not isinstance(file_path, str)
404391
and hasattr(file_path, "read")
405392
and hasattr(file_path, "seek")
406393
):
407-
out = options["object_handler"]
408-
if out is None:
409-
out = uproot.source.object.ObjectSource
410-
else:
411-
warnings.warn(
412-
f"""In version 5.2.0, the 'object_handler' argument will be removed from 'uproot.open'. Use
413-
uproot.open(..., handler={out!r})
414-
instead.
415-
416-
To raise these warnings as errors (and get stack traces to find out where they're called), run
417-
import warnings
418-
warnings.filterwarnings("error", module="uproot.*")
419-
after the first `import uproot` or use `@pytest.mark.filterwarnings("error:::uproot.*")` in pytest.""",
420-
DeprecationWarning,
421-
stacklevel=1,
422-
)
423-
if not (isinstance(out, type) and issubclass(out, uproot.source.chunk.Source)):
424-
raise TypeError(
425-
f"'object_handler' is not a class object inheriting from Source: {out!r}"
426-
)
427-
428-
return out, file_path
394+
source_cls = uproot.source.object.ObjectSource
395+
return source_cls, file_path
429396

430397
windows_absolute_path = None
431398
if win and _windows_absolute_path_pattern.match(file_path) is not None:
@@ -457,107 +424,27 @@ def file_path_to_source_class(file_path, options):
457424
else:
458425
file_path = windows_absolute_path
459426

460-
out = options["file_handler"]
461-
if out is None:
462-
out = uproot.source.file.MemmapSource
463-
else:
464-
warnings.warn(
465-
f"""In version 5.2.0, the 'file_handler' argument will be removed from 'uproot.open'. Use
466-
uproot.open(..., handler={out!r}
467-
instead.
468-
469-
To raise these warnings as errors (and get stack traces to find out where they're called), run
470-
import warnings
471-
warnings.filterwarnings("error", module="uproot.*")
472-
after the first `import uproot` or use `@pytest.mark.filterwarnings("error:::uproot.*")` in pytest.""",
473-
DeprecationWarning,
474-
stacklevel=1,
475-
)
427+
# uproot.source.file.MemmapSource
428+
source_cls = uproot.source.fsspec.FSSpecSource
476429

477-
if not (isinstance(out, type) and issubclass(out, uproot.source.chunk.Source)):
478-
raise TypeError(
479-
"'file_handler' is not a class object inheriting from Source: "
480-
+ repr(out)
481-
)
482-
return out, os.path.expanduser(file_path)
430+
return source_cls, os.path.expanduser(file_path)
483431

484432
elif scheme == "root":
485-
out = options["xrootd_handler"]
486-
if out is None:
487-
out = uproot.source.xrootd.XRootDSource
488-
else:
489-
warnings.warn(
490-
f"""In version 5.2.0, the 'xrootd_handler' argument will be removed from 'uproot.open'. Use
491-
uproot.open(..., handler={out!r}
492-
instead.
493-
494-
To raise these warnings as errors (and get stack traces to find out where they're called), run
495-
import warnings
496-
warnings.filterwarnings("error", module="uproot.*")
497-
after the first `import uproot` or use `@pytest.mark.filterwarnings("error:::uproot.*")` in pytest.""",
498-
DeprecationWarning,
499-
stacklevel=1,
500-
)
501-
if not (isinstance(out, type) and issubclass(out, uproot.source.chunk.Source)):
502-
raise TypeError(
503-
"'xrootd_handler' is not a class object inheriting from Source: "
504-
+ repr(out)
505-
)
506-
return out, file_path
433+
# uproot.source.xrootd.XRootDSource
434+
source_cls = uproot.source.fsspec.FSSpecSource
435+
return source_cls, file_path
507436

508437
elif scheme == "s3":
509-
out = options["s3_handler"]
510-
if out is None:
511-
out = uproot.source.s3.S3Source
512-
else:
513-
warnings.warn(
514-
f"""In version 5.2.0, the 's3_handler' argument will be removed from 'uproot.open'. Use
515-
uproot.open(..., handler={out!r}
516-
instead.
517-
518-
To raise these warnings as errors (and get stack traces to find out where they're called), run
519-
import warnings
520-
warnings.filterwarnings("error", module="uproot.*")
521-
after the first `import uproot` or use `@pytest.mark.filterwarnings("error:::uproot.*")` in pytest.""",
522-
DeprecationWarning,
523-
stacklevel=1,
524-
)
525-
if not (isinstance(out, type) and issubclass(out, uproot.source.chunk.Source)):
526-
raise TypeError(
527-
"'s3' is not a class object inheriting from Source: " + repr(out)
528-
)
529-
return out, file_path
438+
# https://github.com/scikit-hep/uproot5/pull/1012
439+
source_cls = uproot.source.s3.S3Source
440+
return source_cls, file_path
530441

531442
elif scheme in ("http", "https"):
532-
out = options["http_handler"]
533-
if out is None:
534-
out = uproot.source.http.HTTPSource
535-
else:
536-
warnings.warn(
537-
f"""In version 5.2.0, the 'http_handler' argument will be removed from 'uproot.open'. Use
538-
uproot.open(..., handler={out!r}
539-
instead.
540-
541-
To raise these warnings as errors (and get stack traces to find out where they're called), run
542-
import warnings
543-
warnings.filterwarnings("error", module="uproot.*")
544-
after the first `import uproot` or use `@pytest.mark.filterwarnings("error:::uproot.*")` in pytest.""",
545-
DeprecationWarning,
546-
stacklevel=1,
547-
)
548-
if not (isinstance(out, type) and issubclass(out, uproot.source.chunk.Source)):
549-
raise TypeError(
550-
"'http_handler' is not a class object inheriting from Source: "
551-
+ repr(out)
552-
)
553-
return out, file_path
554-
555-
else:
556-
# try to use fsspec before raising an error
557-
if scheme in _schemes:
558-
return uproot.source.fsspec.FSSpecSource, file_path
443+
# uproot.source.http.HTTPSource
444+
source_cls = uproot.source.fsspec.FSSpecSource
445+
return source_cls, file_path
559446

560-
raise ValueError(f"URI scheme not recognized: {file_path}")
447+
return uproot.source.fsspec.FSSpecSource, file_path
561448

562449

563450
if isinstance(__builtins__, dict):

src/uproot/behaviors/TBranch.py

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -154,11 +154,6 @@ def iterate(
154154
Options (type; default):
155155
156156
* handler (:doc:`uproot.source.chunk.Source` class; None)
157-
* file_handler (:doc:`uproot.source.chunk.Source` class; None) (Deprecated: Use `handler` instead. If set, this will take precedence over `handler`)
158-
* xrootd_handler (:doc:`uproot.source.chunk.Source` class; None) (Deprecated: Use `handler` instead. If set, this will take precedence over `handler`)
159-
* s3_handler (:doc:`uproot.source.chunk.Source` class; None) (Deprecated: Use `handler` instead. If set, this will take precedence over `handler`)
160-
* http_handler (:doc:`uproot.source.chunk.Source` class; None) (Deprecated: Use `handler` instead. If set, this will take precedence over `handler`)
161-
* object_handler (:doc:`uproot.source.chunk.Source` class; None) (Deprecated: Use `handler` instead. If set, this will take precedence over `handler`)
162157
* timeout (float for HTTP, int for XRootD; 30)
163158
* max_num_elements (None or int; None)
164159
* num_workers (int; 1)
@@ -328,11 +323,6 @@ def concatenate(
328323
Options (type; default):
329324
330325
* handler (:doc:`uproot.source.chunk.Source` class; None)
331-
* file_handler (:doc:`uproot.source.chunk.Source` class; None) (Deprecated: Use `handler` instead. If set, this will take precedence over `handler`)
332-
* xrootd_handler (:doc:`uproot.source.chunk.Source` class; None) (Deprecated: Use `handler` instead. If set, this will take precedence over `handler`)
333-
* s3_handler (:doc:`uproot.source.chunk.Source` class; None) (Deprecated: Use `handler` instead. If set, this will take precedence over `handler`)
334-
* http_handler (:doc:`uproot.source.chunk.Source` class; None) (Deprecated: Use `handler` instead. If set, this will take precedence over `handler`)
335-
* object_handler (:doc:`uproot.source.chunk.Source` class; None) (Deprecated: Use `handler` instead. If set, this will take precedence over `handler`)
336326
* timeout (float for HTTP, int for XRootD; 30)
337327
* max_num_elements (None or int; None)
338328
* num_workers (int; 1)

src/uproot/reading.py

Lines changed: 20 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -12,11 +12,11 @@
1212
import struct
1313
import sys
1414
import uuid
15-
import warnings
1615
from collections.abc import Mapping, MutableMapping
1716

1817
import uproot
1918
import uproot.behaviors.TBranch
19+
import uproot.source.fsspec
2020
from uproot._util import no_filter
2121

2222

@@ -43,7 +43,7 @@ def open(
4343
``"rel/file.root:tdirectory/ttree"``, ``Path("rel:/file.root")``,
4444
``Path("/abs/path:stuff.root")``
4545
object_cache (None, MutableMapping, or int): Cache of objects drawn
46-
from ROOT directories (e.g histograms, TTrees, other directories);
46+
from ROOT directories (e.g. histograms, TTrees, other directories);
4747
if None, do not use a cache; if an int, create a new cache of this
4848
size.
4949
array_cache (None, MutableMapping, or memory size): Cache of arrays
@@ -78,11 +78,6 @@ def open(
7878
Options (type; default):
7979
8080
* handler (:doc:`uproot.source.chunk.Source` class; None)
81-
* file_handler (:doc:`uproot.source.chunk.Source` class; None) (Deprecated: Use `handler` instead. If set, this will take precedence over `handler`)
82-
* xrootd_handler (:doc:`uproot.source.chunk.Source` class; None) (Deprecated: Use `handler` instead. If set, this will take precedence over `handler`)
83-
* s3_handler (:doc:`uproot.source.chunk.Source` class; None) (Deprecated: Use `handler` instead. If set, this will take precedence over `handler`)
84-
* http_handler (:doc:`uproot.source.chunk.Source` class; None) (Deprecated: Use `handler` instead. If set, this will take precedence over `handler`)
85-
* object_handler (:doc:`uproot.source.chunk.Source` class; None) (Deprecated: Use `handler` instead. If set, this will take precedence over `handler`)
8681
* timeout (float for HTTP, int for XRootD; 30)
8782
* max_num_elements (None or int; None)
8883
* num_workers (int; 1)
@@ -157,43 +152,22 @@ def open(
157152
return file.root_directory[object_path]
158153

159154

155+
open.defaults = {
156+
"handler": None,
157+
"timeout": 30,
158+
"max_num_elements": None,
159+
"num_workers": 1,
160+
"use_threads": sys.platform != "emscripten",
161+
"num_fallback_workers": 10,
162+
"begin_chunk_size": 403, # the smallest a ROOT file can be
163+
"minimal_ttree_metadata": True,
164+
}
165+
166+
160167
class _OpenDefaults(dict):
161-
def __getitem__(self, where):
162-
if where == "xrootd_handler" and where not in self:
163-
# See https://github.com/scikit-hep/uproot5/issues/294
164-
if uproot.extras.older_xrootd("5.2.0"):
165-
message = (
166-
f"XRootD {uproot.extras.xrootd_version()} is not fully supported; "
167-
"""either upgrade to 5.2.0+ or set
168-
169-
open.defaults["xrootd_handler"] = uproot.MultithreadedXRootDSource
170-
"""
171-
)
172-
warnings.warn(message, FutureWarning, stacklevel=1)
173-
174-
# The key should still be set, regardless of whether we see the warning.
175-
self["xrootd_handler"] = uproot.source.xrootd.XRootDSource
176-
177-
return dict.__getitem__(self, where)
178-
179-
180-
open.defaults = _OpenDefaults(
181-
{
182-
"handler": None, # To be updated to fsspec source
183-
"file_handler": None, # Deprecated
184-
"s3_handler": None, # Deprecated
185-
"http_handler": None, # Deprecated
186-
"object_handler": None, # Deprecated
187-
"xrootd_handler": None, # Deprecated
188-
"timeout": 30,
189-
"max_num_elements": None,
190-
"num_workers": 1,
191-
"use_threads": sys.platform != "emscripten",
192-
"num_fallback_workers": 10,
193-
"begin_chunk_size": 403, # the smallest a ROOT file can be
194-
"minimal_ttree_metadata": True,
195-
}
196-
)
168+
def __init__(self):
169+
raise NotImplementedError # kept for backwards compatibility
170+
197171

198172
must_be_attached = [
199173
"TROOT",
@@ -537,11 +511,6 @@ class ReadOnlyFile(CommonFileMethods):
537511
Options (type; default):
538512
539513
* handler (:doc:`uproot.source.chunk.Source` class; None)
540-
* file_handler (:doc:`uproot.source.chunk.Source` class; None) (Deprecated: Use `handler` instead. If set, this will take precedence over `handler`)
541-
* xrootd_handler (:doc:`uproot.source.chunk.Source` class; None) (Deprecated: Use `handler` instead. If set, this will take precedence over `handler`)
542-
* s3_handler (:doc:`uproot.source.chunk.Source` class; None) (Deprecated: Use `handler` instead. If set, this will take precedence over `handler`)
543-
* http_handler (:doc:`uproot.source.chunk.Source` class; None) (Deprecated: Use `handler` instead. If set, this will take precedence over `handler`)
544-
* object_handler (:doc:`uproot.source.chunk.Source` class; None) (Deprecated: Use `handler` instead. If set, this will take precedence over `handler`)
545514
* timeout (float for HTTP, int for XRootD; 30)
546515
* max_num_elements (None or int; None)
547516
* num_workers (int; 1)
@@ -572,7 +541,7 @@ def __init__(
572541
self.decompression_executor = decompression_executor
573542
self.interpretation_executor = interpretation_executor
574543

575-
self._options = _OpenDefaults(open.defaults)
544+
self._options = open.defaults.copy()
576545
self._options.update(options)
577546
for option in ["begin_chunk_size"]:
578547
self._options[option] = uproot._util.memory_size(self._options[option])
@@ -582,10 +551,10 @@ def __init__(
582551

583552
self.hook_before_create_source()
584553

585-
Source, file_path = uproot._util.file_path_to_source_class(
554+
source_cls, file_path = uproot._util.file_path_to_source_class(
586555
file_path, self._options
587556
)
588-
self._source = Source(file_path, **self._options)
557+
self._source = source_cls(file_path, **self._options)
589558

590559
self.hook_before_get_chunks()
591560

tests/test_0017_multi_basket_multi_branch_fetch.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -310,6 +310,7 @@ def test_cache():
310310
skhep_testdata.data_path("uproot-sample-6.20.04-uncompressed.root"),
311311
object_cache=100,
312312
array_cache="100 MB",
313+
handler=uproot.source.file.MemmapSource,
313314
) as f:
314315
assert f.cache_key == "db4be408-93ad-11ea-9027-d201a8c0beef:/"
315316
assert f["sample"].cache_key == "db4be408-93ad-11ea-9027-d201a8c0beef:/sample;1"

tests/test_0066_fix_http_fallback_freeze.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,13 @@
11
# BSD 3-Clause License; see https://github.com/scikit-hep/uproot5/blob/main/LICENSE
22

3-
import numpy
43
import pytest
54

65
import uproot
76

87

98
@pytest.mark.network
109
def test():
10+
pytest.importorskip("aiohttp")
1111
with uproot.open(
1212
{"http://scikit-hep.org/uproot3/examples/HZZ.root": "events"}
1313
) as t:

0 commit comments

Comments
 (0)