s464967/3RNN
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
main
3RNN/Lib/site-packages/sklearn/datasets/_openml.py

1157 lines
40 KiB
Python
Raw Permalink Normal View History

1.0
2024-05-26 19:49:15 +02:00
import gzip
import hashlib
import json
import os
import shutil
import time
from contextlib import closing
from functools import wraps
from os.path import join
from tempfile import TemporaryDirectory
from typing import Any, Callable, Dict, List, Optional, Tuple, Union
from urllib.error import HTTPError, URLError
from urllib.request import Request, urlopen
from warnings import warn
import numpy as np
from ..utils import Bunch
from ..utils._optional_dependencies import check_pandas_support # noqa
from ..utils._param_validation import (
Integral,
Interval,
Real,
StrOptions,
validate_params,
)
from . import get_data_home
from ._arff_parser import load_arff_from_gzip_file
__all__ = ["fetch_openml"]
_OPENML_PREFIX = "https://api.openml.org/"
_SEARCH_NAME = "api/v1/json/data/list/data_name/{}/limit/2"
_DATA_INFO = "api/v1/json/data/{}"
_DATA_FEATURES = "api/v1/json/data/features/{}"
_DATA_QUALITIES = "api/v1/json/data/qualities/{}"
_DATA_FILE = "data/v1/download/{}"
OpenmlQualitiesType = List[Dict[str, str]]
OpenmlFeaturesType = List[Dict[str, str]]
def _get_local_path(openml_path: str, data_home: str) -> str:
return os.path.join(data_home, "openml.org", openml_path + ".gz")
def _retry_with_clean_cache(
openml_path: str,
data_home: Optional[str],
no_retry_exception: Optional[Exception] = None,
) -> Callable:
"""If the first call to the decorated function fails, the local cached
file is removed, and the function is called again. If ``data_home`` is
``None``, then the function is called once. We can provide a specific
exception to not retry on using `no_retry_exception` parameter.
"""
def decorator(f):
@wraps(f)
def wrapper(*args, **kw):
if data_home is None:
return f(*args, **kw)
try:
return f(*args, **kw)
except URLError:
raise
except Exception as exc:
if no_retry_exception is not None and isinstance(
exc, no_retry_exception
):
raise
warn("Invalid cache, redownloading file", RuntimeWarning)
local_path = _get_local_path(openml_path, data_home)
if os.path.exists(local_path):
os.unlink(local_path)
return f(*args, **kw)
return wrapper
return decorator
def _retry_on_network_error(
n_retries: int = 3, delay: float = 1.0, url: str = ""
) -> Callable:
"""If the function call results in a network error, call the function again
up to ``n_retries`` times with a ``delay`` between each call. If the error
has a 412 status code, don't call the function again as this is a specific
OpenML error.
The url parameter is used to give more information to the user about the
error.
"""
def decorator(f):
@wraps(f)
def wrapper(*args, **kwargs):
retry_counter = n_retries
while True:
try:
return f(*args, **kwargs)
except (URLError, TimeoutError) as e:
# 412 is a specific OpenML error code.
if isinstance(e, HTTPError) and e.code == 412:
raise
if retry_counter == 0:
raise
warn(
f"A network error occurred while downloading {url}. Retrying..."
)
retry_counter -= 1
time.sleep(delay)
return wrapper
return decorator
def _open_openml_url(
openml_path: str, data_home: Optional[str], n_retries: int = 3, delay: float = 1.0
):
"""
Returns a resource from OpenML.org. Caches it to data_home if required.
Parameters
----------
openml_path : str
OpenML URL that will be accessed. This will be prefixes with
_OPENML_PREFIX.
data_home : str
Directory to which the files will be cached. If None, no caching will
be applied.
n_retries : int, default=3
Number of retries when HTTP errors are encountered. Error with status
code 412 won't be retried as they represent OpenML generic errors.
delay : float, default=1.0
Number of seconds between retries.
Returns
-------
result : stream
A stream to the OpenML resource.
"""
def is_gzip_encoded(_fsrc):
return _fsrc.info().get("Content-Encoding", "") == "gzip"
req = Request(_OPENML_PREFIX + openml_path)
req.add_header("Accept-encoding", "gzip")
if data_home is None:
fsrc = _retry_on_network_error(n_retries, delay, req.full_url)(urlopen)(req)
if is_gzip_encoded(fsrc):
return gzip.GzipFile(fileobj=fsrc, mode="rb")
return fsrc
local_path = _get_local_path(openml_path, data_home)
dir_name, file_name = os.path.split(local_path)
if not os.path.exists(local_path):
os.makedirs(dir_name, exist_ok=True)
try:
# Create a tmpdir as a subfolder of dir_name where the final file will
# be moved to if the download is successful. This guarantees that the
# renaming operation to the final location is atomic to ensure the
# concurrence safety of the dataset caching mechanism.
with TemporaryDirectory(dir=dir_name) as tmpdir:
with closing(
_retry_on_network_error(n_retries, delay, req.full_url)(urlopen)(
req
)
) as fsrc:
opener: Callable
if is_gzip_encoded(fsrc):
opener = open
else:
opener = gzip.GzipFile
with opener(os.path.join(tmpdir, file_name), "wb") as fdst:
shutil.copyfileobj(fsrc, fdst)
shutil.move(fdst.name, local_path)
except Exception:
if os.path.exists(local_path):
os.unlink(local_path)
raise
# XXX: First time, decompression will not be necessary (by using fsrc), but
# it will happen nonetheless
return gzip.GzipFile(local_path, "rb")
class OpenMLError(ValueError):
"""HTTP 412 is a specific OpenML error code, indicating a generic error"""
pass
def _get_json_content_from_openml_api(
url: str,
error_message: Optional[str],
data_home: Optional[str],
n_retries: int = 3,
delay: float = 1.0,
) -> Dict:
"""
Loads json data from the openml api.
Parameters
----------
url : str
The URL to load from. Should be an official OpenML endpoint.
error_message : str or None
The error message to raise if an acceptable OpenML error is thrown
(acceptable error is, e.g., data id not found. Other errors, like 404's
will throw the native error message).
data_home : str or None
Location to cache the response. None if no cache is required.
n_retries : int, default=3
Number of retries when HTTP errors are encountered. Error with status
code 412 won't be retried as they represent OpenML generic errors.
delay : float, default=1.0
Number of seconds between retries.
Returns
-------
json_data : json
the json result from the OpenML server if the call was successful.
An exception otherwise.
"""
@_retry_with_clean_cache(url, data_home=data_home)
def _load_json():
with closing(
_open_openml_url(url, data_home, n_retries=n_retries, delay=delay)
) as response:
return json.loads(response.read().decode("utf-8"))
try:
return _load_json()
except HTTPError as error:
# 412 is an OpenML specific error code, indicating a generic error
# (e.g., data not found)
if error.code != 412:
raise error
# 412 error, not in except for nicer traceback
raise OpenMLError(error_message)
def _get_data_info_by_name(
name: str,
version: Union[int, str],
data_home: Optional[str],
n_retries: int = 3,
delay: float = 1.0,
):
"""
Utilizes the openml dataset listing api to find a dataset by
name/version
OpenML api function:
https://www.openml.org/api_docs#!/data/get_data_list_data_name_data_name
Parameters
----------
name : str
name of the dataset
version : int or str
If version is an integer, the exact name/version will be obtained from
OpenML. If version is a string (value: "active") it will take the first
version from OpenML that is annotated as active. Any other string
values except "active" are treated as integer.
data_home : str or None
Location to cache the response. None if no cache is required.
n_retries : int, default=3
Number of retries when HTTP errors are encountered. Error with status
code 412 won't be retried as they represent OpenML generic errors.
delay : float, default=1.0
Number of seconds between retries.
Returns
-------
first_dataset : json
json representation of the first dataset object that adhired to the
search criteria
"""
if version == "active":
# situation in which we return the oldest active version
url = _SEARCH_NAME.format(name) + "/status/active/"
error_msg = "No active dataset {} found.".format(name)
json_data = _get_json_content_from_openml_api(
url,
error_msg,
data_home=data_home,
n_retries=n_retries,
delay=delay,
)
res = json_data["data"]["dataset"]
if len(res) > 1:
first_version = version = res[0]["version"]
warning_msg = (
"Multiple active versions of the dataset matching the name"
f" {name} exist. Versions may be fundamentally different, "
f"returning version {first_version}. "
"Available versions:\n"
)
for r in res:
warning_msg += f"- version {r['version']}, status: {r['status']}\n"
warning_msg += (
f" url: https://www.openml.org/search?type=data&id={r['did']}\n"
)
warn(warning_msg)
return res[0]
# an integer version has been provided
url = (_SEARCH_NAME + "/data_version/{}").format(name, version)
try:
json_data = _get_json_content_from_openml_api(
url,
error_message=None,
data_home=data_home,
n_retries=n_retries,
delay=delay,
)
except OpenMLError:
# we can do this in 1 function call if OpenML does not require the
# specification of the dataset status (i.e., return datasets with a
# given name / version regardless of active, deactivated, etc. )
# TODO: feature request OpenML.
url += "/status/deactivated"
error_msg = "Dataset {} with version {} not found.".format(name, version)
json_data = _get_json_content_from_openml_api(
url,
error_msg,
data_home=data_home,
n_retries=n_retries,
delay=delay,
)
return json_data["data"]["dataset"][0]
def _get_data_description_by_id(
data_id: int,
data_home: Optional[str],
n_retries: int = 3,
delay: float = 1.0,
) -> Dict[str, Any]:
# OpenML API function: https://www.openml.org/api_docs#!/data/get_data_id
url = _DATA_INFO.format(data_id)
error_message = "Dataset with data_id {} not found.".format(data_id)
json_data = _get_json_content_from_openml_api(
url,
error_message,
data_home=data_home,
n_retries=n_retries,
delay=delay,
)
return json_data["data_set_description"]
def _get_data_features(
data_id: int,
data_home: Optional[str],
n_retries: int = 3,
delay: float = 1.0,
) -> OpenmlFeaturesType:
# OpenML function:
# https://www.openml.org/api_docs#!/data/get_data_features_id
url = _DATA_FEATURES.format(data_id)
error_message = "Dataset with data_id {} not found.".format(data_id)
json_data = _get_json_content_from_openml_api(
url,
error_message,
data_home=data_home,
n_retries=n_retries,
delay=delay,
)
return json_data["data_features"]["feature"]
def _get_data_qualities(
data_id: int,
data_home: Optional[str],
n_retries: int = 3,
delay: float = 1.0,
) -> OpenmlQualitiesType:
# OpenML API function:
# https://www.openml.org/api_docs#!/data/get_data_qualities_id
url = _DATA_QUALITIES.format(data_id)
error_message = "Dataset with data_id {} not found.".format(data_id)
json_data = _get_json_content_from_openml_api(
url,
error_message,
data_home=data_home,
n_retries=n_retries,
delay=delay,
)
# the qualities might not be available, but we still try to process
# the data
return json_data.get("data_qualities", {}).get("quality", [])
def _get_num_samples(data_qualities: OpenmlQualitiesType) -> int:
"""Get the number of samples from data qualities.
Parameters
----------
data_qualities : list of dict
Used to retrieve the number of instances (samples) in the dataset.
Returns
-------
n_samples : int
The number of samples in the dataset or -1 if data qualities are
unavailable.
"""
# If the data qualities are unavailable, we return -1
default_n_samples = -1
qualities = {d["name"]: d["value"] for d in data_qualities}
return int(float(qualities.get("NumberOfInstances", default_n_samples)))
def _load_arff_response(
url: str,
data_home: Optional[str],
parser: str,
output_type: str,
openml_columns_info: dict,
feature_names_to_select: List[str],
target_names_to_select: List[str],
shape: Optional[Tuple[int, int]],
md5_checksum: str,
n_retries: int = 3,
delay: float = 1.0,
read_csv_kwargs: Optional[Dict] = None,
):
"""Load the ARFF data associated with the OpenML URL.
In addition of loading the data, this function will also check the
integrity of the downloaded file from OpenML using MD5 checksum.
Parameters
----------
url : str
The URL of the ARFF file on OpenML.
data_home : str
The location where to cache the data.
parser : {"liac-arff", "pandas"}
The parser used to parse the ARFF file.
output_type : {"numpy", "pandas", "sparse"}
The type of the arrays that will be returned. The possibilities are:
- `"numpy"`: both `X` and `y` will be NumPy arrays;
- `"sparse"`: `X` will be sparse matrix and `y` will be a NumPy array;
- `"pandas"`: `X` will be a pandas DataFrame and `y` will be either a
pandas Series or DataFrame.
openml_columns_info : dict
The information provided by OpenML regarding the columns of the ARFF
file.
feature_names_to_select : list of str
The list of the features to be selected.
target_names_to_select : list of str
The list of the target variables to be selected.
shape : tuple or None
With `parser="liac-arff"`, when using a generator to load the data,
one needs to provide the shape of the data beforehand.
md5_checksum : str
The MD5 checksum provided by OpenML to check the data integrity.
n_retries : int, default=3
The number of times to retry downloading the data if it fails.
delay : float, default=1.0
The delay between two consecutive downloads in seconds.
read_csv_kwargs : dict, default=None
Keyword arguments to pass to `pandas.read_csv` when using the pandas parser.
It allows to overwrite the default options.
.. versionadded:: 1.3
Returns
-------
X : {ndarray, sparse matrix, dataframe}
The data matrix.
y : {ndarray, dataframe, series}
The target.
frame : dataframe or None
A dataframe containing both `X` and `y`. `None` if
`output_array_type != "pandas"`.
categories : list of str or None
The names of the features that are categorical. `None` if
`output_array_type == "pandas"`.
"""
gzip_file = _open_openml_url(url, data_home, n_retries=n_retries, delay=delay)
with closing(gzip_file):
md5 = hashlib.md5()
for chunk in iter(lambda: gzip_file.read(4096), b""):
md5.update(chunk)
actual_md5_checksum = md5.hexdigest()
if actual_md5_checksum != md5_checksum:
raise ValueError(
f"md5 checksum of local file for {url} does not match description: "
f"expected: {md5_checksum} but got {actual_md5_checksum}. "
"Downloaded file could have been modified / corrupted, clean cache "
"and retry..."
)
def _open_url_and_load_gzip_file(url, data_home, n_retries, delay, arff_params):
gzip_file = _open_openml_url(url, data_home, n_retries=n_retries, delay=delay)
with closing(gzip_file):
return load_arff_from_gzip_file(gzip_file, **arff_params)
arff_params: Dict = dict(
parser=parser,
output_type=output_type,
openml_columns_info=openml_columns_info,
feature_names_to_select=feature_names_to_select,
target_names_to_select=target_names_to_select,
shape=shape,
read_csv_kwargs=read_csv_kwargs or {},
)
try:
X, y, frame, categories = _open_url_and_load_gzip_file(
url, data_home, n_retries, delay, arff_params
)
except Exception as exc:
if parser != "pandas":
raise
from pandas.errors import ParserError
if not isinstance(exc, ParserError):
raise
# A parsing error could come from providing the wrong quotechar
# to pandas. By default, we use a double quote. Thus, we retry
# with a single quote before to raise the error.
arff_params["read_csv_kwargs"].update(quotechar="'")
X, y, frame, categories = _open_url_and_load_gzip_file(
url, data_home, n_retries, delay, arff_params
)
return X, y, frame, categories
def _download_data_to_bunch(
url: str,
sparse: bool,
data_home: Optional[str],
*,
as_frame: bool,
openml_columns_info: List[dict],
data_columns: List[str],
target_columns: List[str],
shape: Optional[Tuple[int, int]],
md5_checksum: str,
n_retries: int = 3,
delay: float = 1.0,
parser: str,
read_csv_kwargs: Optional[Dict] = None,
):
"""Download ARFF data, load it to a specific container and create to Bunch.
This function has a mechanism to retry/cache/clean the data.
Parameters
----------
url : str
The URL of the ARFF file on OpenML.
sparse : bool
Whether the dataset is expected to use the sparse ARFF format.
data_home : str
The location where to cache the data.
as_frame : bool
Whether or not to return the data into a pandas DataFrame.
openml_columns_info : list of dict
The information regarding the columns provided by OpenML for the
ARFF dataset. The information is stored as a list of dictionaries.
data_columns : list of str
The list of the features to be selected.
target_columns : list of str
The list of the target variables to be selected.
shape : tuple or None
With `parser="liac-arff"`, when using a generator to load the data,
one needs to provide the shape of the data beforehand.
md5_checksum : str
The MD5 checksum provided by OpenML to check the data integrity.
n_retries : int, default=3
Number of retries when HTTP errors are encountered. Error with status
code 412 won't be retried as they represent OpenML generic errors.
delay : float, default=1.0
Number of seconds between retries.
parser : {"liac-arff", "pandas"}
The parser used to parse the ARFF file.
read_csv_kwargs : dict, default=None
Keyword arguments to pass to `pandas.read_csv` when using the pandas parser.
It allows to overwrite the default options.
.. versionadded:: 1.3
Returns
-------
data : :class:`~sklearn.utils.Bunch`
Dictionary-like object, with the following attributes.
X : {ndarray, sparse matrix, dataframe}
The data matrix.
y : {ndarray, dataframe, series}
The target.
frame : dataframe or None
A dataframe containing both `X` and `y`. `None` if
`output_array_type != "pandas"`.
categories : list of str or None
The names of the features that are categorical. `None` if
`output_array_type == "pandas"`.
"""
# Prepare which columns and data types should be returned for the X and y
features_dict = {feature["name"]: feature for feature in openml_columns_info}
if sparse:
output_type = "sparse"
elif as_frame:
output_type = "pandas"
else:
output_type = "numpy"
# XXX: target columns should all be categorical or all numeric
_verify_target_data_type(features_dict, target_columns)
for name in target_columns:
column_info = features_dict[name]
n_missing_values = int(column_info["number_of_missing_values"])
if n_missing_values > 0:
raise ValueError(
f"Target column '{column_info['name']}' has {n_missing_values} missing "
"values. Missing values are not supported for target columns."
)
no_retry_exception = None
if parser == "pandas":
# If we get a ParserError with pandas, then we don't want to retry and we raise
# early.
from pandas.errors import ParserError
no_retry_exception = ParserError
X, y, frame, categories = _retry_with_clean_cache(
url, data_home, no_retry_exception
)(_load_arff_response)(
url,
data_home,
parser=parser,
output_type=output_type,
openml_columns_info=features_dict,
feature_names_to_select=data_columns,
target_names_to_select=target_columns,
shape=shape,
md5_checksum=md5_checksum,
n_retries=n_retries,
delay=delay,
read_csv_kwargs=read_csv_kwargs,
)
return Bunch(
data=X,
target=y,
frame=frame,
categories=categories,
feature_names=data_columns,
target_names=target_columns,
)
def _verify_target_data_type(features_dict, target_columns):
# verifies the data type of the y array in case there are multiple targets
# (throws an error if these targets do not comply with sklearn support)
if not isinstance(target_columns, list):
raise ValueError("target_column should be list, got: %s" % type(target_columns))
found_types = set()
for target_column in target_columns:
if target_column not in features_dict:
raise KeyError(f"Could not find target_column='{target_column}'")
if features_dict[target_column]["data_type"] == "numeric":
found_types.add(np.float64)
else:
found_types.add(object)
# note: we compare to a string, not boolean
if features_dict[target_column]["is_ignore"] == "true":
warn(f"target_column='{target_column}' has flag is_ignore.")
if features_dict[target_column]["is_row_identifier"] == "true":
warn(f"target_column='{target_column}' has flag is_row_identifier.")
if len(found_types) > 1:
raise ValueError(
"Can only handle homogeneous multi-target datasets, "
"i.e., all targets are either numeric or "
"categorical."
)
def _valid_data_column_names(features_list, target_columns):
# logic for determining on which columns can be learned. Note that from the
# OpenML guide follows that columns that have the `is_row_identifier` or
# `is_ignore` flag, these can not be learned on. Also target columns are
# excluded.
valid_data_column_names = []
for feature in features_list:
if (
feature["name"] not in target_columns
and feature["is_ignore"] != "true"
and feature["is_row_identifier"] != "true"
):
valid_data_column_names.append(feature["name"])
return valid_data_column_names
@validate_params(
{
"name": [str, None],
"version": [Interval(Integral, 1, None, closed="left"), StrOptions({"active"})],
"data_id": [Interval(Integral, 1, None, closed="left"), None],
"data_home": [str, os.PathLike, None],
"target_column": [str, list, None],
"cache": [bool],
"return_X_y": [bool],
"as_frame": [bool, StrOptions({"auto"})],
"n_retries": [Interval(Integral, 1, None, closed="left")],
"delay": [Interval(Real, 0.0, None, closed="neither")],
"parser": [
StrOptions({"auto", "pandas", "liac-arff"}),
],
"read_csv_kwargs": [dict, None],
},
prefer_skip_nested_validation=True,
)
def fetch_openml(
name: Optional[str] = None,
*,
version: Union[str, int] = "active",
data_id: Optional[int] = None,
data_home: Optional[Union[str, os.PathLike]] = None,
target_column: Optional[Union[str, List]] = "default-target",
cache: bool = True,
return_X_y: bool = False,
as_frame: Union[str, bool] = "auto",
n_retries: int = 3,
delay: float = 1.0,
parser: str = "auto",
read_csv_kwargs: Optional[Dict] = None,
):
"""Fetch dataset from openml by name or dataset id.
Datasets are uniquely identified by either an integer ID or by a
combination of name and version (i.e. there might be multiple
versions of the 'iris' dataset). Please give either name or data_id
(not both). In case a name is given, a version can also be
provided.
Read more in the :ref:`User Guide <openml>`.
.. versionadded:: 0.20
.. note:: EXPERIMENTAL
The API is experimental (particularly the return value structure),
and might have small backward-incompatible changes without notice
or warning in future releases.
Parameters
----------
name : str, default=None
String identifier of the dataset. Note that OpenML can have multiple
datasets with the same name.
version : int or 'active', default='active'
Version of the dataset. Can only be provided if also ``name`` is given.
If 'active' the oldest version that's still active is used. Since
there may be more than one active version of a dataset, and those
versions may fundamentally be different from one another, setting an
exact version is highly recommended.
data_id : int, default=None
OpenML ID of the dataset. The most specific way of retrieving a
dataset. If data_id is not given, name (and potential version) are
used to obtain a dataset.
data_home : str or path-like, default=None
Specify another download and cache folder for the data sets. By default
all scikit-learn data is stored in '~/scikit_learn_data' subfolders.
target_column : str, list or None, default='default-target'
Specify the column name in the data to use as target. If
'default-target', the standard target column a stored on the server
is used. If ``None``, all columns are returned as data and the
target is ``None``. If list (of strings), all columns with these names
are returned as multi-target (Note: not all scikit-learn classifiers
can handle all types of multi-output combinations).
cache : bool, default=True
Whether to cache the downloaded datasets into `data_home`.
return_X_y : bool, default=False
If True, returns ``(data, target)`` instead of a Bunch object. See
below for more information about the `data` and `target` objects.
as_frame : bool or 'auto', default='auto'
If True, the data is a pandas DataFrame including columns with
appropriate dtypes (numeric, string or categorical). The target is
a pandas DataFrame or Series depending on the number of target_columns.
The Bunch will contain a ``frame`` attribute with the target and the
data. If ``return_X_y`` is True, then ``(data, target)`` will be pandas
DataFrames or Series as describe above.
If `as_frame` is 'auto', the data and target will be converted to
DataFrame or Series as if `as_frame` is set to True, unless the dataset
is stored in sparse format.
If `as_frame` is False, the data and target will be NumPy arrays and
the `data` will only contain numerical values when `parser="liac-arff"`
where the categories are provided in the attribute `categories` of the
`Bunch` instance. When `parser="pandas"`, no ordinal encoding is made.
.. versionchanged:: 0.24
The default value of `as_frame` changed from `False` to `'auto'`
in 0.24.
n_retries : int, default=3
Number of retries when HTTP errors or network timeouts are encountered.
Error with status code 412 won't be retried as they represent OpenML
generic errors.
delay : float, default=1.0
Number of seconds between retries.
parser : {"auto", "pandas", "liac-arff"}, default="auto"
Parser used to load the ARFF file. Two parsers are implemented:
- `"pandas"`: this is the most efficient parser. However, it requires
pandas to be installed and can only open dense datasets.
- `"liac-arff"`: this is a pure Python ARFF parser that is much less
memory- and CPU-efficient. It deals with sparse ARFF datasets.
If `"auto"`, the parser is chosen automatically such that `"liac-arff"`
is selected for sparse ARFF datasets, otherwise `"pandas"` is selected.
.. versionadded:: 1.2
.. versionchanged:: 1.4
The default value of `parser` changes from `"liac-arff"` to
`"auto"`.
read_csv_kwargs : dict, default=None
Keyword arguments passed to :func:`pandas.read_csv` when loading the data
from a ARFF file and using the pandas parser. It can allow to
overwrite some default parameters.
.. versionadded:: 1.3
Returns
-------
data : :class:`~sklearn.utils.Bunch`
Dictionary-like object, with the following attributes.
data : np.array, scipy.sparse.csr_matrix of floats, or pandas DataFrame
The feature matrix. Categorical features are encoded as ordinals.
target : np.array, pandas Series or DataFrame
The regression target or classification labels, if applicable.
Dtype is float if numeric, and object if categorical. If
``as_frame`` is True, ``target`` is a pandas object.
DESCR : str
The full description of the dataset.
feature_names : list
The names of the dataset columns.
target_names: list
The names of the target columns.
.. versionadded:: 0.22
categories : dict or None
Maps each categorical feature name to a list of values, such
that the value encoded as i is ith in the list. If ``as_frame``
is True, this is None.
details : dict
More metadata from OpenML.
frame : pandas DataFrame
Only present when `as_frame=True`. DataFrame with ``data`` and
``target``.
(data, target) : tuple if ``return_X_y`` is True
.. note:: EXPERIMENTAL
This interface is **experimental** and subsequent releases may
change attributes without notice (although there should only be
minor changes to ``data`` and ``target``).
Missing values in the 'data' are represented as NaN's. Missing values
in 'target' are represented as NaN's (numerical target) or None
(categorical target).
Notes
-----
The `"pandas"` and `"liac-arff"` parsers can lead to different data types
in the output. The notable differences are the following:
- The `"liac-arff"` parser always encodes categorical features as `str` objects.
To the contrary, the `"pandas"` parser instead infers the type while
reading and numerical categories will be casted into integers whenever
possible.
- The `"liac-arff"` parser uses float64 to encode numerical features
tagged as 'REAL' and 'NUMERICAL' in the metadata. The `"pandas"`
parser instead infers if these numerical features corresponds
to integers and uses panda's Integer extension dtype.
- In particular, classification datasets with integer categories are
typically loaded as such `(0, 1, ...)` with the `"pandas"` parser while
`"liac-arff"` will force the use of string encoded class labels such as
`"0"`, `"1"` and so on.
- The `"pandas"` parser will not strip single quotes - i.e. `'` - from
string columns. For instance, a string `'my string'` will be kept as is
while the `"liac-arff"` parser will strip the single quotes. For
categorical columns, the single quotes are stripped from the values.
In addition, when `as_frame=False` is used, the `"liac-arff"` parser
returns ordinally encoded data where the categories are provided in the
attribute `categories` of the `Bunch` instance. Instead, `"pandas"` returns
a NumPy array were the categories are not encoded.
Examples
--------
>>> from sklearn.datasets import fetch_openml
>>> adult = fetch_openml("adult", version=2) # doctest: +SKIP
>>> adult.frame.info() # doctest: +SKIP
<class 'pandas.core.frame.DataFrame'>
RangeIndex: 48842 entries, 0 to 48841
Data columns (total 15 columns):
# Column Non-Null Count Dtype
--- ------ -------------- -----
0 age 48842 non-null int64
1 workclass 46043 non-null category
2 fnlwgt 48842 non-null int64
3 education 48842 non-null category
4 education-num 48842 non-null int64
5 marital-status 48842 non-null category
6 occupation 46033 non-null category
7 relationship 48842 non-null category
8 race 48842 non-null category
9 sex 48842 non-null category
10 capital-gain 48842 non-null int64
11 capital-loss 48842 non-null int64
12 hours-per-week 48842 non-null int64
13 native-country 47985 non-null category
14 class 48842 non-null category
dtypes: category(9), int64(6)
memory usage: 2.7 MB
"""
if cache is False:
# no caching will be applied
data_home = None
else:
data_home = get_data_home(data_home=data_home)
data_home = join(str(data_home), "openml")
# check valid function arguments. data_id XOR (name, version) should be
# provided
if name is not None:
# OpenML is case-insensitive, but the caching mechanism is not
# convert all data names (str) to lower case
name = name.lower()
if data_id is not None:
raise ValueError(
"Dataset data_id={} and name={} passed, but you can only "
"specify a numeric data_id or a name, not "
"both.".format(data_id, name)
)
data_info = _get_data_info_by_name(
name, version, data_home, n_retries=n_retries, delay=delay
)
data_id = data_info["did"]
elif data_id is not None:
# from the previous if statement, it is given that name is None
if version != "active":
raise ValueError(
"Dataset data_id={} and version={} passed, but you can only "
"specify a numeric data_id or a version, not "
"both.".format(data_id, version)
)
else:
raise ValueError(
"Neither name nor data_id are provided. Please provide name or data_id."
)
data_description = _get_data_description_by_id(data_id, data_home)
if data_description["status"] != "active":
warn(
"Version {} of dataset {} is inactive, meaning that issues have "
"been found in the dataset. Try using a newer version from "
"this URL: {}".format(
data_description["version"],
data_description["name"],
data_description["url"],
)
)
if "error" in data_description:
warn(
"OpenML registered a problem with the dataset. It might be "
"unusable. Error: {}".format(data_description["error"])
)
if "warning" in data_description:
warn(
"OpenML raised a warning on the dataset. It might be "
"unusable. Warning: {}".format(data_description["warning"])
)
return_sparse = data_description["format"].lower() == "sparse_arff"
as_frame = not return_sparse if as_frame == "auto" else as_frame
if parser == "auto":
parser_ = "liac-arff" if return_sparse else "pandas"
else:
parser_ = parser
if parser_ == "pandas":
try:
check_pandas_support("`fetch_openml`")
except ImportError as exc:
if as_frame:
err_msg = (
"Returning pandas objects requires pandas to be installed. "
"Alternatively, explicitly set `as_frame=False` and "
"`parser='liac-arff'`."
)
else:
err_msg = (
f"Using `parser={parser!r}` wit dense data requires pandas to be "
"installed. Alternatively, explicitly set `parser='liac-arff'`."
)
raise ImportError(err_msg) from exc
if return_sparse:
if as_frame:
raise ValueError(
"Sparse ARFF datasets cannot be loaded with as_frame=True. "
"Use as_frame=False or as_frame='auto' instead."
)
if parser_ == "pandas":
raise ValueError(
f"Sparse ARFF datasets cannot be loaded with parser={parser!r}. "
"Use parser='liac-arff' or parser='auto' instead."
)
# download data features, meta-info about column types
features_list = _get_data_features(data_id, data_home)
if not as_frame:
for feature in features_list:
if "true" in (feature["is_ignore"], feature["is_row_identifier"]):
continue
if feature["data_type"] == "string":
raise ValueError(
"STRING attributes are not supported for "
"array representation. Try as_frame=True"
)
if target_column == "default-target":
# determines the default target based on the data feature results
# (which is currently more reliable than the data description;
# see issue: https://github.com/openml/OpenML/issues/768)
target_columns = [
feature["name"]
for feature in features_list
if feature["is_target"] == "true"
]
elif isinstance(target_column, str):
# for code-simplicity, make target_column by default a list
target_columns = [target_column]
elif target_column is None:
target_columns = []
else:
# target_column already is of type list
target_columns = target_column
data_columns = _valid_data_column_names(features_list, target_columns)
shape: Optional[Tuple[int, int]]
# determine arff encoding to return
if not return_sparse:
# The shape must include the ignored features to keep the right indexes
# during the arff data conversion.
data_qualities = _get_data_qualities(data_id, data_home)
shape = _get_num_samples(data_qualities), len(features_list)
else:
shape = None
# obtain the data
url = _DATA_FILE.format(data_description["file_id"])
bunch = _download_data_to_bunch(
url,
return_sparse,
data_home,
as_frame=bool(as_frame),
openml_columns_info=features_list,
shape=shape,
target_columns=target_columns,
data_columns=data_columns,
md5_checksum=data_description["md5_checksum"],
n_retries=n_retries,
delay=delay,
parser=parser_,
read_csv_kwargs=read_csv_kwargs,
)
if return_X_y:
return bunch.data, bunch.target
description = "{}\n\nDownloaded from openml.org.".format(
data_description.pop("description")
)
bunch.update(
DESCR=description,
details=data_description,
url="https://www.openml.org/d/{}".format(data_id),
)
return bunch
Reference in New Issue Copy Permalink