ó
cÂY]c           @   s(  d  d l  Z  d  d l Z d  d l Z d  d l Z d  d l m Z d  d l m Z d  d l m	 Z	 d  d l
 m Z d  d l Z d  d l Z d  d l Z y d  d l m Z m Z Wn' e k
 r× d  d l m Z m Z n Xd  d l Z d  d l Z d  d l m Z d d	 l m Z d
 d l m Z m Z m Z d
 d l m  Z  d
 d l! m" Z" d g Z# d Z$ d Z% d Z& d Z' d Z( d Z) d „  Z* d „  Z+ d „  Z, d „  Z- d „  Z. d „  Z/ d d „ Z1 d „  Z2 d „  Z3 d „  Z4 d „  Z5 d  „  Z6 e7 d! „ Z8 d" „  Z9 d# „  Z: d d$ d d d% e7 e; d& „ Z< d S('   iÿÿÿÿN(   t   join(   t   warn(   t   closing(   t   wraps(   t   urlopent   Request(   t   _arffi   (   t   get_data_homei   (   t   string_typest   PY2t   BytesIO(   t	   HTTPError(   t   Buncht   fetch_openmls   https://openml.org/s*   api/v1/json/data/list/data_name/{}/limit/2s   api/v1/json/data/{}s   api/v1/json/data/features/{}s   api/v1/json/data/qualities/{}s   data/v1/download/{}c         C   s   t  j j | d |  d ƒ S(   Ns
   openml.orgs   .gz(   t   ost   pathR    (   t   openml_patht	   data_home(    (    s6   lib/python2.7/site-packages/sklearn/datasets/openml.pyt   _get_local_path(   s    c            s   ‡  ‡ f d †  } | S(   sÃ   If the first call to the decorated function fails, the local cached
    file is removed, and the function is called again. If ``data_home`` is
    ``None``, then the function is called once.
    c            s%   t  ˆ  ƒ ‡ ‡  ‡ f d †  ƒ } | S(   Nc             sŽ   ˆ  d  k r ˆ ƒ  Sy ˆ ƒ  SWni t k
 r4 ‚  nV t k
 r‰ t j d t ƒ t ˆ ˆ  ƒ }  t j j	 |  ƒ r‚ t j
 |  ƒ n  ˆ ƒ  SXd  S(   Ns!   Invalid cache, redownloading file(   t   NoneR   t	   Exceptiont   warningsR   t   RuntimeWarningR   R   R   t   existst   unlink(   t
   local_path(   R   t   fR   (    s6   lib/python2.7/site-packages/sklearn/datasets/openml.pyt   wrapper2   s    (   R   (   R   R   (   R   R   (   R   s6   lib/python2.7/site-packages/sklearn/datasets/openml.pyt	   decorator1   s    !(    (   R   R   R   (    (   R   R   s6   lib/python2.7/site-packages/sklearn/datasets/openml.pyt   _retry_with_clean_cache,   s    c         C   s¡  d „  } t  t |  ƒ } | j d d ƒ | d k r‚ t | ƒ } | | ƒ r~ t rh t | j ƒ  ƒ } n  t j	 d | d d ƒ S| St
 |  | ƒ } t j j | ƒ s‘y t j t j j | ƒ ƒ Wn t k
 rÓ n Xy t t | ƒ ƒ i } | | ƒ r#t | d ƒ  } t j | | ƒ Wd QXn+ t j	 | d ƒ  } t j | | ƒ Wd QXWd QXWq‘t k
 rt j j | ƒ r‡t j | ƒ n  ‚  q‘Xn  t j	 | d ƒ S(	   s®  
    Returns a resource from OpenML.org. Caches it to data_home if required.

    Parameters
    ----------
    openml_path : str
        OpenML URL that will be accessed. This will be prefixes with
        _OPENML_PREFIX

    data_home : str
        Directory to which the files will be cached. If None, no caching will
        be applied.

    Returns
    -------
    result : stream
        A stream to the OpenML resource
    c         S   s   |  j  ƒ  j d d ƒ d k S(   Ns   Content-Encodingt    t   gzip(   t   infot   get(   t   _fsrc(    (    s6   lib/python2.7/site-packages/sklearn/datasets/openml.pyt   is_gzipY   s    s   Accept-encodingR   t   fileobjt   modet   rbt   wbN(   R   t   _OPENML_PREFIXt
   add_headerR   R   R	   R
   t   readR   t   GzipFileR   R   R   R   t   makedirst   dirnamet   OSErrorR   t   opent   shutilt   copyfileobjR   R   (   R   R   R#   t   reqt   fsrcR   t   fdst(    (    s6   lib/python2.7/site-packages/sklearn/datasets/openml.pyt   _open_openml_urlF   s8    	 
c            ss   t  ˆ ˆ  ƒ ‡  ‡ f d †  ƒ } y | ƒ  SWn+ t k
 rY } | j d k rZ | ‚ qZ n X| ro t | ƒ ‚ n  d S(   sü  
    Loads json data from the openml api

    Parameters
    ----------
    url : str
        The URL to load from. Should be an official OpenML endpoint

    error_message : str or None
        The error message to raise if an acceptable OpenML error is thrown
        (acceptable error is, e.g., data id not found. Other errors, like 404's
        will throw the native error message)

    raise_if_error : bool
        Whether to raise an error if OpenML returns an acceptable error (e.g.,
        date not found). If this argument is set to False, a None is returned
        in case of acceptable errors. Note that all other errors (e.g., 404)
        will still be raised as normal.

    data_home : str or None
        Location to cache the response. None if no cache is required.

    Returns
    -------
    json_data : json or None
        the json result from the OpenML server if the call was successful;
        None otherwise iff raise_if_error was set to False and the error was
        ``acceptable``
    c             s>   t  t ˆ ˆ  ƒ ƒ # }  t j |  j ƒ  j d ƒ ƒ SWd  QXd  S(   Ns   utf-8(   R   R5   t   jsont   loadsR*   t   decode(   t   response(   R   t   url(    s6   lib/python2.7/site-packages/sklearn/datasets/openml.pyt
   _load_json¡   s    iœ  N(   R   R   t   codet
   ValueErrorR   (   R:   t   error_messaget   raise_if_errorR   R;   t   error(    (   R   R:   s6   lib/python2.7/site-packages/sklearn/datasets/openml.pyt!   _get_json_content_from_openml_api   s     !c         C   sª   t  ƒ  t  ƒ  t  ƒ  f } d „  t | ƒ Dƒ } xu t |  d |  d |  d ƒ D]U \ } } } | | k rM | d j | ƒ | d j | ƒ | d j | | ƒ qM qM W| S(   s  
    obtains several columns from sparse arff representation. Additionally, the
    column indices are re-labelled, given the columns that are not included.
    (e.g., when including [1, 2, 3], the columns will be relabelled to
    [0, 1, 2])

    Parameters
    ----------
    arff_data : tuple
        A tuple of three lists of equal size; first list indicating the value,
        second the x coordinate and the third the y coordinate.

    include_columns : list
        A list of columns to include.

    Returns
    -------
    arff_data_new : tuple
        Subset of arff data with only the include columns indicated by the
        include_columns argument.
    c         S   s   i  |  ] \ } } | | “ q S(    (    (   t   .0t	   array_idxt
   column_idx(    (    s6   lib/python2.7/site-packages/sklearn/datasets/openml.pys
   <dictcomp>Ë   s   	 i    i   i   (   t   listt	   enumeratet   zipt   append(   t	   arff_datat   include_columnst   arff_data_newt   reindexed_columnst   valt   row_idxt   col_idx(    (    s6   lib/python2.7/site-packages/sklearn/datasets/openml.pyt   _split_sparse_columns´   s    .c   	      C   s­   t  |  d ƒ d } | t | ƒ f } d „  t | ƒ Dƒ } t j | d t j ƒ} xR t |  d |  d |  d ƒ D]2 \ } } } | | k rs | | | | | f <qs qs W| S(   Ni   c         S   s   i  |  ] \ } } | | “ q S(    (    (   RB   RC   RD   (    (    s6   lib/python2.7/site-packages/sklearn/datasets/openml.pys
   <dictcomp>Ú   s   	 t   dtypei    i   (   t   maxt   lenRF   t   npt   emptyt   float64RG   (	   RI   RJ   t   num_obst   y_shapeRL   t   yRM   RN   RO   (    (    s6   lib/python2.7/site-packages/sklearn/datasets/openml.pyt   _sparse_data_to_arrayÕ   s    .c         C   sX  t  j |  ƒ r¦ | d d k r( d } n | d | d } t j t j j |  ƒ d d d | ƒ} | j | Œ  } | d d … | f } | d d … | f } | | f St |  t	 ƒ rHt
 |  | ƒ } t |  d ƒ d }	 |	 t | ƒ f }
 t j j | d | d | d f f d	 |
 d t j ƒ} | j ƒ  } t |  | ƒ } | | f St d
 ƒ ‚ d S(   sr  
    converts the arff object into the appropriate matrix type (np.array or
    scipy.sparse.csr_matrix) based on the 'data part' (i.e., in the
    liac-arff dict, the object from the 'data' key)

    Parameters
    ----------
    arff_data : list or dict
        as obtained from liac-arff object

    col_slice_x : list
        The column indices that are sliced from the original array to return
        as X data

    col_slice_y : list
        The column indices that are sliced from the original array to return
        as y data

    Returns
    -------
    X : np.array or scipy.sparse.csr_matrix
    y : np.array
    i    iÿÿÿÿi   RQ   RV   t   countNi   t   shapes(   Unexpected Data Type obtained from arff.(   t   inspectt   isgeneratorRT   t   fromitert	   itertoolst   chaint   from_iterablet   reshapet
   isinstancet   tupleRP   RR   RS   t   scipyt   sparset
   coo_matrixRV   t   tocsrRZ   R=   (   RI   t   col_slice_xt   col_slice_yR\   R[   t   datat   XRY   t   arff_data_XRW   t   X_shape(    (    s6   lib/python2.7/site-packages/sklearn/datasets/openml.pyt   _convert_arff_dataä   s*    	
	
c         C   s  | d k r’ t  j |  ƒ d } d j |  ƒ } t | | t | ƒ } | d d } t | ƒ d k rŠ t d j d |  d	 | d
 d	 ƒ ƒ n  | d
 St  d j |  | ƒ } t | d t | ƒ } | d k rý | d 7} d j |  | ƒ } t | | t | ƒ } n  | d d d
 S(   s;  
    Utilizes the openml dataset listing api to find a dataset by
    name/version
    OpenML api function:
    https://www.openml.org/api_docs#!/data/get_data_list_data_name_data_name

    Parameters
    ----------
    name : str
        name of the dataset

    version : int or str
        If version is an integer, the exact name/version will be obtained from
        OpenML. If version is a string (value: "active") it will take the first
        version from OpenML that is annotated as active. Any other string
        values except "active" are treated as integer.

    data_home : str or None
        Location to cache the response. None if no cache is required.

    Returns
    -------
    first_dataset : json
        json representation of the first dataset object that adhired to the
        search criteria

    t   actives   /status/active/s   No active dataset {} found.Rl   t   dataseti   s   Multiple active versions of the dataset matching the name {name} exist. Versions may be fundamentally different, returning version {version}.t   namet   versioni    s   /data_version/{}s   /status/deactivateds%   Dataset {} with version {} not found.N(   t   _SEARCH_NAMEt   formatRA   t   TrueRS   R   R   t   False(   Rs   Rt   R   R:   t	   error_msgt	   json_datat   res(    (    s6   lib/python2.7/site-packages/sklearn/datasets/openml.pyt   _get_data_info_by_name  s(    		
		c         C   s;   t  j |  ƒ } d j |  ƒ } t | | t | ƒ } | d S(   Ns"   Dataset with data_id {} not found.t   data_set_description(   t
   _DATA_INFORv   RA   Rw   (   t   data_idR   R:   R>   Rz   (    (    s6   lib/python2.7/site-packages/sklearn/datasets/openml.pyt   _get_data_description_by_idR  s
    	c         C   s?   t  j |  ƒ } d j |  ƒ } t | | t | ƒ } | d d S(   Ns"   Dataset with data_id {} not found.t   data_featurest   feature(   t   _DATA_FEATURESRv   RA   Rw   (   R   R   R:   R>   Rz   (    (    s6   lib/python2.7/site-packages/sklearn/datasets/openml.pyt   _get_data_features[  s
    	c         C   s\   t  j |  ƒ } d j |  ƒ } t | | t | ƒ } y | d d SWn t k
 rW d  SXd  S(   Ns"   Dataset with data_id {} not found.t   data_qualitiest   quality(   t   _DATA_QUALITIESRv   RA   Rw   t   KeyErrorR   (   R   R   R:   R>   Rz   (    (    s6   lib/python2.7/site-packages/sklearn/datasets/openml.pyt   _get_data_qualitiese  s    	c         C   sg   |  d  k r d  Sd „  |  Dƒ } y. t t | d ƒ ƒ t t | d ƒ ƒ f SWn t k
 rb d  SXd  S(   Nc         S   s!   i  |  ] } | d  | d “ q S(   t   valueRs   (    (   RB   t   d(    (    s6   lib/python2.7/site-packages/sklearn/datasets/openml.pys
   <dictcomp>y  s   	 t   NumberOfInstancest   NumberOfFeatures(   R   t   intt   floatt   AttributeError(   R…   t	   qualities(    (    s6   lib/python2.7/site-packages/sklearn/datasets/openml.pyt   _get_data_shapet  s    c            s=   t  j |  ƒ ‰ t ˆ ˆ  ƒ ‡  ‡ ‡ ‡ f d †  ƒ } | ƒ  S(   Nc       
      s—   t  t ˆ ˆ  ƒ ƒ | }  ˆ t k r0 t j } n	 t j } t rc t j |  j ƒ  d ˆ d | ƒ} n* t j	 |  j ƒ  j
 d ƒ d ˆ d | ƒ} Wd  QX| S(   Nt   encode_nominalt   return_types   utf-8(   R   R5   Rw   R   t   COOt	   DENSE_GENR	   t   loadR*   R7   R8   (   R9   R”   t	   arff_file(   R   R“   Rg   R:   (    s6   lib/python2.7/site-packages/sklearn/datasets/openml.pyt
   _arff_loadˆ  s    	(   t
   _DATA_FILERv   R   (   t   file_idRg   R   R“   R™   (    (   R   R“   Rg   R:   s6   lib/python2.7/site-packages/sklearn/datasets/openml.pyt   _download_data_arff  s    'c         C   s
  t  | t ƒ s( t d t | ƒ ƒ ‚ n  t ƒ  } x± | D]© } | |  k rY t d ƒ ‚ n  |  | d d k r€ | j t j ƒ n | j t	 ƒ |  | d d k r· t
 d j | ƒ ƒ n  |  | d d k r8 t
 d	 j | ƒ ƒ q8 q8 Wt | ƒ d
 k rt d ƒ ‚ n  d  S(   Ns%   target_column should be list, got: %ss   Could not find target_column={}t	   data_typet   numerict	   is_ignoret   trues$   target_column={} has flag is_ignore.t   is_row_identifiers,   target_column={} has flag is_row_identifier.i   sg   Can only handle homogeneous multi-target datasets, i.e., all targets are either numeric or categorical.(   Rd   RE   R=   t   typet   setRˆ   t   addRT   RV   t   objectR   Rv   RS   (   t   features_dictt   target_columnst   found_typest   target_column(    (    s6   lib/python2.7/site-packages/sklearn/datasets/openml.pyt   _verify_target_data_typeŸ  s$    			c         C   s_   g  } xR |  D]J } | d | k r | d d k r | d d k r | j  | d ƒ q q W| S(   NRs   RŸ   R    R¡   (   RH   (   t   features_listR§   t   valid_data_column_namesR‚   (    (    s6   lib/python2.7/site-packages/sklearn/datasets/openml.pyt   _valid_data_column_names»  s    Rq   s   default-targetc            sW  t  d | ƒ } t | d ƒ } | t k r3 d) } n  |  d) k	 r‘ |  j ƒ  }  | d) k	 rr t d j | |  ƒ ƒ ‚ n  t |  | | ƒ } | d } nB | d) k	 rÇ | d k rÓ t d j | |  ƒ ƒ ‚ qÓ n t d ƒ ‚ t | | ƒ } | d d k rt	 d	 j | d
 | d | d ƒ ƒ n  d | k r@t	 d j | d ƒ ƒ n  d | k rft	 d j | d ƒ ƒ n  t
 | | ƒ }	 xM |	 D]E }
 d |
 d |
 d f k r¢q|n  |
 d d k r|t d ƒ ‚ q|q|Wˆ d k rg  |	 D]  }
 |
 d d k rØ|
 d ^ qØ‰ n] t ˆ t ƒ rˆ g ‰ nB ˆ d) k r1g  ‰ n- t ˆ t ƒ s^t d j t ˆ ƒ ƒ ƒ ‚ n  t |	 ˆ ƒ ‰  d „  |	 Dƒ } t | ˆ ƒ g  ˆ D] } t | | d ƒ ^ q‘} g  ˆ  D] } t | | d ƒ ^ q¸} xS | D]K } |	 | } t | d ƒ } | d k rßt d j | d | ƒ ƒ ‚ qßqßWt } | d j ƒ  d  k rSt } n  | s˜t | | ƒ } t | ƒ } | d) k ržd! t |	 ƒ f } qžn d) } t | d" | | ƒ } ‡  ‡ f d# †  | d$ Dƒ ‰ t | d% | | | ƒ \ } } ‡ f d& †  ˆ Dƒ } | snŸ t | ƒ r“t j g  t ˆ ƒ D]Y \ } } t j t j ˆ j | ƒ d' d( ƒ| d) d) … | | d* … f j  t ƒ ƒ ^ q.ƒ } n t! | ƒ r®t d+ ƒ ‚ n  d, j | j d- ƒ ƒ } | j" d* d* k rë| j# d4 ƒ } n | j" d* d k rd) } n  | r| | f St$ d% | d. | d/ ˆ  d0 | d1 | d2 ˆ d d3 j | ƒ ƒ } | S(5   s&  Fetch dataset from openml by name or dataset id.

    Datasets are uniquely identified by either an integer ID or by a
    combination of name and version (i.e. there might be multiple
    versions of the 'iris' dataset). Please give either name or data_id
    (not both). In case a name is given, a version can also be
    provided.

    Read more in the :ref:`User Guide <openml>`.

    .. note:: EXPERIMENTAL

        The API is experimental in version 0.20 (particularly the return value
        structure), and might have small backward-incompatible changes in
        future releases.

    Parameters
    ----------
    name : str or None
        String identifier of the dataset. Note that OpenML can have multiple
        datasets with the same name.

    version : integer or 'active', default='active'
        Version of the dataset. Can only be provided if also ``name`` is given.
        If 'active' the oldest version that's still active is used. Since
        there may be more than one active version of a dataset, and those
        versions may fundamentally be different from one another, setting an
        exact version is highly recommended.

    data_id : int or None
        OpenML ID of the dataset. The most specific way of retrieving a
        dataset. If data_id is not given, name (and potential version) are
        used to obtain a dataset.

    data_home : string or None, default None
        Specify another download and cache folder for the data sets. By default
        all scikit-learn data is stored in '~/scikit_learn_data' subfolders.

    target_column : string, list or None, default 'default-target'
        Specify the column name in the data to use as target. If
        'default-target', the standard target column a stored on the server
        is used. If ``None``, all columns are returned as data and the
        target is ``None``. If list (of strings), all columns with these names
        are returned as multi-target (Note: not all scikit-learn classifiers
        can handle all types of multi-output combinations)

    cache : boolean, default=True
        Whether to cache downloaded datasets using joblib.

    return_X_y : boolean, default=False.
        If True, returns ``(data, target)`` instead of a Bunch object. See
        below for more information about the `data` and `target` objects.

    Returns
    -------

    data : Bunch
        Dictionary-like object, with attributes:

        data : np.array or scipy.sparse.csr_matrix of floats
            The feature matrix. Categorical features are encoded as ordinals.
        target : np.array
            The regression target or classification labels, if applicable.
            Dtype is float if numeric, and object if categorical.
        DESCR : str
            The full description of the dataset
        feature_names : list
            The names of the dataset columns
        categories : dict
            Maps each categorical feature name to a list of values, such
            that the value encoded as i is ith in the list.
        details : dict
            More metadata from OpenML

    (data, target) : tuple if ``return_X_y`` is True

        .. note:: EXPERIMENTAL

            This interface is **experimental** as at version 0.20 and
            subsequent releases may change attributes without notice
            (although there should only be minor changes to ``data``
            and ``target``).

        Missing values in the 'data' are represented as NaN's. Missing values
        in 'target' are represented as NaN's (numerical target) or None
        (categorical target)
    R   t   openmlsf   Dataset data_id={} and name={} passed, but you can only specify a numeric data_id or a name, not both.t   didRq   sl   Dataset data_id={} and version={} passed, but you can only specify a numeric data_id or a version, not both.sF   Neither name nor data_id are provided. Please provide name or data_id.t   statuss…   Version {} of dataset {} is inactive, meaning that issues have been found in the dataset. Try using a newer version from this URL: {}Rt   Rs   R:   R@   sM   OpenML registered a problem with the dataset. It might be unusable. Error: {}t   warningsI   OpenML raised a warning on the dataset. It might be unusable. Warning: {}R    RŸ   R¡   R   t   strings'   STRING attributes are not yet supporteds   default-targett	   is_targetsW   Did not recognize type of target_columnShould be six.string_type, list or None. Got: {}c         S   s   i  |  ] } | | d  “ q S(   Rs   (    (   RB   R‚   (    (    s6   lib/python2.7/site-packages/sklearn/datasets/openml.pys
   <dictcomp>l  s   	 t   indext   number_of_missing_valuesi    s]   Target column {} has {} missing values. Missing values are not supported for target columns. Rv   t   sparse_arffiÿÿÿÿR›   c            s>   i  |  ]4 \ } } t  | t ƒ r | ˆ  ˆ k r | | “ q S(    (   Rd   RE   (   RB   t   kt   v(   t   data_columnsR©   (    s6   lib/python2.7/site-packages/sklearn/datasets/openml.pys
   <dictcomp>”  s   	 	t
   attributesRl   c            s   h  |  ] } | ˆ  k ’ q S(    (    (   RB   t   col_name(   t   nominal_attributes(    s6   lib/python2.7/site-packages/sklearn/datasets/openml.pys	   <setcomp>š  s   	RQ   t   ONi   sA   Mix of nominal and non-nominal targets is not currently supportedu   {}

Downloaded from openml.org.t   descriptiont   targett   feature_namest   DESCRt   detailst
   categoriess   https://www.openml.org/d/{}(   iÿÿÿÿ(%   R   R    Rx   R   t   lowerR=   Rv   R|   R€   R   R„   Rd   R   RE   t	   TypeErrorR¢   R­   Rª   RŽ   Rw   R‰   R’   RS   Rœ   Rp   t   allRT   t   hstackRF   t   taket   asarrayt   popt   astypet   anyR\   Rc   R   (   Rs   Rt   R   R   R©   t   cachet
   return_X_yt	   data_infot   data_descriptionR«   R‚   R¦   R»   Rk   Rj   RO   t   featt
   nr_missingt   return_sparseR…   R\   t   arffRm   RY   t   is_classificationt   iR¾   t   bunch(    (   R¹   R¼   R©   s6   lib/python2.7/site-packages/sklearn/datasets/openml.pyR   É  sÂ    Y					#			$$
			
	o	
(=   R   R6   R   R0   t   os.pathR    R   R   t
   contextlibR   t	   functoolsR   R`   R]   t   urllib.requestR   R   t   ImportErrort   urllib2t   numpyRT   t   scipy.sparseRf   t   sklearn.externalsR   t   baseR   t   externals.sixR   R	   R
   t    externals.six.moves.urllib.errorR   t   utilsR   t   __all__R(   Ru   R~   Rƒ   R‡   Rš   R   R   R5   RA   RP   RZ   R   Rp   R|   R€   R„   R‰   R’   Rw   Rœ   Rª   R­   Rx   R   (    (    (    s6   lib/python2.7/site-packages/sklearn/datasets/openml.pyt   <module>   sZ   				;	3	!	2	<			
				