.. DO NOT EDIT.
.. THIS FILE WAS AUTOMATICALLY GENERATED BY SPHINX-GALLERY.
.. TO MAKE CHANGES, EDIT THE SOURCE PYTHON FILE:
.. "_examples/scikits/plot_data_splitters.py"
.. LINE NUMBERS ARE GIVEN BELOW.

.. only:: html

    .. note::
        :class: sphx-glr-download-link-note

        Click :ref:`here <sphx_glr_download__examples_scikits_plot_data_splitters.py>`
        to download the full example code

.. rst-class:: sphx-glr-example-title

.. _sphx_glr__examples_scikits_plot_data_splitters.py:


02. Data splitters
==================

Useful methods to split data such as (i) hold out and
csv sets or (ii) cross validation folds.

.. warning:: Not completed!

.. GENERATED FROM PYTHON SOURCE LINES 11-198

.. code-block:: default
   :lineno-start: 11

    # Libraries
    import pandas as pd
    import numpy as np

    # Libraries specific
    from sklearn.model_selection import train_test_split
    from sklearn.model_selection import StratifiedKFold
    from sklearn.model_selection import KFold


    try:
        __file__
        TERMINAL = True
    except:
        TERMINAL = False

    def split_dataframe_hos_cvs(dataframe,  **kwargs):
        """This method labels the dataframe hos and cvs sets.

        Parameters
        ----------
        dataframe: np.array or pd.DataFrame
            The data to be divided into HOS/CVS.

        Returns
        -------
        np.array:
            The outcome is a numpy array with rows labelled as
            cvs (cross-validation set) and hos (hold-out set).
            :param data:
            :param inplace:
        """
        # Check it is a dataframe
        if not isinstance(dataframe, pd.DataFrame):
            raise TypeError

        # Length
        n = dataframe.shape[0]

        # Split in hos and training sets
        cvs, hos = train_test_split(np.arange(n), **kwargs)

        # Create result
        empty = np.array([None]*n)
        empty[cvs] = 'cvs'
        empty[hos] = 'hos'

        # Include
        dataframe['sets'] = empty

        # Return
        return dataframe


    def split_dataframe_cvs_folds(dataframe, splitter,
                selected_rows=None, **kwargs):
        """This method labels the different folds.

            .. note:

        Parameters
        ----------
        dataframe: np.array or pd.DataFrame
            The data to be divided into folds.

        splitter: str or splitter
            The splitter which can be an str or an splitter from the
            sklearn library which implementeds the method split.

        selected_rows: array of bools.
            The rows to be considered to create the folds. Note that if
            y is passed (for stratified cross validation) y will also be
            filtered by these rows.

        kwargs:

        Returns
        -------
        pd.DataFrame:
            The outcome is the same dataframe with an additional column
            <set> with the values cvs (cross-validation set) and hos
            (hold-out set).
        """
        # Check it is a dataframe
        if not isinstance(dataframe, pd.DataFrame):
            raise TypeError

        # Get splitter from string
        if isinstance(splitter, str):
            splitter = _DEFAULT_SPLITTERS[splitter]

        # Define X and y
        #X = dataframe[dataframe.sets == 'cvs'].index.to_numpy()
        #y = dataframe[dataframe.sets == 'cvs'][label]

        # Shape
        r, c = dataframe.shape

        # No rows selected (all by default)
        if selected_rows is None:
            selected_rows = np.full(r, True, dtype=bool)

        # Select rows from y
        if 'y' in kwargs:
            if kwargs['y'] is not None:
                kwargs['y'] = kwargs['y'][selected_rows]

        # Create indexes to use for splitting
        idxs = np.arange(r)[selected_rows].reshape(-1, 1)

        # Get splits of idxs
        splits = splitter.split(idxs, **kwargs)

        # Loop and add
        for i, (train, test) in enumerate(splits):
            dataframe['split_{0}'.format(i)] = None
            dataframe.loc[idxs[train].flatten(), 'split_{0}'.format(i)] = 'train'
            dataframe.loc[idxs[test].flatten(), 'split_{0}'.format(i)] = 'test'

        # Return
        return dataframe


    def split_dataframe_completeness(dataframe):
        pass



    class DataframeHOSCSVSplitter():
        """
        """
        col_name = 'sets'
        cvs_name = 'CVS'
        hos_name = 'HOS'

        def __init__(self, col_name=None,
                           cvs_name=None,
                           hos_name=None):
            """Constructor

            :param col_name:
            :param cvs_name:
            :param hos_name:
            """
            if col_name is not None:
                self.col_name = col_name
            if cvs_name is not None:
                self.cvs_name = cvs_name
            if hos_name is not None:
                self.hos_name = hos_name

        def split(self, dataframe, **kwargs):
            """Splits the dataframe...
            """
            # Split
            cvs, hos = train_test_split(dataframe.index.to_numpy(), **kwargs)

            # Fill dataset
            dataframe[self.col_name] = None
            dataframe.loc[cvs, self.col_name] = self.cvs_name
            dataframe.loc[hos, self.col_name] = self.hos_name

            # Return
            return dataframe



    # Default splliters.
    _DEFAULT_SPLITTERS = {
        'skfold10': StratifiedKFold(n_splits=10, shuffle=True),
        'skfold5': StratifiedKFold(n_splits=5, shuffle=True),
        'skfold2': StratifiedKFold(n_splits=2, shuffle=True),
    }


    # --------------------------------------------------
    # Main
    # --------------------------------------------------
    # Libraries
    from sklearn.datasets import load_iris

    # Load data
    bunch = load_iris(as_frame=True)

    # Dataframe
    dataframe = bunch.data








.. GENERATED FROM PYTHON SOURCE LINES 199-200

Lets see the dataset

.. GENERATED FROM PYTHON SOURCE LINES 200-206

.. code-block:: default
   :lineno-start: 201


    if TERMINAL:
        print("\nData")
        print(dataframe)
    dataframe






.. raw:: html

    <div class="output_subarea output_html rendered_html output_result">
    <div>
    <style scoped>
        .dataframe tbody tr th:only-of-type {
            vertical-align: middle;
        }

        .dataframe tbody tr th {
            vertical-align: top;
        }

        .dataframe thead th {
            text-align: right;
        }
    </style>
    <table border="1" class="dataframe">
      <thead>
        <tr style="text-align: right;">
          <th></th>
          <th>sepal length (cm)</th>
          <th>sepal width (cm)</th>
          <th>petal length (cm)</th>
          <th>petal width (cm)</th>
        </tr>
      </thead>
      <tbody>
        <tr>
          <th>0</th>
          <td>5.1</td>
          <td>3.5</td>
          <td>1.4</td>
          <td>0.2</td>
        </tr>
        <tr>
          <th>1</th>
          <td>4.9</td>
          <td>3.0</td>
          <td>1.4</td>
          <td>0.2</td>
        </tr>
        <tr>
          <th>2</th>
          <td>4.7</td>
          <td>3.2</td>
          <td>1.3</td>
          <td>0.2</td>
        </tr>
        <tr>
          <th>3</th>
          <td>4.6</td>
          <td>3.1</td>
          <td>1.5</td>
          <td>0.2</td>
        </tr>
        <tr>
          <th>4</th>
          <td>5.0</td>
          <td>3.6</td>
          <td>1.4</td>
          <td>0.2</td>
        </tr>
        <tr>
          <th>...</th>
          <td>...</td>
          <td>...</td>
          <td>...</td>
          <td>...</td>
        </tr>
        <tr>
          <th>145</th>
          <td>6.7</td>
          <td>3.0</td>
          <td>5.2</td>
          <td>2.3</td>
        </tr>
        <tr>
          <th>146</th>
          <td>6.3</td>
          <td>2.5</td>
          <td>5.0</td>
          <td>1.9</td>
        </tr>
        <tr>
          <th>147</th>
          <td>6.5</td>
          <td>3.0</td>
          <td>5.2</td>
          <td>2.0</td>
        </tr>
        <tr>
          <th>148</th>
          <td>6.2</td>
          <td>3.4</td>
          <td>5.4</td>
          <td>2.3</td>
        </tr>
        <tr>
          <th>149</th>
          <td>5.9</td>
          <td>3.0</td>
          <td>5.1</td>
          <td>1.8</td>
        </tr>
      </tbody>
    </table>
    <p>150 rows × 4 columns</p>
    </div>
    </div>
    <br />
    <br />

.. GENERATED FROM PYTHON SOURCE LINES 207-208

Lets split in HOS and CVS

.. GENERATED FROM PYTHON SOURCE LINES 208-217

.. code-block:: default
   :lineno-start: 209


    # Split in HOS and CVS sets
    df = split_dataframe_hos_cvs(dataframe)

    if TERMINAL:
        print("\nData")
        print(df)
    df






.. raw:: html

    <div class="output_subarea output_html rendered_html output_result">
    <div>
    <style scoped>
        .dataframe tbody tr th:only-of-type {
            vertical-align: middle;
        }

        .dataframe tbody tr th {
            vertical-align: top;
        }

        .dataframe thead th {
            text-align: right;
        }
    </style>
    <table border="1" class="dataframe">
      <thead>
        <tr style="text-align: right;">
          <th></th>
          <th>sepal length (cm)</th>
          <th>sepal width (cm)</th>
          <th>petal length (cm)</th>
          <th>petal width (cm)</th>
          <th>sets</th>
        </tr>
      </thead>
      <tbody>
        <tr>
          <th>0</th>
          <td>5.1</td>
          <td>3.5</td>
          <td>1.4</td>
          <td>0.2</td>
          <td>cvs</td>
        </tr>
        <tr>
          <th>1</th>
          <td>4.9</td>
          <td>3.0</td>
          <td>1.4</td>
          <td>0.2</td>
          <td>cvs</td>
        </tr>
        <tr>
          <th>2</th>
          <td>4.7</td>
          <td>3.2</td>
          <td>1.3</td>
          <td>0.2</td>
          <td>cvs</td>
        </tr>
        <tr>
          <th>3</th>
          <td>4.6</td>
          <td>3.1</td>
          <td>1.5</td>
          <td>0.2</td>
          <td>cvs</td>
        </tr>
        <tr>
          <th>4</th>
          <td>5.0</td>
          <td>3.6</td>
          <td>1.4</td>
          <td>0.2</td>
          <td>cvs</td>
        </tr>
        <tr>
          <th>...</th>
          <td>...</td>
          <td>...</td>
          <td>...</td>
          <td>...</td>
          <td>...</td>
        </tr>
        <tr>
          <th>145</th>
          <td>6.7</td>
          <td>3.0</td>
          <td>5.2</td>
          <td>2.3</td>
          <td>cvs</td>
        </tr>
        <tr>
          <th>146</th>
          <td>6.3</td>
          <td>2.5</td>
          <td>5.0</td>
          <td>1.9</td>
          <td>cvs</td>
        </tr>
        <tr>
          <th>147</th>
          <td>6.5</td>
          <td>3.0</td>
          <td>5.2</td>
          <td>2.0</td>
          <td>hos</td>
        </tr>
        <tr>
          <th>148</th>
          <td>6.2</td>
          <td>3.4</td>
          <td>5.4</td>
          <td>2.3</td>
          <td>cvs</td>
        </tr>
        <tr>
          <th>149</th>
          <td>5.9</td>
          <td>3.0</td>
          <td>5.1</td>
          <td>1.8</td>
          <td>cvs</td>
        </tr>
      </tbody>
    </table>
    <p>150 rows × 5 columns</p>
    </div>
    </div>
    <br />
    <br />

.. GENERATED FROM PYTHON SOURCE LINES 218-219

Lets split the CSV in various folds

.. GENERATED FROM PYTHON SOURCE LINES 219-232

.. code-block:: default
   :lineno-start: 220


    # Split in folds
    df = split_dataframe_cvs_folds(dataframe,
        splitter='skfold5', y=bunch.target,
        selected_rows=(dataframe.sets == 'cvs'))

    if TERMINAL:
        print("\nData")
        print(df)
    df








.. raw:: html

    <div class="output_subarea output_html rendered_html output_result">
    <div>
    <style scoped>
        .dataframe tbody tr th:only-of-type {
            vertical-align: middle;
        }

        .dataframe tbody tr th {
            vertical-align: top;
        }

        .dataframe thead th {
            text-align: right;
        }
    </style>
    <table border="1" class="dataframe">
      <thead>
        <tr style="text-align: right;">
          <th></th>
          <th>sepal length (cm)</th>
          <th>sepal width (cm)</th>
          <th>petal length (cm)</th>
          <th>petal width (cm)</th>
          <th>sets</th>
          <th>split_0</th>
          <th>split_1</th>
          <th>split_2</th>
          <th>split_3</th>
          <th>split_4</th>
        </tr>
      </thead>
      <tbody>
        <tr>
          <th>0</th>
          <td>5.1</td>
          <td>3.5</td>
          <td>1.4</td>
          <td>0.2</td>
          <td>cvs</td>
          <td>train</td>
          <td>test</td>
          <td>train</td>
          <td>train</td>
          <td>train</td>
        </tr>
        <tr>
          <th>1</th>
          <td>4.9</td>
          <td>3.0</td>
          <td>1.4</td>
          <td>0.2</td>
          <td>cvs</td>
          <td>train</td>
          <td>train</td>
          <td>train</td>
          <td>train</td>
          <td>test</td>
        </tr>
        <tr>
          <th>2</th>
          <td>4.7</td>
          <td>3.2</td>
          <td>1.3</td>
          <td>0.2</td>
          <td>cvs</td>
          <td>test</td>
          <td>train</td>
          <td>train</td>
          <td>train</td>
          <td>train</td>
        </tr>
        <tr>
          <th>3</th>
          <td>4.6</td>
          <td>3.1</td>
          <td>1.5</td>
          <td>0.2</td>
          <td>cvs</td>
          <td>train</td>
          <td>train</td>
          <td>train</td>
          <td>train</td>
          <td>test</td>
        </tr>
        <tr>
          <th>4</th>
          <td>5.0</td>
          <td>3.6</td>
          <td>1.4</td>
          <td>0.2</td>
          <td>cvs</td>
          <td>train</td>
          <td>train</td>
          <td>train</td>
          <td>train</td>
          <td>test</td>
        </tr>
        <tr>
          <th>...</th>
          <td>...</td>
          <td>...</td>
          <td>...</td>
          <td>...</td>
          <td>...</td>
          <td>...</td>
          <td>...</td>
          <td>...</td>
          <td>...</td>
          <td>...</td>
        </tr>
        <tr>
          <th>145</th>
          <td>6.7</td>
          <td>3.0</td>
          <td>5.2</td>
          <td>2.3</td>
          <td>cvs</td>
          <td>train</td>
          <td>train</td>
          <td>test</td>
          <td>train</td>
          <td>train</td>
        </tr>
        <tr>
          <th>146</th>
          <td>6.3</td>
          <td>2.5</td>
          <td>5.0</td>
          <td>1.9</td>
          <td>cvs</td>
          <td>train</td>
          <td>test</td>
          <td>train</td>
          <td>train</td>
          <td>train</td>
        </tr>
        <tr>
          <th>147</th>
          <td>6.5</td>
          <td>3.0</td>
          <td>5.2</td>
          <td>2.0</td>
          <td>hos</td>
          <td>None</td>
          <td>None</td>
          <td>None</td>
          <td>None</td>
          <td>None</td>
        </tr>
        <tr>
          <th>148</th>
          <td>6.2</td>
          <td>3.4</td>
          <td>5.4</td>
          <td>2.3</td>
          <td>cvs</td>
          <td>train</td>
          <td>train</td>
          <td>train</td>
          <td>test</td>
          <td>train</td>
        </tr>
        <tr>
          <th>149</th>
          <td>5.9</td>
          <td>3.0</td>
          <td>5.1</td>
          <td>1.8</td>
          <td>cvs</td>
          <td>test</td>
          <td>train</td>
          <td>train</td>
          <td>train</td>
          <td>train</td>
        </tr>
      </tbody>
    </table>
    <p>150 rows × 10 columns</p>
    </div>
    </div>
    <br />
    <br />

.. GENERATED FROM PYTHON SOURCE LINES 233-236

Lets split in HOS and CVS using the class

.. note:: This might not be working properly!

.. GENERATED FROM PYTHON SOURCE LINES 236-244

.. code-block:: default
   :lineno-start: 237


    # Divide in HOS and CSV.
    df = DataframeHOSCSVSplitter().split(dataframe)

    if TERMINAL:
        print("\nData")
        print(df)
    df





.. raw:: html

    <div class="output_subarea output_html rendered_html output_result">
    <div>
    <style scoped>
        .dataframe tbody tr th:only-of-type {
            vertical-align: middle;
        }

        .dataframe tbody tr th {
            vertical-align: top;
        }

        .dataframe thead th {
            text-align: right;
        }
    </style>
    <table border="1" class="dataframe">
      <thead>
        <tr style="text-align: right;">
          <th></th>
          <th>sepal length (cm)</th>
          <th>sepal width (cm)</th>
          <th>petal length (cm)</th>
          <th>petal width (cm)</th>
          <th>sets</th>
          <th>split_0</th>
          <th>split_1</th>
          <th>split_2</th>
          <th>split_3</th>
          <th>split_4</th>
        </tr>
      </thead>
      <tbody>
        <tr>
          <th>0</th>
          <td>5.1</td>
          <td>3.5</td>
          <td>1.4</td>
          <td>0.2</td>
          <td>CVS</td>
          <td>train</td>
          <td>test</td>
          <td>train</td>
          <td>train</td>
          <td>train</td>
        </tr>
        <tr>
          <th>1</th>
          <td>4.9</td>
          <td>3.0</td>
          <td>1.4</td>
          <td>0.2</td>
          <td>CVS</td>
          <td>train</td>
          <td>train</td>
          <td>train</td>
          <td>train</td>
          <td>test</td>
        </tr>
        <tr>
          <th>2</th>
          <td>4.7</td>
          <td>3.2</td>
          <td>1.3</td>
          <td>0.2</td>
          <td>CVS</td>
          <td>test</td>
          <td>train</td>
          <td>train</td>
          <td>train</td>
          <td>train</td>
        </tr>
        <tr>
          <th>3</th>
          <td>4.6</td>
          <td>3.1</td>
          <td>1.5</td>
          <td>0.2</td>
          <td>CVS</td>
          <td>train</td>
          <td>train</td>
          <td>train</td>
          <td>train</td>
          <td>test</td>
        </tr>
        <tr>
          <th>4</th>
          <td>5.0</td>
          <td>3.6</td>
          <td>1.4</td>
          <td>0.2</td>
          <td>CVS</td>
          <td>train</td>
          <td>train</td>
          <td>train</td>
          <td>train</td>
          <td>test</td>
        </tr>
        <tr>
          <th>...</th>
          <td>...</td>
          <td>...</td>
          <td>...</td>
          <td>...</td>
          <td>...</td>
          <td>...</td>
          <td>...</td>
          <td>...</td>
          <td>...</td>
          <td>...</td>
        </tr>
        <tr>
          <th>145</th>
          <td>6.7</td>
          <td>3.0</td>
          <td>5.2</td>
          <td>2.3</td>
          <td>CVS</td>
          <td>train</td>
          <td>train</td>
          <td>test</td>
          <td>train</td>
          <td>train</td>
        </tr>
        <tr>
          <th>146</th>
          <td>6.3</td>
          <td>2.5</td>
          <td>5.0</td>
          <td>1.9</td>
          <td>CVS</td>
          <td>train</td>
          <td>test</td>
          <td>train</td>
          <td>train</td>
          <td>train</td>
        </tr>
        <tr>
          <th>147</th>
          <td>6.5</td>
          <td>3.0</td>
          <td>5.2</td>
          <td>2.0</td>
          <td>CVS</td>
          <td>None</td>
          <td>None</td>
          <td>None</td>
          <td>None</td>
          <td>None</td>
        </tr>
        <tr>
          <th>148</th>
          <td>6.2</td>
          <td>3.4</td>
          <td>5.4</td>
          <td>2.3</td>
          <td>CVS</td>
          <td>train</td>
          <td>train</td>
          <td>train</td>
          <td>test</td>
          <td>train</td>
        </tr>
        <tr>
          <th>149</th>
          <td>5.9</td>
          <td>3.0</td>
          <td>5.1</td>
          <td>1.8</td>
          <td>CVS</td>
          <td>test</td>
          <td>train</td>
          <td>train</td>
          <td>train</td>
          <td>train</td>
        </tr>
      </tbody>
    </table>
    <p>150 rows × 10 columns</p>
    </div>
    </div>
    <br />
    <br />


.. rst-class:: sphx-glr-timing

   **Total running time of the script:** ( 0 minutes  0.033 seconds)


.. _sphx_glr_download__examples_scikits_plot_data_splitters.py:


.. only :: html

 .. container:: sphx-glr-footer
    :class: sphx-glr-footer-example



  .. container:: sphx-glr-download sphx-glr-download-python

     :download:`Download Python source code: plot_data_splitters.py <plot_data_splitters.py>`



  .. container:: sphx-glr-download sphx-glr-download-jupyter

     :download:`Download Jupyter notebook: plot_data_splitters.ipynb <plot_data_splitters.ipynb>`


.. only:: html

 .. rst-class:: sphx-glr-signature

    `Gallery generated by Sphinx-Gallery <https://sphinx-gallery.github.io>`_