## 1. Classification trees
##
## 1.1 Read the documentation at
## http://scikit-learn.org/stable/modules/tree.html

## 1.2 Practical exercise: follow the documentation and try
## the classification on IRIS dataset

## 1.3 Application to a more challenging problem: predict
## onset of diabetes in female patients of Pima Indian heritage.

 
from sklearn.datasets import fetch_mldata
from sklearn import tree
from sklearn.externals.six import StringIO

## The following is for testing various parameters for training
## a tree classifier, not for proper estimation!

pima = fetch_mldata('diabetes_scale')
X = pima.data   # 768 x 8 matrix
y = pima.target # =1 for diabetes, =-1 otherwise

clf = tree.DecisionTreeClassifier()
clf.fit(X, y)

yp = clf.predict(X)

## TODO use yp and y to estimate the performance


## for this you need Graphviz to be installed
with open("pima_tree_1.dot", 'w') as f:
    f = tree.export_graphviz(clf, out_file=f)
## dot -Tpdf pima_tree_1.dot -o pima_tree_1.pdf


## TODO:
## Now try to change the depth (max_depth parameter), the
## number of points in a node for a split (min_samples_split),
## the number of points in a leaf (min_samples_leaf), the
## maximum number of leaves (max_leaf_nodes) and the impurity
## criterion (criterion).
##
## For example,
clf = tree.DecisionTreeClassifier(criterion='entropy', max_depth=3)
clf.fit(X, y)

with open("pima_tree_2.dot", 'w') as f:
    f = tree.export_graphviz(clf, out_file=f)

## 2. Combining the outputs of several classifiers.

## TODO:
## -train an LDA classifier on PIMA dataset
from sklearn.lda import LDA
clf1 = LDA()
clf1.fit(X, y)

## -train a kNN classifier
from sklearn.neighbors import KNeighborsClassifier
clf2 = KNeighborsClassifier(...)
clf2.fit(X, y)

## -train a tree

clf3 = ...
clf3.fit(X, y)

## Now, write a function that combines the predictions of the 
## classifiers:

def combiner(clfs, X, method):
    # clfs - a list of trained classifiers, like [clf1, clf2, clf3]
    # X - a data matrix: each rows is a data vector for which we will
    # predict the label
    # method - a string describing the method to be used
    import numpy as np

    nc = len(clfs)
    method = method.tolower()
    y = np.zeros(X.shape[0])     # final labels

    # simple code, not necessarily the most "Pythonic":
    if method == 'majority':
        labels = [clf.predict(X) for clf in clfs] 
        DP = np.array(labels)    # this will result in a matrix with
                                 # one row for each classifier and
                                 # columns for predictions for each point in X
        # here you should put the code for majoity voting
        y = ...

    elif method == 'average':
    # for continuous outputs
        pass   # replace with your code
    elif method == 'maximum':
    # for continuous outputs
        pass   # replace with your code
    else:
        raise ValueError('Unknown method')

    return y


