This Unit includes main introduction to classification in machine learning, strongly based in [1].
Dirk P. Kroese, Zdravko Botev, Thomas Taimre, and Radislav: Vaisman. Data Science and Machine Learning: Mathematical and Statistical Methods. Machine Learning & Pattern Recognition. Chapman & Hall/CRC, 2020. URL: https://acems.org.au/data-science-machine-learning-book-available-download (visited on 2023-08-15).
In naive Bayes, the value of the overall function \(g\) depends on the product of the function \(g\) for each feature. Thus, the classification is done by choosing the highset \(g\) value.
For example, below we assume we have three features with different means \(\mu\) and standard deviations \(\sigma\). How can a given vector of features \(\uvec{x}\) be classified?
""" naiveBayes.py """
import numpy as np
x = np.array([1.67,2,4.23]).reshape((1,3))
mu = np.array([1.6, 2.4, 4.3,
1.5, 2.9, 6.1,
1.8, 2.5, 4.2,
1.1, 3.1, 5.6]).reshape((4,3))
sig = np.array([0.1, 0.5, 0.2,
0.2, 0.6, 0.9,
0.3, 0.3, 0.3,
0.2, 0.7, 0.3]).reshape((4,3))
f = lambda y: 1/np.prod(sig[y,:]) * np.exp(
-0.5*np.sum((x-mu[y,:])**2/sig[y,:]**2));
for y in range(0,4):
print('{:3.2e}'.format(f(y)))
5.35e+01
2.42e-01
8.37e+00
3.53e-06
Built on the above Bayesian approach, and after some mathematical manipulation, one can draw the lines that discriminate between two groups. This is called the decison boundary, that is quadratic for two given normal distributions but becomes linear if the distributions have the same covariance matrix.
See tha case for a LDA below, with the same \(\sigma\) for two underlying normal distributions:
""" LDAmixture.py """
import numpy as np
import matplotlib.pyplot as plt
from scipy.stats import multivariate_normal
from mpl_toolkits.mplot3d import Axes3D
from matplotlib.colors import LightSource
mu1 = np.array([0,0])
mu2 = np.array([2,4])
Sigma = np.array([[2,0.7],[0.7, 2]])
x, y = np.mgrid[-4:6:150j,-5:8:150j]
mvn1 = multivariate_normal( mu1, Sigma )
mvn2 = multivariate_normal( mu2, Sigma )
xy = np.hstack((x.reshape(-1,1),y.reshape(-1,1)))
z = 0.5*mvn1.pdf(xy).reshape(x.shape) + 0.5*mvn2.pdf(xy).reshape(x.shape)
# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
fig = plt.figure()
ax = fig.add_subplot(projection='3d')
ls = LightSource(azdeg=180, altdeg=65)
cols = ls.shade(z, plt.cm.winter)
surf = ax.plot_surface(x, y, z, rstride=1, cstride=1, linewidth=0,
antialiased=False, facecolors=cols)
plt.savefig('LDApy.png',format='png',dpi=300, transparent=False)
plt.show()
# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
or QDA if \(\sigma_1\neq\sigma_2\):
""" QDA.py """
import numpy as np
import matplotlib.pyplot as plt
from scipy.stats import multivariate_normal
mu1 = np.array([0,0])
mu2 = np.array([2,2])
Sigma1 = np.array([[1,0.3],[0.3, 1]])
Sigma2 = np.array([[0.3,0.3],[0.3, 1]])
x, y = np.mgrid[-2:4:150j,-3:5:150j]
mvn1 = multivariate_normal( mu1, Sigma1 )
mvn2 = multivariate_normal( mu2, Sigma2 )
xy = np.hstack((x.reshape(-1,1),y.reshape(-1,1)))
z = 0.5*mvn1.pdf(xy).reshape(x.shape) + 0.5*mvn2.pdf(xy).reshape(x.shape)
plt.contour(x,y,z)
z1 = 0.5*mvn1.pdf(xy).reshape(x.shape) - 0.5*mvn2.pdf(xy).reshape(x.shape)
plt.contour(x,y,z1, levels=[0],linestyles ='dashed', linewidths = 2, colors = 'm')
# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
plt.savefig('QDApy.pdf')
plt.show()
# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
"""" datared.py """
import numpy as np
from numpy.random import randn
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
n=1000
mu1 = np.array([2,1,-3])
mu2 = np.array([1,-4,0])
mu3 = np.array([2,4,0])
X1 = randn(n,3) + mu1
X2 = randn(n,3) + mu2
X3 = randn(n,3) + mu3
fig = plt.figure()
# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
ax = fig.add_subplot(projection='3d',)
ax.plot(X1[:,0],X1[:,1],X1[:,2],'r.',alpha=0.5,markersize=2)
ax.plot(X2[:,0],X2[:,1],X2[:,2],'b.',alpha=0.5,markersize=2)
ax.plot(X3[:,0],X3[:,1],X3[:,2],'g.',alpha=0.5,markersize=2)
#ax.set_aspect("equal")
ax.set_xlim3d(-4,6)
ax.set_ylim3d(-5,5)
ax.set_zlim3d(-5,2)
plt.savefig('pcaproj1py.pdf')
plt.show()
# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
""" dataproj.py """
from numpy.linalg import svd, pinv
mu21 = (mu2 - mu1).reshape(3,1)
mu31 = (mu3 - mu1).reshape(3,1)
W = np.hstack((mu21, mu31))
U,_,_ = svd(W) # we only need U
P = W @ pinv(W)
R = U.T @ P
RX1 = (R @ X1.T).T
RX2 = (R @ X2.T).T
RX3 = (R @ X3.T).T
# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
plt.plot(RX1[:,0],RX1[:,1],'b.',alpha=0.5,markersize=2)
plt.plot(RX2[:,0],RX2[:,1],'g.',alpha=0.5,markersize=2)
plt.plot(RX3[:,0],RX3[:,1],'r.',alpha=0.5,markersize=2)
plt.savefig('pcaproj2py.pdf')
plt.show()
# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
Figure adapted from https://medium.com/swlh/k-nearest-neighbor-ca2593d7a3c4.
""" nearestnb.py """
import numpy as np
from numpy.random import rand,randn
import matplotlib.pyplot as plt
from scipy.spatial import Voronoi, voronoi_plot_2d
np.random.seed(12345)
M = 80
x = randn(M,2)
y = np.zeros(M) # pre-allocate list
for i in range(M):
if rand()<0.5:
x[i,1], y[i] = x[i,0] + np.abs(randn()), 0
else:
x[i,1], y[i] = x[i,0] - np.abs(randn()), 1
vor = Voronoi(x)
plt_options = {'show_vertices':False, 'show_points':False, 'line_alpha':0.5}
fig = voronoi_plot_2d(vor, **plt_options)
plt.plot(x[y==0,0], x[y==0,1],'bo',
x[y==1,0], x[y==1,1],'rs', markersize=3)
# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
plt.tight_layout()
fig.set_size_inches(5,4)
plt.savefig('nearestnbpy.pdf',format='pdf')
plt.show()
# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
Figure adapted from https://www.javatpoint.com/machine-learning-support-vector-machine-algorithm.
Here is a calculator of the support vectors for a given dataset, taken from here. The data contains information about the purchase or not of an SUV (dependent variable) as a function of gender, age and estimated salary (explanatory or independent variables). We start by importing and visualizing the data, and we will prune the table eliminating userID and gender.
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
# data from https://www.kaggle.com/datasets/rakeshrau/social-network-ads
dataset = pd.read_csv('datasets/Social_Network_Ads.csv')
dataset.head
<bound method NDFrame.head of User ID Gender Age EstimatedSalary Purchased
0 15624510 Male 19 19000 0
1 15810944 Male 35 20000 0
2 15668575 Female 26 43000 0
3 15603246 Female 27 57000 0
4 15804002 Male 19 76000 0
.. ... ... ... ... ...
395 15691863 Female 46 41000 1
396 15706071 Male 51 23000 1
397 15654296 Female 50 20000 1
398 15755018 Male 36 33000 0
399 15594041 Female 49 36000 1
[400 rows x 5 columns]>
we separated the data into two tables for X and y
X = dataset.iloc[:, [2, 3]].values
y = dataset.iloc[:, 4].values
print(X)
print(y)
[[ 19 19000]
[ 35 20000]
[ 26 43000]
[ 27 57000]
[ 19 76000]
[ 27 58000]
[ 27 84000]
[ 32 150000]
[ 25 33000]
[ 35 65000]
[ 26 80000]
[ 26 52000]
[ 20 86000]
[ 32 18000]
[ 18 82000]
[ 29 80000]
[ 47 25000]
[ 45 26000]
[ 46 28000]
[ 48 29000]
[ 45 22000]
[ 47 49000]
[ 48 41000]
[ 45 22000]
[ 46 23000]
[ 47 20000]
[ 49 28000]
[ 47 30000]
[ 29 43000]
[ 31 18000]
[ 31 74000]
[ 27 137000]
[ 21 16000]
[ 28 44000]
[ 27 90000]
[ 35 27000]
[ 33 28000]
[ 30 49000]
[ 26 72000]
[ 27 31000]
[ 27 17000]
[ 33 51000]
[ 35 108000]
[ 30 15000]
[ 28 84000]
[ 23 20000]
[ 25 79000]
[ 27 54000]
[ 30 135000]
[ 31 89000]
[ 24 32000]
[ 18 44000]
[ 29 83000]
[ 35 23000]
[ 27 58000]
[ 24 55000]
[ 23 48000]
[ 28 79000]
[ 22 18000]
[ 32 117000]
[ 27 20000]
[ 25 87000]
[ 23 66000]
[ 32 120000]
[ 59 83000]
[ 24 58000]
[ 24 19000]
[ 23 82000]
[ 22 63000]
[ 31 68000]
[ 25 80000]
[ 24 27000]
[ 20 23000]
[ 33 113000]
[ 32 18000]
[ 34 112000]
[ 18 52000]
[ 22 27000]
[ 28 87000]
[ 26 17000]
[ 30 80000]
[ 39 42000]
[ 20 49000]
[ 35 88000]
[ 30 62000]
[ 31 118000]
[ 24 55000]
[ 28 85000]
[ 26 81000]
[ 35 50000]
[ 22 81000]
[ 30 116000]
[ 26 15000]
[ 29 28000]
[ 29 83000]
[ 35 44000]
[ 35 25000]
[ 28 123000]
[ 35 73000]
[ 28 37000]
[ 27 88000]
[ 28 59000]
[ 32 86000]
[ 33 149000]
[ 19 21000]
[ 21 72000]
[ 26 35000]
[ 27 89000]
[ 26 86000]
[ 38 80000]
[ 39 71000]
[ 37 71000]
[ 38 61000]
[ 37 55000]
[ 42 80000]
[ 40 57000]
[ 35 75000]
[ 36 52000]
[ 40 59000]
[ 41 59000]
[ 36 75000]
[ 37 72000]
[ 40 75000]
[ 35 53000]
[ 41 51000]
[ 39 61000]
[ 42 65000]
[ 26 32000]
[ 30 17000]
[ 26 84000]
[ 31 58000]
[ 33 31000]
[ 30 87000]
[ 21 68000]
[ 28 55000]
[ 23 63000]
[ 20 82000]
[ 30 107000]
[ 28 59000]
[ 19 25000]
[ 19 85000]
[ 18 68000]
[ 35 59000]
[ 30 89000]
[ 34 25000]
[ 24 89000]
[ 27 96000]
[ 41 30000]
[ 29 61000]
[ 20 74000]
[ 26 15000]
[ 41 45000]
[ 31 76000]
[ 36 50000]
[ 40 47000]
[ 31 15000]
[ 46 59000]
[ 29 75000]
[ 26 30000]
[ 32 135000]
[ 32 100000]
[ 25 90000]
[ 37 33000]
[ 35 38000]
[ 33 69000]
[ 18 86000]
[ 22 55000]
[ 35 71000]
[ 29 148000]
[ 29 47000]
[ 21 88000]
[ 34 115000]
[ 26 118000]
[ 34 43000]
[ 34 72000]
[ 23 28000]
[ 35 47000]
[ 25 22000]
[ 24 23000]
[ 31 34000]
[ 26 16000]
[ 31 71000]
[ 32 117000]
[ 33 43000]
[ 33 60000]
[ 31 66000]
[ 20 82000]
[ 33 41000]
[ 35 72000]
[ 28 32000]
[ 24 84000]
[ 19 26000]
[ 29 43000]
[ 19 70000]
[ 28 89000]
[ 34 43000]
[ 30 79000]
[ 20 36000]
[ 26 80000]
[ 35 22000]
[ 35 39000]
[ 49 74000]
[ 39 134000]
[ 41 71000]
[ 58 101000]
[ 47 47000]
[ 55 130000]
[ 52 114000]
[ 40 142000]
[ 46 22000]
[ 48 96000]
[ 52 150000]
[ 59 42000]
[ 35 58000]
[ 47 43000]
[ 60 108000]
[ 49 65000]
[ 40 78000]
[ 46 96000]
[ 59 143000]
[ 41 80000]
[ 35 91000]
[ 37 144000]
[ 60 102000]
[ 35 60000]
[ 37 53000]
[ 36 126000]
[ 56 133000]
[ 40 72000]
[ 42 80000]
[ 35 147000]
[ 39 42000]
[ 40 107000]
[ 49 86000]
[ 38 112000]
[ 46 79000]
[ 40 57000]
[ 37 80000]
[ 46 82000]
[ 53 143000]
[ 42 149000]
[ 38 59000]
[ 50 88000]
[ 56 104000]
[ 41 72000]
[ 51 146000]
[ 35 50000]
[ 57 122000]
[ 41 52000]
[ 35 97000]
[ 44 39000]
[ 37 52000]
[ 48 134000]
[ 37 146000]
[ 50 44000]
[ 52 90000]
[ 41 72000]
[ 40 57000]
[ 58 95000]
[ 45 131000]
[ 35 77000]
[ 36 144000]
[ 55 125000]
[ 35 72000]
[ 48 90000]
[ 42 108000]
[ 40 75000]
[ 37 74000]
[ 47 144000]
[ 40 61000]
[ 43 133000]
[ 59 76000]
[ 60 42000]
[ 39 106000]
[ 57 26000]
[ 57 74000]
[ 38 71000]
[ 49 88000]
[ 52 38000]
[ 50 36000]
[ 59 88000]
[ 35 61000]
[ 37 70000]
[ 52 21000]
[ 48 141000]
[ 37 93000]
[ 37 62000]
[ 48 138000]
[ 41 79000]
[ 37 78000]
[ 39 134000]
[ 49 89000]
[ 55 39000]
[ 37 77000]
[ 35 57000]
[ 36 63000]
[ 42 73000]
[ 43 112000]
[ 45 79000]
[ 46 117000]
[ 58 38000]
[ 48 74000]
[ 37 137000]
[ 37 79000]
[ 40 60000]
[ 42 54000]
[ 51 134000]
[ 47 113000]
[ 36 125000]
[ 38 50000]
[ 42 70000]
[ 39 96000]
[ 38 50000]
[ 49 141000]
[ 39 79000]
[ 39 75000]
[ 54 104000]
[ 35 55000]
[ 45 32000]
[ 36 60000]
[ 52 138000]
[ 53 82000]
[ 41 52000]
[ 48 30000]
[ 48 131000]
[ 41 60000]
[ 41 72000]
[ 42 75000]
[ 36 118000]
[ 47 107000]
[ 38 51000]
[ 48 119000]
[ 42 65000]
[ 40 65000]
[ 57 60000]
[ 36 54000]
[ 58 144000]
[ 35 79000]
[ 38 55000]
[ 39 122000]
[ 53 104000]
[ 35 75000]
[ 38 65000]
[ 47 51000]
[ 47 105000]
[ 41 63000]
[ 53 72000]
[ 54 108000]
[ 39 77000]
[ 38 61000]
[ 38 113000]
[ 37 75000]
[ 42 90000]
[ 37 57000]
[ 36 99000]
[ 60 34000]
[ 54 70000]
[ 41 72000]
[ 40 71000]
[ 42 54000]
[ 43 129000]
[ 53 34000]
[ 47 50000]
[ 42 79000]
[ 42 104000]
[ 59 29000]
[ 58 47000]
[ 46 88000]
[ 38 71000]
[ 54 26000]
[ 60 46000]
[ 60 83000]
[ 39 73000]
[ 59 130000]
[ 37 80000]
[ 46 32000]
[ 46 74000]
[ 42 53000]
[ 41 87000]
[ 58 23000]
[ 42 64000]
[ 48 33000]
[ 44 139000]
[ 49 28000]
[ 57 33000]
[ 56 60000]
[ 49 39000]
[ 39 71000]
[ 47 34000]
[ 48 35000]
[ 48 33000]
[ 47 23000]
[ 45 45000]
[ 60 42000]
[ 39 59000]
[ 46 41000]
[ 51 23000]
[ 50 20000]
[ 36 33000]
[ 49 36000]]
[0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 1 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0
0 1 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 1 0 0 0 0 0 0 0
0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 1 0
0 0 0 0 0 0 0 0 0 0 0 1 1 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0
0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 1 0 1 0 1 0 1 1 0 0 0 1 0 0 0 1 0 1
1 1 0 0 1 1 0 1 1 0 1 1 0 1 0 0 0 1 1 0 1 1 0 1 0 1 0 1 0 0 1 1 0 1 0 0 1
1 0 1 1 0 1 1 0 0 1 0 0 1 1 1 1 1 0 1 1 1 1 0 1 1 0 1 0 1 0 1 1 1 1 0 0 0
1 1 0 1 1 1 1 1 0 0 0 1 1 0 0 1 0 1 0 1 1 0 1 0 1 1 0 1 1 0 0 0 1 1 0 1 0
0 1 0 1 0 0 1 1 0 0 1 1 0 1 1 0 0 1 0 1 0 1 1 1 0 1 0 1 1 1 0 1 1 1 1 0 1
1 1 0 1 0 1 0 0 1 1 0 1 1 1 1 1 1 0 1 1 1 1 1 1 0 1 1 1 0 1]
We now select a training set with 80% of the data
from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2, random_state = 0)
plt.hist(X_train)
plt.legend(['Age','Salary'])
plt.show()
We can see that the data does not have the same scales, so next, we need to scale the features to get normalized data:
from sklearn.preprocessing import StandardScaler
sc = StandardScaler()
X_train = sc.fit_transform(X_train)
X_test = sc.transform(X_test)
plt.hist(X_train)
plt.legend(['Normalized age','Normalized salary'])
plt.show()
We are now ready to fit the SVM to the data. The SVC class allows us to build a kernel SVM model (linear as well as non-linear). The default value of the kernel is the non-linear ‘rbf’.
from sklearn.svm import SVC
classifier = SVC(kernel = 'rbf', random_state = 0)
classifier.fit(X_train, y_train)
SVC(random_state=0)In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
SVC(random_state=0)
After fitting the model, we move to predict the categories for the test cases using such model:
y_pred = classifier.predict(X_test)
y_pred
array([0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 1,
0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0,
1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 1, 0, 0, 1, 0, 0, 1, 0, 1, 0, 1,
0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 1, 1])
and we build the confusion matrix to see how well the prediction went
from sklearn.metrics import confusion_matrix, accuracy_score
cm = confusion_matrix(y_test, y_pred)
print(cm)
accuracy_score(y_test,y_pred)
[[55 3]
[ 1 21]]
0.95
Finally, we visualize the results
from matplotlib.colors import ListedColormap
X_set, y_set = X_test, y_test
X1, X2 = np.meshgrid(np.arange(start = X_set[:, 0].min() - 1, stop = X_set[:, 0].max() + 1, step = 0.01),
np.arange(start = X_set[:, 1].min() - 1, stop = X_set[:, 1].max() + 1, step = 0.01))
plt.contourf(X1, X2, classifier.predict(np.array([X1.ravel(), X2.ravel()]).T).reshape(X1.shape),
alpha = 0.75, cmap = ListedColormap(('red', 'green')))
plt.xlim(X1.min(), X1.max())
plt.ylim(X2.min(), X2.max())
for i, j in enumerate(np.unique(y_set)):
plt.scatter(X_set[y_set == j, 0], X_set[y_set == j, 1],
c = ListedColormap(('red', 'green'))(i), label = j)
plt.title('SVM (Test set)')
plt.xlabel('Age')
plt.ylabel('Estimated Salary')
plt.legend()
plt.show()
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
Cell In[14], line 5
2 X_set, y_set = X_test, y_test
3 X1, X2 = np.meshgrid(np.arange(start = X_set[:, 0].min() - 1, stop = X_set[:, 0].max() + 1, step = 0.01),
4 np.arange(start = X_set[:, 1].min() - 1, stop = X_set[:, 1].max() + 1, step = 0.01))
----> 5 plt.contourf(X1, X2, classifier.predict(np.array([X1.ravel(), X2.ravel()]).T).reshape(X1.shape),
6 alpha = 0.75, cmap = ListedColormap(('red', 'green')))
7 plt.xlim(X1.min(), X1.max())
8 plt.ylim(X2.min(), X2.max())
File ~/miniconda3/lib/python3.11/site-packages/matplotlib/pyplot.py:2938, in contourf(data, *args, **kwargs)
2936 @_copy_docstring_and_deprecators(Axes.contourf)
2937 def contourf(*args, data=None, **kwargs) -> QuadContourSet:
-> 2938 __ret = gca().contourf(
2939 *args, **({"data": data} if data is not None else {}), **kwargs
2940 )
2941 if __ret._A is not None: # type: ignore[attr-defined]
2942 sci(__ret)
File ~/miniconda3/lib/python3.11/site-packages/matplotlib/__init__.py:1465, in _preprocess_data.<locals>.inner(ax, data, *args, **kwargs)
1462 @functools.wraps(func)
1463 def inner(ax, *args, data=None, **kwargs):
1464 if data is None:
-> 1465 return func(ax, *map(sanitize_sequence, args), **kwargs)
1467 bound = new_sig.bind(ax, *args, **kwargs)
1468 auto_label = (bound.arguments.get(label_namer)
1469 or bound.kwargs.get(label_namer))
File ~/miniconda3/lib/python3.11/site-packages/matplotlib/axes/_axes.py:6528, in Axes.contourf(self, *args, **kwargs)
6519 """
6520 Plot filled contours.
6521
(...)
6525 %(contour_doc)s
6526 """
6527 kwargs['filled'] = True
-> 6528 contours = mcontour.QuadContourSet(self, *args, **kwargs)
6529 self._request_autoscale_view()
6530 return contours
File ~/miniconda3/lib/python3.11/site-packages/matplotlib/contour.py:887, in ContourSet.__init__(self, ax, levels, filled, linewidths, linestyles, hatches, alpha, origin, extent, cmap, colors, norm, vmin, vmax, extend, antialiased, nchunk, locator, transform, negative_linestyles, clip_path, *args, **kwargs)
884 self.labelTexts = []
885 self.labelCValues = []
--> 887 self.set_cmap(cmap)
888 if norm is not None:
889 self.set_norm(norm)
File ~/miniconda3/lib/python3.11/site-packages/matplotlib/cm.py:602, in ScalarMappable.set_cmap(self, cmap)
600 self.cmap = _ensure_cmap(cmap)
601 if not in_init:
--> 602 self.changed()
File ~/miniconda3/lib/python3.11/site-packages/matplotlib/contour.py:1145, in ContourSet.changed(self)
1143 def changed(self):
1144 if not hasattr(self, "cvalues"):
-> 1145 self._process_colors() # Sets cvalues.
1146 # Force an autoscale immediately because self.to_rgba() calls
1147 # autoscale_None() internally with the data passed to it,
1148 # so if vmin/vmax are not set yet, this would override them with
1149 # content from *cvalues* rather than levels like we want
1150 self.norm.autoscale_None(self.levels)
File ~/miniconda3/lib/python3.11/site-packages/matplotlib/contour.py:1302, in ContourSet._process_colors(self)
1300 else:
1301 self.cvalues = self.layers
-> 1302 self.norm.autoscale_None(self.levels)
1303 self.set_array(self.cvalues)
1304 self.update_scalarmappable()
File ~/miniconda3/lib/python3.11/site-packages/matplotlib/colors.py:1408, in Normalize.autoscale_None(self, A)
1405 A = A.data
1407 if self.vmin is None and A.size:
-> 1408 self.vmin = A.min()
1409 if self.vmax is None and A.size:
1410 self.vmax = A.max()
File ~/miniconda3/lib/python3.11/site-packages/matplotlib/colors.py:1264, in Normalize.vmin(self, value)
1262 if value != self._vmin:
1263 self._vmin = value
-> 1264 self._changed()
File ~/miniconda3/lib/python3.11/site-packages/matplotlib/colors.py:1292, in Normalize._changed(self)
1287 def _changed(self):
1288 """
1289 Call this whenever the norm is changed to notify all the
1290 callback listeners to the 'changed' signal.
1291 """
-> 1292 self.callbacks.process('changed')
File ~/miniconda3/lib/python3.11/site-packages/matplotlib/cbook.py:303, in CallbackRegistry.process(self, s, *args, **kwargs)
301 except Exception as exc:
302 if self.exception_handler is not None:
--> 303 self.exception_handler(exc)
304 else:
305 raise
File ~/miniconda3/lib/python3.11/site-packages/matplotlib/cbook.py:87, in _exception_printer(exc)
85 def _exception_printer(exc):
86 if _get_running_interactive_framework() in ["headless", None]:
---> 87 raise exc
88 else:
89 traceback.print_exc()
File ~/miniconda3/lib/python3.11/site-packages/matplotlib/cbook.py:298, in CallbackRegistry.process(self, s, *args, **kwargs)
296 if func is not None:
297 try:
--> 298 func(*args, **kwargs)
299 # this does not capture KeyboardInterrupt, SystemExit,
300 # and GeneratorExit
301 except Exception as exc:
File ~/miniconda3/lib/python3.11/site-packages/matplotlib/contour.py:1150, in ContourSet.changed(self)
1145 self._process_colors() # Sets cvalues.
1146 # Force an autoscale immediately because self.to_rgba() calls
1147 # autoscale_None() internally with the data passed to it,
1148 # so if vmin/vmax are not set yet, this would override them with
1149 # content from *cvalues* rather than levels like we want
-> 1150 self.norm.autoscale_None(self.levels)
1151 self.set_array(self.cvalues)
1152 self.update_scalarmappable()
File ~/miniconda3/lib/python3.11/site-packages/matplotlib/colors.py:1410, in Normalize.autoscale_None(self, A)
1408 self.vmin = A.min()
1409 if self.vmax is None and A.size:
-> 1410 self.vmax = A.max()
File ~/miniconda3/lib/python3.11/site-packages/matplotlib/colors.py:1275, in Normalize.vmax(self, value)
1273 if value != self._vmax:
1274 self._vmax = value
-> 1275 self._changed()
File ~/miniconda3/lib/python3.11/site-packages/matplotlib/colors.py:1292, in Normalize._changed(self)
1287 def _changed(self):
1288 """
1289 Call this whenever the norm is changed to notify all the
1290 callback listeners to the 'changed' signal.
1291 """
-> 1292 self.callbacks.process('changed')
File ~/miniconda3/lib/python3.11/site-packages/matplotlib/cbook.py:303, in CallbackRegistry.process(self, s, *args, **kwargs)
301 except Exception as exc:
302 if self.exception_handler is not None:
--> 303 self.exception_handler(exc)
304 else:
305 raise
File ~/miniconda3/lib/python3.11/site-packages/matplotlib/cbook.py:87, in _exception_printer(exc)
85 def _exception_printer(exc):
86 if _get_running_interactive_framework() in ["headless", None]:
---> 87 raise exc
88 else:
89 traceback.print_exc()
File ~/miniconda3/lib/python3.11/site-packages/matplotlib/cbook.py:298, in CallbackRegistry.process(self, s, *args, **kwargs)
296 if func is not None:
297 try:
--> 298 func(*args, **kwargs)
299 # this does not capture KeyboardInterrupt, SystemExit,
300 # and GeneratorExit
301 except Exception as exc:
File ~/miniconda3/lib/python3.11/site-packages/matplotlib/contour.py:1152, in ContourSet.changed(self)
1150 self.norm.autoscale_None(self.levels)
1151 self.set_array(self.cvalues)
-> 1152 self.update_scalarmappable()
1153 alphas = np.broadcast_to(self.get_alpha(), len(self.cvalues))
1154 for label, cv, alpha in zip(self.labelTexts, self.labelCValues, alphas):
File ~/miniconda3/lib/python3.11/site-packages/matplotlib/collections.py:920, in Collection.update_scalarmappable(self)
918 # pcolormesh, scatter, maybe others flatten their _A
919 self._alpha = self._alpha.reshape(self._A.shape)
--> 920 self._mapped_colors = self.to_rgba(self._A, self._alpha)
922 if self._face_is_mapped:
923 self._facecolors = self._mapped_colors
File ~/miniconda3/lib/python3.11/site-packages/matplotlib/cm.py:509, in ScalarMappable.to_rgba(self, x, alpha, bytes, norm)
507 if norm:
508 x = self.norm(x)
--> 509 rgba = self.cmap(x, alpha=alpha, bytes=bytes)
510 return rgba
File ~/miniconda3/lib/python3.11/site-packages/matplotlib/colors.py:725, in Colormap.__call__(self, X, alpha, bytes)
702 r"""
703 Parameters
704 ----------
(...)
722 RGBA values with a shape of ``X.shape + (4, )``.
723 """
724 if not self._isinit:
--> 725 self._init()
727 xa = np.array(X, copy=True)
728 if not xa.dtype.isnative:
729 # Native byteorder is faster.
File ~/miniconda3/lib/python3.11/site-packages/matplotlib/colors.py:1175, in ListedColormap._init(self)
1173 def _init(self):
1174 self._lut = np.zeros((self.N + 3, 4), float)
-> 1175 self._lut[:-3] = to_rgba_array(self.colors)
1176 self._isinit = True
1177 self._set_extremes()
File ~/miniconda3/lib/python3.11/site-packages/matplotlib/colors.py:489, in to_rgba_array(c, alpha)
487 raise e
488 if isinstance(c, str):
--> 489 raise ValueError(f"{c!r} is not a valid color value.")
491 if len(c) == 0:
492 return np.zeros((0, 4), float)
ValueError: 'red' is not a valid color value.
""" svmquad.py """
import numpy as np
from numpy import genfromtxt
import matplotlib.pyplot as plt
from sklearn.svm import SVC
data = genfromtxt('https://raw.githubusercontent.com/DSML-book/Programs/master/Chapter7/svmcirc.csv', delimiter=',')
x = data[:,[0,1]] #vectors are rows
y = data[:,[2]].reshape(len(x),) #labels
tmp = np.sum(np.power(x,2),axis=1).reshape((len(x),1))
z = np.hstack((x,tmp))
#clf = SVC(C = np.inf, kernel='linear')
clf = SVC(C = 1000, kernel='linear')
clf.fit(z,y)
print("Support Vectors \n", clf.support_vectors_)
print("Support Vector Labels ",y[clf.support_])
print("Alpha ",clf.dual_coef_)
print("Bias ",clf.intercept_)
Support Vectors
[[ 0.038758 0.53796 0.29090314]
[-0.49116 -0.20563 0.28352184]
[-0.45068 -0.04797 0.20541358]
[-0.061107 -0.41651 0.17721465]]
Support Vector Labels [-1. -1. 1. 1.]
Alpha [[ -46.49249413 -249.01807328 265.31805855 30.19250886]]
Bias [5.617891]
""" svmkern.py """
import numpy as np
from numpy import genfromtxt
from sklearn.svm import SVC
import matplotlib.pyplot as plt
def mykernel(U,V):
tmpU = np.sum(np.power(U,2),axis=1).reshape((len(U),1))
U = np.hstack((U,tmpU))
tmpV = np.sum(np.power(V,2),axis=1).reshape((len(V),1))
V = np.hstack((V,tmpV))
K = U @ V.T
print(K.shape)
return K
# read in the data
inp = genfromtxt('https://raw.githubusercontent.com/DSML-book/Programs/master/Chapter7/svmcirc.csv', delimiter=',')
data = inp[:,[0,1]] #vectors are rows
y = inp[:,[2]].reshape(len(data),) #labels
#clf = SVC(C = np.inf, kernel=mykernel, gamma='auto')
clf = SVC(C = 1000, kernel="rbf", gamma='scale')
clf.fit(data,y)
print("Support Vectors \n", clf.support_vectors_)
print("Support Vector Labels ",y[clf.support_])
print("Nu",clf.dual_coef_)
print("Bias ",clf.intercept_)
# plot
d = 0.001
fig = plt.figure(figsize=(5,5))
x_min, x_max = -1,1
y_min, y_max = -1,1
xx, yy = np.meshgrid(np.arange(x_min, x_max, d), np.arange(y_min, y_max, d))
plt.plot(data[clf.support_,0],data[clf.support_,1],'go')
plt.plot(data[y==1,0],data[y==1,1],'b.')
plt.plot(data[y==-1,0],data[y==-1,1],'rx')
Z = clf.predict(np.c_[xx.ravel(), yy.ravel()])
Z = Z.reshape(xx.shape)
plt.contour(xx, yy, Z,colors ="k")
plt.show()
# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
fig.savefig("rbf.pdf", bbox_inches='tight')
# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
Support Vectors
[[ 0.038758 0.53796 ]
[-0.49116 -0.20563 ]
[ 0.50858 0.15759 ]
[-0.43742 0.38537 ]
[ 0.34015 -0.57397 ]
[-0.45068 -0.04797 ]
[-0.40977 -0.17625 ]
[ 0.028401 0.41137 ]]
Support Vector Labels [-1. -1. -1. -1. -1. 1. 1. 1.]
Nu [[ -7.79068185 -68.11518916 -1.51725004 -7.03376815 -0.57034285
15.2489841 60.1202509 9.65799706]]
Bias [-3.9763643]
Obtain data from the UCI’s Breast Cancer Wisconsin data set.
Let us have a look at the data with the UCI’s API:
from ucimlrepo import fetch_ucirepo
# fetch dataset
breast_cancer_wisconsin_diagnostic = fetch_ucirepo(id=17)
# data (as pandas dataframes)
X = breast_cancer_wisconsin_diagnostic.data.features
y = breast_cancer_wisconsin_diagnostic.data.targets
# metadata
print(breast_cancer_wisconsin_diagnostic.metadata)
{'uci_id': 17, 'name': 'Breast Cancer Wisconsin (Diagnostic)', 'repository_url': 'https://archive.ics.uci.edu/dataset/17/breast+cancer+wisconsin+diagnostic', 'data_url': 'https://archive.ics.uci.edu/static/public/17/data.csv', 'abstract': 'Diagnostic Wisconsin Breast Cancer Database.', 'area': 'Health and Medicine', 'tasks': ['Classification'], 'characteristics': ['Multivariate'], 'num_instances': 569, 'num_features': 30, 'feature_types': ['Real'], 'demographics': [], 'target_col': ['Diagnosis'], 'index_col': ['ID'], 'has_missing_values': 'no', 'missing_values_symbol': None, 'year_of_dataset_creation': 1993, 'last_updated': 'Fri Nov 03 2023', 'dataset_doi': '10.24432/C5DW2B', 'creators': ['William Wolberg', 'Olvi Mangasarian', 'Nick Street', 'W. Street'], 'intro_paper': {'title': 'Nuclear feature extraction for breast tumor diagnosis', 'authors': 'W. Street, W. Wolberg, O. Mangasarian', 'published_in': 'Electronic imaging', 'year': 1993, 'url': 'https://www.semanticscholar.org/paper/53f0fbb425bc14468eb3bf96b2e1d41ba8087f36', 'doi': '10.1117/12.148698'}, 'additional_info': {'summary': 'Features are computed from a digitized image of a fine needle aspirate (FNA) of a breast mass. They describe characteristics of the cell nuclei present in the image. A few of the images can be found at http://www.cs.wisc.edu/~street/images/\r\n\r\nSeparating plane described above was obtained using Multisurface Method-Tree (MSM-T) [K. P. Bennett, "Decision Tree Construction Via Linear Programming." Proceedings of the 4th Midwest Artificial Intelligence and Cognitive Science Society, pp. 97-101, 1992], a classification method which uses linear programming to construct a decision tree. Relevant features were selected using an exhaustive search in the space of 1-4 features and 1-3 separating planes.\r\n\r\nThe actual linear program used to obtain the separating plane in the 3-dimensional space is that described in: [K. P. Bennett and O. L. Mangasarian: "Robust Linear Programming Discrimination of Two Linearly Inseparable Sets", Optimization Methods and Software 1, 1992, 23-34].\r\n\r\nThis database is also available through the UW CS ftp server:\r\nftp ftp.cs.wisc.edu\r\ncd math-prog/cpo-dataset/machine-learn/WDBC/', 'purpose': None, 'funded_by': None, 'instances_represent': None, 'recommended_data_splits': None, 'sensitive_data': None, 'preprocessing_description': None, 'variable_info': '1) ID number\r\n2) Diagnosis (M = malignant, B = benign)\r\n3-32)\r\n\r\nTen real-valued features are computed for each cell nucleus:\r\n\r\n\ta) radius (mean of distances from center to points on the perimeter)\r\n\tb) texture (standard deviation of gray-scale values)\r\n\tc) perimeter\r\n\td) area\r\n\te) smoothness (local variation in radius lengths)\r\n\tf) compactness (perimeter^2 / area - 1.0)\r\n\tg) concavity (severity of concave portions of the contour)\r\n\th) concave points (number of concave portions of the contour)\r\n\ti) symmetry \r\n\tj) fractal dimension ("coastline approximation" - 1)', 'citation': None}}
# variable information
print(breast_cancer_wisconsin_diagnostic.variables)
name role type demographic description units \
0 ID ID Categorical None None None
1 Diagnosis Target Categorical None None None
2 radius1 Feature Continuous None None None
3 texture1 Feature Continuous None None None
4 perimeter1 Feature Continuous None None None
5 area1 Feature Continuous None None None
6 smoothness1 Feature Continuous None None None
7 compactness1 Feature Continuous None None None
8 concavity1 Feature Continuous None None None
9 concave_points1 Feature Continuous None None None
10 symmetry1 Feature Continuous None None None
11 fractal_dimension1 Feature Continuous None None None
12 radius2 Feature Continuous None None None
13 texture2 Feature Continuous None None None
14 perimeter2 Feature Continuous None None None
15 area2 Feature Continuous None None None
16 smoothness2 Feature Continuous None None None
17 compactness2 Feature Continuous None None None
18 concavity2 Feature Continuous None None None
19 concave_points2 Feature Continuous None None None
20 symmetry2 Feature Continuous None None None
21 fractal_dimension2 Feature Continuous None None None
22 radius3 Feature Continuous None None None
23 texture3 Feature Continuous None None None
24 perimeter3 Feature Continuous None None None
25 area3 Feature Continuous None None None
26 smoothness3 Feature Continuous None None None
27 compactness3 Feature Continuous None None None
28 concavity3 Feature Continuous None None None
29 concave_points3 Feature Continuous None None None
30 symmetry3 Feature Continuous None None None
31 fractal_dimension3 Feature Continuous None None None
missing_values
0 no
1 no
2 no
3 no
4 no
5 no
6 no
7 no
8 no
9 no
10 no
11 no
12 no
13 no
14 no
15 no
16 no
17 no
18 no
19 no
20 no
21 no
22 no
23 no
24 no
25 no
26 no
27 no
28 no
29 no
30 no
31 no
Let us use the direct import of the data in Python to run several classifiers to predict the category of breast masses (bening or malignant):
We create the training set (341 elements) and the test set (228 elements)
""" skclass1.py """
from numpy import genfromtxt
from sklearn.model_selection import train_test_split
# url = "https://archive.ics.uci.edu/dataset/17/breast+cancer+wisconsin+diagnostic"
# name = "wdbc.data"
# data = genfromtxt(url + name,delimiter=',', dtype=str)
data = genfromtxt("datasets/wdbc.data",delimiter=',', dtype=str)
y = data[:,1] #responses
X = data[:,2:].astype('float')#features as an ndarray matrix
X_train , X_test , y_train , y_test = train_test_split(
X, y, test_size = 0.4, random_state = 1234)
print(X_train.shape,X_test.shape)
(341, 30) (228, 30)
""" skclass2.py """
#from skclass1 import X, y
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
import numpy as np
Bidx = np.where(y == 'B')
Midx= np.where(y == 'M')
# plot features Radius (column 0), Texture (1), Concavity (6)
fig = plt.figure()
ax = fig.add_subplot(projection = '3d')
ax.scatter(X[Bidx,0], X[Bidx,1], X[Bidx,6],
c='r', marker='^', label='Benign')
ax.scatter(X[Midx,0], X[Midx,1], X[Midx,6],
c='b', marker='o', label='Malignant')
ax.legend()
ax.set_xlabel('Mean Radius')
ax.set_ylabel('Mean Texture')
ax.set_zlabel('Mean Concavity')
plt.savefig('scatter_bcancer.pdf',format='pdf')
plt.show()
Here we will use severalclassifiers and report their accuracy (percentage of correct predictions). See how QDA perfomrs best for accuracy.
""" skclass3.py """
from sklearn.metrics import accuracy_score
import sklearn.discriminant_analysis as DA
from sklearn.naive_bayes import GaussianNB
from sklearn.neighbors import KNeighborsClassifier
#from sklearn.neighbors import DistanceMetric
from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC
#import numpy as np
#from sklearn.preprocessing import MinMaxScaler
names = ["Logit","NBayes", "LDA", "QDA", "KNN", "SVM"]
#%%
classifiers = [LogisticRegression(C=1e5),
GaussianNB(),
DA.LinearDiscriminantAnalysis(),
DA.QuadraticDiscriminantAnalysis(),
KNeighborsClassifier(n_neighbors=5),
SVC(kernel='rbf', gamma = 1e-4)]
print('Name Accuracy\n'+14*'-')
for name, clf in zip(names, classifiers):
clf.fit(X_train, y_train)
y_pred = clf.predict(X_test)
print('{:6} {:3.3f}'.format(name, accuracy_score(y_test,y_pred)))
Name Accuracy
--------------
Logit 0.934
NBayes 0.908
LDA 0.943
QDA 0.956
/Users/jordivilla/miniconda3/lib/python3.11/site-packages/sklearn/linear_model/_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):
STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.
Increase the number of iterations (max_iter) or scale the data as shown in:
https://scikit-learn.org/stable/modules/preprocessing.html
Please also refer to the documentation for alternative solver options:
https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression
n_iter_i = _check_optimize_result(
KNN 0.925
SVM 0.939
Explore which other types of metrics provided by sklearn may give better appropriate measure for the breast cancer data classification with the different methods. Provide a jupyter notebook with the comparisons, plotting an image similar to this one (taken from the internet from data not related to this exercise at all) but including the methods discussed in the class notes:
Adapted from here.
import matplotlib.pyplot as plt
# roc curve and auc
from sklearn.datasets import make_classification
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split
from sklearn.metrics import roc_curve
from sklearn.metrics import roc_auc_score
# generate 2 class dataset
X, y = make_classification(n_samples=1000, n_classes=2, random_state=1)
# split into train/test sets
trainX, testX, trainy, testy = train_test_split(X, y, test_size=0.5, random_state=2)
# generate a no skill prediction (majority class)
ns_probs = [0 for _ in range(len(testy))]
# fit a model
model = LogisticRegression(solver='lbfgs')
model.fit(trainX, trainy)
# predict probabilities
lr_probs = model.predict_proba(testX)
# keep probabilities for the positive outcome only
lr_probs = lr_probs[:, 1]
# calculate scores
ns_auc = roc_auc_score(testy, ns_probs)
lr_auc = roc_auc_score(testy, lr_probs)
# summarize scores
print('No Skill: ROC AUC=%.3f' % (ns_auc))
print('Logistic: ROC AUC=%.3f' % (lr_auc))
# calculate roc curves
ns_fpr, ns_tpr, _ = roc_curve(testy, ns_probs)
lr_fpr, lr_tpr, _ = roc_curve(testy, lr_probs)
# plot the roc curve for the model
plt.plot(ns_fpr, ns_tpr, linestyle='--', label='No Skill')
plt.plot(lr_fpr, lr_tpr, marker='.', label='Logistic')
# axis labels
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
# show the legend
plt.legend()
# show the plot
plt.show()
No Skill: ROC AUC=0.500
Logistic: ROC AUC=0.903
And the following is an example of a precision-recall curve from the same source:
import matplotlib.pyplot as plt
# precision-recall curve and f1
from sklearn.datasets import make_classification
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split
from sklearn.metrics import precision_recall_curve
from sklearn.metrics import f1_score
from sklearn.metrics import auc
# generate 2 class dataset
X, y = make_classification(n_samples=1000, n_classes=2, random_state=1)
# split into train/test sets
trainX, testX, trainy, testy = train_test_split(X, y, test_size=0.5, random_state=2)
# fit a model
model = LogisticRegression(solver='lbfgs')
model.fit(trainX, trainy)
# predict probabilities
lr_probs = model.predict_proba(testX)
# keep probabilities for the positive outcome only
lr_probs = lr_probs[:, 1]
# predict class values
yhat = model.predict(testX)
lr_precision, lr_recall, _ = precision_recall_curve(testy, lr_probs)
lr_f1, lr_auc = f1_score(testy, yhat), auc(lr_recall, lr_precision)
# summarize scores
print('Logistic: f1=%.3f auc=%.3f' % (lr_f1, lr_auc))
# plot the precision-recall curves
no_skill = len(testy[testy==1]) / len(testy)
plt.plot([0, 1], [no_skill, no_skill], linestyle='--', label='No Skill')
plt.plot(lr_recall, lr_precision, marker='.', label='Logistic')
# axis labels
plt.xlabel('Recall')
plt.ylabel('Precision')
# show the legend
plt.legend()
# show the plot
plt.show()
Logistic: f1=0.841 auc=0.898
