lab_2_Vika

import math
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.preprocessing import StandardScaler
from sklearn.cluster import KMeans
from sklearn import metrics
from sklearn.decomposition import PCA
from seaborn import scatterplot as scatter
from sklearn.manifold import TSNE

# Check if value is nan
def isNa(value):
    if isinstance(value, int) or isinstance(value, float):
        if math.isnan(value) and value != 0:
            return True
    return False

# Drop maxNaPercents percents of nan values in DataFrame
def dropNaRowByPercent (df, maxNaPercents):
    for y, colObject in df.iteritems():
        countOfNa = 0;
        countOfRows = len(colObject);
        for key in colObject:
            if isNa(key):
                countOfNa += 1
        percentNaInRow = (countOfNa / countOfRows) * 100;
        if percentNaInRow > maxNaPercents:
            #print('deleting', colObject.name)
            df = df.drop(colObject.name, axis=1)
    return df 

# Generate value instead nan
def generateValue(naPosX, naPosY, rowCount):
    distances = []
    for x, naFreeRow in df_dropNa.iterrows():
        res = 0.0000000000001
        for y, value in df_res.iloc[naPosX].iteritems():
            if (isNa(value) != True) and (isinstance(value, int) or isinstance(value, float)):
                res += abs(naFreeRow[y] - value)
        distances.append(res/rowCount)
    #print(distances)
    inverseDistancesSum = 0
    for distance in distances:
        inverseDistancesSum += 1/distance 
    affiliationLevels = []
    for distance in distances:
        affiliationLevels.append((1/distance)/inverseDistancesSum)
    
    naValue = 0
    iterator = 0
    for x, value in df_dropNa[naPosY].iteritems():
        naValue += value * affiliationLevels[iterator]
        iterator += 1
    return naValue
                    
#%%
file = pd.ExcelFile('Dataset.xlsx')
df = pd.read_excel(file, sheet_name='Лист1', header=1);
df_res = dropNaRowByPercent(df, 4);     
df_dropNa = df_res.dropna()
resultData = df_res.copy()  

for x, row in df_res.iterrows():
   for y, value in row.iteritems():
        if isNa(value):
            resultData.loc[x, y] = generateValue(x, y, len(df_dropNa.index) + 1)

#lab_2 
#%%        
symptoms = resultData.loc[:].drop(columns=['№иб']).drop(columns=['Діагноз']).values
diagnoses = resultData.loc[:,['Діагноз']].replace('ХОЗЛ', 0).replace('БА', 2).replace('Пневмонія', 1).values
x_norm = StandardScaler().fit_transform(symptoms)
kmeans = KMeans(n_clusters = 3, init = 'k-means++', random_state = 77)
kMeansPredicted = kmeans.fit_predict(x_norm)
expected = diagnoses
predicted = kMeansPredicted
print(metrics.classification_report(expected, predicted))
print(metrics.confusion_matrix(expected, predicted))

#PCA visual function
#%%

def showPCA(data_right, title_right):
    pca = PCA(n_components=2)
    pca_diagnosis = pca.fit_transform(x_norm)
    principalDf = pd.DataFrame(data = pca_diagnosis [:, 0:2], columns = ['principal component 1', 'principal component 2'])
    pcaDF = pd.concat([principalDf, pd.DataFrame(data = diagnoses)], axis=1)                    
    f, axes = plt.subplots(1, 2, figsize=(15,7))
    scatter(pcaDF.loc[:, 'principal component 1'], pcaDF.loc[:, 'principal component 2'], ax=axes[0], hue=diagnoses[:,0])
    scatter(pcaDF.loc[:, 'principal component 1'], pcaDF.loc[:, 'principal component 2'], ax=axes[1], hue=data_right)
    axes[0].set_title('Diagnoses')
    axes[1].set_title(title_right)
    plt.show()
    
 

#PCA diagnose + kmeans
#%%
pca = PCA(n_components=2)
pca_diagnosis = pca.fit_transform(x_norm)
principalDf = pd.DataFrame(data = pca_diagnosis [:, 0:2], columns = ['principal component 1', 'principal component 2'])

pcaDF = pd.concat([principalDf, pd.DataFrame(data = diagnoses)], axis=1)                    
centers = kmeans.cluster_centers_
f, axes = plt.subplots(1, 2, figsize=(15,7))
scatter(pcaDF.loc[:, 'principal component 1'], pcaDF.loc[:, 'principal component 2'], ax=axes[0], hue=diagnoses[:,0])
scatter(pcaDF.loc[:, 'principal component 1'], pcaDF.loc[:, 'principal component 2'], ax=axes[1], hue=kMeansPredicted)
scatter(centers[:,0], centers[:,1], ax=axes[1], marker='s',s=200)
axes[0].set_title('Diagnoses')
axes[1].set_title('Method Kmeans with centers')
plt.show()

#TSNE diagnose + kmeans
#%%
tsne = TSNE (n_components = 2, perplexity = 18, n_iter=1000, random_state = 33, learning_rate = 100)
x_2D = tsne.fit_transform(x_norm)
tsneDf = pd.DataFrame(data = x_2D, columns = ['dim 1', 'dim 2'])
final_tsneDf = pd.concat([tsneDf,  pd.DataFrame(data = diagnoses)], axis = 1)
f, axes = plt.subplots(1, 2, figsize=(15,7))
scatter(final_tsneDf.loc[:, 'dim 1'], final_tsneDf.loc[:, 'dim 2'], ax=axes[0], hue=diagnoses[:,0])
scatter(final_tsneDf.loc[:,'dim 1'], final_tsneDf.loc[:, 'dim 2'], ax=axes[1], hue=kMeansPredicted)
scatter(centers[:,0], centers[:,1], ax=axes[1], marker="s",s=200)
axes[0].set_title('Diagnoses')
axes[1].set_title('Method Kmeans with centers')
plt.show()

#PCA diagnose + DBSCAN
#%%
from sklearn.cluster import DBSCAN
dbscan = DBSCAN(eps=9, min_samples=3)
dbscan.fit(x_norm)
showPCA(dbscan.labels_, 'Method DBSAN')


#TSNE diagnose + DBSCAN
#%%
tsne = TSNE (n_components = 2, perplexity = 18, n_iter=1000, random_state = 33, learning_rate = 100)
x_2D = tsne.fit_transform(x_norm)
tsneDf = pd.DataFrame(data = x_2D, columns = ['dim 1', 'dim 2'])
final_tsneDf = pd.concat([tsneDf,  pd.DataFrame(data = diagnoses)], axis = 1)
f, axes = plt.subplots(1, 2, figsize=(15,7))
scatter(final_tsneDf.loc[:, 'dim 1'], final_tsneDf.loc[:, 'dim 2'], ax=axes[0], hue=diagnoses[:,0])
scatter(final_tsneDf.loc[:, 'dim 1'], final_tsneDf.loc[:, 'dim 2'], ax=axes[1], hue=dbscan.labels_)
axes[0].set_title('Diagnoses')
axes[1].set_title('Method DBSAN')
plt.show()

#Getting Metrics
#%%
dbscan_predicted = dbscan.labels_

def getMetrics(methodName, predictedData):
    print('-------------------------------------', methodName, '-------------------------------------')
    print('adjusted_rand_score: ', metrics.adjusted_rand_score(expected[:,0], predictedData))
    print('adjusted_mutual_info_score: ', metrics.adjusted_mutual_info_score(expected[:,0], predictedData))
    print('homogeneity_score: ', metrics.homogeneity_score(expected[:,0], predictedData))
    print('completeness_score: ', metrics.completeness_score(expected[:,0], predictedData))
    print('v_measure_score: ', metrics.v_measure_score(expected[:,0], predictedData))
    print('silhouette_score: ', metrics.silhouette_score(x_norm, predictedData))

getMetrics('DBSCAN', dbscan_predicted)
getMetrics('KMEANS', kMeansPredicted)