# HG changeset patch
# User Nicolas Saunier <nicolas.saunier@polymtl.ca>
# Date 1601049419 14400
# Node ID f7fbe624fff72b488b4356f00792b96edbe5be72
# Parent  fd729e8f073c0d06307d3c87efa51956ff0c69ff
added helper functions for categorical variables

diff -r fd729e8f073c -r f7fbe624fff7 trafficintelligence/utils.py
--- a/trafficintelligence/utils.py	Wed Sep 09 23:47:57 2020 -0400
+++ b/trafficintelligence/utils.py	Fri Sep 25 11:56:59 2020 -0400
@@ -8,12 +8,12 @@
 from copy import deepcopy, copy
 from collections import Counter
 
-from scipy.stats import rv_continuous, kruskal, shapiro, lognorm, norm, t
+from scipy.stats import rv_continuous, kruskal, shapiro, lognorm, norm, t, chi2_contingency
 from scipy.spatial import distance
 from scipy.sparse import dok_matrix
 from numpy import zeros, array, exp, sum as npsum, int as npint, arange, cumsum, mean, median, percentile, isnan, ones, convolve,  dtype, isnan, NaN, ma, isinf, savez, load as npload, log, polyfit, float as npfloat
 from numpy.random import random_sample, permutation as nppermutation
-from pandas import DataFrame, concat
+from pandas import DataFrame, concat, crosstab
 import matplotlib.pyplot as plt
 
 datetimeFormat = "%Y-%m-%d %H:%M:%S"
@@ -654,7 +654,33 @@
         sample.append(x)
         s += x
     return sample
-    
+
+def cramers_v(x, y):
+    """ calculate Cramers V statistic for categorial-categorial association.
+        uses correction from Bergsma and Wicher,
+        Journal of the Korean Statistical Society 42 (2013): 323-328
+    https://towardsdatascience.com/the-search-for-categorical-correlation-a1cf7f1888c9
+    https://stackoverflow.com/questions/46498455/categorical-features-correlation/46498792#46498792
+    """
+    confusionMatrix = crosstab(x,y)
+    chi2 = chi2_contingency(confusionMatrix)[0]
+    n = confusionMatrix.sum().sum()
+    phi2 = chi2/n
+    r,k = confusionMatrix.shape
+    phi2corr = max(0, phi2-((k-1)*(r-1))/(n-1))
+    rcorr = r-((r-1)**2)/(n-1)
+    kcorr = k-((k-1)**2)/(n-1)
+    return sqrt(phi2corr/min((kcorr-1),(rcorr-1)))
+
+def categoricalCorrelationMatrix(data, categoricalVariables):
+    'Returns correlation matrix for the categorical variables'
+    corr = np.ones((len(categoricalVariables), len(categoricalVariables)))
+    for i in range(len(categoricalVariables)):
+        for j in range(i):
+            corr[i,j] = utils.cramers_v(petDf[categoricalVariables[i]], petDf[categoricalVariables[j]])
+            corr[j,i] = corr[i,j]
+    return corr
+
 #########################
 # regression analysis using statsmodels (and pandas)
 #########################