nsaunier/traffic-intelligence: python/utils.py comparison

comparison python/utils.py @ 1012:01db14e947e4

resolved

author	Wendlasida
date	Fri, 01 Jun 2018 10:47:49 -0400
parents	4f3387a242a1
children	16932cefabc1

comparison

equal deleted inserted replaced

-:4f0312bee393
+:01db14e947e4
 return 1
 def sortByLength(instances, reverse = False):
 '''Returns a new list with the instances sorted by length (method __len__)
 reverse is passed to sorted'''
-return sorted(instances, cmp = compareLengthForSort, reverse = reverse)
+return sorted(instances, key = len, reverse = reverse)
 def ceilDecimals(v, nDecimals):
 '''Rounds the number at the nth decimal
 eg 1.23 at 0 decimal is 2, at 1 decimal is 1.3'''
 tens = 10**nDecimals
 newVariable = (var+'_{}'.format(val)).replace('.','').replace(' ','').replace('-','')
 data[newVariable] = (data[var] == val)
 newVariables.append(newVariable)
 return newVariables
-def kruskalWallis(data, dependentVariable, independentVariable, plotFigure = False, filenamePrefix = None, figureFileType = 'pdf', saveLatex = False, renameVariables = lambda s: s, kwCaption = u''):
+def kruskalWallis(data, dependentVariable, independentVariable, plotFigure = False, filenamePrefix = None, figureFileType = 'pdf', saveLatex = False, renameVariables = lambda s: s, kwCaption = ''):
 '''Studies the influence of (nominal) independent variable over the dependent variable
 Makes tests if the conditional distributions are normal
 using the Shapiro-Wilk test (in which case ANOVA could be used)
 Implements uses the non-parametric Kruskal Wallis test'''
 return result
 def saveDokMatrix(filename, m, lowerTriangle = False):
 'Saves a dok_matrix using savez'
 if lowerTriangle:
-keys = [k for k in m.keys() if k[0] > k[1]]
+keys = [k for k in m if k[0] > k[1]]
 savez(filename, shape = m.shape, keys = keys, values = [m[k[0],k[1]] for k in keys])
 else:
-savez(filename, shape = m.shape, keys = m.keys(), values = m.values())
+savez(filename, shape = m.shape, keys = list(m.keys()), values = list(m.values()))
 def loadDokMatrix(filename):
 'Loads a dok_matrix saved using the above saveDokMatrix'
 data = npload(filename)
 m = dok_matrix(tuple(data['shape']))
 to the binary code derived from the independent variables'''
 from numpy.random import permutation as nppermutation
 if experiments is None:
 experiments = generateExperiments(independentVariables)
 nIndependentVariables = len(independentVariables)
-permutation = nppermutation(range(nIndependentVariables)).tolist()
+permutation = nppermutation(list(range(nIndependentVariables)))
 variableMapping = {j: independentVariables[i] for i,j in enumerate(permutation)}
-print('Tested variables '+', '.join([variableMapping[i] for i in xrange(nIndependentVariables)]))
+print('Tested variables '+', '.join([variableMapping[i] for i in range(nIndependentVariables)]))
 bestModel = [False]*nIndependentVariables
 currentVarNum = 0
 currentR2Adj = 0.
-for currentVarNum in xrange(nIndependentVariables):
+for currentVarNum in range(nIndependentVariables):
 currentModel = [i for i in bestModel]
 currentModel[currentVarNum] = True
-rowIdx = sum([0]+[2**i for i in xrange(nIndependentVariables) if currentModel[permutation[i]]])
+rowIdx = sum([0]+[2**i for i in range(nIndependentVariables) if currentModel[permutation[i]]])
-#print currentVarNum, sum(currentModel), ', '.join([independentVariables[i] for i in xrange(nIndependentVariables) if currentModel[permutation[i]]])
+#print currentVarNum, sum(currentModel), ', '.join([independentVariables[i] for i in range(nIndependentVariables) if currentModel[permutation[i]]])
 if experiments.loc[rowIdx, 'shapiroP'] < 0:
 modelStr = modelString(experiments.loc[rowIdx], dependentVariable, independentVariables)
 model = modelFunc(modelStr, data = data)
 results = model.fit()
 experiments.loc[rowIdx, 'r2adj'] = results.rsquared_adj
 def similarities(self, l1, l2, jshift=0):
 n1 = len(l1)
 n2 = len(l2)
 self.similarityTable = zeros((n1+1,n2+1), dtype = npint)
 if self.similarityFunc is not None:
-for i in xrange(1,n1+1):
+for i in range(1,n1+1):
-for j in xrange(max(1,i-jshift-self.delta),min(n2,i-jshift+self.delta)+1):
+for j in range(max(1,i-jshift-self.delta),min(n2,i-jshift+self.delta)+1):
 if self.similarityFunc(l1[i-1], l2[j-1]):
 self.similarityTable[i,j] = self.similarityTable[i-1,j-1]+1
 else:
 self.similarityTable[i,j] = max(self.similarityTable[i-1,j], self.similarityTable[i,j-1])
 elif self.metric is not None:
 similarElements = distance.cdist(l1, l2, self.metric) <= self.epsilon
-for i in xrange(1,n1+1):
+for i in range(1,n1+1):
-for j in xrange(max(1,i-jshift-self.delta),min(n2,i-jshift+self.delta)+1):
+for j in range(max(1,i-jshift-self.delta),min(n2,i-jshift+self.delta)+1):
 if similarElements[i-1, j-1]:
 self.similarityTable[i,j] = self.similarityTable[i-1,j-1]+1
 else:
 self.similarityTable[i,j] = max(self.similarityTable[i-1,j], self.similarityTable[i,j-1])
 n2 = len(l2)
 if self.aligned:
 lcssValues = {}
 similarityTables = {}
-for i in xrange(-n2-self.delta+1, n1+self.delta): # interval such that [i-shift-delta, i-shift+delta] is never empty, which happens when i-shift+delta < 1 or when i-shift-delta > n2
+for i in range(-n2-self.delta+1, n1+self.delta): # interval such that [i-shift-delta, i-shift+delta] is never empty, which happens when i-shift+delta < 1 or when i-shift-delta > n2
 self.similarities(l1, l2, i)
 lcssValues[i] = self.similarityTable.max()
 similarityTables[i] = self.similarityTable
 #print self.similarityTable
 alignmentShift = argmaxDict(lcssValues) # ideally get the medium alignment shift, the one that minimizes distance
 monochrome = (cycler('color', ['k']) * cycler('linestyle', ['-', '--', ':', '-.']))
 plt.rc('axes', prop_cycle=monochrome)
 def plotIndicatorMap(indicatorMap, squareSize, masked = True, defaultValue=-1):
 from matplotlib.pyplot import pcolor
-coords = array(indicatorMap.keys())
+coords = array(list(indicatorMap.keys()))
 minX = min(coords[:,0])
 minY = min(coords[:,1])
 X = arange(minX, max(coords[:,0])+1.1)*squareSize
 Y = arange(minY, max(coords[:,1])+1.1)*squareSize
 C = defaultValue*ones((len(Y), len(X)))
-for k,v in indicatorMap.iteritems():
+for k,v in indicatorMap.items():
 C[k[1]-minY,k[0]-minX] = v
 if masked:
 pcolor(X, Y, ma.masked_where(C==defaultValue,C))
 else:
 pcolor(X, Y, C)
 To get hourly data for 2009 and 2012, January, March and October, downloadECWeather(10761, [2009,2012], [1,3,10], '/tmp')
 for annee in `seq 2016 2017`;do wget --content-disposition "http://climat.meteo.gc.ca/climate_data/bulk_data_f.html?format=csv&stationID=10761&Year=${annee}&timeframe=2&submit=++T%C3%A9l%C3%A9charger+%0D%0Ades+donn%C3%A9es" ;done
 for annee in `seq 2016 2017`;do for mois in `seq 1 12`;do wget --content-disposition "http://climat.meteo.gc.ca/climate_data/bulk_data_f.html?format=csv&stationID=10761&Year=${annee}&Month=${mois}&timeframe=1&submit=++T%C3%A9l%C3%A9charger+%0D%0Ades+donn%C3%A9es" ;done;done
 '''
-import urllib2
+import urllib.request
 if english:
 language = 'e'
 else:
 language = 'f'
 if len(months) == 0:
 else:
 timeFrame = 1
 for year in years:
 for month in months:
-url = urllib2.urlopen('http://climate.weather.gc.ca/climate_data/bulk_data_{}.html?format=csv&stationID={}&Year={}&Month={}&Day=1&timeframe={}&submit= Download+Data'.format(language, stationID, year, month, timeFrame))
-#http://climat.meteo.gc.ca/climateData/bulkdata_{}.html?format=csv&stationID={}&Year={}&Month={}&Day=1&timeframe={}&submit=++T%C3%A9l%C3%A9charger+%0D%0Ades+donn%C3%A9es
-data = url.read()
 outFilename = '{}/{}-{}'.format(outputDirectoryname, stationID, year)
 if timeFrame == 1:
 outFilename += '-{}-hourly'.format(month)
 else:
 outFilename += '-daily'
 outFilename += '.csv'
-out = open(outFilename, 'w')
+url = urllib.request.urlretrieve('http://climate.weather.gc.ca/climate_data/bulk_data_{}.html?format=csv&stationID={}&Year={}&Month={}&Day=1&timeframe={}&submit=Download+Data'.format(language, stationID, year, month, timeFrame), outFilename)
-out.write(data)
-out.close()
 #########################
 # File I/O
 #########################
 return [float(x) for x in l.split(separator)]
 def line2Ints(l, separator=' '):
 '''Returns the list of ints corresponding to the string'''
 return [int(x) for x in l.split(separator)]
-#########################
-# CLI utils
-#########################
-def parseCLIOptions(helpMessage, options, cliArgs, optionalOptions=[]):
-''' Simple function to handle similar argument parsing
-Returns the dictionary of options and their values
-* cliArgs are most likely directly sys.argv
-(only the elements after the first one are considered)
-* options should be a list of strings for getopt options,
-eg ['frame=','correspondences=','video=']
-A value must be provided for each option, or the program quits'''
-import sys, getopt
-from numpy.core.fromnumeric import all
-optionValues, args = getopt.getopt(cliArgs[1:], 'h', ['help']+options+optionalOptions)
-optionValues = dict(optionValues)
-if '--help' in optionValues.keys() or '-h' in optionValues.keys():
-print(helpMessage+
-'\n - Compulsory options: '+' '.join([opt.replace('=','') for opt in options])+
-'\n - Non-compulsory options: '+' '.join([opt.replace('=','') for opt in optionalOptions]))
-sys.exit()
-missingArgument = [('--'+opt.replace('=','') in optionValues.keys()) for opt in options]
-if not all(missingArgument):
-print('Missing argument')
-print(optionValues)
-sys.exit()
-return optionValues
 #########################
 # Profiling
 #########################

Mercurial > hg > nsaunier > traffic-intelligence

comparison python/utils.py @ 1012:01db14e947e4