Mercurial > hg > nsaunier > traffic-intelligence

comparison scripts/process.py @ 1023:a13f47c8931d

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
work on processing large datasets (generate speed data)
author Nicolas Saunier <nicolas.saunier@polymtl.ca>
date Wed, 06 Jun 2018 16:51:15 -0400
parents 16932cefabc1
children 73b124160911
comparison
equal deleted inserted replaced
1022:b7689372c0ec 1023:a13f47c8931d
2 2
3 import sys, argparse 3 import sys, argparse
4 from pathlib import Path 4 from pathlib import Path
5 from multiprocessing.pool import Pool 5 from multiprocessing.pool import Pool
6 6
7 import matplotlib 7 #import matplotlib
8 matplotlib.use('Agg') 8 #atplotlib.use('Agg')
9 import matplotlib.pyplot as plt 9 import matplotlib.pyplot as plt
10 from numpy import percentile 10 from numpy import percentile
11 11 from pandas import DataFrame
12 import storage, events, prediction, cvutils 12
13 import storage, events, prediction, cvutils, utils
13 from metadata import * 14 from metadata import *
14 15
15 parser = argparse.ArgumentParser(description='This program manages the processing of several files based on a description of the sites and video data in an SQLite database following the metadata module.') 16 parser = argparse.ArgumentParser(description='This program manages the processing of several files based on a description of the sites and video data in an SQLite database following the metadata module.')
17 # input
16 parser.add_argument('--db', dest = 'metadataFilename', help = 'name of the metadata file', required = True) 18 parser.add_argument('--db', dest = 'metadataFilename', help = 'name of the metadata file', required = True)
17 parser.add_argument('--videos', dest = 'videoIds', help = 'indices of the video sequences', nargs = '*', type = int) 19 parser.add_argument('--videos', dest = 'videoIds', help = 'indices of the video sequences', nargs = '*', type = int)
18 parser.add_argument('--sites', dest = 'siteIds', help = 'indices of the video sequences', nargs = '*', type = int) 20 parser.add_argument('--sites', dest = 'siteIds', help = 'indices of the video sequences', nargs = '*', type = int)
19 parser.add_argument('--cfg', dest = 'configFilename', help = 'name of the configuration file') 21
20 parser.add_argument('-n', dest = 'nObjects', help = 'number of objects/interactions to process', type = int) 22 # main function
21 parser.add_argument('--prediction-method', dest = 'predictionMethod', help = 'prediction method (constant velocity (cvd: vector computation (approximate); cve: equation solving; cv: discrete time (approximate)), normal adaptation, point set prediction)', choices = ['cvd', 'cve', 'cv', 'na', 'ps', 'mp'])
22 parser.add_argument('--pet', dest = 'computePET', help = 'computes PET', action = 'store_true')
23 # override other tracking config, erase sqlite?
24 parser.add_argument('--delete', dest = 'delete', help = 'data to delete', choices = ['feature', 'object', 'classification', 'interaction']) 23 parser.add_argument('--delete', dest = 'delete', help = 'data to delete', choices = ['feature', 'object', 'classification', 'interaction'])
25 parser.add_argument('--process', dest = 'process', help = 'data to process', choices = ['feature', 'object', 'classification', 'interaction']) 24 parser.add_argument('--process', dest = 'process', help = 'data to process', choices = ['feature', 'object', 'classification', 'interaction'])
26 parser.add_argument('--display', dest = 'display', help = 'data to display (replay over video)', choices = ['feature', 'object', 'classification', 'interaction']) 25 parser.add_argument('--display', dest = 'display', help = 'data to display (replay over video)', choices = ['feature', 'object', 'classification', 'interaction'])
27 parser.add_argument('--analyze', dest = 'analyze', help = 'data to analyze (results)', choices = ['feature', 'object', 'classification', 'interaction']) 26 parser.add_argument('--analyze', dest = 'analyze', help = 'data to analyze (results)', choices = ['feature', 'object', 'classification', 'interaction'])
27
28 # common options
29 parser.add_argument('--cfg', dest = 'configFilename', help = 'name of the configuration file')
30 parser.add_argument('-n', dest = 'nObjects', help = 'number of objects/interactions to process', type = int)
28 parser.add_argument('--dry', dest = 'dryRun', help = 'dry run of processing', action = 'store_true') 31 parser.add_argument('--dry', dest = 'dryRun', help = 'dry run of processing', action = 'store_true')
29 parser.add_argument('--nthreads', dest = 'nProcesses', help = 'number of processes to run in parallel', type = int, default = 1) 32 parser.add_argument('--nthreads', dest = 'nProcesses', help = 'number of processes to run in parallel', type = int, default = 1)
33
34 # analysis options
35 parser.add_argument('--output', dest = 'output', help = 'kind of output to produce (interval means)', choices = ['figure', 'interval', 'event'])
36 parser.add_argument('--min-user-duration', dest = 'minUserDuration', help = 'mininum duration we have to see the user to take into account in the analysis (s)', type = float, default = 0.1)
37 parser.add_argument('--interval-duration', dest = 'intervalDuration', help = 'length of time interval to aggregate data (min)', type = float, default = 15.)
38 parser.add_argument('--aggregation', dest = 'aggMethod', help = 'aggregation method per user/event and per interval', choices = ['mean', 'median', 'centile'], nargs = '*', default = ['median'])
39 parser.add_argument('--aggregation-centile', dest = 'aggCentiles', help = 'centile(s) to compute from the observations', nargs = '*', type = int)
40 dpi = 150
41 # unit of analysis: site or video sequence?
42
43 # safety analysis
44 parser.add_argument('--prediction-method', dest = 'predictionMethod', help = 'prediction method (constant velocity (cvd: vector computation (approximate); cve: equation solving; cv: discrete time (approximate)), normal adaptation, point set prediction)', choices = ['cvd', 'cve', 'cv', 'na', 'ps', 'mp'])
45 parser.add_argument('--pet', dest = 'computePET', help = 'computes PET', action = 'store_true')
46 # override other tracking config, erase sqlite?
30 47
31 # need way of selecting sites as similar as possible to sql alchemy syntax 48 # need way of selecting sites as similar as possible to sql alchemy syntax
32 # override tracking.cfg from db 49 # override tracking.cfg from db
33 # manage cfg files, overwrite them (or a subset of parameters) 50 # manage cfg files, overwrite them (or a subset of parameters)
34 # delete sqlite files 51 # delete sqlite files
50 for cv in site.cameraViews: 67 for cv in site.cameraViews:
51 videoSequences += cv.videoSequences 68 videoSequences += cv.videoSequences
52 else: 69 else:
53 print('No video/site to process') 70 print('No video/site to process')
54 71
72 if args.nProcesses > 1:
73 pool = Pool(args.nProcesses)
74
55 ################################# 75 #################################
56 # Delete 76 # Delete
57 ################################# 77 #################################
58 if args.delete is not None: 78 if args.delete is not None:
59 if args.delete == 'feature': 79 if args.delete == 'feature':
79 else: 99 else:
80 cvutils.tracking(configFilename, args.process == 'object', str(parentPath.absolute()/vs.getVideoSequenceFilename()), str(parentPath.absolute()/vs.getDatabaseFilename()), str(parentPath.absolute()/vs.cameraView.getHomographyFilename()), str(parentPath.absolute()/vs.cameraView.getMaskFilename()), True, vs.cameraView.cameraType.intrinsicCameraMatrix, vs.cameraView.cameraType.distortionCoefficients, args.dryRun) 100 cvutils.tracking(configFilename, args.process == 'object', str(parentPath.absolute()/vs.getVideoSequenceFilename()), str(parentPath.absolute()/vs.getDatabaseFilename()), str(parentPath.absolute()/vs.cameraView.getHomographyFilename()), str(parentPath.absolute()/vs.cameraView.getMaskFilename()), True, vs.cameraView.cameraType.intrinsicCameraMatrix, vs.cameraView.cameraType.distortionCoefficients, args.dryRun)
81 else: 101 else:
82 print('SQLite already exists: {}'.format(parentPath/vs.getDatabaseFilename())) 102 print('SQLite already exists: {}'.format(parentPath/vs.getDatabaseFilename()))
83 else: 103 else:
84 pool = Pool(args.nProcesses)
85 for vs in videoSequences: 104 for vs in videoSequences:
86 if not (parentPath/vs.getDatabaseFilename()).exists() or args.process == 'object': 105 if not (parentPath/vs.getDatabaseFilename()).exists() or args.process == 'object':
87 if args.configFilename is None: 106 if args.configFilename is None:
88 configFilename = str(parentPath/vs.cameraView.getTrackingConfigurationFilename()) 107 configFilename = str(parentPath/vs.cameraView.getTrackingConfigurationFilename())
89 else: 108 else:
123 # pool.close() 142 # pool.close()
124 143
125 ################################# 144 #################################
126 # Analyze 145 # Analyze
127 ################################# 146 #################################
128 if args.analyze == 'object': # user speed for now 147 if args.analyze == 'object':
129 medianSpeeds = {} 148 # user speeds, accelerations
130 speeds85 = {} 149 # aggregation per site
131 minLength = 2*30 150 data = [] # list of observation per site-user with time
151 headers = ['sites', 'date', 'time', 'user_type']
152 aggFunctions = {}
153 for method in args.aggMethod:
154 if method == 'centile':
155 aggFunctions[method] = utils.aggregationFunction(method, args.aggCentiles)
156 for c in args.aggCentiles:
157 headers.append('{}{}'.format(method,c))
158 else:
159 aggFunctions[method] = utils.aggregationFunction(method)
160 headers.append(method)
132 for vs in videoSequences: 161 for vs in videoSequences:
133 if not vs.cameraView.siteIdx in medianSpeeds: 162 d = vs.startTime.date()
134 medianSpeeds[vs.cameraView.siteIdx] = [] 163 t1 = vs.startTime.time()
135 speeds85[vs.cameraView.siteIdx] = [] 164 minUserDuration = args.minUserDuration*vs.cameraView.cameraType.frameRate
136 print('Extracting speed from '+vs.getDatabaseFilename()) 165 print('Extracting speed from '+vs.getDatabaseFilename())
137 objects = storage.loadTrajectoriesFromSqlite(str(parentPath/vs.getDatabaseFilename()), 'object') 166 objects = storage.loadTrajectoriesFromSqlite(str(parentPath/vs.getDatabaseFilename()), 'object', args.nObjects)
138 for o in objects: 167 for o in objects:
139 if o.length() > minLength: 168 if o.length() > minUserDuration:
140 speeds = 30*3.6*percentile(o.getSpeeds(), [50, 85]) 169 row = [vs.cameraView.siteIdx, d, utils.framesToTime(o.getFirstInstant(), vs.cameraView.cameraType.frameRate, t1), o.getUserType()]
141 medianSpeeds[vs.cameraView.siteIdx].append(speeds[0]) 170 tmp = o.getSpeeds()
142 speeds85[vs.cameraView.siteIdx].append(speeds[1]) 171 for method,func in aggFunctions.items():
143 for speeds, name in zip([medianSpeeds, speeds85], ['Median', '85th Centile']): 172 aggSpeeds = vs.cameraView.cameraType.frameRate*3.6*func(tmp)
144 plt.ioff() 173 if method == 'centile':
145 plt.figure() 174 row += aggSpeeds.tolist()
146 plt.boxplot(list(speeds.values()), labels = [session.query(Site).get(siteId).name for siteId in speeds]) 175 else:
147 plt.ylabel(name+' Speeds (km/h)') 176 row.append(aggSpeeds)
148 plt.savefig(name.lower()+'-speeds.png', dpi=150) 177 data.append(row)
149 plt.close() 178 data = DataFrame(data, columns = headers)
150 179 if args.siteIds is None:
180 siteIds = set([vs.cameraView.siteIdx for vs in videoSequences])
181 else:
182 siteIds = set(args.siteIds)
183 if args.output == 'figure':
184 for name in headers[4:]:
185 plt.ioff()
186 plt.figure()
187 plt.boxplot([data.loc[data['sites']==siteId, name] for siteId in siteIds], labels = [session.query(Site).get(siteId).name for siteId in siteIds])
188 plt.ylabel(name+' Speeds (km/h)')
189 plt.savefig(name.lower()+'-speeds.png', dpi=dpi)
190 plt.close()
191 elif args.output == 'event':
192 data.to_csv('speeds.csv', index = False)
151 if args.analyze == 'interaction': 193 if args.analyze == 'interaction':
152 indicatorIds = [2,5,7,10] 194 indicatorIds = [2,5,7,10]
153 conversionFactors = {2: 1., 5: 30.*3.6, 7:1./30, 10:1./30} 195 conversionFactors = {2: 1., 5: 30.*3.6, 7:1./30, 10:1./30}
154 maxIndicatorValue = {2: float('inf'), 5: float('inf'), 7:10., 10:10.} 196 maxIndicatorValue = {2: float('inf'), 5: float('inf'), 7:10., 10:10.}
155 indicators = {} 197 indicators = {}