Mercurial > hg > nsaunier > traffic-intelligence

comparison scripts/process.py @ 1161:b968c33f8c2f

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
work on producing figures and collision maps for large datasets with multiple sites
author Nicolas Saunier <nicolas.saunier@polymtl.ca>
date Thu, 04 Mar 2021 23:12:34 -0500
parents 8734742c08c0
children
comparison
equal deleted inserted replaced
1160:3a6b00f13e95 1161:b968c33f8c2f
22 # main function 22 # main function
23 parser.add_argument('--delete', dest = 'delete', help = 'data to delete', choices = ['feature', 'object', 'classification', 'interaction']) 23 parser.add_argument('--delete', dest = 'delete', help = 'data to delete', choices = ['feature', 'object', 'classification', 'interaction'])
24 parser.add_argument('--process', dest = 'process', help = 'data to process', choices = ['feature', 'object', 'classification', 'prototype', 'interaction']) 24 parser.add_argument('--process', dest = 'process', help = 'data to process', choices = ['feature', 'object', 'classification', 'prototype', 'interaction'])
25 parser.add_argument('--display', dest = 'display', help = 'data to display (replay over video)', choices = ['feature', 'object', 'classification', 'interaction']) 25 parser.add_argument('--display', dest = 'display', help = 'data to display (replay over video)', choices = ['feature', 'object', 'classification', 'interaction'])
26 parser.add_argument('--progress', dest = 'progress', help = 'information about the progress of processing', action = 'store_true') 26 parser.add_argument('--progress', dest = 'progress', help = 'information about the progress of processing', action = 'store_true')
27 parser.add_argument('--analyze', dest = 'analyze', help = 'data to analyze (results)', choices = ['feature', 'object', 'classification', 'interaction', 'event-speed', 'event-interaction']) 27 parser.add_argument('--analyze', dest = 'analyze', help = 'data to analyze (results)', choices = ['feature', 'object', 'classification', 'interaction', 'collision-map', 'event-speed', 'event-interaction'])
28 28
29 # common options 29 # common options
30 parser.add_argument('--cfg', dest = 'configFilename', help = 'name of the configuration file') 30 parser.add_argument('--cfg', dest = 'configFilename', help = 'name of the configuration file')
31 parser.add_argument('-n', dest = 'nObjects', help = 'number of objects/interactions to process', type = int) 31 parser.add_argument('-n', dest = 'nObjects', help = 'number of objects/interactions to process', type = int)
32 parser.add_argument('-t', dest = 'trajectoryType', help = 'type of trajectories', choices = ['feature', 'object'], default = 'feature') 32 parser.add_argument('-t', dest = 'trajectoryType', help = 'type of trajectories', choices = ['feature', 'object'], default = 'feature')
59 59
60 60
61 # analysis options 61 # analysis options
62 parser.add_argument('--output', dest = 'output', help = 'kind of output to produce (interval means)', choices = ['figure', 'interval', 'event']) 62 parser.add_argument('--output', dest = 'output', help = 'kind of output to produce (interval means)', choices = ['figure', 'interval', 'event'])
63 parser.add_argument('--min-duration', dest = 'minDuration', help = 'mininum duration we have to see the user or interaction to take into account in the analysis (s)', type = float) 63 parser.add_argument('--min-duration', dest = 'minDuration', help = 'mininum duration we have to see the user or interaction to take into account in the analysis (s)', type = float)
64 parser.add_argument('--max-time-indicator-value', dest = 'maxTimeIndicatorValue', help = 'maximum indicator value for time indicators like PET and TTC (s)', type = float)
65 parser.add_argument('--max-speed-indicator-value', dest = 'maxSpeedIndicatorValue', help = 'maximum indicator value for speed indicators like individual speed statistics and speed differential (km/h)', type = float)
64 parser.add_argument('--interval-duration', dest = 'intervalDuration', help = 'length of time interval to aggregate data (min)', type = int, default = 15) 66 parser.add_argument('--interval-duration', dest = 'intervalDuration', help = 'length of time interval to aggregate data (min)', type = int, default = 15)
65 parser.add_argument('--aggregation', dest = 'aggMethods', help = 'aggregation method per user/interaction and per interval', choices = ['mean', 'median', 'centile'], nargs = '*', default = ['median']) 67 parser.add_argument('--aggregation', dest = 'aggMethods', help = 'aggregation method per user/interaction and per interval', choices = ['mean', 'median', 'centile'], nargs = '*', default = ['median'])
66 parser.add_argument('--aggregation-centiles', dest = 'aggCentiles', help = 'centile(s) to compute from the observations', nargs = '*', type = int) 68 parser.add_argument('--aggregation-centiles', dest = 'aggCentiles', help = 'centile(s) to compute from the observations', nargs = '*', type = int)
67 parser.add_argument('--event-thresholds', dest = 'eventThresholds', help = 'threshold to count severe situations', nargs = '*', type = float) 69 parser.add_argument('--event-thresholds', dest = 'eventThresholds', help = 'threshold to count severe situations', nargs = '*', type = float)
68 parser.add_argument('--event-filename', dest = 'eventFilename', help = 'filename of the event data') 70 parser.add_argument('--event-filename', dest = 'eventFilename', help = 'filename of the event data')
136 print('-'*80) 138 print('-'*80)
137 if len(videoSequences) > 0: 139 if len(videoSequences) > 0:
138 print('{} video sequences without a camera view:'.format(len(videoSequences))) 140 print('{} video sequences without a camera view:'.format(len(videoSequences)))
139 print([vs.idx for vs in videoSequences]) 141 print([vs.idx for vs in videoSequences])
140 print('-'*80) 142 print('-'*80)
141 print(data) 143 with pd.option_context('display.max_rows', None, 'display.max_columns', None):
144 print(data)
142 145
143 ################################# 146 #################################
144 # Delete 147 # Delete
145 ################################# 148 #################################
146 if args.delete is not None: 149 if args.delete is not None:
282 elif args.output == 'event': 285 elif args.output == 'event':
283 data.to_csv(args.eventFilename, index = False) 286 data.to_csv(args.eventFilename, index = False)
284 287
285 if args.analyze == 'interaction': # redo as for object, export in dataframe all interaction data 288 if args.analyze == 'interaction': # redo as for object, export in dataframe all interaction data
286 indicatorIds = [2,5,7,10] 289 indicatorIds = [2,5,7,10]
287 conversionFactors = {2: 1., 5: 30.*3.6, 7:1./30, 10:1./30}
288 #maxIndicatorValue = {2: float('inf'), 5: float('inf'), 7:10., 10:10.} 290 #maxIndicatorValue = {2: float('inf'), 5: float('inf'), 7:10., 10:10.}
289 data = [] # list of observation per site-user with time 291 data = [] # list of observation per site-user with time
290 headers = ['site', 'date', 'time', events.Interaction.indicatorNames[10].replace(' ','-')] # user types? 292 headers = ['site', 'date', 'time', events.Interaction.indicatorNames[10].replace(' ','-')] # user types?
291 aggFunctions, tmpheaders = utils.aggregationMethods(args.aggMethods, args.aggCentiles) 293 aggFunctions, tmpheaders = utils.aggregationMethods(args.aggMethods, args.aggCentiles)
294 nAggFunctions = len(tmpheaders)
295 indicatorUnits = [events.Interaction.indicatorUnits[10]] # for PET above
292 for i in indicatorIds[:3]: 296 for i in indicatorIds[:3]:
293 for h in tmpheaders: 297 for h in tmpheaders:
294 headers.append(events.Interaction.indicatorNames[i].replace(' ','-')+'-'+h) 298 headers.append(events.Interaction.indicatorNames[i].replace(' ','-')+'-'+h)
295 indicators = {} 299 indicatorUnits.append(events.Interaction.indicatorUnits[i])
296 interactions = {}
297 for vs in videoSequences: 300 for vs in videoSequences:
298 print('Extracting SMoS from '+vs.getDatabaseFilename()) 301 print('Extracting SMoS from '+vs.getDatabaseFilename())
299 interactions = storage.loadInteractionsFromSqlite(str(parentPath/vs.getDatabaseFilename())) 302 interactions = storage.loadInteractionsFromSqlite(str(parentPath/vs.getDatabaseFilename()))
300 minDuration = vs.cameraView.cameraType.frameRate*args.minDuration 303 minDuration = vs.cameraView.cameraType.frameRate*args.minDuration
304 conversionFactors = {2: 1., 5: 3.6*vs.cameraView.cameraType.frameRate, 7:1./vs.cameraView.cameraType.frameRate, 10:1./vs.cameraView.cameraType.frameRate}
301 for inter in interactions: 305 for inter in interactions:
302 if inter.length() > minDuration: 306 if inter.length() > minDuration:
303 d = vs.startTime.date() 307 d = vs.startTime.date()
304 t = vs.startTime.time() 308 t = vs.startTime.time()
305 row = [vs.cameraView.site.name, d, utils.framesToTime(inter.getFirstInstant(), vs.cameraView.cameraType.frameRate, t)] 309 row = [vs.cameraView.site.name, d, utils.framesToTime(inter.getFirstInstant(), vs.cameraView.cameraType.frameRate, t)]
306 pet = inter.getIndicator('Post Encroachment Time') 310 pet = inter.getIndicator(events.Interaction.indicatorNames[10])
307 if pet is None: 311 if pet is None:
308 row.append(None) 312 row.append(None)
309 else: 313 else:
310 row.append(conversionFactors[10]*pet.getValues()[0]) 314 row.append(conversionFactors[10]*pet.getValues()[0])
311 for i in indicatorIds[:3]: 315 for i in indicatorIds[:3]:
318 if method == 'centile': 322 if method == 'centile':
319 row.extend(agg.tolist()) 323 row.extend(agg.tolist())
320 else: 324 else:
321 row.append(agg) 325 row.append(agg)
322 else: 326 else:
323 row.extend([None]*len(aggFunctions)) 327 row.extend([None]*nAggFunctions)
324 data.append(row) 328 data.append(row)
325 data = pd.DataFrame(data, columns = headers) 329 data = pd.DataFrame(data, columns = headers)
326 if args.output == 'figure': 330 if args.output == 'figure':
327 for i in indicatorIds: 331 plt.ioff()
328 pass # tmp = [indicators[siteId][i] for siteId in indicators] 332 for i, indic in enumerate(headers[3:]):
329 # plt.ioff() 333 if 'Time' in indic and args.maxTimeIndicatorValue is not None:
330 # plt.figure() 334 tmp = data.loc[data[indic] < args.maxTimeIndicatorValue, ['site', indic]]
335 elif 'Speed' in indic and args.maxSpeedIndicatorValue is not None:
336 tmp = data.loc[data[indic] < args.maxSpeedIndicatorValue, ['site', indic]]
337 else:
338 tmp = data[['site', indic]]
339 plt.figure()
340 tmp.boxplot(indic, 'site')
331 # plt.boxplot(tmp, labels = [session.query(Site).get(siteId).name for siteId in indicators]) 341 # plt.boxplot(tmp, labels = [session.query(Site).get(siteId).name for siteId in indicators])
332 # plt.ylabel(events.Interaction.indicatorNames[i]+' ('+events.Interaction.indicatorUnits[i]+')') 342 plt.ylabel(indic+' ('+indicatorUnits[i]+')')
333 # plt.savefig(events.Interaction.indicatorNames[i]+'.png', dpi=150) 343 plt.savefig('boxplot-sites-'+indic+'.pdf')#, dpi=150)
334 # plt.close() 344 plt.close()
345 plt.figure()
346 for site in sorted(tmp.site.unique()):
347 x = sorted(tmp.loc[tmp.site == site, indic])
348 plt.plot(x, np.arange(1,len(x)+1)/len(x), label=site)
349 plt.legend()
350 plt.title('Cumulative Distribution Function by Site')
351 plt.xlabel(indic+' ('+indicatorUnits[i]+')')
352 plt.savefig('cdf-sites-'+indic+'.pdf')
353 plt.close()
335 elif args.output == 'event': 354 elif args.output == 'event':
336 data.to_csv(args.eventFilename, index = False) 355 data.to_csv(args.eventFilename, index = False)
337 356
357 if args.analyze == 'collision-map':
358 predictionParameters = prediction.CVExactPredictionParameters()
359 data = []
360 for vs in videoSequences:
361 print('Extracting potential collision points from '+vs.getDatabaseFilename())
362 interactions = storage.loadInteractionsFromSqlite(str(parentPath/vs.getDatabaseFilename()))
363 objects = storage.loadTrajectoriesFromSqlite(str(parentPath/vs.getDatabaseFilename()), 'object')
364 params = storage.ProcessParameters(str(parentPath/vs.cameraView.getTrackingConfigurationFilename()))
365 minDuration = vs.cameraView.cameraType.frameRate*args.minDuration
366 maxTimeIndicatorValue = vs.cameraView.cameraType.frameRate*args.maxTimeIndicatorValue
367 for inter in interactions:
368 if inter.length() > minDuration:
369 ttc = inter.getIndicator(events.Interaction.indicatorNames[7])
370 if ttc is not None:
371 t = min(ttc.values, key = ttc.values.get)
372 if args.maxTimeIndicatorValue is None or ttc.values[t] < maxTimeIndicatorValue:
373 inter.setRoadUsers(objects)
374 cps, _ = predictionParameters.computeCrossingsCollisionsAtInstant(t, inter.roadUser1, inter.roadUser2, params.collisionDistance, params.predictionTimeHorizon)
375 data.append([vs.cameraView.site.name, cps[0].x, cps[0].y, cps[0].indicator])
376 data = pd.DataFrame(data, columns = ['site', 'x', 'y', 'ttc'])
377 margin = 0.1
378 for site in data.site.unique():
379 s = session.query(Site).filter(Site.name.like('%'+site+'%')).first()
380 img = plt.imread(str(parentPath/s.getMapImageFilename()))
381 tmp = data[data.site == site].copy()
382 tmp.x = tmp.x/s.nUnitsPerPixel
383 tmp.y = tmp.y/s.nUnitsPerPixel
384 h, w, _ = img.shape
385 tmp = tmp[(tmp.x>-margin*w) & (tmp.x < (1+margin)*w) & (tmp.y > -margin*h) & (tmp.y < (1+margin)*h)]
386 plt.figure()
387 plt.imshow(img)
388 plt.hexbin(tmp.x, tmp.y, alpha = 0.5, edgecolors = 'face', mincnt=1, gridsize=50)
389 plt.title('Density of Potential Collision Points at Site '+site)
390 plt.colorbar()
391 plt.axis('equal')
392 plt.savefig('collision-map-'+site+'.pdf')
393 #plt.close()
394
338 if args.analyze == 'event-speed': # aggregate event data by 15 min interval (args.intervalDuration), count events with thresholds 395 if args.analyze == 'event-speed': # aggregate event data by 15 min interval (args.intervalDuration), count events with thresholds
339 data = pd.read_csv(args.eventFilename, parse_dates = [2]) 396 data = pd.read_csv(args.eventFilename, parse_dates = [2])
340 #data = pd.read_csv('./speeds.csv', converters = {'time': lambda s: datetime.datetime.strptime(s, "%H:%M:%S").time()}, nrows = 5000) 397 #data = pd.read_csv('./speeds.csv', converters = {'time': lambda s: datetime.datetime.strptime(s, "%H:%M:%S").time()}, nrows = 5000)
341 # create time for end of each 15 min, then group by, using the agg method for each data column 398 # create time for end of each 15 min, then group by, using the agg method for each data column
342 headers = ['site', 'date', 'intervalend15', 'duration', 'count'] 399 headers = ['site', 'date', 'intervalend15', 'duration', 'count']
375 432
376 elif args.analyze == 'event-interaction': # aggregate event data by 15 min interval (args.intervalDuration), count events with thresholds 433 elif args.analyze == 'event-interaction': # aggregate event data by 15 min interval (args.intervalDuration), count events with thresholds
377 data = pd.read_csv(args.eventFilename, parse_dates = [2]) 434 data = pd.read_csv(args.eventFilename, parse_dates = [2])
378 headers = ['site', 'date', 'intervalend15', 'duration', 'count'] 435 headers = ['site', 'date', 'intervalend15', 'duration', 'count']
379 aggFunctions, tmpheaders = utils.aggregationMethods(args.aggMethods, args.aggCentiles) 436 aggFunctions, tmpheaders = utils.aggregationMethods(args.aggMethods, args.aggCentiles)
437 nAggFunctions = len(tmpheaders)
380 dataColumns = list(data.columns[3:]) 438 dataColumns = list(data.columns[3:])
381 for h in dataColumns: 439 for h in dataColumns:
382 if not 'speed' in h.lower(): # proximity indicators are reversed, taking 85th centile of this column will yield the 15th centile (which we have to take the opposite again) 440 if not 'speed' in h.lower(): # proximity indicators are reversed, taking 85th centile of this column will yield the 15th centile (which we have to take the opposite again)
383 data[h] = -data[h] 441 data[h] = -data[h]
384 for h in dataColumns: 442 for h in dataColumns:
403 if method == 'centile': 461 if method == 'centile':
404 row.extend(np.abs(aggregated)) 462 row.extend(np.abs(aggregated))
405 else: 463 else:
406 row.append(np.abs(aggregated)) 464 row.append(np.abs(aggregated))
407 else: 465 else:
408 row.extend([None]*len(aggFunctions)) 466 row.extend([None]*nAggFunctions)
409 for h,t in zip(dataColumns, args.eventThresholds): # each threshold in this case applies to one indicator 467 for h,t in zip(dataColumns, args.eventThresholds): # each threshold in this case applies to one indicator
410 if 'speed' in h.lower(): 468 if 'speed' in h.lower():
411 row.append((group[h] > t).sum()) 469 row.append((group[h] > t).sum())
412 else: 470 else:
413 row.append((group[h] > -t).sum()) # take larger than than negative threshold for proximity indicators 471 row.append((group[h] > -t).sum()) # take larger than than negative threshold for proximity indicators