Mercurial > hg > nsaunier > traffic-intelligence

comparison scripts/process.py @ 1015:cf9d29de3dbf

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
merge With Pr Saunier's code
author Wendlasida
date Mon, 04 Jun 2018 11:25:49 -0400
parents 0d29b75f74ea
children 16932cefabc1
comparison
equal deleted inserted replaced
1014:026f6b3b122c 1015:cf9d29de3dbf
1 #! /usr/bin/env python3 1 #! /usr/bin/env python3
2 2
3 import sys, argparse 3 import sys, argparse
4 from pathlib import Path 4 from pathlib import Path
5 from multiprocessing.pool import Pool
5 6
6 import matplotlib 7 import matplotlib
7 matplotlib.use('Agg') 8 matplotlib.use('Agg')
8 import matplotlib.pyplot as plt 9 import matplotlib.pyplot as plt
9 from numpy import percentile 10 from numpy import percentile
10 11
11 import storage, events, prediction 12 import storage, events, prediction, cvutils
12 from metadata import * 13 from metadata import *
13 14
14 parser = argparse.ArgumentParser(description='This program manages the processing of several files based on a description of the sites and video data in an SQLite database following the metadata module.') 15 parser = argparse.ArgumentParser(description='This program manages the processing of several files based on a description of the sites and video data in an SQLite database following the metadata module.')
15 parser.add_argument('--db', dest = 'metadataFilename', help = 'name of the metadata file', required = True) 16 parser.add_argument('--db', dest = 'metadataFilename', help = 'name of the metadata file', required = True)
16 parser.add_argument('--videos', dest = 'videoIds', help = 'indices of the video sequences', nargs = '*', type = int) 17 parser.add_argument('--videos', dest = 'videoIds', help = 'indices of the video sequences', nargs = '*', type = int)
18 parser.add_argument('--sites', dest = 'siteIds', help = 'indices of the video sequences', nargs = '*', type = int)
17 parser.add_argument('--cfg', dest = 'configFilename', help = 'name of the configuration file') 19 parser.add_argument('--cfg', dest = 'configFilename', help = 'name of the configuration file')
18 parser.add_argument('-n', dest = 'nObjects', help = 'number of objects/interactions to process', type = int) 20 parser.add_argument('-n', dest = 'nObjects', help = 'number of objects/interactions to process', type = int)
19 parser.add_argument('--prediction-method', dest = 'predictionMethod', help = 'prediction method (constant velocity (cvd: vector computation (approximate); cve: equation solving; cv: discrete time (approximate)), normal adaptation, point set prediction)', choices = ['cvd', 'cve', 'cv', 'na', 'ps', 'mp']) 21 parser.add_argument('--prediction-method', dest = 'predictionMethod', help = 'prediction method (constant velocity (cvd: vector computation (approximate); cve: equation solving; cv: discrete time (approximate)), normal adaptation, point set prediction)', choices = ['cvd', 'cve', 'cv', 'na', 'ps', 'mp'])
20 parser.add_argument('--pet', dest = 'computePET', help = 'computes PET', action = 'store_true') 22 parser.add_argument('--pet', dest = 'computePET', help = 'computes PET', action = 'store_true')
21 # override other tracking config, erase sqlite? 23 # override other tracking config, erase sqlite?
22 parser.add_argument('--delete', dest = 'delete', help = 'data to delete', choices = ['feature', 'object', 'classification', 'interaction']) 24 parser.add_argument('--delete', dest = 'delete', help = 'data to delete', choices = ['feature', 'object', 'classification', 'interaction'])
23 parser.add_argument('--process', dest = 'process', help = 'data to process', choices = ['feature', 'object', 'classification', 'interaction']) 25 parser.add_argument('--process', dest = 'process', help = 'data to process', choices = ['feature', 'object', 'classification', 'interaction'])
24 parser.add_argument('--display', dest = 'display', help = 'data to display (replay over video)', choices = ['feature', 'object', 'classification', 'interaction']) 26 parser.add_argument('--display', dest = 'display', help = 'data to display (replay over video)', choices = ['feature', 'object', 'classification', 'interaction'])
25 parser.add_argument('--analyze', dest = 'analyze', help = 'data to analyze (results)', choices = ['feature', 'object', 'classification', 'interaction']) 27 parser.add_argument('--analyze', dest = 'analyze', help = 'data to analyze (results)', choices = ['feature', 'object', 'classification', 'interaction'])
28 parser.add_argument('--dry', dest = 'dryRun', help = 'dry run of processing', action = 'store_true')
29 parser.add_argument('--nthreads', dest = 'nProcesses', help = 'number of processes to run in parallel', type = int, default = 1)
26 30
27 # need way of selecting sites as similar as possible to sql alchemy syntax 31 # need way of selecting sites as similar as possible to sql alchemy syntax
28 # override tracking.cfg from db 32 # override tracking.cfg from db
29 # manage cfg files, overwrite them (or a subset of parameters) 33 # manage cfg files, overwrite them (or a subset of parameters)
30 # delete sqlite files 34 # delete sqlite files
31
32 # info of metadata 35 # info of metadata
33 36
34 parser.add_argument('--nthreads', dest = 'nProcesses', help = 'number of processes to run in parallel', type = int, default = 1) 37 args = parser.parse_args()
35 38
36 args = parser.parse_args() 39 #################################
37 # files are relative to metadata location 40 # Data preparation
41 #################################
42 session = connectDatabase(args.metadataFilename)
43 parentDir = Path(args.metadataFilename).parent # files are relative to metadata location
44 videoSequences = []
45 if args.videoIds is not None:
46 videoSequences = [session.query(VideoSequence).get(videoId) for videoId in args.videoIds]
47 elif args.siteIds is not None:
48 for siteId in args.siteIds:
49 for site in getSite(session, siteId):
50 for cv in site.cameraViews:
51 videoSequences += cv.videoSequences
52 else:
53 print('No video/site to process')
38 54
39 session = connectDatabase(args.metadataFilename) 55 #################################
40 parentDir = Path(args.metadataFilename).parent 56 # Delete
41 57 #################################
42 if args.delete is not None: 58 if args.delete is not None:
43 if args.delete in ['object', 'interaction']: 59 if args.delete == 'feature':
60 pass
61 elif args.delete in ['object', 'interaction']:
44 #parser.add_argument('-t', dest = 'dataType', help = 'type of the data to remove', required = True, choices = ['object','interaction', 'bb', 'pois', 'prototype']) 62 #parser.add_argument('-t', dest = 'dataType', help = 'type of the data to remove', required = True, choices = ['object','interaction', 'bb', 'pois', 'prototype'])
45 for videoId in args.videoIds: 63 for vs in videoSequences:
46 vs = session.query(VideoSequence).get(videoId)
47 storage.deleteFromSqlite(str(parentDir/vs.getDatabaseFilename()), args.delete) 64 storage.deleteFromSqlite(str(parentDir/vs.getDatabaseFilename()), args.delete)
48 65
66 #################################
67 # Process
68 #################################
49 if args.process in ['feature', 'object']: # tracking 69 if args.process in ['feature', 'object']: # tracking
50 for videoId in args.videoIds: 70 if args.nProcesses == 1:
51 vs = session.query(VideoSequence).get(videoId) 71 for vs in videoSequences:
52 if args.configFilename is None: 72 if not (parentDir/vs.getDatabaseFilename()).exists() or args.process == 'object':
53 configFilename = vs.cameraView.getTrackingConfigurationFilename() 73 if args.configFilename is None:
54 else: 74 configFilename = str(parentDir/vs.cameraView.getTrackingConfigurationFilename())
55 configFilename = args.configFilename 75 else:
56 #todo cvutils.tracking(configFilename, args.process == 'object', str(parentDir/vs.getVideoSequenceFilename(), str(parentDir/vs.getDatabaseFilename(), configFilename = vs.cameraView.getHomographyFilename()) 76 configFilename = args.configFilename
57 77 if vs.cameraView.cameraType is None:
78 cvutils.tracking(configFilename, args.process == 'object', str(parentDir.absolute()/vs.getVideoSequenceFilename()), str(parentDir.absolute()/vs.getDatabaseFilename()), str(parentDir.absolute()/vs.cameraView.getHomographyFilename()), str(parentDir.absolute()/vs.cameraView.getMaskFilename()), False, None, None, args.dryRun)
79 else:
80 cvutils.tracking(configFilename, args.process == 'object', str(parentDir.absolute()/vs.getVideoSequenceFilename()), str(parentDir.absolute()/vs.getDatabaseFilename()), str(parentDir.absolute()/vs.cameraView.getHomographyFilename()), str(parentDir.absolute()/vs.cameraView.getMaskFilename()), True, vs.cameraView.cameraType.intrinsicCameraMatrix, vs.cameraView.cameraType.distortionCoefficients, args.dryRun)
81 else:
82 print('SQLite already exists: {}'.format(parentDir/vs.getDatabaseFilename()))
83 else:
84 pool = Pool(args.nProcesses)
85 for vs in videoSequences:
86 if not (parentDir/vs.getDatabaseFilename()).exists() or args.process == 'object':
87 if args.configFilename is None:
88 configFilename = str(parentDir/vs.cameraView.getTrackingConfigurationFilename())
89 else:
90 configFilename = args.configFilename
91 if vs.cameraView.cameraType is None:
92 pool.apply_async(cvutils.tracking, args = (configFilename, args.process == 'object', str(parentDir.absolute()/vs.getVideoSequenceFilename()), str(parentDir.absolute()/vs.getDatabaseFilename()), str(parentDir.absolute()/vs.cameraView.getHomographyFilename()), str(parentDir.absolute()/vs.cameraView.getMaskFilename()), False, None, None, args.dryRun))
93 else:
94 pool.apply_async(cvutils.tracking, args = (configFilename, args.process == 'object', str(parentDir.absolute()/vs.getVideoSequenceFilename()), str(parentDir.absolute()/vs.getDatabaseFilename()), str(parentDir.absolute()/vs.cameraView.getHomographyFilename()), str(parentDir.absolute()/vs.cameraView.getMaskFilename()), True, vs.cameraView.cameraType.intrinsicCameraMatrix, vs.cameraView.cameraType.distortionCoefficients, args.dryRun))
95 else:
96 print('SQLite already exists: {}'.format(parentDir/vs.getDatabaseFilename()))
97 pool.close()
98 pool.join()
99
58 elif args.process == 'interaction': 100 elif args.process == 'interaction':
59 # safety analysis TODO make function in safety analysis script 101 # safety analysis TODO make function in safety analysis script
60 if args.predictionMethod == 'cvd': 102 if args.predictionMethod == 'cvd':
61 predictionParameters = prediction.CVDirectPredictionParameters() 103 predictionParameters = prediction.CVDirectPredictionParameters()
62 if args.predictionMethod == 'cve': 104 if args.predictionMethod == 'cve':
63 predictionParameters = prediction.CVExactPredictionParameters() 105 predictionParameters = prediction.CVExactPredictionParameters()
64 for videoId in args.videoIds: 106 for vs in videoSequences:
65 vs = session.query(VideoSequence).get(videoId)
66 print('Processing '+vs.getDatabaseFilename()) 107 print('Processing '+vs.getDatabaseFilename())
67 objects = storage.loadTrajectoriesFromSqlite(str(parentDir/vs.getDatabaseFilename()), 'object')#, args.nObjects, withFeatures = (params.useFeaturesForPrediction or predictionMethod == 'ps' or predictionMethod == 'mp')) 108 objects = storage.loadTrajectoriesFromSqlite(str(parentDir/vs.getDatabaseFilename()), 'object')#, args.nObjects, withFeatures = (params.useFeaturesForPrediction or predictionMethod == 'ps' or predictionMethod == 'mp'))
68 interactions = events.createInteractions(objects) 109 interactions = events.createInteractions(objects)
69 #if args.nProcesses == 1: 110 #if args.nProcesses == 1:
70 #print(str(parentDir/vs.cameraView.getTrackingConfigurationFilename())) 111 #print(str(parentDir/vs.cameraView.getTrackingConfigurationFilename()))
79 # processed = [] 120 # processed = []
80 # for job in jobs: 121 # for job in jobs:
81 # processed += job.get() 122 # processed += job.get()
82 # pool.close() 123 # pool.close()
83 124
125 #################################
126 # Analyze
127 #################################
84 if args.analyze == 'object': # user speed for now 128 if args.analyze == 'object': # user speed for now
85 medianSpeeds = {} 129 medianSpeeds = {}
86 speeds85 = {} 130 speeds85 = {}
87 minLength = 2*30 131 minLength = 2*30
88 for videoId in args.videoIds: 132 for vs in videoSequences:
89 vs = session.query(VideoSequence).get(videoId)
90 if not vs.cameraView.siteIdx in medianSpeeds: 133 if not vs.cameraView.siteIdx in medianSpeeds:
91 medianSpeeds[vs.cameraView.siteIdx] = [] 134 medianSpeeds[vs.cameraView.siteIdx] = []
92 speeds85[vs.cameraView.siteIdx] = [] 135 speeds85[vs.cameraView.siteIdx] = []
93 print('Extracting speed from '+vs.getDatabaseFilename()) 136 print('Extracting speed from '+vs.getDatabaseFilename())
94 objects = storage.loadTrajectoriesFromSqlite(str(parentDir/vs.getDatabaseFilename()), 'object') 137 objects = storage.loadTrajectoriesFromSqlite(str(parentDir/vs.getDatabaseFilename()), 'object')
109 indicatorIds = [2,5,7,10] 152 indicatorIds = [2,5,7,10]
110 conversionFactors = {2: 1., 5: 30.*3.6, 7:1./30, 10:1./30} 153 conversionFactors = {2: 1., 5: 30.*3.6, 7:1./30, 10:1./30}
111 maxIndicatorValue = {2: float('inf'), 5: float('inf'), 7:10., 10:10.} 154 maxIndicatorValue = {2: float('inf'), 5: float('inf'), 7:10., 10:10.}
112 indicators = {} 155 indicators = {}
113 interactions = {} 156 interactions = {}
114 for videoId in args.videoIds: 157 for vs in videoSequences:
115 vs = session.query(VideoSequence).get(videoId)
116 if not vs.cameraView.siteIdx in interactions: 158 if not vs.cameraView.siteIdx in interactions:
117 interactions[vs.cameraView.siteIdx] = [] 159 interactions[vs.cameraView.siteIdx] = []
118 indicators[vs.cameraView.siteIdx] = {} 160 indicators[vs.cameraView.siteIdx] = {}
119 for i in indicatorIds: 161 for i in indicatorIds:
120 indicators[vs.cameraView.siteIdx][i] = [] 162 indicators[vs.cameraView.siteIdx][i] = []