Mercurial > hg > nsaunier > traffic-intelligence

comparison scripts/process.py @ 1045:25db2383e7ae

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
work in progress on process.py
author Nicolas Saunier <nicolas.saunier@polymtl.ca>
date Thu, 05 Jul 2018 17:45:18 -0400
parents 75a6ad604cc5
children f2ba9858e6c6
comparison
equal deleted inserted replaced
1044:75a6ad604cc5 1045:25db2383e7ae
8 #atplotlib.use('Agg') 8 #atplotlib.use('Agg')
9 import matplotlib.pyplot as plt 9 import matplotlib.pyplot as plt
10 from numpy import percentile 10 from numpy import percentile
11 from pandas import DataFrame 11 from pandas import DataFrame
12 12
13 from trafficintelligence import storage, events, prediction, cvutils, utils 13 from trafficintelligence import storage, events, prediction, cvutils, utils, moving
14 from trafficintelligence.metadata import * 14 from trafficintelligence.metadata import *
15 15
16 parser = argparse.ArgumentParser(description='This program manages the processing of several files based on a description of the sites and video data in an SQLite database following the metadata module.') 16 parser = argparse.ArgumentParser(description='This program manages the processing of several files based on a description of the sites and video data in an SQLite database following the metadata module.')
17 # input 17 # input
18 parser.add_argument('--db', dest = 'metadataFilename', help = 'name of the metadata file', required = True) 18 parser.add_argument('--db', dest = 'metadataFilename', help = 'name of the metadata file', required = True)
36 36
37 ### process options 37 ### process options
38 # motion pattern learning and assignment 38 # motion pattern learning and assignment
39 parser.add_argument('--prototype-filename', dest = 'outputPrototypeDatabaseFilename', help = 'name of the Sqlite database file to save prototypes') 39 parser.add_argument('--prototype-filename', dest = 'outputPrototypeDatabaseFilename', help = 'name of the Sqlite database file to save prototypes')
40 #parser.add_argument('-i', dest = 'inputPrototypeDatabaseFilename', help = 'name of the Sqlite database file for prototypes to start the algorithm with') 40 #parser.add_argument('-i', dest = 'inputPrototypeDatabaseFilename', help = 'name of the Sqlite database file for prototypes to start the algorithm with')
41 parser.add_argument('--max-nobjectfeatures', dest = 'maxNObjectFeatures', help = 'maximum number of features per object to load', type = int, default = 1) 41 parser.add_argument('--nfeatures-per-object', dest = 'nLongestFeaturesPerObject', help = 'maximum number of features per object to load', type = int)
42 parser.add_argument('--maxdist', dest = 'epsilon', help = 'distance for the similarity of trajectory points', type = float, required = True) 42 parser.add_argument('--epsilon', dest = 'epsilon', help = 'distance for the similarity of trajectory points', type = float, required = True)
43 parser.add_argument('--metric', dest = 'metric', help = 'metric for the similarity of trajectory points', default = 'cityblock') # default is manhattan distance 43 parser.add_argument('--metric', dest = 'metric', help = 'metric for the similarity of trajectory points', default = 'cityblock') # default is manhattan distance
44 parser.add_argument('-minsimil', dest = 'minSimilarity', help = 'minimum similarity to put a trajectory in a cluster', type = float, required = True) 44 parser.add_argument('--minsimil', dest = 'minSimilarity', help = 'minimum similarity to put a trajectory in a cluster', type = float, required = True)
45 parser.add_argument('-min-cluster-size', dest = 'minClusterSize', help = 'minimum cluster size', type = int, default = 0) 45 parser.add_argument('-min-cluster-size', dest = 'minClusterSize', help = 'minimum cluster size', type = int, default = 0)
46 parser.add_argument('--learn', dest = 'learn', help = 'learn', action = 'store_true') 46 #parser.add_argument('--learn', dest = 'learn', help = 'learn', action = 'store_true')
47 parser.add_argument('--optimize', dest = 'optimizeCentroid', help = 'recompute centroid at each assignment', action = 'store_true') 47 parser.add_argument('--optimize', dest = 'optimizeCentroid', help = 'recompute centroid at each assignment', action = 'store_true')
48 parser.add_argument('--random', dest = 'randomInitialization', help = 'random initialization of clustering algorithm', action = 'store_true') 48 parser.add_argument('--random', dest = 'randomInitialization', help = 'random initialization of clustering algorithm', action = 'store_true')
49 #parser.add_argument('--similarities-filename', dest = 'similaritiesFilename', help = 'filename of the similarities') 49 #parser.add_argument('--similarities-filename', dest = 'similaritiesFilename', help = 'filename of the similarities')
50 parser.add_argument('--save-similarities', dest = 'saveSimilarities', help = 'save computed similarities (in addition to prototypes)', action = 'store_true') 50 parser.add_argument('--save-similarities', dest = 'saveSimilarities', help = 'save computed similarities (in addition to prototypes)', action = 'store_true')
51 parser.add_argument('--save-assignments', dest = 'saveAssignments', help = 'saves the assignments of the objects to the prototypes', action = 'store_true') 51 parser.add_argument('--save-assignments', dest = 'saveAssignments', help = 'saves the assignments of the objects to the prototypes', action = 'store_true')
78 # Data preparation 78 # Data preparation
79 ################################# 79 #################################
80 session = connectDatabase(args.metadataFilename) 80 session = connectDatabase(args.metadataFilename)
81 parentPath = Path(args.metadataFilename).parent # files are relative to metadata location 81 parentPath = Path(args.metadataFilename).parent # files are relative to metadata location
82 videoSequences = [] 82 videoSequences = []
83 sites = []
83 if args.videoIds is not None: 84 if args.videoIds is not None:
84 videoSequences = [session.query(VideoSequence).get(videoId) for videoId in args.videoIds] 85 videoSequences = [session.query(VideoSequence).get(videoId) for videoId in args.videoIds]
85 siteIds = set([vs.cameraView.siteIdx for vs in videoSequences]) 86 siteIds = set([vs.cameraView.siteIdx for vs in videoSequences])
86 elif args.siteIds is not None: 87 elif args.siteIds is not None:
87 siteIds = set(args.siteIds) 88 siteIds = set(args.siteIds)
88 for siteId in siteIds: 89 for siteId in siteIds:
89 for site in getSite(session, siteId): 90 tmpsites = getSite(session, siteId)
91 sites.extend(tmpsites)
92 for site in tmpsites:
90 for cv in site.cameraViews: 93 for cv in site.cameraViews:
91 videoSequences += cv.videoSequences 94 videoSequences.extend(cv.videoSequences)
92 else: 95 else:
93 print('No video/site to process') 96 print('No video/site to process')
94 97
95 if args.nProcesses > 1: 98 if args.nProcesses > 1:
96 pool = Pool(args.nProcesses) 99 pool = Pool(args.nProcesses)
143 pool.close() 146 pool.close()
144 pool.join() 147 pool.join()
145 148
146 elif args.process == 'prototype': # motion pattern learning 149 elif args.process == 'prototype': # motion pattern learning
147 # learn by site by default -> group videos by site (or by camera view? TODO add cameraviews) 150 # learn by site by default -> group videos by site (or by camera view? TODO add cameraviews)
148 # by default, load all objects, learn and then assign 151 # by default, load all objects, learn and then assign (BUT not save the assignments)
149 objects = {siteId: [] for siteId in siteIds} 152 for site in sites:
150 for vs in videoSequences: 153 objects = {}
151 print('Loading '+vs.getDatabaseFilename()) 154 object2VideoSequences = {}
152 objects[vs.cameraView.siteIdx] += storage.loadTrajectoriesFromSqlite(str(parentPath/vs.getDatabaseFilename()), args.trajectoryType, args.nTrajectories, timeStep = args.positionSubsamplingRate) 155 for cv in site.cameraViews:
156 for vs in cv.videoSequences:
157 print('Loading '+vs.getDatabaseFilename())
158 objects[vs.idx] = storage.loadTrajectoriesFromSqlite(str(parentPath/vs.getDatabaseFilename()), args.trajectoryType, args.nTrajectories, timeStep = args.positionSubsamplingRate, nLongestFeaturesPerObject = args.nLongestFeaturesPerObject)
159 if args.trajectoryType == 'object' and args.nLongestFeaturesPerObject is not None:
160 objectsWithFeatures = objects[vs.idx]
161 objects[vs.idx] = [f for o in objectsWithFeatures for f in o.getFeatures()]
162 prototypeType = 'feature'
163 else:
164 prototypeType = args.trajectoryType
165 for obj in objects[vs.idx]:
166 object2VideoSequences[obj] = vs
167 lcss = utils.LCSS(metric = args.metric, epsilon = args.epsilon)
168 similarityFunc = lambda x,y : lcss.computeNormalized(x, y)
169 allobjects = [o for tmpobjects in objects.values() for o in tmpobjects]
170 prototypeIndices, labels = processing.learnAssignMotionPatterns(True, True, allobjects, similarities, args.minsimil, similarityFunc, args.minClusterSize, args.optimizeCentroid, args.randomInitialization, True, [])
171 if args.outputPrototypeDatabaseFilename is None:
172 outputPrototypeDatabaseFilename = args.databaseFilename
173 else:
174 outputPrototypeDatabaseFilename = args.outputPrototypeDatabaseFilename
175 # TODO maintain mapping from object prototype to db filename + compute nmatchings before
176 clusterSizes = ml.computeClusterSizes(labels, prototypeIndices, -1)
177 storage.savePrototypesToSqlite(outputPrototypeDatabaseFilename, [moving.Prototype(object2VideoSequences[allobjects[i]].getDatabaseFilename(False), allobjects[i].getNum(), prototypeType) for i in prototypeIndices])
153 178
154 179
155 elif args.process == 'interaction': 180 elif args.process == 'interaction':
156 # safety analysis TODO make function in safety analysis script 181 # safety analysis TODO make function in safety analysis script
157 if args.predictionMethod == 'cvd': 182 if args.predictionMethod == 'cvd':