Mercurial > hg > nsaunier > traffic-intelligence

comparison trafficintelligence/storage.py @ 1030:aafbc0bab925

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
moved method around to avoid cross-dependencies
author Nicolas Saunier <nicolas.saunier@polymtl.ca>
date Tue, 19 Jun 2018 10:04:52 -0400
parents cc5cb04b04b0
children 8ffb3ae9f3d2
comparison
equal deleted inserted replaced
1029:c6cf75a2ed08 1030:aafbc0bab925
1 #! /usr/bin/env python 1 #! /usr/bin/env python
2 # -*- coding: utf-8 -*- 2 # -*- coding: utf-8 -*-
3 '''Various utilities to save and load data''' 3 '''Various utilities to save and load data'''
4
5 from trafficintelligence import utils, moving, events, indicators
6 from trafficintelligence.base import VideoFilenameAddable
7 4
8 from pathlib import Path 5 from pathlib import Path
9 import shutil 6 import shutil
10 from copy import copy 7 from copy import copy
11 import sqlite3, logging 8 import sqlite3, logging
9
12 from numpy import log, min as npmin, max as npmax, round as npround, array, sum as npsum, loadtxt, floor as npfloor, ceil as npceil, linalg 10 from numpy import log, min as npmin, max as npmax, round as npround, array, sum as npsum, loadtxt, floor as npfloor, ceil as npceil, linalg
13 from pandas import read_csv, merge 11 from pandas import read_csv, merge
14 12
15 13 from trafficintelligence import utils, moving, events, indicators
16 commentChar = '#' 14 from trafficintelligence.base import VideoFilenameAddable
17 15
18 delimiterChar = '%';
19 16
20 ngsimUserTypes = {'twowheels':1, 17 ngsimUserTypes = {'twowheels':1,
21 'car':2, 18 'car':2,
22 'truck':3} 19 'truck':3}
23 20
879 876
880 ######################### 877 #########################
881 # txt files 878 # txt files
882 ######################### 879 #########################
883 880
884 def openCheck(filename, option = 'r', quitting = False):
885 '''Open file filename in read mode by default
886 and checks it is open'''
887 try:
888 return open(filename, option)
889 except IOError:
890 print('File {} could not be opened.'.format(filename))
891 if quitting:
892 from sys import exit
893 exit()
894 return None
895
896 def readline(f, commentCharacters = commentChar):
897 '''Modified readline function to skip comments
898 Can take a list of characters or a string (in will work in both)'''
899 s = f.readline()
900 while (len(s) > 0) and s[0] in commentCharacters:
901 s = f.readline()
902 return s.strip()
903
904 def getLines(f, delimiterChar = delimiterChar, commentCharacters = commentChar):
905 '''Gets a complete entry (all the lines) in between delimiterChar.'''
906 dataStrings = []
907 s = readline(f, commentCharacters)
908 while len(s) > 0 and s[0] != delimiterChar:
909 dataStrings += [s.strip()]
910 s = readline(f, commentCharacters)
911 return dataStrings
912
913 def saveList(filename, l): 881 def saveList(filename, l):
914 f = openCheck(filename, 'w') 882 f = utils.openCheck(filename, 'w')
915 for x in l: 883 for x in l:
916 f.write('{}\n'.format(x)) 884 f.write('{}\n'.format(x))
917 f.close() 885 f.close()
918 886
919 def loadListStrings(filename, commentCharacters = commentChar): 887 def loadListStrings(filename, commentCharacters = utils.commentChar):
920 f = openCheck(filename, 'r') 888 f = utils.openCheck(filename, 'r')
921 result = getLines(f, commentCharacters) 889 result = utils.getLines(f, commentCharacters)
922 f.close() 890 f.close()
923 return result 891 return result
924 892
925 def getValuesFromINIFile(filename, option, delimiterChar = '=', commentCharacters = commentChar): 893 def getValuesFromINIFile(filename, option, delimiterChar = '=', commentCharacters = utils.commentChar):
926 values = [] 894 values = []
927 for l in loadListStrings(filename, commentCharacters): 895 for l in loadListStrings(filename, commentCharacters):
928 if l.startswith(option): 896 if l.startswith(option):
929 values.append(l.split(delimiterChar)[1].strip()) 897 values.append(l.split(delimiterChar)[1].strip())
930 return values 898 return values
940 yield line 908 yield line
941 909
942 def loadPemsTraffic(filename): 910 def loadPemsTraffic(filename):
943 '''Loads traffic data downloaded from the http://pems.dot.ca.gov clearinghouse 911 '''Loads traffic data downloaded from the http://pems.dot.ca.gov clearinghouse
944 into pandas dataframe''' 912 into pandas dataframe'''
945 f = openCheck(filename) 913 f = utils.openCheck(filename)
946 l = f.readline().strip() 914 l = f.readline().strip()
947 items = l.split(',') 915 items = l.split(',')
948 headers = ['time', 'station', 'district', 'route', 'direction', 'lanetype', 'length', 'nsamples', 'pctobserved', 'flow', 'occupancy', 'speed', 'delay35', 'delay40', 'delay45', 'delay50', 'delay55', 'delay60'] 916 headers = ['time', 'station', 'district', 'route', 'direction', 'lanetype', 'length', 'nsamples', 'pctobserved', 'flow', 'occupancy', 'speed', 'delay35', 'delay40', 'delay45', 'delay50', 'delay55', 'delay60']
949 nLanes = (len(items)-len(headers))/3 917 nLanes = (len(items)-len(headers))/3
950 for i in range(nLanes): 918 for i in range(nLanes):
958 def convertTrajectoriesVissimToSqlite(filename): 926 def convertTrajectoriesVissimToSqlite(filename):
959 '''Relies on a system call to sqlite3 927 '''Relies on a system call to sqlite3
960 sqlite3 [file.sqlite] < import_fzp.sql''' 928 sqlite3 [file.sqlite] < import_fzp.sql'''
961 sqlScriptFilename = "import_fzp.sql" 929 sqlScriptFilename = "import_fzp.sql"
962 # create sql file 930 # create sql file
963 out = openCheck(sqlScriptFilename, "w") 931 out = utils.openCheck(sqlScriptFilename, "w")
964 out.write(".separator \";\"\n"+ 932 out.write(".separator \";\"\n"+
965 "CREATE TABLE IF NOT EXISTS curvilinear_positions (t REAL, trajectory_id INTEGER, link_id INTEGER, lane_id INTEGER, s_coordinate REAL, y_coordinate REAL, speed REAL, PRIMARY KEY (t, trajectory_id));\n"+ 933 "CREATE TABLE IF NOT EXISTS curvilinear_positions (t REAL, trajectory_id INTEGER, link_id INTEGER, lane_id INTEGER, s_coordinate REAL, y_coordinate REAL, speed REAL, PRIMARY KEY (t, trajectory_id));\n"+
966 ".import "+filename+" curvilinear_positions\n"+ 934 ".import "+filename+" curvilinear_positions\n"+
967 "DELETE FROM curvilinear_positions WHERE trajectory_id IS NULL OR trajectory_id = \"NO\";\n") 935 "DELETE FROM curvilinear_positions WHERE trajectory_id IS NULL OR trajectory_id = \"NO\";\n")
968 out.close() 936 out.close()
969 # system call 937 # system call
970 from subprocess import run 938 from subprocess import run
971 out = openCheck("err.log", "w") 939 out = utils.openCheck("err.log", "w")
972 run("sqlite3 "+utils.removeExtension(filename)+".sqlite < "+sqlScriptFilename, stderr = out) 940 run("sqlite3 "+utils.removeExtension(filename)+".sqlite < "+sqlScriptFilename, stderr = out)
973 out.close() 941 out.close()
974 shutil.os.remove(sqlScriptFilename) 942 shutil.os.remove(sqlScriptFilename)
975 943
976 def loadObjectNumbersInLinkFromVissimFile(filename, linkIds): 944 def loadObjectNumbersInLinkFromVissimFile(filename, linkIds):
1025 objects[objNum].curvilinearPositions = moving.CurvilinearTrajectory(S = npround(tmp['POS'].tolist(), nDecimals), Y = npround(tmp['POSLAT'].tolist(), nDecimals), lanes = tmp['LANE'].tolist()) 993 objects[objNum].curvilinearPositions = moving.CurvilinearTrajectory(S = npround(tmp['POS'].tolist(), nDecimals), Y = npround(tmp['POSLAT'].tolist(), nDecimals), lanes = tmp['LANE'].tolist())
1026 if objectNumbers is not None and objectNumbers > 0 and len(objects) >= objectNumbers: 994 if objectNumbers is not None and objectNumbers > 0 and len(objects) >= objectNumbers:
1027 return list(objects.values()) 995 return list(objects.values())
1028 else: 996 else:
1029 if filename.endswith(".fzp"): 997 if filename.endswith(".fzp"):
1030 inputfile = openCheck(filename, quitting = True) 998 inputfile = utils.openCheck(filename, quitting = True)
1031 line = readline(inputfile, '*$') 999 line = utils.readline(inputfile, '*$')
1032 while len(line) > 0:#for line in inputfile: 1000 while len(line) > 0:#for line in inputfile:
1033 data = line.strip().split(';') 1001 data = line.strip().split(';')
1034 objNum = int(data[1]) 1002 objNum = int(data[1])
1035 instant = float(data[0])*simulationStepsPerTimeUnit 1003 instant = float(data[0])*simulationStepsPerTimeUnit
1036 s = float(data[4]) 1004 s = float(data[4])
1042 objects[objNum] = moving.MovingObject(num = objNum, timeInterval = moving.TimeInterval(instant, instant)) 1010 objects[objNum] = moving.MovingObject(num = objNum, timeInterval = moving.TimeInterval(instant, instant))
1043 objects[objNum].curvilinearPositions = moving.CurvilinearTrajectory() 1011 objects[objNum].curvilinearPositions = moving.CurvilinearTrajectory()
1044 if (warmUpLastInstant is None or instant >= warmUpLastInstant) and objNum in objects: 1012 if (warmUpLastInstant is None or instant >= warmUpLastInstant) and objNum in objects:
1045 objects[objNum].timeInterval.last = instant 1013 objects[objNum].timeInterval.last = instant
1046 objects[objNum].curvilinearPositions.addPositionSYL(s, y, lane) 1014 objects[objNum].curvilinearPositions.addPositionSYL(s, y, lane)
1047 line = readline(inputfile, '*$') 1015 line = utils.readline(inputfile, '*$')
1048 elif filename.endswith(".sqlite"): 1016 elif filename.endswith(".sqlite"):
1049 with sqlite3.connect(filename) as connection: 1017 with sqlite3.connect(filename) as connection:
1050 cursor = connection.cursor() 1018 cursor = connection.cursor()
1051 queryStatement = 'SELECT t, trajectory_id, link_id, lane_id, s_coordinate, y_coordinate FROM curvilinear_positions' 1019 queryStatement = 'SELECT t, trajectory_id, link_id, lane_id, s_coordinate, y_coordinate FROM curvilinear_positions'
1052 if objectNumbers is not None: 1020 if objectNumbers is not None:
1154 def loadTrajectoriesFromNgsimFile(filename, nObjects = -1, sequenceNum = -1): 1122 def loadTrajectoriesFromNgsimFile(filename, nObjects = -1, sequenceNum = -1):
1155 '''Reads data from the trajectory data provided by NGSIM project 1123 '''Reads data from the trajectory data provided by NGSIM project
1156 and returns the list of Feature objects''' 1124 and returns the list of Feature objects'''
1157 objects = [] 1125 objects = []
1158 1126
1159 inputfile = openCheck(filename, quitting = True) 1127 inputfile = utils.openCheck(filename, quitting = True)
1160 1128
1161 def createObject(numbers): 1129 def createObject(numbers):
1162 firstFrameNum = int(numbers[1]) 1130 firstFrameNum = int(numbers[1])
1163 # do the geometry and usertype 1131 # do the geometry and usertype
1164 1132
1178 obj.curvilinearPositions = moving.CurvilinearTrajectory([float(numbers[5])],[float(numbers[4])], obj.laneNums) # X is the longitudinal coordinate 1146 obj.curvilinearPositions = moving.CurvilinearTrajectory([float(numbers[5])],[float(numbers[4])], obj.laneNums) # X is the longitudinal coordinate
1179 obj.speeds = [float(numbers[11])] 1147 obj.speeds = [float(numbers[11])]
1180 obj.size = [float(numbers[8]), float(numbers[9])] # 8 lengh, 9 width # TODO: temporary, should use a geometry object 1148 obj.size = [float(numbers[8]), float(numbers[9])] # 8 lengh, 9 width # TODO: temporary, should use a geometry object
1181 return obj 1149 return obj
1182 1150
1183 numbers = readline(inputfile).strip().split() 1151 numbers = utils.readline(inputfile).strip().split()
1184 if (len(numbers) > 0): 1152 if (len(numbers) > 0):
1185 obj = createObject(numbers) 1153 obj = createObject(numbers)
1186 1154
1187 for line in inputfile: 1155 for line in inputfile:
1188 numbers = line.strip().split() 1156 numbers = line.strip().split()
1216 1184
1217 def convertNgsimFile(inputfile, outputfile, append = False, nObjects = -1, sequenceNum = 0): 1185 def convertNgsimFile(inputfile, outputfile, append = False, nObjects = -1, sequenceNum = 0):
1218 '''Reads data from the trajectory data provided by NGSIM project 1186 '''Reads data from the trajectory data provided by NGSIM project
1219 and converts to our current format.''' 1187 and converts to our current format.'''
1220 if append: 1188 if append:
1221 out = openCheck(outputfile,'a') 1189 out = utils.openCheck(outputfile,'a')
1222 else: 1190 else:
1223 out = openCheck(outputfile,'w') 1191 out = utils.openCheck(outputfile,'w')
1224 nObjectsPerType = [0,0,0] 1192 nObjectsPerType = [0,0,0]
1225 1193
1226 features = loadNgsimFile(inputfile, sequenceNum) 1194 features = loadNgsimFile(inputfile, sequenceNum)
1227 for f in features: 1195 for f in features:
1228 nObjectsPerType[f.userType-1] += 1 1196 nObjectsPerType[f.userType-1] += 1
1235 def loadPinholeCameraModel(filename, tanalystFormat = True): 1203 def loadPinholeCameraModel(filename, tanalystFormat = True):
1236 '''Loads the data from a file containing the camera parameters 1204 '''Loads the data from a file containing the camera parameters
1237 (pinhole camera model, http://docs.opencv.org/2.4/modules/calib3d/doc/camera_calibration_and_3d_reconstruction.html) 1205 (pinhole camera model, http://docs.opencv.org/2.4/modules/calib3d/doc/camera_calibration_and_3d_reconstruction.html)
1238 and returns a dictionary''' 1206 and returns a dictionary'''
1239 if tanalystFormat: 1207 if tanalystFormat:
1240 f = openCheck(filename, quitting = True) 1208 f = utils.openCheck(filename, quitting = True)
1241 content = getLines(f) 1209 content = utils.getLines(f)
1242 cameraData = {} 1210 cameraData = {}
1243 for l in content: 1211 for l in content:
1244 tmp = l.split(':') 1212 tmp = l.split(':')
1245 cameraData[tmp[0]] = float(tmp[1].strip().replace(',','.')) 1213 cameraData[tmp[0]] = float(tmp[1].strip().replace(',','.'))
1246 return cameraData 1214 return cameraData
1259 p2 = curvilinearPositions[i] 1227 p2 = curvilinearPositions[i]
1260 s += ',{},{}'.format(p2[0],p2[1]) 1228 s += ',{},{}'.format(p2[0],p2[1])
1261 f.write(s+'\n') 1229 f.write(s+'\n')
1262 1230
1263 def saveTrajectoriesToCsv(filename, objects): 1231 def saveTrajectoriesToCsv(filename, objects):
1264 f = openCheck(filename, 'w') 1232 f = utils.openCheck(filename, 'w')
1265 for i,obj in enumerate(objects): 1233 for i,obj in enumerate(objects):
1266 savePositionsToCsv(f, obj) 1234 savePositionsToCsv(f, obj)
1267 f.close() 1235 f.close()
1268 1236
1269 1237
1275 'Class for the parameters of object classifiers' 1243 'Class for the parameters of object classifiers'
1276 def loadConfigFile(self, filename): 1244 def loadConfigFile(self, filename):
1277 from configparser import ConfigParser 1245 from configparser import ConfigParser
1278 1246
1279 config = ConfigParser() 1247 config = ConfigParser()
1280 config.read_file(addSectionHeader(openCheck(filename))) 1248 config.read_file(addSectionHeader(utils.openCheck(filename)))
1281 1249
1282 parentPath = Path(filename).parent 1250 parentPath = Path(filename).parent
1283 self.sectionHeader = config.sections()[0] 1251 self.sectionHeader = config.sections()[0]
1284 1252
1285 self.pedBikeCarSVMFilename = utils.getRelativeFilename(parentPath, config.get(self.sectionHeader, 'pbv-svm-filename')) 1253 self.pedBikeCarSVMFilename = utils.getRelativeFilename(parentPath, config.get(self.sectionHeader, 'pbv-svm-filename'))
1344 1312
1345 def loadConfigFile(self, filename): 1313 def loadConfigFile(self, filename):
1346 from configparser import ConfigParser 1314 from configparser import ConfigParser
1347 1315
1348 config = ConfigParser(strict=False) 1316 config = ConfigParser(strict=False)
1349 config.read_file(addSectionHeader(openCheck(filename))) 1317 config.read_file(addSectionHeader(utils.openCheck(filename)))
1350 1318
1351 parentPath = Path(filename).parent 1319 parentPath = Path(filename).parent
1352 self.sectionHeader = config.sections()[0] 1320 self.sectionHeader = config.sections()[0]
1353 # Tracking/display parameters 1321 # Tracking/display parameters
1354 self.videoFilename = utils.getRelativeFilename(parentPath, config.get(self.sectionHeader, 'video-filename')) 1322 self.videoFilename = utils.getRelativeFilename(parentPath, config.get(self.sectionHeader, 'video-filename'))
1460 1428
1461 @staticmethod 1429 @staticmethod
1462 def loadConfigFile(filename): 1430 def loadConfigFile(filename):
1463 from configparser import ConfigParser 1431 from configparser import ConfigParser
1464 config = ConfigParser() 1432 config = ConfigParser()
1465 config.readfp(openCheck(filename)) 1433 config.readfp(utils.openCheck(filename))
1466 configDict = dict() 1434 configDict = dict()
1467 for sectionName in config.sections(): 1435 for sectionName in config.sections():
1468 configDict[sectionName] = SceneParameters(config, sectionName) 1436 configDict[sectionName] = SceneParameters(config, sectionName)
1469 return configDict 1437 return configDict
1470 1438