Mercurial > hg > nsaunier > traffic-intelligence

comparison python/storage.py @ 756:a73f43aac00e dev

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
moved pandas imports, sql in comments
author Nicolas Saunier <nicolas.saunier@polymtl.ca>
date Fri, 30 Oct 2015 17:57:09 -0400
parents f3aeb0b47eff
children a05b70f307dd
comparison
equal deleted inserted replaced
755:f3aeb0b47eff 756:a73f43aac00e
5 import utils, moving, events, indicators, shutil 5 import utils, moving, events, indicators, shutil
6 from base import VideoFilenameAddable 6 from base import VideoFilenameAddable
7 7
8 import sqlite3, logging 8 import sqlite3, logging
9 from numpy import log, min as npmin, max as npmax, round as npround, array, sum as npsum, loadtxt 9 from numpy import log, min as npmin, max as npmax, round as npround, array, sum as npsum, loadtxt
10 from pandas import read_csv, merge
10 11
11 12
12 commentChar = '#' 13 commentChar = '#'
13 14
14 delimiterChar = '%'; 15 delimiterChar = '%';
774 Assumed to be sorted over time 775 Assumed to be sorted over time
775 Warning: if reading from SQLite a limited number of objects, objectNumbers will be the maximum object id''' 776 Warning: if reading from SQLite a limited number of objects, objectNumbers will be the maximum object id'''
776 objects = {} # dictionary of objects index by their id 777 objects = {} # dictionary of objects index by their id
777 778
778 if usePandas: 779 if usePandas:
779 from pandas import read_csv
780 data = read_csv(filename, delimiter=';', comment='*', header=0, skiprows = 1, low_memory = lowMemory) 780 data = read_csv(filename, delimiter=';', comment='*', header=0, skiprows = 1, low_memory = lowMemory)
781 generatePDLaneColumn(data) 781 generatePDLaneColumn(data)
782 data['TIME'] = data['$VEHICLE:SIMSEC']*simulationStepsPerTimeUnit 782 data['TIME'] = data['$VEHICLE:SIMSEC']*simulationStepsPerTimeUnit
783 if warmUpLastInstant is not None: 783 if warmUpLastInstant is not None:
784 data = data[data['TIME']>=warmUpLastInstant] 784 data = data[data['TIME']>=warmUpLastInstant]
861 861
862 Vehicles are considered finally stationary 862 Vehicles are considered finally stationary
863 if more than proportionStationaryTime of their total time 863 if more than proportionStationaryTime of their total time
864 If lanes is not None, only the data for the selected lanes will be provided 864 If lanes is not None, only the data for the selected lanes will be provided
865 (format as string x_y where x is link index and y is lane index)''' 865 (format as string x_y where x is link index and y is lane index)'''
866 from pandas import read_csv 866 if filename.endswith(".fzp"):
867 columns = ['NO', '$VEHICLE:SIMSEC', 'POS'] 867 columns = ['NO', '$VEHICLE:SIMSEC', 'POS']
868 if lanes is not None: 868 if lanes is not None:
869 columns += ['LANE\LINK\NO', 'LANE\INDEX'] 869 columns += ['LANE\LINK\NO', 'LANE\INDEX']
870 data = read_csv(filename, delimiter=';', comment='*', header=0, skiprows = 1, usecols = columns, low_memory = lowMemory) 870 data = read_csv(filename, delimiter=';', comment='*', header=0, skiprows = 1, usecols = columns, low_memory = lowMemory)
871 data = selectPDLanes(data, lanes) 871 data = selectPDLanes(data, lanes)
872 data.sort(['$VEHICLE:SIMSEC'], inplace = True) 872 data.sort(['$VEHICLE:SIMSEC'], inplace = True)
873 873
874 nStationary = 0 874 nStationary = 0
875 nVehicles = 0 875 nVehicles = 0
876 for name, group in data.groupby(['NO'], sort = False): 876 for name, group in data.groupby(['NO'], sort = False):
877 nVehicles += 1 877 nVehicles += 1
878 positions = array(group['POS']) 878 positions = array(group['POS'])
879 diff = positions[1:]-positions[:-1] 879 diff = positions[1:]-positions[:-1]
880 if npsum(diff == 0.) >= proportionStationaryTime*len(positions): 880 if npsum(diff == 0.) >= proportionStationaryTime*(len(positions)-1):
881 nStationary += 1 881 nStationary += 1
882 elif filename.endswith(".sqlite"):
883 # select trajectory_id, t, s_coordinate, speed from curvilinear_positions where trajectory_id between 1860 and 1870 and speed < 0.1
884 # pb of the meaning of proportionStationaryTime in arterial network? Why proportion of existence time?
885 pass
886 else:
887 print("File type of "+filename+" not supported (only .sqlite and .fzp files)")
882 888
883 return nStationary, nVehicles 889 return nStationary, nVehicles
884 890
885 def countCollisionsVissim(filename, lanes = None, collisionTimeDifference = 0.2, lowMemory = True): 891 def countCollisionsVissim(filename, lanes = None, collisionTimeDifference = 0.2, lowMemory = True):
886 '''Counts the number of collisions per lane in a VISSIM trajectory file 892 '''Counts the number of collisions per lane in a VISSIM trajectory file
888 To distinguish between cars passing and collision, 894 To distinguish between cars passing and collision,
889 one checks when the sign of the position difference inverts 895 one checks when the sign of the position difference inverts
890 (if the time are closer than collisionTimeDifference) 896 (if the time are closer than collisionTimeDifference)
891 If lanes is not None, only the data for the selected lanes will be provided 897 If lanes is not None, only the data for the selected lanes will be provided
892 (format as string x_y where x is link index and y is lane index)''' 898 (format as string x_y where x is link index and y is lane index)'''
893 from pandas import read_csv, merge
894 data = read_csv(filename, delimiter=';', comment='*', header=0, skiprows = 1, usecols = ['LANE\LINK\NO', 'LANE\INDEX', '$VEHICLE:SIMSEC', 'NO', 'POS'], low_memory = lowMemory) 899 data = read_csv(filename, delimiter=';', comment='*', header=0, skiprows = 1, usecols = ['LANE\LINK\NO', 'LANE\INDEX', '$VEHICLE:SIMSEC', 'NO', 'POS'], low_memory = lowMemory)
895 data = selectPDLanes(data, lanes) 900 data = selectPDLanes(data, lanes)
896 data = data.convert_objects(convert_numeric=True) 901 data = data.convert_objects(convert_numeric=True)
897 902
898 merged = merge(data, data, how='inner', left_on=['LANE\LINK\NO', 'LANE\INDEX', '$VEHICLE:SIMSEC'], right_on=['LANE\LINK\NO', 'LANE\INDEX', '$VEHICLE:SIMSEC'], sort = False) 903 merged = merge(data, data, how='inner', left_on=['LANE\LINK\NO', 'LANE\INDEX', '$VEHICLE:SIMSEC'], right_on=['LANE\LINK\NO', 'LANE\INDEX', '$VEHICLE:SIMSEC'], sort = False)
905 if len(diff) >= 2 and npmin(diff) < 0 and npmax(diff) > 0: 910 if len(diff) >= 2 and npmin(diff) < 0 and npmax(diff) > 0:
906 xidx = diff[diff < 0].argmax() 911 xidx = diff[diff < 0].argmax()
907 yidx = diff[diff > 0].argmin() 912 yidx = diff[diff > 0].argmin()
908 if abs(group.loc[xidx, '$VEHICLE:SIMSEC'] - group.loc[yidx, '$VEHICLE:SIMSEC']) <= collisionTimeDifference: 913 if abs(group.loc[xidx, '$VEHICLE:SIMSEC'] - group.loc[yidx, '$VEHICLE:SIMSEC']) <= collisionTimeDifference:
909 nCollisions += 1 914 nCollisions += 1
915
916 # select TD1.link_id, TD1.lane_id from temp.diff_positions as TD1, temp.diff_positions as TD2 where TD1.link_id = TD2.link_id and TD1.lane_id = TD2.lane_id and TD1.id1 = TD2.id1 and TD1.id2 = TD2.id2 and TD1.t = TD2.t+0.1 and TD1.diff*TD2.diff < 0; # besoin de faire un group by??
917 # create temp table diff_positions as select CP1.t as t, CP1.link_id as link_id, CP1.lane_id as lane_id, CP1.trajectory_id as id1, CP2.trajectory_id as id2, CP1.s_coordinate - CP2.s_coordinate as diff from curvilinear_positions CP1, curvilinear_positions CP2 where CP1.link_id = CP2.link_id and CP1.lane_id = CP2.lane_id and CP1.t = CP2.t and CP1.trajectory_id > CP2.trajectory_id;
918 # SQL select link_id, lane_id, id1, id2, min(diff), max(diff) from (select CP1.t as t, CP1.link_id as link_id, CP1.lane_id as lane_id, CP1.trajectory_id as id1, CP2.trajectory_id as id2, CP1.s_coordinate - CP2.s_coordinate as diff from curvilinear_positions CP1, curvilinear_positions CP2 where CP1.link_id = CP2.link_id and CP1.lane_id = CP2.lane_id and CP1.t = CP2.t and CP1.trajectory_id > CP2.trajectory_id) group by link_id, lane_id, id1, id2 having min(diff)*max(diff) < 0
910 return nCollisions 919 return nCollisions
911 920
912 def loadTrajectoriesFromNgsimFile(filename, nObjects = -1, sequenceNum = -1): 921 def loadTrajectoriesFromNgsimFile(filename, nObjects = -1, sequenceNum = -1):
913 '''Reads data from the trajectory data provided by NGSIM project 922 '''Reads data from the trajectory data provided by NGSIM project
914 and returns the list of Feature objects''' 923 and returns the list of Feature objects'''