Mercurial > hg > nsaunier > traffic-intelligence

comparison python/storage.py @ 564:36605d843be5

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
modified bug for reading vissim files, cleaned use of readline with multiple type of characters for comments (to ignore)
author Nicolas Saunier <nicolas.saunier@polymtl.ca>
date Mon, 21 Jul 2014 16:17:22 -0400
parents ca6bded754ac
children 5bda87ac0a69 84690dfe5560
comparison
equal deleted inserted replaced
563:39de5c532559 564:36605d843be5
33 cursor.execute('DROP TABLE IF EXISTS '+tableName) 33 cursor.execute('DROP TABLE IF EXISTS '+tableName)
34 except sqlite3.OperationalError as error: 34 except sqlite3.OperationalError as error:
35 printDBError(error) 35 printDBError(error)
36 36
37 # IO to sqlite 37 # IO to sqlite
38 def writeTrajectoriesToSqlite(objects, outFilename, trajectoryType, objectNumbers = -1): 38 def writeTrajectoriesToSqlite(objects, outputFilename, trajectoryType, objectNumbers = -1):
39 """ 39 """
40 This function writers trajectories to a specified sqlite file 40 This function writers trajectories to a specified sqlite file
41 @param[in] objects -> a list of trajectories 41 @param[in] objects -> a list of trajectories
42 @param[in] trajectoryType - 42 @param[in] trajectoryType -
43 @param[out] outFile -> the .sqlite file containting the written objects 43 @param[out] outputFilename -> the .sqlite file containting the written objects
44 @param[in] objectNumber : number of objects loaded 44 @param[in] objectNumber : number of objects loaded
45 """ 45 """
46 connection = sqlite3.connect(outFilename) 46 connection = sqlite3.connect(outputFilename)
47 cursor = connection.cursor() 47 cursor = connection.cursor()
48 48
49 schema = "CREATE TABLE IF NOT EXISTS \"positions\"(trajectory_id INTEGER,frame_number INTEGER, x_coordinate REAL, y_coordinate REAL, PRIMARY KEY(trajectory_id, frame_number))" 49 schema = "CREATE TABLE IF NOT EXISTS \"positions\"(trajectory_id INTEGER,frame_number INTEGER, x_coordinate REAL, y_coordinate REAL, PRIMARY KEY(trajectory_id, frame_number))"
50 cursor.execute(schema) 50 cursor.execute(schema)
51 51
62 cursor.execute(query,(trajectory_id,frame_number,position.x,position.y)) 62 cursor.execute(query,(trajectory_id,frame_number,position.x,position.y))
63 63
64 connection.commit() 64 connection.commit()
65 connection.close() 65 connection.close()
66 66
67 def writeFeaturesToSqlite(objects, outFilename, trajectoryType, objectNumbers = -1): 67 def writeFeaturesToSqlite(objects, outputFilename, trajectoryType, objectNumbers = -1):
68 '''write features trajectories maintain trajectory ID,velocities dataset ''' 68 '''write features trajectories maintain trajectory ID,velocities dataset '''
69 connection = sqlite3.connect(outFilename) 69 connection = sqlite3.connect(outputFilename)
70 cursor = connection.cursor() 70 cursor = connection.cursor()
71 71
72 cursor.execute("CREATE TABLE IF NOT EXISTS \"positions\"(trajectory_id INTEGER,frame_number INTEGER, x_coordinate REAL, y_coordinate REAL, PRIMARY KEY(trajectory_id, frame_number))") 72 cursor.execute("CREATE TABLE IF NOT EXISTS \"positions\"(trajectory_id INTEGER,frame_number INTEGER, x_coordinate REAL, y_coordinate REAL, PRIMARY KEY(trajectory_id, frame_number))")
73 cursor.execute("CREATE TABLE IF NOT EXISTS \"velocities\"(trajectory_id INTEGER,frame_number INTEGER, x_coordinate REAL, y_coordinate REAL, PRIMARY KEY(trajectory_id, frame_number))") 73 cursor.execute("CREATE TABLE IF NOT EXISTS \"velocities\"(trajectory_id INTEGER,frame_number INTEGER, x_coordinate REAL, y_coordinate REAL, PRIMARY KEY(trajectory_id, frame_number))")
74 74
83 frame_number += 1 83 frame_number += 1
84 84
85 connection.commit() 85 connection.commit()
86 connection.close() 86 connection.close()
87 87
88 def writePrototypesToSqlite(prototypes,nMatching, outFilename): 88 def writePrototypesToSqlite(prototypes,nMatching, outputFilename):
89 """ prototype dataset is a dictionary with keys== routes, values== prototypes Ids """ 89 """ prototype dataset is a dictionary with keys== routes, values== prototypes Ids """
90 connection = sqlite3.connect(outFilename) 90 connection = sqlite3.connect(outputFilename)
91 cursor = connection.cursor() 91 cursor = connection.cursor()
92 92
93 cursor.execute("CREATE TABLE IF NOT EXISTS \"prototypes\"(prototype_id INTEGER,routeIDstart INTEGER,routeIDend INTEGER, nMatching INTEGER, PRIMARY KEY(prototype_id))") 93 cursor.execute("CREATE TABLE IF NOT EXISTS \"prototypes\"(prototype_id INTEGER,routeIDstart INTEGER,routeIDend INTEGER, nMatching INTEGER, PRIMARY KEY(prototype_id))")
94 94
95 for route in prototypes.keys(): 95 for route in prototypes.keys():
125 nMatching[row[0]]=row[3] 125 nMatching[row[0]]=row[3]
126 126
127 connection.close() 127 connection.close()
128 return prototypes,nMatching 128 return prototypes,nMatching
129 129
130 def writeLabelsToSqlite(labels, outFilename): 130 def writeLabelsToSqlite(labels, outputFilename):
131 """ labels is a dictionary with keys: routes, values: prototypes Ids 131 """ labels is a dictionary with keys: routes, values: prototypes Ids
132 """ 132 """
133 connection = sqlite3.connect(outFilename) 133 connection = sqlite3.connect(outputFilename)
134 cursor = connection.cursor() 134 cursor = connection.cursor()
135 135
136 cursor.execute("CREATE TABLE IF NOT EXISTS \"labels\"(object_id INTEGER,routeIDstart INTEGER,routeIDend INTEGER, prototype_id INTEGER, PRIMARY KEY(object_id))") 136 cursor.execute("CREATE TABLE IF NOT EXISTS \"labels\"(object_id INTEGER,routeIDstart INTEGER,routeIDend INTEGER, prototype_id INTEGER, PRIMARY KEY(object_id))")
137 137
138 for route in labels.keys(): 138 for route in labels.keys():
166 labels[route][p].append(row[0]) 166 labels[route][p].append(row[0])
167 167
168 connection.close() 168 connection.close()
169 return labels 169 return labels
170 170
171 def writeRoutesToSqlite(Routes, outFilename): 171 def writeRoutesToSqlite(Routes, outputFilename):
172 """ This function writes the activity path define by start and end IDs""" 172 """ This function writes the activity path define by start and end IDs"""
173 connection = sqlite3.connect(outFilename) 173 connection = sqlite3.connect(outputFilename)
174 cursor = connection.cursor() 174 cursor = connection.cursor()
175 175
176 cursor.execute("CREATE TABLE IF NOT EXISTS \"routes\"(object_id INTEGER,routeIDstart INTEGER,routeIDend INTEGER, PRIMARY KEY(object_id))") 176 cursor.execute("CREATE TABLE IF NOT EXISTS \"routes\"(object_id INTEGER,routeIDstart INTEGER,routeIDend INTEGER, PRIMARY KEY(object_id))")
177 177
178 for route in Routes.keys(): 178 for route in Routes.keys():
515 if quitting: 515 if quitting:
516 from sys import exit 516 from sys import exit
517 exit() 517 exit()
518 return None 518 return None
519 519
520 def readline(f, commentChar = commentChar): 520 def readline(f, commentCharacters = commentChar):
521 '''Modified readline function to skip comments.''' 521 '''Modified readline function to skip comments
522 Can take a list of characters or a string (in will work in both)'''
522 s = f.readline() 523 s = f.readline()
523 while (len(s) > 0) and s.startswith(commentChar): 524 while (len(s) > 0) and s[0] in commentCharacters:
524 s = f.readline() 525 s = f.readline()
525 return s.strip() 526 return s.strip()
526 527
527 def getLines(f, commentChar = commentChar): 528 def getLines(f, commentCharacters = commentChar):
528 '''Gets a complete entry (all the lines) in between delimiterChar.''' 529 '''Gets a complete entry (all the lines) in between delimiterChar.'''
529 dataStrings = [] 530 dataStrings = []
530 s = readline(f, commentChar) 531 s = readline(f, commentCharacters)
531 while len(s) > 0: 532 while len(s) > 0:
532 dataStrings += [s.strip()] 533 dataStrings += [s.strip()]
533 s = readline(f, commentChar) 534 s = readline(f, commentCharacters)
534 return dataStrings 535 return dataStrings
535 536
536 def writeList(filename, l): 537 def writeList(filename, l):
537 f = openCheck(filename, 'w') 538 f = openCheck(filename, 'w')
538 for x in l: 539 for x in l:
539 f.write('{}\n'.format(x)) 540 f.write('{}\n'.format(x))
540 f.close() 541 f.close()
541 542
542 def loadListStrings(filename, commentChar = commentChar): 543 def loadListStrings(filename, commentCharacters = commentChar):
543 f = openCheck(filename, 'r') 544 f = openCheck(filename, 'r')
544 result = getLines(f, commentChar) 545 result = getLines(f, commentCharacters)
545 f.close() 546 f.close()
546 return result 547 return result
547 548
548 def getValuesFromINIFile(filename, option, delimiterChar = '=', commentChar = commentChar): 549 def getValuesFromINIFile(filename, option, delimiterChar = '=', commentCharacters = commentChar):
549 values = [] 550 values = []
550 for l in loadListStrings(filename, commentChar): 551 for l in loadListStrings(filename, commentCharacters):
551 if l.startswith(option): 552 if l.startswith(option):
552 values.append(l.split(delimiterChar)[1].strip()) 553 values.append(l.split(delimiterChar)[1].strip())
553 return values 554 return values
554 555
555 class FakeSecHead(object): 556 class FakeSecHead(object):
577 578
578 Assumed to be sorted over time''' 579 Assumed to be sorted over time'''
579 objects = {} # dictionary of objects index by their id 580 objects = {} # dictionary of objects index by their id
580 firstInstants = {} 581 firstInstants = {}
581 582
582 infile = openCheck(filename, quitting = True) 583 inputfile = openCheck(filename, quitting = True)
583 584
584 # data = pd.read_csv(filename, skiprows=15, delimiter=';') 585 # data = pd.read_csv(filename, skiprows=15, delimiter=';')
585 # skip header: 15 lines + 1 586 # skip header: 15 lines + 1
586 for i in range(16): 587 line = readline(inputfile, '*$')
587 readline(infile) 588 while len(line) > 0:#for line in inputfile:
588
589 for line in infile:
590 data = line.strip().split(';') 589 data = line.strip().split(';')
591 objNum = int(data[1]) 590 objNum = int(data[1])
592 instant = int(float(data[0])*simulationStepsPerTimeUnit) 591 instant = int(float(data[0])*simulationStepsPerTimeUnit)
593 s = float(data[4]) 592 s = float(data[4])
594 y = float(data[5]) 593 y = float(data[5])
600 objects[objNum] = moving.MovingObject(num = objNum, timeInterval = moving.TimeInterval(instant, instant)) 599 objects[objNum] = moving.MovingObject(num = objNum, timeInterval = moving.TimeInterval(instant, instant))
601 objects[objNum].curvilinearPositions = moving.CurvilinearTrajectory() 600 objects[objNum].curvilinearPositions = moving.CurvilinearTrajectory()
602 if (warmUpLastInstant == None or firstInstants[objNum] >= warmUpLastInstant) and objNum in objects: 601 if (warmUpLastInstant == None or firstInstants[objNum] >= warmUpLastInstant) and objNum in objects:
603 objects[objNum].timeInterval.last = instant 602 objects[objNum].timeInterval.last = instant
604 objects[objNum].curvilinearPositions.addPositionSYL(s, y, lane) 603 objects[objNum].curvilinearPositions.addPositionSYL(s, y, lane)
604 line = readline(inputfile, '*$')
605 605
606 return objects.values() 606 return objects.values()
607 607
608 def loadTrajectoriesFromNgsimFile(filename, nObjects = -1, sequenceNum = -1): 608 def loadTrajectoriesFromNgsimFile(filename, nObjects = -1, sequenceNum = -1):
609 '''Reads data from the trajectory data provided by NGSIM project 609 '''Reads data from the trajectory data provided by NGSIM project
610 and returns the list of Feature objects''' 610 and returns the list of Feature objects'''
611 objects = [] 611 objects = []
612 612
613 infile = openCheck(filename, quitting = True) 613 inputfile = openCheck(filename, quitting = True)
614 614
615 def createObject(numbers): 615 def createObject(numbers):
616 firstFrameNum = int(numbers[1]) 616 firstFrameNum = int(numbers[1])
617 # do the geometry and usertype 617 # do the geometry and usertype
618 618
632 obj.curvilinearPositions = moving.CurvilinearTrajectory([float(numbers[5])],[float(numbers[4])], obj.laneNums) # X is the longitudinal coordinate 632 obj.curvilinearPositions = moving.CurvilinearTrajectory([float(numbers[5])],[float(numbers[4])], obj.laneNums) # X is the longitudinal coordinate
633 obj.speeds = [float(numbers[11])] 633 obj.speeds = [float(numbers[11])]
634 obj.size = [float(numbers[8]), float(numbers[9])] # 8 lengh, 9 width # TODO: temporary, should use a geometry object 634 obj.size = [float(numbers[8]), float(numbers[9])] # 8 lengh, 9 width # TODO: temporary, should use a geometry object
635 return obj 635 return obj
636 636
637 numbers = infile.readline().strip().split() 637 numbers = readline(inputfile).strip().split()
638 if (len(numbers) > 0): 638 if (len(numbers) > 0):
639 obj = createObject(numbers) 639 obj = createObject(numbers)
640 640
641 for line in infile: 641 for line in inputfile:
642 numbers = line.strip().split() 642 numbers = line.strip().split()
643 if obj.getNum() != int(numbers[0]): 643 if obj.getNum() != int(numbers[0]):
644 # check and adapt the length to deal with issues in NGSIM data 644 # check and adapt the length to deal with issues in NGSIM data
645 if (obj.length() != obj.positions.length()): 645 if (obj.length() != obj.positions.length()):
646 print 'length pb with object %s (%d,%d)' % (obj.getNum(),obj.length(),obj.positions.length()) 646 print 'length pb with object %s (%d,%d)' % (obj.getNum(),obj.length(),obj.positions.length())
663 if (obj.size[0] != float(numbers[8])): 663 if (obj.size[0] != float(numbers[8])):
664 print 'changed length obj %d' % (obj.getNum()) 664 print 'changed length obj %d' % (obj.getNum())
665 if (obj.size[1] != float(numbers[9])): 665 if (obj.size[1] != float(numbers[9])):
666 print 'changed width obj %d' % (obj.getNum()) 666 print 'changed width obj %d' % (obj.getNum())
667 667
668 infile.close() 668 inputfile.close()
669 return objects 669 return objects
670 670
671 def convertNgsimFile(inFile, outFile, append = False, nObjects = -1, sequenceNum = 0): 671 def convertNgsimFile(inputfile, outputfile, append = False, nObjects = -1, sequenceNum = 0):
672 '''Reads data from the trajectory data provided by NGSIM project 672 '''Reads data from the trajectory data provided by NGSIM project
673 and converts to our current format.''' 673 and converts to our current format.'''
674 if append: 674 if append:
675 out = openCheck(outFile,'a') 675 out = openCheck(outputfile,'a')
676 else: 676 else:
677 out = openCheck(outFile,'w') 677 out = openCheck(outputfile,'w')
678 nObjectsPerType = [0,0,0] 678 nObjectsPerType = [0,0,0]
679 679
680 features = loadNgsimFile(inFile, sequenceNum) 680 features = loadNgsimFile(inputfile, sequenceNum)
681 for f in features: 681 for f in features:
682 nObjectsPerType[f.userType-1] += 1 682 nObjectsPerType[f.userType-1] += 1
683 f.write(out) 683 f.write(out)
684 684
685 print nObjectsPerType 685 print nObjectsPerType