--- a/scripts/process.py	Thu Jul 05 22:24:31 2018 -0400
+++ b/scripts/process.py	Fri Jul 06 18:42:58 2018 -0400
@@ -7,17 +7,17 @@
 #import matplotlib
 #atplotlib.use('Agg')
 import matplotlib.pyplot as plt
-from numpy import percentile
+import numpy as np
 from pandas import DataFrame
 
-from trafficintelligence import storage, events, prediction, cvutils, utils
+from trafficintelligence import storage, events, prediction, cvutils, utils, moving, processing, ml
 from trafficintelligence.metadata import *
 
 parser = argparse.ArgumentParser(description='This program manages the processing of several files based on a description of the sites and video data in an SQLite database following the metadata module.')
 # input
 parser.add_argument('--db', dest = 'metadataFilename', help = 'name of the metadata file', required = True)
 parser.add_argument('--videos', dest = 'videoIds', help = 'indices of the video sequences', nargs = '*', type = int)
-parser.add_argument('--sites', dest = 'siteIds', help = 'indices of the video sequences', nargs = '*', type = int)
+parser.add_argument('--sites', dest = 'siteIds', help = 'indices of the video sequences', nargs = '*')
 
 # main function
 parser.add_argument('--delete', dest = 'delete', help = 'data to delete', choices = ['feature', 'object', 'classification', 'interaction'])
@@ -28,8 +28,34 @@
 # common options
 parser.add_argument('--cfg', dest = 'configFilename', help = 'name of the configuration file')
 parser.add_argument('-n', dest = 'nObjects', help = 'number of objects/interactions to process', type = int)
+parser.add_argument('-t', dest = 'trajectoryType', help = 'type of trajectories', choices = ['feature', 'object'], default = 'feature')
 parser.add_argument('--dry', dest = 'dryRun', help = 'dry run of processing', action = 'store_true')
 parser.add_argument('--nthreads', dest = 'nProcesses', help = 'number of processes to run in parallel', type = int, default = 1)
+parser.add_argument('--subsample', dest = 'positionSubsamplingRate', help = 'rate of position subsampling (1 every n positions)', type = int)
+
+### process options
+# motion pattern learning and assignment
+parser.add_argument('--prototype-filename', dest = 'outputPrototypeDatabaseFilename', help = 'name of the Sqlite database file to save prototypes', default = 'prototypes.sqlite')
+#parser.add_argument('-i', dest = 'inputPrototypeDatabaseFilename', help = 'name of the Sqlite database file for prototypes to start the algorithm with')
+parser.add_argument('--nobjects-mp', dest = 'nMPObjects', help = 'number of objects/interactions to process', type = int)
+parser.add_argument('--nfeatures-per-object', dest = 'nLongestFeaturesPerObject', help = 'maximum number of features per object to load', type = int)
+parser.add_argument('--epsilon', dest = 'epsilon', help = 'distance for the similarity of trajectory points', type = float)
+parser.add_argument('--metric', dest = 'metric', help = 'metric for the similarity of trajectory points', default = 'cityblock') # default is manhattan distance
+parser.add_argument('--minsimil', dest = 'minSimilarity', help = 'minimum similarity to put a trajectory in a cluster', type = float)
+parser.add_argument('--min-cluster-size', dest = 'minClusterSize', help = 'minimum cluster size', type = int, default = 0)
+#parser.add_argument('--learn', dest = 'learn', help = 'learn', action = 'store_true')
+parser.add_argument('--optimize', dest = 'optimizeCentroid', help = 'recompute centroid at each assignment', action = 'store_true')
+parser.add_argument('--random', dest = 'randomInitialization', help = 'random initialization of clustering algorithm', action = 'store_true')
+#parser.add_argument('--similarities-filename', dest = 'similaritiesFilename', help = 'filename of the similarities')
+parser.add_argument('--save-similarities', dest = 'saveSimilarities', help = 'save computed similarities (in addition to prototypes)', action = 'store_true')
+parser.add_argument('--save-assignments', dest = 'saveAssignments', help = 'saves the assignments of the objects to the prototypes', action = 'store_true')
+parser.add_argument('--assign', dest = 'assign', help = 'assigns the objects to the prototypes and saves the assignments', action = 'store_true')
+
+# safety analysis
+parser.add_argument('--prediction-method', dest = 'predictionMethod', help = 'prediction method (constant velocity (cvd: vector computation (approximate); cve: equation solving; cv: discrete time (approximate)), normal adaptation, point set prediction)', choices = ['cvd', 'cve', 'cv', 'na', 'ps', 'mp'])
+parser.add_argument('--pet', dest = 'computePET', help = 'computes PET', action = 'store_true')
+# override other tracking config, erase sqlite?
+
 
 # analysis options
 parser.add_argument('--output', dest = 'output', help = 'kind of output to produce (interval means)', choices = ['figure', 'interval', 'event'])
@@ -40,11 +66,6 @@
 dpi = 150
 # unit of analysis: site or video sequence?
 
-# safety analysis
-parser.add_argument('--prediction-method', dest = 'predictionMethod', help = 'prediction method (constant velocity (cvd: vector computation (approximate); cve: equation solving; cv: discrete time (approximate)), normal adaptation, point set prediction)', choices = ['cvd', 'cve', 'cv', 'na', 'ps', 'mp'])
-parser.add_argument('--pet', dest = 'computePET', help = 'computes PET', action = 'store_true')
-# override other tracking config, erase sqlite?
-
 # need way of selecting sites as similar as possible to sql alchemy syntax
 # override tracking.cfg from db
 # manage cfg files, overwrite them (or a subset of parameters)
@@ -59,13 +80,18 @@
 session = connectDatabase(args.metadataFilename)
 parentPath = Path(args.metadataFilename).parent # files are relative to metadata location
 videoSequences = []
+sites = []
 if args.videoIds is not None:
     videoSequences = [session.query(VideoSequence).get(videoId) for videoId in args.videoIds]
+    siteIds = set([vs.cameraView.siteIdx for vs in videoSequences])
 elif args.siteIds is not None:
-    for siteId in args.siteIds:
-        for site in getSite(session, siteId):
+    siteIds = set(args.siteIds)
+    for siteId in siteIds:
+        tmpsites = getSite(session, siteId)
+        sites.extend(tmpsites)
+        for site in tmpsites:
             for cv in site.cameraViews:
-                videoSequences += cv.videoSequences
+                videoSequences.extend(cv.videoSequences)
 else:
     print('No video/site to process')
 
@@ -121,7 +147,40 @@
         pool.join()
 
 elif args.process == 'prototype': # motion pattern learning
-    pass
+    # learn by site by default -> group videos by site (or by camera view? TODO add cameraviews)
+    # by default, load all objects, learn and then assign (BUT not save the assignments)
+    for site in sites:
+        print('Learning motion patterns for site {} ({})'.format(site.idx, site.name))
+        objects = {}
+        object2VideoSequences = {}
+        for cv in site.cameraViews:
+            for vs in cv.videoSequences:
+                print('Loading '+vs.getDatabaseFilename())
+                objects[vs.idx] = storage.loadTrajectoriesFromSqlite(str(parentPath/vs.getDatabaseFilename()), args.trajectoryType, args.nObjects, timeStep = args.positionSubsamplingRate, nLongestFeaturesPerObject = args.nLongestFeaturesPerObject)
+                if args.trajectoryType == 'object' and args.nLongestFeaturesPerObject is not None:
+                    objectsWithFeatures = objects[vs.idx]
+                    objects[vs.idx] = [f for o in objectsWithFeatures for f in o.getFeatures()]
+                    prototypeType = 'feature'
+                else:
+                    prototypeType = args.trajectoryType
+                for obj in objects[vs.idx]:
+                    object2VideoSequences[obj] = vs
+        lcss = utils.LCSS(metric = args.metric, epsilon = args.epsilon)
+        similarityFunc = lambda x,y : lcss.computeNormalized(x, y)
+        trainingObjects = [o for tmpobjects in objects.values() for o in tmpobjects]
+        if args.nMPObjects is not None and args.nMPObjects < len(trainingObjects):
+            m = int(np.floor(float(len(trainingObjects))/args.nMPObjects))
+            trainingObjects = trainingObjects[::m]
+        similarities = -np.ones((len(trainingObjects), len(trainingObjects)))
+        prototypeIndices, labels = processing.learnAssignMotionPatterns(True, True, trainingObjects, similarities, args.minSimilarity, similarityFunc, args.minClusterSize, args.optimizeCentroid, args.randomInitialization, True, [])
+        if args.outputPrototypeDatabaseFilename is None:
+            outputPrototypeDatabaseFilename = args.databaseFilename
+        else:
+            outputPrototypeDatabaseFilename = args.outputPrototypeDatabaseFilename
+        # TODO maintain mapping from object prototype to db filename + compute nmatchings before
+        clusterSizes = ml.computeClusterSizes(labels, prototypeIndices, -1)
+        storage.savePrototypesToSqlite(str(parentPath/site.getPath()/outputPrototypeDatabaseFilename), [moving.Prototype(object2VideoSequences[trainingObjects[i]].getDatabaseFilename(False), trainingObjects[i].getNum(), prototypeType, clusterSizes[i]) for i in prototypeIndices])
+
 
 elif args.process == 'interaction':
     # safety analysis TODO make function in safety analysis script
@@ -183,10 +242,6 @@
                         row.append(aggSpeeds)
             data.append(row)
     data = DataFrame(data, columns = headers)
-    if args.siteIds is None:
-        siteIds = set([vs.cameraView.siteIdx for vs in videoSequences])
-    else:
-        siteIds = set(args.siteIds)
     if args.output == 'figure':
         for name in headers[4:]:
             plt.ioff()