From 79118f2406042535aa6e7713e44e0135dce85808 Mon Sep 17 00:00:00 2001 From: wilsonwong2104 Date: Sun, 24 May 2020 18:28:01 +0800 Subject: [PATCH 01/24] update --- MyReadMe.txt | 6 + my_deep_sort_app.py | 264 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 270 insertions(+) create mode 100644 MyReadMe.txt create mode 100644 my_deep_sort_app.py diff --git a/MyReadMe.txt b/MyReadMe.txt new file mode 100644 index 00000000..30c513d3 --- /dev/null +++ b/MyReadMe.txt @@ -0,0 +1,6 @@ +python deep_sort_app.py \ + --sequence_dir=/home/hjw/e/dataset_tiptical/MOT16/test/MOT16-06 \ + --detection_file=/home/hjw/e/dataset_tiptical/MOT16/test/MOT16-06/MOT16-06.npy \ + --min_confidence=0.3 \ + --nn_budget=100 \ + --display=True diff --git a/my_deep_sort_app.py b/my_deep_sort_app.py new file mode 100644 index 00000000..b78b1310 --- /dev/null +++ b/my_deep_sort_app.py @@ -0,0 +1,264 @@ +# vim: expandtab:ts=4:sw=4 +from __future__ import division, print_function, absolute_import + +import argparse +import os + +import cv2 +import numpy as np + +from application_util import preprocessing +from application_util import visualization +from deep_sort import nn_matching +from deep_sort.detection import Detection +from deep_sort.tracker import Tracker + + +def gather_sequence_info(sequence_dir, detection_file): + """Gather sequence information, such as image filenames, detections, + groundtruth (if available). + + Parameters + ---------- + sequence_dir : str + Path to the MOTChallenge sequence directory. + detection_file : str + Path to the detection file. + + Returns + ------- + Dict + A dictionary of the following sequence information: + + * sequence_name: Name of the sequence + * image_filenames: A dictionary that maps frame indices to image + filenames. + * detections: A numpy array of detections in MOTChallenge format. + * groundtruth: A numpy array of ground truth in MOTChallenge format. + * image_size: Image size (height, width). + * min_frame_idx: Index of the first frame. + * max_frame_idx: Index of the last frame. + + """ + image_dir = os.path.join(sequence_dir, "img1") + image_filenames = { + int(os.path.splitext(f)[0]): os.path.join(image_dir, f) + for f in os.listdir(image_dir)} + groundtruth_file = os.path.join(sequence_dir, "gt/gt.txt") + + detections = None + if detection_file is not None: + detections = np.load(detection_file) + groundtruth = None + if os.path.exists(groundtruth_file): + groundtruth = np.loadtxt(groundtruth_file, delimiter=',') + + if len(image_filenames) > 0: + image = cv2.imread(next(iter(image_filenames.values())), + cv2.IMREAD_GRAYSCALE) + image_size = image.shape + else: + image_size = None + + if len(image_filenames) > 0: + min_frame_idx = min(image_filenames.keys()) + max_frame_idx = max(image_filenames.keys()) + else: + min_frame_idx = int(detections[:, 0].min()) + max_frame_idx = int(detections[:, 0].max()) + + info_filename = os.path.join(sequence_dir, "seqinfo.ini") + if os.path.exists(info_filename): + with open(info_filename, "r") as f: + line_splits = [l.split('=') for l in f.read().splitlines()[1:]] + info_dict = dict( + s for s in line_splits if isinstance(s, list) and len(s) == 2) + + update_ms = 1000 / int(info_dict["frameRate"]) + else: + update_ms = None + + feature_dim = detections.shape[1] - 10 if detections is not None else 0 + seq_info = { + "sequence_name": os.path.basename(sequence_dir), + "image_filenames": image_filenames, + "detections": detections, + "groundtruth": groundtruth, + "image_size": image_size, + "min_frame_idx": min_frame_idx, + "max_frame_idx": max_frame_idx, + "feature_dim": feature_dim, + "update_ms": update_ms + } + return seq_info + + +def create_detections(detection_mat, frame_idx, min_height=0): + """Create detections for given frame index from the raw detection matrix. + + Parameters + ---------- + detection_mat : ndarray + Matrix of detections. The first 10 columns of the detection matrix are + in the standard MOTChallenge detection format. In the remaining columns + store the feature vector associated with each detection. + frame_idx : int + The frame index. + min_height : Optional[int] + A minimum detection bounding box height. Detections that are smaller + than this value are disregarded. + + Returns + ------- + List[tracker.Detection] + Returns detection responses at given frame index. + + """ + frame_indices = detection_mat[:, 0].astype(np.int) + mask = frame_indices == frame_idx + + detection_list = [] + for row in detection_mat[mask]: + bbox, confidence, feature = row[2:6], row[6], row[10:] + if bbox[3] < min_height: + continue + detection_list.append(Detection(bbox, confidence, feature)) + return detection_list + + +def run(sequence_dir, detection_file, output_file, min_confidence, + nms_max_overlap, min_detection_height, max_cosine_distance, + nn_budget, display): + """Run multi-target tracker on a particular sequence. + + Parameters + ---------- + sequence_dir : str + Path to the MOTChallenge sequence directory. + detection_file : str + Path to the detections file. + output_file : str + Path to the tracking output file. This file will contain the tracking + results on completion. + min_confidence : float + Detection confidence threshold. Disregard all detections that have + a confidence lower than this value. + nms_max_overlap: float + Maximum detection overlap (non-maxima suppression threshold). + min_detection_height : int + Detection height threshold. Disregard all detections that have + a height lower than this value. + max_cosine_distance : float + Gating threshold for cosine distance metric (object appearance). + nn_budget : Optional[int] + Maximum size of the appearance descriptor gallery. If None, no budget + is enforced. + display : bool + If True, show visualization of intermediate tracking results. + + """ + seq_info = gather_sequence_info(sequence_dir, detection_file) + metric = nn_matching.NearestNeighborDistanceMetric( + "cosine", max_cosine_distance, nn_budget) + tracker = Tracker(metric) + results = [] + + def frame_callback(vis, frame_idx): + print("Processing frame %05d" % frame_idx) + + # Load image and generate detections. + detections = create_detections( + seq_info["detections"], frame_idx, min_detection_height) + detections = [d for d in detections if d.confidence >= min_confidence] + + # Run non-maxima suppression. + boxes = np.array([d.tlwh for d in detections]) + scores = np.array([d.confidence for d in detections]) + indices = preprocessing.non_max_suppression( + boxes, nms_max_overlap, scores) + detections = [detections[i] for i in indices] + + # Update tracker. + tracker.predict() + tracker.update(detections) + + # Update visualization. + if display: + image = cv2.imread( + seq_info["image_filenames"][frame_idx], cv2.IMREAD_COLOR) + vis.set_image(image.copy()) + vis.draw_detections(detections) + vis.draw_trackers(tracker.tracks) + + # Store results. + for track in tracker.tracks: + if not track.is_confirmed() or track.time_since_update > 1: + continue + bbox = track.to_tlwh() + results.append([ + frame_idx, track.track_id, bbox[0], bbox[1], bbox[2], bbox[3]]) + + # Run tracker. + if display: + visualizer = visualization.Visualization(seq_info, update_ms=5) + else: + visualizer = visualization.NoVisualization(seq_info) + visualizer.run(frame_callback) + + # Store results. + f = open(output_file, 'w') + for row in results: + print('%d,%d,%.2f,%.2f,%.2f,%.2f,1,-1,-1,-1' % ( + row[0], row[1], row[2], row[3], row[4], row[5]),file=f) + + +def bool_string(input_string): + if input_string not in {"True","False"}: + raise ValueError("Please Enter a valid Ture/False choice") + else: + return (input_string == "True") + +def parse_args(): + """ Parse command line arguments. + """ + parser = argparse.ArgumentParser(description="Deep SORT") + parser.add_argument( + "--sequence_dir", help="Path to MOTChallenge sequence directory", + default=None, required=True) + parser.add_argument( + "--detection_file", help="Path to custom detections.", default=None, + required=True) + parser.add_argument( + "--output_file", help="Path to the tracking output file. This file will" + " contain the tracking results on completion.", + default="/tmp/hypotheses.txt") + parser.add_argument( + "--min_confidence", help="Detection confidence threshold. Disregard " + "all detections that have a confidence lower than this value.", + default=0.8, type=float) + parser.add_argument( + "--min_detection_height", help="Threshold on the detection bounding " + "box height. Detections with height smaller than this value are " + "disregarded", default=0, type=int) + parser.add_argument( + "--nms_max_overlap", help="Non-maxima suppression threshold: Maximum " + "detection overlap.", default=1.0, type=float) + parser.add_argument( + "--max_cosine_distance", help="Gating threshold for cosine distance " + "metric (object appearance).", type=float, default=0.2) + parser.add_argument( + "--nn_budget", help="Maximum size of the appearance descriptors " + "gallery. If None, no budget is enforced.", type=int, default=None) + parser.add_argument( + "--display", help="Show intermediate tracking results", + default=True, type=bool_string) + return parser.parse_args() + + +if __name__ == "__main__": + args = parse_args() + print('Parameters:') + run( + args.sequence_dir, args.detection_file, args.output_file, + args.min_confidence, args.nms_max_overlap, args.min_detection_height, + args.max_cosine_distance, args.nn_budget, args.display) From 0587e8a8615bc4683b220c05e99f624f9899e1d1 Mon Sep 17 00:00:00 2001 From: wilsonwnog2014 Date: Mon, 25 May 2020 18:36:57 +0800 Subject: [PATCH 02/24] update --- MyReadMe.txt | 4 ++-- my_deep_sort_app.py | 14 +++++++++++++- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/MyReadMe.txt b/MyReadMe.txt index 30c513d3..f59d1552 100644 --- a/MyReadMe.txt +++ b/MyReadMe.txt @@ -1,6 +1,6 @@ python deep_sort_app.py \ - --sequence_dir=/home/hjw/e/dataset_tiptical/MOT16/test/MOT16-06 \ - --detection_file=/home/hjw/e/dataset_tiptical/MOT16/test/MOT16-06/MOT16-06.npy \ + --sequence_dir=$HOME/e/dataset_tiptical/MOT16/MOT16-06 \ + --detection_file=/home/hjw/e/dataset_tiptical/MOT16/MOT16-06/MOT16-06.npy \ --min_confidence=0.3 \ --nn_budget=100 \ --display=True diff --git a/my_deep_sort_app.py b/my_deep_sort_app.py index b78b1310..2f70eaeb 100644 --- a/my_deep_sort_app.py +++ b/my_deep_sort_app.py @@ -257,7 +257,19 @@ def parse_args(): if __name__ == "__main__": args = parse_args() - print('Parameters:') + print('Parameters') + print('==========') + print('sequence_dir: ', args.sequence_dir) + print('detection_file: ', args.detection_file) + print('output_file: ', args.output_file) + print('min_confidence: ', args.min_confidence) + print('min_detection_height: ', args.min_detection_height) + print('nms_max_overlap: ', args.nms_max_overlap) + print('max_cosine_distance: ', args.max_cosine_distance) + print('nn_budget: ', args.nn_budget) + print('display: ', args.display) + print('') + run( args.sequence_dir, args.detection_file, args.output_file, args.min_confidence, args.nms_max_overlap, args.min_detection_height, From f9183991de74213959e306eace8dc709a9ace6aa Mon Sep 17 00:00:00 2001 From: wilsonwnog2014 Date: Mon, 25 May 2020 19:03:34 +0800 Subject: [PATCH 03/24] update --- my_deep_sort_app.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/my_deep_sort_app.py b/my_deep_sort_app.py index 2f70eaeb..a87eb2e8 100644 --- a/my_deep_sort_app.py +++ b/my_deep_sort_app.py @@ -13,6 +13,23 @@ from deep_sort.detection import Detection from deep_sort.tracker import Tracker +def get_feature(sequence_dir, frame_idx, bbox, feature_dim): + '''提取检测目标特征 + 通过直方图描述目标特征 + @sequence_dir - 测试图像序列存放目录 + @frame_idx - 图像序号 + @bbox - 检测目标bbox + @feature_dim - 特征数 + @return feature - 特征向量 + ''' + image_dir = os.path.join(sequence_dir, "img1") + img_file= os.path.join(image_dir, ('000000'+str(frame_idx))[-6:]+'.jpg') + img = cv2.imread(img_file, cv2.IMREAD_GRAYSCALE) + t, l, w, h = bbox + roi = img[t:t+h;l:l+w] # tlwh + hists, bins = np.histogram(roi, bins=feature_dim) + return hists + def gather_sequence_info(sequence_dir, detection_file): """Gather sequence information, such as image filenames, detections, From 0d6e4c40350d17c885f0da05c7b1c1b65488d74e Mon Sep 17 00:00:00 2001 From: wilsonwnog2014 Date: Tue, 26 May 2020 10:07:15 +0800 Subject: [PATCH 04/24] 1. Bug Fixed zero divided. 2. Add np.histogram feature type --- MyReadMe.txt | 2 +- deep_sort/nn_matching.py | 4 ++-- env.bashrc | 3 +++ my_deep_sort_app.py | 43 +++++++++++++++++++++++++++------------- 4 files changed, 35 insertions(+), 17 deletions(-) create mode 100644 env.bashrc diff --git a/MyReadMe.txt b/MyReadMe.txt index f59d1552..95589f27 100644 --- a/MyReadMe.txt +++ b/MyReadMe.txt @@ -1,6 +1,6 @@ python deep_sort_app.py \ --sequence_dir=$HOME/e/dataset_tiptical/MOT16/MOT16-06 \ - --detection_file=/home/hjw/e/dataset_tiptical/MOT16/MOT16-06/MOT16-06.npy \ + --detection_file=$HOME/e/dataset_tiptical/MOT16/MOT16-06/MOT16-06.npy \ --min_confidence=0.3 \ --nn_budget=100 \ --display=True diff --git a/deep_sort/nn_matching.py b/deep_sort/nn_matching.py index 2e7bfea4..c1f20b5e 100644 --- a/deep_sort/nn_matching.py +++ b/deep_sort/nn_matching.py @@ -49,8 +49,8 @@ def _cosine_distance(a, b, data_is_normalized=False): """ if not data_is_normalized: - a = np.asarray(a) / np.linalg.norm(a, axis=1, keepdims=True) - b = np.asarray(b) / np.linalg.norm(b, axis=1, keepdims=True) + a = np.asarray(a) / (np.linalg.norm(a, axis=1, keepdims=True)+1e-5) + b = np.asarray(b) / (np.linalg.norm(b, axis=1, keepdims=True)+1e-5) return 1. - np.dot(a, b.T) diff --git a/env.bashrc b/env.bashrc new file mode 100644 index 00000000..167da416 --- /dev/null +++ b/env.bashrc @@ -0,0 +1,3 @@ +source ~/.bashrc +source ~/miniconda3/bin/activate tf1.14 + diff --git a/my_deep_sort_app.py b/my_deep_sort_app.py index a87eb2e8..3bfbac4f 100644 --- a/my_deep_sort_app.py +++ b/my_deep_sort_app.py @@ -13,7 +13,10 @@ from deep_sort.detection import Detection from deep_sort.tracker import Tracker -def get_feature(sequence_dir, frame_idx, bbox, feature_dim): +def get_feature1_dim(): + return 20 + +def get_feature1(sequence_dir, frame_idx, bbox): '''提取检测目标特征 通过直方图描述目标特征 @sequence_dir - 测试图像序列存放目录 @@ -25,13 +28,14 @@ def get_feature(sequence_dir, frame_idx, bbox, feature_dim): image_dir = os.path.join(sequence_dir, "img1") img_file= os.path.join(image_dir, ('000000'+str(frame_idx))[-6:]+'.jpg') img = cv2.imread(img_file, cv2.IMREAD_GRAYSCALE) - t, l, w, h = bbox - roi = img[t:t+h;l:l+w] # tlwh - hists, bins = np.histogram(roi, bins=feature_dim) + t, l, w, h = [int(x) for x in bbox] + print('[DEBUG] img.shape: %s,bbox: %s' %(str(img.shape), bbox)) + roi = img[t:t+h,l:l+w] # tlwh + hists, bins = np.histogram(roi, bins=get_feature1_dim()) return hists -def gather_sequence_info(sequence_dir, detection_file): +def gather_sequence_info(sequence_dir, detection_file, feature_type=0): """Gather sequence information, such as image filenames, detections, groundtruth (if available). @@ -95,7 +99,10 @@ def gather_sequence_info(sequence_dir, detection_file): else: update_ms = None - feature_dim = detections.shape[1] - 10 if detections is not None else 0 + if feature_type==0: + feature_dim = detections.shape[1] - 10 if detections is not None else 0 + elif feature_type==1: + feature_dim = get_feature1_dim() seq_info = { "sequence_name": os.path.basename(sequence_dir), "image_filenames": image_filenames, @@ -110,7 +117,7 @@ def gather_sequence_info(sequence_dir, detection_file): return seq_info -def create_detections(detection_mat, frame_idx, min_height=0): +def create_detections(detection_mat, frame_idx, min_height=0, feature_type=0, sequence_dir='model_data'): """Create detections for given frame index from the raw detection matrix. Parameters @@ -137,6 +144,8 @@ def create_detections(detection_mat, frame_idx, min_height=0): detection_list = [] for row in detection_mat[mask]: bbox, confidence, feature = row[2:6], row[6], row[10:] + if feature_type==1: + feature = get_feature1(sequence_dir, frame_idx, bbox) if bbox[3] < min_height: continue detection_list.append(Detection(bbox, confidence, feature)) @@ -145,7 +154,7 @@ def create_detections(detection_mat, frame_idx, min_height=0): def run(sequence_dir, detection_file, output_file, min_confidence, nms_max_overlap, min_detection_height, max_cosine_distance, - nn_budget, display): + nn_budget, display, feature_type): """Run multi-target tracker on a particular sequence. Parameters @@ -172,9 +181,11 @@ def run(sequence_dir, detection_file, output_file, min_confidence, is enforced. display : bool If True, show visualization of intermediate tracking results. + feature_type: int + Indicate how to get feature from bbox area. """ - seq_info = gather_sequence_info(sequence_dir, detection_file) + seq_info = gather_sequence_info(sequence_dir, detection_file, feature_type) metric = nn_matching.NearestNeighborDistanceMetric( "cosine", max_cosine_distance, nn_budget) tracker = Tracker(metric) @@ -185,7 +196,7 @@ def frame_callback(vis, frame_idx): # Load image and generate detections. detections = create_detections( - seq_info["detections"], frame_idx, min_detection_height) + seq_info["detections"], frame_idx, min_detection_height, feature_type, sequence_dir) detections = [d for d in detections if d.confidence >= min_confidence] # Run non-maxima suppression. @@ -241,10 +252,9 @@ def parse_args(): parser = argparse.ArgumentParser(description="Deep SORT") parser.add_argument( "--sequence_dir", help="Path to MOTChallenge sequence directory", - default=None, required=True) + default='model_data') parser.add_argument( - "--detection_file", help="Path to custom detections.", default=None, - required=True) + "--detection_file", help="Path to custom detections.", default='model_data/MOT16-06.npy') parser.add_argument( "--output_file", help="Path to the tracking output file. This file will" " contain the tracking results on completion.", @@ -269,6 +279,10 @@ def parse_args(): parser.add_argument( "--display", help="Show intermediate tracking results", default=True, type=bool_string) + parser.add_argument( + "--feature_type", help="feature type", + default=0, type=int) + return parser.parse_args() @@ -285,9 +299,10 @@ def parse_args(): print('max_cosine_distance: ', args.max_cosine_distance) print('nn_budget: ', args.nn_budget) print('display: ', args.display) + print('feature_type: ', args.feature_type) print('') run( args.sequence_dir, args.detection_file, args.output_file, args.min_confidence, args.nms_max_overlap, args.min_detection_height, - args.max_cosine_distance, args.nn_budget, args.display) + args.max_cosine_distance, args.nn_budget, args.display, args.feature_type) From bec59629c838da41a6bb43b91a2c37e9add4df31 Mon Sep 17 00:00:00 2001 From: wilsonwnog2014 Date: Tue, 26 May 2020 17:54:19 +0800 Subject: [PATCH 05/24] add model_data dirpath; add .gitignore --- my_deep_sort_app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/my_deep_sort_app.py b/my_deep_sort_app.py index 3bfbac4f..25fee226 100644 --- a/my_deep_sort_app.py +++ b/my_deep_sort_app.py @@ -29,7 +29,7 @@ def get_feature1(sequence_dir, frame_idx, bbox): img_file= os.path.join(image_dir, ('000000'+str(frame_idx))[-6:]+'.jpg') img = cv2.imread(img_file, cv2.IMREAD_GRAYSCALE) t, l, w, h = [int(x) for x in bbox] - print('[DEBUG] img.shape: %s,bbox: %s' %(str(img.shape), bbox)) + #print('[DEBUG] img.shape: %s,bbox: %s' %(str(img.shape), bbox)) roi = img[t:t+h,l:l+w] # tlwh hists, bins = np.histogram(roi, bins=get_feature1_dim()) return hists From 940048a5797cefd2db6059be025fdd12592ac9b2 Mon Sep 17 00:00:00 2001 From: wilsonwnog2014 Date: Tue, 26 May 2020 17:56:47 +0800 Subject: [PATCH 06/24] add .gitignore --- .gitignore | 46 +++------------------------------------------- 1 file changed, 3 insertions(+), 43 deletions(-) diff --git a/.gitignore b/.gitignore index 72364f99..7e78d1e1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,6 @@ # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] -*$py.class # C extensions *.so @@ -12,9 +11,7 @@ env/ build/ develop-eggs/ dist/ -downloads/ eggs/ -.eggs/ lib/ lib64/ parts/ @@ -24,12 +21,6 @@ var/ .installed.cfg *.egg -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - # Installer logs pip-log.txt pip-delete-this-directory.txt @@ -38,12 +29,9 @@ pip-delete-this-directory.txt htmlcov/ .tox/ .coverage -.coverage.* .cache nosetests.xml coverage.xml -*,cover -.hypothesis/ # Translations *.mo @@ -51,39 +39,11 @@ coverage.xml # Django stuff: *.log -local_settings.py - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy # Sphinx documentation docs/_build/ -# PyBuilder -target/ - -# IPython Notebook -.ipynb_checkpoints - -# pyenv -.python-version - -# celery beat schedule file -celerybeat-schedule - -# dotenv -.env - -# virtualenv -venv/ -ENV/ - -# Spyder project settings -.spyderproject +model_data +model_data/ +.ipynb_checkpoints/ -# Rope project settings -.ropeproject From 957606c12033cb5a4763fba090db225cbabcf6ce Mon Sep 17 00:00:00 2001 From: wilsonwnog2014 Date: Wed, 3 Jun 2020 10:35:54 +0800 Subject: [PATCH 07/24] =?UTF-8?q?=E6=B7=BB=E5=8A=A0tensorflow1.15=E7=89=88?= =?UTF-8?q?=E6=9C=AC=E6=94=AF=E6=8C=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- deep_sort/linear_assignment.py | 12 +++++++++++- env.bashrc | 3 ++- env_tf1.14.bashrc | 4 ++++ 3 files changed, 17 insertions(+), 2 deletions(-) create mode 100644 env_tf1.14.bashrc diff --git a/deep_sort/linear_assignment.py b/deep_sort/linear_assignment.py index 178456cf..d432a08c 100644 --- a/deep_sort/linear_assignment.py +++ b/deep_sort/linear_assignment.py @@ -1,7 +1,13 @@ # vim: expandtab:ts=4:sw=4 from __future__ import absolute_import import numpy as np -from sklearn.utils.linear_assignment_ import linear_assignment + +import sklearn +#FutureWarning: The linear_assignment function is deprecated in 0.21 and will be removed from 0.23. Use scipy.optimize.linear_sum_assignment instead. +if sklearn.__version__>'0.22': + from scipy.optimize import linear_sum_assignment as linear_assignment # tf1.15, sklearn=>0.23 +else: + from sklearn.utils.linear_assignment_ import linear_assignment # tf1.14, sklearn=>0.22 from . import kalman_filter @@ -57,6 +63,10 @@ def min_cost_matching( cost_matrix[cost_matrix > max_distance] = max_distance + 1e-5 indices = linear_assignment(cost_matrix) + # Fix for tf1.15 envirement when use scipy.optimize.linear_sum_assignment instead from sklearn.utils.linear_assignment_ + if type(indices)==tuple: + indices = np.array([[x,y] for x,y in zip(*indices)]) + matches, unmatched_tracks, unmatched_detections = [], [], [] for col, detection_idx in enumerate(detection_indices): if col not in indices[:, 1]: diff --git a/env.bashrc b/env.bashrc index 167da416..0bf7d08c 100644 --- a/env.bashrc +++ b/env.bashrc @@ -1,3 +1,4 @@ source ~/.bashrc -source ~/miniconda3/bin/activate tf1.14 +#source ~/miniconda3/bin/activate tf1.14 +source ~/miniconda3/bin/activate tf1.15 diff --git a/env_tf1.14.bashrc b/env_tf1.14.bashrc new file mode 100644 index 00000000..25dd5463 --- /dev/null +++ b/env_tf1.14.bashrc @@ -0,0 +1,4 @@ +source ~/.bashrc +source ~/miniconda3/bin/activate tf1.14 +#source ~/miniconda3/bin/activate tf1.15 + From d0c4e1ecdd9b69daf1b597935aae07ba76040934 Mon Sep 17 00:00:00 2001 From: wilsonwnog2014 Date: Mon, 8 Jun 2020 18:52:29 +0800 Subject: [PATCH 08/24] add deep_sort_ex --- MyReadMe.txt | 2 +- deep_sort_ex/__init__.py | 1 + deep_sort_ex/detection.py | 54 +++++++ deep_sort_ex/iou_matching.py | 81 ++++++++++ deep_sort_ex/kalman_filter.py | 248 ++++++++++++++++++++++++++++++ deep_sort_ex/linear_assignment.py | 200 ++++++++++++++++++++++++ deep_sort_ex/nn_matching.py | 177 +++++++++++++++++++++ deep_sort_ex/track.py | 170 ++++++++++++++++++++ deep_sort_ex/tracker.py | 138 +++++++++++++++++ my_deep_sort_app.py | 4 +- 10 files changed, 1072 insertions(+), 3 deletions(-) create mode 100644 deep_sort_ex/__init__.py create mode 100644 deep_sort_ex/detection.py create mode 100644 deep_sort_ex/iou_matching.py create mode 100644 deep_sort_ex/kalman_filter.py create mode 100644 deep_sort_ex/linear_assignment.py create mode 100644 deep_sort_ex/nn_matching.py create mode 100644 deep_sort_ex/track.py create mode 100644 deep_sort_ex/tracker.py diff --git a/MyReadMe.txt b/MyReadMe.txt index 95589f27..4fb1f21d 100644 --- a/MyReadMe.txt +++ b/MyReadMe.txt @@ -1,4 +1,4 @@ -python deep_sort_app.py \ +python my_deep_sort_app.py \ --sequence_dir=$HOME/e/dataset_tiptical/MOT16/MOT16-06 \ --detection_file=$HOME/e/dataset_tiptical/MOT16/MOT16-06/MOT16-06.npy \ --min_confidence=0.3 \ diff --git a/deep_sort_ex/__init__.py b/deep_sort_ex/__init__.py new file mode 100644 index 00000000..43e08fb8 --- /dev/null +++ b/deep_sort_ex/__init__.py @@ -0,0 +1 @@ +# vim: expandtab:ts=4:sw=4 diff --git a/deep_sort_ex/detection.py b/deep_sort_ex/detection.py new file mode 100644 index 00000000..e09c637c --- /dev/null +++ b/deep_sort_ex/detection.py @@ -0,0 +1,54 @@ +# vim: expandtab:ts=4:sw=4 +import numpy as np + + +class Detection(object): + """ + This class represents a bounding box detection in a single image. + + Parameters + ---------- + tlwh : array_like + Bounding box in format `(x, y, w, h)`. + confidence : float + Detector confidence score. + feature : array_like + A feature vector that describes the object contained in this image. + + Attributes + ---------- + tlwh : ndarray + Bounding box in format `(top left x, top left y, width, height)`. + confidence : ndarray + Detector confidence score. + feature : ndarray | NoneType + A feature vector that describes the object contained in this image. + + """ + + def __init__(self, tlwh, confidence, feature, exts=None, binding_obj=None): + self.tlwh = np.asarray(tlwh, dtype=np.float) + self.exts = exts + self.binding_obj = binding_obj + self.confidence = float(confidence) + self.feature = np.asarray(feature, dtype=np.float32) + + def to_tlbr(self): + """Convert bounding box to format `(min x, min y, max x, max y)`, i.e., + `(top left, bottom right)`. + """ + ret = self.tlwh.copy() + ret[2:] += ret[:2] + return ret + + def to_xyah(self): + """Convert bounding box to format `(center x, center y, aspect ratio, + height)`, where the aspect ratio is `width / height`. + """ + ret = self.tlwh.copy() + ret[:2] += ret[2:] / 2 + ret[2] /= ret[3] + if self.exts is None: + return ret + else: + return np.hstack([ret, self.exts]) diff --git a/deep_sort_ex/iou_matching.py b/deep_sort_ex/iou_matching.py new file mode 100644 index 00000000..c4dd0b88 --- /dev/null +++ b/deep_sort_ex/iou_matching.py @@ -0,0 +1,81 @@ +# vim: expandtab:ts=4:sw=4 +from __future__ import absolute_import +import numpy as np +from . import linear_assignment + + +def iou(bbox, candidates): + """Computer intersection over union. + + Parameters + ---------- + bbox : ndarray + A bounding box in format `(top left x, top left y, width, height)`. + candidates : ndarray + A matrix of candidate bounding boxes (one per row) in the same format + as `bbox`. + + Returns + ------- + ndarray + The intersection over union in [0, 1] between the `bbox` and each + candidate. A higher score means a larger fraction of the `bbox` is + occluded by the candidate. + + """ + bbox_tl, bbox_br = bbox[:2], bbox[:2] + bbox[2:] + candidates_tl = candidates[:, :2] + candidates_br = candidates[:, :2] + candidates[:, 2:] + + tl = np.c_[np.maximum(bbox_tl[0], candidates_tl[:, 0])[:, np.newaxis], + np.maximum(bbox_tl[1], candidates_tl[:, 1])[:, np.newaxis]] + br = np.c_[np.minimum(bbox_br[0], candidates_br[:, 0])[:, np.newaxis], + np.minimum(bbox_br[1], candidates_br[:, 1])[:, np.newaxis]] + wh = np.maximum(0., br - tl) + + area_intersection = wh.prod(axis=1) + area_bbox = bbox[2:].prod() + area_candidates = candidates[:, 2:].prod(axis=1) + return area_intersection / (area_bbox + area_candidates - area_intersection) + + +def iou_cost(tracks, detections, track_indices=None, + detection_indices=None): + """An intersection over union distance metric. + + Parameters + ---------- + tracks : List[deep_sort.track.Track] + A list of tracks. + detections : List[deep_sort.detection.Detection] + A list of detections. + track_indices : Optional[List[int]] + A list of indices to tracks that should be matched. Defaults to + all `tracks`. + detection_indices : Optional[List[int]] + A list of indices to detections that should be matched. Defaults + to all `detections`. + + Returns + ------- + ndarray + Returns a cost matrix of shape + len(track_indices), len(detection_indices) where entry (i, j) is + `1 - iou(tracks[track_indices[i]], detections[detection_indices[j]])`. + + """ + if track_indices is None: + track_indices = np.arange(len(tracks)) + if detection_indices is None: + detection_indices = np.arange(len(detections)) + + cost_matrix = np.zeros((len(track_indices), len(detection_indices))) + for row, track_idx in enumerate(track_indices): + if tracks[track_idx].time_since_update > 1: + cost_matrix[row, :] = linear_assignment.INFTY_COST + continue + + bbox = tracks[track_idx].to_tlwh() + candidates = np.asarray([detections[i].tlwh for i in detection_indices]) + cost_matrix[row, :] = 1. - iou(bbox, candidates) + return cost_matrix diff --git a/deep_sort_ex/kalman_filter.py b/deep_sort_ex/kalman_filter.py new file mode 100644 index 00000000..ce23ed79 --- /dev/null +++ b/deep_sort_ex/kalman_filter.py @@ -0,0 +1,248 @@ +# vim: expandtab:ts=4:sw=4 +import numpy as np +import scipy.linalg + + +""" +Table for the 0.95 quantile of the chi-square distribution with N degrees of +freedom (contains values for N=1, ..., 9). Taken from MATLAB/Octave's chi2inv +function and used as Mahalanobis gating threshold. +""" +chi2inv95 = { + 1: 3.8415, + 2: 5.9915, + 3: 7.8147, + 4: 9.4877, + 5: 11.070, + 6: 12.592, + 7: 14.067, + 8: 15.507, + 9: 16.919} + + +class KalmanFilter(object): + """ + A simple Kalman filter for tracking bounding boxes in image space. + + The 8-dimensional state space + + x, y, a, h, vx, vy, va, vh + + contains the bounding box center position (x, y), aspect ratio a, height h, + and their respective velocities. + + Object motion follows a constant velocity model. The bounding box location + (x, y, a, h) is taken as direct observation of the state space (linear + observation model). + + """ + + def __init__(self, n_extend=0): + ''' + @param n_extend - 扩展通道数 + ''' + ndim, dt = 4, 1. + ndim += n_extend # 扩展通道数 + self.n_extend = n_extend + + # Create Kalman filter model matrices. + self._motion_mat = np.eye(2 * ndim, 2 * ndim) + for i in range(ndim): + self._motion_mat[i, ndim + i] = dt + self._update_mat = np.eye(ndim, 2 * ndim) + + # Motion and observation uncertainty are chosen relative to the current + # state estimate. These weights control the amount of uncertainty in + # the model. This is a bit hacky. + self._std_weight_position = 1. / 20 + self._std_weight_velocity = 1. / 160 + + def initiate(self, measurement): + """Create track from unassociated measurement. + + Parameters + ---------- + measurement : ndarray + Bounding box coordinates (x, y, a, h) with center position (x, y), + aspect ratio a, and height h. + + Returns + ------- + (ndarray, ndarray) + Returns the mean vector (8 dimensional) and covariance matrix (8x8 + dimensional) of the new track. Unobserved velocities are initialized + to 0 mean. + + """ + mean_pos = measurement + mean_vel = np.zeros_like(mean_pos) + mean = np.r_[mean_pos, mean_vel] + + std = [ + 2 * self._std_weight_position * measurement[3], + 2 * self._std_weight_position * measurement[3], + 1e-2, + 2 * self._std_weight_position * measurement[3], + + 10 * self._std_weight_velocity * measurement[3], + 10 * self._std_weight_velocity * measurement[3], + 1e-5, + 10 * self._std_weight_velocity * measurement[3]] + + for i in range(self.n_extend): + std.insert(4, 2 * self._std_weight_position * measurement[3]) + std.append(10 * self._std_weight_velocity * measurement[3]) + + covariance = np.diag(np.square(std)) + return mean, covariance + + def predict(self, mean, covariance): + """Run Kalman filter prediction step. + + Parameters + ---------- + mean : ndarray + The 8 dimensional mean vector of the object state at the previous + time step. + covariance : ndarray + The 8x8 dimensional covariance matrix of the object state at the + previous time step. + + Returns + ------- + (ndarray, ndarray) + Returns the mean vector and covariance matrix of the predicted + state. Unobserved velocities are initialized to 0 mean. + + """ + std_pos = [ + self._std_weight_position * mean[3], + self._std_weight_position * mean[3], + 1e-2, + self._std_weight_position * mean[3]] + for i in range(self.n_extend): + std_pos.append(self._std_weight_position * mean[3]) + + std_vel = [ + self._std_weight_velocity * mean[3], + self._std_weight_velocity * mean[3], + 1e-5, + self._std_weight_velocity * mean[3]] + for i in range(self.n_extend): + std_vel.append(self._std_weight_velocity * mean[3]) + + motion_cov = np.diag(np.square(np.r_[std_pos, std_vel])) + + mean = np.dot(self._motion_mat, mean) + covariance = np.linalg.multi_dot(( + self._motion_mat, covariance, self._motion_mat.T)) + motion_cov + + return mean, covariance + + def project(self, mean, covariance): + """Project state distribution to measurement space. + + Parameters + ---------- + mean : ndarray + The state's mean vector (8 dimensional array). + covariance : ndarray + The state's covariance matrix (8x8 dimensional). + + Returns + ------- + (ndarray, ndarray) + Returns the projected mean and covariance matrix of the given state + estimate. + + """ + std = [ + self._std_weight_position * mean[3], + self._std_weight_position * mean[3], + 1e-1, + self._std_weight_position * mean[3]] + for i in range(self.n_extend): + std.append(self._std_weight_position * mean[3]) + innovation_cov = np.diag(np.square(std)) + + mean = np.dot(self._update_mat, mean) + covariance = np.linalg.multi_dot(( + self._update_mat, covariance, self._update_mat.T)) + return mean, covariance + innovation_cov + + def update(self, mean, covariance, measurement): + """Run Kalman filter correction step. + + Parameters + ---------- + mean : ndarray + The predicted state's mean vector (8 dimensional). + covariance : ndarray + The state's covariance matrix (8x8 dimensional). + measurement : ndarray + The 4 dimensional measurement vector (x, y, a, h), where (x, y) + is the center position, a the aspect ratio, and h the height of the + bounding box. + + Returns + ------- + (ndarray, ndarray) + Returns the measurement-corrected state distribution. + + """ + projected_mean, projected_cov = self.project(mean, covariance) + + chol_factor, lower = scipy.linalg.cho_factor( + projected_cov, lower=True, check_finite=False) + kalman_gain = scipy.linalg.cho_solve( + (chol_factor, lower), np.dot(covariance, self._update_mat.T).T, + check_finite=False).T + innovation = measurement - projected_mean + + new_mean = mean + np.dot(innovation, kalman_gain.T) + new_covariance = covariance - np.linalg.multi_dot(( + kalman_gain, projected_cov, kalman_gain.T)) + return new_mean, new_covariance + + def gating_distance(self, mean, covariance, measurements, + only_position=False): + """Compute gating distance between state distribution and measurements. + + A suitable distance threshold can be obtained from `chi2inv95`. If + `only_position` is False, the chi-square distribution has 4 degrees of + freedom, otherwise 2. + + Parameters + ---------- + mean : ndarray + Mean vector over the state distribution (8 dimensional). + covariance : ndarray + Covariance of the state distribution (8x8 dimensional). + measurements : ndarray + An Nx4 dimensional matrix of N measurements, each in + format (x, y, a, h) where (x, y) is the bounding box center + position, a the aspect ratio, and h the height. + only_position : Optional[bool] + If True, distance computation is done with respect to the bounding + box center position only. + + Returns + ------- + ndarray + Returns an array of length N, where the i-th element contains the + squared Mahalanobis distance between (mean, covariance) and + `measurements[i]`. + + """ + mean, covariance = self.project(mean, covariance) + if only_position: + mean, covariance = mean[:2], covariance[:2, :2] + measurements = measurements[:, :2] + + cholesky_factor = np.linalg.cholesky(covariance) + d = measurements - mean + z = scipy.linalg.solve_triangular( + cholesky_factor, d.T, lower=True, check_finite=False, + overwrite_b=True) + squared_maha = np.sum(z * z, axis=0) + return squared_maha diff --git a/deep_sort_ex/linear_assignment.py b/deep_sort_ex/linear_assignment.py new file mode 100644 index 00000000..d432a08c --- /dev/null +++ b/deep_sort_ex/linear_assignment.py @@ -0,0 +1,200 @@ +# vim: expandtab:ts=4:sw=4 +from __future__ import absolute_import +import numpy as np + +import sklearn +#FutureWarning: The linear_assignment function is deprecated in 0.21 and will be removed from 0.23. Use scipy.optimize.linear_sum_assignment instead. +if sklearn.__version__>'0.22': + from scipy.optimize import linear_sum_assignment as linear_assignment # tf1.15, sklearn=>0.23 +else: + from sklearn.utils.linear_assignment_ import linear_assignment # tf1.14, sklearn=>0.22 +from . import kalman_filter + + +INFTY_COST = 1e+5 + + +def min_cost_matching( + distance_metric, max_distance, tracks, detections, track_indices=None, + detection_indices=None): + """Solve linear assignment problem. + + Parameters + ---------- + distance_metric : Callable[List[Track], List[Detection], List[int], List[int]) -> ndarray + The distance metric is given a list of tracks and detections as well as + a list of N track indices and M detection indices. The metric should + return the NxM dimensional cost matrix, where element (i, j) is the + association cost between the i-th track in the given track indices and + the j-th detection in the given detection_indices. + max_distance : float + Gating threshold. Associations with cost larger than this value are + disregarded. + tracks : List[track.Track] + A list of predicted tracks at the current time step. + detections : List[detection.Detection] + A list of detections at the current time step. + track_indices : List[int] + List of track indices that maps rows in `cost_matrix` to tracks in + `tracks` (see description above). + detection_indices : List[int] + List of detection indices that maps columns in `cost_matrix` to + detections in `detections` (see description above). + + Returns + ------- + (List[(int, int)], List[int], List[int]) + Returns a tuple with the following three entries: + * A list of matched track and detection indices. + * A list of unmatched track indices. + * A list of unmatched detection indices. + + """ + if track_indices is None: + track_indices = np.arange(len(tracks)) + if detection_indices is None: + detection_indices = np.arange(len(detections)) + + if len(detection_indices) == 0 or len(track_indices) == 0: + return [], track_indices, detection_indices # Nothing to match. + + cost_matrix = distance_metric( + tracks, detections, track_indices, detection_indices) + cost_matrix[cost_matrix > max_distance] = max_distance + 1e-5 + indices = linear_assignment(cost_matrix) + + # Fix for tf1.15 envirement when use scipy.optimize.linear_sum_assignment instead from sklearn.utils.linear_assignment_ + if type(indices)==tuple: + indices = np.array([[x,y] for x,y in zip(*indices)]) + + matches, unmatched_tracks, unmatched_detections = [], [], [] + for col, detection_idx in enumerate(detection_indices): + if col not in indices[:, 1]: + unmatched_detections.append(detection_idx) + for row, track_idx in enumerate(track_indices): + if row not in indices[:, 0]: + unmatched_tracks.append(track_idx) + for row, col in indices: + track_idx = track_indices[row] + detection_idx = detection_indices[col] + if cost_matrix[row, col] > max_distance: + unmatched_tracks.append(track_idx) + unmatched_detections.append(detection_idx) + else: + matches.append((track_idx, detection_idx)) + return matches, unmatched_tracks, unmatched_detections + + +def matching_cascade( + distance_metric, max_distance, cascade_depth, tracks, detections, + track_indices=None, detection_indices=None): + """Run matching cascade. + + Parameters + ---------- + distance_metric : Callable[List[Track], List[Detection], List[int], List[int]) -> ndarray + The distance metric is given a list of tracks and detections as well as + a list of N track indices and M detection indices. The metric should + return the NxM dimensional cost matrix, where element (i, j) is the + association cost between the i-th track in the given track indices and + the j-th detection in the given detection indices. + max_distance : float + Gating threshold. Associations with cost larger than this value are + disregarded. + cascade_depth: int + The cascade depth, should be se to the maximum track age. + tracks : List[track.Track] + A list of predicted tracks at the current time step. + detections : List[detection.Detection] + A list of detections at the current time step. + track_indices : Optional[List[int]] + List of track indices that maps rows in `cost_matrix` to tracks in + `tracks` (see description above). Defaults to all tracks. + detection_indices : Optional[List[int]] + List of detection indices that maps columns in `cost_matrix` to + detections in `detections` (see description above). Defaults to all + detections. + + Returns + ------- + (List[(int, int)], List[int], List[int]) + Returns a tuple with the following three entries: + * A list of matched track and detection indices. + * A list of unmatched track indices. + * A list of unmatched detection indices. + + """ + if track_indices is None: + track_indices = list(range(len(tracks))) + if detection_indices is None: + detection_indices = list(range(len(detections))) + + unmatched_detections = detection_indices + matches = [] + for level in range(cascade_depth): + if len(unmatched_detections) == 0: # No detections left + break + + track_indices_l = [ + k for k in track_indices + if tracks[k].time_since_update == 1 + level + ] + if len(track_indices_l) == 0: # Nothing to match at this level + continue + + matches_l, _, unmatched_detections = \ + min_cost_matching( + distance_metric, max_distance, tracks, detections, + track_indices_l, unmatched_detections) + matches += matches_l + unmatched_tracks = list(set(track_indices) - set(k for k, _ in matches)) + return matches, unmatched_tracks, unmatched_detections + + +def gate_cost_matrix( + kf, cost_matrix, tracks, detections, track_indices, detection_indices, + gated_cost=INFTY_COST, only_position=False): + """Invalidate infeasible entries in cost matrix based on the state + distributions obtained by Kalman filtering. + + Parameters + ---------- + kf : The Kalman filter. + cost_matrix : ndarray + The NxM dimensional cost matrix, where N is the number of track indices + and M is the number of detection indices, such that entry (i, j) is the + association cost between `tracks[track_indices[i]]` and + `detections[detection_indices[j]]`. + tracks : List[track.Track] + A list of predicted tracks at the current time step. + detections : List[detection.Detection] + A list of detections at the current time step. + track_indices : List[int] + List of track indices that maps rows in `cost_matrix` to tracks in + `tracks` (see description above). + detection_indices : List[int] + List of detection indices that maps columns in `cost_matrix` to + detections in `detections` (see description above). + gated_cost : Optional[float] + Entries in the cost matrix corresponding to infeasible associations are + set this value. Defaults to a very large value. + only_position : Optional[bool] + If True, only the x, y position of the state distribution is considered + during gating. Defaults to False. + + Returns + ------- + ndarray + Returns the modified cost matrix. + + """ + gating_dim = 2 if only_position else 4 + gating_threshold = kalman_filter.chi2inv95[gating_dim] + measurements = np.asarray( + [detections[i].to_xyah() for i in detection_indices]) + for row, track_idx in enumerate(track_indices): + track = tracks[track_idx] + gating_distance = kf.gating_distance( + track.mean, track.covariance, measurements, only_position) + cost_matrix[row, gating_distance > gating_threshold] = gated_cost + return cost_matrix diff --git a/deep_sort_ex/nn_matching.py b/deep_sort_ex/nn_matching.py new file mode 100644 index 00000000..c1f20b5e --- /dev/null +++ b/deep_sort_ex/nn_matching.py @@ -0,0 +1,177 @@ +# vim: expandtab:ts=4:sw=4 +import numpy as np + + +def _pdist(a, b): + """Compute pair-wise squared distance between points in `a` and `b`. + + Parameters + ---------- + a : array_like + An NxM matrix of N samples of dimensionality M. + b : array_like + An LxM matrix of L samples of dimensionality M. + + Returns + ------- + ndarray + Returns a matrix of size len(a), len(b) such that eleement (i, j) + contains the squared distance between `a[i]` and `b[j]`. + + """ + a, b = np.asarray(a), np.asarray(b) + if len(a) == 0 or len(b) == 0: + return np.zeros((len(a), len(b))) + a2, b2 = np.square(a).sum(axis=1), np.square(b).sum(axis=1) + r2 = -2. * np.dot(a, b.T) + a2[:, None] + b2[None, :] + r2 = np.clip(r2, 0., float(np.inf)) + return r2 + + +def _cosine_distance(a, b, data_is_normalized=False): + """Compute pair-wise cosine distance between points in `a` and `b`. + + Parameters + ---------- + a : array_like + An NxM matrix of N samples of dimensionality M. + b : array_like + An LxM matrix of L samples of dimensionality M. + data_is_normalized : Optional[bool] + If True, assumes rows in a and b are unit length vectors. + Otherwise, a and b are explicitly normalized to lenght 1. + + Returns + ------- + ndarray + Returns a matrix of size len(a), len(b) such that eleement (i, j) + contains the squared distance between `a[i]` and `b[j]`. + + """ + if not data_is_normalized: + a = np.asarray(a) / (np.linalg.norm(a, axis=1, keepdims=True)+1e-5) + b = np.asarray(b) / (np.linalg.norm(b, axis=1, keepdims=True)+1e-5) + return 1. - np.dot(a, b.T) + + +def _nn_euclidean_distance(x, y): + """ Helper function for nearest neighbor distance metric (Euclidean). + + Parameters + ---------- + x : ndarray + A matrix of N row-vectors (sample points). + y : ndarray + A matrix of M row-vectors (query points). + + Returns + ------- + ndarray + A vector of length M that contains for each entry in `y` the + smallest Euclidean distance to a sample in `x`. + + """ + distances = _pdist(x, y) + return np.maximum(0.0, distances.min(axis=0)) + + +def _nn_cosine_distance(x, y): + """ Helper function for nearest neighbor distance metric (cosine). + + Parameters + ---------- + x : ndarray + A matrix of N row-vectors (sample points). + y : ndarray + A matrix of M row-vectors (query points). + + Returns + ------- + ndarray + A vector of length M that contains for each entry in `y` the + smallest cosine distance to a sample in `x`. + + """ + distances = _cosine_distance(x, y) + return distances.min(axis=0) + + +class NearestNeighborDistanceMetric(object): + """ + A nearest neighbor distance metric that, for each target, returns + the closest distance to any sample that has been observed so far. + + Parameters + ---------- + metric : str + Either "euclidean" or "cosine". + matching_threshold: float + The matching threshold. Samples with larger distance are considered an + invalid match. + budget : Optional[int] + If not None, fix samples per class to at most this number. Removes + the oldest samples when the budget is reached. + + Attributes + ---------- + samples : Dict[int -> List[ndarray]] + A dictionary that maps from target identities to the list of samples + that have been observed so far. + + """ + + def __init__(self, metric, matching_threshold, budget=None): + + + if metric == "euclidean": + self._metric = _nn_euclidean_distance + elif metric == "cosine": + self._metric = _nn_cosine_distance + else: + raise ValueError( + "Invalid metric; must be either 'euclidean' or 'cosine'") + self.matching_threshold = matching_threshold + self.budget = budget + self.samples = {} + + def partial_fit(self, features, targets, active_targets): + """Update the distance metric with new data. + + Parameters + ---------- + features : ndarray + An NxM matrix of N features of dimensionality M. + targets : ndarray + An integer array of associated target identities. + active_targets : List[int] + A list of targets that are currently present in the scene. + + """ + for feature, target in zip(features, targets): + self.samples.setdefault(target, []).append(feature) + if self.budget is not None: + self.samples[target] = self.samples[target][-self.budget:] + self.samples = {k: self.samples[k] for k in active_targets} + + def distance(self, features, targets): + """Compute distance between features and targets. + + Parameters + ---------- + features : ndarray + An NxM matrix of N features of dimensionality M. + targets : List[int] + A list of targets to match the given `features` against. + + Returns + ------- + ndarray + Returns a cost matrix of shape len(targets), len(features), where + element (i, j) contains the closest squared distance between + `targets[i]` and `features[j]`. + + """ + cost_matrix = np.zeros((len(targets), len(features))) + for i, target in enumerate(targets): + cost_matrix[i, :] = self._metric(self.samples[target], features) + return cost_matrix diff --git a/deep_sort_ex/track.py b/deep_sort_ex/track.py new file mode 100644 index 00000000..58b714ee --- /dev/null +++ b/deep_sort_ex/track.py @@ -0,0 +1,170 @@ +# vim: expandtab:ts=4:sw=4 + + +class TrackState: + """ + Enumeration type for the single target track state. Newly created tracks are + classified as `tentative` until enough evidence has been collected. Then, + the track state is changed to `confirmed`. Tracks that are no longer alive + are classified as `deleted` to mark them for removal from the set of active + tracks. + + """ + + Tentative = 1 + Confirmed = 2 + Deleted = 3 + + +class Track: + """ + A single target track with state space `(x, y, a, h)` and associated + velocities, where `(x, y)` is the center of the bounding box, `a` is the + aspect ratio and `h` is the height. + + Parameters + ---------- + mean : ndarray + Mean vector of the initial state distribution. + covariance : ndarray + Covariance matrix of the initial state distribution. + track_id : int + A unique track identifier. + n_init : int + Number of consecutive detections before the track is confirmed. The + track state is set to `Deleted` if a miss occurs within the first + `n_init` frames. + max_age : int + The maximum number of consecutive misses before the track state is + set to `Deleted`. + feature : Optional[ndarray] + Feature vector of the detection this track originates from. If not None, + this feature is added to the `features` cache. + + Attributes + ---------- + mean : ndarray + Mean vector of the initial state distribution. + covariance : ndarray + Covariance matrix of the initial state distribution. + track_id : int + A unique track identifier. + hits : int + Total number of measurement updates. + age : int + Total number of frames since first occurance. + time_since_update : int + Total number of frames since last measurement update. + state : TrackState + The current track state. + features : List[ndarray] + A cache of features. On each measurement update, the associated feature + vector is added to this list. + + """ + + def __init__(self, mean, covariance, track_id, n_init, max_age, + feature=None, binding_obj=None): + self.mean = mean + self.covariance = covariance + self.track_id = track_id + self.hits = 1 + self.age = 1 + self.time_since_update = 0 + + self.state = TrackState.Tentative + self.features = [] + if feature is not None: + self.features.append(feature) + + self._n_init = n_init + self._max_age = max_age + self.binding_obj = binding_obj + + def to_tlwh(self): + """Get current position in bounding box format `(top left x, top left y, + width, height)`. + + Returns + ------- + ndarray + The bounding box. + + """ + ret = self.mean[:4].copy() + ret[2] *= ret[3] + ret[:2] -= ret[2:4] / 2 + return ret + + def to_tlbr(self): + """Get current position in bounding box format `(min x, miny, max x, + max y)`. + + Returns + ------- + ndarray + The bounding box. + + """ + ret = self.to_tlwh() + ret[2:4] = ret[:2] + ret[2:4] + return ret + + def get_exts(self): + return self.mean[4:].copy() + + def predict(self, kf): + """Propagate the state distribution to the current time step using a + Kalman filter prediction step. + + Parameters + ---------- + kf : kalman_filter.KalmanFilter + The Kalman filter. + + """ + self.mean, self.covariance = kf.predict(self.mean, self.covariance) + self.age += 1 + self.time_since_update += 1 + + def update(self, kf, detection): + """Perform Kalman filter measurement update step and update the feature + cache. + + Parameters + ---------- + kf : kalman_filter.KalmanFilter + The Kalman filter. + detection : Detection + The associated detection. + + """ + self.mean, self.covariance = kf.update( + self.mean, self.covariance, detection.to_xyah()) + self.features.append(detection.feature) + + self.hits += 1 + self.time_since_update = 0 + if self.state == TrackState.Tentative and self.hits >= self._n_init: + self.state = TrackState.Confirmed + + def mark_missed(self): + """Mark this track as missed (no association at the current time step). + """ + if self.state == TrackState.Tentative: + self.state = TrackState.Deleted + elif self.time_since_update > self._max_age: + self.state = TrackState.Deleted + + def is_tentative(self): + """Returns True if this track is tentative (unconfirmed). + """ + return self.state == TrackState.Tentative + + def is_confirmed(self): + """Returns True if this track is confirmed.""" + return self.state == TrackState.Confirmed + + def is_deleted(self): + """Returns True if this track is dead and should be deleted.""" + return self.state == TrackState.Deleted diff --git a/deep_sort_ex/tracker.py b/deep_sort_ex/tracker.py new file mode 100644 index 00000000..112f2d4c --- /dev/null +++ b/deep_sort_ex/tracker.py @@ -0,0 +1,138 @@ +# vim: expandtab:ts=4:sw=4 +from __future__ import absolute_import +import numpy as np +from . import kalman_filter +from . import linear_assignment +from . import iou_matching +from .track import Track + + +class Tracker: + """ + This is the multi-target tracker. + + Parameters + ---------- + metric : nn_matching.NearestNeighborDistanceMetric + A distance metric for measurement-to-track association. + max_age : int + Maximum number of missed misses before a track is deleted. + n_init : int + Number of consecutive detections before the track is confirmed. The + track state is set to `Deleted` if a miss occurs within the first + `n_init` frames. + + Attributes + ---------- + metric : nn_matching.NearestNeighborDistanceMetric + The distance metric used for measurement to track association. + max_age : int + Maximum number of missed misses before a track is deleted. + n_init : int + Number of frames that a track remains in initialization phase. + kf : kalman_filter.KalmanFilter + A Kalman filter to filter target trajectories in image space. + tracks : List[Track] + The list of active tracks at the current time step. + + """ + + def __init__(self, metric, max_iou_distance=0.7, max_age=30, n_init=3, n_extend=0): + self.metric = metric + self.max_iou_distance = max_iou_distance + self.max_age = max_age + self.n_init = n_init + + self.kf = kalman_filter.KalmanFilter(n_extend=n_extend) + self.tracks = [] + self._next_id = 1 + + def predict(self): + """Propagate track state distributions one time step forward. + + This function should be called once every time step, before `update`. + """ + for track in self.tracks: + track.predict(self.kf) + + def update(self, detections): + """Perform measurement update and track management. + + Parameters + ---------- + detections : List[deep_sort.detection.Detection] + A list of detections at the current time step. + + """ + # Run matching cascade. + matches, unmatched_tracks, unmatched_detections = \ + self._match(detections) + + # Update track set. + for track_idx, detection_idx in matches: + self.tracks[track_idx].update( + self.kf, detections[detection_idx]) + for track_idx in unmatched_tracks: + self.tracks[track_idx].mark_missed() + for detection_idx in unmatched_detections: + self._initiate_track(detections[detection_idx]) + self.tracks = [t for t in self.tracks if not t.is_deleted()] + + # Update distance metric. + active_targets = [t.track_id for t in self.tracks if t.is_confirmed()] + features, targets = [], [] + for track in self.tracks: + if not track.is_confirmed(): + continue + features += track.features + targets += [track.track_id for _ in track.features] + track.features = [] + self.metric.partial_fit( + np.asarray(features), np.asarray(targets), active_targets) + + def _match(self, detections): + + def gated_metric(tracks, dets, track_indices, detection_indices): + features = np.array([dets[i].feature for i in detection_indices]) + targets = np.array([tracks[i].track_id for i in track_indices]) + cost_matrix = self.metric.distance(features, targets) + cost_matrix = linear_assignment.gate_cost_matrix( + self.kf, cost_matrix, tracks, dets, track_indices, + detection_indices) + + return cost_matrix + + # Split track set into confirmed and unconfirmed tracks. + confirmed_tracks = [ + i for i, t in enumerate(self.tracks) if t.is_confirmed()] + unconfirmed_tracks = [ + i for i, t in enumerate(self.tracks) if not t.is_confirmed()] + + # Associate confirmed tracks using appearance features. + matches_a, unmatched_tracks_a, unmatched_detections = \ + linear_assignment.matching_cascade( + gated_metric, self.metric.matching_threshold, self.max_age, + self.tracks, detections, confirmed_tracks) + + # Associate remaining tracks together with unconfirmed tracks using IOU. + iou_track_candidates = unconfirmed_tracks + [ + k for k in unmatched_tracks_a if + self.tracks[k].time_since_update == 1] + unmatched_tracks_a = [ + k for k in unmatched_tracks_a if + self.tracks[k].time_since_update != 1] + matches_b, unmatched_tracks_b, unmatched_detections = \ + linear_assignment.min_cost_matching( + iou_matching.iou_cost, self.max_iou_distance, self.tracks, + detections, iou_track_candidates, unmatched_detections) + + matches = matches_a + matches_b + unmatched_tracks = list(set(unmatched_tracks_a + unmatched_tracks_b)) + return matches, unmatched_tracks, unmatched_detections + + def _initiate_track(self, detection): + mean, covariance = self.kf.initiate(detection.to_xyah()) + self.tracks.append(Track( + mean, covariance, self._next_id, self.n_init, self.max_age, + detection.feature, binding_obj=detection.binding_obj)) + self._next_id += 1 diff --git a/my_deep_sort_app.py b/my_deep_sort_app.py index 25fee226..eb66e27e 100644 --- a/my_deep_sort_app.py +++ b/my_deep_sort_app.py @@ -148,7 +148,7 @@ def create_detections(detection_mat, frame_idx, min_height=0, feature_type=0, se feature = get_feature1(sequence_dir, frame_idx, bbox) if bbox[3] < min_height: continue - detection_list.append(Detection(bbox, confidence, feature)) + detection_list.append(Detection(bbox, confidence, feature, exts=[0.5])) return detection_list @@ -188,7 +188,7 @@ def run(sequence_dir, detection_file, output_file, min_confidence, seq_info = gather_sequence_info(sequence_dir, detection_file, feature_type) metric = nn_matching.NearestNeighborDistanceMetric( "cosine", max_cosine_distance, nn_budget) - tracker = Tracker(metric) + tracker = Tracker(metric, n_extend=1) results = [] def frame_callback(vis, frame_idx): From ca40bd32df41c6abffeaab1e24384c8b02fe4f34 Mon Sep 17 00:00:00 2001 From: wilsonwnog2014 Date: Tue, 9 Jun 2020 18:02:03 +0800 Subject: [PATCH 09/24] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=20deep=5Fsort=5Fex/?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- my_deep_sort_app.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/my_deep_sort_app.py b/my_deep_sort_app.py index eb66e27e..dd41d28d 100644 --- a/my_deep_sort_app.py +++ b/my_deep_sort_app.py @@ -9,9 +9,9 @@ from application_util import preprocessing from application_util import visualization -from deep_sort import nn_matching -from deep_sort.detection import Detection -from deep_sort.tracker import Tracker +from deep_sort_ex import nn_matching +from deep_sort_ex.detection import Detection +from deep_sort_ex.tracker import Tracker def get_feature1_dim(): return 20 From e726774b650f81d6d19df60c4b0c36a4ad6eb933 Mon Sep 17 00:00:00 2001 From: wilsonwnog2014 Date: Wed, 10 Jun 2020 18:21:24 +0800 Subject: [PATCH 10/24] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=20exts2=20=E6=BB=A4?= =?UTF-8?q?=E6=B3=A2=E5=99=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- deep_sort_ex/detection.py | 17 ++++-- deep_sort_ex/track.py | 117 +++++++++++++++++++++++++++++++++++++- deep_sort_ex/tracker.py | 7 ++- my_deep_sort_app.py | 4 +- 4 files changed, 134 insertions(+), 11 deletions(-) diff --git a/deep_sort_ex/detection.py b/deep_sort_ex/detection.py index e09c637c..c8c65709 100644 --- a/deep_sort_ex/detection.py +++ b/deep_sort_ex/detection.py @@ -26,9 +26,18 @@ class Detection(object): """ - def __init__(self, tlwh, confidence, feature, exts=None, binding_obj=None): + def __init__(self, tlwh, confidence, feature, exts1=None, exts2=None, binding_obj=None): + ''' + @param tlwh - bbox: top, left, width, height + @param confidence - 目标检测置信度 + @param ffeature - 目标图像特征码 + @param exts1 - 扩展属性: 扩展卡尔曼滤波器的向量(mean)， 需要与 tracker,track的n_extend参数配合使用 + @param exts2 - 扩展属性: 独立于卡尔曼滤波器，对扩展通道单独处理 + @param binding_obj - 绑定原始目标检测的序号，方便数据源跟踪 + ''' self.tlwh = np.asarray(tlwh, dtype=np.float) - self.exts = exts + self.exts1 = exts1 + self.exts2 = exts2 self.binding_obj = binding_obj self.confidence = float(confidence) self.feature = np.asarray(feature, dtype=np.float32) @@ -48,7 +57,7 @@ def to_xyah(self): ret = self.tlwh.copy() ret[:2] += ret[2:] / 2 ret[2] /= ret[3] - if self.exts is None: + if self.exts1 is None: return ret else: - return np.hstack([ret, self.exts]) + return np.hstack([ret, self.exts1]) diff --git a/deep_sort_ex/track.py b/deep_sort_ex/track.py index 58b714ee..cb05fc1f 100644 --- a/deep_sort_ex/track.py +++ b/deep_sort_ex/track.py @@ -1,5 +1,111 @@ # vim: expandtab:ts=4:sw=4 +import queue +import numpy as np + +# 滤波方案0 +class Filter0(object): + '''均值滤波，设定队列长度，取队列均值作为输出 + ''' + def __init__(self, N=3): + ''' + @param N - 队列长度 + ''' + self.q = queue.Queue() + self.N = N + + def filter(self, data): + ''' + @param data - np.array: channels x data_len + ''' + for i in range(data.shape[1]): + self.q.put(data[:, i]) + data[:, i] = np.array(self.q.queue).mean(axis=0) + if self.q.qsize()>=self.N: + self.q.get() + +# 滤波方案1 +class Filter1(object): + '''依据方差剔除奇点数据 + ''' + def __init__(self, N=4, std_th=0.05): + ''' + @param N - 队列长度 + @param std_th - 方差域值 + ''' + self.q_size = N + self.std_th = std_th + self.q = queue.Queue() + + def filter(self, data): + ''' + @param data - np.array: channels x data_len + ''' + for i in range(data.shape[1]): + if self.q.qsize()==self.q_size: + q_data = np.array(self.q.queue) + mean_vals = np.array(q_data).mean(axis=0) + err_vals = [data[:, i]-mean_vals]/mean_vals + filter_ids = err_vals>self.std_th + data[filter_ids[0], i] = mean_vals[filter_ids[0]]*0.8 + data[filter_ids[0], i]*0.2 + self.q.put(data[:, i]) + if self.q.qsize()>self.q_size: + self.q.get() + +# 滤波方案2 +class Filter2(object): + def __init__(self, Q=1e-6, R=4e-4): + ''' + @param Q - Q参数, channel x 1 + @param R - R参数, channel x 1 + ''' + self.Q = Q + self.R = R + self.K_prev = np.zeros_like(Q) + self.X_prev = np.zeros_like(Q) + self.P_prev = np.zeros_like(Q) + self.b_first = True + + def filter(self, data): + ''' + @param data - [InPlace], channel x data_len + ''' + if self.b_first: + self.b_first = False + self.X_prev = data[:,0] + self.P_prev = np.zeros_like(self.X_prev) + else: + self.K_prev = self.P_prev / (self.P_prev + self.R) + data[:, 0] = self.X_prev + self.K_prev * (data[:, 0] - self.X_prev) + self.P_prev = self.P_prev - self.K_prev * self.P_prev + self.Q + for i in range(data.shape[1]): + K = self.P_prev / (self.P_prev + self.R) + data[:, i] = data[:, i-1] + K * (data[:, i] - data[:, i-1]) + P = self.P_prev - K * self.P_prev + self.Q + self.P_prev = P + self.K_prev = K + self.X_prev = data[:, i] + +# 滤波方案总成 +class Filter(object): + def __init__(self, filter_type=0): + ''' + @param fitler_type - 滤波器方案,对应 FilterX + @param + ''' + if filter_type==0: + self.objFilter = Filter0() + elif filter_type==1: + self.objFilter = Filter1() + elif filter_type==2: + self.objFilter = Filter2() + + def filter(self, det): + if not det.exts2 is None: + data = np.expand_dims(det.exts2, axis=1) + self.objFilter.filter(data) + det.exts2 = data.reshape(-1) + class TrackState: """ @@ -64,7 +170,7 @@ class Track: """ def __init__(self, mean, covariance, track_id, n_init, max_age, - feature=None, binding_obj=None): + feature=None, binding_obj=None, filter_type=0): self.mean = mean self.covariance = covariance self.track_id = track_id @@ -80,6 +186,8 @@ def __init__(self, mean, covariance, track_id, n_init, max_age, self._n_init = n_init self._max_age = max_age self.binding_obj = binding_obj + self.objFilter = Filter(filter_type=filter_type) + self.exts2 = None def to_tlwh(self): """Get current position in bounding box format `(top left x, top left y, @@ -110,9 +218,12 @@ def to_tlbr(self): ret[2:4] = ret[:2] + ret[2:4] return ret - def get_exts(self): + def get_exts1(self): return self.mean[4:].copy() + def get_exts2(self): + return np.array(self.exts2) + def predict(self, kf): """Propagate the state distribution to the current time step using a Kalman filter prediction step. @@ -142,6 +253,8 @@ def update(self, kf, detection): self.mean, self.covariance = kf.update( self.mean, self.covariance, detection.to_xyah()) self.features.append(detection.feature) + self.objFilter.filter(detection) + self.exts2 = detection.exts2 self.hits += 1 self.time_since_update = 0 diff --git a/deep_sort_ex/tracker.py b/deep_sort_ex/tracker.py index 112f2d4c..30241827 100644 --- a/deep_sort_ex/tracker.py +++ b/deep_sort_ex/tracker.py @@ -37,13 +37,14 @@ class Tracker: """ - def __init__(self, metric, max_iou_distance=0.7, max_age=30, n_init=3, n_extend=0): + def __init__(self, metric, max_iou_distance=0.7, max_age=30, n_init=3, n_extend=0, filter_type=0): self.metric = metric self.max_iou_distance = max_iou_distance self.max_age = max_age self.n_init = n_init - + self.kf = kalman_filter.KalmanFilter(n_extend=n_extend) + self.filter_type = filter_type # exts2 滤波器类型 self.tracks = [] self._next_id = 1 @@ -134,5 +135,5 @@ def _initiate_track(self, detection): mean, covariance = self.kf.initiate(detection.to_xyah()) self.tracks.append(Track( mean, covariance, self._next_id, self.n_init, self.max_age, - detection.feature, binding_obj=detection.binding_obj)) + detection.feature, binding_obj=detection.binding_obj, filter_type=self.filter_type)) self._next_id += 1 diff --git a/my_deep_sort_app.py b/my_deep_sort_app.py index dd41d28d..33deb2a2 100644 --- a/my_deep_sort_app.py +++ b/my_deep_sort_app.py @@ -148,7 +148,7 @@ def create_detections(detection_mat, frame_idx, min_height=0, feature_type=0, se feature = get_feature1(sequence_dir, frame_idx, bbox) if bbox[3] < min_height: continue - detection_list.append(Detection(bbox, confidence, feature, exts=[0.5])) + detection_list.append(Detection(bbox, confidence, feature, exts1=[0.5], exts2=[0.6])) return detection_list @@ -188,7 +188,7 @@ def run(sequence_dir, detection_file, output_file, min_confidence, seq_info = gather_sequence_info(sequence_dir, detection_file, feature_type) metric = nn_matching.NearestNeighborDistanceMetric( "cosine", max_cosine_distance, nn_budget) - tracker = Tracker(metric, n_extend=1) + tracker = Tracker(metric, n_extend=1, filter_type=2) results = [] def frame_callback(vis, frame_idx): From 8d91095104933ff8111c3904a1112ee807ae35ed Mon Sep 17 00:00:00 2001 From: wilsonwnog2014 Date: Thu, 11 Jun 2020 18:06:33 +0800 Subject: [PATCH 11/24] =?UTF-8?q?=E6=89=A9=E5=B1=95=20tracker,=20track=20?= =?UTF-8?q?=E7=9A=84=20=5F=5Finit=5F=5F=E7=9A=84=E5=8F=82=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- deep_sort_ex/track.py | 13 ++++++------- deep_sort_ex/tracker.py | 18 ++++++++++++++++-- unit_test.sh | 6 ++++++ 3 files changed, 28 insertions(+), 9 deletions(-) create mode 100644 unit_test.sh diff --git a/deep_sort_ex/track.py b/deep_sort_ex/track.py index cb05fc1f..3cec1585 100644 --- a/deep_sort_ex/track.py +++ b/deep_sort_ex/track.py @@ -88,17 +88,16 @@ def filter(self, data): # 滤波方案总成 class Filter(object): - def __init__(self, filter_type=0): + def __init__(self, filter_type=0, q_size=4, std_th=0.05, Q=1e-6, R=4e-4): ''' @param fitler_type - 滤波器方案,对应 FilterX - @param ''' if filter_type==0: - self.objFilter = Filter0() + self.objFilter = Filter0(N=q_size) elif filter_type==1: - self.objFilter = Filter1() + self.objFilter = Filter1(N=q_size, std_th=std_th) elif filter_type==2: - self.objFilter = Filter2() + self.objFilter = Filter2(Q=Q, R=R) def filter(self, det): if not det.exts2 is None: @@ -170,7 +169,7 @@ class Track: """ def __init__(self, mean, covariance, track_id, n_init, max_age, - feature=None, binding_obj=None, filter_type=0): + feature=None, binding_obj=None, filter_type=0, q_size=4, std_th=0.05, Q=1e-6, R=4e-4): self.mean = mean self.covariance = covariance self.track_id = track_id @@ -186,7 +185,7 @@ def __init__(self, mean, covariance, track_id, n_init, max_age, self._n_init = n_init self._max_age = max_age self.binding_obj = binding_obj - self.objFilter = Filter(filter_type=filter_type) + self.objFilter = Filter(filter_type=filter_type, q_size=q_size, std_th=std_th, Q=Q, R=R) self.exts2 = None def to_tlwh(self): diff --git a/deep_sort_ex/tracker.py b/deep_sort_ex/tracker.py index 30241827..247c8ed4 100644 --- a/deep_sort_ex/tracker.py +++ b/deep_sort_ex/tracker.py @@ -37,7 +37,17 @@ class Tracker: """ - def __init__(self, metric, max_iou_distance=0.7, max_age=30, n_init=3, n_extend=0, filter_type=0): + def __init__(self, metric, max_iou_distance=0.7, max_age=30, n_init=3, n_extend=0, filter_type=0, q_size=4, std_th=0.05, Q=1e-6, R=4e-4): + ''' + 扩展属性 + ----- + @param n_extend - mean扩展属性数目 + @param filter_type - [Track] exts2滤波器类型 + @param q_size - [Track] 队列长度 + @param std_th - [Track] 相对误差域值 + @param Q - [Track] 卡尔曼滤波器参数 + @param R - [Track] 卡尔曼滤波器参数 + ''' self.metric = metric self.max_iou_distance = max_iou_distance self.max_age = max_age @@ -45,6 +55,10 @@ def __init__(self, metric, max_iou_distance=0.7, max_age=30, n_init=3, n_extend= self.kf = kalman_filter.KalmanFilter(n_extend=n_extend) self.filter_type = filter_type # exts2 滤波器类型 + self.q_size = q_size + self.std_th = std_th + self.Q = Q + self.R = R self.tracks = [] self._next_id = 1 @@ -135,5 +149,5 @@ def _initiate_track(self, detection): mean, covariance = self.kf.initiate(detection.to_xyah()) self.tracks.append(Track( mean, covariance, self._next_id, self.n_init, self.max_age, - detection.feature, binding_obj=detection.binding_obj, filter_type=self.filter_type)) + detection.feature, binding_obj=detection.binding_obj, filter_type=self.filter_type, q_size=self.q_size, std_th=self.std_th, Q=self.Q, R=self.R)) self._next_id += 1 diff --git a/unit_test.sh b/unit_test.sh new file mode 100644 index 00000000..39ac8649 --- /dev/null +++ b/unit_test.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env sh +# -*- coding: utf-8 -*- + +# 单元测试 +source env.bashrc +python3 my_deep_sort_app.py --feature_type 1 --display False From 9d4661d0eb9abef115b6e27f1fd19a8597e2fab0 Mon Sep 17 00:00:00 2001 From: wilsonwnog2014 Date: Fri, 12 Jun 2020 18:46:33 +0800 Subject: [PATCH 12/24] update env.bashrc, unit_test.sh --- env.bashrc | 7 ++++++- unit_test.sh | 7 ++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/env.bashrc b/env.bashrc index 0bf7d08c..ab569f85 100644 --- a/env.bashrc +++ b/env.bashrc @@ -1,4 +1,9 @@ +tf="tf1.15" +if [[ $# -gt 0 ]]; then + tf=$1 +fi + source ~/.bashrc #source ~/miniconda3/bin/activate tf1.14 -source ~/miniconda3/bin/activate tf1.15 +source ~/miniconda3/bin/activate $tf diff --git a/unit_test.sh b/unit_test.sh index 39ac8649..189df59d 100644 --- a/unit_test.sh +++ b/unit_test.sh @@ -1,6 +1,11 @@ #!/usr/bin/env sh # -*- coding: utf-8 -*- +tf="tf1.15" +if [[ $# -gt 0 ]]; then + tf=$1 +fi + # 单元测试 -source env.bashrc +source env.bashrc $tf python3 my_deep_sort_app.py --feature_type 1 --display False From 7ec1ee2a7a62ca7145fc382cdb5822ab42d16ed1 Mon Sep 17 00:00:00 2001 From: wilsonwnog2014 Date: Mon, 15 Jun 2020 19:09:11 +0800 Subject: [PATCH 13/24] update *.bashrc --- env_tf1.14.bashrc | 4 ---- 1 file changed, 4 deletions(-) delete mode 100644 env_tf1.14.bashrc diff --git a/env_tf1.14.bashrc b/env_tf1.14.bashrc deleted file mode 100644 index 25dd5463..00000000 --- a/env_tf1.14.bashrc +++ /dev/null @@ -1,4 +0,0 @@ -source ~/.bashrc -source ~/miniconda3/bin/activate tf1.14 -#source ~/miniconda3/bin/activate tf1.15 - From 97b0b8224f747e5dac9e7f2ff2bb3525a2836c37 Mon Sep 17 00:00:00 2001 From: wilsonwnog2014 Date: Wed, 17 Jun 2020 18:29:25 +0800 Subject: [PATCH 14/24] update --- deep_sort_ex/track.py | 64 ++++++++++++++++++++++++++++++++++++++++- deep_sort_ex/tracker.py | 14 +++++---- 2 files changed, 71 insertions(+), 7 deletions(-) diff --git a/deep_sort_ex/track.py b/deep_sort_ex/track.py index 3cec1585..eae8c536 100644 --- a/deep_sort_ex/track.py +++ b/deep_sort_ex/track.py @@ -86,6 +86,26 @@ def filter(self, data): self.K_prev = K self.X_prev = data[:, i] +# 滤波方案3 +class Filter3(object): + def __init__(self, N=4, std_th=0.05, Q=1e-6, R=4e-4): + ''' + @param N - 队列长度 + @param std_th - 方差域值 + @param Q - Q参数, channel x 1 + @param R - R参数, channel x 1 + ''' + self.filter1 = Filter1(N=N, std_th=std_th) + self.filter2 = Filter2(Q=Q, R=R) + + def filter(self, data): + ''' + @param data - np.array: channels x data_len + ''' + self.filter1.filter(data) + self.filter2.filter(data) + + # 滤波方案总成 class Filter(object): def __init__(self, filter_type=0, q_size=4, std_th=0.05, Q=1e-6, R=4e-4): @@ -98,6 +118,8 @@ def __init__(self, filter_type=0, q_size=4, std_th=0.05, Q=1e-6, R=4e-4): self.objFilter = Filter1(N=q_size, std_th=std_th) elif filter_type==2: self.objFilter = Filter2(Q=Q, R=R) + elif filter_type==3: + self.objFilter = Filter3(N=q_size, std_th=std_th, Q=Q, R=R) def filter(self, det): if not det.exts2 is None: @@ -169,7 +191,19 @@ class Track: """ def __init__(self, mean, covariance, track_id, n_init, max_age, - feature=None, binding_obj=None, filter_type=0, q_size=4, std_th=0.05, Q=1e-6, R=4e-4): + feature=None, binding_obj=None, filter_type=0, q_size=4, std_th=0.05, Q=1e-6, R=4e-4, save_to=None): + ''' + 扩展属性 + ----- + @param n_extend - mean扩展属性数目 + @param filter_type - [Track] exts2滤波器类型 + @param q_size - [Track] 队列长度 + @param std_th - [Track] 相对误差域值 + @param Q - [Track] 卡尔曼滤波器参数 + @param R - [Track] 卡尔曼滤波器参数 + @param save_to - [Track] 采集数据保存目录 + ''' + self.mean = mean self.covariance = covariance self.track_id = track_id @@ -187,6 +221,10 @@ def __init__(self, mean, covariance, track_id, n_init, max_age, self.binding_obj = binding_obj self.objFilter = Filter(filter_type=filter_type, q_size=q_size, std_th=std_th, Q=Q, R=R) self.exts2 = None + self.save_to = save_to + if not save_to is None: + import os + os.makedirs(save_to, exist_ok=True) def to_tlwh(self): """Get current position in bounding box format `(top left x, top left y, @@ -249,12 +287,36 @@ def update(self, kf, detection): The associated detection. """ + # 数据采集: 滤波前 + # --------------- + if not self.save_to is None: + with open('%s/mean-orign-%d.txt' % (self.save_to, self.track_id), 'a+') as f: + f.write(str(list(detection.to_xyah()))[1:-1]) + f.write('\n') + with open('%s/exts2-orign-%d.txt' % (self.save_to, self.track_id), 'a+') as f: + f.write(str(list(detection.exts2))[1:-1]) + f.write('\n') + + # 跟踪/滤波处理 + # ------------ self.mean, self.covariance = kf.update( self.mean, self.covariance, detection.to_xyah()) self.features.append(detection.feature) self.objFilter.filter(detection) self.exts2 = detection.exts2 + # 数据采集: 滤波后 + # --------------- + if not self.save_to is None: + with open('%s/mean-filter-%d.txt' % (self.save_to, self.track_id), 'a+') as f: + f.write(str(list(self.mean))[1:-1]) + f.write('\n') + with open('%s/exts2-filter-%d.txt' % (self.save_to, self.track_id), 'a+') as f: + f.write(str(list(self.exts2))[1:-1]) + f.write('\n') + + # 状态更新 + # -------- self.hits += 1 self.time_since_update = 0 if self.state == TrackState.Tentative and self.hits >= self._n_init: diff --git a/deep_sort_ex/tracker.py b/deep_sort_ex/tracker.py index 247c8ed4..1fe46a17 100644 --- a/deep_sort_ex/tracker.py +++ b/deep_sort_ex/tracker.py @@ -37,7 +37,7 @@ class Tracker: """ - def __init__(self, metric, max_iou_distance=0.7, max_age=30, n_init=3, n_extend=0, filter_type=0, q_size=4, std_th=0.05, Q=1e-6, R=4e-4): + def __init__(self, metric, max_iou_distance=0.7, max_age=30, n_init=3, n_extend=0, filter_type=0, q_size=4, std_th=0.05, Q=1e-6, R=4e-4, save_to=None): ''' 扩展属性 ----- @@ -47,6 +47,7 @@ def __init__(self, metric, max_iou_distance=0.7, max_age=30, n_init=3, n_extend= @param std_th - [Track] 相对误差域值 @param Q - [Track] 卡尔曼滤波器参数 @param R - [Track] 卡尔曼滤波器参数 + @param save_to - [Track] 采集数据保存目录 ''' self.metric = metric self.max_iou_distance = max_iou_distance @@ -55,10 +56,11 @@ def __init__(self, metric, max_iou_distance=0.7, max_age=30, n_init=3, n_extend= self.kf = kalman_filter.KalmanFilter(n_extend=n_extend) self.filter_type = filter_type # exts2 滤波器类型 - self.q_size = q_size - self.std_th = std_th - self.Q = Q - self.R = R + self.q_size = q_size # 队列长度 + self.std_th = std_th # 方差域值 + self.Q = Q # 卡尔曼参数 + self.R = R # 卡尔曼参数 + self.save_to = save_to # 采集数据保存目录 self.tracks = [] self._next_id = 1 @@ -149,5 +151,5 @@ def _initiate_track(self, detection): mean, covariance = self.kf.initiate(detection.to_xyah()) self.tracks.append(Track( mean, covariance, self._next_id, self.n_init, self.max_age, - detection.feature, binding_obj=detection.binding_obj, filter_type=self.filter_type, q_size=self.q_size, std_th=self.std_th, Q=self.Q, R=self.R)) + detection.feature, binding_obj=detection.binding_obj, filter_type=self.filter_type, q_size=self.q_size, std_th=self.std_th, Q=self.Q, R=self.R, save_to=self.save_to)) self._next_id += 1 From 6d4ecf4aaaf865c1854722d9ec998df491d2d4e4 Mon Sep 17 00:00:00 2001 From: wilsonwnog2014 Date: Fri, 19 Jun 2020 16:44:49 +0800 Subject: [PATCH 15/24] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=20=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E9=87=87=E9=9B=86[=E8=B0=83=E8=AF=95]=E7=9A=84=20flag=E6=A0=87?= =?UTF-8?q?=E8=AE=B0=E5=8F=82=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- deep_sort_ex/detection.py | 4 +++- deep_sort_ex/track.py | 36 ++++++++++++++++++++---------------- deep_sort_ex/tracker.py | 6 ++++-- 3 files changed, 27 insertions(+), 19 deletions(-) diff --git a/deep_sort_ex/detection.py b/deep_sort_ex/detection.py index c8c65709..7f1fd35e 100644 --- a/deep_sort_ex/detection.py +++ b/deep_sort_ex/detection.py @@ -26,7 +26,7 @@ class Detection(object): """ - def __init__(self, tlwh, confidence, feature, exts1=None, exts2=None, binding_obj=None): + def __init__(self, tlwh, confidence, feature, exts1=None, exts2=None, binding_obj=None, flag=None): ''' @param tlwh - bbox: top, left, width, height @param confidence - 目标检测置信度 @@ -34,11 +34,13 @@ def __init__(self, tlwh, confidence, feature, exts1=None, exts2=None, binding_ob @param exts1 - 扩展属性: 扩展卡尔曼滤波器的向量(mean)， 需要与 tracker,track的n_extend参数配合使用 @param exts2 - 扩展属性: 独立于卡尔曼滤波器，对扩展通道单独处理 @param binding_obj - 绑定原始目标检测的序号，方便数据源跟踪 + @param flag - 跟踪目标标记， 如用于目标类别，由外部解析 ''' self.tlwh = np.asarray(tlwh, dtype=np.float) self.exts1 = exts1 self.exts2 = exts2 self.binding_obj = binding_obj + self.flag = flag self.confidence = float(confidence) self.feature = np.asarray(feature, dtype=np.float32) diff --git a/deep_sort_ex/track.py b/deep_sort_ex/track.py index eae8c536..92526b93 100644 --- a/deep_sort_ex/track.py +++ b/deep_sort_ex/track.py @@ -28,13 +28,15 @@ def filter(self, data): class Filter1(object): '''依据方差剔除奇点数据 ''' - def __init__(self, N=4, std_th=0.05): + def __init__(self, N=4, std_th=0.05, percent=0.8): ''' - @param N - 队列长度 - @param std_th - 方差域值 + @param N - 队列长度 + @param std_th - 方差域值 + @param percent - 奇异点保留前置能量比，当设为1.0即为完全用前置点替换奇异点 ''' self.q_size = N self.std_th = std_th + self.percent = percent self.q = queue.Queue() def filter(self, data): @@ -47,7 +49,7 @@ def filter(self, data): mean_vals = np.array(q_data).mean(axis=0) err_vals = [data[:, i]-mean_vals]/mean_vals filter_ids = err_vals>self.std_th - data[filter_ids[0], i] = mean_vals[filter_ids[0]]*0.8 + data[filter_ids[0], i]*0.2 + data[filter_ids[0], i] = mean_vals[filter_ids[0]]*self.percent + data[filter_ids[0], i]*(1.0-self.percent) self.q.put(data[:, i]) if self.q.qsize()>self.q_size: self.q.get() @@ -88,14 +90,15 @@ def filter(self, data): # 滤波方案3 class Filter3(object): - def __init__(self, N=4, std_th=0.05, Q=1e-6, R=4e-4): + def __init__(self, N=4, std_th=0.05, percent=0.8, Q=1e-6, R=4e-4): ''' - @param N - 队列长度 - @param std_th - 方差域值 + @param N - 队列长度 + @param std_th - 方差域值 + @param percent - 奇异点保留前置能量比，当设为1.0即为完全用前置点替换奇异点 @param Q - Q参数, channel x 1 @param R - R参数, channel x 1 ''' - self.filter1 = Filter1(N=N, std_th=std_th) + self.filter1 = Filter1(N=N, std_th=std_th, percent=percent) self.filter2 = Filter2(Q=Q, R=R) def filter(self, data): @@ -108,18 +111,18 @@ def filter(self, data): # 滤波方案总成 class Filter(object): - def __init__(self, filter_type=0, q_size=4, std_th=0.05, Q=1e-6, R=4e-4): + def __init__(self, filter_type=0, q_size=4, std_th=0.05, percent=0.8, Q=1e-6, R=4e-4): ''' @param fitler_type - 滤波器方案,对应 FilterX ''' if filter_type==0: self.objFilter = Filter0(N=q_size) elif filter_type==1: - self.objFilter = Filter1(N=q_size, std_th=std_th) + self.objFilter = Filter1(N=q_size, std_th=std_th, percent=percent) elif filter_type==2: self.objFilter = Filter2(Q=Q, R=R) elif filter_type==3: - self.objFilter = Filter3(N=q_size, std_th=std_th, Q=Q, R=R) + self.objFilter = Filter3(N=q_size, std_th=std_th, percent=percent, Q=Q, R=R) def filter(self, det): if not det.exts2 is None: @@ -191,7 +194,7 @@ class Track: """ def __init__(self, mean, covariance, track_id, n_init, max_age, - feature=None, binding_obj=None, filter_type=0, q_size=4, std_th=0.05, Q=1e-6, R=4e-4, save_to=None): + feature=None, binding_obj=None, filter_type=0, q_size=4, std_th=0.05, percent=0.8, Q=1e-6, R=4e-4, save_to=None): ''' 扩展属性 ----- @@ -199,6 +202,7 @@ def __init__(self, mean, covariance, track_id, n_init, max_age, @param filter_type - [Track] exts2滤波器类型 @param q_size - [Track] 队列长度 @param std_th - [Track] 相对误差域值 + @param percent - [Track] 奇异点保留前置能量比，当设为1.0即为完全用前置点替换奇异点 @param Q - [Track] 卡尔曼滤波器参数 @param R - [Track] 卡尔曼滤波器参数 @param save_to - [Track] 采集数据保存目录 @@ -290,10 +294,10 @@ def update(self, kf, detection): # 数据采集: 滤波前 # --------------- if not self.save_to is None: - with open('%s/mean-orign-%d.txt' % (self.save_to, self.track_id), 'a+') as f: + with open('%s/mean-orign-%s-%d.txt' % (self.save_to, detection.flag, self.track_id), 'a+') as f: f.write(str(list(detection.to_xyah()))[1:-1]) f.write('\n') - with open('%s/exts2-orign-%d.txt' % (self.save_to, self.track_id), 'a+') as f: + with open('%s/exts2-orign-%s-%d.txt' % (self.save_to, detection.flag, self.track_id), 'a+') as f: f.write(str(list(detection.exts2))[1:-1]) f.write('\n') @@ -308,10 +312,10 @@ def update(self, kf, detection): # 数据采集: 滤波后 # --------------- if not self.save_to is None: - with open('%s/mean-filter-%d.txt' % (self.save_to, self.track_id), 'a+') as f: + with open('%s/mean-filter-%s-%d.txt' % (self.save_to, detection.flag, self.track_id), 'a+') as f: f.write(str(list(self.mean))[1:-1]) f.write('\n') - with open('%s/exts2-filter-%d.txt' % (self.save_to, self.track_id), 'a+') as f: + with open('%s/exts2-filter-%s-%d.txt' % (self.save_to, detection.flag, self.track_id), 'a+') as f: f.write(str(list(self.exts2))[1:-1]) f.write('\n') diff --git a/deep_sort_ex/tracker.py b/deep_sort_ex/tracker.py index 1fe46a17..9d139827 100644 --- a/deep_sort_ex/tracker.py +++ b/deep_sort_ex/tracker.py @@ -37,7 +37,7 @@ class Tracker: """ - def __init__(self, metric, max_iou_distance=0.7, max_age=30, n_init=3, n_extend=0, filter_type=0, q_size=4, std_th=0.05, Q=1e-6, R=4e-4, save_to=None): + def __init__(self, metric, max_iou_distance=0.7, max_age=30, n_init=3, n_extend=0, filter_type=0, q_size=4, std_th=0.05, percent=0.8, Q=1e-6, R=4e-4, save_to=None): ''' 扩展属性 ----- @@ -45,6 +45,7 @@ def __init__(self, metric, max_iou_distance=0.7, max_age=30, n_init=3, n_extend= @param filter_type - [Track] exts2滤波器类型 @param q_size - [Track] 队列长度 @param std_th - [Track] 相对误差域值 + @param percent - [Track] 奇异点保留前置能量比，当设为1.0即为完全用前置点替换奇异点 @param Q - [Track] 卡尔曼滤波器参数 @param R - [Track] 卡尔曼滤波器参数 @param save_to - [Track] 采集数据保存目录 @@ -58,6 +59,7 @@ def __init__(self, metric, max_iou_distance=0.7, max_age=30, n_init=3, n_extend= self.filter_type = filter_type # exts2 滤波器类型 self.q_size = q_size # 队列长度 self.std_th = std_th # 方差域值 + self.percent = percent # 奇异点保留前置能量比，当设为1.0即为完全用前置点替换奇异点 self.Q = Q # 卡尔曼参数 self.R = R # 卡尔曼参数 self.save_to = save_to # 采集数据保存目录 @@ -151,5 +153,5 @@ def _initiate_track(self, detection): mean, covariance = self.kf.initiate(detection.to_xyah()) self.tracks.append(Track( mean, covariance, self._next_id, self.n_init, self.max_age, - detection.feature, binding_obj=detection.binding_obj, filter_type=self.filter_type, q_size=self.q_size, std_th=self.std_th, Q=self.Q, R=self.R, save_to=self.save_to)) + detection.feature, binding_obj=detection.binding_obj, filter_type=self.filter_type, q_size=self.q_size, std_th=self.std_th, percent=self.percent, Q=self.Q, R=self.R, save_to=self.save_to)) self._next_id += 1 From 9e392d72c23c3002873513d496cef23fefaba082 Mon Sep 17 00:00:00 2001 From: wilsonwnog2014 Date: Fri, 3 Jul 2020 09:38:24 +0800 Subject: [PATCH 16/24] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=20self.samples.get(tar?= =?UTF-8?q?get)=E4=B8=BANone=E9=97=AE=E9=A2=98=E5=AF=BC=E8=87=B4=E7=9A=84?= =?UTF-8?q?=E5=BC=82=E5=B8=B8,=20=E5=8E=9F=E5=9B=A0=E5=BE=85=E6=9F=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- deep_sort_ex/nn_matching.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/deep_sort_ex/nn_matching.py b/deep_sort_ex/nn_matching.py index c1f20b5e..779ad4b5 100644 --- a/deep_sort_ex/nn_matching.py +++ b/deep_sort_ex/nn_matching.py @@ -151,7 +151,8 @@ def partial_fit(self, features, targets, active_targets): self.samples.setdefault(target, []).append(feature) if self.budget is not None: self.samples[target] = self.samples[target][-self.budget:] - self.samples = {k: self.samples[k] for k in active_targets} + # self.samples = {k: self.samples[k] for k in active_targets} + self.samples = {k: self.samples.get(k) for k in active_targets} # Modify def distance(self, features, targets): """Compute distance between features and targets. @@ -173,5 +174,10 @@ def distance(self, features, targets): """ cost_matrix = np.zeros((len(targets), len(features))) for i, target in enumerate(targets): + # modefy begin + if self.samples.get(target) is None: + print('[WARNING] self.samples.get(%d) is None, %s: %s here.' %(target, __file__, str(sys._getframe().f_lineno))) + continue + # modefy end cost_matrix[i, :] = self._metric(self.samples[target], features) return cost_matrix From 6e81f9b61d53de1afcd9a220e1bdc67b58220b32 Mon Sep 17 00:00:00 2001 From: wilsonwnog2014 Date: Wed, 8 Jul 2020 19:01:20 +0800 Subject: [PATCH 17/24] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E4=BD=8E=E9=80=9A?= =?UTF-8?q?=E6=BB=A4=E6=B3=A2=E5=99=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- deep_sort_ex/nn_matching.py | 2 +- deep_sort_ex/track.py | 94 ++++++++++++++++++++++++++++++------- deep_sort_ex/tracker.py | 10 +++- 3 files changed, 87 insertions(+), 19 deletions(-) diff --git a/deep_sort_ex/nn_matching.py b/deep_sort_ex/nn_matching.py index 779ad4b5..918c0a19 100644 --- a/deep_sort_ex/nn_matching.py +++ b/deep_sort_ex/nn_matching.py @@ -1,6 +1,6 @@ # vim: expandtab:ts=4:sw=4 import numpy as np - +import sys def _pdist(a, b): """Compute pair-wise squared distance between points in `a` and `b`. diff --git a/deep_sort_ex/track.py b/deep_sort_ex/track.py index 92526b93..aa99ba45 100644 --- a/deep_sort_ex/track.py +++ b/deep_sort_ex/track.py @@ -2,7 +2,7 @@ import queue import numpy as np - +from scipy.signal import butter, lfilter, lfilter_zi, freqz # 滤波方案0 class Filter0(object): '''均值滤波，设定队列长度，取队列均值作为输出 @@ -56,6 +56,8 @@ def filter(self, data): # 滤波方案2 class Filter2(object): + '''简易的卡尔曼滤波 + ''' def __init__(self, Q=1e-6, R=4e-4): ''' @param Q - Q参数, channel x 1 @@ -88,18 +90,25 @@ def filter(self, data): self.K_prev = K self.X_prev = data[:, i] -# 滤波方案3 -class Filter3(object): - def __init__(self, N=4, std_th=0.05, percent=0.8, Q=1e-6, R=4e-4): +# 滤波方案3_del +class Filter3_del(object): + '''集成奇异值过滤和卡尔曼滤波 + ''' + def __init__(self, N=4, std_th=0.05, percent=0.8, Q=1e-6, R=4e-4, fs=5., cutoff=1.0, order=5): ''' @param N - 队列长度 @param std_th - 方差域值 @param percent - 奇异点保留前置能量比，当设为1.0即为完全用前置点替换奇异点 @param Q - Q参数, channel x 1 @param R - R参数, channel x 1 + @param fs - 采样率 + @param cutoff - 截止频率, Hz + @param order - 滤波器阶数 + ''' self.filter1 = Filter1(N=N, std_th=std_th, percent=percent) self.filter2 = Filter2(Q=Q, R=R) + self.filter4 = Filter4(fs=fs, cutoff=cutoff, order=order) def filter(self, data): ''' @@ -107,29 +116,79 @@ def filter(self, data): ''' self.filter1.filter(data) self.filter2.filter(data) + self.filter4.filter(data) +# 滤波方案3 +class Filter3(object): + '''Buffer低通滤波器 + ''' + def __init__(self, fs, cutoff=2.0, order=5): + ''' + @param fs - 采样率 + @param cutoff - 截止频率, Hz + @param order - 滤波器阶数 + ''' + self.order = order + b, a = self.butter_lowpass(cutoff, fs, order) + self.b = b + self.a = a + self.zi = lfilter_zi(b, a) + self.index = 0 + + def butter_lowpass(self, cutoff, fs, order=5): + nyq = 0.5 * fs + normal_cutoff = cutoff / nyq + b, a = butter(order, normal_cutoff, btype='low', analog=False) + return b, a + + def filter(self, data): + ''' + @param data - [InPlace], channel x data_len + ''' + index = self.index + for data_chl in data: + for i in range(len(data_chl)): + z, self.zi = lfilter(self.b, self.a, [data_chl[i]], zi=self.zi) + if index+i Date: Fri, 14 Aug 2020 19:21:19 +0800 Subject: [PATCH 18/24] =?UTF-8?q?=E4=BC=98=E5=8C=96=E5=99=AA=E5=A3=B0?= =?UTF-8?q?=E8=BF=87=E6=BB=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- deep_sort_ex/track.py | 121 ++++++++++++++++++++++++++++++++++++------ 1 file changed, 104 insertions(+), 17 deletions(-) diff --git a/deep_sort_ex/track.py b/deep_sort_ex/track.py index aa99ba45..e3b7d581 100644 --- a/deep_sort_ex/track.py +++ b/deep_sort_ex/track.py @@ -19,13 +19,17 @@ def filter(self, data): @param data - np.array: channels x data_len ''' for i in range(data.shape[1]): - self.q.put(data[:, i]) - data[:, i] = np.array(self.q.queue).mean(axis=0) + if np.isnan(data[:, i]).any(): + data[:, i] = np.array(self.q.queue).mean(axis=0) + self.q.put(data[:, i]) + else: + self.q.put(data[:, i]) + data[:, i] = np.array(self.q.queue).mean(axis=0) if self.q.qsize()>=self.N: self.q.get() # 滤波方案1 -class Filter1(object): +class Filter1_del(object): '''依据方差剔除奇点数据 ''' def __init__(self, N=4, std_th=0.05, percent=0.8): @@ -44,12 +48,64 @@ def filter(self, data): @param data - np.array: channels x data_len ''' for i in range(data.shape[1]): + if np.isnan(data[:, i]): + data[:, i] = np.array(self.q.queue).mean(axis=0) if self.q.qsize()==self.q_size: q_data = np.array(self.q.queue) mean_vals = np.array(q_data).mean(axis=0) - err_vals = [data[:, i]-mean_vals]/mean_vals + err_vals = (data[:, i]-mean_vals)/mean_vals filter_ids = err_vals>self.std_th - data[filter_ids[0], i] = mean_vals[filter_ids[0]]*self.percent + data[filter_ids[0], i]*(1.0-self.percent) + #data[filter_ids[0], i] = mean_vals[filter_ids[0]]*self.percent + data[filter_ids[0], i]*(1.0-self.percent) + for j, t in enumerate(filter_ids): + if t: + data[j, i] = mean_vals[j]*self.percent + data[j, i]*(1.0-self.percent) + self.q.put(data[:, i]) + if self.q.qsize()>self.q_size: + self.q.get() + +# 滤波方案1 +class Filter1(object): + '''依据方差剔除奇点数据 + 根据方差统计动态调整percent值 + 设置队列记录过去N个数据值 queue[N] + 计算标准差 std(queue) => std_val + 动态计算percnet: 标准差换算 + 相对误差 err = (det-mean)/mean + 设定sigmod曲线表 tbl_percent = np.exp(range(-100, 1)) + 设定相对误差最大阈值 std_val = 0.1 + 相对误差换算曲线表序号 tbl_index = int(err*(100/std_val)) + tbl_index修正处理: >100 => 设置为100 + percent = tbl_percent[tbl_index] + + ''' + def __init__(self, N=4, std_th=0.1, percent=0.8): + ''' + @param N - 队列长度 + @param std_th - 方差阈值 + @param percent - 奇异点保留前置能量比，当设为1.0即为完全用前置点替换奇异点 + ''' + self.q_size = N + self.std_th = std_th + self.percent = percent + self.q = queue.Queue() + self.tbl_percent = np.exp(range(-100, 1)) # + self.max_val = 100/std_th + + def filter(self, data): + ''' + @param data - np.array: channels x data_len + ''' + for i in range(data.shape[1]): + if np.isnan(data[:, i]).any(): + data[:, i] = np.array(self.q.queue).mean(axis=0) + if self.q.qsize()==self.q_size: + q_data = np.array(self.q.queue) + mean_vals = np.array(q_data).mean(axis=0) + err_vals = abs(data[:, i]-mean_vals)/abs(mean_vals) + percent_ids = (err_vals*self.max_val).astype(np.int) # 相对误差截止点 0.1 + percent_ids[percent_ids>100]=100 # sigmod曲线 + percents = self.tbl_percent[percent_ids] + data[:, i] = mean_vals*percents + data[:, i]*(1.0-percents) self.q.put(data[:, i]) if self.q.qsize()>self.q_size: self.q.get() @@ -75,14 +131,28 @@ def filter(self, data): @param data - [InPlace], channel x data_len ''' if self.b_first: - self.b_first = False - self.X_prev = data[:,0] - self.P_prev = np.zeros_like(self.X_prev) + # 查找第一个非 NaN数据 + next_i = None + for i in range(data.shape[1]): + if not np.isnan(data[:, i]): + next_i=i+1 + break + if not next_i is None: + self.b_first = False + self.X_prev = data[:, next_i-1] + self.P_prev = np.zeros_like(self.X_prev) + else: + next_i = data.shape[1]+1 else: + if np.isnan(data[:, 0]): + data[:, 0] = self.X_prev self.K_prev = self.P_prev / (self.P_prev + self.R) data[:, 0] = self.X_prev + self.K_prev * (data[:, 0] - self.X_prev) self.P_prev = self.P_prev - self.K_prev * self.P_prev + self.Q - for i in range(data.shape[1]): + next_i = 1 + for i in range(next_i, data.shape[1]): + if np.isnan(data[:, i]): + data[:, i] = self.X_prev K = self.P_prev / (self.P_prev + self.R) data[:, i] = data[:, i-1] + K * (data[:, i] - data[:, i-1]) P = self.P_prev - K * self.P_prev + self.Q @@ -133,6 +203,9 @@ def __init__(self, fs, cutoff=2.0, order=5): self.b = b self.a = a self.zi = lfilter_zi(b, a) + self.data_first_len = int(order*2) + self.data_first = np.zeros((self.data_first_len,), dtype=np.float32) + self.prev_val = None self.index = 0 def butter_lowpass(self, cutoff, fs, order=5): @@ -145,14 +218,28 @@ def filter(self, data): ''' @param data - [InPlace], channel x data_len ''' + if self.prev_val is None: + self.prev_val = np.zeros((data.shape[0],)) index = self.index - for data_chl in data: - for i in range(len(data_chl)): + for chl, data_chl in enumerate(data): + for i in range(len(data_chl)): + if np.isnan(data_chl[i]): + data_chl[i] = self.prev_val[chl] z, self.zi = lfilter(self.b, self.a, [data_chl[i]], zi=self.zi) - if index+i Date: Fri, 4 Sep 2020 20:18:48 +0800 Subject: [PATCH 19/24] update --- deep_sort_ex/tracker.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/deep_sort_ex/tracker.py b/deep_sort_ex/tracker.py index 6cc039c8..707324a7 100644 --- a/deep_sort_ex/tracker.py +++ b/deep_sort_ex/tracker.py @@ -97,10 +97,15 @@ def update(self, detections): for track_idx, detection_idx in matches: self.tracks[track_idx].update( self.kf, detections[detection_idx]) + # update track.binding_obj + self.tracks[track_idx].binding_obj = detections[detection_idx].binding_obj + for track_idx in unmatched_tracks: self.tracks[track_idx].mark_missed() + for detection_idx in unmatched_detections: self._initiate_track(detections[detection_idx]) + self.tracks = [t for t in self.tracks if not t.is_deleted()] # Update distance metric. From 20eab64bf0c55d92cf4e48993c851a64e004b91a Mon Sep 17 00:00:00 2001 From: wilsonwnog2014 Date: Mon, 7 Sep 2020 21:30:34 +0800 Subject: [PATCH 20/24] =?UTF-8?q?=E4=BF=AE=E6=94=B9=20=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E9=87=87=E9=9B=86=E4=BF=9D=E5=AD=98=E7=9A=84=E8=B7=AF=E5=BE=84?= =?UTF-8?q?=E8=A7=84=E5=88=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- deep_sort_ex/track.py | 129 +++++++++++++++------------------------- deep_sort_ex/tracker.py | 12 +++- 2 files changed, 58 insertions(+), 83 deletions(-) diff --git a/deep_sort_ex/track.py b/deep_sort_ex/track.py index e3b7d581..a43561f7 100644 --- a/deep_sort_ex/track.py +++ b/deep_sort_ex/track.py @@ -1,5 +1,6 @@ # vim: expandtab:ts=4:sw=4 +import os import queue import numpy as np from scipy.signal import butter, lfilter, lfilter_zi, freqz @@ -18,51 +19,27 @@ def filter(self, data): ''' @param data - np.array: channels x data_len ''' + #for i in range(data.shape[1]): + # if np.isnan(data[:, i]).any(): + # data[:, i] = np.array(self.q.queue).mean(axis=0) + # self.q.put(data[:, i]) + # else: + # self.q.put(data[:, i]) + # data[:, i] = np.array(self.q.queue).mean(axis=0) + # if self.q.qsize()>=self.N: + # self.q.get() + for i in range(data.shape[1]): - if np.isnan(data[:, i]).any(): - data[:, i] = np.array(self.q.queue).mean(axis=0) - self.q.put(data[:, i]) + if np.isnan(data[:, i]).any() or (data[:, i]==9999.0).any(): + if self.q.qsize()>0: + data[:, i] = np.array(self.q.queue).mean(axis=0) + self.q.put(data[:, i]) else: self.q.put(data[:, i]) data[:, i] = np.array(self.q.queue).mean(axis=0) if self.q.qsize()>=self.N: self.q.get() -# 滤波方案1 -class Filter1_del(object): - '''依据方差剔除奇点数据 - ''' - def __init__(self, N=4, std_th=0.05, percent=0.8): - ''' - @param N - 队列长度 - @param std_th - 方差域值 - @param percent - 奇异点保留前置能量比，当设为1.0即为完全用前置点替换奇异点 - ''' - self.q_size = N - self.std_th = std_th - self.percent = percent - self.q = queue.Queue() - - def filter(self, data): - ''' - @param data - np.array: channels x data_len - ''' - for i in range(data.shape[1]): - if np.isnan(data[:, i]): - data[:, i] = np.array(self.q.queue).mean(axis=0) - if self.q.qsize()==self.q_size: - q_data = np.array(self.q.queue) - mean_vals = np.array(q_data).mean(axis=0) - err_vals = (data[:, i]-mean_vals)/mean_vals - filter_ids = err_vals>self.std_th - #data[filter_ids[0], i] = mean_vals[filter_ids[0]]*self.percent + data[filter_ids[0], i]*(1.0-self.percent) - for j, t in enumerate(filter_ids): - if t: - data[j, i] = mean_vals[j]*self.percent + data[j, i]*(1.0-self.percent) - self.q.put(data[:, i]) - if self.q.qsize()>self.q_size: - self.q.get() - # 滤波方案1 class Filter1(object): '''依据方差剔除奇点数据 @@ -96,8 +73,11 @@ def filter(self, data): @param data - np.array: channels x data_len ''' for i in range(data.shape[1]): - if np.isnan(data[:, i]).any(): - data[:, i] = np.array(self.q.queue).mean(axis=0) + exist_nan = False + if np.isnan(data[:, i]).any() or (data[:, i]==9999.0).any(): + if self.q.qsize()>0: + data[:, i] = np.array(self.q.queue).mean(axis=0) + exist_nan = True if self.q.qsize()==self.q_size: q_data = np.array(self.q.queue) mean_vals = np.array(q_data).mean(axis=0) @@ -106,9 +86,11 @@ def filter(self, data): percent_ids[percent_ids>100]=100 # sigmod曲线 percents = self.tbl_percent[percent_ids] data[:, i] = mean_vals*percents + data[:, i]*(1.0-percents) - self.q.put(data[:, i]) - if self.q.qsize()>self.q_size: - self.q.get() + + if not exist_nan: + self.q.put(data[:, i]) + if self.q.qsize()>self.q_size: + self.q.get() # 滤波方案2 class Filter2(object): @@ -134,7 +116,7 @@ def filter(self, data): # 查找第一个非 NaN数据 next_i = None for i in range(data.shape[1]): - if not np.isnan(data[:, i]): + if not np.isnan(data[:, i]).any() and not (data[:, i]==9999.0).any() : next_i=i+1 break if not next_i is None: @@ -144,14 +126,14 @@ def filter(self, data): else: next_i = data.shape[1]+1 else: - if np.isnan(data[:, 0]): + if np.isnan(data[:, 0]).any() or (data[:, 0]==9999.0).any(): data[:, 0] = self.X_prev self.K_prev = self.P_prev / (self.P_prev + self.R) data[:, 0] = self.X_prev + self.K_prev * (data[:, 0] - self.X_prev) self.P_prev = self.P_prev - self.K_prev * self.P_prev + self.Q next_i = 1 for i in range(next_i, data.shape[1]): - if np.isnan(data[:, i]): + if np.isnan(data[:, i]).any() or (data[:, i]==9999.0).any(): data[:, i] = self.X_prev K = self.P_prev / (self.P_prev + self.R) data[:, i] = data[:, i-1] + K * (data[:, i] - data[:, i-1]) @@ -160,33 +142,6 @@ def filter(self, data): self.K_prev = K self.X_prev = data[:, i] -# 滤波方案3_del -class Filter3_del(object): - '''集成奇异值过滤和卡尔曼滤波 - ''' - def __init__(self, N=4, std_th=0.05, percent=0.8, Q=1e-6, R=4e-4, fs=5., cutoff=1.0, order=5): - ''' - @param N - 队列长度 - @param std_th - 方差域值 - @param percent - 奇异点保留前置能量比，当设为1.0即为完全用前置点替换奇异点 - @param Q - Q参数, channel x 1 - @param R - R参数, channel x 1 - @param fs - 采样率 - @param cutoff - 截止频率, Hz - @param order - 滤波器阶数 - - ''' - self.filter1 = Filter1(N=N, std_th=std_th, percent=percent) - self.filter2 = Filter2(Q=Q, R=R) - self.filter4 = Filter4(fs=fs, cutoff=cutoff, order=order) - - def filter(self, data): - ''' - @param data - np.array: channels x data_len - ''' - self.filter1.filter(data) - self.filter2.filter(data) - self.filter4.filter(data) # 滤波方案3 class Filter3(object): @@ -223,7 +178,7 @@ def filter(self, data): index = self.index for chl, data_chl in enumerate(data): for i in range(len(data_chl)): - if np.isnan(data_chl[i]): + if np.isnan(data_chl[i]) or data_chl[i]==9999.0: data_chl[i] = self.prev_val[chl] z, self.zi = lfilter(self.b, self.a, [data_chl[i]], zi=self.zi) #if index+i Date: Tue, 15 Sep 2020 19:38:25 +0800 Subject: [PATCH 21/24] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E9=80=9F=E5=BA=A6?= =?UTF-8?q?=E4=BC=B0=E7=AE=97=E7=9B=B8=E5=85=B3=E4=BF=A1=E6=81=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- deep_sort_ex/detection.py | 6 ++++-- deep_sort_ex/kalman_filter.py | 14 +++++++------- deep_sort_ex/track.py | 18 ++++++++++++++++-- deep_sort_ex/tracker.py | 2 +- 4 files changed, 28 insertions(+), 12 deletions(-) diff --git a/deep_sort_ex/detection.py b/deep_sort_ex/detection.py index 7f1fd35e..95c48d56 100644 --- a/deep_sort_ex/detection.py +++ b/deep_sort_ex/detection.py @@ -26,17 +26,19 @@ class Detection(object): """ - def __init__(self, tlwh, confidence, feature, exts1=None, exts2=None, binding_obj=None, flag=None): + def __init__(self, tlwh, confidence, feature, t=None, exts1=None, exts2=None, binding_obj=None, flag=None): ''' @param tlwh - bbox: top, left, width, height @param confidence - 目标检测置信度 @param ffeature - 目标图像特征码 + @param t - 检测时间(秒) @param exts1 - 扩展属性: 扩展卡尔曼滤波器的向量(mean)， 需要与 tracker,track的n_extend参数配合使用 @param exts2 - 扩展属性: 独立于卡尔曼滤波器，对扩展通道单独处理 - @param binding_obj - 绑定原始目标检测的序号，方便数据源跟踪 + @param binding_obj - 绑定原始目标检测的序号(或结构体对象)，方便数据源跟踪 @param flag - 跟踪目标标记， 如用于目标类别，由外部解析 ''' self.tlwh = np.asarray(tlwh, dtype=np.float) + self.t = t self.exts1 = exts1 self.exts2 = exts2 self.binding_obj = binding_obj diff --git a/deep_sort_ex/kalman_filter.py b/deep_sort_ex/kalman_filter.py index ce23ed79..3088dfc1 100644 --- a/deep_sort_ex/kalman_filter.py +++ b/deep_sort_ex/kalman_filter.py @@ -46,10 +46,10 @@ def __init__(self, n_extend=0): self.n_extend = n_extend # Create Kalman filter model matrices. - self._motion_mat = np.eye(2 * ndim, 2 * ndim) + self._motion_mat = np.eye(2 * ndim, 2 * ndim) # 预测矩阵F for i in range(ndim): self._motion_mat[i, ndim + i] = dt - self._update_mat = np.eye(ndim, 2 * ndim) + self._update_mat = np.eye(ndim, 2 * ndim) # # Motion and observation uncertainty are chosen relative to the current # state estimate. These weights control the amount of uncertainty in @@ -133,9 +133,9 @@ def predict(self, mean, covariance): motion_cov = np.diag(np.square(np.r_[std_pos, std_vel])) - mean = np.dot(self._motion_mat, mean) + mean = np.dot(self._motion_mat, mean) # x = F*x covariance = np.linalg.multi_dot(( - self._motion_mat, covariance, self._motion_mat.T)) + motion_cov + self._motion_mat, covariance, self._motion_mat.T)) + motion_cov # P = F*P*F' + Q return mean, covariance @@ -165,10 +165,10 @@ def project(self, mean, covariance): std.append(self._std_weight_position * mean[3]) innovation_cov = np.diag(np.square(std)) - mean = np.dot(self._update_mat, mean) + mean = np.dot(self._update_mat, mean) # x = H * x covariance = np.linalg.multi_dot(( - self._update_mat, covariance, self._update_mat.T)) - return mean, covariance + innovation_cov + self._update_mat, covariance, self._update_mat.T)) # P = H*P*H' + R + return mean, covariance + innovation_cov # def update(self, mean, covariance, measurement): """Run Kalman filter correction step. diff --git a/deep_sort_ex/track.py b/deep_sort_ex/track.py index a43561f7..7332936d 100644 --- a/deep_sort_ex/track.py +++ b/deep_sort_ex/track.py @@ -299,7 +299,8 @@ def __init__(self, mean, covariance, track_id, n_init, max_age, ''' 扩展属性 ----- - @param n_extend - mean扩展属性数目 + @param feature - [list like] 目标特征向量 + @param binding_obj - [int or object] 绑定对象 @param filter_type - [Track] exts2滤波器类型 @param q_size - [Track] 队列长度 @param std_th - [Track] 相对误差域值 @@ -328,7 +329,10 @@ def __init__(self, mean, covariance, track_id, n_init, max_age, self._max_age = max_age self.binding_obj = binding_obj self.objFilter = Filter(filter_type=filter_type, q_size=q_size, std_th=std_th, Q=Q, R=R, fs=fs, cutoff=cutoff, order=order) - self.exts2 = None + self.exts2_prev = None # 上一帧扩展信息 + self.exts2 = None # 当前帧扩展信息 + self.t_prev = None # 上一帧时间(秒) + self.t = None # 当前帧时间(秒) self.save_to = save_to if not save_to is None: import os @@ -369,6 +373,9 @@ def get_exts1(self): def get_exts2(self): return np.array(self.exts2) + def get_exts2_prev(self): + return np.array(self.exts2_prev) + def predict(self, kf): """Propagate the state distribution to the current time step using a Kalman filter prediction step. @@ -417,7 +424,14 @@ def update(self, kf, detection, save_to=None): self.mean, self.covariance, detection.to_xyah()) self.features.append(detection.feature) self.objFilter.filter(detection) + + if not self.exts2 is None: + self.exts2_prev = self.exts2.copy() + self.t_prev = self.t + self.exts2 = detection.exts2 + self.t = detection.t + # 数据采集: 滤波后 # --------------- diff --git a/deep_sort_ex/tracker.py b/deep_sort_ex/tracker.py index 816144e6..cd7003ad 100644 --- a/deep_sort_ex/tracker.py +++ b/deep_sort_ex/tracker.py @@ -172,5 +172,5 @@ def _initiate_track(self, detection): mean, covariance = self.kf.initiate(detection.to_xyah()) self.tracks.append(Track( mean, covariance, self._next_id, self.n_init, self.max_age, - detection.feature, binding_obj=detection.binding_obj, filter_type=self.filter_type, q_size=self.q_size, std_th=self.std_th, percent=self.percent, Q=self.Q, R=self.R, fs=self.fs, cutoff=self.cutoff, order=self.order, save_to=self.save_to)) + feature=detection.feature, binding_obj=detection.binding_obj, filter_type=self.filter_type, q_size=self.q_size, std_th=self.std_th, percent=self.percent, Q=self.Q, R=self.R, fs=self.fs, cutoff=self.cutoff, order=self.order, save_to=self.save_to)) self._next_id += 1 From 3ba5d0469fba1c49a596536c9b885b997cfa4586 Mon Sep 17 00:00:00 2001 From: wilsonwnog2014 Date: Thu, 17 Sep 2020 18:55:30 +0800 Subject: [PATCH 22/24] update --- deep_sort_ex/track.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deep_sort_ex/track.py b/deep_sort_ex/track.py index 7332936d..0711b431 100644 --- a/deep_sort_ex/track.py +++ b/deep_sort_ex/track.py @@ -59,7 +59,7 @@ def __init__(self, N=4, std_th=0.1, percent=0.8): ''' @param N - 队列长度 @param std_th - 方差阈值 - @param percent - 奇异点保留前置能量比，当设为1.0即为完全用前置点替换奇异点 + @param percent - [弃用，用sigmod自适应替代]奇异点保留前置能量比，当设为1.0即为完全用前置点替换奇异点 ''' self.q_size = N self.std_th = std_th From 02e10de4c12aed36168ab0aa4c0e8302baa527e4 Mon Sep 17 00:00:00 2001 From: wilsonwnog2014 Date: Tue, 22 Sep 2020 19:22:03 +0800 Subject: [PATCH 23/24] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E6=B3=A8=E9=87=8A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- deep_sort_ex/kalman_filter.py | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/deep_sort_ex/kalman_filter.py b/deep_sort_ex/kalman_filter.py index 3088dfc1..90d05470 100644 --- a/deep_sort_ex/kalman_filter.py +++ b/deep_sort_ex/kalman_filter.py @@ -46,16 +46,16 @@ def __init__(self, n_extend=0): self.n_extend = n_extend # Create Kalman filter model matrices. - self._motion_mat = np.eye(2 * ndim, 2 * ndim) # 预测矩阵F + self._motion_mat = np.eye(2 * ndim, 2 * ndim) # 预测矩阵 F: 匀速模型 for i in range(ndim): self._motion_mat[i, ndim + i] = dt - self._update_mat = np.eye(ndim, 2 * ndim) # + self._update_mat = np.eye(ndim, 2 * ndim) # 传感器读数矩阵(测量矩阵) H # Motion and observation uncertainty are chosen relative to the current # state estimate. These weights control the amount of uncertainty in # the model. This is a bit hacky. - self._std_weight_position = 1. / 20 - self._std_weight_velocity = 1. / 160 + self._std_weight_position = 1. / 20 # 系统误差权重 + self._std_weight_velocity = 1. / 160 # 系统误差权重 def initiate(self, measurement): """Create track from unassociated measurement. @@ -131,11 +131,11 @@ def predict(self, mean, covariance): for i in range(self.n_extend): std_vel.append(self._std_weight_velocity * mean[3]) - motion_cov = np.diag(np.square(np.r_[std_pos, std_vel])) + motion_cov = np.diag(np.square(np.r_[std_pos, std_vel])) # 系统误差 Q - mean = np.dot(self._motion_mat, mean) # x = F*x + mean = np.dot(self._motion_mat, mean) # 状态预测: x = F*x covariance = np.linalg.multi_dot(( - self._motion_mat, covariance, self._motion_mat.T)) + motion_cov # P = F*P*F' + Q + self._motion_mat, covariance, self._motion_mat.T)) + motion_cov # 更新协方差矩阵 P = F*P*F' + Q return mean, covariance @@ -163,12 +163,12 @@ def project(self, mean, covariance): self._std_weight_position * mean[3]] for i in range(self.n_extend): std.append(self._std_weight_position * mean[3]) - innovation_cov = np.diag(np.square(std)) + innovation_cov = np.diag(np.square(std)) # 传感器噪声 R - mean = np.dot(self._update_mat, mean) # x = H * x + mean = np.dot(self._update_mat, mean) # 提取前面4个分量: x = H * x covariance = np.linalg.multi_dot(( - self._update_mat, covariance, self._update_mat.T)) # P = H*P*H' + R - return mean, covariance + innovation_cov # + self._update_mat, covariance, self._update_mat.T)) # P = H*P*H' + return mean, covariance + innovation_cov # P = H*P*H' + R def update(self, mean, covariance, measurement): """Run Kalman filter correction step. @@ -190,16 +190,18 @@ def update(self, mean, covariance, measurement): Returns the measurement-corrected state distribution. """ - projected_mean, projected_cov = self.project(mean, covariance) - + projected_mean, projected_cov = self.project(mean, covariance) # 投影到测量空间 + # Choresky分解: A=>L*L', projected_cov: P = H*P*H' + R chol_factor, lower = scipy.linalg.cho_factor( projected_cov, lower=True, check_finite=False) + # 求卡尔曼增益K: (H*P*H'+R)*K=P*H' kalman_gain = scipy.linalg.cho_solve( (chol_factor, lower), np.dot(covariance, self._update_mat.T).T, check_finite=False).T - innovation = measurement - projected_mean + innovation = measurement - projected_mean # 变化值 - new_mean = mean + np.dot(innovation, kalman_gain.T) + new_mean = mean + np.dot(innovation, kalman_gain.T) # 新的均值向量 mean + # 新的协方差矩阵 P new_covariance = covariance - np.linalg.multi_dot(( kalman_gain, projected_cov, kalman_gain.T)) return new_mean, new_covariance From c437d01e491832ca2fb7f3e1b7c59062b68c7f55 Mon Sep 17 00:00:00 2001 From: wilsonwnog2014 Date: Fri, 16 Oct 2020 19:08:27 +0800 Subject: [PATCH 24/24] =?UTF-8?q?=E4=BB=A3=E7=A0=81=E6=95=B4=E7=90=86?= =?UTF-8?q?=EF=BC=8C=E5=88=A0=E9=99=A4=E5=A4=9A=E4=BD=99=E5=8F=82=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- deep_sort_ex/track.py | 42 +++++++++++++++++++++-------------------- deep_sort_ex/tracker.py | 7 ++----- 2 files changed, 24 insertions(+), 25 deletions(-) diff --git a/deep_sort_ex/track.py b/deep_sort_ex/track.py index 0711b431..25c5f9d1 100644 --- a/deep_sort_ex/track.py +++ b/deep_sort_ex/track.py @@ -19,16 +19,6 @@ def filter(self, data): ''' @param data - np.array: channels x data_len ''' - #for i in range(data.shape[1]): - # if np.isnan(data[:, i]).any(): - # data[:, i] = np.array(self.q.queue).mean(axis=0) - # self.q.put(data[:, i]) - # else: - # self.q.put(data[:, i]) - # data[:, i] = np.array(self.q.queue).mean(axis=0) - # if self.q.qsize()>=self.N: - # self.q.get() - for i in range(data.shape[1]): if np.isnan(data[:, i]).any() or (data[:, i]==9999.0).any(): if self.q.qsize()>0: @@ -295,7 +285,7 @@ class Track: """ def __init__(self, mean, covariance, track_id, n_init, max_age, - feature=None, binding_obj=None, filter_type=0, q_size=4, std_th=0.05, percent=0.8, Q=1e-6, R=4e-4, fs=5., cutoff=1., order=5, save_to=None): + feature=None, binding_obj=None, filter_type=0, q_size=4, std_th=0.05, percent=0.8, Q=1e-6, R=4e-4, fs=5., cutoff=1., order=5): ''' 扩展属性 ----- @@ -310,7 +300,6 @@ def __init__(self, mean, covariance, track_id, n_init, max_age, @param fs - [Track] 采样率 @param cutoff - [Track] 截止频率, Hz @param order - [Track] 滤波器阶数 - @param save_to - [Track] 采集数据保存目录 ''' self.mean = mean @@ -333,10 +322,7 @@ def __init__(self, mean, covariance, track_id, n_init, max_age, self.exts2 = None # 当前帧扩展信息 self.t_prev = None # 上一帧时间(秒) self.t = None # 当前帧时间(秒) - self.save_to = save_to - if not save_to is None: - import os - os.makedirs(save_to, exist_ok=True) + def to_tlwh(self): """Get current position in bounding box format `(top left x, top left y, @@ -404,13 +390,11 @@ def update(self, kf, detection, save_to=None): """ # 数据采集: 滤波前 # --------------- - if not save_to is None: + if False and not save_to is None: save_file_origin_mean = '%s/mean/%s-%d/origin.txt' % (save_to, detection.flag, self.track_id) save_file_origin_ext2 = '%s/exts2/%s-%d/origin.txt' % (save_to, detection.flag, self.track_id) os.makedirs(os.path.dirname(save_file_origin_mean), exist_ok=True) os.makedirs(os.path.dirname(save_file_origin_ext2), exist_ok=True) - #print('save_file_origin_mean: ', save_file_origin_mean) - #print('save_file_origin_ext2: ', save_file_origin_ext2) with open(save_file_origin_mean, 'a+') as f: f.write(str(list(detection.to_xyah()))[1:-1]) f.write('\n') @@ -418,6 +402,13 @@ def update(self, kf, detection, save_to=None): f.write(str(list(detection.exts2))[1:-1]) f.write('\n') + if not save_to is None: + save_file = '%s/%s_%d/origin.txt' % (save_to, detection.flag, self.track_id) + os.makedirs(os.path.dirname(save_file), exist_ok=True) + with open(save_file, 'a+') as f: + f.write(str(list(np.r_[self.to_tlbr(), detection.exts2]))[1:-1]) + f.write('\n') + # 跟踪/滤波处理 # ------------ self.mean, self.covariance = kf.update( @@ -435,7 +426,11 @@ def update(self, kf, detection, save_to=None): # 数据采集: 滤波后 # --------------- - if not save_to is None: + if False and not save_to is None: + t, l, b, r = self.to_tlbr() + t, l, b, r = int(t), int(l), int(b), int(r) + #save_file_filter_mean = '%s/mean/%s-%d-%d-%d-%d-%d/filter.txt' % (save_to, detection.flag, self.track_id, t, l, b, r) + #save_file_filter_ext2 = '%s/exts2/%s-%d-%d-%d-%d-%d/filter.txt' % (save_to, detection.flag, self.track_id, t, l, b, r) save_file_filter_mean = '%s/mean/%s-%d/filter.txt' % (save_to, detection.flag, self.track_id) save_file_filter_ext2 = '%s/exts2/%s-%d/filter.txt' % (save_to, detection.flag, self.track_id) os.makedirs(os.path.dirname(save_file_filter_mean), exist_ok=True) @@ -449,6 +444,13 @@ def update(self, kf, detection, save_to=None): f.write(str(list(self.exts2))[1:-1]) f.write('\n') + if not save_to is None: + save_file = '%s/%s_%d/filter.txt' % (save_to, detection.flag, self.track_id) + os.makedirs(os.path.dirname(save_file), exist_ok=True) + with open(save_file, 'a+') as f: + f.write(str(list(np.r_[self.to_tlbr(), detection.exts2]))[1:-1]) + f.write('\n') + # 状态更新 # -------- self.hits += 1 diff --git a/deep_sort_ex/tracker.py b/deep_sort_ex/tracker.py index cd7003ad..a73ac917 100644 --- a/deep_sort_ex/tracker.py +++ b/deep_sort_ex/tracker.py @@ -37,7 +37,7 @@ class Tracker: """ - def __init__(self, metric, max_iou_distance=0.7, max_age=30, n_init=3, n_extend=0, filter_type=0, q_size=4, std_th=0.05, percent=0.8, Q=1e-6, R=4e-4, fs=5., cutoff=1., order=5, save_to=None): + def __init__(self, metric, max_iou_distance=0.7, max_age=30, n_init=3, n_extend=0, filter_type=0, q_size=4, std_th=0.05, percent=0.8, Q=1e-6, R=4e-4, fs=5., cutoff=1., order=5): ''' 扩展属性 ----- @@ -51,7 +51,6 @@ def __init__(self, metric, max_iou_distance=0.7, max_age=30, n_init=3, n_extend= @param fs - [Track] 采样率 @param cutoff - [Track] 截止频率, Hz @param order - [Track] 滤波器阶数 - @param save_to - [Track] 采集数据保存目录 ''' self.metric = metric self.max_iou_distance = max_iou_distance @@ -68,7 +67,6 @@ def __init__(self, metric, max_iou_distance=0.7, max_age=30, n_init=3, n_extend= self.fs = fs self.cutoff = cutoff self.order = order - self.save_to = save_to # 采集数据保存目录 self.tracks = [] self._next_id = 1 @@ -129,7 +127,6 @@ def update(self, detections, save_to=None): def _match(self, detections): - def gated_metric(tracks, dets, track_indices, detection_indices): features = np.array([dets[i].feature for i in detection_indices]) targets = np.array([tracks[i].track_id for i in track_indices]) @@ -172,5 +169,5 @@ def _initiate_track(self, detection): mean, covariance = self.kf.initiate(detection.to_xyah()) self.tracks.append(Track( mean, covariance, self._next_id, self.n_init, self.max_age, - feature=detection.feature, binding_obj=detection.binding_obj, filter_type=self.filter_type, q_size=self.q_size, std_th=self.std_th, percent=self.percent, Q=self.Q, R=self.R, fs=self.fs, cutoff=self.cutoff, order=self.order, save_to=self.save_to)) + feature=detection.feature, binding_obj=detection.binding_obj, filter_type=self.filter_type, q_size=self.q_size, std_th=self.std_th, percent=self.percent, Q=self.Q, R=self.R, fs=self.fs, cutoff=self.cutoff, order=self.order)) self._next_id += 1