moonq/tsmark
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

changed data format

Browse Source
This commit is contained in:
q
2025-06-27 14:53:40 +03:00
parent d7d2e7549e
commit 0ab46e4af2
2 changed files with 377 additions and 226 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
tsmark/__init__.py
View File

@@ -40,6 +40,15 @@ def get_options():
required=False, required=False,
help="Load points from a JSON file", help="Load points from a JSON file",
) )
parser.add_argument(
"--max-track",
action="store",
dest="max_track",
type=float,
default=4,
required=False,
help="Length of tracking segment in seconds: %(default)s",
)
parser.add_argument( parser.add_argument(
"--fps", "--fps",
action="store", action="store",

564
tsmark/video_annotator.py
View File

@@ -3,6 +3,7 @@ import os
import shlex import shlex
import subprocess import subprocess
import sys import sys
import threading
import time import time
import cv2 import cv2
@@ -37,7 +38,7 @@ class Marker:
self.crop_click = 0 self.crop_click = 0
self.point_click = 0 self.point_click = 0
self.point_tracking = 0 self.point_tracking = 0
self.point_tracking_length = 4 self.point_tracking_length = float(self.opts.max_track)
self.points = {} self.points = {}
self.points_interpolated = {} self.points_interpolated = {}
self.point_index = None self.point_index = None
@@ -45,6 +46,9 @@ class Marker:
self.message = None self.message = None
self.message_timer = time.time() self.message_timer = time.time()
self.autosave_interval = 60
self.autosave_timer = time.time()
self.forced_fps = opts.fps self.forced_fps = opts.fps
try: try:
@@ -234,19 +238,22 @@ class Marker:
continue continue
current = self.get_interpolated_point(index=index) current = self.get_interpolated_point(index=index)
if current[5] == 0: if current["type"] in ("pre", "post"):
continue continue
color = (0, 192, 192) color = (0, 192, 192)
if current[5] == 2: if current["type"] == "key":
color = (60, 205, 60) color = (60, 205, 60)
if current[5] == 1: if current["type"] == "interp":
color = (192, 0, 192) color = (192, 0, 192)
cv2.circle(frame, (current[6], current[7]), 10, (0, 0, 0), 2) if current["visible"] == 0:
cv2.circle(frame, (current[6], current[7]), 10, color, 1) color = (96, 96, 96)
continue
cv2.circle(frame, (current["cx"], current["cy"]), 10, (0, 0, 0), 2)
cv2.circle(frame, (current["cx"], current["cy"]), 10, color, 1)
self.shadow_text( self.shadow_text(
frame, frame,
index, index,
(current[6], current[7]), (current["cx"], current["cy"]),
0.5, 0.5,
1, 1,
color, color,
@@ -261,89 +268,109 @@ class Marker:
frame, (0, self.mouse_position[1]), (self.video_res[0], self.mouse_position[1]), (128, 128, 128), 1 frame, (0, self.mouse_position[1]), (self.video_res[0], self.mouse_position[1]), (128, 128, 128), 1
) )
# Show current track # Show current track
x, y = [self.video_res[0] - 120, 50] x, y = [20, 70]
self.shadow_text( self.shadow_text(
frame, frame,
"Points: " + str(self.point_index), "P:" + str(self.point_index),
(x, y), (x, y),
0.5, 0.5,
1, 1,
(0, 192, 192), (255, 255, 255),
) )
try: try:
current = self.get_interpolated_point() # self.points_interpolated[self.point_index][self.nr] current = self.get_interpolated_point()
if current["type"] is not None:
color = (0, 192, 192) color = (0, 192, 192)
if current[5] == 2: if current["type"] == "key":
color = (60, 205, 60) color = (60, 205, 60)
if current[5] == 1: if current["type"] == "interp":
color = (192, 0, 192) color = (192, 0, 192)
if current["visible"] == 0:
color = (96, 96, 96)
cv2.rectangle( cv2.rectangle(
frame, frame,
(current[1], current[2]), (current["x0"], current["y0"]),
(current[3], current[4]), (current["x1"], current["y1"]),
color, color,
2, 2,
) )
cv2.circle(frame, (current[6], current[7]), 10, color, 1)
history = list(range(max(1, int(self.nr - self.viewer_fps)), self.nr + 1)) cv2.circle(frame, (current["cx"], current["cy"]), 10, color, 1)
for p in history:
current = self.get_interpolated_point(p) history = []
past = self.get_interpolated_point(p - 1) for p in range(max(1, int(self.nr - self.viewer_fps)), self.nr + 1):
cv2.line( po = self.get_interpolated_point(p)
frame, history.append([po["cx"], po["cy"]])
(past[6], past[7]), history = np.array(history, np.int32).reshape((-1, 1, 2))
(current[6], current[7]), cv2.polylines(frame, [history], False, (192, 0, 192), 1)
(192, 0, 192),
1,
)
except KeyError: except KeyError:
print(current, self.nr)
pass pass
except IndexError: except IndexError:
print(current, self.nr) print(current, self.nr)
pass pass
try: try:
# ~ point_keys = list(sorted(self.points[self.point_index].keys())) current = self.get_point()
# current = self.points[self.point_index][self.nr] if current["x0"] is not None:
current = self.get_point() # self.points_interpolated[self.point_index][self.nr] cv2.circle(frame, (current["cx"], current["cy"]), 13, (60, 205, 60), 2)
color = (60, 205, 60)
cv2.circle(frame, (current[4], current[5]), 13, color, 2)
except KeyError: except KeyError:
pass pass
except IndexError: except IndexError:
# ~ print(self.points[self.point_index]) print(self.points[self.point_index])
# ~ print(self.nr) print(self.nr)
pass pass
def scan_point(self, direction): def scan_point(self, direction):
def set_nr(ts):
self.nr = ts - 1
self.read_next = True
try: try:
if direction == "first":
return set_nr(min(list(self.points[self.point_index].keys())))
if direction == "last":
return set_nr(max(list(self.points[self.point_index].keys())))
if direction == "next": if direction == "next":
for ts in sorted(list(self.points[self.point_index].keys())): for ts in sorted(list(self.points[self.point_index].keys())):
if ts > self.nr: if ts > self.nr:
self.nr = ts - 1 return set_nr(ts)
self.read_next = True
return
if direction == "previous": if direction == "previous":
for ts in reversed(sorted(list(self.points[self.point_index].keys()))): for ts in reversed(sorted(list(self.points[self.point_index].keys()))):
if ts < self.nr - 1: if ts < self.nr - 1:
self.nr = ts - 1 return set_nr(ts)
self.read_next = True
return
except Exception: except Exception:
pass pass
def del_point(self, ts): def toggle_point(self, ts):
try: try:
if ts in self.points[self.point_index]:
# Remove point
del self.points[self.point_index][ts] del self.points[self.point_index][ts]
else:
# Introduce point from interpolated
ip = self.get_interpolated_point()
if ip["type"] is None:
return
{"x0": None, "y0": None, "x1": None, "y1": None, "cx": None, "cy": None, "visible": 0, "type": None}
self.points[self.point_index][self.nr] = {
"x0": ip["x0"],
"y0": ip["y0"],
"x1": ip["x1"],
"y1": ip["y1"],
"visible": 1,
}
self.interpolate_points() self.interpolate_points()
except Exception: except Exception:
pass pass
def get_point(self, nr=None, index=None): def get_point(self, nr=None, index=None):
"""[x,y,x2,y2, cx, cy, w, h]""" """{x0,y0,x1,y1, cx, cy, w, h, visible}"""
if nr is None: if nr is None:
nr = self.nr nr = self.nr
if index is None: if index is None:
@@ -351,105 +378,130 @@ class Marker:
if index in self.points: if index in self.points:
if nr in self.points[index]: if nr in self.points[index]:
return [ value = self.points[index][nr].copy()
*self.points[index][nr], value.update(
int((self.points[index][nr][0] + self.points[index][nr][2]) / 2), {
int((self.points[index][nr][1] + self.points[index][nr][3]) / 2), "cx": int((value["x0"] + value["x1"]) / 2),
int(abs(self.points[index][nr][0] - self.points[index][nr][2])), "cy": int((value["y0"] + value["y1"]) / 2),
int(abs(self.points[index][nr][1] - self.points[index][nr][3])), "w": int(abs(value["x0"] - value["x1"])),
] "h": int(abs(value["y0"] - value["y1"])),
}
)
return value
return [None, None, None, None, None, None] return {
"x0": None,
"y0": None,
"x1": None,
"y1": None,
"cx": None,
"cy": None,
"w": None,
"h": None,
"visible": 0,
}
def get_interpolated_point(self, nr=None, index=None): def get_interpolated_point(self, nr=None, index=None):
"""{x0,y0,x1,y1, cx, cy, visible,type}"""
if nr is None: if nr is None:
nr = self.nr nr = self.nr
if index is None: if index is None:
index = self.point_index index = self.point_index
if index in self.points: if index in self.points_interpolated:
if nr in self.points_interpolated[index]: if nr in self.points_interpolated[index]:
return [ value = self.points_interpolated[index][nr].copy()
*self.points_interpolated[index][nr], value.update(
int((self.points_interpolated[index][nr][1] + self.points_interpolated[index][nr][3]) / 2), {
int((self.points_interpolated[index][nr][2] + self.points_interpolated[index][nr][4]) / 2), "cx": int((value["x0"] + value["x1"]) / 2),
] "cy": int((value["y0"] + value["y1"]) / 2),
}
)
return value
return [None, None, None, None, None, None, None, None] return {"x0": None, "y0": None, "x1": None, "y1": None, "cx": None, "cy": None, "visible": 0, "type": None}
def modify_point(self, position, x, y): def modify_point(self, position, x, y):
"""position: tl topleft, br bottomright, c center""" """position: tl topleft, br bottomright, c center"""
if position == "tl":
ix = 0
iy = 1
if position == "br":
ix = 2
iy = 3
if not self.point_index in self.points: if not self.point_index in self.points:
self.points[self.point_index] = {} self.points[self.point_index] = {}
if not self.nr in self.points[self.point_index]: if not self.nr in self.points[self.point_index]:
if len(self.points[self.point_index]) > 0: if len(self.points[self.point_index]) > 0:
keys = sorted(list(self.points[self.point_index].keys())) keys = sorted(list(self.points[self.point_index].keys()))
if self.nr > keys[-1]: if self.nr > keys[-1]: # last point if at end of track
last_p = self.points[self.point_index][keys[-1]] last_p = self.points[self.point_index][keys[-1]]
elif self.nr < keys[0]: elif self.nr < keys[0]: # first point if before track
last_p = self.points[self.point_index][keys[0]] last_p = self.points[self.point_index][keys[0]]
else: else: # previous point if in the middle of track
prev_key = keys[0] prev_key = keys[0]
for key in keys: for key in keys:
if key > self.nr: if key > self.nr:
last_p = self.points[self.point_index][prev_key] last_p = self.points[self.point_index][prev_key]
break break
prev_key = key prev_key = key
w = abs(last_p[2] - last_p[0]) w = abs(last_p["x1"] - last_p["x0"])
h = abs(last_p[3] - last_p[1]) h = abs(last_p["y1"] - last_p["y0"])
else: else:
w = 50 w = 50
h = 50 h = 50
if position == "tl": if position == "tl":
self.points[self.point_index][self.nr] = [ self.points[self.point_index][self.nr] = {
x, "x0": x,
y, "y0": y,
min(self.video_res[0] - 1, x + w), "x1": min(self.video_res[0] - 1, x + w),
min(self.video_res[1] - 1, y + h), "y1": min(self.video_res[1] - 1, y + h),
] "visible": 1,
}
if position == "br": if position == "br":
self.points[self.point_index][self.nr] = [max(0, x - w), max(0, y - h), x, y] self.points[self.point_index][self.nr] = {
"x0": max(0, x - w),
"y0": max(0, y - h),
"x1": x,
"y1": y,
"visible": 1,
}
if position == "c": if position == "c":
self.points[self.point_index][self.nr] = [ self.points[self.point_index][self.nr] = {
max(0, int(x - w / 2)), "x0": max(0, int(x - w / 2)),
max(0, int(y - h / 2)), "y0": max(0, int(y - h / 2)),
min(self.video_res[0] - 1, int(x + w / 2)), "x1": min(self.video_res[0] - 1, int(x + w / 2)),
min(self.video_res[1] - 1, int(y + h / 2)), "y1": min(self.video_res[1] - 1, int(y + h / 2)),
] "visible": 1,
}
else: else:
# not a new point # not a new point
self.points[self.point_index][self.nr]["visible"] = 1
if position == "c": if position == "c":
current = self.points[self.point_index][self.nr] current = self.points[self.point_index][self.nr]
w = abs(current[2] - current[0]) w = abs(current["x1"] - current["x0"])
h = abs(current[3] - current[1]) h = abs(current["y1"] - current["y0"])
self.points[self.point_index][self.nr] = [ self.points[self.point_index][self.nr] = {
max(0, int(x - w / 2)), "x0": max(0, int(x - w / 2)),
max(0, int(y - h / 2)), "y0": max(0, int(y - h / 2)),
min(self.video_res[0] - 1, int(x + w / 2)), "x1": min(self.video_res[0] - 1, int(x + w / 2)),
min(self.video_res[1] - 1, int(y + h / 2)), "y1": min(self.video_res[1] - 1, int(y + h / 2)),
] "visible": 1,
else: }
self.points[self.point_index][self.nr][ix] = x elif position == "tl":
self.points[self.point_index][self.nr][iy] = y self.points[self.point_index][self.nr]["x0"] = x
self.points[self.point_index][self.nr]["y0"] = y
if self.points[self.point_index][self.nr][0] > self.points[self.point_index][self.nr][2]: elif position == "br":
self.points[self.point_index][self.nr][2], self.points[self.point_index][self.nr][0] = ( self.points[self.point_index][self.nr]["x1"] = x
self.points[self.point_index][self.nr][0], self.points[self.point_index][self.nr]["y1"] = y
self.points[self.point_index][self.nr][2],
if self.points[self.point_index][self.nr]["x0"] > self.points[self.point_index][self.nr]["x1"]:
self.points[self.point_index][self.nr]["x1"], self.points[self.point_index][self.nr]["x0"] = (
self.points[self.point_index][self.nr]["x0"],
self.points[self.point_index][self.nr]["x1"],
) )
if self.points[self.point_index][self.nr][1] > self.points[self.point_index][self.nr][3]: if self.points[self.point_index][self.nr]["y0"] > self.points[self.point_index][self.nr]["y1"]:
self.points[self.point_index][self.nr][3], self.points[self.point_index][self.nr][1] = ( self.points[self.point_index][self.nr]["y1"], self.points[self.point_index][self.nr]["y0"] = (
self.points[self.point_index][self.nr][1], self.points[self.point_index][self.nr]["y0"],
self.points[self.point_index][self.nr][3], self.points[self.point_index][self.nr]["y1"],
) )
self.interpolate_points() self.interpolate_points()
@@ -462,17 +514,32 @@ class Marker:
if self.opts.output_points is None: if self.opts.output_points is None:
return return
curr_point = self.get_point() curr_point = self.get_point()
if curr_point[0] is None: if curr_point["x0"] is None:
self.add_message("Not in point frame (green)") self.add_message("Not in point frame (green)")
return return
new_wh = abs(self.mouse_position[0] - curr_point[4]) new_wh = abs(self.mouse_position[0] - curr_point["cx"])
new_hh = abs(self.mouse_position[1] - curr_point[5]) new_hh = abs(self.mouse_position[1] - curr_point["cy"])
x1 = int(curr_point[4] - new_wh) self.points[self.point_index][self.nr]["x0"] = int(curr_point["cx"] - new_wh)
y1 = int(curr_point[5] - new_hh) self.points[self.point_index][self.nr]["y0"] = int(curr_point["cy"] - new_hh)
x2 = int(curr_point[4] + new_wh) self.points[self.point_index][self.nr]["x1"] = int(curr_point["cx"] + new_wh)
y2 = int(curr_point[5] + new_hh) self.points[self.point_index][self.nr]["y1"] = int(curr_point["cy"] + new_hh)
self.points[self.point_index][self.nr] = [x1, y1, x2, y2] self.points[self.point_index][self.nr]["visible"] = 1
self.interpolate_points()
def toggle_point_visibility(self):
if self.point_click == 0:
self.add_message("Not in point clicking mode")
return
if self.opts.output_points is None:
return
curr_point = self.get_point()
if curr_point["x0"] is None:
self.add_message("Not in point frame (green)")
return
self.points[self.point_index][self.nr]["visible"] = 1 - self.points[self.point_index][self.nr]["visible"]
self.interpolate_points() self.interpolate_points()
def track_point(self): def track_point(self):
@@ -488,60 +555,77 @@ class Marker:
for nr in tracker_gui.points: for nr in tracker_gui.points:
self.points[self.point_index][nr] = tracker_gui.points[nr] self.points[self.point_index][nr] = tracker_gui.points[nr]
self.interpolate_points() self.interpolate_points()
self.nr = max(tracker_gui.points)-1 self.nr = max(tracker_gui.points) - 1
self.read_next = True self.read_next = True
def interpolate_points(self): def interpolate_points(self):
"""types:
key: user clicked / accepted frame
interp: interpolated frame
pre: before any keyframes
post: after any keyframes
"""
def i_point(x0=None, y0=None, x1=None, y1=None, t=None, visible=None):
return {"x0": x0, "y0": y0, "x1": x1, "y1": y1, "type": t, "visible": visible}
def point2array(p):
return [p["x0"], p["y0"], p["x1"], p["y1"]]
if not self.point_index in self.points_interpolated: if not self.point_index in self.points_interpolated:
self.points_interpolated[self.point_index] = {key: [] for key in range(self.frames)} self.points_interpolated[self.point_index] = {key: {} for key in range(self.frames)}
if len(self.points[self.point_index]) == 1: if len(self.points[self.point_index]) == 1: # only one point added
key = list(self.points[self.point_index].keys())[0] key = list(self.points[self.point_index].keys())[0]
x, y, x2, y2 = self.points[self.point_index][key] vals = self.points[self.point_index][key]
# x, y, x2, y2 = self.points[self.point_index][key]
for key in range(self.frames): for key in range(self.frames):
self.points_interpolated[self.point_index][key] = [False, int(x), int(y), x2, y2, 0] self.points_interpolated[self.point_index][key] = i_point()
self.points_interpolated[self.point_index][self.nr][5] = 2 self.points_interpolated[self.point_index][key].update(vals)
self.points_interpolated[self.point_index][key]["type"] = "pre"
self.points_interpolated[self.point_index][self.nr]["type"] = "key"
else: # more points else: # more points
point_keys = list(sorted(list(self.points[self.point_index].keys()))) point_keys = list(sorted(list(self.points[self.point_index].keys())))
point_values = [self.points[self.point_index][k] for k in point_keys] point_values = [point2array(self.points[self.point_index][k]) for k in point_keys]
xyxy = np.array(point_values).T xyxy = np.array(point_values).T
spline = PchipInterpolator(point_keys, xyxy, axis=1) spline = PchipInterpolator(point_keys, xyxy, axis=1)
start_key = min(point_keys) start_key = min(point_keys)
end_key = max(point_keys) + 1 end_key = max(point_keys) + 1
t2 = np.arange(start_key, end_key) t2 = np.arange(start_key, end_key)
# Pre points
for key in range(0, start_key): for key in range(0, start_key):
self.points_interpolated[self.point_index][key] = [ self.points_interpolated[self.point_index][key]["type"] = "pre"
False, self.points_interpolated[self.point_index][key].update(self.points[self.point_index][start_key])
self.points[self.point_index][start_key][0],
self.points[self.point_index][start_key][1],
self.points[self.point_index][start_key][2],
self.points[self.point_index][start_key][3],
0,
]
# interpolated points # interpolated points
visible = self.points[self.point_index][start_key]["visible"]
for row in np.vstack((t2, spline(t2))).T: for row in np.vstack((t2, spline(t2))).T:
self.points_interpolated[self.point_index][row[0]] = [ if row[0] in point_keys:
True, visible = self.points[self.point_index][row[0]]["visible"]
int(row[1]), self.points_interpolated[self.point_index][row[0]] = {
int(row[2]), "type": "interp",
int(row[3]), "x0": int(row[1]),
int(row[4]), "y0": int(row[2]),
1, "x1": int(row[3]),
] "y1": int(row[4]),
"visible": visible,
}
# post points
for key in range(end_key, self.frames + 1): for key in range(end_key, self.frames + 1):
self.points_interpolated[self.point_index][key] = [ self.points_interpolated[self.point_index][key] = {
False, "type": "post",
int(row[1]), "x0": int(row[1]),
int(row[2]), "y0": int(row[2]),
int(row[3]), "x1": int(row[3]),
int(row[4]), "y1": int(row[4]),
3, "visible": visible,
] }
# clicked points (not necessary, could determine at draw time!) # clicked points (not necessary, could determine at draw time!)
for key in point_keys: for key in point_keys:
self.points_interpolated[self.point_index][key][5] = 2 self.points_interpolated[self.point_index][key]["type"] = "key"
def draw_help(self, frame): def draw_help(self, frame):
@@ -598,7 +682,7 @@ class Marker:
def get_help(self): def get_help(self):
return """Keyboard help: return """Keyboard help:
(Note: after mouse click, arrows stop working due to unknown bug: use j,l,i,k)
Arrows, PgUp, PgDn, Home, End or click mouse in position bar Arrows, PgUp, PgDn, Home, End or click mouse in position bar
j l i k [ ] j l i k [ ]
jump in video position jump in video position
@@ -610,11 +694,27 @@ class Marker:
space or click video space or click video
pause pause
a and s modify crop offset or size a and s modify crop offset or size
p toggle bounding box drawing. follow with any key as index. left/middle/right mouse sets box position
f toggle 0.25x 1x or 4x FPS f toggle 0.25x 1x or 4x FPS
v toggle HUD v toggle HUD
h toggle help h toggle help
q or esc quit q or esc quit
Bounding box editor:
p toggle bounding box drawing. follow with any key as index.
o toggle object is occluded
x toggle (delete) key frame
mouse left: set top-left corner of box
mouse middle: set center of box
mouse right: set lower right corner of box
e set width/height of box symmetric around center
z c Home End move between key-frames
t start optical flow tracker
Color codes:
green keyframe
purple interpolated frame
gray object is occluded
""" """
def mouse_click(self, event, x, y, flags, param): def mouse_click(self, event, x, y, flags, param):
@@ -713,10 +813,13 @@ class Marker:
self.point_index = index self.point_index = index
self.points[index] = {int(k): v for k, v in self.points[index].items()} self.points[index] = {int(k): v for k, v in self.points[index].items()}
for key in self.points[index]: for key in self.points[index]:
self.points[index][key] = [ self.points[index][key]["x0"], self.points[index][key]["y0"] = self.original_to_visual(
*self.original_to_visual((self.points[index][key][0], self.points[index][key][1])), (self.points[index][key]["x0"], self.points[index][key]["y0"])
*self.original_to_visual((self.points[index][key][2], self.points[index][key][3])), )
] self.points[index][key]["x1"], self.points[index][key]["y1"] = self.original_to_visual(
(self.points[index][key]["x1"], self.points[index][key]["y1"])
)
self.interpolate_points() self.interpolate_points()
print(f"Loaded points with index: {index}") print(f"Loaded points with index: {index}")
self.point_index = None self.point_index = None
@@ -795,10 +898,13 @@ class Marker:
for index in self.points.keys(): for index in self.points.keys():
points[index] = {} points[index] = {}
for key in sorted(self.points[index].keys()): for key in sorted(self.points[index].keys()):
points[index][key] = [ points[index][key] = self.points[index][key].copy()
*self.visual_to_original((self.points[index][key][0], self.points[index][key][1])), points[index][key]["x0"], points[index][key]["y0"] = self.visual_to_original(
*self.visual_to_original((self.points[index][key][2], self.points[index][key][3])), (self.points[index][key]["x0"], self.points[index][key]["y0"])
] )
points[index][key]["x1"], points[index][key]["y1"] = self.visual_to_original(
(self.points[index][key]["x1"], self.points[index][key]["y1"])
)
with open(self.opts.output_points, "wt") as fp: with open(self.opts.output_points, "wt") as fp:
json.dump(points, fp, indent=2) json.dump(points, fp, indent=2)
@@ -863,13 +969,14 @@ class Marker:
FPS_modifier = 1 FPS_modifier = 1
FPS_modifiers = [0.25, 1, 4] FPS_modifiers = [0.25, 1, 4]
read_fails = 0
while self.video_reader.isOpened(): while self.video_reader.isOpened():
show_time = time.time() show_time = time.time()
if (not self.paused) or self.read_next: if (not self.paused) or self.read_next:
ret, frame = self.video_reader.read() ret, frame = self.video_reader.read()
if ret == True: if ret == True:
read_fails = 0
draw_wait = 200 if self.paused or ( self.paused and self.point_click == 0 ) else 1 draw_wait = 200 if self.paused or (self.paused and self.point_click == 0) else 1
if (not self.paused) or self.read_next: if (not self.paused) or self.read_next:
self.read_next = False self.read_next = False
@@ -890,6 +997,7 @@ class Marker:
break break
cv2.imshow("tsmark", frame_visu) cv2.imshow("tsmark", frame_visu)
k = cv2.waitKey(draw_wait) k = cv2.waitKey(draw_wait)
if k & 0xFF == ord("q") or k & 0xFF == 27: if k & 0xFF == ord("q") or k & 0xFF == 27:
break break
elif k & 0xFF == 32: # space elif k & 0xFF == 32: # space
@@ -897,10 +1005,16 @@ class Marker:
# Movement ================= # Movement =================
elif k & 0xFF == 80: # home key elif k & 0xFF == 80: # home key
if self.point_click == 1:
self.scan_point("first")
else:
self.nr = -1 self.nr = -1
self.read_next = True self.read_next = True
elif k & 0xFF == 87: # end key elif k & 0xFF == 87: # end key
if self.point_click == 1:
self.scan_point("last")
else:
self.nr = self.frames - 1 self.nr = self.frames - 1
self.paused = True self.paused = True
self.read_next = True self.read_next = True
@@ -977,6 +1091,10 @@ class Marker:
elif k & 0xFF == ord("s"): # toggle crop size elif k & 0xFF == ord("s"): # toggle crop size
self.crop_click = 0 if self.crop_click == 2 else 2 self.crop_click = 0 if self.crop_click == 2 else 2
self.crop[2] = True self.crop[2] = True
elif k & 0xFF == ord("o"): # toggle point visibility (occlusion)
if self.opts.output_points is None:
continue
self.toggle_point_visibility()
elif k & 0xFF == ord("p"): # toggle points elif k & 0xFF == ord("p"): # toggle points
if self.opts.output_points is None: if self.opts.output_points is None:
continue continue
@@ -985,10 +1103,10 @@ class Marker:
self.shadow_text( self.shadow_text(
frame_visu, frame_visu,
"Enter point index", "Enter point index",
(self.video_res[0] - 200, 50), (20, 70),
0.5, 0.9,
1, 2,
(0, 192, 192), (255, 255, 255),
) )
cv2.imshow("tsmark", frame_visu) cv2.imshow("tsmark", frame_visu)
@@ -1004,10 +1122,10 @@ class Marker:
(20, 70), (20, 70),
0.9, 0.9,
2, 2,
(255,255,255), (255, 255, 255),
) )
cv2.imshow("tsmark", frame_visu) cv2.imshow("tsmark", frame_visu)
entered_chars="" entered_chars = ""
while True: while True:
frame_query = frame_visu.copy() frame_query = frame_visu.copy()
self.shadow_text( self.shadow_text(
@@ -1016,7 +1134,7 @@ class Marker:
(20, 100), (20, 100),
0.9, 0.9,
2, 2,
(255,255,255), (255, 255, 255),
) )
cv2.imshow("tsmark", frame_query) cv2.imshow("tsmark", frame_query)
del frame_query del frame_query
@@ -1025,7 +1143,7 @@ class Marker:
break break
elif k2 & 0xFF == ord("g") or k2 & 0xFF == 13: elif k2 & 0xFF == ord("g") or k2 & 0xFF == 13:
try: try:
self.nr = int(entered_chars) -1 self.nr = int(entered_chars) - 1
except ValueError: except ValueError:
try: try:
self.nr = self.parse_time(entered_chars) self.nr = self.parse_time(entered_chars)
@@ -1045,7 +1163,6 @@ class Marker:
else: else:
pass pass
elif k & 0xFF == ord("t"): # tracking elif k & 0xFF == ord("t"): # tracking
self.track_point() self.track_point()
elif k & 0xFF == ord("e"): # point edit (width height) elif k & 0xFF == ord("e"): # point edit (width height)
@@ -1053,7 +1170,7 @@ class Marker:
elif k & 0xFF == ord("x"): # toggle ts elif k & 0xFF == ord("x"): # toggle ts
if self.point_click == 1: if self.point_click == 1:
self.del_point(self.nr) self.toggle_point(self.nr)
else: else:
self.toggle_stamp() self.toggle_stamp()
@@ -1078,11 +1195,23 @@ class Marker:
time.sleep(time_to_wait) time.sleep(time_to_wait)
else: else:
self.nr = self.frames - 2 self.nr = self.frames - 2 - read_fails
self.video_reader.set(cv2.CAP_PROP_POS_FRAMES, self.nr) self.video_reader.set(cv2.CAP_PROP_POS_FRAMES, self.nr)
read_fails += 1
if read_fails > self.frames:
self.nr = 0
self.open()
self.paused = True self.paused = True
self.read_next = True self.read_next = True
if time.time() > self.autosave_timer + self.autosave_interval:
self.autosave_timer = time.time()
try:
print("Autosave timestamps / points")
self.save_timestamps()
except Exception as e:
print(e)
self.video_reader.release() self.video_reader.release()
cv2.destroyAllWindows() cv2.destroyAllWindows()
self.print_timestamps() self.print_timestamps()
@@ -1093,28 +1222,41 @@ class TrackerGUI:
def __init__(self, marker): def __init__(self, marker):
self.marker = marker self.marker = marker
self.points = {}
try:
cv2.TrackerKCF_create()
except AttributeError:
marker.add_message("Tracking failed: missing opencv contrib")
return
self.start() self.start()
def start(self): def start(self):
old_nr = self.marker.nr old_nr = self.marker.nr
curr_point = self.marker.get_point() curr_point = self.marker.get_point()
if curr_point[0] is None: if curr_point["x0"] is None:
self.marker.add_message("Not in point frame (green)") self.marker.add_message("Not in point frame (green)")
return return
max_frames = int(min(self.marker.point_tracking_length * self.marker.fps, self.marker.frames - self.marker.nr - 1)) max_frames = int(
min(self.marker.point_tracking_length * self.marker.fps, self.marker.frames - self.marker.nr - 1)
)
cv2.namedWindow("tsmark - tracker", flags=cv2.WINDOW_AUTOSIZE | cv2.WINDOW_KEEPRATIO | cv2.WINDOW_GUI_NORMAL) cv2.namedWindow("tsmark - tracker", flags=cv2.WINDOW_AUTOSIZE | cv2.WINDOW_KEEPRATIO | cv2.WINDOW_GUI_NORMAL)
tracker = cv2.TrackerKCF_create() tracker = cv2.TrackerKCF_create()
self.marker.video_reader.set(cv2.CAP_PROP_POS_FRAMES, self.marker.nr) self.marker.video_reader.set(cv2.CAP_PROP_POS_FRAMES, self.marker.nr)
# TODO: track using original video resolution!
ok, frame = self.marker.video_reader.read() ok, frame = self.marker.video_reader.read()
frame = cv2.resize(frame.copy(), self.marker.video_res) frame = cv2.resize(frame.copy(), self.marker.video_res)
bbox = tuple([*curr_point[0:2], *curr_point[6:8]]) bbox = tuple([curr_point["x0"], curr_point["y0"], curr_point["w"], curr_point["h"]])
ok = tracker.init(frame, bbox) ok = tracker.init(frame, bbox)
visu_interval = 0.2
show_time = 0
show_message = ""
tracked = {} tracked = {}
tracked[0] = [*bbox, 1]
for i in range(max_frames): for i in range(max_frames):
# Read a new frame # Read a new frame
ok, frame = self.marker.video_reader.read() ok, frame = self.marker.video_reader.read()
@@ -1123,25 +1265,28 @@ class TrackerGUI:
break break
ok, bbox = tracker.update(frame) ok, bbox = tracker.update(frame)
# Draw bounding box # ~ print(f"Tracking... ({i}/{max_frames})")
if ok: if ok:
# Tracking success # Tracking success
tracked[i + 1] = [*bbox, 1]
show_message = f"Tracking... ({i}/{max_frames})"
else:
# Tracking failure
show_message = f"Tracking failure detected ({i}/{max_frames})"
bbox = None
if time.time() > show_time + visu_interval:
# Display result
if bbox is not None:
p1 = (int(bbox[0]), int(bbox[1])) p1 = (int(bbox[0]), int(bbox[1]))
p2 = (int(bbox[0] + bbox[2]), int(bbox[1] + bbox[3])) p2 = (int(bbox[0] + bbox[2]), int(bbox[1] + bbox[3]))
cv2.rectangle(frame, p1, p2, (255, 0, 0), 2, 1) cv2.rectangle(frame, p1, p2, (255, 0, 0), 2, 1)
tracked[i] = [*bbox,1] self.marker.shadow_text(frame, show_message, (100, 80), 0.75, 2, (255, 255, 255))
cv2.putText(frame, "Tracking...", (100, 80), cv2.FONT_HERSHEY_SIMPLEX, 0.75, (0, 0, 255), 2)
else:
# Tracking failure
cv2.putText(
frame, "Tracking failure detected", (100, 80), cv2.FONT_HERSHEY_SIMPLEX, 0.75, (0, 0, 255), 2
)
# Display result
cv2.imshow("tsmark - tracker", frame) cv2.imshow("tsmark - tracker", frame)
show_time = time.time()
# Exit if ESC pressed k = cv2.waitKey(1)
if cv2.waitKey(1) & 0xFF == ord("q"): # if press SPACE bar # break tracking if ESC pressed, q, space or enter
if k & 0xFF == ord("q") or k & 0xFF == 32 or k & 0xFF == 27 or k & 0xFF == 13:
break break
done = False done = False
@@ -1151,8 +1296,8 @@ class TrackerGUI:
while True: while True:
if done: if done:
break break
self.marker.video_reader.set(cv2.CAP_PROP_POS_FRAMES, self.marker.nr + 1) self.marker.video_reader.set(cv2.CAP_PROP_POS_FRAMES, self.marker.nr)
i = 0 i = -1
while True: while True:
show_time = time.time() show_time = time.time()
if done: if done:
@@ -1165,20 +1310,25 @@ class TrackerGUI:
frame_copy = frame.copy() frame_copy = frame.copy()
i += 1 i += 1
seek = False seek = False
self.marker.shadow_text(frame, f"Accept? ({i+1}/{max_frames})", (100, 80), 0.75, 2, (255, 255, 255))
cv2.putText(frame, f"Replay... {i}", (100, 80), cv2.FONT_HERSHEY_SIMPLEX, 0.75, (0, 0, 255), 2)
if i in tracked: if i in tracked:
bbox = tracked[i] bbox = tracked[i]
p1 = (int(bbox[0]), int(bbox[1])) p1 = (int(bbox[0]), int(bbox[1]))
p2 = (int(bbox[0] + bbox[2]), int(bbox[1] + bbox[3])) p2 = (int(bbox[0] + bbox[2]), int(bbox[1] + bbox[3]))
color = (0,255,0) if cut_after > i else (0, 192, 192) color = (0, 255, 0) if cut_after > i else (0, 192, 192)
thicc = 2 if cut_after > i else 1 thicc = 2 if cut_after > i else 1
cv2.rectangle(frame, p1, p2, color, thicc, 1) cv2.rectangle(frame, p1, p2, color, thicc, 1)
cv2.imshow("tsmark - tracker", frame) cv2.imshow("tsmark - tracker", frame)
k = cv2.waitKey(1) # speed up fps by 2
if k & 0xFF == ord("q"): # if press SPACE bar time_to_wait = self.marker.viewer_spf / 2 - time.time() + show_time
k = cv2.waitKey(max(1, int(time_to_wait * 1000)))
if k & 0xFF == ord("q") or k & 0xFF == 13: # accept with q or enter
done = True done = True
break break
if k & 0xFF == 27: # decline with escape
done = True
cut_after = 0
break
elif k & 0xFF == 32: # space elif k & 0xFF == 32: # space
paused = not paused paused = not paused
# Movement ================= # Movement =================
@@ -1189,48 +1339,40 @@ class TrackerGUI:
i -= int(self.marker.fps) + 1 i -= int(self.marker.fps) + 1
seek = True seek = True
# Move by frame # Move by frame
elif k & 0xFF == ord("."): elif k & 0xFF == ord(".") or k & 0xFF == ord("c"):
paused = True paused = True
seek = True seek = True
elif k & 0xFF == ord(","): elif k & 0xFF == ord(",") or k & 0xFF == ord("z"):
paused = True paused = True
i -= 2 i -= 2
seek = True seek = True
elif k & 0xFF == ord("x"): elif k & 0xFF == ord("x"):
cut_after = i cut_after = i
#if i in tracked: # TODO: ord("h") for help!
# tracked[i][4] = 1 - tracked[i][4]
time_to_wait = self.marker.viewer_spf - time.time() + show_time if i >= max_frames - 1:
if time_to_wait > 0: i = max_frames - 2
time.sleep(time_to_wait)
if i >= max_frames:
i = max_frames-1
paused = True paused = True
seek = True seek = True
if i<0: if i < 0:
i=0 i = -1
paused = True paused = True
seek = True seek = True
if seek: if seek:
self.marker.video_reader.set(cv2.CAP_PROP_POS_FRAMES, self.marker.nr + 1 + i) self.marker.video_reader.set(cv2.CAP_PROP_POS_FRAMES, self.marker.nr + i + 1)
cv2.destroyWindow("tsmark - tracker") cv2.destroyWindow("tsmark - tracker")
self.marker.nr = old_nr - 1 self.marker.nr = old_nr - 1
self.marker.read_next = True self.marker.read_next = True
self.points = {} self.points = {}
for i in sorted(list(tracked.keys())): for i in sorted(list(tracked.keys())):
if i >= cut_after: if i >= cut_after:
continue continue
self.points[self.marker.nr+1+i] = [ self.points[self.marker.nr + i + 1] = {
tracked[i][0], "x0": tracked[i][0],
tracked[i][1], "y0": tracked[i][1],
tracked[i][0]+tracked[i][2], "x1": tracked[i][0] + tracked[i][2],
tracked[i][1]+tracked[i][3], "y1": tracked[i][1] + tracked[i][3],
] "visible": 1,
}