Raincloud/GigaMux
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

the audio is synced?

Browse Source
This commit is contained in:
Raincloud
2026-02-19 07:45:42 -07:00
parent 0201d3e8bd
commit da21cafe1c
2 changed files with 1451 additions and 8143 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.specstory/history/2026-01-21_08-11Z-check-files-py-small-output-error-handling.md
+1357 -8130
View File
File diff suppressed because one or more lines are too long
ffmpeg_distributed.py
+94 -13
View File
@@ -281,6 +281,7 @@ class TqdmAbsolute(tqdm):
HOST_COLORS = ['\033[94m', '\033[92m', '\033[93m', '\033[95m', '\033[96m', '\033[91m'] # blue, green, yellow, magenta, cyan, red
RESET = '\033[0m'
SEGMENT_STALL_TIMEOUT = int(environ.get('SEGMENT_STALL_TIMEOUT', '300')) # no encoder progress for this many sec -> kill and re-queue
class TaskThread(Thread):
def __init__(self, host: str, gpu_id: int, source_file: str, task_queue: SimpleQueue, bar_pos: int, remote_ffmpeg_path: str = None):
@@ -303,9 +304,18 @@ class TaskThread(Thread):
def stop(self):
self._should_stop = True
if getattr(self, '_reader_proc', None) is not None and self._reader_proc.poll() is None:
rp = getattr(self, '_reader_proc', None)
if rp is not None and rp.poll() is None:
try:
self._reader_proc.terminate()
if rp.stdin is not None:
rp.stdin.write(b'q')
rp.stdin.flush()
sleep(0.5)
except (OSError, BrokenPipeError):
pass
try:
if rp.poll() is None:
rp.terminate()
except OSError:
pass
if self._ffmpeg:
@@ -313,28 +323,54 @@ class TaskThread(Thread):
def run(self):
last_log = [0.0] # mutable for progress heartbeat
last_progress = [time()] # for stall detection
last_encoder_t = [0.0] # last encoder progress position (sec) for stall log
segment_start_time = [0.0] # when current segment started
first_progress_logged = [False] # one-time "first frame" log per segment
verbose = environ.get('VERBOSE', '').lower() in ('1', 'true', 'yes') or environ.get('DISTRIBUTED_DEBUG', '').lower() in ('1', 'true', 'yes')
def upd(frames, fps, t, duration, speed):
now = time()
last_progress[0] = now
last_encoder_t[0] = t
self._bar.total = duration or 999
self._bar.desc = self._host_desc + ': ' + (self._current_file or '')
self._bar.update(t)
if duration and duration > 0 and (time() - last_log[0]) >= 30:
if not first_progress_logged[0] and (frames > 0 or t > 0):
elapsed = now - segment_start_time[0]
tqdm.write(f' {self._host_tag()}: {self._current_file} first frame after {elapsed:.1f}s', file=stderr)
stderr.flush()
first_progress_logged[0] = True
if duration and duration > 0 and (now - last_log[0]) >= 30:
tqdm.write(f' {self._host_tag()}: {self._current_file} {t:.0f}s / {duration:.0f}s ({speed:.1f}x)', file=stderr)
stderr.flush()
last_log[0] = time()
last_log[0] = now
try:
while not self._should_stop:
task = self._task_queue.get(False)
self._current_file = basename(task.output_file)
tqdm.write(f' {self._host_tag()}: starting {self._current_file} (t={task.start_sec:.0f}-{task.start_sec+task.duration_sec:.0f}s)', file=stderr)
stderr.flush()
# -i then -ss then -frames:v so we send exactly N frames (avoids VFR/source timestamps giving short segments)
last_progress[0] = time()
segment_start_time[0] = last_progress[0]
first_progress_logged[0] = False
last_encoder_t[0] = 0.0
n_frames = round(task.duration_sec * task.fps)
reader_cmd = [
'ffmpeg', '-i', self._source_file,
'-ss', str(task.start_sec), '-frames:v', str(n_frames),
'-an', '-sn', '-c:v', 'copy', '-f', 'mpegts', 'pipe:1'
]
# Default fast_seek=True: -ss before -i (low RAM, quick). Set READER_FAST_SEEK=0 for frame-accurate (high RAM, slow for late segments).
fast_seek = environ.get('READER_FAST_SEEK', '1').lower() not in ('0', 'false', 'no')
tqdm.write(f' {self._host_tag()}: starting {self._current_file} (t={task.start_sec:.0f}-{task.start_sec+task.duration_sec:.0f}s, n_frames={n_frames}, fast_seek={fast_seek})', file=stderr)
stderr.flush()
# READER_FAST_SEEK=1: -ss before -i (keyframe seek, low RAM, no decode from 0). Else -i then -ss (frame-accurate but decodes 0..start = high RAM for late segments).
if fast_seek:
reader_cmd = [
'ffmpeg', '-ss', str(task.start_sec), '-noaccurate_seek', '-i', self._source_file,
'-frames:v', str(n_frames), '-an', '-sn', '-c:v', 'copy', '-f', 'mpegts', 'pipe:1'
]
else:
reader_cmd = [
'ffmpeg', '-i', self._source_file,
'-ss', str(task.start_sec), '-frames:v', str(n_frames),
'-an', '-sn', '-c:v', 'copy', '-f', 'mpegts', 'pipe:1'
]
ffmpeg_bin = (self._remote_ffmpeg_path or 'ffmpeg') if self._host != 'localhost' else 'ffmpeg'
# -r only: lock output to CFR; do not use -frames:v or segments get truncated when reader sends fewer frames
encoder_cmd = [
@@ -349,8 +385,44 @@ class TaskThread(Thread):
if self._host != 'localhost':
encoder_cmd = ['ssh', '-o', 'ConnectTimeout=15', self._host, join(encoder_cmd)]
self._reader_proc = Popen(reader_cmd, stdout=PIPE, stderr=DEVNULL)
reader_stderr_lines = []
if verbose:
self._reader_proc = Popen(reader_cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE)
def read_reader_stderr():
rp = getattr(self, '_reader_proc', None)
if rp and rp.stderr:
for line in iter(rp.stderr.readline, b''):
reader_stderr_lines.append(line.decode(errors='replace').rstrip())
Thread(target=read_reader_stderr, daemon=True).start()
else:
self._reader_proc = Popen(reader_cmd, stdin=PIPE, stdout=PIPE, stderr=DEVNULL)
ret = -1
stall_done = [False] # watchdog sets True when it kills
def stall_watchdog():
if SEGMENT_STALL_TIMEOUT <= 0:
return
while not self._should_stop and not stall_done[0]:
sleep(60)
if stall_done[0] or self._should_stop:
break
if time() - last_progress[0] > SEGMENT_STALL_TIMEOUT:
ago = time() - last_progress[0]
enc_t = last_encoder_t[0]
tqdm.write(f' {self._host_tag()}: STALLED {self._current_file} (no progress {ago:.0f}s; last encoder at {enc_t:.1f}s into segment)', file=stderr)
if not fast_seek and task.start_sec > 60:
tqdm.write(f' {self._host_tag()}: (reader -ss {task.start_sec:.0f} after -i may still be decoding 0..{task.start_sec:.0f}s from source)', file=stderr)
tqdm.write(f' {self._host_tag()}: killing and re-queuing', file=stderr)
stderr.flush()
stall_done[0] = True
self.stop()
break
proc = getattr(self._ffmpeg, '_proc', None) if self._ffmpeg else None
if proc is not None and proc.poll() is not None:
break
watchdog = Thread(target=stall_watchdog, daemon=True)
watchdog.start()
try:
with open(task.output_file, 'wb') as outfile:
self._ffmpeg = FFMPEGProc(
@@ -359,6 +431,8 @@ class TaskThread(Thread):
)
ret = self._ffmpeg.run()
finally:
stall_done[0] = True
watchdog.join(timeout=2)
rp = getattr(self, '_reader_proc', None)
if rp is not None:
try:
@@ -377,6 +451,11 @@ class TaskThread(Thread):
print(f' {self._host_tag()}: FAILED {self._current_file}', file=stderr, flush=True)
if self._ffmpeg.stderr:
print(self._ffmpeg.stderr, file=stderr, end='', flush=True)
if reader_stderr_lines:
tqdm.write(f' {self._host_tag()}: Reader stderr:', file=stderr)
for line in reader_stderr_lines[-50:]: # last 50 lines
tqdm.write(f' {line}', file=stderr)
stderr.flush()
self._task_queue.put(task)
else:
tqdm.write(f' {self._host_tag()}: done {self._current_file}', file=stderr)
@@ -438,6 +517,8 @@ def encode(workers: List[Tuple[str, int]], input_file: str, output_file: str, se
dprint(f'Segments: {len(segments)} total, {n_tasks} tasks queued')
tqdm.write(f'[3/4] Encoding segments on {len(workers)} worker(s)...', file=stderr)
if environ.get('READER_FAST_SEEK', '1').lower() not in ('0', 'false', 'no'):
tqdm.write(' (Reader: fast seek -ss before -i; set READER_FAST_SEEK=0 for frame-accurate)', file=stderr)
stderr.flush()
threads = [TaskThread(host, gpu_id, input_file, task_queue, pos, remote_ffmpeg_path) for pos, (host, gpu_id) in enumerate(workers, 0)]