Searching for an alternative as OpenCV would not provide timestamps for live camera stream (on Windows), which are required in my computer vision algorithm, I found ffmpeg and this excellent article https://zulko.github.io/blog/2013/09/27/read-and-write-video-frames-in-python-using-ffmpeg/
The solution uses ffmpeg, accessing its standard output (stdout) stream. I extended it to read the standard error (stderr) stream as well.
Working up the python code on windows, while I received the video frames from ffmpeg stdout, but the stderr freezes after delivering the showinfo videofilter details (timestamp) for first frame.
I recollected seeing on ffmpeg forum somewhere that the video filters like showinfo are bypassed when redirected. Is this why the following code does not work as expected?
Expected: It should write video frames to disk as well as print timestamp details.
Actual: It writes video files but does not get the timestamp (showinfo) details.
Here's the code I tried:
import subprocess as sp
import numpy
import cv2
command = [ 'ffmpeg',
'-i', 'e:\sample.wmv',
'-pix_fmt', 'rgb24',
'-vcodec', 'rawvideo',
'-vf', 'showinfo', # video filter - showinfo will provide frame timestamps
'-an','-sn', #-an, -sn disables audio and sub-title processing respectively
'-f', 'image2pipe', '-'] # we need to output to a pipe
pipe = sp.Popen(command, stdout = sp.PIPE, stderr = sp.PIPE) # TODO someone on ffmpeg forum said video filters (e.g. showinfo) are bypassed when stdout is redirected to pipes???
for i in range(10):
raw_image = pipe.stdout.read(1280*720*3)
img_info = pipe.stderr.read(244) # 244 characters is the current output of showinfo video filter
print "showinfo output", img_info
image1 = numpy.fromstring(raw_image, dtype='uint8')
image2 = image1.reshape((720,1280,3))
# write video frame to file just to verify
videoFrameName = 'Video_Frame{0}.png'.format(i)
cv2.imwrite(videoFrameName,image2)
# throw away the data in the pipe's buffer.
pipe.stdout.flush()
pipe.stderr.flush()
So how to still get the frame timestamps from ffmpeg into python code so that it can be used in my computer vision algorithm...
You can use MoviePy:
import moviepy.editor as mpy
vid = mpy.VideoFileClip('e:\\sample.wmv')
for timestamp, raw_img in vid.iter_frames(with_times=True):
# do stuff
Redirecting stderr works in python.
So instead of this pipe = sp.Popen(command, stdout = sp.PIPE, stderr = sp.PIPE)
do this pipe = sp.Popen(command, stdout = sp.PIPE, stderr = sp.STDOUT)
We could avoid redirection by adding an asynchronous call to read both the standard streams (stdout and stderr) of ffmpeg. This would avoid any mixing of the video frame and timestamp and thus the error prone seperation.
So modifying the original code to use threading module would look like this:
# Python script to read video frames and timestamps using ffmpeg
import subprocess as sp
import threading
import matplotlib.pyplot as plt
import numpy
import cv2
ffmpeg_command = [ 'ffmpeg',
'-nostats', # do not print extra statistics
#'-debug_ts', # -debug_ts could provide timestamps avoiding showinfo filter (-vcodec copy). Need to check by providing expected fps TODO
'-r', '30', # output 30 frames per second
'-i', 'e:\sample.wmv',
'-an','-sn', #-an, -sn disables audio and sub-title processing respectively
'-pix_fmt', 'rgb24',
'-vcodec', 'rawvideo',
#'-vcodec', 'copy', # very fast!, direct copy - Note: No Filters, No Decode/Encode, no quality loss
#'-vframes', '20', # process n video frames only. For Debugging
'-vf', 'showinfo', # showinfo videofilter provides frame timestamps as pts_time
'-f', 'image2pipe', 'pipe:1' ] # outputs to stdout pipe. can also use '-' which is redirected to pipe
# seperate method to read images on stdout asynchronously
def AppendProcStdout(proc, nbytes, AppendList):
while proc.poll() is None: # continue while the process is alive
AppendList.append(proc.stdout.read(nbytes)) # read image bytes at a time
# seperate method to read image info. on stderr asynchronously
def AppendProcStderr(proc, AppendList):
while proc.poll() is None: # continue while the process is alive
try: AppendList.append(proc.stderr.next()) # read stderr until empty
except StopIteration: continue # ignore stderr empty exception and continue
if __name__ == '__main__':
# run ffmpeg command
pipe = sp.Popen(ffmpeg_command, stdout=sp.PIPE, stderr=sp.PIPE)
# 2 threads to talk with ffmpeg stdout and stderr pipes
framesList = [];
frameDetailsList = []
appendFramesThread = threading.Thread(group=None, target=AppendProcStdout, name='FramesThread', args=(pipe, 1280*720*3, framesList), kwargs=None, verbose=None) # assuming rgb video frame with size 1280*720
appendInfoThread = threading.Thread(group=None, target=AppendProcStderr, name='InfoThread', args=(pipe, frameDetailsList), kwargs=None, verbose=None)
# start threads to capture ffmpeg frames and info.
appendFramesThread.start()
appendInfoThread.start()
# wait for few seconds and close - simulating cancel
import time; time.sleep(2)
pipe.terminate()
# check if threads finished and close
appendFramesThread.join()
appendInfoThread.join()
# save an image per 30 frames to disk
savedList = []
for cnt,raw_image in enumerate(framesList):
if (cnt%30 != 0): continue
image1 = numpy.fromstring(raw_image, dtype='uint8')
image2 = image1.reshape((720,1280,3)) # assuming rgb image with size 1280 X 720
# write video frame to file just to verify
videoFrameName = 'video_frame{0}.png'.format(cnt)
cv2.imwrite(videoFrameName,image2)
savedList.append('{} {}'.format(videoFrameName, image2.shape))
print '### Results ###'
print 'Images captured: ({}) \nImages saved to disk:{}\n'.format(len(framesList), savedList) # framesList contains all the video frames got from the ffmpeg
print 'Images info captured: \n', ''.join(frameDetailsList) # this contains all the timestamp details got from the ffmpeg showinfo videofilter and some initial noise text which can be easily removed while parsing
You can try to specify the buffer size so you're sure the whole frame fits in it :
bufsize = w*h*3 + 100
pipe = sp.Popen(command, bufsize=bufsize, stdout = sp.PIPE, stderr = sp.PIPE)
with this set up, you can normally read on pipe.stdout for your frames and pipe.stderr for its info
