From cc37b311db158ddb52fb3aedd9e41e137a17b43d Mon Sep 17 00:00:00 2001 From: Michael Zucchi Date: Mon, 21 Oct 2019 11:44:13 +1030 Subject: [PATCH] Fixes for seeking. - Change convertPTS() to not offset by the stream start. - When seeking use an offset of ~32 frames to in order to hit a key frame. - Provide another seek function which takes the keyframe for seek. Some formatting changes. Revert some java9+ code that broke jjoctave. --- contrib/octave/jjoctave/VideoReader.java | 2 +- .../classes/au/notzed/jjmpeg/AVRational.java | 4 + .../au/notzed/jjmpeg/io/JJMediaReader.java | 202 +++++++++++------- 3 files changed, 131 insertions(+), 77 deletions(-) diff --git a/contrib/octave/jjoctave/VideoReader.java b/contrib/octave/jjoctave/VideoReader.java index dca883c..cd3b2f0 100644 --- a/contrib/octave/jjoctave/VideoReader.java +++ b/contrib/octave/jjoctave/VideoReader.java @@ -163,7 +163,7 @@ public class VideoReader { * @return Frame timestamp in milliseconds. */ public long getTimeStamp(int sid) { - return videos[sid].absolutePTS(frames[sid].getPTS()); + return videos[sid].convertPTS(frames[sid].getPTS()); } /** diff --git a/src/notzed.jjmpeg/classes/au/notzed/jjmpeg/AVRational.java b/src/notzed.jjmpeg/classes/au/notzed/jjmpeg/AVRational.java index 7dd1674..3120144 100644 --- a/src/notzed.jjmpeg/classes/au/notzed/jjmpeg/AVRational.java +++ b/src/notzed.jjmpeg/classes/au/notzed/jjmpeg/AVRational.java @@ -39,6 +39,10 @@ public class AVRational extends Number implements Comparable { AVObject.init(); } + public AVRational inv() { + return new AVRational(den, num); + } + /** * Calls av_d2q * diff --git a/src/notzed.jjmpeg/classes/au/notzed/jjmpeg/io/JJMediaReader.java b/src/notzed.jjmpeg/classes/au/notzed/jjmpeg/io/JJMediaReader.java index 9892eaf..ad679da 100644 --- a/src/notzed.jjmpeg/classes/au/notzed/jjmpeg/io/JJMediaReader.java +++ b/src/notzed.jjmpeg/classes/au/notzed/jjmpeg/io/JJMediaReader.java @@ -35,6 +35,7 @@ import au.notzed.jjmpeg.AVRational; import au.notzed.jjmpeg.AVSampleFormat; import au.notzed.jjmpeg.AVStream; import java.io.FileNotFoundException; +import java.util.AbstractList; import java.util.Iterator; import java.util.List; import java.util.logging.Logger; @@ -42,9 +43,29 @@ import java.util.logging.Logger; /** * High level interface for scanning audio and video frames. *

- * TODO: handle all frames. - * - * @author notzed + *

Seeking

+ *

+ * {@link AVFormatContext#seekFile} has unreliable behaviour + * depending on the codec and container. In general it seems + * to seek to the keyframe at or after the requested timestamp. + * As such, for reliable seeking the seek must be offset by a + * delay and then the correct frame found by decoding frames + * one by one. + *

+ * Each JJReaderStream has a pts and millisecond offset which is + * estimated based on the frame-rate of the video file. A + * conservative estimate is used which means seeking may be slower + * than possible. + * For all-keyframe formats this shouldn't be required but it has not + * been special-cased. + *

+ * An alternative is provided by the 2-argument {@link #seekMS} + * method which takes the timestamp of a keyframe together with + * the timestamp of a specific frame. This does the obvious and + * seeks to the keyframe and then single-steps to the desired frame + * and provides the highest performance general seek function but + * requires an external index. + *

*/ public class JJMediaReader implements AutoCloseable { @@ -139,9 +160,20 @@ public class JJMediaReader implements AutoCloseable { } public List getStreams() { - return List.of(streams); + // Ideally List.of(streams) but that's java 9+ + return new AbstractList() { + @Override + public JJReaderStream get(int index) { + return streams[index]; + } + + @Override + public int size() { + return streams.length; + } + }; } - + public Iterable streams() { return () -> { return new Iterator() { @@ -194,6 +226,16 @@ public class JJMediaReader implements AutoCloseable { return pts; } + /** + * Retrieve current timestamp against the given stream. + * + * @param rs + * @return time in milliseconds + */ + public long convertPTS(JJReaderStream rs) { + return rs.convertPTS(pts); + } + /** * call flushBuffers() on all opened streams codecs. *

@@ -205,37 +247,41 @@ public class JJMediaReader implements AutoCloseable { } } - private long currentVideoMS() { - for (JJReaderStream r: streams) { - if (r instanceof JJReaderVideo) { - return r.convertPTS(getPTS()); - } - } - return -1; - } - /** * Attempt to seek to the nearest millisecond. *

* The next frame will have pts (in milliseconds) ≥ stamp. * + * @param rs Stream to seek against. * @param stamp * @throws AVIOException */ - public void seekMS(long stamp) throws AVIOException { - // This is an attempt at an optimisation, for small seeks (< 1 second) - // just parse/discard frames. - // This is going to be faster on many codecs than a flush/seek which will probably - // have to jump through more frames after a keyframe anyway - long current = currentVideoMS(); - if (current != -1 && (stamp - current) >= 0 && (stamp - current) < 1000) { - seekms = stamp; - } else { - format.seekFile(-1, 0, stamp * 1000, stamp * 1000, 0); + public void seekMS(JJReaderStream rs, long stamp) throws AVIOException { + long current = rs.convertPTS(getPTS()); + + if (current != -1 && (stamp - current) > 0 && (stamp - current) < 1000) { seekms = stamp; + return; + } + + format.seekFile(-1, 0, Math.max(0, stamp - rs.seekOffsetMS) * 1000, stamp * 1000, 0); + seekms = stamp; + + flushCodec(); + } + + public void seekMS(JJReaderStream rs, long keyStamp, long stamp) throws AVIOException { + long current = rs.convertPTS(getPTS()); - flushCodec(); + if (current != -1 && (stamp - current) > 0 && (stamp - current) < 2000) { + seekms = stamp; + return; } + + format.seekFile(-1, 0, keyStamp * 1000, keyStamp * 1000, 0); + seekms = stamp; + + flushCodec(); } /** @@ -243,11 +289,12 @@ public class JJMediaReader implements AutoCloseable { *

* The next frame will have pts ≥ stamp * + * @param rs Stream to seek against. * @param stamp * @throws AVIOException */ - public void seek(long stamp) throws AVIOException { - format.seekFile(-1, 0, stamp, stamp, 0); + public void seek(JJReaderStream rs, long stamp) throws AVIOException { + format.seekFile(rs.stream.getIndex(), 0, Math.max(0, stamp - rs.seekOffset), stamp, 4); seekid = stamp; flushCodec(); @@ -270,24 +317,34 @@ public class JJMediaReader implements AutoCloseable { int index = packet.getStreamIndex(); JJReaderStream ms = streams[index]; - if (ms != null) { - if (ms.isOpened() && ms.decode(packet)) { - pts = packet.getDTS(); + if (ms != null && ms.isOpened() && ms.decode(packet)) { + pts = packet.getDTS(); - // If seeking, attempt to get to the exact frame - if (seekid != -1 - && pts < seekid) { + // If seeking, attempt to get to the exact frame + if (seekid != -1) { + if (pts < seekid) { + Logger.getLogger("jjmpeg.io").fine(() + -> String.format("seek=%d but @ %d flags=%d", seekid, pts, packet.getFlags())); continue; - } else if (seekms != -1 - && ms.convertPTS(pts) < seekms) { + } + } else if (seekms != -1) { + long ptsms = ms.convertPTS(pts); + + if (ptsms < seekms) { + Logger.getLogger("jjmpeg.io").fine(() + -> String.format("seekms=%d but @ %d flags=%d", seekms, ms.convertPTS(pts), packet.getFlags())); continue; } - seekid = -1; - seekms = -1; - freePacket = true; - return ms; + Logger.getLogger("jjmpeg.io").fine(() + -> String.format("seekms=%d now @ %d flags=%d *", seekms, ms.convertPTS(pts), packet.getFlags())); } + + seekid = -1; + seekms = -1; + freePacket = true; + return ms; } + } finally { if (!freePacket) { packet.clear(); @@ -348,6 +405,7 @@ public class JJMediaReader implements AutoCloseable { } return null; + } static public abstract class JJReaderStream { @@ -355,7 +413,6 @@ public class JJMediaReader implements AutoCloseable { AVStream stream; AVCodecContext c; AVCodecParameters cp; - int streamID = -1; protected AVCodec codec; protected boolean opened = false; // timebase @@ -371,6 +428,9 @@ public class JJMediaReader implements AutoCloseable { final AVFrame frame; // Incremented every time a frame is decoded long frameIndex; + // Some formats (mpeg?) always seek to the next keyframe, which is really not desirable. + long seekOffset; + long seekOffsetMS; public JJReaderStream(AVStream stream, AVCodecParameters cp) throws AVIOException { this.stream = stream; @@ -393,6 +453,10 @@ public class JJMediaReader implements AutoCloseable { startms = tb.rescale(startpts, 1000); duration = stream.getDuration(); durationms = tb.rescale(duration, 1000); + + // conservatively assume keyframe every 32 frames + seekOffsetMS = stream.getAverageFrameRate().inv().rescale(1000, 32); + seekOffset = (long) tb.inv().mul(new AVRational((int) seekOffsetMS, 1000)).doubleValue(); } public void open() throws AVIOException { @@ -408,7 +472,8 @@ public class JJMediaReader implements AutoCloseable { codec.release(); c.release(); } - frame.release(); + if (frame != null) + frame.release(); stream.release(); } @@ -467,22 +532,12 @@ public class JJMediaReader implements AutoCloseable { } /** - * Convert the 'pts' provided to milliseconds relative to the start of the stream. + * Convert the 'pts' provided to milliseconds. * * @param pts * @return */ public long convertPTS(long pts) { - return tb.rescale(pts - startpts, 1000); - } - - /** - * Convert the 'pts' to milliseconds as presented in the stream. - * - * @param pts - * @return - */ - public long absolutePTS(long pts) { return tb.rescale(pts, 1000); } @@ -500,9 +555,8 @@ public class JJMediaReader implements AutoCloseable { boolean done = false; if (isOpened()) { if (packet != null && packet.getSize() == 0) { - Logger.getLogger("jjmpeg.io") - .fine(() - -> String.format("stream %d packet size 0", streamID)); + Logger.getLogger("jjmpeg.io").fine(() + -> String.format("stream %d packet size 0", stream.getIndex())); return done; } @@ -579,17 +633,15 @@ public class JJMediaReader implements AutoCloseable { // throw new AVIOException(AVError.AVERROR_INVALIDDATA, "No decodable video present"); //} - Logger.getLogger("jjmpeg.io") - .fine(() -> { - return String.format("Open video reader\n" - + " video %dx%d %s [aspect %s]\n" - + " codec %s\n", - cp.getWidth(), - cp.getHeight(), - AVPixelFormat.toString(cp.getPixelFormat()), - cp.getAspectRatio(), - AVCodecID.toString(cp.getCodecID())); - }); + Logger.getLogger("jjmpeg.io").fine(() + -> String.format("Open video reader\n" + + " video %dx%d %s [aspect %s]\n" + + " codec %s\n", + cp.getWidth(), + cp.getHeight(), + AVPixelFormat.toString(cp.getPixelFormat()), + cp.getAspectRatio(), + AVCodecID.toString(cp.getCodecID()))); super.open(); } @@ -635,16 +687,14 @@ public class JJMediaReader implements AutoCloseable { @Override public void open() throws AVIOException { - Logger.getLogger("jjmpeg.io") - .fine(() -> { - return String.format("Open audio reader\n" - + " audio x%d %dHz %s\n" - + " codec %s\n", - cp.getNumChannels(), - cp.getSampleRate(), - AVSampleFormat.toString(cp.getSampleFormat()), - AVCodecID.toString(cp.getCodecID())); - }); + Logger.getLogger("jjmpeg.io").fine(() + -> String.format("Open audio reader\n" + + " audio x%d %dHz %s\n" + + " codec %s\n", + cp.getNumChannels(), + cp.getSampleRate(), + AVSampleFormat.toString(cp.getSampleFormat()), + AVCodecID.toString(cp.getCodecID()))); super.open(); } -- 2.39.2