From a295cf975f226736f3ea114629f91c213b6f7c30 Mon Sep 17 00:00:00 2001 From: Michael Zucchi Date: Tue, 22 Oct 2019 08:15:05 +1030 Subject: [PATCH] Mostly documentation updates. Some minor cleanups. --- contrib/octave/jjVideoOpen.m | 33 ++- contrib/octave/jjoctave/VideoReader.java | 27 ++- .../au/notzed/jjmpeg/AVDictionary.java | 48 ++-- .../au/notzed/jjmpeg/io/JJMediaReader.java | 222 +++++++++--------- .../au/notzed/jjmpeg/io/JJMediaWriter.java | 2 +- .../au/notzed/jjmpeg/io/package-info.java | 7 + .../au/notzed/jjmpeg/package-info.java | 15 ++ src/notzed.jjmpeg/gen/extract-defines.pl | 8 +- src/notzed.jjmpeg/gen/extract-enum.pl | 4 +- 9 files changed, 209 insertions(+), 157 deletions(-) create mode 100644 src/notzed.jjmpeg/classes/au/notzed/jjmpeg/io/package-info.java create mode 100644 src/notzed.jjmpeg/classes/au/notzed/jjmpeg/package-info.java diff --git a/contrib/octave/jjVideoOpen.m b/contrib/octave/jjVideoOpen.m index 2aa284e..7f1b49a 100644 --- a/contrib/octave/jjVideoOpen.m +++ b/contrib/octave/jjVideoOpen.m @@ -9,7 +9,38 @@ % % video The video handle (object). % -% See also jjVideoClose(). +% This creates a Java jjoctave.VideoReader object which +% provides methods for reading multi-stream video files. +% +% In octave the methods can be invoked using: +% +% result = javaMethod('methodName', video [, arguments]) +% +% In MATLAB they can be invoked directly as: +% +% video.methodName([arguments]) +% +% This is a summary, see the javadoc for details. +% +% sid = video.getStreamCount() Get number of video streams. +% width = video.getWidth(sid) Get image width. +% height = video.getHeight(sid) Get image height. +% ms = video.getDuration(sid) Get video duration from header. +% video.close() Close video. +% +% For reading frames. +% +% sid = video.nextFrame() Next frame, returns stream or -1 +% sid = video.nextFrame(sid) Next frame in stream, returns tream or -1 +% ms = video.getTimeStamp(sid) Get current frame timestamp +% array = video.getPixels(sid, fmt) +% Get copy of pixel data +% +% Access to low-level interface which provides every detail available. +% +% stream = video.getStream(sid) Get JJMediaReader.JJReaderStream object +% +% See also jjVideoClose(), jjVideoNext(), jjVideoFrame() and jjoctave.VideoReader. % % Copyright (C) 2019 Michael Zucchi diff --git a/contrib/octave/jjoctave/VideoReader.java b/contrib/octave/jjoctave/VideoReader.java index cd3b2f0..e6f11b0 100644 --- a/contrib/octave/jjoctave/VideoReader.java +++ b/contrib/octave/jjoctave/VideoReader.java @@ -59,7 +59,7 @@ import java.util.List; * end * */ -public class VideoReader { +public class VideoReader extends JJMediaReader { JJMediaReader reader; final JJMediaReader.JJReaderVideo[] videos; @@ -79,10 +79,11 @@ public class VideoReader { * @see #close */ public VideoReader(String path) throws IOException { - reader = new JJMediaReader(path); + super(path); + List streams = new ArrayList<>(); - for (JJMediaReader.JJReaderStream rs: reader.streams()) { + for (JJMediaReader.JJReaderStream rs: getStreams()) { if (rs.getType() == AVMediaTypeBits.AVMEDIA_TYPE_VIDEO) { JJMediaReader.JJReaderVideo rv = (JJMediaReader.JJReaderVideo) rs; rs.open(); @@ -127,6 +128,13 @@ public class VideoReader { return videos.length; } + /** + * Retrieves the raw stream object. + */ + public JJMediaReader.JJReaderVideo getStream(int sid) { + return videos[sid]; + } + /** * Retrieve the frame width of the specified stream. *

@@ -153,6 +161,19 @@ public class VideoReader { return videos[sid].getHeight(); } + /** + * Retrieve the stream duration in milliseconds. + *

+ * This may be an estimated value. + * + * @param sid Stream id. + * @return The stream dursation. + * @throws ArrayIndexOutOfBoundsException if sid is invalid. + */ + public long getDuration(int sid) { + return videos[sid].getDurationMS(); + } + /** * Retrieve the timestamp of the current frame in milliseconds. *

diff --git a/src/notzed.jjmpeg/classes/au/notzed/jjmpeg/AVDictionary.java b/src/notzed.jjmpeg/classes/au/notzed/jjmpeg/AVDictionary.java index 44e05c0..6110f45 100644 --- a/src/notzed.jjmpeg/classes/au/notzed/jjmpeg/AVDictionary.java +++ b/src/notzed.jjmpeg/classes/au/notzed/jjmpeg/AVDictionary.java @@ -18,47 +18,31 @@ */ package au.notzed.jjmpeg; -import java.util.Collection; -import java.util.Map.Entry; import java.util.TreeMap; /** - * AVDictionary is a remarkably shithouse "ADT" inherited by ffmpeg from libav. + * AVDictionary is a remarkably shithouse "ADT" used for configuration parameters. *

- * Because the pointer can be changed by any of the update functions this - * doesn't bother trying to wrap the C api. Instead it maintains it's own - * set of data and converts it to an array of strings on demand. + * This does not wrap it, it simply provides an equivalent functionality + * using a TreeMap. + *

+ *

Historical Note

*

- * It's a tree because ... I don't know. + * AVDictionary was inherited from libav. It's a miserable + * design, it's simply a pointer to an array which updates itself by copying the whole array + * every time it's length is changed. + *

*/ public class AVDictionary extends TreeMap { + /** + * Convert the dictionary to an array of {@code Entry}. + *

+ * This is used by the native binding to read elements. + * + * @return An array containing {@link java.util.Map.Entry} objects. + */ public Object[] toArray() { return entrySet().toArray(); } - - public void setAll(Collection set) { - this.entrySet().retainAll(set); - } - - public String[] toDictionary() { - String[] list = new String[size() * 2]; - int i = 0; - - for (Entry e: entrySet()) { - list[i++] = e.getKey(); - list[i++] = e.getValue(); - } - - return list; - } - - public void fromDictionary(String[] set) { - clear(); - if (set != null) { - for (int i = 0; i < set.length; i += 2) { - put(set[i], set[i + 1]); - } - } - } } diff --git a/src/notzed.jjmpeg/classes/au/notzed/jjmpeg/io/JJMediaReader.java b/src/notzed.jjmpeg/classes/au/notzed/jjmpeg/io/JJMediaReader.java index ad679da..41b61bc 100644 --- a/src/notzed.jjmpeg/classes/au/notzed/jjmpeg/io/JJMediaReader.java +++ b/src/notzed.jjmpeg/classes/au/notzed/jjmpeg/io/JJMediaReader.java @@ -36,7 +36,6 @@ import au.notzed.jjmpeg.AVSampleFormat; import au.notzed.jjmpeg.AVStream; import java.io.FileNotFoundException; import java.util.AbstractList; -import java.util.Iterator; import java.util.List; import java.util.logging.Logger; @@ -69,14 +68,22 @@ import java.util.logging.Logger; */ public class JJMediaReader implements AutoCloseable { - JJReaderStream[] streams; + List streams; AVFormatContext format; private AVPacket packet; private boolean freePacket = false; - // - long seekid = -1; + /** + * Last decoded pts. + */ + long pts; + /** + * Tracks the last seek() + */ + long seek = -1; + /** + * Tracks the last seekMS() + */ long seekms = -1; - // /** * VIDEO Type indicator for openDefaultStream(). @@ -88,10 +95,17 @@ public class JJMediaReader implements AutoCloseable { public static final Class TYPE_AUDIO = JJReaderAudio.class; /** - * Create a new media reader, will scan the file for available streams. + * Open a media file. + *

+ * This opens a media file and scans it's format. After it has been opened + * one must open the individual streams of interest and then use readFrame + * to decode the media frames. * - * @param name + * @param name Filename. * @throws AVIOException + * @throws java.io.FileNotFoundException + * @see #getStreams + * @see #readFrame */ public JJMediaReader(String name) throws AVIOException, FileNotFoundException { this(name, null, null); @@ -111,24 +125,36 @@ public class JJMediaReader implements AutoCloseable { // find all streams and map to something int nstreams = format.getNumStreams(); - streams = new JJReaderStream[nstreams]; + JJReaderStream[] list = new JJReaderStream[nstreams]; for (int i = 0; i < nstreams; i++) { AVStream s = format.getStream(i); AVCodecParameters cp = s.getCodecParameters(); switch (cp.getCodecType()) { case AVMediaType.AVMEDIA_TYPE_VIDEO: - streams[i] = new JJReaderVideo(s, cp); + list[i] = new JJReaderVideo(s, cp); break; case AVMediaType.AVMEDIA_TYPE_AUDIO: - streams[i] = new JJReaderAudio(s, cp); + list[i] = new JJReaderAudio(s, cp); break; default: - streams[i] = new JJReaderUnknown(s, cp); + list[i] = new JJReaderUnknown(s, cp); s.setDiscard(AVDiscardBits.AVDISCARD_ALL); break; } } + // Ideally List.of(streams) but that's java 9+ + streams = new AbstractList() { + @Override + public JJReaderStream get(int index) { + return list[index]; + } + + @Override + public int size() { + return list.length; + } + }; packet = AVPacket.alloc(); } @@ -159,39 +185,23 @@ public class JJMediaReader implements AutoCloseable { return null; } + /** + * Get the list of streams. + * + * @return + */ public List getStreams() { - // Ideally List.of(streams) but that's java 9+ - return new AbstractList() { - @Override - public JJReaderStream get(int index) { - return streams[index]; - } - - @Override - public int size() { - return streams.length; - } - }; - } - - public Iterable streams() { - return () -> { - return new Iterator() { - int i = 0; - - @Override - public boolean hasNext() { - return i < streams.length; - } - - @Override - public JJReaderStream next() { - return streams[i++]; - } - }; - }; + return streams; } + /** + * Release and close all resources. + *

+ * Resources will be automatically reclaimed by the JVM but this can be used + * to control when they are released. + *

+ * The object or any sourced from it of the package au.notzed.jjmpeg must not be accessed thereafter. + */ public void release() { for (JJReaderStream m: streams) { m.release(); @@ -200,23 +210,27 @@ public class JJMediaReader implements AutoCloseable { packet.release(); } + /** + * Close the media reader. + *

+ * Calls {@link #release}. + */ @Override public void close() { release(); } /** - * Get source AVFormatContext. + * Get underlying AVFormatContext object. * * @return */ public AVFormatContext getFormat() { return format; } - long pts; /** - * Retrieve (calculated) pts of the last frame decoded. + * Get pts of the last frame decoded. *

* Well be -1 at EOF * @@ -229,7 +243,7 @@ public class JJMediaReader implements AutoCloseable { /** * Retrieve current timestamp against the given stream. * - * @param rs + * @param rs Stream of interest * @return time in milliseconds */ public long convertPTS(JJReaderStream rs) { @@ -237,9 +251,9 @@ public class JJMediaReader implements AutoCloseable { } /** - * call flushBuffers() on all opened streams codecs. + * Call flushBuffers() on all opened streams codecs. *

- * e.g. after a seek. + * This is called automatically after a seek. */ public void flushCodec() { for (JJReaderStream rs: streams) { @@ -248,13 +262,14 @@ public class JJMediaReader implements AutoCloseable { } /** - * Attempt to seek to the nearest millisecond. + * Seek in milliseconds. *

* The next frame will have pts (in milliseconds) ≥ stamp. * * @param rs Stream to seek against. * @param stamp * @throws AVIOException + * @see #seekMS(JJReaderStream,long,long) */ public void seekMS(JJReaderStream rs, long stamp) throws AVIOException { long current = rs.convertPTS(getPTS()); @@ -270,6 +285,17 @@ public class JJMediaReader implements AutoCloseable { flushCodec(); } + /** + * Seek in milliseconds via key-frame. + *

+ * This will first seek to a key-frame whose time-stamp is ≥ keyStamp + * and then advance by single frames until pts (in milliseconds) ≥ stamp; + * + * @param rs Stream to seek against. + * @param keyStamp Timestamp of a keyframe. + * @param stamp Timestamp of desired frame. + * @throws AVIOException + */ public void seekMS(JJReaderStream rs, long keyStamp, long stamp) throws AVIOException { long current = rs.convertPTS(getPTS()); @@ -295,7 +321,7 @@ public class JJMediaReader implements AutoCloseable { */ public void seek(JJReaderStream rs, long stamp) throws AVIOException { format.seekFile(rs.stream.getIndex(), 0, Math.max(0, stamp - rs.seekOffset), stamp, 4); - seekid = stamp; + seek = stamp; flushCodec(); } @@ -315,16 +341,16 @@ public class JJMediaReader implements AutoCloseable { while (format.readFrame(packet)) { try { int index = packet.getStreamIndex(); - JJReaderStream ms = streams[index]; + JJReaderStream ms = streams.get(index); if (ms != null && ms.isOpened() && ms.decode(packet)) { pts = packet.getDTS(); // If seeking, attempt to get to the exact frame - if (seekid != -1) { - if (pts < seekid) { + if (seek != -1) { + if (pts < seek) { Logger.getLogger("jjmpeg.io").fine(() - -> String.format("seek=%d but @ %d flags=%d", seekid, pts, packet.getFlags())); + -> String.format("seek=%d but @ %d flags=%d", seek, pts, packet.getFlags())); continue; } } else if (seekms != -1) { @@ -339,7 +365,7 @@ public class JJMediaReader implements AutoCloseable { -> String.format("seekms=%d now @ %d flags=%d *", seekms, ms.convertPTS(pts), packet.getFlags())); } - seekid = -1; + seek = -1; seekms = -1; freePacket = true; return ms; @@ -355,59 +381,6 @@ public class JJMediaReader implements AutoCloseable { return null; } - JJReaderStream last; - - public JJReaderStream readFrameX() throws AVIOException { - if (freePacket) { - packet.clear(); - } - freePacket = false; - - // Drain all frames from last stream first - if (last != null && last.c.receiveFrame(last.frame)) { - last.frameIndex += 1; - pts = packet.getDTS(); - return last; - } - last = null; - - while (format.readFrame(packet)) { - try { - int index = packet.getStreamIndex(); - JJReaderStream ms = streams[index]; - - if (ms != null && ms.isOpened()) { - ms.c.sendPacket(packet); - while (ms.c.receiveFrame(ms.frame)) { - ms.frameIndex += 1; - pts = packet.getDTS(); - - // If seeking, attempt to get to the exact frame - if (seekid != -1 - && pts < seekid) { - continue; - } else if (seekms != -1 - && ms.convertPTS(pts) < seekms) { - continue; - } - last = ms; - seekid = -1; - seekms = -1; - freePacket = true; - return ms; - } - } - } finally { - if (!freePacket) { - packet.clear(); - } - } - } - - return null; - - } - static public abstract class JJReaderStream { AVStream stream; @@ -488,20 +461,35 @@ public class JJMediaReader implements AutoCloseable { return cp; } + /** + * Get the underlying AVStream object. + * + * @return + */ public AVStream getStream() { return stream; } + /** + * Get the underlying AVCodec object. + * + * @return + */ public AVCodec getCodec() { return codec; } + /** + * Get the underlying AVCodecContext object. + * + * @return + */ public AVCodecContext getContext() { return c; } /** - * Retrieve duration of sequence, in milliseconds. + * Get stream duration in milliseconds. * * @return */ @@ -510,7 +498,7 @@ public class JJMediaReader implements AutoCloseable { } /** - * Get duration in timebase units (i.e. frames?) + * Get stream duration in timebase units. * * @return */ @@ -542,14 +530,13 @@ public class JJMediaReader implements AutoCloseable { } /** - * Decode a packet. Returns true if data is now ready. - *

- * It is ok to call this on an unopened stream: return false. + * Decode a packet into the current frame. *

- * TODO: this is now common across different formats, up-class the code. + * It is ok to call this on an unopened stream. * * @param packet - * @return + * @return true if the decoded data is now ready. + * @see #getFrame */ public boolean decode(AVPacket packet) throws AVIOException { boolean done = false; @@ -569,7 +556,12 @@ public class JJMediaReader implements AutoCloseable { } /** - * Retrieve the decoded frame. This is only valid after a call to decode() returns true. + * Retrieve the decoded frame. + *

+ * The frame is reused for each decoded frame so the value returned remains + * static for the lifetime of the JJReaderStream. + *

+ * This is only valid after a call to decode() returns true. * * @return */ diff --git a/src/notzed.jjmpeg/classes/au/notzed/jjmpeg/io/JJMediaWriter.java b/src/notzed.jjmpeg/classes/au/notzed/jjmpeg/io/JJMediaWriter.java index 4b7ba98..4c43f7d 100644 --- a/src/notzed.jjmpeg/classes/au/notzed/jjmpeg/io/JJMediaWriter.java +++ b/src/notzed.jjmpeg/classes/au/notzed/jjmpeg/io/JJMediaWriter.java @@ -47,7 +47,7 @@ import java.util.List; import java.util.logging.Logger; /** - * High level av file writer interface. + * High level media file writer interface. *

* This is still work in progress and subject to change. */ diff --git a/src/notzed.jjmpeg/classes/au/notzed/jjmpeg/io/package-info.java b/src/notzed.jjmpeg/classes/au/notzed/jjmpeg/io/package-info.java new file mode 100644 index 0000000..03c9d84 --- /dev/null +++ b/src/notzed.jjmpeg/classes/au/notzed/jjmpeg/io/package-info.java @@ -0,0 +1,7 @@ +/** + * Provides an easy to use high-level interface to reading and writing media files. + *

+ * The aim is that they take care of some of the fiddly details in using + * libavformat and libavcodec. + */ +package au.notzed.jjmpeg.io; diff --git a/src/notzed.jjmpeg/classes/au/notzed/jjmpeg/package-info.java b/src/notzed.jjmpeg/classes/au/notzed/jjmpeg/package-info.java new file mode 100644 index 0000000..bda6651 --- /dev/null +++ b/src/notzed.jjmpeg/classes/au/notzed/jjmpeg/package-info.java @@ -0,0 +1,15 @@ +/** + * An object-oriented wrapper for FFmpeg libav*. + *

+ * This package provides java bindings for parts of libavformat, libavcontext, libavutil, and so on. + *

+ * It is a mixture of lightweight binding which simply invokes the respective C library + * and higher level interfaces which provide extended functionality. Or fixes for abominations + * like {@code struct AVDictionary}. + *

+ * In most cases the classes are implemented as native methods which allows methods and + * objects to be used directly without a messy c-like interface layer. All objects derive from + * {@link au.notzed.nativez.NativeZ} which means they are (generally) mapped to singleton + * instances of the native objects and are automatically reclaimed. + */ +package au.notzed.jjmpeg; diff --git a/src/notzed.jjmpeg/gen/extract-defines.pl b/src/notzed.jjmpeg/gen/extract-defines.pl index b4be487..b0fca92 100644 --- a/src/notzed.jjmpeg/gen/extract-defines.pl +++ b/src/notzed.jjmpeg/gen/extract-defines.pl @@ -107,18 +107,20 @@ print C "int main(int argc, char **argv) {\n"; $date = `date`; chop $date; -print C "\tprintf(\"\\n/* This file was autogenerated on $date: */\\n\");\n"; +print C "\tprintf(\"\\npackage au.notzed.jjmpeg;\\n\");\n"; + +print C "\tprintf(\"\\n/**\\n * Autogenerated constants.\\n

This file was autogenerated on $date from:\\n

    \\n\");\n"; %included = (); foreach $define (@defines) { %proc = %{$process{$define}}; if ($proc{"header"} ne "" && !$included{$proc{header}}) { - print C "printf(\"/* from $ff/include/$proc{header} */\\n\");\n"; + print C "printf(\" *
  • $ff/include/$proc{header}\\n\");\n"; $included{$proc{header}} = 1; } } +print C "\tprintf(\" *
\\n */\\n\");\n"; -print C "\tprintf(\"\\npackage au.notzed.jjmpeg;\\n\");\n"; print C "\tprintf(\"public $class {\\n\");\n"; foreach $define (@defines) { diff --git a/src/notzed.jjmpeg/gen/extract-enum.pl b/src/notzed.jjmpeg/gen/extract-enum.pl index 37eff67..e0edf33 100644 --- a/src/notzed.jjmpeg/gen/extract-enum.pl +++ b/src/notzed.jjmpeg/gen/extract-enum.pl @@ -43,9 +43,9 @@ print C "#include <$h>\n"; print C "int main(int argc, char **argv) {\n"; $date = `date`; chop $date; -print C "printf(\"\\n/* This file was autogenerated on $date */\\n\");\n"; -print C "printf(\"/* from $h */\\n\");\n"; print C "printf(\"package au.notzed.jjmpeg;\\n\");\n"; +print C "printf(\"\\n/**\\n Autogenerated constants.\\n

This file was autogenerated on $date from:\\n\");\n"; +print C "printf(\"

    \\n
  • $h\\n
*/\\n\");\n"; print C "printf(\"public interface $t"."Bits {\\n\");\n"; scanCopyright($h); -- 2.39.2