From e1e2763f86e7c99227a2afce0675499609b489fc Mon Sep 17 00:00:00 2001 From: Michael Zucchi Date: Sat, 13 Jul 2019 12:47:19 +0930 Subject: [PATCH] Added an octave interface for reading multi-stream video files. --- contrib/octave/Makefile | 39 ++++ contrib/octave/README | 74 +++++++ contrib/octave/javaclasspath.txt | 1 + contrib/octave/jjPlayAll.m | 56 +++++ contrib/octave/jjVideoClose.m | 31 +++ contrib/octave/jjVideoFrame.m | 86 ++++++++ contrib/octave/jjVideoNext.m | 41 ++++ contrib/octave/jjVideoOpen.m | 35 ++++ contrib/octave/jjoctave/VideoReader.java | 251 +++++++++++++++++++++++ 9 files changed, 614 insertions(+) create mode 100644 contrib/octave/Makefile create mode 100644 contrib/octave/README create mode 100644 contrib/octave/javaclasspath.txt create mode 100644 contrib/octave/jjPlayAll.m create mode 100644 contrib/octave/jjVideoClose.m create mode 100644 contrib/octave/jjVideoFrame.m create mode 100644 contrib/octave/jjVideoNext.m create mode 100644 contrib/octave/jjVideoOpen.m create mode 100644 contrib/octave/jjoctave/VideoReader.java diff --git a/contrib/octave/Makefile b/contrib/octave/Makefile new file mode 100644 index 0000000..19758f8 --- /dev/null +++ b/contrib/octave/Makefile @@ -0,0 +1,39 @@ +# +# Copyright (C) 2019 Michael Zucchi +# +# This is the copyright for jjmpeg/contrib/octave +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# + +TOP=../.. + +jjmpeg-octave_JAVA := jjoctave/VideoReader.java \ + $(shell find \ + $(TOP)/../nativez/src/notzed.nativez/classes \ + $(TOP)/src/notzed.jjmpeg/classes \ + $(TOP)/bin/gen/notzed.jjmpeg/classes \ + -name '*.java' -a \! -name 'module-info.java') + +$(TOP)/bin/jjmpeg-octave.jar: $(jjmpeg-octave_JAVA) + rm -rf $(TOP)/bin/octave + mkdir -p $(TOP)/bin/octave + javac -source 8 \ + -d $(TOP)/bin/octave \ + $(jjmpeg-octave_JAVA) + jar cf $(TOP)/bin/jjmpeg-octave.jar \ + -C $(TOP)/bin/octave . + +clean: + rm -rf $(TOP)/bin/jjmpeg-octave.jar $(TOP)/bin/octave diff --git a/contrib/octave/README b/contrib/octave/README new file mode 100644 index 0000000..4eced80 --- /dev/null +++ b/contrib/octave/README @@ -0,0 +1,74 @@ + +Copyright (C) 2017-2019 Michael Zucchi + +See the section LICENSE for details. + +INTRODUCTION +------------ + +This is a module for reading video data inside octave. + +It supports multiple streams and 16-bit frames and all of the input +formats and protocols available in jjmpeg. + +There is a simplified interface in jjoctave/VideoReader.java and the +jj*m files define a basic interface for reading frames and handling +data layout. + +The octave sources are portable and work with a similar +mostly-compatible tool but require different installation +instructions. + +COMPILE +------- + +First the normal build process must be completed successfully. This +is required to create the native libraries and generated sources. + +Octave uses Java 8 and is thus incompatible with the standard +modular-Java build of both jjmpeg and nativez. + +A trivial Makefile is included in this directory to recompile the core +classes of jjmpeg and nativez into a single jar file. It requires +that the nativez source is available in the same directory as jjmpeg +and that the jdk-8 javac and jar are on the path. + +$ make + +It creates a new jar in `$(TOP)/bin/jjmpeg-octave.jar'. + +jjPlayAll.m can be used to check the build. libjjmpeg.so and +libnativez.so must be in the library path before invoking octave. + +INSTALL +------- + +Copy the jj*m files to a directory and add a javaclasspath.txt file +which includes the path to the jjmpeg-octave.jar. + +libnativez.so and libjjmpeg.so must be in the LD_LIBRARY_PATH. + +For usage read the .m files. + +LICENSE +------- + +This is licensed under GNU General Public License version 3, see +../../src/jjmpeg.nativez/legal/LICENSE for the full details. + + Copyright (C) 2019 Michael Zucchi + + This program is free software: you can redistribute it and/or + modify it under the terms of the GNU General Public License as + published by the Free Software Foundation, either version 3 of the + License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see + . + diff --git a/contrib/octave/javaclasspath.txt b/contrib/octave/javaclasspath.txt new file mode 100644 index 0000000..677b0e8 --- /dev/null +++ b/contrib/octave/javaclasspath.txt @@ -0,0 +1 @@ +../../bin/jjmpeg-octave.jar diff --git a/contrib/octave/jjPlayAll.m b/contrib/octave/jjPlayAll.m new file mode 100644 index 0000000..412c8a0 --- /dev/null +++ b/contrib/octave/jjPlayAll.m @@ -0,0 +1,56 @@ + +%% jjPlayAll play multi-stream video file. +% +% This simple demo will play all streams of a +% video simultaneously in a single figure. +% +% jjPlayAll(src) +% +% src Path to video file. + +% +% Copyright (C) 2019 Michael Zucchi +% +% This is the copyright for jjmpeg octave functions. +% +% This program is free software: you can redistribute it and/or modify +% it under the terms of the GNU General Public License as published by +% the Free Software Foundation, either version 3 of the License, or +% (at your option) any later version. +% +% This program is distributed in the hope that it will be useful, +% but WITHOUT ANY WARRANTY; without even the implied warranty of +% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +% GNU General Public License for more details. +% +% You should have received a copy of the GNU General Public License +% along with this program. If not, see . +% + +function jjPlayAll(src, sid) + + v = jjVideoOpen(src); + + streams = javaMethod('getStreamCount', v); + + images = cell(streams, 1); + ncols = ceil(sqrt(streams)); + nrows = ceil(streams/ncols); + + sid = jjVideoNext(v); + while sid ~= -1 + did = sid+1; + frame = jjVideoFrame(v, sid, 'rgb24'); + if isempty(images{did}) + subplot(nrows, ncols, did); + images{did} = imshow(frame); + else + set(images{did}, 'CData', frame); + end + drawnow; + sid = jjVideoNext(v); + end + + jjVideoClose(v); +end + diff --git a/contrib/octave/jjVideoClose.m b/contrib/octave/jjVideoClose.m new file mode 100644 index 0000000..00e1913 --- /dev/null +++ b/contrib/octave/jjVideoClose.m @@ -0,0 +1,31 @@ + +%% jjVideoClose Close video file. +% +% jjVideoClose(video) +% +% Closes video file, it should no longer be used. +% +% See also jjVideoOpen(). + +% +% Copyright (C) 2019 Michael Zucchi +% +% This is the copyright for jjmpeg octave functions. +% +% This program is free software: you can redistribute it and/or modify +% it under the terms of the GNU General Public License as published by +% the Free Software Foundation, either version 3 of the License, or +% (at your option) any later version. +% +% This program is distributed in the hope that it will be useful, +% but WITHOUT ANY WARRANTY; without even the implied warranty of +% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +% GNU General Public License for more details. +% +% You should have received a copy of the GNU General Public License +% along with this program. If not, see . +% + +function jjVideoClose(video) + javaMethod('close', video); +end diff --git a/contrib/octave/jjVideoFrame.m b/contrib/octave/jjVideoFrame.m new file mode 100644 index 0000000..92110e8 --- /dev/null +++ b/contrib/octave/jjVideoFrame.m @@ -0,0 +1,86 @@ +%% jjVideoFrame Read video frame pixels. +% +% [frame] = jjVideoFrame(video, streamid, format) +% +% Retrieve the video frame pixels for the current frame. +% +% video A previously opened video. +% +% streamid The last streamid returned by the last call to +% jjNextFrame(video). Default is 0. +% +% format The desired pixel format. Default is 'rgb24'. +% +% 'rgb24' 3 channel 'uint8' +% 'rgb48' 3 channel 'uint16' +% 'grey8' 1 channel 'uint8' +% 'grey16' 1 channel 'uint16' +% +% frame The image pixels as a [height width depth] matrix of the +% given type. +% +% See also jjVideoNext() + +% +% Copyright (C) 2019 Michael Zucchi +% +% This is the copyright for jjmpeg octave functions. +% +% This program is free software: you can redistribute it and/or modify +% it under the terms of the GNU General Public License as published by +% the Free Software Foundation, either version 3 of the License, or +% (at your option) any later version. +% +% This program is distributed in the hope that it will be useful, +% but WITHOUT ANY WARRANTY; without even the implied warranty of +% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +% GNU General Public License for more details. +% +% You should have received a copy of the GNU General Public License +% along with this program. If not, see . +% + +function [frame] = jjVideoFrame(v, streamid, format) + switch nargin + case 1 + streamid = 1; + format = 'rgb24'; + case 2 + format = 'rgb24'; + end + + % This is hardcoded as matlab isn't compatible with octave for field refs + switch format + case 'rgb24' + fmt = 0; + case 'grey8' + fmt = 1; + case 'rgb48' + fmt = 2; + case 'grey16' + fmt = 3; + otherwise + error('unknown pixel format'); + end + + frame = javaMethod('getPixels', v, streamid, fmt); + width = javaMethod('getWidth', v, streamid); + height = javaMethod('getHeight', v, streamid); + + switch fmt + case 0 + frame = typecast(frame, 'uint8'); + frame = reshape(frame, [3 width height]); + frame = permute(frame, [3 2 1]); + case 1 + frame = typecast(frame, 'uint8'); + frame = reshape(frame, [height width]); + case 2 + frame = typecast(frame, 'uint16'); + frame = reshape(frame, [3 width height]); + frame = permute(frame, [3 2 1]); + case 3 + frame = typecast(frame, 'uint16'); + frame = reshape(frame, [height width]); + end +end diff --git a/contrib/octave/jjVideoNext.m b/contrib/octave/jjVideoNext.m new file mode 100644 index 0000000..eaefd75 --- /dev/null +++ b/contrib/octave/jjVideoNext.m @@ -0,0 +1,41 @@ +%% jjVideoNext Advance to the next frame. +% +% streamid = jjVideoNext(video) +% +% Advance to the next frame in the video. +% +% video An opened video. +% +% streamid[in] The desired streamid, -1 or missing for any stream. +% +% streamid[out] The index of the stream the frame belongs to. +% +% See also jjVideoOpen(), jjVideoFrame() + +% +% Copyright (C) 2019 Michael Zucchi +% +% This is the copyright for jjmpeg octave functions. +% +% This program is free software: you can redistribute it and/or modify +% it under the terms of the GNU General Public License as published by +% the Free Software Foundation, either version 3 of the License, or +% (at your option) any later version. +% +% This program is distributed in the hope that it will be useful, +% but WITHOUT ANY WARRANTY; without even the implied warranty of +% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +% GNU General Public License for more details. +% +% You should have received a copy of the GNU General Public License +% along with this program. If not, see . +% + +function [streamid] = jjVideoNext(video, streamid) + if nargin == 1 + streamid = javaMethod('nextFrame', video); + else + streamid = javaMethod('nextFrame', video, streamid); + end +end + diff --git a/contrib/octave/jjVideoOpen.m b/contrib/octave/jjVideoOpen.m new file mode 100644 index 0000000..2aa284e --- /dev/null +++ b/contrib/octave/jjVideoOpen.m @@ -0,0 +1,35 @@ + +%% jjVideoOpen Open video file. +% +% video = jjVideoOpen(path) +% +% Open video file for reading. +% +% path The file path. +% +% video The video handle (object). +% +% See also jjVideoClose(). + +% +% Copyright (C) 2019 Michael Zucchi +% +% This is the copyright for jjmpeg octave functions. +% +% This program is free software: you can redistribute it and/or modify +% it under the terms of the GNU General Public License as published by +% the Free Software Foundation, either version 3 of the License, or +% (at your option) any later version. +% +% This program is distributed in the hope that it will be useful, +% but WITHOUT ANY WARRANTY; without even the implied warranty of +% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +% GNU General Public License for more details. +% +% You should have received a copy of the GNU General Public License +% along with this program. If not, see . +% + +function [video] = jjVideoOpen(path) + video = javaObject('jjoctave.VideoReader', path); +end diff --git a/contrib/octave/jjoctave/VideoReader.java b/contrib/octave/jjoctave/VideoReader.java new file mode 100644 index 0000000..150d88e --- /dev/null +++ b/contrib/octave/jjoctave/VideoReader.java @@ -0,0 +1,251 @@ +/* + * Copyright (c) 2019 Michael Zucchi + * + * This file is part of jjmpeg, a java binding to ffmpeg's libraries. + * + * jjmpeg is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * jjmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with jjmpeg. If not, see . + */ +package jjoctave; + +import au.notzed.jjmpeg.AVFrame; +import au.notzed.jjmpeg.AVMediaTypeBits; +import au.notzed.jjmpeg.AVPixelFormat; +import au.notzed.jjmpeg.AVPixelReader; +import au.notzed.jjmpeg.io.JJMediaReader; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/** + * Object for managing multi-stream video sources for octave use. + *

+ * This exports the frame pixels as native AVPixelFormat formats as signed java types, + * i.e. channel-packed in row order. + *

+ * octave expects images to be in planar format using unsigned types. The following code + * can be used to transform the data, if required. + * 

+ *   frame = javaMethod('getPixels', v, streamid, fmt);
+ *   width = javaMethod('getWidth', v, streamid);
+ *   height = javaMethod('getHeight', v, streamid);
+ *
+ *   switch fmt
+ *     case 0
+ *       frame = typecast(frame, 'uint8');
+ *       frame = reshape(frame, [3 width height]);
+ *       frame = permute(frame, [3 2 1]);
+ *     case 1
+ *       frame = typecast(frame, 'uint8');
+ *       frame = reshape(frame, [height width]);
+ *     case 2
+ *       frame = typecast(frame, 'uint16');
+ *       frame = reshape(frame, [3 width height]);
+ *       frame = permute(frame, [3 2 1]);
+ *     case 3
+ *       frame = typecast(frame, 'uint16');
+ *       frame = reshape(frame, [height width]);
+ *   end
+ *
+ */ +public class VideoReader { + + JJMediaReader reader; + final JJMediaReader.JJReaderVideo[] videos; + final AVFrame[] frames; + final AVPixelReader[] pixels; + + public static final int FMT_UINT8_RGB = 0; + public static final int FMT_UINT8_I = 1; + public static final int FMT_UINT16_RGB = 2; + public static final int FMT_UINT16_I = 3; + + /** + * Open a video file for reading frames. + * + * @param path Path to file. + * @throws IOException On an i/o error. + * @see #close + */ + public VideoReader(String path) throws IOException { + reader = new JJMediaReader(path); + List streams = new ArrayList<>(); + + for (JJMediaReader.JJReaderStream rs: reader.streams()) { + if (rs.getType() == AVMediaTypeBits.AVMEDIA_TYPE_VIDEO) { + JJMediaReader.JJReaderVideo rv = (JJMediaReader.JJReaderVideo) rs; + rs.open(); + rs.getContext().setThreadCount(4); + streams.add(rv); + } + } + + videos = new JJMediaReader.JJReaderVideo[streams.size()]; + frames = new AVFrame[videos.length]; + pixels = new AVPixelReader[videos.length]; + for (int i = 0; i < streams.size(); i++) { + JJMediaReader.JJReaderVideo rv = streams.get(i); + videos[i] = rv; + frames[i] = rv.getFrame(); + pixels[i] = frames[i].getPixelReader(rv.getWidth(), rv.getHeight(), 0); + } + } + + /** + * Close the video stream. + *

+ * Video streams should be closed explicitly when they are no longer needed, + * however they will be garabage collected either way. + */ + public void close() { + if (reader != null) { + reader.close(); + reader = null; + Arrays.fill(videos, null); + Arrays.fill(frames, null); + Arrays.fill(pixels, null); + } + } + + /** + * Retrieve the number of video streams in the file. + * + * @return The number of streams found. + */ + public int getStreamCount() { + return videos.length; + } + + /** + * Retrieve the frame width of the specified stream. + *

+ * This may be called at any time. + * + * @param sid Stream id. + * @return The video frame width. + * @throws ArrayIndexOutOfBoundsException if sid is invalid. + */ + public int getWidth(int sid) { + return videos[sid].getWidth(); + } + + /** + * Retrieve the frame height of the specified stream. + *

+ * This may be called at any time. + * + * @param sid Stream id. + * @return The video frame height. + * @throws ArrayIndexOutOfBoundsException if sid is invalid. + */ + public int getHeight(int sid) { + return videos[sid].getHeight(); + } + + /** + * Retrieve the timestamp of the current frame in milliseconds. + *

+ * The timestamp is only valid if the sid matches the last sid + * returned by nextFrame(). + * + * @param sid Stream to query. + * @return Frame timestamp in milliseconds. + */ + public long getTimeStamp(int sid) { + return videos[sid].convertPTS(frames[sid].getPTS()); + } + + /** + * Retrieve frame pixels in a specific format. + *

+ * This converts the frame to the specified format and + * returns it as a single-dimensional vector of the specified type. + *

+ * The frame data is only valid if the sid matches the last sid + * returned by nextFrame(). + * + * @param sid Stream id of frame. + * @param format One of the FMT_* constants. + * @return A primitive array containing the pixels. It will contain width * height * depth + * elements in channel-packed row order. + * @see #nextFrame() + */ + public Object getPixels(int sid, int format) { + JJMediaReader.JJReaderVideo vs = videos[sid]; + int width = vs.getWidth(); + int height = vs.getHeight(); + byte[] bytes; + short[] shorts; + + switch (format) { + case FMT_UINT8_RGB: + bytes = new byte[width * height * 3]; + pixels[sid].getPixels(0, height, AVPixelFormat.AV_PIX_FMT_RGB24, bytes, 0, width * 3); + return bytes; + case FMT_UINT8_I: + bytes = new byte[width * height]; + pixels[sid].getPixels(0, height, AVPixelFormat.AV_PIX_FMT_GRAY8, bytes, 0, width); + return bytes; + case FMT_UINT16_RGB: + shorts = new short[width * height * 3]; + pixels[sid].getPixels(0, height, AVPixelFormat.AV_PIX_FMT_RGB48, shorts, 0, width * 3); + return shorts; + case FMT_UINT16_I: + shorts = new short[width * height]; + pixels[sid].getPixels(0, height, AVPixelFormat.AV_PIX_FMT_GRAY16, shorts, 0, width); + return shorts; + } + return null; + } + + /** + * Advance to the next frame of the specified stream. + *

+ * Intervening frames from other streams will be dropped. + * + * @param sid Desired stream, or -1 for any stream. + * @return The streamid of the frame received or -1 for end of file. + * @throws IOException On an i/o error. + */ + public int nextFrame(int sid) throws IOException { + int next; + + if (sid == -1) + return nextFrame(); + + do { + next = nextFrame(); + } while (next != -1 && next != sid); + + return next; + } + + /** + * Advance to the next frame in the file. + * + * @return The streamid of the frame received or -1 for end of file. + * @throws IOException On an i/o error. + */ + public int nextFrame() throws IOException { + while (true) { + JJMediaReader.JJReaderStream rs = reader.readFrame(); + if (rs == null) + return -1; + for (int i = 0; i < videos.length; i++) + if (videos[i] == rs) { + return i; + } + } + } +} -- 2.39.5