From c57b3751d9be3c717634b752f8afcdb56506dadb Mon Sep 17 00:00:00 2001 From: Not Zed Date: Wed, 18 Nov 2020 22:06:08 +1030 Subject: [PATCH] Added a volume normalisation filter. Use filter graph for rate and format conversion. Change to using fixed output frequency. --- music-player.c | 248 ++++++++++++++++++++++++++++++++++--------------- 1 file changed, 175 insertions(+), 73 deletions(-) diff --git a/music-player.c b/music-player.c index f22632b..1e4d275 100644 --- a/music-player.c +++ b/music-player.c @@ -18,6 +18,9 @@ #include #include +#include +#include +#include #include #include @@ -59,12 +62,13 @@ struct voice_audio_msg { int audio_init_voice(struct audio_player *ap); int audio_close_voice(struct audio_player *ap); int audio_voice_speak(struct audio_player *ap, const char *text); -static void handle_audio_msg(struct audio_player *ap, struct voice_audio_msg *msg); #ifndef VOICE_MT #define audio_close_voice close_voice #define audio_init_voice init_voice #define voice_speak_text audio_voice_speak +#else +static void handle_audio_msg(struct audio_player *ap, struct voice_audio_msg *msg); #endif /* @@ -86,8 +90,9 @@ struct audio_player { char *device; snd_pcm_t *aud; // output device snd_mixer_t *amixer; // volume control + int sample_rate; // desired/actual sample rate of hardware - // voice output + // voice output TODO: can this go into the filter graph source? pthread_t voice_thread; struct ez_port *voice_port; // player -> voice thread SwrContext *voice_swr; @@ -97,6 +102,9 @@ struct audio_player { volatile int voice_rate; volatile int voice_cur_seq; + size_t buffer_size; // for output conversion if required + uint8_t *buffer; + // playlist management dbindex *index; dbfile *playing; @@ -124,9 +132,12 @@ struct audio_player { AVCodecContext *cc; AVStream *audio; - SwrContext *swr; // if needed for output - size_t buffer_size; // for output conversion if required - uint8_t *buffer; + // filter graph for volume normalisation and rate conversion + AVFilterGraph *fg; + AVFilterContext *asource_ctx; + AVFilterContext *anorm_ctx; + AVFilterContext *aformat_ctx; + AVFilterContext *asink_ctx; // pre-frame state //AVFrame *frame;// unused? @@ -142,6 +153,8 @@ int audio_mixer_adjust(struct audio_player *ap, int delta); struct audio_player *audio_player_new(const char *device) { struct audio_player *ap = calloc(sizeof(*ap), 1); + ap->sample_rate = 48000; + ap->player = notify_reader_new(NOTIFY_PLAYER); ap->device = strdup(device); ap->index = dbindex_open(MAIN_INDEX); @@ -221,10 +234,10 @@ void audio_player_free(struct audio_player *ap) { dbindex_close(ap->index); - swr_free(&ap->swr); + free(ap->playing_path); + dbfile_free(ap->playing); free(ap->poll); - free(ap->buffer); free(ap->device); free(ap); } @@ -319,9 +332,7 @@ static unsigned int hw_sample_rate(struct audio_player *ap) { * The audio device will be in the READY state when this returns successfully. */ int audio_init_pcm(struct audio_player *ap) { - AVCodecContext *cc = ap->cc; int res; - int init = 1; printf("audio_init_pcm\n"); @@ -329,55 +340,17 @@ int audio_init_pcm(struct audio_player *ap) { res = snd_pcm_open(&ap->aud, ap->device ? ap->device : "default", SND_PCM_STREAM_PLAYBACK, 0); if (res < 0) return res; - } else { - if (hw_sample_rate(ap) == cc->sample_rate) { - printf(" rate unchanged\n"); - init = 0; - } else if (snd_pcm_state(ap->aud) == SND_PCM_STATE_RUNNING) { - // other states? - res = snd_pcm_drain(ap->aud); - } - } - if (init) { res = snd_pcm_set_params(ap->aud, SND_PCM_FORMAT_S16_LE, SND_PCM_ACCESS_RW_INTERLEAVED, 2, - cc->sample_rate, + ap->sample_rate, 1, 500000); if (res < 0) return res; - } - - // Check actual sample rate of hardware - int actual_rate = hw_sample_rate(ap); - - // Create resampler if needed - if (cc->sample_fmt != AV_SAMPLE_FMT_S16 - || cc->channel_layout != AV_CH_LAYOUT_STEREO - || cc->sample_rate != actual_rate) { - printf(" soft resample %ld:%s:%d -> %d:%s:%d\n", - cc->channel_layout, av_get_sample_fmt_name(cc->sample_fmt), cc->sample_rate, - AV_CH_LAYOUT_STEREO, av_get_sample_fmt_name(AV_SAMPLE_FMT_S16), actual_rate); - - ap->swr = swr_alloc_set_opts(ap->swr, - AV_CH_LAYOUT_STEREO, // out_ch_layout - AV_SAMPLE_FMT_S16, // out_sample_fmt - actual_rate, // out_sample_rate - cc->channel_layout, // in_ch_layout - cc->sample_fmt, // in_sample_fmt - cc->sample_rate, // in_sample_rate - 0, // log_offset - NULL); // log_ctx - if (ap->swr == NULL) { - printf("swr create failed\n"); - return -1; - } - swr_init(ap->swr); - } else { - swr_free(&ap->swr); + ap->sample_rate = hw_sample_rate(ap); } return audio_init_poll(ap); @@ -400,34 +373,117 @@ int audio_stop_pcm(struct audio_player *ap) { void audio_close_pcm(struct audio_player *ap) { int res; - snd_pcm_drop(ap->aud); - if (ap->paused) { - res = snd_pcm_pause(ap->aud, 0); + if (ap->aud) { + snd_pcm_drop(ap->aud); + if (ap->paused) { + res = snd_pcm_pause(ap->aud, 0); + } + res = snd_pcm_drain(ap->aud); + snd_pcm_close(ap->aud); + ap->aud = NULL; } - res = snd_pcm_drain(ap->aud); - snd_pcm_close(ap->aud); - ap->aud = NULL; +} + +int audio_init_filter(struct audio_player *ap) { + char tmp[256]; + int res = -1; + + if (ap->fg) { + // FIXME: only reset if the input rate changed + avfilter_graph_free(&ap->fg); + } + + ap->fg = avfilter_graph_alloc(); + if (!ap->fg) + goto fail; + + const AVFilter *asource = avfilter_get_by_name("abuffer"); + const AVFilter *anorm = avfilter_get_by_name("loudnorm"); + const AVFilter *aformat = avfilter_get_by_name("aformat"); + const AVFilter *asink = avfilter_get_by_name("abuffersink"); + + AVCodecContext *avctx = ap->cc; + AVRational time_base = ap->audio->time_base; + + sprintf(tmp, "time_base=%d/%d:sample_rate=%d:sample_fmt=%s:channel_layout=0x%"PRIx64, + time_base.num, time_base.den, avctx->sample_rate, + av_get_sample_fmt_name(avctx->sample_fmt), + avctx->channel_layout); + fprintf(stderr, "asource: %s\n", tmp); + res = avfilter_graph_create_filter(&ap->asource_ctx, asource, "src", tmp, NULL, ap->fg); + if (res < 0) + goto fail; + + res = avfilter_graph_create_filter(&ap->anorm_ctx, anorm, "norm", "", NULL, ap->fg); + if (res < 0) + goto fail; + + sprintf(tmp, "sample_fmts=%s:sample_rates=%d:channel_layouts=0x%lx", + av_get_sample_fmt_name(AV_SAMPLE_FMT_S16), ap->sample_rate, + (uint64_t)AV_CH_LAYOUT_STEREO); + fprintf(stderr, "aformat: %s\n", tmp); + res = avfilter_graph_create_filter(&ap->aformat_ctx, aformat, "format", tmp, NULL, ap->fg); + if (res < 0) + goto fail; + + res = avfilter_graph_create_filter(&ap->asink_ctx, asink, "sink", "", NULL, ap->fg); + if (res < 0) + goto fail; + + if ((res = avfilter_link(ap->asource_ctx, 0, ap->anorm_ctx, 0)) < 0) + goto fail; + printf("link: %d\n", res); + if ((res = avfilter_link(ap->anorm_ctx, 0, ap->aformat_ctx, 0)) < 0) + goto fail; + printf("link: %d\n", res); + if ((res = avfilter_link(ap->aformat_ctx, 0, ap->asink_ctx, 0)) < 0) + goto fail; + printf("link: %d\n", res); + + if ((res = avfilter_graph_config(ap->fg, NULL)) < 0) + goto fail; + + printf("setup ok\n"); + + printf("output channels: %d\n", av_buffersink_get_channels(ap->asink_ctx)); + printf("output format: %s %d\n", + av_get_sample_fmt_name(av_buffersink_get_format(ap->asink_ctx)), + av_buffersink_get_format(ap->asink_ctx)); + + printf("output rate: %d\n", av_buffersink_get_sample_rate(ap->asink_ctx)); + return 0; + +fail: + av_log(NULL, AV_LOG_ERROR, "error configuring the filter graph\n"); + return res; } /** * New frame ready. * Set up pointers for pcm output. */ -int audio_init_frame(struct audio_player *ap, AVFrame *frame) { - if (ap->swr) { - int samples = swr_get_out_samples(ap->swr, frame->nb_samples); - - if (ap->buffer_size < samples * 4) { - ap->buffer_size = samples * 4; - ap->buffer = realloc(ap->buffer, ap->buffer_size); - } - ap->nsamples = swr_convert(ap->swr, &ap->buffer, samples, (const uint8_t **)frame->data, frame->nb_samples); - ap->data[0] = (void *)ap->buffer; - } else { - ap->data[0] = (void *)frame->data[0]; - ap->nsamples = frame->nb_samples; +int audio_init_frame(struct audio_player *ap, AVFrame *iframe, AVFrame *frame) { + ap->data[0] = (void *)frame->data[0]; + ap->nsamples = frame->nb_samples; + ap->pos = iframe->pts; + + // Convert pts from filter timebase to source timebase, for approximate seeking to match the audio output + // TODO: only do this when seeking? + if (ap->asink_ctx) { + AVRational sb = ap->audio->time_base; + AVRational tb = av_buffersink_get_time_base(ap->asink_ctx); + uint64_t p = frame->pts; + + p = p * sb.den * tb.num; + p = p / sb.num / tb.den; + ap->pos = p; + //printf("init frame ipos=%ld * %d/%d opos=%ld * %d/%d = %ld\n", iframe->pts, sb.num, sb.den, frame->pts, tb.num, tb.den, p); } - ap->pos = frame->pts; + + //init frame ipos=197222400 * 1/14112000 opos=527984 * 1/48000 + // rebase + // + return 0; } @@ -458,6 +514,13 @@ int audio_send_pcm(struct audio_player *ap) { } int audio_close_media(struct audio_player *ap) { + // TODO: just keep it around and reset if required? + avfilter_graph_free(&ap->fg); + ap->asource_ctx = NULL; + ap->anorm_ctx = NULL; + ap->aformat_ctx = NULL; + ap->asink_ctx = NULL; + avcodec_free_context(&ap->cc); avformat_close_input(&ap->fc); ap->audio = NULL; @@ -508,7 +571,11 @@ int audio_init_media(struct audio_player *ap, const char *path) { printf("codec layout: %lu\n", ap->cc->channel_layout); printf("codec format: %s\n", av_get_sample_fmt_name(ap->cc->sample_fmt)); - return audio_init_pcm(ap); + res = audio_init_pcm(ap); + if (res < 0) + goto fail; + + return audio_init_filter(ap); fail: perror("init media"); audio_close_media(ap); @@ -565,7 +632,9 @@ int audio_prev_file(struct audio_player *ap) { void audio_player_control(struct audio_player *ap) { int ready = notify_msg_ready(ap->player); +#ifdef VOICE_MT mainloop: +#endif while (!ap->quit && (ap->cc == NULL || ap->paused || ready || ap->paused_tmp)) { int res; @@ -645,7 +714,7 @@ void audio_player_control(struct audio_player *ap) { int64_t min = rel > 0 ? (int64_t)((now - rel) * AV_TIME_BASE) + 2 : INT64_MIN; int64_t max = rel < 0 ? (int64_t)((now - rel) * AV_TIME_BASE) - 2 : INT64_MAX; - printf("seek %f%+f %ld %ld %ld\n", now, rel, min, seek, max); + printf("seek res %f%+f %ld %ld %ld\n", now, rel, min, seek, max); res = avformat_seek_file(ap->fc, -1, min, seek, max, 0); } else { double rel = s->stamp; @@ -653,10 +722,17 @@ void audio_player_control(struct audio_player *ap) { int64_t min = rel > now ? seek - 2 : INT64_MIN; int64_t max = rel < now ? seek + 2 : INT64_MAX; - printf("seek %f %ld %ld %ld\n", rel, min, seek, max); + printf("seek abs %f %ld %ld %ld\n", rel, min, seek, max); res = avformat_seek_file(ap->fc, -1, min, seek, max, 0); } avcodec_flush_buffers(ap->cc); + // flush filter graph as well + if (ap->fg) { + // TODO: or should it just drain the contents? that might be expensive + avfilter_graph_free(&ap->fg); + audio_init_filter(ap); + ap->nsamples = 0; + } } break; case NOTIFY_PLAY_NEXT: @@ -766,6 +842,7 @@ void audio_player_control(struct audio_player *ap) { // TODO: the play queue / play list / etc void audio_player_loop(struct audio_player *ap) { AVFrame *frame = av_frame_alloc(); + AVFrame *oframe = av_frame_alloc(); audio_next_file(ap); @@ -784,13 +861,37 @@ void audio_player_loop(struct audio_player *ap) { continue; } + // Check for anything from the filter + if (ap->fg) { + av_frame_unref(oframe); + res = av_buffersink_get_frame(ap->asink_ctx, oframe); + //printf("1 read filter samples (%d %s) %d\n", res, strerror(res), oframe->nb_samples); + if ((res != AVERROR_EOF && res != AVERROR(EAGAIN)) || res >= 0) { + res = audio_init_frame(ap, frame, oframe); + res = audio_send_pcm(ap); + continue; + } + } + // Have file open / reading if (ap->cc) { AVPacket pkt; res = avcodec_receive_frame(ap->cc, frame); if (res >= 0) { - res = audio_init_frame(ap, frame); + if (ap->fg) { + //printf("0 write frame samples %d\n", frame->nb_samples); + res = av_buffersrc_write_frame(ap->asource_ctx, frame); + av_frame_unref(oframe); + res = av_buffersink_get_frame(ap->asink_ctx, oframe); + //printf("0 read filter samples (%d %s) %d\n", res, strerror(res), oframe->nb_samples); + if ((res != AVERROR_EOF && res != AVERROR(EAGAIN)) || res >= 0) { + res = audio_init_frame(ap, frame, oframe); + res = audio_send_pcm(ap); + } + continue; + } + res = audio_init_frame(ap, frame, frame); res = audio_send_pcm(ap); continue; } @@ -808,6 +909,7 @@ void audio_player_loop(struct audio_player *ap) { } av_frame_free(&frame); + av_frame_free(&oframe); // shutdown shit } -- 2.39.2