Refactor TmpFile into an abstract base class

This allows a TmpFile pointer to be either a Sync or Async (Threaded)
writer. As result we must be able to handle both RT and non RT processing.

Still, post-processing (normalization and encoding) should always
happen faster than realtime (freewheeling).

Since jack does not allow a client to change to freewheeling from within
the process-callback, the async-writer disk-thread FileFlushed is used
to initiate post-processing.

libs/ardour/ardour/export_graph_builder.h

@@ -41,7 +41,6 @@ namespace AudioGrapher {
 	template <typename T> class SndfileWriter;
 	template <typename T> class SilenceTrimmer;
 	template <typename T> class TmpFile;
-	template <typename T> class TmpFileRt;
 	template <typename T> class Threader;
 	template <typename T> class AllocatingProcessContext;
 }
@@ -165,11 +164,11 @@ class LIBARDOUR_API ExportGraphBuilder
 		typedef boost::shared_ptr<AudioGrapher::LoudnessReader> LoudnessReaderPtr;
 		typedef boost::shared_ptr<AudioGrapher::Normalizer> NormalizerPtr;
 		typedef boost::shared_ptr<AudioGrapher::TmpFile<Sample> > TmpFilePtr;
-		typedef boost::shared_ptr<AudioGrapher::TmpFileRt<Sample> > TmpFileRtPtr;
 		typedef boost::shared_ptr<AudioGrapher::Threader<Sample> > ThreaderPtr;
 		typedef boost::shared_ptr<AudioGrapher::AllocatingProcessContext<Sample> > BufferPtr;
 
-		void start_post_processing();
+		void prepare_post_processing ();
+		void start_post_processing ();
 
 		ExportGraphBuilder & parent;
 
@@ -184,7 +183,7 @@ class LIBARDOUR_API ExportGraphBuilder
 		LoudnessReaderPtr    loudness_reader;
 		boost::ptr_list<SFC> children;
 
-		PBD::ScopedConnection post_processing_connection;
+		PBD::ScopedConnectionList post_processing_connection;
 	};
 
 	// sample rate converter

libs/ardour/export_graph_builder.cc

@@ -37,6 +37,7 @@
 #include "audiographer/general/threader.h"
 #include "audiographer/sndfile/tmp_file.h"
 #include "audiographer/sndfile/tmp_file_rt.h"
+#include "audiographer/sndfile/tmp_file_sync.h"
 #include "audiographer/sndfile/sndfile_writer.h"
 
 #include "ardour/audioengine.h"
@@ -434,8 +435,10 @@ ExportGraphBuilder::Normalizer::Normalizer (ExportGraphBuilder & parent, FileSpe
 	normalizer->add_output (threader);
 
 	int format = ExportFormatBase::F_RAW | ExportFormatBase::SF_Float;
-	tmp_file.reset (new TmpFile<float> (&tmpfile_path_buf[0], format, channels, config.format->sample_rate()));
+	tmp_file.reset (new TmpFileSync<float> (&tmpfile_path_buf[0], format, channels, config.format->sample_rate()));
 	tmp_file->FileWritten.connect_same_thread (post_processing_connection,
+	                                           boost::bind (&Normalizer::prepare_post_processing, this));
+	tmp_file->FileFlushed.connect_same_thread (post_processing_connection,
 	                                           boost::bind (&Normalizer::start_post_processing, this));
 
 	add_child (new_config);
@@ -509,8 +512,9 @@ ExportGraphBuilder::Normalizer::process()
 }
 
 void
-ExportGraphBuilder::Normalizer::start_post_processing()
+ExportGraphBuilder::Normalizer::prepare_post_processing()
 {
+	// called in sync rt-context
 	float gain;
 	if (use_loudness) {
 		gain = normalizer->set_peak (loudness_reader->get_peak (config.format->normalize_lufs (), config.format->normalize_dbtp ()));
@@ -520,11 +524,20 @@ ExportGraphBuilder::Normalizer::start_post_processing()
 	for (boost::ptr_list<SFC>::iterator i = children.begin(); i != children.end(); ++i) {
 		(*i).set_peak (gain);
 	}
-	tmp_file->seek (0, SEEK_SET);
 	tmp_file->add_output (normalizer);
 	parent.normalizers.push_back (this);
 }
 
+void
+ExportGraphBuilder::Normalizer::start_post_processing()
+{
+	// called in disk-thread (when exporting in realtime)
+	tmp_file->seek (0, SEEK_SET);
+	if (!AudioEngine::instance()->freewheeling ()) {
+		AudioEngine::instance()->freewheel (true);
+	}
+}
+
 /* SRC */
 
 ExportGraphBuilder::SRC::SRC (ExportGraphBuilder & parent, FileSpec const & new_config, framecnt_t max_frames)

libs/ardour/export_handler.cc

@@ -26,6 +26,7 @@
 
 #include "pbd/convert.h"
 
+#include "ardour/audioengine.h"
 #include "ardour/audiofile_tagger.h"
 #include "ardour/debug.h"
 #include "ardour/export_graph_builder.h"
@@ -232,7 +233,12 @@ ExportHandler::process (framecnt_t frames)
 		return 0;
 	} else if (normalizing) {
 		Glib::Threads::Mutex::Lock l (export_status->lock());
-		return process_normalize ();
+		if (AudioEngine::instance()->freewheeling ()) {
+			return process_normalize ();
+		} else {
+			// wait until we're freewheeling
+			return 0;
+		}
 	} else {
 		Glib::Threads::Mutex::Lock l (export_status->lock());
 		return process_timespan (frames);

libs/ardour/session_export.cc

@@ -177,13 +177,14 @@ Session::start_audio_export (framepos_t position, bool realtime)
 		return -1;
 	}
 
+	_engine.Freewheel.connect_same_thread (export_freewheel_connection, boost::bind (&Session::process_export_fw, this, _1));
+
 	if (_realtime_export) {
 		Glib::Threads::Mutex::Lock lm (AudioEngine::instance()->process_lock ());
 		_export_rolling = true;
 		process_function = &Session::process_export_fw;
 		return 0;
 	} else {
-		_engine.Freewheel.connect_same_thread (export_freewheel_connection, boost::bind (&Session::process_export_fw, this, _1));
 		_export_rolling = true;
 		return _engine.freewheel (true);
 	}
@@ -197,14 +198,18 @@ Session::process_export (pframes_t nframes)
 	}
 
 	if (_export_rolling) {
-		/* make sure we've caught up with disk i/o, since
+		if (!_realtime_export)  {
-		we're running faster than realtime c/o JACK.
+			/* make sure we've caught up with disk i/o, since
-		*/
+			 * we're running faster than realtime c/o JACK.
-		_butler->wait_until_finished ();
+			 */
+			_butler->wait_until_finished ();
+		}
 
 		/* do the usual stuff */
 
 		process_without_events (nframes);
+	} else if (_realtime_export) {
+		fail_roll (nframes); // somehow we need to silence _ALL_ output buffers
 	}
 
 	try {
@@ -221,13 +226,14 @@ Session::process_export (pframes_t nframes)
 void
 Session::process_export_fw (pframes_t nframes)
 {
+	const bool need_buffers = _engine.freewheeling ();
 	if (_export_preroll > 0) {
 
-		if (!_realtime_export) {
+		if (need_buffers) {
 			_engine.main_thread()->get_buffers ();
 		}
 		fail_roll (nframes);
-		if (!_realtime_export) {
+		if (need_buffers) {
 			_engine.main_thread()->drop_buffers ();
 		}
 
@@ -249,11 +255,11 @@ Session::process_export_fw (pframes_t nframes)
 	if (_export_latency > 0) {
 		framepos_t remain = std::min ((framepos_t)nframes, _export_latency);
 
-		if (!_realtime_export) {
+		if (need_buffers) {
 			_engine.main_thread()->get_buffers ();
 		}
 		process_without_events (remain);
-		if (!_realtime_export) {
+		if (need_buffers) {
 			_engine.main_thread()->drop_buffers ();
 		}
 
@@ -264,11 +270,12 @@ Session::process_export_fw (pframes_t nframes)
 			return;
 		}
 	}
-	if (!_realtime_export) {
+
+	if (need_buffers) {
 		_engine.main_thread()->get_buffers ();
 	}
 	process_export (nframes);
-	if (!_realtime_export) {
+	if (need_buffers) {
 		_engine.main_thread()->drop_buffers ();
 	}
 
@@ -304,10 +311,9 @@ Session::finalize_audio_export ()
 	if (_realtime_export) {
 		Glib::Threads::Mutex::Lock lm (AudioEngine::instance()->process_lock ());
 		process_function = &Session::process_with_events;
-	} else {
-		_engine.freewheel (false);
-		export_freewheel_connection.disconnect();
 	}
+	_engine.freewheel (false);
+	export_freewheel_connection.disconnect();
 
 	_mmc->enable_send (_pre_export_mmc_enabled);
 

libs/audiographer/audiographer/sndfile/sndfile_writer.h

@@ -40,19 +40,13 @@ class SndfileWriter
 
 	virtual ~SndfileWriter () {}
 
-	SndfileWriter (SndfileWriter const & other)
-		: SndfileHandle (other)
-	{
-		init();
-	}
-
 	using SndfileHandle::operator=;
 
 	framecnt_t get_frames_written() const { return frames_written; }
 	void       reset_frames_written_count() { frames_written = 0; }
 
 	/// Writes data to file
-	void process (ProcessContext<T> const & c)
+	virtual void process (ProcessContext<T> const & c)
 	{
 		check_flags (*this, c);
 
@@ -88,7 +82,7 @@ class SndfileWriter
 		init();
 	}
 
-	void init()
+	virtual void init()
 	{
 		frames_written = 0;
 		add_supported_flag (ProcessContext<T>::EndOfInput);
@@ -97,6 +91,9 @@ class SndfileWriter
   protected:
 	std::string path;
 	framecnt_t frames_written;
+
+	private:
+	SndfileWriter (SndfileWriter const & other) {}
 };
 
 } // namespace

libs/audiographer/audiographer/sndfile/tmp_file.h

@@ -15,36 +15,14 @@ namespace AudioGrapher
 
 /// A temporary file deleted after this class is destructed
 template<typename T = DefaultSampleType>
-class TmpFile : public SndfileWriter<T>, public SndfileReader<T>
+class TmpFile
+	: public SndfileWriter<T>
+	, public SndfileReader<T>
 {
   public:
+	virtual ~TmpFile () {}
+	PBD::Signal0<void> FileFlushed;
 
-	/// \a filename_template must match the requirements for mkstemp, i.e. end in "XXXXXX"
-	TmpFile (char * filename_template, int format, ChannelCount channels, framecnt_t samplerate)
-		: SndfileHandle (g_mkstemp(filename_template), true, SndfileBase::ReadWrite, format, channels, samplerate)
-		, filename (filename_template)
-	{}
-
-	TmpFile (int format, ChannelCount channels, framecnt_t samplerate)
-	  : SndfileHandle (fileno (tmpfile()), true, SndfileBase::ReadWrite, format, channels, samplerate)
-	{}
-
-	TmpFile (TmpFile const & other) : SndfileHandle (other) {}
-	using SndfileHandle::operator=;
-
-	~TmpFile()
-	{
-		/* explicitly close first, some OS (yes I'm looking at you windows)
-		 * cannot delete files that are still open
-		 */
-		if (!filename.empty()) {
-			SndfileBase::close();
-			std::remove(filename.c_str());
-		}
-	}
-
-  private:
-	std::string filename;
 };
 
 } // namespace

libs/audiographer/audiographer/sndfile/tmp_file_rt.h

@@ -10,18 +10,21 @@
 
 #include "audiographer/flag_debuggable.h"
 #include "audiographer/sink.h"
+#include "sndfile_writer.h"
 #include "sndfile_reader.h"
+#include "tmp_file.h"
 
 namespace AudioGrapher
 {
 
-/// A temporary file deleted after this class is destructed
+	static const framecnt_t rb_chunksize = 8192; // samples
+
+/** A temporary file deleted after this class is destructed
+ * with realtime safe background thread writer.
+ */
 template<typename T = DefaultSampleType>
 class TmpFileRt
-	: public virtual SndfileReader<T>
+	: public TmpFile<T>
-  , public virtual SndfileBase
-  , public Sink<T>
-  , public FlagDebuggable<>
 {
   public:
 
@@ -29,7 +32,8 @@ class TmpFileRt
 	TmpFileRt (char * filename_template, int format, ChannelCount channels, framecnt_t samplerate)
 		: SndfileHandle (g_mkstemp(filename_template), true, SndfileBase::ReadWrite, format, channels, samplerate)
 		, filename (filename_template)
-  , _rb (samplerate * channels)
+  , _chunksize (rb_chunksize * channels)
+  , _rb (std::max (_chunksize * 16, 5 * samplerate * channels))
 	{
 		init ();
 	}
@@ -50,81 +54,76 @@ class TmpFileRt
 		pthread_cond_destroy  (&_data_ready);
 	}
 
-	framecnt_t get_frames_written() const { return frames_written; }
-	void       reset_frames_written_count() { frames_written = 0; }
-
 	/// Writes data to file
 	void process (ProcessContext<T> const & c)
 	{
-		check_flags (*this, c);
+		SndfileWriter<T>::check_flags (*this, c);
 
-		if (SndfileReader<T>::throw_level (ThrowStrict) && c.channels() != channels()) {
+		if (SndfileWriter<T>::throw_level (ThrowStrict) && c.channels() != SndfileHandle::channels()) {
 			throw Exception (*this, boost::str (boost::format
 				("Wrong number of channels given to process(), %1% instead of %2%")
-				% c.channels() % channels()));
+				% c.channels() % SndfileHandle::channels()));
 		}
 
-		if (SndfileReader<T>::throw_level (ThrowProcess) && _rb.write_space() < c.frames()) {
+		if (SndfileWriter<T>::throw_level (ThrowProcess) && _rb.write_space() < c.frames()) {
 			throw Exception (*this, boost::str (boost::format
 				("Could not write data to ringbuffer/output file (%1%)")
-				% strError()));
+				% SndfileHandle::strError()));
 		}
 
 		_rb.write (c.data(), c.frames());
 
+		if (c.has_flag(ProcessContext<T>::EndOfInput)) {
+			_capture = false;
+			SndfileWriter<T>::FileWritten (filename);
+		}
+
 		if (pthread_mutex_trylock (&_disk_thread_lock) == 0) {
 			pthread_cond_signal (&_data_ready);
 			pthread_mutex_unlock (&_disk_thread_lock);
 		}
-
-		if (c.has_flag(ProcessContext<T>::EndOfInput)) {
-			end_write (); // XXX not rt-safe -- TODO add API call to flush
-			FileWritten (filename);
-		}
 	}
 
 	using Sink<T>::process;
 
-	PBD::Signal1<void, std::string> FileWritten;
-
 	void disk_thread ()
 	{
-		const size_t chunksize = 8192; // samples
+		T *framebuf = (T*) malloc (_chunksize * sizeof (T));
-		T *framebuf = (T*) malloc (chunksize * sizeof (T));
 
 		pthread_mutex_lock (&_disk_thread_lock);
 
-		while (1) {
+		while (_capture) {
+			if ((framecnt_t)_rb.read_space () >= _chunksize) {
+				_rb.read (framebuf, _chunksize);
+				framecnt_t const written = SndfileBase::write (framebuf, _chunksize);
+				assert (written == _chunksize);
+				SndfileWriter<T>::frames_written += written;
+			}
 			if (!_capture) {
 				break;
 			}
-			if (_rb.read_space () >= chunksize) {
-				_rb.read (framebuf, chunksize);
-				framecnt_t const written = write (framebuf, chunksize);
-				assert (written == chunksize);
-				frames_written += written;
-			}
 			pthread_cond_wait (&_data_ready, &_disk_thread_lock);
 		}
 
 		// flush ringbuffer
 		while (_rb.read_space () > 0) {
-			size_t remain = std::min ((size_t)_rb.read_space (), chunksize);
+			size_t remain = std::min ((framecnt_t)_rb.read_space (), _chunksize);
 			_rb.read (framebuf, remain);
-			framecnt_t const written = write (framebuf, remain);
+			framecnt_t const written = SndfileBase::write (framebuf, remain);
-			frames_written += written;
+			SndfileWriter<T>::frames_written += written;
 		}
-		writeSync();
 
+		SndfileWriter<T>::writeSync();
 		pthread_mutex_unlock (&_disk_thread_lock);
 		free (framebuf);
+		TmpFile<T>::FileFlushed ();
 	}
 
   protected:
 	std::string filename;
-	framecnt_t frames_written;
 
 	bool _capture;
+	framecnt_t _chunksize;
 	RingBuffer<T> _rb;
 
 	pthread_mutex_t _disk_thread_lock;
@@ -141,10 +140,6 @@ class TmpFileRt
 
 	void end_write () {
 		pthread_mutex_lock (&_disk_thread_lock);
-		if (!_capture) {
-			pthread_mutex_unlock (&_disk_thread_lock);
-			return;
-		}
 		_capture = false;
 		pthread_cond_signal (&_data_ready);
 		pthread_mutex_unlock (&_disk_thread_lock);
@@ -153,13 +148,18 @@ class TmpFileRt
 
 	void init()
 	{
-		frames_written = 0;
+		SndfileWriter<T>::frames_written = 0;
 		_capture = true;
-		add_supported_flag (ProcessContext<T>::EndOfInput);
+		SndfileWriter<T>::add_supported_flag (ProcessContext<T>::EndOfInput);
 		pthread_mutex_init (&_disk_thread_lock, 0);
 		pthread_cond_init  (&_data_ready, 0);
 
-		pthread_create (&_thread_id, NULL, _disk_thread, this);
+		if (pthread_create (&_thread_id, NULL, _disk_thread, this)) {
+			_capture = false;
+			if (SndfileWriter<T>::throw_level (ThrowStrict)) {
+				throw Exception (*this, "Cannot create export disk writer");
+			}
+		}
 	}
 
 	private:

libs/audiographer/audiographer/sndfile/tmp_file_sync.h

@@ -0,0 +1,65 @@
+#ifndef AUDIOGRAPHER_TMP_FILE_SYNC_H
+#define AUDIOGRAPHER_TMP_FILE_SYNC_H
+
+#include <cstdio>
+#include <string>
+
+#include <glib.h>
+#include "pbd/gstdio_compat.h"
+
+#include "sndfile_writer.h"
+#include "sndfile_reader.h"
+#include "tmp_file.h"
+
+namespace AudioGrapher
+{
+
+/// A temporary file deleted after this class is destructed
+template<typename T = DefaultSampleType>
+class TmpFileSync
+	: public TmpFile<T>
+{
+  public:
+
+	/// \a filename_template must match the requirements for mkstemp, i.e. end in "XXXXXX"
+	TmpFileSync (char * filename_template, int format, ChannelCount channels, framecnt_t samplerate)
+		: SndfileHandle (g_mkstemp(filename_template), true, SndfileBase::ReadWrite, format, channels, samplerate)
+		, filename (filename_template)
+	{}
+
+	TmpFileSync (int format, ChannelCount channels, framecnt_t samplerate)
+	  : SndfileHandle (fileno (tmpfile()), true, SndfileBase::ReadWrite, format, channels, samplerate)
+	{}
+
+	TmpFileSync (TmpFileSync const & other) : SndfileHandle (other) {}
+	using SndfileHandle::operator=;
+
+	~TmpFileSync()
+	{
+		/* explicitly close first, some OS (yes I'm looking at you windows)
+		 * cannot delete files that are still open
+		 */
+		if (!filename.empty()) {
+			SndfileBase::close();
+			std::remove(filename.c_str());
+		}
+	}
+
+	void process (ProcessContext<T> const & c)
+	{
+		SndfileWriter<T>::process (c);
+
+		if (c.has_flag(ProcessContext<T>::EndOfInput)) {
+			TmpFile<T>::FileFlushed ();
+		}
+	}
+
+	using Sink<T>::process;
+
+  private:
+	std::string filename;
+};
+
+} // namespace
+
+#endif // AUDIOGRAPHER_TMP_FILE_SYNC_H