From d36afeebc4cb15b7660bc031a8b6ca483ad83c36 Mon Sep 17 00:00:00 2001 From: Besmir Beqiri Date: Fri, 12 Jan 2024 12:39:31 +0100 Subject: [PATCH 1/9] Use `FFmpeg` frame grabber to retrieve audio samples from the microphone --- .../media/recorder/impl/FXMediaRecorder.java | 455 ++++++++---------- 1 file changed, 197 insertions(+), 258 deletions(-) diff --git a/jpro-media/src/main/java/one/jpro/platform/media/recorder/impl/FXMediaRecorder.java b/jpro-media/src/main/java/one/jpro/platform/media/recorder/impl/FXMediaRecorder.java index 38792a55..4a10428e 100644 --- a/jpro-media/src/main/java/one/jpro/platform/media/recorder/impl/FXMediaRecorder.java +++ b/jpro-media/src/main/java/one/jpro/platform/media/recorder/impl/FXMediaRecorder.java @@ -14,11 +14,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.sound.sampled.*; import java.io.File; import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; import java.nio.CharBuffer; import java.nio.ShortBuffer; import java.nio.file.Files; @@ -26,7 +23,6 @@ import java.util.Arrays; import java.util.Comparator; import java.util.concurrent.*; -import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.stream.Stream; /** @@ -36,49 +32,48 @@ */ public final class FXMediaRecorder extends BaseMediaRecorder { - private final Logger log = LoggerFactory.getLogger(FXMediaRecorder.class); + private static final Logger logger = LoggerFactory.getLogger(FXMediaRecorder.class); private static final Path RECORDING_PATH = Path.of(System.getProperty("user.home"), ".jpro", "video", "capture"); private static final String DIVIDER_LINE = - CharBuffer.allocate(80).toString().replace( '\0', '*' ); + CharBuffer.allocate(80).toString().replace('\0', '*'); // Video resources - private static final int WEBCAM_DEVICE_INDEX = 0; + private static final int CAMERA_DEVICE_INDEX = 0; // Use the default system camera + private static final String MICROPHONE_DEVICE_NAME = ":0"; // Use the default system microphone private static final int FRAME_RATE = 30; private static final String MP4_FILE_EXTENSION = ".mp4"; - private final FrameGrabber webcamGrabber; - private final JavaFXFrameConverter frameConverter; - private final ImageView frameView; + private FrameGrabber cameraGrabber; + private FFmpegFrameGrabber micGrabber; + private final JavaFXFrameConverter cameraFrameConverter; + private final ImageView cameraView; private double frameRate; // Audio resources + private static final int DEFAULT_AUDIO_CHANNELS = 1; // Mono audio private static final int DEFAULT_AUDIO_SAMPLE_RATE = 44100; // 44.1 KHz - private static final int DEFAULT_AUDIO_CHANNELS = 0; // no audioHz - private static final int DEFAULT_AUDIO_FRAME_SIZE = 1; // 1 byte - private AudioFormat audioFormat; - private TargetDataLine micLine; - private int audioSampleRate; - private int audioNumChannels; // Storage resources private FFmpegFrameRecorder recorder; private Path tempVideoFile; // Recording state - private volatile boolean recordingStarted = false; - private volatile boolean recordingStopped = false; + private volatile long startTime = 0; + private volatile boolean recording = false; // Concurrency and locking resources private final ThreadGroup scheduledThreadGroup = new ThreadGroup("Media Recorder thread pool"); private int threadCounter; private final ExecutorService videoExecutorService; - private final ScheduledExecutorService audioExecutorService; + private final ExecutorService audioExecutorService; private final ExecutorService startStopRecordingExecutorService; - private final ReentrantReadWriteLock readWriteLock = new ReentrantReadWriteLock(); - private final ReentrantReadWriteLock.ReadLock readLock = readWriteLock.readLock(); - private final ReentrantReadWriteLock.WriteLock writeLock = readWriteLock.writeLock(); + private final Semaphore semaphore = new Semaphore(2); + /** + * Default constructor for FXMediaRecorder. + * Initializes the video and audio capture resources. + */ public FXMediaRecorder() { // Set native log level to error avutil.av_log_set_level(avutil.AV_LOG_ERROR); @@ -91,16 +86,22 @@ public FXMediaRecorder() { return thread; }; videoExecutorService = Executors.newSingleThreadExecutor(threadFactory); - audioExecutorService = Executors.newSingleThreadScheduledExecutor(threadFactory); + audioExecutorService = Executors.newSingleThreadExecutor(threadFactory); startStopRecordingExecutorService = Executors.newSingleThreadExecutor(threadFactory); - // Initialize webcam frame grabber - webcamGrabber = (isOsWindows()) ? new VideoInputFrameGrabber(WEBCAM_DEVICE_INDEX) - : new OpenCVFrameGrabber(WEBCAM_DEVICE_INDEX); + // Initialize camera frame grabber + cameraGrabber = (isOsWindows()) ? new VideoInputFrameGrabber(CAMERA_DEVICE_INDEX) + : new OpenCVFrameGrabber(CAMERA_DEVICE_INDEX); // Frame to JavaFX image converter - frameConverter = new JavaFXFrameConverter(); + cameraFrameConverter = new JavaFXFrameConverter(); // Use ImageView to show camera grabbed frames - frameView = new ImageView(); + cameraView = new ImageView(); + + // Initialize audio frame grabber + micGrabber = new FFmpegFrameGrabber(MICROPHONE_DEVICE_NAME); + micGrabber.setFormat("avfoundation"); + micGrabber.setAudioChannels(DEFAULT_AUDIO_CHANNELS); + micGrabber.setSampleRate(DEFAULT_AUDIO_SAMPLE_RATE); // Stop and release native resources on exit Runtime.getRuntime().addShutdownHook(new Thread(() -> { @@ -114,192 +115,161 @@ public FXMediaRecorder() { .map(Path::toFile) .forEach(File::delete); } catch (IOException ex) { - log.error(ex.getMessage(), ex); + logger.error(ex.getMessage(), ex); } } })); } ImageView getCameraView() { - return frameView; + return cameraView; } @Override public void enable() { if (recorderReady) { - log.info("Media recorder is already enabled."); + logger.info("Media recorder is already enabled."); } else { - log.debug("Enabling media recorder..."); - // Camera frame grabber runnable - final Runnable frameGrabber = () -> { - try { - // start the video capture - webcamGrabber.start(); - frameRate = (webcamGrabber.getFrameRate() < FRAME_RATE) ? FRAME_RATE : webcamGrabber.getFrameRate(); - printCaptureDeviceDescription(); - } catch (FrameGrabber.Exception ex) { - setError("Exception during the enabling of video camera stream.", ex); - release(); - } + logger.debug("Enabling media recorder..."); + enableVideoCapture(); + enableAudioCapture(); + } - try { - // start the audio capture - enableAudioCapture(); - } catch (LineUnavailableException ex) { - setError("Exception on creating audio input line from the microphone.", ex); - if (micLine != null) { - micLine.close(); - } - } + try { + // Wait for the video and audio capture to be enabled + semaphore.acquire(); + } catch (InterruptedException ex) { + setError("Exception during the enabling of video and audio capture.", ex); + } - // Set recorder ready - recorderReady = true; - Platform.runLater(() -> { - // Set status to ready - setStatus(Status.READY); - - // Fire ready event - Event.fireEvent(FXMediaRecorder.this, - new MediaRecorderEvent(FXMediaRecorder.this, - MediaRecorderEvent.MEDIA_RECORDER_READY)); - }); - - // Start the camera frame grabbing, showing and recording task - try { - while (recorderReady) { - // effectively grab a single frame - final Frame frame; - readLock.lock(); - try { - frame = webcamGrabber.grab(); - } finally { - readLock.unlock(); - } - - if (frame != null) { - // convert and show the frame - updateCameraView(frameView, frameConverter.convert(frame)); - - // write the webcam frame if recording started - writeVideoFrame(frame); - } - } - } catch (FrameGrabber.Exception ex) { - setError("Exception during camera stream frame grabbing.", ex); - release(); - } - }; + // Set recorder ready + recorderReady = true; - videoExecutorService.execute(frameGrabber); - } + // Set status to ready + setStatus(Status.READY); + + // Fire ready event + Event.fireEvent(FXMediaRecorder.this, + new MediaRecorderEvent(FXMediaRecorder.this, + MediaRecorderEvent.MEDIA_RECORDER_READY)); } /** - * Records a video frame. - * - * @param frame the frame to record + * Initializes and starts video capture from the camera. */ - private void writeVideoFrame(Frame frame) { - if (!recordingStopped && recorder != null) { - writeLock.lock(); + private void enableVideoCapture() { + logger.debug("Enabling video capture..."); + // Camera frame grabber runnable + final Runnable frameGrabber = () -> { try { - recorder.record(frame); - } catch (FFmpegFrameRecorder.Exception ex) { - setError("Exception during video frame recording.", ex); - } finally { - writeLock.unlock(); - } - } - } + // Acquire the semaphore + semaphore.acquire(); - /** - * Records an audio data. - * - * @throws LineUnavailableException if the mic line is unavailable - */ - private void enableAudioCapture() throws LineUnavailableException { - final Mixer.Info micDevice = getDefaultMicDevice(); - if (micDevice != null) { - final Mixer micMixer = AudioSystem.getMixer(micDevice); - // Audio format: 44.1k sample rate, 16 bits, mono, signed, little endian - audioFormat = new AudioFormat(DEFAULT_AUDIO_SAMPLE_RATE, 16, 1, true, false); - DataLine.Info dataLineInfo = new DataLine.Info(TargetDataLine.class, audioFormat); - - if (micMixer.isLineSupported(dataLineInfo)) { - micLine = (TargetDataLine) micMixer.getLine(dataLineInfo); - micLine.open(audioFormat); - micLine.start(); - - audioSampleRate = (int) audioFormat.getSampleRate(); - audioNumChannels = audioFormat.getChannels(); + // start the video capture + cameraGrabber.start(); + frameRate = (cameraGrabber.getFrameRate() < FRAME_RATE) ? FRAME_RATE : cameraGrabber.getFrameRate(); + printCaptureDeviceDescription(); + } catch (FrameGrabber.Exception | InterruptedException ex) { + setError("Exception during the enabling of video stream from the camera.", ex); + release(); + } finally { + // Release the semaphore + semaphore.release(); + logger.debug("Video capture enabled."); } - final Runnable audioSampleGrabber = () -> { - if (recordingStarted) { - // Initialize audio buffer - final int audioBufferSize = audioSampleRate * audioNumChannels; - final byte[] audioBytes = new byte[audioBufferSize]; - - // Read from the line - int nBytesRead = 0; - while (nBytesRead == 0) { - nBytesRead = micLine.read(audioBytes, 0, micLine.available()); + // Start the camera frame grabbing, showing and recording task + try { + while (recorderReady) { + // effectively grab a single frame + final Frame videoFrame = cameraGrabber.grabFrame(); + if (videoFrame != null) { + // convert and show the frame + updateCameraView(cameraView, cameraFrameConverter.convert(videoFrame)); + // write the webcam frame if recording started + writeVideoFrame(videoFrame); } + } + } catch (FrameGrabber.Exception ex) { + setError("Exception during camera stream frame grabbing.", ex); + release(); + } + }; - final int nSamplesRead = nBytesRead / 2; - final short[] samples = new short[nSamplesRead]; - - final ByteOrder byteOrder = audioFormat.isBigEndian() ? - ByteOrder.BIG_ENDIAN : ByteOrder.LITTLE_ENDIAN; + videoExecutorService.execute(frameGrabber); + } - // Wrap our short[] into a ShortBuffer - ByteBuffer.wrap(audioBytes).order(byteOrder).asShortBuffer().get(samples); - ShortBuffer samplesBuff = ShortBuffer.wrap(samples, 0, nSamplesRead); + /** + * Initializes and starts audio capture from the microphone. + */ + private void enableAudioCapture() { + logger.debug("Enabling audio capture..."); + final Runnable audioSampleGrabber = () -> { + try { + // Acquire the semaphore + semaphore.acquire(); -// log.trace("Record audio samples: {}", nSamplesRead); + // start the audio capture + micGrabber.start(); + } catch (FrameGrabber.Exception | InterruptedException ex) { + setError("Exception during the enabling of audio stream from the microphone.", ex); + release(); + } finally { + // Release the semaphore + semaphore.release(); + logger.debug("Audio capture enabled."); + } - // recorder audio data - if (!recordingStopped && recorder != null) { - writeLock.lock(); - try { - recorder.recordSamples(audioSampleRate, audioNumChannels, samplesBuff); - } catch (FFmpegFrameRecorder.Exception ex) { - setError("Exception on recording the audio samples.", ex); - } finally { - writeLock.unlock(); - } + try { + while (recorderReady) { + // effectively grab a single frame + final Frame audioFrame = micGrabber.grabSamples(); + if (audioFrame != null) { + ShortBuffer audioData = (ShortBuffer) audioFrame.samples[0]; + short[] shorts = new short[audioData.limit()]; + audioData.get(shorts); + audioData.flip(); + + writeAudioSamples(audioData); } } - }; + } catch (FrameGrabber.Exception ex) { + setError("Exception during microphone stream frame grabbing.", ex); + release(); + } + }; + + audioExecutorService.execute(audioSampleGrabber); + } - final long period = (long) (1000.0 / frameRate); - audioExecutorService.scheduleAtFixedRate(audioSampleGrabber, 0, period, TimeUnit.MILLISECONDS); + /** + * Records a single video frame. + * + * @param frame the video frame to be recorded. + */ + private void writeVideoFrame(Frame frame) { + if (recorder != null && recording) { + try { + recorder.record(frame); + } catch (FFmpegFrameRecorder.Exception ex) { + setError("Exception during video frame recording.", ex); + } } } /** - * Retries the list of available audio input devices and returns the default microphone device. + * Records audio samples. This method is called for each audio sample captured from the microphone. * - * @return the default microphone device + * @param audioSamples the buffer containing audio samples to be recorded */ - private Mixer.Info getDefaultMicDevice() { - final Mixer.Info[] mixerInfos = AudioSystem.getMixerInfo(); - for (Mixer.Info info : mixerInfos) { - final Mixer mixer = AudioSystem.getMixer(info); - final Line.Info[] lineInfos = mixer.getTargetLineInfo(); - // Only prints out info is it is a Microphone - if (lineInfos.length >= 1 && lineInfos[0].getLineClass().equals(TargetDataLine.class)) { - log.debug(DIVIDER_LINE); - for (Line.Info lineInfo : lineInfos) { - log.debug("Mic Line Name: " + info.getName()); // The audio device name - log.debug("Mic Line Description: " + info.getDescription()); // The type of audio device - printSupportedAudioFormats(lineInfo); - } - log.debug(DIVIDER_LINE); - return info; + private void writeAudioSamples(ShortBuffer audioSamples) { + if (recorder != null && recording) { + try { + recorder.recordSamples(audioSamples); + } catch (FFmpegFrameRecorder.Exception ex) { + setError("Exception during audio samples recording.", ex); } } - return null; } @Override @@ -310,13 +280,12 @@ public void start() { if (recorderReady) { tempVideoFile = createTempFilename("video_", MP4_FILE_EXTENSION); - recorder = new FFmpegFrameRecorder(tempVideoFile.toString(), - webcamGrabber.getImageWidth(), webcamGrabber.getImageHeight()); + recorder = new FFmpegFrameRecorder(tempVideoFile.toString(), cameraGrabber.getImageWidth(), + cameraGrabber.getImageHeight(), micGrabber.getAudioChannels()); recorder.setInterleaved(true); recorder.setVideoOption("tune", "zerolatency"); // low latency for webcam streaming recorder.setVideoOption("preset", "ultrafast"); // low cpu usage for the encoder recorder.setVideoOption("crf", "28"); // video quality -// recorder.setVideoBitrate(webcamGrabber.getVideoBitrate()); recorder.setVideoBitrate(8 * 1024 * 1024); // 8 Mbps for 1080p recorder.setVideoCodec(avcodec.AV_CODEC_ID_H264); recorder.setFormat("mp4"); @@ -324,17 +293,15 @@ public void start() { recorder.setGopSize((int) (frameRate * 2)); recorder.setAudioOption("crf", "0"); // no variable bitrate audio recorder.setAudioQuality(0); // highest quality -// recorder.setAudioBitrate(getAudioSampleRate() * getFrameSize() * getAudioChannels()); recorder.setAudioBitrate(192000); // 192 kbps recorder.setAudioCodec(avcodec.AV_CODEC_ID_AAC); - recorder.setSampleRate(getAudioSampleRate()); - recorder.setAudioChannels(getAudioChannels()); + recorder.setSampleRate(micGrabber.getSampleRate()); try { // Enable recording recorder.start(); - recordingStopped = false; - recordingStarted = true; + startTime = System.currentTimeMillis(); + recording = true; Platform.runLater(() -> { // Set status to recording @@ -349,7 +316,7 @@ public void start() { setError("Exception on starting the audio/video recorder.", ex); } } else { - log.info("Please, enable the camera first!"); + logger.info("Please, enable the camera first!"); } }; @@ -357,7 +324,7 @@ public void start() { } else if (getStatus().equals(Status.PAUSED)) { if (recorderReady) { // enable recording - recordingStarted = true; + recording = true; // Set status to recording setStatus(Status.RECORDING); @@ -373,7 +340,7 @@ public void start() { @Override public void pause() { // Disable recording - recordingStarted = false; + recording = false; // Set status to paused setStatus(Status.PAUSED); @@ -410,18 +377,14 @@ public void stop() { */ private void stopRecording() { // Stop recording - recordingStarted = false; - recordingStopped = true; + recording = false; // Release video recorder resources if (recorder != null) { - writeLock.lock(); try { recorder.close(); // This call stops the recorder and releases all resources used by it. } catch (FrameRecorder.Exception ex) { setError("Exception on stopping the audio/video recorder", ex); - } finally { - writeLock.unlock(); } } } @@ -431,19 +394,21 @@ private void stopRecording() { */ public void release() { recorderReady = false; + recording = false; releaseVideoResources(); releaseAudioResources(); } /** - * Release the video resources. + * Releases the resources associated with the video capture. */ private void releaseVideoResources() { if (videoExecutorService != null && !videoExecutorService.isShutdown()) { try { // release video grabber - if (webcamGrabber != null) { - webcamGrabber.release(); + if (cameraGrabber != null) { + cameraGrabber.close(); + cameraGrabber = null; } // stop the video recoding service @@ -457,67 +422,27 @@ private void releaseVideoResources() { } /** - * Release the audio resources. + * Releases the resources associated with the audio capture. */ private void releaseAudioResources() { if (audioExecutorService != null && !audioExecutorService.isShutdown()) { try { // release audio grabber - if (micLine != null) { - micLine.close(); + if (micGrabber != null) { + micGrabber.close(); + micGrabber = null; } // stop the audio recording service audioExecutorService.shutdown(); //noinspection ResultOfMethodCallIgnored audioExecutorService.awaitTermination(100, TimeUnit.MILLISECONDS); - } catch (InterruptedException ex) { + } catch (FrameGrabber.Exception | InterruptedException ex) { setError("Exception in stopping the audio frame capture service.", ex); } } } - /** - * Obtains the audio sample rate from the specified audio format - * used for recoding. If not specified, then default value (44100) is returned. - * - * @return the audio sample rate - */ - private int getAudioSampleRate() { - if (audioFormat == null) return DEFAULT_AUDIO_SAMPLE_RATE; - - final int sampleRate = (int) audioFormat.getSampleRate(); - return (sampleRate == AudioSystem.NOT_SPECIFIED) ? DEFAULT_AUDIO_SAMPLE_RATE : sampleRate; - } - - /** - * Obtains the number of audio channels from the specified audio format - * used for recoding. If not specified, then 0 is returned, - * meaning there no audio input device available. - * - * @return the number of audio channels (1 for mono, 2 for stereo, etc.) - */ - private int getAudioChannels() { - if (audioFormat == null) return DEFAULT_AUDIO_CHANNELS; - - final int audioChannels = audioFormat.getChannels(); - return (audioChannels == AudioSystem.NOT_SPECIFIED) ? DEFAULT_AUDIO_CHANNELS : audioChannels; - } - - /** - * Obtains the frame size in bytes. For compressed formats, the return value is - * the frame size of the uncompressed audio data. {@code AudioSystem.NOT_SPECIFIED} - * is returned when the frame size is not defined for this audio format. - * - * @return the number of bytes per frame, or {@code AudioSystem.NOT_SPECIFIED} - */ - private int getFrameSize() { - if (audioFormat == null) return DEFAULT_AUDIO_FRAME_SIZE; - - final int frameSize = audioFormat.getFrameSize(); - return (frameSize == AudioSystem.NOT_SPECIFIED) ? DEFAULT_AUDIO_FRAME_SIZE : frameSize; - } - /** * Update the {@link ImageView} in the JavaFX main thread * @@ -552,14 +477,14 @@ private Path createTempFilename(final String prefix, final String postfix) { try { Files.createDirectories(parentDir); } catch (IOException ex) { - log.error(ex.getMessage(), ex); + logger.error(ex.getMessage(), ex); } } return tempFile; } /** - * Check if the OS is Windows. + * Checks if the operating system is Windows. * * @return true if the OS is Windows, false otherwise */ @@ -580,36 +505,50 @@ private void setError(String message, Exception ex) { } else { Platform.runLater(() -> setError(new MediaRecorderException(message, ex))); } - log.error(message, ex); + logger.error(message, ex); } /** - * Print to terminal the capture device description. + * Prints to terminal the capture device description. This includes details about the camera and + * microphone being used for recording. */ private void printCaptureDeviceDescription() { - log.debug(DIVIDER_LINE); - if (isOsWindows()) { + logger.debug(DIVIDER_LINE); + + // Print camera device description + if (cameraGrabber != null) { try { - log.debug("Capture devices: " + Arrays.toString(VideoInputFrameGrabber.getDeviceDescriptions())); + if (isOsWindows()) { + logger.debug("Camera Device Description: " + + Arrays.toString(VideoInputFrameGrabber.getDeviceDescriptions())); + } + + logger.debug("Image Width: " + cameraGrabber.getImageWidth()); + logger.debug("Image Height: " + cameraGrabber.getImageHeight()); + logger.debug("Frame Rate: " + frameRate); } catch (FrameGrabber.Exception ex) { - log.error(ex.getMessage(), ex); + logger.error("Error getting camera device description: " + ex.getMessage(), ex); } } - log.debug("Capture Device Info:"); - log.debug("Image Width: " + webcamGrabber.getImageWidth()); - log.debug("Image Height: " + webcamGrabber.getImageHeight()); - log.debug("Frame Rate: " + frameRate); + + // Print microphone device description + if (micGrabber != null) { + String microphoneDescription = micGrabber.getFormat() + " - " + MICROPHONE_DEVICE_NAME; + logger.debug("Microphone Device Description: " + microphoneDescription); + logger.debug("Audio Channels: " + micGrabber.getAudioChannels()); + logger.debug("Sample Rate: " + micGrabber.getSampleRate()); + } } /** - * Print to terminal the supported audio formats. - * - * @param lineInfo the line info + * Adjusts the timestamp of the recording to match the system time. Ensures synchronization + * between video and audio streams. */ - private void printSupportedAudioFormats(final Line.Info lineInfo) { - if (lineInfo instanceof final DataLine.Info dataLineInfo) { - log.debug("Supported Audio Formats:"); - Arrays.stream(dataLineInfo.getFormats()).forEach(format -> log.debug("{}", format)); + private void adjustTimestamp() { + long t = 1000 * (System.currentTimeMillis() - startTime); + if (t > recorder.getTimestamp()) { + logger.debug("Correct recorder timestamp: " + t); + recorder.setTimestamp(t); } } } From aeaf85a97d6355cd7b7aa08ce5646807c44b7aac Mon Sep 17 00:00:00 2001 From: Besmir Beqiri Date: Fri, 12 Jan 2024 12:51:31 +0100 Subject: [PATCH 2/9] Rename native implementations of media recorder and media recorder view to `NativeMediaRecorder` and `NativeMediaRecorderView` --- .../one/jpro/platform/media/MediaView.java | 8 +++--- .../media/recorder/MediaRecorder.java | 4 +-- ...Recorder.java => NativeMediaRecorder.java} | 26 +++++++++---------- ...View.java => NativeMediaRecorderView.java} | 16 ++++++------ 4 files changed, 27 insertions(+), 27 deletions(-) rename jpro-media/src/main/java/one/jpro/platform/media/recorder/impl/{FXMediaRecorder.java => NativeMediaRecorder.java} (96%) rename jpro-media/src/main/java/one/jpro/platform/media/recorder/impl/{FXMediaRecorderView.java => NativeMediaRecorderView.java} (87%) diff --git a/jpro-media/src/main/java/one/jpro/platform/media/MediaView.java b/jpro-media/src/main/java/one/jpro/platform/media/MediaView.java index 5c71e40a..14cfe682 100644 --- a/jpro-media/src/main/java/one/jpro/platform/media/MediaView.java +++ b/jpro-media/src/main/java/one/jpro/platform/media/MediaView.java @@ -15,8 +15,8 @@ import one.jpro.platform.media.player.impl.WebMediaPlayer; import one.jpro.platform.media.player.impl.WebMediaPlayerView; import one.jpro.platform.media.recorder.MediaRecorder; -import one.jpro.platform.media.recorder.impl.FXMediaRecorder; -import one.jpro.platform.media.recorder.impl.FXMediaRecorderView; +import one.jpro.platform.media.recorder.impl.NativeMediaRecorder; +import one.jpro.platform.media.recorder.impl.NativeMediaRecorderView; import one.jpro.platform.media.recorder.impl.WebMediaRecorder; import one.jpro.platform.media.recorder.impl.WebMediaRecorderView; @@ -136,8 +136,8 @@ public static MediaView create(MediaPlayer mediaPlayer) { * @return a {@link MediaView} object. */ public static MediaView create(MediaRecorder mediaRecorder) { - if (mediaRecorder instanceof FXMediaRecorder fxMediaRecorder) { - return new FXMediaRecorderView(fxMediaRecorder); + if (mediaRecorder instanceof NativeMediaRecorder fxMediaRecorder) { + return new NativeMediaRecorderView(fxMediaRecorder); } else if (mediaRecorder instanceof WebMediaRecorder webMediaRecorder) { return new WebMediaRecorderView(webMediaRecorder); } else { diff --git a/jpro-media/src/main/java/one/jpro/platform/media/recorder/MediaRecorder.java b/jpro-media/src/main/java/one/jpro/platform/media/recorder/MediaRecorder.java index c77c5857..a4f50019 100644 --- a/jpro-media/src/main/java/one/jpro/platform/media/recorder/MediaRecorder.java +++ b/jpro-media/src/main/java/one/jpro/platform/media/recorder/MediaRecorder.java @@ -11,7 +11,7 @@ import one.jpro.platform.media.MediaSource; import one.jpro.platform.media.MediaView; import one.jpro.platform.media.event.MediaRecorderEvent; -import one.jpro.platform.media.recorder.impl.FXMediaRecorder; +import one.jpro.platform.media.recorder.impl.NativeMediaRecorder; import one.jpro.platform.media.recorder.impl.WebMediaRecorder; import java.util.Optional; @@ -60,7 +60,7 @@ static MediaRecorder create(Stage stage) { final WebAPI webAPI = WebAPI.getWebAPI(stage); return new WebMediaRecorder(webAPI); } - return new FXMediaRecorder(); + return new NativeMediaRecorder(); } /** diff --git a/jpro-media/src/main/java/one/jpro/platform/media/recorder/impl/FXMediaRecorder.java b/jpro-media/src/main/java/one/jpro/platform/media/recorder/impl/NativeMediaRecorder.java similarity index 96% rename from jpro-media/src/main/java/one/jpro/platform/media/recorder/impl/FXMediaRecorder.java rename to jpro-media/src/main/java/one/jpro/platform/media/recorder/impl/NativeMediaRecorder.java index 4a10428e..4960dbb3 100644 --- a/jpro-media/src/main/java/one/jpro/platform/media/recorder/impl/FXMediaRecorder.java +++ b/jpro-media/src/main/java/one/jpro/platform/media/recorder/impl/NativeMediaRecorder.java @@ -30,9 +30,9 @@ * * @author Besmir Beqiri */ -public final class FXMediaRecorder extends BaseMediaRecorder { +public final class NativeMediaRecorder extends BaseMediaRecorder { - private static final Logger logger = LoggerFactory.getLogger(FXMediaRecorder.class); + private static final Logger logger = LoggerFactory.getLogger(NativeMediaRecorder.class); private static final Path RECORDING_PATH = Path.of(System.getProperty("user.home"), ".jpro", "video", "capture"); @@ -74,7 +74,7 @@ public final class FXMediaRecorder extends BaseMediaRecorder { * Default constructor for FXMediaRecorder. * Initializes the video and audio capture resources. */ - public FXMediaRecorder() { + public NativeMediaRecorder() { // Set native log level to error avutil.av_log_set_level(avutil.AV_LOG_ERROR); @@ -149,8 +149,8 @@ public void enable() { setStatus(Status.READY); // Fire ready event - Event.fireEvent(FXMediaRecorder.this, - new MediaRecorderEvent(FXMediaRecorder.this, + Event.fireEvent(NativeMediaRecorder.this, + new MediaRecorderEvent(NativeMediaRecorder.this, MediaRecorderEvent.MEDIA_RECORDER_READY)); } @@ -308,8 +308,8 @@ public void start() { setStatus(Status.RECORDING); // Fire start event - Event.fireEvent(FXMediaRecorder.this, - new MediaRecorderEvent(FXMediaRecorder.this, + Event.fireEvent(NativeMediaRecorder.this, + new MediaRecorderEvent(NativeMediaRecorder.this, MediaRecorderEvent.MEDIA_RECORDER_START)); }); } catch (FFmpegFrameRecorder.Exception ex) { @@ -330,8 +330,8 @@ public void start() { setStatus(Status.RECORDING); // Fire start event - Event.fireEvent(FXMediaRecorder.this, - new MediaRecorderEvent(FXMediaRecorder.this, + Event.fireEvent(NativeMediaRecorder.this, + new MediaRecorderEvent(NativeMediaRecorder.this, MediaRecorderEvent.MEDIA_RECORDER_RESUME)); } } @@ -346,8 +346,8 @@ public void pause() { setStatus(Status.PAUSED); // Fire start event - Event.fireEvent(FXMediaRecorder.this, - new MediaRecorderEvent(FXMediaRecorder.this, + Event.fireEvent(NativeMediaRecorder.this, + new MediaRecorderEvent(NativeMediaRecorder.this, MediaRecorderEvent.MEDIA_RECORDER_PAUSE)); } @@ -364,8 +364,8 @@ public void stop() { setStatus(Status.INACTIVE); // Fire start event - Event.fireEvent(FXMediaRecorder.this, - new MediaRecorderEvent(FXMediaRecorder.this, + Event.fireEvent(NativeMediaRecorder.this, + new MediaRecorderEvent(NativeMediaRecorder.this, MediaRecorderEvent.MEDIA_RECORDER_STOP)); }); }; diff --git a/jpro-media/src/main/java/one/jpro/platform/media/recorder/impl/FXMediaRecorderView.java b/jpro-media/src/main/java/one/jpro/platform/media/recorder/impl/NativeMediaRecorderView.java similarity index 87% rename from jpro-media/src/main/java/one/jpro/platform/media/recorder/impl/FXMediaRecorderView.java rename to jpro-media/src/main/java/one/jpro/platform/media/recorder/impl/NativeMediaRecorderView.java index e581e403..e2a237ad 100644 --- a/jpro-media/src/main/java/one/jpro/platform/media/recorder/impl/FXMediaRecorderView.java +++ b/jpro-media/src/main/java/one/jpro/platform/media/recorder/impl/NativeMediaRecorderView.java @@ -15,17 +15,17 @@ * * @author Besmir Beqiri */ -public class FXMediaRecorderView extends MediaView { +public class NativeMediaRecorderView extends MediaView { - private final Logger log = LoggerFactory.getLogger(FXMediaRecorderView.class); + private static final Logger logger = LoggerFactory.getLogger(NativeMediaRecorderView.class); private ImageView fxFrameView; - public FXMediaRecorderView() { + public NativeMediaRecorderView() { getStyleClass().add(DEFAULT_STYLE_CLASS); } - public FXMediaRecorderView(FXMediaRecorder mediaRecorder) { + public NativeMediaRecorderView(NativeMediaRecorder mediaRecorder) { this(); setMediaEngine(mediaRecorder); } @@ -84,7 +84,7 @@ protected void invalidated() { if (fxFrameView != null) { fxFrameView.setPreserveRatio(get()); } - log.trace("preserve ratio: {}", isPreserveRatio()); + logger.trace("preserve ratio: {}", isPreserveRatio()); } }; } @@ -99,7 +99,7 @@ private void setInternalFitWidth(double fitWidth) { getChildren().add(fxFrameView); } fxFrameView.setFitWidth(fitWidth); - log.trace("video width: {}", fitWidth); + logger.trace("video width: {}", fitWidth); } } @@ -111,7 +111,7 @@ private void setInternalFitHeight(double fitHeight) { getChildren().add(fxFrameView); } fxFrameView.setFitHeight(fitHeight); - log.trace("video height: " + fitHeight); + logger.trace("video height: " + fitHeight); } } @@ -120,7 +120,7 @@ private void setInternalFitHeight(double fitHeight) { new WeakInvalidationListener(updateViewContainerListener); private void updateViewContainer() { - if (getScene() != null && getMediaEngine() instanceof FXMediaRecorder fxMediaRecorder) { + if (getScene() != null && getMediaEngine() instanceof NativeMediaRecorder fxMediaRecorder) { fxFrameView = fxMediaRecorder.getCameraView(); fxFrameView.setPreserveRatio(isPreserveRatio()); setInternalFitWidth(getFitWidth()); From ae62ff5070128445dee12051b1a670dc0c3181c0 Mon Sep 17 00:00:00 2001 From: Besmir Beqiri Date: Fri, 12 Jan 2024 12:58:42 +0100 Subject: [PATCH 3/9] Expose media recorder native and web implementations --- .../src/main/java/one/jpro/platform/media/MediaView.java | 8 ++++---- .../main/java/one/jpro/platform/media/WebMediaView.java | 2 +- .../media/recorder/{impl => }/BaseMediaRecorder.java | 4 +--- .../one/jpro/platform/media/recorder/MediaRecorder.java | 2 -- .../media/recorder/{impl => }/MediaRecorderOptions.java | 2 +- .../media/recorder/{impl => }/NativeMediaRecorder.java | 6 ++---- .../recorder/{impl => }/NativeMediaRecorderView.java | 3 +-- .../media/recorder/{impl => }/WebMediaRecorder.java | 4 +--- .../media/recorder/{impl => }/WebMediaRecorderView.java | 3 +-- 9 files changed, 12 insertions(+), 22 deletions(-) rename jpro-media/src/main/java/one/jpro/platform/media/recorder/{impl => }/BaseMediaRecorder.java (98%) rename jpro-media/src/main/java/one/jpro/platform/media/recorder/{impl => }/MediaRecorderOptions.java (98%) rename jpro-media/src/main/java/one/jpro/platform/media/recorder/{impl => }/NativeMediaRecorder.java (98%) rename jpro-media/src/main/java/one/jpro/platform/media/recorder/{impl => }/NativeMediaRecorderView.java (97%) rename jpro-media/src/main/java/one/jpro/platform/media/recorder/{impl => }/WebMediaRecorder.java (98%) rename jpro-media/src/main/java/one/jpro/platform/media/recorder/{impl => }/WebMediaRecorderView.java (83%) diff --git a/jpro-media/src/main/java/one/jpro/platform/media/MediaView.java b/jpro-media/src/main/java/one/jpro/platform/media/MediaView.java index 14cfe682..448d5018 100644 --- a/jpro-media/src/main/java/one/jpro/platform/media/MediaView.java +++ b/jpro-media/src/main/java/one/jpro/platform/media/MediaView.java @@ -15,10 +15,10 @@ import one.jpro.platform.media.player.impl.WebMediaPlayer; import one.jpro.platform.media.player.impl.WebMediaPlayerView; import one.jpro.platform.media.recorder.MediaRecorder; -import one.jpro.platform.media.recorder.impl.NativeMediaRecorder; -import one.jpro.platform.media.recorder.impl.NativeMediaRecorderView; -import one.jpro.platform.media.recorder.impl.WebMediaRecorder; -import one.jpro.platform.media.recorder.impl.WebMediaRecorderView; +import one.jpro.platform.media.recorder.NativeMediaRecorder; +import one.jpro.platform.media.recorder.NativeMediaRecorderView; +import one.jpro.platform.media.recorder.WebMediaRecorder; +import one.jpro.platform.media.recorder.WebMediaRecorderView; /** * Provides a view of {@link MediaSource} being played by a {@link MediaPlayer} diff --git a/jpro-media/src/main/java/one/jpro/platform/media/WebMediaView.java b/jpro-media/src/main/java/one/jpro/platform/media/WebMediaView.java index fc437c18..385a6d57 100644 --- a/jpro-media/src/main/java/one/jpro/platform/media/WebMediaView.java +++ b/jpro-media/src/main/java/one/jpro/platform/media/WebMediaView.java @@ -9,7 +9,7 @@ import javafx.geometry.HPos; import javafx.geometry.VPos; import javafx.scene.Node; -import one.jpro.platform.media.recorder.impl.WebMediaRecorder; +import one.jpro.platform.media.recorder.WebMediaRecorder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/jpro-media/src/main/java/one/jpro/platform/media/recorder/impl/BaseMediaRecorder.java b/jpro-media/src/main/java/one/jpro/platform/media/recorder/BaseMediaRecorder.java similarity index 98% rename from jpro-media/src/main/java/one/jpro/platform/media/recorder/impl/BaseMediaRecorder.java rename to jpro-media/src/main/java/one/jpro/platform/media/recorder/BaseMediaRecorder.java index c20319df..b2629705 100644 --- a/jpro-media/src/main/java/one/jpro/platform/media/recorder/impl/BaseMediaRecorder.java +++ b/jpro-media/src/main/java/one/jpro/platform/media/recorder/BaseMediaRecorder.java @@ -1,4 +1,4 @@ -package one.jpro.platform.media.recorder.impl; +package one.jpro.platform.media.recorder; import com.sun.javafx.event.EventHandlerManager; import javafx.application.Platform; @@ -12,8 +12,6 @@ import javafx.util.Duration; import one.jpro.platform.media.MediaSource; import one.jpro.platform.media.event.MediaRecorderEvent; -import one.jpro.platform.media.recorder.MediaRecorder; -import one.jpro.platform.media.recorder.MediaRecorderException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/jpro-media/src/main/java/one/jpro/platform/media/recorder/MediaRecorder.java b/jpro-media/src/main/java/one/jpro/platform/media/recorder/MediaRecorder.java index a4f50019..daae7ea8 100644 --- a/jpro-media/src/main/java/one/jpro/platform/media/recorder/MediaRecorder.java +++ b/jpro-media/src/main/java/one/jpro/platform/media/recorder/MediaRecorder.java @@ -11,8 +11,6 @@ import one.jpro.platform.media.MediaSource; import one.jpro.platform.media.MediaView; import one.jpro.platform.media.event.MediaRecorderEvent; -import one.jpro.platform.media.recorder.impl.NativeMediaRecorder; -import one.jpro.platform.media.recorder.impl.WebMediaRecorder; import java.util.Optional; diff --git a/jpro-media/src/main/java/one/jpro/platform/media/recorder/impl/MediaRecorderOptions.java b/jpro-media/src/main/java/one/jpro/platform/media/recorder/MediaRecorderOptions.java similarity index 98% rename from jpro-media/src/main/java/one/jpro/platform/media/recorder/impl/MediaRecorderOptions.java rename to jpro-media/src/main/java/one/jpro/platform/media/recorder/MediaRecorderOptions.java index 5b2c0d5d..f91279f4 100644 --- a/jpro-media/src/main/java/one/jpro/platform/media/recorder/impl/MediaRecorderOptions.java +++ b/jpro-media/src/main/java/one/jpro/platform/media/recorder/MediaRecorderOptions.java @@ -1,4 +1,4 @@ -package one.jpro.platform.media.recorder.impl; +package one.jpro.platform.media.recorder; import org.json.JSONObject; diff --git a/jpro-media/src/main/java/one/jpro/platform/media/recorder/impl/NativeMediaRecorder.java b/jpro-media/src/main/java/one/jpro/platform/media/recorder/NativeMediaRecorder.java similarity index 98% rename from jpro-media/src/main/java/one/jpro/platform/media/recorder/impl/NativeMediaRecorder.java rename to jpro-media/src/main/java/one/jpro/platform/media/recorder/NativeMediaRecorder.java index 4960dbb3..f211e93b 100644 --- a/jpro-media/src/main/java/one/jpro/platform/media/recorder/impl/NativeMediaRecorder.java +++ b/jpro-media/src/main/java/one/jpro/platform/media/recorder/NativeMediaRecorder.java @@ -1,4 +1,4 @@ -package one.jpro.platform.media.recorder.impl; +package one.jpro.platform.media.recorder; import javafx.application.Platform; import javafx.event.Event; @@ -6,8 +6,6 @@ import javafx.scene.image.ImageView; import one.jpro.platform.media.MediaSource; import one.jpro.platform.media.event.MediaRecorderEvent; -import one.jpro.platform.media.recorder.MediaRecorder; -import one.jpro.platform.media.recorder.MediaRecorderException; import org.bytedeco.ffmpeg.global.avcodec; import org.bytedeco.ffmpeg.global.avutil; import org.bytedeco.javacv.*; @@ -30,7 +28,7 @@ * * @author Besmir Beqiri */ -public final class NativeMediaRecorder extends BaseMediaRecorder { +public class NativeMediaRecorder extends BaseMediaRecorder { private static final Logger logger = LoggerFactory.getLogger(NativeMediaRecorder.class); diff --git a/jpro-media/src/main/java/one/jpro/platform/media/recorder/impl/NativeMediaRecorderView.java b/jpro-media/src/main/java/one/jpro/platform/media/recorder/NativeMediaRecorderView.java similarity index 97% rename from jpro-media/src/main/java/one/jpro/platform/media/recorder/impl/NativeMediaRecorderView.java rename to jpro-media/src/main/java/one/jpro/platform/media/recorder/NativeMediaRecorderView.java index e2a237ad..d76a3097 100644 --- a/jpro-media/src/main/java/one/jpro/platform/media/recorder/impl/NativeMediaRecorderView.java +++ b/jpro-media/src/main/java/one/jpro/platform/media/recorder/NativeMediaRecorderView.java @@ -1,4 +1,4 @@ -package one.jpro.platform.media.recorder.impl; +package one.jpro.platform.media.recorder; import javafx.beans.InvalidationListener; import javafx.beans.WeakInvalidationListener; @@ -6,7 +6,6 @@ import javafx.scene.image.ImageView; import one.jpro.platform.media.MediaEngine; import one.jpro.platform.media.MediaView; -import one.jpro.platform.media.recorder.MediaRecorder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/jpro-media/src/main/java/one/jpro/platform/media/recorder/impl/WebMediaRecorder.java b/jpro-media/src/main/java/one/jpro/platform/media/recorder/WebMediaRecorder.java similarity index 98% rename from jpro-media/src/main/java/one/jpro/platform/media/recorder/impl/WebMediaRecorder.java rename to jpro-media/src/main/java/one/jpro/platform/media/recorder/WebMediaRecorder.java index 7ba794e6..0076a598 100644 --- a/jpro-media/src/main/java/one/jpro/platform/media/recorder/impl/WebMediaRecorder.java +++ b/jpro-media/src/main/java/one/jpro/platform/media/recorder/WebMediaRecorder.java @@ -1,4 +1,4 @@ -package one.jpro.platform.media.recorder.impl; +package one.jpro.platform.media.recorder; import com.jpro.webapi.JSVariable; import com.jpro.webapi.WebAPI; @@ -8,8 +8,6 @@ import one.jpro.platform.media.MediaSource; import one.jpro.platform.media.WebMediaEngine; import one.jpro.platform.media.event.MediaRecorderEvent; -import one.jpro.platform.media.recorder.MediaRecorder; -import one.jpro.platform.media.recorder.MediaRecorderException; import org.json.JSONObject; import java.util.Objects; diff --git a/jpro-media/src/main/java/one/jpro/platform/media/recorder/impl/WebMediaRecorderView.java b/jpro-media/src/main/java/one/jpro/platform/media/recorder/WebMediaRecorderView.java similarity index 83% rename from jpro-media/src/main/java/one/jpro/platform/media/recorder/impl/WebMediaRecorderView.java rename to jpro-media/src/main/java/one/jpro/platform/media/recorder/WebMediaRecorderView.java index d2c9ae5f..21886bc6 100644 --- a/jpro-media/src/main/java/one/jpro/platform/media/recorder/impl/WebMediaRecorderView.java +++ b/jpro-media/src/main/java/one/jpro/platform/media/recorder/WebMediaRecorderView.java @@ -1,9 +1,8 @@ -package one.jpro.platform.media.recorder.impl; +package one.jpro.platform.media.recorder; import com.jpro.webapi.WebAPI; import one.jpro.platform.media.MediaView; import one.jpro.platform.media.WebMediaView; -import one.jpro.platform.media.recorder.MediaRecorder; /** * {@link MediaView} implementation for a web {@link MediaRecorder}. From 1c4e6d3000b642aa8ba4f5905eea8c74fb285b7e Mon Sep 17 00:00:00 2001 From: Besmir Beqiri Date: Fri, 12 Jan 2024 13:11:22 +0100 Subject: [PATCH 4/9] Make some internal implementations final and apply some cleanup on the media recorder classes --- .../media/recorder/BaseMediaRecorder.java | 10 +++++----- .../media/recorder/NativeMediaRecorder.java | 19 +++++++++++------- .../recorder/NativeMediaRecorderView.java | 4 ++-- .../media/recorder/WebMediaRecorder.java | 20 +++++++++---------- 4 files changed, 29 insertions(+), 24 deletions(-) diff --git a/jpro-media/src/main/java/one/jpro/platform/media/recorder/BaseMediaRecorder.java b/jpro-media/src/main/java/one/jpro/platform/media/recorder/BaseMediaRecorder.java index b2629705..5969dee8 100644 --- a/jpro-media/src/main/java/one/jpro/platform/media/recorder/BaseMediaRecorder.java +++ b/jpro-media/src/main/java/one/jpro/platform/media/recorder/BaseMediaRecorder.java @@ -26,14 +26,14 @@ */ abstract class BaseMediaRecorder implements MediaRecorder { + private static final Logger logger = LoggerFactory.getLogger(BaseMediaRecorder.class); + private RecorderTimerTask recorderTimerTask; volatile boolean recorderReady; volatile boolean isUpdateDurationEnabled; long startRecordingTime = 0; private double pauseDurationTime = 0; - private final Logger log = LoggerFactory.getLogger(BaseMediaRecorder.class); - // media source property (read-only) ReadOnlyObjectWrapper mediaSource; @@ -117,12 +117,12 @@ public Duration getDuration() { return (duration == null) ? Duration.ZERO : duration.get(); } - void setDuration(Duration value) { + final void setDuration(Duration value) { durationPropertyImpl().set(value); } @Override - public ReadOnlyObjectProperty durationProperty() { + public final ReadOnlyObjectProperty durationProperty() { return durationPropertyImpl().getReadOnlyProperty(); } @@ -132,7 +132,7 @@ private ReadOnlyObjectWrapper durationPropertyImpl() { @Override protected void invalidated() { - log.trace("Recording duration: {} s", get().toSeconds()); + logger.trace("Recording duration: {} s", get().toSeconds()); } }; } diff --git a/jpro-media/src/main/java/one/jpro/platform/media/recorder/NativeMediaRecorder.java b/jpro-media/src/main/java/one/jpro/platform/media/recorder/NativeMediaRecorder.java index f211e93b..55ab729a 100644 --- a/jpro-media/src/main/java/one/jpro/platform/media/recorder/NativeMediaRecorder.java +++ b/jpro-media/src/main/java/one/jpro/platform/media/recorder/NativeMediaRecorder.java @@ -119,12 +119,17 @@ public NativeMediaRecorder() { })); } - ImageView getCameraView() { + /** + * Returns the {@link ImageView} instance used to display video frames captured from the camera. + * + * @return ImageView instance showing the captured video frames + */ + public final ImageView getCameraView() { return cameraView; } @Override - public void enable() { + public final void enable() { if (recorderReady) { logger.info("Media recorder is already enabled."); } else { @@ -243,7 +248,7 @@ private void enableAudioCapture() { /** * Records a single video frame. * - * @param frame the video frame to be recorded. + * @param frame the video frame to be recorded */ private void writeVideoFrame(Frame frame) { if (recorder != null && recording) { @@ -271,7 +276,7 @@ private void writeAudioSamples(ShortBuffer audioSamples) { } @Override - public void start() { + public final void start() { if (getStatus().equals(Status.INACTIVE) || getStatus().equals(Status.READY)) { final Runnable startRecordingRunnable = () -> { // Start the recording if the camera is enabled @@ -336,7 +341,7 @@ public void start() { } @Override - public void pause() { + public final void pause() { // Disable recording recording = false; @@ -350,7 +355,7 @@ public void pause() { } @Override - public void stop() { + public final void stop() { final Runnable startRecordingRunnable = () -> { stopRecording(); @@ -390,7 +395,7 @@ private void stopRecording() { /** * Stop the acquisition from the camera and microphone and release all the resources. */ - public void release() { + public final void release() { recorderReady = false; recording = false; releaseVideoResources(); diff --git a/jpro-media/src/main/java/one/jpro/platform/media/recorder/NativeMediaRecorderView.java b/jpro-media/src/main/java/one/jpro/platform/media/recorder/NativeMediaRecorderView.java index d76a3097..878ded28 100644 --- a/jpro-media/src/main/java/one/jpro/platform/media/recorder/NativeMediaRecorderView.java +++ b/jpro-media/src/main/java/one/jpro/platform/media/recorder/NativeMediaRecorderView.java @@ -30,7 +30,7 @@ public NativeMediaRecorderView(NativeMediaRecorder mediaRecorder) { } @Override - public ObjectProperty mediaEngineProperty() { + public final ObjectProperty mediaEngineProperty() { if (mediaEngine == null) { mediaEngine = new SimpleObjectProperty<>(this, "mediaEngine") { @@ -60,7 +60,7 @@ protected void invalidated() { } @Override - public DoubleProperty fitHeightProperty() { + public final DoubleProperty fitHeightProperty() { if (fitHeight == null) { fitHeight = new SimpleDoubleProperty(this, "fitHeight") { diff --git a/jpro-media/src/main/java/one/jpro/platform/media/recorder/WebMediaRecorder.java b/jpro-media/src/main/java/one/jpro/platform/media/recorder/WebMediaRecorder.java index 0076a598..519895a5 100644 --- a/jpro-media/src/main/java/one/jpro/platform/media/recorder/WebMediaRecorder.java +++ b/jpro-media/src/main/java/one/jpro/platform/media/recorder/WebMediaRecorder.java @@ -17,7 +17,7 @@ * * @author Besmir Beqiri */ -public final class WebMediaRecorder extends BaseMediaRecorder implements WebMediaEngine { +public class WebMediaRecorder extends BaseMediaRecorder implements WebMediaEngine { private static final String DEFAULT_MIME_TYPE = "video/webm"; @@ -118,12 +118,12 @@ public WebMediaRecorder(WebAPI webAPI) { } @Override - public WebAPI getWebAPI() { + public final WebAPI getWebAPI() { return webAPI; } @Override - public JSVariable getVideoElement() { + public final JSVariable getVideoElement() { return recorderVideoElement; } @@ -138,7 +138,7 @@ public JSVariable getVideoElement() { * specified format. * @throws Exception */ - public boolean isTypeSupported(String mimeType) throws Exception { + public final boolean isTypeSupported(String mimeType) throws Exception { return Boolean.getBoolean(webAPI.executeScriptWithReturn(""" MediaRecorder.isTypeSupported("%s") """.formatted(mimeType))); @@ -147,7 +147,7 @@ public boolean isTypeSupported(String mimeType) throws Exception { // mimeType property (read-only) private ReadOnlyStringWrapper mimeType; - public String getMimeType() { + public final String getMimeType() { return (mimeType == null) ? DEFAULT_MIME_TYPE : mimeType.get(); } @@ -155,7 +155,7 @@ private void setMimeType(String value) { mimeTypePropertyImpl().set(value); } - public ReadOnlyStringProperty mimeTypeProperty() { + public final ReadOnlyStringProperty mimeTypeProperty() { return mimeTypePropertyImpl().getReadOnlyProperty(); } @@ -168,7 +168,7 @@ private ReadOnlyStringWrapper mimeTypePropertyImpl() { // Recorder controller methods @Override - public void enable() { + public final void enable() { final var mediaRecorderOptions = new MediaRecorderOptions().mimeType(getMimeType()); webAPI.executeScript(""" $blobsRecorded = []; // stream buffer @@ -218,7 +218,7 @@ public void enable() { } @Override - public void start() { + public final void start() { if (recorderReady) { if (getStatus().equals(Status.INACTIVE) || getStatus().equals(Status.READY)) { webAPI.executeScript(""" @@ -238,7 +238,7 @@ public void start() { } @Override - public void pause() { + public final void pause() { if (recorderReady) { webAPI.executeScript(""" if ($mediaRecorder.state === "recording") { @@ -249,7 +249,7 @@ public void pause() { } @Override - public void stop() { + public final void stop() { if (recorderReady) { webAPI.executeScript(""" $mediaRecorder.stop(); From 01dc65894a4ec7dc24f0a5c2ff0af110a68e70a0 Mon Sep 17 00:00:00 2001 From: Besmir Beqiri Date: Fri, 12 Jan 2024 15:08:28 +0100 Subject: [PATCH 5/9] Automatically determine the correct audio input device name for recording based on the current running operating system --- .../media/recorder/NativeMediaRecorder.java | 44 +++++++++++++++++-- 1 file changed, 41 insertions(+), 3 deletions(-) diff --git a/jpro-media/src/main/java/one/jpro/platform/media/recorder/NativeMediaRecorder.java b/jpro-media/src/main/java/one/jpro/platform/media/recorder/NativeMediaRecorder.java index 55ab729a..935cd782 100644 --- a/jpro-media/src/main/java/one/jpro/platform/media/recorder/NativeMediaRecorder.java +++ b/jpro-media/src/main/java/one/jpro/platform/media/recorder/NativeMediaRecorder.java @@ -39,7 +39,6 @@ public class NativeMediaRecorder extends BaseMediaRecorder { // Video resources private static final int CAMERA_DEVICE_INDEX = 0; // Use the default system camera - private static final String MICROPHONE_DEVICE_NAME = ":0"; // Use the default system microphone private static final int FRAME_RATE = 30; private static final String MP4_FILE_EXTENSION = ".mp4"; private FrameGrabber cameraGrabber; @@ -96,7 +95,7 @@ public NativeMediaRecorder() { cameraView = new ImageView(); // Initialize audio frame grabber - micGrabber = new FFmpegFrameGrabber(MICROPHONE_DEVICE_NAME); + micGrabber = new FFmpegFrameGrabber(getDefaultAudioInputDevice()); micGrabber.setFormat("avfoundation"); micGrabber.setAudioChannels(DEFAULT_AUDIO_CHANNELS); micGrabber.setSampleRate(DEFAULT_AUDIO_SAMPLE_RATE); @@ -536,7 +535,7 @@ private void printCaptureDeviceDescription() { // Print microphone device description if (micGrabber != null) { - String microphoneDescription = micGrabber.getFormat() + " - " + MICROPHONE_DEVICE_NAME; + String microphoneDescription = micGrabber.getFormat() + " - " + getDefaultAudioInputDevice(); logger.debug("Microphone Device Description: " + microphoneDescription); logger.debug("Audio Channels: " + micGrabber.getAudioChannels()); logger.debug("Sample Rate: " + micGrabber.getSampleRate()); @@ -554,4 +553,43 @@ private void adjustTimestamp() { recorder.setTimestamp(t); } } + + /** + * Gets the default audio input device name based on the operating system. + *

+ * This method determines the appropriate audio input device for use with {@link FFmpegFrameGrabber}, + * based on the operating system where the application is running: + *

+ *
    + *
  • Windows: Returns "null", which typically lets the system select the default device. + * Note: This might need to be adjusted based on the system configuration.
    • + *
  • macOS: Returns ":0" to refer to the default audio input device as per AVFoundation's standard.
    • + *
  • Linux: Returns "default", which typically refers to the default ALSA audio input device.
    • + *
  • Other: Returns "default" as a generic fallback. This should be validated or adjusted + * based on the specific requirements or environment.
    • + *
+ * + * @return the name of the default audio input device + */ + public static String getDefaultAudioInputDevice() { + String OS = System.getProperty("os.name").toLowerCase(); + + String audioDevice; + if (OS.contains("win")) { + // Windows - use device name or leave it null for default + audioDevice = "null"; // TODO: might need to adjust this based on the windows system + } else if (OS.contains("mac")) { + // macOS - default device + audioDevice = ":0"; + } else if (OS.contains("nix") || OS.contains("nux") || OS.contains("aix")) { + // Linux - default device + audioDevice = "default"; + } else { + // Unknown OS - fallback + audioDevice = "default"; + } + + return audioDevice; + } + } From 5d2377ee6e4517594e389f72d041b49dd2c09f8e Mon Sep 17 00:00:00 2001 From: Besmir Beqiri Date: Fri, 12 Jan 2024 15:13:07 +0100 Subject: [PATCH 6/9] Determine the correct audio format for recording based on the current running operating system --- .../media/recorder/NativeMediaRecorder.java | 39 ++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/jpro-media/src/main/java/one/jpro/platform/media/recorder/NativeMediaRecorder.java b/jpro-media/src/main/java/one/jpro/platform/media/recorder/NativeMediaRecorder.java index 935cd782..3090540b 100644 --- a/jpro-media/src/main/java/one/jpro/platform/media/recorder/NativeMediaRecorder.java +++ b/jpro-media/src/main/java/one/jpro/platform/media/recorder/NativeMediaRecorder.java @@ -96,7 +96,7 @@ public NativeMediaRecorder() { // Initialize audio frame grabber micGrabber = new FFmpegFrameGrabber(getDefaultAudioInputDevice()); - micGrabber.setFormat("avfoundation"); + micGrabber.setFormat(getDefaultAudioInputFormat()); micGrabber.setAudioChannels(DEFAULT_AUDIO_CHANNELS); micGrabber.setSampleRate(DEFAULT_AUDIO_SAMPLE_RATE); @@ -592,4 +592,41 @@ public static String getDefaultAudioInputDevice() { return audioDevice; } + /** + * Gets the default audio input format based on the operating system. + *

+ * This method determines the appropriate audio input format for use with {@link FFmpegFrameGrabber}, + * based on the operating system where the application is running: + *

+ *
    + *
  • Windows: Returns "dshow" for DirectShow.
    • + *
  • macOS: Returns "avfoundation" for AVFoundation.
    • + *
  • Linux: Returns "alsa" for ALSA.
    • + *
  • Other: Returns "default" as a fallback, + * but this should be adjusted based on specific requirements or environment.
    • + *
+ * + * @return the format string for audio input format + */ + public static String getDefaultAudioInputFormat() { + String OS = System.getProperty("os.name").toLowerCase(); + String format; + + if (OS.contains("win")) { + // Windows - DirectShow + format = "dshow"; + } else if (OS.contains("mac")) { + // macOS - AVFoundation + format = "avfoundation"; + } else if (OS.contains("nix") || OS.contains("nux") || OS.contains("aix")) { + // Linux - ALSA + format = "alsa"; + } else { + // Unknown OS - fallback + format = "default"; + } + + return format; + } + } From aca176c3fbcefffb5cbce88f7bddeff2da228bb1 Mon Sep 17 00:00:00 2001 From: Besmir Beqiri Date: Fri, 12 Jan 2024 18:44:25 +0100 Subject: [PATCH 7/9] Retrieve audio input devices list using Java Sound API --- .../media/recorder/NativeMediaRecorder.java | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/jpro-media/src/main/java/one/jpro/platform/media/recorder/NativeMediaRecorder.java b/jpro-media/src/main/java/one/jpro/platform/media/recorder/NativeMediaRecorder.java index 3090540b..0b2c224c 100644 --- a/jpro-media/src/main/java/one/jpro/platform/media/recorder/NativeMediaRecorder.java +++ b/jpro-media/src/main/java/one/jpro/platform/media/recorder/NativeMediaRecorder.java @@ -12,14 +12,17 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.sound.sampled.*; import java.io.File; import java.io.IOException; import java.nio.CharBuffer; import java.nio.ShortBuffer; import java.nio.file.Files; import java.nio.file.Path; +import java.util.ArrayList; import java.util.Arrays; import java.util.Comparator; +import java.util.List; import java.util.concurrent.*; import java.util.stream.Stream; @@ -144,6 +147,11 @@ public final void enable() { setError("Exception during the enabling of video and audio capture.", ex); } + final var micDevices = getAudioInputDevices(); + for (Mixer.Info micDevice : micDevices) { + logger.info("Mic device: {}", micDevice.getName()); + } + // Set recorder ready recorderReady = true; @@ -510,6 +518,44 @@ private void setError(String message, Exception ex) { logger.error(message, ex); } + /** + * Retries the list of available audio input devices used as microphones. + * + * @return the list of available audio input devices + */ + private List getAudioInputDevices() { + final Mixer.Info[] mixerInfos = AudioSystem.getMixerInfo(); + List result = new ArrayList<>(); + for (Mixer.Info info : mixerInfos) { + final Mixer mixer = AudioSystem.getMixer(info); + final Line.Info[] lineInfos = mixer.getTargetLineInfo(); + // Only prints out info is it is a Microphone + if (lineInfos.length >= 1 && lineInfos[0].getLineClass().equals(TargetDataLine.class)) { + logger.debug(DIVIDER_LINE); + for (Line.Info lineInfo : lineInfos) { + logger.debug("Mic Line Name: " + info.getName()); // The audio device name + logger.debug("Mic Line Description: " + info.getDescription()); // The type of audio device + printSupportedAudioFormats(lineInfo); + } + logger.debug(DIVIDER_LINE); + result.add(info); + } + } + return result; + } + + /** + * Print to terminal the supported audio formats. + * + * @param lineInfo the line info + */ + private void printSupportedAudioFormats(final Line.Info lineInfo) { + if (lineInfo instanceof final DataLine.Info dataLineInfo) { + logger.debug("Supported Audio Formats:"); + Arrays.stream(dataLineInfo.getFormats()).forEach(format -> logger.debug("{}", format)); + } + } + /** * Prints to terminal the capture device description. This includes details about the camera and * microphone being used for recording. From 11058947e3827a07477961d7649ca8fc947c44e7 Mon Sep 17 00:00:00 2001 From: Besmir Beqiri Date: Fri, 12 Jan 2024 19:00:02 +0100 Subject: [PATCH 8/9] Fix audio recoding device name retrieval for Windows platforms --- .../media/recorder/NativeMediaRecorder.java | 79 ++++++++----------- 1 file changed, 33 insertions(+), 46 deletions(-) diff --git a/jpro-media/src/main/java/one/jpro/platform/media/recorder/NativeMediaRecorder.java b/jpro-media/src/main/java/one/jpro/platform/media/recorder/NativeMediaRecorder.java index 0b2c224c..b3354d0a 100644 --- a/jpro-media/src/main/java/one/jpro/platform/media/recorder/NativeMediaRecorder.java +++ b/jpro-media/src/main/java/one/jpro/platform/media/recorder/NativeMediaRecorder.java @@ -147,11 +147,6 @@ public final void enable() { setError("Exception during the enabling of video and audio capture.", ex); } - final var micDevices = getAudioInputDevices(); - for (Mixer.Info micDevice : micDevices) { - logger.info("Mic device: {}", micDevice.getName()); - } - // Set recorder ready recorderReady = true; @@ -518,44 +513,6 @@ private void setError(String message, Exception ex) { logger.error(message, ex); } - /** - * Retries the list of available audio input devices used as microphones. - * - * @return the list of available audio input devices - */ - private List getAudioInputDevices() { - final Mixer.Info[] mixerInfos = AudioSystem.getMixerInfo(); - List result = new ArrayList<>(); - for (Mixer.Info info : mixerInfos) { - final Mixer mixer = AudioSystem.getMixer(info); - final Line.Info[] lineInfos = mixer.getTargetLineInfo(); - // Only prints out info is it is a Microphone - if (lineInfos.length >= 1 && lineInfos[0].getLineClass().equals(TargetDataLine.class)) { - logger.debug(DIVIDER_LINE); - for (Line.Info lineInfo : lineInfos) { - logger.debug("Mic Line Name: " + info.getName()); // The audio device name - logger.debug("Mic Line Description: " + info.getDescription()); // The type of audio device - printSupportedAudioFormats(lineInfo); - } - logger.debug(DIVIDER_LINE); - result.add(info); - } - } - return result; - } - - /** - * Print to terminal the supported audio formats. - * - * @param lineInfo the line info - */ - private void printSupportedAudioFormats(final Line.Info lineInfo) { - if (lineInfo instanceof final DataLine.Info dataLineInfo) { - logger.debug("Supported Audio Formats:"); - Arrays.stream(dataLineInfo.getFormats()).forEach(format -> logger.debug("{}", format)); - } - } - /** * Prints to terminal the capture device description. This includes details about the camera and * microphone being used for recording. @@ -581,7 +538,8 @@ private void printCaptureDeviceDescription() { // Print microphone device description if (micGrabber != null) { - String microphoneDescription = micGrabber.getFormat() + " - " + getDefaultAudioInputDevice(); + String microphoneDescription = "format: " + micGrabber.getFormat() + + " - device: " + getDefaultAudioInputDevice(); logger.debug("Microphone Device Description: " + microphoneDescription); logger.debug("Audio Channels: " + micGrabber.getAudioChannels()); logger.debug("Sample Rate: " + micGrabber.getSampleRate()); @@ -622,8 +580,8 @@ public static String getDefaultAudioInputDevice() { String audioDevice; if (OS.contains("win")) { - // Windows - use device name or leave it null for default - audioDevice = "null"; // TODO: might need to adjust this based on the windows system + // Windows - use device name + audioDevice = "audio=" + getAudioInputDevices().get(1).getName(); } else if (OS.contains("mac")) { // macOS - default device audioDevice = ":0"; @@ -638,6 +596,35 @@ public static String getDefaultAudioInputDevice() { return audioDevice; } + /** + * Retries the list of available audio input devices used as microphones. + * + * @return the list of available audio input devices + */ + private static List getAudioInputDevices() { + final Mixer.Info[] mixerInfos = AudioSystem.getMixerInfo(); + List result = new ArrayList<>(); + for (Mixer.Info info : mixerInfos) { + final Mixer mixer = AudioSystem.getMixer(info); + final Line.Info[] lineInfos = mixer.getTargetLineInfo(); + // Only prints out info is it is a Microphone + if (lineInfos.length >= 1 && lineInfos[0].getLineClass().equals(TargetDataLine.class)) { + logger.debug(DIVIDER_LINE); + for (Line.Info lineInfo : lineInfos) { + logger.debug("Mic Line Name: " + info.getName()); // The audio device name + logger.debug("Mic Line Description: " + info.getDescription()); // The type of audio device + logger.debug("Supported Audio Formats:"); + if (lineInfo instanceof final DataLine.Info dataLineInfo) { + Arrays.stream(dataLineInfo.getFormats()).forEach(format -> logger.debug("{}", format)); + } + } + logger.debug(DIVIDER_LINE); + result.add(info); + } + } + return result; + } + /** * Gets the default audio input format based on the operating system. *

From d56b57d0e21702e4134c04be012ca0d069133230 Mon Sep 17 00:00:00 2001 From: Besmir Beqiri Date: Fri, 19 Jan 2024 15:40:01 +0100 Subject: [PATCH 9/9] Hide externally `flandmark-platform` needed dependency --- README.md | 9 --------- jpro-media/README.md | 1 - jpro-media/build.gradle | 2 +- jpro-media/example/build.gradle | 1 - 4 files changed, 1 insertion(+), 12 deletions(-) diff --git a/README.md b/README.md index 0f1c8ff3..81717d64 100644 --- a/README.md +++ b/README.md @@ -172,14 +172,6 @@ all while utilizing the same codebase. - - - - org.bytedeco - flandmark-platform - 1.07-1.5.8 - runtime - ``` @@ -192,7 +184,6 @@ plugins { dependencies { implementation("one.jpro.platform:jpro-media:0.2.11-SNAPSHOT") implementation "org.bytedeco:javacv-platform:1.5.9" // use compileOnly configuration when running/deploying with JPro - runtimeOnly 'org.bytedeco:flandmark-platform:1.07-1.5.8' // when running on desktop/device only } ``` diff --git a/jpro-media/README.md b/jpro-media/README.md index 604eb1cd..cbdcd7a2 100644 --- a/jpro-media/README.md +++ b/jpro-media/README.md @@ -38,7 +38,6 @@ plugins { dependencies { implementation 'one.jpro.platform:jpro-media:0.2.11-SNAPSHOT' - runtimeOnly 'org.bytedeco:flandmark-platform:1.07-1.5.8' // when running on desktop/device only // use compileOnly configuration when running/deploying with JPro, // since the platform specific libraries are no more needed diff --git a/jpro-media/build.gradle b/jpro-media/build.gradle index 558333e8..0c601432 100644 --- a/jpro-media/build.gradle +++ b/jpro-media/build.gradle @@ -5,12 +5,12 @@ plugins { dependencies { implementation "com.sandec.jpro:jpro-webapi:$JPRO_VERSION" compileOnly "org.bytedeco:javacv-platform:$JAVACV_VERSION" + runtimeOnly 'org.bytedeco:flandmark-platform:1.07-1.5.8' api "org.slf4j:slf4j-api:$SLF4J_API_VERSION" api "org.json:json:$JSON_VERSION" testImplementation "org.testfx:testfx-junit5:$TESTFX_VERSION" testImplementation "org.bytedeco:javacv-platform:$JAVACV_VERSION" - testRuntimeOnly 'org.bytedeco:flandmark-platform:1.07-1.5.8' } compileJava { diff --git a/jpro-media/example/build.gradle b/jpro-media/example/build.gradle index edc8e4f6..b3553226 100644 --- a/jpro-media/example/build.gradle +++ b/jpro-media/example/build.gradle @@ -11,7 +11,6 @@ dependencies { implementation "org.bytedeco:javacv-platform:$JAVACV_VERSION" // compileOnly "org.bytedeco:javacv-platform:$JAVACV_VERSION" // when running on JPro - runtimeOnly 'org.bytedeco:flandmark-platform:1.07-1.5.8' // when running on desktop/device only implementation "io.github.mkpaz:atlantafx-base:$ATLANTAFX_VERSION" runtimeOnly "ch.qos.logback:logback-classic:$LOGBACK_VERSION" }