/* * Copyright (C) 2014 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package android.hardware.camera2.legacy; import android.hardware.camera2.impl.CameraDeviceImpl; import android.util.Log; import android.util.MutableLong; import android.util.Pair; import android.view.Surface; import java.util.ArrayDeque; import java.util.ArrayList; import java.util.TreeSet; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.Condition; import java.util.concurrent.locks.ReentrantLock; /** * Collect timestamps and state for each {@link CaptureRequest} as it passes through * the Legacy camera pipeline. */ public class CaptureCollector { private static final String TAG = "CaptureCollector"; private static final boolean DEBUG = false; private static final int FLAG_RECEIVED_JPEG = 1; private static final int FLAG_RECEIVED_JPEG_TS = 2; private static final int FLAG_RECEIVED_PREVIEW = 4; private static final int FLAG_RECEIVED_PREVIEW_TS = 8; private static final int FLAG_RECEIVED_ALL_JPEG = FLAG_RECEIVED_JPEG | FLAG_RECEIVED_JPEG_TS; private static final int FLAG_RECEIVED_ALL_PREVIEW = FLAG_RECEIVED_PREVIEW | FLAG_RECEIVED_PREVIEW_TS; private static final int MAX_JPEGS_IN_FLIGHT = 1; private class CaptureHolder implements Comparable{ private final RequestHolder mRequest; private final LegacyRequest mLegacy; public final boolean needsJpeg; public final boolean needsPreview; private long mTimestamp = 0; private int mReceivedFlags = 0; private boolean mHasStarted = false; private boolean mFailedJpeg = false; private boolean mFailedPreview = false; private boolean mCompleted = false; private boolean mPreviewCompleted = false; public CaptureHolder(RequestHolder request, LegacyRequest legacyHolder) { mRequest = request; mLegacy = legacyHolder; needsJpeg = request.hasJpegTargets(); needsPreview = request.hasPreviewTargets(); } public boolean isPreviewCompleted() { return (mReceivedFlags & FLAG_RECEIVED_ALL_PREVIEW) == FLAG_RECEIVED_ALL_PREVIEW; } public boolean isJpegCompleted() { return (mReceivedFlags & FLAG_RECEIVED_ALL_JPEG) == FLAG_RECEIVED_ALL_JPEG; } public boolean isCompleted() { return (needsJpeg == isJpegCompleted()) && (needsPreview == isPreviewCompleted()); } public void tryComplete() { if (!mPreviewCompleted && needsPreview && isPreviewCompleted()) { CaptureCollector.this.onPreviewCompleted(); mPreviewCompleted = true; } if (isCompleted() && !mCompleted) { if (mFailedPreview || mFailedJpeg) { if (!mHasStarted) { // Send a request error if the capture has not yet started. mRequest.failRequest(); CaptureCollector.this.mDeviceState.setCaptureStart(mRequest, mTimestamp, CameraDeviceImpl.CameraDeviceCallbacks.ERROR_CAMERA_REQUEST); } else { // Send buffer dropped errors for each pending buffer if the request has // started. for (Surface targetSurface : mRequest.getRequest().getTargets() ) { try { if (mRequest.jpegType(targetSurface)) { if (mFailedJpeg) { CaptureCollector.this.mDeviceState.setCaptureResult(mRequest, /*result*/null, CameraDeviceImpl.CameraDeviceCallbacks. ERROR_CAMERA_BUFFER, targetSurface); } } else { // preview buffer if (mFailedPreview) { CaptureCollector.this.mDeviceState.setCaptureResult(mRequest, /*result*/null, CameraDeviceImpl.CameraDeviceCallbacks. ERROR_CAMERA_BUFFER, targetSurface); } } } catch (LegacyExceptionUtils.BufferQueueAbandonedException e) { Log.e(TAG, "Unexpected exception when querying Surface: " + e); } } } } CaptureCollector.this.onRequestCompleted(CaptureHolder.this); mCompleted = true; } } public void setJpegTimestamp(long timestamp) { if (DEBUG) { Log.d(TAG, "setJpegTimestamp - called for request " + mRequest.getRequestId()); } if (!needsJpeg) { throw new IllegalStateException( "setJpegTimestamp called for capture with no jpeg targets."); } if (isCompleted()) { throw new IllegalStateException( "setJpegTimestamp called on already completed request."); } mReceivedFlags |= FLAG_RECEIVED_JPEG_TS; if (mTimestamp == 0) { mTimestamp = timestamp; } if (!mHasStarted) { mHasStarted = true; CaptureCollector.this.mDeviceState.setCaptureStart(mRequest, mTimestamp, CameraDeviceState.NO_CAPTURE_ERROR); } tryComplete(); } public void setJpegProduced() { if (DEBUG) { Log.d(TAG, "setJpegProduced - called for request " + mRequest.getRequestId()); } if (!needsJpeg) { throw new IllegalStateException( "setJpegProduced called for capture with no jpeg targets."); } if (isCompleted()) { throw new IllegalStateException( "setJpegProduced called on already completed request."); } mReceivedFlags |= FLAG_RECEIVED_JPEG; tryComplete(); } public void setJpegFailed() { if (DEBUG) { Log.d(TAG, "setJpegFailed - called for request " + mRequest.getRequestId()); } if (!needsJpeg || isJpegCompleted()) { return; } mFailedJpeg = true; mReceivedFlags |= FLAG_RECEIVED_JPEG; mReceivedFlags |= FLAG_RECEIVED_JPEG_TS; tryComplete(); } public void setPreviewTimestamp(long timestamp) { if (DEBUG) { Log.d(TAG, "setPreviewTimestamp - called for request " + mRequest.getRequestId()); } if (!needsPreview) { throw new IllegalStateException( "setPreviewTimestamp called for capture with no preview targets."); } if (isCompleted()) { throw new IllegalStateException( "setPreviewTimestamp called on already completed request."); } mReceivedFlags |= FLAG_RECEIVED_PREVIEW_TS; if (mTimestamp == 0) { mTimestamp = timestamp; } if (!needsJpeg) { if (!mHasStarted) { mHasStarted = true; CaptureCollector.this.mDeviceState.setCaptureStart(mRequest, mTimestamp, CameraDeviceState.NO_CAPTURE_ERROR); } } tryComplete(); } public void setPreviewProduced() { if (DEBUG) { Log.d(TAG, "setPreviewProduced - called for request " + mRequest.getRequestId()); } if (!needsPreview) { throw new IllegalStateException( "setPreviewProduced called for capture with no preview targets."); } if (isCompleted()) { throw new IllegalStateException( "setPreviewProduced called on already completed request."); } mReceivedFlags |= FLAG_RECEIVED_PREVIEW; tryComplete(); } public void setPreviewFailed() { if (DEBUG) { Log.d(TAG, "setPreviewFailed - called for request " + mRequest.getRequestId()); } if (!needsPreview || isPreviewCompleted()) { return; } mFailedPreview = true; mReceivedFlags |= FLAG_RECEIVED_PREVIEW; mReceivedFlags |= FLAG_RECEIVED_PREVIEW_TS; tryComplete(); } // Comparison and equals based on frame number. @Override public int compareTo(CaptureHolder captureHolder) { return (mRequest.getFrameNumber() > captureHolder.mRequest.getFrameNumber()) ? 1 : ((mRequest.getFrameNumber() == captureHolder.mRequest.getFrameNumber()) ? 0 : -1); } // Comparison and equals based on frame number. @Override public boolean equals(Object o) { return o instanceof CaptureHolder && compareTo((CaptureHolder) o) == 0; } } private final TreeSet mActiveRequests; private final ArrayDeque mJpegCaptureQueue; private final ArrayDeque mJpegProduceQueue; private final ArrayDeque mPreviewCaptureQueue; private final ArrayDeque mPreviewProduceQueue; private final ArrayList mCompletedRequests = new ArrayList<>(); private final ReentrantLock mLock = new ReentrantLock(); private final Condition mIsEmpty; private final Condition mPreviewsEmpty; private final Condition mNotFull; private final CameraDeviceState mDeviceState; private int mInFlight = 0; private int mInFlightPreviews = 0; private final int mMaxInFlight; /** * Create a new {@link CaptureCollector} that can modify the given {@link CameraDeviceState}. * * @param maxInFlight max allowed in-flight requests. * @param deviceState the {@link CameraDeviceState} to update as requests are processed. */ public CaptureCollector(int maxInFlight, CameraDeviceState deviceState) { mMaxInFlight = maxInFlight; mJpegCaptureQueue = new ArrayDeque<>(MAX_JPEGS_IN_FLIGHT); mJpegProduceQueue = new ArrayDeque<>(MAX_JPEGS_IN_FLIGHT); mPreviewCaptureQueue = new ArrayDeque<>(mMaxInFlight); mPreviewProduceQueue = new ArrayDeque<>(mMaxInFlight); mActiveRequests = new TreeSet<>(); mIsEmpty = mLock.newCondition(); mNotFull = mLock.newCondition(); mPreviewsEmpty = mLock.newCondition(); mDeviceState = deviceState; } /** * Queue a new request. * *

* For requests that use the Camera1 API preview output stream, this will block if there are * already {@code maxInFlight} requests in progress (until at least one prior request has * completed). For requests that use the Camera1 API jpeg callbacks, this will block until * all prior requests have been completed to avoid stopping preview for * {@link android.hardware.Camera#takePicture} before prior preview requests have been * completed. *

* @param holder the {@link RequestHolder} for this request. * @param legacy the {@link LegacyRequest} for this request; this will not be mutated. * @param timeout a timeout to use for this call. * @param unit the units to use for the timeout. * @return {@code false} if this method timed out. * @throws InterruptedException if this thread is interrupted. */ public boolean queueRequest(RequestHolder holder, LegacyRequest legacy, long timeout, TimeUnit unit) throws InterruptedException { CaptureHolder h = new CaptureHolder(holder, legacy); long nanos = unit.toNanos(timeout); final ReentrantLock lock = this.mLock; lock.lock(); try { if (DEBUG) { Log.d(TAG, "queueRequest for request " + holder.getRequestId() + " - " + mInFlight + " requests remain in flight."); } if (!(h.needsJpeg || h.needsPreview)) { throw new IllegalStateException("Request must target at least one output surface!"); } if (h.needsJpeg) { // Wait for all current requests to finish before queueing jpeg. while (mInFlight > 0) { if (nanos <= 0) { return false; } nanos = mIsEmpty.awaitNanos(nanos); } mJpegCaptureQueue.add(h); mJpegProduceQueue.add(h); } if (h.needsPreview) { while (mInFlight >= mMaxInFlight) { if (nanos <= 0) { return false; } nanos = mNotFull.awaitNanos(nanos); } mPreviewCaptureQueue.add(h); mPreviewProduceQueue.add(h); mInFlightPreviews++; } mActiveRequests.add(h); mInFlight++; return true; } finally { lock.unlock(); } } /** * Wait all queued requests to complete. * * @param timeout a timeout to use for this call. * @param unit the units to use for the timeout. * @return {@code false} if this method timed out. * @throws InterruptedException if this thread is interrupted. */ public boolean waitForEmpty(long timeout, TimeUnit unit) throws InterruptedException { long nanos = unit.toNanos(timeout); final ReentrantLock lock = this.mLock; lock.lock(); try { while (mInFlight > 0) { if (nanos <= 0) { return false; } nanos = mIsEmpty.awaitNanos(nanos); } return true; } finally { lock.unlock(); } } /** * Wait all queued requests that use the Camera1 API preview output to complete. * * @param timeout a timeout to use for this call. * @param unit the units to use for the timeout. * @return {@code false} if this method timed out. * @throws InterruptedException if this thread is interrupted. */ public boolean waitForPreviewsEmpty(long timeout, TimeUnit unit) throws InterruptedException { long nanos = unit.toNanos(timeout); final ReentrantLock lock = this.mLock; lock.lock(); try { while (mInFlightPreviews > 0) { if (nanos <= 0) { return false; } nanos = mPreviewsEmpty.awaitNanos(nanos); } return true; } finally { lock.unlock(); } } /** * Wait for the specified request to be completed (all buffers available). * *

May not wait for the same request more than once, since a successful wait * will erase the history of that request.

* * @param holder the {@link RequestHolder} for this request. * @param timeout a timeout to use for this call. * @param unit the units to use for the timeout. * @param timestamp the timestamp of the request will be written out to here, in ns * * @return {@code false} if this method timed out. * * @throws InterruptedException if this thread is interrupted. */ public boolean waitForRequestCompleted(RequestHolder holder, long timeout, TimeUnit unit, MutableLong timestamp) throws InterruptedException { long nanos = unit.toNanos(timeout); final ReentrantLock lock = this.mLock; lock.lock(); try { while (!removeRequestIfCompleted(holder, /*out*/timestamp)) { if (nanos <= 0) { return false; } nanos = mNotFull.awaitNanos(nanos); } return true; } finally { lock.unlock(); } } private boolean removeRequestIfCompleted(RequestHolder holder, MutableLong timestamp) { int i = 0; for (CaptureHolder h : mCompletedRequests) { if (h.mRequest.equals(holder)) { timestamp.value = h.mTimestamp; mCompletedRequests.remove(i); return true; } i++; } return false; } /** * Called to alert the {@link CaptureCollector} that the jpeg capture has begun. * * @param timestamp the time of the jpeg capture. * @return the {@link RequestHolder} for the request associated with this capture. */ public RequestHolder jpegCaptured(long timestamp) { final ReentrantLock lock = this.mLock; lock.lock(); try { CaptureHolder h = mJpegCaptureQueue.poll(); if (h == null) { Log.w(TAG, "jpegCaptured called with no jpeg request on queue!"); return null; } h.setJpegTimestamp(timestamp); return h.mRequest; } finally { lock.unlock(); } } /** * Called to alert the {@link CaptureCollector} that the jpeg capture has completed. * * @return a pair containing the {@link RequestHolder} and the timestamp of the capture. */ public Pair jpegProduced() { final ReentrantLock lock = this.mLock; lock.lock(); try { CaptureHolder h = mJpegProduceQueue.poll(); if (h == null) { Log.w(TAG, "jpegProduced called with no jpeg request on queue!"); return null; } h.setJpegProduced(); return new Pair<>(h.mRequest, h.mTimestamp); } finally { lock.unlock(); } } /** * Check if there are any pending capture requests that use the Camera1 API preview output. * * @return {@code true} if there are pending preview requests. */ public boolean hasPendingPreviewCaptures() { final ReentrantLock lock = this.mLock; lock.lock(); try { return !mPreviewCaptureQueue.isEmpty(); } finally { lock.unlock(); } } /** * Called to alert the {@link CaptureCollector} that the preview capture has begun. * * @param timestamp the time of the preview capture. * @return a pair containing the {@link RequestHolder} and the timestamp of the capture. */ public Pair previewCaptured(long timestamp) { final ReentrantLock lock = this.mLock; lock.lock(); try { CaptureHolder h = mPreviewCaptureQueue.poll(); if (h == null) { if (DEBUG) { Log.d(TAG, "previewCaptured called with no preview request on queue!"); } return null; } h.setPreviewTimestamp(timestamp); return new Pair<>(h.mRequest, h.mTimestamp); } finally { lock.unlock(); } } /** * Called to alert the {@link CaptureCollector} that the preview capture has completed. * * @return the {@link RequestHolder} for the request associated with this capture. */ public RequestHolder previewProduced() { final ReentrantLock lock = this.mLock; lock.lock(); try { CaptureHolder h = mPreviewProduceQueue.poll(); if (h == null) { Log.w(TAG, "previewProduced called with no preview request on queue!"); return null; } h.setPreviewProduced(); return h.mRequest; } finally { lock.unlock(); } } /** * Called to alert the {@link CaptureCollector} that the next pending preview capture has failed. */ public void failNextPreview() { final ReentrantLock lock = this.mLock; lock.lock(); try { CaptureHolder h1 = mPreviewCaptureQueue.peek(); CaptureHolder h2 = mPreviewProduceQueue.peek(); // Find the request with the lowest frame number. CaptureHolder h = (h1 == null) ? h2 : ((h2 == null) ? h1 : ((h1.compareTo(h2) <= 0) ? h1 : h2)); if (h != null) { mPreviewCaptureQueue.remove(h); mPreviewProduceQueue.remove(h); mActiveRequests.remove(h); h.setPreviewFailed(); } } finally { lock.unlock(); } } /** * Called to alert the {@link CaptureCollector} that the next pending jpeg capture has failed. */ public void failNextJpeg() { final ReentrantLock lock = this.mLock; lock.lock(); try { CaptureHolder h1 = mJpegCaptureQueue.peek(); CaptureHolder h2 = mJpegProduceQueue.peek(); // Find the request with the lowest frame number. CaptureHolder h = (h1 == null) ? h2 : ((h2 == null) ? h1 : ((h1.compareTo(h2) <= 0) ? h1 : h2)); if (h != null) { mJpegCaptureQueue.remove(h); mJpegProduceQueue.remove(h); mActiveRequests.remove(h); h.setJpegFailed(); } } finally { lock.unlock(); } } /** * Called to alert the {@link CaptureCollector} all pending captures have failed. */ public void failAll() { final ReentrantLock lock = this.mLock; lock.lock(); try { CaptureHolder h; while ((h = mActiveRequests.pollFirst()) != null) { h.setPreviewFailed(); h.setJpegFailed(); } mPreviewCaptureQueue.clear(); mPreviewProduceQueue.clear(); mJpegCaptureQueue.clear(); mJpegProduceQueue.clear(); } finally { lock.unlock(); } } private void onPreviewCompleted() { mInFlightPreviews--; if (mInFlightPreviews < 0) { throw new IllegalStateException( "More preview captures completed than requests queued."); } if (mInFlightPreviews == 0) { mPreviewsEmpty.signalAll(); } } private void onRequestCompleted(CaptureHolder capture) { RequestHolder request = capture.mRequest; mInFlight--; if (DEBUG) { Log.d(TAG, "Completed request " + request.getRequestId() + ", " + mInFlight + " requests remain in flight."); } if (mInFlight < 0) { throw new IllegalStateException( "More captures completed than requests queued."); } mCompletedRequests.add(capture); mActiveRequests.remove(capture); mNotFull.signalAll(); if (mInFlight == 0) { mIsEmpty.signalAll(); } } }