001/**
002 * Licensed to the Apache Software Foundation (ASF) under one or more
003 * contributor license agreements.  See the NOTICE file distributed with
004 * this work for additional information regarding copyright ownership.
005 * The ASF licenses this file to You under the Apache License, Version 2.0
006 * (the "License"); you may not use this file except in compliance with
007 * the License.  You may obtain a copy of the License at
008 *
009 *      http://www.apache.org/licenses/LICENSE-2.0
010 *
011 * Unless required by applicable law or agreed to in writing, software
012 * distributed under the License is distributed on an "AS IS" BASIS,
013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
014 * See the License for the specific language governing permissions and
015 * limitations under the License.
016 */
017package org.apache.camel.impl;
018
019import java.util.List;
020import java.util.Timer;
021import java.util.TimerTask;
022import java.util.concurrent.atomic.AtomicInteger;
023import java.util.concurrent.locks.Lock;
024import java.util.concurrent.locks.ReentrantLock;
025
026import org.apache.camel.CamelContext;
027import org.apache.camel.CamelContextAware;
028import org.apache.camel.Exchange;
029import org.apache.camel.Route;
030import org.apache.camel.processor.loadbalancer.CircuitBreakerLoadBalancer;
031import org.apache.camel.spi.RoutePolicy;
032import org.apache.camel.support.RoutePolicySupport;
033import org.slf4j.Logger;
034import org.slf4j.LoggerFactory;
035
036/**
037 * Modeled after the {@link CircuitBreakerLoadBalancer} and {@link ThrottlingInflightRoutePolicy}
038 * this {@link RoutePolicy} will stop consuming from an endpoint based on the type of exceptions that are
039 * thrown and the threshold setting. 
040 * 
041 * the scenario: if a route cannot process data from an endpoint due to problems with resources used by the route
042 * (ie database down) then it will stop consuming new messages from the endpoint by stopping the consumer. 
043 * The implementation is comparable to the Circuit Breaker pattern. After a set amount of time, it will move 
044 * to a half open state and attempt to determine if the consumer can be started.
045 * There are two ways to determine if a route can be closed after being opened
046 * (1) start the consumer and check the failure threshold
047 * (2) call the {@link ThrottlingExceptionHalfOpenHandler} 
048 * The second option allows a custom check to be performed without having to take on the possibility of
049 * multiple messages from the endpoint. The idea is that a handler could run a simple test (ie select 1 from dual)
050 * to determine if the processes that cause the route to be open are now available  
051 */
052public class ThrottlingExceptionRoutePolicy extends RoutePolicySupport implements CamelContextAware {
053    private static final Logger LOG = LoggerFactory.getLogger(ThrottlingExceptionRoutePolicy.class);
054    
055    private static final int STATE_CLOSED = 0;
056    private static final int STATE_HALF_OPEN = 1;
057    private static final int STATE_OPEN = 2;
058    
059    private CamelContext camelContext;
060    private final Lock lock = new ReentrantLock();
061    
062    // configuration
063    private int failureThreshold;
064    private long failureWindow;
065    private long halfOpenAfter;
066    private final List<Class<?>> throttledExceptions;
067    
068    // handler for half open circuit
069    // can be used instead of resuming route
070    // to check on resources
071    private ThrottlingExceptionHalfOpenHandler halfOpenHandler;
072
073    // stateful information
074    private final AtomicInteger failures = new AtomicInteger();
075    private final AtomicInteger state = new AtomicInteger(STATE_CLOSED);
076    private volatile Timer halfOpenTimer;
077    private volatile long lastFailure;
078    private volatile long openedAt;
079    
080    public ThrottlingExceptionRoutePolicy(int threshold, long failureWindow, long halfOpenAfter, List<Class<?>> handledExceptions) {
081        this.throttledExceptions = handledExceptions;
082        this.failureWindow = failureWindow;
083        this.halfOpenAfter = halfOpenAfter;
084        this.failureThreshold = threshold;
085    }
086    
087    @Override
088    public void setCamelContext(CamelContext camelContext) {
089        this.camelContext = camelContext;
090    }
091
092    @Override
093    public CamelContext getCamelContext() {
094        return camelContext;
095    }
096
097    @Override
098    public void onInit(Route route) {
099        LOG.debug("Initializing ThrottlingExceptionRoutePolicy route policy...");
100        logState();
101    }
102    
103    @Override
104    public void onExchangeDone(Route route, Exchange exchange) {
105        if (hasFailed(exchange)) {
106            // record the failure
107            failures.incrementAndGet();
108            lastFailure = System.currentTimeMillis();
109        } 
110        
111        // check for state change
112        calculateState(route);
113    }
114    
115    /**
116     * uses similar approach as {@link CircuitBreakerLoadBalancer}
117     * if the exchange has an exception that we are watching 
118     * then we count that as a failure otherwise we ignore it
119     */
120    private boolean hasFailed(Exchange exchange) {
121        if (exchange == null) {
122            return false;
123        }
124
125        boolean answer = false;
126
127        if (exchange.getException() != null) {
128            if (throttledExceptions == null || throttledExceptions.isEmpty()) {
129                // if no exceptions defined then always fail 
130                // (ie) assume we throttle on all exceptions
131                answer = true;
132            } else {
133                for (Class<?> exception : throttledExceptions) {
134                    // will look in exception hierarchy
135                    if (exchange.getException(exception) != null) {
136                        answer = true;
137                        break;
138                    }
139                }
140            }
141        }
142
143        if (LOG.isDebugEnabled()) {
144            String exceptionName = exchange.getException() == null ? "none" : exchange.getException().getClass().getSimpleName();
145            LOG.debug("hasFailed ({}) with Throttled Exception: {} for exchangeId: {}", answer, exceptionName, exchange.getExchangeId());
146        }
147        return answer;
148    }
149
150    private void calculateState(Route route) {
151        
152        // have we reached the failure limit?
153        boolean failureLimitReached = isThresholdExceeded();
154        
155        if (state.get() == STATE_CLOSED) {
156            if (failureLimitReached) {
157                LOG.debug("Opening circuit...");
158                openCircuit(route);
159            }
160        } else if (state.get() == STATE_HALF_OPEN) {
161            if (failureLimitReached) {
162                LOG.debug("Opening circuit...");
163                openCircuit(route);
164            } else {
165                LOG.debug("Closing circuit...");
166                closeCircuit(route);
167            }
168        } else if (state.get() == STATE_OPEN) {
169            long elapsedTimeSinceOpened = System.currentTimeMillis() - openedAt;
170            if (halfOpenAfter <= elapsedTimeSinceOpened) {
171                LOG.debug("Checking an open circuit...");
172                if (halfOpenHandler != null) {
173                    if (halfOpenHandler.isReadyToBeClosed()) {
174                        LOG.debug("Closing circuit...");
175                        closeCircuit(route);
176                    } else {
177                        LOG.debug("Opening circuit...");
178                        openCircuit(route);
179                    }
180                } else {
181                    LOG.debug("Half opening circuit...");
182                    halfOpenCircuit(route);                    
183                }
184            } 
185        }
186        
187    }
188    
189    protected boolean isThresholdExceeded() {
190        boolean output = false;
191        logState();
192        // failures exceed the threshold 
193        // AND the last of those failures occurred within window
194        if ((failures.get() >= failureThreshold) && (lastFailure >= System.currentTimeMillis() - failureWindow)) {
195            output = true;
196        }
197        
198        return output;
199    }
200        
201    protected void openCircuit(Route route) {
202        try {
203            lock.lock();
204            suspendOrStopConsumer(route.getConsumer());
205            state.set(STATE_OPEN);
206            openedAt = System.currentTimeMillis();
207            halfOpenTimer = new Timer();
208            halfOpenTimer.schedule(new HalfOpenTask(route), halfOpenAfter);
209            logState();
210        } catch (Exception e) {
211            handleException(e);
212        } finally {
213            lock.unlock();
214        }
215    }
216
217    protected void halfOpenCircuit(Route route) {
218        try {
219            lock.lock();
220            resumeOrStartConsumer(route.getConsumer());
221            state.set(STATE_HALF_OPEN);
222            logState();
223        } catch (Exception e) {
224            handleException(e);
225        } finally {
226            lock.unlock();
227        }
228    }
229    
230    protected void closeCircuit(Route route) {
231        try {
232            lock.lock();
233            resumeOrStartConsumer(route.getConsumer());
234            failures.set(0);
235            lastFailure = 0;
236            openedAt = 0;
237            state.set(STATE_CLOSED);
238            logState();
239        } catch (Exception e) {
240            handleException(e);
241        } finally {
242            lock.unlock();
243        }
244    }
245    
246    private void logState() {
247        if (LOG.isDebugEnabled()) {
248            LOG.debug(dumpState());
249        }
250    }
251    
252    public String dumpState() {
253        int num = state.get();
254        String routeState = stateAsString(num);
255        if (failures.get() > 0) {
256            return String.format("State %s, failures %d, last failure %d ms ago", routeState, failures.get(), System.currentTimeMillis() - lastFailure);
257        } else {
258            return String.format("State %s, failures %d", routeState, failures.get());
259        }
260    }
261    
262    private static String stateAsString(int num) {
263        if (num == STATE_CLOSED) {
264            return "closed";
265        } else if (num == STATE_HALF_OPEN) {
266            return "half opened";
267        } else {
268            return "opened";
269        }
270    }
271    
272    class HalfOpenTask extends TimerTask {
273        private final Route route;
274        
275        HalfOpenTask(Route route) {
276            this.route = route;
277        }
278        
279        @Override
280        public void run() {
281            calculateState(route);
282            halfOpenTimer.cancel();
283        }
284    }
285    
286    public ThrottlingExceptionHalfOpenHandler getHalfOpenHandler() {
287        return halfOpenHandler;
288    }
289
290    public void setHalfOpenHandler(ThrottlingExceptionHalfOpenHandler halfOpenHandler) {
291        this.halfOpenHandler = halfOpenHandler;
292    }
293
294    public int getFailureThreshold() {
295        return failureThreshold;
296    }
297
298    public void setFailureThreshold(int failureThreshold) {
299        this.failureThreshold = failureThreshold;
300    }
301
302    public long getFailureWindow() {
303        return failureWindow;
304    }
305
306    public void setFailureWindow(long failureWindow) {
307        this.failureWindow = failureWindow;
308    }
309
310    public long getHalfOpenAfter() {
311        return halfOpenAfter;
312    }
313
314    public void setHalfOpenAfter(long halfOpenAfter) {
315        this.halfOpenAfter = halfOpenAfter;
316    }
317
318    public int getFailures() {
319        return failures.get();
320    }
321
322    public long getLastFailure() {
323        return lastFailure;
324    }
325
326    public long getOpenedAt() {
327        return openedAt;
328    }
329
330}