001/** 002 * Licensed to the Apache Software Foundation (ASF) under one or more 003 * contributor license agreements. See the NOTICE file distributed with 004 * this work for additional information regarding copyright ownership. 005 * The ASF licenses this file to You under the Apache License, Version 2.0 006 * (the "License"); you may not use this file except in compliance with 007 * the License. You may obtain a copy of the License at 008 * 009 * http://www.apache.org/licenses/LICENSE-2.0 010 * 011 * Unless required by applicable law or agreed to in writing, software 012 * distributed under the License is distributed on an "AS IS" BASIS, 013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 014 * See the License for the specific language governing permissions and 015 * limitations under the License. 016 */ 017package org.apache.camel.processor; 018 019import java.util.ArrayList; 020import java.util.Collection; 021import java.util.Iterator; 022import java.util.LinkedList; 023import java.util.List; 024import java.util.Queue; 025import java.util.concurrent.ConcurrentLinkedQueue; 026import java.util.concurrent.TimeUnit; 027import java.util.concurrent.atomic.AtomicBoolean; 028import java.util.concurrent.locks.Condition; 029import java.util.concurrent.locks.Lock; 030import java.util.concurrent.locks.ReentrantLock; 031 032import org.apache.camel.AsyncCallback; 033import org.apache.camel.AsyncProcessor; 034import org.apache.camel.CamelContext; 035import org.apache.camel.CamelExchangeException; 036import org.apache.camel.Exchange; 037import org.apache.camel.Expression; 038import org.apache.camel.Navigate; 039import org.apache.camel.Predicate; 040import org.apache.camel.Processor; 041import org.apache.camel.spi.ExceptionHandler; 042import org.apache.camel.spi.IdAware; 043import org.apache.camel.support.LoggingExceptionHandler; 044import org.apache.camel.support.ServiceSupport; 045import org.apache.camel.util.AsyncProcessorHelper; 046import org.apache.camel.util.ObjectHelper; 047import org.apache.camel.util.ServiceHelper; 048import org.slf4j.Logger; 049import org.slf4j.LoggerFactory; 050 051/** 052 * A base class for any kind of {@link Processor} which implements some kind of batch processing. 053 * 054 * @version 055 * @deprecated may be removed in the future when we overhaul the resequencer EIP 056 */ 057@Deprecated 058public class BatchProcessor extends ServiceSupport implements AsyncProcessor, Navigate<Processor>, IdAware { 059 060 public static final long DEFAULT_BATCH_TIMEOUT = 1000L; 061 public static final int DEFAULT_BATCH_SIZE = 100; 062 063 private static final Logger LOG = LoggerFactory.getLogger(BatchProcessor.class); 064 065 private String id; 066 private long batchTimeout = DEFAULT_BATCH_TIMEOUT; 067 private int batchSize = DEFAULT_BATCH_SIZE; 068 private int outBatchSize; 069 private boolean groupExchanges; 070 private boolean batchConsumer; 071 private boolean ignoreInvalidExchanges; 072 private boolean reverse; 073 private boolean allowDuplicates; 074 private Predicate completionPredicate; 075 private Expression expression; 076 077 private final CamelContext camelContext; 078 private final Processor processor; 079 private final Collection<Exchange> collection; 080 private ExceptionHandler exceptionHandler; 081 082 private final BatchSender sender; 083 084 public BatchProcessor(CamelContext camelContext, Processor processor, Collection<Exchange> collection, Expression expression) { 085 ObjectHelper.notNull(camelContext, "camelContext"); 086 ObjectHelper.notNull(processor, "processor"); 087 ObjectHelper.notNull(collection, "collection"); 088 ObjectHelper.notNull(expression, "expression"); 089 090 // wrap processor in UnitOfWork so what we send out of the batch runs in a UoW 091 this.camelContext = camelContext; 092 this.processor = processor; 093 this.collection = collection; 094 this.expression = expression; 095 this.sender = new BatchSender(); 096 this.exceptionHandler = new LoggingExceptionHandler(camelContext, getClass()); 097 } 098 099 @Override 100 public String toString() { 101 return "BatchProcessor[to: " + processor + "]"; 102 } 103 104 // Properties 105 // ------------------------------------------------------------------------- 106 107 108 public Expression getExpression() { 109 return expression; 110 } 111 112 public ExceptionHandler getExceptionHandler() { 113 return exceptionHandler; 114 } 115 116 public void setExceptionHandler(ExceptionHandler exceptionHandler) { 117 this.exceptionHandler = exceptionHandler; 118 } 119 120 public int getBatchSize() { 121 return batchSize; 122 } 123 124 /** 125 * Sets the <b>in</b> batch size. This is the number of incoming exchanges that this batch processor will 126 * process before its completed. The default value is {@link #DEFAULT_BATCH_SIZE}. 127 * 128 * @param batchSize the size 129 */ 130 public void setBatchSize(int batchSize) { 131 // setting batch size to 0 or negative is like disabling it, so we set it as the max value 132 // as the code logic is dependent on a batch size having 1..n value 133 if (batchSize <= 0) { 134 LOG.debug("Disabling batch size, will only be triggered by timeout"); 135 this.batchSize = Integer.MAX_VALUE; 136 } else { 137 this.batchSize = batchSize; 138 } 139 } 140 141 public int getOutBatchSize() { 142 return outBatchSize; 143 } 144 145 /** 146 * Sets the <b>out</b> batch size. If the batch processor holds more exchanges than this out size then the 147 * completion is triggered. Can for instance be used to ensure that this batch is completed when a certain 148 * number of exchanges has been collected. By default this feature is <b>not</b> enabled. 149 * 150 * @param outBatchSize the size 151 */ 152 public void setOutBatchSize(int outBatchSize) { 153 this.outBatchSize = outBatchSize; 154 } 155 156 public long getBatchTimeout() { 157 return batchTimeout; 158 } 159 160 public void setBatchTimeout(long batchTimeout) { 161 this.batchTimeout = batchTimeout; 162 } 163 164 public boolean isGroupExchanges() { 165 return groupExchanges; 166 } 167 168 public void setGroupExchanges(boolean groupExchanges) { 169 this.groupExchanges = groupExchanges; 170 } 171 172 public boolean isBatchConsumer() { 173 return batchConsumer; 174 } 175 176 public void setBatchConsumer(boolean batchConsumer) { 177 this.batchConsumer = batchConsumer; 178 } 179 180 public boolean isIgnoreInvalidExchanges() { 181 return ignoreInvalidExchanges; 182 } 183 184 public void setIgnoreInvalidExchanges(boolean ignoreInvalidExchanges) { 185 this.ignoreInvalidExchanges = ignoreInvalidExchanges; 186 } 187 188 public boolean isReverse() { 189 return reverse; 190 } 191 192 public void setReverse(boolean reverse) { 193 this.reverse = reverse; 194 } 195 196 public boolean isAllowDuplicates() { 197 return allowDuplicates; 198 } 199 200 public void setAllowDuplicates(boolean allowDuplicates) { 201 this.allowDuplicates = allowDuplicates; 202 } 203 204 public Predicate getCompletionPredicate() { 205 return completionPredicate; 206 } 207 208 public void setCompletionPredicate(Predicate completionPredicate) { 209 this.completionPredicate = completionPredicate; 210 } 211 212 public Processor getProcessor() { 213 return processor; 214 } 215 216 public List<Processor> next() { 217 if (!hasNext()) { 218 return null; 219 } 220 List<Processor> answer = new ArrayList<Processor>(1); 221 answer.add(processor); 222 return answer; 223 } 224 225 public boolean hasNext() { 226 return processor != null; 227 } 228 229 public String getId() { 230 return id; 231 } 232 233 public void setId(String id) { 234 this.id = id; 235 } 236 237 /** 238 * A strategy method to decide if the "in" batch is completed. That is, whether the resulting exchanges in 239 * the in queue should be drained to the "out" collection. 240 */ 241 private boolean isInBatchCompleted(int num) { 242 return num >= batchSize; 243 } 244 245 /** 246 * A strategy method to decide if the "out" batch is completed. That is, whether the resulting exchange in 247 * the out collection should be sent. 248 */ 249 private boolean isOutBatchCompleted() { 250 if (outBatchSize == 0) { 251 // out batch is disabled, so go ahead and send. 252 return true; 253 } 254 return collection.size() > 0 && collection.size() >= outBatchSize; 255 } 256 257 /** 258 * Strategy Method to process an exchange in the batch. This method allows derived classes to perform 259 * custom processing before or after an individual exchange is processed 260 */ 261 protected void processExchange(Exchange exchange) throws Exception { 262 processor.process(exchange); 263 if (exchange.getException() != null) { 264 getExceptionHandler().handleException("Error processing aggregated exchange: " + exchange, exchange.getException()); 265 } 266 } 267 268 protected void doStart() throws Exception { 269 ServiceHelper.startServices(processor); 270 sender.start(); 271 } 272 273 protected void doStop() throws Exception { 274 sender.cancel(); 275 ServiceHelper.stopServices(processor); 276 collection.clear(); 277 } 278 279 public void process(Exchange exchange) throws Exception { 280 AsyncProcessorHelper.process(this, exchange); 281 } 282 283 /** 284 * Enqueues an exchange for later batch processing. 285 */ 286 public boolean process(Exchange exchange, AsyncCallback callback) { 287 try { 288 // if batch consumer is enabled then we need to adjust the batch size 289 // with the size from the batch consumer 290 if (isBatchConsumer()) { 291 int size = exchange.getProperty(Exchange.BATCH_SIZE, Integer.class); 292 if (batchSize != size) { 293 batchSize = size; 294 LOG.trace("Using batch consumer completion, so setting batch size to: {}", batchSize); 295 } 296 } 297 298 // validate that the exchange can be used 299 if (!isValid(exchange)) { 300 if (isIgnoreInvalidExchanges()) { 301 LOG.debug("Invalid Exchange. This Exchange will be ignored: {}", exchange); 302 } else { 303 throw new CamelExchangeException("Exchange is not valid to be used by the BatchProcessor", exchange); 304 } 305 } else { 306 // exchange is valid so enqueue the exchange 307 sender.enqueueExchange(exchange); 308 } 309 } catch (Throwable e) { 310 exchange.setException(e); 311 } 312 callback.done(true); 313 return true; 314 } 315 316 /** 317 * Is the given exchange valid to be used. 318 * 319 * @param exchange the given exchange 320 * @return <tt>true</tt> if valid, <tt>false</tt> otherwise 321 */ 322 private boolean isValid(Exchange exchange) { 323 Object result = null; 324 try { 325 result = expression.evaluate(exchange, Object.class); 326 } catch (Exception e) { 327 // ignore 328 } 329 return result != null; 330 } 331 332 /** 333 * Sender thread for queued-up exchanges. 334 */ 335 private class BatchSender extends Thread { 336 337 private Queue<Exchange> queue; 338 private Lock queueLock = new ReentrantLock(); 339 private final AtomicBoolean exchangeEnqueued = new AtomicBoolean(); 340 private final Queue<String> completionPredicateMatched = new ConcurrentLinkedQueue<String>(); 341 private Condition exchangeEnqueuedCondition = queueLock.newCondition(); 342 343 BatchSender() { 344 super(camelContext.getExecutorServiceManager().resolveThreadName("Batch Sender")); 345 this.queue = new LinkedList<Exchange>(); 346 } 347 348 @Override 349 public void run() { 350 // Wait until one of either: 351 // * an exchange being queued; 352 // * the batch timeout expiring; or 353 // * the thread being cancelled. 354 // 355 // If an exchange is queued then we need to determine whether the 356 // batch is complete. If it is complete then we send out the batched 357 // exchanges. Otherwise we move back into our wait state. 358 // 359 // If the batch times out then we send out the batched exchanges 360 // collected so far. 361 // 362 // If we receive an interrupt then all blocking operations are 363 // interrupted and our thread terminates. 364 // 365 // The goal of the following algorithm in terms of synchronisation 366 // is to provide fine grained locking i.e. retaining the lock only 367 // when required. Special consideration is given to releasing the 368 // lock when calling an overloaded method i.e. sendExchanges. 369 // Unlocking is important as the process of sending out the exchanges 370 // would otherwise block new exchanges from being queued. 371 372 queueLock.lock(); 373 try { 374 do { 375 try { 376 if (!exchangeEnqueued.get()) { 377 LOG.trace("Waiting for new exchange to arrive or batchTimeout to occur after {} ms.", batchTimeout); 378 exchangeEnqueuedCondition.await(batchTimeout, TimeUnit.MILLISECONDS); 379 } 380 381 // if the completion predicate was triggered then there is an exchange id which denotes when to complete 382 String id = null; 383 if (!completionPredicateMatched.isEmpty()) { 384 id = completionPredicateMatched.poll(); 385 } 386 387 if (id != null || !exchangeEnqueued.get()) { 388 if (id != null) { 389 LOG.trace("Collecting exchanges to be aggregated triggered by completion predicate"); 390 } else { 391 LOG.trace("Collecting exchanges to be aggregated triggered by batch timeout"); 392 } 393 drainQueueTo(collection, batchSize, id); 394 } else { 395 exchangeEnqueued.set(false); 396 boolean drained = false; 397 while (isInBatchCompleted(queue.size())) { 398 drained = true; 399 drainQueueTo(collection, batchSize, id); 400 } 401 if (drained) { 402 LOG.trace("Collecting exchanges to be aggregated triggered by new exchanges received"); 403 } 404 405 if (!isOutBatchCompleted()) { 406 continue; 407 } 408 } 409 410 queueLock.unlock(); 411 try { 412 try { 413 sendExchanges(); 414 } catch (Throwable t) { 415 // a fail safe to handle all exceptions being thrown 416 getExceptionHandler().handleException(t); 417 } 418 } finally { 419 queueLock.lock(); 420 } 421 422 } catch (InterruptedException e) { 423 break; 424 } 425 426 } while (isRunAllowed()); 427 428 } finally { 429 queueLock.unlock(); 430 } 431 } 432 433 /** 434 * This method should be called with queueLock held 435 */ 436 private void drainQueueTo(Collection<Exchange> collection, int batchSize, String exchangeId) { 437 for (int i = 0; i < batchSize; ++i) { 438 Exchange e = queue.poll(); 439 if (e != null) { 440 try { 441 collection.add(e); 442 } catch (Exception t) { 443 e.setException(t); 444 } catch (Throwable t) { 445 getExceptionHandler().handleException(t); 446 } 447 if (exchangeId != null && exchangeId.equals(e.getExchangeId())) { 448 // this batch is complete so stop draining 449 break; 450 } 451 } else { 452 break; 453 } 454 } 455 } 456 457 public void cancel() { 458 interrupt(); 459 } 460 461 public void enqueueExchange(Exchange exchange) { 462 LOG.debug("Received exchange to be batched: {}", exchange); 463 queueLock.lock(); 464 try { 465 // pre test whether the completion predicate matched 466 if (completionPredicate != null) { 467 boolean matches = completionPredicate.matches(exchange); 468 if (matches) { 469 LOG.trace("Exchange matched completion predicate: {}", exchange); 470 // add this exchange to the list of exchanges which marks the batch as complete 471 completionPredicateMatched.add(exchange.getExchangeId()); 472 } 473 } 474 queue.add(exchange); 475 exchangeEnqueued.set(true); 476 exchangeEnqueuedCondition.signal(); 477 } finally { 478 queueLock.unlock(); 479 } 480 } 481 482 private void sendExchanges() throws Exception { 483 Iterator<Exchange> iter = collection.iterator(); 484 while (iter.hasNext()) { 485 Exchange exchange = iter.next(); 486 iter.remove(); 487 try { 488 LOG.debug("Sending aggregated exchange: {}", exchange); 489 processExchange(exchange); 490 } catch (Throwable t) { 491 // must catch throwable to avoid growing memory 492 getExceptionHandler().handleException("Error processing aggregated exchange: " + exchange, t); 493 } 494 } 495 } 496 } 497 498}