001/** 002 * Licensed to the Apache Software Foundation (ASF) under one or more 003 * contributor license agreements. See the NOTICE file distributed with 004 * this work for additional information regarding copyright ownership. 005 * The ASF licenses this file to You under the Apache License, Version 2.0 006 * (the "License"); you may not use this file except in compliance with 007 * the License. You may obtain a copy of the License at 008 * 009 * http://www.apache.org/licenses/LICENSE-2.0 010 * 011 * Unless required by applicable law or agreed to in writing, software 012 * distributed under the License is distributed on an "AS IS" BASIS, 013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 014 * See the License for the specific language governing permissions and 015 * limitations under the License. 016 */ 017package org.apache.camel.impl; 018 019import java.time.Duration; 020import java.util.ArrayList; 021import java.util.Collection; 022import java.util.Collections; 023import java.util.EventObject; 024import java.util.HashMap; 025import java.util.List; 026import java.util.Map; 027import java.util.Optional; 028import java.util.Set; 029import java.util.TreeSet; 030import java.util.concurrent.ConcurrentHashMap; 031import java.util.concurrent.ConcurrentMap; 032import java.util.concurrent.ScheduledExecutorService; 033import java.util.concurrent.TimeUnit; 034import java.util.concurrent.atomic.AtomicBoolean; 035import java.util.concurrent.atomic.AtomicInteger; 036import java.util.function.Function; 037import java.util.stream.Collectors; 038 039import org.apache.camel.CamelContext; 040import org.apache.camel.Exchange; 041import org.apache.camel.Experimental; 042import org.apache.camel.Route; 043import org.apache.camel.RuntimeCamelException; 044import org.apache.camel.ServiceStatus; 045import org.apache.camel.StartupListener; 046import org.apache.camel.management.event.CamelContextStartedEvent; 047import org.apache.camel.model.RouteDefinition; 048import org.apache.camel.spi.HasId; 049import org.apache.camel.spi.RouteContext; 050import org.apache.camel.spi.RouteController; 051import org.apache.camel.spi.RoutePolicy; 052import org.apache.camel.spi.RoutePolicyFactory; 053import org.apache.camel.support.EventNotifierSupport; 054import org.apache.camel.util.ObjectHelper; 055import org.apache.camel.util.backoff.BackOff; 056import org.apache.camel.util.backoff.BackOffTimer; 057import org.apache.camel.util.function.ThrowingConsumer; 058import org.slf4j.Logger; 059import org.slf4j.LoggerFactory; 060 061/** 062 * A simple implementation of the {@link RouteController} that delays the startup 063 * of the routes after the camel context startup and retries to start failing routes. 064 * 065 * NOTE: this is experimental/unstable. 066 */ 067@Experimental 068public class SupervisingRouteController extends DefaultRouteController { 069 private static final Logger LOGGER = LoggerFactory.getLogger(SupervisingRouteController.class); 070 private final Object lock; 071 private final AtomicBoolean contextStarted; 072 private final AtomicInteger routeCount; 073 private final List<Filter> filters; 074 private final Set<RouteHolder> routes; 075 private final CamelContextStartupListener listener; 076 private final RouteManager routeManager; 077 private BackOffTimer timer; 078 private ScheduledExecutorService executorService; 079 private BackOff defaultBackOff; 080 private Map<String, BackOff> backOffConfigurations; 081 private Duration initialDelay; 082 083 public SupervisingRouteController() { 084 this.lock = new Object(); 085 this.contextStarted = new AtomicBoolean(false); 086 this.filters = new ArrayList<>(); 087 this.routeCount = new AtomicInteger(0); 088 this.routes = new TreeSet<>(); 089 this.routeManager = new RouteManager(); 090 this.defaultBackOff = BackOff.builder().build(); 091 this.backOffConfigurations = new HashMap<>(); 092 this.initialDelay = Duration.ofMillis(0); 093 094 try { 095 this.listener = new CamelContextStartupListener(); 096 this.listener.start(); 097 } catch (Exception e) { 098 throw new RuntimeException(e); 099 } 100 } 101 102 // ********************************* 103 // Properties 104 // ********************************* 105 106 public BackOff getDefaultBackOff() { 107 return defaultBackOff; 108 } 109 110 /** 111 * Sets the default back-off. 112 */ 113 public void setDefaultBackOff(BackOff defaultBackOff) { 114 this.defaultBackOff = defaultBackOff; 115 } 116 117 public Map<String, BackOff> getBackOffConfigurations() { 118 return backOffConfigurations; 119 } 120 121 /** 122 * Set the back-off for the given IDs. 123 */ 124 public void setBackOffConfigurations(Map<String, BackOff> backOffConfigurations) { 125 this.backOffConfigurations = backOffConfigurations; 126 } 127 128 public BackOff getBackOff(String id) { 129 return backOffConfigurations.getOrDefault(id, defaultBackOff); 130 } 131 132 /** 133 * Sets the back-off to be applied to the given <code>id</code>. 134 */ 135 public void setBackOff(String id, BackOff backOff) { 136 backOffConfigurations.put(id, backOff); 137 } 138 139 public Duration getInitialDelay() { 140 return initialDelay; 141 } 142 143 /** 144 * Set the amount of time the route controller should wait before to start 145 * the routes after the camel context is started or after the route is 146 * initialized if the route is created after the camel context is started. 147 * 148 * @param initialDelay the initial delay. 149 */ 150 public void setInitialDelay(Duration initialDelay) { 151 this.initialDelay = initialDelay; 152 } 153 154 /** 155 * #see {@link this#setInitialDelay(Duration)} 156 * 157 * @param initialDelay the initial delay amount. 158 */ 159 public void setInitialDelay(long initialDelay, TimeUnit initialDelayUnit) { 160 this.initialDelay = Duration.ofMillis(initialDelayUnit.toMillis(initialDelay)); 161 } 162 163 /** 164 * #see {@link this#setInitialDelay(Duration)} 165 * 166 * @param initialDelay the initial delay in milliseconds. 167 */ 168 public void setInitialDelay(long initialDelay) { 169 this.initialDelay = Duration.ofMillis(initialDelay); 170 } 171 172 /** 173 * Add a filter used to determine the routes to supervise. 174 */ 175 public void addFilter(Filter filter) { 176 this.filters.add(filter); 177 } 178 179 /** 180 * Sets the filters user to determine the routes to supervise. 181 */ 182 public void setFilters(Collection<Filter> filters) { 183 this.filters.clear(); 184 this.filters.addAll(filters); 185 } 186 187 public Collection<Filter> getFilters() { 188 return Collections.unmodifiableList(filters); 189 } 190 191 public Optional<BackOffTimer.Task> getBackOffContext(String id) { 192 return routeManager.getBackOffContext(id); 193 } 194 195 // ********************************* 196 // Lifecycle 197 // ********************************* 198 199 @Override 200 protected void doStart() throws Exception { 201 final CamelContext context = getCamelContext(); 202 203 context.setAutoStartup(false); 204 context.addRoutePolicyFactory(new ManagedRoutePolicyFactory()); 205 context.addStartupListener(this.listener); 206 context.getManagementStrategy().addEventNotifier(this.listener); 207 208 executorService = context.getExecutorServiceManager().newSingleThreadScheduledExecutor(this, "SupervisingRouteController"); 209 timer = new BackOffTimer(executorService); 210 } 211 212 @Override 213 protected void doStop() throws Exception { 214 if (getCamelContext() != null && executorService != null) { 215 getCamelContext().getExecutorServiceManager().shutdown(executorService); 216 executorService = null; 217 timer = null; 218 } 219 } 220 221 @Override 222 protected void doShutdown() throws Exception { 223 if (getCamelContext() != null) { 224 getCamelContext().getManagementStrategy().removeEventNotifier(listener); 225 } 226 } 227 228 // ********************************* 229 // Route management 230 // ********************************* 231 232 @Override 233 public void startRoute(String routeId) throws Exception { 234 final Optional<RouteHolder> route = routes.stream().filter(r -> r.getId().equals(routeId)).findFirst(); 235 236 if (!route.isPresent()) { 237 // This route is unknown to this controller, apply default behaviour 238 // from super class. 239 super.startRoute(routeId); 240 } else { 241 doStartRoute(route.get(), true, r -> super.startRoute(routeId)); 242 } 243 } 244 245 @Override 246 public void stopRoute(String routeId) throws Exception { 247 final Optional<RouteHolder> route = routes.stream().filter(r -> r.getId().equals(routeId)).findFirst(); 248 249 if (!route.isPresent()) { 250 // This route is unknown to this controller, apply default behaviour 251 // from super class. 252 super.stopRoute(routeId); 253 } else { 254 doStopRoute(route.get(), true, r -> super.stopRoute(routeId)); 255 } 256 } 257 258 @Override 259 public void stopRoute(String routeId, long timeout, TimeUnit timeUnit) throws Exception { 260 final Optional<RouteHolder> route = routes.stream().filter(r -> r.getId().equals(routeId)).findFirst(); 261 262 if (!route.isPresent()) { 263 // This route is unknown to this controller, apply default behaviour 264 // from super class. 265 super.stopRoute(routeId, timeout, timeUnit); 266 } else { 267 doStopRoute(route.get(), true, r -> super.stopRoute(r.getId(), timeout, timeUnit)); 268 } 269 } 270 271 @Override 272 public boolean stopRoute(String routeId, long timeout, TimeUnit timeUnit, boolean abortAfterTimeout) throws Exception { 273 final Optional<RouteHolder> route = routes.stream().filter(r -> r.getId().equals(routeId)).findFirst(); 274 275 if (!route.isPresent()) { 276 // This route is unknown to this controller, apply default behaviour 277 // from super class. 278 return super.stopRoute(routeId, timeout, timeUnit, abortAfterTimeout); 279 } else { 280 final AtomicBoolean result = new AtomicBoolean(false); 281 282 doStopRoute(route.get(), true, r -> result.set(super.stopRoute(r.getId(), timeout, timeUnit, abortAfterTimeout))); 283 return result.get(); 284 } 285 } 286 287 @Override 288 public void suspendRoute(String routeId) throws Exception { 289 final Optional<RouteHolder> route = routes.stream().filter(r -> r.getId().equals(routeId)).findFirst(); 290 291 if (!route.isPresent()) { 292 // This route is unknown to this controller, apply default behaviour 293 // from super class. 294 super.suspendRoute(routeId); 295 } else { 296 doStopRoute(route.get(), true, r -> super.suspendRoute(r.getId())); 297 } 298 } 299 300 @Override 301 public void suspendRoute(String routeId, long timeout, TimeUnit timeUnit) throws Exception { 302 final Optional<RouteHolder> route = routes.stream().filter(r -> r.getId().equals(routeId)).findFirst(); 303 304 if (!route.isPresent()) { 305 // This route is unknown to this controller, apply default behaviour 306 // from super class. 307 super.suspendRoute(routeId, timeout, timeUnit); 308 } else { 309 doStopRoute(route.get(), true, r -> super.suspendRoute(r.getId(), timeout, timeUnit)); 310 } 311 } 312 313 @Override 314 public void resumeRoute(String routeId) throws Exception { 315 final Optional<RouteHolder> route = routes.stream().filter(r -> r.getId().equals(routeId)).findFirst(); 316 317 if (!route.isPresent()) { 318 // This route is unknown to this controller, apply default behaviour 319 // from super class. 320 super.resumeRoute(routeId); 321 } else { 322 doStartRoute(route.get(), true, r -> super.startRoute(routeId)); 323 } 324 } 325 326 @Override 327 public Collection<Route> getControlledRoutes() { 328 return routes.stream() 329 .map(RouteHolder::get) 330 .collect(Collectors.toList()); 331 } 332 333 // ********************************* 334 // Helpers 335 // ********************************* 336 337 private void doStopRoute(RouteHolder route, boolean checker, ThrowingConsumer<RouteHolder, Exception> consumer) throws Exception { 338 synchronized (lock) { 339 if (checker) { 340 // remove it from checked routes so the route don't get started 341 // by the routes manager task as a manual operation on the routes 342 // indicates that the route is then managed manually 343 routeManager.release(route); 344 } 345 346 LOGGER.info("Route {} has been requested to stop: stop supervising it", route.getId()); 347 348 // Mark the route as un-managed 349 route.getContext().setRouteController(null); 350 351 consumer.accept(route); 352 } 353 } 354 355 private void doStartRoute(RouteHolder route, boolean checker, ThrowingConsumer<RouteHolder, Exception> consumer) throws Exception { 356 synchronized (lock) { 357 // If a manual start is triggered, then the controller should take 358 // care that the route is started 359 route.getContext().setRouteController(this); 360 361 try { 362 if (checker) { 363 // remove it from checked routes as a manual start may trigger 364 // a new back off task if start fails 365 routeManager.release(route); 366 } 367 368 consumer.accept(route); 369 } catch (Exception e) { 370 371 if (checker) { 372 // if start fails the route is moved to controller supervision 373 // so its get (eventually) restarted 374 routeManager.start(route); 375 } 376 377 throw e; 378 } 379 } 380 } 381 382 private void startRoutes() { 383 if (!isRunAllowed()) { 384 return; 385 } 386 387 final List<String> routeList; 388 389 synchronized (lock) { 390 routeList = routes.stream() 391 .filter(r -> r.getStatus() == ServiceStatus.Stopped) 392 .map(RouteHolder::getId) 393 .collect(Collectors.toList()); 394 } 395 396 for (String route: routeList) { 397 try { 398 startRoute(route); 399 } catch (Exception e) { 400 // ignored, exception handled by startRoute 401 } 402 } 403 404 LOGGER.info("Total managed routes: {} of which {} successfully started and {} re-starting", 405 routes.size(), 406 routes.stream().filter(r -> r.getStatus() == ServiceStatus.Started).count(), 407 routeManager.routes.size() 408 ); 409 } 410 411 private synchronized void stopRoutes() { 412 if (!isRunAllowed()) { 413 return; 414 } 415 416 final List<String> routeList; 417 418 synchronized (lock) { 419 routeList = routes.stream() 420 .filter(r -> r.getStatus() == ServiceStatus.Started) 421 .map(RouteHolder::getId) 422 .collect(Collectors.toList()); 423 } 424 425 for (String route: routeList) { 426 try { 427 stopRoute(route); 428 } catch (Exception e) { 429 // ignored, exception handled by stopRoute 430 } 431 } 432 } 433 434 // ********************************* 435 // RouteChecker 436 // ********************************* 437 438 private class RouteManager { 439 private final Logger logger; 440 private final ConcurrentMap<RouteHolder, BackOffTimer.Task> routes; 441 442 RouteManager() { 443 this.logger = LoggerFactory.getLogger(RouteManager.class); 444 this.routes = new ConcurrentHashMap<>(); 445 } 446 447 void start(RouteHolder route) { 448 route.getContext().setRouteController(SupervisingRouteController.this); 449 450 routes.computeIfAbsent( 451 route, 452 r -> { 453 BackOff backOff = getBackOff(r.getId()); 454 455 logger.info("Start supervising route: {} with back-off: {}", r.getId(), backOff); 456 457 BackOffTimer.Task task = timer.schedule(backOff, context -> { 458 try { 459 logger.info("Try to restart route: {}", r.getId()); 460 461 doStartRoute(r, false, rx -> SupervisingRouteController.super.startRoute(rx.getId())); 462 return false; 463 } catch (Exception e) { 464 return true; 465 } 466 }); 467 468 task.whenComplete((backOffTask, throwable) -> { 469 if (backOffTask == null || backOffTask.getStatus() != BackOffTimer.Task.Status.Active) { 470 // This indicates that the task has been cancelled 471 // or that back-off retry is exhausted thus if the 472 // route is not started it is moved out of the 473 // supervisor control. 474 475 synchronized (lock) { 476 final ServiceStatus status = route.getStatus(); 477 final boolean stopped = status.isStopped() || status.isStopping(); 478 479 if (backOffTask != null && backOffTask.getStatus() == BackOffTimer.Task.Status.Exhausted && stopped) { 480 LOGGER.info("Back-off for route {} is exhausted, no more attempts will be made and stop supervising it", route.getId()); 481 r.getContext().setRouteController(null); 482 } 483 } 484 } 485 486 routes.remove(r); 487 }); 488 489 return task; 490 } 491 ); 492 } 493 494 boolean release(RouteHolder route) { 495 BackOffTimer.Task task = routes.remove(route); 496 if (task != null) { 497 LOGGER.info("Cancel restart task for route {}", route.getId()); 498 task.cancel(); 499 } 500 501 return task != null; 502 } 503 504 void clear() { 505 routes.values().forEach(BackOffTimer.Task::cancel); 506 routes.clear(); 507 } 508 509 public Optional<BackOffTimer.Task> getBackOffContext(String id) { 510 return routes.entrySet().stream() 511 .filter(e -> ObjectHelper.equal(e.getKey().getId(), id)) 512 .findFirst() 513 .map(Map.Entry::getValue); 514 } 515 } 516 517 // ********************************* 518 // 519 // ********************************* 520 521 private class RouteHolder implements HasId, Comparable<RouteHolder> { 522 private final int order; 523 private final Route route; 524 525 RouteHolder(Route route, int order) { 526 this.route = route; 527 this.order = order; 528 } 529 530 @Override 531 public String getId() { 532 return this.route.getId(); 533 } 534 535 public Route get() { 536 return this.route; 537 } 538 539 public RouteContext getContext() { 540 return this.route.getRouteContext(); 541 } 542 543 public RouteDefinition getDefinition() { 544 return this.route.getRouteContext().getRoute(); 545 } 546 547 public ServiceStatus getStatus() { 548 return getContext().getCamelContext().getRouteStatus(getId()); 549 } 550 551 public int getInitializationOrder() { 552 return order; 553 } 554 555 public int getStartupOrder() { 556 Integer order = getDefinition().getStartupOrder(); 557 if (order == null) { 558 order = Integer.MAX_VALUE; 559 } 560 561 return order; 562 } 563 564 @Override 565 public int compareTo(RouteHolder o) { 566 int answer = Integer.compare(getStartupOrder(), o.getStartupOrder()); 567 if (answer == 0) { 568 answer = Integer.compare(getInitializationOrder(), o.getInitializationOrder()); 569 } 570 571 return answer; 572 } 573 574 @Override 575 public boolean equals(Object o) { 576 if (this == o) { 577 return true; 578 } 579 if (o == null || getClass() != o.getClass()) { 580 return false; 581 } 582 583 return this.route.equals(((RouteHolder)o).route); 584 } 585 586 @Override 587 public int hashCode() { 588 return route.hashCode(); 589 } 590 } 591 592 // ********************************* 593 // Policies 594 // ********************************* 595 596 private class ManagedRoutePolicyFactory implements RoutePolicyFactory { 597 private final RoutePolicy policy = new ManagedRoutePolicy(); 598 599 @Override 600 public RoutePolicy createRoutePolicy(CamelContext camelContext, String routeId, RouteDefinition route) { 601 return policy; 602 } 603 } 604 605 private class ManagedRoutePolicy implements RoutePolicy { 606 607 private void startRoute(RouteHolder holder) { 608 try { 609 SupervisingRouteController.this.doStartRoute( 610 holder, 611 true, 612 r -> SupervisingRouteController.super.startRoute(r.getId()) 613 ); 614 } catch (Exception e) { 615 throw new RuntimeCamelException(e); 616 } 617 } 618 619 @Override 620 public void onInit(Route route) { 621 final String autoStartup = route.getRouteContext().getRoute().getAutoStartup(); 622 if (ObjectHelper.equalIgnoreCase("false", autoStartup)) { 623 LOGGER.info("Route {} won't be supervised (reason: has explicit auto-startup flag set to false)", route.getId()); 624 return; 625 } 626 627 for (Filter filter : filters) { 628 FilterResult result = filter.apply(route); 629 630 if (!result.supervised()) { 631 LOGGER.info("Route {} won't be supervised (reason: {})", route.getId(), result.reason()); 632 return; 633 } 634 } 635 636 RouteHolder holder = new RouteHolder(route, routeCount.incrementAndGet()); 637 if (routes.add(holder)) { 638 holder.getContext().setRouteController(SupervisingRouteController.this); 639 holder.getDefinition().setAutoStartup("false"); 640 641 if (contextStarted.get()) { 642 LOGGER.info("Context is already started: attempt to start route {}", route.getId()); 643 644 // Eventually delay the startup of the route a later time 645 if (initialDelay.toMillis() > 0) { 646 LOGGER.debug("Route {} will be started in {}", holder.getId(), initialDelay); 647 executorService.schedule(() -> startRoute(holder), initialDelay.toMillis(), TimeUnit.MILLISECONDS); 648 } else { 649 startRoute(holder); 650 } 651 } else { 652 LOGGER.info("Context is not yet started: defer route {} start", holder.getId()); 653 } 654 } 655 } 656 657 @Override 658 public void onRemove(Route route) { 659 synchronized (lock) { 660 routes.removeIf( 661 r -> ObjectHelper.equal(r.get(), route) || ObjectHelper.equal(r.getId(), route.getId()) 662 ); 663 } 664 } 665 666 @Override 667 public void onStart(Route route) { 668 } 669 670 @Override 671 public void onStop(Route route) { 672 } 673 674 @Override 675 public void onSuspend(Route route) { 676 } 677 678 @Override 679 public void onResume(Route route) { 680 } 681 682 @Override 683 public void onExchangeBegin(Route route, Exchange exchange) { 684 // NO-OP 685 } 686 687 @Override 688 public void onExchangeDone(Route route, Exchange exchange) { 689 // NO-OP 690 } 691 } 692 693 private class CamelContextStartupListener extends EventNotifierSupport implements StartupListener { 694 @Override 695 public void notify(EventObject event) throws Exception { 696 onCamelContextStarted(); 697 } 698 699 @Override 700 public boolean isEnabled(EventObject event) { 701 return event instanceof CamelContextStartedEvent; 702 } 703 704 @Override 705 public void onCamelContextStarted(CamelContext context, boolean alreadyStarted) throws Exception { 706 if (alreadyStarted) { 707 // Invoke it only if the context was already started as this 708 // method is not invoked at last event as documented but after 709 // routes warm-up so this is useful for routes deployed after 710 // the camel context has been started-up. For standard routes 711 // configuration the notification of the camel context started 712 // is provided by EventNotifier. 713 // 714 // We should check why this callback is not invoked at latest 715 // stage, or maybe rename it as it is misleading and provide a 716 // better alternative for intercept camel events. 717 onCamelContextStarted(); 718 } 719 } 720 721 private void onCamelContextStarted() { 722 // Start managing the routes only when the camel context is started 723 // so start/stop of managed routes do not clash with CamelContext 724 // startup 725 if (contextStarted.compareAndSet(false, true)) { 726 727 // Eventually delay the startup of the routes a later time 728 if (initialDelay.toMillis() > 0) { 729 LOGGER.debug("Routes will be started in {}", initialDelay); 730 executorService.schedule(SupervisingRouteController.this::startRoutes, initialDelay.toMillis(), TimeUnit.MILLISECONDS); 731 } else { 732 startRoutes(); 733 } 734 } 735 } 736 } 737 738 // ********************************* 739 // Filter 740 // ********************************* 741 742 @Experimental 743 public static class FilterResult { 744 public static final FilterResult SUPERVISED = new FilterResult(true, null); 745 746 private final boolean controlled; 747 private final String reason; 748 749 public FilterResult(boolean controlled, String reason) { 750 this.controlled = controlled; 751 this.reason = reason; 752 } 753 754 public FilterResult(boolean controlled, String format, Object... args) { 755 this(controlled, String.format(format, args)); 756 } 757 758 public boolean supervised() { 759 return controlled; 760 } 761 762 public String reason() { 763 return reason; 764 } 765 } 766 767 @Experimental 768 public interface Filter extends Function<Route, FilterResult> { 769 } 770}