001/** 002 * Licensed to the Apache Software Foundation (ASF) under one or more 003 * contributor license agreements. See the NOTICE file distributed with 004 * this work for additional information regarding copyright ownership. 005 * The ASF licenses this file to You under the Apache License, Version 2.0 006 * (the "License"); you may not use this file except in compliance with 007 * the License. You may obtain a copy of the License at 008 * 009 * http://www.apache.org/licenses/LICENSE-2.0 010 * 011 * Unless required by applicable law or agreed to in writing, software 012 * distributed under the License is distributed on an "AS IS" BASIS, 013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 014 * See the License for the specific language governing permissions and 015 * limitations under the License. 016 */ 017package org.apache.camel.impl; 018 019import java.time.Duration; 020import java.util.ArrayList; 021import java.util.Collection; 022import java.util.Collections; 023import java.util.EventObject; 024import java.util.HashMap; 025import java.util.List; 026import java.util.Map; 027import java.util.Optional; 028import java.util.Set; 029import java.util.TreeSet; 030import java.util.concurrent.ConcurrentHashMap; 031import java.util.concurrent.ConcurrentMap; 032import java.util.concurrent.ScheduledExecutorService; 033import java.util.concurrent.TimeUnit; 034import java.util.concurrent.atomic.AtomicBoolean; 035import java.util.concurrent.atomic.AtomicInteger; 036import java.util.function.Function; 037import java.util.stream.Collectors; 038 039import org.apache.camel.CamelContext; 040import org.apache.camel.Exchange; 041import org.apache.camel.Experimental; 042import org.apache.camel.Route; 043import org.apache.camel.RuntimeCamelException; 044import org.apache.camel.ServiceStatus; 045import org.apache.camel.StartupListener; 046import org.apache.camel.management.event.CamelContextStartedEvent; 047import org.apache.camel.model.RouteDefinition; 048import org.apache.camel.spi.HasId; 049import org.apache.camel.spi.RouteContext; 050import org.apache.camel.spi.RouteController; 051import org.apache.camel.spi.RoutePolicy; 052import org.apache.camel.spi.RoutePolicyFactory; 053import org.apache.camel.support.EventNotifierSupport; 054import org.apache.camel.util.ObjectHelper; 055import org.apache.camel.util.backoff.BackOff; 056import org.apache.camel.util.backoff.BackOffTimer; 057import org.apache.camel.util.function.ThrowingConsumer; 058import org.slf4j.Logger; 059import org.slf4j.LoggerFactory; 060 061/** 062 * A simple implementation of the {@link RouteController} that delays the startup 063 * of the routes after the camel context startup and retries to start failing routes. 064 * 065 * NOTE: this is experimental/unstable. 066 */ 067@Experimental 068public class SupervisingRouteController extends DefaultRouteController { 069 private static final Logger LOGGER = LoggerFactory.getLogger(SupervisingRouteController.class); 070 private final Object lock; 071 private final AtomicBoolean contextStarted; 072 private final AtomicInteger routeCount; 073 private final List<Filter> filters; 074 private final Set<RouteHolder> routes; 075 private final CamelContextStartupListener listener; 076 private final RouteManager routeManager; 077 private BackOffTimer timer; 078 private ScheduledExecutorService executorService; 079 private BackOff defaultBackOff; 080 private Map<String, BackOff> backOffConfigurations; 081 private Duration initialDelay; 082 083 public SupervisingRouteController() { 084 this.lock = new Object(); 085 this.contextStarted = new AtomicBoolean(false); 086 this.filters = new ArrayList<>(); 087 this.routeCount = new AtomicInteger(0); 088 this.routes = new TreeSet<>(); 089 this.routeManager = new RouteManager(); 090 this.defaultBackOff = BackOff.builder().build(); 091 this.backOffConfigurations = new HashMap<>(); 092 this.initialDelay = Duration.ofMillis(0); 093 094 try { 095 this.listener = new CamelContextStartupListener(); 096 this.listener.start(); 097 } catch (Exception e) { 098 throw new RuntimeException(e); 099 } 100 } 101 102 // ********************************* 103 // Properties 104 // ********************************* 105 106 public BackOff getDefaultBackOff() { 107 return defaultBackOff; 108 } 109 110 /** 111 * Sets the default back-off. 112 */ 113 public void setDefaultBackOff(BackOff defaultBackOff) { 114 this.defaultBackOff = defaultBackOff; 115 } 116 117 public Map<String, BackOff> getBackOffConfigurations() { 118 return backOffConfigurations; 119 } 120 121 /** 122 * Set the back-off for the given IDs. 123 */ 124 public void setBackOffConfigurations(Map<String, BackOff> backOffConfigurations) { 125 this.backOffConfigurations = backOffConfigurations; 126 } 127 128 public BackOff getBackOff(String id) { 129 return backOffConfigurations.getOrDefault(id, defaultBackOff); 130 } 131 132 /** 133 * Sets the back-off to be applied to the given <code>id</code>. 134 */ 135 public void setBackOff(String id, BackOff backOff) { 136 backOffConfigurations.put(id, backOff); 137 } 138 139 public Duration getInitialDelay() { 140 return initialDelay; 141 } 142 143 /** 144 * Set the amount of time the route controller should wait before to start 145 * the routes after the camel context is started or after the route is 146 * initialized if the route is created after the camel context is started. 147 * 148 * @param initialDelay the initial delay. 149 */ 150 public void setInitialDelay(Duration initialDelay) { 151 this.initialDelay = initialDelay; 152 } 153 154 /** 155 * #see {@link this#setInitialDelay(Duration)} 156 * 157 * @param initialDelay the initial delay amount. 158 * @param initialDelay the initial delay time unit. 159 */ 160 public void setInitialDelay(long initialDelay, TimeUnit initialDelayUnit) { 161 this.initialDelay = Duration.ofMillis(initialDelayUnit.toMillis(initialDelay)); 162 } 163 164 /** 165 * #see {@link this#setInitialDelay(Duration)} 166 * 167 * @param initialDelay the initial delay in milliseconds. 168 */ 169 public void setInitialDelay(long initialDelay) { 170 this.initialDelay = Duration.ofMillis(initialDelay); 171 } 172 173 /** 174 * Add a filter used to determine the routes to supervise. 175 */ 176 public void addFilter(Filter filter) { 177 this.filters.add(filter); 178 } 179 180 /** 181 * Sets the filters user to determine the routes to supervise. 182 */ 183 public void setFilters(Collection<Filter> filters) { 184 this.filters.clear(); 185 this.filters.addAll(filters); 186 } 187 188 public Collection<Filter> getFilters() { 189 return Collections.unmodifiableList(filters); 190 } 191 192 public Optional<BackOffTimer.Task> getBackOffContext(String id) { 193 return routeManager.getBackOffContext(id); 194 } 195 196 // ********************************* 197 // Lifecycle 198 // ********************************* 199 200 @Override 201 protected void doStart() throws Exception { 202 final CamelContext context = getCamelContext(); 203 204 context.setAutoStartup(false); 205 context.addRoutePolicyFactory(new ManagedRoutePolicyFactory()); 206 context.addStartupListener(this.listener); 207 context.getManagementStrategy().addEventNotifier(this.listener); 208 209 executorService = context.getExecutorServiceManager().newSingleThreadScheduledExecutor(this, "SupervisingRouteController"); 210 timer = new BackOffTimer(executorService); 211 } 212 213 @Override 214 protected void doStop() throws Exception { 215 if (getCamelContext() != null && executorService != null) { 216 getCamelContext().getExecutorServiceManager().shutdown(executorService); 217 executorService = null; 218 timer = null; 219 } 220 } 221 222 @Override 223 protected void doShutdown() throws Exception { 224 if (getCamelContext() != null) { 225 getCamelContext().getManagementStrategy().removeEventNotifier(listener); 226 } 227 } 228 229 // ********************************* 230 // Route management 231 // ********************************* 232 233 @Override 234 public void startRoute(String routeId) throws Exception { 235 final Optional<RouteHolder> route = routes.stream().filter(r -> r.getId().equals(routeId)).findFirst(); 236 237 if (!route.isPresent()) { 238 // This route is unknown to this controller, apply default behaviour 239 // from super class. 240 super.startRoute(routeId); 241 } else { 242 doStartRoute(route.get(), true, r -> super.startRoute(routeId)); 243 } 244 } 245 246 @Override 247 public void stopRoute(String routeId) throws Exception { 248 final Optional<RouteHolder> route = routes.stream().filter(r -> r.getId().equals(routeId)).findFirst(); 249 250 if (!route.isPresent()) { 251 // This route is unknown to this controller, apply default behaviour 252 // from super class. 253 super.stopRoute(routeId); 254 } else { 255 doStopRoute(route.get(), true, r -> super.stopRoute(routeId)); 256 } 257 } 258 259 @Override 260 public void stopRoute(String routeId, long timeout, TimeUnit timeUnit) throws Exception { 261 final Optional<RouteHolder> route = routes.stream().filter(r -> r.getId().equals(routeId)).findFirst(); 262 263 if (!route.isPresent()) { 264 // This route is unknown to this controller, apply default behaviour 265 // from super class. 266 super.stopRoute(routeId, timeout, timeUnit); 267 } else { 268 doStopRoute(route.get(), true, r -> super.stopRoute(r.getId(), timeout, timeUnit)); 269 } 270 } 271 272 @Override 273 public boolean stopRoute(String routeId, long timeout, TimeUnit timeUnit, boolean abortAfterTimeout) throws Exception { 274 final Optional<RouteHolder> route = routes.stream().filter(r -> r.getId().equals(routeId)).findFirst(); 275 276 if (!route.isPresent()) { 277 // This route is unknown to this controller, apply default behaviour 278 // from super class. 279 return super.stopRoute(routeId, timeout, timeUnit, abortAfterTimeout); 280 } else { 281 final AtomicBoolean result = new AtomicBoolean(false); 282 283 doStopRoute(route.get(), true, r -> result.set(super.stopRoute(r.getId(), timeout, timeUnit, abortAfterTimeout))); 284 return result.get(); 285 } 286 } 287 288 @Override 289 public void suspendRoute(String routeId) throws Exception { 290 final Optional<RouteHolder> route = routes.stream().filter(r -> r.getId().equals(routeId)).findFirst(); 291 292 if (!route.isPresent()) { 293 // This route is unknown to this controller, apply default behaviour 294 // from super class. 295 super.suspendRoute(routeId); 296 } else { 297 doStopRoute(route.get(), true, r -> super.suspendRoute(r.getId())); 298 } 299 } 300 301 @Override 302 public void suspendRoute(String routeId, long timeout, TimeUnit timeUnit) throws Exception { 303 final Optional<RouteHolder> route = routes.stream().filter(r -> r.getId().equals(routeId)).findFirst(); 304 305 if (!route.isPresent()) { 306 // This route is unknown to this controller, apply default behaviour 307 // from super class. 308 super.suspendRoute(routeId, timeout, timeUnit); 309 } else { 310 doStopRoute(route.get(), true, r -> super.suspendRoute(r.getId(), timeout, timeUnit)); 311 } 312 } 313 314 @Override 315 public void resumeRoute(String routeId) throws Exception { 316 final Optional<RouteHolder> route = routes.stream().filter(r -> r.getId().equals(routeId)).findFirst(); 317 318 if (!route.isPresent()) { 319 // This route is unknown to this controller, apply default behaviour 320 // from super class. 321 super.resumeRoute(routeId); 322 } else { 323 doStartRoute(route.get(), true, r -> super.startRoute(routeId)); 324 } 325 } 326 327 @Override 328 public Collection<Route> getControlledRoutes() { 329 return routes.stream() 330 .map(RouteHolder::get) 331 .collect(Collectors.toList()); 332 } 333 334 // ********************************* 335 // Helpers 336 // ********************************* 337 338 private void doStopRoute(RouteHolder route, boolean checker, ThrowingConsumer<RouteHolder, Exception> consumer) throws Exception { 339 synchronized (lock) { 340 if (checker) { 341 // remove it from checked routes so the route don't get started 342 // by the routes manager task as a manual operation on the routes 343 // indicates that the route is then managed manually 344 routeManager.release(route); 345 } 346 347 LOGGER.info("Route {} has been requested to stop: stop supervising it", route.getId()); 348 349 // Mark the route as un-managed 350 route.getContext().setRouteController(null); 351 352 consumer.accept(route); 353 } 354 } 355 356 private void doStartRoute(RouteHolder route, boolean checker, ThrowingConsumer<RouteHolder, Exception> consumer) throws Exception { 357 synchronized (lock) { 358 // If a manual start is triggered, then the controller should take 359 // care that the route is started 360 route.getContext().setRouteController(this); 361 362 try { 363 if (checker) { 364 // remove it from checked routes as a manual start may trigger 365 // a new back off task if start fails 366 routeManager.release(route); 367 } 368 369 consumer.accept(route); 370 } catch (Exception e) { 371 372 if (checker) { 373 // if start fails the route is moved to controller supervision 374 // so its get (eventually) restarted 375 routeManager.start(route); 376 } 377 378 throw e; 379 } 380 } 381 } 382 383 private void startRoutes() { 384 if (!isRunAllowed()) { 385 return; 386 } 387 388 final List<String> routeList; 389 390 synchronized (lock) { 391 routeList = routes.stream() 392 .filter(r -> r.getStatus() == ServiceStatus.Stopped) 393 .map(RouteHolder::getId) 394 .collect(Collectors.toList()); 395 } 396 397 for (String route: routeList) { 398 try { 399 startRoute(route); 400 } catch (Exception e) { 401 // ignored, exception handled by startRoute 402 } 403 } 404 405 LOGGER.info("Total managed routes: {} of which {} successfully started and {} re-starting", 406 routes.size(), 407 routes.stream().filter(r -> r.getStatus() == ServiceStatus.Started).count(), 408 routeManager.routes.size() 409 ); 410 } 411 412 private synchronized void stopRoutes() { 413 if (!isRunAllowed()) { 414 return; 415 } 416 417 final List<String> routeList; 418 419 synchronized (lock) { 420 routeList = routes.stream() 421 .filter(r -> r.getStatus() == ServiceStatus.Started) 422 .map(RouteHolder::getId) 423 .collect(Collectors.toList()); 424 } 425 426 for (String route: routeList) { 427 try { 428 stopRoute(route); 429 } catch (Exception e) { 430 // ignored, exception handled by stopRoute 431 } 432 } 433 } 434 435 // ********************************* 436 // RouteChecker 437 // ********************************* 438 439 private class RouteManager { 440 private final Logger logger; 441 private final ConcurrentMap<RouteHolder, BackOffTimer.Task> routes; 442 443 RouteManager() { 444 this.logger = LoggerFactory.getLogger(RouteManager.class); 445 this.routes = new ConcurrentHashMap<>(); 446 } 447 448 void start(RouteHolder route) { 449 route.getContext().setRouteController(SupervisingRouteController.this); 450 451 routes.computeIfAbsent( 452 route, 453 r -> { 454 BackOff backOff = getBackOff(r.getId()); 455 456 logger.info("Start supervising route: {} with back-off: {}", r.getId(), backOff); 457 458 BackOffTimer.Task task = timer.schedule(backOff, context -> { 459 try { 460 logger.info("Try to restart route: {}", r.getId()); 461 462 doStartRoute(r, false, rx -> SupervisingRouteController.super.startRoute(rx.getId())); 463 return false; 464 } catch (Exception e) { 465 return true; 466 } 467 }); 468 469 task.whenComplete((backOffTask, throwable) -> { 470 if (backOffTask == null || backOffTask.getStatus() != BackOffTimer.Task.Status.Active) { 471 // This indicates that the task has been cancelled 472 // or that back-off retry is exhausted thus if the 473 // route is not started it is moved out of the 474 // supervisor control. 475 476 synchronized (lock) { 477 final ServiceStatus status = route.getStatus(); 478 final boolean stopped = status.isStopped() || status.isStopping(); 479 480 if (backOffTask != null && backOffTask.getStatus() == BackOffTimer.Task.Status.Exhausted && stopped) { 481 LOGGER.info("Back-off for route {} is exhausted, no more attempts will be made and stop supervising it", route.getId()); 482 r.getContext().setRouteController(null); 483 } 484 } 485 } 486 487 routes.remove(r); 488 }); 489 490 return task; 491 } 492 ); 493 } 494 495 boolean release(RouteHolder route) { 496 BackOffTimer.Task task = routes.remove(route); 497 if (task != null) { 498 LOGGER.info("Cancel restart task for route {}", route.getId()); 499 task.cancel(); 500 } 501 502 return task != null; 503 } 504 505 void clear() { 506 routes.values().forEach(BackOffTimer.Task::cancel); 507 routes.clear(); 508 } 509 510 public Optional<BackOffTimer.Task> getBackOffContext(String id) { 511 return routes.entrySet().stream() 512 .filter(e -> ObjectHelper.equal(e.getKey().getId(), id)) 513 .findFirst() 514 .map(Map.Entry::getValue); 515 } 516 } 517 518 // ********************************* 519 // 520 // ********************************* 521 522 private class RouteHolder implements HasId, Comparable<RouteHolder> { 523 private final int order; 524 private final Route route; 525 526 RouteHolder(Route route, int order) { 527 this.route = route; 528 this.order = order; 529 } 530 531 @Override 532 public String getId() { 533 return this.route.getId(); 534 } 535 536 public Route get() { 537 return this.route; 538 } 539 540 public RouteContext getContext() { 541 return this.route.getRouteContext(); 542 } 543 544 public RouteDefinition getDefinition() { 545 return this.route.getRouteContext().getRoute(); 546 } 547 548 public ServiceStatus getStatus() { 549 return getContext().getCamelContext().getRouteStatus(getId()); 550 } 551 552 public int getInitializationOrder() { 553 return order; 554 } 555 556 public int getStartupOrder() { 557 Integer order = getDefinition().getStartupOrder(); 558 if (order == null) { 559 order = Integer.MAX_VALUE; 560 } 561 562 return order; 563 } 564 565 @Override 566 public int compareTo(RouteHolder o) { 567 int answer = Integer.compare(getStartupOrder(), o.getStartupOrder()); 568 if (answer == 0) { 569 answer = Integer.compare(getInitializationOrder(), o.getInitializationOrder()); 570 } 571 572 return answer; 573 } 574 575 @Override 576 public boolean equals(Object o) { 577 if (this == o) { 578 return true; 579 } 580 if (o == null || getClass() != o.getClass()) { 581 return false; 582 } 583 584 return this.route.equals(((RouteHolder)o).route); 585 } 586 587 @Override 588 public int hashCode() { 589 return route.hashCode(); 590 } 591 } 592 593 // ********************************* 594 // Policies 595 // ********************************* 596 597 private class ManagedRoutePolicyFactory implements RoutePolicyFactory { 598 private final RoutePolicy policy = new ManagedRoutePolicy(); 599 600 @Override 601 public RoutePolicy createRoutePolicy(CamelContext camelContext, String routeId, RouteDefinition route) { 602 return policy; 603 } 604 } 605 606 private class ManagedRoutePolicy implements RoutePolicy { 607 608 private void startRoute(RouteHolder holder) { 609 try { 610 SupervisingRouteController.this.doStartRoute( 611 holder, 612 true, 613 r -> SupervisingRouteController.super.startRoute(r.getId()) 614 ); 615 } catch (Exception e) { 616 throw new RuntimeCamelException(e); 617 } 618 } 619 620 @Override 621 public void onInit(Route route) { 622 final String autoStartup = route.getRouteContext().getRoute().getAutoStartup(); 623 if (ObjectHelper.equalIgnoreCase("false", autoStartup)) { 624 LOGGER.info("Route {} won't be supervised (reason: has explicit auto-startup flag set to false)", route.getId()); 625 return; 626 } 627 628 for (Filter filter : filters) { 629 FilterResult result = filter.apply(route); 630 631 if (!result.supervised()) { 632 LOGGER.info("Route {} won't be supervised (reason: {})", route.getId(), result.reason()); 633 return; 634 } 635 } 636 637 RouteHolder holder = new RouteHolder(route, routeCount.incrementAndGet()); 638 if (routes.add(holder)) { 639 holder.getContext().setRouteController(SupervisingRouteController.this); 640 holder.getDefinition().setAutoStartup("false"); 641 642 if (contextStarted.get()) { 643 LOGGER.info("Context is already started: attempt to start route {}", route.getId()); 644 645 // Eventually delay the startup of the route a later time 646 if (initialDelay.toMillis() > 0) { 647 LOGGER.debug("Route {} will be started in {}", holder.getId(), initialDelay); 648 executorService.schedule(() -> startRoute(holder), initialDelay.toMillis(), TimeUnit.MILLISECONDS); 649 } else { 650 startRoute(holder); 651 } 652 } else { 653 LOGGER.info("Context is not yet started: defer route {} start", holder.getId()); 654 } 655 } 656 } 657 658 @Override 659 public void onRemove(Route route) { 660 synchronized (lock) { 661 routes.removeIf( 662 r -> ObjectHelper.equal(r.get(), route) || ObjectHelper.equal(r.getId(), route.getId()) 663 ); 664 } 665 } 666 667 @Override 668 public void onStart(Route route) { 669 } 670 671 @Override 672 public void onStop(Route route) { 673 } 674 675 @Override 676 public void onSuspend(Route route) { 677 } 678 679 @Override 680 public void onResume(Route route) { 681 } 682 683 @Override 684 public void onExchangeBegin(Route route, Exchange exchange) { 685 // NO-OP 686 } 687 688 @Override 689 public void onExchangeDone(Route route, Exchange exchange) { 690 // NO-OP 691 } 692 } 693 694 private class CamelContextStartupListener extends EventNotifierSupport implements StartupListener { 695 @Override 696 public void notify(EventObject event) throws Exception { 697 onCamelContextStarted(); 698 } 699 700 @Override 701 public boolean isEnabled(EventObject event) { 702 return event instanceof CamelContextStartedEvent; 703 } 704 705 @Override 706 public void onCamelContextStarted(CamelContext context, boolean alreadyStarted) throws Exception { 707 if (alreadyStarted) { 708 // Invoke it only if the context was already started as this 709 // method is not invoked at last event as documented but after 710 // routes warm-up so this is useful for routes deployed after 711 // the camel context has been started-up. For standard routes 712 // configuration the notification of the camel context started 713 // is provided by EventNotifier. 714 // 715 // We should check why this callback is not invoked at latest 716 // stage, or maybe rename it as it is misleading and provide a 717 // better alternative for intercept camel events. 718 onCamelContextStarted(); 719 } 720 } 721 722 private void onCamelContextStarted() { 723 // Start managing the routes only when the camel context is started 724 // so start/stop of managed routes do not clash with CamelContext 725 // startup 726 if (contextStarted.compareAndSet(false, true)) { 727 728 // Eventually delay the startup of the routes a later time 729 if (initialDelay.toMillis() > 0) { 730 LOGGER.debug("Routes will be started in {}", initialDelay); 731 executorService.schedule(SupervisingRouteController.this::startRoutes, initialDelay.toMillis(), TimeUnit.MILLISECONDS); 732 } else { 733 startRoutes(); 734 } 735 } 736 } 737 } 738 739 // ********************************* 740 // Filter 741 // ********************************* 742 743 @Experimental 744 public static class FilterResult { 745 public static final FilterResult SUPERVISED = new FilterResult(true, null); 746 747 private final boolean controlled; 748 private final String reason; 749 750 public FilterResult(boolean controlled, String reason) { 751 this.controlled = controlled; 752 this.reason = reason; 753 } 754 755 public FilterResult(boolean controlled, String format, Object... args) { 756 this(controlled, String.format(format, args)); 757 } 758 759 public boolean supervised() { 760 return controlled; 761 } 762 763 public String reason() { 764 return reason; 765 } 766 } 767 768 @Experimental 769 public interface Filter extends Function<Route, FilterResult> { 770 } 771}