001/** 002 * Licensed to the Apache Software Foundation (ASF) under one or more 003 * contributor license agreements. See the NOTICE file distributed with 004 * this work for additional information regarding copyright ownership. 005 * The ASF licenses this file to You under the Apache License, Version 2.0 006 * (the "License"); you may not use this file except in compliance with 007 * the License. You may obtain a copy of the License at 008 * 009 * http://www.apache.org/licenses/LICENSE-2.0 010 * 011 * Unless required by applicable law or agreed to in writing, software 012 * distributed under the License is distributed on an "AS IS" BASIS, 013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 014 * See the License for the specific language governing permissions and 015 * limitations under the License. 016 */ 017package org.apache.camel.component.file.strategy; 018 019import java.io.File; 020import java.util.concurrent.ScheduledExecutorService; 021import java.util.concurrent.TimeUnit; 022 023import org.apache.camel.CamelContext; 024import org.apache.camel.CamelContextAware; 025import org.apache.camel.Exchange; 026import org.apache.camel.LoggingLevel; 027import org.apache.camel.component.file.GenericFile; 028import org.apache.camel.component.file.GenericFileEndpoint; 029import org.apache.camel.component.file.GenericFileExclusiveReadLockStrategy; 030import org.apache.camel.component.file.GenericFileOperations; 031import org.apache.camel.spi.IdempotentRepository; 032import org.apache.camel.support.ServiceSupport; 033import org.apache.camel.util.CamelLogger; 034import org.apache.camel.util.ObjectHelper; 035import org.slf4j.Logger; 036import org.slf4j.LoggerFactory; 037 038/** 039 * A file read lock that uses an {@link IdempotentRepository} and {@link FileChangedExclusiveReadLockStrategy changed} as the lock strategy. 040 * This allows to plugin and use existing idempotent repositories that for example supports clustering. 041 * The other read lock strategies that are using marker files or file locks, are not guaranteed to work in clustered setup with various platform and file systems. 042 */ 043public class FileIdempotentChangedRepositoryReadLockStrategy extends ServiceSupport implements GenericFileExclusiveReadLockStrategy<File>, CamelContextAware { 044 045 private static final transient Logger LOG = LoggerFactory.getLogger(FileIdempotentChangedRepositoryReadLockStrategy.class); 046 047 private final FileChangedExclusiveReadLockStrategy changed; 048 private GenericFileEndpoint<File> endpoint; 049 private LoggingLevel readLockLoggingLevel = LoggingLevel.DEBUG; 050 private CamelContext camelContext; 051 private IdempotentRepository<String> idempotentRepository; 052 private boolean removeOnRollback = true; 053 private boolean removeOnCommit; 054 private int readLockIdempotentReleaseDelay; 055 private boolean readLockIdempotentReleaseAsync; 056 private int readLockIdempotentReleaseAsyncPoolSize; 057 private ScheduledExecutorService readLockIdempotentReleaseExecutorService; 058 private boolean shutdownExecutorService; 059 060 public FileIdempotentChangedRepositoryReadLockStrategy() { 061 this.changed = new FileChangedExclusiveReadLockStrategy(); 062 // no need to use marker file as idempotent ensures exclusive read-lock 063 this.changed.setMarkerFiler(false); 064 this.changed.setDeleteOrphanLockFiles(false); 065 } 066 067 @Override 068 public void prepareOnStartup(GenericFileOperations<File> operations, GenericFileEndpoint<File> endpoint) throws Exception { 069 this.endpoint = endpoint; 070 LOG.info("Using FileIdempotentRepositoryReadLockStrategy: {} on endpoint: {}", idempotentRepository, endpoint); 071 072 changed.prepareOnStartup(operations, endpoint); 073 } 074 075 @Override 076 public boolean acquireExclusiveReadLock(GenericFileOperations<File> operations, GenericFile<File> file, Exchange exchange) throws Exception { 077 // in clustered mode then another node may have processed the file so we must check here again if the file exists 078 File path = file.getFile(); 079 if (!path.exists()) { 080 return false; 081 } 082 083 // check if we can begin on this file 084 String key = asKey(file); 085 boolean answer = idempotentRepository.add(key); 086 if (!answer) { 087 // another node is processing the file so skip 088 CamelLogger.log(LOG, readLockLoggingLevel, "Cannot acquire read lock. Will skip the file: " + file); 089 } 090 091 if (answer) { 092 // if we acquired during idempotent then check changed also 093 answer = changed.acquireExclusiveReadLock(operations, file, exchange); 094 if (!answer) { 095 // remove from idempontent as we did not acquire it from changed 096 idempotentRepository.remove(key); 097 } 098 } 099 return answer; 100 } 101 102 @Override 103 public void releaseExclusiveReadLockOnAbort(GenericFileOperations<File> operations, GenericFile<File> file, Exchange exchange) throws Exception { 104 changed.releaseExclusiveReadLockOnAbort(operations, file, exchange); 105 } 106 107 @Override 108 public void releaseExclusiveReadLockOnRollback(GenericFileOperations<File> operations, GenericFile<File> file, Exchange exchange) throws Exception { 109 String key = asKey(file); 110 Runnable r = () -> { 111 if (removeOnRollback) { 112 idempotentRepository.remove(key); 113 } else { 114 // okay we should not remove then confirm it instead 115 idempotentRepository.confirm(key); 116 } 117 118 try { 119 changed.releaseExclusiveReadLockOnRollback(operations, file, exchange); 120 } catch (Exception e) { 121 LOG.warn("Error during releasing exclusive readlock on rollback. This exception is ignored.", e); 122 } 123 }; 124 125 if (readLockIdempotentReleaseDelay > 0 && readLockIdempotentReleaseExecutorService != null) { 126 LOG.debug("Scheduling readlock release task to run asynchronous delayed after {} millis", readLockIdempotentReleaseDelay); 127 readLockIdempotentReleaseExecutorService.schedule(r, readLockIdempotentReleaseDelay, TimeUnit.MILLISECONDS); 128 } else if (readLockIdempotentReleaseDelay > 0) { 129 LOG.debug("Delaying readlock release task {} millis", readLockIdempotentReleaseDelay); 130 Thread.sleep(readLockIdempotentReleaseDelay); 131 r.run(); 132 } else { 133 r.run(); 134 } 135 } 136 137 @Override 138 public void releaseExclusiveReadLockOnCommit(GenericFileOperations<File> operations, GenericFile<File> file, Exchange exchange) throws Exception { 139 String key = asKey(file); 140 Runnable r = () -> { 141 if (removeOnCommit) { 142 idempotentRepository.remove(key); 143 } else { 144 // confirm on commit 145 idempotentRepository.confirm(key); 146 } 147 148 try { 149 changed.releaseExclusiveReadLockOnCommit(operations, file, exchange); 150 } catch (Exception e) { 151 LOG.warn("Error during releasing exclusive readlock on rollback. This exception is ignored.", e); 152 } 153 }; 154 155 if (readLockIdempotentReleaseDelay > 0 && readLockIdempotentReleaseExecutorService != null) { 156 LOG.debug("Scheduling readlock release task to run asynchronous delayed after {} millis", readLockIdempotentReleaseDelay); 157 readLockIdempotentReleaseExecutorService.schedule(r, readLockIdempotentReleaseDelay, TimeUnit.MILLISECONDS); 158 } else if (readLockIdempotentReleaseDelay > 0) { 159 LOG.debug("Delaying readlock release task {} millis", readLockIdempotentReleaseDelay); 160 Thread.sleep(readLockIdempotentReleaseDelay); 161 r.run(); 162 } else { 163 r.run(); 164 } 165 } 166 167 public void setTimeout(long timeout) { 168 changed.setTimeout(timeout); 169 } 170 171 public void setCheckInterval(long checkInterval) { 172 changed.setCheckInterval(checkInterval); 173 } 174 175 public void setReadLockLoggingLevel(LoggingLevel readLockLoggingLevel) { 176 this.readLockLoggingLevel = readLockLoggingLevel; 177 changed.setReadLockLoggingLevel(readLockLoggingLevel); 178 } 179 180 public void setMarkerFiler(boolean markerFile) { 181 // we do not use marker files 182 } 183 184 public void setDeleteOrphanLockFiles(boolean deleteOrphanLockFiles) { 185 // we do not use marker files 186 } 187 188 public void setMinLength(long minLength) { 189 changed.setMinLength(minLength); 190 } 191 192 public void setMinAge(long minAge) { 193 changed.setMinAge(minAge); 194 } 195 196 public CamelContext getCamelContext() { 197 return camelContext; 198 } 199 200 public void setCamelContext(CamelContext camelContext) { 201 this.camelContext = camelContext; 202 } 203 204 /** 205 * The idempotent repository to use as the store for the read locks. 206 */ 207 public IdempotentRepository<String> getIdempotentRepository() { 208 return idempotentRepository; 209 } 210 211 /** 212 * The idempotent repository to use as the store for the read locks. 213 */ 214 public void setIdempotentRepository(IdempotentRepository<String> idempotentRepository) { 215 this.idempotentRepository = idempotentRepository; 216 } 217 218 /** 219 * Whether to remove the file from the idempotent repository when doing a rollback. 220 * <p/> 221 * By default this is true. 222 */ 223 public boolean isRemoveOnRollback() { 224 return removeOnRollback; 225 } 226 227 /** 228 * Whether to remove the file from the idempotent repository when doing a rollback. 229 * <p/> 230 * By default this is true. 231 */ 232 public void setRemoveOnRollback(boolean removeOnRollback) { 233 this.removeOnRollback = removeOnRollback; 234 } 235 236 /** 237 * Whether to remove the file from the idempotent repository when doing a commit. 238 * <p/> 239 * By default this is false. 240 */ 241 public boolean isRemoveOnCommit() { 242 return removeOnCommit; 243 } 244 245 /** 246 * Whether to remove the file from the idempotent repository when doing a commit. 247 * <p/> 248 * By default this is false. 249 */ 250 public void setRemoveOnCommit(boolean removeOnCommit) { 251 this.removeOnCommit = removeOnCommit; 252 } 253 254 /** 255 * Whether to delay the release task for a period of millis. 256 */ 257 public void setReadLockIdempotentReleaseDelay(int readLockIdempotentReleaseDelay) { 258 this.readLockIdempotentReleaseDelay = readLockIdempotentReleaseDelay; 259 } 260 261 public boolean isReadLockIdempotentReleaseAsync() { 262 return readLockIdempotentReleaseAsync; 263 } 264 265 /** 266 * Whether the delayed release task should be synchronous or asynchronous. 267 */ 268 public void setReadLockIdempotentReleaseAsync(boolean readLockIdempotentReleaseAsync) { 269 this.readLockIdempotentReleaseAsync = readLockIdempotentReleaseAsync; 270 } 271 272 public int getReadLockIdempotentReleaseAsyncPoolSize() { 273 return readLockIdempotentReleaseAsyncPoolSize; 274 } 275 276 /** 277 * The number of threads in the scheduled thread pool when using asynchronous release tasks. 278 */ 279 public void setReadLockIdempotentReleaseAsyncPoolSize(int readLockIdempotentReleaseAsyncPoolSize) { 280 this.readLockIdempotentReleaseAsyncPoolSize = readLockIdempotentReleaseAsyncPoolSize; 281 } 282 283 public ScheduledExecutorService getReadLockIdempotentReleaseExecutorService() { 284 return readLockIdempotentReleaseExecutorService; 285 } 286 287 /** 288 * To use a custom and shared thread pool for asynchronous release tasks. 289 */ 290 public void setReadLockIdempotentReleaseExecutorService(ScheduledExecutorService readLockIdempotentReleaseExecutorService) { 291 this.readLockIdempotentReleaseExecutorService = readLockIdempotentReleaseExecutorService; 292 } 293 294 protected String asKey(GenericFile<File> file) { 295 // use absolute file path as default key, but evaluate if an expression key was configured 296 String key = file.getAbsoluteFilePath(); 297 if (endpoint.getIdempotentKey() != null) { 298 Exchange dummy = endpoint.createExchange(file); 299 key = endpoint.getIdempotentKey().evaluate(dummy, String.class); 300 } 301 return key; 302 } 303 304 @Override 305 protected void doStart() throws Exception { 306 ObjectHelper.notNull(camelContext, "camelContext", this); 307 ObjectHelper.notNull(idempotentRepository, "idempotentRepository", this); 308 309 if (readLockIdempotentReleaseAsync && readLockIdempotentReleaseExecutorService == null) { 310 readLockIdempotentReleaseExecutorService = camelContext.getExecutorServiceManager().newScheduledThreadPool( 311 this, "ReadLockChangedIdempotentReleaseTask", readLockIdempotentReleaseAsyncPoolSize); 312 shutdownExecutorService = true; 313 } 314 } 315 316 @Override 317 protected void doStop() throws Exception { 318 if (shutdownExecutorService && readLockIdempotentReleaseExecutorService != null) { 319 camelContext.getExecutorServiceManager().shutdownGraceful(readLockIdempotentReleaseExecutorService, 30000); 320 readLockIdempotentReleaseExecutorService = null; 321 } 322 } 323 324}