001/**
002 * Licensed to the Apache Software Foundation (ASF) under one or more
003 * contributor license agreements.  See the NOTICE file distributed with
004 * this work for additional information regarding copyright ownership.
005 * The ASF licenses this file to You under the Apache License, Version 2.0
006 * (the "License"); you may not use this file except in compliance with
007 * the License.  You may obtain a copy of the License at
008 *
009 *      http://www.apache.org/licenses/LICENSE-2.0
010 *
011 * Unless required by applicable law or agreed to in writing, software
012 * distributed under the License is distributed on an "AS IS" BASIS,
013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
014 * See the License for the specific language governing permissions and
015 * limitations under the License.
016 */
017package org.apache.camel.component.file.strategy;
018
019import java.io.File;
020import java.util.concurrent.ScheduledExecutorService;
021import java.util.concurrent.TimeUnit;
022
023import org.apache.camel.CamelContext;
024import org.apache.camel.CamelContextAware;
025import org.apache.camel.Exchange;
026import org.apache.camel.LoggingLevel;
027import org.apache.camel.component.file.GenericFile;
028import org.apache.camel.component.file.GenericFileEndpoint;
029import org.apache.camel.component.file.GenericFileExclusiveReadLockStrategy;
030import org.apache.camel.component.file.GenericFileOperations;
031import org.apache.camel.spi.IdempotentRepository;
032import org.apache.camel.support.ServiceSupport;
033import org.apache.camel.util.CamelLogger;
034import org.apache.camel.util.ObjectHelper;
035import org.slf4j.Logger;
036import org.slf4j.LoggerFactory;
037
038/**
039 * A file read lock that uses an {@link IdempotentRepository} and {@link FileChangedExclusiveReadLockStrategy changed} as the lock strategy.
040 * This allows to plugin and use existing idempotent repositories that for example supports clustering.
041 * The other read lock strategies that are using marker files or file locks, are not guaranteed to work in clustered setup with various platform and file systems.
042 */
043public class FileIdempotentChangedRepositoryReadLockStrategy extends ServiceSupport implements GenericFileExclusiveReadLockStrategy<File>, CamelContextAware {
044
045    private static final transient Logger LOG = LoggerFactory.getLogger(FileIdempotentChangedRepositoryReadLockStrategy.class);
046
047    private final FileChangedExclusiveReadLockStrategy changed;
048    private GenericFileEndpoint<File> endpoint;
049    private LoggingLevel readLockLoggingLevel = LoggingLevel.DEBUG;
050    private CamelContext camelContext;
051    private IdempotentRepository<String> idempotentRepository;
052    private boolean removeOnRollback = true;
053    private boolean removeOnCommit;
054    private int readLockIdempotentReleaseDelay;
055    private boolean readLockIdempotentReleaseAsync;
056    private int readLockIdempotentReleaseAsyncPoolSize;
057    private ScheduledExecutorService readLockIdempotentReleaseExecutorService;
058    private boolean shutdownExecutorService;
059
060    public FileIdempotentChangedRepositoryReadLockStrategy() {
061        this.changed = new FileChangedExclusiveReadLockStrategy();
062        // no need to use marker file as idempotent ensures exclusive read-lock
063        this.changed.setMarkerFiler(false);
064        this.changed.setDeleteOrphanLockFiles(false);
065    }
066
067    @Override
068    public void prepareOnStartup(GenericFileOperations<File> operations, GenericFileEndpoint<File> endpoint) throws Exception {
069        this.endpoint = endpoint;
070        LOG.info("Using FileIdempotentRepositoryReadLockStrategy: {} on endpoint: {}", idempotentRepository, endpoint);
071
072        changed.prepareOnStartup(operations, endpoint);
073    }
074
075    @Override
076    public boolean acquireExclusiveReadLock(GenericFileOperations<File> operations, GenericFile<File> file, Exchange exchange) throws Exception {
077        // in clustered mode then another node may have processed the file so we must check here again if the file exists
078        File path = file.getFile();
079        if (!path.exists()) {
080            return false;
081        }
082
083        // check if we can begin on this file
084        String key = asKey(file);
085        boolean answer = idempotentRepository.add(key);
086        if (!answer) {
087            // another node is processing the file so skip
088            CamelLogger.log(LOG, readLockLoggingLevel, "Cannot acquire read lock. Will skip the file: " + file);
089        }
090
091        if (answer) {
092            // if we acquired during idempotent then check changed also
093            answer = changed.acquireExclusiveReadLock(operations, file, exchange);
094            if (!answer) {
095                // remove from idempontent as we did not acquire it from changed
096                idempotentRepository.remove(key);
097            }
098        }
099        return answer;
100    }
101
102    @Override
103    public void releaseExclusiveReadLockOnAbort(GenericFileOperations<File> operations, GenericFile<File> file, Exchange exchange) throws Exception {
104        changed.releaseExclusiveReadLockOnAbort(operations, file, exchange);
105    }
106
107    @Override
108    public void releaseExclusiveReadLockOnRollback(GenericFileOperations<File> operations, GenericFile<File> file, Exchange exchange) throws Exception {
109        String key = asKey(file);
110        Runnable r = () -> {
111            if (removeOnRollback) {
112                idempotentRepository.remove(key);
113            } else {
114                // okay we should not remove then confirm it instead
115                idempotentRepository.confirm(key);
116            }
117
118            try {
119                changed.releaseExclusiveReadLockOnRollback(operations, file, exchange);
120            } catch (Exception e) {
121                LOG.warn("Error during releasing exclusive readlock on rollback. This exception is ignored.", e);
122            }
123        };
124
125        if (readLockIdempotentReleaseDelay > 0 && readLockIdempotentReleaseExecutorService != null) {
126            LOG.debug("Scheduling readlock release task to run asynchronous delayed after {} millis", readLockIdempotentReleaseDelay);
127            readLockIdempotentReleaseExecutorService.schedule(r, readLockIdempotentReleaseDelay, TimeUnit.MILLISECONDS);
128        } else if (readLockIdempotentReleaseDelay > 0) {
129            LOG.debug("Delaying readlock release task {} millis", readLockIdempotentReleaseDelay);
130            Thread.sleep(readLockIdempotentReleaseDelay);
131            r.run();
132        } else {
133            r.run();
134        }
135    }
136
137    @Override
138    public void releaseExclusiveReadLockOnCommit(GenericFileOperations<File> operations, GenericFile<File> file, Exchange exchange) throws Exception {
139        String key = asKey(file);
140        Runnable r = () -> {
141            if (removeOnCommit) {
142                idempotentRepository.remove(key);
143            } else {
144                // confirm on commit
145                idempotentRepository.confirm(key);
146            }
147
148            try {
149                changed.releaseExclusiveReadLockOnCommit(operations, file, exchange);
150            } catch (Exception e) {
151                LOG.warn("Error during releasing exclusive readlock on rollback. This exception is ignored.", e);
152            }
153        };
154
155        if (readLockIdempotentReleaseDelay > 0 && readLockIdempotentReleaseExecutorService != null) {
156            LOG.debug("Scheduling readlock release task to run asynchronous delayed after {} millis", readLockIdempotentReleaseDelay);
157            readLockIdempotentReleaseExecutorService.schedule(r, readLockIdempotentReleaseDelay, TimeUnit.MILLISECONDS);
158        } else if (readLockIdempotentReleaseDelay > 0) {
159            LOG.debug("Delaying readlock release task {} millis", readLockIdempotentReleaseDelay);
160            Thread.sleep(readLockIdempotentReleaseDelay);
161            r.run();
162        } else {
163            r.run();
164        }
165    }
166
167    public void setTimeout(long timeout) {
168        changed.setTimeout(timeout);
169    }
170
171    public void setCheckInterval(long checkInterval) {
172        changed.setCheckInterval(checkInterval);
173    }
174
175    public void setReadLockLoggingLevel(LoggingLevel readLockLoggingLevel) {
176        this.readLockLoggingLevel = readLockLoggingLevel;
177        changed.setReadLockLoggingLevel(readLockLoggingLevel);
178    }
179
180    public void setMarkerFiler(boolean markerFile) {
181        // we do not use marker files
182    }
183
184    public void setDeleteOrphanLockFiles(boolean deleteOrphanLockFiles) {
185        // we do not use marker files
186    }
187
188    public void setMinLength(long minLength) {
189        changed.setMinLength(minLength);
190    }
191
192    public void setMinAge(long minAge) {
193        changed.setMinAge(minAge);
194    }
195
196    public CamelContext getCamelContext() {
197        return camelContext;
198    }
199
200    public void setCamelContext(CamelContext camelContext) {
201        this.camelContext = camelContext;
202    }
203
204    /**
205     * The idempotent repository to use as the store for the read locks.
206     */
207    public IdempotentRepository<String> getIdempotentRepository() {
208        return idempotentRepository;
209    }
210
211    /**
212     * The idempotent repository to use as the store for the read locks.
213     */
214    public void setIdempotentRepository(IdempotentRepository<String> idempotentRepository) {
215        this.idempotentRepository = idempotentRepository;
216    }
217
218    /**
219     * Whether to remove the file from the idempotent repository when doing a rollback.
220     * <p/>
221     * By default this is true.
222     */
223    public boolean isRemoveOnRollback() {
224        return removeOnRollback;
225    }
226
227    /**
228     * Whether to remove the file from the idempotent repository when doing a rollback.
229     * <p/>
230     * By default this is true.
231     */
232    public void setRemoveOnRollback(boolean removeOnRollback) {
233        this.removeOnRollback = removeOnRollback;
234    }
235
236    /**
237     * Whether to remove the file from the idempotent repository when doing a commit.
238     * <p/>
239     * By default this is false.
240     */
241    public boolean isRemoveOnCommit() {
242        return removeOnCommit;
243    }
244
245    /**
246     * Whether to remove the file from the idempotent repository when doing a commit.
247     * <p/>
248     * By default this is false.
249     */
250    public void setRemoveOnCommit(boolean removeOnCommit) {
251        this.removeOnCommit = removeOnCommit;
252    }
253
254    /**
255     * Whether to delay the release task for a period of millis.
256     */
257    public void setReadLockIdempotentReleaseDelay(int readLockIdempotentReleaseDelay) {
258        this.readLockIdempotentReleaseDelay = readLockIdempotentReleaseDelay;
259    }
260
261    public boolean isReadLockIdempotentReleaseAsync() {
262        return readLockIdempotentReleaseAsync;
263    }
264
265    /**
266     * Whether the delayed release task should be synchronous or asynchronous.
267     */
268    public void setReadLockIdempotentReleaseAsync(boolean readLockIdempotentReleaseAsync) {
269        this.readLockIdempotentReleaseAsync = readLockIdempotentReleaseAsync;
270    }
271
272    public int getReadLockIdempotentReleaseAsyncPoolSize() {
273        return readLockIdempotentReleaseAsyncPoolSize;
274    }
275
276    /**
277     * The number of threads in the scheduled thread pool when using asynchronous release tasks.
278     */
279    public void setReadLockIdempotentReleaseAsyncPoolSize(int readLockIdempotentReleaseAsyncPoolSize) {
280        this.readLockIdempotentReleaseAsyncPoolSize = readLockIdempotentReleaseAsyncPoolSize;
281    }
282
283    public ScheduledExecutorService getReadLockIdempotentReleaseExecutorService() {
284        return readLockIdempotentReleaseExecutorService;
285    }
286
287    /**
288     * To use a custom and shared thread pool for asynchronous release tasks.
289     */
290    public void setReadLockIdempotentReleaseExecutorService(ScheduledExecutorService readLockIdempotentReleaseExecutorService) {
291        this.readLockIdempotentReleaseExecutorService = readLockIdempotentReleaseExecutorService;
292    }
293
294    protected String asKey(GenericFile<File> file) {
295        // use absolute file path as default key, but evaluate if an expression key was configured
296        String key = file.getAbsoluteFilePath();
297        if (endpoint.getIdempotentKey() != null) {
298            Exchange dummy = endpoint.createExchange(file);
299            key = endpoint.getIdempotentKey().evaluate(dummy, String.class);
300        }
301        return key;
302    }
303
304    @Override
305    protected void doStart() throws Exception {
306        ObjectHelper.notNull(camelContext, "camelContext", this);
307        ObjectHelper.notNull(idempotentRepository, "idempotentRepository", this);
308
309        if (readLockIdempotentReleaseAsync && readLockIdempotentReleaseExecutorService == null) {
310            readLockIdempotentReleaseExecutorService = camelContext.getExecutorServiceManager().newScheduledThreadPool(
311                this, "ReadLockChangedIdempotentReleaseTask", readLockIdempotentReleaseAsyncPoolSize);
312            shutdownExecutorService = true;
313        }
314    }
315
316    @Override
317    protected void doStop() throws Exception {
318        if (shutdownExecutorService && readLockIdempotentReleaseExecutorService != null) {
319            camelContext.getExecutorServiceManager().shutdownGraceful(readLockIdempotentReleaseExecutorService, 30000);
320            readLockIdempotentReleaseExecutorService = null;
321        }
322    }
323
324}