I have developed a backend for a photo sharing app, with Node.js.
I use Firebase as the database.
Here is a piece of the screenshots. 'Posts' section in Firebase
I am going to index the 'Posts' section to ElasticSearch( https://www.firebase.com/blog/2014-01-02-queries-part-two.html ). Here is the indexing codes:
// initialize our ElasticSearch API var client = new ElasticClient({ host: 'localhost', port: 9200 }); // listen for changes to Firebase data var fb = new Firebase('https://mydb.firebaseio.com/Posts'); fb.on('child_added', createOrUpdateIndex); fb.on('child_changed', createOrUpdateIndex); fb.on('child_removed', removeIndex); var index = 'firebase'; var type = 'post'; function createOrUpdateIndex(snap) { //var data = snap.val(); //console.log(data); client.index(index, type, snap.val(), snap.key()) .on('data', function(data) { console.log('indexed ', snap.key()); }) .on('error', function(err) { console.log(err); }).exec(); } function removeIndex(snap) { client.deleteDocument(index, type, snap.key(), function(error, data) { if( error ) console.error('failed to delete', snap.key(), error); else console.log('deleted', snap.key()); }); }
[2016-04-07 16:15:32,851][WARN ][indices.cluster ] [Caretaker] [[firebase][1]] marking and sending shard failed due to [engine failure, reason [index]] java.nio.file.FileSystemException: /Users/user/Downloads/elasticsearch-2.3.1/data/elasticsearch/nodes/0/indices/firebase/1/index/_a.fdt: Too many open files in system at sun.nio.fs.UnixException.translateToIOException(UnixException.java:91) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:214) at java.nio.file.spi.FileSystemProvider.newOutputStream(FileSystemProvider.java:430) at java.nio.file.Files.newOutputStream(Files.java:172) at org.apache.lucene.store.FSDirectory$FSIndexOutput.<init>(FSDirectory.java:271) at org.apache.lucene.store.FSDirectory.createOutput(FSDirectory.java:224) at org.apache.lucene.store.FileSwitchDirectory.createOutput(FileSwitchDirectory.java:155) at org.apache.lucene.store.RateLimitedFSDirectory.createOutput(RateLimitedFSDirectory.java:40) at org.apache.lucene.store.FilterDirectory.createOutput(FilterDirectory.java:73) at org.apache.lucene.store.LockValidatingDirectoryWrapper.createOutput(LockValidatingDirectoryWrapper.java:44) at org.apache.lucene.store.TrackingDirectoryWrapper.createOutput(TrackingDirectoryWrapper.java:43) at org.apache.lucene.codecs.compressing.CompressingStoredFieldsWriter.<init>(CompressingStoredFieldsWriter.java:111) at org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat.fieldsWriter(CompressingStoredFieldsFormat.java:128) at org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.fieldsWriter(Lucene50StoredFieldsFormat.java:183) at org.apache.lucene.index.DefaultIndexingChain.initStoredFieldsWriter(DefaultIndexingChain.java:81) at org.apache.lucene.index.DefaultIndexingChain.startStoredFields(DefaultIndexingChain.java:279) at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:316) at org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:234) at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:450) at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1477) at org.elasticsearch.index.engine.InternalEngine.innerIndex(InternalEngine.java:541) at org.elasticsearch.index.engine.InternalEngine.index(InternalEngine.java:457) at org.elasticsearch.index.shard.IndexShard.index(IndexShard.java:601) at org.elasticsearch.index.engine.Engine$Index.execute(Engine.java:836) at org.elasticsearch.action.index.TransportIndexAction.executeIndexRequestOnPrimary(TransportIndexAction.java:237) at org.elasticsearch.action.index.TransportIndexAction.shardOperationOnPrimary(TransportIndexAction.java:158) at org.elasticsearch.action.index.TransportIndexAction.shardOperationOnPrimary(TransportIndexAction.java:66) at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryPhase.doRun(TransportReplicationAction.java:639) at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:279) at org.elasticsearch.action.support.replication.TransportReplicationAction$PrimaryOperationTransportHandler.messageReceived(TransportReplicationAction.java:271) at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:75) at org.elasticsearch.transport.TransportService$4.doRun(TransportService.java:376) at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745)
I have struggled for this error and now I see it is because of maximum number of opened files in the ElasticSearch module when a great number of 'child_added' events are triggered in a short time.
I think I need to cache the Index operations to avoid this error keeping the limit of opened files as the default value. How can I do that?
You may use a Semaphore to limit the number of running index action to a specified capacity. Look at my post : https://stackoverflow.com/a/37456691/2733216
The code is not tested, but should work.
// initialize our ElasticSearch API
var client = new ElasticClient({ host: 'localhost', port: 9200 });
// listen for changes to Firebase data
var fb = new Firebase('https://mydb.firebaseio.com/Posts');
fb.on('child_added', createOrUpdateIndex);
fb.on('child_changed', createOrUpdateIndex);
fb.on('child_removed', removeIndex);
var index = 'firebase';
var type = 'post';
// Create a semaphore of capacity 1
var semaphore = require ('semaphore');
function createOrUpdateIndex(snap) {
semaphore.take(function () {
//var data = snap.val();
//console.log(data);
client.index(index, type, snap.val(), snap.key())
.on('data', function(data) {
semaphore.leave();
console.log('indexed ', snap.key());
})
.on('error', function(err) {
semaphore.leave();
console.log(err);
}).exec();
});
}
function removeIndex(snap) {
client.deleteDocument(index, type, snap.key(), function(error, data) {
if( error ) console.error('failed to delete', snap.key(), error);
else console.log('deleted', snap.key());
});
}
Then following your system concurrency capacity, you may adjust 1 to x
The technical post webpages of this site follow the CC BY-SA 4.0 protocol. If you need to reprint, please indicate the site URL or the original address.Any question please contact:yoyou2525@163.com.