Cloud Function for Firebase timeout after 60 seconds while running a bunch of Firebase queries

Question

I am using Firebase for a group collaboration app (like Whatsapp) and I am using a Cloud Function to figure out which of the phone contacts are also using my app (again similar to Whatsapp). The Cloud Function ran fine till I started to see the following log in the Functions Log for some invocations.

Function execution took 60023 ms, finished with status: 'timeout'

I did some debugging and found that for this particular user, he has a lot of contacts on his phone's contacts book and so obviously the work required to figure out which of those contacts are using the app also increased to a point that it took more than 60 secs. Below is the code for the Cloud Function

      // contactsData is an array of contacts on the user's phone
      // Each contact can contain one more phone numbers which are
      // present in the phoneNumbers array. So, essentially, we need
      // to query over all the phone numbers in the user's contact book
      contactsData.forEach((contact) => {
        contact.phoneNumbers.forEach((phoneNumber) => {
          // Find if user with this phoneNumber is using the app
          // Check against mobileNumber and mobileNumberWithCC
          promises.push(ref.child('users').orderByChild("mobileNumber").
            equalTo(phoneNumber.number).once("value").then(usersSnapshot => {
              // usersSnapshot should contain just one entry assuming
              // that the phoneNumber will be unique to the user
              if(!usersSnapshot.exists()) {
                return null
              }
              var user = null
              usersSnapshot.forEach(userSnapshot => {
                user = userSnapshot.val()
              })
              return {
                name: contact.name,
                mobileNumber: phoneNumber.number,
                id: user.id
              }
            }))
          promises.push(ref.child('users').orderByChild("mobileNumberWithCC").
            equalTo(phoneNumber.number).once("value").then(usersSnapshot => {
              // usersSnapshot should contain just one entry assuming
              // that the phoneNumber will be unique to the user
              if(!usersSnapshot.exists()) {
                return null
              }
              var user = null
              usersSnapshot.forEach(userSnapshot => {
                user = userSnapshot.val()
              })
              return {
                name: contact.name,
                mobileNumber: phoneNumber.number,
                id: user.id
              }
            }))
        });
      });
      return Promise.all(promises)
    }).then(allContacts => {
      // allContacts is an array of nulls and contacts using the app
      // Get rid of null and any duplicate entries in the returned array
      currentContacts = arrayCompact(allContacts)

      // Create contactsObj which will the user's contacts that are using the app
      currentContacts.forEach(contact => {
        contactsObj[contact.id] = contact
      })
      // Return the currently present contacts
      return ref.child('userInfos').child(uid).child('contacts').once('value')
    }).then((contactsSnapshot) => {
      if(contactsSnapshot.exists()) {
        contactsSnapshot.forEach((contactSnapshot) => {
          previousContacts.push(contactSnapshot.val())
        })
      }
      // Update the contacts on firease asap after reading the previous contacts
      ref.child('userInfos').child(uid).child('contacts').set(contactsObj)

      // Figure out the new, deleted and renamed contacts
      newContacts = arrayDifferenceWith(currentContacts, previousContacts, 
        (obj1, obj2) => (obj1.id === obj2.id))
      deletedContacts = arrayDifferenceWith(previousContacts, currentContacts,
        (obj1, obj2) => (obj1.id === obj2.id))
      renamedContacts = arrayIntersectionWith(currentContacts, previousContacts,
        (obj1, obj2) => (obj1.id === obj2.id && obj1.name !== obj2.name))
      // Create the deletedContactsObj to store on firebase
      deletedContacts.forEach((deletedContact) => {
        deletedContactsObj[deletedContact.id] = deletedContact
      })
      // Get the deleted contacts
      return ref.child('userInfos').child(uid).child('deletedContacts').once('value')
    }).then((deletedContactsSnapshot) => {
      if(deletedContactsSnapshot.exists()) {
        deletedContactsSnapshot.forEach((deletedContactSnapshot) => {
          previouslyDeletedContacts.push(deletedContactSnapshot.val())
        })
      }
      // Contacts that were previously deleted but now added again
      restoredContacts = arrayIntersectionWith(newContacts, previouslyDeletedContacts,
        (obj1, obj2) => (obj1.id === obj2.id))
      // Removed the restored contacts from the deletedContacts
      restoredContacts.forEach((restoredContact) => {
        deletedContactsObj[restoredContact.id] = null
      })
      // Update groups using any of the deleted, new or renamed contacts
      return ContactsHelper.processContactsData(uid, deletedContacts, newContacts, renamedContacts)
    }).then(() => {
      // Set after retrieving the previously deletedContacts
      return ref.child('userInfos').child(uid).child('deletedContacts').update(deletedContactsObj)
    })

Below is some sample data

// This is a sample contactsData
[
  {
    "phoneNumbers": [
      {
        "number": "12324312321",
        "label": "home"
      },
      {
        "number": "2322412132",
        "label": "work"
      }
    ],
    "givenName": "blah5",
    "familyName": "",
    "middleName": ""
  },
  {
    "phoneNumbers": [
      {
        "number": "1231221221",
        "label": "mobile"
      }
    ],
    "givenName": "blah3",
    "familyName": "blah4",
    "middleName": ""
  },
  {
    "phoneNumbers": [
      {
        "number": "1234567890",
        "label": "mobile"
      }
    ],
    "givenName": "blah1",
    "familyName": "blah2",
    "middleName": ""
  }
]



// This is how users are stored on Firebase. This could a lot of users
  "users": {
    "id1" : {
      "countryCode" : "91",
      "id" : "id1",
      "mobileNumber" : "1231211232",
      "mobileNumberWithCC" : "911231211232",
      "name" : "Varun"
    },
    "id2" : {
      "countryCode" : "1",
      "id" : "id2",
      "mobileNumber" : "2342112133",
      "mobileNumberWithCC" : "12342112133",
      "name" : "Ashish"
    },
    "id3" : {
      "countryCode" : "1",
      "id" : "id3",
      "mobileNumber" : "123213421",
      "mobileNumberWithCC" : "1123213421",
      "name" : "Pradeep Singh"
    }
  }

In this particular case, the contactsData contained 1046 entries and for some of those entries, there were two phoneNumbers . So, let's assume there were a total of 1500 phone numbers that I need to check. I am creating queries to compare against the mobileNumber and mobileNumberWithCC for the users in the database. So, there are a total of 3000 queries that the function will make before the promise finishes and I am guessing it is taking more than 60 seconds to finish up all those queries and hence the Cloud Function timed out.

My few questions are:

1. Is it expected for all those queries to take more than 60 secs? I was expecting it to finish much faster given that it is running within the Firebase infrastructure.
2. Is there a way to increase the timeout limit for a function? I am currently on Blaze plan.

I will also appreciate any alternate implementation suggestions for the above function in order alleviate the problem. Thanks!

Answer 1

If you cannot avoid querying so much data, you can change the timeout of a function in the Cloud Console for your project using the Functions product on the left. Currently, you will have to reset the timeout with each new deploy.

Answer 2

The Problem

Your performance problem you experience comes from the query ref.child('users').orderByChild("mobileNumber").equalTo(phon‌​eNumber.number).once‌​("value") which you are calling from within a forEach() inside another forEach() .

To break down this query, you are essentially asking the database to iterate through the children of /users , compare the key mobileNumber to phon‌​eNumber.number and if they match, return a value. However, not only are you calling this for both mobileNumber and mobileNumberWithCC , you are calling this on every iteration of forEach() . So this means that you are looking over X amount of users for Y amount of phone numbers, for Z amount of contacts, therefore performing up to X*Y*Z internal database operations. This is obviously taxing and hence why your query takes over 60 seconds to process.

Potential Fix

I would recommend implementing an index on your database, called /phoneNumbers . Each key in /phoneNumbers is to be named n########### or c########### and contains an "array" of user IDs associated with that phone number.

This structure would look similar to:

"phoneNumbers": {
  "n1234567890": { // without CC, be warned of overlap
    "userId1": true,
    "userId3": true
  },
  "c011234567890": { // with CC for US
    "userId1": true
  },
  "c611234567890": { // with CC for AU
    "userId3": true
  },
  ...
}

Notes:

Why are the phone numbers stored in the format n########### and c########### ?

This is because Firebase treats numeric keys as indexes of an array. This doesn't make sense for this use case, so we add the n / c at the start to suppress this behaviour.

Why use both n########### and c########### ?

If all entries simply used a n prefix, an 11-digit phone number might overlap with a 10-digit phone number that has it's country code added. So we use n for normal phone numbers and c for numbers that include their country code.

Why did you say that each key of /phoneNumbers contains an "array" of user IDs?

This is because you should avoid using numeric-index arrays in Firebase databases (and arrays in general). Say that two separate processes wanted to update /phoneNumbers/n1234567890 by removing user IDs. If one wants to remove the ID at position 1 and the other at position 2; they would end up removing the IDs at position 1 and 3 instead. This can be overcome by storing the user ID as the key instead which allows you to add/remove it by ID rather than position.

Implementation

As you are already using Cloud Functions, implementing such an index is relatively simple. This code can easily be adapted to suit any sort of automatically generated index based on user data.

// Initialize functions and admin.
const functions = require('firebase-functions');
const admin = require('firebase-admin');
admin.initializeApp(functions.config().firebase);

/**
 * Listens to operations on the children of `/users` and updates the `/phoneNumbers` index appropriately.
 */
exports.handleNewUser = functions.database.ref('/users/{userId}')
  .onWrite(event => {
    var deltaSnapshot = event.data,
        userId = event.params.userId,
        tasks = []; // for returned promises

    if (!deltaSnapshot.exists()) {
      // This user has been deleted.
      var previousData = deltaSnapshot.previous.val();
      if (previousData.number) {
        tasks.push(removeUserFromPhoneNumber(userId, previousData.number, false));
      }
      if (previousData.numberWithCC) {
        tasks.push(removeUserFromPhoneNumber(userId, previousData.numberWithCC, true));
      }
      // Handle other cleanup tasks.
      return Promise.all(tasks).then(() => {
        console.log('User "' + userId + '" deleted successfully.');
      });
    }

    var currentData = deltaSnapshot.val();

    if (deltaSnapshot.previous.exists()) {
      // This is an update to existing data.
      var previousData = deltaSnapshot.previous.val();

      if (currentData.number != previousData.number) { // Phone number changed.
        tasks.push(removeUserFromPhoneNumber(userId, previousData.number, false));
        tasks.push(addUserToPhoneNumber(userId, currentData.number, false));
      }
      if (currentData.numberWithCC != previousData.numberWithCC) { // Phone number changed.
        tasks.push(removeUserFromPhoneNumber(userId, previousData.numberWithCC, true));
        tasks.push(addUserToPhoneNumber(userId, currentData.numberWithCC, true));
      }
      // Handle other tasks related to update.
      return Promise.all(tasks).then(() => {
        console.log('User "' + userId + '" updated successfully.');
      });
    }

    // If here, this is a new user.
    tasks.push(addUserToPhoneNumber(userId, currentData.number, false));
    tasks.push(addUserToPhoneNumber(userId, currentData.numberWithCC, true));
    // Handle other tasks related to addition of new user.
    return Promise.all(tasks).then(() => {
      console.log('User "' + userId + '" created successfully.');
    });
  );

/* Phone Number Index Helper Functions */

/**
 * Returns an array of user IDs linked to the specified phone number.
 * @param {String} number - the phone number
 * @param {Boolean} withCountryCode - true, if the phone number includes a country code
 * @return {Promise} - a promise returning an array of user IDs, may be empty.
 */
function lookupUsersByPhoneNumber(number, withCountryCode) {
  // Error out before corrupting data.
  if (!number) return Promise.reject(new TypeError('number cannot be falsy.');
  return lookupIdsByIndex('phoneNumbers', (withCountryCode ? 'c' : 'n') + number);
}

/**
 * Adds the user ID under the specified phone number's index.
 * @param {String} userId - the user ID
 * @param {String} number - the phone number
 * @param {Boolean} withCountryCode - true, if the phone number includes a country code
 * @return {Promise} - the promise returned by transaction()
 */
function addUserToPhoneNumber(userId, number, withCountryCode) {
  // Error out before corrupting data.
  if (!number) return Promise.reject(new TypeError('number cannot be falsy.');
  return addIdToIndex(userId, 'phoneNumbers', (withCountryCode ? 'c' : 'n') + number)
}

/**
 * Removes the user ID under the specified phone number's index.
 * @param {String} userId - the user ID
 * @param {String} number - the phone number
 * @param {Boolean} withCountryCode - true, if the phone number includes a country code
 * @return {Promise} - the promise returned by transaction()
 */
function removeUserFromPhoneNumber(userId, number, withCountryCode) {
  // Error out before corrupting data.
  if (!number) return Promise.reject(new TypeError('number cannot be falsy.');
  return removeIdFromIndex(userId, 'phoneNumbers', (withCountryCode ? 'c' : 'n') + number)
}

/* General Firebase Index CRUD APIs */
/* Credit: @samthecodingman */

/**
 * Returns an array of IDs linked to the specified key in the given index.
 * @param {String} indexName - the index name
 * @param {String} keyName - the key name
 * @return {Promise} - the promise returned by transaction()
 */
function lookupIdsByIndex(indexName, keyName) {
  // Error out before corrupting data.
  if (!indexName) return Promise.reject(new TypeError('indexName cannot be falsy.');
  if (!keyName) return Promise.reject(new TypeError('keyName cannot be falsy.');
  return admin.database().ref(indexName).child(keyName).once("value")
  .then(snapshot => {
    if (!snapshot.exists()) return []; // Use empty array for 'no data'
    var idsObject = snapshot.val();
    if (idsObject == null) return [];
    return Object.keys(idsObject); // return array of IDs
  });
}

/**
 * Adds the ID to the index under the named key.
 * @param {String} id - the entry ID
 * @param {String} indexName - the index name
 * @param {String} keyName - the key name
 * @return {Promise} - the promise returned by transaction()
 */
function addIdToIndex(id, indexName, keyName) {
  // Error out before corrupting data.
  if (!id) return Promise.reject(new TypeError('id cannot be falsy.');
  if (!indexName) return Promise.reject(new TypeError('indexName cannot be falsy.');
  if (!keyName) return Promise.reject(new TypeError('keyName cannot be falsy.');
  return admin.database().ref(indexName).child(keyName)
  .transaction(function(idsObject) {
    idsObject = idsObject || {}; // Create data if it doesn't exist.
    if (idsObject.hasOwnProperty(id)) return; // No update needed.
    idsObject[id] = true; // Add ID.
    return idsObject;
  });
}

/**
 * Removes the ID from the index under the named key.
 * @param {String} id - the entry ID
 * @param {String} indexName - the index name
 * @param {String} keyName - the key name
 * @return {Promise} - the promise returned by transaction()
 */
function removeIdFromIndex(id, indexName, keyName) {
  // Error out before corrupting data.
  if (!id) return Promise.reject(new TypeError('id cannot be falsy.');
  if (!indexName) return Promise.reject(new TypeError('indexName cannot be falsy.');
  if (!keyName) return Promise.reject(new TypeError('keyName cannot be falsy.');
  return admin.database().ref(indexName).child(keyName)
  .transaction(function(idsObject) {
    if (idsObject === null) return; // No data to update.
    if (!idsObject.hasOwnProperty(id)) return; // No update needed.
    delete idsObject[id]; // Remove ID.
    if (Object.keys(idsObject).length === 0) return null; // Delete entire entry.
    return idsObject;
  });
}

The handleNewUser function in the above snippet doesn't catch errors. It will just let Firebase deal with them (by default FB will just log the error). I would recommend implementing appropriate fallbacks as you desire (as you should with any Cloud Function).

Regarding the source code in your question, it would become something similar to:

contactsData.forEach((contact) => {
  contact.phoneNumbers.forEach((phoneNumber) => {
    var tasks = [];
    tasks.push(lookupUsersByPhoneNumber(phoneNumber.number, false)); // Lookup without CC
    tasks.push(lookupUsersByPhoneNumber(phoneNumber.number, true)); // Lookup with CC
    Promise.all(tasks).then(taskResults => {
      var i = 0;
      // Elements of taskResults are arrays of strings from the lookup functions.
      // Flatten and dedupe strings arrays
      var userIds = taskResults.reduce((arr, results) => {
        for (i=0;i<results.length;i++) {
          if (results[i] !== null && ~arr.indexOf(results[i])) {
            arr.push(results[i]); // Add if not already added.
          }              
        }
        return arr;
      }, []);

      // Build 'contacts' array (Doesn't need a database lookup!)
      return userIds.map(uid => ({
        name: contact.name,
        phone: phoneNumber.number,
        id: uid
      }));
    }).then(currentContacts => {
      currentContacts.forEach(contact => {
        contactsObj[contact.id] = contact
      });

      // do original code from question here.
      // I'm not 100% on what it does, so I'll leave it to you.
      // It currently uses an array which is a bad implementation (see notes above). Use PUSH to update the contacts rather than deleting and readding them constantly.
    });
  });
});

Note on Security/Privacy

I would highly recommend restricting read and write access of /phoneNumbers to just the cloud functions service worker for privacy reasons. This may also require portions of your program logic to be moved to the server depending on permissions problems.

To achieve this, replace:

admin.initializeApp(functions.config().firebase);

with:

admin.initializeApp(Object.assign({}, functions.config().firebase, {
  databaseAuthVariableOverride: {
    uid: "cloudfunc-service-worker" // change as desired
  }
});

and to enable it, you would need to configure your Firebase Database rules as follows:

"rules": {
    "phoneNumbers": {
      ".read": "'cloudfunc-service-worker' === auth.uid",
      ".write": "'cloudfunc-service-worker' === auth.uid"
    }
  }