Mongodb Sorting Desc for _id field very slow

Question

I have a mongodb database which has 30 million Dictionary and Each day of month has 1 million rows so all of document count 30x1=30 Million, database just has data for 1 month and i want to list and sort desc records of between 2018-07-01 and 2018-07-03 so i have 2 million rows between of that two days My each collection like below:

{
"_id":"5c66cf5b67011aa76ca597b6",
"timestamp":"2018-07-01 15:45:37.000",
"category":"category_1"
}

I added sorting desc index for timestamp column

When i try sort asc i get response 0,1 seconds but i try sort desc i am getting response 702 seconds

I am building python

from pymongo import MongoClient
import datetime
import time
client = MongoClient()
client = MongoClient('localhost', 27017)
db = client.MongoBencmarkTestDB

indicator_collections = db.IndicatorCollections

dstart = datetime.datetime(2018, 7, 1,0, 0, 0)
dfinish = datetime.datetime(2018, 7, 3,0, 0, 0)

for indicator_collection in indicator_collections.find({
    "$and":
        [
            {
                "timestamp": {"$lte": dfinish, "$gte": dstart}
            },
        ]

}).sort([("_id", -1)]).skip(0).limit(1000):
    print(indicator_collection['_id'])

When i explain _id field for sorting desc:

db.IndicatorCollections.find().sort({_id : -1}).explain()

Getting response:

{
    "queryPlanner" : {
        "plannerVersion" : 1,
        "namespace" : "MongoBencmarkTestDB.IndicatorCollections",
        "indexFilterSet" : false,
        "parsedQuery" : {

        },
        "winningPlan" : {
            "stage" : "FETCH",
            "inputStage" : {
                "stage" : "IXSCAN",
                "keyPattern" : {
                    "_id" : 1
                },
                "indexName" : "_id_",
                "isMultiKey" : false,
                "multiKeyPaths" : {
                    "_id" : [ ]
                },
                "isUnique" : true,
                "isSparse" : false,
                "isPartial" : false,
                "indexVersion" : 2,
                "direction" : "backward",
                "indexBounds" : {
                    "_id" : [
                        "[MaxKey, MinKey]"
                    ]
                }
            }
        },
        "rejectedPlans" : [ ]
    },
    "serverInfo" : {
        "host" : "reterius-pc-MacBook-Pro.local",
        "port" : 27017,
        "version" : "4.0.3",
        "gitVersion" : "7ea530946fa7880364d88c8d8b6026bbc9ffa48c"
    },
    "ok" : 1
}

My indexes:

[
    {
        "v" : 2,
        "key" : {
            "_id" : 1
        },
        "name" : "_id_",
        "ns" : "MongoBencmarkTestDB.IndicatorCollections"
    },
    {
        "v" : 2,
        "key" : {
            "timestamp" : -1
        },
        "name" : "timestamp_-1",
        "ns" : "MongoBencmarkTestDB.IndicatorCollections"
    }
]

I want to get response quickly because its very important.

Answer 1

So you need to run db.collection.createIndex( { timestamp: 1, _id: -1 } ) and check again if its faster (as it should be). As I have written before, Mongo uses just one index for a query, and if there is no index with timestamp and descending _id field, it is slow.

Answer 2

When i explained my query on pymongo result is below:

{'queryPlanner': {'plannerVersion': 1, 'namespace': 'MongoBencmarkTestDB.IndicatorCollections', 'indexFilterSet': False, 'parsedQuery': {'$and': [{'timestamp': {'$lte': datetime.datetime(2018, 7, 3, 0, 0)}}, {'timestamp': {'$gte': datetime.datetime(2018, 7, 1, 0, 0)}}]}, 'winningPlan': {'stage': 'SORT', 'sortPattern': {'_id': -1}, 'limitAmount': 1000, 'inputStage': {'stage': 'SORT_KEY_GENERATOR', 'inputStage': {'stage': 'FETCH', 'inputStage': {'stage': 'IXSCAN', 'keyPattern': {'timestamp': -1.0}, 'indexName': 'timestamp_-1', 'isMultiKey': False, 'multiKeyPaths': {'timestamp': []}, 'isUnique': False, 'isSparse': False, 'isPartial': False, 'indexVersion': 2, 'direction': 'forward', 'indexBounds': {'timestamp': ['[new Date(1530576000000), new Date(1530403200000)]']}}}}}, 'rejectedPlans': [{'stage': 'SORT', 'sortPattern': {'_id': -1}, 'limitAmount': 1000, 'inputStage': {'stage': 'SORT_KEY_GENERATOR', 'inputStage': {'stage': 'FETCH', 'inputStage': {'stage': 'IXSCAN', 'keyPattern': {'timesta mp': 1.0, '_id': -1.0}, 'indexName': 'timestamp_1__id_-1', 'isMultiKey': False, 'multiKeyPaths': {'timestamp': [], '_id': []}, 'isUnique': False, 'isSparse': False, 'isPartial': False, 'indexVersion': 2, 'direction': 'forward', 'indexBounds': {'timestamp': ['[new Date(1530403200000), new Date(1530576000000)]'], '_id': ['[MaxKey, MinKey]']}}}}}, {'stage': 'LIMIT', 'limitAmount': 1000, 'inputStage': {'stage': 'FETCH', 'filter': {'$and': [{'timestamp': {'$lte': datetime.datetime(2018, 7, 3, 0, 0)}}, {'timestamp': {'$gte': datetime.datetime(2018, 7, 1, 0, 0)}}]}, 'inputStage': {'stage': 'IXSCAN', 'keyPattern': {'_id': 1}, 'indexName': ' id ', 'isMultiKey': False, 'multiKeyPaths': {'_id': []}, 'isUnique': True, 'isSparse': False, 'isPartial': False, 'indexVersion': 2, 'direction': 'backward', 'indexBounds': {'_id': ['[MaxKey, MinKey]']}}}}]}, 'executionStats': {'executionSuccess': True, 'nReturned': 1000, 'executionTimeMillis': 552284, 'totalKeysExamined': 2000000, 'totalDocsExamined': 2000 000, 'executionStages': {'stage': 'SORT', 'nReturned': 1000, 'executionTimeMillisEstimate': 137134, 'works': 2001003, 'advanced': 1000, 'needTime': 2000002, 'needYield': 0, 'saveState': 53750, 'restoreState': 53750, 'isEOF': 1, 'invalidates': 0, 'sortPattern': {'_id': -1}, 'memUsage': 9056307, 'memLimit': 33554432, 'limitAmount': 1000, 'inputStage': {'stage': 'SORT_KEY_GENERATOR', 'nReturned': 2000000, 'executionTimeMillisEstimate': 134910, 'works': 2000002, 'advanced': 2000000, 'needTime': 1, 'needYield': 0, 'saveState': 53750, 'restoreState': 53750, 'isEOF': 1, 'invalidates': 0, 'inputStage': {'stage': 'FETCH', 'nReturned': 2000000, 'executionTimeMillisEstimate': 132229, 'works': 2000001, 'advanced': 2000000, 'needTime': 0, 'needYield': 0, 'saveState': 53750, 'restoreState': 53750, 'isEOF': 1, 'invalidates': 0, 'docsExamined': 2000000, 'alreadyHasObj': 0, 'inputStage': {'stage': 'IXSCAN', 'nReturned': 2000000, 'executionTimeMillisEstimate': 2077, 'works': 2000001, 'advanced': 2000000 , 'needTime': 0, 'needYield': 0, 'saveState': 53750, 'restoreState': 53750, 'isEOF': 1, 'invalidates': 0, 'keyPattern': {'timestamp': -1.0}, 'indexName': 'timestamp_-1', 'isMultiKey': False, 'multiKeyPaths': {'timestamp': []}, 'isUnique': False, 'isSparse': False, 'isPartial': False, 'indexVersion': 2, 'direction': 'forward', 'indexBounds': {'timestamp': ['[new Date(1530576000000), new Date(1530403200000)]']}, 'keysExamined': 2000000, 'seeks': 1, 'dupsTested': 0, 'dupsDropped': 0, 'seenInvalidated': 0}}}}, 'allPlansExecution': [{'nReturned': 101, 'executionTimeMillisEstimate': 137134, 'totalKeysExamined': 2000000, 'totalDocsExamined': 2000000, 'executionStages': {'stage': 'SORT', 'nReturned': 101, 'executionTimeMillisEstimate': 137134, 'works': 2000103, 'advanced': 101, 'needTime': 2000002, 'needYield': 0, 'saveState': 53742, 'restoreState': 53742, 'isEOF': 0, 'invalidates': 0, 'sortPattern': {'_id': -1}, 'memUsage': 9056307, 'memLimit': 33554432, 'limitAmount': 1000, 'inputStage': {'s tage': 'SORT_KEY_GENERATOR', 'nReturned': 2000000, 'executionTimeMillisEstimate': 134910, 'works': 2000002, 'advanced': 2000000, 'needTime': 1, 'needYield': 0, 'saveState': 53742, 'restoreState': 53742, 'isEOF': 1, 'invalidates': 0, 'inputStage': {'stage': 'FETCH', 'nReturned': 2000000, 'executionTimeMillisEstimate': 132229, 'works': 2000001, 'advanced': 2000000, 'needTime': 0, 'needYield': 0, 'saveState': 53742, 'restoreState': 53742, 'isEOF': 1, 'invalidates': 0, 'docsExamined': 2000000, 'alreadyHasObj': 0, 'inputStage': {'stage': 'IXSCAN', 'nReturned': 2000000, 'executionTimeMillisEstimate': 2077, 'works': 2000001, 'advanced': 2000000, 'needTime': 0, 'needYield': 0, 'saveState': 53742, 'restoreState': 53742, 'isEOF': 1, 'invalidates': 0, 'keyPattern': {'timestamp': -1.0}, 'indexName': 'timestamp_-1', 'isMultiKey': False, 'multiKeyPaths': {'timestamp': []}, 'isUnique': False, 'isSparse': False, 'isPartial': False, 'indexVersion': 2, 'direction': 'forward', 'indexBounds': {'timestamp' : ['[new Date(1530576000000), new Date(1530403200000)]']}, 'keysExamined': 2000000, 'seeks': 1, 'dupsTested': 0, 'dupsDropped': 0, 'seenInvalidated': 0}}}}}, {'nReturned': 101, 'executionTimeMillisEstimate': 286826, 'totalKeysExamined': 2000000, 'totalDocsExamined': 2000000, 'executionStages': {'stage': 'SORT', 'nReturned': 101, 'executionTimeMillisEstimate': 286826, 'works': 2000103, 'advanced': 101, 'needTime': 2000002, 'needYield': 0, 'saveState': 53750, 'restoreState': 53750, 'isEOF': 0, 'invalidates': 0, 'sortPattern': {'_id': -1}, 'memUsage': 9056307, 'memLimit': 33554432, 'limitAmount': 1000, 'inputStage': {'stage': 'SORT_KEY_GENERATOR', 'nReturned': 2000000, 'executionTimeMillisEstimate': 284785, 'works': 2000002, 'advanced': 2000000, 'needTime': 1, 'needYield': 0, 'saveState': 53750, 'restoreState': 53750, 'isEOF': 1, 'invalidates': 0, 'inputStage': {'stage': 'FETCH', 'nReturned': 2000000, 'executionTimeMillisEstimate': 128225, 'works': 2000001, 'advanced': 2000000, 'needTime' : 0, 'needYield': 0, 'saveState': 53750, 'restoreState': 53750, 'isEOF': 1, 'invalidates': 0, 'docsExamined': 2000000, 'alreadyHasObj': 0, 'inputStage': {'stage': 'IXSCAN', 'nReturned': 2000000, 'executionTimeMillisEstimate': 1579, 'works': 2000001, 'advanced': 2000000, 'needTime': 0, 'needYield': 0, 'saveState': 53750, 'restoreState': 53750, 'isEOF': 1, 'invalidates': 0, 'keyPattern': {'timestamp': 1.0, '_id': -1.0}, 'indexName': 'timestamp_1__id_-1', 'isMultiKey': False, 'multiKeyPaths': {'timestamp': [], '_id': []}, 'isUnique': False, 'isSparse': False, 'isPartial': False, 'indexVersion': 2, 'direction': 'forward', 'indexBounds': {'timestamp': ['[new Date(1530403200000), new Date(1530576000000)]'], '_id': ['[MaxKey, MinKey]']}, 'keysExamined': 2000000, 'seeks': 1, 'dupsTested': 0, 'dupsDropped': 0, 'seenInvalidated': 0}}}}}, {'nReturned': 0, 'executionTimeMillisEstimate': 126373, 'totalKeysExamined': 2000103, 'totalDocsExamined': 2000103, 'executionStages': {'stage': 'LIMIT', 'nRetu rned': 0, 'executionTimeMillisEstimate': 126373, 'works': 2000103, 'advanced': 0, 'needTime': 2000103, 'needYield': 0, 'saveState': 53750, 'restoreState': 53750, 'isEOF': 0, 'invalidates': 0, 'limitAmount': 1000, 'inputStage': {'stage': 'FETCH', 'filter': {'$and': [{'timestamp': {'$lte': datetime.datetime(2018, 7, 3, 0, 0)}}, {'timestamp': {'$gte': datetime.datetime(2018, 7, 1, 0, 0)}}]}, 'nReturned': 0, 'executionTimeMillisEstimate': 126232, 'works': 2000103, 'advanced': 0, 'needTime': 2000103, 'needYield': 0, 'saveState': 53750, 'restoreState': 53750, 'isEOF': 0, 'invalidates': 0, 'docsExamined': 2000103, 'alreadyHasObj': 0, 'inputStage': {'stage': 'IXSCAN', 'nReturned': 2000103, 'executionTimeMillisEstimate': 2205, 'works': 2000103, 'advanced': 2000103, 'needTime': 0, 'needYield': 0, 'saveState': 53750, 'restoreState': 53750, 'isEOF': 0, 'invalidates': 0, 'keyPattern': {'_id': 1}, 'indexName': ' id ', 'isMultiKey': False, 'multiKeyPaths': {'_id': []}, 'isUnique': True, 'isSparse': F alse, 'isPartial': False, 'indexVersion': 2, 'direction': 'backward', 'indexBounds': {'_id': ['[MaxKey, MinKey]']}, 'keysExamined': 2000103, 'seeks': 1, 'dupsTested': 0, 'dupsDropped': 0, 'seenInvalidated': 0}}}}]}, 'serverInfo': {'host': 'reterius-pc-MacBook-Pro.local', 'port': 27017, 'version': '4.0.3', 'gitVersion': '7ea530946fa7880364d88c8d8b6026bbc9ffa48c'}, 'ok': 1.0}