QUERY_EXCEEDED_MAX_MATCHES_ALLOWED error on Kaltura API (Python)

Question

I'm unable to generate all entries in Kaltura. An ApiException with the message "Unable to generate list. max matches value was reached" (Error: QUERY_EXCEEDED_MAX_MATCHES_ALLOWED) gets triggered.

I tried to work around such issue by setting my sessionPrivileges to disableentitlement

class class_chk_integrity():
    client = None
    pagesize = 0

    def __init__(self,worker_num, progress):
        self.pagesize = 30
        self.worker_num = worker_num
        self.progress = progress

        config = KalturaConfiguration(2723521)
        config.serviceUrl = "https://www.kaltura.com/"
        self.client = KalturaClient(config)
        ks = self.client.session.start("KALTURA_ADMIN_SECRET",
            "email@email.com",
            KalturaPluginsCore.KalturaSessionType.ADMIN,
            "KALTURA_PARTNER_ID",
            432000,
            "disableentitlement")
        self.client.setKs(ks)

I also tried to filter based on the id's. However, I can't manage to put the filter.idNotIn to work properly.

   def get_total_reg(self, cont, lastEntryIds, lastEntryCreatedAt):
        
        
        filter = KalturaPluginsCore.KalturaBaseEntryFilter()

        if lastEntryIds != "":
            filter.idNotIn = lastEntryIds

        filter.orderBy = KalturaBaseEntryOrderBy.CREATED_AT_DESC

        pager = KalturaPluginsCore.KalturaFilterPager()
        

        pageIndex = 1
        entriesGot = 0

        pager.pageSize = self.pagesize
        pager.setPageIndex = pageIndex
        result = self.client.baseEntry.list(filter, pager)
        totalCount = result.totalCount

        if totalCount > 10000:
            totalCount = 9970

        if totalCount <= 0:
            cont = False
        
        while entriesGot < totalCount:

            pager.pageSize = self.pagesize
            pageIndex += 1
            pager.pageIndex = pageIndex
            result = self.client.baseEntry.list(filter, pager)
            entriesGot += len(result.objects)

        
            for e in result.objects:
                if lastEntryIds == "":
                    lastEntryIds.append(e.id)
                else:
                    lastEntryIds.append(e.id)

                lastEntryCreatedAt = e.createdAt
        

        return result.totalCount, self.pagesize, cont, lastEntryIds, lastEntryCreatedAt

This is my how I'm calling the functions

if __name__ == '__main__':
    try:
        log = _ServiceUtils.log()
        log.setup('all', 'integrity')
        cont = True
        lastEntryIds = []
        lastEntryCreatedAt = 0


        while cont is True:

            kmc = class_chk_integrity(0,0)
        
            kmc_total_reg, kmc_page_size, cont, lastEntryIds, lastEntryCreatedAt = kmc.get_total_reg(cont, lastEntryIds, lastEntryCreatedAt)

            interval = 10
            max_threads = math.ceil(kmc_total_reg / (interval * kmc_page_size))
            # max_threads = 1
            threads_list = []

            print('TOTAL REG : %s | PAGE_SIZE : %s | INTERVAL : %s | THREADS : %s' % (kmc_total_reg,kmc_page_size,interval,max_threads))  

            progress = class_progress_thread(max_threads)

            for index in range(0,max_threads):
                page_ini = index * interval
                page_end = index * interval + interval
                progress.add_worker_progress(index,datetime.now())

                threads_list.append(threading.Thread(target=thread_chk_integrity, args=(index, log, index * interval + 1,index * interval + interval,progress)))
        
            threads_list.append(threading.Thread(target=thread_output_progress, args=(progress,max_threads)))    

            for thread in threads_list:
                thread.start()

            for thread in threads_list:
                thread.join()

            while not progress.stop(): time.sleep(30)


    except KeyboardInterrupt:
        try:
            sys.exit(0)
        except SystemExit:
            os._exit(0)

I'd appreciate any help with this. Thank you for your attention.

Answer 1

 if totalCount > 10000:
     totalCount = 9970

I'm curious to know why you are changing the totalCount this way.

Short answer - paging works as long as the result set is up to 10K.

To work around that, sort the result by creation date (as you did), and when you get to 10K, start with a new search where the created_at date in the filter is the last value you got in the previous search. Reset your paging of course.