1

I am receiving the following error for some of my batches every here and there:

You've exceeded the limit of 100 jobs in the flex queue for org 00D6A000000vKLe. Wait for some of your batch jobs to finish before adding more.

I know exactly why this is happening. I have 5 classes that are of this size:

global class SDR_Calculator_This_Week implements Schedulable
{
    public Integer batchSize = 100;

    global void execute(SchedulableContext sc)
    {

     List<Id> SDR_IDs = new List<Id>(new Map<Id, User>([SELECT Id FROM user WHERE ( UserRoleId = '00E6A000000UW7gUAG' OR UserRoleId = '00E3s000000sA9DEAU') and IsActive = true]).keySet());


    // This Week
    String Contacts_Created_This_Week_Query = 'SELECT CreatedByID SDR_ID, COUNT( id ) result FROM Contact Where CreatedDate = THIS_WEEK AND CreatedById in :ListOfIDs GROUP BY CreatedById';
    DynamicBatchApexAR batch1_This_Week      =   new DynamicBatchApexAR('SDR_ID', 'result', 'Contacts_Created_This_Week__c', Contacts_Created_This_Week_Query, SDR_IDs);

    String Accounts_Created_This_Week_Query = 'SELECT CreatedByID SDR_ID, COUNT( id ) result FROM Account Where CreatedDate = THIS_WEEK AND CreatedById in :ListOfIDs GROUP BY CreatedById';
    DynamicBatchApexAR batch2_This_Week      =   new DynamicBatchApexAR('SDR_ID', 'result', 'Accounts_Created_This_Week__c', Accounts_Created_This_Week_Query, SDR_IDs);

    String Contacts_Reached_This_Week_Query = 'SELECT Ownerid, whoid FROM Task Where ActivityDate = THIS_WEEK AND Whoid != null AND OwnerId in :ListOfIDs';
    DynamicBatchApexCountDistinct_Tasks batch3_This_Week      =   new DynamicBatchApexCountDistinct_Tasks('Ownerid', 'whoid', 'Contacts_Reached_This_Week__c', Contacts_Reached_This_Week_Query, SDR_IDs);

    String Accounts_Reached_This_Week_Query = 'SELECT Ownerid, accountid FROM Task Where ActivityDate = THIS_WEEK AND AccountId != Null AND OwnerId in :ListOfIDs';
    DynamicBatchApexCountDistinct_Tasks batch4_This_Week      =   new DynamicBatchApexCountDistinct_Tasks('Ownerid', 'accountid', 'Accounts_Reached_This_Week__c', Accounts_Reached_This_Week_Query, SDR_IDs);

    String Manual_Emails_This_Week_Query     = 'SELECT Ownerid SDR_ID, COUNT( id ) result FROM Task Where ActivityDate = THIS_WEEK AND Subject like \'%[Out]%\' AND ( Subject like \'%[Manual]%\' OR Subject like \'%[Gmail]%\' ) AND OwnerId in :ListOfIDs GROUP BY Ownerid';
    DynamicBatchApexAR batch5_This_Week      =   new DynamicBatchApexAR('SDR_ID', 'result', 'Manual_Emails_This_Week__c',  Manual_Emails_This_Week_Query, SDR_IDs);

    String Emails_This_Week_Query           = 'SELECT Ownerid SDR_ID, COUNT( id ) result FROM Task Where ActivityDate = THIS_WEEK AND Subject like \'%[Out]%\' AND OwnerId in :ListOfIDs GROUP BY Ownerid';
    DynamicBatchApexAR batch6_This_Week      =   new DynamicBatchApexAR('SDR_ID', 'result', 'Emails_This_Week__c',  Emails_This_Week_Query, SDR_IDs);

    String Calls_This_Week_Query            = 'SELECT Ownerid SDR_ID, COUNT( id ) result FROM Task Where ActivityDate = THIS_WEEK AND Subject like \'%[Outbound]%\' AND OwnerId in :ListOfIDs GROUP BY Ownerid';
    DynamicBatchApexAR batch7_This_Week      =   new DynamicBatchApexAR('SDR_ID', 'result', 'Calls_This_Week__c',  Calls_This_Week_Query, SDR_IDs);

    String Other_Tasks_This_Week_Query      = 'SELECT Ownerid SDR_ID, COUNT( id ) result FROM Task Where ActivityDate = THIS_WEEK AND Outreach_Task_Type__c != NULL AND OwnerId in :ListOfIDs GROUP BY Ownerid';
    DynamicBatchApexAR batch8_This_Week      =   new DynamicBatchApexAR('SDR_ID', 'result', 'Other_Tasks_This_Week__c',  Other_Tasks_This_Week_Query, SDR_IDs);

    String Email_Opens_This_Week_Query      = 'SELECT Ownerid SDR_ID, COUNT( id ) result FROM Task Where ActivityDate = THIS_WEEK AND Opened_At__c != NULL AND OwnerId in :ListOfIDs GROUP BY Ownerid';
    DynamicBatchApexAR batch9_This_Week      =   new DynamicBatchApexAR('SDR_ID', 'result', 'Email_Opens_This_Week__c',  Email_Opens_This_Week_Query, SDR_IDs);

    String Email_Clicks_This_Week_Query     = 'SELECT Ownerid SDR_ID, COUNT( id ) result FROM Task Where ActivityDate = THIS_WEEK AND Number_Of_Clicks__c > 0 AND OwnerId in :ListOfIDs GROUP BY Ownerid';
    DynamicBatchApexAR batch10_This_Week      =   new DynamicBatchApexAR('SDR_ID', 'result', 'Email_Clicks_This_Week__c',  Email_Clicks_This_Week_Query, SDR_IDs);

    String Email_Replies_This_Week_Query    = 'SELECT Ownerid SDR_ID, COUNT( id ) result FROM Task Where ActivityDate = THIS_WEEK AND Subject like \'%[In]%\' AND ( NOT Subject like \'%Automatic%\' ) AND OwnerId in :ListOfIDs GROUP BY Ownerid';
    DynamicBatchApexAR batch11_This_Week      =   new DynamicBatchApexAR('SDR_ID', 'result', 'Email_Replies_This_Week__c',  Email_Replies_This_Week_Query, SDR_IDs);

    String Demo_Sets_This_Week_Query        = 'SELECT SDR_Owner__c SDR_ID, COUNT(id) result FROM Opportunity where CreatedDate = THIS_WEEK AND ( RecordType.Name = \'Pre Opportunity\' OR RecordType.Name = \'New Business\' ) AND SDR_Owner__c in :ListOfIDs GROUP BY SDR_Owner__c';
    DynamicBatchApexAR batch12_This_Week      = a  new DynamicBatchApexAR('SDR_ID', 'result', 'Demo_Sets_This_Week__c',  Demo_Sets_This_Week_Query, SDR_IDs);

    String Demo_No_Shows_This_Week_Query        = 'SELECT SDR_Owner__c SDR_ID, COUNT(id) result FROM Opportunity where CloseDate = THIS_WEEK AND RecordType.Name = \'Pre Opportunity\' AND Rejection_Reason__c = \'No Show\' AND SDR_Owner__c in :ListOfIDs GROUP BY SDR_Owner__c';
    DynamicBatchApexAR batch13_This_Week      =   new DynamicBatchApexAR('SDR_ID', 'result', 'Demo_No_Shows_This_Week__c',  Demo_No_Shows_This_Week_Query, SDR_IDs);

    String Demo_Completes_This_Week_Query        = 'SELECT SDR_Owner__c SDR_ID, COUNT(id) result FROM Opportunity where ( CloseDate = LAST_WEEK OR New_Business_Date__c = LAST_WEEK )  AND Demo_Completed__c = true AND SDR_Owner__c in :ListOfIDs GROUP BY SDR_Owner__c';
    DynamicBatchApexAR batch14_This_Week      =   new DynamicBatchApexAR('SDR_ID', 'result', 'Demo_Completes_This_Week__c',  Demo_Completes_This_Week_Query, SDR_IDs);

    String Requalifying_Demos_This_Week_Query        = 'SELECT SDR_Owner__c SDR_ID, COUNT(id) result FROM Opportunity where CloseDate = THIS_WEEK AND Recordtype.Name = \'Pre Opportunity\' AND reengagement__c = \'Yes\' AND SDR_Owner__c in :ListOfIDs GROUP BY SDR_Owner__c';
    DynamicBatchApexAR batch15_This_Week      =   new DynamicBatchApexAR('SDR_ID', 'result', 'Requalifying_Demos_This_Week__c',  Requalifying_Demos_This_Week_Query, SDR_IDs);

    String Unqualified_Demos_This_Week_Query   = 'SELECT SDR_Owner__c SDR_ID, COUNT(id) result FROM Opportunity where CloseDate = THIS_WEEK AND Recordtype.Name = \'Pre Opportunity\' AND rejection_reason__c = \'Unqualified Account\' AND SDR_Owner__c in :ListOfIDs GROUP BY SDR_Owner__c';
    DynamicBatchApexAR batch16_This_Week      =   new DynamicBatchApexAR('SDR_ID', 'result', 'Unqualified_Demos_This_Week__c',  Unqualified_Demos_This_Week_Query, SDR_IDs);

    String Qualified_Demos_This_Week_Query   = 'SELECT SDR_Owner__c SDR_ID, COUNT(id) result FROM Opportunity where New_Business_Date__c = THIS_WEEK AND Recordtype.Name = \'New Business\' AND SDR_Owner__c in :ListOfIDs GROUP BY SDR_Owner__c';
    DynamicBatchApexAR batch17_This_Week      =   new DynamicBatchApexAR('SDR_ID', 'result', 'Qualified_Demos_This_Week__c',  Qualified_Demos_This_Week_Query, SDR_IDs);

    String Closed_Won_This_Week_Query   = 'SELECT SDR_Owner__c SDR_ID, COUNT(id) result FROM Opportunity where CreatedDate = THIS_WEEK AND Recordtype.Name = \'New Business\' AND IsWon = true AND SDR_Owner__c in :ListOfIDs GROUP BY SDR_Owner__c';
    DynamicBatchApexAR batch18_This_Week      =   new DynamicBatchApexAR('SDR_ID', 'result', 'Closed_Won_This_Week__c',  Closed_Won_This_Week_Query, SDR_IDs);

    String Closed_Lost_This_Week_Query   = 'SELECT SDR_Owner__c SDR_ID, COUNT(id) result FROM Opportunity where CreatedDate = THIS_WEEK AND Recordtype.Name = \'New Business\' AND IsClosed = true AND IsWon = false AND SDR_Owner__c in :ListOfIDs GROUP BY SDR_Owner__c';
    DynamicBatchApexAR batch19_This_Week      =   new DynamicBatchApexAR('SDR_ID', 'result', 'Closed_Lost_This_Week__c',  Closed_Lost_This_Week_Query, SDR_IDs);


    // TR 120 Batches
    Database.executeBatch(batch1_This_Week, batchSize);
    Database.executeBatch(batch2_This_Week, batchSize);
    Database.executeBatch(batch3_This_Week, batchSize);
    Database.executeBatch(batch4_This_Week, batchSize);
    Database.executeBatch(batch5_This_Week, batchSize);
    Database.executeBatch(batch6_This_Week, batchSize);
    Database.executeBatch(batch7_This_Week, batchSize);
    Database.executeBatch(batch8_This_Week, batchSize);
    Database.executeBatch(batch9_This_Week, batchSize);
    Database.executeBatch(batch10_This_Week, batchSize);
    Database.executeBatch(batch11_This_Week, batchSize);
    Database.executeBatch(batch12_This_Week, batchSize);
    Database.executeBatch(batch13_This_Week, batchSize);
    Database.executeBatch(batch14_This_Week, batchSize);
    Database.executeBatch(batch15_This_Week, batchSize);
    Database.executeBatch(batch16_This_Week, batchSize);
    Database.executeBatch(batch17_This_Week, batchSize);
    Database.executeBatch(batch18_This_Week, batchSize);
    Database.executeBatch(batch19_This_Week, batchSize);


    }
}

So I did some digging - I came across chaining batch jobs in the Finish Method and queueable jobs. Really not sure which one to use. The issue with starting a batch job in the finish method is that the batch apex class I am using is dynamic and is used by so many different schedulabe interfaces across my org. So by me chaining the job in the finish statement, it loses its abstraction. I am interested to see how others would go about this.

5
  • Does your batch class implement Database.Stateful?
    – David Reed
    Oct 23 '19 at 17:20
  • @DavidReed Yes and no. The DynamicBatchApexAR class does not and the DynamicBatchApexCountDistinct_Tasks does. Oct 23 '19 at 17:23
  • @DavidReed I would love it if the code could wait - similar to how the code times out after a certain number of milliseconds during an http request. req.setTimeout(2000); // timeout in milliseconds Oct 23 '19 at 17:24
  • @DavidReed Do you think I could just create a queue data structure in the scheduled class that gets passed into each batch. Then in the finish method it checks to see if there is anything left in the data structure. If there is, it just calls the batch that is up next. Oct 23 '19 at 17:53
  • @DavidReed Looks like we might be able to dynamically call classes with the Callable interface salesforce.stackexchange.com/questions/1035/… Oct 23 '19 at 17:55
1

I suspect that the needed approach looks something like this:

public class DynamicBatchApexAR implements Database.Batchable, Database.Stateful {
    private List<String> queries;
    private List<Id> SDR_IDs;

    public DynamicBatchApexAR(List<String> queries, List<Id> SDR_IDs) {
        this.queries = queries;
        this.SDR_IDs = SDR_IDs;
    }

    public QueryLocator start(Database.BatchableContext bc) {
        String thisQuery = queries.remove(0);
        return Database.query(thisQuery);
    }

    // etc...

    public void finish(Database.BatchableContext bc) {
        if (queries.size() > 0) {
            Database.executeBatch(new DynamicBatchApexAR(queries, SDR_IDs), 100);
        }
    }
}

That is, the batch class accepts a list of queries that it needs to run against, along with the apparently constant parameter SDR_IDs. When it runs, it pops a query off the stack and executes against it. When it finishes, if there are queries left on the stack, it chains into a new batch, passing along the entire parameter data structure.

Then you can enqueue one (or five, or however many) chains of batches, instead of >100 individual batches, and ensure that your flex queue utilization stays under control.

2
  • There is no way to abstract this out of the actual batch? Oct 23 '19 at 21:47
  • @MatthewMetros I mean, you could abstract the query and Id details into your own context object. But if you want to do batch chaining, you've got to do it in finish().
    – David Reed
    Oct 24 '19 at 1:53

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Not the answer you're looking for? Browse other questions tagged or ask your own question.