1

I am receiving the following error for some of my batches every here and there:

You've exceeded the limit of 100 jobs in the flex queue for org 00D6A000000vKLe. Wait for some of your batch jobs to finish before adding more.

I know exactly why this is happening. I have 5 classes that are of this size:

global class SDR_Calculator_This_Week implements Schedulable
{
    public Integer batchSize = 100;

    global void execute(SchedulableContext sc)
    {

     List<Id> SDR_IDs = new List<Id>(new Map<Id, User>([SELECT Id FROM user WHERE ( UserRoleId = '00E6A000000UW7gUAG' OR UserRoleId = '00E3s000000sA9DEAU') and IsActive = true]).keySet());


    // This Week
    String Contacts_Created_This_Week_Query = 'SELECT CreatedByID SDR_ID, COUNT( id ) result FROM Contact Where CreatedDate = THIS_WEEK AND CreatedById in :ListOfIDs GROUP BY CreatedById';
    DynamicBatchApexAR batch1_This_Week      =   new DynamicBatchApexAR('SDR_ID', 'result', 'Contacts_Created_This_Week__c', Contacts_Created_This_Week_Query, SDR_IDs);

    String Accounts_Created_This_Week_Query = 'SELECT CreatedByID SDR_ID, COUNT( id ) result FROM Account Where CreatedDate = THIS_WEEK AND CreatedById in :ListOfIDs GROUP BY CreatedById';
    DynamicBatchApexAR batch2_This_Week      =   new DynamicBatchApexAR('SDR_ID', 'result', 'Accounts_Created_This_Week__c', Accounts_Created_This_Week_Query, SDR_IDs);

    String Contacts_Reached_This_Week_Query = 'SELECT Ownerid, whoid FROM Task Where ActivityDate = THIS_WEEK AND Whoid != null AND OwnerId in :ListOfIDs';
    DynamicBatchApexCountDistinct_Tasks batch3_This_Week      =   new DynamicBatchApexCountDistinct_Tasks('Ownerid', 'whoid', 'Contacts_Reached_This_Week__c', Contacts_Reached_This_Week_Query, SDR_IDs);

    String Accounts_Reached_This_Week_Query = 'SELECT Ownerid, accountid FROM Task Where ActivityDate = THIS_WEEK AND AccountId != Null AND OwnerId in :ListOfIDs';
    DynamicBatchApexCountDistinct_Tasks batch4_This_Week      =   new DynamicBatchApexCountDistinct_Tasks('Ownerid', 'accountid', 'Accounts_Reached_This_Week__c', Accounts_Reached_This_Week_Query, SDR_IDs);

    String Manual_Emails_This_Week_Query     = 'SELECT Ownerid SDR_ID, COUNT( id ) result FROM Task Where ActivityDate = THIS_WEEK AND Subject like \'%[Out]%\' AND ( Subject like \'%[Manual]%\' OR Subject like \'%[Gmail]%\' ) AND OwnerId in :ListOfIDs GROUP BY Ownerid';
    DynamicBatchApexAR batch5_This_Week      =   new DynamicBatchApexAR('SDR_ID', 'result', 'Manual_Emails_This_Week__c',  Manual_Emails_This_Week_Query, SDR_IDs);

    String Emails_This_Week_Query           = 'SELECT Ownerid SDR_ID, COUNT( id ) result FROM Task Where ActivityDate = THIS_WEEK AND Subject like \'%[Out]%\' AND OwnerId in :ListOfIDs GROUP BY Ownerid';
    DynamicBatchApexAR batch6_This_Week      =   new DynamicBatchApexAR('SDR_ID', 'result', 'Emails_This_Week__c',  Emails_This_Week_Query, SDR_IDs);

    String Calls_This_Week_Query            = 'SELECT Ownerid SDR_ID, COUNT( id ) result FROM Task Where ActivityDate = THIS_WEEK AND Subject like \'%[Outbound]%\' AND OwnerId in :ListOfIDs GROUP BY Ownerid';
    DynamicBatchApexAR batch7_This_Week      =   new DynamicBatchApexAR('SDR_ID', 'result', 'Calls_This_Week__c',  Calls_This_Week_Query, SDR_IDs);

    String Other_Tasks_This_Week_Query      = 'SELECT Ownerid SDR_ID, COUNT( id ) result FROM Task Where ActivityDate = THIS_WEEK AND Outreach_Task_Type__c != NULL AND OwnerId in :ListOfIDs GROUP BY Ownerid';
    DynamicBatchApexAR batch8_This_Week      =   new DynamicBatchApexAR('SDR_ID', 'result', 'Other_Tasks_This_Week__c',  Other_Tasks_This_Week_Query, SDR_IDs);

    String Email_Opens_This_Week_Query      = 'SELECT Ownerid SDR_ID, COUNT( id ) result FROM Task Where ActivityDate = THIS_WEEK AND Opened_At__c != NULL AND OwnerId in :ListOfIDs GROUP BY Ownerid';
    DynamicBatchApexAR batch9_This_Week      =   new DynamicBatchApexAR('SDR_ID', 'result', 'Email_Opens_This_Week__c',  Email_Opens_This_Week_Query, SDR_IDs);

    String Email_Clicks_This_Week_Query     = 'SELECT Ownerid SDR_ID, COUNT( id ) result FROM Task Where ActivityDate = THIS_WEEK AND Number_Of_Clicks__c > 0 AND OwnerId in :ListOfIDs GROUP BY Ownerid';
    DynamicBatchApexAR batch10_This_Week      =   new DynamicBatchApexAR('SDR_ID', 'result', 'Email_Clicks_This_Week__c',  Email_Clicks_This_Week_Query, SDR_IDs);

    String Email_Replies_This_Week_Query    = 'SELECT Ownerid SDR_ID, COUNT( id ) result FROM Task Where ActivityDate = THIS_WEEK AND Subject like \'%[In]%\' AND ( NOT Subject like \'%Automatic%\' ) AND OwnerId in :ListOfIDs GROUP BY Ownerid';
    DynamicBatchApexAR batch11_This_Week      =   new DynamicBatchApexAR('SDR_ID', 'result', 'Email_Replies_This_Week__c',  Email_Replies_This_Week_Query, SDR_IDs);

    String Demo_Sets_This_Week_Query        = 'SELECT SDR_Owner__c SDR_ID, COUNT(id) result FROM Opportunity where CreatedDate = THIS_WEEK AND ( RecordType.Name = \'Pre Opportunity\' OR RecordType.Name = \'New Business\' ) AND SDR_Owner__c in :ListOfIDs GROUP BY SDR_Owner__c';
    DynamicBatchApexAR batch12_This_Week      = a  new DynamicBatchApexAR('SDR_ID', 'result', 'Demo_Sets_This_Week__c',  Demo_Sets_This_Week_Query, SDR_IDs);

    String Demo_No_Shows_This_Week_Query        = 'SELECT SDR_Owner__c SDR_ID, COUNT(id) result FROM Opportunity where CloseDate = THIS_WEEK AND RecordType.Name = \'Pre Opportunity\' AND Rejection_Reason__c = \'No Show\' AND SDR_Owner__c in :ListOfIDs GROUP BY SDR_Owner__c';
    DynamicBatchApexAR batch13_This_Week      =   new DynamicBatchApexAR('SDR_ID', 'result', 'Demo_No_Shows_This_Week__c',  Demo_No_Shows_This_Week_Query, SDR_IDs);

    String Demo_Completes_This_Week_Query        = 'SELECT SDR_Owner__c SDR_ID, COUNT(id) result FROM Opportunity where ( CloseDate = LAST_WEEK OR New_Business_Date__c = LAST_WEEK )  AND Demo_Completed__c = true AND SDR_Owner__c in :ListOfIDs GROUP BY SDR_Owner__c';
    DynamicBatchApexAR batch14_This_Week      =   new DynamicBatchApexAR('SDR_ID', 'result', 'Demo_Completes_This_Week__c',  Demo_Completes_This_Week_Query, SDR_IDs);

    String Requalifying_Demos_This_Week_Query        = 'SELECT SDR_Owner__c SDR_ID, COUNT(id) result FROM Opportunity where CloseDate = THIS_WEEK AND Recordtype.Name = \'Pre Opportunity\' AND reengagement__c = \'Yes\' AND SDR_Owner__c in :ListOfIDs GROUP BY SDR_Owner__c';
    DynamicBatchApexAR batch15_This_Week      =   new DynamicBatchApexAR('SDR_ID', 'result', 'Requalifying_Demos_This_Week__c',  Requalifying_Demos_This_Week_Query, SDR_IDs);

    String Unqualified_Demos_This_Week_Query   = 'SELECT SDR_Owner__c SDR_ID, COUNT(id) result FROM Opportunity where CloseDate = THIS_WEEK AND Recordtype.Name = \'Pre Opportunity\' AND rejection_reason__c = \'Unqualified Account\' AND SDR_Owner__c in :ListOfIDs GROUP BY SDR_Owner__c';
    DynamicBatchApexAR batch16_This_Week      =   new DynamicBatchApexAR('SDR_ID', 'result', 'Unqualified_Demos_This_Week__c',  Unqualified_Demos_This_Week_Query, SDR_IDs);

    String Qualified_Demos_This_Week_Query   = 'SELECT SDR_Owner__c SDR_ID, COUNT(id) result FROM Opportunity where New_Business_Date__c = THIS_WEEK AND Recordtype.Name = \'New Business\' AND SDR_Owner__c in :ListOfIDs GROUP BY SDR_Owner__c';
    DynamicBatchApexAR batch17_This_Week      =   new DynamicBatchApexAR('SDR_ID', 'result', 'Qualified_Demos_This_Week__c',  Qualified_Demos_This_Week_Query, SDR_IDs);

    String Closed_Won_This_Week_Query   = 'SELECT SDR_Owner__c SDR_ID, COUNT(id) result FROM Opportunity where CreatedDate = THIS_WEEK AND Recordtype.Name = \'New Business\' AND IsWon = true AND SDR_Owner__c in :ListOfIDs GROUP BY SDR_Owner__c';
    DynamicBatchApexAR batch18_This_Week      =   new DynamicBatchApexAR('SDR_ID', 'result', 'Closed_Won_This_Week__c',  Closed_Won_This_Week_Query, SDR_IDs);

    String Closed_Lost_This_Week_Query   = 'SELECT SDR_Owner__c SDR_ID, COUNT(id) result FROM Opportunity where CreatedDate = THIS_WEEK AND Recordtype.Name = \'New Business\' AND IsClosed = true AND IsWon = false AND SDR_Owner__c in :ListOfIDs GROUP BY SDR_Owner__c';
    DynamicBatchApexAR batch19_This_Week      =   new DynamicBatchApexAR('SDR_ID', 'result', 'Closed_Lost_This_Week__c',  Closed_Lost_This_Week_Query, SDR_IDs);


    // TR 120 Batches
    Database.executeBatch(batch1_This_Week, batchSize);
    Database.executeBatch(batch2_This_Week, batchSize);
    Database.executeBatch(batch3_This_Week, batchSize);
    Database.executeBatch(batch4_This_Week, batchSize);
    Database.executeBatch(batch5_This_Week, batchSize);
    Database.executeBatch(batch6_This_Week, batchSize);
    Database.executeBatch(batch7_This_Week, batchSize);
    Database.executeBatch(batch8_This_Week, batchSize);
    Database.executeBatch(batch9_This_Week, batchSize);
    Database.executeBatch(batch10_This_Week, batchSize);
    Database.executeBatch(batch11_This_Week, batchSize);
    Database.executeBatch(batch12_This_Week, batchSize);
    Database.executeBatch(batch13_This_Week, batchSize);
    Database.executeBatch(batch14_This_Week, batchSize);
    Database.executeBatch(batch15_This_Week, batchSize);
    Database.executeBatch(batch16_This_Week, batchSize);
    Database.executeBatch(batch17_This_Week, batchSize);
    Database.executeBatch(batch18_This_Week, batchSize);
    Database.executeBatch(batch19_This_Week, batchSize);


    }
}

So I did some digging - I came across chaining batch jobs in the Finish Method and queueable jobs. Really not sure which one to use. The issue with starting a batch job in the finish method is that the batch apex class I am using is dynamic and is used by so many different schedulabe interfaces across my org. So by me chaining the job in the finish statement, it loses its abstraction. I am interested to see how others would go about this.

5
  • Does your batch class implement Database.Stateful?
    – David Reed
    Commented Oct 23, 2019 at 17:20
  • @DavidReed Yes and no. The DynamicBatchApexAR class does not and the DynamicBatchApexCountDistinct_Tasks does. Commented Oct 23, 2019 at 17:23
  • @DavidReed I would love it if the code could wait - similar to how the code times out after a certain number of milliseconds during an http request. req.setTimeout(2000); // timeout in milliseconds Commented Oct 23, 2019 at 17:24
  • @DavidReed Do you think I could just create a queue data structure in the scheduled class that gets passed into each batch. Then in the finish method it checks to see if there is anything left in the data structure. If there is, it just calls the batch that is up next. Commented Oct 23, 2019 at 17:53
  • @DavidReed Looks like we might be able to dynamically call classes with the Callable interface salesforce.stackexchange.com/questions/1035/… Commented Oct 23, 2019 at 17:55

1 Answer 1

1

I suspect that the needed approach looks something like this:

public class DynamicBatchApexAR implements Database.Batchable, Database.Stateful {
    private List<String> queries;
    private List<Id> SDR_IDs;

    public DynamicBatchApexAR(List<String> queries, List<Id> SDR_IDs) {
        this.queries = queries;
        this.SDR_IDs = SDR_IDs;
    }

    public QueryLocator start(Database.BatchableContext bc) {
        String thisQuery = queries.remove(0);
        return Database.query(thisQuery);
    }

    // etc...

    public void finish(Database.BatchableContext bc) {
        if (queries.size() > 0) {
            Database.executeBatch(new DynamicBatchApexAR(queries, SDR_IDs), 100);
        }
    }
}

That is, the batch class accepts a list of queries that it needs to run against, along with the apparently constant parameter SDR_IDs. When it runs, it pops a query off the stack and executes against it. When it finishes, if there are queries left on the stack, it chains into a new batch, passing along the entire parameter data structure.

Then you can enqueue one (or five, or however many) chains of batches, instead of >100 individual batches, and ensure that your flex queue utilization stays under control.

2
  • There is no way to abstract this out of the actual batch? Commented Oct 23, 2019 at 21:47
  • @MatthewMetros I mean, you could abstract the query and Id details into your own context object. But if you want to do batch chaining, you've got to do it in finish().
    – David Reed
    Commented Oct 24, 2019 at 1:53

You must log in to answer this question.

Not the answer you're looking for? Browse other questions tagged .