0

Edits : Adding the code snippet here for review

 global class Agg_Batch implements Database.Batchable<sObject>, Database.Stateful, System.Schedulable {

 global Agg_Batch() {

        //Initialize the collections and set
        grpLfcMappedQuarterImports = new List<ParentObject>();
        setLFCMappedQyStrings = new Set<String>();
        updatedLfcMappedQuarterImports = new List<ParentObject>(); // this is to update Sync to report flag on the ParentObject

    }

     public static void schedule(){

        String CRON_EXP = '0 0 * * * ?';//every hour
        //cannot be tested in Apex tests, causes AsyncException if class already scheduled.
        if( !Test.isRunningTest() )
            System.schedule(JOB_NAME, CRON_EXP, new Agg_Batch() );

    }


    private Id enqueueBatchProcess(){

        if( BatchUtilityService.getBatchJobRunningCount( CLASS_NAME ) > 0 )
            return null;
        return Database.executeBatch( new Agg_Batch(), 1 );

    }


    global void execute( SchedulableContext sc ){
       enqueueBatchProcess();
    }


//ParentObject - This is parent object, and each of the associated child object can have more than 10000 records.
//ChildObject - This is child records
//AggObject - Aggregate object.
global List<ParentObject> start( Database.BatchableContext BC ){


        for( ParentObject mappedImport1 :
                    [SELECT Id, Revision_Status__c, ImportID__c, AggView_Period__c,
                    Year__c, Quarter__c, Account_ID__C FROM ParentObject WHEN Sync_To_Report__c = false
                    ])
        {

            updatedLfcMappedQuarterImports.Add(new ParentObject(id = mappedImport1.Id, Sync_To_Report__c = false)); // the parent record, once read needs to be flagged

            if (!setLFCMappedQyStrings.contains(mappedImport1.ImportID__c)) {
                setLFCMappedQyStrings.add(mappedImport1.ImportID__c);
                ParentObject imp = new ParentObject(LFC_Quarter_Id__c = mappedImport1.LFC_Quarter_Id__c,
                        Account_Id__c = mappedImport1.Account_Id__c, Year__c=mappedImport1.Year__c, Quarter__c = mappedImport1.Quarter__c); 

                grpLfcMappedQuarterImports.add(imp); // Grouping unique ParentObject by an Alternate ID i.e. ImportID__c
            }
        }

        return grpLfcMappedQuarterImports;


}

global void execute(Database.BatchableContext BC, List<sObject> scope) {

        for ( ParentObject mappedImport : grpLfcMappedQuarterImports) {
            generateAggregateReportData(mappedImport);
        }
    }

 public  void generateAggregateReportData( ParentObject mappedImport ) {


        String importID = mappedImport.Account_Id__c + '^' + String.valueOf(mappedImport.Quarter__c ) + '^' + (mappedImport.Year__c == null ? '': String.valueOf(Integer.valueOf(mappedImport.Year__c)));
        String periodString = String.valueOf(mappedImport.Quarter__c ) + ' ' + (mappedImport.Year__c == null ? '': String.valueOf(Integer.valueOf(mappedImport.Year__c)));

        // Generate new AggObject records
        Map<String, Integer> keyToOccurenceCountMap = new Map<String, Integer>();

        Map<String, List<AggregateLineGroup>> aggregateMap = new Map<String, List<AggregateLineGroup>>(); 
        List<ChildObject> lines = LFCMappedLinesSelector.getMappedLineItemsByMappedKeyImportId( importID ); // This call gets more than 10000 records
        List<AggObject> aggrInsertList = new List<AggObject>();

        for( ChildObject line : lines ){

            //Custom agg Logic to aggregate into aggrInsertList

        }

        if (aggrInsertList.isEmpty()) {
            insert aggrInsertList; // this call FAILS [wondering if this can be BATCHED Too]
        }

}

 global void finish(Database.BatchableContext BC) {

        if ( !updatedLfcMappedQuarterImports.isEmpty())
            update updatedLfcMappedQuarterImports;

        System.debug('MappedLineAggregateBatch2 finished Processing');

}
}

Here are the sequence of events I am trying to execute :

In the start method of the Batch, I am trying to query Parent Object ( Keeping Batch Size =1, this is kept minimum as we are subsequently querying a very large set of child record later ).

We are passing each Parent Record into the execute method to query for child records and then Aggregating results of the child records and finally inserting into a Aggregate table ( this is a new object that needs to store the aggregate results from the child rows).

Now issue is, in step 2 I am getting from than 10,000 records ( we have to get all records as we are running aggregating on the entire set), and while trying to insert it-I am running into governors limit. I am wondering if there is way the aggregate collection in step2 above also be Batched or if there is any other option to consider. I even tried to implement queyarable interface, but running into governor limit which is 100 max job.

Any advice?

6
  • 1
    Are you trying to insert 10k records against 1 parent record? If so that's not very scalable. Also If you could share some code that would be helpful, I'm struggling to follow as is.
    – Girbot
    Jun 26, 2019 at 17:18
  • why would you be inserting 10000+ recs if you are aggregating them in the execute() method? Please use edit to clarify
    – cropredy
    Jun 26, 2019 at 17:55
  • @Gitbot, just added the code - hope it helps to understand the issue here.
    – Raj
    Jun 26, 2019 at 18:20
  • @cropredy - I am not trying to, the aggregate logic is a custom logic, and does not significantly bring down the number of records that need to be inserted/
    – Raj
    Jun 26, 2019 at 18:22
  • If I read this correctly is your call to generateAggregateReportData not doing DML in a for loop? Or have you sanitised the code?
    – Girbot
    Jun 26, 2019 at 19:07

3 Answers 3

-1

You should do by using queuable implementation but you have to chain the job one after other.Each job can process 10k and I believe we can chain 50 jobs please check documentation. The challenge here is you need to pass only subset of records for each job from the remaining records..!

4
  • thanks Prakash, instead of chaining the queuable interface, does it make more sense to chain another batch job to avoid any apex limits.
    – Raj
    Jun 26, 2019 at 19:43
  • Channing will be sequential , so there is a guarantee .But batch execution will be not sequential , that may execute whenever resources are available .So if you are not worried About the order then yes it will work
    – Prakash
    Jun 27, 2019 at 3:33
  • Ok thanks, that gives me some idea. I think I may be able to structure the code a little bit, and then possibly chain the batches....I will give it a try..let you know.
    – Raj
    Jun 27, 2019 at 16:21
  • Ok, it worked for me - I ending up calling Querable interface in the execute method of the main batch and then chaining a SubBatch Job in the Execute method of the Queryable interface.
    – Raj
    Jun 28, 2019 at 16:15
3

I had a similar issue and wrote the following class. Hope it helps someone.

global class UT_BatchDML implements Queueable { 
global enum Operation { INS, UPD, DEL }
global Integer BatchSize = 10000;
private Operation operation;
private List<SObject> recordList;
private UT_BatchDML followupJob;

global UT_BatchDML(Operation operation, List<SObject> recordList) {
    this.operation = operation;
    this.recordList = recordList;
}

// Process the records
global void execute(QueueableContext context) {         
    if (recordList == null || recordList.isEmpty()) { return; }

    Integer size = this.recordList.size();      
    Schema.SObjectType sObjectType = recordList.getSObjectType();

    List<SObject> activeBatch = (List<SObject>)Type.forName('List<' + sObjectType + '>').newInstance();
    System.debug('UT_BatchDML for ' + size + ' ' + sObjectType + ' records.');

    if (size > BatchSize) {
        // take the last <BatchSize> records from the top and put the rest in the next queueable
        for (Integer i = size-1; i >= size-BatchSize; i--) {
            activeBatch.add(this.recordList.remove(i));
        }
        followupJob = new UT_BatchDML(this.operation, this.recordList);

    } else {
        activeBatch = this.recordList;
    }

    switch on this.operation {
        when INS {
            System.debug(LoggingLevel.INFO, 'Inserting ' + activeBatch.size() + ' ' + sObjectType + ' records.');
            insert activeBatch;
        }
        when UPD {
            System.debug(LoggingLevel.INFO, 'Updating ' + activeBatch.size() + ' ' + sObjectType + ' records.');
            update activeBatch;
        }
        when DEL {
            System.debug(LoggingLevel.INFO, 'Deleting ' + activeBatch.size() + ' ' + sObjectType + ' records.');
            delete activeBatch;
        }
    }

    if (followupJob != null) {
        if(!Test.isRunningTest()) {                        
            System.enqueueJob(followupJob);
        }
    }
}
1
  • Thanks Marco, I was looking for exactly this little bit of code, so I appreciate you posting it:) Oct 19, 2021 at 17:25
0

Three pieces of advice:

  • Structure your queries so that a determinate number of records are always handled in the execute method. That way you can tune the batch size to reliably stay within the various governor limits in each execute call. In your case that could mean querying the child records in the start method along with the relevant parent fields instead of querying the parent.
  • Favor returning a QueryLocator from the start method so that up to 50 million records can be processed.
  • While you may have existing classes that perform parts of the logic, it is not uncommon (and may be unavoidable) to have to refactor those to deal with the large scale case.

You must log in to answer this question.

Not the answer you're looking for? Browse other questions tagged .