WSProxy HTTP POST Request only runs a handful of records at a time before erroring and pausing

Question

I'm leveraging SSJS WSProxy's HTTP Request to send a payload over to a third-party server. The payload is essentially a data extension's information that is being looped over until all records are sent over the third-party server.

I don't think the issue is the payload because if I keep refreshing this script, it will eventually be able to complete the entire list of records that need to be injected.

The issue is that it's almost as if I'm limited- sometimes the script will run 50 records at once, most times it only runs 10 records until erroring out. It's a random number of records everytime. This can be a pain as when I have 3000 records, having to refresh until the entirety of that data extension is passed over the third-party takes an hour of manual refresh.
I've tried splitting my data extensions down to small lists of 50 and I still face the same issue.

Please note that the access token is retrieved in a separate step and stored in a data extension to avoid having to retrieve it multiple times

Is there a better way that I can be debug this? I'm not really getting back a meaningful error message right now.

Here is my code - altered a few things to protect sensitive data.

<script runat="server" type="text/javascript">
Platform.Load("core", "1.1.5");

var SessionID = DataExtension.Init("external key here");
var Points = DataExtension.Init("external key here");
var data = SessionID.Rows.Retrieve();
var dataDE = data.length -1;
var datas = data[dataDE];
var accessToken = datas.Codefrom;

var filter = {
            LeftOperand:{
                Property:"Status",
                SimpleOperator:"notEquals",
                Value: "1"
            },
            LogicalOperator:"OR",
            RightOperand:{
                    Property:"Status",
                    SimpleOperator:"isNull"
            }
           };
var pointsdata = Points.Rows.Retrieve(filter);

try{
   for (var i=0; i < pointsdata.length; i++)
       {
    var pointdata = pointsdata[i];
    var card_number = pointdata.card_number;
    var transaction_id = pointdata.transaction_id;
    var point_value = pointdata.point_value
    

    //PROD
    var url = 'endpoint URL here';
    //PROD 
    var payload={
        "PointValue":point_value,
        "ProgramGuid":"a2fe52ae-0000-0000-0000-000000000",
        "Reward":1,
        "CardNumber":card_number,
        "SessionGuid":accessToken,
        "TransactionMetadataDescription":"Offer"
     };
     
     var req = new Script.Util.HttpRequest(url);
     req.emptyContentHandling = 0;
     req.retries = 2;
     req.continueOnError = true;
     req.contentType = "application/json"
     req.method = "POST";
     req.postData = Stringify(payload);

     var resp = req.send();

         
     var resultJSON = Platform.Function.ParseJSON(String(resp.content));

     var message=resultJSON.Result.Message;
     var messageDetails=resultJSON.Result.MessageDetails;
     var status=resultJSON.Result.Status;
     var brokertransactionId=resultJSON.BrokerTransactionId;
Write(status+"<br>");
         Write(Stringify(resultJSON)+"<br>");
         
     
     var upsert = Platform.Function.UpsertDE("de_to_log_status_to",["card_number","transaction_id"],[card_number,transaction_id],["Message","Status","MessageDetails","BrokerTransactionId"],[message,status,messagedetails,brokertransactionId]);
    };
} catch(error) {
  Write(Stringify(error))
};

</script>

Answer 1

I don't see any WSProxy in your post.

You could use it to retrieve your Points DE rows instead of Points.Rows.Retrieve(filter) as it provides an explicit view into the continuing a request to traverse all of the results the filter selects.

Here's an example from the Advanced Retrieves with WSProxy page

var prox = new Script.Util.WSProxy(),
    objectType = "Email",
    cols = ["Name"],
    moreData = true,
    reqID = null,
    numItems = 0;

while(moreData) {
    moreData = false;
    var data = reqID == null ?
           prox.retrieve(objectType, cols) :
           prox.getNextBatch(objectType, reqID);

    if(data != null) {
        moreData = data.HasMoreRows;
        reqID = data.RequestID;
        if(data && data.Results) {
            for(var i=0; i< data.Results.length; i++) {
                Platform.Response.Write(data.Results[i].Name);
                numItems++;
            }
        }
    }
}
Platform.Response.Write("<br />" + numItems + " total " + objectType);

In the above example, it's iterating through Email objects, but you could iterate through DE rows which are DataExtensionObject SOAP objects (DataExtensionObject[DataExtensionCustomerKey]).

The core functions are nice and short, but since WSProxy is for SOAP, expect it to be much more verbose to do operations that are one-liners in the core and platform libraries.

Edit:

Here's a Data Extension example that I wrote a while back -- it's actually doing an HTTP get like you're doing.

<script runat="server">

Platform.Load("Core","1");

var debug = false;
var MID = '11111111'  // CLIENT MID
var sourceDEName = "PI_HistoricalPurchases_scriptSource";
var sourceDEKey = "419C299A-3A8D-47C1-BC80-60D4D06AA05B";
var masterDEName = "PI_HistoricalPurchases_master";
var logDEName = "PI_Offline_Import_Log";
var recordCount = 0;

var prox = new Script.Util.WSProxy(),
    objectType = "DataExtensionObject[" + sourceDEKey + "]",
    cols = ["EMAIL_ADDRESS", "SKU", "recordKey", "PI_Processed"],
    filter = {Property: "PI_Processed", SimpleOperator: "equals", Value: 0},
    moreData = true,
    reqID = null,
    numItems = 0;

while (moreData) {

    moreData = false;

    var data = reqID == null ? prox.retrieve(objectType, cols) : prox.getNextBatch(objectType, reqID);

    if (data != null) {

        moreData = data.HasMoreRows;
        reqID = data.RequestID;

        if (data && data.Results) {

            for (var i=0; i < data.Results.length; i++) {

                var email = data.Results[i].Properties[0].Value;
                var sku = data.Results[i].Properties[1].Value;
                var recordKey = data.Results[i].Properties[2].Value;
                var url = 'https://' + MID + '.collect.igodigital.com/c2/' + MID + '/track_conversion?payload={"cart":[{"item":"' + sku + '","unique_id":"' + sku +'","quantity":"1"}],"special":"ownit","user_info":{"email":"'+ email +'"}}';

                if (!debug) {

                    var result = HTTP.Get(url);

                    var sourceUpdate = DataExtension.Init(sourceDEName);
                    sourceUpdate.Rows.Update({PI_Processed: 1}, ["recordKey"], [recordKey]);

                    var masterUpdate = DataExtension.Init(masterDEName);
                    masterUpdate.Rows.Update({PI_Processed: 1}, ["recordKey"], [recordKey]);

                } else {

                    Platform.Response.Write("<br>email: " + email);
                    Platform.Response.Write("<br>sku: " + sku);
                    Platform.Response.Write("<br>recordKey: " + recordKey);
                    Platform.Response.Write("<br>url: " + url);

                }

                recordCount++;

            }
        }
    }
}

var todaysDate = new Date();

if (!debug) {

    var logDE = DataExtension.Init(logDEName);
    logDE.Rows.Add({TimeStamp: todaysDate, RecordsImported: recordCount, ImportType: "Offline Purchase Script"});

} else {

    Platform.Response.Write("<br>recordCount: " + recordCount);

}

</script>