I'm starting to use Test.loadData and am facing an issue when creating Portal User data. Code below works very well, not only creating and inserting Accounts and Contacts, but also establishing proper Contact-Account references based on Id in Account CSV and AccountId in Contact CSV (please note that the Ids in test context after insert are different than what was specified in Id fields in CSVs).
Test.loadData(
Account.sObjectType,
'testdataAccount'
);
Test.loadData(
Contact.sObjectType,
'testdataContact'
);
The issue appears here:
Test.loadData(
User.sObjectType,
'testdataPortalUser'
);
I'm constantly getting System.UnexpectedException: Salesforce System Error: 1431980718-510656 (-1625830553) (-1625830553). A part of User CSV with only one record is below:
ID,USERNAME,LASTNAME,FIRSTNAME,POSTALCODE,COUNTRY,EMAIL,PHONE,MOBILEPHONE,ALIAS,COMMUNITYNICKNAME,TIMEZONESIDKEY,LOCALESIDKEY,PROFILEID,LANGUAGELOCALEKEY,EMAILENCODINGKEY,CONTACTID,LMSCONS__CORNERSTONE_ID__C,LICENSE_OWNER__C,LICENSE_TYPE__C
005G00000030XXxXXX,john.censored@censored.com.lmc,Censored,John,,,john.censored=company.com@example.com,,,jcnsrd,john.censored@all44-31,2,2,00eG0000000eXXxXXX,2,3,003G0000018tXXxXXX,censored,Censored,CNSRD
The total amount of records in User CSV file is 9. The references to Contact records in User CSV were validated and are correct. ProfileIds are also correct.
I followed advises from this knowledge base article, but it still didn't help:
Try to edit your an user record, and click 'Email Encoding' field. For instance, first value is 'UTF-8' and second value is 'ISO-8859-1'. In this case EmailEncodingKey should be 1 and 2 respectively.
Based on article above the way how Test.loadData CSV parsing was designed in Salesforce might confuse a lot of developers. I've spent many hours constantly using 'ISO-8859-1' before finding the article. Error handling is also something Salesforce needs to improve. I would really like to see why data load is failing instead of just 'Salesforce System Error'.
Update 2
There was a functionality developed that is based on code sample from Update 1, but has a lot more features is stable and is currently used on Production.
Further read:
Update 1
Managed to develop custom code that works similar to Test.loadData, but is more flexible. It needs some improvements, methods are too large, haven't had enough time to make it better.
AF_TestDataFactory.cls
private static final String LOCALE_EN_US = 'en_US';
private static final Set<Schema.DisplayType> STRING_TYPES = new Set<Schema.DisplayType> {
Schema.DisplayType.base64,
Schema.DisplayType.Email,
Schema.DisplayType.MultiPicklist,
Schema.DisplayType.Phone,
Schema.DisplayType.Picklist,
Schema.DisplayType.String,
Schema.DisplayType.TextArea,
Schema.DisplayType.URL
};
private static final Set<Schema.DisplayType> INTEGER_TYPES = new Set<Schema.DisplayType> {
Schema.DisplayType.Integer
};
private static final Set<Schema.DisplayType> ID_TYPES = new Set<Schema.DisplayType> {
Schema.DisplayType.ID,
Schema.DisplayType.Reference
};
private static final Set<Schema.DisplayType> DOUBLE_TYPES = new Set<Schema.DisplayType> {
Schema.DisplayType.Currency,
Schema.DisplayType.Double,
Schema.DisplayType.Percent
};
private static final Set<Schema.DisplayType> DATETIME_TYPES = new Set<Schema.DisplayType> {
Schema.DisplayType.DateTime
};
private static final Set<Schema.DisplayType> DATE_TYPES = new Set<Schema.DisplayType> {
Schema.DisplayType.Date
};
private static final Set<Schema.DisplayType> BOOLEAN_TYPES = new Set<Schema.DisplayType> {
Schema.DisplayType.Boolean,
Schema.DisplayType.Combobox
};
private static Map<String, String> csvToDatabaseIdMap = new Map<String, String>();
public static Map<String, List<SObject>> loadDataFromCSVString(
List<ObjectCSVData> objctCSVDataList) {
Map<String, List<SObject>> objctMap;
for (ObjectCSVData objctCSVData : objctCSVDataList) {
List<Map<String, String>> csvRowMapList;
List<String> csvRowList;
List<String> fieldList;
csvRowList = objctCSVData.csvObjctData.split('\n');
fieldList = csvRowList[0]
.replace('\n','')
.replace('\r','')
.toLowerCase()
.split(',');
AF_CSVHandler.parameterMap = new Map<String, Object> {
'fieldList' => fieldList
};
csvRowMapList = AF_CSVHandler.createTable(csvRowList, null);
objctMap.put(
objctCSVData.objctName,
createObjectListFromCSV(
objctCSVData.objctName,
csvRowMapList
)
);
}
return objctMap;
}
public static Map<String, List<SObject>> loadDataFromStaticResource(
List<ObjectStaticResource> objctStaticRsrsNameList) {
Integer objctStaticRsrsNameListSize;
Integer staticRsrsListSize;
List<StaticResource> orderedStaticRsrsList;
List<StaticResource> unorderedStaticRsrsList;
Map<String, List<SObject>> objctMap;
Map<String, StaticResource> orderStaticRsrsMap;
Set<String> staticRsrsNameSet;
staticRsrsNameSet = new Set<String>();
for (ObjectStaticResource objctStaticRsrs : objctStaticRsrsNameList) {
staticRsrsNameSet.add(objctStaticRsrs.staticRsrsName);
}
unorderedStaticRsrsList = [
SELECT Name,
Body
FROM StaticResource
WHERE Name IN :staticRsrsNameSet
];
objctStaticRsrsNameListSize = objctStaticRsrsNameList.size();
staticRsrsListSize = unorderedStaticRsrsList.size();
System.assertEquals(objctStaticRsrsNameListSize, staticRsrsListSize);
orderStaticRsrsMap = new Map<String, StaticResource>();
for (StaticResource staticRsrs : unorderedStaticRsrsList) {
orderStaticRsrsMap.put(staticRsrs.Name, staticRsrs);
}
orderedStaticRsrsList = new List<StaticResource>();
for (ObjectStaticResource objctStaticRsrs : objctStaticRsrsNameList) {
orderedStaticRsrsList.add(
orderStaticRsrsMap.get(objctStaticRsrs.staticRsrsName)
);
}
objctMap = new Map<String, List<SObject>>();
for (Integer i = 0; i < staticRsrsListSize; i++) {
List<Map<String, String>> csvRowMapList;
List<String> csvRowList;
List<String> fieldList;
String objctName;
csvRowList = orderedStaticRsrsList[i].Body.toString().split('\n');
fieldList = csvRowList[0]
.replace('\n','')
.replace('\r','')
.split(',');
objctName = objctStaticRsrsNameList[i].objctName;
AF_CSVHandler.parameterMap = new Map<String, Object> {
'fieldList' => fieldList
};
csvRowMapList = AF_CSVHandler.createTable(csvRowList, null);
objctMap.put(
objctName,
createObjectListFromCSV(
objctName,
csvRowMapList
)
);
}
return objctMap;
}
public static List<SObject> createObjectListFromCSV(
String objctName,
List<Map<String, String>> csvRowMapList) {
Integer csvRowMapListSize;
List<SObject> objctList;
Map<String, Schema.SObjectField> fldMap;
Schema.DescribeSobjectResult dscrObjctReslt;
Set<String> booleanTypeFieldSet;
Set<String> dateTypeFieldSet;
Set<String> datetimeTypeFieldSet;
Set<String> doubleTypeFieldSet;
Set<String> fldAPINameSet;
Set<String> idTypeFieldSet;
Set<String> integerTypeFieldSet;
Set<String> referencesFldSet;
Set<String> stringTypeFldSet;
booleanTypeFieldSet = new Set<String>();
dateTypeFieldSet = new Set<String>();
datetimeTypeFieldSet = new Set<String>();
doubleTypeFieldSet = new Set<String>();
fldAPINameSet = new Set<String>();
idTypeFieldSet = new Set<String>();
integerTypeFieldSet = new Set<String>();
referencesFldSet = new Set<String>();
stringTypeFldSet = new Set<String>();
fldAPINameSet = csvRowMapList[0].keySet();
dscrObjctReslt = Schema.describeSObjects(
new List<String>{ objctName }
)[0];
fldMap = dscrObjctReslt.fields.getMap();
referencesFldSet = new Set<String>();
for (String fldAPIName : fldAPINameSet) {
Schema.DescribeFieldResult dscrbFldRslt;
Schema.SObjectField field;
field = fldMap.get(fldAPIName);
if (field != null && (dscrbFldRslt = field.getDescribe()).isCreateable()) {
Schema.DisplayType displayType;
displayType = dscrbFldRslt.getType();
if (STRING_TYPES.contains(displayType)) {
stringTypeFldSet.add(fldAPIName);
} else if (INTEGER_TYPES.contains(displayType)) {
integerTypeFieldSet.add(fldAPIName);
} else if (ID_TYPES.contains(displayType)) {
idTypeFieldSet.add(fldAPIName);
} else if (DOUBLE_TYPES.contains(displayType)) {
doubleTypeFieldSet.add(fldAPIName);
} else if (DATETIME_TYPES.contains(displayType)) {
datetimeTypeFieldSet.add(fldAPIName);
} else if (DATE_TYPES.contains(displayType)) {
dateTypeFieldSet.add(fldAPIName);
} else if (BOOLEAN_TYPES.contains(displayType)) {
booleanTypeFieldSet.add(fldAPIName);
}
} else {
fldAPINameSet.remove(fldAPIName);
}
}
objctList = new List<SObject>();
csvRowMapListSize = csvRowMapList.size();
for (Integer i = 1; i < csvRowMapListSize; i++) {
SObject sobjct = (SObject) Type.forName(objctName).newInstance();
for (String fldAPIName : fldAPINameSet) {
Object value;
String valueStr;
value = csvRowMapList[i].get(fldAPIName);
valueStr = (String) value;
if (String.isNotBlank(valueStr)
&& fldAPIName != 'id') {
if (!stringTypeFldSet.contains(fldAPIName)) {
if (integerTypeFieldSet.contains(fldAPIName)) {
value = Integer.valueOf(value);
} else if (idTypeFieldSet.contains(fldAPIName)) {
String valueFromMap;
valueFromMap = csvToDatabaseIdMap.get(valueStr);
value = Id.valueOf(
valueFromMap != null ? valueFromMap : valueStr
);
} else if (doubleTypeFieldSet.contains(fldAPIName)) {
value = Double.valueOf(value);
} else if (datetimeTypeFieldSet.contains(fldAPIName)) {
value = (Datetime) JSON.deserialize(valueStr, Datetime.class);
} else if (dateTypeFieldSet.contains(fldAPIName)) {
value = (Date) JSON.deserialize(valueStr, Date.class);
} else if (booleanTypeFieldSet.contains(fldAPIName)) {
value = Boolean.valueOf(value);
}
}
sobjct.put(
fldAPIName,
value
);
}
}
objctList.add(sobjct);
}
System.debug(JSON.serializePretty(objctList));
insert objctList;
for (Integer i = 0; i < objctList.size(); i++) {
System.debug( + ', ' + objctList[i].Id);
csvToDatabaseIdMap.put(csvRowMapList[i + 1].get('id'), objctList[i].Id);
}
return objctList;
}
AF_CSVHandler.class
public static List<Map<String, String>> createTable(
List<String> scope,
Map<String, Object> statefulBatchMap) {
List<Map<String, String>> table = new List<Map<String, String>>();
try {
String header = (String) parameterMap.get('csvHeader');
String log = header + ',Message';
for (String row : scope) {
if (String.isNotEmpty(row)) {
try {
Integer i;
Map<String, String> columnMap;
columnMap = new Map<String, String>();
row = row
.replace('\n','')
.replace('\r','');
row = escapeCommas(row);
i = 0;
for (String column : row.split(',')) {
columnMap.put(
fieldList[i],
column.replace('ESCAPEDCOMMA', ','));
i++;
}
table.add(columnMap);
}
catch (Exception e) {
log += row + ','
+ e.getStackTraceString() + '\n'
+ e.getMessage()+ '\n';
}
}
}
if (log != header + ',Message') {
createDocumentLog(
new Log (
(String)parameterMap.get('folderName'),
(String)parameterMap.get('logName'),
log
)
);
}
}
catch (Exception e) {
new DealerCRMException().processErrors(e, true, false, false);
}
return table;
}
public static String escapeCommas(String row) {
List<String> charList = row.split('');
charList.remove(0);
Integer charListLength = charList.size();
Boolean rangeValidationStarted = false;
for (Integer i = 0; i < charListLength; i++) {
String str = charList[i];
if (!rangeValidationStarted) {
if (str == '"') {
rangeValidationStarted = true;
}
} else if (str == '"') {
rangeValidationStarted = false;
} else if (str == ',') {
charList[i] = 'ESCAPEDCOMMA';
}
}
return String.join(charList, '');
}
