Try This:
trigger checkaccount on contact(before insert, before update){
for(contact con : trigger.new){
if(con.accountId != null){
Id a = con.accountId;
list<contact> cList = [SELECT Id, Check_box_name__c FROM Contact where Check_box_name__c = true and AccountId = :a];
if(cList.size() > 0){
con.addError('no more primary contacts');
}
}
}
}
Update
Above code, I got SOQL inside for loop, if you wish to avoid that... and also as per the suggestion in below comment:
trigger checkaccount on contact(before insert, before update){
List<Id> accID = new List<Id>();
for(contact c : trigger.new){
if(c.accountId != null){
accID.add(c.accountId);
}
}
List<account> acc = [select id, (select id, Check_box_name__c from contacts where Check_box_name__c = true) from Account WHERE Id In: accID];
Map<id, boolean> bool = new map<id,boolean>();
for(account a : acc){
bool.put(a.id, a.contacts.size()>0 ? true : false);
}
for(Contact c : trigger.new){
if(bool.get(c.AccountId) == true){
c.addError('no more primary fields');
}
}
}