This seems to be possible. I just performed this simple test as a user without that permission without any problems, so if I understand the question correctly I think the answer is yes.
Page:
<apex:page controller="TestQueueMembership">
<apex:messages />
<apex:form >
<apex:commandButton value="Test" action="{!toggle}"/>
</apex:form>
</apex:page>
Controller:
public class TestQueueMembership
{
public PageReference toggle()
{
List<GroupMember> groups = [select Id from GroupMember where Group.Type = 'Queue' and GroupId = '00Gb0000000RFBC' and UserOrGroupId = '005b0000000TPaL'];
if( !groups.isEmpty() )
{
delete groups;
ApexPages.addMessage( new ApexPages.Message(ApexPages.Severity.INFO, 'Removed user from Queue') );
}
else
{
insert new GroupMember( GroupId = '00Gb0000000RFBC', UserOrGroupId = '005b0000000TPaL' );
ApexPages.addMessage( new ApexPages.Message(ApexPages.Severity.INFO, 'Added user to Queue') );
}
return null;
}
}