You have to build the logic like this.
> 1. Find Previous owner and Current Owner
2. Verify if previous owner is a 'Queue'
3. If point 2 satisfies then find the current user is part of that queue.
4. If point 3 satisfies then allow update the ownership, otherwise throw the error message.
5. Also, wise to check, if current and previous owner are not same.


    {!REQUIRESCRIPT("/soap/ajax/31.0/connection.js")} 
    var ownershipChangeRequest = new sforce.SObject("Case"); 
    var previousOwner = "{!Case.OwnerId}"; 
    var currentOwner = "{!$User.Id}"; 
    ownershipChangeRequest.Id = "{!Case.Id}"; 
    ownershipChangeRequest.OwnerId = "{!$User.Id}"; 
    
    //if previousOwner is queue 
    var ownerRec = sforce.connection.query("SELECT owner.type, ownerid, Id from Case where  OwnerId='"+ previousOwner + "' AND " + Id='" + {!Case.Id} + "'"); 
    var records1 = accountRec.getArray('ownerRec');
    if(records1 !=null)
    {	
    	if(ownerRec.records.owner.type == 'Queue')
    	{
    		//currentOwner is part of queue
    		var currentOwnerRec = sforce.connection.query("SELECT g.UserOrGroupId From GroupMember g WHERE groupId ='" + previousOwner + "' AND g.UserOrGroupId ='" + currentOwner + "'");
    		var records2 = accountRec.getArray('currentOwnerRec');
    		if(records2 !=null)
    		{
    			var result = sforce.connection.update([ownershipChangeRequest]);	
    			location.reload(); 	
    		}
    		else
    		{
    			alert("You cannot take this case because you are not part of the queue");
    		}
    	}
    }
    if(previousOwner == currentOwner) 
    {	
    	alert("You are already the Case Owner ");	
    }