As far as I have searched documentation, there is no standard way of knowing in which container a component is being loaded.

Solution: We can create a @api property called container through which we can tell the component about the container it is being loaded in. You can check the below sample LWC component which has been designed for both visualforce and lightning:

HTML:

<template>
    <div>
        recordPageUrl - {recordPageUrl}
    </div>
    <div>
        <a href={recordPageUrl}>Go to Url</a>
    </div>
    <div>
        <lightning-button label="Navigate to account" onclick={navToAcc}></lightning-button>
    </div>
</template>

JS:

import { LightningElement, wire, api, track } from 'lwc';
import { NavigationMixin } from 'lightning/navigation';

export default class Poc extends NavigationMixin(LightningElement) {
    @api container = 'Lightning';
    @track recordPageUrl;
    pageRef = {
        type: 'standard__recordPage',
        attributes: {
            recordId: '0010K00001eKzKgQAK',
            actionName: 'view',
        },
    };

    connectedCallback() {
        switch (this.container) {
            case 'Lightning':
                this[NavigationMixin.GenerateUrl](this.pageRef).then(url => {
                    this.recordPageUrl = url;
                }).catch(err => {
                    console.error("ERROR => ", JSON.stringify(err))
                });
                break;
            case 'Visualforce':
                this.recordPageUrl = '/0010K00001eKzKgQAK';
                break;
            default:
                console.error('container context not provided');
        }
    }

    navToAcc() {
        switch (this.container) {
            case 'Lightning':
                this[NavigationMixin.Navigate](this.pageRef);
                break;
            case 'Visualforce':
                window.open('/0010K00001eKzKgQAK');
                break;
            default:
                console.error('container context not provided');
        }
    }
}

We are saying it will be loaded in Lightning by default unless stated other-wise.

When loading the component in visualforce, we can specify it as 'Visualforce' container:

vf page:

<apex:page >
    <apex:includeLightning />

    <div id="lightningvf" />

    <script>
        $Lightning.use("c:TestCompApp", function() {
          $Lightning.createComponent("c:poc",
              { container:'Visualforce' },
              "lightningvf",
              function(cmp) {
                console.log("poc was created");
                // do some stuff
              }
          );
        });
    </script>
</apex:page>